sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-09-22 05:16:22 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat(dictionary): add Hachidori backend support

Browse Source 
- Add backend selection, setup gating, Anki integration, and external host support
- Add launcher flags, documentation, packaging, and focused tests
- Open on-demand overlay modals on the first attempt
This commit is contained in:
sudacode
2026-09-22 00:21:19 -07:00
parent 1508863dbb
commit d9fdc7ef6d
446 changed files with 109060 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vendor/hachidori/third_party/hoshidicts/.clang-format
Vendored
+3
View File
@@ -0,0 +1,3 @@
BasedOnStyle: Google
ColumnLimit: 120
FixNamespaceComments: false
vendor/hachidori/third_party/hoshidicts/.clang-tidy
Vendored
+32
View File
@@ -0,0 +1,32 @@
Checks: >
-*,
bugprone-*,
modernize-*,
readability-*,
performance-*,
-modernize-use-nodiscard,
-modernize-use-trailing-return-type,
-readability-identifier-length,
-readability-magic-numbers,
-readability-function-cognitive-complexity,
-bugprone-easily-swappable-parameters,
-readability-math-missing-parentheses,
-modernize-use-using,
-modernize-deprecated-headers
CheckOptions:
- { key: readability-identifier-naming.NamespaceCase, value: lower_case }
- { key: readability-identifier-naming.ClassCase, value: CamelCase }
- { key: readability-identifier-naming.StructCase, value: CamelCase }
- { key: readability-identifier-naming.TemplateParameterCase, value: CamelCase }
- { key: readability-identifier-naming.ParameterCase, value: lower_case }
- { key: readability-identifier-naming.FunctionCase, value: aNy_CasE }
- { key: readability-identifier-naming.VariableCase, value: lower_case }
- { key: readability-identifier-naming.ClassMemberCase, value: lower_case }
- { key: readability-identifier-naming.StructMemberCase, value: lower_case }
- { key: readability-identifier-naming.ClassMemberSuffix, value: _ }
- { key: readability-identifier-naming.PrivateMemberSuffix, value: _ }
- { key: readability-identifier-naming.ProtectedMemberSuffix, value: _ }
- { key: readability-implicit-bool-conversion.AllowIntegerConditions, value: 1 }
- { key: readability-implicit-bool-conversion.AllowPointerConditions, value: 1 }
- { key: readability-function-cognitive-complexity.IgnoreMacros, value: 1 }
vendor/hachidori/third_party/hoshidicts/.gitignore
+56
View File
@@ -0,0 +1,56 @@
# Prerequisites
*.d
# Compiled Object files
*.slo
*.lo
*.o
*.obj
# Precompiled Headers
*.gch
*.pch
# Linker files
*.ilk
# Debugger Files
*.pdb
# Compiled Dynamic libraries
*.so
*.dylib
*.dll
# Fortran module files
*.mod
*.smod
# Compiled Static libraries
*.lai
*.la
*.a
*.lib
# Executables
*.exe
*.out
*.app
# debug information files
*.dwo
# directories
build/
.build/
.cache/
# macos ds_store
.DS_Store
# Swift
Package.resolved
.swiftpm/
# python fixture generators
__pycache__/
vendor/hachidori/third_party/hoshidicts/.gitmodules
+24
View File
@@ -0,0 +1,24 @@
[submodule "external/utfcpp"]
path = external/utfcpp
url = https://github.com/nemtrif/utfcpp.git
[submodule "external/glaze"]
path = external/glaze
url = https://github.com/stephenberry/glaze
[submodule "external/zstd"]
path = external/zstd
url = https://github.com/facebook/zstd.git
[submodule "external/unordered_dense"]
path = external/unordered_dense
url = https://github.com/martinus/unordered_dense.git
[submodule "external/xxHash"]
path = external/xxHash
url = https://github.com/Cyan4973/xxHash.git
[submodule "external/libdeflate"]
path = external/libdeflate
url = https://github.com/ebiggers/libdeflate.git
[submodule "external/utf8proc"]
path = external/utf8proc
url = https://github.com/JuliaStrings/utf8proc.git
[submodule "external/kanji-processor"]
path = external/kanji-processor
url = https://github.com/yomidevs/kanji-processor.git
vendor/hachidori/third_party/hoshidicts/CMakeLists.txt
Vendored
+199
View File
@@ -0,0 +1,199 @@
cmake_minimum_required(VERSION 3.22.1)
cmake_policy(SET CMP0077 NEW)
project(hoshidicts LANGUAGES C CXX)
set(CMAKE_CXX_STANDARD 23)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_C_STANDARD 99)
set(CMAKE_C_STANDARD_REQUIRED ON)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
option(HOSHIDICTS_CLI "hoshidicts cli" OFF)
option(HOSHIDICTS_BENCHMARK "hoshidicts import and lookup benchmarks" OFF)
option(HOSHIDICTS_TESTS "hoshidicts unit tests (ctest)" OFF)
# Emscripten hosts that link with -sWASMFS: read-only descriptors are served
# from a Blob there, so dictionary files are opened read-write for the copy
# (see src/memory/memory.cpp).
option(HOSHIDICTS_WASMFS "The Emscripten build links against WasmFS" OFF)
set(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "")
set(ZSTD_BUILD_TESTS OFF CACHE BOOL "")
set(ZSTD_BUILD_SHARED OFF CACHE BOOL "")
set(ZSTD_BUILD_STATIC ON CACHE BOOL "")
set(LIBDEFLATE_BUILD_SHARED_LIB OFF CACHE BOOL "")
set(LIBDEFLATE_BUILD_GZIP OFF CACHE BOOL "")
set(UTF8PROC_INSTALL OFF CACHE BOOL "")
add_subdirectory(external/glaze)
add_subdirectory(external/zstd)
# Override one zstd translation unit: src/zstd/zstd_compress_sequences.c is the
# pinned upstream file plus a cache for the default FSE table (see its header
# comment). Re-copy it when the zstd submodule moves.
get_target_property(HOSHIDICTS_ZSTD_SOURCES libzstd_static SOURCES)
list(FILTER HOSHIDICTS_ZSTD_SOURCES EXCLUDE REGEX "compress/zstd_compress_sequences\\.c$")
set_property(TARGET libzstd_static PROPERTY SOURCES ${HOSHIDICTS_ZSTD_SOURCES})
target_sources(libzstd_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/zstd/zstd_compress_sequences.c)
target_include_directories(libzstd_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/external/zstd/lib/compress)
add_subdirectory(external/unordered_dense)
add_subdirectory(external/libdeflate)
add_subdirectory(external/utf8proc)
# Vendored (not submodules): external/lzokay and external/gumbo-parser carry
# their own LICENSE and README with the upstream commit they were copied from.
add_library(lzokay STATIC external/lzokay/lzokay.cpp)
target_include_directories(lzokay PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/external/lzokay)
set(GUMBO_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/gumbo-parser/src)
add_library(gumbo STATIC
${GUMBO_DIR}/ascii.c
${GUMBO_DIR}/attribute.c
${GUMBO_DIR}/char_ref.c
${GUMBO_DIR}/error.c
${GUMBO_DIR}/foreign_attrs.c
${GUMBO_DIR}/hashmap.c
${GUMBO_DIR}/parser.c
${GUMBO_DIR}/string_buffer.c
${GUMBO_DIR}/string_piece.c
${GUMBO_DIR}/string_set.c
${GUMBO_DIR}/svg_attrs.c
${GUMBO_DIR}/svg_tags.c
${GUMBO_DIR}/tag.c
${GUMBO_DIR}/tag_lookup.c
${GUMBO_DIR}/token_buffer.c
${GUMBO_DIR}/tokenizer.c
${GUMBO_DIR}/utf8.c
${GUMBO_DIR}/util.c
${GUMBO_DIR}/vector.c
)
target_include_directories(gumbo PUBLIC ${GUMBO_DIR})
if(MSVC)
target_compile_definitions(gumbo PRIVATE _CRT_SECURE_NO_WARNINGS)
endif()
add_library(hoshidicts
src/hash/hash.cpp
src/hash/bloom.cpp
src/importer.cpp
src/memory/memory.cpp
src/zip/zip.cpp
src/source/zip_source.cpp
src/mdict/ripemd128.cpp
src/mdict/mdict_reader.cpp
src/mdict/html_to_structured.cpp
src/mdict/mdict_source.cpp
src/json/yomitan_parser.cpp
src/json/json_skip.cpp
src/text_processor/text_processor.cpp
src/text_processor/kanji_variants.cpp
src/deinflector.cpp
src/query.cpp
src/lookup.cpp
src/hoshidicts_c.cpp
)
if(HOSHIDICTS_WASMFS)
target_compile_definitions(hoshidicts PRIVATE HOSHIDICTS_WASMFS=1)
endif()
target_include_directories(hoshidicts PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/external/utfcpp/source
${CMAKE_CURRENT_SOURCE_DIR}/external/xxHash
)
target_link_libraries(hoshidicts PRIVATE
glaze::glaze
libzstd_static
libdeflate_static
unordered_dense
utf8proc::utf8proc
lzokay
gumbo
)
if(MSVC)
target_compile_options(hoshidicts PRIVATE /utf-8)
endif()
if(HOSHIDICTS_CLI)
add_executable(hoshidicts-cli
cli/main.cpp
)
add_executable(hoshidicts-cli-c
cli/main.c
)
target_link_libraries(hoshidicts-cli PRIVATE
hoshidicts
)
target_link_libraries(hoshidicts-cli-c PRIVATE
hoshidicts
)
endif()
if(HOSHIDICTS_BENCHMARK)
add_executable(benchmark-import
benchmark/import.cpp
)
target_link_libraries(benchmark-import PRIVATE
hoshidicts
)
add_executable(benchmark-lookup
benchmark/lookup.cpp
)
target_link_libraries(benchmark-lookup PRIVATE
hoshidicts
)
endif()
if(HOSHIDICTS_TESTS)
enable_testing()
foreach(variant simd scalar)
add_executable(json-skip-test-${variant} tests/json_skip_test.cpp src/json/json_skip.cpp)
target_include_directories(json-skip-test-${variant} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
target_link_libraries(json-skip-test-${variant} PRIVATE glaze::glaze)
add_test(NAME json-skip-${variant} COMMAND json-skip-test-${variant})
endforeach()
target_compile_definitions(json-skip-test-scalar PRIVATE HOSHIDICTS_JSON_SKIP_SCALAR=1)
add_executable(score-roundtrip-test tests/score_roundtrip_test.cpp)
target_link_libraries(score-roundtrip-test PRIVATE hoshidicts)
add_test(NAME score-roundtrip COMMAND score-roundtrip-test)
add_executable(long-key-scan-test tests/long_key_scan_test.cpp)
target_link_libraries(long-key-scan-test PRIVATE hoshidicts)
add_test(NAME long-key-scan COMMAND long-key-scan-test)
add_executable(deps-smoke-test tests/deps_smoke_test.cpp)
target_link_libraries(deps-smoke-test PRIVATE lzokay gumbo)
add_test(NAME deps-smoke COMMAND deps-smoke-test)
add_executable(mdict-reader-test tests/mdict_reader_test.cpp)
target_include_directories(mdict-reader-test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
target_link_libraries(mdict-reader-test PRIVATE hoshidicts)
add_test(NAME mdict-reader COMMAND mdict-reader-test ${CMAKE_CURRENT_SOURCE_DIR}/tests/fixtures/mdict)
add_executable(html-to-structured-test tests/html_to_structured_test.cpp)
target_include_directories(html-to-structured-test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
target_link_libraries(html-to-structured-test PRIVATE hoshidicts glaze::glaze)
add_test(NAME html-to-structured COMMAND html-to-structured-test)
add_executable(mdict-test tests/mdict_test.cpp)
target_link_libraries(mdict-test PRIVATE hoshidicts)
add_test(NAME mdict-import COMMAND mdict-test ${CMAKE_CURRENT_SOURCE_DIR}/tests/fixtures/mdict)
add_executable(import-equivalence-test tests/import_equivalence_test.cpp)
target_link_libraries(import-equivalence-test PRIVATE hoshidicts)
add_test(NAME import-equivalence
COMMAND import-equivalence-test
${CMAKE_CURRENT_SOURCE_DIR}/tests/fixtures/yomitan/small_dict.zip
${CMAKE_CURRENT_SOURCE_DIR}/tests/fixtures/yomitan/golden.sha256)
endif()
vendor/hachidori/third_party/hoshidicts/LICENSE
Vendored
+674
View File
@@ -0,0 +1,674 @@
GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU General Public License is a free, copyleft license for
software and other kinds of works.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
the GNU General Public License is intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users. We, the Free Software Foundation, use the
GNU General Public License for most of our software; it applies also to
any other work released this way by its authors. You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
To protect your rights, we need to prevent others from denying you
these rights or asking you to surrender the rights. Therefore, you have
certain responsibilities if you distribute copies of the software, or if
you modify it: responsibilities to respect the freedom of others.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must pass on to the recipients the same
freedoms that you received. You must make sure that they, too, receive
or can get the source code. And you must show them these terms so they
know their rights.
Developers that use the GNU GPL protect your rights with two steps:
(1) assert copyright on the software, and (2) offer you this License
giving you legal permission to copy, distribute and/or modify it.
For the developers' and authors' protection, the GPL clearly explains
that there is no warranty for this free software. For both users' and
authors' sake, the GPL requires that modified versions be marked as
changed, so that their problems will not be attributed erroneously to
authors of previous versions.
Some devices are designed to deny users access to install or run
modified versions of the software inside them, although the manufacturer
can do so. This is fundamentally incompatible with the aim of
protecting users' freedom to change the software. The systematic
pattern of such abuse occurs in the area of products for individuals to
use, which is precisely where it is most unacceptable. Therefore, we
have designed this version of the GPL to prohibit the practice for those
products. If such problems arise substantially in other domains, we
stand ready to extend this provision to those domains in future versions
of the GPL, as needed to protect the freedom of users.
Finally, every program is threatened constantly by software patents.
States should not allow patents to restrict development and use of
software on general-purpose computers, but in those that do, we wish to
avoid the special danger that patents applied to a free program could
make it effectively proprietary. To prevent this, the GPL assures that
patents cannot be used to render the program non-free.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Use with the GNU Affero General Public License.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU Affero General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the special requirements of the GNU Affero General Public License,
section 13, concerning interaction through a network will apply to the
combination as such.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If the program does terminal interaction, make it output a short
notice like this when it starts in an interactive mode:
<program> Copyright (C) <year> <name of author>
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, your program's commands
might be different; for a GUI interface, you would use an "about box".
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU GPL, see
<https://www.gnu.org/licenses/>.
The GNU General Public License does not permit incorporating your program
into proprietary programs. If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications with
the library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License. But first, please read
<https://www.gnu.org/licenses/why-not-lgpl.html>.
vendor/hachidori/third_party/hoshidicts/Package.swift
Vendored
+58
View File
@@ -0,0 +1,58 @@
// swift-tools-version: 6.2
import PackageDescription
let package = Package(
name: "hoshidicts",
platforms: [.iOS(.v18), .macOS(.v15)],
products: [
.library(name: "CHoshiDicts", targets: ["CHoshiDicts"]),
.executable(name: "hoshidicts", targets: ["hoshidicts"]),
],
dependencies: [
.package(url: "https://github.com/facebook/zstd.git", from: "1.5.7"),
],
targets: [
.target(
name: "CHoshiDicts",
dependencies: [
.product(name: "libzstd", package: "zstd"),
],
path: ".",
sources: [
"src",
"external/libdeflate/lib",
"external/utf8proc/utf8proc.c",
"external/lzokay/lzokay.cpp",
"external/gumbo-parser/src",
],
publicHeadersPath: "include",
cxxSettings: [
.headerSearchPath("include"),
.headerSearchPath("external/libdeflate"),
.headerSearchPath("external/libdeflate/lib"),
.headerSearchPath("external/utfcpp/source"),
.headerSearchPath("external/glaze/include"),
.headerSearchPath("external/xxHash"),
.headerSearchPath("external/unordered_dense/include"),
.headerSearchPath("external/utf8proc"),
.headerSearchPath("external/lzokay"),
.headerSearchPath("external/gumbo-parser/src"),
.unsafeFlags(["-Wno-missing-braces"]),
],
swiftSettings: [
.interoperabilityMode(.Cxx)
]
),
.executableTarget(
name: "hoshidicts",
dependencies: ["CHoshiDicts"],
path: ".",
sources: ["cli/main.cpp"],
cxxSettings: [
.headerSearchPath("include"),
.headerSearchPath("external/utfcpp/source"),
]
),
],
cxxLanguageStandard: .cxx2b
)
vendor/hachidori/third_party/hoshidicts/README.md
Vendored
+116
View File
@@ -0,0 +1,116 @@
# hoshidicts
This library implements a dictionary backend that works similarly to [Yomitan](https://github.com/yomidevs/yomitan). This was made for [Hoshi Reader](https://github.com/Manhhao/Hoshi-Reader) and was only tested with Japanese. Other languages might need their own deinflector or adjustments to the lookup strategy.
A MIT version of the library is available on the [main-mit](https://github.com/Manhhao/hoshidicts/tree/main-mit) branch.
## Reference
### importer
```cpp
ImportResult dictionary_importer::import(const std::string& source_path, const std::string& output_dir, bool low_ram = false)
```
Imports a Yomitan `.zip` dictionary file or an MDict `.mdx` dictionary into a custom format. The resulting folder is stored in `output_dir/<dict_title>`. Glossaries are compressed using zstd. Term, frequency and pitch dictionaries are generally supported, but only a small part of the pitch accent spec was implemented. Setting `low_ram` to `true` can reduce memory usage significantly at the cost of slightly lower import speed.
The format is detected from the file contents, not the extension.
#### MDX / MDD import
An `.mdx` file is imported directly, without an intermediate Yomitan archive: its entries are converted to Yomitan term banks of 10,000 rows as they are read, so memory use is bounded by the bank size, not the dictionary. The conversion follows [manabitan](https://github.com/ManabiIO/manabitan)'s MDX importer so the result renders the same way.
- Container: MDict engine versions 1.x and 2.0 (`GeneratedByEngineVersion`). Version 3 files are rejected with a clear error.
- Encodings: UTF-8 and UTF-16. `GBK`, `GB18030` and `Big5` dictionaries are rejected (`unsupported MDX encoding: <name>`).
- Compression: none, LZO1X and zlib, per block, with every block's Adler-32 verified.
- Encryption: `Encrypted="2"` (ciphered key index) is supported. `Encrypted="1"` (registration-protected record blocks) needs a user key and is rejected.
- Resource files: `X.mdd`, `X.1.mdd`, `X.2.mdd`, ... next to `X.mdx` are read automatically (file name case does not matter); a missing MDD is not an error. Only assets the glossaries or stylesheets refer to are imported, under `mdict-media/<path>`; every `*.css` in the MDD plus inline `<style>` blocks become the dictionary's stylesheet. Keys containing `..`, a drive letter or NUL are dropped.
- Entries: `@@@LINK=target` redirects become extra headwords of the target (one hop; a redirect to a missing target is dropped). Duplicate headwords stay separate entries. `Format="Text"` definitions become plain string glossaries; HTML definitions become structured content.
- HTML fidelity: the MDX `StyleSheet` backtick markup is expanded; `b/i/em/strong/u/s/sub/sup/h1-6/p/pre/font/...` map to styled `span`/`div`; inline `style` keeps the properties Yomitan's structured content supports; `entry://`, `bword://`, `d:`, `x:` links search the term; `sound://` links are disabled (rendered as `#`); `javascript:` and friends are neutralised; `<script>` is dropped; unsupported elements keep their text; nesting deeper than 20 is flattened. CSS from the MDD is passed through as the dictionary stylesheet, so selectors that depend on tags Yomitan does not render (`<b>`, `<p>`, ...) will not match.
- The key index of the MDX (and of each MDD) is held in memory during the import; records are streamed block by block.
```
hoshidicts-cli import path/to/dictionary.mdx
```
### query
```cpp
void DictionaryQuery::add_term_dict(const std::string& path)
```
Adds an imported term dictionary to the query.
```cpp
void DictionaryQuery::add_freq_dict(const std::string& path)
```
Adds an imported frequency dictionary to the query.
```cpp
void DictionaryQuery::add_pitch_dict(const std::string& path)
```
Adds an imported pitch dictionary to the query.
```cpp
std::vector<TermResult> DictionaryQuery::query(const std::string& expression) const
```
Queries all added dictionaries for the given expression. TermResult includes glossary, frequency and pitch data in the order dictionaries were added. Glossaries are decompressed.
```cpp
std::vector<DictionaryStyle> DictionaryQuery::get_styles() const
```
Returns CSS styles for all dictionaries, if present.
```cpp
std::vector<char> DictionaryQuery::get_media_file(const std::string& dict_name, const std::string& media_path) const
```
Returns raw bytes for file originally stored at `media_path` in term dictionary `dict_name` or an empty vector if the file does not exist.
### deinflector
```cpp
std::vector<DeinflectionResult> Deinflector::deinflect(const std::string& text) const
```
Deinflects a given Japanese string using rules from the Yomitan deinflector. As this doesn't use any dictionary data, the result may include invalid deinflections.
```cpp
static uint32_t Deinflector::pos_to_conditions(const std::vector<std::string>& part_of_speech)
```
Converts a vector of part-of-speech tags into a bitmask used for deinflection filtering.
### lookup
```cpp
Lookup::Lookup(DictionaryQuery& query, Deinflector& deinflector)
```
Creates a Lookup object using a given query with dictionaries added and a deinflector.
```cpp
std::vector<LookupResult> Lookup::lookup(const std::string& lookup_string, int max_results = 16, size_t scan_length = 16) const
```
Follows a parsing strategy similar to Yomitan. Substrings of `lookup_string` are tested from length `scan_length` down to 1. Each substring is preprocessed, deinflected then queried using the query object.
Keys longer than `scan_length` are still found when `scan_length` is at least 8 and `lookup_string` is long enough to contain them: the importer records every key longer than 16 code points by its first eight code points in `scan.idx`, and when the input begins like such a key the scan extends to that key's length plus eight code points for an inflected ending. Inputs that do not begin like a long key keep the cost of `scan_length`. `DictionaryQuery::max_long_key_length()` returns the longest such key across the loaded term dictionaries so a caller can size `lookup_string`. Dictionaries imported before `scan.idx` existed simply never extend.
Results are filtered by part-of-speech tags defined in dictionaries, or added directly if none are present. The results are sorted by matched length first, then by preprocessing steps, then deinflection trace length and finally by frequency.
```cpp
std::vector<LookupResult> Lookup::lookup_dictionary(const std::string& lookup_string,
const std::string& dictionary_path,
int max_results = 16,
size_t scan_length = 16) const
```
Runs the same lookup and ranking pipeline while restricting term matches to one
already-added dictionary. Frequency and pitch metadata still come from every
added metadata dictionary.
## Acknowledgements
- [Yomitan](https://github.com/yomidevs/yomitan): Dictionary format, Japanese deinflection rules and descriptions, Japanese preprocessor | GPL-3.0
- [glaze](https://github.com/stephenberry/glaze): MIT
- [libdeflate](https://github.com/ebiggers/libdeflate.git): MIT
- [xxHash](https://github.com/Cyan4973/xxHash): BSD-2-Clause
- [zstd](https://github.com/facebook/zstd): BSD
- [utfcpp](https://github.com/nemtrif/utfcpp): BSL-1.0
- [unordered_dense](https://github.com/martinus/unordered_dense.git): MIT
- [utf8proc](https://github.com/JuliaStrings/utf8proc): MIT
- [kanji-processor](https://github.com/yomidevs/kanji-processor): MIT
- [lzokay](https://github.com/AxioDL/lzokay): MIT (vendored in `external/lzokay`)
- [gumbo-parser](https://github.com/sparklemotion/nokogiri/tree/main/gumbo-parser) (Nokogiri's fork of Google's gumbo): Apache-2.0, `hashmap.c` MIT (vendored in `external/gumbo-parser`)
## License
hoshidicts (main) is licensed under the GNU General Public License v3.0. See [LICENSE](LICENSE) for details.
vendor/hachidori/third_party/hoshidicts/benchmark/import.cpp
Vendored
+59
View File
@@ -0,0 +1,59 @@
#include <algorithm>
#include <chrono>
#include <filesystem>
#include <iostream>
#include <numeric>
#include <vector>
#include "hoshidicts/importer.hpp"
int main(int argc, char** argv) {
if (argc < 3) {
std::cout << std::format("{} <dictionary.zip|dictionary.mdx> <iterations>\n", argv[0]);
return 1;
}
const std::string source_path = argv[1];
const int iterations = std::stoi(argv[2]);
std::vector<double> durations;
std::string dict_title;
size_t term_count = 0;
size_t media_count = 0;
for (int i = 0; i < iterations; ++i) {
const auto start = std::chrono::high_resolution_clock::now();
const auto result = dictionary_importer::import(source_path, ".");
const auto end = std::chrono::high_resolution_clock::now();
if (result.success) {
if (dict_title.empty()) {
dict_title = result.title;
}
if (term_count == 0) {
term_count = result.summary.counts.terms.total;
media_count = result.summary.counts.media.total;
}
const std::chrono::duration<double, std::milli> elapsed = end - start;
durations.push_back(elapsed.count());
std::filesystem::remove_all(result.title);
}
}
if (durations.empty()) {
return 1;
}
const auto [min, max] = std::ranges::minmax_element(durations);
const double total = std::accumulate(durations.begin(), durations.end(), 0.0);
const double average = total / durations.size();
std::cout << std::format("dict: {} iterations: {}\n", dict_title, iterations);
std::cout << std::format("term_count: {}\n", term_count);
std::cout << std::format("media_count: {}\n", media_count);
std::cout << std::format("total: {:.2f}ms\n", total);
std::cout << std::format("avg: {:.2f}ms\n", average);
std::cout << std::format("min: {:.2f}ms\n", *min);
std::cout << std::format("max: {:.2f}ms\n", *max);
return 0;
}
vendor/hachidori/third_party/hoshidicts/benchmark/lookup.cpp
Vendored
+103
View File
@@ -0,0 +1,103 @@
#include "hoshidicts/lookup.hpp"
#include <algorithm>
#include <chrono>
#include <format>
#include <fstream>
#include <iostream>
#include <numeric>
#include <string_view>
#include <vector>
#include "hoshidicts/deinflector.hpp"
#include "hoshidicts/query.hpp"
namespace {
std::vector<std::string> read_word_list(const std::string& path) {
std::vector<std::string> words;
std::ifstream file(path);
std::string line;
while (std::getline(file, line)) {
if (line.ends_with('\r')) {
line.pop_back();
}
std::string word(line, 0, line.find(','));
if (!word.empty() && word != "Word") {
words.push_back(std::move(word));
}
}
return words;
}
}
int main(int argc, char** argv) {
if (argc < 5) {
std::cout << std::format(
"{} <csv_path> <iterations> --term <dict_path>... [--freq <dict_path>...] [--pitch <dict_path>...]\n", argv[0]);
return 1;
}
const std::string csv_path = argv[1];
const int iterations = std::stoi(argv[2]);
const std::vector<std::string> words = read_word_list(csv_path);
std::vector<std::string> term_paths;
std::vector<std::string> freq_paths;
std::vector<std::string> pitch_paths;
std::vector<std::string>* current = &term_paths;
for (int i = 3; i < argc; ++i) {
const std::string_view arg = argv[i];
if (arg == "--term") {
current = &term_paths;
} else if (arg == "--freq") {
current = &freq_paths;
} else if (arg == "--pitch") {
current = &pitch_paths;
} else {
current->emplace_back(arg);
}
}
DictionaryQuery query;
for (const auto& path : term_paths) {
query.add_term_dict(path);
}
for (const auto& path : freq_paths) {
query.add_freq_dict(path);
}
for (const auto& path : pitch_paths) {
query.add_pitch_dict(path);
}
Deinflector deinflector;
Lookup lookup(query, deinflector);
std::vector<double> durations;
durations.reserve(static_cast<size_t>(iterations) * words.size());
for (int i = 0; i < iterations; ++i) {
for (const auto& word : words) {
const auto start = std::chrono::high_resolution_clock::now();
const auto results = lookup.lookup(word);
const auto end = std::chrono::high_resolution_clock::now();
const std::chrono::duration<double, std::milli> elapsed = end - start;
durations.push_back(elapsed.count());
}
}
if (durations.empty()) {
return 1;
}
const auto [min, max] = std::ranges::minmax_element(durations);
const double total = std::accumulate(durations.begin(), durations.end(), 0.0);
const double average = total / durations.size();
std::cout << std::format("words: {} ({}) iterations: {}\n", csv_path, words.size(), iterations);
std::cout << std::format("total: {:.2f}ms\n", total);
std::cout << std::format("avg: {:.2f}ms\n", average);
std::cout << std::format("min: {:.2f}ms\n", *min);
std::cout << std::format("max: {:.2f}ms\n", *max);
return 0;
}
vendor/hachidori/third_party/hoshidicts/cli/main.c
Vendored
+232
View File
@@ -0,0 +1,232 @@
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#include "hoshidicts_c.h"
static void print_usage(const char* program) {
printf("Usage:\n");
printf("%s import <path/to/dictionary.zip|dictionary.mdx>\n", program);
printf("%s query <path/to/dictionary> <word>\n", program);
printf("%s lookup <path/to/dictionary> <lookup_string>\n", program);
printf("%s kanji <path/to/dictionary> <kanji>\n", program);
}
static size_t utf8_length(const char* text) {
size_t length = 0;
for (const char* c = text; *c != '\0'; c++) {
if ((*c & 0xC0) != 0x80) {
length++;
}
}
return length;
}
static int cmd_import(const char* path) {
char output_dir[256] = ".";
const char* parent = strrchr(path, '/');
if (parent) {
memcpy(output_dir, path, parent - path);
output_dir[parent - path] = '\0';
}
hd_import_result* ir = hd_import(path, output_dir, false);
if (ir == NULL) {
printf("failed to import dictionary\n");
return 1;
}
if (hd_import_result_success(ir)) {
printf("title: %s\n", hd_import_result_title(ir));
printf("term_count: %llu\n", hd_import_result_term_count(ir));
printf("meta_count: %llu\n", hd_import_result_meta_count(ir));
printf("freq_count: %llu\n", hd_import_result_freq_count(ir));
printf("pitch_count: %llu\n", hd_import_result_pitch_count(ir));
printf("kanji_count: %llu\n", hd_import_result_kanji_count(ir));
printf("media_count: %llu\n", hd_import_result_media_count(ir));
} else {
printf("could not import dictionary: %s\n", hd_import_result_error(ir));
}
hd_import_result_free(ir);
return 0;
}
static int cmd_query(const char* db_path, const char* expression) {
hd_query* q = hd_query_new();
if (hd_query_add_term_dict(q, db_path) != 0) {
printf("could not open dictionary: %s\n", db_path);
hd_query_free(q);
return 1;
}
const hd_term_result* terms = NULL;
size_t count = 0;
hd_results* r = hd_query_run(q, expression, &terms, &count);
if (r == NULL) {
printf("query failed\n");
hd_query_free(q);
return 1;
}
printf("query results for: %s length: %zu\n", expression, utf8_length(expression));
printf("%zu entries\n", count);
for (size_t i = 0; i < count; i++) {
printf("---------------------------------------------------------------\n");
printf("%.*s %.*s %.*s\n", (int)terms[i].expression.len, terms[i].expression.ptr, (int)terms[i].reading.len,
terms[i].reading.ptr, (int)terms[i].rules.len, terms[i].rules.ptr);
printf("%zu glossary entries\n", terms[i].glossaries_count);
for (size_t j = 0; j < terms[i].glossaries_count; j++) {
printf("------\n");
printf("%.*s\n", (int)terms[i].glossaries[j].dict_name.len, terms[i].glossaries[j].dict_name.ptr);
printf("%.*s\n", (int)terms[i].glossaries[j].glossary.len, terms[i].glossaries[j].glossary.ptr);
}
}
hd_results_free(r);
hd_query_free(q);
return 0;
}
static int cmd_lookup(const char* const* db_paths, int db_count, const char* lookup_string) {
const int max_results = 8;
const size_t scan_length = 16;
hd_query* q = hd_query_new();
for (int i = 0; i < db_count; i++) {
if (hd_query_add_term_dict(q, db_paths[i]) != 0) {
printf("could not open dictionary: %s\n", db_paths[i]);
hd_query_free(q);
return 1;
}
}
hd_deinflector* d = hd_deinflector_new();
hd_lookup* l = hd_lookup_new(q, d);
const hd_lookup_result* results = NULL;
size_t count = 0;
hd_lookup_results* r = hd_lookup_run(l, lookup_string, max_results, scan_length, &results, &count);
if (r == NULL) {
printf("lookup failed\n");
hd_lookup_free(l);
hd_deinflector_free(d);
hd_query_free(q);
return 1;
}
printf("lookup results for: %s max_results: %d scan_length: %zu\n", lookup_string, max_results, scan_length);
printf("%zu results\n", count);
for (size_t i = 0; i < count; i++) {
printf("---------------------------------------------------------------\n");
printf("%.*s\n", (int)results[i].matched.len, results[i].matched.ptr);
if (results[i].trace_count > 0) {
printf(" ");
for (size_t j = 0; j < results[i].trace_count; j++) {
printf("%.*s%s", (int)results[i].trace[j].name.len, results[i].trace[j].name.ptr,
j < results[i].trace_count - 1 ? " -> " : "");
}
printf("\n");
}
printf("%.*s %.*s\n", (int)results[i].term.expression.len, results[i].term.expression.ptr,
(int)results[i].term.reading.len, results[i].term.reading.ptr);
for (size_t j = 0; j < results[i].term.glossaries_count; j++) {
printf("------\n");
printf("%.*s\n", (int)results[i].term.glossaries[j].dict_name.len, results[i].term.glossaries[j].dict_name.ptr);
printf("%.*s\n", (int)results[i].term.glossaries[j].glossary.len, results[i].term.glossaries[j].glossary.ptr);
}
}
const hd_dictionary_style* styles = NULL;
size_t styles_count = 0;
hd_styles* s = hd_query_get_styles(q, &styles, &styles_count);
printf("styles: \n");
for (size_t i = 0; i < styles_count; i++) {
printf("%.*s\n", (int)styles[i].dict_name.len, styles[i].dict_name.ptr);
printf("%.*s\n", (int)styles[i].styles.len, styles[i].styles.ptr);
}
hd_styles_free(s);
hd_lookup_results_free(r);
hd_lookup_free(l);
hd_deinflector_free(d);
hd_query_free(q);
return 0;
}
static int cmd_kanji(const char* db_path, const char* kanji) {
hd_query* q = hd_query_new();
if (hd_query_add_kanji_dict(q, db_path) != 0) {
printf("could not open dictionary: %s\n", db_path);
hd_query_free(q);
return 1;
}
const hd_kanji_entry* entries = NULL;
size_t count = 0;
hd_kanji_results* r = hd_query_run_kanji(q, kanji, &entries, &count);
if (r == NULL) {
printf("kanji query failed\n");
hd_query_free(q);
return 1;
}
printf("kanji result for: %s\n", kanji);
printf("%zu entries\n", count);
for (size_t i = 0; i < count; i++) {
printf("---------------------------------------------------------------\n");
printf("dict: %.*s\n", (int)entries[i].dict_name.len, entries[i].dict_name.ptr);
printf("onyomi: %.*s\n", (int)entries[i].onyomi.len, entries[i].onyomi.ptr);
printf("kunyomi: %.*s\n", (int)entries[i].kunyomi.len, entries[i].kunyomi.ptr);
printf("tags: %.*s\n", (int)entries[i].tags.len, entries[i].tags.ptr);
printf("definitions:\n");
for (size_t j = 0; j < entries[i].definitions_count; j++) {
printf(" - %.*s\n", (int)entries[i].definitions[j].len, entries[i].definitions[j].ptr);
}
if (entries[i].stats_count > 0) {
printf("stats:\n");
for (size_t j = 0; j < entries[i].stats_count; j++) {
printf(" %.*s: %.*s\n", (int)entries[i].stats[j].key.len, entries[i].stats[j].key.ptr,
(int)entries[i].stats[j].value.len, entries[i].stats[j].value.ptr);
}
}
}
hd_kanji_results_free(r);
hd_query_free(q);
return 0;
}
int main(int argc, char* argv[]) {
if (argc < 2) {
print_usage(argv[0]);
return 1;
}
struct timespec t0;
struct timespec t1;
clock_gettime(CLOCK_MONOTONIC, &t0);
int ret;
const char* command = argv[1];
if (strcmp(command, "import") == 0 && argc >= 3) {
ret = cmd_import(argv[2]);
} else if (strcmp(command, "query") == 0 && argc >= 4) {
ret = cmd_query(argv[2], argv[3]);
} else if (strcmp(command, "lookup") == 0 && argc >= 4) {
ret = cmd_lookup((const char* const*)argv + 2, argc - 3, argv[argc - 1]);
} else if (strcmp(command, "kanji") == 0 && argc >= 4) {
ret = cmd_kanji(argv[2], argv[3]);
} else {
print_usage(argv[0]);
return 1;
}
clock_gettime(CLOCK_MONOTONIC, &t1);
double ms = (double)(t1.tv_sec - t0.tv_sec) * 1000.0 + (double)(t1.tv_nsec - t0.tv_nsec) / 1e6;
printf("runtime: %.2fms\n", ms);
return ret;
}
vendor/hachidori/third_party/hoshidicts/cli/main.cpp
Vendored
+222
View File
@@ -0,0 +1,222 @@
#include <utf8.h>
#include <chrono>
#include <filesystem>
#include <iostream>
#include <ranges>
#include <string>
#include "../src/path_utils.hpp"
#include "../src/text_processor/text_processor.hpp"
#include "hoshidicts/deinflector.hpp"
#include "hoshidicts/importer.hpp"
#include "hoshidicts/lookup.hpp"
#include "hoshidicts/query.hpp"
void print_usage(const char* program) {
std::cout << std::format("Usage:\n");
std::cout << std::format("{} import <path/to/dictionary.zip|dictionary.mdx>\n", program);
std::cout << std::format("{} deinflect <word>\n", program);
std::cout << std::format("{} preprocess <word>\n", program);
std::cout << std::format("{} query <path/to/dictionary> <word>\n", program);
std::cout << std::format("{} lookup <path/to/dictionary> <lookup_string>\n", program);
std::cout << std::format("{} freq <path/to/dictionary> <word>\n", program);
std::cout << std::format("{} kanji <path/to/dictionary> <kanji>\n", program);
}
void cmd_import(const std::string& path) {
std::filesystem::path source_path = path_utils::from_utf8(path);
std::string output_dir = path_utils::to_utf8(source_path.parent_path());
if (output_dir.empty()) {
output_dir = ".";
}
ImportResult result = dictionary_importer::import(path, output_dir);
if (result.success) {
std::cout << std::format("title: {}\n", result.title);
std::cout << std::format("term_count: {}\n", result.summary.counts.terms.total);
std::cout << std::format("meta_count: {}\n", result.summary.counts.termMeta["total"]);
std::cout << std::format("freq_count: {}\n", result.summary.counts.termMeta["freq"]);
std::cout << std::format("pitch_count: {}\n",
result.summary.counts.termMeta["pitch"] + result.summary.counts.termMeta["ipa"]);
std::cout << std::format("kanji_count: {}\n", result.summary.counts.kanji.total);
std::cout << std::format("media_count: {}\n", result.summary.counts.media.total);
} else {
std::cout << std::format("could not import dictionary: {}\n", result.error);
}
}
void cmd_deinflect(const std::string& inflected) {
Deinflector deinflector;
auto results = deinflector.deinflect(inflected);
std::cout << std::format("deinflections for: {} length: {}\n", inflected,
utf8::distance(inflected.begin(), inflected.end()));
std::cout << std::format("found {} candidates\n\n", results.size());
for (const auto& r : results) {
std::cout << std::format("{} (conditions: {})", r.text, r.conditions);
if (!r.trace.empty()) {
std::cout << std::format(" ");
for (size_t i = 0; i < r.trace.size(); ++i) {
std::cout << std::format("{}{}", r.trace[i].name, i < r.trace.size() - 1 ? " -> " : "");
}
std::cout << std::format("\n");
}
}
}
void cmd_preprocess(const std::string& text) {
auto results = text_processor::process(text);
std::cout << std::format("preproccesing for: {} length: {}\n", text, utf8::distance(text.begin(), text.end()));
std::cout << std::format("found {} variants\n", results.size());
for (const auto& r : results) {
std::cout << std::format("{}\n", r.text);
}
}
void cmd_query(const std::string& db_path, const std::string& expression) {
DictionaryQuery dict_query;
dict_query.add_term_dict(db_path);
auto result = dict_query.query(expression);
std::cout << std::format("query results for: {} length: {}\n", expression,
utf8::distance(expression.begin(), expression.end()));
std::cout << std::format("{} entries\n", result.size());
for (const auto& r : result) {
std::cout << std::format("---------------------------------------------------------------\n");
std::cout << std::format("{} {} {}\n", r.expression, r.reading, r.rules);
std::cout << std::format("{} glossary entries\n", r.glossaries.size());
for (const auto& g : r.glossaries) {
std::cout << std::format("------\n");
std::cout << std::format("{}\n", g.dict_name);
std::cout << std::format("{}\n", g.glossary);
}
}
}
void cmd_freq(const std::string& path, const std::string& expression, const std::string& reading) {
std::vector<TermResult> terms;
terms.emplace_back(TermResult{.expression = expression, .reading = reading});
DictionaryQuery query;
query.add_freq_dict(path);
query.query_freq(terms);
std::cout << std::format("frequency entries for: {}\n", expression);
int count = 0;
for (auto& freq : terms[0].frequencies) {
std::cout << std::format("dict: {}\n", freq.dict_name);
for (auto& freq_entry : freq.frequencies) {
std::cout << std::format("val: {} display_val: {}\n", freq_entry.value, freq_entry.display_value);
count++;
}
}
std::cout << std::format("count: {}\n", count);
}
void cmd_kanji(const std::string& path, const std::string& kanji) {
DictionaryQuery query;
query.add_kanji_dict(path);
auto result = query.query_kanji(kanji);
std::cout << std::format("kanji result for: {}\n", kanji);
std::cout << std::format("{} entries\n", result.entries.size());
for (const auto& e : result.entries) {
std::cout << std::format("---------------------------------------------------------------\n");
std::cout << std::format("dict: {}\n", e.dict_name);
std::cout << std::format("onyomi: {}\n", e.onyomi);
std::cout << std::format("kunyomi: {}\n", e.kunyomi);
std::cout << std::format("tags: {}\n", e.tags);
std::cout << std::format("definitions:\n");
for (const auto& def : e.definitions) {
std::cout << std::format(" - {}\n", def);
}
if (!e.stats.empty()) {
std::cout << std::format("stats:\n");
for (const auto& [k, v] : e.stats) {
std::cout << std::format(" {}: {}\n", k, v);
}
}
}
}
void cmd_lookup(const std::vector<std::string>& db_paths, const std::string& lookup_string, int max_results = 8,
int scan_length = 16) {
DictionaryQuery dict_query;
for (const auto& path : db_paths) {
dict_query.add_term_dict(path);
}
Deinflector deinflect;
Lookup lookup(dict_query, deinflect);
auto result = lookup.lookup(lookup_string, max_results, scan_length);
std::cout << std::format("lookup results for: {} max_results: {} scan_length: {}\n", lookup_string, max_results,
scan_length);
std::cout << std::format("{} results\n", result.size());
for (const auto& r : result) {
std::cout << std::format("---------------------------------------------------------------\n");
std::cout << std::format("{}\n", r.matched);
if (!r.trace.empty()) {
std::cout << std::format(" ");
for (size_t i = 0; i < r.trace.size(); ++i) {
std::cout << std::format("{}{}", r.trace[i].name, i < r.trace.size() - 1 ? " -> " : "");
}
std::cout << std::format("\n");
}
std::cout << std::format("{} {}\n", r.term.expression, r.term.reading);
for (const auto& g : r.term.glossaries) {
std::cout << std::format("------\n");
std::cout << std::format("{}\n", g.dict_name);
std::cout << std::format("{}\n", g.glossary);
}
}
std::cout << std::format("styles: \n");
for (const auto& s : dict_query.get_styles()) {
std::cout << std::format("{}\n", s.dict_name);
std::cout << std::format("{}\n", s.styles);
}
}
int main(int argc, char* argv[]) {
if (argc < 2) {
print_usage(argv[0]);
return 1;
}
const auto begin = std::chrono::steady_clock::now();
std::string_view command = argv[1];
if (command == "import" && argc >= 3) {
cmd_import(argv[2]);
} else if (command == "deinflect" && argc >= 3) {
cmd_deinflect(argv[2]);
} else if (command == "preprocess" && argc >= 3) {
cmd_preprocess(argv[2]);
} else if (command == "query" && argc >= 4) {
cmd_query(argv[2], argv[3]);
} else if (command == "lookup" && argc >= 4) {
auto db_paths = std::views::counted(argv + 2, argc - 3) |
std::views::transform([](const char* arg) { return std::string(arg); }) |
std::ranges::to<std::vector>();
std::string term = argv[argc - 1];
cmd_lookup(db_paths, term);
} else if (command == "freq" && argc >= 5) {
cmd_freq(argv[2], argv[3], argv[4]);
} else if (command == "kanji" && argc >= 4) {
cmd_kanji(argv[2], argv[3]);
} else {
print_usage(argv[0]);
return 1;
}
const auto end = std::chrono::steady_clock::now();
std::chrono::duration<double, std::milli> duration = end - begin;
std::cout << std::format("runtime: {}ms\n", duration.count());
return 0;
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/LICENSE
Vendored
+179
View File
@@ -0,0 +1,179 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/LICENSE-hashmap.c
Vendored
+24
View File
@@ -0,0 +1,24 @@
hashmap.c is MIT, Copyright (c) 2020 Joshua J Baker (https://github.com/tidwall/hashmap.c):
The MIT License (MIT)
Copyright (c) 2020 Joshua J Baker
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/README.md
Vendored
+17
View File
@@ -0,0 +1,17 @@
# gumbo-parser (vendored)
HTML5 parser used by the MDX importer to turn MDX glossary HTML into Yomitan
structured content.
Source: the `gumbo-parser/src` tree of Nokogiri
(https://github.com/sparklemotion/nokogiri, commit
da64104acfdc8b595e49935499c9cc357ebcac7a), itself a maintained fork of
google/gumbo-parser via lua-gumbo. Only the library sources (`src/*.c`,
`src/*.h`) are copied; Nokogiri's tests, fuzzers and the gperf/ragel inputs
are not. No file is modified. See UPSTREAM-README.md for the fork's history.
Licence: Apache-2.0 (LICENSE); `src/hashmap.c` is MIT (LICENSE-hashmap.c).
To update: copy `gumbo-parser/src/*.{c,h}` and `gumbo-parser/src/README.md`
(as UPSTREAM-README.md) from a newer Nokogiri checkout and bump the commit
above.
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/UPSTREAM-README.md
Vendored
+41
View File
@@ -0,0 +1,41 @@
libgumbo
========
This is an internal fork of the [libgumbo] library, which was copied and
later modified under the terms of the Apache 2.0 [license]. See `lua-gumbo`
commit [`0a04728`] for details of the original import.
Since importing the code, the following notable fixes and improvements
have been made:
* `91cef89`: Re-implement `adjust_foreign_attributes()` with a gperf hash
* `b11abe7`: Pass `TagSet` arrays into functions by reference instead of value
* `b73dc03`: Simplify `maybe_replace_codepoint()` function
* `d5d0bb3`: Remove special handling of `<menuitem>` tag
* `7bd5162`: Remove special handling of `<isindex>` tag
* `a5c1b0e`: Use `realloc(3)` instead of `malloc(3)` in `enlarge_vector_if_full()`
* `dcbebd7`: Use `realloc(3)` instead of `malloc(3)` in `maybe_resize_string_buffer()`
* `df15262`: Make `destroy_node()` function non-recursive
* `2df37f5`: Fix signedness of some format specifiers
* `176553e`: Add maximum element nesting limit
* `bed0f4a`: Annotate `gumbo_debug()` with `PRINTF` macro and fix warnings
* `7ffc218`: Annotate `print_message()` with `PRINTF` macro and fix warnings
* `1bd8ab5`, `9136507`, `53a1f9a`: Deduplicate some identical `TagSet` arrays
* `a7a9065`: Add some GCC/Clang function attributes
* `8d3d4e4`: Remove custom allocator support
* `8d3b006`: Fix recording of source positions for `</form>` end tags
* `1a8d763`: Replace linear search in `maybe_replace_codepoint()` with a lookup table
* `6dca79e`: Replace `strcasecmp()` and `strncasecmp()` with ascii-only equivalents
* `17ab1d2`: Fix `TAGSET_INCLUDES` macro to work properly with multiple bit flags
* `7e56d45`: Re-implement `gumbo_normalize_svg_tagname()` with a gperf hash
* `a518d35`: Replace linear array search in `adjust_svg_attributes()` with a gperf hash
* `a4a7433`: Fix duplicate `TagSet` initializer being ignored in `is_special_node()`
* `8137fcd`: Add support for `<dialog>` tag
* `4b35471`: Add missing `static` qualifiers to hide symbols that shouldn't be extern
* `df57c59`, `03101f3`, `ea62330`: Replace use of locale-dependant `ctype.h` functions
with custom, ASCII-only equivalents
[libgumbo]: https://github.com/google/gumbo-parser/tree/aa91b27b02c0c80c482e24348a457ed7c3c088e0/src
[license]: https://github.com/google/gumbo-parser/blob/aa91b27b02c0c80c482e24348a457ed7c3c088e0/COPYING
[`0a04728`]: https://gitlab.com/craigbarnes/lua-gumbo/commit/0a047282815af86f3367a7d95fefcfe5723ece48
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/ascii.c
Vendored
+75
View File
@@ -0,0 +1,75 @@
#include "ascii.h"
int gumbo_ascii_strcasecmp(const char *s1, const char *s2) {
int c1, c2;
while (*s1 && *s2) {
c1 = (int)(unsigned char) gumbo_ascii_tolower(*s1);
c2 = (int)(unsigned char) gumbo_ascii_tolower(*s2);
if (c1 != c2) {
return (c1 - c2);
}
s1++;
s2++;
}
return (((int)(unsigned char) *s1) - ((int)(unsigned char) *s2));
}
int gumbo_ascii_strncasecmp(const char *s1, const char *s2, size_t n) {
int c1, c2;
while (n && *s1 && *s2) {
n -= 1;
c1 = (int)(unsigned char) gumbo_ascii_tolower(*s1);
c2 = (int)(unsigned char) gumbo_ascii_tolower(*s2);
if (c1 != c2) {
return (c1 - c2);
}
s1++;
s2++;
}
if (n) {
return (((int)(unsigned char) *s1) - ((int)(unsigned char) *s2));
}
return 0;
}
const unsigned char _gumbo_ascii_table[0x80] = {
0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x03,0x03,0x01,0x03,0x03,0x01,0x01,
0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x00,
0x00,0x28,0x28,0x28,0x28,0x28,0x28,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,
0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x00,0x00,0x00,0x00,0x00,
0x00,0x50,0x50,0x50,0x50,0x50,0x50,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,
0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x00,0x00,0x00,0x00,0x00,
};
// Table generation code.
// clang -DGUMBO_GEN_TABLE ascii.c && ./a.out && rm a.out
#ifdef GUMBO_GEN_TABLE
#include <stdio.h>
int main() {
printf("const unsigned char _gumbo_ascii_table[0x80] = {");
for (int c = 0; c < 0x80; ++c) {
unsigned int x = 0;
// https://infra.spec.whatwg.org/#ascii-code-point
if (c <= 0x1f)
x |= GUMBO_ASCII_CNTRL;
if (c == 0x09 || c == 0x0a || c == 0x0c || c == 0x0d || c == 0x20)
x |= GUMBO_ASCII_SPACE;
if (c >= 0x30 && c <= 0x39)
x |= GUMBO_ASCII_DIGIT;
if ((c >= 0x30 && c <= 0x39) || (c >= 0x41 && c <= 0x46))
x |= GUMBO_ASCII_UPPER_XDIGIT;
if ((c >= 0x30 && c <= 0x39) || (c >= 0x61 && c <= 0x66))
x |= GUMBO_ASCII_LOWER_XDIGIT;
if (c >= 0x41 && c <= 0x5a)
x |= GUMBO_ASCII_UPPER_ALPHA;
if (c >= 0x61 && c <= 0x7a)
x |= GUMBO_ASCII_LOWER_ALPHA;
printf("%s0x%02x,", (c % 16 == 0? "\n " : ""), x);
}
printf("\n};\n");
return 0;
}
#endif
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/ascii.h
Vendored
+115
View File
@@ -0,0 +1,115 @@
#ifndef GUMBO_ASCII_H_
#define GUMBO_ASCII_H_
#include <stddef.h>
#include "macros.h"
#ifdef __cplusplus
extern "C" {
#endif
PURE NONNULL_ARGS
int gumbo_ascii_strcasecmp(const char *s1, const char *s2);
PURE NONNULL_ARGS
int gumbo_ascii_strncasecmp(const char *s1, const char *s2, size_t n);
// If these values change, then _gumbo_ascii_table needs to be regenerated.
#define GUMBO_ASCII_CNTRL 1
#define GUMBO_ASCII_SPACE 2
#define GUMBO_ASCII_DIGIT 4
#define GUMBO_ASCII_UPPER_XDIGIT 8
#define GUMBO_ASCII_LOWER_XDIGIT 16
#define GUMBO_ASCII_UPPER_ALPHA 32
#define GUMBO_ASCII_LOWER_ALPHA 64
#define GUMBO_ASCII_XDIGIT (GUMBO_ASCII_LOWER_XDIGIT | GUMBO_ASCII_UPPER_XDIGIT)
#define GUMBO_ASCII_ALPHA (GUMBO_ASCII_UPPER_ALPHA | GUMBO_ASCII_LOWER_ALPHA)
#define GUMBO_ASCII_ALNUM (GUMBO_ASCII_DIGIT | GUMBO_ASCII_ALPHA)
extern const unsigned char _gumbo_ascii_table[0x80];
CONST_FN
static inline int gumbo_ascii_isascii(int c) {
return ((unsigned int)c & ~0x7fu) == 0;
}
// 0x00 -- 0x1F (A C0 control)
CONST_FN
static inline int gumbo_ascii_iscntrl(int c) {
return gumbo_ascii_isascii(c)
&& (_gumbo_ascii_table[c] & GUMBO_ASCII_CNTRL);
}
// 0x09, 0x0a, 0x0c, 0x0d, 0x20
CONST_FN
static inline int gumbo_ascii_isspace(int c) {
return gumbo_ascii_isascii(c)
&& (_gumbo_ascii_table[c] & GUMBO_ASCII_SPACE);
}
CONST_FN
static inline int gumbo_ascii_istab_or_newline(int c) {
return c == 0x09 || c == 0x0a || c == 0x0d;
}
CONST_FN
static inline int gumbo_ascii_isdigit(int c) {
return c >= 0x30 && c <= 0x39;
}
CONST_FN
static inline int gumbo_ascii_isalpha(int c) {
return gumbo_ascii_isascii(c)
&& (_gumbo_ascii_table[c] & GUMBO_ASCII_ALPHA);
}
CONST_FN
static inline int gumbo_ascii_isxdigit(int c) {
return gumbo_ascii_isascii(c)
&& (_gumbo_ascii_table[c] & GUMBO_ASCII_XDIGIT);
}
CONST_FN
static inline int gumbo_ascii_isupper_xdigit(int c) {
return gumbo_ascii_isascii(c)
&& (_gumbo_ascii_table[c] & GUMBO_ASCII_UPPER_XDIGIT);
}
CONST_FN
static inline int gumbo_ascii_islower_xdigit(int c) {
return gumbo_ascii_isascii(c)
&& (_gumbo_ascii_table[c] & GUMBO_ASCII_LOWER_XDIGIT);
}
CONST_FN
static inline int gumbo_ascii_isupper(int c) {
return ((unsigned)(c) - 'A') < 26;
}
CONST_FN
static inline int gumbo_ascii_islower(int c) {
return gumbo_ascii_isascii(c)
&& (_gumbo_ascii_table[c] & GUMBO_ASCII_LOWER_ALPHA);
}
CONST_FN
static inline int gumbo_ascii_isalnum(int c) {
return gumbo_ascii_isascii(c)
&& (_gumbo_ascii_table[c] & GUMBO_ASCII_ALNUM);
}
CONST_FN
static inline int gumbo_ascii_tolower(int c) {
if (gumbo_ascii_isupper(c)) {
return c | 32;
}
return c;
}
#ifdef __cplusplus
}
#endif
#endif // GUMBO_ASCII_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/attribute.c
Vendored
+42
View File
@@ -0,0 +1,42 @@
/*
Copyright 2018 Craig Barnes.
Copyright 2010 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include "attribute.h"
#include "ascii.h"
#include "util.h"
GumboAttribute* gumbo_get_attribute (
const GumboVector* attributes,
const char* name
) {
for (unsigned int i = 0; i < attributes->length; ++i) {
GumboAttribute* attr = attributes->data[i];
if (!gumbo_ascii_strcasecmp(attr->name, name)) {
return attr;
}
}
return NULL;
}
void gumbo_destroy_attribute(GumboAttribute* attribute) {
gumbo_free((void*) attribute->name);
gumbo_free((void*) attribute->value);
gumbo_free((void*) attribute);
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/attribute.h
Vendored
+17
View File
@@ -0,0 +1,17 @@
#ifndef GUMBO_ATTRIBUTE_H_
#define GUMBO_ATTRIBUTE_H_
#include "nokogiri_gumbo.h"
#ifdef __cplusplus
extern "C" {
#endif
// Release the memory used for a GumboAttribute, including the attribute itself
void gumbo_destroy_attribute(GumboAttribute* attribute);
#ifdef __cplusplus
}
#endif
#endif // GUMBO_ATTRIBUTE_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/char_ref.c
Vendored
+22225
View File
File diff suppressed because it is too large Load Diff
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/char_ref.h
Vendored
+29
View File
@@ -0,0 +1,29 @@
#ifndef GUMBO_CHAR_REF_H_
#define GUMBO_CHAR_REF_H_
#include <stdlib.h>
#ifdef __cplusplus
extern "C" {
#endif
// Value that indicates no character was produced.
#define kGumboNoChar (-1)
// On input, str points to the start of the string to match and size is the
// size of the string.
//
// Returns the length of the match or 0 if there is no match.
// output[0] contains the first codepoint and output[1] contains the second if
// there are two, otherwise output[1] contains kGumboNoChar.
size_t match_named_char_ref (
const char *str,
size_t size,
int output[2]
);
#ifdef __cplusplus
}
#endif
#endif // GUMBO_CHAR_REF_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/error.c
Vendored
+658
View File
@@ -0,0 +1,658 @@
/*
Copyright 2010 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include <assert.h>
#include <inttypes.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#include "ascii.h"
#include "error.h"
#include "nokogiri_gumbo.h"
#include "macros.h"
#include "parser.h"
#include "string_buffer.h"
#include "util.h"
#include "vector.h"
// Prints a formatted message to a StringBuffer. This automatically resizes the
// StringBuffer as necessary to fit the message. Returns the number of bytes
// written.
static int PRINTF(2) print_message (
GumboStringBuffer* output,
const char* format,
...
) {
va_list args;
int remaining_capacity = output->capacity - output->length;
va_start(args, format);
int bytes_written = vsnprintf (
output->data + output->length,
remaining_capacity,
format,
args
);
va_end(args);
#if (defined(_MSC_VER) && (_MSC_VER < 1900)) || defined(_RUBY_MSVCRT)
if (bytes_written == -1) {
// vsnprintf returns -1 on older MSVC++ if there's not enough capacity,
// instead of returning the number of bytes that would've been written had
// there been enough. In this case, we can call vsnprintf() again but
// with a count of 0 to get the number of bytes written, not including
// the null terminator.
// https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/vsnprintf-vsnprintf-vsnprintf-l-vsnwprintf-vsnwprintf-l?view=msvc-140#behavior-summary
va_start(args, format);
bytes_written = vsnprintf (
NULL,
0,
format,
args
);
va_end(args);
}
#endif
// -1 in standard C99 indicates an encoding error. Return 0 and do nothing.
if (bytes_written == -1) {
return 0;
}
if (bytes_written >= remaining_capacity) {
// At least double the size of the buffer.
size_t new_capacity = output->capacity * 2;
if (new_capacity < output->length + bytes_written + 1) {
// The +1 is for the null terminator.
new_capacity = output->length + bytes_written + 1;
}
gumbo_string_buffer_reserve(new_capacity, output);
remaining_capacity = output->capacity - output->length;
va_start(args, format);
bytes_written = vsnprintf (
output->data + output->length,
remaining_capacity,
format,
args
);
va_end(args);
}
output->length += bytes_written;
return bytes_written;
}
static void print_tag_stack (
const GumboParserError* error,
GumboStringBuffer* output
) {
print_message(output, " Currently open tags: ");
for (unsigned int i = 0; i < error->tag_stack.length; ++i) {
if (i) {
print_message(output, ", ");
}
uintptr_t tag = (uintptr_t) error->tag_stack.data[i];
const char* tag_name;
if (tag > GUMBO_TAG_UNKNOWN) {
tag_name = error->tag_stack.data[i];
} else {
tag_name = gumbo_normalized_tagname((GumboTag)tag);
}
print_message(output, "%s", tag_name);
}
gumbo_string_buffer_append_codepoint('.', output);
}
static void handle_tokenizer_error (
const GumboError* error,
GumboStringBuffer* output
) {
switch (error->type) {
case GUMBO_ERR_ABRUPT_CLOSING_OF_EMPTY_COMMENT:
print_message(output, "Empty comment abruptly closed by '%s', use '-->'.",
error->v.tokenizer.state == GUMBO_LEX_COMMENT_START? ">" : "->");
break;
case GUMBO_ERR_ABRUPT_DOCTYPE_PUBLIC_IDENTIFIER:
print_message (
output,
"DOCTYPE public identifier missing closing %s.",
error->v.tokenizer.state == GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED?
"quotation mark (\")" : "apostrophe (')"
);
break;
case GUMBO_ERR_ABRUPT_DOCTYPE_SYSTEM_IDENTIFIER:
print_message (
output,
"DOCTYPE system identifier missing closing %s.",
error->v.tokenizer.state == GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED?
"quotation mark (\")" : "apostrophe (')"
);
break;
case GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE:
print_message (
output,
"Numeric character reference '%.*s' does not contain any %sdigits.",
(int)error->original_text.length, error->original_text.data,
error->v.tokenizer.state == GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE_START? "hexadecimal " : ""
);
break;
case GUMBO_ERR_CDATA_IN_HTML_CONTENT:
print_message(output, "CDATA section outside foreign (SVG or MathML) content.");
break;
case GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE:
print_message (
output,
"Numeric character reference '%.*s' references a code point that is outside the valid Unicode range.",
(int)error->original_text.length, error->original_text.data
);
break;
case GUMBO_ERR_CONTROL_CHARACTER_IN_INPUT_STREAM:
print_message (
output,
"Input contains prohibited control code point U+%04X.",
error->v.tokenizer.codepoint
);
break;
case GUMBO_ERR_CONTROL_CHARACTER_REFERENCE:
print_message (
output,
"Numeric character reference '%.*s' references prohibited control code point U+%04X.",
(int)error->original_text.length, error->original_text.data,
error->v.tokenizer.codepoint
);
break;
case GUMBO_ERR_END_TAG_WITH_ATTRIBUTES:
print_message(output, "End tag contains attributes.");
break;
case GUMBO_ERR_DUPLICATE_ATTRIBUTE:
print_message(output, "Tag contains multiple attributes with the same name.");
break;
case GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS:
print_message(output, "End tag ends with '/>', use '>'.");
break;
case GUMBO_ERR_EOF_BEFORE_TAG_NAME:
print_message(output, "End of input where a tag name is expected.");
break;
case GUMBO_ERR_EOF_IN_CDATA:
print_message(output, "End of input in CDATA section.");
break;
case GUMBO_ERR_EOF_IN_COMMENT:
print_message(output, "End of input in comment.");
break;
case GUMBO_ERR_EOF_IN_DOCTYPE:
print_message(output, "End of input in DOCTYPE.");
break;
case GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT:
print_message(output, "End of input in text that resembles an HTML comment inside script element content.");
break;
case GUMBO_ERR_EOF_IN_TAG:
print_message(output, "End of input in tag.");
break;
case GUMBO_ERR_INCORRECTLY_CLOSED_COMMENT:
print_message(output, "Comment closed incorrectly by '--!>', use '-->'.");
break;
case GUMBO_ERR_INCORRECTLY_OPENED_COMMENT:
print_message(output, "Comment, DOCTYPE, or CDATA opened incorrectly, use '<!--', '<!DOCTYPE', or '<![CDATA['.");
break;
case GUMBO_ERR_INVALID_CHARACTER_SEQUENCE_AFTER_DOCTYPE_NAME:
print_message(output, "Invalid character sequence after DOCTYPE name, expected 'PUBLIC', 'SYSTEM', or '>'.");
break;
case GUMBO_ERR_INVALID_FIRST_CHARACTER_OF_TAG_NAME:
if (gumbo_ascii_isascii(error->v.tokenizer.codepoint)
&& !gumbo_ascii_iscntrl(error->v.tokenizer.codepoint))
print_message(output, "Invalid first character of tag name '%c'.", error->v.tokenizer.codepoint);
else
print_message(output, "Invalid first code point of tag name U+%04X.", error->v.tokenizer.codepoint);
break;
case GUMBO_ERR_MISSING_ATTRIBUTE_VALUE:
print_message(output, "Missing attribute value.");
break;
case GUMBO_ERR_MISSING_DOCTYPE_NAME:
print_message(output, "Missing DOCTYPE name.");
break;
case GUMBO_ERR_MISSING_DOCTYPE_PUBLIC_IDENTIFIER:
print_message(output, "Missing DOCTYPE public identifier.");
break;
case GUMBO_ERR_MISSING_DOCTYPE_SYSTEM_IDENTIFIER:
print_message(output, "Missing DOCTYPE system identifier.");
break;
case GUMBO_ERR_MISSING_END_TAG_NAME:
print_message(output, "Missing end tag name.");
break;
case GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_PUBLIC_IDENTIFIER:
print_message(output, "Missing quote before DOCTYPE public identifier.");
break;
case GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER:
print_message(output, "Missing quote before DOCTYPE system identifier.");
break;
case GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE:
print_message(output, "Missing semicolon after character reference '%.*s'.",
(int)error->original_text.length, error->original_text.data);
break;
case GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_PUBLIC_KEYWORD:
print_message(output, "Missing whitespace after 'PUBLIC' keyword.");
break;
case GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_SYSTEM_KEYWORD:
print_message(output, "Missing whitespace after 'SYSTEM' keyword.");
break;
case GUMBO_ERR_MISSING_WHITESPACE_BEFORE_DOCTYPE_NAME:
print_message(output, "Missing whitespace between 'DOCTYPE' keyword and DOCTYPE name.");
break;
case GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_ATTRIBUTES:
print_message(output, "Missing whitespace between attributes.");
break;
case GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS:
print_message(output, "Missing whitespace between DOCTYPE public and system identifiers.");
break;
case GUMBO_ERR_NESTED_COMMENT:
print_message(output, "Nested comment.");
break;
case GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE:
print_message (
output,
"Numeric character reference '%.*s' references noncharacter U+%04X.",
(int)error->original_text.length, error->original_text.data,
error->v.tokenizer.codepoint
);
break;
case GUMBO_ERR_NONCHARACTER_IN_INPUT_STREAM:
print_message(output, "Input contains noncharacter U+%04X.", error->v.tokenizer.codepoint);
break;
case GUMBO_ERR_NON_VOID_HTML_ELEMENT_START_TAG_WITH_TRAILING_SOLIDUS:
print_message(output, "Start tag of nonvoid HTML element ends with '/>', use '>'.");
break;
case GUMBO_ERR_NULL_CHARACTER_REFERENCE:
print_message(output, "Numeric character reference '%.*s' references U+0000.",
(int)error->original_text.length, error->original_text.data);
break;
case GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE:
print_message (
output,
"Numeric character reference '%.*s' references surrogate U+%4X.",
(int)error->original_text.length, error->original_text.data,
error->v.tokenizer.codepoint
);
break;
case GUMBO_ERR_SURROGATE_IN_INPUT_STREAM:
print_message(output, "Input contains surrogate U+%04X.", error->v.tokenizer.codepoint);
break;
case GUMBO_ERR_UNEXPECTED_CHARACTER_AFTER_DOCTYPE_SYSTEM_IDENTIFIER:
print_message(output, "Unexpected character after DOCTYPE system identifier.");
break;
case GUMBO_ERR_UNEXPECTED_CHARACTER_IN_ATTRIBUTE_NAME:
print_message(output, "Unexpected character (%c) in attribute name.", error->v.tokenizer.codepoint);
break;
case GUMBO_ERR_UNEXPECTED_CHARACTER_IN_UNQUOTED_ATTRIBUTE_VALUE:
print_message(output, "Unexpected character (%c) in unquoted attribute value.", error->v.tokenizer.codepoint);
break;
case GUMBO_ERR_UNEXPECTED_EQUALS_SIGN_BEFORE_ATTRIBUTE_NAME:
print_message(output, "Unexpected '=' before an attribute name.");
break;
case GUMBO_ERR_UNEXPECTED_NULL_CHARACTER:
print_message(output, "Input contains unexpected U+0000.");
break;
case GUMBO_ERR_UNEXPECTED_QUESTION_MARK_INSTEAD_OF_TAG_NAME:
print_message(output, "Unexpected '?' where start tag name is expected.");
break;
case GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG:
print_message(output, "Unexpected '/' in tag.");
break;
case GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE:
print_message(output, "Unknown named character reference '%.*s'.",
(int)error->original_text.length, error->original_text.data);
break;
case GUMBO_ERR_UTF8_INVALID:
print_message(output, "Invalid UTF8 encoding.");
break;
case GUMBO_ERR_UTF8_TRUNCATED:
print_message(output, "UTF8 character truncated.");
break;
case GUMBO_ERR_PARSER:
assert(0 && "Unreachable.");
}
}
static void handle_parser_error (
const GumboParserError* error,
GumboStringBuffer* output
) {
if (
error->parser_state == GUMBO_INSERTION_MODE_INITIAL
&& error->input_type != GUMBO_TOKEN_DOCTYPE
) {
print_message (
output,
"Expected a doctype token"
);
return;
}
switch (error->input_type) {
case GUMBO_TOKEN_DOCTYPE:
print_message(output, "This is not a legal doctype");
return;
case GUMBO_TOKEN_COMMENT:
// Should never happen; comments are always legal.
assert(0);
// But just in case...
print_message(output, "Comments aren't legal here");
return;
case GUMBO_TOKEN_CDATA:
case GUMBO_TOKEN_WHITESPACE:
case GUMBO_TOKEN_CHARACTER:
print_message(output, "Character tokens aren't legal here");
return;
case GUMBO_TOKEN_NULL:
print_message(output, "Null bytes are not allowed in HTML5");
return;
case GUMBO_TOKEN_EOF:
if (error->parser_state == GUMBO_INSERTION_MODE_INITIAL) {
print_message(output, "You must provide a doctype");
} else {
print_message(output, "Premature end of file.");
print_tag_stack(error, output);
}
return;
case GUMBO_TOKEN_START_TAG:
case GUMBO_TOKEN_END_TAG:
{
const char* tag_name;
const char* which = error->input_type == GUMBO_TOKEN_START_TAG ? "Start" : "End";
if (error->input_name) {
tag_name = error->input_name;
} else {
tag_name = gumbo_normalized_tagname(error->input_tag);
}
print_message(output, "%s tag '%s' isn't allowed here.", which, tag_name);
print_tag_stack(error, output);
return;
}
}
}
// Finds the preceding newline in an original source buffer from a given byte
// location. Returns a character pointer to the character after that, or a
// pointer to the beginning of the string if this is the first line.
static const char* find_prev_newline (
const char* source_text,
size_t source_length,
const char* error_location
) {
const char* source_end = source_text + source_length;
assert(error_location >= source_text);
assert(error_location <= source_end);
const char* c = error_location;
if (c != source_text && (error_location == source_end || *c == '\n'))
--c;
while (c != source_text && *c != '\n')
--c;
return c == source_text ? c : c + 1;
}
// Finds the next newline in the original source buffer from a given byte
// location. Returns a character pointer to that newline, or a pointer to
// source_text + source_length if this is the last line.
static const char* find_next_newline(
const char* source_text,
size_t source_length,
const char* error_location
) {
const char* source_end = source_text + source_length;
assert(error_location >= source_text);
assert(error_location <= source_end);
const char* c = error_location;
while (c != source_end && *c != '\n')
++c;
return c;
}
GumboError* gumbo_add_error(GumboParser* parser) {
parser->_output->document_error = true;
int max_errors = parser->_options->max_errors;
if (max_errors >= 0 && parser->_output->errors.length >= (unsigned int) max_errors) {
return NULL;
}
GumboError* error = gumbo_alloc(sizeof(GumboError));
gumbo_vector_add(error, &parser->_output->errors);
return error;
}
GumboSourcePosition gumbo_error_position(const GumboError* error) {
return error->position;
}
const char* gumbo_error_code(const GumboError* error) {
switch (error->type) {
// Defined tokenizer errors.
case GUMBO_ERR_ABRUPT_CLOSING_OF_EMPTY_COMMENT:
return "abrupt-closing-of-empty-comment";
case GUMBO_ERR_ABRUPT_DOCTYPE_PUBLIC_IDENTIFIER:
return "abrupt-doctype-public-identifier";
case GUMBO_ERR_ABRUPT_DOCTYPE_SYSTEM_IDENTIFIER:
return "abrupt-doctype-system-identifier";
case GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE:
return "absence-of-digits-in-numeric-character-reference";
case GUMBO_ERR_CDATA_IN_HTML_CONTENT:
return "cdata-in-html-content";
case GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE:
return "character-reference-outside-unicode-range";
case GUMBO_ERR_CONTROL_CHARACTER_IN_INPUT_STREAM:
return "control-character-in-input-stream";
case GUMBO_ERR_CONTROL_CHARACTER_REFERENCE:
return "control-character-reference";
case GUMBO_ERR_END_TAG_WITH_ATTRIBUTES:
return "end-tag-with-attributes";
case GUMBO_ERR_DUPLICATE_ATTRIBUTE:
return "duplicate-attribute";
case GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS:
return "end-tag-with-trailing-solidus";
case GUMBO_ERR_EOF_BEFORE_TAG_NAME:
return "eof-before-tag-name";
case GUMBO_ERR_EOF_IN_CDATA:
return "eof-in-cdata";
case GUMBO_ERR_EOF_IN_COMMENT:
return "eof-in-comment";
case GUMBO_ERR_EOF_IN_DOCTYPE:
return "eof-in-doctype";
case GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT:
return "eof-in-script-html-comment-like-text";
case GUMBO_ERR_EOF_IN_TAG:
return "eof-in-tag";
case GUMBO_ERR_INCORRECTLY_CLOSED_COMMENT:
return "incorrectly-closed-comment";
case GUMBO_ERR_INCORRECTLY_OPENED_COMMENT:
return "incorrectly-opened-comment";
case GUMBO_ERR_INVALID_CHARACTER_SEQUENCE_AFTER_DOCTYPE_NAME:
return "invalid-character-sequence-after-doctype-name";
case GUMBO_ERR_INVALID_FIRST_CHARACTER_OF_TAG_NAME:
return "invalid-first-character-of-tag-name";
case GUMBO_ERR_MISSING_ATTRIBUTE_VALUE:
return "missing-attribute-value";
case GUMBO_ERR_MISSING_DOCTYPE_NAME:
return "missing-doctype-name";
case GUMBO_ERR_MISSING_DOCTYPE_PUBLIC_IDENTIFIER:
return "missing-doctype-public-identifier";
case GUMBO_ERR_MISSING_DOCTYPE_SYSTEM_IDENTIFIER:
return "missing-doctype-system-identifier";
case GUMBO_ERR_MISSING_END_TAG_NAME:
return "missing-end-tag-name";
case GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_PUBLIC_IDENTIFIER:
return "missing-quote-before-doctype-public-identifier";
case GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER:
return "missing-quote-before-doctype-system-identifier";
case GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE:
return "missing-semicolon-after-character-reference";
case GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_PUBLIC_KEYWORD:
return "missing-whitespace-after-doctype-public-keyword";
case GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_SYSTEM_KEYWORD:
return "missing-whitespace-after-doctype-system-keyword";
case GUMBO_ERR_MISSING_WHITESPACE_BEFORE_DOCTYPE_NAME:
return "missing-whitespace-before-doctype-name";
case GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_ATTRIBUTES:
return "missing-whitespace-between-attributes";
case GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS:
return "missing-whitespace-between-doctype-public-and-system-identifiers";
case GUMBO_ERR_NESTED_COMMENT:
return "nested-comment";
case GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE:
return "noncharacter-character-reference";
case GUMBO_ERR_NONCHARACTER_IN_INPUT_STREAM:
return "noncharacter-in-input-stream";
case GUMBO_ERR_NON_VOID_HTML_ELEMENT_START_TAG_WITH_TRAILING_SOLIDUS:
return "non-void-html-element-start-tag-with-trailing-solidus";
case GUMBO_ERR_NULL_CHARACTER_REFERENCE:
return "null-character-reference";
case GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE:
return "surrogate-character-reference";
case GUMBO_ERR_SURROGATE_IN_INPUT_STREAM:
return "surrogate-in-input-stream";
case GUMBO_ERR_UNEXPECTED_CHARACTER_AFTER_DOCTYPE_SYSTEM_IDENTIFIER:
return "unexpected-character-after-doctype-system-identifier";
case GUMBO_ERR_UNEXPECTED_CHARACTER_IN_ATTRIBUTE_NAME:
return "unexpected-character-in-attribute-name";
case GUMBO_ERR_UNEXPECTED_CHARACTER_IN_UNQUOTED_ATTRIBUTE_VALUE:
return "unexpected-character-in-unquoted-attribute-value";
case GUMBO_ERR_UNEXPECTED_EQUALS_SIGN_BEFORE_ATTRIBUTE_NAME:
return "unexpected-equals-sign-before-attribute-name";
case GUMBO_ERR_UNEXPECTED_NULL_CHARACTER:
return "unexpected-null-character";
case GUMBO_ERR_UNEXPECTED_QUESTION_MARK_INSTEAD_OF_TAG_NAME:
return "unexpected-question-mark-instead-of-tag-name";
case GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG:
return "unexpected-solidus-in-tag";
case GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE:
return "unknown-named-character-reference";
// Encoding errors.
case GUMBO_ERR_UTF8_INVALID:
return "utf8-invalid";
case GUMBO_ERR_UTF8_TRUNCATED:
return "utf8-truncated";
// Generic parser error.
case GUMBO_ERR_PARSER:
return "generic-parser";
}
// Silence warning about control reaching end of non-void function.
// All errors _should_ be handled in the switch statement.
return "generic-parser";
}
static void error_to_string (
const GumboError* error,
GumboStringBuffer* output
) {
if (error->type < GUMBO_ERR_PARSER)
handle_tokenizer_error(error, output);
else
handle_parser_error(&error->v.parser, output);
}
size_t gumbo_error_to_string(const GumboError* error, char** output) {
GumboStringBuffer sb;
gumbo_string_buffer_init(&sb);
error_to_string(error, &sb);
*output = sb.data;
return sb.length;
}
void caret_diagnostic_to_string (
const GumboError* error,
const char* source_text,
size_t source_length,
GumboStringBuffer* output
) {
error_to_string(error, output);
const char* error_text = error->original_text.data;
const char* line_start = find_prev_newline(source_text, source_length, error_text);
const char* line_end = find_next_newline(source_text, source_length, error_text);
GumboStringPiece original_line;
original_line.data = line_start;
original_line.length = line_end - line_start;
gumbo_string_buffer_append_codepoint('\n', output);
gumbo_string_buffer_append_string(&original_line, output);
gumbo_string_buffer_append_codepoint('\n', output);
gumbo_string_buffer_reserve(output->length + error->position.column, output);
if (error->position.column >= 2) {
size_t num_spaces = error->position.column - 1;
memset(output->data + output->length, ' ', num_spaces);
output->length += num_spaces;
}
gumbo_string_buffer_append_codepoint('^', output);
gumbo_string_buffer_append_codepoint('\n', output);
}
size_t gumbo_caret_diagnostic_to_string (
const GumboError* error,
const char* source_text,
size_t source_length,
char **output
) {
GumboStringBuffer sb;
gumbo_string_buffer_init(&sb);
caret_diagnostic_to_string(error, source_text, source_length, &sb);
*output = sb.data;
return sb.length;
}
void gumbo_print_caret_diagnostic (
const GumboError* error,
const char* source_text,
size_t source_length
) {
GumboStringBuffer text;
gumbo_string_buffer_init(&text);
print_message (
&text,
"%lu:%lu: ",
(unsigned long)error->position.line,
(unsigned long)error->position.column
);
caret_diagnostic_to_string(error, source_text, source_length, &text);
printf("%.*s", (int) text.length, text.data);
gumbo_string_buffer_destroy(&text);
}
void gumbo_error_destroy(GumboError* error) {
if (error->type == GUMBO_ERR_PARSER) {
// Free the tag name.
if (error->v.parser.input_name) {
gumbo_free(error->v.parser.input_name);
}
for (unsigned int i = 0; i < error->v.parser.tag_stack.length; ++i) {
intptr_t tag = (intptr_t) error->v.parser.tag_stack.data[i];
if (tag > GUMBO_TAG_UNKNOWN) {
gumbo_free(error->v.parser.tag_stack.data[i]);
}
}
gumbo_vector_destroy(&error->v.parser.tag_stack);
}
gumbo_free(error);
}
void gumbo_init_errors(GumboParser* parser) {
gumbo_vector_init(5, &parser->_output->errors);
}
void gumbo_destroy_errors(GumboParser* parser) {
for (unsigned int i = 0; i < parser->_output->errors.length; ++i) {
gumbo_error_destroy(parser->_output->errors.data[i]);
}
gumbo_vector_destroy(&parser->_output->errors);
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/error.h
Vendored
+152
View File
@@ -0,0 +1,152 @@
#ifndef GUMBO_ERROR_H_
#define GUMBO_ERROR_H_
#include <stdint.h>
#include "nokogiri_gumbo.h"
#include "insertion_mode.h"
#include "string_buffer.h"
#include "token_type.h"
#include "tokenizer_states.h"
#ifdef __cplusplus
extern "C" {
#endif
struct GumboInternalParser;
typedef enum {
// Defined errors.
// https://html.spec.whatwg.org/multipage/parsing.html#parse-errors
GUMBO_ERR_ABRUPT_CLOSING_OF_EMPTY_COMMENT,
GUMBO_ERR_ABRUPT_DOCTYPE_PUBLIC_IDENTIFIER,
GUMBO_ERR_ABRUPT_DOCTYPE_SYSTEM_IDENTIFIER,
GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE,
GUMBO_ERR_CDATA_IN_HTML_CONTENT,
GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE,
GUMBO_ERR_CONTROL_CHARACTER_IN_INPUT_STREAM,
GUMBO_ERR_CONTROL_CHARACTER_REFERENCE,
GUMBO_ERR_END_TAG_WITH_ATTRIBUTES,
GUMBO_ERR_DUPLICATE_ATTRIBUTE,
GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS,
GUMBO_ERR_EOF_BEFORE_TAG_NAME,
GUMBO_ERR_EOF_IN_CDATA,
GUMBO_ERR_EOF_IN_COMMENT,
GUMBO_ERR_EOF_IN_DOCTYPE,
GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT,
GUMBO_ERR_EOF_IN_TAG,
GUMBO_ERR_INCORRECTLY_CLOSED_COMMENT,
GUMBO_ERR_INCORRECTLY_OPENED_COMMENT,
GUMBO_ERR_INVALID_CHARACTER_SEQUENCE_AFTER_DOCTYPE_NAME,
GUMBO_ERR_INVALID_FIRST_CHARACTER_OF_TAG_NAME,
GUMBO_ERR_MISSING_ATTRIBUTE_VALUE,
GUMBO_ERR_MISSING_DOCTYPE_NAME,
GUMBO_ERR_MISSING_DOCTYPE_PUBLIC_IDENTIFIER,
GUMBO_ERR_MISSING_DOCTYPE_SYSTEM_IDENTIFIER,
GUMBO_ERR_MISSING_END_TAG_NAME,
GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_PUBLIC_IDENTIFIER,
GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER,
GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE,
GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_PUBLIC_KEYWORD,
GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_SYSTEM_KEYWORD,
GUMBO_ERR_MISSING_WHITESPACE_BEFORE_DOCTYPE_NAME,
GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_ATTRIBUTES,
GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS,
GUMBO_ERR_NESTED_COMMENT,
GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE,
GUMBO_ERR_NONCHARACTER_IN_INPUT_STREAM,
GUMBO_ERR_NON_VOID_HTML_ELEMENT_START_TAG_WITH_TRAILING_SOLIDUS,
GUMBO_ERR_NULL_CHARACTER_REFERENCE,
GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE,
GUMBO_ERR_SURROGATE_IN_INPUT_STREAM,
GUMBO_ERR_UNEXPECTED_CHARACTER_AFTER_DOCTYPE_SYSTEM_IDENTIFIER,
GUMBO_ERR_UNEXPECTED_CHARACTER_IN_ATTRIBUTE_NAME,
GUMBO_ERR_UNEXPECTED_CHARACTER_IN_UNQUOTED_ATTRIBUTE_VALUE,
GUMBO_ERR_UNEXPECTED_EQUALS_SIGN_BEFORE_ATTRIBUTE_NAME,
GUMBO_ERR_UNEXPECTED_NULL_CHARACTER,
GUMBO_ERR_UNEXPECTED_QUESTION_MARK_INSTEAD_OF_TAG_NAME,
GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG,
GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE,
// Encoding errors.
GUMBO_ERR_UTF8_INVALID,
GUMBO_ERR_UTF8_TRUNCATED,
// Generic parser error.
GUMBO_ERR_PARSER,
} GumboErrorType;
// Additional data for tokenizer errors.
// This records the current state and codepoint encountered - this is usually
// enough to reconstruct what went wrong and provide a friendly error message.
typedef struct GumboInternalTokenizerError {
// The bad codepoint encountered.
int codepoint;
// The state that the tokenizer was in at the time.
GumboTokenizerEnum state;
} GumboTokenizerError;
// Additional data for parse errors.
typedef struct GumboInternalParserError {
// The type of input token that resulted in this error.
GumboTokenType input_type;
// The HTML tag of the input token. TAG_UNKNOWN if this was not a tag token.
GumboTag input_tag;
// The HTML tag of the input token if it was nonstandard tag token. NULL otherwise.
char *input_name;
// The insertion mode that the parser was in at the time.
GumboInsertionMode parser_state;
// The tag stack at the point of the error. Note that this is an GumboVector
// of GumboTag's *stored by value* - cast the void* to an GumboTag directly to
// get at the tag. For nonstandard tags, this is a pointer to an owned char *
// containing the tag name.
GumboVector /* GumboTag */ tag_stack;
} GumboParserError;
// The overall error struct representing an error in decoding/tokenizing/parsing
// the HTML. This contains an enumerated type flag, a source position, and then
// a union of fields containing data specific to the error.
struct GumboInternalError {
// The type of error.
GumboErrorType type;
// The position within the source file where the error occurred.
GumboSourcePosition position;
// The piece of text that caused the error.
GumboStringPiece original_text;
// Type-specific error information.
union {
// Tokenizer errors.
GumboTokenizerError tokenizer;
// Parser errors.
GumboParserError parser;
} v;
};
// Adds a new error to the parser's error list, and returns a pointer to it so
// that clients can fill out the rest of its fields. May return NULL if we're
// already over the max_errors field specified in GumboOptions.
GumboError* gumbo_add_error(struct GumboInternalParser* parser);
// Initializes the errors vector in the parser.
void gumbo_init_errors(struct GumboInternalParser* errors);
// Frees all the errors in the 'errors_' field of the parser.
void gumbo_destroy_errors(struct GumboInternalParser* errors);
// Frees the memory used for a single GumboError.
void gumbo_error_destroy(GumboError* error);
#ifdef __cplusplus
}
#endif
#endif // GUMBO_ERROR_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/foreign_attrs.c
Vendored
+103
View File
@@ -0,0 +1,103 @@
/* ANSI-C code produced by gperf version 3.1 */
/* Command-line: gperf -m100 -n src/foreign_attrs.gperf */
/* Computed positions: -k'8-9' */
/* Filtered by: gperf-filter.sed */
#include "replacement.h"
#include "macros.h"
#include <string.h>
#define TOTAL_KEYWORDS 11
#define MIN_WORD_LENGTH 5
#define MAX_WORD_LENGTH 13
#define MIN_HASH_VALUE 0
#define MAX_HASH_VALUE 10
/* maximum key range = 11, duplicates = 0 */
static inline unsigned int
hash (register const char *str, register size_t len)
{
static const unsigned char asso_values[] =
{
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 2,
11, 1, 11, 10, 4, 4, 11, 11, 3, 11,
11, 5, 3, 11, 0, 11, 2, 11, 11, 11,
11, 2, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11
};
register unsigned int hval = 0;
switch (len)
{
default:
hval += asso_values[(unsigned char)str[8]];
/*FALLTHROUGH*/
case 8:
hval += asso_values[(unsigned char)str[7]];
/*FALLTHROUGH*/
case 7:
case 6:
case 5:
break;
}
return hval;
}
const ForeignAttrReplacement *
gumbo_get_foreign_attr_replacement (register const char *str, register size_t len)
{
static const unsigned char lengthtable[] =
{
5, 10, 13, 9, 13, 10, 11, 11, 10, 10, 8
};
static const ForeignAttrReplacement wordlist[] =
{
{"xmlns", "xmlns", GUMBO_ATTR_NAMESPACE_XMLNS},
{"xlink:href", "href", GUMBO_ATTR_NAMESPACE_XLINK},
{"xlink:arcrole", "arcrole", GUMBO_ATTR_NAMESPACE_XLINK},
{"xml:space", "space", GUMBO_ATTR_NAMESPACE_XML},
{"xlink:actuate", "actuate", GUMBO_ATTR_NAMESPACE_XLINK},
{"xlink:type", "type", GUMBO_ATTR_NAMESPACE_XLINK},
{"xlink:title", "title", GUMBO_ATTR_NAMESPACE_XLINK},
{"xmlns:xlink", "xlink", GUMBO_ATTR_NAMESPACE_XMLNS},
{"xlink:role", "role", GUMBO_ATTR_NAMESPACE_XLINK},
{"xlink:show", "show", GUMBO_ATTR_NAMESPACE_XLINK},
{"xml:lang", "lang", GUMBO_ATTR_NAMESPACE_XML}
};
if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
{
register unsigned int key = hash (str, len);
if (key <= MAX_HASH_VALUE)
if (len == lengthtable[key])
{
register const char *s = wordlist[key].from;
if (s && *str == *s && !memcmp (str + 1, s + 1, len - 1))
return &wordlist[key];
}
}
return 0;
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/hashmap.c
Vendored
+1154
View File
File diff suppressed because it is too large Load Diff
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/hashmap.h
Vendored
+62
View File
@@ -0,0 +1,62 @@
// Copyright 2020 Joshua J Baker. All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
#ifndef HASHMAP_H
#define HASHMAP_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#if defined(__cplusplus)
extern "C" {
#endif // __cplusplus
struct hashmap;
struct hashmap *hashmap_new(size_t elsize, size_t cap, uint64_t seed0,
uint64_t seed1,
uint64_t (*hash)(const void *item, uint64_t seed0, uint64_t seed1),
int (*compare)(const void *a, const void *b, void *udata),
void (*elfree)(void *item),
void *udata);
struct hashmap *hashmap_new_with_allocator(void *(*malloc)(size_t),
void *(*realloc)(void *, size_t), void (*free)(void*), size_t elsize,
size_t cap, uint64_t seed0, uint64_t seed1,
uint64_t (*hash)(const void *item, uint64_t seed0, uint64_t seed1),
int (*compare)(const void *a, const void *b, void *udata),
void (*elfree)(void *item),
void *udata);
void hashmap_free(struct hashmap *map);
void hashmap_clear(struct hashmap *map, bool update_cap);
size_t hashmap_count(struct hashmap *map);
bool hashmap_oom(struct hashmap *map);
const void *hashmap_get(struct hashmap *map, const void *item);
const void *hashmap_set(struct hashmap *map, const void *item);
const void *hashmap_delete(struct hashmap *map, const void *item);
const void *hashmap_probe(struct hashmap *map, uint64_t position);
bool hashmap_scan(struct hashmap *map, bool (*iter)(const void *item, void *udata), void *udata);
bool hashmap_iter(struct hashmap *map, size_t *i, void **item);
uint64_t hashmap_sip(const void *data, size_t len, uint64_t seed0, uint64_t seed1);
uint64_t hashmap_murmur(const void *data, size_t len, uint64_t seed0, uint64_t seed1);
uint64_t hashmap_xxhash3(const void *data, size_t len, uint64_t seed0, uint64_t seed1);
const void *hashmap_get_with_hash(struct hashmap *map, const void *key, uint64_t hash);
const void *hashmap_delete_with_hash(struct hashmap *map, const void *key, uint64_t hash);
const void *hashmap_set_with_hash(struct hashmap *map, const void *item, uint64_t hash);
void hashmap_set_grow_by_power(struct hashmap *map, size_t power);
void hashmap_set_load_factor(struct hashmap *map, double load_factor);
// DEPRECATED: use `hashmap_new_with_allocator`
void hashmap_set_allocator(void *(*malloc)(size_t), void (*free)(void*));
#if defined(__cplusplus)
}
#endif // __cplusplus
#endif // HASHMAP_H
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/insertion_mode.h
Vendored
+33
View File
@@ -0,0 +1,33 @@
#ifndef GUMBO_INSERTION_MODE_H_
#define GUMBO_INSERTION_MODE_H_
// https://html.spec.whatwg.org/multipage/parsing.html#insertion-mode
// If new enum values are added, be sure to update the kTokenHandlers
// dispatch table in parser.c.
typedef enum {
GUMBO_INSERTION_MODE_INITIAL,
GUMBO_INSERTION_MODE_BEFORE_HTML,
GUMBO_INSERTION_MODE_BEFORE_HEAD,
GUMBO_INSERTION_MODE_IN_HEAD,
GUMBO_INSERTION_MODE_IN_HEAD_NOSCRIPT,
GUMBO_INSERTION_MODE_AFTER_HEAD,
GUMBO_INSERTION_MODE_IN_BODY,
GUMBO_INSERTION_MODE_TEXT,
GUMBO_INSERTION_MODE_IN_TABLE,
GUMBO_INSERTION_MODE_IN_TABLE_TEXT,
GUMBO_INSERTION_MODE_IN_CAPTION,
GUMBO_INSERTION_MODE_IN_COLUMN_GROUP,
GUMBO_INSERTION_MODE_IN_TABLE_BODY,
GUMBO_INSERTION_MODE_IN_ROW,
GUMBO_INSERTION_MODE_IN_CELL,
GUMBO_INSERTION_MODE_IN_SELECT,
GUMBO_INSERTION_MODE_IN_SELECT_IN_TABLE,
GUMBO_INSERTION_MODE_IN_TEMPLATE,
GUMBO_INSERTION_MODE_AFTER_BODY,
GUMBO_INSERTION_MODE_IN_FRAMESET,
GUMBO_INSERTION_MODE_AFTER_FRAMESET,
GUMBO_INSERTION_MODE_AFTER_AFTER_BODY,
GUMBO_INSERTION_MODE_AFTER_AFTER_FRAMESET
} GumboInsertionMode;
#endif // GUMBO_INSERTION_MODE_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/macros.h
Vendored
+91
View File
@@ -0,0 +1,91 @@
#ifndef MACROS_H
#define MACROS_H
#if (!defined(__STDC_VERSION__) || !(__STDC_VERSION__ >= 199901L)) \
&& !defined(_WIN32) && !defined(__cplusplus)
# error C99 compiler required
#endif
#if defined(_WIN32)
# define inline __inline
# define __func__ __FUNCTION__
#endif
// Calculate the number of elements in an array.
// The extra division on the third line is a trick to help prevent
// passing a pointer to the first element of an array instead of a
// reference to the array itself.
#define ARRAY_COUNT(x) ( \
(sizeof(x) / sizeof((x)[0])) \
/ ((size_t)(!(sizeof(x) % sizeof((x)[0])))) \
)
#ifdef NDEBUG
#define UNUSED_IF_NDEBUG(x) (void)(x)
#else
#define UNUSED_IF_NDEBUG(x)
#endif
#ifdef __GNUC__
#define GNUC_AT_LEAST(major, minor) ( \
(__GNUC__ > major) \
|| ((__GNUC__ == major) && (__GNUC_MINOR__ >= minor)) )
#else
#define GNUC_AT_LEAST(major, minor) 0
#endif
#ifdef __has_attribute
#define HAS_ATTRIBUTE(x) __has_attribute(x)
#else
#define HAS_ATTRIBUTE(x) 0
#endif
#if GNUC_AT_LEAST(3, 0) || HAS_ATTRIBUTE(unused) || defined(__TINYC__)
#define UNUSED __attribute__((__unused__))
#else
#define UNUSED
#endif
#if GNUC_AT_LEAST(3, 0)
#define MALLOC __attribute__((__malloc__))
#define PRINTF(x) __attribute__((__format__(__printf__, (x), (x + 1))))
#define PURE __attribute__((__pure__))
#define CONST_FN __attribute__((__const__))
#else
#define MALLOC
#define PRINTF(x)
#define PURE
#define CONST_FN
#endif
#define UNUSED_ARG(x) unused__ ## x UNUSED
#if GNUC_AT_LEAST(3, 0) && defined(__OPTIMIZE__)
#define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
#else
#define likely(x) (x)
#define unlikely(x) (x)
#endif
#if GNUC_AT_LEAST(3, 3) || HAS_ATTRIBUTE(nonnull)
#define NONNULL_ARGS __attribute__((__nonnull__))
#else
#define NONNULL_ARGS
#endif
#if GNUC_AT_LEAST(3, 4) || HAS_ATTRIBUTE(warn_unused_result)
#define WARN_UNUSED_RESULT __attribute__((__warn_unused_result__))
#else
#define WARN_UNUSED_RESULT
#endif
#if GNUC_AT_LEAST(5, 0) || HAS_ATTRIBUTE(returns_nonnull)
#define RETURNS_NONNULL __attribute__((__returns_nonnull__))
#else
#define RETURNS_NONNULL
#endif
#define XMALLOC MALLOC RETURNS_NONNULL
#endif // ndef MACROS_H
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/nokogiri_gumbo.h
Vendored
+938
View File
@@ -0,0 +1,938 @@
// Copyright 2010 Google Inc.
// Copyright 2018 Craig Barnes.
// Licensed under the Apache License, version 2.0.
// We use Gumbo as a prefix for types, gumbo_ as a prefix for functions,
// GUMBO_ as a prefix for enum constants and kGumbo as a prefix for
// static constants
/**
* @file
* @mainpage Gumbo HTML Parser
*
* This provides a conformant, no-dependencies implementation of the
* [HTML5] parsing algorithm. It supports only UTF-8 -- if you need
* to parse a different encoding, run a preprocessing step to convert
* to UTF-8. It returns a parse tree made of the structs in this file.
*
* Example:
* @code
* GumboOutput* output = gumbo_parse(input);
* do_something_with_doctype(output->document);
* do_something_with_html_tree(output->root);
* gumbo_destroy_output(output);
* @endcode
*
* [HTML5]: https://html.spec.whatwg.org/multipage/
*/
#ifndef GUMBO_H
#define GUMBO_H
#include <stdbool.h>
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
#endif
/**
* A struct representing a character position within the original text
* buffer. Line and column numbers are 1-based and offsets are 0-based,
* which matches how most editors and command-line tools work.
*/
typedef struct {
size_t line;
size_t column;
size_t offset;
} GumboSourcePosition;
/**
* A struct representing a string or part of a string. Strings within
* the parser are represented by a `char*` and a length; the `char*`
* points into an existing data buffer owned by some other code (often
* the original input). `GumboStringPiece`s are assumed (by convention)
* to be immutable, because they may share data. Clients should assume
* that it is not NUL-terminated and should always use explicit lengths
* when manipulating them.
*/
typedef struct {
/** A pointer to the beginning of the string. `NULL` if `length == 0`. */
const char* data;
/** The length of the string fragment, in bytes (may be zero). */
size_t length;
} GumboStringPiece;
#define GUMBO_EMPTY_STRING_INIT { .data = NULL, .length = 0 }
/** A constant to represent a 0-length null string. */
#define kGumboEmptyString (const GumboStringPiece)GUMBO_EMPTY_STRING_INIT
/**
* Compares two `GumboStringPiece`s, and returns `true` if they're
* equal or `false` otherwise.
*/
bool gumbo_string_equals (
const GumboStringPiece* str1,
const GumboStringPiece* str2
);
/**
* Compares two `GumboStringPiece`s, ignoring case, and returns `true`
* if they're equal or `false` otherwise.
*/
bool gumbo_string_equals_ignore_case (
const GumboStringPiece* str1,
const GumboStringPiece* str2
);
/**
* Check if the first `GumboStringPiece` is a prefix of the second, ignoring
* case.
*/
bool gumbo_string_prefix_ignore_case (
const GumboStringPiece* prefix,
const GumboStringPiece* str
);
/**
* A simple vector implementation. This stores a pointer to a data array
* and a length. All elements are stored as `void*`; client code must
* cast to the appropriate type. Overflows upon addition result in
* reallocation of the data array, with the size doubling to maintain
* `O(1)` amortized cost. There is no removal function, as this isn't
* needed for any of the operations within this library. Iteration can
* be done through inspecting the structure directly in a `for` loop.
*/
typedef struct {
/**
* Data elements. This points to a dynamically-allocated array of
* `capacity` elements, each a `void*` to the element itself.
*/
void** data;
/** Number of elements currently in the vector. */
unsigned int length;
/** Current array capacity. */
unsigned int capacity;
} GumboVector;
# define GUMBO_EMPTY_VECTOR_INIT { .data = NULL, .length = 0, .capacity = 0 }
/** An empty (0-length, 0-capacity) `GumboVector`. */
#define kGumboEmptyVector (const GumboVector)GUMBO_EMPTY_VECTOR_INIT
/**
* Returns the first index at which an element appears in this vector
* (testing by pointer equality), or `-1` if it never does.
*/
int gumbo_vector_index_of(GumboVector* vector, const void* element);
/**
* An `enum` for all the tags defined in the HTML5 standard. These
* correspond to the tag names themselves. Enum constants exist only
* for tags that appear in the spec itself (or for tags with special
* handling in the SVG and MathML namespaces). Any other tags appear
* as `GUMBO_TAG_UNKNOWN` and the actual tag name can be obtained
* through `original_tag`.
*
* This is mostly for API convenience, so that clients of this library
* don't need to perform a `strcasecmp` to find the normalized tag
* name. It also has efficiency benefits, by letting the parser work
* with enums instead of strings.
*/
typedef enum {
GUMBO_TAG_HTML,
GUMBO_TAG_HEAD,
GUMBO_TAG_TITLE,
GUMBO_TAG_BASE,
GUMBO_TAG_LINK,
GUMBO_TAG_META,
GUMBO_TAG_STYLE,
GUMBO_TAG_SCRIPT,
GUMBO_TAG_NOSCRIPT,
GUMBO_TAG_TEMPLATE,
GUMBO_TAG_BODY,
GUMBO_TAG_ARTICLE,
GUMBO_TAG_SECTION,
GUMBO_TAG_NAV,
GUMBO_TAG_ASIDE,
GUMBO_TAG_H1,
GUMBO_TAG_H2,
GUMBO_TAG_H3,
GUMBO_TAG_H4,
GUMBO_TAG_H5,
GUMBO_TAG_H6,
GUMBO_TAG_HGROUP,
GUMBO_TAG_HEADER,
GUMBO_TAG_FOOTER,
GUMBO_TAG_ADDRESS,
GUMBO_TAG_P,
GUMBO_TAG_HR,
GUMBO_TAG_PRE,
GUMBO_TAG_BLOCKQUOTE,
GUMBO_TAG_OL,
GUMBO_TAG_UL,
GUMBO_TAG_LI,
GUMBO_TAG_DL,
GUMBO_TAG_DT,
GUMBO_TAG_DD,
GUMBO_TAG_FIGURE,
GUMBO_TAG_FIGCAPTION,
GUMBO_TAG_MAIN,
GUMBO_TAG_DIV,
GUMBO_TAG_A,
GUMBO_TAG_EM,
GUMBO_TAG_STRONG,
GUMBO_TAG_SMALL,
GUMBO_TAG_S,
GUMBO_TAG_CITE,
GUMBO_TAG_Q,
GUMBO_TAG_DFN,
GUMBO_TAG_ABBR,
GUMBO_TAG_DATA,
GUMBO_TAG_TIME,
GUMBO_TAG_CODE,
GUMBO_TAG_VAR,
GUMBO_TAG_SAMP,
GUMBO_TAG_KBD,
GUMBO_TAG_SUB,
GUMBO_TAG_SUP,
GUMBO_TAG_I,
GUMBO_TAG_B,
GUMBO_TAG_U,
GUMBO_TAG_MARK,
GUMBO_TAG_RUBY,
GUMBO_TAG_RT,
GUMBO_TAG_RP,
GUMBO_TAG_BDI,
GUMBO_TAG_BDO,
GUMBO_TAG_SPAN,
GUMBO_TAG_BR,
GUMBO_TAG_WBR,
GUMBO_TAG_INS,
GUMBO_TAG_DEL,
GUMBO_TAG_IMAGE,
GUMBO_TAG_IMG,
GUMBO_TAG_IFRAME,
GUMBO_TAG_EMBED,
GUMBO_TAG_OBJECT,
GUMBO_TAG_PARAM,
GUMBO_TAG_VIDEO,
GUMBO_TAG_AUDIO,
GUMBO_TAG_SOURCE,
GUMBO_TAG_TRACK,
GUMBO_TAG_CANVAS,
GUMBO_TAG_MAP,
GUMBO_TAG_AREA,
GUMBO_TAG_MATH,
GUMBO_TAG_MI,
GUMBO_TAG_MO,
GUMBO_TAG_MN,
GUMBO_TAG_MS,
GUMBO_TAG_MTEXT,
GUMBO_TAG_MGLYPH,
GUMBO_TAG_MALIGNMARK,
GUMBO_TAG_ANNOTATION_XML,
GUMBO_TAG_SVG,
GUMBO_TAG_FOREIGNOBJECT,
GUMBO_TAG_DESC,
GUMBO_TAG_TABLE,
GUMBO_TAG_CAPTION,
GUMBO_TAG_COLGROUP,
GUMBO_TAG_COL,
GUMBO_TAG_TBODY,
GUMBO_TAG_THEAD,
GUMBO_TAG_TFOOT,
GUMBO_TAG_TR,
GUMBO_TAG_TD,
GUMBO_TAG_TH,
GUMBO_TAG_FORM,
GUMBO_TAG_FIELDSET,
GUMBO_TAG_LEGEND,
GUMBO_TAG_LABEL,
GUMBO_TAG_INPUT,
GUMBO_TAG_BUTTON,
GUMBO_TAG_SELECT,
GUMBO_TAG_DATALIST,
GUMBO_TAG_OPTGROUP,
GUMBO_TAG_OPTION,
GUMBO_TAG_TEXTAREA,
GUMBO_TAG_KEYGEN,
GUMBO_TAG_OUTPUT,
GUMBO_TAG_PROGRESS,
GUMBO_TAG_METER,
GUMBO_TAG_DETAILS,
GUMBO_TAG_SUMMARY,
GUMBO_TAG_MENU,
GUMBO_TAG_MENUITEM,
GUMBO_TAG_APPLET,
GUMBO_TAG_ACRONYM,
GUMBO_TAG_BGSOUND,
GUMBO_TAG_DIR,
GUMBO_TAG_FRAME,
GUMBO_TAG_FRAMESET,
GUMBO_TAG_NOFRAMES,
GUMBO_TAG_LISTING,
GUMBO_TAG_XMP,
GUMBO_TAG_NEXTID,
GUMBO_TAG_NOEMBED,
GUMBO_TAG_PLAINTEXT,
GUMBO_TAG_RB,
GUMBO_TAG_STRIKE,
GUMBO_TAG_BASEFONT,
GUMBO_TAG_BIG,
GUMBO_TAG_BLINK,
GUMBO_TAG_CENTER,
GUMBO_TAG_FONT,
GUMBO_TAG_MARQUEE,
GUMBO_TAG_MULTICOL,
GUMBO_TAG_NOBR,
GUMBO_TAG_SPACER,
GUMBO_TAG_TT,
GUMBO_TAG_RTC,
GUMBO_TAG_DIALOG,
GUMBO_TAG_SEARCH,
// Used for all tags that don't have special handling in HTML.
GUMBO_TAG_UNKNOWN,
// A marker value to indicate the end of the enum, for iterating over it.
GUMBO_TAG_LAST,
} GumboTag;
/**
* Returns the normalized (all lower case) tag name for a `GumboTag` enum. The
* return value is static data owned by the library.
*/
const char* gumbo_normalized_tagname(GumboTag tag);
/**
* Extracts the tag name from the `original_text` field of an element
* or token by stripping off `</>` characters and attributes and
* adjusting the passed-in `GumboStringPiece` appropriately. The tag
* name is in the original case and shares a buffer with the original
* text, to simplify memory management. Behavior is undefined if a
* string piece that doesn't represent an HTML tag (`<tagname>` or
* `</tagname>`) is passed in. If the string piece is completely
* empty (`NULL` data pointer), then this function will exit
* successfully as a no-op.
*/
void gumbo_tag_from_original_text(GumboStringPiece* text);
/**
* Converts a tag name string (which may be in upper or mixed case) to a
* tag enum.
*/
GumboTag gumbo_tagn_enum(const char* tagname, size_t length);
/**
* Attribute namespaces.
* HTML includes special handling for XLink, XML, and XMLNS namespaces
* on attributes. Everything else goes in the generic "NONE" namespace.
*/
typedef enum {
GUMBO_ATTR_NAMESPACE_NONE,
GUMBO_ATTR_NAMESPACE_XLINK,
GUMBO_ATTR_NAMESPACE_XML,
GUMBO_ATTR_NAMESPACE_XMLNS,
} GumboAttributeNamespaceEnum;
/**
* A struct representing a single attribute on a HTML tag. This is a
* name-value pair, but also includes information about source locations
* and original source text.
*/
typedef struct {
/**
* The namespace for the attribute. This will usually be
* `GUMBO_ATTR_NAMESPACE_NONE`, but some XLink/XMLNS/XML attributes
* take special values, per:
* https://html.spec.whatwg.org/multipage/parsing.html#adjust-foreign-attributes
*/
GumboAttributeNamespaceEnum attr_namespace;
/**
* The name of the attribute. This is in a freshly-allocated buffer to
* deal with case-normalization and is null-terminated.
*/
const char* name;
/**
* The original text of the attribute name, as a pointer into the
* original source buffer.
*/
GumboStringPiece original_name;
/**
* The value of the attribute. This is in a freshly-allocated buffer
* to deal with unescaping and is null-terminated. It does not include
* any quotes that surround the attribute. If the attribute has no
* value (for example, `selected` on a checkbox) this will be an empty
* string.
*/
const char* value;
/**
* The original text of the value of the attribute. This points into
* the original source buffer. It includes any quotes that surround
* the attribute and you can look at `original_value.data[0]` and
* `original_value.data[original_value.length - 1]` to determine what
* the quote characters were. If the attribute has no value this will
* be a 0-length string.
*/
GumboStringPiece original_value;
/** The starting position of the attribute name. */
GumboSourcePosition name_start;
/**
* The ending position of the attribute name. This is not always derivable
* from the starting position of the value because of the possibility of
* whitespace around the `=` sign.
*/
GumboSourcePosition name_end;
/** The starting position of the attribute value. */
GumboSourcePosition value_start;
/** The ending position of the attribute value. */
GumboSourcePosition value_end;
} GumboAttribute;
/**
* Given a vector of `GumboAttribute`s, look up the one with the
* specified name and return it, or `NULL` if no such attribute exists.
* This uses a case-insensitive match, as HTML is case-insensitive.
*/
GumboAttribute* gumbo_get_attribute(const GumboVector* attrs, const char* name);
/**
* Enum denoting the type of node. This determines the type of the
* `node.v` union.
*/
typedef enum {
/** Document node. `v` will be a `GumboDocument`. */
GUMBO_NODE_DOCUMENT,
/** Element node. `v` will be a `GumboElement`. */
GUMBO_NODE_ELEMENT,
/** Text node. `v` will be a `GumboText`. */
GUMBO_NODE_TEXT,
/** CDATA node. `v` will be a `GumboText`. */
GUMBO_NODE_CDATA,
/** Comment node. `v` will be a `GumboText`, excluding comment delimiters. */
GUMBO_NODE_COMMENT,
/** Text node, where all contents is whitespace. `v` will be a `GumboText`. */
GUMBO_NODE_WHITESPACE,
/**
* Template node. This is separate from `GUMBO_NODE_ELEMENT` because
* many client libraries will want to ignore the contents of template
* nodes, as the spec suggests. Recursing on `GUMBO_NODE_ELEMENT` will
* do the right thing here, while clients that want to include template
* contents should also check for `GUMBO_NODE_TEMPLATE`. `v` will be a
* `GumboElement`.
*/
GUMBO_NODE_TEMPLATE
} GumboNodeType;
/**
* Forward declaration of GumboNode so it can be used recursively in
* GumboNode.parent.
*/
typedef struct GumboInternalNode GumboNode;
/** https://dom.spec.whatwg.org/#concept-document-quirks */
typedef enum {
GUMBO_DOCTYPE_NO_QUIRKS,
GUMBO_DOCTYPE_QUIRKS,
GUMBO_DOCTYPE_LIMITED_QUIRKS
} GumboQuirksModeEnum;
/**
* Namespaces.
* Unlike in X(HT)ML, namespaces in HTML5 are not denoted by a prefix.
* Rather, anything inside an `<svg>` tag is in the SVG namespace,
* anything inside the `<math>` tag is in the MathML namespace, and
* anything else is inside the HTML namespace. No other namespaces are
* supported, so this can be an `enum`.
*/
typedef enum {
GUMBO_NAMESPACE_HTML,
GUMBO_NAMESPACE_SVG,
GUMBO_NAMESPACE_MATHML
} GumboNamespaceEnum;
/**
* Parse flags.
* We track the reasons for parser insertion of nodes and store them in
* a bitvector in the node itself. This lets client code optimize out
* nodes that are implied by the HTML structure of the document, or flag
* constructs that may not be allowed by a style guide, or track the
* prevalence of incorrect or tricky HTML code.
*/
typedef enum {
/**
* A normal node -- both start and end tags appear in the source,
* nothing has been reparented.
*/
GUMBO_INSERTION_NORMAL = 0,
/**
* A node inserted by the parser to fulfill some implicit insertion
* rule. This is usually set in addition to some other flag giving a
* more specific insertion reason; it's a generic catch-all term
* meaning "The start tag for this node did not appear in the document
* source".
*/
GUMBO_INSERTION_BY_PARSER = 1 << 0,
/**
* A flag indicating that the end tag for this node did not appear in
* the document source. Note that in some cases, you can still have
* parser-inserted nodes with an explicit end tag. For example,
* `Text</html>` has `GUMBO_INSERTED_BY_PARSER` set on the `<html>`
* node, but `GUMBO_INSERTED_END_TAG_IMPLICITLY` is unset, as the
* `</html>` tag actually exists.
*
* This flag will be set only if the end tag is completely missing.
* In some cases, the end tag may be misplaced (e.g. a `</body>` tag
* with text afterwards), which will leave this flag unset and require
* clients to inspect the parse errors for that case.
*/
GUMBO_INSERTION_IMPLICIT_END_TAG = 1 << 1,
// Value 1 << 2 was for a flag that has since been removed.
/**
* A flag for nodes that are inserted because their presence is
* implied by other tags, e.g. `<html>`, `<head>`, `<body>`,
* `<tbody>`, etc.
*/
GUMBO_INSERTION_IMPLIED = 1 << 3,
/**
* A flag for nodes that are converted from their end tag equivalents.
* For example, `</p>` when no paragraph is open implies that the
* parser should create a `<p>` tag and immediately close it, while
* `</br>` means the same thing as `<br>`.
*/
GUMBO_INSERTION_CONVERTED_FROM_END_TAG = 1 << 4,
// Value 1 << 5 was for a flag that has since been removed.
/** A flag for `<image>` tags that are rewritten as `<img>`. */
GUMBO_INSERTION_FROM_IMAGE = 1 << 6,
/**
* A flag for nodes that are cloned as a result of the reconstruction
* of active formatting elements. This is set only on the clone; the
* initial portion of the formatting run is a NORMAL node with an
* `IMPLICIT_END_TAG`.
*/
GUMBO_INSERTION_RECONSTRUCTED_FORMATTING_ELEMENT = 1 << 7,
/** A flag for nodes that are cloned by the adoption agency algorithm. */
GUMBO_INSERTION_ADOPTION_AGENCY_CLONED = 1 << 8,
/** A flag for nodes that are moved by the adoption agency algorithm. */
GUMBO_INSERTION_ADOPTION_AGENCY_MOVED = 1 << 9,
/**
* A flag for nodes that have been foster-parented out of a table (or
* should've been foster-parented, if verbatim mode is set).
*/
GUMBO_INSERTION_FOSTER_PARENTED = 1 << 10,
} GumboParseFlags;
/** Information specific to document nodes. */
typedef struct {
/**
* An array of `GumboNode`s, containing the children of this element.
* This will normally consist of the `<html>` element and any comment
* nodes found. Pointers are owned.
*/
GumboVector /* GumboNode* */ children;
/**
* `true` if there was an explicit doctype token, as opposed to it
* being omitted.
*/
bool has_doctype;
// Fields from the doctype token, copied verbatim.
const char* name;
const char* public_identifier;
const char* system_identifier;
/**
* Whether or not the document is in QuirksMode, as determined by the
* values in the GumboTokenDocType template.
*/
GumboQuirksModeEnum doc_type_quirks_mode;
} GumboDocument;
/**
* The struct used to represent TEXT, CDATA, COMMENT, and WHITESPACE
* elements. This contains just a block of text and its position.
*/
typedef struct {
/**
* The text of this node, after entities have been parsed and decoded.
* For comment and cdata nodes, this does not include the comment
* delimiters.
*/
const char* text;
/**
* The original text of this node, as a pointer into the original
* buffer. For comment/cdata nodes, this includes the comment
* delimiters.
*/
GumboStringPiece original_text;
/**
* The starting position of this node. This corresponds to the
* position of `original_text`, before entities are decoded.
* */
GumboSourcePosition start_pos;
} GumboText;
/**
* The struct used to represent all HTML elements. This contains
* information about the tag, attributes, and child nodes.
*/
typedef struct {
/**
* An array of `GumboNode`s, containing the children of this element.
* Pointers are owned.
*/
GumboVector /* GumboNode* */ children;
/** The GumboTag enum for this element. */
GumboTag tag;
/** The name for this element. */
const char* name;
/** The GumboNamespaceEnum for this element. */
GumboNamespaceEnum tag_namespace;
/**
* A `GumboStringPiece` pointing to the original tag text for this
* element, pointing directly into the source buffer. If the tag was
* inserted algorithmically (for example, `<head>` or `<tbody>`
* insertion), this will be a zero-length string.
*/
GumboStringPiece original_tag;
/**
* A `GumboStringPiece` pointing to the original end tag text for this
* element. If the end tag was inserted algorithmically, (for example,
* closing a self-closing tag), this will be a zero-length string.
*/
GumboStringPiece original_end_tag;
/** The source position for the start of the start tag. */
GumboSourcePosition start_pos;
/** The source position for the start of the end tag. */
GumboSourcePosition end_pos;
/**
* An array of `GumboAttribute`s, containing the attributes for this
* tag in the order that they were parsed. Pointers are owned.
*/
GumboVector /* GumboAttribute* */ attributes;
} GumboElement;
/**
* A supertype for `GumboElement` and `GumboText`, so that we can
* include one generic type in lists of children and cast as necessary
* to subtypes.
*/
struct GumboInternalNode {
/** The type of node that this is. */
GumboNodeType type;
/** Pointer back to parent node. Not owned. */
GumboNode* parent;
/** The index within the parent's children vector of this node. */
unsigned int index_within_parent;
/**
* A bitvector of flags containing information about why this element
* was inserted into the parse tree, including a variety of special
* parse situations.
*/
GumboParseFlags parse_flags;
/** The actual node data. */
union {
GumboDocument document; // For GUMBO_NODE_DOCUMENT.
GumboElement element; // For GUMBO_NODE_ELEMENT.
GumboText text; // For everything else.
} v;
};
/**
* Input struct containing configuration options for the parser.
* These let you specify alternate memory managers, provide different
* error handling, etc. Use `kGumboDefaultOptions` for sensible
* defaults and only set what you need.
*/
typedef struct GumboInternalOptions {
/**
* The tab-stop size, for computing positions in HTML files that
* use tabs. Default: `8`.
*/
int tab_stop;
/**
* Whether or not to stop parsing when the first error is encountered.
* Default: `false`.
*/
bool stop_on_first_error;
/**
* Maximum allowed number of attributes per element. If this limit is
* exceeded, the parser will return early with a partial document and
* the returned `GumboOutput` will have its `status` field set to
* `GUMBO_STATUS_TOO_MANY_ATTRIBUTES`. Set to `-1` to disable the limit.
* Default: `400`.
*/
int max_attributes;
/**
* Maximum allowed depth for the parse tree. If this limit is exceeded,
* the parser will return early with a partial document and the returned
* `GumboOutput` will have its `status` field set to
* `GUMBO_STATUS_TREE_TOO_DEEP`.
* Default: `400`.
*/
unsigned int max_tree_depth;
/**
* The maximum number of errors before the parser stops recording
* them. This is provided so that if the page is totally borked, we
* don't completely fill up the errors vector and exhaust memory with
* useless redundant errors. Set to `-1` to disable the limit.
* Default: `-1`.
*/
int max_errors;
/**
* The fragment context for parsing:
* https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments
*
* If `NULL` is passed here, it is assumed to be "no
* fragment", i.e. the regular parsing algorithm. Otherwise, pass the
* tag name for the intended parent of the parsed fragment. We use the
* tag name, namespace, and encoding attribute which are sufficient to
* set all of the parsing context needed for fragment parsing.
*
* Default: `NULL`.
*/
const char* fragment_context;
/**
* The namespace for the fragment context. This lets client code
* differentiate between, say, parsing a `<title>` tag in SVG vs.
* parsing it in HTML.
*
* Default: `GUMBO_NAMESPACE_HTML`.
*/
GumboNamespaceEnum fragment_namespace;
/**
* The value of the fragment context's `encoding` attribute, if any.
* Set to `NULL` for no `encoding` attribute.
*
* Default: `NULL`.
*/
const char* fragment_encoding;
/**
* Quirks mode for fragment parsing. The quirks mode for a given DOCTYPE can
* be looked up using `gumbo_compute_quirks_mode()`.
*
* Default: `GUMBO_DOCTYPE_NO_QUIRKS`.
*/
GumboQuirksModeEnum quirks_mode;
/**
* For fragment parsing. Set this to true if the context node has a form
* element as an ancestor.
*
* Default: `false`.
*/
bool fragment_context_has_form_ancestor;
/**
* Parse `noscript` elements as if scripting was enabled. This causes the
* contents of the `noscript` element to be parsed as raw text, rather
* than as HTML elements.
*
* Default: `false`.
*/
bool parse_noscript_content_as_text;
} GumboOptions;
/** Default options struct; use this with gumbo_parse_with_options. */
extern const GumboOptions kGumboDefaultOptions;
/**
* Status code indicating whether parsing finished successfully or
* was stopped mid-document due to exceptional circumstances.
*/
typedef enum {
/**
* Indicates that parsing completed successfully. The resulting tree
* will be a complete document.
*/
GUMBO_STATUS_OK,
/**
* Indicates that the maximum element nesting limit
* (`GumboOptions::max_tree_depth`) was reached during parsing. The
* resulting tree will be a partial document, with no further nodes
* created after the point where the limit was reached. The partial
* document may be useful for constructing an error message but
* typically shouldn't be used for other purposes.
*/
GUMBO_STATUS_TREE_TOO_DEEP,
/**
* Indicates that the maximum number of attributes per element
* (`GumboOptions::max_attributes`) was reached during parsing. The
* resulting tree will be a partial document, with no further nodes
* created after the point where the limit was reached. The partial
* document may be useful for constructing an error message but
* typically shouldn't be used for other purposes.
*/
GUMBO_STATUS_TOO_MANY_ATTRIBUTES,
// Currently unused
GUMBO_STATUS_OUT_OF_MEMORY,
} GumboOutputStatus;
/** The output struct containing the results of the parse. */
typedef struct GumboInternalOutput {
/**
* Pointer to the document node. This is a `GumboNode` of type
* `NODE_DOCUMENT` that contains the entire document as its child.
*/
GumboNode* document;
/**
* Pointer to the root node. This is the `<html>` tag that forms the
* root of the document.
*/
GumboNode* root;
/**
* A list of errors that occurred during the parse.
*/
GumboVector /* GumboError */ errors;
/**
* True if the parser encountered an error.
*
* This can be true and `errors` an empty `GumboVector` if the `max_errors`
* option was set to 0.
*/
bool document_error;
/**
* A status code indicating whether parsing finished successfully or was
* stopped mid-document due to exceptional circumstances.
*/
GumboOutputStatus status;
} GumboOutput;
/**
* Parses a buffer of UTF-8 text into an `GumboNode` parse tree. The
* buffer must live at least as long as the parse tree, as some fields
* (eg. `original_text`) point directly into the original buffer.
*
* This doesn't support buffers longer than 4 gigabytes.
*/
GumboOutput* gumbo_parse(const char* buffer);
/**
* Extended version of `gumbo_parse` that takes an explicit options
* structure, buffer, and length.
*/
GumboOutput* gumbo_parse_with_options (
const GumboOptions* options,
const char* buffer,
size_t buffer_length
);
/**
* Compute the quirks mode based on the name, public identifier, and system
* identifier. Any of these may be `NULL` to indicate a missing value.
*/
GumboQuirksModeEnum gumbo_compute_quirks_mode (
const char *name,
const char *pubid,
const char *sysid
);
/** Convert a `GumboOutputStatus` code into a readable description. */
const char* gumbo_status_to_string(GumboOutputStatus status);
/** Release the memory used for the parse tree and parse errors. */
void gumbo_destroy_output(GumboOutput* output);
/** Opaque GumboError type */
typedef struct GumboInternalError GumboError;
/**
* Returns the position of the error.
*/
GumboSourcePosition gumbo_error_position(const GumboError* error);
/**
* Returns a constant string representation of the error's code. This is owned
* by the library and should not be freed by the caller.
*/
const char* gumbo_error_code(const GumboError* error);
/**
* Prints an error to a string. This stores a freshly-allocated buffer
* containing the error message text in output. The caller is responsible for
* freeing the buffer. The size of the error message is returned. The error
* message itself may not be NULL-terminated and may contain NULL bytes so the
* returned size must be used.
*/
size_t gumbo_error_to_string(const GumboError* error, char **output);
/**
* Prints a caret diagnostic to a string. This stores a freshly-allocated
* buffer containing the error message text in output. The caller is responsible for
* freeing the buffer. The size of the error message is returned. The error
* message itself may not be NULL-terminated and may contain NULL bytes so the
* returned size must be used.
*/
size_t gumbo_caret_diagnostic_to_string (
const GumboError* error,
const char* source_text,
size_t source_length,
char** output
);
/**
* Like gumbo_caret_diagnostic_to_string, but prints the text to stdout
* instead of writing to a string.
*/
void gumbo_print_caret_diagnostic (
const GumboError* error,
const char* source_text,
size_t source_length
);
#ifdef __cplusplus
}
#endif
#endif // GUMBO_H
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/parser.c
Vendored
+4924
View File
File diff suppressed because it is too large Load Diff
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/parser.h
Vendored
+41
View File
@@ -0,0 +1,41 @@
#ifndef GUMBO_PARSER_H_
#define GUMBO_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
// Contains the definition of the top-level GumboParser structure that's
// threaded through basically every internal function in the library.
struct GumboInternalParserState;
struct GumboInternalOutput;
struct GumboInternalOptions;
struct GumboInternalTokenizerState;
// An overarching struct that's threaded through (nearly) all functions in the
// library, OOP-style. This gives each function access to the options and
// output, along with any internal state needed for the parse.
typedef struct GumboInternalParser {
// Settings for this parse run.
const struct GumboInternalOptions* _options;
// Output for the parse.
struct GumboInternalOutput* _output;
// The internal tokenizer state, defined as a pointer to avoid a cyclic
// dependency on html5tokenizer.h. The main parse routine is responsible for
// initializing this on parse start, and destroying it on parse end.
// End-users will never see a non-garbage value in this pointer.
struct GumboInternalTokenizerState* _tokenizer_state;
// The internal parser state. Initialized on parse start and destroyed on
// parse end; end-users will never see a non-garbage value in this pointer.
struct GumboInternalParserState* _parser_state;
} GumboParser;
#ifdef __cplusplus
}
#endif
#endif // GUMBO_PARSER_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/replacement.h
Vendored
+33
View File
@@ -0,0 +1,33 @@
#ifndef GUMBO_REPLACEMENT_H_
#define GUMBO_REPLACEMENT_H_
#include <stddef.h>
#include "nokogiri_gumbo.h"
typedef struct {
const char *const from;
const char *const to;
} StringReplacement;
const StringReplacement *gumbo_get_svg_tag_replacement (
const char* str,
size_t len
);
const StringReplacement *gumbo_get_svg_attr_replacement (
const char* str,
size_t len
);
typedef struct {
const char *const from;
const char *const local_name;
const GumboAttributeNamespaceEnum attr_namespace;
} ForeignAttrReplacement;
const ForeignAttrReplacement *gumbo_get_foreign_attr_replacement (
const char* str,
size_t len
);
#endif // GUMBO_REPLACEMENT_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/string_buffer.c
Vendored
+103
View File
@@ -0,0 +1,103 @@
/*
Copyright 2010 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include <string.h>
#include "string_buffer.h"
#include "util.h"
// Size chosen via statistical analysis of ~60K websites.
// 99% of text nodes and 98% of attribute names/values fit in this initial size.
static const size_t kDefaultStringBufferSize = 5;
static void maybe_resize_string_buffer (
size_t additional_chars,
GumboStringBuffer* buffer
) {
size_t new_length = buffer->length + additional_chars;
size_t new_capacity = buffer->capacity;
while (new_capacity < new_length) {
new_capacity *= 2;
}
if (new_capacity != buffer->capacity) {
buffer->data = gumbo_realloc(buffer->data, new_capacity);
buffer->capacity = new_capacity;
}
}
void gumbo_string_buffer_init(GumboStringBuffer* output) {
output->data = gumbo_alloc(kDefaultStringBufferSize);
output->length = 0;
output->capacity = kDefaultStringBufferSize;
}
void gumbo_string_buffer_reserve (
size_t min_capacity,
GumboStringBuffer* output
) {
maybe_resize_string_buffer(min_capacity - output->length, output);
}
void gumbo_string_buffer_append_codepoint (
int c,
GumboStringBuffer* output
) {
// num_bytes is actually the number of continuation bytes, 1 less than the
// total number of bytes. This is done to keep the loop below simple and
// should probably change if we unroll it.
int num_bytes, prefix;
if (c <= 0x7f) {
num_bytes = 0;
prefix = 0;
} else if (c <= 0x7ff) {
num_bytes = 1;
prefix = 0xc0;
} else if (c <= 0xffff) {
num_bytes = 2;
prefix = 0xe0;
} else {
num_bytes = 3;
prefix = 0xf0;
}
maybe_resize_string_buffer(num_bytes + 1, output);
output->data[output->length++] = prefix | (c >> (num_bytes * 6));
for (int i = num_bytes - 1; i >= 0; --i) {
output->data[output->length++] = 0x80 | (0x3f & (c >> (i * 6)));
}
}
void gumbo_string_buffer_append_string (
const GumboStringPiece* str,
GumboStringBuffer* output
) {
maybe_resize_string_buffer(str->length, output);
memcpy(output->data + output->length, str->data, str->length);
output->length += str->length;
}
char* gumbo_string_buffer_to_string(const GumboStringBuffer* input) {
char* buffer = gumbo_alloc(input->length + 1);
memcpy(buffer, input->data, input->length);
buffer[input->length] = '\0';
return buffer;
}
void gumbo_string_buffer_clear(GumboStringBuffer* input) {
input->length = 0;
}
void gumbo_string_buffer_destroy(GumboStringBuffer* buffer) {
gumbo_free(buffer->data);
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/string_buffer.h
Vendored
+68
View File
@@ -0,0 +1,68 @@
#ifndef GUMBO_STRING_BUFFER_H_
#define GUMBO_STRING_BUFFER_H_
#include <stdbool.h>
#include <stddef.h>
#include "nokogiri_gumbo.h"
#ifdef __cplusplus
extern "C" {
#endif
// A struct representing a mutable, growable string. This consists of a
// heap-allocated buffer that may grow (by doubling) as necessary. When
// converting to a string, this allocates a new buffer that is only as long as
// it needs to be. Note that the internal buffer here is *not* nul-terminated,
// so be sure not to use ordinary string manipulation functions on it.
typedef struct {
// A pointer to the beginning of the string. NULL if length == 0.
char* data;
// The length of the string fragment, in bytes. May be zero.
size_t length;
// The capacity of the buffer, in bytes.
size_t capacity;
} GumboStringBuffer;
// Initializes a new GumboStringBuffer.
void gumbo_string_buffer_init(GumboStringBuffer* output);
// Ensures that the buffer contains at least a certain amount of space. Most
// useful with snprintf and the other length-delimited string functions, which
// may want to write directly into the buffer.
void gumbo_string_buffer_reserve (
size_t min_capacity,
GumboStringBuffer* output
);
// Appends a single Unicode codepoint onto the end of the GumboStringBuffer.
// This is essentially a UTF-8 encoder, and may add 1-4 bytes depending on the
// value of the codepoint.
void gumbo_string_buffer_append_codepoint (
int c,
GumboStringBuffer* output
);
// Appends a string onto the end of the GumboStringBuffer.
void gumbo_string_buffer_append_string (
const GumboStringPiece* str,
GumboStringBuffer* output
);
// Converts this string buffer to const char*, alloctaing a new buffer for it.
char* gumbo_string_buffer_to_string(const GumboStringBuffer* input);
// Reinitialize this string buffer. This clears it by setting length=0. It
// does not zero out the buffer itself.
void gumbo_string_buffer_clear(GumboStringBuffer* input);
// Deallocates this GumboStringBuffer.
void gumbo_string_buffer_destroy(GumboStringBuffer* buffer);
#ifdef __cplusplus
}
#endif
#endif // GUMBO_STRING_BUFFER_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/string_piece.c
Vendored
+48
View File
@@ -0,0 +1,48 @@
/*
Copyright 2018 Craig Barnes.
Copyright 2010 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include <stddef.h>
#include <string.h>
#include "nokogiri_gumbo.h"
#include "ascii.h"
bool gumbo_string_equals (
const GumboStringPiece* str1,
const GumboStringPiece* str2
) {
return
str1->length == str2->length
&& !memcmp(str1->data, str2->data, str1->length);
}
bool gumbo_string_equals_ignore_case (
const GumboStringPiece* str1,
const GumboStringPiece* str2
) {
return
str1->length == str2->length
&& !gumbo_ascii_strncasecmp(str1->data, str2->data, str1->length);
}
bool gumbo_string_prefix_ignore_case (
const GumboStringPiece* prefix,
const GumboStringPiece* str
) {
return
prefix->length <= str->length
&& !gumbo_ascii_strncasecmp(prefix->data, str->data, prefix->length);
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/string_set.c
Vendored
+41
View File
@@ -0,0 +1,41 @@
#include "string_set.h"
#include <string.h>
#include "hashmap.h"
#define SEED0 0xf00ba2
#define SEED1 0xfa1afe1
static int
string_compare(const void *a, const void *b, void *udata) {
return strcmp(*(const char **)a, *(const char **)b);
}
static uint64_t
string_hash(const void *item, uint64_t seed0, uint64_t seed1) {
const char *str = *(const char **)item;
return hashmap_xxhash3(str, strlen(str), seed0, seed1);
}
GumboStringSet *
gumbo_string_set_new(size_t cap)
{
return hashmap_new(sizeof(char *), cap, SEED0, SEED1, string_hash, string_compare, NULL, NULL);
}
void gumbo_string_set_free(GumboStringSet *set)
{
hashmap_free(set);
}
void
gumbo_string_set_insert(GumboStringSet *set, const char *str)
{
hashmap_set(set, &str);
}
int
gumbo_string_set_contains(GumboStringSet *set, const char *str)
{
return hashmap_get(set, &str) == NULL ? 0 : 1;
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/string_set.h
Vendored
+21
View File
@@ -0,0 +1,21 @@
#ifndef STRING_SET_H
#define STRING_SET_H
#include <stddef.h>
#if defined(__cplusplus)
extern "C" {
#endif // __cplusplus
typedef struct hashmap GumboStringSet;
GumboStringSet* gumbo_string_set_new(size_t cap);
void gumbo_string_set_free(GumboStringSet *set);
void gumbo_string_set_insert(GumboStringSet *set, const char *str);
int gumbo_string_set_contains(GumboStringSet *set, const char *str);
#if defined(__cplusplus)
}
#endif // __cplusplus
#endif // STRING_SET_H
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/svg_attrs.c
Vendored
+174
View File
@@ -0,0 +1,174 @@
/* ANSI-C code produced by gperf version 3.1 */
/* Command-line: gperf -m100 src/svg_attrs.gperf */
/* Computed positions: -k'1,10,$' */
/* Filtered by: gperf-filter.sed */
#include "replacement.h"
#include "macros.h"
#include "ascii.h"
#include <string.h>
#define TOTAL_KEYWORDS 58
#define MIN_WORD_LENGTH 4
#define MAX_WORD_LENGTH 19
#define MIN_HASH_VALUE 5
#define MAX_HASH_VALUE 77
/* maximum key range = 73, duplicates = 0 */
static inline unsigned int
hash (register const char *str, register size_t len)
{
static const unsigned char asso_values[] =
{
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 5, 78, 39, 14, 1,
31, 31, 13, 13, 78, 78, 22, 25, 10, 2,
7, 78, 22, 0, 1, 3, 1, 78, 0, 36,
14, 17, 20, 78, 78, 78, 78, 5, 78, 39,
14, 1, 31, 31, 13, 13, 78, 78, 22, 25,
10, 2, 7, 78, 22, 0, 1, 3, 1, 78,
0, 36, 14, 17, 20, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78, 78, 78,
78, 78, 78, 78, 78, 78, 78, 78
};
register unsigned int hval = len;
switch (hval)
{
default:
hval += asso_values[(unsigned char)str[9]];
/*FALLTHROUGH*/
case 9:
case 8:
case 7:
case 6:
case 5:
case 4:
case 3:
case 2:
case 1:
hval += asso_values[(unsigned char)str[0]+2];
break;
}
return hval + asso_values[(unsigned char)str[len - 1]];
}
const StringReplacement *
gumbo_get_svg_attr_replacement (register const char *str, register size_t len)
{
static const unsigned char lengthtable[] =
{
0, 0, 0, 0, 0, 4, 0, 7, 7, 0, 8, 9, 10, 11,
11, 11, 11, 10, 16, 18, 16, 12, 16, 11, 13, 11, 12, 11,
16, 0, 17, 9, 9, 8, 9, 10, 13, 10, 12, 14, 8, 4,
12, 19, 7, 9, 12, 12, 11, 14, 10, 19, 8, 16, 13, 16,
16, 15, 10, 12, 0, 0, 13, 13, 13, 0, 0, 9, 16, 0,
0, 0, 0, 0, 0, 0, 0, 17
};
static const StringReplacement wordlist[] =
{
{(char*)0,(char*)0}, {(char*)0,(char*)0},
{(char*)0,(char*)0}, {(char*)0,(char*)0},
{(char*)0,(char*)0},
{"refx", "refX"},
{(char*)0,(char*)0},
{"viewbox", "viewBox"},
{"targetx", "targetX"},
{(char*)0,(char*)0},
{"calcmode", "calcMode"},
{"maskunits", "maskUnits"},
{"viewtarget", "viewTarget"},
{"tablevalues", "tableValues"},
{"markerunits", "markerUnits"},
{"stitchtiles", "stitchTiles"},
{"startoffset", "startOffset"},
{"numoctaves", "numOctaves"},
{"requiredfeatures", "requiredFeatures"},
{"requiredextensions", "requiredExtensions"},
{"specularexponent", "specularExponent"},
{"surfacescale", "surfaceScale"},
{"specularconstant", "specularConstant"},
{"repeatcount", "repeatCount"},
{"clippathunits", "clipPathUnits"},
{"filterunits", "filterUnits"},
{"lengthadjust", "lengthAdjust"},
{"markerwidth", "markerWidth"},
{"maskcontentunits", "maskContentUnits"},
{(char*)0,(char*)0},
{"limitingconeangle", "limitingConeAngle"},
{"pointsatx", "pointsAtX"},
{"repeatdur", "repeatDur"},
{"keytimes", "keyTimes"},
{"keypoints", "keyPoints"},
{"keysplines", "keySplines"},
{"gradientunits", "gradientUnits"},
{"textlength", "textLength"},
{"stddeviation", "stdDeviation"},
{"primitiveunits", "primitiveUnits"},
{"edgemode", "edgeMode"},
{"refy", "refY"},
{"spreadmethod", "spreadMethod"},
{"preserveaspectratio", "preserveAspectRatio"},
{"targety", "targetY"},
{"pointsatz", "pointsAtZ"},
{"markerheight", "markerHeight"},
{"patternunits", "patternUnits"},
{"baseprofile", "baseProfile"},
{"systemlanguage", "systemLanguage"},
{"zoomandpan", "zoomAndPan"},
{"patterncontentunits", "patternContentUnits"},
{"glyphref", "glyphRef"},
{"xchannelselector", "xChannelSelector"},
{"attributetype", "attributeType"},
{"kernelunitlength", "kernelUnitLength"},
{"ychannelselector", "yChannelSelector"},
{"diffuseconstant", "diffuseConstant"},
{"pathlength", "pathLength"},
{"kernelmatrix", "kernelMatrix"},
{(char*)0,(char*)0}, {(char*)0,(char*)0},
{"preservealpha", "preserveAlpha"},
{"attributename", "attributeName"},
{"basefrequency", "baseFrequency"},
{(char*)0,(char*)0}, {(char*)0,(char*)0},
{"pointsaty", "pointsAtY"},
{"patterntransform", "patternTransform"},
{(char*)0,(char*)0}, {(char*)0,(char*)0},
{(char*)0,(char*)0}, {(char*)0,(char*)0},
{(char*)0,(char*)0}, {(char*)0,(char*)0},
{(char*)0,(char*)0}, {(char*)0,(char*)0},
{"gradienttransform", "gradientTransform"}
};
if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
{
register unsigned int key = hash (str, len);
if (key <= MAX_HASH_VALUE)
if (len == lengthtable[key])
{
register const char *s = wordlist[key].from;
if (s && (((unsigned char)*str ^ (unsigned char)*s) & ~32) == 0 && !gumbo_ascii_strncasecmp(str, s, len))
return &wordlist[key];
}
}
return 0;
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/svg_tags.c
Vendored
+137
View File
@@ -0,0 +1,137 @@
/* ANSI-C code produced by gperf version 3.1 */
/* Command-line: gperf -m100 src/svg_tags.gperf */
/* Computed positions: -k'3,7' */
/* Filtered by: gperf-filter.sed */
#include "replacement.h"
#include "macros.h"
#include "ascii.h"
#include <string.h>
#define TOTAL_KEYWORDS 36
#define MIN_WORD_LENGTH 6
#define MAX_WORD_LENGTH 19
#define MIN_HASH_VALUE 6
#define MAX_HASH_VALUE 42
/* maximum key range = 37, duplicates = 0 */
static inline unsigned int
hash (register const char *str, register size_t len)
{
static const unsigned char asso_values[] =
{
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 12, 2, 10, 22,
1, 28, 15, 1, 43, 43, 43, 0, 9, 26,
3, 17, 1, 11, 0, 22, 5, 43, 3, 2,
43, 43, 43, 43, 43, 43, 43, 43, 12, 2,
10, 22, 1, 28, 15, 1, 43, 43, 43, 0,
9, 26, 3, 17, 1, 11, 0, 22, 5, 43,
3, 2, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43, 43, 43, 43,
43, 43, 43, 43, 43, 43, 43
};
register unsigned int hval = len;
switch (hval)
{
default:
hval += asso_values[(unsigned char)str[6]+1];
/*FALLTHROUGH*/
case 6:
case 5:
case 4:
case 3:
hval += asso_values[(unsigned char)str[2]];
break;
}
return hval;
}
const StringReplacement *
gumbo_get_svg_tag_replacement (register const char *str, register size_t len)
{
static const unsigned char lengthtable[] =
{
0, 0, 0, 0, 0, 0, 6, 0, 7, 7, 7, 8, 11, 12,
12, 13, 11, 12, 16, 7, 7, 16, 11, 7, 19, 8, 13, 17,
11, 12, 7, 8, 17, 8, 18, 8, 14, 12, 14, 14, 13, 7,
14
};
static const StringReplacement wordlist[] =
{
{(char*)0,(char*)0}, {(char*)0,(char*)0},
{(char*)0,(char*)0}, {(char*)0,(char*)0},
{(char*)0,(char*)0}, {(char*)0,(char*)0},
{"fetile", "feTile"},
{(char*)0,(char*)0},
{"femerge", "feMerge"},
{"feimage", "feImage"},
{"fefuncb", "feFuncB"},
{"glyphref", "glyphRef"},
{"femergenode", "feMergeNode"},
{"femorphology", "feMorphology"},
{"animatecolor", "animateColor"},
{"animatemotion", "animateMotion"},
{"fecomposite", "feComposite"},
{"feturbulence", "feTurbulence"},
{"animatetransform", "animateTransform"},
{"fefuncr", "feFuncR"},
{"fefunca", "feFuncA"},
{"feconvolvematrix", "feConvolveMatrix"},
{"fespotlight", "feSpotLight"},
{"fefuncg", "feFuncG"},
{"fecomponenttransfer", "feComponentTransfer"},
{"altglyph", "altGlyph"},
{"fecolormatrix", "feColorMatrix"},
{"fedisplacementmap", "feDisplacementMap"},
{"altglyphdef", "altGlyphDef"},
{"altglyphitem", "altGlyphItem"},
{"feflood", "feFlood"},
{"clippath", "clipPath"},
{"fediffuselighting", "feDiffuseLighting"},
{"textpath", "textPath"},
{"fespecularlighting", "feSpecularLighting"},
{"feoffset", "feOffset"},
{"fedistantlight", "feDistantLight"},
{"fepointlight", "fePointLight"},
{"lineargradient", "linearGradient"},
{"radialgradient", "radialGradient"},
{"foreignobject", "foreignObject"},
{"feblend", "feBlend"},
{"fegaussianblur", "feGaussianBlur"}
};
if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
{
register unsigned int key = hash (str, len);
if (key <= MAX_HASH_VALUE)
if (len == lengthtable[key])
{
register const char *s = wordlist[key].from;
if (s && (((unsigned char)*str ^ (unsigned char)*s) & ~32) == 0 && !gumbo_ascii_strncasecmp(str, s, len))
return &wordlist[key];
}
}
return 0;
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/tag.c
Vendored
+223
View File
@@ -0,0 +1,223 @@
/*
Copyright 2011 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include "nokogiri_gumbo.h"
#include "util.h"
#include "tag_lookup.h"
#include <assert.h>
#include <string.h>
static const char kGumboTagNames[GUMBO_TAG_LAST+1][15] = {
[GUMBO_TAG_HTML] = "html",
[GUMBO_TAG_HEAD] = "head",
[GUMBO_TAG_TITLE] = "title",
[GUMBO_TAG_BASE] = "base",
[GUMBO_TAG_LINK] = "link",
[GUMBO_TAG_META] = "meta",
[GUMBO_TAG_STYLE] = "style",
[GUMBO_TAG_SCRIPT] = "script",
[GUMBO_TAG_NOSCRIPT] = "noscript",
[GUMBO_TAG_TEMPLATE] = "template",
[GUMBO_TAG_BODY] = "body",
[GUMBO_TAG_ARTICLE] = "article",
[GUMBO_TAG_SECTION] = "section",
[GUMBO_TAG_NAV] = "nav",
[GUMBO_TAG_ASIDE] = "aside",
[GUMBO_TAG_H1] = "h1",
[GUMBO_TAG_H2] = "h2",
[GUMBO_TAG_H3] = "h3",
[GUMBO_TAG_H4] = "h4",
[GUMBO_TAG_H5] = "h5",
[GUMBO_TAG_H6] = "h6",
[GUMBO_TAG_HGROUP] = "hgroup",
[GUMBO_TAG_HEADER] = "header",
[GUMBO_TAG_FOOTER] = "footer",
[GUMBO_TAG_ADDRESS] = "address",
[GUMBO_TAG_P] = "p",
[GUMBO_TAG_HR] = "hr",
[GUMBO_TAG_PRE] = "pre",
[GUMBO_TAG_BLOCKQUOTE] = "blockquote",
[GUMBO_TAG_OL] = "ol",
[GUMBO_TAG_UL] = "ul",
[GUMBO_TAG_LI] = "li",
[GUMBO_TAG_DL] = "dl",
[GUMBO_TAG_DT] = "dt",
[GUMBO_TAG_DD] = "dd",
[GUMBO_TAG_FIGURE] = "figure",
[GUMBO_TAG_FIGCAPTION] = "figcaption",
[GUMBO_TAG_MAIN] = "main",
[GUMBO_TAG_DIV] = "div",
[GUMBO_TAG_A] = "a",
[GUMBO_TAG_EM] = "em",
[GUMBO_TAG_STRONG] = "strong",
[GUMBO_TAG_SMALL] = "small",
[GUMBO_TAG_S] = "s",
[GUMBO_TAG_CITE] = "cite",
[GUMBO_TAG_Q] = "q",
[GUMBO_TAG_DFN] = "dfn",
[GUMBO_TAG_ABBR] = "abbr",
[GUMBO_TAG_DATA] = "data",
[GUMBO_TAG_TIME] = "time",
[GUMBO_TAG_CODE] = "code",
[GUMBO_TAG_VAR] = "var",
[GUMBO_TAG_SAMP] = "samp",
[GUMBO_TAG_KBD] = "kbd",
[GUMBO_TAG_SUB] = "sub",
[GUMBO_TAG_SUP] = "sup",
[GUMBO_TAG_I] = "i",
[GUMBO_TAG_B] = "b",
[GUMBO_TAG_U] = "u",
[GUMBO_TAG_MARK] = "mark",
[GUMBO_TAG_RUBY] = "ruby",
[GUMBO_TAG_RT] = "rt",
[GUMBO_TAG_RP] = "rp",
[GUMBO_TAG_BDI] = "bdi",
[GUMBO_TAG_BDO] = "bdo",
[GUMBO_TAG_SPAN] = "span",
[GUMBO_TAG_BR] = "br",
[GUMBO_TAG_WBR] = "wbr",
[GUMBO_TAG_INS] = "ins",
[GUMBO_TAG_DEL] = "del",
[GUMBO_TAG_IMAGE] = "image",
[GUMBO_TAG_IMG] = "img",
[GUMBO_TAG_IFRAME] = "iframe",
[GUMBO_TAG_EMBED] = "embed",
[GUMBO_TAG_OBJECT] = "object",
[GUMBO_TAG_PARAM] = "param",
[GUMBO_TAG_VIDEO] = "video",
[GUMBO_TAG_AUDIO] = "audio",
[GUMBO_TAG_SOURCE] = "source",
[GUMBO_TAG_TRACK] = "track",
[GUMBO_TAG_CANVAS] = "canvas",
[GUMBO_TAG_MAP] = "map",
[GUMBO_TAG_AREA] = "area",
[GUMBO_TAG_MATH] = "math",
[GUMBO_TAG_MI] = "mi",
[GUMBO_TAG_MO] = "mo",
[GUMBO_TAG_MN] = "mn",
[GUMBO_TAG_MS] = "ms",
[GUMBO_TAG_MTEXT] = "mtext",
[GUMBO_TAG_MGLYPH] = "mglyph",
[GUMBO_TAG_MALIGNMARK] = "malignmark",
[GUMBO_TAG_ANNOTATION_XML] = "annotation-xml",
[GUMBO_TAG_SVG] = "svg",
[GUMBO_TAG_FOREIGNOBJECT] = "foreignobject",
[GUMBO_TAG_DESC] = "desc",
[GUMBO_TAG_TABLE] = "table",
[GUMBO_TAG_CAPTION] = "caption",
[GUMBO_TAG_COLGROUP] = "colgroup",
[GUMBO_TAG_COL] = "col",
[GUMBO_TAG_TBODY] = "tbody",
[GUMBO_TAG_THEAD] = "thead",
[GUMBO_TAG_TFOOT] = "tfoot",
[GUMBO_TAG_TR] = "tr",
[GUMBO_TAG_TD] = "td",
[GUMBO_TAG_TH] = "th",
[GUMBO_TAG_FORM] = "form",
[GUMBO_TAG_FIELDSET] = "fieldset",
[GUMBO_TAG_LEGEND] = "legend",
[GUMBO_TAG_LABEL] = "label",
[GUMBO_TAG_INPUT] = "input",
[GUMBO_TAG_BUTTON] = "button",
[GUMBO_TAG_SELECT] = "select",
[GUMBO_TAG_DATALIST] = "datalist",
[GUMBO_TAG_OPTGROUP] = "optgroup",
[GUMBO_TAG_OPTION] = "option",
[GUMBO_TAG_TEXTAREA] = "textarea",
[GUMBO_TAG_KEYGEN] = "keygen",
[GUMBO_TAG_OUTPUT] = "output",
[GUMBO_TAG_PROGRESS] = "progress",
[GUMBO_TAG_METER] = "meter",
[GUMBO_TAG_DETAILS] = "details",
[GUMBO_TAG_SUMMARY] = "summary",
[GUMBO_TAG_MENU] = "menu",
[GUMBO_TAG_MENUITEM] = "menuitem",
[GUMBO_TAG_APPLET] = "applet",
[GUMBO_TAG_ACRONYM] = "acronym",
[GUMBO_TAG_BGSOUND] = "bgsound",
[GUMBO_TAG_DIR] = "dir",
[GUMBO_TAG_FRAME] = "frame",
[GUMBO_TAG_FRAMESET] = "frameset",
[GUMBO_TAG_NOFRAMES] = "noframes",
[GUMBO_TAG_LISTING] = "listing",
[GUMBO_TAG_XMP] = "xmp",
[GUMBO_TAG_NEXTID] = "nextid",
[GUMBO_TAG_NOEMBED] = "noembed",
[GUMBO_TAG_PLAINTEXT] = "plaintext",
[GUMBO_TAG_RB] = "rb",
[GUMBO_TAG_STRIKE] = "strike",
[GUMBO_TAG_BASEFONT] = "basefont",
[GUMBO_TAG_BIG] = "big",
[GUMBO_TAG_BLINK] = "blink",
[GUMBO_TAG_CENTER] = "center",
[GUMBO_TAG_FONT] = "font",
[GUMBO_TAG_MARQUEE] = "marquee",
[GUMBO_TAG_MULTICOL] = "multicol",
[GUMBO_TAG_NOBR] = "nobr",
[GUMBO_TAG_SPACER] = "spacer",
[GUMBO_TAG_TT] = "tt",
[GUMBO_TAG_RTC] = "rtc",
[GUMBO_TAG_DIALOG] = "dialog",
[GUMBO_TAG_SEARCH] = "search",
[GUMBO_TAG_UNKNOWN] = "",
[GUMBO_TAG_LAST] = "",
};
const char* gumbo_normalized_tagname(GumboTag tag) {
assert(tag <= GUMBO_TAG_LAST);
const char *tagname = kGumboTagNames[tag];
assert(tagname);
return tagname;
}
void gumbo_tag_from_original_text(GumboStringPiece* text) {
if (text->data == NULL) {
return;
}
assert(text->length >= 2);
assert(text->data[0] == '<');
assert(text->data[text->length - 1] == '>');
if (text->data[1] == '/') {
// End tag
assert(text->length >= 3);
text->data += 2; // Move past </
text->length -= 3;
} else {
// Start tag
text->data += 1; // Move past <
text->length -= 2;
for (const char* c = text->data; c != text->data + text->length; ++c) {
switch (*c) {
case '\t':
case '\n':
case '\f':
case ' ':
case '/':
text->length = c - text->data;
return;
}
}
}
}
GumboTag gumbo_tagn_enum(const char *tagname, size_t tagname_length) {
const TagHashSlot *slot = gumbo_tag_lookup(tagname, tagname_length);
return slot ? slot->tag : GUMBO_TAG_UNKNOWN;
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/tag_lookup.c
Vendored
+382
View File
@@ -0,0 +1,382 @@
/* ANSI-C code produced by gperf version 3.1 */
/* Command-line: gperf -m100 src/tag_lookup.gperf */
/* Computed positions: -k'1-2,$' */
/* Filtered by: gperf-filter.sed */
#include "tag_lookup.h"
#include "macros.h"
#include "ascii.h"
#include <string.h>
#define TOTAL_KEYWORDS 151
#define MIN_WORD_LENGTH 1
#define MAX_WORD_LENGTH 14
#define MIN_HASH_VALUE 9
#define MAX_HASH_VALUE 271
/* maximum key range = 263, duplicates = 0 */
static inline unsigned int
hash (register const char *str, register size_t len)
{
static const unsigned short asso_values[] =
{
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 11,
7, 6, 4, 4, 3, 4, 3, 3, 272, 272,
272, 272, 272, 272, 272, 70, 83, 152, 7, 16,
61, 98, 5, 76, 102, 126, 12, 19, 54, 54,
31, 97, 3, 4, 9, 33, 136, 113, 86, 15,
272, 272, 272, 272, 272, 272, 272, 70, 83, 152,
7, 16, 61, 98, 5, 76, 102, 126, 12, 19,
54, 54, 31, 97, 3, 4, 9, 33, 136, 113,
86, 15, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
272, 272, 272, 272, 272, 272, 272, 272, 272
};
register unsigned int hval = len;
switch (hval)
{
default:
hval += asso_values[(unsigned char)str[1]+3];
/*FALLTHROUGH*/
case 1:
hval += asso_values[(unsigned char)str[0]];
break;
}
return hval + asso_values[(unsigned char)str[len - 1]];
}
const TagHashSlot *
gumbo_tag_lookup (register const char *str, register size_t len)
{
static const unsigned char lengthtable[] =
{
0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2,
2, 2, 2, 6, 2, 6, 6, 4, 2, 7, 6, 3, 0, 3,
0, 6, 6, 8, 5, 0, 0, 4, 5, 5, 8, 0, 2, 4,
5, 2, 0, 5, 4, 2, 0, 7, 0, 8, 5, 0, 0, 0,
0, 0, 0, 5, 3, 4, 5, 1, 4, 0, 4, 1, 2, 8,
7, 7, 6, 6, 8, 2, 8, 4, 2, 0, 6, 0, 0, 3,
4, 6, 13, 4, 4, 6, 8, 0, 8, 4, 0, 6, 0, 8,
4, 5, 0, 2, 2, 9, 2, 4, 0, 8, 4, 2, 4, 8,
7, 0, 2, 5, 2, 0, 6, 0, 3, 2, 2, 6, 3, 8,
7, 2, 5, 7, 0, 2, 6, 2, 4, 3, 0, 10, 5, 6,
3, 1, 2, 0, 6, 0, 5, 5, 0, 3, 0, 3, 3, 1,
4, 6, 4, 7, 3, 0, 0, 2, 10, 10, 0, 0, 6, 1,
4, 6, 3, 0, 2, 5, 6, 4, 3, 4, 0, 7, 3, 0,
0, 0, 4, 0, 0, 5, 0, 0, 0, 6, 0, 14, 8, 1,
3, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 5, 3, 0,
0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 7, 6, 0, 0,
0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0,
3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0,
0, 0, 5, 0, 0, 3
};
static const TagHashSlot wordlist[] =
{
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"s", GUMBO_TAG_S},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"h6", GUMBO_TAG_H6},
{"h5", GUMBO_TAG_H5},
{"h4", GUMBO_TAG_H4},
{"h3", GUMBO_TAG_H3},
{"spacer", GUMBO_TAG_SPACER},
{"h2", GUMBO_TAG_H2},
{"header", GUMBO_TAG_HEADER},
{"search", GUMBO_TAG_SEARCH},
{"head", GUMBO_TAG_HEAD},
{"h1", GUMBO_TAG_H1},
{"details", GUMBO_TAG_DETAILS},
{"select", GUMBO_TAG_SELECT},
{"dir", GUMBO_TAG_DIR},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"del", GUMBO_TAG_DEL},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"source", GUMBO_TAG_SOURCE},
{"legend", GUMBO_TAG_LEGEND},
{"datalist", GUMBO_TAG_DATALIST},
{"meter", GUMBO_TAG_METER},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"math", GUMBO_TAG_MATH},
{"label", GUMBO_TAG_LABEL},
{"table", GUMBO_TAG_TABLE},
{"template", GUMBO_TAG_TEMPLATE},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"rp", GUMBO_TAG_RP},
{"time", GUMBO_TAG_TIME},
{"title", GUMBO_TAG_TITLE},
{"hr", GUMBO_TAG_HR},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"tbody", GUMBO_TAG_TBODY},
{"samp", GUMBO_TAG_SAMP},
{"tr", GUMBO_TAG_TR},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"marquee", GUMBO_TAG_MARQUEE},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"menuitem", GUMBO_TAG_MENUITEM},
{"small", GUMBO_TAG_SMALL},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"embed", GUMBO_TAG_EMBED},
{"map", GUMBO_TAG_MAP},
{"menu", GUMBO_TAG_MENU},
{"param", GUMBO_TAG_PARAM},
{"p", GUMBO_TAG_P},
{"nobr", GUMBO_TAG_NOBR},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"span", GUMBO_TAG_SPAN},
{"u", GUMBO_TAG_U},
{"em", GUMBO_TAG_EM},
{"noframes", GUMBO_TAG_NOFRAMES},
{"section", GUMBO_TAG_SECTION},
{"noembed", GUMBO_TAG_NOEMBED},
{"nextid", GUMBO_TAG_NEXTID},
{"footer", GUMBO_TAG_FOOTER},
{"noscript", GUMBO_TAG_NOSCRIPT},
{"dl", GUMBO_TAG_DL},
{"progress", GUMBO_TAG_PROGRESS},
{"font", GUMBO_TAG_FONT},
{"mo", GUMBO_TAG_MO},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"script", GUMBO_TAG_SCRIPT},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"pre", GUMBO_TAG_PRE},
{"main", GUMBO_TAG_MAIN},
{"object", GUMBO_TAG_OBJECT},
{"foreignobject", GUMBO_TAG_FOREIGNOBJECT},
{"form", GUMBO_TAG_FORM},
{"data", GUMBO_TAG_DATA},
{"applet", GUMBO_TAG_APPLET},
{"fieldset", GUMBO_TAG_FIELDSET},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"textarea", GUMBO_TAG_TEXTAREA},
{"abbr", GUMBO_TAG_ABBR},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"figure", GUMBO_TAG_FIGURE},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"optgroup", GUMBO_TAG_OPTGROUP},
{"meta", GUMBO_TAG_META},
{"tfoot", GUMBO_TAG_TFOOT},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"ul", GUMBO_TAG_UL},
{"li", GUMBO_TAG_LI},
{"plaintext", GUMBO_TAG_PLAINTEXT},
{"rb", GUMBO_TAG_RB},
{"body", GUMBO_TAG_BODY},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"basefont", GUMBO_TAG_BASEFONT},
{"ruby", GUMBO_TAG_RUBY},
{"mi", GUMBO_TAG_MI},
{"base", GUMBO_TAG_BASE},
{"frameset", GUMBO_TAG_FRAMESET},
{"summary", GUMBO_TAG_SUMMARY},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"dd", GUMBO_TAG_DD},
{"frame", GUMBO_TAG_FRAME},
{"td", GUMBO_TAG_TD},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"option", GUMBO_TAG_OPTION},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"svg", GUMBO_TAG_SVG},
{"br", GUMBO_TAG_BR},
{"ol", GUMBO_TAG_OL},
{"dialog", GUMBO_TAG_DIALOG},
{"sup", GUMBO_TAG_SUP},
{"multicol", GUMBO_TAG_MULTICOL},
{"article", GUMBO_TAG_ARTICLE},
{"rt", GUMBO_TAG_RT},
{"image", GUMBO_TAG_IMAGE},
{"listing", GUMBO_TAG_LISTING},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"dt", GUMBO_TAG_DT},
{"mglyph", GUMBO_TAG_MGLYPH},
{"tt", GUMBO_TAG_TT},
{"html", GUMBO_TAG_HTML},
{"wbr", GUMBO_TAG_WBR},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"figcaption", GUMBO_TAG_FIGCAPTION},
{"style", GUMBO_TAG_STYLE},
{"strike", GUMBO_TAG_STRIKE},
{"dfn", GUMBO_TAG_DFN},
{"a", GUMBO_TAG_A},
{"th", GUMBO_TAG_TH},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"hgroup", GUMBO_TAG_HGROUP},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"mtext", GUMBO_TAG_MTEXT},
{"thead", GUMBO_TAG_THEAD},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"var", GUMBO_TAG_VAR},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"xmp", GUMBO_TAG_XMP},
{"kbd", GUMBO_TAG_KBD},
{"i", GUMBO_TAG_I},
{"link", GUMBO_TAG_LINK},
{"output", GUMBO_TAG_OUTPUT},
{"mark", GUMBO_TAG_MARK},
{"acronym", GUMBO_TAG_ACRONYM},
{"div", GUMBO_TAG_DIV},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"ms", GUMBO_TAG_MS},
{"malignmark", GUMBO_TAG_MALIGNMARK},
{"blockquote", GUMBO_TAG_BLOCKQUOTE},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"center", GUMBO_TAG_CENTER},
{"b", GUMBO_TAG_B},
{"desc", GUMBO_TAG_DESC},
{"canvas", GUMBO_TAG_CANVAS},
{"col", GUMBO_TAG_COL},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"mn", GUMBO_TAG_MN},
{"track", GUMBO_TAG_TRACK},
{"iframe", GUMBO_TAG_IFRAME},
{"code", GUMBO_TAG_CODE},
{"sub", GUMBO_TAG_SUB},
{"area", GUMBO_TAG_AREA},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"address", GUMBO_TAG_ADDRESS},
{"ins", GUMBO_TAG_INS},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"cite", GUMBO_TAG_CITE},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"input", GUMBO_TAG_INPUT},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"keygen", GUMBO_TAG_KEYGEN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"annotation-xml", GUMBO_TAG_ANNOTATION_XML},
{"colgroup", GUMBO_TAG_COLGROUP},
{"q", GUMBO_TAG_Q},
{"big", GUMBO_TAG_BIG},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"bgsound", GUMBO_TAG_BGSOUND},
{"nav", GUMBO_TAG_NAV},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"video", GUMBO_TAG_VIDEO},
{"img", GUMBO_TAG_IMG},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"audio", GUMBO_TAG_AUDIO},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"caption", GUMBO_TAG_CAPTION},
{"strong", GUMBO_TAG_STRONG},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"aside", GUMBO_TAG_ASIDE},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"button", GUMBO_TAG_BUTTON},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"bdo", GUMBO_TAG_BDO},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"bdi", GUMBO_TAG_BDI},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"blink", GUMBO_TAG_BLINK},
{(char*)0,GUMBO_TAG_UNKNOWN},
{(char*)0,GUMBO_TAG_UNKNOWN},
{"rtc", GUMBO_TAG_RTC}
};
if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH)
{
register unsigned int key = hash (str, len);
if (key <= MAX_HASH_VALUE)
if (len == lengthtable[key])
{
register const char *s = wordlist[key].key;
if (s && (((unsigned char)*str ^ (unsigned char)*s) & ~32) == 0 && !gumbo_ascii_strncasecmp(str, s, len))
return &wordlist[key];
}
}
return 0;
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/tag_lookup.h
Vendored
+13
View File
@@ -0,0 +1,13 @@
#ifndef GUMBO_TAG_LOOKUP_H_
#define GUMBO_TAG_LOOKUP_H_
#include "nokogiri_gumbo.h"
typedef struct {
const char *key;
const GumboTag tag;
} TagHashSlot;
const TagHashSlot *gumbo_tag_lookup(const char *str, size_t len);
#endif // GUMBO_TAG_LOOKUP_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/token_buffer.c
Vendored
+79
View File
@@ -0,0 +1,79 @@
/*
Copyright 2018 Stephen Checkoway
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include <assert.h>
#include "ascii.h"
#include "token_buffer.h"
#include "tokenizer.h"
#include "util.h"
struct GumboInternalCharacterToken {
GumboSourcePosition position;
GumboStringPiece original_text;
int c;
};
void gumbo_character_token_buffer_init(GumboCharacterTokenBuffer* buffer) {
buffer->data = NULL;
buffer->length = 0;
buffer->capacity = 0;
}
void gumbo_character_token_buffer_append (
const GumboToken* token,
GumboCharacterTokenBuffer* buffer
) {
assert(token->type == GUMBO_TOKEN_WHITESPACE
|| token->type == GUMBO_TOKEN_CHARACTER);
if (buffer->length == buffer->capacity) {
if (buffer->capacity == 0)
buffer->capacity = 10;
else
buffer->capacity *= 2;
size_t bytes = sizeof(*buffer->data) * buffer->capacity;
buffer->data = gumbo_realloc(buffer->data, bytes);
}
size_t index = buffer->length++;
buffer->data[index].position = token->position;
buffer->data[index].original_text = token->original_text;
buffer->data[index].c = token->v.character;
}
void gumbo_character_token_buffer_get (
const GumboCharacterTokenBuffer* buffer,
size_t index,
struct GumboInternalToken* output
) {
assert(index < buffer->length);
int c = buffer->data[index].c;
output->type = gumbo_ascii_isspace(c)?
GUMBO_TOKEN_WHITESPACE : GUMBO_TOKEN_CHARACTER;
output->position = buffer->data[index].position;
output->original_text = buffer->data[index].original_text;
output->v.character = c;
}
void gumbo_character_token_buffer_clear(GumboCharacterTokenBuffer* buffer) {
buffer->length = 0;
}
void gumbo_character_token_buffer_destroy(GumboCharacterTokenBuffer* buffer) {
gumbo_free(buffer->data);
buffer->data = NULL;
buffer->length = 0;
buffer->capacity = 0;
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/token_buffer.h
Vendored
+71
View File
@@ -0,0 +1,71 @@
/*
Copyright 2018 Stephen Checkoway
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#ifndef GUMBO_TOKEN_BUFFER_H
#define GUMBO_TOKEN_BUFFER_H
#include <stdbool.h>
#include <stddef.h>
#include "nokogiri_gumbo.h"
#ifdef __cplusplus
extern "C" {
#endif
struct GumboInternalCharacterToken;
struct GumboInternalToken;
// A struct representing a growable sequence of character (and whitespace)
// tokens.
typedef struct {
// A pointer to the start of the sequence.
struct GumboInternalCharacterToken* data;
// The length of the sequence.
size_t length;
// The capacity of the buffer.
size_t capacity;
} GumboCharacterTokenBuffer;
// Initializes a new GumboCharacterTokenBuffer.
void gumbo_character_token_buffer_init(GumboCharacterTokenBuffer* buffer);
// Appends a character (or whitespace) token.
void gumbo_character_token_buffer_append (
const struct GumboInternalToken* token,
GumboCharacterTokenBuffer* buffer
);
void gumbo_character_token_buffer_get (
const GumboCharacterTokenBuffer* buffer,
size_t index,
struct GumboInternalToken* output
);
// Reinitialize this string buffer. This clears it by setting length=0. It
// does not zero out the buffer itself.
void gumbo_character_token_buffer_clear(GumboCharacterTokenBuffer* buffer);
// Deallocates this GumboCharacterTokenBuffer.
void gumbo_character_token_buffer_destroy(GumboCharacterTokenBuffer* buffer);
#ifdef __cplusplus
}
#endif
#endif // GUMBO_TOKEN_BUFFER_H
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/token_type.h
Vendored
+17
View File
@@ -0,0 +1,17 @@
#ifndef GUMBO_TOKEN_TYPE_H_
#define GUMBO_TOKEN_TYPE_H_
// An enum representing the type of token.
typedef enum {
GUMBO_TOKEN_DOCTYPE,
GUMBO_TOKEN_START_TAG,
GUMBO_TOKEN_END_TAG,
GUMBO_TOKEN_COMMENT,
GUMBO_TOKEN_WHITESPACE,
GUMBO_TOKEN_CHARACTER,
GUMBO_TOKEN_CDATA,
GUMBO_TOKEN_NULL,
GUMBO_TOKEN_EOF
} GumboTokenType;
#endif // GUMBO_TOKEN_TYPE_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/tokenizer.c
Vendored
+3494
View File
File diff suppressed because it is too large Load Diff
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/tokenizer.h
Vendored
+112
View File
@@ -0,0 +1,112 @@
#ifndef GUMBO_TOKENIZER_H_
#define GUMBO_TOKENIZER_H_
// This contains an implementation of a tokenizer for HTML5. It consumes a
// buffer of UTF-8 characters, and then emits a stream of tokens.
#include <stdbool.h>
#include <stddef.h>
#include "nokogiri_gumbo.h"
#include "token_type.h"
#include "tokenizer_states.h"
#ifdef __cplusplus
extern "C" {
#endif
struct GumboInternalParser;
// Struct containing all information pertaining to doctype tokens.
typedef struct GumboInternalTokenDocType {
const char* name;
const char* public_identifier;
const char* system_identifier;
bool force_quirks;
// There's no way to tell a 0-length public or system ID apart from the
// absence of a public or system ID, but they're handled different by the
// spec, so we need bool flags for them.
bool has_public_identifier;
bool has_system_identifier;
} GumboTokenDocType;
// Struct containing all information pertaining to start tag tokens.
typedef struct GumboInternalTokenStartTag {
GumboTag tag;
// NULL unless tag is GUMBO_TAG_UNKNOWN
char *name;
GumboVector /* GumboAttribute */ attributes;
bool is_self_closing;
} GumboTokenStartTag;
// Struct containing all information pertaining to end tag tokens.
typedef struct GumboInternalTokenEndTag {
GumboTag tag;
// NULL unless tag is GUMBO_TAG_UNKNOWN
char *name;
} GumboTokenEndTag;
// A data structure representing a single token in the input stream. This
// contains an enum for the type, the source position, a GumboStringPiece
// pointing to the original text, and then a union for any parsed data.
typedef struct GumboInternalToken {
GumboTokenType type;
GumboSourcePosition position;
GumboStringPiece original_text;
union {
GumboTokenDocType doc_type;
GumboTokenStartTag start_tag;
GumboTokenEndTag end_tag;
const char* text; // For comments.
int character; // For character, whitespace, null, and EOF tokens.
} v;
} GumboToken;
// Initializes the tokenizer state within the GumboParser object, setting up a
// parse of the specified text.
void gumbo_tokenizer_state_init (
struct GumboInternalParser* parser,
const char* text,
size_t text_length
);
// Destroys the tokenizer state within the GumboParser object, freeing any
// dynamically-allocated structures within it.
void gumbo_tokenizer_state_destroy(struct GumboInternalParser* parser);
// Sets the tokenizer state to the specified value. This is needed by some
// parser states, which alter the state of the tokenizer in response to tags
// seen.
void gumbo_tokenizer_set_state (
struct GumboInternalParser* parser,
GumboTokenizerEnum state
);
// Flags whether the adjusted current node is a foreign content element. This
// is necessary for the markup declaration open state, where the tokenizer
// must be aware of the state of the parser to properly tokenize bad comment
// tags.
// https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state
void gumbo_tokenizer_set_is_adjusted_current_node_foreign (
struct GumboInternalParser* parser,
bool is_foreign
);
// Lexes a single token from the specified buffer, filling the output with the
// parsed GumboToken data structure.
void gumbo_lex(struct GumboInternalParser* parser, GumboToken* output);
// Frees the internally-allocated pointers within a GumboToken. Note that this
// doesn't free the token itself, since oftentimes it will be allocated on the
// stack.
//
// Note that if you are handing over ownership of the internal strings to some
// other data structure - for example, a parse tree - these do not need to be
// freed.
void gumbo_token_destroy(GumboToken* token);
#ifdef __cplusplus
}
#endif
#endif // GUMBO_TOKENIZER_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/tokenizer_states.h
Vendored
+339
View File
@@ -0,0 +1,339 @@
#ifndef GUMBO_TOKENIZER_STATES_H_
#define GUMBO_TOKENIZER_STATES_H_
// This contains the list of states used in the tokenizer. Although at first
// glance it seems like these could be kept internal to the tokenizer, several
// of the actions in the parser require that it reach into the tokenizer and
// reset the tokenizer state. For that to work, it needs to have the
// definitions of individual states available.
//
// This may also be useful for providing more detailed error messages for parse
// errors, as we can match up states and inputs in a table without having to
// clutter the tokenizer code with lots of precise error messages.
// The ordering of this enum is also used to build the dispatch table for the
// tokenizer state machine, so if it is changed, be sure to update that too.
typedef enum {
// 12.2.5.1 Data state
// https://html.spec.whatwg.org/multipage/parsing.html#data-state
GUMBO_LEX_DATA,
// 12.2.5.2 RCDATA state
// https://html.spec.whatwg.org/multipage/parsing.html#rcdata-state
GUMBO_LEX_RCDATA,
// 12.2.5.3 RAWTEXT state
// https://html.spec.whatwg.org/multipage/parsing.html#rawtext-state<Paste>
GUMBO_LEX_RAWTEXT,
// 12.2.5.4 Script data state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-state
GUMBO_LEX_SCRIPT_DATA,
// 12.2.5.5 PLAINTEXT state
// https://html.spec.whatwg.org/multipage/parsing.html#plaintext-state
GUMBO_LEX_PLAINTEXT,
// 12.2.5.6 Tag open state
// https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
GUMBO_LEX_TAG_OPEN,
// 12.2.5.7 End tag open state
// https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state
GUMBO_LEX_END_TAG_OPEN,
// 12.2.5.8 Tag name state
// https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
GUMBO_LEX_TAG_NAME,
// 12.2.5.9 RCDATA less-than sign state
// https://html.spec.whatwg.org/multipage/parsing.html#rcdata-less-than-sign-state
GUMBO_LEX_RCDATA_LT,
// 12.2.5.10 RCDATA end tag open state
// https://html.spec.whatwg.org/multipage/parsing.html#rcdata-end-tag-open-state
GUMBO_LEX_RCDATA_END_TAG_OPEN,
// 12.2.5.11 RCDATA end tag name state
// https://html.spec.whatwg.org/multipage/parsing.html#rcdata-end-tag-name-state
GUMBO_LEX_RCDATA_END_TAG_NAME,
// 12.2.5.12 RAWTEXT less-than sign state
// https://html.spec.whatwg.org/multipage/parsing.html#rawtext-less-than-sign-state
GUMBO_LEX_RAWTEXT_LT,
// 12.2.5.13 RAWTEXT end tag open state
// https://html.spec.whatwg.org/multipage/parsing.html#rawtext-end-tag-open-state
GUMBO_LEX_RAWTEXT_END_TAG_OPEN,
// 12.2.5.14 RAWTEXT end tag name state
// https://html.spec.whatwg.org/multipage/parsing.html#rawtext-end-tag-name-state
GUMBO_LEX_RAWTEXT_END_TAG_NAME,
// 12.2.5.15 Script data less-than sign state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-less-than-sign-state
GUMBO_LEX_SCRIPT_DATA_LT,
// 12.2.5.16 Script data end tag open state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-end-tag-open-state
GUMBO_LEX_SCRIPT_DATA_END_TAG_OPEN,
// 12.2.5.17 Script data end tag name state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-end-tag-name-state
GUMBO_LEX_SCRIPT_DATA_END_TAG_NAME,
// 12.2.5.18 Script data escape start state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escape-start-state
GUMBO_LEX_SCRIPT_DATA_ESCAPED_START,
// 12.2.5.19 Script data escape start dash state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escape-start-dash-state
GUMBO_LEX_SCRIPT_DATA_ESCAPED_START_DASH,
// 12.2.5.20 Script data escaped state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-state
GUMBO_LEX_SCRIPT_DATA_ESCAPED,
// 12.2.5.21 Script data escaped dash state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-dash-state
GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH,
// 12.2.5.22 Script data escaped dash dash state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-dash-dash-state
GUMBO_LEX_SCRIPT_DATA_ESCAPED_DASH_DASH,
// 12.2.5.23 Script data escaped less than sign state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-less-than-sign-state
GUMBO_LEX_SCRIPT_DATA_ESCAPED_LT,
// 12.2.5.24 Script data escaped end tag open state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-end-tag-open-state
GUMBO_LEX_SCRIPT_DATA_ESCAPED_END_TAG_OPEN,
// 12.2.5.25 Script data escaped end tag name state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-escaped-end-tag-name-state
GUMBO_LEX_SCRIPT_DATA_ESCAPED_END_TAG_NAME,
// 12.2.5.26 Script data double escape start state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escape-start-state
GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_START,
// 12.2.5.27 Script data double escaped state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-state
GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED,
// 12.2.5.28 Script data double escaped dash state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-dash-state
GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_DASH,
// 12.2.5.29 Script data double escaped dash dash state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-dash-dash-state
GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH,
// 12.2.5.30 Script data double escaped less-than sign state
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escaped-less-than-sign-state
GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_LT,
// 12.2.5.31 Script data double escape end state (XXX: spec bug with the
// name?)
// https://html.spec.whatwg.org/multipage/parsing.html#script-data-double-escape-end-state
GUMBO_LEX_SCRIPT_DATA_DOUBLE_ESCAPED_END,
// 12.2.5.32 Before attribute name state
// https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-name-state
GUMBO_LEX_BEFORE_ATTR_NAME,
// 12.2.5.33 Attributet name state
// https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state
GUMBO_LEX_ATTR_NAME,
// 12.2.5.34 After attribute name state
// https://html.spec.whatwg.org/multipage/parsing.html#after-attribute-name-state
GUMBO_LEX_AFTER_ATTR_NAME,
// 12.2.5.35 Before attribute value state
// https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-value-state
GUMBO_LEX_BEFORE_ATTR_VALUE,
// 12.2.5.36 Attribute value (double-quoted) state
// https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-(double-quoted)-state
GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED,
// 12.2.5.37 Attribute value (single-quoted) state
// https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-(single-quoted)-state
GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED,
// 12.2.5.38 Attribute value (unquoted) state
// https://html.spec.whatwg.org/multipage/parsing.html#attribute-value-(unquoted)-state
GUMBO_LEX_ATTR_VALUE_UNQUOTED,
// 12.2.5.39 After attribute value (quoted) state
// https://html.spec.whatwg.org/multipage/parsing.html#after-attribute-value-(quoted)-state
GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED,
// 12.2.5.40 Self-closing start tag state
// https://html.spec.whatwg.org/multipage/parsing.html#self-closing-start-tag-state
GUMBO_LEX_SELF_CLOSING_START_TAG,
// 12.2.5.41 Bogus comment state
// https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state
GUMBO_LEX_BOGUS_COMMENT,
// 12.2.5.42 Markup declaration open state
// https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state
GUMBO_LEX_MARKUP_DECLARATION_OPEN,
// 12.2.5.43 Comment start state
// https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state
GUMBO_LEX_COMMENT_START,
// 12.2.5.44 Comment start dash state
// https://html.spec.whatwg.org/multipage/parsing.html#comment-start-dash-state
GUMBO_LEX_COMMENT_START_DASH,
// 12.2.5.45 Comment state
// https://html.spec.whatwg.org/multipage/parsing.html#comment-state
GUMBO_LEX_COMMENT,
// 12.2.5.46 Comment less-than sign state
// https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-state
GUMBO_LEX_COMMENT_LT,
// 12.2.5.47 Comment less-than sign bang state
// https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-state
GUMBO_LEX_COMMENT_LT_BANG,
// 12.2.5.48 Comment less-than sign bang dash state
// https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-dash-state
GUMBO_LEX_COMMENT_LT_BANG_DASH,
// 12.2.5.49 Comment less-than sign bang dash dash state
// https://html.spec.whatwg.org/multipage/parsing.html#comment-less-than-sign-bang-dash-dash-state
GUMBO_LEX_COMMENT_LT_BANG_DASH_DASH,
// 12.2.5.50 Comment end dash state
// https://html.spec.whatwg.org/multipage/parsing.html#comment-end-dash-state
GUMBO_LEX_COMMENT_END_DASH,
// 12.2.5.51 Comment end state
// https://html.spec.whatwg.org/multipage/parsing.html#comment-end-state
GUMBO_LEX_COMMENT_END,
// 12.2.5.52 Comment end bang state
// https://html.spec.whatwg.org/multipage/parsing.html#comment-end-bang-state
GUMBO_LEX_COMMENT_END_BANG,
// 12.2.5.53 DOCTYPE state
// https://html.spec.whatwg.org/multipage/parsing.html#doctype-state
GUMBO_LEX_DOCTYPE,
// 12.2.5.54 Before DOCTYPE name state
// https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-name-state
GUMBO_LEX_BEFORE_DOCTYPE_NAME,
// 12.2.5.55 DOCTYPE name state
// https://html.spec.whatwg.org/multipage/parsing.html#doctype-name-state
GUMBO_LEX_DOCTYPE_NAME,
// 12.2.5.56 After DOCTYPE name state
// https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-name-state
GUMBO_LEX_AFTER_DOCTYPE_NAME,
// 12.2.5.57 After DOCTYPE public keyword state
// https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-public-keyword-state
GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_KEYWORD,
// 12.2.5.58 Before DOCTYPE public identifier state
// https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-public-identifier-state
GUMBO_LEX_BEFORE_DOCTYPE_PUBLIC_ID,
// 12.2.5.59 DOCTYPE public identifier (double-quoted) state
// https://html.spec.whatwg.org/multipage/parsing.html#doctype-public-identifier-(double-quoted)-state
GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED,
// 12.2.5.60 DOCTYPE public identifier (single-quoted) state
// https://html.spec.whatwg.org/multipage/parsing.html#doctype-public-identifier-(single-quoted)-state
GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED,
// 12.2.5.61 After DOCTYPE public identifier state
// https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-public-identifier-state
GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID,
// 12.2.5.62 Between DOCTYPE public and system identifiers state
// https://html.spec.whatwg.org/multipage/parsing.html#between-doctype-public-and-system-identifiers-state
GUMBO_LEX_BETWEEN_DOCTYPE_PUBLIC_SYSTEM_ID,
// 12.2.5.63 After DOCTYPE system keyword state
// https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-system-keyword-state
GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_KEYWORD,
// 12.2.5.64 Before DOCTYPE system identifier state
// https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-system-identifier-state
GUMBO_LEX_BEFORE_DOCTYPE_SYSTEM_ID,
// 12.2.5.65 DOCTYPE system identifier (double-quoted) state
// https://html.spec.whatwg.org/multipage/parsing.html#doctype-system-identifier-(double-quoted)-state
GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED,
// 12.2.5.66 DOCTYPE system identifier (single-quoted) state
// https://html.spec.whatwg.org/multipage/parsing.html#doctype-system-identifier-(single-quoted)-state
GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED,
// 12.2.5.67 After DOCTYPE system identifier state
// https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-system-identifier-state
GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID,
// 12.2.5.68 Bogus DOCTYPE state
// https://html.spec.whatwg.org/multipage/parsing.html#bogus-doctype-state
GUMBO_LEX_BOGUS_DOCTYPE,
// 12.2.5.69 CDATA section state
// https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-state
GUMBO_LEX_CDATA_SECTION,
// 12.2.5.70 CDATA section bracket state
// https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-bracket-state
GUMBO_LEX_CDATA_SECTION_BRACKET,
// 12.2.5.71 CDATA section end state
// https://html.spec.whatwg.org/multipage/parsing.html#cdata-section-end-state
GUMBO_LEX_CDATA_SECTION_END,
// 12.2.5.72 Character reference state
// https://html.spec.whatwg.org/multipage/parsing.html#character-reference-state
GUMBO_LEX_CHARACTER_REFERENCE,
// 12.2.5.73 Named character reference state
// https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
GUMBO_LEX_NAMED_CHARACTER_REFERENCE,
// 12.2.5.74 Ambiguous ampersand state
// https://html.spec.whatwg.org/multipage/parsing.html#ambiguous-ampersand-state
GUMBO_LEX_AMBIGUOUS_AMPERSAND,
// 12.2.5.75 Numeric character reference state
// https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-state
GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE,
// 12.2.5.76 Hexadecimal character reference start state
// https://html.spec.whatwg.org/multipage/parsing.html#hexadecimal-character-reference-start-state
GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE_START,
// 12.2.5.77 Decimal character reference start state
// https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-start-state
GUMBO_LEX_DECIMAL_CHARACTER_REFERENCE_START,
// 12.2.5.78 Hexadecimal character reference state
// https://html.spec.whatwg.org/multipage/parsing.html#hexadecimal-character-reference-state
GUMBO_LEX_HEXADECIMAL_CHARACTER_REFERENCE,
// 12.2.5.79 Decimal character reference state
// https://html.spec.whatwg.org/multipage/parsing.html#decimal-character-reference-state
GUMBO_LEX_DECIMAL_CHARACTER_REFERENCE,
// 12.2.5.80 Numeric character reference end state
// https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
GUMBO_LEX_NUMERIC_CHARACTER_REFERENCE_END
} GumboTokenizerEnum;
#endif // GUMBO_TOKENIZER_STATES_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/utf8.c
Vendored
+245
View File
@@ -0,0 +1,245 @@
/*
Copyright 2018 Craig Barnes.
Copyright 2010 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include "utf8.h"
#include <assert.h>
#include <stdint.h>
#include <string.h>
#include "error.h"
#include "nokogiri_gumbo.h"
#include "parser.h"
#include "ascii.h"
#include "vector.h"
// References:
// * https://tools.ietf.org/html/rfc3629
// * https://html.spec.whatwg.org/multipage/parsing.html#preprocessing-the-input-stream
// The following code is a DFA-based UTF-8 decoder by Bjoern Hoehrmann.
// We wrap the inner table-based decoder routine in our own handling for
// newlines, tabs, invalid continuation bytes, and other conditions that
// the HTML5 spec fully specifies but normal UTF-8 decoders do not handle.
// See https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
// Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
#define UTF8_ACCEPT 0
#define UTF8_REJECT 12
static const uint8_t utf8d[] = {
// The first part of the table maps bytes to character classes that
// to reduce the size of the transition table and create bitmasks.
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
// The second part is a transition table that maps a combination
// of a state of the automaton and a character class to a state.
0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
12,36,12,12,12,12,12,12,12,12,12,12,
};
static inline uint32_t decode(uint32_t* state, uint32_t* codep, uint32_t byte) {
uint32_t type = utf8d[byte];
*codep =
(*state != UTF8_ACCEPT)
? (byte & 0x3fu) | (*codep << 6)
: (0xff >> type) & (byte);
*state = utf8d[256 + *state + type];
return *state;
}
// END COPIED CODE.
// Adds a decoding error to the parser's error list, based on the current state
// of the Utf8Iterator.
static void add_error(Utf8Iterator* iter, GumboErrorType type) {
GumboParser* parser = iter->_parser;
GumboError* error = gumbo_add_error(parser);
if (!error) {
return;
}
error->type = type;
error->position = iter->_pos;
error->original_text.data = iter->_start;
error->original_text.length = iter->_width;
error->v.tokenizer.codepoint = iter->_current;
}
// Reads the next UTF-8 character in the iter.
// This assumes that iter->_start points to the beginning of the character.
// When this method returns, iter->_width and iter->_current will be set
// appropriately, as well as any error flags.
static void read_char(Utf8Iterator* iter) {
if (iter->_start >= iter->_end) {
// No input left to consume; emit an EOF and set width = 0.
iter->_current = -1;
iter->_width = 0;
return;
}
uint32_t code_point = 0;
uint32_t state = UTF8_ACCEPT;
for (const char* c = iter->_start; c < iter->_end; ++c) {
decode(&state, &code_point, (uint32_t)(unsigned char) (*c));
if (state == UTF8_ACCEPT) {
iter->_width = c - iter->_start + 1;
// This is the special handling for carriage returns that is mandated by
// the HTML5 spec. Since we're looking for particular 7-bit literal
// characters, we operate in terms of chars and only need a check for iter
// overrun, instead of having to read in a full next code point.
// https://html.spec.whatwg.org/multipage/parsing.html#preprocessing-the-input-stream
if (code_point == '\r') {
assert(iter->_width == 1);
const char* next = c + 1;
if (next < iter->_end && *next == '\n') {
// Advance the iter, as if the carriage return didn't exist.
++iter->_start;
// Preserve the true offset, since other tools that look at it may be
// unaware of HTML5's rules for converting \r into \n.
++iter->_pos.offset;
}
code_point = '\n';
}
iter->_current = code_point;
if (utf8_is_surrogate(code_point)) {
add_error(iter, GUMBO_ERR_SURROGATE_IN_INPUT_STREAM);
} else if (utf8_is_noncharacter(code_point)) {
add_error(iter, GUMBO_ERR_NONCHARACTER_IN_INPUT_STREAM);
} else if (utf8_is_control(code_point)
&& !(gumbo_ascii_isspace(code_point) || code_point == 0)) {
add_error(iter, GUMBO_ERR_CONTROL_CHARACTER_IN_INPUT_STREAM);
}
return;
} else if (state == UTF8_REJECT) {
// We don't want to consume the invalid continuation byte of a multi-byte
// run, but we do want to skip past an invalid first byte.
iter->_width = c - iter->_start + (c == iter->_start);
iter->_current = kUtf8ReplacementChar;
add_error(iter, GUMBO_ERR_UTF8_INVALID);
return;
}
}
// If we got here without exiting early, then we've reached the end of the
// iterator. Add an error for truncated input, set the width to consume the
// rest of the iterator, and emit a replacement character. The next time we
// enter this method, it will detect that there's no input to consume and
// output an EOF.
iter->_width = iter->_end - iter->_start;
iter->_current = kUtf8ReplacementChar;
add_error(iter, GUMBO_ERR_UTF8_TRUNCATED);
}
static void update_position(Utf8Iterator* iter) {
iter->_pos.offset += iter->_width;
if (iter->_current == '\n') {
++iter->_pos.line;
iter->_pos.column = 1;
} else if (iter->_current == '\t') {
int tab_stop = iter->_parser->_options->tab_stop;
iter->_pos.column = ((iter->_pos.column / tab_stop) + 1) * tab_stop;
} else if (iter->_current != -1) {
++iter->_pos.column;
}
}
void utf8iterator_init (
GumboParser* parser,
const char* source,
size_t source_length,
Utf8Iterator* iter
) {
iter->_start = source;
iter->_end = source + source_length;
iter->_pos.line = 1;
iter->_pos.column = 1;
iter->_pos.offset = 0;
iter->_parser = parser;
read_char(iter);
if (iter->_current == kUtf8BomChar) {
iter->_start += iter->_width;
iter->_pos.offset += iter->_width;
read_char(iter);
}
}
void utf8iterator_next(Utf8Iterator* iter) {
// We update positions based on the *last* character read, so that the first
// character following a newline is at column 1 in the next line.
update_position(iter);
iter->_start += iter->_width;
read_char(iter);
}
bool utf8iterator_maybe_consume_match (
Utf8Iterator* iter,
const char* prefix,
size_t length,
bool case_sensitive
) {
bool matched =
(iter->_start + length <= iter->_end)
&& (
case_sensitive
? !strncmp(iter->_start, prefix, length)
: !gumbo_ascii_strncasecmp(iter->_start, prefix, length)
)
;
if (matched) {
for (size_t i = 0; i < length; ++i) {
utf8iterator_next(iter);
}
return true;
} else {
return false;
}
}
void utf8iterator_mark(Utf8Iterator* iter) {
iter->_mark = iter->_start;
iter->_mark_pos = iter->_pos;
}
// Returns the current input stream position to the mark.
void utf8iterator_reset(Utf8Iterator* iter) {
iter->_start = iter->_mark;
iter->_pos = iter->_mark_pos;
read_char(iter);
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/utf8.h
Vendored
+164
View File
@@ -0,0 +1,164 @@
#ifndef GUMBO_UTF8_H_
#define GUMBO_UTF8_H_
// This contains an implementation of a UTF-8 iterator and decoder suitable for
// a HTML5 parser. This does a bit more than straight UTF-8 decoding. The
// HTML5 spec specifies that:
// 1. Decoding errors are parse errors.
// 2. Certain other codepoints (e.g. control characters) are parse errors.
// 3. Carriage returns and CR/LF groups are converted to line feeds.
// https://encoding.spec.whatwg.org/#utf-8-decode
//
// Also, we want to keep track of source positions for error handling. As a
// result, we fold all that functionality into this decoder, and can't use an
// off-the-shelf library.
//
// This header is internal-only, which is why we prefix functions with only
// utf8_ or utf8_iterator_ instead of gumbo_utf8_.
#include <stdbool.h>
#include <stddef.h>
#include "nokogiri_gumbo.h"
#include "macros.h"
#ifdef __cplusplus
extern "C" {
#endif
struct GumboInternalError;
struct GumboInternalParser;
// Unicode replacement char.
#define kUtf8ReplacementChar 0xFFFD
#define kUtf8BomChar 0xFEFF
#define kUtf8MaxChar 0x10FFFF
typedef struct GumboInternalUtf8Iterator {
// Points at the start of the code point most recently read into 'current'.
const char* _start;
// Points at the mark. The mark is initially set to the beginning of the
// input.
const char* _mark;
// Points past the end of the iter, like a past-the-end iterator in the STL.
const char* _end;
// The code point under the cursor.
int _current;
// The width in bytes of the current code point.
size_t _width;
// The SourcePosition for the current location.
GumboSourcePosition _pos;
// The SourcePosition for the mark.
GumboSourcePosition _mark_pos;
// Pointer back to the GumboParser instance, for configuration options and
// error recording.
struct GumboInternalParser* _parser;
} Utf8Iterator;
// Returns true if this Unicode code point is a surrogate.
CONST_FN static inline bool utf8_is_surrogate(int c) {
return c >= 0xD800 && c <= 0xDFFF;
}
// Returns true if this Unicode code point is a noncharacter.
CONST_FN static inline bool utf8_is_noncharacter(int c) {
return
(c >= 0xFDD0 && c <= 0xFDEF)
|| ((c & 0xFFFF) == 0xFFFE)
|| ((c & 0xFFFF) == 0xFFFF);
}
// Returns true if this Unicode code point is a control.
CONST_FN static inline bool utf8_is_control(int c) {
return ((unsigned int)c < 0x1Fu) || (c >= 0x7F && c <= 0x9F);
}
// Initializes a new Utf8Iterator from the given byte buffer. The source does
// not have to be NUL-terminated, but the length must be passed in explicitly.
void utf8iterator_init (
struct GumboInternalParser* parser,
const char* source,
size_t source_length,
Utf8Iterator* iter
);
// Advances the current position by one code point.
void utf8iterator_next(Utf8Iterator* iter);
// Returns the current code point as an integer.
static inline int utf8iterator_current(const Utf8Iterator* iter) {
return iter->_current;
}
// Retrieves and fills the output parameter with the current source position.
static inline void utf8iterator_get_position (
const Utf8Iterator* iter,
GumboSourcePosition* output
) {
*output = iter->_pos;
}
// Retrieves the marked position.
static inline GumboSourcePosition utf8iterator_get_mark_position (
const Utf8Iterator* iter
) {
return iter->_mark_pos;
}
// Retrieves a character pointer to the start of the current character.
static inline const char* utf8iterator_get_char_pointer(const Utf8Iterator* iter) {
return iter->_start;
}
// Retrieves the width of the current character.
static inline size_t utf8iterator_get_width(const Utf8Iterator* iter) {
return iter->_width;
}
// Retrieves a character pointer to 1 past the end of the buffer. This is
// necessary for certain state machines and string comparisons that would like
// to look directly for ASCII text in the buffer without going through the
// decoder.
static inline const char* utf8iterator_get_end_pointer(const Utf8Iterator* iter) {
return iter->_end;
}
// Retrieves a character pointer to the marked position.
static inline const char* utf8iterator_get_mark_pointer(const Utf8Iterator* iter) {
return iter->_mark;
}
// If the upcoming text in the buffer matches the specified prefix (which has
// length 'length'), consume it and return true. Otherwise, return false with
// no other effects. If the length of the string would overflow the buffer,
// this returns false. Note that prefix should not contain null bytes because
// of the use of strncmp/strncasecmp internally. All existing use-cases adhere
// to this.
bool utf8iterator_maybe_consume_match (
Utf8Iterator* iter,
const char* prefix,
size_t length,
bool case_sensitive
);
// "Marks" a particular location of interest in the input stream, so that it can
// later be reset() to. There's also the ability to record an error at the
// point that was marked, as oftentimes that's more useful than the last
// character before the error was detected.
void utf8iterator_mark(Utf8Iterator* iter);
// Returns the current input stream position to the mark.
void utf8iterator_reset(Utf8Iterator* iter);
#ifdef __cplusplus
}
#endif
#endif // GUMBO_UTF8_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/util.c
Vendored
+66
View File
@@ -0,0 +1,66 @@
/*
Copyright 2017-2018 Craig Barnes.
Copyright 2010 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "util.h"
#include "nokogiri_gumbo.h"
void* gumbo_alloc(size_t size) {
void* ptr = malloc(size);
if (unlikely(ptr == NULL)) {
perror(__func__);
abort();
}
return ptr;
}
void* gumbo_realloc(void* ptr, size_t size) {
ptr = realloc(ptr, size);
if (unlikely(ptr == NULL)) {
perror(__func__);
abort();
}
return ptr;
}
void gumbo_free(void* ptr) {
free(ptr);
}
char* gumbo_strdup(const char* str) {
const size_t size = strlen(str) + 1;
// The strdup(3) function isn't available in strict "-std=c99" mode
// (it's part of POSIX, not C99), so use malloc(3) and memcpy(3)
// instead:
char* buffer = gumbo_alloc(size);
return memcpy(buffer, str, size);
}
#ifdef GUMBO_DEBUG
#include <stdarg.h>
// Debug function to trace operation of the parser
// (define GUMBO_DEBUG to use).
void gumbo_debug(const char* format, ...) {
va_list args;
va_start(args, format);
vprintf(format, args);
va_end(args);
fflush(stdout);
}
#endif
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/util.h
Vendored
+34
View File
@@ -0,0 +1,34 @@
#ifndef GUMBO_UTIL_H_
#define GUMBO_UTIL_H_
#include <stdbool.h>
#include <stddef.h>
#include "macros.h"
#ifdef __cplusplus
extern "C" {
#endif
// Utility function for allocating & copying a null-terminated string into a
// freshly-allocated buffer. This is necessary for proper memory management; we
// have the convention that all const char* in parse tree structures are
// freshly-allocated, so if we didn't copy, we'd try to delete a literal string
// when the parse tree is destroyed.
char* gumbo_strdup(const char* str) XMALLOC NONNULL_ARGS;
void* gumbo_alloc(size_t size) XMALLOC;
void* gumbo_realloc(void* ptr, size_t size) RETURNS_NONNULL;
void gumbo_free(void* ptr);
// Debug wrapper for printf
#ifdef GUMBO_DEBUG
void gumbo_debug(const char* format, ...) PRINTF(1);
#else
static inline void PRINTF(1) gumbo_debug(const char* UNUSED_ARG(format), ...) {};
#endif
#ifdef __cplusplus
}
#endif
#endif // GUMBO_UTIL_H_
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/vector.c
Vendored
+111
View File
@@ -0,0 +1,111 @@
/*
Copyright 2018 Craig Barnes.
Copyright 2010 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include "vector.h"
#include "util.h"
void gumbo_vector_init(unsigned int initial_capacity, GumboVector* vector) {
vector->length = 0;
vector->capacity = initial_capacity;
if (initial_capacity > 0) {
vector->data = gumbo_alloc(sizeof(void*) * initial_capacity);
} else {
vector->data = NULL;
}
}
void gumbo_vector_destroy(GumboVector* vector) {
if (vector->capacity > 0) {
gumbo_free(vector->data);
}
}
static void enlarge_vector_if_full(GumboVector* vector) {
if (vector->length >= vector->capacity) {
if (vector->capacity) {
vector->capacity *= 2;
size_t num_bytes = sizeof(void*) * vector->capacity;
vector->data = gumbo_realloc(vector->data, num_bytes);
} else {
// 0-capacity vector; no previous array to deallocate.
vector->capacity = 2;
vector->data = gumbo_alloc(sizeof(void*) * vector->capacity);
}
}
}
void gumbo_vector_add(void* element, GumboVector* vector) {
enlarge_vector_if_full(vector);
assert(vector->data);
assert(vector->length < vector->capacity);
vector->data[vector->length++] = element;
}
void* gumbo_vector_pop(GumboVector* vector) {
if (vector->length == 0) {
return NULL;
}
return vector->data[--vector->length];
}
int gumbo_vector_index_of(GumboVector* vector, const void* element) {
for (unsigned int i = 0; i < vector->length; ++i) {
if (vector->data[i] == element) {
return i;
}
}
return -1;
}
void gumbo_vector_insert_at (
void* element,
unsigned int index,
GumboVector* vector
) {
assert(index <= vector->length);
enlarge_vector_if_full(vector);
++vector->length;
memmove (
&vector->data[index + 1],
&vector->data[index],
sizeof(void*) * (vector->length - index - 1)
);
vector->data[index] = element;
}
void gumbo_vector_remove(void* node, GumboVector* vector) {
int index = gumbo_vector_index_of(vector, node);
if (index == -1) {
return;
}
gumbo_vector_remove_at(index, vector);
}
void* gumbo_vector_remove_at(unsigned int index, GumboVector* vector) {
assert(index < vector->length);
void* result = vector->data[index];
memmove (
&vector->data[index],
&vector->data[index + 1],
sizeof(void*) * (vector->length - index - 1)
);
--vector->length;
return result;
}
vendor/hachidori/third_party/hoshidicts/external/gumbo-parser/src/vector.h
Vendored
+45
View File
@@ -0,0 +1,45 @@
#ifndef GUMBO_VECTOR_H_
#define GUMBO_VECTOR_H_
#include "nokogiri_gumbo.h"
#ifdef __cplusplus
extern "C" {
#endif
// Initializes a new GumboVector with the specified initial capacity.
void gumbo_vector_init(unsigned int initial_capacity, GumboVector* vector);
// Frees the memory used by a GumboVector. Does not free the contained
// pointers.
void gumbo_vector_destroy(GumboVector* vector);
// Adds a new element to a GumboVector.
void gumbo_vector_add(void* element, GumboVector* vector);
// Removes and returns the element most recently added to the GumboVector.
// Ownership is transferred to caller. Capacity is unchanged. If the vector is
// empty, NULL is returned.
void* gumbo_vector_pop(GumboVector* vector);
// Inserts an element at a specific index. This is potentially O(N) time, but
// is necessary for some of the spec's behavior.
void gumbo_vector_insert_at (
void* element,
unsigned int index,
GumboVector* vector
);
// Removes an element from the vector, or does nothing if the element is not in
// the vector.
void gumbo_vector_remove(void* element, GumboVector* vector);
// Removes and returns an element at a specific index. Note that this is
// potentially O(N) time and should be used sparingly.
void* gumbo_vector_remove_at(unsigned int index, GumboVector* vector);
#ifdef __cplusplus
}
#endif
#endif // GUMBO_VECTOR_H_
vendor/hachidori/third_party/hoshidicts/external/lzokay/LICENSE
Vendored
+22
View File
@@ -0,0 +1,22 @@
The MIT License
Copyright (c) 2018 Jack Andersen
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
vendor/hachidori/third_party/hoshidicts/external/lzokay/README.md
Vendored
+8
View File
@@ -0,0 +1,8 @@
# lzokay (vendored)
LZO1X decompressor/compressor used for MDX/MDD record blocks with compression
type 1.
Source: https://github.com/AxioDL/lzokay, commit
db2df1fcbebc2ed06c10f727f72567d40f06a2be (`lzokay.cpp`, `lzokay.hpp`,
`LICENSE`). Unmodified. Licence: MIT.
vendor/hachidori/third_party/hoshidicts/external/lzokay/lzokay.cpp
Vendored
+647
View File
@@ -0,0 +1,647 @@
#include "lzokay.hpp"
#include <cstring>
#include <algorithm>
#include <iterator>
/*
* Based on documentation from the Linux sources: Documentation/lzo.txt
* https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/lzo.txt
*/
namespace lzokay {
#if _WIN32
#define HOST_BIG_ENDIAN 0
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#define HOST_BIG_ENDIAN 1
#else
#define HOST_BIG_ENDIAN 0
#endif
#if HOST_BIG_ENDIAN
static uint16_t get_le16(const uint8_t* p) {
uint16_t val = *reinterpret_cast<const uint16_t*>(p);
#if __GNUC__
return __builtin_bswap16(val);
#elif _WIN32
return _byteswap_ushort(val);
#else
return (val = (val << 8) | ((val >> 8) & 0xFF));
#endif
}
#else
static uint16_t get_le16(const uint8_t* p) {
return *reinterpret_cast<const uint16_t*>(p);
}
#endif
constexpr std::size_t Max255Count = std::size_t(~0) / 255 - 2;
#define NEEDS_IN(count) \
if (inp + (count) > inp_end) { \
dst_size = outp - dst; \
return EResult::InputOverrun; \
}
#define NEEDS_OUT(count) \
if (outp + (count) > outp_end) { \
dst_size = outp - dst; \
return EResult::OutputOverrun; \
}
#define CONSUME_ZERO_BYTE_LENGTH \
std::size_t offset; \
{ \
const uint8_t *old_inp = inp; \
while (*inp == 0) ++inp; \
offset = inp - old_inp; \
if (offset > Max255Count) { \
dst_size = outp - dst; \
return EResult::Error; \
} \
}
#define WRITE_ZERO_BYTE_LENGTH(length) \
{ \
std::size_t l; \
for (l = length; l > 255; l -= 255) { *outp++ = 0; } \
*outp++ = l; \
}
constexpr uint32_t M1MaxOffset = 0x0400;
constexpr uint32_t M2MaxOffset = 0x0800;
constexpr uint32_t M3MaxOffset = 0x4000;
constexpr uint32_t M4MaxOffset = 0xbfff;
constexpr uint32_t M1MinLen = 2;
constexpr uint32_t M1MaxLen = 2;
constexpr uint32_t M2MinLen = 3;
constexpr uint32_t M2MaxLen = 8;
constexpr uint32_t M3MinLen = 3;
constexpr uint32_t M3MaxLen = 33;
constexpr uint32_t M4MinLen = 3;
constexpr uint32_t M4MaxLen = 9;
constexpr uint32_t M1Marker = 0x0;
constexpr uint32_t M2Marker = 0x40;
constexpr uint32_t M3Marker = 0x20;
constexpr uint32_t M4Marker = 0x10;
constexpr uint32_t MaxMatchByLengthLen = 34; /* Max M3 len + 1 */
EResult decompress(const uint8_t* src, std::size_t src_size,
uint8_t* dst, std::size_t init_dst_size,
std::size_t& dst_size) {
dst_size = init_dst_size;
if (src_size < 3) {
dst_size = 0;
return EResult::InputOverrun;
}
const uint8_t* inp = src;
const uint8_t* inp_end = src + src_size;
uint8_t* outp = dst;
uint8_t* outp_end = dst + dst_size;
uint8_t* lbcur;
std::size_t lblen;
std::size_t state = 0;
std::size_t nstate = 0;
/* First byte encoding */
if (*inp >= 22) {
/* 22..255 : copy literal string
* length = (byte - 17) = 4..238
* state = 4 [ don't copy extra literals ]
* skip byte
*/
std::size_t len = *inp++ - uint8_t(17);
NEEDS_IN(len)
NEEDS_OUT(len)
for (std::size_t i = 0; i < len; ++i)
*outp++ = *inp++;
state = 4;
} else if (*inp >= 18) {
/* 18..21 : copy 0..3 literals
* state = (byte - 17) = 0..3 [ copy <state> literals ]
* skip byte
*/
nstate = *inp++ - uint8_t(17);
state = nstate;
NEEDS_IN(nstate)
NEEDS_OUT(nstate)
for (std::size_t i = 0; i < nstate; ++i)
*outp++ = *inp++;
}
/* 0..17 : follow regular instruction encoding, see below. It is worth
* noting that codes 16 and 17 will represent a block copy from
* the dictionary which is empty, and that they will always be
* invalid at this place.
*/
while (true) {
NEEDS_IN(1)
uint8_t inst = *inp++;
if (inst & 0xC0) {
/* [M2]
* 1 L L D D D S S (128..255)
* Copy 5-8 bytes from block within 2kB distance
* state = S (copy S literals after this block)
* length = 5 + L
* Always followed by exactly one byte : H H H H H H H H
* distance = (H << 3) + D + 1
*
* 0 1 L D D D S S (64..127)
* Copy 3-4 bytes from block within 2kB distance
* state = S (copy S literals after this block)
* length = 3 + L
* Always followed by exactly one byte : H H H H H H H H
* distance = (H << 3) + D + 1
*/
NEEDS_IN(1)
lbcur = outp - ((*inp++ << 3) + ((inst >> 2) & 0x7) + 1);
lblen = std::size_t(inst >> 5) + 1;
nstate = inst & uint8_t(0x3);
} else if (inst & M3Marker) {
/* [M3]
* 0 0 1 L L L L L (32..63)
* Copy of small block within 16kB distance (preferably less than 34B)
* length = 2 + (L ?: 31 + (zero_bytes * 255) + non_zero_byte)
* Always followed by exactly one LE16 : D D D D D D D D : D D D D D D S S
* distance = D + 1
* state = S (copy S literals after this block)
*/
lblen = std::size_t(inst & uint8_t(0x1f)) + 2;
if (lblen == 2) {
CONSUME_ZERO_BYTE_LENGTH
NEEDS_IN(1)
lblen += offset * 255 + 31 + *inp++;
}
NEEDS_IN(2)
nstate = get_le16(inp);
inp += 2;
lbcur = outp - ((nstate >> 2) + 1);
nstate &= 0x3;
} else if (inst & M4Marker) {
/* [M4]
* 0 0 0 1 H L L L (16..31)
* Copy of a block within 16..48kB distance (preferably less than 10B)
* length = 2 + (L ?: 7 + (zero_bytes * 255) + non_zero_byte)
* Always followed by exactly one LE16 : D D D D D D D D : D D D D D D S S
* distance = 16384 + (H << 14) + D
* state = S (copy S literals after this block)
* End of stream is reached if distance == 16384
*/
lblen = std::size_t(inst & uint8_t(0x7)) + 2;
if (lblen == 2) {
CONSUME_ZERO_BYTE_LENGTH
NEEDS_IN(1)
lblen += offset * 255 + 7 + *inp++;
}
NEEDS_IN(2)
nstate = get_le16(inp);
inp += 2;
lbcur = outp - (((inst & 0x8) << 11) + (nstate >> 2));
nstate &= 0x3;
if (lbcur == outp)
break; /* Stream finished */
lbcur -= 16384;
} else {
/* [M1] Depends on the number of literals copied by the last instruction. */
if (state == 0) {
/* If last instruction did not copy any literal (state == 0), this
* encoding will be a copy of 4 or more literal, and must be interpreted
* like this :
*
* 0 0 0 0 L L L L (0..15) : copy long literal string
* length = 3 + (L ?: 15 + (zero_bytes * 255) + non_zero_byte)
* state = 4 (no extra literals are copied)
*/
std::size_t len = inst + 3;
if (len == 3) {
CONSUME_ZERO_BYTE_LENGTH
NEEDS_IN(1)
len += offset * 255 + 15 + *inp++;
}
/* copy_literal_run */
NEEDS_IN(len)
NEEDS_OUT(len)
for (std::size_t i = 0; i < len; ++i)
*outp++ = *inp++;
state = 4;
continue;
} else if (state != 4) {
/* If last instruction used to copy between 1 to 3 literals (encoded in
* the instruction's opcode or distance), the instruction is a copy of a
* 2-byte block from the dictionary within a 1kB distance. It is worth
* noting that this instruction provides little savings since it uses 2
* bytes to encode a copy of 2 other bytes but it encodes the number of
* following literals for free. It must be interpreted like this :
*
* 0 0 0 0 D D S S (0..15) : copy 2 bytes from <= 1kB distance
* length = 2
* state = S (copy S literals after this block)
* Always followed by exactly one byte : H H H H H H H H
* distance = (H << 2) + D + 1
*/
NEEDS_IN(1)
nstate = inst & uint8_t(0x3);
lbcur = outp - ((inst >> 2) + (*inp++ << 2) + 1);
lblen = 2;
} else {
/* If last instruction used to copy 4 or more literals (as detected by
* state == 4), the instruction becomes a copy of a 3-byte block from the
* dictionary from a 2..3kB distance, and must be interpreted like this :
*
* 0 0 0 0 D D S S (0..15) : copy 3 bytes from 2..3 kB distance
* length = 3
* state = S (copy S literals after this block)
* Always followed by exactly one byte : H H H H H H H H
* distance = (H << 2) + D + 2049
*/
NEEDS_IN(1)
nstate = inst & uint8_t(0x3);
lbcur = outp - ((inst >> 2) + (*inp++ << 2) + 2049);
lblen = 3;
}
}
if (lbcur < dst) {
dst_size = outp - dst;
return EResult::LookbehindOverrun;
}
NEEDS_IN(nstate)
NEEDS_OUT(lblen + nstate)
/* Copy lookbehind */
for (std::size_t i = 0; i < lblen; ++i)
*outp++ = *lbcur++;
state = nstate;
/* Copy literal */
for (std::size_t i = 0; i < nstate; ++i)
*outp++ = *inp++;
}
dst_size = outp - dst;
if (lblen != 3) /* Ensure terminating M4 was encountered */
return EResult::Error;
if (inp == inp_end)
return EResult::Success;
else if (inp < inp_end)
return EResult::InputNotConsumed;
else
return EResult::InputOverrun;
}
struct State {
const uint8_t* src;
const uint8_t* src_end;
const uint8_t* inp;
uint32_t wind_sz;
uint32_t wind_b;
uint32_t wind_e;
uint32_t cycle1_countdown;
const uint8_t* bufp;
uint32_t buf_sz;
/* Access next input byte and advance both ends of circular buffer */
void get_byte(uint8_t* buf) {
if (inp >= src_end) {
if (wind_sz > 0)
--wind_sz;
buf[wind_e] = 0;
if (wind_e < DictBase::MaxMatchLen)
buf[DictBase::BufSize + wind_e] = 0;
} else {
buf[wind_e] = *inp;
if (wind_e < DictBase::MaxMatchLen)
buf[DictBase::BufSize + wind_e] = *inp;
++inp;
}
if (++wind_e == DictBase::BufSize)
wind_e = 0;
if (++wind_b == DictBase::BufSize)
wind_b = 0;
}
uint32_t pos2off(uint32_t pos) const {
return wind_b > pos ? wind_b - pos : DictBase::BufSize - (pos - wind_b);
}
};
class DictImpl : public DictBase {
public:
struct Match3Impl : DictBase::Match3 {
static uint32_t make_key(const uint8_t* data) {
return ((0x9f5f * (((uint32_t(data[0]) << 5 ^ uint32_t(data[1])) << 5) ^ data[2])) >> 5) & 0x3fff;
}
uint16_t get_head(uint32_t key) const {
return (chain_sz[key] == 0) ? uint16_t(UINT16_MAX) : head[key];
}
void init() {
std::fill(std::begin(chain_sz), std::end(chain_sz), 0);
}
void remove(uint32_t pos, const uint8_t* b) {
--chain_sz[make_key(b + pos)];
}
void advance(State& s, uint32_t& match_pos, uint32_t& match_count, const uint8_t* b) {
uint32_t key = make_key(b + s.wind_b);
match_pos = chain[s.wind_b] = get_head(key);
match_count = chain_sz[key]++;
if (match_count > DictBase::MaxMatchLen)
match_count = DictBase::MaxMatchLen;
head[key] = uint16_t(s.wind_b);
}
void skip_advance(State& s, const uint8_t* b) {
uint32_t key = make_key(b + s.wind_b);
chain[s.wind_b] = get_head(key);
head[key] = uint16_t(s.wind_b);
best_len[s.wind_b] = uint16_t(DictBase::MaxMatchLen + 1);
chain_sz[key]++;
}
};
struct Match2Impl : DictBase::Match2 {
static uint32_t make_key(const uint8_t* data) {
return uint32_t(data[0]) ^ (uint32_t(data[1]) << 8);
}
void init() {
std::fill(std::begin(head), std::end(head), UINT16_MAX);
}
void add(uint16_t pos, const uint8_t* b) {
head[make_key(b + pos)] = pos;
}
void remove(uint32_t pos, const uint8_t* b) {
uint16_t& p = head[make_key(b + pos)];
if (p == pos)
p = UINT16_MAX;
}
bool search(State& s, uint32_t& lb_pos, uint32_t& lb_len,
uint32_t best_pos[MaxMatchByLengthLen], const uint8_t* b) const {
uint16_t pos = head[make_key(b + s.wind_b)];
if (pos == UINT16_MAX)
return false;
if (best_pos[2] == 0)
best_pos[2] = pos + 1;
if (lb_len < 2) {
lb_len = 2;
lb_pos = pos;
}
return true;
}
};
void init(State& s, const uint8_t* src, std::size_t src_size) {
auto& match3 = static_cast<Match3Impl&>(_storage->match3);
auto& match2 = static_cast<Match2Impl&>(_storage->match2);
s.cycle1_countdown = DictBase::MaxDist;
match3.init();
match2.init();
s.src = src;
s.src_end = src + src_size;
s.inp = src;
s.wind_sz = uint32_t(std::min(src_size, std::size_t(MaxMatchLen)));
s.wind_b = 0;
s.wind_e = s.wind_sz;
std::copy_n(s.inp, s.wind_sz, _storage->buffer);
s.inp += s.wind_sz;
if (s.wind_e == DictBase::BufSize)
s.wind_e = 0;
if (s.wind_sz < 3)
std::fill_n(_storage->buffer + s.wind_b + s.wind_sz, 3, 0);
}
void reset_next_input_entry(State& s, Match3Impl& match3, Match2Impl& match2) {
/* Remove match from about-to-be-clobbered buffer entry */
if (s.cycle1_countdown == 0) {
match3.remove(s.wind_e, _storage->buffer);
match2.remove(s.wind_e, _storage->buffer);
} else {
--s.cycle1_countdown;
}
}
void advance(State& s, uint32_t& lb_off, uint32_t& lb_len,
uint32_t best_off[MaxMatchByLengthLen], bool skip) {
auto& match3 = static_cast<Match3Impl&>(_storage->match3);
auto& match2 = static_cast<Match2Impl&>(_storage->match2);
if (skip) {
for (uint32_t i = 0; i < lb_len - 1; ++i) {
reset_next_input_entry(s, match3, match2);
match3.skip_advance(s, _storage->buffer);
match2.add(uint16_t(s.wind_b), _storage->buffer);
s.get_byte(_storage->buffer);
}
}
lb_len = 1;
lb_off = 0;
uint32_t lb_pos;
uint32_t best_pos[MaxMatchByLengthLen] = {};
uint32_t match_pos, match_count;
match3.advance(s, match_pos, match_count, _storage->buffer);
int best_char = _storage->buffer[s.wind_b];
uint32_t best_len = lb_len;
if (lb_len >= s.wind_sz) {
if (s.wind_sz == 0)
best_char = -1;
lb_off = 0;
match3.best_len[s.wind_b] = DictBase::MaxMatchLen + 1;
} else {
if (match2.search(s, lb_pos, lb_len, best_pos, _storage->buffer) && s.wind_sz >= 3) {
for (uint32_t i = 0; i < match_count; ++i, match_pos = match3.chain[match_pos]) {
auto ref_ptr = _storage->buffer + s.wind_b;
auto match_ptr = _storage->buffer + match_pos;
auto mismatch = std::mismatch(ref_ptr, ref_ptr + s.wind_sz, match_ptr);
auto match_len = uint32_t(mismatch.first - ref_ptr);
if (match_len < 2)
continue;
if (match_len < MaxMatchByLengthLen && best_pos[match_len] == 0)
best_pos[match_len] = match_pos + 1;
if (match_len > lb_len) {
lb_len = match_len;
lb_pos = match_pos;
if (match_len == s.wind_sz || match_len > match3.best_len[match_pos])
break;
}
}
}
if (lb_len > best_len)
lb_off = s.pos2off(lb_pos);
match3.best_len[s.wind_b] = uint16_t(lb_len);
for (auto posit = std::begin(best_pos) + 2, offit = best_off + 2;
posit != std::end(best_pos); ++posit, ++offit) {
*offit = (*posit > 0) ? s.pos2off(*posit - 1) : 0;
}
}
reset_next_input_entry(s, match3, match2);
match2.add(uint16_t(s.wind_b), _storage->buffer);
s.get_byte(_storage->buffer);
if (best_char < 0) {
s.buf_sz = 0;
lb_len = 0;
/* Signal exit */
} else {
s.buf_sz = s.wind_sz + 1;
}
s.bufp = s.inp - s.buf_sz;
}
};
static void find_better_match(const uint32_t best_off[MaxMatchByLengthLen], uint32_t& lb_len, uint32_t& lb_off) {
if (lb_len <= M2MinLen || lb_off <= M2MaxOffset)
return;
if (lb_off > M2MaxOffset && lb_len >= M2MinLen + 1 && lb_len <= M2MaxLen + 1 &&
best_off[lb_len - 1] != 0 && best_off[lb_len - 1] <= M2MaxOffset) {
lb_len -= 1;
lb_off = best_off[lb_len];
} else if (lb_off > M3MaxOffset && lb_len >= M4MaxLen + 1 && lb_len <= M2MaxLen + 2 &&
best_off[lb_len - 2] && best_off[lb_len] <= M2MaxOffset) {
lb_len -= 2;
lb_off = best_off[lb_len];
} else if (lb_off > M3MaxOffset && lb_len >= M4MaxLen + 1 && lb_len <= M3MaxLen + 1 &&
best_off[lb_len - 1] != 0 && best_off[lb_len - 2] <= M3MaxOffset) {
lb_len -= 1;
lb_off = best_off[lb_len];
}
}
static EResult encode_literal_run(uint8_t*& outp, const uint8_t* outp_end, const uint8_t* dst, std::size_t& dst_size,
const uint8_t* lit_ptr, uint32_t lit_len) {
if (outp == dst && lit_len <= 238) {
NEEDS_OUT(1);
*outp++ = uint8_t(17 + lit_len);
} else if (lit_len <= 3) {
outp[-2] = uint8_t(outp[-2] | lit_len);
} else if (lit_len <= 18) {
NEEDS_OUT(1);
*outp++ = uint8_t(lit_len - 3);
} else {
NEEDS_OUT((lit_len - 18) / 255 + 2);
*outp++ = 0;
WRITE_ZERO_BYTE_LENGTH(lit_len - 18);
}
NEEDS_OUT(lit_len);
outp = std::copy_n(lit_ptr, lit_len, outp);
return EResult::Success;
}
static EResult encode_lookback_match(uint8_t*& outp, const uint8_t* outp_end, const uint8_t* dst, std::size_t& dst_size,
uint32_t lb_len, uint32_t lb_off, uint32_t last_lit_len) {
if (lb_len == 2) {
lb_off -= 1;
NEEDS_OUT(2);
*outp++ = uint8_t(M1Marker | ((lb_off & 0x3) << 2));
*outp++ = uint8_t(lb_off >> 2);
} else if (lb_len <= M2MaxLen && lb_off <= M2MaxOffset) {
lb_off -= 1;
NEEDS_OUT(2);
*outp++ = uint8_t((lb_len - 1) << 5 | ((lb_off & 0x7) << 2));
*outp++ = uint8_t(lb_off >> 3);
} else if (lb_len == M2MinLen && lb_off <= M1MaxOffset + M2MaxOffset && last_lit_len >= 4) {
lb_off -= 1 + M2MaxOffset;
NEEDS_OUT(2);
*outp++ = uint8_t(M1Marker | ((lb_off & 0x3) << 2));
*outp++ = uint8_t(lb_off >> 2);
} else if (lb_off <= M3MaxOffset) {
lb_off -= 1;
if (lb_len <= M3MaxLen) {
NEEDS_OUT(1);
*outp++ = uint8_t(M3Marker | (lb_len - 2));
} else {
lb_len -= M3MaxLen;
NEEDS_OUT(lb_len / 255 + 2);
*outp++ = uint8_t(M3Marker);
WRITE_ZERO_BYTE_LENGTH(lb_len);
}
NEEDS_OUT(2);
*outp++ = uint8_t(lb_off << 2);
*outp++ = uint8_t(lb_off >> 6);
} else {
lb_off -= 0x4000;
if (lb_len <= M4MaxLen) {
NEEDS_OUT(1);
*outp++ = uint8_t(M4Marker | ((lb_off & 0x4000) >> 11) | (lb_len - 2));
} else {
lb_len -= M4MaxLen;
NEEDS_OUT(lb_len / 255 + 2);
*outp++ = uint8_t(M4Marker | ((lb_off & 0x4000) >> 11));
WRITE_ZERO_BYTE_LENGTH(lb_len);
}
NEEDS_OUT(2);
*outp++ = uint8_t(lb_off << 2);
*outp++ = uint8_t(lb_off >> 6);
}
return EResult::Success;
}
EResult compress(const uint8_t* src, std::size_t src_size,
uint8_t* dst, std::size_t init_dst_size,
std::size_t& dst_size, DictBase& dict) {
EResult err;
State s;
auto& d = static_cast<DictImpl&>(dict);
dst_size = init_dst_size;
uint8_t* outp = dst;
uint8_t* outp_end = dst + dst_size;
uint32_t lit_len = 0;
uint32_t lb_off, lb_len;
uint32_t best_off[MaxMatchByLengthLen];
d.init(s, src, src_size);
const uint8_t* lit_ptr = s.inp;
d.advance(s, lb_off, lb_len, best_off, false);
while (s.buf_sz > 0) {
if (lit_len == 0)
lit_ptr = s.bufp;
if (lb_len < 2 || (lb_len == 2 && (lb_off > M1MaxOffset || lit_len == 0 || lit_len >= 4)) ||
(lb_len == 2 && outp == dst) || (outp == dst && lit_len == 0)) {
lb_len = 0;
} else if (lb_len == M2MinLen && lb_off > M1MaxOffset + M2MaxOffset && lit_len >= 4) {
lb_len = 0;
}
if (lb_len == 0) {
++lit_len;
d.advance(s, lb_off, lb_len, best_off, false);
continue;
}
find_better_match(best_off, lb_len, lb_off);
if ((err = encode_literal_run(outp, outp_end, dst, dst_size, lit_ptr, lit_len)) < EResult::Success)
return err;
if ((err = encode_lookback_match(outp, outp_end, dst, dst_size, lb_len, lb_off, lit_len)) < EResult::Success)
return err;
lit_len = 0;
d.advance(s, lb_off, lb_len, best_off, true);
}
if ((err = encode_literal_run(outp, outp_end, dst, dst_size, lit_ptr, lit_len)) < EResult::Success)
return err;
/* Terminating M4 */
NEEDS_OUT(3);
*outp++ = M4Marker | 1;
*outp++ = 0;
*outp++ = 0;
dst_size = outp - dst;
return EResult::Success;
}
}
vendor/hachidori/third_party/hoshidicts/external/lzokay/lzokay.hpp
Vendored
+79
View File
@@ -0,0 +1,79 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <memory>
namespace lzokay {
enum class EResult {
LookbehindOverrun = -4,
OutputOverrun = -3,
InputOverrun = -2,
Error = -1,
Success = 0,
InputNotConsumed = 1,
};
class DictBase {
protected:
static constexpr uint32_t HashSize = 0x4000;
static constexpr uint32_t MaxDist = 0xbfff;
static constexpr uint32_t MaxMatchLen = 0x800;
static constexpr uint32_t BufSize = MaxDist + MaxMatchLen;
/* List encoding of previous 3-byte data matches */
struct Match3 {
uint16_t head[HashSize]; /* key -> chain-head-pos */
uint16_t chain_sz[HashSize]; /* key -> chain-size */
uint16_t chain[BufSize]; /* chain-pos -> next-chain-pos */
uint16_t best_len[BufSize]; /* chain-pos -> best-match-length */
};
/* Encoding of 2-byte data matches */
struct Match2 {
uint16_t head[1 << 16]; /* 2-byte-data -> head-pos */
};
struct Data {
Match3 match3;
Match2 match2;
/* Circular buffer caching enough data to access the maximum lookback
* distance of 48K + maximum match length of 2K. An additional 2K is
* allocated so the start of the buffer may be replicated at the end,
* therefore providing efficient circular access.
*/
uint8_t buffer[BufSize + MaxMatchLen];
};
using storage_type = Data;
storage_type* _storage;
DictBase() = default;
friend struct State;
friend EResult compress(const uint8_t* src, std::size_t src_size,
uint8_t* dst, std::size_t& dst_size, DictBase& dict);
};
template <template<typename> class _Alloc = std::allocator>
class Dict : public DictBase {
_Alloc<DictBase::storage_type> _allocator;
public:
Dict() { _storage = _allocator.allocate(1); }
~Dict() { _allocator.deallocate(_storage, 1); }
};
EResult decompress(const uint8_t* src, std::size_t src_size,
uint8_t* dst, std::size_t dst_size,
std::size_t& out_size);
EResult compress(const uint8_t* src, std::size_t src_size,
uint8_t* dst, std::size_t dst_size,
std::size_t& out_size, DictBase& dict);
inline EResult compress(const uint8_t* src, std::size_t src_size,
uint8_t* dst, std::size_t dst_size,
std::size_t& out_size) {
Dict<> dict;
return compress(src, src_size, dst, dst_size, out_size, dict);
}
constexpr std::size_t compress_worst_size(std::size_t s) {
return s + s / 16 + 64 + 3;
}
}
vendor/hachidori/third_party/hoshidicts/include/hoshidicts.h
Vendored
+6
View File
@@ -0,0 +1,6 @@
#pragma once
#include "hoshidicts/deinflector.hpp"
#include "hoshidicts/importer.hpp"
#include "hoshidicts/lookup.hpp"
#include "hoshidicts/query.hpp"
vendor/hachidori/third_party/hoshidicts/include/hoshidicts/deinflector.hpp
Vendored
+80
View File
@@ -0,0 +1,80 @@
#pragma once
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include <cstdint>
#include <cstddef>
struct TransformGroup {
std::string name;
std::string description;
};
struct DeinflectionResult {
std::string text;
uint32_t conditions;
std::vector<TransformGroup> trace;
};
class Deinflector {
public:
Deinflector();
std::vector<DeinflectionResult> deinflect(const std::string& text) const;
static uint32_t pos_to_conditions(const std::vector<std::string>& part_of_speech);
private:
struct Rule {
std::string from;
std::string to;
uint32_t conditions_in;
uint32_t conditions_out;
int group_id;
};
enum Conditions : uint32_t {
NONE = 0,
V1D = 1 << 0,
V1P = 1 << 1,
V5D = 1 << 2,
V5SS = 1 << 3,
V5SP = 1 << 4,
VK = 1 << 5,
VS = 1 << 6,
VZ = 1 << 7,
ADJ_I = 1 << 8,
MASU = 1 << 9,
MASEN = 1 << 10,
TE = 1 << 11,
BA = 1 << 12,
KU = 1 << 13,
TA = 1 << 14,
NN = 1 << 15,
NASAI = 1 << 16,
YA = 1 << 17,
V1 = V1D | V1P,
V5S = V5SS | V5SP,
V5 = V5D | V5S,
V = V1 | V5 | VK | VS | VZ,
};
void deinflect_recursive(const std::string& text, uint32_t conditions, std::vector<int>& trace,
std::vector<DeinflectionResult>& results) const;
void init_transforms();
int add_group(const TransformGroup& group);
void add_rule(const Rule& rule);
void add_irregular(std::string_view suffix, uint32_t conditions_in, uint32_t conditions_out, int group_id);
struct TrieNode {
std::vector<std::pair<char32_t, uint32_t>> children;
int rules = -1;
};
std::vector<TrieNode> trie_;
std::vector<std::vector<Rule>> rule_lists_;
std::vector<TransformGroup> groups_;
size_t max_length_;
};
vendor/hachidori/third_party/hoshidicts/include/hoshidicts/importer.hpp
Vendored
+56
View File
@@ -0,0 +1,56 @@
#pragma once
#include <cstdint>
#include <map>
#include <optional>
#include <string>
#include <vector>
struct SummaryItemCount {
size_t total = 0;
};
using SummaryMetaCount = std::map<std::string, size_t>;
struct SummaryCounts {
SummaryItemCount terms;
SummaryMetaCount termMeta;
SummaryItemCount kanji;
SummaryMetaCount kanjiMeta;
SummaryItemCount tagMeta;
SummaryItemCount media;
};
struct Summary {
std::string title;
std::string revision;
bool sequenced = false;
std::optional<std::string> minimumYomitanVersion;
int version = 3;
uint64_t importDate = 0;
bool prefixWildcardsSupported = false;
SummaryCounts counts;
std::string styles;
std::optional<bool> isUpdatable;
std::optional<std::string> indexUrl;
std::optional<std::string> downloadUrl;
std::optional<std::string> author;
std::optional<std::string> url;
std::optional<std::string> description;
std::optional<std::string> attribution;
std::optional<std::string> sourceLanguage;
std::optional<std::string> targetLanguage;
std::optional<std::string> frequencyMode;
std::optional<bool> importSuccess;
};
struct ImportResult {
bool success = false;
std::string title;
Summary summary;
std::string error;
};
namespace dictionary_importer {
ImportResult import(const std::string& source_path, const std::string& output_dir, bool low_ram = false);
};
vendor/hachidori/third_party/hoshidicts/include/hoshidicts/lookup.hpp
Vendored
+42
View File
@@ -0,0 +1,42 @@
#pragma once
#include <optional>
#include <string>
#include <vector>
#include "deinflector.hpp"
#include "query.hpp"
struct LookupResult {
std::string matched;
std::string deinflected;
std::vector<TransformGroup> trace;
TermResult term;
int preprocessor_steps;
};
enum class LookupFrequencyOrder { Auto, Ascending, Descending, Disabled };
struct LookupOptions {
std::optional<std::string> frequency_dictionary;
LookupFrequencyOrder frequency_order = LookupFrequencyOrder::Auto;
std::optional<std::string> primary_reading;
};
class Lookup {
public:
Lookup(DictionaryQuery& query, Deinflector& deinflector) : query_(query), deinflector_(deinflector) {};
std::vector<LookupResult> lookup(const std::string& lookup_string, int max_results = 16, size_t scan_length = 16,
const LookupOptions& options = {}) const;
std::vector<LookupResult> lookup_dictionary(const std::string& lookup_string, const std::string& dictionary_path,
int max_results = 16, size_t scan_length = 16,
const LookupOptions& options = {}) const;
private:
std::vector<LookupResult> lookup_impl(const std::string& lookup_string, const std::string* dictionary_path,
int max_results, size_t scan_length, const LookupOptions& options) const;
static void filter_by_pos(RawTerms& terms, const DeinflectionResult& d);
DictionaryQuery& query_;
Deinflector& deinflector_;
};
vendor/hachidori/third_party/hoshidicts/include/hoshidicts/query.hpp
Vendored
+174
View File
@@ -0,0 +1,174 @@
#pragma once
#include <cstdint>
#include <memory>
#include <string>
#include <string_view>
#include <unordered_map>
#include <vector>
#if defined(__clang__) && defined(__APPLE__)
#define SWIFT_IMPORT_UNSAFE __attribute__((swift_attr("import_unsafe")))
#else
#define SWIFT_IMPORT_UNSAFE
#endif
struct Frequency {
int value;
std::string display_value;
std::string reading;
};
struct DictionaryStyle {
std::string dict_name;
std::string styles;
};
struct MediaFileView {
const char* data;
size_t size;
};
struct ZSTD_DDict_s;
struct GlossaryEntry {
std::string dict_name;
std::string glossary;
std::string definition_tags;
std::string term_tags;
const uint8_t* compressed_data = nullptr;
uint32_t compressed_size = 0;
const ZSTD_DDict_s* zstd_dict = nullptr;
};
struct FrequencyEntry {
std::string dict_name;
std::vector<Frequency> frequencies;
};
struct Pitch {
int position = 0;
std::string pattern;
std::vector<int> nasal;
std::vector<int> devoice;
};
struct PitchEntry {
std::string dict_name;
std::vector<Pitch> pitches;
std::vector<std::string> transcriptions;
};
struct TermResult {
std::string expression;
std::string reading;
std::string rules;
double score = 0;
std::vector<GlossaryEntry> glossaries;
std::vector<FrequencyEntry> frequencies;
std::vector<PitchEntry> pitches;
};
struct KanjiEntry {
std::string dict_name;
std::string onyomi;
std::string kunyomi;
std::string tags;
std::vector<std::string> definitions;
std::unordered_map<std::string, std::string> stats;
};
struct KanjiResult {
std::string character;
std::vector<KanjiEntry> entries;
};
struct RawTerm;
struct RawTerms;
class DictionaryQuery {
public:
DictionaryQuery();
~DictionaryQuery();
DictionaryQuery(const DictionaryQuery&) = delete;
DictionaryQuery& operator=(const DictionaryQuery&) = delete;
DictionaryQuery(DictionaryQuery&&) noexcept;
DictionaryQuery& operator=(DictionaryQuery&&) noexcept;
bool add_term_dict(const std::string& path);
bool add_freq_dict(const std::string& path);
bool add_pitch_dict(const std::string& path);
bool add_kanji_dict(const std::string& path);
// Drops every loaded kind of the dictionary at `path` and returns how many
// entries were removed (0 when the path is not loaded). The other
// dictionaries keep their relative order, so a caller can reshape the loaded
// set without rebuilding it.
size_t remove_dict(const std::string& path);
// Reorders every kind so the dictionaries appear in the order of `paths`.
// Dictionaries not listed keep their relative order after the listed ones.
// Returns false, changing nothing, when a listed path is not loaded.
bool set_dict_order(const std::vector<std::string>& paths);
// Long-key scan index (see src/scan_index.hpp). `long_key_length` returns
// the longest term-dictionary key, in code points, that begins with the
// first eight code points of `text` and is longer than 16, or 0 when there
// is none or `text` is shorter than eight code points. `max_long_key_length`
// is the longest such key any loaded term dictionary records, so a host can
// size the text it hands to Lookup; 0 when no dictionary has an index.
// Both accept a term dictionary path to consult that dictionary alone.
size_t long_key_length(std::string_view text, const std::string* term_dictionary_path = nullptr) const;
size_t max_long_key_length(const std::string* term_dictionary_path = nullptr) const;
void query_freq(std::vector<TermResult>& terms, bool match_reading = true) const;
void query_pitch(std::vector<TermResult>& terms) const;
KanjiResult query_kanji(const std::string& kanji) const;
std::vector<TermResult> query(const std::string& expression) const;
std::vector<char> get_media_file(const std::string& dict_name, const std::string& media_path) const;
SWIFT_IMPORT_UNSAFE
MediaFileView get_media_file_view(const std::string& dict_name, const std::string& media_path) const;
std::vector<DictionaryStyle> get_styles() const;
std::vector<std::string> get_freq_dict_order() const;
private:
friend class Lookup;
RawTerms query_raw(const std::string& expression,
const std::string* term_dictionary_path = nullptr) const;
TermResult build_term(const RawTerms& raw, RawTerm& term) const;
void collect_frequencies(std::string_view expression, std::string_view reading,
std::vector<FrequencyEntry>& out, bool match_reading = true) const;
void collect_pitches(std::string_view expression, std::string_view reading, std::vector<PitchEntry>& out) const;
void materialize(TermResult& term) const;
struct DictionaryData;
struct Dictionary {
Dictionary();
~Dictionary();
Dictionary(const Dictionary&) = delete;
Dictionary& operator=(const Dictionary&) = delete;
Dictionary(Dictionary&&) noexcept;
Dictionary& operator=(Dictionary&&) noexcept;
std::string path;
std::string name;
std::string styles;
std::unique_ptr<DictionaryData> data;
};
enum DictionaryType : uint8_t { TERM, FREQ, PITCH, KANJI };
bool add_dict(const std::string& path, DictionaryType);
bool add_dict_(const std::string& path, DictionaryType);
static std::string decompress_glossary(const void* data, size_t size, const ZSTD_DDict_s* dict);
std::vector<Dictionary> term_dicts_;
std::vector<Dictionary> freq_dicts_;
std::vector<Dictionary> pitch_dicts_;
std::vector<Dictionary> kanji_dicts_;
};
vendor/hachidori/third_party/hoshidicts/include/hoshidicts_c.h
Vendored
+196
View File
@@ -0,0 +1,196 @@
#ifndef HOSHIDICTS_C_H
#define HOSHIDICTS_C_H
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef struct hd_str {
const char* ptr;
size_t len;
} hd_str;
// importer
typedef struct hd_import_result hd_import_result;
hd_import_result* hd_import(const char* source_path, const char* output_dir, int low_ram);
void hd_import_result_free(hd_import_result* r);
int hd_import_result_success(const hd_import_result* r);
const char* hd_import_result_title(const hd_import_result* r);
uint64_t hd_import_result_term_count(const hd_import_result* r);
uint64_t hd_import_result_meta_count(const hd_import_result* r);
uint64_t hd_import_result_freq_count(const hd_import_result* r);
uint64_t hd_import_result_pitch_count(const hd_import_result* r);
uint64_t hd_import_result_kanji_count(const hd_import_result* r);
uint64_t hd_import_result_media_count(const hd_import_result* r);
const char* hd_import_result_error(const hd_import_result* r);
// deinflector
typedef struct hd_deinflector hd_deinflector;
hd_deinflector* hd_deinflector_new(void);
void hd_deinflector_free(hd_deinflector* d);
// query
typedef struct hd_query hd_query;
typedef struct hd_results hd_results;
typedef struct hd_kanji_results hd_kanji_results;
typedef struct hd_styles hd_styles;
typedef struct hd_frequency {
int32_t value;
hd_str display_value;
} hd_frequency;
typedef struct hd_dictionary_style {
hd_str dict_name;
hd_str styles;
} hd_dictionary_style;
typedef struct hd_media_file {
const uint8_t* data;
size_t size;
} hd_media_file;
typedef struct hd_glossary_entry {
hd_str dict_name;
hd_str glossary;
hd_str definition_tags;
hd_str term_tags;
} hd_glossary_entry;
typedef struct hd_frequency_entry {
hd_str dict_name;
const hd_frequency* frequencies;
size_t frequencies_count;
} hd_frequency_entry;
typedef struct hd_pitch {
int32_t position;
hd_str pattern;
const int32_t* nasal;
size_t nasal_count;
const int32_t* devoice;
size_t devoice_count;
} hd_pitch;
typedef struct hd_pitch_entry {
hd_str dict_name;
const hd_pitch* pitches;
size_t pitches_count;
const hd_str* transcriptions;
size_t transcriptions_count;
} hd_pitch_entry;
typedef struct hd_term_result {
hd_str expression;
hd_str reading;
hd_str rules;
double score;
const hd_glossary_entry* glossaries;
size_t glossaries_count;
const hd_frequency_entry* frequencies;
size_t frequencies_count;
const hd_pitch_entry* pitches;
size_t pitches_count;
} hd_term_result;
typedef struct hd_kanji_stat {
hd_str key;
hd_str value;
} hd_kanji_stat;
typedef struct hd_kanji_entry {
hd_str dict_name;
hd_str onyomi;
hd_str kunyomi;
hd_str tags;
const hd_str* definitions;
size_t definitions_count;
const hd_kanji_stat* stats;
size_t stats_count;
} hd_kanji_entry;
hd_query* hd_query_new(void);
void hd_query_free(hd_query* q);
int hd_query_add_term_dict(hd_query* q, const char* path);
int hd_query_add_freq_dict(hd_query* q, const char* path);
int hd_query_add_pitch_dict(hd_query* q, const char* path);
int hd_query_add_kanji_dict(hd_query* q, const char* path);
// Removes every loaded kind of the dictionary at path; returns the number removed.
size_t hd_query_remove_dict(hd_query* q, const char* path);
// Reorders the loaded dictionaries to follow paths (see DictionaryQuery::set_dict_order).
// Returns 0 on success, 1 when a listed path is not loaded.
int hd_query_set_dict_order(hd_query* q, const char* const* paths, size_t count);
// Longest term-dictionary key, in code points, recorded in the loaded
// dictionaries' long-key scan indexes (keys longer than 16 code points; 0 when
// none is loaded). A host should hand hd_lookup_run at least this many code
// points plus eight so that an extended scan can reach such a key.
size_t hd_query_max_long_key_length(const hd_query* q);
hd_results* hd_query_run(const hd_query* q, const char* expression, const hd_term_result** out_terms,
size_t* out_count);
void hd_results_free(hd_results* r);
hd_kanji_results* hd_query_run_kanji(const hd_query* q, const char* kanji, const hd_kanji_entry** out_entries,
size_t* out_count);
void hd_kanji_results_free(hd_kanji_results* r);
hd_media_file hd_query_get_media_file(const hd_query* q, const char* dict_name, const char* media_path);
hd_styles* hd_query_get_styles(const hd_query* q, const hd_dictionary_style** out_styles, size_t* out_count);
void hd_styles_free(hd_styles* s);
// lookup
typedef struct hd_lookup hd_lookup;
typedef struct hd_lookup_results hd_lookup_results;
typedef struct hd_transform_group {
hd_str name;
hd_str description;
} hd_transform_group;
typedef struct hd_lookup_result {
hd_str matched;
hd_str deinflected;
const hd_transform_group* trace;
size_t trace_count;
hd_term_result term;
int32_t preprocessor_steps;
} hd_lookup_result;
typedef enum hd_lookup_frequency_order {
HD_LOOKUP_FREQUENCY_ORDER_AUTO = 0,
HD_LOOKUP_FREQUENCY_ORDER_ASCENDING = 1,
HD_LOOKUP_FREQUENCY_ORDER_DESCENDING = 2,
HD_LOOKUP_FREQUENCY_ORDER_DISABLED = 3,
} hd_lookup_frequency_order;
typedef struct hd_lookup_options {
hd_str frequency_dictionary;
int32_t frequency_order;
hd_str primary_reading;
} hd_lookup_options;
hd_lookup* hd_lookup_new(hd_query* q, hd_deinflector* d);
void hd_lookup_free(hd_lookup* l);
hd_lookup_results* hd_lookup_run(const hd_lookup* l, const char* lookup_string, int max_results, size_t scan_length,
const hd_lookup_result** out_results, size_t* out_count);
hd_lookup_results* hd_lookup_run_with_options(const hd_lookup* l, const char* lookup_string, int max_results,
size_t scan_length, const hd_lookup_options* options,
const hd_lookup_result** out_results, size_t* out_count);
void hd_lookup_results_free(hd_lookup_results* r);
#ifdef __cplusplus
}
#endif
#endif
vendor/hachidori/third_party/hoshidicts/include/module.modulemap
Vendored
+5
View File
@@ -0,0 +1,5 @@
module CHoshiDicts {
header "hoshidicts.h"
requires cplusplus
export *
}
vendor/hachidori/third_party/hoshidicts/src/deinflector.cpp
Vendored
+1361
View File
File diff suppressed because it is too large Load Diff
vendor/hachidori/third_party/hoshidicts/src/hash/bloom.cpp
Vendored
+91
View File
@@ -0,0 +1,91 @@
#include "bloom.hpp"
#include <algorithm>
#include <bit>
#include <cstdint>
#include <cstring>
#include <stdexcept>
#include <vector>
#include "../memory/memory.hpp"
namespace hash {
namespace {
constexpr uint64_t num_hashes = 7;
}
bool bloom::load(const uint8_t* ptr, size_t size) {
uint64_t num_bits = *reinterpret_cast<const uint64_t*>(ptr);
if (size != 2 * sizeof(uint64_t) + num_bits / 8) {
return false;
}
num_hashes_ = *reinterpret_cast<const uint64_t*>(ptr + sizeof(uint64_t));
mask_ = num_bits - 1;
bits_ = reinterpret_cast<const uint64_t*>(ptr + 2 * sizeof(uint64_t));
return true;
}
void bloom::build_to_file(const std::vector<uint64_t>& hashes, const std::filesystem::path& path, size_t threads,
const spawn_fn& spawn) {
uint64_t num_bits = std::bit_ceil(std::max<uint64_t>(hashes.size() * 10, 64));
uint64_t mask = num_bits - 1;
size_t bits_size = num_bits / 8;
auto out = memory::map_rw(path, 2 * sizeof(uint64_t) + bits_size);
if (!out) {
throw std::runtime_error("failed to create bloom filter");
}
std::memcpy(out.data, &num_bits, sizeof(uint64_t));
std::memcpy(out.data + sizeof(uint64_t), &num_hashes, sizeof(uint64_t));
auto* bits = reinterpret_cast<uint64_t*>(out.data + 2 * sizeof(uint64_t));
std::memset(bits, 0, bits_size);
// Each thread owns a range of words and scans every hash, setting only the
// bits that fall into its range: no atomics, no per-thread copies, and each
// thread's writes stay within a slice small enough to sit in its cache. The
// position arithmetic is repeated per thread, but it is a few ALU operations
// against a random write that would otherwise miss the cache.
const uint64_t words = bits_size / sizeof(uint64_t);
const auto set_range = [bits, mask, &hashes](uint64_t word_begin, uint64_t word_end) {
for (uint64_t h : hashes) {
auto h1 = static_cast<uint32_t>(h);
auto h2 = static_cast<uint32_t>(h >> 32);
for (uint64_t k = 0; k < num_hashes; k++) {
uint64_t bit = (h1 + k * h2) & mask;
uint64_t word = bit >> 6;
if (word >= word_begin && word < word_end) {
bits[word] |= 1ULL << (bit & 63);
}
}
}
};
// Splitting only pays for large filters; below that the scan repeats cost
// more than the cache misses they avoid.
const size_t ranges = (spawn && hashes.size() >= 262144) ? std::max<size_t>(1, std::min<uint64_t>(threads, words / 4096)) : 1;
if (ranges == 1) {
for (uint64_t h : hashes) {
auto h1 = static_cast<uint32_t>(h);
auto h2 = static_cast<uint32_t>(h >> 32);
for (uint64_t k = 0; k < num_hashes; k++) {
uint64_t bit = (h1 + k * h2) & mask;
bits[bit >> 6] |= 1ULL << (bit & 63);
}
}
} else {
const uint64_t per_range = (words + ranges - 1) / ranges;
std::vector<std::future<void>> futures;
for (size_t r = 1; r < ranges; r++) {
const uint64_t begin = std::min<uint64_t>(r * per_range, words);
const uint64_t end = std::min<uint64_t>(begin + per_range, words);
if (begin >= end) break;
futures.push_back(spawn([&set_range, begin, end]() { set_range(begin, end); }));
}
set_range(0, std::min<uint64_t>(per_range, words));
for (auto& future : futures) future.get();
}
memory::unmap(out);
}
}
vendor/hachidori/third_party/hoshidicts/src/hash/bloom.hpp
Vendored
+37
View File
@@ -0,0 +1,37 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <functional>
#include <future>
#include <vector>
namespace hash {
class bloom {
public:
// Runs a task on another thread and returns its future; when absent, the
// filter is built on the calling thread alone.
using spawn_fn = std::function<std::future<void>(std::function<void()>)>;
// Setting bits is order-independent, so up to `threads` ranges of the
// filter are filled concurrently; the result is the same bits.
static void build_to_file(const std::vector<uint64_t>& hashes, const std::filesystem::path& path,
size_t threads = 1, const spawn_fn& spawn = nullptr);
bool load(const uint8_t* ptr, size_t size);
bool contains(uint64_t h) const {
auto h1 = static_cast<uint32_t>(h);
auto h2 = static_cast<uint32_t>(h >> 32);
for (uint64_t k = 0; k < num_hashes_; k++) {
uint64_t bit = (h1 + k * h2) & mask_;
if (!(bits_[bit >> 6] & (1ULL << (bit & 63)))) {
return false;
}
}
return true;
}
private:
uint64_t mask_ = 0;
uint64_t num_hashes_ = 0;
const uint64_t* bits_ = nullptr;
};
}
vendor/hachidori/third_party/hoshidicts/src/hash/hash.cpp
Vendored
+75
View File
@@ -0,0 +1,75 @@
#include "hash.hpp"
#include <xxh3.h>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <memory>
#include <stdexcept>
#include "../memory/memory.hpp"
namespace hash {
linear::linear() : ptr_(std::make_unique<table>()) {};
linear::~linear() = default;
uint64_t linear::operator()(std::string_view key) const {
uint64_t h = XXH3_64bits(key.data(), key.size());
if (!bloom_->contains(h)) {
return 0;
}
uint64_t pos = h % ptr_->capacity;
while (true) {
if (ptr_->table[pos].hash == 0) {
return 0;
}
if (ptr_->table[pos].hash == h) {
return ptr_->table[pos].offset;
}
if (++pos == ptr_->capacity) {
pos = 0;
}
}
}
void linear::build_to_file(const std::vector<std::pair<uint64_t, uint64_t>>& hash_entries,
const std::filesystem::path& path) {
ptr_->capacity = std::max<uint64_t>(hash_entries.size() * 10 / 7, 16);
size_t file_size = sizeof(uint32_t) + ptr_->capacity * sizeof(slot);
auto out = memory::map_rw(path, file_size);
if (!out) {
throw std::runtime_error("failed to create hash table");
}
std::memcpy(out.data, &ptr_->capacity, sizeof(uint32_t));
ptr_->table = reinterpret_cast<slot*>(out.data + sizeof(uint32_t));
std::memset(ptr_->table, 0, ptr_->capacity * sizeof(slot));
for (const auto& he : hash_entries) {
uint64_t h = he.first;
uint64_t pos = h % ptr_->capacity;
while (true) {
if (ptr_->table[pos].hash == 0) {
ptr_->table[pos] = {.hash = h, .offset = he.second};
break;
}
pos = (pos + 1) % ptr_->capacity;
}
}
memory::unmap(out);
ptr_->table = nullptr;
ptr_->capacity = 0;
}
bool linear::load(uint8_t* ptr, size_t size) {
uint32_t capacity = *reinterpret_cast<uint32_t*>(ptr);
if (size != sizeof(uint32_t) + static_cast<size_t>(capacity) * sizeof(slot)) {
return false;
}
ptr_->capacity = capacity;
ptr_->table = reinterpret_cast<slot*>(ptr + sizeof(uint32_t));
return true;
}
}
vendor/hachidori/third_party/hoshidicts/src/hash/hash.hpp
Vendored
+33
View File
@@ -0,0 +1,33 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <memory>
#include <vector>
#include "bloom.hpp"
namespace hash {
class linear {
public:
linear();
~linear();
uint64_t operator()(std::string_view key) const;
void build_to_file(const std::vector<std::pair<uint64_t, uint64_t>>& hash_entries, const std::filesystem::path& path);
bool load(uint8_t* ptr, size_t size);
void set_bloom(const bloom* b) { bloom_ = b; }
private:
struct slot {
uint64_t hash;
uint64_t offset;
};
struct table {
uint32_t capacity = 0;
slot* table;
};
std::unique_ptr<table> ptr_;
const bloom* bloom_ = nullptr;
};
}
vendor/hachidori/third_party/hoshidicts/src/hoshidicts_c.cpp
Vendored
+512
View File
@@ -0,0 +1,512 @@
#include "hoshidicts_c.h"
#include <cstdint>
#include <memory>
#include <optional>
#include <stdexcept>
#include <string>
#include <string_view>
#include <vector>
#include "hoshidicts/deinflector.hpp"
#include "hoshidicts/importer.hpp"
#include "hoshidicts/lookup.hpp"
#include "hoshidicts/query.hpp"
// importer
struct hd_import_result {
ImportResult result;
};
hd_import_result* hd_import(const char* source_path, const char* output_dir, int low_ram) {
try {
return new hd_import_result{dictionary_importer::import(source_path, output_dir, low_ram != 0)};
} catch (...) {
return nullptr;
}
}
void hd_import_result_free(hd_import_result* r) { delete r; }
int hd_import_result_success(const hd_import_result* r) { return static_cast<int>(r->result.success); }
const char* hd_import_result_title(const hd_import_result* r) { return r->result.title.c_str(); }
static uint64_t meta_count(const SummaryMetaCount& counts, const std::string& mode) {
auto it = counts.find(mode);
return it == counts.end() ? 0 : it->second;
}
uint64_t hd_import_result_term_count(const hd_import_result* r) { return r->result.summary.counts.terms.total; }
uint64_t hd_import_result_meta_count(const hd_import_result* r) {
return meta_count(r->result.summary.counts.termMeta, "total");
}
uint64_t hd_import_result_freq_count(const hd_import_result* r) {
return meta_count(r->result.summary.counts.termMeta, "freq");
}
uint64_t hd_import_result_pitch_count(const hd_import_result* r) {
return meta_count(r->result.summary.counts.termMeta, "pitch") + meta_count(r->result.summary.counts.termMeta, "ipa");
}
uint64_t hd_import_result_kanji_count(const hd_import_result* r) { return r->result.summary.counts.kanji.total; }
uint64_t hd_import_result_media_count(const hd_import_result* r) { return r->result.summary.counts.media.total; }
const char* hd_import_result_error(const hd_import_result* r) { return r->result.error.c_str(); }
// deinflector
struct hd_deinflector {
Deinflector deinflector;
};
hd_deinflector* hd_deinflector_new(void) {
try {
return new hd_deinflector;
} catch (...) {
return nullptr;
}
}
void hd_deinflector_free(hd_deinflector* d) { delete d; }
// query
struct hd_query {
DictionaryQuery query;
};
struct hd_results {
std::vector<TermResult> res;
std::vector<hd_term_result> term_results;
std::vector<hd_glossary_entry> glossary_entries;
std::vector<hd_frequency_entry> frequency_entries;
std::vector<hd_pitch_entry> pitch_entries;
std::vector<hd_frequency> frequencies;
std::vector<hd_pitch> pitches;
std::vector<hd_str> transcriptions;
};
struct hd_kanji_results {
KanjiResult res;
std::vector<hd_kanji_entry> entries;
std::vector<hd_str> definitions;
std::vector<hd_kanji_stat> stats;
};
struct hd_styles {
std::vector<DictionaryStyle> res;
std::vector<hd_dictionary_style> styles;
};
hd_query* hd_query_new(void) {
try {
return new hd_query;
} catch (...) {
return nullptr;
}
}
void hd_query_free(hd_query* q) { delete q; }
size_t hd_query_max_long_key_length(const hd_query* q) { return q->query.max_long_key_length(); }
int hd_query_add_term_dict(hd_query* q, const char* path) {
try {
return q->query.add_term_dict(path) ? 0 : 1;
} catch (...) {
return 1;
}
}
int hd_query_add_freq_dict(hd_query* q, const char* path) {
try {
return q->query.add_freq_dict(path) ? 0 : 1;
} catch (...) {
return 1;
}
}
int hd_query_add_pitch_dict(hd_query* q, const char* path) {
try {
return q->query.add_pitch_dict(path) ? 0 : 1;
} catch (...) {
return 1;
}
}
int hd_query_add_kanji_dict(hd_query* q, const char* path) {
try {
return q->query.add_kanji_dict(path) ? 0 : 1;
} catch (...) {
return 1;
}
}
size_t hd_query_remove_dict(hd_query* q, const char* path) {
try {
return q->query.remove_dict(path);
} catch (...) {
return 0;
}
}
int hd_query_set_dict_order(hd_query* q, const char* const* paths, size_t count) {
try {
std::vector<std::string> order;
order.reserve(count);
for (size_t i = 0; i < count; i++) {
order.emplace_back(paths[i]);
}
return q->query.set_dict_order(order) ? 0 : 1;
} catch (...) {
return 1;
}
}
static void build_glossaries(std::vector<hd_glossary_entry>& entries, const TermResult& term_result,
hd_term_result& tr) {
size_t glossaries_start = entries.size();
for (const auto& gloss_entry : term_result.glossaries) {
hd_glossary_entry gls;
gls.dict_name = hd_str{gloss_entry.dict_name.c_str(), gloss_entry.dict_name.size()};
gls.glossary = hd_str{gloss_entry.glossary.c_str(), gloss_entry.glossary.size()};
gls.definition_tags = hd_str{gloss_entry.definition_tags.c_str(), gloss_entry.definition_tags.size()};
gls.term_tags = hd_str{gloss_entry.term_tags.c_str(), gloss_entry.term_tags.size()};
entries.push_back(gls);
}
tr.glossaries = entries.data() + glossaries_start;
tr.glossaries_count = term_result.glossaries.size();
}
static void build_frequencies(std::vector<hd_frequency_entry>& entries, std::vector<hd_frequency>& frequencies,
const TermResult& term_result, hd_term_result& tr) {
size_t frequency_entry_start = entries.size();
for (const auto& freq_entry : term_result.frequencies) {
hd_frequency_entry frq;
frq.dict_name = hd_str{freq_entry.dict_name.c_str(), freq_entry.dict_name.size()};
size_t frequencies_start = frequencies.size();
for (const auto& freq : freq_entry.frequencies) {
frequencies.push_back(hd_frequency{freq.value, hd_str{freq.display_value.c_str(), freq.display_value.size()}});
}
frq.frequencies = frequencies.data() + frequencies_start;
frq.frequencies_count = freq_entry.frequencies.size();
entries.push_back(frq);
}
tr.frequencies = entries.data() + frequency_entry_start;
tr.frequencies_count = term_result.frequencies.size();
}
static void build_pitches(std::vector<hd_pitch_entry>& entries, std::vector<hd_pitch>& pitches,
std::vector<hd_str>& transcriptions, const TermResult& term_result, hd_term_result& tr) {
size_t pitch_entry_start = entries.size();
for (const auto& pitch_entry : term_result.pitches) {
hd_pitch_entry p;
p.dict_name = hd_str{pitch_entry.dict_name.c_str(), pitch_entry.dict_name.size()};
size_t pitches_start = pitches.size();
for (const auto& pitch : pitch_entry.pitches) {
pitches.push_back(hd_pitch{pitch.position, hd_str{pitch.pattern.c_str(), pitch.pattern.size()},
pitch.nasal.data(), pitch.nasal.size(), pitch.devoice.data(), pitch.devoice.size()});
}
p.pitches = pitches.data() + pitches_start;
p.pitches_count = pitch_entry.pitches.size();
size_t transcription_start = transcriptions.size();
for (const auto& transcription : pitch_entry.transcriptions) {
transcriptions.push_back(hd_str{transcription.c_str(), transcription.size()});
}
p.transcriptions = transcriptions.data() + transcription_start;
p.transcriptions_count = pitch_entry.transcriptions.size();
entries.push_back(p);
}
tr.pitches = entries.data() + pitch_entry_start;
tr.pitches_count = term_result.pitches.size();
}
hd_results* hd_query_run(const hd_query* q, const char* expression, const hd_term_result** out_terms,
size_t* out_count) {
try {
auto r = std::make_unique<hd_results>();
r->res = q->query.query(expression);
size_t glossaries_count = 0;
size_t freq_entry_count = 0;
size_t freq_count = 0;
size_t pitch_entry_count = 0;
size_t pitch_count = 0;
size_t transcription_count = 0;
for (const auto& term_result : r->res) {
glossaries_count += term_result.glossaries.size();
freq_entry_count += term_result.frequencies.size();
pitch_entry_count += term_result.pitches.size();
for (const auto& freq_entry : term_result.frequencies) {
freq_count += freq_entry.frequencies.size();
}
for (const auto& pitch_entry : term_result.pitches) {
pitch_count += pitch_entry.pitches.size();
transcription_count += pitch_entry.transcriptions.size();
}
}
r->glossary_entries.reserve(glossaries_count);
r->frequency_entries.reserve(freq_entry_count);
r->frequencies.reserve(freq_count);
r->pitch_entries.reserve(pitch_entry_count);
r->pitches.reserve(pitch_count);
r->transcriptions.reserve(transcription_count);
for (const auto& term_result : r->res) {
hd_term_result tr;
tr.expression = hd_str{term_result.expression.c_str(), term_result.expression.size()};
tr.reading = hd_str{term_result.reading.c_str(), term_result.reading.size()};
tr.rules = hd_str{term_result.rules.c_str(), term_result.rules.size()};
tr.score = term_result.score;
build_glossaries(r->glossary_entries, term_result, tr);
build_frequencies(r->frequency_entries, r->frequencies, term_result, tr);
build_pitches(r->pitch_entries, r->pitches, r->transcriptions, term_result, tr);
r->term_results.push_back(tr);
}
*out_terms = r->term_results.data();
*out_count = r->term_results.size();
return r.release();
} catch (...) {
return nullptr;
}
}
void hd_results_free(hd_results* r) { delete r; }
hd_kanji_results* hd_query_run_kanji(const hd_query* q, const char* kanji, const hd_kanji_entry** out_entries,
size_t* out_count) {
try {
auto r = std::make_unique<hd_kanji_results>();
r->res = q->query.query_kanji(kanji);
size_t definitions_count = 0;
size_t stats_count = 0;
for (const auto& entry : r->res.entries) {
definitions_count += entry.definitions.size();
stats_count += entry.stats.size();
}
r->definitions.reserve(definitions_count);
r->stats.reserve(stats_count);
for (const auto& entry : r->res.entries) {
hd_kanji_entry ke;
ke.dict_name = hd_str{entry.dict_name.c_str(), entry.dict_name.size()};
ke.onyomi = hd_str{entry.onyomi.c_str(), entry.onyomi.size()};
ke.kunyomi = hd_str{entry.kunyomi.c_str(), entry.kunyomi.size()};
ke.tags = hd_str{entry.tags.c_str(), entry.tags.size()};
size_t definitions_start = r->definitions.size();
for (const auto& definition : entry.definitions) {
r->definitions.push_back(hd_str{definition.c_str(), definition.size()});
}
ke.definitions = r->definitions.data() + definitions_start;
ke.definitions_count = entry.definitions.size();
size_t stats_start = r->stats.size();
for (const auto& [key, value] : entry.stats) {
r->stats.push_back(hd_kanji_stat{hd_str{key.c_str(), key.size()}, hd_str{value.c_str(), value.size()}});
}
ke.stats = r->stats.data() + stats_start;
ke.stats_count = entry.stats.size();
r->entries.push_back(ke);
}
*out_entries = r->entries.data();
*out_count = r->entries.size();
return r.release();
} catch (...) {
return nullptr;
}
}
void hd_kanji_results_free(hd_kanji_results* r) { delete r; }
hd_media_file hd_query_get_media_file(const hd_query* q, const char* dict_name, const char* media_path) {
try {
auto view = q->query.get_media_file_view(dict_name, media_path);
return hd_media_file{reinterpret_cast<const uint8_t*>(view.data), view.size};
} catch (...) {
return hd_media_file{nullptr, 0};
}
}
hd_styles* hd_query_get_styles(const hd_query* q, const hd_dictionary_style** out_styles, size_t* out_count) {
try {
auto s = std::make_unique<hd_styles>();
s->res = q->query.get_styles();
s->styles.reserve(s->res.size());
for (const auto& style : s->res) {
s->styles.push_back(hd_dictionary_style{hd_str{style.dict_name.c_str(), style.dict_name.size()},
hd_str{style.styles.c_str(), style.styles.size()}});
}
*out_styles = s->styles.data();
*out_count = s->styles.size();
return s.release();
} catch (...) {
return nullptr;
}
}
void hd_styles_free(hd_styles* s) { delete s; }
// lookup
struct hd_lookup {
Lookup lookup;
};
struct hd_lookup_results {
std::vector<LookupResult> res;
std::vector<hd_lookup_result> results;
std::vector<hd_transform_group> trace;
std::vector<hd_glossary_entry> glossary_entries;
std::vector<hd_frequency_entry> frequency_entries;
std::vector<hd_pitch_entry> pitch_entries;
std::vector<hd_frequency> frequencies;
std::vector<hd_pitch> pitches;
std::vector<hd_str> transcriptions;
};
hd_lookup* hd_lookup_new(hd_query* q, hd_deinflector* d) {
try {
return new hd_lookup{Lookup(q->query, d->deinflector)};
} catch (...) {
return nullptr;
}
}
void hd_lookup_free(hd_lookup* l) { delete l; }
static void marshal_lookup_results(hd_lookup_results* r) {
size_t trace_count = 0;
size_t glossaries_count = 0;
size_t freq_entry_count = 0;
size_t freq_count = 0;
size_t pitch_entry_count = 0;
size_t pitch_count = 0;
size_t transcription_count = 0;
for (const auto& lookup_result : r->res) {
trace_count += lookup_result.trace.size();
glossaries_count += lookup_result.term.glossaries.size();
freq_entry_count += lookup_result.term.frequencies.size();
pitch_entry_count += lookup_result.term.pitches.size();
for (const auto& freq_entry : lookup_result.term.frequencies) {
freq_count += freq_entry.frequencies.size();
}
for (const auto& pitch_entry : lookup_result.term.pitches) {
pitch_count += pitch_entry.pitches.size();
transcription_count += pitch_entry.transcriptions.size();
}
}
r->trace.reserve(trace_count);
r->glossary_entries.reserve(glossaries_count);
r->frequency_entries.reserve(freq_entry_count);
r->frequencies.reserve(freq_count);
r->pitch_entries.reserve(pitch_entry_count);
r->pitches.reserve(pitch_count);
r->transcriptions.reserve(transcription_count);
for (const auto& lookup_result : r->res) {
const auto& term_result = lookup_result.term;
hd_lookup_result lr;
lr.matched = hd_str{lookup_result.matched.c_str(), lookup_result.matched.size()};
lr.deinflected = hd_str{lookup_result.deinflected.c_str(), lookup_result.deinflected.size()};
lr.preprocessor_steps = lookup_result.preprocessor_steps;
size_t trace_start = r->trace.size();
for (const auto& group : lookup_result.trace) {
r->trace.push_back(hd_transform_group{hd_str{group.name.c_str(), group.name.size()},
hd_str{group.description.c_str(), group.description.size()}});
}
lr.trace = r->trace.data() + trace_start;
lr.trace_count = lookup_result.trace.size();
lr.term.expression = hd_str{term_result.expression.c_str(), term_result.expression.size()};
lr.term.reading = hd_str{term_result.reading.c_str(), term_result.reading.size()};
lr.term.rules = hd_str{term_result.rules.c_str(), term_result.rules.size()};
lr.term.score = term_result.score;
build_glossaries(r->glossary_entries, term_result, lr.term);
build_frequencies(r->frequency_entries, r->frequencies, term_result, lr.term);
build_pitches(r->pitch_entries, r->pitches, r->transcriptions, term_result, lr.term);
r->results.push_back(lr);
}
}
hd_lookup_results* hd_lookup_run(const hd_lookup* l, const char* lookup_string, int max_results, size_t scan_length,
const hd_lookup_result** out_results, size_t* out_count) {
try {
auto r = std::make_unique<hd_lookup_results>();
r->res = l->lookup.lookup(lookup_string, max_results, scan_length);
marshal_lookup_results(r.get());
*out_results = r->results.data();
*out_count = r->results.size();
return r.release();
} catch (...) {
return nullptr;
}
}
static std::optional<std::string_view> optional_string_view(hd_str value) {
if (value.len == 0) {
return std::nullopt;
}
if (value.ptr == nullptr) {
throw std::invalid_argument("non-empty lookup option has a null pointer");
}
return std::string_view(value.ptr, value.len);
}
hd_lookup_results* hd_lookup_run_with_options(const hd_lookup* l, const char* lookup_string, int max_results,
size_t scan_length, const hd_lookup_options* options,
const hd_lookup_result** out_results, size_t* out_count) {
try {
LookupOptions native_options;
if (options != nullptr) {
native_options.frequency_dictionary = optional_string_view(options->frequency_dictionary);
native_options.primary_reading = optional_string_view(options->primary_reading);
switch (options->frequency_order) {
case HD_LOOKUP_FREQUENCY_ORDER_AUTO:
native_options.frequency_order = LookupFrequencyOrder::Auto;
break;
case HD_LOOKUP_FREQUENCY_ORDER_ASCENDING:
native_options.frequency_order = LookupFrequencyOrder::Ascending;
break;
case HD_LOOKUP_FREQUENCY_ORDER_DESCENDING:
native_options.frequency_order = LookupFrequencyOrder::Descending;
break;
case HD_LOOKUP_FREQUENCY_ORDER_DISABLED:
native_options.frequency_order = LookupFrequencyOrder::Disabled;
break;
default:
return nullptr;
}
}
auto r = std::make_unique<hd_lookup_results>();
r->res = l->lookup.lookup(lookup_string, max_results, scan_length, native_options);
marshal_lookup_results(r.get());
*out_results = r->results.data();
*out_count = r->results.size();
return r.release();
} catch (...) {
return nullptr;
}
}
void hd_lookup_results_free(hd_lookup_results* r) { delete r; }
vendor/hachidori/third_party/hoshidicts/src/importer.cpp
Vendored
+1180
View File
File diff suppressed because it is too large Load Diff
vendor/hachidori/third_party/hoshidicts/src/json/json_skip.cpp
Vendored
+165
View File
@@ -0,0 +1,165 @@
#include "json_skip.hpp"
#include <bit>
#include <cstdint>
#include <cstring>
// The portable classifier is used where neither wasm simd128 nor SSE2 is
// available; HOSHIDICTS_JSON_SKIP_SCALAR forces it (its test does).
#if defined(HOSHIDICTS_JSON_SKIP_SCALAR) || !(defined(__wasm_simd128__) || defined(__SSE2__))
#define HOSHIDICTS_JSON_SKIP_USE_SCALAR 1
#elif defined(__wasm_simd128__)
#include <wasm_simd128.h>
#else
#include <emmintrin.h>
#endif
namespace hoshidicts {
namespace {
struct Masks {
uint64_t quote;
uint64_t backslash;
uint64_t open;
uint64_t close;
};
#if defined(HOSHIDICTS_JSON_SKIP_USE_SCALAR)
inline Masks classify64(const char* p, char open, char close) noexcept {
Masks m{};
for (int i = 0; i < 64; ++i) {
const uint64_t bit = uint64_t{1} << i;
const char c = p[i];
if (c == '"') m.quote |= bit;
else if (c == '\\') m.backslash |= bit;
else if (c == open) m.open |= bit;
else if (c == close) m.close |= bit;
}
return m;
}
#elif defined(__wasm_simd128__)
inline uint64_t eq_bits16(v128_t v, char c) noexcept {
return static_cast<uint64_t>(static_cast<uint16_t>(wasm_i8x16_bitmask(wasm_i8x16_eq(v, wasm_i8x16_splat(c)))));
}
inline Masks classify64(const char* p, char open, char close) noexcept {
Masks m{};
for (int i = 0; i < 4; ++i) {
const v128_t v = wasm_v128_load(p + 16 * i);
const int shift = 16 * i;
m.quote |= eq_bits16(v, '"') << shift;
m.backslash |= eq_bits16(v, '\\') << shift;
m.open |= eq_bits16(v, open) << shift;
m.close |= eq_bits16(v, close) << shift;
}
return m;
}
#else
inline uint64_t eq_bits16(__m128i v, char c) noexcept {
return static_cast<uint64_t>(static_cast<uint16_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(v, _mm_set1_epi8(c)))));
}
inline Masks classify64(const char* p, char open, char close) noexcept {
Masks m{};
for (int i = 0; i < 4; ++i) {
const __m128i v = _mm_loadu_si128(reinterpret_cast<const __m128i*>(p + 16 * i));
const int shift = 16 * i;
m.quote |= eq_bits16(v, '"') << shift;
m.backslash |= eq_bits16(v, '\\') << shift;
m.open |= eq_bits16(v, open) << shift;
m.close |= eq_bits16(v, close) << shift;
}
return m;
}
#endif
// Positions escaped by a preceding odd-length run of backslashes (after
// simdjson's json_escape_scanner). `next_escaped` carries whether a run at the
// end of this block escapes the first byte of the next one.
inline uint64_t find_escaped(uint64_t backslash, uint64_t& next_escaped) noexcept {
const uint64_t prev_escaped = next_escaped;
if (backslash == 0) {
next_escaped = 0;
return prev_escaped;
}
backslash &= ~prev_escaped;
const uint64_t follows_escape = (backslash << 1) | prev_escaped;
constexpr uint64_t even_bits = 0x5555555555555555ULL;
const uint64_t odd_sequence_starts = backslash & ~even_bits & ~follows_escape;
uint64_t sequences_starting_on_even_bits = 0;
next_escaped = __builtin_add_overflow(odd_sequence_starts, backslash, &sequences_starting_on_even_bits) ? 1 : 0;
const uint64_t invert_mask = sequences_starting_on_even_bits << 1;
return (even_bits ^ invert_mask) & follows_escape;
}
inline uint64_t prefix_xor(uint64_t x) noexcept {
x ^= x << 1;
x ^= x << 2;
x ^= x << 4;
x ^= x << 8;
x ^= x << 16;
x ^= x << 32;
return x;
}
} // namespace
const char* skip_json_container(const char* begin, const char* end) noexcept {
const char open = *begin;
const char close = open == '[' ? ']' : '}';
const char* p = begin + 1;
int64_t depth = 1;
uint64_t next_escaped = 0;
uint64_t in_string = 0; // all ones while inside a string at a block boundary
while (end - p >= 64) {
const Masks m = classify64(p, open, close);
const uint64_t escaped = find_escaped(m.backslash, next_escaped);
const uint64_t quotes = m.quote & ~escaped;
// Bits set from each opening quote (inclusive) to its closing quote (exclusive).
const uint64_t inside = prefix_xor(quotes) ^ in_string;
in_string = static_cast<uint64_t>(static_cast<int64_t>(inside) >> 63);
const uint64_t open = m.open & ~inside;
const uint64_t close = m.close & ~inside;
if (depth - std::popcount(close) <= 0) {
// The matching bracket may be in this block: walk the structural bits in order.
uint64_t structural = open | close;
while (structural != 0) {
const int i = std::countr_zero(structural);
if ((open >> i) & 1) {
++depth;
} else if (--depth == 0) {
return p + i + 1;
}
structural &= structural - 1;
}
} else {
depth += std::popcount(open) - std::popcount(close);
}
p += 64;
}
bool in_str = in_string != 0;
bool escaped = (next_escaped & 1) != 0;
for (; p < end; ++p) {
const char c = *p;
if (in_str) {
if (escaped) {
escaped = false;
} else if (c == '\\') {
escaped = true;
} else if (c == '"') {
in_str = false;
}
} else if (c == '"') {
in_str = true;
} else if (c == open) {
++depth;
} else if (c == close) {
if (--depth == 0) {
return p + 1;
}
}
}
return nullptr;
}
} // namespace hoshidicts
vendor/hachidori/third_party/hoshidicts/src/json/json_skip.hpp
Vendored
+48
View File
@@ -0,0 +1,48 @@
#pragma once
#include <glaze/glaze.hpp>
#include <string_view>
namespace hoshidicts {
// Given `begin` pointing at the '[' or '{' that opens a JSON array or object,
// returns the pointer one past its matching ']' or '}', or nullptr when the
// input ends first. Strings are honoured (quotes, backslash escapes); nothing
// else is validated, which is also what glaze's non-validating skip does for
// raw_json_view. The scan classifies 64 bytes at a time (wasm simd128 or SSE2
// where available).
const char* skip_json_container(const char* begin, const char* end) noexcept;
// A raw JSON value captured as a view, like glz::raw_json_view, whose reader
// uses skip_json_container for arrays and objects. Term glossaries and meta
// data make up most of a bank's bytes; glaze's byte-and-switch skip was the
// largest item in the import profile.
struct raw_value_view {
std::string_view str;
};
} // namespace hoshidicts
template <>
struct glz::from<glz::JSON, hoshidicts::raw_value_view> {
template <auto Opts>
GLZ_ALWAYS_INLINE static void op(hoshidicts::raw_value_view& value, glz::is_context auto&& ctx, auto&& it,
auto end) {
if (*it == '[' || *it == '{') {
const char* const start = it;
const char* const stop = hoshidicts::skip_json_container(start, end);
if (stop == nullptr) [[unlikely]] {
ctx.error = glz::error_code::unexpected_end;
return;
}
value.str = {start, static_cast<size_t>(stop - start)};
it = stop;
return;
}
glz::raw_json_view other;
glz::from<glz::JSON, glz::raw_json_view>::template op<Opts>(other, ctx, it, end);
if (bool(ctx.error)) [[unlikely]] {
return;
}
value.str = other.str;
}
};
vendor/hachidori/third_party/hoshidicts/src/json/yomitan_parser.cpp
Vendored
+244
View File
@@ -0,0 +1,244 @@
#include "yomitan_parser.hpp"
#include <string_view>
#include <variant>
template <>
struct glz::meta<Index> {
using T = Index;
static constexpr auto value =
object("title", glz::raw_string<&T::title>, "format", &T::format, "version", &T::version,
"revision", glz::raw_string<&T::revision>, "minimumYomitanVersion", glz::raw_string<&T::minimumYomitanVersion>,
"sequenced", &T::sequenced, "isUpdatable", &T::isUpdatable, "indexUrl", glz::raw_string<&T::indexUrl>,
"downloadUrl", glz::raw_string<&T::downloadUrl>, "author", glz::raw_string<&T::author>,
"url", glz::raw_string<&T::url>, "description", glz::raw_string<&T::description>,
"attribution", glz::raw_string<&T::attribution>, "sourceLanguage", glz::raw_string<&T::sourceLanguage>,
"targetLanguage", glz::raw_string<&T::targetLanguage>, "frequencyMode", glz::raw_string<&T::frequencyMode>);
};
template <>
struct glz::meta<Term> {
using T = Term;
static constexpr auto value =
array(glz::raw_string<&T::expression>, glz::raw_string<&T::reading>, glz::raw_string<&T::definition_tags>,
glz::raw_string<&T::rules>, &T::score, &T::glossary, &T::sequence, glz::raw_string<&T::term_tags>);
};
template <>
struct glz::meta<Meta> {
using T = Meta;
static constexpr auto value = array(glz::raw_string<&T::expression>, glz::raw_string<&T::mode>, &T::data);
};
template <>
struct glz::meta<Kanji> {
using T = Kanji;
static constexpr auto value = array(glz::raw_string<&T::character>, glz::raw_string<&T::onyomi>,
glz::raw_string<&T::kunyomi>, glz::raw_string<&T::tags>, &T::definitions, &T::stats);
};
template <>
struct glz::meta<Tag> {
using T = Tag;
static constexpr auto value =
array(glz::raw_string<&T::name>, glz::raw_string<&T::category>, &T::order, glz::raw_string<&T::notes>, &T::score);
};
namespace internal {
struct FrequencyValue {
int value;
std::optional<std::string> display_value;
};
struct RawFrequencyFlat {
std::optional<std::string_view> reading;
int value;
std::optional<std::string> display_value;
};
struct RawFrequency {
std::optional<std::string_view> reading;
std::variant<int, FrequencyValue> frequency;
};
struct PitchesArray {
std::variant<int, std::string> position;
std::optional<std::variant<int, std::vector<int>>> nasal;
std::optional<std::variant<int, std::vector<int>>> devoice;
};
struct RawPitch {
std::string_view reading;
std::vector<PitchesArray> pitches;
};
struct TranscriptionsArray {
std::string_view ipa;
};
struct RawIPA {
std::string_view reading;
std::vector<TranscriptionsArray> transcriptions;
};
};
template <>
struct glz::meta<internal::RawFrequencyFlat> {
using T = internal::RawFrequencyFlat;
static constexpr auto value = object("reading", &T::reading, "value", &T::value, "displayValue", &T::display_value);
};
template <>
struct glz::meta<internal::FrequencyValue> {
using T = internal::FrequencyValue;
static constexpr auto value = object("value", &T::value, "displayValue", &T::display_value);
};
template <>
struct glz::meta<internal::RawFrequency> {
using T = internal::RawFrequency;
static constexpr auto value = object("reading", &T::reading, "frequency", &T::frequency);
};
template <>
struct glz::meta<internal::PitchesArray> {
using T = internal::PitchesArray;
static constexpr auto value = object("position", &T::position, "nasal", &T::nasal, "devoice", &T::devoice);
};
template <>
struct glz::meta<internal::RawPitch> {
using T = internal::RawPitch;
static constexpr auto value = object("reading", glz::raw_string<&T::reading>, "pitches", &T::pitches);
};
template <>
struct glz::meta<internal::TranscriptionsArray> {
using T = internal::TranscriptionsArray;
static constexpr auto value = object("ipa", glz::raw_string<&T::ipa>);
};
template <>
struct glz::meta<internal::RawIPA> {
using T = internal::RawIPA;
static constexpr auto value = object("reading", glz::raw_string<&T::reading>, "transcriptions", &T::transcriptions);
};
bool yomitan_parser::parse_index(std::string_view content, Index& out) {
auto error = glz::read<glz::opts{.error_on_unknown_keys = false, .error_on_missing_keys = false}>(out, content);
return !error;
}
namespace {
// Bank strings are captured raw (raw_string / raw_json_view) and copied through
// unchanged, so glaze's UTF-8 validation of every skipped string bought
// nothing but time: it was ~12-14% of a Jitendex or Pixiv Light import. A bank
// with malformed UTF-8 now imports with the bytes as they are (a renderer
// shows U+FFFD, as Yomitan does) instead of being dropped whole.
struct BankOpts : glz::opts {
bool validate_utf8 = false;
};
constexpr BankOpts bank_opts{{.error_on_unknown_keys = false, .error_on_missing_keys = false}};
} // namespace
bool yomitan_parser::parse_term_bank(std::string_view content, std::vector<Term>& out) {
auto error = glz::read<bank_opts>(out, content);
return !error;
}
bool yomitan_parser::parse_meta_bank(std::string_view content, std::vector<Meta>& out) {
auto error = glz::read<bank_opts>(out, content);
return !error;
}
bool yomitan_parser::parse_kanji_bank(std::string_view content, std::vector<Kanji>& out) {
auto error = glz::read<bank_opts>(out, content);
return !error;
}
bool yomitan_parser::parse_tag_bank(std::string_view content, std::vector<Tag>& out) {
auto error = glz::read<bank_opts>(out, content);
return !error;
}
bool yomitan_parser::parse_frequency(std::string_view content, ParsedFrequency& out) {
internal::RawFrequencyFlat parsed_flat;
auto error =
glz::read<glz::opts{.error_on_unknown_keys = false, .error_on_missing_keys = true}>(parsed_flat, content);
if (!error) {
out.reading = parsed_flat.reading.value_or("");
out.value = parsed_flat.value;
out.display_value = parsed_flat.display_value.value_or(std::to_string(parsed_flat.value));
return true;
}
int val;
error = glz::read_json(val, content);
if (!error) {
out.value = val;
out.display_value = std::to_string(val);
out.reading = "";
return true;
}
internal::RawFrequency parsed;
error = glz::read<glz::opts{.error_on_unknown_keys = false, .error_on_missing_keys = true}>(parsed, content);
if (error) {
return false;
}
out.reading = parsed.reading.value_or("");
if (std::holds_alternative<int>(parsed.frequency)) {
int freq = std::get<int>(parsed.frequency);
out.value = freq;
out.display_value = std::to_string(freq);
} else {
auto& freq = std::get<internal::FrequencyValue>(parsed.frequency);
out.value = freq.value;
out.display_value = freq.display_value.value_or(std::to_string(freq.value));
}
return true;
}
bool yomitan_parser::parse_pitch(std::string_view content, ParsedPitch& out) {
internal::RawPitch parsed;
auto error = glz::read<glz::opts{.error_on_unknown_keys = false, .error_on_missing_keys = true}>(parsed, content);
if (error) {
return false;
}
auto to_number_array = [](const std::optional<std::variant<int, std::vector<int>>>& value) -> std::vector<int> {
if (!value) {
return {};
}
if (std::holds_alternative<int>(*value)) {
return {std::get<int>(*value)};
}
return std::get<std::vector<int>>(*value);
};
out.reading = parsed.reading;
for (auto& pitch : parsed.pitches) {
ParsedAccent accent{.nasal = to_number_array(pitch.nasal), .devoice = to_number_array(pitch.devoice)};
if (std::holds_alternative<int>(pitch.position)) {
accent.position = std::get<int>(pitch.position);
} else {
accent.pattern = std::move(std::get<std::string>(pitch.position));
}
out.pitches.emplace_back(std::move(accent));
}
return true;
}
bool yomitan_parser::parse_ipa(std::string_view content, ParsedPitch& out) {
internal::RawIPA parsed;
auto error = glz::read<glz::opts{.error_on_unknown_keys = false, .error_on_missing_keys = false}>(parsed, content);
if (error) {
return false;
}
out.reading = parsed.reading;
out.transcriptions =
parsed.transcriptions | std::views::transform(&internal::TranscriptionsArray::ipa) | std::ranges::to<std::vector>();
return true;
}
vendor/hachidori/third_party/hoshidicts/src/json/yomitan_parser.hpp
Vendored
+91
View File
@@ -0,0 +1,91 @@
#pragma once
#include <cstdint>
#include <glaze/glaze.hpp>
#include "json_skip.hpp"
#include <optional>
#include <string_view>
#include <vector>
struct Index {
std::string_view title;
std::optional<int> format;
std::optional<int> version;
std::string_view revision;
std::optional<std::string_view> minimumYomitanVersion;
bool sequenced = false;
std::optional<bool> isUpdatable;
std::optional<std::string_view> indexUrl;
std::optional<std::string_view> downloadUrl;
std::optional<std::string_view> author;
std::optional<std::string_view> url;
std::optional<std::string_view> description;
std::optional<std::string_view> attribution;
std::optional<std::string_view> sourceLanguage;
std::optional<std::string_view> targetLanguage;
std::optional<std::string_view> frequencyMode;
};
struct Term {
std::string_view expression;
std::string_view reading;
std::optional<std::string_view> definition_tags;
std::string_view rules;
double score = 0;
hoshidicts::raw_value_view glossary;
int64_t sequence = 0;
std::string_view term_tags;
};
struct Meta {
std::string_view expression;
std::string_view mode;
hoshidicts::raw_value_view data;
};
struct Kanji {
std::string_view character;
std::string_view onyomi;
std::string_view kunyomi;
std::string_view tags;
std::vector<std::string_view> definitions;
std::unordered_map<std::string, std::string> stats;
};
struct Tag {
std::string_view name;
std::string_view category;
int order = 0;
std::string_view notes;
int score = 0;
};
struct ParsedFrequency {
std::string_view reading;
int value;
std::string display_value;
};
struct ParsedAccent {
int position = 0;
std::string pattern;
std::vector<int> nasal;
std::vector<int> devoice;
};
struct ParsedPitch {
std::string_view reading;
std::vector<ParsedAccent> pitches;
std::vector<std::string_view> transcriptions;
};
namespace yomitan_parser {
bool parse_index(std::string_view content, Index& out);
bool parse_term_bank(std::string_view content, std::vector<Term>& out);
bool parse_meta_bank(std::string_view content, std::vector<Meta>& out);
bool parse_kanji_bank(std::string_view content, std::vector<Kanji>& out);
bool parse_tag_bank(std::string_view content, std::vector<Tag>& out);
bool parse_frequency(std::string_view content, ParsedFrequency& out);
bool parse_pitch(std::string_view content, ParsedPitch& out);
bool parse_ipa(std::string_view content, ParsedPitch& out);
};
vendor/hachidori/third_party/hoshidicts/src/lookup.cpp
Vendored
+323
View File
@@ -0,0 +1,323 @@
#include "hoshidicts/lookup.hpp"
#include <ankerl/unordered_dense.h>
#include <utf8.h>
#include <xxh3.h>
#include <algorithm>
#include <climits>
#include <numeric>
#include <optional>
#include <ranges>
#include <string_view>
#include <vector>
#include "query_internal.hpp"
#include "scan_index.hpp"
#include "text_processor/text_processor.hpp"
namespace {
bool is_space(char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r'; }
void split_whitespace(std::string_view str, std::vector<std::string>& result) {
result.clear();
size_t i = 0;
const size_t n = str.size();
while (i < n) {
while (i < n && is_space(str[i])) {
++i;
}
const size_t begin = i;
while (i < n && !is_space(str[i])) {
++i;
}
if (i > begin) {
result.emplace_back(str, begin, i - begin);
}
}
}
struct Candidate {
size_t matched_len;
const DeinflectionResult* deinflection;
RawTerm* term;
uint32_t store_index;
int steps;
};
uint64_t key_hash(std::string_view expression, std::string_view reading) {
return XXH3_64bits_withSeed(reading.data(), reading.size(), XXH3_64bits(expression.data(), expression.size()));
}
bool key_less(const RawTerm& a, const RawTerm& b) {
const int c = a.expression.compare(b.expression);
return c != 0 ? c < 0 : a.reading < b.reading;
}
std::optional<int> get_freq_value_for_dict(const RawTerm& term, std::string_view dictionary_name, bool descending) {
std::optional<int> frequency;
for (const auto& frequency_entry : term.frequencies) {
if (frequency_entry.dict_name != dictionary_name || frequency_entry.frequencies.empty()) {
continue;
}
for (const auto& candidate : frequency_entry.frequencies) {
if (candidate.value < 0) {
continue;
}
frequency = frequency.has_value() ? std::optional<int>(descending ? std::max(*frequency, candidate.value)
: std::min(*frequency, candidate.value))
: std::optional<int>(candidate.value);
}
}
return frequency;
}
bool matches_primary_reading(const RawTerm& term, std::string_view primary_reading) {
return term.reading == primary_reading;
}
}
std::vector<LookupResult> Lookup::lookup(const std::string& lookup_string, int max_results, size_t scan_length,
const LookupOptions& options) const {
return lookup_impl(lookup_string, nullptr, max_results, scan_length, options);
}
std::vector<LookupResult> Lookup::lookup_dictionary(const std::string& lookup_string,
const std::string& dictionary_path, int max_results,
size_t scan_length, const LookupOptions& options) const {
return lookup_impl(lookup_string, &dictionary_path, max_results, scan_length, options);
}
std::vector<LookupResult> Lookup::lookup_impl(const std::string& lookup_string, const std::string* dictionary_path,
int max_results, size_t scan_length,
const LookupOptions& options) const {
std::vector<Candidate> candidates;
ankerl::unordered_dense::map<uint64_t, uint32_t> index;
std::vector<std::vector<DeinflectionResult>> deinflection_store;
std::vector<RawTerms> term_store;
size_t text_len = utf8::distance(lookup_string.begin(), lookup_string.end());
size_t start = std::min(scan_length, text_len);
auto search_str_it = lookup_string.begin();
utf8::advance(search_str_it, start, lookup_string.end());
auto find_candidate = [&](uint64_t h, const RawTerm& term) -> Candidate* {
auto it = index.find(h);
if (it == index.end()) {
return nullptr;
}
Candidate& hit = candidates[it->second];
if (hit.term->expression == term.expression && hit.term->reading == term.reading) {
return &hit;
}
for (auto& c : candidates) {
if (c.term->expression == term.expression && c.term->reading == term.reading) {
return &c;
}
}
return nullptr;
};
// The processed variants of the input's first eight code points, kept from
// the ordinary scan for the long-key check below.
std::vector<std::string> prefix_variants;
// Scans one prefix of the input: every processed variant, deinflected, looked
// up, and merged into the candidates keeping the longest matched form.
auto scan_prefix = [&](std::string_view search_str, size_t codepoints) {
auto processor_results = text_processor::process(search_str);
if (codepoints == scan_index::long_key_prefix_codepoints) {
prefix_variants.reserve(processor_results.size());
for (const auto& variant : processor_results) {
prefix_variants.push_back(variant.text);
}
}
for (auto& variant : processor_results) {
auto deinflection_results = deinflector_.deinflect(variant.text);
for (auto& deinflection : deinflection_results) {
auto terms = query_.query_raw(deinflection.text, dictionary_path);
filter_by_pos(terms, deinflection);
const auto store_index = static_cast<uint32_t>(term_store.size());
for (auto& term : terms.terms) {
const uint64_t h = key_hash(term.expression, term.reading);
Candidate* existing = find_candidate(h, term);
if (existing != nullptr) {
if (search_str.size() > existing->matched_len) {
existing->matched_len = search_str.size();
existing->deinflection = &deinflection;
existing->term = &term;
existing->store_index = store_index;
existing->steps = variant.steps;
}
} else {
index.try_emplace(h, static_cast<uint32_t>(candidates.size()));
candidates.push_back(Candidate{.matched_len = search_str.size(),
.deinflection = &deinflection,
.term = &term,
.store_index = store_index,
.steps = variant.steps});
}
}
term_store.push_back(std::move(terms));
}
deinflection_store.push_back(std::move(deinflection_results));
}
};
for (size_t i = start; i > 0; i--) {
scan_prefix(std::string_view(lookup_string.begin(), search_str_it), i);
if (i > 1) {
utf8::prior(search_str_it, lookup_string.begin());
}
}
// Long keys (see src/scan_index.hpp): when the input begins like a key that
// is longer than the scan just done, scan the longer prefixes too, up to that
// key's length plus room for an inflected ending. A scan shorter than eight
// code points never produced the variants, so it never extends -- a caller
// asking for one character wants one character.
if (!prefix_variants.empty() && text_len > scan_length) {
size_t long_key = 0;
for (const auto& variant : prefix_variants) {
long_key = std::max(long_key, query_.long_key_length(variant, dictionary_path));
}
if (long_key > scan_length) {
const size_t extended = std::min(long_key + scan_index::inflection_slack_codepoints, text_len);
auto extended_it = lookup_string.begin();
utf8::advance(extended_it, extended, lookup_string.end());
for (size_t i = extended; i > scan_length; i--) {
scan_prefix(std::string_view(lookup_string.begin(), extended_it), i);
utf8::prior(extended_it, lookup_string.begin());
}
}
}
std::vector<std::string> auto_frequency_dictionaries;
std::optional<std::string_view> frequency_dictionary;
bool frequency_descending = false;
switch (options.frequency_order) {
case LookupFrequencyOrder::Auto:
auto_frequency_dictionaries = query_.get_freq_dict_order();
break;
case LookupFrequencyOrder::Ascending:
case LookupFrequencyOrder::Descending:
if (options.frequency_dictionary.has_value()) {
const auto selected =
std::ranges::find(query_.freq_dicts_, *options.frequency_dictionary, &DictionaryQuery::Dictionary::name);
if (selected != query_.freq_dicts_.end()) {
frequency_dictionary = selected->name;
frequency_descending = options.frequency_order == LookupFrequencyOrder::Descending;
}
}
break;
case LookupFrequencyOrder::Disabled:
break;
}
std::string_view primary_reading;
if (options.primary_reading.has_value()) {
primary_reading = *options.primary_reading;
}
const size_t retained_count = std::min(candidates.size(), static_cast<size_t>(max_results));
auto less = [&auto_frequency_dictionaries, frequency_dictionary, frequency_descending, primary_reading](
const Candidate& a, const Candidate& b) {
if (!primary_reading.empty()) {
const bool primary_a = matches_primary_reading(*a.term, primary_reading);
const bool primary_b = matches_primary_reading(*b.term, primary_reading);
if (primary_a != primary_b) {
return primary_a;
}
}
if (a.matched_len != b.matched_len) {
return a.matched_len > b.matched_len;
}
auto steps_a = a.steps;
auto steps_b = b.steps;
if (steps_a != steps_b) {
return steps_a < steps_b;
}
auto trace_len_a = a.deinflection->trace.size();
auto trace_len_b = b.deinflection->trace.size();
if (trace_len_a != trace_len_b) {
return trace_len_a < trace_len_b;
}
auto match_a = a.term->expression == a.deinflection->text;
auto match_b = b.term->expression == b.deinflection->text;
if (match_a != match_b) {
return match_a > match_b;
}
for (const auto& dictionary_name : auto_frequency_dictionaries) {
const int freq_a = get_freq_value_for_dict(*a.term, dictionary_name, false).value_or(INT_MAX);
const int freq_b = get_freq_value_for_dict(*b.term, dictionary_name, false).value_or(INT_MAX);
if (freq_a != freq_b) {
return freq_a < freq_b;
}
}
if (frequency_dictionary.has_value()) {
const auto freq_a = get_freq_value_for_dict(*a.term, *frequency_dictionary, frequency_descending);
const auto freq_b = get_freq_value_for_dict(*b.term, *frequency_dictionary, frequency_descending);
if (freq_a.has_value() != freq_b.has_value()) {
return freq_a.has_value();
}
if (freq_a.has_value() && *freq_a != *freq_b) {
return frequency_descending ? *freq_a > *freq_b : *freq_a < *freq_b;
}
}
if (a.term->score != b.term->score) {
return a.term->score > b.term->score;
}
auto a_reading_expr_match = a.term->expression == a.term->reading;
auto b_reading_expr_match = b.term->expression == b.term->reading;
return a_reading_expr_match > b_reading_expr_match;
};
std::vector<uint32_t> order(candidates.size());
std::iota(order.begin(), order.end(), uint32_t{0});
std::ranges::sort(order,
[&](uint32_t ia, uint32_t ib) { return key_less(*candidates[ia].term, *candidates[ib].term); });
auto order_middle = std::ranges::next(order.begin(), static_cast<std::ptrdiff_t>(retained_count));
std::ranges::partial_sort(order, order_middle,
[&](uint32_t ia, uint32_t ib) { return less(candidates[ia], candidates[ib]); });
std::vector<LookupResult> retained;
retained.reserve(retained_count);
for (auto it = order.begin(); it != order_middle; ++it) {
Candidate& c = candidates[*it];
retained.push_back(LookupResult{.matched = lookup_string.substr(0, c.matched_len),
.deinflected = c.deinflection->text,
.trace = c.deinflection->trace,
.term = query_.build_term(term_store[c.store_index], *c.term),
.preprocessor_steps = c.steps});
}
for (auto& r : retained) {
query_.materialize(r.term);
}
return retained;
}
void Lookup::filter_by_pos(RawTerms& terms, const DeinflectionResult& d) {
if (d.conditions == 0) {
return;
}
std::vector<std::string> tokens;
std::erase_if(terms.terms, [&](const RawTerm& term) {
uint32_t dict_conditions = 0;
for (uint32_t i = term.first_glossary; i != UINT32_MAX; i = terms.glossaries[i].next) {
split_whitespace(terms.glossaries[i].rules, tokens);
dict_conditions |= Deinflector::pos_to_conditions(tokens);
}
return (dict_conditions & d.conditions) == 0;
});
}
vendor/hachidori/third_party/hoshidicts/src/mdict/html_to_structured.cpp
Vendored
+1068
View File
File diff suppressed because it is too large Load Diff
vendor/hachidori/third_party/hoshidicts/src/mdict/html_to_structured.hpp
Vendored
+76
View File
@@ -0,0 +1,76 @@
#pragma once
#include <cstdint>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
// MDX glossary HTML -> Yomitan structured content, a port of manabitan's
// ext/js/dictionary/mdx/mdx-converter.js (the structured-content half).
//
// Supported HTML elements keep their tag; b/i/em/strong/h1-6/p/pre/font/...
// become span or div with the tag's default style; unsupported elements are
// unwrapped (their children are kept). Inline styles are mapped to the
// structured-content style properties Yomitan knows. Links are rewritten:
// entry:// bword:// d: x: -> ?query=, sound:// -> media: (only when audio is
// enabled), data: -> an extracted embedded asset, http(s)/mailto/tel kept,
// javascript:/vbscript:/about:/# -> #, anything else -> media:<prefix>path.
// <script>/<noscript>/<link rel=stylesheet> are dropped, <style> blocks are
// collected, <audio>/<video> become links.
//
// Beyond manabitan: the MDX StyleSheet backtick substitution is applied by
// apply_stylesheet(), and nesting deeper than ConvertOptions::max_depth is
// flattened because Yomitan rejects structured content nested past 24.
namespace mdict {
struct ConvertOptions {
// Prefix under which MDD assets appear in the imported dictionary's media.
std::string asset_prefix = "mdict-media/";
// sound:// links become media: links only when set (manabitan's default is off).
bool enable_audio = false;
// Elements nested deeper than this (root div = 1) are unwrapped.
int max_depth = 20;
};
struct EmbeddedAsset {
std::string path; // "<prefix>embedded/<category>/<hash>.<ext>"
std::vector<uint8_t> data;
};
struct ConvertResult {
// The {"type":"structured-content", ...} glossary object as JSON text.
std::string glossary_json;
// <style> blocks in document order, named "inline/1.css", "inline/2.css", ...
std::vector<std::pair<std::string, std::string>> inline_stylesheets;
// Assets decoded from data: URLs.
std::vector<EmbeddedAsset> embedded_assets;
// Normalised MDD keys (without the prefix) the glossary refers to, in first-use order.
std::vector<std::string> asset_references;
};
// Parses `html` as a body fragment and converts it.
ConvertResult convert_html(std::string_view html, const ConvertOptions& options);
// Expands the `N` markers of an MDX with a StyleSheet header attribute
// ("N\nbegin\nend\n" triples). Returns the text unchanged when there is no
// stylesheet or no markers.
std::string apply_stylesheet(std::string_view text, std::string_view stylesheet);
// Rewrites url(...) references inside CSS text to "<prefix>key", collecting
// the keys in `references`. `source_asset_path` resolves ./ and ../ relative
// to the stylesheet's own MDD path (empty for inline styles).
std::string rewrite_css_asset_urls(std::string_view css, std::string_view asset_prefix,
std::string_view source_asset_path, std::vector<std::string>& references);
// MDD key normalisation shared with MdictSource: backslashes to slashes, no
// leading slash, ./ and ../ collapsed, query/fragment dropped, percent
// escapes decoded. Empty when the path is not an asset path (schemes, data:,
// #, javascript:, ...).
std::string normalize_asset_path(std::string_view path, std::string_view source_asset_path = {});
// encodeURIComponent.
std::string encode_uri_component(std::string_view value);
// `value` as a quoted JSON string literal.
std::string json_quote(std::string_view value);
}
vendor/hachidori/third_party/hoshidicts/src/mdict/mdict_reader.cpp
Vendored
+632
View File
@@ -0,0 +1,632 @@
#include "mdict_reader.hpp"
#include <libdeflate.h>
#include <lzokay.hpp>
#include <algorithm>
#include <array>
#include <cctype>
#include <charconv>
#include <cstdlib>
#include <cstring>
#include <format>
#include "ripemd128.hpp"
namespace mdict {
namespace {
// A single block, key index or record block, is never anywhere near this in
// practice (MdxBuilder writes blocks of a few hundred KiB); the cap keeps a
// corrupt size field from turning into a giant allocation.
constexpr uint64_t max_block_size = 256ULL * 1024 * 1024;
constexpr uint64_t max_header_size = 16ULL * 1024 * 1024;
uint32_t be32(const uint8_t* p) {
return (uint32_t{p[0]} << 24) | (uint32_t{p[1]} << 16) | (uint32_t{p[2]} << 8) | uint32_t{p[3]};
}
uint32_t le32(const uint8_t* p) {
return uint32_t{p[0]} | (uint32_t{p[1]} << 8) | (uint32_t{p[2]} << 16) | (uint32_t{p[3]} << 24);
}
uint64_t be64(const uint8_t* p) { return (uint64_t{be32(p)} << 32) | be32(p + 4); }
uint32_t adler32(const uint8_t* data, size_t size) { return libdeflate_adler32(1, data, size); }
std::string lower(std::string_view s) {
std::string out(s);
for (auto& c : out) {
c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
}
return out;
}
void append_utf8(std::string& out, uint32_t cp) {
if (cp < 0x80) {
out += static_cast<char>(cp);
} else if (cp < 0x800) {
out += static_cast<char>(0xc0 | (cp >> 6));
out += static_cast<char>(0x80 | (cp & 0x3f));
} else if (cp < 0x10000) {
out += static_cast<char>(0xe0 | (cp >> 12));
out += static_cast<char>(0x80 | ((cp >> 6) & 0x3f));
out += static_cast<char>(0x80 | (cp & 0x3f));
} else {
out += static_cast<char>(0xf0 | (cp >> 18));
out += static_cast<char>(0x80 | ((cp >> 12) & 0x3f));
out += static_cast<char>(0x80 | ((cp >> 6) & 0x3f));
out += static_cast<char>(0x80 | (cp & 0x3f));
}
}
// XML character and entity references in header attribute values.
std::string unescape_xml(std::string_view s) {
std::string out;
out.reserve(s.size());
for (size_t i = 0; i < s.size();) {
if (s[i] != '&') {
out += s[i++];
continue;
}
const size_t end = s.find(';', i);
if (end == std::string_view::npos || end - i > 10) {
out += s[i++];
continue;
}
const std::string_view name = s.substr(i + 1, end - i - 1);
if (name == "lt") {
out += '<';
} else if (name == "gt") {
out += '>';
} else if (name == "amp") {
out += '&';
} else if (name == "quot") {
out += '"';
} else if (name == "apos") {
out += '\'';
} else if (name.size() > 1 && name[0] == '#') {
const bool hex = name[1] == 'x' || name[1] == 'X';
const std::string_view digits = name.substr(hex ? 2 : 1);
uint32_t cp = 0;
auto [ptr, ec] = std::from_chars(digits.data(), digits.data() + digits.size(), cp, hex ? 16 : 10);
if (ec != std::errc{} || ptr != digits.data() + digits.size() || cp > 0x10ffff) {
out += s[i++];
continue;
}
append_utf8(out, cp);
} else {
out += s[i++];
continue;
}
i = end + 1;
}
return out;
}
// Parses `<Name attr="value" attr2='value2' ... />`. Returns the element name.
std::string parse_element(std::string_view xml, std::map<std::string, std::string>& attributes) {
size_t pos = xml.find('<');
if (pos == std::string_view::npos) {
throw Error("not an MDict file: header is not an XML element");
}
pos++;
const size_t name_end = xml.find_first_of(" \t\r\n/>", pos);
if (name_end == std::string_view::npos) {
throw Error("not an MDict file: unterminated header element");
}
std::string name(xml.substr(pos, name_end - pos));
pos = name_end;
while (pos < xml.size()) {
pos = xml.find_first_not_of(" \t\r\n", pos);
if (pos == std::string_view::npos || xml[pos] == '/' || xml[pos] == '>') {
break;
}
const size_t eq = xml.find('=', pos);
if (eq == std::string_view::npos) {
break;
}
std::string attr(xml.substr(pos, eq - pos));
while (!attr.empty() && std::isspace(static_cast<unsigned char>(attr.back()))) {
attr.pop_back();
}
size_t value_start = xml.find_first_not_of(" \t\r\n", eq + 1);
if (value_start == std::string_view::npos || (xml[value_start] != '"' && xml[value_start] != '\'')) {
throw Error(std::format("malformed MDict header: attribute {} has no quoted value", attr));
}
const char quote = xml[value_start];
const size_t value_end = xml.find(quote, value_start + 1);
if (value_end == std::string_view::npos) {
throw Error(std::format("malformed MDict header: attribute {} is not closed", attr));
}
attributes[attr] = unescape_xml(xml.substr(value_start + 1, value_end - value_start - 1));
pos = value_end + 1;
}
return name;
}
// The key-block index cipher of Encrypted="2": key = RIPEMD-128(adler32 bytes
// || 0x3695 LE), then a byte-wise nibble swap XOR chain.
void decrypt_key_index(std::vector<uint8_t>& block) {
std::array<uint8_t, 8> seed{};
std::memcpy(seed.data(), block.data() + 4, 4);
seed[4] = 0x95;
seed[5] = 0x36;
const std::array<uint8_t, 16> key = ripemd128(seed.data(), seed.size());
uint8_t previous = 0x36;
for (size_t i = 8; i < block.size(); ++i) {
const uint8_t b = block[i];
uint8_t t = static_cast<uint8_t>((b >> 4) | (b << 4));
t = static_cast<uint8_t>(t ^ previous ^ static_cast<uint8_t>((i - 8) & 0xff) ^ key[(i - 8) % key.size()]);
previous = b;
block[i] = t;
}
}
// Inflates/copies the payload of a framed block into exactly `unpacked_size`
// bytes and verifies the stored Adler-32.
std::vector<uint8_t> decode_framed(const std::vector<uint8_t>& framed, uint64_t unpacked_size, const char* what) {
if (framed.size() < 8) {
throw Error(std::format("truncated {}: {} bytes, framing needs 8", what, framed.size()));
}
if (unpacked_size > max_block_size) {
throw Error(std::format("{} claims {} bytes, over the {} MiB limit", what, unpacked_size,
max_block_size / (1024 * 1024)));
}
const uint32_t compression = le32(framed.data());
const uint32_t expected = be32(framed.data() + 4);
const uint8_t* payload = framed.data() + 8;
const size_t payload_size = framed.size() - 8;
std::vector<uint8_t> out(static_cast<size_t>(unpacked_size));
if (compression == 0) {
if (payload_size != unpacked_size) {
throw Error(std::format("{}: stored size {} does not match declared size {}", what, payload_size,
unpacked_size));
}
std::memcpy(out.data(), payload, payload_size);
} else if (compression == 1) {
size_t produced = 0;
const auto result = lzokay::decompress(payload, payload_size, out.data(), out.size(), produced);
if (result != lzokay::EResult::Success || produced != unpacked_size) {
throw Error(std::format("{}: LZO data is corrupt or does not match declared size {}", what, unpacked_size));
}
} else if (compression == 2) {
struct Decompressor {
libdeflate_decompressor* handle = libdeflate_alloc_decompressor();
~Decompressor() { libdeflate_free_decompressor(handle); }
};
thread_local Decompressor decompressor;
if (!decompressor.handle) {
throw Error("out of memory allocating a zlib decompressor");
}
size_t produced = 0;
const auto result = libdeflate_zlib_decompress(decompressor.handle, payload, payload_size, out.data(),
out.size(), &produced);
if (result != LIBDEFLATE_SUCCESS || produced != unpacked_size) {
throw Error(std::format("{}: zlib data is corrupt or does not match declared size {}", what, unpacked_size));
}
} else {
throw Error(std::format("unsupported compression type {} in {}", compression, what));
}
if (adler32(out.data(), out.size()) != expected) {
throw Error(std::format("{}: Adler-32 checksum mismatch", what));
}
return out;
}
}
std::string utf16le_to_utf8(const uint8_t* data, size_t size) {
std::string out;
out.reserve(size);
const size_t units = size / 2;
for (size_t i = 0; i < units; ++i) {
uint32_t cp = uint32_t{data[2 * i]} | (uint32_t{data[2 * i + 1]} << 8);
if (cp >= 0xd800 && cp <= 0xdbff) {
if (i + 1 < units) {
const uint32_t low = uint32_t{data[2 * i + 2]} | (uint32_t{data[2 * i + 3]} << 8);
if (low >= 0xdc00 && low <= 0xdfff) {
cp = 0x10000 + ((cp - 0xd800) << 10) + (low - 0xdc00);
i++;
} else {
cp = 0xfffd;
}
} else {
cp = 0xfffd;
}
} else if (cp >= 0xdc00 && cp <= 0xdfff) {
cp = 0xfffd;
}
append_utf8(out, cp);
}
return out;
}
bool looks_like_mdict(const uint8_t* data, size_t size) {
static constexpr std::string_view roots[] = {"<Dictionary", "<Library_Data"};
if (size < 4) {
return false;
}
const uint32_t length = be32(data);
if (length < 22 || length > max_header_size) {
return false;
}
for (std::string_view root : roots) {
if (size < 4 + root.size() * 2) {
continue;
}
bool match = true;
for (size_t i = 0; i < root.size() && match; ++i) {
match = data[4 + 2 * i] == static_cast<uint8_t>(root[i]) && data[5 + 2 * i] == 0;
}
if (match) {
return true;
}
}
return false;
}
// Bounds-checked reads from the mapping; every offset the reader follows goes
// through here.
class Cursor {
public:
Cursor(const memory::mapped_file& file, uint64_t offset, const char* what)
: file_(file), pos_(offset), what_(what) {}
uint64_t pos() const { return pos_; }
const uint8_t* take(uint64_t n) {
if (pos_ > file_.size || file_.size - pos_ < n) {
throw Error(std::format("truncated file: {} needs {} bytes at offset {}, file has {}", what_, n, pos_,
file_.size));
}
const uint8_t* p = file_.data + pos_;
pos_ += n;
return p;
}
uint64_t number(size_t width) {
const uint8_t* p = take(width);
return width == 8 ? be64(p) : be32(p);
}
std::vector<uint8_t> bytes(uint64_t n) {
const uint8_t* p = take(n);
return std::vector<uint8_t>(p, p + n);
}
private:
const memory::mapped_file& file_;
uint64_t pos_;
const char* what_;
};
Reader::~Reader() { memory::unmap(file_); }
void Reader::open(const std::filesystem::path& path) {
memory::unmap(file_);
key_blocks_.clear();
record_blocks_.clear();
header_ = Header{};
file_ = memory::map_rd(path);
if (!file_) {
throw Error("could not open file");
}
parse_header();
parse_key_section();
parse_record_section();
}
void Reader::parse_header() {
Cursor cursor(file_, 0, "header");
const uint64_t length = cursor.number(4);
if (length > max_header_size) {
throw Error(std::format("header claims {} bytes, over the {} MiB limit", length, max_header_size / (1024 * 1024)));
}
const uint8_t* text = cursor.take(length);
const uint32_t checksum = le32(cursor.take(4));
if (adler32(text, length) != checksum) {
throw Error("header Adler-32 checksum mismatch");
}
const std::string xml = utf16le_to_utf8(text, length);
const std::string root = parse_element(xml, header_.attributes);
if (root == "Dictionary") {
header_.kind = Kind::Mdx;
} else if (root == "Library_Data") {
header_.kind = Kind::Mdd;
} else {
throw Error(std::format("not an MDict file: header element is <{}>", root));
}
auto attr = [this](const char* name) -> std::string {
auto it = header_.attributes.find(name);
return it == header_.attributes.end() ? std::string() : it->second;
};
header_.engine_version = attr("GeneratedByEngineVersion");
{
const std::string_view text = header_.engine_version;
const auto [end, ec] = std::from_chars(text.data(), text.data() + text.size(), header_.version);
if (ec != std::errc{} || end == text.data()) {
header_.version = 0;
}
}
if (header_.version < 1.0) {
throw Error(std::format("unsupported MDX engine version \"{}\"", header_.engine_version));
}
if (header_.version >= 3.0) {
throw Error(std::format("unsupported MDX engine version {} (only 1.x and 2.0 are readable)",
header_.engine_version));
}
num_width_ = header_.version >= 2.0 ? 8 : 4;
const std::string encrypted = attr("Encrypted");
if (encrypted.empty() || lower(encrypted) == "no") {
header_.encrypted = 0;
} else if (lower(encrypted) == "yes") {
header_.encrypted = 1;
} else {
header_.encrypted = std::atoi(encrypted.c_str());
}
if (header_.encrypted & 1) {
throw Error("unsupported: registration-protected MDX (Encrypted=1 needs a user key)");
}
if (header_.kind == Kind::Mdd) {
header_.encoding = Encoding::Utf16le;
} else {
const std::string encoding = lower(attr("Encoding"));
if (encoding.empty() || encoding == "utf-8" || encoding == "utf8") {
header_.encoding = Encoding::Utf8;
} else if (encoding == "utf-16" || encoding == "utf16" || encoding == "utf-16le") {
header_.encoding = Encoding::Utf16le;
} else {
throw Error(std::format("unsupported MDX encoding: {}", attr("Encoding")));
}
}
header_.format = attr("Format");
header_.compact = lower(attr("Compact")) == "yes" || lower(attr("Compat")) == "yes";
header_.stylesheet = attr("StyleSheet");
header_.title = attr("Title");
header_.description = attr("Description");
key_section_offset_ = cursor.pos();
}
void Reader::parse_key_section() {
Cursor cursor(file_, key_section_offset_, "key section header");
const bool v2 = num_width_ == 8;
const uint8_t* counts = cursor.take(v2 ? 40 : 16);
size_t p = 0;
auto next = [&]() {
const uint64_t v = v2 ? be64(counts + p) : be32(counts + p);
p += num_width_;
return v;
};
const uint64_t num_blocks = next();
key_count_ = next();
const uint64_t info_unpacked = v2 ? next() : 0;
const uint64_t info_packed = next();
const uint64_t blocks_packed = next();
if (v2) {
const uint32_t checksum = be32(cursor.take(4));
if (adler32(counts, 40) != checksum) {
throw Error("key section header Adler-32 checksum mismatch");
}
}
if (num_blocks == 0 || key_count_ == 0) {
throw Error("empty dictionary: no key blocks");
}
if (info_packed > max_block_size || info_unpacked > max_block_size) {
throw Error("key-block index claims a size over the 256 MiB limit");
}
std::vector<uint8_t> info = cursor.bytes(info_packed);
const uint64_t blocks_start = cursor.pos();
if (v2) {
if (header_.encrypted & 2) {
if (info.size() < 8) {
throw Error("truncated key-block index");
}
decrypt_key_index(info);
}
info = decode_framed(info, info_unpacked, "key-block index");
}
const size_t width = header_.encoding == Encoding::Utf16le ? 2 : 1;
const size_t size_field = num_width_ / 4;
const size_t terminator = v2 ? 1 : 0;
size_t pos = 0;
auto need = [&](size_t n) {
if (info.size() - pos < n) {
throw Error("truncated key-block index");
}
};
auto number = [&]() {
need(num_width_);
const uint64_t v = v2 ? be64(info.data() + pos) : be32(info.data() + pos);
pos += num_width_;
return v;
};
auto key_text = [&]() {
need(size_field);
const uint64_t chars = size_field == 2 ? (uint64_t{info[pos]} << 8) | info[pos + 1] : info[pos];
pos += size_field;
const size_t bytes = static_cast<size_t>(chars + terminator) * width;
need(bytes);
// The stored text is NUL terminated in v2; the terminator is not part of the key.
std::string text = decode_key(info.data() + pos, static_cast<size_t>(chars) * width);
pos += bytes;
return text;
};
key_blocks_.reserve(static_cast<size_t>(std::min<uint64_t>(num_blocks, info.size())));
uint64_t file_offset = blocks_start;
uint64_t entries_total = 0;
for (uint64_t i = 0; i < num_blocks; ++i) {
KeyBlockInfo block;
block.entries = number();
block.first_key = key_text();
block.last_key = key_text();
block.packed_size = number();
block.unpacked_size = number();
block.file_offset = file_offset;
const uint64_t used = file_offset - blocks_start;
// Every entry needs at least its offset number and a terminated key.
if (block.packed_size < 8 || block.unpacked_size > max_block_size || block.packed_size > blocks_packed ||
used > blocks_packed - block.packed_size || block.entries > block.unpacked_size / (num_width_ + width)) {
throw Error(std::format("key block {} has an impossible size", i));
}
file_offset += block.packed_size;
entries_total += block.entries;
key_blocks_.push_back(std::move(block));
}
if (file_offset - blocks_start != blocks_packed) {
throw Error("key blocks do not add up to the declared key section size");
}
if (entries_total != key_count_) {
throw Error(std::format("key-block index lists {} entries, header says {}", entries_total, key_count_));
}
if (file_offset > file_.size) {
throw Error("truncated file: key blocks run past the end");
}
record_section_offset_ = file_offset;
}
void Reader::parse_record_section() {
Cursor cursor(file_, record_section_offset_, "record section header");
const uint64_t num_blocks = cursor.number(num_width_);
const uint64_t num_entries = cursor.number(num_width_);
const uint64_t info_size = cursor.number(num_width_);
const uint64_t blocks_size = cursor.number(num_width_);
if (num_entries != key_count_) {
throw Error(std::format("record section lists {} entries, key section {}", num_entries, key_count_));
}
if (num_blocks == 0 || num_blocks > file_.size / (2 * num_width_) || info_size != num_blocks * 2 * num_width_) {
throw Error("record-block index size does not match its block count");
}
Cursor info(file_, cursor.pos(), "record-block index");
const uint64_t blocks_start = cursor.pos() + info_size;
record_blocks_.reserve(static_cast<size_t>(num_blocks));
uint64_t file_offset = blocks_start;
uint64_t unpacked_offset = 0;
for (uint64_t i = 0; i < num_blocks; ++i) {
RecordBlockInfo block;
block.packed_size = info.number(num_width_);
block.unpacked_size = info.number(num_width_);
block.file_offset = file_offset;
block.unpacked_offset = unpacked_offset;
const uint64_t used = file_offset - blocks_start;
if (block.packed_size < 8 || block.unpacked_size > max_block_size || block.packed_size > blocks_size ||
used > blocks_size - block.packed_size) {
throw Error(std::format("record block {} has an impossible size", i));
}
file_offset += block.packed_size;
unpacked_offset += block.unpacked_size;
record_blocks_.push_back(block);
}
if (file_offset - blocks_start != blocks_size) {
throw Error("record blocks do not add up to the declared record section size");
}
if (file_offset > file_.size) {
throw Error(std::format("truncated file: record blocks end at {}, file has {} bytes", file_offset, file_.size));
}
record_space_size_ = unpacked_offset;
}
std::vector<uint8_t> Reader::unpack_block(uint64_t offset, uint64_t packed_size, uint64_t unpacked_size,
const char* what) const {
Cursor cursor(file_, offset, what);
return decode_framed(cursor.bytes(packed_size), unpacked_size, what);
}
std::string Reader::decode_key(const uint8_t* data, size_t size) const {
if (header_.encoding == Encoding::Utf16le) {
return utf16le_to_utf8(data, size);
}
return std::string(reinterpret_cast<const char*>(data), size);
}
void Reader::read_key_block(size_t index, std::vector<KeyEntry>& out) const {
const KeyBlockInfo& info = key_blocks_.at(index);
const std::vector<uint8_t> block = unpack_block(info.file_offset, info.packed_size, info.unpacked_size, "key block");
const size_t width = header_.encoding == Encoding::Utf16le ? 2 : 1;
size_t pos = 0;
uint64_t produced = 0;
while (pos < block.size()) {
if (block.size() - pos < num_width_) {
throw Error(std::format("truncated key block {}", index));
}
KeyEntry entry;
entry.record_offset = num_width_ == 8 ? be64(block.data() + pos) : be32(block.data() + pos);
pos += num_width_;
size_t end = pos;
while (true) {
if (block.size() - end < width) {
throw Error(std::format("key block {}: key text is not terminated", index));
}
if (block[end] == 0 && (width == 1 || block[end + 1] == 0)) {
break;
}
end += width;
}
entry.key = decode_key(block.data() + pos, end - pos);
pos = end + width;
out.push_back(std::move(entry));
produced++;
}
if (produced != info.entries) {
throw Error(std::format("key block {} holds {} entries, index says {}", index, produced, info.entries));
}
}
std::vector<KeyEntry> Reader::read_all_keys() const {
std::vector<KeyEntry> keys;
keys.reserve(static_cast<size_t>(key_count_));
for (size_t i = 0; i < key_blocks_.size(); ++i) {
read_key_block(i, keys);
}
return keys;
}
size_t Reader::record_block_for(uint64_t offset) const {
auto it = std::upper_bound(record_blocks_.begin(), record_blocks_.end(), offset,
[](uint64_t value, const RecordBlockInfo& block) {
return value < block.unpacked_offset;
});
if (it == record_blocks_.begin() || offset >= record_space_size_) {
throw Error(std::format("record offset {} is outside the record space of {} bytes", offset, record_space_size_));
}
return static_cast<size_t>(std::distance(record_blocks_.begin(), it) - 1);
}
std::vector<uint8_t> Reader::read_record_block(size_t index) const {
const RecordBlockInfo& info = record_blocks_.at(index);
return unpack_block(info.file_offset, info.packed_size, info.unpacked_size, "record block");
}
std::string_view Reader::record_in_block(const std::vector<uint8_t>& block, size_t block_index, uint64_t offset,
uint64_t next_offset) const {
const RecordBlockInfo& info = record_blocks_.at(block_index);
if (offset < info.unpacked_offset || offset - info.unpacked_offset > block.size()) {
throw Error(std::format("record offset {} is not inside record block {}", offset, block_index));
}
const uint64_t block_end = info.unpacked_offset + block.size();
const uint64_t end = std::clamp(next_offset, offset, block_end);
const auto* begin = reinterpret_cast<const char*>(block.data()) + (offset - info.unpacked_offset);
return std::string_view(begin, static_cast<size_t>(end - offset));
}
std::string Reader::record_text(std::string_view record) const {
if (header_.encoding == Encoding::Utf16le) {
while (record.size() >= 2 && record[record.size() - 1] == '\0' && record[record.size() - 2] == '\0') {
record.remove_suffix(2);
}
return utf16le_to_utf8(reinterpret_cast<const uint8_t*>(record.data()), record.size());
}
while (!record.empty() && record.back() == '\0') {
record.remove_suffix(1);
}
return std::string(record);
}
}
vendor/hachidori/third_party/hoshidicts/src/mdict/mdict_reader.hpp
Vendored
+150
View File
@@ -0,0 +1,150 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <filesystem>
#include <map>
#include <stdexcept>
#include <string>
#include <string_view>
#include <vector>
#include "../memory/memory.hpp"
// Reader for the MDict container format (.mdx dictionaries and .mdd resource
// archives), engine versions 1.x and 2.0. Layout after
// https://github.com/zhansliu/writemdict/blob/master/fileformat.md:
//
// header BE u32 length, UTF-16LE XML element, LE adler32
// key section counts, key-block index (v2: zlib, optionally XOR-ciphered),
// key blocks (each: LE u32 compression, BE adler32, payload)
// record section counts, record-block index, record blocks (same framing)
//
// Every key entry is (record offset, key text); the record offset addresses the
// concatenation of all decompressed record blocks. Keys are stored sorted, and
// records in key order, so an entry's record ends where the next entry's
// begins (or at the end of its block).
//
// The reader maps the file and decodes on demand: opening reads the header
// and both indexes; key blocks and record blocks are decompressed per call,
// so callers can stream a large dictionary without holding every record.
// All const methods are safe to call concurrently.
//
// Every failure throws mdict::Error with a message that names what was
// unsupported or malformed (engine version, encoding, compression type,
// encryption mode, checksum, truncation).
namespace mdict {
class Error : public std::runtime_error {
public:
using std::runtime_error::runtime_error;
};
enum class Kind : uint8_t { Mdx, Mdd };
enum class Encoding : uint8_t { Utf8, Utf16le };
struct Header {
Kind kind = Kind::Mdx;
// Numeric engine version (1.x or 2.0) and the raw attribute.
double version = 0;
std::string engine_version;
Encoding encoding = Encoding::Utf8;
// Bit 0: record blocks ciphered (needs a registration code; rejected).
// Bit 1: key-block index ciphered (handled).
int encrypted = 0;
std::string format; // "Html" or "Text" for MDX
bool compact = false;
std::string stylesheet; // raw StyleSheet attribute (backtick substitution table)
std::string title;
std::string description;
// Every attribute of the root element with XML entities decoded.
std::map<std::string, std::string> attributes;
};
struct KeyEntry {
uint64_t record_offset = 0;
std::string key; // UTF-8 whatever the file encoding
};
struct KeyBlockInfo {
uint64_t entries = 0;
uint64_t packed_size = 0;
uint64_t unpacked_size = 0;
// Offset of the block's framing within the file.
uint64_t file_offset = 0;
std::string first_key;
std::string last_key;
};
struct RecordBlockInfo {
uint64_t packed_size = 0;
uint64_t unpacked_size = 0;
uint64_t file_offset = 0;
// Offset of the block's first byte in the decompressed record space.
uint64_t unpacked_offset = 0;
};
class Reader {
public:
Reader() = default;
~Reader();
Reader(const Reader&) = delete;
Reader& operator=(const Reader&) = delete;
// Maps the file and decodes the header, key-block index and record-block
// index. Throws mdict::Error.
void open(const std::filesystem::path& path);
const Header& header() const { return header_; }
uint64_t key_count() const { return key_count_; }
const std::vector<KeyBlockInfo>& key_blocks() const { return key_blocks_; }
const std::vector<RecordBlockInfo>& record_blocks() const { return record_blocks_; }
// Total size of the decompressed record space.
uint64_t record_space_size() const { return record_space_size_; }
// Decodes key block `index` and appends its entries to `out` in file order.
void read_key_block(size_t index, std::vector<KeyEntry>& out) const;
// Every key entry of the file in file order.
std::vector<KeyEntry> read_all_keys() const;
// Index of the record block containing decompressed-space `offset`.
size_t record_block_for(uint64_t offset) const;
// Decompresses (and verifies) record block `index`.
std::vector<uint8_t> read_record_block(size_t index) const;
// The record starting at `offset` in the decompressed space. `next_offset`
// is the following entry's record offset (or record_space_size() for the
// last entry); the record is clipped to its block. `block` must be the
// block returned by read_record_block(record_block_for(offset)).
std::string_view record_in_block(const std::vector<uint8_t>& block, size_t block_index, uint64_t offset,
uint64_t next_offset) const;
// MDX record bytes -> UTF-8 text with the terminating NUL(s) removed.
std::string record_text(std::string_view record) const;
private:
void parse_header();
void parse_key_section();
void parse_record_section();
std::vector<uint8_t> unpack_block(uint64_t offset, uint64_t packed_size, uint64_t unpacked_size,
const char* what) const;
std::string decode_key(const uint8_t* data, size_t size) const;
memory::mapped_file file_;
Header header_;
size_t num_width_ = 8;
uint64_t key_section_offset_ = 0;
uint64_t record_section_offset_ = 0;
uint64_t key_count_ = 0;
std::vector<KeyBlockInfo> key_blocks_;
std::vector<RecordBlockInfo> record_blocks_;
uint64_t record_space_size_ = 0;
};
// Decodes a UTF-16LE byte sequence to UTF-8, replacing unpaired surrogates
// with U+FFFD. Exposed for the MDD stylesheet sniffing in MdictSource.
std::string utf16le_to_utf8(const uint8_t* data, size_t size);
// True when the first bytes look like an MDict header (BE length followed by
// UTF-16LE "<Dictionary" or "<Library_Data"). Used by import() to dispatch.
bool looks_like_mdict(const uint8_t* data, size_t size);
}
vendor/hachidori/third_party/hoshidicts/src/mdict/mdict_source.cpp
Vendored
+577
View File
@@ -0,0 +1,577 @@
#include "mdict_source.hpp"
#include <algorithm>
#include <cctype>
#include <format>
#include <utility>
#include "../path_utils.hpp"
namespace mdict {
namespace {
constexpr std::string_view link_prefix = "@@@LINK=";
constexpr std::string_view placeholder_title = "Title (No HTML code allowed)";
std::string_view trim_nul_and_space(std::string_view s) {
while (!s.empty() && (s.back() == '\0' || std::isspace(static_cast<unsigned char>(s.back())))) {
s.remove_suffix(1);
}
while (!s.empty() && (s.front() == '\0' || std::isspace(static_cast<unsigned char>(s.front())))) {
s.remove_prefix(1);
}
return s;
}
std::string lower(std::string_view s) {
std::string out(s);
for (auto& c : out) {
c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
}
return out;
}
std::string strip_tags(std::string_view s) {
std::string out;
bool in_tag = false;
for (char c : s) {
if (c == '<') {
in_tag = true;
} else if (c == '>') {
in_tag = false;
} else if (!in_tag) {
out += c;
}
}
return out;
}
// The title becomes the output directory name, so it must be a single plain
// path component.
std::string sanitize_title(std::string_view raw, const std::string& fallback) {
std::string title(trim_nul_and_space(strip_tags(raw)));
title = std::string(trim_nul_and_space(title));
if (title.empty() || title == placeholder_title) {
title = fallback;
}
for (auto& c : title) {
if (c == '/' || c == '\\' || c == '\0') {
c = '_';
}
}
if (title.empty() || title == "." || title == "..") {
title = "mdx-dictionary";
}
return title;
}
// MDD keys are file paths written by the dictionary author. Backslashes become
// slashes and leading slashes go; a key that tries to leave its root, names a
// drive or embeds a NUL is dropped rather than repaired.
std::optional<std::string> normalize_mdd_key(std::string_view raw) {
std::string key(raw);
if (key.find('\0') != std::string::npos) {
return std::nullopt;
}
std::replace(key.begin(), key.end(), '\\', '/');
size_t start = 0;
while (start < key.size() && key[start] == '/') {
start++;
}
key.erase(0, start);
if (key.empty()) {
return std::nullopt;
}
if (key.size() >= 2 && std::isalpha(static_cast<unsigned char>(key[0])) && key[1] == ':') {
return std::nullopt;
}
size_t pos = 0;
while (pos <= key.size()) {
const size_t end = key.find('/', pos);
const std::string_view part =
std::string_view(key).substr(pos, end == std::string::npos ? std::string::npos : end - pos);
if (part == "..") {
return std::nullopt;
}
if (end == std::string::npos) {
break;
}
pos = end + 1;
}
return key;
}
// Stylesheet bytes from an MDD are UTF-8 or UTF-16 with or without a BOM.
std::optional<std::string> decode_stylesheet(const std::vector<char>& bytes) {
const auto* data = reinterpret_cast<const uint8_t*>(bytes.data());
const size_t size = bytes.size();
std::string text;
if (size >= 2 && data[0] == 0xff && data[1] == 0xfe) {
text = utf16le_to_utf8(data + 2, size - 2);
} else if (size >= 2 && data[0] == 0xfe && data[1] == 0xff) {
std::vector<uint8_t> swapped(data + 2, data + size);
for (size_t i = 0; i + 1 < swapped.size(); i += 2) {
std::swap(swapped[i], swapped[i + 1]);
}
text = utf16le_to_utf8(swapped.data(), swapped.size());
} else if (size >= 3 && data[0] == 0xef && data[1] == 0xbb && data[2] == 0xbf) {
text.assign(bytes.begin() + 3, bytes.end());
} else if (std::find(bytes.begin(), bytes.end(), '\0') != bytes.end()) {
// No BOM: a stylesheet never contains NUL, so any NUL means UTF-16LE.
text = utf16le_to_utf8(data, size);
} else {
text.assign(bytes.begin(), bytes.end());
}
text = std::string(trim_nul_and_space(text));
if (text.empty() || text.find('\0') != std::string::npos) {
return std::nullopt;
}
return text;
}
bool starts_with_link(std::string_view record, Encoding encoding) {
if (encoding == Encoding::Utf8) {
return record.starts_with(link_prefix);
}
if (record.size() < link_prefix.size() * 2) {
return false;
}
for (size_t i = 0; i < link_prefix.size(); ++i) {
if (record[2 * i] != link_prefix[i] || record[2 * i + 1] != '\0') {
return false;
}
}
return true;
}
// Reads consecutive records with one decompressed block in hand.
class RecordCursor {
public:
explicit RecordCursor(const Reader& reader) : reader_(reader) {}
std::string_view record(uint64_t offset, uint64_t next_offset) {
const size_t block = reader_.record_block_for(offset);
if (!have_ || block != block_index_) {
block_ = reader_.read_record_block(block);
block_index_ = block;
have_ = true;
}
return reader_.record_in_block(block_, block_index_, offset, next_offset);
}
private:
const Reader& reader_;
std::vector<uint8_t> block_;
size_t block_index_ = 0;
bool have_ = false;
};
}
void MdictSource::open(const std::filesystem::path& mdx_path, std::string fallback_title) {
mdx_.open(mdx_path);
if (mdx_.header().kind == Kind::Mdd) {
throw Error("this is an MDD resource file; import the .mdx dictionary next to it instead");
}
title_ = sanitize_title(mdx_.header().title, fallback_title);
keys_ = mdx_.read_all_keys();
for (auto& entry : keys_) {
entry.key = std::string(trim_nul_and_space(entry.key));
}
index_redirects();
if (terms_.empty()) {
if (redirect_count_ > 0) {
throw Error("MDX has no usable entries: every entry is a redirect whose target is missing");
}
throw Error("MDX has no usable entries");
}
discover_mdds(mdx_path);
bank_count_ = (terms_.size() + bank_size - 1) / bank_size;
entries_.push_back(SourceEntry{"index.json", build_index_json().size()});
for (size_t bank = 0; bank < bank_count_; ++bank) {
uint64_t bytes = 0;
const size_t begin = bank * bank_size;
const size_t end = std::min(terms_.size(), begin + bank_size);
for (size_t i = begin; i < end; ++i) {
const uint32_t key = terms_[i];
const uint64_t next = key + 1 < keys_.size() ? keys_[key + 1].record_offset : mdx_.record_space_size();
bytes += next > keys_[key].record_offset ? next - keys_[key].record_offset : 0;
}
// Structured content is a few times the size of the HTML it came from;
// the importer only uses this to pace how many banks are in flight.
const bool html = lower(mdx_.header().format) != "text";
entries_.push_back(SourceEntry{std::format("term_bank_{}.json", bank + 1), html ? bytes * 3 : bytes});
}
}
void MdictSource::index_redirects() {
RecordCursor cursor(mdx_);
const Encoding encoding = mdx_.header().encoding;
terms_.reserve(keys_.size());
for (size_t i = 0; i < keys_.size(); ++i) {
const KeyEntry& entry = keys_[i];
if (entry.key.empty()) {
continue;
}
const uint64_t next = i + 1 < keys_.size() ? keys_[i + 1].record_offset : mdx_.record_space_size();
const std::string_view record = cursor.record(entry.record_offset, next);
if (!starts_with_link(record, encoding)) {
terms_.push_back(static_cast<uint32_t>(i));
continue;
}
const std::string text = mdx_.record_text(record);
const std::string target(trim_nul_and_space(std::string_view(text).substr(link_prefix.size())));
if (target.empty() || target == entry.key) {
continue;
}
auto& aliases = redirects_[target];
if (std::find(aliases.begin(), aliases.end(), entry.key) == aliases.end()) {
aliases.push_back(entry.key);
redirect_count_++;
}
}
}
// `base` + `suffix`, matched exactly first and then ignoring case, so
// Dict.MDD next to Dict.mdx is found on a case-sensitive file system.
std::optional<std::filesystem::path> find_sibling(const std::filesystem::path& base, const std::string& suffix) {
const std::filesystem::path exact = std::filesystem::path(base).concat(suffix);
std::error_code ec;
if (std::filesystem::is_regular_file(exact, ec)) {
return exact;
}
const std::string wanted = lower(path_utils::to_utf8(exact.filename()));
const std::filesystem::path dir = exact.parent_path().empty() ? "." : exact.parent_path();
for (const auto& entry : std::filesystem::directory_iterator(dir, ec)) {
if (entry.is_regular_file(ec) && lower(path_utils::to_utf8(entry.path().filename())) == wanted) {
return entry.path();
}
}
return std::nullopt;
}
void MdictSource::discover_mdds(const std::filesystem::path& mdx_path) {
std::vector<std::filesystem::path> candidates;
std::filesystem::path base = mdx_path;
base.replace_extension();
if (auto plain = find_sibling(base, ".mdd")) {
candidates.push_back(*plain);
}
for (int n = 1;; ++n) {
auto numbered = find_sibling(base, std::format(".{}.mdd", n));
if (!numbered) {
break;
}
candidates.push_back(*numbered);
}
for (const auto& path : candidates) {
Mdd mdd;
mdd.reader = std::make_unique<Reader>();
try {
mdd.reader->open(path);
} catch (const Error& e) {
throw Error(std::format("{}: {}", path_utils::to_utf8(path.filename()), e.what()));
}
if (mdd.reader->header().kind != Kind::Mdd) {
throw Error(std::format("{}: not an MDD resource file", path_utils::to_utf8(path.filename())));
}
mdd.keys = mdd.reader->read_all_keys();
const size_t mdd_index = mdds_.size();
for (size_t k = 0; k < mdd.keys.size(); ++k) {
auto key = normalize_mdd_key(mdd.keys[k].key);
if (!key) {
continue;
}
const MddAsset asset{mdd_index, k};
if (!assets_.try_emplace(*key, asset).second) {
continue;
}
assets_lowercase_.try_emplace(lower(*key), asset);
if (lower(*key).ends_with(".css")) {
css_keys_.push_back(*key);
}
}
mdds_.push_back(std::move(mdd));
mdd_paths_.push_back(path);
}
std::sort(css_keys_.begin(), css_keys_.end());
}
std::string MdictSource::build_index_json() const {
const std::string description(trim_nul_and_space(mdx_.header().description));
return std::format(R"({{"title":{},"revision":"mdx import","sequenced":true,"format":3,"description":{}}})",
json_quote(title_), json_quote(description));
}
int MdictSource::find(std::string_view name) const {
for (size_t i = 0; i < entries_.size(); ++i) {
if (entries_[i].name == name) {
return static_cast<int>(i);
}
}
return -1;
}
std::string MdictSource::read(int index) const {
if (index < 0 || static_cast<size_t>(index) >= entries_.size()) {
return {};
}
if (index == 0) {
return build_index_json();
}
if (static_cast<size_t>(index) <= bank_count_) {
const size_t bank = static_cast<size_t>(index) - 1;
{
std::lock_guard lock(mutex_);
if (bank_cache_ && bank_cache_->first == bank) {
std::string json = std::move(bank_cache_->second);
bank_cache_.reset();
return json;
}
}
std::string json = build_bank(bank);
if (bank == 0) {
// The importer reads the first bank twice (zstd trainer, then the
// import proper); keep it for the second read.
std::lock_guard lock(mutex_);
bank_cache_ = std::make_pair(bank, json);
}
return json;
}
if (index == styles_index_) {
return styles_;
}
return {};
}
std::string MdictSource::build_bank(size_t bank) const {
const Header& header = mdx_.header();
const bool text_format = lower(header.format) == "text";
ConvertOptions options;
options.asset_prefix = std::string(asset_prefix);
options.enable_audio = false;
RecordCursor cursor(mdx_);
std::string json = "[";
const size_t begin = bank * bank_size;
const size_t end = std::min(terms_.size(), begin + bank_size);
std::vector<std::pair<std::string, std::string>> stylesheets;
std::vector<EmbeddedAsset> embedded;
std::vector<std::string> references;
for (size_t i = begin; i < end; ++i) {
const uint32_t key = terms_[i];
const KeyEntry& entry = keys_[key];
const uint64_t next = key + 1 < keys_.size() ? keys_[key + 1].record_offset : mdx_.record_space_size();
const std::string text = mdx_.record_text(cursor.record(entry.record_offset, next));
const std::string_view definition = trim_nul_and_space(text);
std::string glossary;
if (text_format) {
glossary = json_quote(definition);
} else {
const std::string html = apply_stylesheet(definition, header.stylesheet);
ConvertResult converted = convert_html(html, options);
glossary = std::move(converted.glossary_json);
for (auto& [name, css] : converted.inline_stylesheets) {
stylesheets.emplace_back(std::format("{:08}/{}/{}", i, entry.key, name), std::move(css));
}
for (auto& asset : converted.embedded_assets) {
embedded.push_back(std::move(asset));
}
for (auto& reference : converted.asset_references) {
references.push_back(std::move(reference));
}
}
std::vector<std::string_view> expressions{entry.key};
if (auto aliases = redirects_.find(entry.key); aliases != redirects_.end()) {
for (const std::string& alias : aliases->second) {
if (std::find(expressions.begin(), expressions.end(), alias) == expressions.end()) {
expressions.push_back(alias);
}
}
}
for (std::string_view expression : expressions) {
if (json.size() > 1) {
json += ',';
}
json += '[';
json += json_quote(expression);
json += R"(,"","","",0,[)";
json += glossary;
json += "],";
json += std::to_string(i);
json += R"(,""])";
}
}
json += ']';
if (!stylesheets.empty() || !embedded.empty() || !references.empty()) {
std::lock_guard lock(mutex_);
for (auto& [name, css] : stylesheets) {
if (inline_stylesheet_names_.insert(name).second) {
inline_stylesheets_.emplace_back(std::move(name), std::move(css));
}
}
for (auto& asset : embedded) {
if (embedded_asset_paths_.insert(asset.path).second) {
embedded_assets_.push_back(std::move(asset));
}
}
for (auto& reference : references) {
asset_references_.insert(std::move(reference));
}
}
return json;
}
const MdictSource::MddAsset* MdictSource::find_asset(const std::string& key) const {
if (auto it = assets_.find(key); it != assets_.end()) {
return &it->second;
}
if (auto it = assets_lowercase_.find(lower(key)); it != assets_lowercase_.end()) {
return &it->second;
}
return nullptr;
}
std::vector<char> MdictSource::asset_bytes(const MddAsset& asset) const {
const Mdd& mdd = mdds_[asset.mdd];
const KeyEntry& entry = mdd.keys[asset.key];
const uint64_t next = asset.key + 1 < mdd.keys.size() ? mdd.keys[asset.key + 1].record_offset
: mdd.reader->record_space_size();
const size_t block_index = mdd.reader->record_block_for(entry.record_offset);
const std::vector<uint8_t> block = mdd.reader->read_record_block(block_index);
const std::string_view record = mdd.reader->record_in_block(block, block_index, entry.record_offset, next);
return std::vector<char>(record.begin(), record.end());
}
std::string MdictSource::build_styles() const {
std::vector<std::string> sections;
std::vector<std::string> references;
for (const std::string& key : css_keys_) {
const MddAsset* asset = find_asset(key);
if (!asset) {
continue;
}
auto css = decode_stylesheet(asset_bytes(*asset));
if (!css) {
continue;
}
sections.push_back(std::format("/* Source: {} */\n{}", key,
rewrite_css_asset_urls(*css, asset_prefix, key, references)));
}
// Inline blocks were collected from several threads; their names start with
// the term's sequence so the order is the dictionary's, not the threads'.
std::vector<std::pair<std::string, std::string>> inline_sheets;
{
std::lock_guard lock(mutex_);
inline_sheets = inline_stylesheets_;
}
std::sort(inline_sheets.begin(), inline_sheets.end());
for (const auto& [name, css] : inline_sheets) {
const std::string_view display = std::string_view(name).substr(name.find('/') + 1);
sections.push_back(std::format("/* Source: {} */\n{}", display,
rewrite_css_asset_urls(css, asset_prefix, {}, references)));
}
{
std::lock_guard lock(mutex_);
for (auto& reference : references) {
asset_references_.insert(std::move(reference));
}
}
if (sections.empty()) {
return {};
}
std::string out;
for (size_t i = 0; i < sections.size(); ++i) {
if (i) {
out += "\n\n";
}
out += sections[i];
}
out += '\n';
return out;
}
void MdictSource::finish_banks() {
if (banks_finished_) {
return;
}
banks_finished_ = true;
styles_ = build_styles();
if (!styles_.empty()) {
styles_index_ = static_cast<int>(entries_.size());
entries_.push_back(SourceEntry{"styles.css", styles_.size()});
}
std::set<std::string> added;
auto add_asset = [&](const std::string& key) {
const MddAsset* asset = find_asset(key);
// media.bin stores the path length in 16 bits.
if (!asset || asset_prefix.size() + key.size() > 0xffff || !added.insert(key).second) {
return;
}
const Mdd& mdd = mdds_[asset->mdd];
const KeyEntry& entry = mdd.keys[asset->key];
const uint64_t next = asset->key + 1 < mdd.keys.size() ? mdd.keys[asset->key + 1].record_offset
: mdd.reader->record_space_size();
media_.push_back(MediaEntry{*asset, std::nullopt});
entries_.push_back(SourceEntry{std::string(asset_prefix) + key, next - entry.record_offset});
};
// Every stylesheet asset, then whatever the glossaries and stylesheets refer to.
for (const std::string& key : css_keys_) {
add_asset(key);
}
std::set<std::string> references;
std::vector<EmbeddedAsset> embedded;
{
std::lock_guard lock(mutex_);
references = asset_references_;
embedded = embedded_assets_;
}
for (const std::string& key : references) {
if (lower(key).ends_with(".css")) {
continue;
}
add_asset(key);
}
std::sort(embedded.begin(), embedded.end(),
[](const EmbeddedAsset& a, const EmbeddedAsset& b) { return a.path < b.path; });
for (size_t i = 0; i < embedded.size(); ++i) {
if (embedded[i].path.size() > 0xffff) {
continue;
}
media_.push_back(MediaEntry{std::nullopt, i});
entries_.push_back(SourceEntry{embedded[i].path, embedded[i].data.size()});
}
{
std::lock_guard lock(mutex_);
embedded_assets_ = std::move(embedded);
}
}
std::optional<SourceMediaFile> MdictSource::read_media(int index) const {
const size_t first_media = entries_.size() - media_.size();
if (index < 0 || static_cast<size_t>(index) < first_media || static_cast<size_t>(index) >= entries_.size()) {
return std::nullopt;
}
const MediaEntry& media = media_[static_cast<size_t>(index) - first_media];
SourceMediaFile out;
out.path = entries_[static_cast<size_t>(index)].name;
if (media.asset) {
try {
out.blob = asset_bytes(*media.asset);
} catch (const Error&) {
// A corrupt block loses this asset, not the whole dictionary.
return std::nullopt;
}
} else {
std::lock_guard lock(mutex_);
const auto& data = embedded_assets_[*media.embedded].data;
out.blob.assign(data.begin(), data.end());
}
return out;
}
}
vendor/hachidori/third_party/hoshidicts/src/mdict/mdict_source.hpp
Vendored
+112
View File
@@ -0,0 +1,112 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <set>
#include <string>
#include <string_view>
#include <vector>
#include "../source/dictionary_source.hpp"
#include "html_to_structured.hpp"
#include "mdict_reader.hpp"
// An MDX dictionary (plus its sibling MDD resource files) presented to the
// importer as a Yomitan dictionary: index.json, term_bank_N.json, styles.css
// and mdict-media/... entries. Nothing is written to disk; each bank is
// produced from the MDX record blocks when the importer asks for it.
//
// Opening reads the key index and makes one pass over the record blocks to
// find @@@LINK= redirects (an alias is emitted as an extra headword of its
// target, one hop, missing targets dropped). Banks are then materialised per
// read(): bank N holds terms [N*10000, (N+1)*10000) of the non-redirect
// entries in file order, each as [expression, "", "", "", 0, [glossary],
// sequence, ""], glossary being structured content converted from the HTML
// (or a plain string for Format=Text).
//
// Media is discovered while the banks convert (which MDD assets the glossaries
// reference, plus data: URLs), so the media entries and the final styles.css
// (MDD *.css plus inline <style> blocks) exist only after finish_banks().
//
// Errors are mdict::Error with a specific message; a file whose header parses
// as an MDD is rejected with a hint to import the .mdx instead.
namespace mdict {
class MdictSource final : public DictionarySource {
public:
static constexpr size_t bank_size = 10000;
static constexpr std::string_view asset_prefix = "mdict-media/";
// `fallback_title` is used when the header has no usable Title (typically
// the file's stem).
void open(const std::filesystem::path& mdx_path, std::string fallback_title);
const std::vector<SourceEntry>& entries() const override { return entries_; }
int find(std::string_view name) const override;
std::string read(int index) const override;
std::optional<SourceMediaFile> read_media(int index) const override;
void finish_banks() override;
const Header& header() const { return mdx_.header(); }
const std::string& title() const { return title_; }
size_t term_count() const { return terms_.size(); }
size_t redirect_count() const { return redirect_count_; }
const std::vector<std::filesystem::path>& mdd_paths() const { return mdd_paths_; }
private:
struct MddAsset {
size_t mdd = 0;
size_t key = 0;
};
struct Mdd {
std::unique_ptr<Reader> reader;
std::vector<KeyEntry> keys;
};
struct MediaEntry {
// Exactly one of the two is set.
std::optional<MddAsset> asset;
std::optional<size_t> embedded;
};
void discover_mdds(const std::filesystem::path& mdx_path);
void index_redirects();
std::string build_index_json() const;
std::string build_bank(size_t bank) const;
std::string build_styles() const;
const MddAsset* find_asset(const std::string& key) const;
std::vector<char> asset_bytes(const MddAsset& asset) const;
Reader mdx_;
std::vector<std::filesystem::path> mdd_paths_;
std::vector<Mdd> mdds_;
std::map<std::string, MddAsset> assets_;
std::map<std::string, MddAsset> assets_lowercase_;
std::vector<std::string> css_keys_;
std::string title_;
std::vector<KeyEntry> keys_;
// Indices into keys_ of the entries that become terms, in file order.
std::vector<uint32_t> terms_;
std::map<std::string, std::vector<std::string>> redirects_;
size_t redirect_count_ = 0;
std::vector<SourceEntry> entries_;
size_t bank_count_ = 0;
int styles_index_ = -1;
std::string styles_;
std::vector<MediaEntry> media_;
bool banks_finished_ = false;
// Collected while banks convert, from several threads.
mutable std::mutex mutex_;
mutable std::vector<std::pair<std::string, std::string>> inline_stylesheets_;
mutable std::set<std::string> inline_stylesheet_names_;
mutable std::vector<EmbeddedAsset> embedded_assets_;
mutable std::set<std::string> embedded_asset_paths_;
mutable std::set<std::string> asset_references_;
mutable std::optional<std::pair<size_t, std::string>> bank_cache_;
};
}
vendor/hachidori/third_party/hoshidicts/src/mdict/ripemd128.cpp
Vendored
+100
View File
@@ -0,0 +1,100 @@
#include "ripemd128.hpp"
#include <cstring>
namespace mdict {
namespace {
constexpr std::array<uint8_t, 64> left_index = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
7, 4, 13, 1, 10, 6, 15, 3, 12, 0, 9, 5, 2, 14, 11, 8,
3, 10, 14, 4, 9, 15, 8, 1, 2, 7, 0, 6, 13, 11, 5, 12,
1, 9, 11, 10, 0, 8, 12, 4, 13, 3, 7, 15, 14, 5, 6, 2};
constexpr std::array<uint8_t, 64> right_index = {5, 14, 7, 0, 9, 2, 11, 4, 13, 6, 15, 8, 1, 10, 3, 12,
6, 11, 3, 7, 0, 13, 5, 10, 14, 15, 8, 12, 4, 9, 1, 2,
15, 5, 1, 3, 7, 14, 6, 9, 11, 8, 12, 2, 10, 0, 4, 13,
8, 6, 4, 1, 3, 11, 15, 0, 5, 12, 2, 13, 9, 7, 10, 14};
constexpr std::array<uint8_t, 64> left_shift = {11, 14, 15, 12, 5, 8, 7, 9, 11, 13, 14, 15, 6, 7, 9, 8,
7, 6, 8, 13, 11, 9, 7, 15, 7, 12, 15, 9, 11, 7, 13, 12,
11, 13, 6, 7, 14, 9, 13, 15, 14, 8, 13, 6, 5, 12, 7, 5,
11, 12, 14, 15, 14, 15, 9, 8, 9, 14, 5, 6, 8, 6, 5, 12};
constexpr std::array<uint8_t, 64> right_shift = {8, 9, 9, 11, 13, 15, 15, 5, 7, 7, 8, 11, 14, 14, 12, 6,
9, 13, 15, 7, 12, 8, 9, 11, 7, 7, 12, 7, 6, 15, 13, 11,
9, 7, 15, 11, 8, 6, 6, 14, 12, 13, 5, 14, 13, 13, 7, 5,
15, 5, 8, 11, 14, 14, 6, 14, 6, 9, 12, 9, 12, 5, 15, 8};
constexpr std::array<uint32_t, 4> left_k = {0x00000000, 0x5a827999, 0x6ed9eba1, 0x8f1bbcdc};
constexpr std::array<uint32_t, 4> right_k = {0x50a28be6, 0x5c4dd124, 0x6d703ef3, 0x00000000};
uint32_t rol(uint32_t x, unsigned n) { return (x << n) | (x >> (32 - n)); }
uint32_t f(unsigned round, uint32_t x, uint32_t y, uint32_t z) {
switch (round) {
case 0:
return x ^ y ^ z;
case 1:
return (x & y) | (~x & z);
case 2:
return (x | ~y) ^ z;
default:
return (x & z) | (y & ~z);
}
}
void compress(std::array<uint32_t, 4>& h, const uint8_t* block) {
std::array<uint32_t, 16> x{};
for (size_t i = 0; i < 16; ++i) {
x[i] = uint32_t{block[4 * i]} | (uint32_t{block[4 * i + 1]} << 8) | (uint32_t{block[4 * i + 2]} << 16) |
(uint32_t{block[4 * i + 3]} << 24);
}
uint32_t al = h[0], bl = h[1], cl = h[2], dl = h[3];
uint32_t ar = h[0], br = h[1], cr = h[2], dr = h[3];
for (unsigned j = 0; j < 64; ++j) {
const unsigned round = j / 16;
uint32_t t = rol(al + f(round, bl, cl, dl) + x[left_index[j]] + left_k[round], left_shift[j]);
al = dl;
dl = cl;
cl = bl;
bl = t;
t = rol(ar + f(3 - round, br, cr, dr) + x[right_index[j]] + right_k[round], right_shift[j]);
ar = dr;
dr = cr;
cr = br;
br = t;
}
const uint32_t t = h[1] + cl + dr;
h[1] = h[2] + dl + ar;
h[2] = h[3] + al + br;
h[3] = h[0] + bl + cr;
h[0] = t;
}
}
std::array<uint8_t, 16> ripemd128(const uint8_t* data, size_t size) {
std::array<uint32_t, 4> h = {0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476};
size_t offset = 0;
while (size - offset >= 64) {
compress(h, data + offset);
offset += 64;
}
std::array<uint8_t, 128> tail{};
const size_t rest = size - offset;
std::memcpy(tail.data(), data + offset, rest);
tail[rest] = 0x80;
const size_t padded = rest < 56 ? 64 : 128;
const uint64_t bits = static_cast<uint64_t>(size) * 8;
for (size_t i = 0; i < 8; ++i) {
tail[padded - 8 + i] = static_cast<uint8_t>(bits >> (8 * i));
}
compress(h, tail.data());
if (padded == 128) {
compress(h, tail.data() + 64);
}
std::array<uint8_t, 16> digest{};
for (size_t i = 0; i < 4; ++i) {
for (size_t b = 0; b < 4; ++b) {
digest[4 * i + b] = static_cast<uint8_t>(h[i] >> (8 * b));
}
}
return digest;
}
}
vendor/hachidori/third_party/hoshidicts/src/mdict/ripemd128.hpp
Vendored
+13
View File
@@ -0,0 +1,13 @@
#pragma once
#include <array>
#include <cstddef>
#include <cstdint>
namespace mdict {
// RIPEMD-128 (Dobbertin, Bosselaers, Preneel 1996). MDX files with
// Encrypted="2" derive the key-index cipher key from it; nothing else in the
// project needs it, so this is a small standalone implementation rather than
// a crypto dependency.
std::array<uint8_t, 16> ripemd128(const uint8_t* data, size_t size);
}
vendor/hachidori/third_party/hoshidicts/src/memory/memory.cpp
Vendored
+164
View File
@@ -0,0 +1,164 @@
#include "memory.hpp"
#ifdef _WIN32
#include <windows.h>
#else
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#endif
namespace memory {
mapped_file map_rd(const std::filesystem::path& path) {
#ifdef _WIN32
HANDLE file =
CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
if (file == INVALID_HANDLE_VALUE) {
return {};
}
LARGE_INTEGER file_size;
if (!GetFileSizeEx(file, &file_size) || file_size.QuadPart == 0) {
CloseHandle(file);
return {};
}
HANDLE mapping = CreateFileMappingW(file, nullptr, PAGE_READONLY, 0, 0, nullptr);
CloseHandle(file);
if (!mapping) {
return {};
}
auto* data = static_cast<uint8_t*>(MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0));
CloseHandle(mapping);
if (!data) {
return {};
}
return {.data = data, .size = static_cast<size_t>(file_size.QuadPart)};
#else
#if defined(__EMSCRIPTEN__) && HOSHIDICTS_WASMFS
// WasmFS's OPFS backend serves a read-only descriptor from a Blob: the whole
// file is copied into a JavaScript ArrayBuffer and then into the heap, with
// an async round trip in between. A read-write descriptor uses a sync access
// handle that reads straight into the heap, about twice as fast. WasmFS's
// mmap copies the file either way and its munmap never touches a read-only
// mapping's descriptor, so the handle (and its lock on the file) is released
// as soon as the copy is done. The classic Emscripten FS is different on
// both counts, hence the gate.
int fd = open(path.c_str(), O_RDWR);
if (fd < 0) {
fd = open(path.c_str(), O_RDONLY);
}
constexpr bool keep_fd = false;
#else
int fd = open(path.c_str(), O_RDONLY);
#ifdef __EMSCRIPTEN__
constexpr bool keep_fd = true;
#else
constexpr bool keep_fd = false;
#endif
#endif
if (fd < 0) {
return {};
}
struct stat st{};
if (fstat(fd, &st) != 0 || st.st_size == 0) {
close(fd);
return {};
}
auto* data = static_cast<uint8_t*>(mmap(nullptr, st.st_size, PROT_READ, MAP_SHARED, fd, 0));
if (!keep_fd) {
close(fd);
fd = -1;
}
if (data == reinterpret_cast<uint8_t*>(MAP_FAILED)) {
if (fd >= 0) {
close(fd);
}
return {};
}
return {.data = data, .size = static_cast<size_t>(st.st_size), .fd = fd};
#endif
}
mapped_file map_rw(const std::filesystem::path& path, size_t file_size) {
if (file_size == 0) {
return {};
}
#ifdef _WIN32
HANDLE file = CreateFileW(path.c_str(), GENERIC_READ | GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS,
FILE_ATTRIBUTE_NORMAL, nullptr);
if (file == INVALID_HANDLE_VALUE) {
return {};
}
LARGE_INTEGER size;
size.QuadPart = static_cast<LONGLONG>(file_size);
if (!SetFilePointerEx(file, size, nullptr, FILE_BEGIN) || !SetEndOfFile(file)) {
CloseHandle(file);
return {};
}
HANDLE mapping = CreateFileMappingW(file, nullptr, PAGE_READWRITE, size.HighPart, size.LowPart, nullptr);
CloseHandle(file);
if (!mapping) {
return {};
}
auto* data = static_cast<uint8_t*>(MapViewOfFile(mapping, FILE_MAP_WRITE, 0, 0, file_size));
CloseHandle(mapping);
if (!data) {
return {};
}
return {.data = data, .size = file_size};
#else
int fd = open(path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644);
if (fd < 0) {
return {};
}
if (ftruncate(fd, static_cast<off_t>(file_size)) < 0) {
close(fd);
return {};
}
auto* data = static_cast<uint8_t*>(mmap(nullptr, file_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
#ifndef __EMSCRIPTEN__
close(fd);
fd = -1;
#endif
if (data == reinterpret_cast<uint8_t*>(MAP_FAILED)) {
if (fd >= 0) {
close(fd);
}
return {};
}
return {.data = data, .size = file_size, .fd = fd};
#endif
}
void unmap(mapped_file mapping) {
if (!mapping.data) {
return;
}
#ifdef _WIN32
FlushViewOfFile(mapping.data, 0);
UnmapViewOfFile(mapping.data);
#else
msync(mapping.data, mapping.size, MS_SYNC);
munmap(mapping.data, mapping.size);
if (mapping.fd >= 0) {
close(mapping.fd);
}
#endif
}
}
vendor/hachidori/third_party/hoshidicts/src/memory/memory.hpp
Vendored
+22
View File
@@ -0,0 +1,22 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <filesystem>
namespace memory {
struct mapped_file {
uint8_t* data = nullptr;
size_t size = 0;
// Emscripten's mmap emulation flushes MAP_SHARED writes through the file
// descriptor when msync/munmap runs, so the descriptor has to outlive the
// mapping there. Left at -1 on platforms that close it eagerly.
int fd = -1;
explicit operator bool() const { return data != nullptr; }
};
mapped_file map_rd(const std::filesystem::path& path);
mapped_file map_rw(const std::filesystem::path& path, size_t file_size);
void unmap(mapped_file mapping);
}
vendor/hachidori/third_party/hoshidicts/src/path_utils.hpp
Vendored
+16
View File
@@ -0,0 +1,16 @@
#pragma once
#include <filesystem>
#include <string>
#include <string_view>
namespace path_utils {
inline std::filesystem::path from_utf8(std::string_view value) {
return std::filesystem::path(std::u8string(reinterpret_cast<const char8_t*>(value.data()), value.size()));
}
inline std::string to_utf8(const std::filesystem::path& value) {
const std::u8string encoded = value.u8string();
return {reinterpret_cast<const char*>(encoded.data()), encoded.size()};
}
}
vendor/hachidori/third_party/hoshidicts/src/query.cpp
Vendored
+752
View File
@@ -0,0 +1,752 @@
#include "hoshidicts/query.hpp"
#include <ankerl/unordered_dense.h>
#define ZSTD_STATIC_LINKING_ONLY
#include <zstd.h>
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <memory>
#include <ranges>
#include <string_view>
#include <vector>
#include "hash/hash.hpp"
#include "hoshidicts/importer.hpp"
#include "json/yomitan_parser.hpp"
#include "memory/memory.hpp"
#include "path_utils.hpp"
#include "query_internal.hpp"
#include "scan_index.hpp"
namespace {
template <typename T>
T read_val(const uint8_t*& addr) {
T val;
std::memcpy(&val, addr, sizeof(T));
addr += sizeof(T);
return val;
}
std::string_view read_str(const uint8_t*& addr, uint32_t len) {
std::string_view result(reinterpret_cast<const char*>(addr), len);
addr += len;
return result;
}
ZSTD_DCtx* thread_dctx() {
static thread_local std::unique_ptr<ZSTD_DCtx, decltype(&ZSTD_freeDCtx)> ctx(ZSTD_createDCtx(), ZSTD_freeDCtx);
return ctx.get();
}
}
struct DictionaryQuery::DictionaryData {
int version;
hash::linear table;
hash::bloom bloom;
memory::mapped_file blobs;
memory::mapped_file hash_table;
memory::mapped_file bloom_filter;
memory::mapped_file media;
memory::mapped_file media_index;
// Optional long-key scan index (see src/scan_index.hpp); absent for
// dictionaries imported before it existed and for ones without long keys.
memory::mapped_file scan_index;
ZSTD_DDict* zstd_dict = nullptr;
~DictionaryData() {
memory::unmap(blobs);
memory::unmap(hash_table);
memory::unmap(bloom_filter);
memory::unmap(media);
memory::unmap(media_index);
memory::unmap(scan_index);
ZSTD_freeDDict(zstd_dict);
}
struct ScanIndexView {
uint32_t count = 0;
uint16_t max_key_length = 0;
const uint8_t* hashes = nullptr;
const uint8_t* lengths = nullptr;
};
// A view over the mapped file, or an empty one when the file is missing,
// has an unknown version, or is not the size its header claims.
ScanIndexView scan_index_view() const {
ScanIndexView view;
if (!scan_index || scan_index.size < scan_index::header_bytes) {
return view;
}
const uint8_t* addr = scan_index.data;
if (read_val<uint32_t>(addr) != scan_index::magic || read_val<uint32_t>(addr) != scan_index::version) {
return view;
}
const auto count = read_val<uint32_t>(addr);
const auto max_key_length = read_val<uint16_t>(addr);
const size_t expected = scan_index::header_bytes + static_cast<size_t>(count) * (sizeof(uint64_t) + sizeof(uint16_t));
if (scan_index.size != expected) {
return view;
}
view.count = count;
view.max_key_length = max_key_length;
view.hashes = scan_index.data + scan_index::header_bytes;
view.lengths = view.hashes + static_cast<size_t>(count) * sizeof(uint64_t);
return view;
}
// Longest key sharing the hashed prefix, or 0.
size_t long_key_length(uint64_t prefix_hash) const {
const ScanIndexView view = scan_index_view();
size_t lo = 0;
size_t hi = view.count;
while (lo < hi) {
const size_t mid = lo + (hi - lo) / 2;
uint64_t hash;
std::memcpy(&hash, view.hashes + mid * sizeof(uint64_t), sizeof(hash));
if (hash < prefix_hash) {
lo = mid + 1;
} else if (hash > prefix_hash) {
hi = mid;
} else {
uint16_t length;
std::memcpy(&length, view.lengths + mid * sizeof(uint16_t), sizeof(length));
return length;
}
}
return 0;
}
};
DictionaryQuery::DictionaryQuery() = default;
DictionaryQuery::~DictionaryQuery() = default;
DictionaryQuery::DictionaryQuery(DictionaryQuery&&) noexcept = default;
DictionaryQuery& DictionaryQuery::operator=(DictionaryQuery&&) noexcept = default;
DictionaryQuery::Dictionary::Dictionary() = default;
DictionaryQuery::Dictionary::~Dictionary() = default;
DictionaryQuery::Dictionary::Dictionary(Dictionary&&) noexcept = default;
DictionaryQuery::Dictionary& DictionaryQuery::Dictionary::operator=(Dictionary&&) noexcept = default;
bool DictionaryQuery::add_dict(const std::string& path_utf8, DictionaryType type) {
try {
return add_dict_(path_utf8, type);
} catch (const std::exception&) {
return false;
}
}
bool DictionaryQuery::add_dict_(const std::string& path_utf8, DictionaryType type) {
const std::filesystem::path path = path_utils::from_utf8(path_utf8);
// Marker layout: _1/_2 are legacy; _3 and _4 store the term score as an int32
// and differ only in whether dict.zstd was trained (_4); _5 and _6 are the
// same pair with the score stored as a double, which is what the Yomitan
// schema's JSON number can hold (fractions, and magnitudes beyond int32).
int version = 0;
if (std::filesystem::is_regular_file(path / ".hoshidicts_6")) {
version = 6;
} else if (std::filesystem::is_regular_file(path / ".hoshidicts_5")) {
version = 5;
} else if (std::filesystem::is_regular_file(path / ".hoshidicts_4")) {
version = 4;
} else if (std::filesystem::is_regular_file(path / ".hoshidicts_3")) {
version = 3;
} else if (std::filesystem::is_regular_file(path / ".hoshidicts_2")) {
version = 2;
} else if (std::filesystem::is_regular_file(path / ".hoshidicts_1")) {
version = 1;
} else {
return false;
}
Dictionary dict;
dict.path = path_utf8;
Summary summary;
std::ifstream index_file(path / "index.json", std::ios::binary);
if (!index_file) {
return false;
}
std::string buf(std::istreambuf_iterator<char>(index_file), {});
if (glz::read<glz::opts{.error_on_unknown_keys = false}>(summary, buf)) {
return false;
}
dict.name = summary.title.empty() ? path_utils::to_utf8(path.stem()) : summary.title;
dict.styles = summary.styles;
if (dict.styles.empty() && std::filesystem::exists(path / "styles.css")) {
std::ifstream f(path / "styles.css");
dict.styles = std::string(std::istreambuf_iterator<char>(f), {});
}
dict.data = std::make_unique<DictionaryData>();
dict.data->version = version;
dict.data->hash_table = memory::map_rd(path / "hash.table");
if (!dict.data->hash_table) {
return false;
}
if (!dict.data->table.load(dict.data->hash_table.data, dict.data->hash_table.size)) {
return false;
}
dict.data->bloom_filter = memory::map_rd(path / "bloom.filter");
if (!dict.data->bloom_filter) {
return false;
}
if (!dict.data->bloom.load(dict.data->bloom_filter.data, dict.data->bloom_filter.size)) {
return false;
}
dict.data->table.set_bloom(&dict.data->bloom);
dict.data->blobs = memory::map_rd(path / "blobs.bin");
if (!dict.data->blobs) {
return false;
}
dict.data->media = memory::map_rd(path / "media.bin");
if (dict.data->media) {
dict.data->media_index = memory::map_rd(path / "media.idx");
}
if (type == TERM && std::filesystem::is_regular_file(path / scan_index::file_name)) {
dict.data->scan_index = memory::map_rd(path / scan_index::file_name);
}
if (version == 4 || version == 6) {
std::ifstream f(path / "dict.zstd", std::ios::binary);
const std::string blob(std::istreambuf_iterator<char>(f), {});
dict.data->zstd_dict =
ZSTD_createDDict_advanced(blob.data(), blob.size(), ZSTD_dlm_byCopy, ZSTD_dct_fullDict, ZSTD_defaultCMem);
if (dict.data->zstd_dict == nullptr) {
return false;
}
}
switch (type) {
case TERM:
term_dicts_.push_back(std::move(dict));
break;
case FREQ:
freq_dicts_.push_back(std::move(dict));
break;
case PITCH:
pitch_dicts_.push_back(std::move(dict));
break;
case KANJI:
kanji_dicts_.push_back(std::move(dict));
break;
}
return true;
}
bool DictionaryQuery::add_term_dict(const std::string& path) {
return add_dict(path, DictionaryQuery::DictionaryType::TERM);
}
bool DictionaryQuery::add_freq_dict(const std::string& path) {
return add_dict(path, DictionaryQuery::DictionaryType::FREQ);
}
bool DictionaryQuery::add_pitch_dict(const std::string& path) {
return add_dict(path, DictionaryQuery::DictionaryType::PITCH);
}
bool DictionaryQuery::add_kanji_dict(const std::string& path) {
return add_dict(path, DictionaryQuery::DictionaryType::KANJI);
}
size_t DictionaryQuery::remove_dict(const std::string& path) {
size_t removed = 0;
for (auto* dicts : {&term_dicts_, &freq_dicts_, &pitch_dicts_, &kanji_dicts_}) {
removed += std::erase_if(*dicts, [&path](const Dictionary& d) { return d.path == path; });
}
return removed;
}
bool DictionaryQuery::set_dict_order(const std::vector<std::string>& paths) {
// A listed path is rejected unless some kind of it is loaded; otherwise the
// caller's view of the loaded set has drifted and it should rebuild instead.
for (const auto& path : paths) {
const auto loaded = [&path](const std::vector<Dictionary>& dicts) {
return std::ranges::any_of(dicts, [&path](const Dictionary& d) { return d.path == path; });
};
if (!loaded(term_dicts_) && !loaded(freq_dicts_) && !loaded(pitch_dicts_) && !loaded(kanji_dicts_)) {
return false;
}
}
const auto rank = [&paths](const Dictionary& d) {
const auto it = std::ranges::find(paths, d.path);
return it == paths.end() ? paths.size() : static_cast<size_t>(it - paths.begin());
};
for (auto* dicts : {&term_dicts_, &freq_dicts_, &pitch_dicts_, &kanji_dicts_}) {
std::ranges::stable_sort(*dicts, {}, rank);
}
return true;
}
size_t DictionaryQuery::long_key_length(std::string_view text, const std::string* term_dictionary_path) const {
const auto hash = scan_index::prefix_hash(text);
if (!hash) {
return 0;
}
size_t longest = 0;
for (const auto& dict : term_dicts_) {
if (term_dictionary_path != nullptr && dict.path != *term_dictionary_path) {
continue;
}
longest = std::max(longest, dict.data->long_key_length(*hash));
}
return longest;
}
size_t DictionaryQuery::max_long_key_length(const std::string* term_dictionary_path) const {
size_t longest = 0;
for (const auto& dict : term_dicts_) {
if (term_dictionary_path != nullptr && dict.path != *term_dictionary_path) {
continue;
}
longest = std::max<size_t>(longest, dict.data->scan_index_view().max_key_length);
}
return longest;
}
std::vector<TermResult> DictionaryQuery::query(const std::string& expression) const {
RawTerms raw = query_raw(expression);
std::vector<TermResult> results;
results.reserve(raw.terms.size());
for (auto& term : raw.terms) {
results.push_back(build_term(raw, term));
}
std::ranges::sort(results, [](const TermResult& a, const TermResult& b) {
return a.expression != b.expression ? a.expression < b.expression : a.reading < b.reading;
});
for (auto& term : results) {
materialize(term);
}
return results;
}
RawTerms DictionaryQuery::query_raw(const std::string& expression,
const std::string* term_dictionary_path) const {
RawTerms raw;
auto find_term = [&raw](std::string_view expr, std::string_view reading) -> RawTerm* {
for (auto& term : raw.terms) {
if (term.expression == expr && term.reading == reading) {
return &term;
}
}
return nullptr;
};
for (const auto& [path, name, styles, data] : term_dicts_) {
if (term_dictionary_path != nullptr && path != *term_dictionary_path) {
continue;
}
uint64_t offset_addr = data->table(expression);
if (offset_addr == 0) {
continue;
}
const uint8_t* index_addr = data->blobs.data + offset_addr;
auto count = read_val<uint32_t>(index_addr);
raw.terms.reserve(raw.terms.size() + count);
raw.glossaries.reserve(raw.glossaries.size() + count);
for (uint32_t i = 0; i < count; i++) {
auto offset = read_val<uint64_t>(index_addr);
const uint8_t* blob_addr = data->blobs.data + offset;
// first byte encodes term (0) or meta (1) entry
auto type = read_val<uint8_t>(blob_addr);
if (type != 0) {
continue;
}
auto expr_len = read_val<uint16_t>(blob_addr);
std::string_view expr = read_str(blob_addr, expr_len);
auto reading_len = read_val<uint16_t>(blob_addr);
std::string_view reading = read_str(blob_addr, reading_len);
if (expr != expression && reading != expression) {
continue;
}
auto glossary_offset = read_val<uint64_t>(blob_addr);
auto glossary_size = read_val<uint32_t>(blob_addr);
auto def_tags_size = read_val<uint8_t>(blob_addr);
std::string_view definition_tags = read_str(blob_addr, def_tags_size);
auto rules_size = read_val<uint8_t>(blob_addr);
std::string_view rules = read_str(blob_addr, rules_size);
auto term_tag_size = read_val<uint8_t>(blob_addr);
std::string_view term_tags = read_str(blob_addr, term_tag_size);
if (data->version >= 2) {
auto redirect_count = read_val<uint32_t>(blob_addr);
for (uint32_t r = 0; r < redirect_count; r++) {
auto form_of_len = read_val<uint32_t>(blob_addr);
read_str(blob_addr, form_of_len);
auto rule_count = read_val<uint32_t>(blob_addr);
for (uint32_t j = 0; j < rule_count; j++) {
auto rule_len = read_val<uint32_t>(blob_addr);
read_str(blob_addr, rule_len);
}
}
}
double score = 0;
if (data->version >= 5) {
score = read_val<double>(blob_addr);
} else if (data->version >= 3) {
score = read_val<int32_t>(blob_addr);
}
const auto glossary_index = static_cast<uint32_t>(raw.glossaries.size());
raw.glossaries.push_back(RawGlossary{.dict_name = &name,
.definition_tags = definition_tags,
.term_tags = term_tags,
.rules = rules,
.compressed_data = data->blobs.data + glossary_offset,
.compressed_size = glossary_size,
.zstd_dict = data->zstd_dict,
.next = UINT32_MAX});
RawTerm* term = find_term(expr, reading);
if (term == nullptr) {
raw.terms.push_back(RawTerm{.expression = expr,
.reading = reading,
.score = score,
.first_glossary = glossary_index,
.last_glossary = glossary_index,
.frequencies = {},
.pitches = {}});
} else {
raw.glossaries[term->last_glossary].next = glossary_index;
term->last_glossary = glossary_index;
term->score = std::max(term->score, score);
}
}
}
for (auto& term : raw.terms) {
collect_frequencies(term.expression, term.reading, term.frequencies);
collect_pitches(term.expression, term.reading, term.pitches);
}
return raw;
}
TermResult DictionaryQuery::build_term(const RawTerms& raw, RawTerm& term) const {
TermResult result{.expression = std::string(term.expression),
.reading = std::string(term.reading),
.rules = {},
.score = term.score,
.glossaries = {},
.frequencies = std::move(term.frequencies),
.pitches = std::move(term.pitches)};
size_t glossary_count = 0;
for (uint32_t i = term.first_glossary; i != UINT32_MAX; i = raw.glossaries[i].next) {
++glossary_count;
}
result.glossaries.reserve(glossary_count);
for (uint32_t i = term.first_glossary; i != UINT32_MAX; i = raw.glossaries[i].next) {
const RawGlossary& g = raw.glossaries[i];
if (!g.rules.empty()) {
if (!result.rules.empty()) {
result.rules += " ";
}
result.rules += g.rules;
}
GlossaryEntry& entry = result.glossaries.emplace_back();
entry.dict_name = *g.dict_name;
entry.definition_tags = g.definition_tags;
entry.term_tags = g.term_tags;
entry.compressed_data = g.compressed_data;
entry.compressed_size = g.compressed_size;
entry.zstd_dict = g.zstd_dict;
}
return result;
}
void DictionaryQuery::query_freq(std::vector<TermResult>& terms, bool match_reading) const {
for (auto& term : terms) {
collect_frequencies(term.expression, term.reading, term.frequencies, match_reading);
}
}
void DictionaryQuery::collect_frequencies(std::string_view expression, std::string_view reading,
std::vector<FrequencyEntry>& out, bool match_reading) const {
for (const auto& [path, name, styles, data] : freq_dicts_) {
uint64_t offset_addr = data->table(expression);
if (offset_addr == 0) {
continue;
}
const uint8_t* index_addr = data->blobs.data + offset_addr;
auto count = read_val<uint32_t>(index_addr);
std::vector<Frequency> frequencies;
for (uint32_t i = 0; i < count; i++) {
auto offset = read_val<uint64_t>(index_addr);
const uint8_t* blob_addr = data->blobs.data + offset;
auto type = read_val<uint8_t>(blob_addr);
if (type != 1) {
continue;
}
auto expr_len = read_val<uint16_t>(blob_addr);
std::string_view expr = read_str(blob_addr, expr_len);
if (expr != expression) {
continue;
}
auto mode_len = read_val<uint8_t>(blob_addr);
std::string_view mode = read_str(blob_addr, mode_len);
if (mode != "freq") {
continue;
}
auto freq_data_size = read_val<uint32_t>(blob_addr);
std::string_view freq_data = read_str(blob_addr, freq_data_size);
ParsedFrequency parsed;
if (yomitan_parser::parse_frequency(freq_data, parsed)) {
if (match_reading && !parsed.reading.empty() && parsed.reading != reading) {
continue;
}
frequencies.emplace_back(
Frequency{.value = parsed.value, .display_value = std::string(parsed.display_value),
.reading = std::string(parsed.reading)});
}
}
if (!frequencies.empty()) {
out.emplace_back(FrequencyEntry{.dict_name = name, .frequencies = std::move(frequencies)});
}
}
}
void DictionaryQuery::query_pitch(std::vector<TermResult>& terms) const {
for (auto& term : terms) {
collect_pitches(term.expression, term.reading, term.pitches);
}
}
void DictionaryQuery::collect_pitches(std::string_view expression, std::string_view reading,
std::vector<PitchEntry>& out) const {
for (const auto& [path, name, styles, data] : pitch_dicts_) {
uint64_t offset_addr = data->table(expression);
if (offset_addr == 0) {
continue;
}
const uint8_t* index_addr = data->blobs.data + offset_addr;
auto count = read_val<uint32_t>(index_addr);
std::vector<Pitch> pitches;
std::vector<std::string> transcriptions;
for (uint32_t i = 0; i < count; i++) {
auto offset = read_val<uint64_t>(index_addr);
const uint8_t* blob_addr = data->blobs.data + offset;
auto type = read_val<uint8_t>(blob_addr);
if (type != 1) {
continue;
}
auto expr_len = read_val<uint16_t>(blob_addr);
std::string_view expr = read_str(blob_addr, expr_len);
if (expr != expression) {
continue;
}
auto mode_len = read_val<uint8_t>(blob_addr);
std::string_view mode = read_str(blob_addr, mode_len);
ParsedPitch parsed;
if (mode == "pitch") {
auto pitch_data_size = read_val<uint32_t>(blob_addr);
std::string_view pitch_data = read_str(blob_addr, pitch_data_size);
if (yomitan_parser::parse_pitch(pitch_data, parsed)) {
if (!parsed.reading.empty() && parsed.reading != reading) {
continue;
}
for (auto& accent : parsed.pitches) {
pitches.emplace_back(Pitch{.position = accent.position,
.pattern = std::move(accent.pattern),
.nasal = std::move(accent.nasal),
.devoice = std::move(accent.devoice)});
}
}
} else if (mode == "ipa") {
auto transcriptions_data_size = read_val<uint32_t>(blob_addr);
std::string_view transcriptions_data = read_str(blob_addr, transcriptions_data_size);
if (yomitan_parser::parse_ipa(transcriptions_data, parsed)) {
if (!parsed.reading.empty() && parsed.reading != reading) {
continue;
}
for (std::string_view transcription : parsed.transcriptions) {
transcriptions.emplace_back(transcription);
}
}
}
}
if (!pitches.empty() || !transcriptions.empty()) {
out.emplace_back(PitchEntry{
.dict_name = name,
.pitches = std::move(pitches),
.transcriptions = std::move(transcriptions),
});
}
}
}
KanjiResult DictionaryQuery::query_kanji(const std::string& kanji) const {
KanjiResult result;
result.character = kanji;
for (const auto& [path, name, styles, data] : kanji_dicts_) {
uint64_t offset_addr = data->table(kanji);
if (offset_addr == 0) {
continue;
}
const uint8_t* index_addr = data->blobs.data + offset_addr;
auto count = read_val<uint32_t>(index_addr);
for (uint32_t i = 0; i < count; i++) {
auto offset = read_val<uint64_t>(index_addr);
const uint8_t* blob_addr = data->blobs.data + offset;
auto type = read_val<uint8_t>(blob_addr);
if (type != 2) {
continue;
}
auto char_len = read_val<uint8_t>(blob_addr);
std::string_view char_sv = read_str(blob_addr, char_len);
if (char_sv != kanji) {
continue;
}
auto onyomi_len = read_val<uint16_t>(blob_addr);
std::string_view onyomi = read_str(blob_addr, onyomi_len);
auto kunyomi_len = read_val<uint16_t>(blob_addr);
std::string_view kunyomi = read_str(blob_addr, kunyomi_len);
auto tags_len = read_val<uint16_t>(blob_addr);
std::string_view tags = read_str(blob_addr, tags_len);
KanjiEntry entry;
entry.dict_name = name;
entry.onyomi = onyomi;
entry.kunyomi = kunyomi;
entry.tags = tags;
auto def_count = read_val<uint16_t>(blob_addr);
for (uint16_t j = 0; j < def_count; j++) {
auto def_len = read_val<uint16_t>(blob_addr);
std::string_view def = read_str(blob_addr, def_len);
entry.definitions.emplace_back(def);
}
auto stat_count = read_val<uint16_t>(blob_addr);
for (uint16_t j = 0; j < stat_count; j++) {
auto key_len = read_val<uint16_t>(blob_addr);
std::string_view key = read_str(blob_addr, key_len);
auto val_len = read_val<uint16_t>(blob_addr);
std::string_view val = read_str(blob_addr, val_len);
entry.stats.emplace(key, val);
}
result.entries.push_back(std::move(entry));
}
}
return result;
}
std::string DictionaryQuery::decompress_glossary(const void* data, size_t size, const ZSTD_DDict_s* dict) {
if (!data || size == 0) {
return "";
}
unsigned long long decompressed_size = ZSTD_getFrameContentSize(data, size);
if (decompressed_size == ZSTD_CONTENTSIZE_ERROR || decompressed_size == ZSTD_CONTENTSIZE_UNKNOWN) {
return "";
}
std::string result;
size_t actual_size = 0;
result.resize_and_overwrite(decompressed_size, [&](char* buf, size_t capacity) {
actual_size = ZSTD_decompress_usingDDict(thread_dctx(), buf, capacity, data, size, dict);
return ZSTD_isError(actual_size) ? size_t{0} : actual_size;
});
if (ZSTD_isError(actual_size)) {
return "";
}
return result;
}
void DictionaryQuery::materialize(TermResult& term) const {
for (auto& g : term.glossaries) {
g.glossary = decompress_glossary(g.compressed_data, g.compressed_size, g.zstd_dict);
}
}
std::vector<char> DictionaryQuery::get_media_file(const std::string& dict_name, const std::string& media_path) const {
auto view = get_media_file_view(dict_name, media_path);
return {view.data, view.data + view.size};
}
MediaFileView DictionaryQuery::get_media_file_view(const std::string& dict_name, const std::string& media_path) const {
for (const auto& [path, name, styles, data] : term_dicts_) {
if (name != dict_name) {
continue;
}
if (!data->media || !data->media_index) {
return {};
}
const uint8_t* ptr = data->media_index.data;
auto count = read_val<uint32_t>(ptr);
size_t left = 0;
size_t right = count;
while (left < right) {
const size_t mid = left + (right - left) / 2;
uint64_t record_offset;
std::memcpy(&record_offset, data->media_index.data + sizeof(uint32_t) + mid * sizeof(uint64_t), sizeof(uint64_t));
const uint8_t* record = data->media.data + record_offset;
auto path_size = read_val<uint16_t>(record);
std::string_view indexed_path = read_str(record, path_size);
if (indexed_path < media_path) {
left = mid + 1;
} else if (indexed_path > media_path) {
right = mid;
} else {
auto blob_size = read_val<uint32_t>(record);
const char* blob_data = reinterpret_cast<const char*>(record);
return {.data = blob_data, .size = blob_size};
}
}
return {};
}
return {};
}
std::vector<DictionaryStyle> DictionaryQuery::get_styles() const {
return term_dicts_ | std::views::filter([](const auto& d) { return !d.styles.empty(); }) |
std::views::transform([](const auto& d) { return DictionaryStyle{d.name, d.styles}; }) |
std::ranges::to<std::vector>();
}
std::vector<std::string> DictionaryQuery::get_freq_dict_order() const {
return freq_dicts_ | std::views::transform([](const auto& d) { return d.name; }) | std::ranges::to<std::vector>();
}
vendor/hachidori/third_party/hoshidicts/src/query_internal.hpp
Vendored
+34
View File
@@ -0,0 +1,34 @@
#pragma once
#include <cstdint>
#include <string>
#include <string_view>
#include <vector>
#include "hoshidicts/query.hpp"
struct RawGlossary {
const std::string* dict_name;
std::string_view definition_tags;
std::string_view term_tags;
std::string_view rules;
const uint8_t* compressed_data;
uint32_t compressed_size;
const ZSTD_DDict_s* zstd_dict;
uint32_t next;
};
struct RawTerm {
std::string_view expression;
std::string_view reading;
double score;
uint32_t first_glossary;
uint32_t last_glossary;
std::vector<FrequencyEntry> frequencies;
std::vector<PitchEntry> pitches;
};
struct RawTerms {
std::vector<RawTerm> terms;
std::vector<RawGlossary> glossaries;
};
vendor/hachidori/third_party/hoshidicts/src/scan_index.hpp
Vendored
+80
View File
@@ -0,0 +1,80 @@
#pragma once
// Long-key scan index (`scan.idx`, optional per term dictionary).
//
// Lookup::lookup scans the input from `scan_length` code points down to one,
// running the text processors and the deinflector on every prefix, so its cost
// is linear in the scan length. A reader that wants to find a 30-character
// proverb by scanning 30 characters pays that on every hover, although almost
// no dictionary key is that long (Jitendex: 0.26% of keys are longer than 16).
//
// The importer therefore records, for every key longer than
// `long_key_min_codepoints`, the hash of its first `long_key_prefix_codepoints`
// code points and the key's length. After the ordinary scan the lookup hashes
// the processed variants of the input's first eight code points, and only when
// one of them is the prefix of some long key does it extend the scan to that
// key's length (plus room for an inflected ending). Everything else keeps the
// cost of the configured scan length.
//
// Layout (little-endian):
// u32 magic 'HDSI' u32 version (1) u32 count u16 max_key_length u16 0
// u64 prefix_hash[count] (sorted ascending)
// u16 key_length[count] (maximum length among keys sharing the prefix)
#include <xxh3.h>
#include <cstddef>
#include <cstdint>
#include <optional>
#include <string_view>
namespace scan_index {
inline constexpr uint32_t magic = 0x49534448; // "HDSI"
inline constexpr uint32_t version = 1;
inline constexpr size_t header_bytes = 16;
inline constexpr const char* file_name = "scan.idx";
// Keys at most this long are the ordinary scan's business; the index only
// knows about longer ones. 16 is the default scan length of every host.
inline constexpr size_t long_key_min_codepoints = 16;
// The prefix a long key is recognised by. Eight code points is short enough
// that an inflected ending never reaches into it for keys longer than 16, and
// long enough that ordinary text rarely shares it with a long key by chance.
inline constexpr size_t long_key_prefix_codepoints = 8;
// How far past a long key's length the extended scan looks, so that an
// inflected form (食べさせられなかった for 食べる: +7) is still covered.
inline constexpr size_t inflection_slack_codepoints = 8;
inline size_t codepoint_length(std::string_view utf8) {
size_t n = 0;
for (unsigned char c : utf8) {
n += (c & 0xC0) != 0x80;
}
return n;
}
// Byte length of the first `codepoints` code points, or nullopt when the text
// has fewer than that.
inline std::optional<size_t> prefix_bytes(std::string_view utf8, size_t codepoints) {
size_t seen = 0;
for (size_t i = 0; i < utf8.size(); ++i) {
if ((static_cast<unsigned char>(utf8[i]) & 0xC0) != 0x80) {
if (seen == codepoints) {
return i;
}
++seen;
}
}
return seen == codepoints ? std::optional<size_t>(utf8.size()) : std::nullopt;
}
inline std::optional<uint64_t> prefix_hash(std::string_view utf8) {
const auto bytes = prefix_bytes(utf8, long_key_prefix_codepoints);
if (!bytes) {
return std::nullopt;
}
return XXH3_64bits(utf8.data(), *bytes);
}
} // namespace scan_index
vendor/hachidori/third_party/hoshidicts/src/source/dictionary_source.hpp
Vendored
+50
View File
@@ -0,0 +1,50 @@
#pragma once
#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
// One file of a dictionary source as the importer sees it: a name in the
// Yomitan layout (index.json, styles.css, term_bank_N.json, media paths) and
// the size of its contents, which the importer uses to schedule work.
struct SourceEntry {
std::string name;
uint64_t uncompressed_size = 0;
};
struct SourceMediaFile {
std::string path;
std::vector<char> blob;
};
// The importer's whole view of a dictionary. A source is opened once and then
// read concurrently from the import workers, so read() and read_media() must be
// safe to call from several threads at the same time on a const source.
//
// ZipSource is the Yomitan archive; other formats (MDX) present themselves as
// the same virtual file list so everything past this seam stays format-agnostic.
class DictionarySource {
public:
virtual ~DictionarySource() = default;
virtual const std::vector<SourceEntry>& entries() const = 0;
// Index of the entry called `name`, or -1.
virtual int find(std::string_view name) const = 0;
// The contents of a text entry, empty when it cannot be read.
virtual std::string read(int index) const = 0;
// The contents of a media entry together with the path it is stored under,
// or nullopt when it cannot be read.
virtual std::optional<SourceMediaFile> read_media(int index) const = 0;
// Called once, after every bank has been read and before styles.css is read
// and media is enumerated. A source that only learns what media and styles
// it has while producing banks (MdictSource) appends those entries here;
// the importer re-scans entries() for media afterwards. ZipSource has
// nothing to do.
virtual void finish_banks() {}
};
vendor/hachidori/third_party/hoshidicts/src/source/zip_source.cpp
Vendored
+34
View File
@@ -0,0 +1,34 @@
#include "zip_source.hpp"
#include <utility>
bool ZipSource::open(const std::filesystem::path& path) {
if (!zip_.open(path)) {
return false;
}
entries_.clear();
entries_.reserve(zip_.entries.size());
for (const auto& entry : zip_.entries) {
entries_.push_back(SourceEntry{entry.name, entry.uncompressed_size});
}
return true;
}
int ZipSource::find(std::string_view name) const {
for (int i = 0; i < static_cast<int>(entries_.size()); ++i) {
if (entries_[static_cast<size_t>(i)].name == name) {
return i;
}
}
return -1;
}
std::string ZipSource::read(int index) const { return zip_.read(index); }
std::optional<SourceMediaFile> ZipSource::read_media(int index) const {
auto media = zip_.read_media(index);
if (!media.has_value()) {
return std::nullopt;
}
return SourceMediaFile{std::move(media->path), std::move(media->blob)};
}
vendor/hachidori/third_party/hoshidicts/src/source/zip_source.hpp
Vendored
+28
View File
@@ -0,0 +1,28 @@
#pragma once
#include <filesystem>
#include <string>
#include <vector>
#include "../zip/zip.hpp"
#include "dictionary_source.hpp"
// A Yomitan dictionary archive. Wraps Zip without changing how it is parsed
// or read; the entry list is a copy of the archive's central directory names
// and sizes in archive order, so indices are shared with the Zip.
class ZipSource final : public DictionarySource {
public:
// False when the archive cannot be mapped or parsed; error() then names the
// reason when the parser had one.
bool open(const std::filesystem::path& path);
const std::string& error() const { return zip_.error; }
const std::vector<SourceEntry>& entries() const override { return entries_; }
int find(std::string_view name) const override;
std::string read(int index) const override;
std::optional<SourceMediaFile> read_media(int index) const override;
private:
Zip zip_;
std::vector<SourceEntry> entries_;
};
vendor/hachidori/third_party/hoshidicts/src/text_processor/kanji_variants.cpp
Vendored
+358
View File
@@ -0,0 +1,358 @@
// generated from https://github.com/yomidevs/kanji-processor/blob/main/src/full_list.json
extern const char32_t kanji_variants[][2] = {
{0x2f49, 0x6708}, {0x2f5b, 0x7259}, {0x2fb3, 0x97f3}, {0x2fb7, 0x98df}, {0x2fc7, 0x9ebb}, {0x2fd0, 0x9f3b},
{0x3400, 0x4e18}, {0x3402, 0x559c}, {0x3406, 0x200a3}, {0x342c, 0x65d2}, {0x342d, 0x5ee9}, {0x342e, 0x8944},
{0x3452, 0x50c9}, {0x3468, 0x4fe3}, {0x346a, 0x5115}, {0x3475, 0x5ac9}, {0x3492, 0x50d5}, {0x349e, 0x5101},
{0x34b5, 0x8c8c}, {0x34db, 0x529f}, {0x34dd, 0x5211}, {0x34fb, 0x525b}, {0x351f, 0x52e2}, {0x352b, 0x5de6},
{0x3541, 0x5374}, {0x3551, 0x53a8}, {0x355c, 0x53f2}, {0x355d, 0x4e8b}, {0x3581, 0x541d}, {0x35be, 0x54a2},
{0x35cb, 0x5589}, {0x3605, 0x929c}, {0x3634, 0x5750}, {0x3644, 0x965b}, {0x3652, 0x91ce}, {0x3662, 0x5881},
{0x3672, 0x58c5}, {0x367b, 0x5edb}, {0x36f0, 0x5a5a}, {0x3707, 0x605a}, {0x373d, 0x5b50}, {0x375b, 0x5bbf},
{0x3760, 0x51a5}, {0x3761, 0x6700}, {0x3762, 0x5b87}, {0x376b, 0x7abf}, {0x376c, 0x7c20}, {0x3774, 0x5213},
{0x3775, 0x5f97}, {0x378d, 0x5c3b}, {0x37c1, 0x5cb8}, {0x37e2, 0x5d0e}, {0x37f4, 0x5d6c}, {0x3800, 0x5cf6},
{0x382f, 0x4ee5}, {0x3861, 0x5e6e}, {0x386f, 0x5b85}, {0x387f, 0x65a5}, {0x38a3, 0x64ce}, {0x38ae, 0x5f1b},
{0x3905, 0x611b}, {0x3907, 0x61ca}, {0x3917, 0x6cf0}, {0x3929, 0x606a}, {0x392c, 0x605a}, {0x393a, 0x614c},
{0x3943, 0x60b6}, {0x3966, 0x611c}, {0x3971, 0x60b1}, {0x3994, 0x61d5}, {0x39a4, 0x61ff}, {0x39af, 0x6216},
{0x39d6, 0x627c}, {0x3a14, 0x64bc}, {0x3a57, 0x643a}, {0x3a5c, 0x652c}, {0x3a85, 0x66f4}, {0x3a9f, 0x6566},
{0x3aa3, 0x6572}, {0x3abf, 0x6298}, {0x3ac4, 0x65c1}, {0x3ad6, 0x65e8}, {0x3ad7, 0x539a}, {0x3aea, 0x8202},
{0x3b05, 0x66fc}, {0x3b22, 0x66e6}, {0x3b25, 0x66b4}, {0x3b30, 0x81fe}, {0x3b31, 0x66ff}, {0x3b52, 0x801c},
{0x3b68, 0x6930}, {0x3b88, 0x67f0}, {0x3b8d, 0x7b8b}, {0x3b8e, 0x676f}, {0x3b9a, 0x6817}, {0x3ba4, 0x677e},
{0x3bae, 0x6854}, {0x3bb5, 0x6834}, {0x3bc3, 0x6f06}, {0x3bcd, 0x69f3}, {0x3bf0, 0x690d}, {0x3c0d, 0x6af3},
{0x3c0f, 0x7bd9}, {0x3c16, 0x6b16}, {0x3c41, 0x6b3e}, {0x3cb1, 0x9b23}, {0x3cc2, 0x6cbf}, {0x3cc3, 0x6dec},
{0x3cd2, 0x6cd5}, {0x3cd9, 0x6d93}, {0x3cfa, 0x6e38}, {0x3cfd, 0x7030}, {0x3d11, 0x6cdd}, {0x3d1e, 0x6ed4},
{0x3d31, 0x6df1}, {0x3d4e, 0x6f97}, {0x3d9a, 0x705e}, {0x3db3, 0x71fc}, {0x3dd4, 0x7130}, {0x3ded, 0x70fd},
{0x3e3f, 0x72a2}, {0x3e60, 0x8c5a}, {0x3e83, 0x72d7}, {0x3e85, 0x7334}, {0x3e9a, 0x737a}, {0x3f1c, 0x76ce},
{0x3f57, 0x755d}, {0x3fd7, 0x3fc9}, {0x4093, 0x898f}, {0x40c9, 0x73c9}, {0x417a, 0x79d2}, {0x41ab, 0x7a93},
{0x41b4, 0x7ac8}, {0x41bf, 0x5bf1}, {0x41f3, 0x7b8b}, {0x4207, 0x7f69}, {0x4264, 0x7c54}, {0x4275, 0x994c},
{0x42aa, 0x7cf2}, {0x42c6, 0x7e91}, {0x42dd, 0x7e93}, {0x432b, 0x7e9c}, {0x43b1, 0x8019}, {0x43ee, 0x8107},
{0x4407, 0x543b}, {0x4451, 0x50d5}, {0x445b, 0x8210}, {0x4468, 0x8235}, {0x44b3, 0x6736}, {0x450d, 0x7be4},
{0x4525, 0x856d}, {0x4543, 0x852d}, {0x45a4, 0x873f}, {0x45b8, 0x86fe}, {0x45f6, 0x881f}, {0x460f, 0x6064},
{0x461a, 0x5352}, {0x462e, 0x55aa}, {0x4633, 0x895f}, {0x46e1, 0x8a71}, {0x471b, 0x8b92}, {0x471f, 0x8b96},
{0x475c, 0x72fb}, {0x475d, 0x7317}, {0x4764, 0x7360}, {0x477f, 0x8cb4}, {0x4793, 0x8d6c}, {0x47a8, 0x8dec},
{0x47e6, 0x8dcb}, {0x47fd, 0x758f}, {0x4816, 0x8d91}, {0x4827, 0x8e87}, {0x4844, 0x8ecc}, {0x484e, 0x8f64},
{0x4890, 0x5f82}, {0x489b, 0x8fe5}, {0x48b5, 0x9116}, {0x48e9, 0x9187}, {0x48f1, 0x9157}, {0x495d, 0x93d6},
{0x49b0, 0x9b2e}, {0x49fa, 0x96c4}, {0x49ff, 0x9d72}, {0x4a04, 0x9d6a}, {0x4a7f, 0x9f16}, {0x4a8d, 0x97ca},
{0x4aa1, 0x9f4f}, {0x4ac9, 0x8c8c}, {0x4b34, 0x79e3}, {0x4b3b, 0x991e}, {0x4baf, 0x9a41}, {0x4bbd, 0x9463},
{0x4c17, 0x9b2e}, {0x4c1b, 0x9b32}, {0x4c1d, 0x7511}, {0x4c1f, 0x9b42}, {0x4c48, 0x9ba8}, {0x4cc4, 0x96cc},
{0x4cc7, 0x9d61}, {0x4d21, 0x9e97}, {0x4d77, 0x86d9}, {0x4db5, 0x7bea}, {0x4e12, 0x4e11}, {0x4e17, 0x4e16},
{0x4e46, 0x4e45}, {0x4e48, 0x5e7a}, {0x4e55, 0x864e}, {0x4e58, 0x4e57}, {0x4e79, 0x4e7e}, {0x4e82, 0x4e71},
{0x4e8a, 0x4e8b}, {0x4e8f, 0x4e8e}, {0x4e90, 0x4e8e}, {0x4e9e, 0x4e9c}, {0x4eaf, 0x4ea8}, {0x4eb0, 0x4eac},
{0x4ebb, 0x4eba}, {0x4ebc, 0x96c6}, {0x4ed0, 0x4eca}, {0x4edd, 0x540c}, {0x4eed, 0x4ede}, {0x4f16, 0x5e11},
{0x4f1c, 0x5005}, {0x4f37, 0x80c4}, {0x4f40, 0x4f3c}, {0x4f5b, 0x4ecf}, {0x4f60, 0x511e}, {0x4f84, 0x59ea},
{0x4f86, 0x6765}, {0x4fa0, 0x4fe0}, {0x4fab, 0x4f5e}, {0x4fad, 0x5118}, {0x4fd6, 0x500d}, {0x4fde, 0x516a},
{0x4fe6, 0x5114}, {0x4ff2, 0x509a}, {0x5002, 0x4f75}, {0x500a, 0x506c}, {0x5010, 0x500f}, {0x5036, 0x4ff1},
{0x5040, 0x82f1}, {0x5047, 0x4eee}, {0x5058, 0x4f83}, {0x505a, 0x4f5c}, {0x5077, 0x5078}, {0x5079, 0x5099},
{0x507b, 0x50c2}, {0x509c, 0x5fad}, {0x50a5, 0x513b}, {0x50af, 0x506c}, {0x50b3, 0x4f1d}, {0x50c3, 0x5099},
{0x50de, 0x507d}, {0x50e3, 0x50ed}, {0x50f9, 0x4fa1}, {0x5109, 0x5039}, {0x514a, 0x514c}, {0x5151, 0x514c},
{0x5152, 0x5150}, {0x5154, 0x514e}, {0x5156, 0x5157}, {0x5160, 0x515c}, {0x5167, 0x5185}, {0x5169, 0x4e21},
{0x517e, 0x5180}, {0x5184, 0x5189}, {0x518b, 0x5182}, {0x518c, 0x518a}, {0x519d, 0x5b9c}, {0x51a3, 0x6700},
{0x51a6, 0x5bc7}, {0x51a8, 0x5bcc}, {0x51ad, 0x6cf0}, {0x51b0, 0x6c37}, {0x51b2, 0x6c96}, {0x51b3, 0x6c7a},
{0x51b5, 0x6cc1}, {0x51c3, 0x6d82}, {0x51c9, 0x6dbc}, {0x51cf, 0x6e1b}, {0x51d6, 0x6e96}, {0x51db, 0x51dc},
{0x51ec, 0x98a8}, {0x51ee, 0x98a8}, {0x51f4, 0x51ed}, {0x51fe, 0x51fd}, {0x5205, 0x5275}, {0x520b, 0x520a},
{0x5226, 0x52ab}, {0x5227, 0x52ab}, {0x522a, 0x5220}, {0x524f, 0x5231}, {0x5259, 0x5231}, {0x5265, 0x525d},
{0x5269, 0x5270}, {0x5284, 0x7b9a}, {0x528b, 0x527f}, {0x528d, 0x5263}, {0x5291, 0x5264}, {0x52a4, 0x52c1},
{0x52ca, 0x524b}, {0x52cc, 0x5026}, {0x52d7, 0x52d6}, {0x52de, 0x52b4}, {0x52f3, 0x52f2}, {0x52f5, 0x52b1},
{0x52f8, 0x52e7}, {0x5300, 0x52fb}, {0x5304, 0x5303}, {0x5307, 0x5306}, {0x530a, 0x63ac}, {0x531b, 0x67e9},
{0x5332, 0x5333}, {0x5344, 0x5eff}, {0x5346, 0x5352}, {0x534b, 0x4e16}, {0x535d, 0x4e31}, {0x5367, 0x81e5},
{0x536d, 0x909b}, {0x5377, 0x5dfb}, {0x5379, 0x6064}, {0x537b, 0x5374}, {0x537d, 0x5373}, {0x5380, 0x819d},
{0x5389, 0x53b2}, {0x53a0, 0x5ec1}, {0x53a1, 0x539f}, {0x53a2, 0x5ec2}, {0x53a6, 0x5ec8}, {0x53ae, 0x5edd},
{0x53b7, 0x80b1}, {0x53ba, 0x53bb}, {0x53c3, 0x53c2}, {0x53d3, 0x4e8b}, {0x53dc, 0x53df}, {0x53f1, 0x20b9f},
{0x5412, 0x54a4}, {0x5433, 0x5449}, {0x543f, 0x544a}, {0x544d, 0x543d}, {0x5451, 0x541e}, {0x5455, 0x5614},
{0x5467, 0x8a46}, {0x548a, 0x548c}, {0x548f, 0x8a60}, {0x5492, 0x546a}, {0x549c, 0x54a4}, {0x5516, 0x555e},
{0x5557, 0x5556}, {0x5560, 0x54f2}, {0x5563, 0x929c}, {0x5586, 0x54f2}, {0x55ae, 0x5358}, {0x55bb, 0x55a9},
{0x55e5, 0x5637}, {0x55f8, 0x55f7}, {0x560a, 0x5540}, {0x5618, 0x5653}, {0x5628, 0x562f}, {0x5649, 0x5556},
{0x565b, 0x5699}, {0x568f, 0x5694}, {0x569e, 0x54f2}, {0x56a0, 0x700f}, {0x56a2, 0x56ca}, {0x56b4, 0x53b3},
{0x56bb, 0x56c2}, {0x56cf, 0x8271}, {0x56d1, 0x5631}, {0x56d3, 0x9f67}, {0x56d8, 0x56de}, {0x56d9, 0x56e0},
{0x56e7, 0x518f}, {0x56ec, 0x56de}, {0x56f1, 0x5306}, {0x5705, 0x51fd}, {0x5708, 0x570f}, {0x570b, 0x56fd},
{0x570d, 0x56f2}, {0x5713, 0x5186}, {0x5716, 0x56f3}, {0x5718, 0x56e3}, {0x5721, 0x571f}, {0x574b, 0x574c},
{0x576f, 0x574f}, {0x5775, 0x4e18}, {0x579c, 0x579b}, {0x57a8, 0x5b88}, {0x57c0, 0x5782}, {0x57c6, 0x786e},
{0x57d3, 0x57d2}, {0x57d7, 0x57e0}, {0x57dc, 0x91ce}, {0x57de, 0x5824}, {0x57f3, 0x574e}, {0x57ff, 0x6ce5},
{0x5818, 0x584d}, {0x5822, 0x5821}, {0x5826, 0x968e}, {0x582f, 0x5c2d}, {0x583a, 0x754c}, {0x583d, 0x5ca1},
{0x585f, 0x846c}, {0x5864, 0x58ce}, {0x586b, 0x5861}, {0x5872, 0x5834}, {0x589d, 0x78fd}, {0x589e, 0x5897},
{0x58aa, 0x58a9}, {0x58ab, 0x6a3d}, {0x58ae, 0x5815}, {0x58b8, 0x8e87}, {0x58bb, 0x7246}, {0x58c3, 0x7586},
{0x58c4, 0x91ce}, {0x58cd, 0x5879}, {0x58d3, 0x5727}, {0x58d8, 0x5841}, {0x58de, 0x58ca}, {0x58e0, 0x58df},
{0x58e4, 0x58cc}, {0x58e5, 0x5edb}, {0x58ef, 0x58ee}, {0x58f7, 0x58fa}, {0x58f9, 0x58f1}, {0x58fb, 0x5a7f},
{0x58fd, 0x5bff}, {0x5914, 0x21582}, {0x5918, 0x536f}, {0x591b, 0x591a}, {0x5967, 0x5965}, {0x5969, 0x5333},
{0x596c, 0x5968}, {0x598d, 0x59f8}, {0x5992, 0x59ac}, {0x59b8, 0x5a3f}, {0x59ca, 0x59c9}, {0x59d9, 0x598a},
{0x59e2, 0x5a1f}, {0x59e7, 0x59e6}, {0x5a1a, 0x5b32}, {0x5a1b, 0x5a2f}, {0x5a24, 0x599d}, {0x5a2c, 0x5af5},
{0x5a40, 0x5a3f}, {0x5a63, 0x59fb}, {0x5a86, 0x5ae9}, {0x5aaa, 0x5abc}, {0x5abf, 0x6127}, {0x5ad0, 0x5b32},
{0x5ae6, 0x59ee}, {0x5af0, 0x5ae9}, {0x5afb, 0x5afa}, {0x5b34, 0x218cd}, {0x5b43, 0x5b22}, {0x5b6f, 0x6394},
{0x5b73, 0x5b76}, {0x5b78, 0x5b66}, {0x5b7d, 0x5b7c}, {0x5b82, 0x5197}, {0x5b8d, 0x8089}, {0x5b90, 0x5b9c},
{0x5bc3, 0x51a4}, {0x5bc9, 0x96ba}, {0x5be2, 0x5bdd}, {0x5be6, 0x5b9f}, {0x5beb, 0x5199}, {0x5bec, 0x5bdb},
{0x5bf6, 0x5b9d}, {0x5c05, 0x524b}, {0x5c07, 0x5c06}, {0x5c08, 0x5c02}, {0x5c0d, 0x5bfe}, {0x5c12, 0x723e},
{0x5c13, 0x723e}, {0x5c19, 0x5c1a}, {0x5c1f, 0x5c20}, {0x5c23, 0x5c22}, {0x5c2a, 0x5c29}, {0x5c2b, 0x5c29},
{0x5c46, 0x5c4a}, {0x5c4f, 0x5c5b}, {0x5c53, 0x5c6d}, {0x5c5a, 0x6f0f}, {0x5c61, 0x5c62}, {0x5c6c, 0x5c5e},
{0x5cba, 0x5cad}, {0x5ce9, 0x5ce8}, {0x5cef, 0x5cf0}, {0x5cfd, 0x5ce1}, {0x5d10, 0x5d11}, {0x5d15, 0x5d16},
{0x5d17, 0x5ca1}, {0x5d18, 0x5d19}, {0x5d27, 0x5d69}, {0x5d2a, 0x5d12}, {0x5d2b, 0x5d1b}, {0x5d47, 0x5d46},
{0x5d53, 0x5d52}, {0x5d5c, 0x5d0e}, {0x5d73, 0x5d6f}, {0x5d84, 0x5d83}, {0x5d8b, 0x5cf6}, {0x5d8c, 0x5cf6},
{0x5da4, 0x5da2}, {0x5db9, 0x5cf6}, {0x5dbd, 0x5cb3}, {0x5dd6, 0x5dcc}, {0x5ddb, 0x5ddd}, {0x5de2, 0x5de3},
{0x5df5, 0x536e}, {0x5df9, 0x537a}, {0x5e00, 0x531d}, {0x5e08, 0x5e2b}, {0x5e0b, 0x7d19}, {0x5e12, 0x888b},
{0x5e2c, 0x88d9}, {0x5e2e, 0x5e6b}, {0x5e36, 0x5e2f}, {0x5e42, 0x51aa}, {0x5e47, 0x5e6b}, {0x5e59, 0x5e55},
{0x5e76, 0x5e77}, {0x5e99, 0x5edf}, {0x5ebb, 0x5eb6}, {0x5ebd, 0x5bd3}, {0x5ebf, 0x5edf}, {0x5ecf, 0x53a9},
{0x5ed0, 0x53a9}, {0x5eda, 0x53a8}, {0x5ee2, 0x5ec3}, {0x5ee3, 0x5e83}, {0x5eea, 0x5ee9}, {0x5ef3, 0x5e81},
{0x5efc, 0x8ffa}, {0x5efd, 0x5efb}, {0x5f00, 0x5e75}, {0x5f09, 0x5958}, {0x5f0c, 0x4e00}, {0x5f0d, 0x4e8c},
{0x5f0e, 0x4e09}, {0x5f12, 0x5f11}, {0x5f16, 0x6c10}, {0x5f2f, 0x5f4e}, {0x5f3a, 0x5f37}, {0x5f3b, 0x5f3c},
{0x5f48, 0x5f3e}, {0x5f4c, 0x5f25}, {0x5f51, 0x5f50}, {0x5f5a, 0x5f59}, {0x5f5c, 0x5f5d}, {0x5f65, 0x5f66},
{0x5f72, 0x87ad}, {0x5f8f, 0x965f}, {0x5f91, 0x5f84}, {0x5f9e, 0x5f93}, {0x5fa4, 0x5065}, {0x5fb5, 0x5fb4},
{0x5fb7, 0x5fb3}, {0x5fe2, 0x609f}, {0x5fe9, 0x6031}, {0x5ff0, 0x60b4}, {0x5ff7, 0x605f}, {0x5ffc, 0x6177},
{0x6046, 0x6052}, {0x604a, 0x5354}, {0x6053, 0x60bd}, {0x6056, 0x601d}, {0x6060, 0x602a}, {0x6061, 0x608b},
{0x6085, 0x60a6}, {0x608a, 0x54f2}, {0x6090, 0x60d5}, {0x6091, 0x6016}, {0x60a4, 0x6031}, {0x60a7, 0x4fd0},
{0x60e0, 0x6075}, {0x60e1, 0x60aa}, {0x60ee, 0x619a}, {0x60f1, 0x60a9}, {0x60fd, 0x60db}, {0x6119, 0x606a},
{0x611e, 0x61e6}, {0x6120, 0x614d}, {0x6121, 0x6181}, {0x6133, 0x61fc}, {0x613c, 0x614e}, {0x6158, 0x60e8},
{0x615a, 0x6159}, {0x6160, 0x50b2}, {0x6164, 0x6128}, {0x617d, 0x617c}, {0x617f, 0x6191}, {0x6187, 0x61a9},
{0x6197, 0x6196}, {0x6198, 0x6199}, {0x619c, 0x60f0}, {0x619e, 0x619d}, {0x61c9, 0x5fdc}, {0x61d4, 0x61cd},
{0x61f4, 0x61fa}, {0x61f7, 0x61d0}, {0x6200, 0x604b}, {0x621d, 0x8ca1}, {0x621e, 0x621b}, {0x6227, 0x5275},
{0x6230, 0x6226}, {0x6232, 0x622f}, {0x6236, 0x6238}, {0x623c, 0x536f}, {0x623e, 0x623b}, {0x6261, 0x62d5},
{0x6275, 0x65bc}, {0x6285, 0x62d8}, {0x628d, 0x62ef}, {0x629b, 0x62cb}, {0x62ac, 0x64e1}, {0x62c2, 0x6255},
{0x62cf, 0x62ff}, {0x62d4, 0x629c}, {0x62d6, 0x62d5}, {0x62dc, 0x62dd}, {0x630a, 0x5f04}, {0x630d, 0x6821},
{0x6318, 0x6bdf}, {0x6335, 0x5f04}, {0x633e, 0x631f}, {0x6353, 0x63f6}, {0x6359, 0x62fd}, {0x63b4, 0x6451},
{0x63bb, 0x6414}, {0x63d2, 0x633f}, {0x63ed, 0x63b2}, {0x6416, 0x63fa}, {0x6417, 0x64e3}, {0x641e, 0x6572},
{0x6439, 0x627c}, {0x6491, 0x6490}, {0x6498, 0x642d}, {0x64a1, 0x647b}, {0x64b9, 0x652a}, {0x64c7, 0x629e},
{0x64ca, 0x6483}, {0x64cf, 0x64ce}, {0x64d4, 0x62c5}, {0x64d5, 0x643a}, {0x64da, 0x62e0}, {0x64e5, 0x652c},
{0x64e7, 0x6319}, {0x64f4, 0x62e1}, {0x6502, 0x64c2}, {0x6505, 0x6522}, {0x651c, 0x643a}, {0x651d, 0x6442},
{0x6535, 0x6534}, {0x6536, 0x53ce}, {0x6548, 0x52b9}, {0x654d, 0x53d9}, {0x654e, 0x6559}, {0x6555, 0x52c5},
{0x6578, 0x6570}, {0x658c, 0x5f6c}, {0x65b5, 0x65b2}, {0x65b7, 0x65ad}, {0x65be, 0x65c6}, {0x65d9, 0x65db},
{0x65dc, 0x65c3}, {0x65e3, 0x65e2}, {0x65ea, 0x5354}, {0x65f6, 0x6642}, {0x65f9, 0x6642}, {0x65fe, 0x6625},
{0x661e, 0x663a}, {0x662c, 0x660f}, {0x6630, 0x662f}, {0x663b, 0x6602}, {0x663f, 0x66e0}, {0x6644, 0x6643},
{0x6648, 0x768e}, {0x6649, 0x664b}, {0x665a, 0x6669}, {0x665d, 0x663c}, {0x6663, 0x6662}, {0x6665, 0x7696},
{0x6673, 0x6670}, {0x668e, 0x6620}, {0x669c, 0x666e}, {0x66a4, 0x769e}, {0x66ad, 0x769e}, {0x66b1, 0x6635},
{0x66c6, 0x66a6}, {0x66c9, 0x6681}, {0x66d0, 0x661f}, {0x66ec, 0x6652}, {0x66f5, 0x66f3}, {0x66fa, 0x66f9},
{0x66fb, 0x6607}, {0x66fe, 0x66fd}, {0x6703, 0x4f1a}, {0x6735, 0x6736}, {0x674d, 0x6893}, {0x6764, 0x6803},
{0x6766, 0x6749}, {0x6780, 0x677e}, {0x678f, 0x6960}, {0x67a6, 0x6ae8}, {0x67a9, 0x677e}, {0x67b4, 0x67fa},
{0x67c8, 0x69c3}, {0x67d2, 0x6f06}, {0x67d7, 0x677e}, {0x67df, 0x6960}, {0x67e0, 0x696e}, {0x67e5, 0x67fb},
{0x67f9, 0x67ff}, {0x6800, 0x6894}, {0x6801, 0x67f3}, {0x6805, 0x67f5}, {0x6816, 0x68f2}, {0x6822, 0x67cf},
{0x6827, 0x67bb}, {0x6830, 0x7b4f}, {0x683e, 0x6b12}, {0x6849, 0x6848}, {0x684c, 0x5353}, {0x6852, 0x6851},
{0x685d, 0x67a1}, {0x6863, 0x6a94}, {0x6867, 0x6a9c}, {0x686e, 0x676f}, {0x687a, 0x67f3}, {0x687f, 0x6746},
{0x688d, 0x7681}, {0x6899, 0x69f5}, {0x689d, 0x6761}, {0x68a5, 0x677e}, {0x68b9, 0x6ab3}, {0x68bc, 0x6aae},
{0x68c3, 0x68a8}, {0x68ca, 0x68cb}, {0x68d5, 0x6936}, {0x68de, 0x68b1}, {0x68e7, 0x685f}, {0x6922, 0x69f6},
{0x6926, 0x68ec}, {0x699f, 0x6893}, {0x69a6, 0x5e79}, {0x69ae, 0x6804}, {0x69c0, 0x69c1}, {0x69c7, 0x69d9},
{0x69c8, 0x8028}, {0x69d6, 0x6a50}, {0x69d7, 0x6a4b}, {0x69de, 0x6af3}, {0x69e8, 0x6901}, {0x69ea, 0x6982},
{0x6a02, 0x697d}, {0x6a12, 0x6993}, {0x6a13, 0x697c}, {0x6a1e, 0x67a2}, {0x6a23, 0x69d8}, {0x6a2f, 0x6aa3},
{0x6a37, 0x53e2}, {0x6a45, 0x6a21}, {0x6a4a, 0x69b4}, {0x6a5c, 0x6a5b}, {0x6a62, 0x6955}, {0x6a6b, 0x6a2a},
{0x6a7a, 0x6a4c}, {0x6a8f, 0x6a38}, {0x6a98, 0x6a97}, {0x6a9d, 0x696b}, {0x6aa2, 0x691c}, {0x6aaa, 0x6adf},
{0x6ac1, 0x6993}, {0x6afb, 0x685c}, {0x6afd, 0x6a83}, {0x6b0a, 0x6a29}, {0x6b1d, 0x9b31}, {0x6b1e, 0x6afa},
{0x6b35, 0x6b3e}, {0x6b50, 0x6b27}, {0x6b56, 0x559c}, {0x6b58, 0x6b3b}, {0x6b61, 0x6b53}, {0x6b65, 0x6b69},
{0x6b72, 0x6b73}, {0x6b77, 0x6b74}, {0x6b78, 0x5e30}, {0x6b7a, 0x6b79}, {0x6b7e, 0x6b7f}, {0x6b81, 0x6b7f},
{0x6b98, 0x6b8b}, {0x6bb1, 0x6bb2}, {0x6bbc, 0x6bbb}, {0x6bc1, 0x6bc0}, {0x6bc6, 0x6bb4}, {0x6bcf, 0x6bce},
{0x6bd3, 0x80b2}, {0x6bd7, 0x6bd8}, {0x6be1, 0x6c08}, {0x6bee, 0x6bdf}, {0x6bf1, 0x97a0}, {0x6c0a, 0x6c08},
{0x6c23, 0x6c17}, {0x6c59, 0x6c5a}, {0x6c61, 0x6c5a}, {0x6c73, 0x6c74}, {0x6c89, 0x6c88}, {0x6c92, 0x6ca1},
{0x6caa, 0x6ffe}, {0x6cb2, 0x6cb1}, {0x6cfb, 0x7009}, {0x6d24, 0x6cc9}, {0x6d3f, 0x6c5a}, {0x6d43, 0x6d79},
{0x6d4a, 0x6fc1}, {0x6d81, 0x6ef2}, {0x6d89, 0x6e09}, {0x6d9b, 0x6fe4}, {0x6d9c, 0x7006}, {0x6dd2, 0x51c4},
{0x6dda, 0x6d99}, {0x6ddb, 0x6d59}, {0x6de8, 0x6d44}, {0x6df8, 0x6e05}, {0x6dfa, 0x6d45}, {0x6e0a, 0x6df5},
{0x6e15, 0x6df5}, {0x6e17, 0x6ef2}, {0x6e34, 0x6e07}, {0x6e4c, 0x9910}, {0x6e7b, 0x6df3}, {0x6e8c, 0x6f51},
{0x6eaa, 0x6e13}, {0x6eab, 0x6e29}, {0x6eb0, 0x769a}, {0x6ed9, 0x532f}, {0x6eda, 0x6efe}, {0x6eef, 0x6ede},
{0x6eff, 0x6e80}, {0x6f25, 0x7aaa}, {0x6f40, 0x6f68}, {0x6f45, 0x704c}, {0x6f5b, 0x6f5c}, {0x6f74, 0x7026},
{0x6f81, 0x6e0b}, {0x6f82, 0x6f84}, {0x6f91, 0x6e9c}, {0x6f98, 0x6f78}, {0x6f9f, 0x51dc}, {0x6fa3, 0x6d63},
{0x6fa4, 0x6ca2}, {0x6fd5, 0x6e7f}, {0x6fda, 0x6ece}, {0x6fdf, 0x6e08}, {0x6ff1, 0x6d5c}, {0x6ff6, 0x95ca},
{0x7027, 0x6edd}, {0x7028, 0x702c}, {0x7047, 0x6f68}, {0x704b, 0x6cd5}, {0x704e, 0x7069}, {0x7054, 0x7069},
{0x7063, 0x6e7e}, {0x706e, 0x5149}, {0x7076, 0x7ac8}, {0x707e, 0x707d}, {0x7097, 0x5149}, {0x70b2, 0x70b1},
{0x70d6, 0x707d}, {0x70ec, 0x71fc}, {0x70ed, 0x71b1}, {0x70f1, 0x70af}, {0x7102, 0x500f}, {0x710f, 0x4e9f},
{0x7114, 0x7130}, {0x714a, 0x6684}, {0x7155, 0x7199}, {0x7196, 0x7130}, {0x71a2, 0x70fd}, {0x71c4, 0x7130},
{0x71c8, 0x706f}, {0x71d2, 0x713c}, {0x71d3, 0x711a}, {0x71d7, 0x721b}, {0x71df, 0x55b6}, {0x71fb, 0x718f},
{0x7200, 0x8d6b}, {0x7210, 0x7089}, {0x7215, 0x71ee}, {0x7224, 0x721b}, {0x722d, 0x4e89}, {0x7232, 0x70ba},
{0x7234, 0x652b}, {0x723c, 0x4fce}, {0x724b, 0x7b8b}, {0x7255, 0x7a93}, {0x7257, 0x7256}, {0x7281, 0x7282},
{0x72a7, 0x72a0}, {0x72b2, 0x8c7a}, {0x72b4, 0x8c7b}, {0x72c0, 0x72b6}, {0x72e2, 0x8c89}, {0x72f9, 0x72ed},
{0x730f, 0x8c63}, {0x7312, 0x53ad}, {0x7328, 0x733f}, {0x732c, 0x875f}, {0x7333, 0x8c6d}, {0x7350, 0x9e9e},
{0x7368, 0x72ec}, {0x7375, 0x731f}, {0x7378, 0x7363}, {0x737b, 0x732e}, {0x7385, 0x5999}, {0x739f, 0x73c9},
{0x73b3, 0x7447}, {0x73ce, 0x73cd}, {0x73cf, 0x73a8}, {0x73d0, 0x743a}, {0x73e1, 0x7434}, {0x73ea, 0x572d},
{0x73ee, 0x4f69}, {0x73f1, 0x74d4}, {0x7439, 0x7434}, {0x7449, 0x73c9}, {0x7459, 0x78af}, {0x7464, 0x7476},
{0x746f, 0x7405}, {0x74a2, 0x7460}, {0x74bf, 0x7487}, {0x74c8, 0x7483}, {0x74c9, 0x74da}, {0x74cc, 0x7470},
{0x74e3, 0x5f01}, {0x74ef, 0x750c}, {0x7501, 0x74f6}, {0x7506, 0x74f7}, {0x7516, 0x7f4c}, {0x751b, 0x751c},
{0x751e, 0x5617}, {0x7522, 0x7523}, {0x753c, 0x753a}, {0x753d, 0x754e}, {0x7541, 0x7540}, {0x7544, 0x7559},
{0x7546, 0x755d}, {0x754d, 0x754c}, {0x7552, 0x755d}, {0x755e, 0x755d}, {0x7561, 0x5793}, {0x7567, 0x7565},
{0x756b, 0x753b}, {0x756d, 0x756c}, {0x756e, 0x755d}, {0x7571, 0x7559}, {0x7572, 0x756c}, {0x7574, 0x7587},
{0x7576, 0x5f53}, {0x757a, 0x7586}, {0x757b, 0x584d}, {0x7585, 0x7586}, {0x758a, 0x7573}, {0x75a9, 0x7601},
{0x75b4, 0x75fe}, {0x7618, 0x763b}, {0x7626, 0x75e9}, {0x7645, 0x7624}, {0x7655, 0x7670}, {0x7661, 0x75f4},
{0x767c, 0x767a}, {0x7682, 0x7681}, {0x7683, 0x8c8c}, {0x768b, 0x7690}, {0x76a1, 0x769e}, {0x76a5, 0x769e},
{0x76a8, 0x661f}, {0x76b7, 0x9f13}, {0x76b9, 0x76b8}, {0x76c3, 0x676f}, {0x76c7, 0x76cd}, {0x76cb, 0x9262},
{0x76cc, 0x7897}, {0x76d6, 0x84cb}, {0x76dc, 0x76d7}, {0x76e1, 0x5c3d}, {0x771e, 0x771f}, {0x7726, 0x7725},
{0x773e, 0x8846}, {0x7758, 0x778f}, {0x7764, 0x7765}, {0x777f, 0x53e1}, {0x77a9, 0x77da}, {0x77aa, 0x77a0},
{0x77d9, 0x77b0}, {0x77e4, 0x77e7}, {0x77e6, 0x4faf}, {0x77f4, 0x7887}, {0x7806, 0x739e}, {0x783a, 0x792a},
{0x783f, 0x7926}, {0x784f, 0x7814}, {0x7851, 0x7830}, {0x788e, 0x7815}, {0x78aa, 0x7827}, {0x78b5, 0x78a9},
{0x78bb, 0x78ba}, {0x78c7, 0x7812}, {0x78ce, 0x8c3f}, {0x7900, 0x6f97}, {0x7955, 0x79d8}, {0x7958, 0x7b97},
{0x7962, 0x79b0}, {0x7977, 0x79b1}, {0x797f, 0x7984}, {0x7980, 0x7a1f}, {0x799d, 0x7a37}, {0x79a9, 0x7940},
{0x79aa, 0x7985}, {0x79ae, 0x793c}, {0x79c6, 0x7a08}, {0x79c7, 0x57f7}, {0x79ca, 0x5e74}, {0x79cc, 0x79cb},
{0x79d4, 0x7cb3}, {0x7a05, 0x7a0e}, {0x7a09, 0x7cb3}, {0x7a2c, 0x7cef}, {0x7a31, 0x79f0}, {0x7a38, 0x84c4},
{0x7a3a, 0x7a1a}, {0x7a3b, 0x7a32}, {0x7a3e, 0x7a3f}, {0x7a45, 0x7ce0}, {0x7a49, 0x7a1a}, {0x7a4c, 0x8607},
{0x7a50, 0x79cb}, {0x7a57, 0x7a42}, {0x7a64, 0x7cef}, {0x7a69, 0x7a4f}, {0x7a70, 0x7a63}, {0x7a82, 0x7262},
{0x7a91, 0x7aaf}, {0x7a97, 0x7a93}, {0x7ab0, 0x7aaf}, {0x7abb, 0x7a93}, {0x7ac3, 0x7ac8}, {0x7ac6, 0x7aae},
{0x7aca, 0x7a83}, {0x7ad2, 0x5947}, {0x7ada, 0x4f47}, {0x7add, 0x4e26}, {0x7ae2, 0x4fdf}, {0x7af8, 0x7af6},
{0x7b04, 0x7b53}, {0x7b0b, 0x7b4d}, {0x7b14, 0x7b46}, {0x7b36, 0x77e2}, {0x7b3b, 0x7b47}, {0x7b5d, 0x7b8f},
{0x7b69, 0x7b52}, {0x7b71, 0x7be0}, {0x7b79, 0x7c4c}, {0x7b7a, 0x7b50}, {0x7b7f, 0x7be0}, {0x7b86, 0x7be6},
{0x7b8e, 0x7bea}, {0x7b93, 0x7c0f}, {0x7b9f, 0x7b98}, {0x7baa, 0x7c1e}, {0x7bcf, 0x5d4c}, {0x7bd7, 0x7c70},
{0x7bed, 0x7c60}, {0x7c11, 0x84d1}, {0x7c12, 0x7be1}, {0x7c14, 0x84d1}, {0x7c31, 0x65d7}, {0x7c40, 0x7c52},
{0x7c4f, 0x65d7}, {0x7c56, 0x7c64}, {0x7c62, 0x5333}, {0x7c6d, 0x7be9}, {0x7c74, 0x7cf4}, {0x7c75, 0x8e6f},
{0x7c83, 0x79d5}, {0x7c9c, 0x7cf6}, {0x7cae, 0x7ce7}, {0x7cb9, 0x7c8b}, {0x7cc7, 0x9931}, {0x7cc9, 0x7cbd},
{0x7cdd, 0x7cc2}, {0x7ce2, 0x6a21}, {0x7ce6, 0x994e}, {0x7cf5, 0x7cf1}, {0x7d23, 0x7db7}, {0x7d25, 0x7d2e},
{0x7d4b, 0x7e8a}, {0x7d4d, 0x7d1d}, {0x7d4f, 0x7d32}, {0x7d55, 0x7d76}, {0x7d56, 0x7e8a}, {0x7d5a, 0x7dea},
{0x7d5d, 0x88b4}, {0x7d6b, 0x7d2f}, {0x7d72, 0x7cf8}, {0x7d89, 0x7e61}, {0x7d8b, 0x7d18}, {0x7d93, 0x7d4c},
{0x7da0, 0x7dd1}, {0x7dab, 0x7dda}, {0x7dcd, 0x7de1}, {0x7dd5, 0x7e83}, {0x7dd6, 0x7dd2}, {0x7ddc, 0x7dbf},
{0x7de3, 0x7e01}, {0x7de4, 0x7d32}, {0x7de5, 0x8913}, {0x7dfc, 0x7e15}, {0x7e23, 0x770c}, {0x7e27, 0x7d5b},
{0x7e31, 0x7e26}, {0x7e3d, 0x7dcf}, {0x7e4b, 0x7e6b}, {0x7e4d, 0x7e61}, {0x7e66, 0x7e48}, {0x7e69, 0x7e04},
{0x7e6a, 0x7d75}, {0x7e7c, 0x7d99}, {0x7e7f, 0x8964}, {0x7e89, 0x7e98}, {0x7e8c, 0x7d9a}, {0x7e8d, 0x7d2f},
{0x7e92, 0x7e8f}, {0x7e96, 0x7e4a}, {0x7f3a, 0x6b20}, {0x7f3d, 0x9262}, {0x7f47, 0x6a3d}, {0x7f4b, 0x7515},
{0x7f4e, 0x58dc}, {0x7f50, 0x7f36}, {0x7f78, 0x7f70}, {0x7f83, 0x51aa}, {0x7f90, 0x7f91}, {0x7f97, 0x7f8c},
{0x7fa3, 0x7fa4}, {0x7fae, 0x7fb9}, {0x7fe6, 0x526a}, {0x7ff1, 0x7ffa}, {0x8008, 0x8007}, {0x800a, 0x800b},
{0x8011, 0x7aef}, {0x803b, 0x6065}, {0x8043, 0x803c}, {0x805d, 0x9998}, {0x805f, 0x5a7f}, {0x8068, 0x806f},
{0x8070, 0x8061}, {0x8072, 0x58f0}, {0x807d, 0x8074}, {0x8085, 0x7c9b}, {0x808a, 0x81c6}, {0x808e, 0x80af},
{0x80a4, 0x819a}, {0x80a7, 0x80da}, {0x80dc, 0x8165}, {0x80df, 0x62c7}, {0x80f7, 0x80f8}, {0x80fc, 0x8141},
{0x8103, 0x8106}, {0x810d, 0x81be}, {0x8117, 0x543b}, {0x8123, 0x5507}, {0x812b, 0x8131}, {0x8166, 0x8133},
{0x816d, 0x9f76}, {0x816e, 0x984b}, {0x8173, 0x811a}, {0x8193, 0x8178}, {0x81bd, 0x80c6}, {0x81c3, 0x7670},
{0x81c8, 0x81d8}, {0x81d9, 0x80ed}, {0x81df, 0x81d3}, {0x81ef, 0x7690}, {0x81fa, 0x53f0}, {0x8204, 0x8203},
{0x8207, 0x4e0e}, {0x820a, 0x65e7}, {0x820d, 0x820e}, {0x8213, 0x8210}, {0x8216, 0x8217}, {0x8229, 0x8239},
{0x822e, 0x826b}, {0x8263, 0x826a}, {0x8277, 0x8276}, {0x8294, 0x5349}, {0x82c5, 0x5208}, {0x82e2, 0x82e1},
{0x82fd, 0x83f0}, {0x8330, 0x8438}, {0x8346, 0x834a}, {0x8354, 0x8318}, {0x8355, 0x7b4b}, {0x838a, 0x8358},
{0x8393, 0x82fa}, {0x8395, 0x8347}, {0x8396, 0x830e}, {0x83b1, 0x840a}, {0x83ed, 0x82d4}, {0x83f7, 0x5e1a},
{0x8405, 0x6625}, {0x8415, 0x85ba}, {0x8417, 0x7b56}, {0x842c, 0x4e07}, {0x8445, 0x83f9}, {0x844a, 0x83f4},
{0x8458, 0x83d1}, {0x8462, 0x84cb}, {0x8482, 0x8515}, {0x848b, 0x8523}, {0x849e, 0x8385}, {0x84ad, 0x82bb},
{0x84da, 0x84e8}, {0x84f1, 0x840d}, {0x84f3, 0x83eb}, {0x8525, 0x8471}, {0x854b, 0x854a}, {0x855a, 0x843c},
{0x856f, 0x85a9}, {0x857f, 0x8431}, {0x8593, 0x8518}, {0x8597, 0x5712}, {0x85ae, 0x85ea}, {0x85b0, 0x85ab},
{0x85b2, 0x860b}, {0x85b6, 0x57cb}, {0x85c1, 0x7a3f}, {0x85c2, 0x53e2}, {0x85cf, 0x8535}, {0x85dd, 0x82b8},
{0x85e1, 0x837b}, {0x85e5, 0x85ac}, {0x85f3, 0x7a3f}, {0x85fc, 0x8431}, {0x8602, 0x854a}, {0x8606, 0x82a6},
{0x8610, 0x8431}, {0x8612, 0xfa20}, {0x8613, 0x8607}, {0x8616, 0x6af1}, {0x8617, 0x6a97}, {0x861c, 0x83ca},
{0x862f, 0x8569}, {0x8640, 0x9f4f}, {0x8641, 0x21582}, {0x8655, 0x51e6}, {0x865b, 0x865a}, {0x865f, 0x53f7},
{0x866c, 0x866f}, {0x8671, 0x8768}, {0x8675, 0x86c7}, {0x8689, 0x868a}, {0x8698, 0x86d4}, {0x86ab, 0x9b91},
{0x86ce, 0x8823}, {0x86d5, 0x86d4}, {0x86fd, 0x8c9d}, {0x8704, 0x8703}, {0x8719, 0x86a3}, {0x872f, 0x868c},
{0x8739, 0x868b}, {0x8749, 0x87ec}, {0x874b, 0x881f}, {0x8750, 0x7441}, {0x8761, 0x8815}, {0x8771, 0x867b},
{0x877c, 0x87bb}, {0x877f, 0x8805}, {0x8782, 0x870b}, {0x8798, 0x87fb}, {0x8799, 0x8839}, {0x87a2, 0x86cd},
{0x87be, 0x8693}, {0x87c1, 0x868a}, {0x87c6, 0x87c7}, {0x87ca, 0x8765}, {0x87f2, 0x866b}, {0x8807, 0x8823},
{0x880f, 0x87f9}, {0x8827, 0x8839}, {0x882d, 0x8702}, {0x8836, 0x8695}, {0x883b, 0x86ee}, {0x8842, 0x8844},
{0x8856, 0x5df7}, {0x8858, 0x929c}, {0x885e, 0x885b}, {0x889e, 0x886e}, {0x889f, 0x88a0}, {0x88ae, 0x8897},
{0x88b5, 0x887d}, {0x88cc, 0x88b7}, {0x88dd, 0x88c5}, {0x88e0, 0x88d9}, {0x88e1, 0x88cf}, {0x88e9, 0x890c},
{0x88f5, 0x88f4}, {0x88f6, 0x88f4}, {0x890f, 0x890e}, {0x891d, 0x894c}, {0x892d, 0x88ca}, {0x8932, 0x88b4},
{0x8943, 0x8912}, {0x8989, 0x7f87}, {0x898a, 0x7f88}, {0x8994, 0x8993}, {0x89a9, 0x7779}, {0x89b7, 0x89b0},
{0x89ba, 0x899a}, {0x89bd, 0x89a7}, {0x89c0, 0x89b3}, {0x89d3, 0x89e9}, {0x89d4, 0x7b4b}, {0x89d7, 0x89f6},
{0x89e7, 0x89e3}, {0x89f5, 0x89e5}, {0x89f8, 0x89e6}, {0x89fd, 0x89ff}, {0x8a21, 0x541f}, {0x8a3d, 0x8a6c},
{0x8a3f, 0x8a3e}, {0x8aaa, 0x8aac}, {0x8ad0, 0x6106}, {0x8b0c, 0x6b4c}, {0x8b20, 0x8b21}, {0x8b2d, 0x8b7e},
{0x8b38, 0x8b37}, {0x8b3c, 0x5611}, {0x8b49, 0x8a3c}, {0x8b4b, 0x8b95}, {0x8b4c, 0x8a1b}, {0x8b5b, 0x8b56},
{0x8b6f, 0x8a33}, {0x8b71, 0x5584}, {0x8b7d, 0x8a89}, {0x8b80, 0x8aad}, {0x8b81, 0x8b2b}, {0x8b87, 0x8ac2},
{0x8b8a, 0x5909}, {0x8b8d, 0x8ae4}, {0x8b8e, 0x8b90}, {0x8b93, 0x8b72}, {0x8b9a, 0x8b83}, {0x8c4e, 0x7aea},
{0x8c50, 0x8c4a}, {0x8c58, 0x8c5a}, {0x8c6b, 0x4e88}, {0x8c7c, 0x8c94}, {0x8c7e, 0x72c9}, {0x8c83, 0x8c8a},
{0x8c8d, 0x72f8}, {0x8c8e, 0x730a}, {0x8c93, 0x732b}, {0x8c98, 0x734f}, {0x8c9f, 0x54e1}, {0x8cad, 0x8cea},
{0x8cb3, 0x5f10}, {0x8ccd, 0x8d13}, {0x8cce, 0x8ce4}, {0x8cd2, 0x8cd6}, {0x8ce3, 0x58f2}, {0x8cee, 0x8d10},
{0x8cf4, 0x983c}, {0x8d0a, 0x8cdb}, {0x8d12, 0x8ce2}, {0x8d17, 0x8d0b}, {0x8d71, 0x8d70}, {0x8d82, 0x8d81},
{0x8da6, 0x8d91}, {0x8dae, 0x8e81}, {0x8e08, 0x758e}, {0x8e0c, 0x8e8a}, {0x8e10, 0x8df5}, {0x8e34, 0x8e0a},
{0x8e4f, 0x8e44}, {0x8e5e, 0x8dec}, {0x8e60, 0x8dd6}, {0x8e64, 0x8e2a}, {0x8e6e, 0x8e9a}, {0x8e75, 0x8e74},
{0x8e77, 0x8e76}, {0x8e78, 0x8e99}, {0x8e79, 0x8e4b}, {0x8e83, 0x8e84}, {0x8eaa, 0x8e99}, {0x8ead, 0x803d},
{0x8eaf, 0x8ec0}, {0x8eb3, 0x8eac}, {0x8eb6, 0x88f8}, {0x8ec4, 0x8077}, {0x8ee3, 0x8f5f}, {0x8ef0, 0x8f29},
{0x8ef6, 0x8edb}, {0x8f00, 0x8f5c}, {0x8f0c, 0x8f1b}, {0x8f15, 0x8efd}, {0x8f19, 0x8f12}, {0x8f2d, 0x8edf},
{0x8f3a, 0x8f1c}, {0x8f49, 0x8ee2}, {0x8f5d, 0x8f3f}, {0x8fa0, 0x7f6a}, {0x8fa8, 0x5f01}, {0x8fac, 0x6591},
{0x8fad, 0x8f9e}, {0x8faf, 0x5f01}, {0x8fc6, 0x8fe4}, {0x8fca, 0x531d}, {0x8fd9, 0x9019}, {0x8fe9, 0x9087},
{0x8ff4, 0x5efb}, {0x8ff8, 0x902c}, {0x9008, 0x8fe5}, {0x900e, 0x9052}, {0x9037, 0x9016}, {0x9059, 0x9065},
{0x905e, 0x9013}, {0x9072, 0x9045}, {0x908a, 0x8fba}, {0x90a8, 0x6751}, {0x90b1, 0x4e18}, {0x90c4, 0x90e4},
{0x90c9, 0x90a2}, {0x90de, 0x90ce}, {0x90f6, 0x90e8}, {0x9115, 0x90f7}, {0x9137, 0x9146}, {0x9139, 0x90f0},
{0x913d, 0x5edb}, {0x9159, 0x659f}, {0x9167, 0x916c}, {0x9186, 0x76de}, {0x9189, 0x9154}, {0x918b, 0x9162},
{0x918e, 0x9e79}, {0x9195, 0x9187}, {0x9197, 0x91b1}, {0x91a4, 0x91ac}, {0x91ab, 0x533b}, {0x91bb, 0x916c},
{0x91bc, 0x8b8c}, {0x91c0, 0x91b8}, {0x91c4, 0x91be}, {0x91cb, 0x91c8}, {0x91d6, 0x5200}, {0x91e1, 0x91dc},
{0x9206, 0x925b}, {0x920e, 0x9264}, {0x9229, 0x946a}, {0x922c, 0x9438}, {0x9247, 0x9248}, {0x92b2, 0x91ec},
{0x92b3, 0x92ed}, {0x92bf, 0x93de}, {0x9304, 0x9332}, {0x9322, 0x92ad}, {0x934a, 0x932c}, {0x936b, 0x936c},
{0x9373, 0x9451}, {0x93ad, 0x93ae}, {0x93b8, 0x942b}, {0x93e0, 0x92d2}, {0x93e5, 0x92b9}, {0x93e9, 0x93e8},
{0x93fd, 0x92b9}, {0x9435, 0x9244}, {0x9441, 0x9350}, {0x9444, 0x92f3}, {0x9452, 0x9451}, {0x945a, 0x947d},
{0x945b, 0x9271}, {0x9578, 0x9577}, {0x9579, 0x4e45}, {0x9586, 0x95bb}, {0x9587, 0x9589}, {0x9592, 0x9593},
{0x9599, 0x9b27}, {0x95a0, 0x958f}, {0x95b1, 0x95b2}, {0x95b4, 0x95c3}, {0x95da, 0x7aba}, {0x95dc, 0x95a2},
{0x962b, 0x574f}, {0x962c, 0x5751}, {0x962f, 0x5740}, {0x9631, 0x7a7d}, {0x9633, 0x967d}, {0x9634, 0x9670},
{0x9638, 0x9628}, {0x9641, 0x9624}, {0x9666, 0x5cf6}, {0x966d, 0x5d0e}, {0x9677, 0x9665}, {0x9681, 0x5830},
{0x9682, 0x9670}, {0x969d, 0x5cf6}, {0x96a8, 0x968f}, {0x96aa, 0x967a}, {0x96af, 0x5cf6}, {0x96b1, 0x96a0},
{0x96b2, 0x9a2d}, {0x96b8, 0x96b7}, {0x96bd, 0x96cb}, {0x96d9, 0x53cc}, {0x96da, 0x9e1b}, {0x96dc, 0x96d1},
{0x96dd, 0x96cd}, {0x96e7, 0x96c6}, {0x9721, 0x9722}, {0x9724, 0x29178}, {0x9736, 0x6ec2}, {0x973b, 0x974a},
{0x9741, 0x96f7}, {0x9748, 0x970a}, {0x974d, 0x9db4}, {0x974e, 0x9db4}, {0x974f, 0x9db4}, {0x9751, 0x9752},
{0x975c, 0x9759}, {0x9763, 0x9762}, {0x9764, 0x76b0}, {0x976d, 0x9771}, {0x978c, 0x978d}, {0x979f, 0x97b9},
{0x97b2, 0x97dd}, {0x97b5, 0x978b}, {0x97b8, 0x97e0}, {0x97be, 0x9774}, {0x97c8, 0x896a}, {0x97cc, 0x9771},
{0x97d2, 0x9798}, {0x97e4, 0x896a}, {0x97ee, 0x97ed}, {0x97f2, 0x9f4f}, {0x97f5, 0x97fb}, {0x981a, 0x9838},
{0x9825, 0x9824}, {0x982c, 0x9830}, {0x9833, 0x8d6c}, {0x9834, 0x7a4e}, {0x9839, 0x983d}, {0x983a, 0x983d},
{0x983e, 0x9aed}, {0x984f, 0x9854}, {0x9856, 0x56df}, {0x985b, 0x985a}, {0x986f, 0x9855}, {0x98b7, 0x98c6},
{0x98c3, 0x98c4}, {0x98c7, 0x98c6}, {0x98c8, 0x98c6}, {0x98cc, 0x98a8}, {0x98e1, 0x98e7}, {0x98e6, 0x9958},
{0x98ee, 0x98f2}, {0x98f1, 0x98e7}, {0x9918, 0x4f59}, {0x991a, 0x80b4}, {0x9920, 0x9905}, {0x993b, 0x7cd5},
{0x993e, 0x297b7}, {0x994d, 0x81b3}, {0x9959, 0x9934}, {0x995f, 0x9909}, {0x99af, 0x99fb}, {0x99b1, 0x99c4},
{0x99bd, 0x7e36}, {0x99bf, 0x9a62}, {0x99de, 0x99dd}, {0x99e2, 0x9a08}, {0x9a28, 0x9a52}, {0x9a2e, 0x9a51},
{0x9a37, 0x9a12}, {0x9a45, 0x99c6}, {0x9a57, 0x9a13}, {0x9a58, 0x9a3e}, {0x9a5b, 0x99c5}, {0x9ac8, 0x8180},
{0x9ad3, 0x9ac4}, {0x9ad4, 0x4f53}, {0x9ad5, 0x81cf}, {0x9ad9, 0x9ad8}, {0x9ae0, 0x9ae1}, {0x9ae8, 0x9ae1},
{0x9ae9, 0x9b22}, {0x9aee, 0x9aea}, {0x9aef, 0x9ae5}, {0x9b00, 0x5243}, {0x9b02, 0x9b22}, {0x9b2a, 0x95d8},
{0x9b3d, 0x9b45}, {0x9b57, 0x919c}, {0x9b66, 0x9bca}, {0x9b72, 0x9c78}, {0x9b86, 0x9c6d}, {0x9bcb, 0x9bca},
{0x9bd7, 0x9c76}, {0x9bd8, 0x9bbe}, {0x9bf5, 0x9c3a}, {0x9c0d, 0x9c0c}, {0x9c1b, 0x9c2e}, {0x9c32, 0x9f07},
{0x9c49, 0x9f08}, {0x9c60, 0x81be}, {0x9c70, 0x9c6a}, {0x9c77, 0x9c10}, {0x9ceb, 0x96c1}, {0x9cec, 0x9ce7},
{0x9d02, 0x9d03}, {0x9d08, 0x96c1}, {0x9d0e, 0x9dd7}, {0x9d2a, 0x9d25}, {0x9d2c, 0x9daf}, {0x9d44, 0x9d1f},
{0x9d5e, 0x9d5d}, {0x9d70, 0x96d5}, {0x9d76, 0x9d09}, {0x9d7e, 0x9da4}, {0x9daa, 0x9d59}, {0x9db5, 0x96db},
{0x9dbf, 0x9dc0}, {0x9dc4, 0x9d8f}, {0x9dcf, 0x9dc6}, {0x9e0e, 0x9daf}, {0x9e1c, 0x9d1d}, {0x9e78, 0x9e7c},
{0x9e7b, 0x9e7c}, {0x9e7d, 0x5869}, {0x9e81, 0x9ea4}, {0x9e95, 0x9e87}, {0x9ea5, 0x9ea6}, {0x9eaf, 0x9eb4},
{0x9eb5, 0x9eba}, {0x9eb9, 0x9eb4}, {0x9ebd, 0x9ebc}, {0x9ec3, 0x9ec4}, {0x9ecf, 0x7c98}, {0x9ed1, 0x9ed2},
{0x9ed8, 0x9ed9}, {0x9ede, 0x70b9}, {0x9ee8, 0x515a}, {0x9f03, 0x86d9}, {0x9f21, 0x9f20}, {0x9f26, 0x8c82},
{0x9f39, 0x9f34}, {0x9f45, 0x55c5}, {0x9f4a, 0x6589}, {0x9f4b, 0x658e}, {0x9f52, 0x6b6f}, {0x9f53, 0x9f54},
{0x9f61, 0x9f62}, {0x9f8d, 0x7adc}, {0x9f9c, 0x4e80}, {0x9f9d, 0x79cb}, {0x9fa1, 0x5439}, {0x9fa2, 0x548c},
{0x9fa5, 0x25e24}, {0xf909, 0x5951}, {0xf91d, 0x6b04}, {0xf91f, 0x862d}, {0xf928, 0x5eca}, {0xf929, 0x6717},
{0xf936, 0x865c}, {0xf95f, 0x5be7}, {0xf970, 0x6bba}, {0xf983, 0x65c5}, {0xf999, 0x84ee}, {0xf99a, 0x9023},
{0xf9a2, 0x5ec9}, {0xf9c3, 0x907c}, {0xf9d0, 0x985e}, {0xf9dc, 0x9686}, {0xfa03, 0x7cd6}, {0xfa10, 0x585a},
{0xfa11, 0x5d0e}, {0xfa12, 0x6674}, {0xfa14, 0x6b05}, {0xfa16, 0x732a}, {0xfa17, 0x76ca}, {0xfa19, 0x795e},
{0xfa1a, 0x7965}, {0xfa1b, 0x798f}, {0xfa1c, 0x9756}, {0xfa1d, 0x7cbe}, {0xfa1e, 0x7fbd}, {0xfa1f, 0x81d8},
{0xfa22, 0x8af8}, {0xfa26, 0x90fd}, {0xfa2a, 0x98ef}, {0xfa2b, 0x98fc}, {0xfa2c, 0x9928}, {0xfa2d, 0x9db4},
{0xfa30, 0x4fae}, {0xfa31, 0x50e7}, {0xfa32, 0x514d}, {0xfa33, 0x52c9}, {0xfa34, 0x52e4}, {0xfa35, 0x5351},
{0xfa36, 0x559d}, {0xfa37, 0x5606}, {0xfa38, 0x5668}, {0xfa39, 0x5840}, {0xfa3a, 0x58a8}, {0xfa3b, 0x5c64},
{0xfa3d, 0x6094}, {0xfa3f, 0x618e}, {0xfa40, 0x61f2}, {0xfa41, 0x654f}, {0xfa43, 0x6691}, {0xfa44, 0x6885},
{0xfa45, 0x6d77}, {0xfa46, 0x6e1a}, {0xfa47, 0x6f22}, {0xfa48, 0x716e}, {0xfa4a, 0x7422}, {0xfa4b, 0x7891},
{0xfa4c, 0x793e}, {0xfa4d, 0x7949}, {0xfa4e, 0x7948}, {0xfa4f, 0x7950}, {0xfa50, 0x7956}, {0xfa51, 0x795d},
{0xfa52, 0x798d}, {0xfa53, 0x798e}, {0xfa54, 0x7a40}, {0xfa55, 0x7a81}, {0xfa56, 0x7bc0}, {0xfa57, 0x7df4},
{0xfa58, 0x7e09}, {0xfa59, 0x7e41}, {0xfa5a, 0x7f72}, {0xfa5b, 0x8005}, {0xfa5c, 0x81ed}, {0xfa5f, 0x8457},
{0xfa60, 0x8910}, {0xfa61, 0x8996}, {0xfa62, 0x8b01}, {0xfa63, 0x8b39}, {0xfa64, 0x8cd3}, {0xfa65, 0x8d08},
{0xfa67, 0x9038}, {0xfa68, 0x96e3}, {0xfa69, 0x97ff}, {0xfa6a, 0x983b}, {0xfa71, 0x6cc1}, {0xfa74, 0x5145},
{0xfa75, 0x5180}, {0xfa7d, 0x58b3}, {0xfa7f, 0x5954}, {0xfa8c, 0x6234}, {0xfa97, 0x6d41}, {0xfa9b, 0x701e},
{0xfab4, 0x83ef}, {0x2000b, 0x4e08}, {0x200a4, 0x5806}, {0x200b7, 0x5e78}, {0x20158, 0x4ead}, {0x20159, 0x4eae},
{0x201a2, 0x4eba}, {0x20209, 0x4f83}, {0x20213, 0x5006}, {0x20232, 0x4f78}, {0x20255, 0x4f7f}, {0x202b1, 0x5019},
{0x202b3, 0x4fbf}, {0x202e3, 0x501a}, {0x2032b, 0x8e55}, {0x20358, 0x5098}, {0x20371, 0x96c6}, {0x2037b, 0x5114},
{0x20427, 0x511a}, {0x2044a, 0x513c}, {0x205b1, 0x51b1}, {0x20621, 0x51dc}, {0x206c4, 0x5208}, {0x206e0, 0x525c},
{0x206ec, 0x5211}, {0x206f0, 0x5225}, {0x20702, 0x5220}, {0x20713, 0x5254}, {0x2071c, 0x522e}, {0x2073d, 0x5239},
{0x20765, 0x5232}, {0x207b0, 0x527f}, {0x207d0, 0x5275}, {0x207ed, 0x5247}, {0x20807, 0x65b2}, {0x208de, 0x8ecd},
{0x208e5, 0x51a2}, {0x20914, 0x65e8}, {0x20983, 0x5345}, {0x20991, 0x53d4}, {0x209de, 0x5146}, {0x209ea, 0x5363},
{0x20afb, 0x755a}, {0x20b56, 0x6562}, {0x20bb7, 0x5409}, {0x20bcf, 0x5403}, {0x20d45, 0x5629}, {0x20dd4, 0x55aa},
{0x20e49, 0x5617}, {0x2122d, 0x7344}, {0x2123d, 0x571f}, {0x21274, 0x58da}, {0x2127b, 0x57e3}, {0x212d7, 0x2146d},
{0x212e4, 0x58d7}, {0x212f3, 0x57a0}, {0x2131b, 0x91ce}, {0x21369, 0x577c}, {0x213f2, 0x588d}, {0x2141b, 0x5879},
{0x216b4, 0x5b89}, {0x21742, 0x599d}, {0x21769, 0x5996}, {0x2181c, 0x5aeb}, {0x21883, 0x5afa}, {0x21894, 0x218cd},
{0x21998, 0x5b7a}, {0x219c3, 0x5b87}, {0x21a25, 0x5bc7}, {0x21bea, 0x5c29}, {0x21c41, 0x5c4e}, {0x21d78, 0x5da7},
{0x21d92, 0x5caa}, {0x21d9c, 0x5cb8}, {0x21e33, 0x5da2}, {0x22034, 0x5384}, {0x2203e, 0x537a}, {0x2206b, 0x5e0c},
{0x220d6, 0x5e6c}, {0x220f6, 0x7e9b}, {0x221b2, 0x5e7c}, {0x22218, 0x9e7f}, {0x2231e, 0x5efb}, {0x22392, 0x77f0},
{0x22400, 0x5f3c}, {0x22488, 0x5f62}, {0x224c8, 0x5f87}, {0x22536, 0x5f97}, {0x225b9, 0x6084}, {0x225cc, 0x6015},
{0x225d8, 0x5ffd}, {0x22643, 0x6068}, {0x22644, 0x606d}, {0x2264a, 0x6063}, {0x226c5, 0x6019}, {0x2272f, 0x8b28},
{0x2274a, 0x6182}, {0x2279b, 0x60f8}, {0x227e1, 0x618a}, {0x22835, 0x60dd}, {0x22846, 0x61c7}, {0x2285b, 0x61e3},
{0x2287b, 0x61ab}, {0x22894, 0x6162}, {0x2289a, 0x6112}, {0x228a3, 0x61c8}, {0x229a7, 0x77db}, {0x229e2, 0x621f},
{0x229f5, 0x622a}, {0x22ab8, 0x6365}, {0x22ae6, 0x62cd}, {0x22afe, 0x6307}, {0x22b46, 0x62bd}, {0x22b4f, 0x64e3},
{0x22b50, 0x6488}, {0x22b94, 0x6412}, {0x22ba6, 0x64bf}, {0x22bf2, 0x8155}, {0x22c24, 0x638e}, {0x22c67, 0x6279},
{0x22d96, 0x63c4}, {0x22e42, 0x643a}, {0x22e6c, 0x64c1}, {0x22fe5, 0x64ad}, {0x22fea, 0x6575}, {0x22feb, 0x6574},
{0x230fc, 0x65c3}, {0x23166, 0x6603}, {0x231bb, 0x6634}, {0x231c3, 0x8202}, {0x231c4, 0x9f0e}, {0x231f5, 0x6676},
{0x23358, 0x66f9}, {0x2335f, 0x80af}, {0x23392, 0x584d}, {0x233d0, 0x6753}, {0x233d3, 0x673d}, {0x233d5, 0x676e},
{0x233dd, 0x6795}, {0x23465, 0x6832}, {0x234e4, 0x6a48}, {0x23581, 0x69bc}, {0x235f3, 0x6a39}, {0x2363a, 0x6a4b},
{0x2371c, 0x6803}, {0x2383d, 0x6b1b}, {0x23878, 0x9b31}, {0x239d1, 0x51f6}, {0x239eb, 0x6b82}, {0x23a98, 0x7c0b},
{0x23b08, 0x6bd8}, {0x23c66, 0x6c1b}, {0x23cd8, 0x6cc4}, {0x23cfe, 0x6cf0}, {0x23d0e, 0x7f95}, {0x23d20, 0x6d3b},
{0x23d40, 0x6d85}, {0x23d7a, 0x6cb1}, {0x23d7d, 0x6c83}, {0x23e01, 0x6d25}, {0x23e08, 0x6eb2}, {0x23e62, 0x6db5},
{0x23f4a, 0x6fec}, {0x24096, 0x705d}, {0x240a2, 0x7051}, {0x24103, 0x95ca}, {0x2414d, 0x6f15}, {0x24261, 0x5155},
{0x243bc, 0x719f}, {0x2442b, 0x7968}, {0x24553, 0x723a}, {0x24660, 0x89e9}, {0x248d0, 0x736e}, {0x2493b, 0x73ed},
{0x2493d, 0x73cd}, {0x24a0f, 0x7463}, {0x24b26, 0x74e6}, {0x24b56, 0x74fa}, {0x24b6f, 0x697e}, {0x24bf3, 0x7526},
{0x24c16, 0x755d}, {0x24c1d, 0x754e}, {0x24d21, 0x7590}, {0x24e6a, 0x75c0}, {0x24e8b, 0x75b9}, {0x25044, 0x76c2},
{0x2504a, 0x9262}, {0x250e9, 0x826e}, {0x25122, 0x76f8}, {0x2515a, 0x76f1}, {0x251a2, 0x773c}, {0x251a9, 0x7768},
{0x251e5, 0x77bc}, {0x2521e, 0x779e}, {0x2521f, 0x778d}, {0x252ba, 0x77af}, {0x253a6, 0x4faf}, {0x2544a, 0x78be},
{0x2548e, 0x788a}, {0x2550e, 0x7895}, {0x25535, 0x78c1}, {0x2562c, 0x7953}, {0x25755, 0x8292}, {0x25764, 0x5229},
{0x25771, 0x79ed}, {0x25874, 0x7a3d}, {0x259c4, 0x7ac8}, {0x259d4, 0x5c41}, {0x25ae1, 0x7afd}, {0x25ae3, 0x7c45},
{0x25cd1, 0x7c21}, {0x25d61, 0x7bdd}, {0x25dda, 0x97ab}, {0x25e62, 0x805e}, {0x2600c, 0x88bd}, {0x26017, 0x7e54},
{0x26231, 0x74f7}, {0x26286, 0x7f61}, {0x26292, 0x7b31}, {0x2634c, 0x7f8a}, {0x26469, 0x7fec}, {0x2657e, 0x8052},
{0x2667e, 0x811b}, {0x266b0, 0x8107}, {0x2671d, 0x81cd}, {0x26878, 0x81a9}, {0x268dd, 0x9824}, {0x26936, 0x81f4},
{0x2695b, 0x81ff}, {0x26aff, 0x827e}, {0x26e65, 0x852d}, {0x26ff6, 0x5f45}, {0x27080, 0x85d0}, {0x270ce, 0x859b},
{0x2726e, 0x86c7}, {0x272cb, 0x8708}, {0x27312, 0x872e}, {0x273fe, 0x460c}, {0x27449, 0x86d7}, {0x2770e, 0x8977},
{0x2776c, 0x8952}, {0x27985, 0x8b33}, {0x2799c, 0x8a3a}, {0x279fc, 0x8abc}, {0x27a84, 0x8a18}, {0x27baf, 0x8c39},
{0x27bbe, 0x8c3f}, {0x27bc7, 0x8c45}, {0x27be1, 0x8c4c}, {0x27be2, 0x537a}, {0x27c3c, 0x8c61}, {0x27c3d, 0x5155},
{0x27c5a, 0x8c63}, {0x27d9b, 0x8cda}, {0x27de4, 0x8cfe}, {0x27e10, 0x81a9}, {0x27e16, 0x8cfa}, {0x27e86, 0x8d70},
{0x27f8d, 0x9085}, {0x2801a, 0x8e27}, {0x280bb, 0x8e48}, {0x281e0, 0x8e94}, {0x282cf, 0x8eeb}, {0x28330, 0x8e2a},
{0x2840c, 0x8f9b}, {0x28452, 0x5f92}, {0x2846d, 0x5f99}, {0x2848c, 0x5f81}, {0x28637, 0x9088}, {0x28946, 0x91e4},
{0x28968, 0x77db}, {0x2896b, 0x925a}, {0x28987, 0x92cf}, {0x28a1e, 0x65a7}, {0x28a50, 0x93e4}, {0x28a71, 0x93df},
{0x28a99, 0x93d8}, {0x28acd, 0x93f8}, {0x28add, 0x9475}, {0x28bc1, 0x28ae4}, {0x28bef, 0x9453}, {0x28e1f, 0x49e2},
{0x28e5f, 0x964b}, {0x28eac, 0x754c}, {0x28eca, 0x761e}, {0x28eeb, 0x96b4}, {0x28ef6, 0x9699}, {0x28f32, 0x852d},
{0x28fa6, 0x9d1f}, {0x28fe1, 0x9d89}, {0x28ffd, 0x96d6}, {0x29346, 0x8203}, {0x296a9, 0x992c}, {0x2970b, 0x9909},
{0x29719, 0x9952}, {0x29750, 0x7ccd}, {0x297cb, 0x8d0d}, {0x29d4b, 0x9b5a}, {0x29dfa, 0x9b92}, {0x29ec4, 0x9c25},
{0x29fce, 0x9d09}, {0x2a0f9, 0x9dba}, {0x2a176, 0x6c05}, {0x2a369, 0x9945}, {0x2a38c, 0x9eb3}, {0x2a502, 0x9f0e},
{0x2a535, 0x9f16}, {0x2a5f1, 0x9f5f}, {0x2a602, 0x9f67}, {0x2a61a, 0x9f6c}, {0x2a6c1, 0x79cb}, {0x2a818, 0x5ee8},
{0x2b746, 0x4eca}, {0x2b777, 0x5ea7}, {0x2f80b, 0x50cf}, {0x2f80f, 0x514e}, {0x2f817, 0x5197}, {0x2f81a, 0x51ac},
{0x2f822, 0x5272}, {0x2f828, 0x52fa}, {0x2f82a, 0x5306}, {0x2f82c, 0x5349}, {0x2f833, 0x537f}, {0x2f839, 0x53eb},
{0x2f83f, 0x5468}, {0x2f84f, 0x5674}, {0x2f852, 0x57ce}, {0x2f862, 0x59eb}, {0x2f86b, 0x5b3e}, {0x2f884, 0x5dfd},
{0x2f899, 0x5f62}, {0x2f89a, 0x5f6b}, {0x2f8a5, 0x60c7}, {0x2f8a6, 0x6148}, {0x2f8ac, 0x61b2}, {0x2f8ad, 0x61a4},
{0x2f8b1, 0x61f6}, {0x2f8b2, 0x6210}, {0x2f8b7, 0x6350}, {0x2f8bf, 0x6422}, {0x2f8e1, 0x6851}, {0x2f8e5, 0x681f},
{0x2f8ed, 0x6adb}, {0x2f8f1, 0x6b54}, {0x2f8fc, 0x6cbf}, {0x2f903, 0x6d69}, {0x2f90b, 0x6ecb}, {0x2f90f, 0x6f6e},
{0x2f938, 0x7570}, {0x2f93d, 0x76c2}, {0x2f941, 0x76f1}, {0x2f96c, 0x7d63}, {0x2f96e, 0x7dc7}, {0x2f995, 0x82bd},
{0x2f999, 0x831d}, {0x2f9d0, 0x8aed}, {0x2f9df, 0x8f38}, {0x2f9ea, 0x927c},
};
extern const unsigned kanji_variants_count = 2122;
vendor/hachidori/third_party/hoshidicts/src/text_processor/text_processor.cpp
Vendored
+462
View File
@@ -0,0 +1,462 @@
#include "text_processor.hpp"
#include <ankerl/unordered_dense.h>
#include <utf8.h>
#include <utf8proc.h>
#include <algorithm>
#include <cstdint>
#include <ranges>
#include <string>
#include <string_view>
#include <unordered_map>
#include <utility>
#include <vector>
extern const char32_t kanji_variants[][2];
extern const unsigned kanji_variants_count;
namespace {
struct TextProcessor {
std::vector<int> options;
void (*process)(const std::u32string&, int, std::u32string&);
};
// https://github.com/yomidevs/yomitan/blob/81d17d877fb18c62ba826210bf6db2b7f4d4deed/ext/js/language/ja/japanese.js#L21
constexpr uint32_t KATAKANA_SMALL_KA = 0x30f5;
constexpr uint32_t KATAKANA_SMALL_KE = 0x30f6;
constexpr uint32_t KANA_PROLONGED_SOUND_MARK = 0x30fc;
constexpr uint32_t HIRAGANA_SMALL_TSU = 0x3063;
constexpr uint32_t KATAKANA_SMALL_TSU = 0x30c3;
constexpr char32_t KATAKANA_MIDDLE_DOT = 0x30fb;
constexpr uint32_t HIRAGANA_CONVERSION_RANGE_START = 0x3041;
constexpr uint32_t HIRAGANA_CONVERSION_RANGE_END = 0x3096;
constexpr uint32_t KATAKANA_CONVERSION_RANGE_START = 0x30a1;
constexpr uint32_t KATAKANA_CONVERSION_RANGE_END = 0x30f6;
constexpr char32_t KANJI_ITERATION_MARK = 0x3005;
constexpr char32_t HIRAGANA_ITERATION_MARK = 0x309d;
constexpr char32_t HIRAGANA_VOICED_ITERATION_MARK = 0x309e;
constexpr char32_t KATAKANA_ITERATION_MARK = 0x30fd;
constexpr char32_t KATAKANA_VOICED_ITERATION_MARK = 0x30fe;
constexpr char32_t DAKUTEN = 0x3099;
// https://github.com/yomidevs/yomitan/blob/81d17d877fb18c62ba826210bf6db2b7f4d4deed/ext/js/language/ja/japanese.js#L121
const std::unordered_map<char32_t, std::u32string> VOWEL_TO_KANA{
{U'a', U"ぁあかがさざただなはばぱまゃやらゎわヵァアカガサザタダナハバパマャヤラヮワヵヷ"},
{U'i', U"ぃいきぎしじちぢにひびぴみりゐィイキギシジチヂニヒビピミリヰヸ"},
{U'u', U"ぅうくぐすずっつづぬふぶぷむゅゆるゥウクグスズッツヅヌフブプムュユルヴ"},
{U'e', U"ぇえけげせぜてでねへべぺめれゑヶェエケゲセゼテデネヘベペメレヱヶヹ"},
{U'o', U"ぉおこごそぞとどのほぼぽもょよろをォオコゴソゾトドノホボポモョヨロヲヺ"}};
// https://github.com/yomidevs/yomitan/blob/81d17d877fb18c62ba826210bf6db2b7f4d4deed/ext/js/language/ja/japanese.js#L131
std::unordered_map<char32_t, char32_t> build_kana_to_vowel_map() {
std::unordered_map<char32_t, char32_t> map;
for (const auto& [vowel, kana_string] : VOWEL_TO_KANA) {
for (char32_t c : kana_string) {
map.try_emplace(c, vowel);
}
}
return map;
}
char32_t kana_to_vowel(char32_t kana) {
static const auto KANA_TO_VOWEL = build_kana_to_vowel_map();
auto it = KANA_TO_VOWEL.find(kana);
if (it != KANA_TO_VOWEL.end()) {
return it->second;
}
return 0;
}
// https://github.com/yomidevs/yomitan/blob/81d17d877fb18c62ba826210bf6db2b7f4d4deed/ext/js/language/ja/japanese.js#L155
char32_t get_prolonged_hiragana(char32_t prev) {
switch (kana_to_vowel(prev)) {
case U'a':
return U'あ';
case U'i':
return U'い';
case U'u':
return U'う';
case U'e':
return U'え';
case U'o':
return U'う';
default:
return 0;
}
}
bool is_in_range(uint32_t c, uint32_t range_start, uint32_t range_end) { return c >= range_start && c <= range_end; }
// https://github.com/yomidevs/yomitan/blob/81d17d877fb18c62ba826210bf6db2b7f4d4deed/ext/js/language/ja/japanese.js#L472
void hiragana_to_katakana(const std::u32string& text, std::u32string& result) {
result.assign(text);
const uint32_t offset = (KATAKANA_CONVERSION_RANGE_START - HIRAGANA_CONVERSION_RANGE_START);
for (char32_t& c : result) {
if (is_in_range(c, HIRAGANA_CONVERSION_RANGE_START, HIRAGANA_CONVERSION_RANGE_END)) {
c = static_cast<char32_t>(c + offset);
}
}
}
// https://github.com/yomidevs/yomitan/blob/81d17d877fb18c62ba826210bf6db2b7f4d4deed/ext/js/language/ja/japanese.js#L441
void katakana_to_hiragana(const std::u32string& text, std::u32string& result) {
result.assign(text);
const uint32_t offset = (HIRAGANA_CONVERSION_RANGE_START - KATAKANA_CONVERSION_RANGE_START);
for (size_t i = 0; i < result.size(); ++i) {
char32_t c = result[i];
switch (c) {
case KATAKANA_SMALL_KA:
case KATAKANA_SMALL_KE:
break;
case KANA_PROLONGED_SOUND_MARK:
if (i > 0) {
const auto prolonged = get_prolonged_hiragana(result[i - 1]);
if (prolonged != 0) {
c = prolonged;
}
}
break;
default:
if (is_in_range(c, KATAKANA_CONVERSION_RANGE_START, KATAKANA_CONVERSION_RANGE_END)) {
c = static_cast<char32_t>(c + offset);
}
break;
}
result[i] = c;
}
}
bool is_emphatic(char32_t c) {
return c == HIRAGANA_SMALL_TSU || c == KATAKANA_SMALL_TSU || c == KANA_PROLONGED_SOUND_MARK;
}
// https://github.com/yomidevs/yomitan/blob/81d17d877fb18c62ba826210bf6db2b7f4d4deed/ext/js/language/ja/japanese.js#L776
void collapse_emphatic_sequences(const std::u32string& text, bool full_collapse, std::u32string& result) {
ptrdiff_t left = 0;
while (left < static_cast<ptrdiff_t>(text.size()) && is_emphatic(text[left])) {
++left;
}
ptrdiff_t right = static_cast<ptrdiff_t>(text.size()) - 1;
while (right >= 0 && is_emphatic(text[right])) {
--right;
}
if (left > right) {
result = text;
return;
}
result.clear();
result.reserve(text.size());
result.append(text, 0, static_cast<size_t>(left));
auto current_collapsed_code_point = static_cast<char32_t>(-1);
for (ptrdiff_t i = left; i <= right; ++i) {
char32_t c = text[i];
if (is_emphatic(c)) {
if (current_collapsed_code_point != c) {
current_collapsed_code_point = c;
if (!full_collapse) {
result += c;
continue;
}
}
} else {
current_collapsed_code_point = static_cast<char32_t>(-1);
result += c;
}
}
result.append(text, static_cast<size_t>(right + 1), std::u32string::npos);
}
void nfkc(const std::u32string& text, std::u32string& result) {
constexpr auto options = static_cast<utf8proc_option_t>(UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
static thread_local std::vector<utf8proc_int32_t> buffer;
buffer.clear();
int boundclass = UTF8PROC_BOUNDCLASS_START;
for (char32_t c : text) {
if (c == 0) {
break;
}
utf8proc_int32_t tmp[32];
utf8proc_ssize_t n = utf8proc_decompose_char(static_cast<utf8proc_int32_t>(c), tmp, 32, options, &boundclass);
if (n < 0) {
result = text;
return;
}
if (n <= 32) {
buffer.insert(buffer.end(), tmp, tmp + n);
} else {
const size_t pos = buffer.size();
buffer.resize(pos + static_cast<size_t>(n));
n = utf8proc_decompose_char(static_cast<utf8proc_int32_t>(c), buffer.data() + pos, n, options, &boundclass);
if (n < 0) {
result = text;
return;
}
}
}
utf8proc_ssize_t len = static_cast<utf8proc_ssize_t>(buffer.size());
for (utf8proc_ssize_t pos = 0; pos < len - 1;) {
const utf8proc_int32_t uc1 = buffer[pos];
const utf8proc_int32_t uc2 = buffer[pos + 1];
const utf8proc_property_t* p1 = utf8proc_get_property(uc1);
const utf8proc_property_t* p2 = utf8proc_get_property(uc2);
if (p1->combining_class > p2->combining_class && p2->combining_class > 0) {
buffer[pos] = uc2;
buffer[pos + 1] = uc1;
if (pos > 0) {
pos--;
} else {
pos++;
}
} else {
pos++;
}
}
len = utf8proc_normalize_utf32(buffer.data(), len, options);
if (len < 0) {
result = text;
return;
}
result.assign(buffer.begin(), buffer.begin() + len);
}
// https://github.com/yomidevs/yomitan/blob/3440451aecb23a43f308857969c890a55ce34a91/ext/js/language/ja/japanese.js#L489
void alphanumeric_to_fullwidth(const std::u32string& text, std::u32string& result) {
result.assign(text);
for (char32_t& c : result) {
if (is_in_range(c, U'0', U'9')) {
c = static_cast<char32_t>(c + (0xff10 - 0x30));
} else if (is_in_range(c, U'A', U'Z')) {
c = static_cast<char32_t>(c + (0xff21 - 0x41));
} else if (is_in_range(c, U'a', U'z')) {
c = static_cast<char32_t>(c + (0xff41 - 0x61));
}
}
}
struct KanjiVariantTable {
ankerl::unordered_dense::map<char32_t, char32_t> map;
std::vector<bool> blocks;
};
const KanjiVariantTable& kanji_variant_table() {
static const KanjiVariantTable table = [] {
KanjiVariantTable t;
t.map.reserve(kanji_variants_count);
t.blocks.assign(0x1100, false);
for (unsigned i = 0; i < kanji_variants_count; ++i) {
const char32_t from = kanji_variants[i][0];
t.map[from] = kanji_variants[i][1];
t.blocks[from >> 8] = true;
}
return t;
}();
return table;
}
void standardize_kanji(const std::u32string& text, std::u32string& result) {
const auto& table = kanji_variant_table();
result.assign(text);
for (char32_t& c : result) {
if (c >= 0x110000 || !table.blocks[c >> 8]) {
continue;
}
auto it = table.map.find(c);
if (it != table.map.end()) {
c = it->second;
}
}
}
char32_t add_dakuten(char32_t kana) {
std::u32string pair = {kana, DAKUTEN};
std::string utf8 = utf8::utf32to8(pair);
utf8proc_uint8_t* out = utf8proc_NFC(reinterpret_cast<const utf8proc_uint8_t*>(utf8.c_str()));
if (!out) {
return kana;
}
std::u32string composed = utf8::utf8to32(std::string(reinterpret_cast<char*>(out)));
utf8proc_free(out);
return composed.size() == 1 ? composed.front() : kana;
}
char32_t expand_mark(char32_t prev, char32_t mark) {
switch (mark) {
case KANJI_ITERATION_MARK:
case HIRAGANA_ITERATION_MARK:
case KATAKANA_ITERATION_MARK:
return prev;
case HIRAGANA_VOICED_ITERATION_MARK:
case KATAKANA_VOICED_ITERATION_MARK:
return add_dakuten(prev);
default:
return 0;
}
}
void expand_iteration_marks(const std::u32string& text, std::u32string& result) {
result.clear();
result.reserve(text.size());
for (size_t i = 0; i < text.size(); ++i) {
result += text[i];
if (i + 1 < text.size()) {
char32_t expanded = expand_mark(text[i], text[i + 1]);
if (expanded != 0) {
result += expanded;
++i;
}
}
}
}
constexpr std::u32string_view KANJI_NUMBERS = U"〇一二三四五六七八九";
void numbers_to_kanji(const std::u32string& text, std::u32string& result) {
result.assign(text);
for (char32_t& c : result) {
if (is_in_range(c, 0xff10, 0xff19)) {
c = KANJI_NUMBERS[c - 0xff10];
}
}
}
void strip_middle_dots(const std::u32string& text, std::u32string& result) {
result.clear();
result.reserve(text.size());
for (char32_t c : text) {
if (c != KATAKANA_MIDDLE_DOT) {
result += c;
}
}
}
const std::vector<TextProcessor>& get_japanese_processors() {
static const std::vector<TextProcessor> processors = {
{.options = {0, 1},
.process = [](const std::u32string& text, int opt, std::u32string& out) { nfkc(text, out); }},
// https://github.com/yomidevs/yomitan/blob/81d17d877fb18c62ba826210bf6db2b7f4d4deed/ext/js/language/ja/japanese-text-preprocessors.js#L66
{.options = {0, 1, 2},
.process =
[](const std::u32string& text, int opt, std::u32string& out) {
if (opt == 1) {
katakana_to_hiragana(text, out);
} else {
hiragana_to_katakana(text, out);
}
}},
{.options = {0, 1, 2},
.process =
[](const std::u32string& text, int opt, std::u32string& out) {
collapse_emphatic_sequences(text, opt == 2, out);
}},
{.options = {0, 1},
.process = [](const std::u32string& text, int opt, std::u32string& out) { alphanumeric_to_fullwidth(text, out); }},
{.options = {0, 1},
.process = [](const std::u32string& text, int opt, std::u32string& out) { standardize_kanji(text, out); }},
{.options = {0, 1},
.process = [](const std::u32string& text, int opt, std::u32string& out) { expand_iteration_marks(text, out); }},
{.options = {0, 1},
.process = [](const std::u32string& text, int opt, std::u32string& out) { numbers_to_kanji(text, out); }},
{.options = {0, 1}, .process = [](const std::u32string& text, int opt, std::u32string& out) {
strip_middle_dots(text, out);
}}};
return processors;
}
}
// https://github.com/yomidevs/yomitan/blob/81d17d877fb18c62ba826210bf6db2b7f4d4deed/ext/js/language/translator.js#L564
std::vector<TextVariant> text_processor::process(std::string_view src) {
using Variant = std::pair<std::u32string, int>;
static thread_local std::vector<Variant> variants_pool;
static thread_local std::vector<Variant> next_pool;
static thread_local std::u32string scratch;
std::vector<Variant>& variants = variants_pool;
std::vector<Variant>& next = next_pool;
if (variants.empty()) {
variants.emplace_back();
}
variants[0].first.clear();
utf8::utf8to32(src.begin(), src.end(), std::back_inserter(variants[0].first));
variants[0].second = 0;
size_t variant_count = 1;
for (const auto& processor : get_japanese_processors()) {
size_t next_count = 0;
auto find_next = [&](const std::u32string& text) {
return std::find_if(next.begin(), next.begin() + static_cast<std::ptrdiff_t>(next_count),
[&](const Variant& entry) { return entry.first == text; });
};
auto next_end = [&] { return next.begin() + static_cast<std::ptrdiff_t>(next_count); };
for (size_t vi = 0; vi < variant_count; ++vi) {
std::u32string& variant = variants[vi].first;
const int steps = variants[vi].second;
for (int option : processor.options) {
if (option == 0) {
continue;
}
processor.process(variant, option, scratch);
if (scratch == variant) {
continue;
}
int new_steps = steps + 1;
auto it = find_next(scratch);
if (it == next_end()) {
if (next_count < next.size()) {
next[next_count].first.assign(scratch);
next[next_count].second = new_steps;
} else {
next.emplace_back(scratch, new_steps);
}
++next_count;
} else if (new_steps < it->second) {
it->second = new_steps;
}
}
auto it = find_next(variant);
if (it == next_end()) {
if (next_count < next.size()) {
std::swap(next[next_count].first, variant);
next[next_count].second = steps;
} else {
next.emplace_back(std::move(variant), steps);
}
++next_count;
} else if (steps < it->second) {
it->second = steps;
}
}
std::sort(next.begin(), next_end(), [](const Variant& a, const Variant& b) { return a.first < b.first; });
std::swap(variants, next);
variant_count = next_count;
}
std::vector<TextVariant> result;
result.reserve(variant_count);
for (size_t vi = 0; vi < variant_count; ++vi) {
const std::u32string& variant = variants[vi].first;
const int steps = variants[vi].second;
size_t bytes = 0;
for (char32_t c : variant) {
bytes += c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4;
}
std::string utf8;
utf8.resize_and_overwrite(bytes, [&](char* out, size_t) {
char* end = utf8::utf32to8(variant.begin(), variant.end(), out);
return static_cast<size_t>(end - out);
});
result.emplace_back(std::move(utf8), steps);
}
return result;
}
vendor/hachidori/third_party/hoshidicts/src/text_processor/text_processor.hpp
Vendored
+14
View File
@@ -0,0 +1,14 @@
#pragma once
#include <string>
#include <string_view>
#include <vector>
struct TextVariant {
std::string text;
int steps;
};
namespace text_processor {
std::vector<TextVariant> process(std::string_view src);
}
vendor/hachidori/third_party/hoshidicts/src/zip/zip.cpp
Vendored
+167
View File
@@ -0,0 +1,167 @@
#include "zip.hpp"
#include <libdeflate.h>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include "../memory/memory.hpp"
namespace {
template <typename T>
T read_at(const uint8_t* base, size_t offset) {
T val;
std::memcpy(&val, base + offset, sizeof(T));
return val;
}
}
Zip::~Zip() { memory::unmap(file); }
bool Zip::open(const std::filesystem::path& path) {
file = memory::map_rd(path);
if (!file) {
return false;
}
return parse_central_directory();
}
int Zip::find(const std::string& name) const {
for (int i = 0; i < static_cast<int>(entries.size()); ++i) {
if (entries[i].name == name) {
return i;
}
}
return -1;
}
std::string Zip::read(int index) const {
const auto& e = entries[index];
if (e.uncompressed_size == 0) {
return "";
}
std::string result;
result.resize(e.uncompressed_size);
const auto* src = file.data + e.data_offset;
if (e.compression_method == 0) {
std::memcpy(result.data(), src, e.uncompressed_size);
} else if (e.compression_method == 8) {
thread_local auto* d = libdeflate_alloc_decompressor();
if (libdeflate_deflate_decompress(d, src, e.compressed_size, result.data(), e.uncompressed_size, nullptr) !=
LIBDEFLATE_SUCCESS) {
return "";
}
} else {
return "";
}
return result;
}
std::optional<Zip::MediaResult> Zip::read_media(int index) const {
const auto& e = entries[index];
MediaResult out;
out.path = e.name;
out.blob.resize(e.uncompressed_size);
if (e.uncompressed_size == 0) {
return out;
}
const auto* src = file.data + e.data_offset;
if (e.compression_method == 0) {
std::memcpy(out.blob.data(), src, e.uncompressed_size);
} else if (e.compression_method == 8) {
thread_local auto* d = libdeflate_alloc_decompressor();
if (libdeflate_deflate_decompress(d, src, e.compressed_size, out.blob.data(), e.uncompressed_size, nullptr) !=
LIBDEFLATE_SUCCESS) {
return std::nullopt;
}
} else {
return std::nullopt;
}
return out;
}
// https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
bool Zip::parse_central_directory() {
const auto* base = file.data;
if (file.size < 22) {
return false;
}
size_t eocd = file.size - 22;
while (eocd > 0 && read_at<uint32_t>(base, eocd) != 0x06054b50) {
eocd--;
}
if (read_at<uint32_t>(base, eocd) != 0x06054b50) {
return false;
}
uint64_t total_entries = read_at<uint16_t>(base, eocd + 10);
uint64_t cd_offset = read_at<uint32_t>(base, eocd + 16);
if (eocd >= 20 && read_at<uint32_t>(base, eocd - 20) == 0x07064b50) {
auto eocd64_offset = read_at<uint64_t>(base, eocd - 12);
if (eocd64_offset <= file.size && file.size - eocd64_offset >= 56 &&
read_at<uint32_t>(base, eocd64_offset) == 0x06064b50) {
total_entries = read_at<uint64_t>(base, eocd64_offset + 32);
cd_offset = read_at<uint64_t>(base, eocd64_offset + 48);
}
}
entries.reserve(total_entries);
size_t pos = cd_offset;
for (uint64_t i = 0; i < total_entries; ++i) {
if (pos > file.size || file.size - pos < 46) {
return false;
}
if (read_at<uint32_t>(base, pos) != 0x02014b50) {
return false;
}
ZipEntry e;
e.compression_method = read_at<uint16_t>(base, pos + 10);
e.compressed_size = read_at<uint32_t>(base, pos + 20);
e.uncompressed_size = read_at<uint32_t>(base, pos + 24);
auto name_len = read_at<uint16_t>(base, pos + 28);
auto extra_len = read_at<uint16_t>(base, pos + 30);
auto comment_len = read_at<uint16_t>(base, pos + 32);
auto lfh_offset = read_at<uint32_t>(base, pos + 42);
e.name.assign(reinterpret_cast<const char*>(base + pos + 46), name_len);
if (lfh_offset > file.size || file.size - lfh_offset < 30) {
return false;
}
if (read_at<uint32_t>(base, lfh_offset + 18) != e.compressed_size ||
read_at<uint32_t>(base, lfh_offset + 22) != e.uncompressed_size) {
error = "archive entry sizes disagree between headers";
return false;
}
auto lfh_name_len = read_at<uint16_t>(base, lfh_offset + 26);
auto lfh_extra_len = read_at<uint16_t>(base, lfh_offset + 28);
e.data_offset = static_cast<size_t>(lfh_offset) + 30 + lfh_name_len + lfh_extra_len;
const uint32_t data_size = e.compression_method == 0 ? e.uncompressed_size : e.compressed_size;
if (e.data_offset > file.size || file.size - e.data_offset < data_size) {
return false;
}
if (e.compression_method != 0 && e.uncompressed_size > 0) {
if (e.compressed_size == 0) {
error = "archive entry has no compressed data for its declared size";
return false;
}
}
entries.push_back(std::move(e));
pos += static_cast<size_t>(46) + name_len + extra_len + comment_len;
}
return true;
}
vendor/hachidori/third_party/hoshidicts/src/zip/zip.hpp
Vendored
+38
View File
@@ -0,0 +1,38 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <optional>
#include <string>
#include <vector>
#include "../memory/memory.hpp"
struct ZipEntry {
std::string name;
uint16_t compression_method;
uint32_t compressed_size;
uint32_t uncompressed_size;
size_t data_offset;
};
struct Zip {
memory::mapped_file file;
std::vector<ZipEntry> entries;
std::string error;
~Zip();
bool open(const std::filesystem::path& path);
int find(const std::string& name) const;
std::string read(int index) const;
struct MediaResult {
std::string path;
std::vector<char> blob;
};
std::optional<MediaResult> read_media(int index) const;
private:
bool parse_central_directory();
};

Some files were not shown because too many files have changed in this diff Show More