mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-09-22 17:16:19 -07:00
feat(dictionary): add Hachidori backend support
- Add backend selection, setup gating, Anki integration, and external host support - Add launcher flags, documentation, packaging, and focused tests - Open on-demand overlay modals on the first attempt
This commit is contained in:
+232
@@ -0,0 +1,232 @@
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
#include "hoshidicts_c.h"
|
||||
|
||||
static void print_usage(const char* program) {
|
||||
printf("Usage:\n");
|
||||
printf("%s import <path/to/dictionary.zip|dictionary.mdx>\n", program);
|
||||
printf("%s query <path/to/dictionary> <word>\n", program);
|
||||
printf("%s lookup <path/to/dictionary> <lookup_string>\n", program);
|
||||
printf("%s kanji <path/to/dictionary> <kanji>\n", program);
|
||||
}
|
||||
|
||||
static size_t utf8_length(const char* text) {
|
||||
size_t length = 0;
|
||||
for (const char* c = text; *c != '\0'; c++) {
|
||||
if ((*c & 0xC0) != 0x80) {
|
||||
length++;
|
||||
}
|
||||
}
|
||||
return length;
|
||||
}
|
||||
|
||||
static int cmd_import(const char* path) {
|
||||
char output_dir[256] = ".";
|
||||
const char* parent = strrchr(path, '/');
|
||||
if (parent) {
|
||||
memcpy(output_dir, path, parent - path);
|
||||
output_dir[parent - path] = '\0';
|
||||
}
|
||||
|
||||
hd_import_result* ir = hd_import(path, output_dir, false);
|
||||
if (ir == NULL) {
|
||||
printf("failed to import dictionary\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (hd_import_result_success(ir)) {
|
||||
printf("title: %s\n", hd_import_result_title(ir));
|
||||
printf("term_count: %llu\n", hd_import_result_term_count(ir));
|
||||
printf("meta_count: %llu\n", hd_import_result_meta_count(ir));
|
||||
printf("freq_count: %llu\n", hd_import_result_freq_count(ir));
|
||||
printf("pitch_count: %llu\n", hd_import_result_pitch_count(ir));
|
||||
printf("kanji_count: %llu\n", hd_import_result_kanji_count(ir));
|
||||
printf("media_count: %llu\n", hd_import_result_media_count(ir));
|
||||
} else {
|
||||
printf("could not import dictionary: %s\n", hd_import_result_error(ir));
|
||||
}
|
||||
|
||||
hd_import_result_free(ir);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int cmd_query(const char* db_path, const char* expression) {
|
||||
hd_query* q = hd_query_new();
|
||||
if (hd_query_add_term_dict(q, db_path) != 0) {
|
||||
printf("could not open dictionary: %s\n", db_path);
|
||||
hd_query_free(q);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const hd_term_result* terms = NULL;
|
||||
size_t count = 0;
|
||||
hd_results* r = hd_query_run(q, expression, &terms, &count);
|
||||
if (r == NULL) {
|
||||
printf("query failed\n");
|
||||
hd_query_free(q);
|
||||
return 1;
|
||||
}
|
||||
|
||||
printf("query results for: %s length: %zu\n", expression, utf8_length(expression));
|
||||
printf("%zu entries\n", count);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
printf("---------------------------------------------------------------\n");
|
||||
printf("%.*s %.*s %.*s\n", (int)terms[i].expression.len, terms[i].expression.ptr, (int)terms[i].reading.len,
|
||||
terms[i].reading.ptr, (int)terms[i].rules.len, terms[i].rules.ptr);
|
||||
printf("%zu glossary entries\n", terms[i].glossaries_count);
|
||||
for (size_t j = 0; j < terms[i].glossaries_count; j++) {
|
||||
printf("------\n");
|
||||
printf("%.*s\n", (int)terms[i].glossaries[j].dict_name.len, terms[i].glossaries[j].dict_name.ptr);
|
||||
printf("%.*s\n", (int)terms[i].glossaries[j].glossary.len, terms[i].glossaries[j].glossary.ptr);
|
||||
}
|
||||
}
|
||||
|
||||
hd_results_free(r);
|
||||
hd_query_free(q);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int cmd_lookup(const char* const* db_paths, int db_count, const char* lookup_string) {
|
||||
const int max_results = 8;
|
||||
const size_t scan_length = 16;
|
||||
|
||||
hd_query* q = hd_query_new();
|
||||
for (int i = 0; i < db_count; i++) {
|
||||
if (hd_query_add_term_dict(q, db_paths[i]) != 0) {
|
||||
printf("could not open dictionary: %s\n", db_paths[i]);
|
||||
hd_query_free(q);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
hd_deinflector* d = hd_deinflector_new();
|
||||
hd_lookup* l = hd_lookup_new(q, d);
|
||||
|
||||
const hd_lookup_result* results = NULL;
|
||||
size_t count = 0;
|
||||
hd_lookup_results* r = hd_lookup_run(l, lookup_string, max_results, scan_length, &results, &count);
|
||||
if (r == NULL) {
|
||||
printf("lookup failed\n");
|
||||
hd_lookup_free(l);
|
||||
hd_deinflector_free(d);
|
||||
hd_query_free(q);
|
||||
return 1;
|
||||
}
|
||||
|
||||
printf("lookup results for: %s max_results: %d scan_length: %zu\n", lookup_string, max_results, scan_length);
|
||||
printf("%zu results\n", count);
|
||||
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
printf("---------------------------------------------------------------\n");
|
||||
printf("%.*s\n", (int)results[i].matched.len, results[i].matched.ptr);
|
||||
if (results[i].trace_count > 0) {
|
||||
printf(" ");
|
||||
for (size_t j = 0; j < results[i].trace_count; j++) {
|
||||
printf("%.*s%s", (int)results[i].trace[j].name.len, results[i].trace[j].name.ptr,
|
||||
j < results[i].trace_count - 1 ? " -> " : "");
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
printf("%.*s %.*s\n", (int)results[i].term.expression.len, results[i].term.expression.ptr,
|
||||
(int)results[i].term.reading.len, results[i].term.reading.ptr);
|
||||
for (size_t j = 0; j < results[i].term.glossaries_count; j++) {
|
||||
printf("------\n");
|
||||
printf("%.*s\n", (int)results[i].term.glossaries[j].dict_name.len, results[i].term.glossaries[j].dict_name.ptr);
|
||||
printf("%.*s\n", (int)results[i].term.glossaries[j].glossary.len, results[i].term.glossaries[j].glossary.ptr);
|
||||
}
|
||||
}
|
||||
|
||||
const hd_dictionary_style* styles = NULL;
|
||||
size_t styles_count = 0;
|
||||
hd_styles* s = hd_query_get_styles(q, &styles, &styles_count);
|
||||
printf("styles: \n");
|
||||
for (size_t i = 0; i < styles_count; i++) {
|
||||
printf("%.*s\n", (int)styles[i].dict_name.len, styles[i].dict_name.ptr);
|
||||
printf("%.*s\n", (int)styles[i].styles.len, styles[i].styles.ptr);
|
||||
}
|
||||
|
||||
hd_styles_free(s);
|
||||
hd_lookup_results_free(r);
|
||||
hd_lookup_free(l);
|
||||
hd_deinflector_free(d);
|
||||
hd_query_free(q);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int cmd_kanji(const char* db_path, const char* kanji) {
|
||||
hd_query* q = hd_query_new();
|
||||
if (hd_query_add_kanji_dict(q, db_path) != 0) {
|
||||
printf("could not open dictionary: %s\n", db_path);
|
||||
hd_query_free(q);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const hd_kanji_entry* entries = NULL;
|
||||
size_t count = 0;
|
||||
hd_kanji_results* r = hd_query_run_kanji(q, kanji, &entries, &count);
|
||||
if (r == NULL) {
|
||||
printf("kanji query failed\n");
|
||||
hd_query_free(q);
|
||||
return 1;
|
||||
}
|
||||
|
||||
printf("kanji result for: %s\n", kanji);
|
||||
printf("%zu entries\n", count);
|
||||
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
printf("---------------------------------------------------------------\n");
|
||||
printf("dict: %.*s\n", (int)entries[i].dict_name.len, entries[i].dict_name.ptr);
|
||||
printf("onyomi: %.*s\n", (int)entries[i].onyomi.len, entries[i].onyomi.ptr);
|
||||
printf("kunyomi: %.*s\n", (int)entries[i].kunyomi.len, entries[i].kunyomi.ptr);
|
||||
printf("tags: %.*s\n", (int)entries[i].tags.len, entries[i].tags.ptr);
|
||||
printf("definitions:\n");
|
||||
for (size_t j = 0; j < entries[i].definitions_count; j++) {
|
||||
printf(" - %.*s\n", (int)entries[i].definitions[j].len, entries[i].definitions[j].ptr);
|
||||
}
|
||||
if (entries[i].stats_count > 0) {
|
||||
printf("stats:\n");
|
||||
for (size_t j = 0; j < entries[i].stats_count; j++) {
|
||||
printf(" %.*s: %.*s\n", (int)entries[i].stats[j].key.len, entries[i].stats[j].key.ptr,
|
||||
(int)entries[i].stats[j].value.len, entries[i].stats[j].value.ptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hd_kanji_results_free(r);
|
||||
hd_query_free(q);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
if (argc < 2) {
|
||||
print_usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
struct timespec t0;
|
||||
struct timespec t1;
|
||||
clock_gettime(CLOCK_MONOTONIC, &t0);
|
||||
int ret;
|
||||
|
||||
const char* command = argv[1];
|
||||
if (strcmp(command, "import") == 0 && argc >= 3) {
|
||||
ret = cmd_import(argv[2]);
|
||||
} else if (strcmp(command, "query") == 0 && argc >= 4) {
|
||||
ret = cmd_query(argv[2], argv[3]);
|
||||
} else if (strcmp(command, "lookup") == 0 && argc >= 4) {
|
||||
ret = cmd_lookup((const char* const*)argv + 2, argc - 3, argv[argc - 1]);
|
||||
} else if (strcmp(command, "kanji") == 0 && argc >= 4) {
|
||||
ret = cmd_kanji(argv[2], argv[3]);
|
||||
} else {
|
||||
print_usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
clock_gettime(CLOCK_MONOTONIC, &t1);
|
||||
double ms = (double)(t1.tv_sec - t0.tv_sec) * 1000.0 + (double)(t1.tv_nsec - t0.tv_nsec) / 1e6;
|
||||
printf("runtime: %.2fms\n", ms);
|
||||
return ret;
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
#include <utf8.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <ranges>
|
||||
#include <string>
|
||||
|
||||
#include "../src/path_utils.hpp"
|
||||
#include "../src/text_processor/text_processor.hpp"
|
||||
#include "hoshidicts/deinflector.hpp"
|
||||
#include "hoshidicts/importer.hpp"
|
||||
#include "hoshidicts/lookup.hpp"
|
||||
#include "hoshidicts/query.hpp"
|
||||
|
||||
void print_usage(const char* program) {
|
||||
std::cout << std::format("Usage:\n");
|
||||
std::cout << std::format("{} import <path/to/dictionary.zip|dictionary.mdx>\n", program);
|
||||
std::cout << std::format("{} deinflect <word>\n", program);
|
||||
std::cout << std::format("{} preprocess <word>\n", program);
|
||||
std::cout << std::format("{} query <path/to/dictionary> <word>\n", program);
|
||||
std::cout << std::format("{} lookup <path/to/dictionary> <lookup_string>\n", program);
|
||||
std::cout << std::format("{} freq <path/to/dictionary> <word>\n", program);
|
||||
std::cout << std::format("{} kanji <path/to/dictionary> <kanji>\n", program);
|
||||
}
|
||||
|
||||
void cmd_import(const std::string& path) {
|
||||
std::filesystem::path source_path = path_utils::from_utf8(path);
|
||||
std::string output_dir = path_utils::to_utf8(source_path.parent_path());
|
||||
if (output_dir.empty()) {
|
||||
output_dir = ".";
|
||||
}
|
||||
ImportResult result = dictionary_importer::import(path, output_dir);
|
||||
|
||||
if (result.success) {
|
||||
std::cout << std::format("title: {}\n", result.title);
|
||||
std::cout << std::format("term_count: {}\n", result.summary.counts.terms.total);
|
||||
std::cout << std::format("meta_count: {}\n", result.summary.counts.termMeta["total"]);
|
||||
std::cout << std::format("freq_count: {}\n", result.summary.counts.termMeta["freq"]);
|
||||
std::cout << std::format("pitch_count: {}\n",
|
||||
result.summary.counts.termMeta["pitch"] + result.summary.counts.termMeta["ipa"]);
|
||||
std::cout << std::format("kanji_count: {}\n", result.summary.counts.kanji.total);
|
||||
std::cout << std::format("media_count: {}\n", result.summary.counts.media.total);
|
||||
} else {
|
||||
std::cout << std::format("could not import dictionary: {}\n", result.error);
|
||||
}
|
||||
}
|
||||
|
||||
void cmd_deinflect(const std::string& inflected) {
|
||||
Deinflector deinflector;
|
||||
auto results = deinflector.deinflect(inflected);
|
||||
|
||||
std::cout << std::format("deinflections for: {} length: {}\n", inflected,
|
||||
utf8::distance(inflected.begin(), inflected.end()));
|
||||
std::cout << std::format("found {} candidates\n\n", results.size());
|
||||
|
||||
for (const auto& r : results) {
|
||||
std::cout << std::format("{} (conditions: {})", r.text, r.conditions);
|
||||
if (!r.trace.empty()) {
|
||||
std::cout << std::format(" ");
|
||||
for (size_t i = 0; i < r.trace.size(); ++i) {
|
||||
std::cout << std::format("{}{}", r.trace[i].name, i < r.trace.size() - 1 ? " -> " : "");
|
||||
}
|
||||
std::cout << std::format("\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void cmd_preprocess(const std::string& text) {
|
||||
auto results = text_processor::process(text);
|
||||
|
||||
std::cout << std::format("preproccesing for: {} length: {}\n", text, utf8::distance(text.begin(), text.end()));
|
||||
std::cout << std::format("found {} variants\n", results.size());
|
||||
|
||||
for (const auto& r : results) {
|
||||
std::cout << std::format("{}\n", r.text);
|
||||
}
|
||||
}
|
||||
|
||||
void cmd_query(const std::string& db_path, const std::string& expression) {
|
||||
DictionaryQuery dict_query;
|
||||
dict_query.add_term_dict(db_path);
|
||||
auto result = dict_query.query(expression);
|
||||
|
||||
std::cout << std::format("query results for: {} length: {}\n", expression,
|
||||
utf8::distance(expression.begin(), expression.end()));
|
||||
std::cout << std::format("{} entries\n", result.size());
|
||||
for (const auto& r : result) {
|
||||
std::cout << std::format("---------------------------------------------------------------\n");
|
||||
std::cout << std::format("{} {} {}\n", r.expression, r.reading, r.rules);
|
||||
std::cout << std::format("{} glossary entries\n", r.glossaries.size());
|
||||
for (const auto& g : r.glossaries) {
|
||||
std::cout << std::format("------\n");
|
||||
std::cout << std::format("{}\n", g.dict_name);
|
||||
std::cout << std::format("{}\n", g.glossary);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void cmd_freq(const std::string& path, const std::string& expression, const std::string& reading) {
|
||||
std::vector<TermResult> terms;
|
||||
terms.emplace_back(TermResult{.expression = expression, .reading = reading});
|
||||
|
||||
DictionaryQuery query;
|
||||
query.add_freq_dict(path);
|
||||
query.query_freq(terms);
|
||||
std::cout << std::format("frequency entries for: {}\n", expression);
|
||||
int count = 0;
|
||||
for (auto& freq : terms[0].frequencies) {
|
||||
std::cout << std::format("dict: {}\n", freq.dict_name);
|
||||
for (auto& freq_entry : freq.frequencies) {
|
||||
std::cout << std::format("val: {} display_val: {}\n", freq_entry.value, freq_entry.display_value);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
std::cout << std::format("count: {}\n", count);
|
||||
}
|
||||
|
||||
void cmd_kanji(const std::string& path, const std::string& kanji) {
|
||||
DictionaryQuery query;
|
||||
query.add_kanji_dict(path);
|
||||
auto result = query.query_kanji(kanji);
|
||||
|
||||
std::cout << std::format("kanji result for: {}\n", kanji);
|
||||
std::cout << std::format("{} entries\n", result.entries.size());
|
||||
|
||||
for (const auto& e : result.entries) {
|
||||
std::cout << std::format("---------------------------------------------------------------\n");
|
||||
std::cout << std::format("dict: {}\n", e.dict_name);
|
||||
std::cout << std::format("onyomi: {}\n", e.onyomi);
|
||||
std::cout << std::format("kunyomi: {}\n", e.kunyomi);
|
||||
std::cout << std::format("tags: {}\n", e.tags);
|
||||
std::cout << std::format("definitions:\n");
|
||||
for (const auto& def : e.definitions) {
|
||||
std::cout << std::format(" - {}\n", def);
|
||||
}
|
||||
if (!e.stats.empty()) {
|
||||
std::cout << std::format("stats:\n");
|
||||
for (const auto& [k, v] : e.stats) {
|
||||
std::cout << std::format(" {}: {}\n", k, v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void cmd_lookup(const std::vector<std::string>& db_paths, const std::string& lookup_string, int max_results = 8,
|
||||
int scan_length = 16) {
|
||||
DictionaryQuery dict_query;
|
||||
for (const auto& path : db_paths) {
|
||||
dict_query.add_term_dict(path);
|
||||
}
|
||||
Deinflector deinflect;
|
||||
Lookup lookup(dict_query, deinflect);
|
||||
auto result = lookup.lookup(lookup_string, max_results, scan_length);
|
||||
|
||||
std::cout << std::format("lookup results for: {} max_results: {} scan_length: {}\n", lookup_string, max_results,
|
||||
scan_length);
|
||||
std::cout << std::format("{} results\n", result.size());
|
||||
|
||||
for (const auto& r : result) {
|
||||
std::cout << std::format("---------------------------------------------------------------\n");
|
||||
std::cout << std::format("{}\n", r.matched);
|
||||
if (!r.trace.empty()) {
|
||||
std::cout << std::format(" ");
|
||||
for (size_t i = 0; i < r.trace.size(); ++i) {
|
||||
std::cout << std::format("{}{}", r.trace[i].name, i < r.trace.size() - 1 ? " -> " : "");
|
||||
}
|
||||
std::cout << std::format("\n");
|
||||
}
|
||||
std::cout << std::format("{} {}\n", r.term.expression, r.term.reading);
|
||||
for (const auto& g : r.term.glossaries) {
|
||||
std::cout << std::format("------\n");
|
||||
std::cout << std::format("{}\n", g.dict_name);
|
||||
std::cout << std::format("{}\n", g.glossary);
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << std::format("styles: \n");
|
||||
for (const auto& s : dict_query.get_styles()) {
|
||||
std::cout << std::format("{}\n", s.dict_name);
|
||||
std::cout << std::format("{}\n", s.styles);
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
if (argc < 2) {
|
||||
print_usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const auto begin = std::chrono::steady_clock::now();
|
||||
std::string_view command = argv[1];
|
||||
|
||||
if (command == "import" && argc >= 3) {
|
||||
cmd_import(argv[2]);
|
||||
} else if (command == "deinflect" && argc >= 3) {
|
||||
cmd_deinflect(argv[2]);
|
||||
} else if (command == "preprocess" && argc >= 3) {
|
||||
cmd_preprocess(argv[2]);
|
||||
} else if (command == "query" && argc >= 4) {
|
||||
cmd_query(argv[2], argv[3]);
|
||||
} else if (command == "lookup" && argc >= 4) {
|
||||
auto db_paths = std::views::counted(argv + 2, argc - 3) |
|
||||
std::views::transform([](const char* arg) { return std::string(arg); }) |
|
||||
std::ranges::to<std::vector>();
|
||||
std::string term = argv[argc - 1];
|
||||
cmd_lookup(db_paths, term);
|
||||
} else if (command == "freq" && argc >= 5) {
|
||||
cmd_freq(argv[2], argv[3], argv[4]);
|
||||
} else if (command == "kanji" && argc >= 4) {
|
||||
cmd_kanji(argv[2], argv[3]);
|
||||
} else {
|
||||
print_usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const auto end = std::chrono::steady_clock::now();
|
||||
std::chrono::duration<double, std::milli> duration = end - begin;
|
||||
std::cout << std::format("runtime: {}ms\n", duration.count());
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user