feat(dictionary): add Hachidori backend support

- Add backend selection, setup gating, Anki integration, and external host support
- Add launcher flags, documentation, packaging, and focused tests
- Open on-demand overlay modals on the first attempt

vendor/hachidori/third_party/hoshidicts/include/hoshidicts.h

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "hoshidicts/deinflector.hpp"
+#include "hoshidicts/importer.hpp"
+#include "hoshidicts/lookup.hpp"
+#include "hoshidicts/query.hpp"

vendor/hachidori/third_party/hoshidicts/include/hoshidicts/deinflector.hpp

@@ -0,0 +1,80 @@
+#pragma once
+
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+#include <cstdint>
+#include <cstddef>
+
+struct TransformGroup {
+  std::string name;
+  std::string description;
+};
+
+struct DeinflectionResult {
+  std::string text;
+  uint32_t conditions;
+  std::vector<TransformGroup> trace;
+};
+
+class Deinflector {
+ public:
+  Deinflector();
+  std::vector<DeinflectionResult> deinflect(const std::string& text) const;
+  static uint32_t pos_to_conditions(const std::vector<std::string>& part_of_speech);
+
+ private:
+  struct Rule {
+    std::string from;
+    std::string to;
+    uint32_t conditions_in;
+    uint32_t conditions_out;
+    int group_id;
+  };
+
+  enum Conditions : uint32_t {
+    NONE = 0,
+    V1D = 1 << 0,
+    V1P = 1 << 1,
+    V5D = 1 << 2,
+    V5SS = 1 << 3,
+    V5SP = 1 << 4,
+    VK = 1 << 5,
+    VS = 1 << 6,
+    VZ = 1 << 7,
+    ADJ_I = 1 << 8,
+    MASU = 1 << 9,
+    MASEN = 1 << 10,
+    TE = 1 << 11,
+    BA = 1 << 12,
+    KU = 1 << 13,
+    TA = 1 << 14,
+    NN = 1 << 15,
+    NASAI = 1 << 16,
+    YA = 1 << 17,
+    V1 = V1D | V1P,
+    V5S = V5SS | V5SP,
+    V5 = V5D | V5S,
+    V = V1 | V5 | VK | VS | VZ,
+  };
+
+  void deinflect_recursive(const std::string& text, uint32_t conditions, std::vector<int>& trace,
+                           std::vector<DeinflectionResult>& results) const;
+
+  void init_transforms();
+
+  int add_group(const TransformGroup& group);
+  void add_rule(const Rule& rule);
+  void add_irregular(std::string_view suffix, uint32_t conditions_in, uint32_t conditions_out, int group_id);
+
+  struct TrieNode {
+    std::vector<std::pair<char32_t, uint32_t>> children;
+    int rules = -1;
+  };
+
+  std::vector<TrieNode> trie_;
+  std::vector<std::vector<Rule>> rule_lists_;
+  std::vector<TransformGroup> groups_;
+  size_t max_length_;
+};

vendor/hachidori/third_party/hoshidicts/include/hoshidicts/importer.hpp

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <optional>
+#include <string>
+#include <vector>
+
+struct SummaryItemCount {
+  size_t total = 0;
+};
+
+using SummaryMetaCount = std::map<std::string, size_t>;
+
+struct SummaryCounts {
+  SummaryItemCount terms;
+  SummaryMetaCount termMeta;
+  SummaryItemCount kanji;
+  SummaryMetaCount kanjiMeta;
+  SummaryItemCount tagMeta;
+  SummaryItemCount media;
+};
+
+struct Summary {
+  std::string title;
+  std::string revision;
+  bool sequenced = false;
+  std::optional<std::string> minimumYomitanVersion;
+  int version = 3;
+  uint64_t importDate = 0;
+  bool prefixWildcardsSupported = false;
+  SummaryCounts counts;
+  std::string styles;
+  std::optional<bool> isUpdatable;
+  std::optional<std::string> indexUrl;
+  std::optional<std::string> downloadUrl;
+  std::optional<std::string> author;
+  std::optional<std::string> url;
+  std::optional<std::string> description;
+  std::optional<std::string> attribution;
+  std::optional<std::string> sourceLanguage;
+  std::optional<std::string> targetLanguage;
+  std::optional<std::string> frequencyMode;
+  std::optional<bool> importSuccess;
+};
+
+struct ImportResult {
+  bool success = false;
+  std::string title;
+  Summary summary;
+  std::string error;
+};
+
+namespace dictionary_importer {
+ImportResult import(const std::string& source_path, const std::string& output_dir, bool low_ram = false);
+};

vendor/hachidori/third_party/hoshidicts/include/hoshidicts/lookup.hpp

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "deinflector.hpp"
+#include "query.hpp"
+
+struct LookupResult {
+  std::string matched;
+  std::string deinflected;
+  std::vector<TransformGroup> trace;
+  TermResult term;
+  int preprocessor_steps;
+};
+
+enum class LookupFrequencyOrder { Auto, Ascending, Descending, Disabled };
+
+struct LookupOptions {
+  std::optional<std::string> frequency_dictionary;
+  LookupFrequencyOrder frequency_order = LookupFrequencyOrder::Auto;
+  std::optional<std::string> primary_reading;
+};
+
+class Lookup {
+ public:
+  Lookup(DictionaryQuery& query, Deinflector& deinflector) : query_(query), deinflector_(deinflector) {};
+  std::vector<LookupResult> lookup(const std::string& lookup_string, int max_results = 16, size_t scan_length = 16,
+                                   const LookupOptions& options = {}) const;
+  std::vector<LookupResult> lookup_dictionary(const std::string& lookup_string, const std::string& dictionary_path,
+                                              int max_results = 16, size_t scan_length = 16,
+                                              const LookupOptions& options = {}) const;
+
+ private:
+  std::vector<LookupResult> lookup_impl(const std::string& lookup_string, const std::string* dictionary_path,
+                                        int max_results, size_t scan_length, const LookupOptions& options) const;
+  static void filter_by_pos(RawTerms& terms, const DeinflectionResult& d);
+
+  DictionaryQuery& query_;
+  Deinflector& deinflector_;
+};

vendor/hachidori/third_party/hoshidicts/include/hoshidicts/query.hpp

@@ -0,0 +1,174 @@
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <vector>
+
+#if defined(__clang__) && defined(__APPLE__)
+#define SWIFT_IMPORT_UNSAFE __attribute__((swift_attr("import_unsafe")))
+#else
+#define SWIFT_IMPORT_UNSAFE
+#endif
+
+struct Frequency {
+  int value;
+  std::string display_value;
+  std::string reading;
+};
+
+struct DictionaryStyle {
+  std::string dict_name;
+  std::string styles;
+};
+
+struct MediaFileView {
+  const char* data;
+  size_t size;
+};
+
+struct ZSTD_DDict_s;
+
+struct GlossaryEntry {
+  std::string dict_name;
+  std::string glossary;
+  std::string definition_tags;
+  std::string term_tags;
+  const uint8_t* compressed_data = nullptr;
+  uint32_t compressed_size = 0;
+  const ZSTD_DDict_s* zstd_dict = nullptr;
+};
+
+struct FrequencyEntry {
+  std::string dict_name;
+  std::vector<Frequency> frequencies;
+};
+
+struct Pitch {
+  int position = 0;
+  std::string pattern;
+  std::vector<int> nasal;
+  std::vector<int> devoice;
+};
+
+struct PitchEntry {
+  std::string dict_name;
+  std::vector<Pitch> pitches;
+  std::vector<std::string> transcriptions;
+};
+
+struct TermResult {
+  std::string expression;
+  std::string reading;
+  std::string rules;
+  double score = 0;
+  std::vector<GlossaryEntry> glossaries;
+  std::vector<FrequencyEntry> frequencies;
+  std::vector<PitchEntry> pitches;
+};
+
+struct KanjiEntry {
+  std::string dict_name;
+  std::string onyomi;
+  std::string kunyomi;
+  std::string tags;
+  std::vector<std::string> definitions;
+  std::unordered_map<std::string, std::string> stats;
+};
+
+struct KanjiResult {
+  std::string character;
+  std::vector<KanjiEntry> entries;
+};
+
+struct RawTerm;
+struct RawTerms;
+
+class DictionaryQuery {
+ public:
+  DictionaryQuery();
+  ~DictionaryQuery();
+
+  DictionaryQuery(const DictionaryQuery&) = delete;
+  DictionaryQuery& operator=(const DictionaryQuery&) = delete;
+
+  DictionaryQuery(DictionaryQuery&&) noexcept;
+  DictionaryQuery& operator=(DictionaryQuery&&) noexcept;
+
+  bool add_term_dict(const std::string& path);
+  bool add_freq_dict(const std::string& path);
+  bool add_pitch_dict(const std::string& path);
+  bool add_kanji_dict(const std::string& path);
+
+  // Drops every loaded kind of the dictionary at `path` and returns how many
+  // entries were removed (0 when the path is not loaded). The other
+  // dictionaries keep their relative order, so a caller can reshape the loaded
+  // set without rebuilding it.
+  size_t remove_dict(const std::string& path);
+
+  // Reorders every kind so the dictionaries appear in the order of `paths`.
+  // Dictionaries not listed keep their relative order after the listed ones.
+  // Returns false, changing nothing, when a listed path is not loaded.
+  bool set_dict_order(const std::vector<std::string>& paths);
+
+  // Long-key scan index (see src/scan_index.hpp). `long_key_length` returns
+  // the longest term-dictionary key, in code points, that begins with the
+  // first eight code points of `text` and is longer than 16, or 0 when there
+  // is none or `text` is shorter than eight code points. `max_long_key_length`
+  // is the longest such key any loaded term dictionary records, so a host can
+  // size the text it hands to Lookup; 0 when no dictionary has an index.
+  // Both accept a term dictionary path to consult that dictionary alone.
+  size_t long_key_length(std::string_view text, const std::string* term_dictionary_path = nullptr) const;
+  size_t max_long_key_length(const std::string* term_dictionary_path = nullptr) const;
+
+  void query_freq(std::vector<TermResult>& terms, bool match_reading = true) const;
+  void query_pitch(std::vector<TermResult>& terms) const;
+  KanjiResult query_kanji(const std::string& kanji) const;
+
+  std::vector<TermResult> query(const std::string& expression) const;
+
+  std::vector<char> get_media_file(const std::string& dict_name, const std::string& media_path) const;
+  SWIFT_IMPORT_UNSAFE
+  MediaFileView get_media_file_view(const std::string& dict_name, const std::string& media_path) const;
+  std::vector<DictionaryStyle> get_styles() const;
+  std::vector<std::string> get_freq_dict_order() const;
+
+ private:
+  friend class Lookup;
+  RawTerms query_raw(const std::string& expression,
+                     const std::string* term_dictionary_path = nullptr) const;
+  TermResult build_term(const RawTerms& raw, RawTerm& term) const;
+  void collect_frequencies(std::string_view expression, std::string_view reading,
+                           std::vector<FrequencyEntry>& out, bool match_reading = true) const;
+  void collect_pitches(std::string_view expression, std::string_view reading, std::vector<PitchEntry>& out) const;
+  void materialize(TermResult& term) const;
+
+  struct DictionaryData;
+  struct Dictionary {
+    Dictionary();
+    ~Dictionary();
+
+    Dictionary(const Dictionary&) = delete;
+    Dictionary& operator=(const Dictionary&) = delete;
+
+    Dictionary(Dictionary&&) noexcept;
+    Dictionary& operator=(Dictionary&&) noexcept;
+
+    std::string path;
+    std::string name;
+    std::string styles;
+    std::unique_ptr<DictionaryData> data;
+  };
+  enum DictionaryType : uint8_t { TERM, FREQ, PITCH, KANJI };
+
+  bool add_dict(const std::string& path, DictionaryType);
+  bool add_dict_(const std::string& path, DictionaryType);
+
+  static std::string decompress_glossary(const void* data, size_t size, const ZSTD_DDict_s* dict);
+  std::vector<Dictionary> term_dicts_;
+  std::vector<Dictionary> freq_dicts_;
+  std::vector<Dictionary> pitch_dicts_;
+  std::vector<Dictionary> kanji_dicts_;
+};

vendor/hachidori/third_party/hoshidicts/include/hoshidicts_c.h

@@ -0,0 +1,196 @@
+#ifndef HOSHIDICTS_C_H
+#define HOSHIDICTS_C_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct hd_str {
+  const char* ptr;
+  size_t len;
+} hd_str;
+
+// importer
+typedef struct hd_import_result hd_import_result;
+
+hd_import_result* hd_import(const char* source_path, const char* output_dir, int low_ram);
+void hd_import_result_free(hd_import_result* r);
+
+int hd_import_result_success(const hd_import_result* r);
+const char* hd_import_result_title(const hd_import_result* r);
+uint64_t hd_import_result_term_count(const hd_import_result* r);
+uint64_t hd_import_result_meta_count(const hd_import_result* r);
+uint64_t hd_import_result_freq_count(const hd_import_result* r);
+uint64_t hd_import_result_pitch_count(const hd_import_result* r);
+uint64_t hd_import_result_kanji_count(const hd_import_result* r);
+uint64_t hd_import_result_media_count(const hd_import_result* r);
+
+const char* hd_import_result_error(const hd_import_result* r);
+
+// deinflector
+typedef struct hd_deinflector hd_deinflector;
+
+hd_deinflector* hd_deinflector_new(void);
+void hd_deinflector_free(hd_deinflector* d);
+
+// query
+typedef struct hd_query hd_query;
+typedef struct hd_results hd_results;
+typedef struct hd_kanji_results hd_kanji_results;
+typedef struct hd_styles hd_styles;
+
+typedef struct hd_frequency {
+  int32_t value;
+  hd_str display_value;
+} hd_frequency;
+
+typedef struct hd_dictionary_style {
+  hd_str dict_name;
+  hd_str styles;
+} hd_dictionary_style;
+
+typedef struct hd_media_file {
+  const uint8_t* data;
+  size_t size;
+} hd_media_file;
+
+typedef struct hd_glossary_entry {
+  hd_str dict_name;
+  hd_str glossary;
+  hd_str definition_tags;
+  hd_str term_tags;
+} hd_glossary_entry;
+
+typedef struct hd_frequency_entry {
+  hd_str dict_name;
+  const hd_frequency* frequencies;
+  size_t frequencies_count;
+} hd_frequency_entry;
+
+typedef struct hd_pitch {
+  int32_t position;
+  hd_str pattern;
+  const int32_t* nasal;
+  size_t nasal_count;
+  const int32_t* devoice;
+  size_t devoice_count;
+} hd_pitch;
+
+typedef struct hd_pitch_entry {
+  hd_str dict_name;
+  const hd_pitch* pitches;
+  size_t pitches_count;
+  const hd_str* transcriptions;
+  size_t transcriptions_count;
+} hd_pitch_entry;
+
+typedef struct hd_term_result {
+  hd_str expression;
+  hd_str reading;
+  hd_str rules;
+  double score;
+  const hd_glossary_entry* glossaries;
+  size_t glossaries_count;
+  const hd_frequency_entry* frequencies;
+  size_t frequencies_count;
+  const hd_pitch_entry* pitches;
+  size_t pitches_count;
+} hd_term_result;
+
+typedef struct hd_kanji_stat {
+  hd_str key;
+  hd_str value;
+} hd_kanji_stat;
+
+typedef struct hd_kanji_entry {
+  hd_str dict_name;
+  hd_str onyomi;
+  hd_str kunyomi;
+  hd_str tags;
+  const hd_str* definitions;
+  size_t definitions_count;
+  const hd_kanji_stat* stats;
+  size_t stats_count;
+} hd_kanji_entry;
+
+hd_query* hd_query_new(void);
+void hd_query_free(hd_query* q);
+
+int hd_query_add_term_dict(hd_query* q, const char* path);
+int hd_query_add_freq_dict(hd_query* q, const char* path);
+int hd_query_add_pitch_dict(hd_query* q, const char* path);
+int hd_query_add_kanji_dict(hd_query* q, const char* path);
+// Removes every loaded kind of the dictionary at path; returns the number removed.
+size_t hd_query_remove_dict(hd_query* q, const char* path);
+// Reorders the loaded dictionaries to follow paths (see DictionaryQuery::set_dict_order).
+// Returns 0 on success, 1 when a listed path is not loaded.
+int hd_query_set_dict_order(hd_query* q, const char* const* paths, size_t count);
+
+// Longest term-dictionary key, in code points, recorded in the loaded
+// dictionaries' long-key scan indexes (keys longer than 16 code points; 0 when
+// none is loaded). A host should hand hd_lookup_run at least this many code
+// points plus eight so that an extended scan can reach such a key.
+size_t hd_query_max_long_key_length(const hd_query* q);
+
+hd_results* hd_query_run(const hd_query* q, const char* expression, const hd_term_result** out_terms,
+                         size_t* out_count);
+void hd_results_free(hd_results* r);
+
+hd_kanji_results* hd_query_run_kanji(const hd_query* q, const char* kanji, const hd_kanji_entry** out_entries,
+                                     size_t* out_count);
+void hd_kanji_results_free(hd_kanji_results* r);
+
+hd_media_file hd_query_get_media_file(const hd_query* q, const char* dict_name, const char* media_path);
+
+hd_styles* hd_query_get_styles(const hd_query* q, const hd_dictionary_style** out_styles, size_t* out_count);
+void hd_styles_free(hd_styles* s);
+
+// lookup
+typedef struct hd_lookup hd_lookup;
+typedef struct hd_lookup_results hd_lookup_results;
+
+typedef struct hd_transform_group {
+  hd_str name;
+  hd_str description;
+} hd_transform_group;
+
+typedef struct hd_lookup_result {
+  hd_str matched;
+  hd_str deinflected;
+  const hd_transform_group* trace;
+  size_t trace_count;
+  hd_term_result term;
+  int32_t preprocessor_steps;
+} hd_lookup_result;
+
+typedef enum hd_lookup_frequency_order {
+  HD_LOOKUP_FREQUENCY_ORDER_AUTO = 0,
+  HD_LOOKUP_FREQUENCY_ORDER_ASCENDING = 1,
+  HD_LOOKUP_FREQUENCY_ORDER_DESCENDING = 2,
+  HD_LOOKUP_FREQUENCY_ORDER_DISABLED = 3,
+} hd_lookup_frequency_order;
+
+typedef struct hd_lookup_options {
+  hd_str frequency_dictionary;
+  int32_t frequency_order;
+  hd_str primary_reading;
+} hd_lookup_options;
+
+hd_lookup* hd_lookup_new(hd_query* q, hd_deinflector* d);
+void hd_lookup_free(hd_lookup* l);
+
+hd_lookup_results* hd_lookup_run(const hd_lookup* l, const char* lookup_string, int max_results, size_t scan_length,
+                                 const hd_lookup_result** out_results, size_t* out_count);
+hd_lookup_results* hd_lookup_run_with_options(const hd_lookup* l, const char* lookup_string, int max_results,
+                                              size_t scan_length, const hd_lookup_options* options,
+                                              const hd_lookup_result** out_results, size_t* out_count);
+void hd_lookup_results_free(hd_lookup_results* r);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

vendor/hachidori/third_party/hoshidicts/include/module.modulemap

@@ -0,0 +1,5 @@
+module CHoshiDicts {
+    header "hoshidicts.h"
+    requires cplusplus
+    export *
+}