refactor(dictionary): use Hachidori fork submodule and upstream API

src/core/services/tokenizer.test.ts

@@ -4362,6 +4362,26 @@ test('tokenizeSubtitle keeps Yomitan frequency for noun-particle-noun compounds'
   assert.equal(result.tokens?.[0]?.frequencyRank, 581);
 });
 
+test('tokenizeSubtitle skips frequency requests for ranks supplied by the scanner', async () => {
+  const deps = makeDepsFromYomitanTokens(
+    [{ surface: '猫', reading: 'ねこ', headword: '猫', frequencyRank: 42 }],
+    { getFrequencyDictionaryEnabled: () => true },
+  );
+  const parserWindow = deps.getYomitanParserWindow();
+  assert.ok(parserWindow);
+  deps.getYomitanParserWindow = () => parserWindow;
+  const scripts: string[] = [];
+  const execute = parserWindow.webContents.executeJavaScript.bind(parserWindow.webContents);
+  parserWindow.webContents.executeJavaScript = async (script) => {
+    scripts.push(script);
+    return execute(script);
+  };
+  const result = await tokenizeSubtitle('猫', deps);
+  assert.equal(result.tokens?.[0]?.frequencyRank, 42);
+  assert.ok(scripts.length > 0);
+  assert.equal(scripts.filter((script) => script.includes('getTermFrequencies')).length, 0);
+});
+
 test('tokenizeSubtitle keeps frequency for ordinal prefix-noun tokens', async () => {
   const result = await tokenizeSubtitle(
     '第二走者',

src/core/services/tokenizer.ts

@@ -494,6 +494,7 @@ function buildYomitanFrequencyTermReadingList(
 ): Array<{ term: string; reading: string | null }> {
   const termReadingList: Array<{ term: string; reading: string | null }> = [];
   for (const token of tokens) {
+    if (normalizePositiveFrequencyRank(token.frequencyRank) !== null) continue;
     const readingRaw =
       token.reading && token.reading.trim().length > 0 ? token.reading.trim() : null;
     for (const term of resolveYomitanFrequencyLookupTexts(token, matchMode)) {

src/core/services/tokenizer/hachidori-parser-bridge.test.ts

@@ -116,20 +116,6 @@ async function createHarness(emptyLibrary = false) {
                 results: candidates.filter((result) => text.startsWith(result.matched)),
               };
             }
-            case 'hd_frequencies':
-              return {
-                ok: true,
-                frequencies: [
-                  {
-                    term: '食べる',
-                    reading: 'たべる',
-                    hasReading: true,
-                    dictionary: 'Frequency',
-                    frequency: 42,
-                    displayValue: null,
-                  },
-                ],
-              };
             case 'hd_options_write': {
               if (message.baseRevision !== optionRevision) return { ok: false, error: 'conflict' };
               const update = message.options;
@@ -254,6 +240,28 @@ test('Hachidori runs the shared scanner with inflected offsets, headwords, names
   assert.equal(frequencies[0]?.dictionary, 'Frequency');
 });
 
+test('Hachidori frequency lookups match API headwords, readings and requested dictionaries', async () => {
+  const harness = await createHarness();
+  const query = (term: string, reading: string | null, dictionaries = ['Frequency']) =>
+    harness.invoke('getTermFrequencies', { termReadingList: [{ term, reading }], dictionaries });
+  assert.deepEqual(await query('食べる', 'たべる'), [
+    {
+      term: '食べる',
+      reading: 'たべる',
+      hasReading: false,
+      dictionary: 'Frequency',
+      frequency: 42,
+      displayValue: null,
+      displayValueParsed: false,
+    },
+  ]);
+  assert.deepEqual(await query('食べる', 'べつのよみ'), []);
+  assert.deepEqual(await query('食べる', null, ['Other frequency']), []);
+  assert.deepEqual(await query('食べるだけ', null), []);
+  assert.deepEqual(await query('頻度だけ', null), []);
+  assert.deepEqual(await query('食べる', null), await query('食べる', 'たべる'));
+});
+
 test('Hachidori syncs the Anki endpoint and every term template through revisioned writes', async () => {
   const harness = await createHarness();
   const synced = await syncYomitanDefaultAnkiServer(

src/core/services/tokenizer/hachidori-parser-bridge.ts

@@ -138,30 +138,27 @@ export const HACHIDORI_PARSER_BRIDGE_SCRIPT = String.raw`
       }
     }
     async function getTermFrequencies({ termReadingList, dictionaries }) {
-      let frequencies;
-      try {
-        ({ frequencies } = await engine('hd_frequencies', { termReadingList }));
-      } catch (error) {
-        const { sharing } = await send('hd_sharing_status', {}, 'hachidori-sharing');
-        if (!sharing?.client?.connected || !/unknown|unsupported|not supported/i.test(error.message)) throw error;
-        // Older external hosts expose frequency data through term lookups only.
-        frequencies = [];
-        for (const { term, reading } of termReadingList) {
-          const { results } = await engine('hd_lookup', { text: term, maxResults: 100 });
-          for (const result of results) {
-            if (result.term.expression !== term || (reading !== null && result.term.reading !== reading)) continue;
-            for (const group of result.term.frequencies) {
-              for (const value of group.frequencies) {
-                frequencies.push({ term, reading: value.reading || null,
-                  hasReading: typeof value.reading === 'string' && value.reading.length > 0,
-                  dictionary: group.dictionary, frequency: value.value,
-                  displayValue: value.displayValue || null, displayValueParsed: false });
-              }
-            }
+      const terms = [...new Set(termReadingList.map(pair => pair.term))];
+      const { results } = await api({ type: 'hd_api_term_entries', terms });
+      const frequencies = [];
+      for (const result of results) {
+        const term = terms[result.index];
+        const pairs = termReadingList.filter(pair => pair.term === term);
+        for (const entry of result.dictionaryEntries) {
+          for (const value of entry.frequencies) {
+            const headword = entry.headwords[value.headwordIndex];
+            if (!headword || headword.term !== term || !dictionaries.includes(value.dictionary)) continue;
+            if (!pairs.some(pair => pair.reading === null || pair.reading === headword.reading)) continue;
+            // Upstream does not expose the frequency entry's original reading.
+            // Keep its API flag and associate the value with the matched headword.
+            frequencies.push({ term, reading: headword.reading || null,
+              hasReading: value.hasReading, dictionary: value.dictionary,
+              frequency: value.frequency, displayValue: value.displayValue,
+              displayValueParsed: value.displayValueParsed });
           }
         }
       }
-      return frequencies.filter(frequency => dictionaries.includes(frequency.dictionary));
+      return frequencies;
     }
     // Hachidori's public tokenize API emits display furigana without headwords.
     // SubMiner's fallback requires one group per token and a dictionary form.