fix(tokenizer): lazy yomitan term-only frequency fallback

This commit is contained in:
2026-03-02 01:45:37 -08:00
parent 629fe97ef7
commit 737101fe9e
5 changed files with 298 additions and 79 deletions

View File

@@ -329,11 +329,6 @@ function buildYomitanFrequencyTermReadingList(
const readingRaw =
token.reading && token.reading.trim().length > 0 ? token.reading.trim() : null;
termReadingList.push({ term, reading: readingRaw });
// Yomitan parse readings can be noisy/truncated on merged tokens; include term-only fallback.
if (readingRaw !== null) {
termReadingList.push({ term, reading: null });
}
}
return termReadingList;