sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-07-31 19:21:33 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix(tokenizer): stop partial furigana readings from marking words known (#137)

Browse Source
This commit is contained in:
Kyle Yasuda
2026-07-05 17:02:44 -07:00
committed by GitHub
parent eef4500599
commit b14f977e33
5 changed files with 150 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/services/tokenizer/yomitan-parser-runtime.ts
+7 -1
View File
@@ -817,6 +817,12 @@ const YOMITAN_SCANNING_HELPERS = String.raw`
function isCodePointKana(codePoint) { return isCodePointInRanges(codePoint, KANA_RANGES); }
function isCodePointJapanese(codePoint) { return isCodePointInRanges(codePoint, JAPANESE_RANGES); }
function createFuriganaSegment(text, reading) { return {text, reading}; }
function getSegmentReadingContribution(segment) {
if (typeof segment.reading === "string" && segment.reading.length > 0) { return segment.reading; }
const segmentText = typeof segment.text === "string" ? segment.text : "";
const isKanaOnly = segmentText.length > 0 && [...segmentText].every((char) => isCodePointKana(char.codePointAt(0)));
return isKanaOnly ? segmentText : "";
}
function getProlongedHiragana(previousCharacter) {
switch (previousCharacter) {
case "あ": case "か": case "が": case "さ": case "ざ": case "た": case "だ": case "な": case "は": case "ば": case "ぱ": case "ま": case "や": case "ら": case "わ": case "ぁ": case "ゃ": case "ゎ": return "あ";
@@ -1310,7 +1316,7 @@ ${YOMITAN_SCANNING_HELPERS}
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
const tokenPayload = {
surface: segments.map((segment) => segment.text).join("") || source,
reading: segments.map((segment) => typeof segment.reading === "string" ? segment.reading : "").join(""),
reading: segments.map(getSegmentReadingContribution).join(""),
headword: preferredHeadword.term,
startPos: i,
endPos: i + originalTextLength,