diff --git a/changes/subtitle-tokenization-performance.md b/changes/subtitle-tokenization-performance.md index 41ef3f0e..40ae971b 100644 --- a/changes/subtitle-tokenization-performance.md +++ b/changes/subtitle-tokenization-performance.md @@ -15,6 +15,6 @@ area: subtitles - Character name annotations no longer cost a dictionary lookup at every position in a line. The scanner now knows which name forms the current title's character dictionary actually contains and only checks where one can start, which removes the whole overhead of having the character dictionary enabled (measured: 21 lookups per line down to 10, the same as with it disabled). Titles with no cached character data keep the previous exhaustive scan, so a missing snapshot costs speed rather than a missing name. - The cross-line termsFind cache is now bounded by the number of retained dictionary entries as well as by key count, so a run of lookups that each carry hundreds of entries with full glossaries cannot grow the parser window's memory without limit. The budget is re-checked when a lookup resolves, so a single oversized response is dropped rather than parked in the cache and reused. - The unnamed-mob disambiguator filter (Girl A / Girl B) now only drops a single letter or digit split off a name, instead of every one-character term: a name that is genuinely one character keeps its terms whatever the script (𠮷, あ, 별 김, ア・ベ). The character dictionary and the scanner's name pre-pass also share one Han code-point table now, so a name the dictionary accepts is a name the scanner will look for. -- A character name written in halfwidth katakana takes part in the greedy name pre-pass again, so a longer generic word can no longer swallow the start of it, and it now carries a reading (it used to come out blank, which disables known-word matching and frequency lookups for the token). Voiced halfwidth kana compose properly, so ｶﾞｸ reads ガク rather than カﾞク, and kana normalization folds halfwidth throughout so those tokens compare equal to the same word written fullwidth. +- A character name written in halfwidth katakana takes part in the greedy name pre-pass again, so a longer generic word can no longer swallow the start of it, and it now carries a reading (it used to come out blank, which disables known-word matching and frequency lookups for the token). Voiced halfwidth kana compose properly, so ｶﾞｸ reads ガク rather than カﾞク, and kana normalization folds halfwidth throughout so those tokens compare equal to the same word written fullwidth. Because the fold makes halfwidth text indexable, the character-name prefilter now judges halfwidth spellings like any other, and only a position whose candidate-sized region holds an unfoldable voiced mark bypasses it. That also covers a name that starts on a kanji and turns halfwidth later (山ｶﾞｸ), which the earlier first-character rule dropped. - Dictionary-entry classification (source dictionaries, character-dictionary media ids) is memoized per entry object for as long as the entry is cached, instead of being recomputed for every headword comparison and every retry window. - Autoplay priming no longer broadcasts the plain subtitle twice: it tells the processing controller the line has already been painted, so the controller goes straight to the annotated payload. diff --git a/src/core/services/tokenizer/character-dictionary-title.ts b/src/core/services/tokenizer/character-dictionary-title.ts new file mode 100644 index 00000000..d61133da --- /dev/null +++ b/src/core/services/tokenizer/character-dictionary-title.ts @@ -0,0 +1,4 @@ +// Title prefix of the dictionaries SubMiner generates per media. Lives on its +// own because both the main process and the injected scan runtime match on it, +// and the injected fragments interpolate it into their own source. +export const CHARACTER_DICTIONARY_TITLE_PREFIX = 'SubMiner Character Dictionary'; diff --git a/src/core/services/tokenizer/yomitan-dictionary-classification-script.ts b/src/core/services/tokenizer/yomitan-dictionary-classification-script.ts new file mode 100644 index 00000000..68c9823b --- /dev/null +++ b/src/core/services/tokenizer/yomitan-dictionary-classification-script.ts @@ -0,0 +1,141 @@ +// Dictionary classification for the injected scan runtime: which dictionaries +// an entry came from, and whether it is a SubMiner character entry for the +// media being watched. Both walk nested entry data, so both are memoized on the +// entry object by the runtime that hosts them. + +import { CHARACTER_DICTIONARY_TITLE_PREFIX } from './character-dictionary-title'; + +export const YOMITAN_DICTIONARY_CLASSIFICATION_HELPERS = String.raw` + function normalizeWordClasses(headword) { + if (!Array.isArray(headword?.wordClasses)) { return undefined; } + const classes = headword.wordClasses.filter((wordClass) => typeof wordClass === "string" && wordClass.trim().length > 0); + return classes.length > 0 ? classes : undefined; + } + function appendDictionaryNames(target, value) { + if (!value || typeof value !== 'object') { + return; + } + const candidates = [ + value.dictionary, + value.dictionaryName, + value.name, + value.title, + value.dictionaryTitle, + value.dictionaryAlias + ]; + for (const candidate of candidates) { + if (typeof candidate === 'string' && candidate.trim().length > 0) { + target.push(candidate.trim()); + } + } + } + // Memoized on the entry object: termsFind results are cached across + // lines, so the same entries come back for every repeated lookup, and + // each one is classified several times per scan (name pre-pass, + // headword preference, every retry window). + function getDictionaryEntryNames(entry) { + if (!entry || typeof entry !== 'object') { return []; } + const cached = dictionaryEntryNamesCache.get(entry); + if (cached !== undefined) { return cached; } + const names = []; + appendDictionaryNames(names, entry); + for (const definition of entry?.definitions || []) { + appendDictionaryNames(names, definition); + } + for (const frequency of entry?.frequencies || []) { + appendDictionaryNames(names, frequency); + } + for (const pronunciation of entry?.pronunciations || []) { + appendDictionaryNames(names, pronunciation); + } + dictionaryEntryNamesCache.set(entry, names); + return names; + } + // Cached per scan rather than per runtime: the answer depends on + // includeNameMatchMetadata, which is a per-call parameter. + const nameDictionaryEntryCache = new WeakMap(); + function isNameDictionaryEntry(entry) { + if (!includeNameMatchMetadata || !entry || typeof entry !== 'object') { + return false; + } + const cached = nameDictionaryEntryCache.get(entry); + if (cached !== undefined) { return cached; } + const isName = getDictionaryEntryNames(entry).some((name) => name.startsWith(${JSON.stringify(CHARACTER_DICTIONARY_TITLE_PREFIX)})); + nameDictionaryEntryCache.set(entry, isName); + return isName; + } + function parseSubMinerMediaIdFromString(value) { + const imageMatch = value.match(/\bimg\/m(\d+)-/i); + if (imageMatch) { + const parsed = Number.parseInt(imageMatch[1], 10); + if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; } + } + const titleMatch = value.match(/${CHARACTER_DICTIONARY_TITLE_PREFIX}[^\d]*(?:AniList\s*)?(\d+)/i); + if (titleMatch) { + const parsed = Number.parseInt(titleMatch[1], 10); + if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; } + } + return null; + } + function parseSubMinerMediaIdCandidate(value) { + if (typeof value === 'number' && Number.isSafeInteger(value) && value > 0) { + return value; + } + if (typeof value === 'string' && /^\d+$/.test(value.trim())) { + const parsed = Number.parseInt(value.trim(), 10); + if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; } + } + return null; + } + function collectSubMinerMediaIds(value, target) { + if (typeof value === 'string') { + const parsed = parseSubMinerMediaIdFromString(value); + if (parsed !== null) { target.add(parsed); } + return; + } + if (!value || typeof value !== 'object') { + return; + } + if (Array.isArray(value)) { + for (const item of value) { collectSubMinerMediaIds(item, target); } + return; + } + const mediaIdCandidates = [ + value.subminerMediaId, + value.subMinerMediaId, + value.characterDictionaryMediaId, + value.data?.subminerMediaId, + value.data?.subMinerMediaId, + value.data?.characterDictionaryMediaId + ]; + for (const candidate of mediaIdCandidates) { + const parsed = parseSubMinerMediaIdCandidate(candidate); + if (parsed !== null) { target.add(parsed); } + } + for (const child of Object.values(value)) { + collectSubMinerMediaIds(child, target); + } + } + // Walking an entry collects media ids from every nested value, so this + // is the most expensive classification step; memoized on the entry for + // the same reason as the dictionary names above. + function getSubMinerMediaIds(entry) { + if (!entry || typeof entry !== 'object') { return EMPTY_MEDIA_ID_SET; } + const cached = subMinerMediaIdsCache.get(entry); + if (cached !== undefined) { return cached; } + const mediaIds = new Set(); + collectSubMinerMediaIds(entry, mediaIds); + subMinerMediaIdsCache.set(entry, mediaIds); + return mediaIds; + } + function isCurrentMediaNameDictionaryEntry(entry) { + if (!isNameDictionaryEntry(entry)) { + return false; + } + if (currentCharacterDictionaryMediaId === null) { + return true; + } + const mediaIds = getSubMinerMediaIds(entry); + return mediaIds.size === 0 || mediaIds.has(currentCharacterDictionaryMediaId); + } +`; diff --git a/src/core/services/tokenizer/yomitan-frequency-script.ts b/src/core/services/tokenizer/yomitan-frequency-script.ts new file mode 100644 index 00000000..90955fb4 --- /dev/null +++ b/src/core/services/tokenizer/yomitan-frequency-script.ts @@ -0,0 +1,135 @@ +// Frequency-rank resolution for the injected scan runtime: reads the many +// shapes a Yomitan frequency entry can take and picks the best rank for a +// headword, honouring per-dictionary priority and occurrence-vs-rank mode. + +export const YOMITAN_FREQUENCY_HELPERS = String.raw` + function parsePositiveFrequencyNumber(value) { + if (typeof value === 'number' && Number.isFinite(value) && value > 0) { + return Math.max(1, Math.floor(value)); + } + if (typeof value === 'string') { + const numericMatch = value.trim().match(/[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?/)?.[0]; + if (!numericMatch) { return null; } + const parsed = Number.parseFloat(numericMatch); + if (!Number.isFinite(parsed) || parsed <= 0) { return null; } + return Math.max(1, Math.floor(parsed)); + } + if (Array.isArray(value)) { + for (const item of value) { + const parsed = parsePositiveFrequencyNumber(item); + if (parsed !== null) { return parsed; } + } + } + return null; + } + function parseDisplayFrequencyNumber(value) { + if (typeof value === 'string') { + const leadingDigits = value.trim().match(/^\d+/)?.[0]; + if (!leadingDigits) { return null; } + const parsed = Number.parseInt(leadingDigits, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : null; + } + return parsePositiveFrequencyNumber(value); + } + function getFrequencyDictionaryName(frequency) { + const candidates = [ + frequency?.dictionary, + frequency?.dictionaryName, + frequency?.name, + frequency?.title, + frequency?.dictionaryTitle, + frequency?.dictionaryAlias + ]; + for (const candidate of candidates) { + if (typeof candidate === 'string' && candidate.trim().length > 0) { + return candidate.trim(); + } + } + return null; + } + function getBestFrequencyRank(dictionaryEntry, headwordIndex, dictionaryPriorityByName, dictionaryFrequencyModeByName) { + let best = null; + const headwordCount = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords.length : 0; + for (const frequency of dictionaryEntry?.frequencies || []) { + if (!frequency || typeof frequency !== 'object') { continue; } + const frequencyHeadwordIndex = frequency.headwordIndex; + if (typeof frequencyHeadwordIndex === 'number') { + if (frequencyHeadwordIndex !== headwordIndex) { continue; } + } else if (headwordCount > 1) { + continue; + } + const dictionary = getFrequencyDictionaryName(frequency); + if (!dictionary) { continue; } + if (dictionaryFrequencyModeByName[dictionary] === 'occurrence-based') { continue; } + const rank = + parseDisplayFrequencyNumber(frequency.displayValue) ?? + parsePositiveFrequencyNumber(frequency.frequency); + if (rank === null) { continue; } + const priorityRaw = dictionaryPriorityByName[dictionary]; + const fallbackPriority = + typeof frequency.dictionaryIndex === 'number' && Number.isFinite(frequency.dictionaryIndex) + ? Math.max(0, Math.floor(frequency.dictionaryIndex)) + : Number.MAX_SAFE_INTEGER; + const priority = + typeof priorityRaw === 'number' && Number.isFinite(priorityRaw) + ? Math.max(0, Math.floor(priorityRaw)) + : fallbackPriority; + if (best === null || priority < best.priority || (priority === best.priority && rank < best.rank)) { + best = { priority, rank }; + } + } + return best?.rank ?? null; + } + function hasExactSource(headword, token, requirePrimary) { + for (const src of headword.sources || []) { + if (src.originalText !== token) { continue; } + if (requirePrimary && !src.isPrimary) { continue; } + if (src.matchType !== 'exact') { continue; } + return true; + } + return false; + } + function collectExactHeadwordMatches(dictionaryEntries, token, requirePrimary) { + const matches = []; + for (const dictionaryEntry of dictionaryEntries || []) { + const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : []; + for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) { + const headword = headwords[headwordIndex]; + if (!hasExactSource(headword, token, requirePrimary)) { continue; } + matches.push({ dictionaryEntry, headword, headwordIndex }); + } + } + return matches; + } + function sameHeadword(match, preferredMatch) { + if (!match || !preferredMatch) { + return false; + } + if (match.headword?.term !== preferredMatch.headword?.term) { + return false; + } + const matchReading = typeof match.headword?.reading === 'string' ? match.headword.reading : ''; + const preferredReading = + typeof preferredMatch.headword?.reading === 'string' ? preferredMatch.headword.reading : ''; + if (!matchReading || !preferredReading) { + return true; + } + return matchReading === preferredReading; + } + function getBestFrequencyRankForMatches(matches, dictionaryPriorityByName, dictionaryFrequencyModeByName) { + let best = null; + for (const match of matches) { + const rank = getBestFrequencyRank( + match.dictionaryEntry, + match.headwordIndex, + dictionaryPriorityByName, + dictionaryFrequencyModeByName + ); + if (rank === null) { continue; } + if (best === null || rank < best) { + best = rank; + } + } + return best; + } +`; diff --git a/src/core/services/tokenizer/yomitan-furigana-script.ts b/src/core/services/tokenizer/yomitan-furigana-script.ts new file mode 100644 index 00000000..62dc2bf0 --- /dev/null +++ b/src/core/services/tokenizer/yomitan-furigana-script.ts @@ -0,0 +1,170 @@ +// Furigana distribution for the injected scan runtime: splits a headword and +// its reading into the segments a token carries, including the inflected case +// where the matched source text differs from the dictionary form. + +export const YOMITAN_FURIGANA_HELPERS = String.raw` + function createFuriganaSegment(text, reading) { return {text, reading}; } + function getSegmentReadingContribution(segment) { + if (typeof segment.reading === "string" && segment.reading.length > 0) { return segment.reading; } + const segmentText = typeof segment.text === "string" ? segment.text : ""; + const isKanaOnly = segmentText.length > 0 && [...segmentText].every((char) => isCodePointKana(char.codePointAt(0))); + return isKanaOnly ? convertHalfwidthKanaToKatakana(segmentText) : ""; + } + function getProlongedHiragana(previousCharacter) { + switch (previousCharacter) { + case "あ": case "か": case "が": case "さ": case "ざ": case "た": case "だ": case "な": case "は": case "ば": case "ぱ": case "ま": case "や": case "ら": case "わ": case "ぁ": case "ゃ": case "ゎ": return "あ"; + case "い": case "き": case "ぎ": case "し": case "じ": case "ち": case "ぢ": case "に": case "ひ": case "び": case "ぴ": case "み": case "り": case "ぃ": return "い"; + case "う": case "く": case "ぐ": case "す": case "ず": case "つ": case "づ": case "ぬ": case "ふ": case "ぶ": case "ぷ": case "む": case "ゆ": case "る": case "ぅ": case "ゅ": return "う"; + case "え": case "け": case "げ": case "せ": case "ぜ": case "て": case "で": case "ね": case "へ": case "べ": case "ぺ": case "め": case "れ": case "ぇ": return "え"; + case "お": case "こ": case "ご": case "そ": case "ぞ": case "と": case "ど": case "の": case "ほ": case "ぼ": case "ぽ": case "も": case "よ": case "ろ": case "を": case "ぉ": case "ょ": return "う"; + default: return null; + } + } + function getFuriganaKanaSegments(text, reading) { + const newSegments = []; + let start = 0; + let state = (reading[0] === text[0]); + for (let i = 1; i < text.length; ++i) { + const newState = (reading[i] === text[i]); + if (state === newState) { continue; } + newSegments.push(createFuriganaSegment(text.substring(start, i), state ? '' : reading.substring(start, i))); + state = newState; + start = i; + } + newSegments.push(createFuriganaSegment(text.substring(start), state ? '' : reading.substring(start))); + return newSegments; + } + function convertKatakanaToHiragana(text, keepProlongedSoundMarks = false) { + let result = ''; + const offset = (HIRAGANA_CONVERSION_RANGE[0] - KATAKANA_CONVERSION_RANGE[0]); + for (let char of text) { + const codePoint = char.codePointAt(0); + switch (codePoint) { + case KATAKANA_SMALL_KA_CODE_POINT: + case KATAKANA_SMALL_KE_CODE_POINT: + break; + case KANA_PROLONGED_SOUND_MARK_CODE_POINT: + case HALFWIDTH_KANA_PROLONGED_SOUND_MARK_CODE_POINT: + char = "ー"; + if (!keepProlongedSoundMarks && result.length > 0) { + const char2 = getProlongedHiragana(result[result.length - 1]); + if (char2 !== null) { char = char2; } + } + break; + default: + if (isCodePointInRange(codePoint, KATAKANA_CONVERSION_RANGE)) { + char = String.fromCodePoint(codePoint + offset); + break; + } + // Halfwidth katakana folds too, or a name written that way would + // match neither a candidate form nor its own reading. + const halfwidthHiragana = convertHalfwidthKanaCodePointToHiragana(codePoint); + if (halfwidthHiragana !== null) { char = halfwidthHiragana; } + break; + } + result += char; + } + return result; + } + function segmentizeFurigana(reading, readingNormalized, groups, groupsStart) { + const groupCount = groups.length - groupsStart; + if (groupCount <= 0) { return reading.length === 0 ? [] : null; } + const group = groups[groupsStart]; + const {isKana, text} = group; + if (isKana) { + if (group.textNormalized !== null && readingNormalized.startsWith(group.textNormalized)) { + const segments = segmentizeFurigana(reading.substring(text.length), readingNormalized.substring(text.length), groups, groupsStart + 1); + if (segments !== null) { + if (reading.startsWith(text)) { segments.unshift(createFuriganaSegment(text, '')); } + else { segments.unshift(...getFuriganaKanaSegments(text, reading)); } + return segments; + } + } + return null; + } + let result = null; + for (let i = reading.length; i >= text.length; --i) { + const segments = segmentizeFurigana(reading.substring(i), readingNormalized.substring(i), groups, groupsStart + 1); + if (segments !== null) { + if (result !== null) { return null; } + segments.unshift(createFuriganaSegment(text, reading.substring(0, i))); + result = segments; + } + if (groupCount === 1) { break; } + } + return result; + } + function distributeFurigana(term, reading) { + if (reading === term) { return [createFuriganaSegment(term, '')]; } + const groups = []; + let groupPre = null; + let isKanaPre = null; + for (const c of term) { + const isKana = isCodePointKana(c.codePointAt(0)); + if (isKana === isKanaPre) { groupPre.text += c; } + else { + groupPre = {isKana, text: c, textNormalized: null}; + groups.push(groupPre); + isKanaPre = isKana; + } + } + for (const group of groups) { + if (group.isKana) { group.textNormalized = convertKatakanaToHiragana(group.text); } + } + const segments = segmentizeFurigana(reading, convertKatakanaToHiragana(reading), groups, 0); + return segments !== null ? segments : [createFuriganaSegment(term, reading)]; + } + function getStemLength(text1, text2) { + const minLength = Math.min(text1.length, text2.length); + if (minLength === 0) { return 0; } + let i = 0; + while (true) { + const char1 = text1.codePointAt(i); + const char2 = text2.codePointAt(i); + if (char1 !== char2) { break; } + const charLength = String.fromCodePoint(char1).length; + i += charLength; + if (i >= minLength) { + if (i > minLength) { i -= charLength; } + break; + } + } + return i; + } + function distributeFuriganaInflected(term, reading, source) { + const termNormalized = convertKatakanaToHiragana(term); + const readingNormalized = convertKatakanaToHiragana(reading); + const sourceNormalized = convertKatakanaToHiragana(source); + let mainText = term; + let stemLength = getStemLength(termNormalized, sourceNormalized); + const readingStemLength = getStemLength(readingNormalized, sourceNormalized); + if (readingStemLength > 0 && readingStemLength >= stemLength) { + mainText = reading; + stemLength = readingStemLength; + reading = source.substring(0, stemLength) + reading.substring(stemLength); + } + const segments = []; + if (stemLength > 0) { + mainText = source.substring(0, stemLength) + mainText.substring(stemLength); + const segments2 = distributeFurigana(mainText, reading); + let consumed = 0; + for (const segment of segments2) { + const start = consumed; + consumed += segment.text.length; + if (consumed < stemLength) { segments.push(segment); } + else if (consumed === stemLength) { segments.push(segment); break; } + else { + if (start < stemLength) { segments.push(createFuriganaSegment(mainText.substring(start, stemLength), '')); } + break; + } + } + } + if (stemLength < source.length) { + const remainder = source.substring(stemLength); + const last = segments[segments.length - 1]; + if (last && last.reading.length === 0) { last.text += remainder; } + else { segments.push(createFuriganaSegment(remainder, '')); } + } + return segments; + } +`; diff --git a/src/core/services/tokenizer/yomitan-kana-script.ts b/src/core/services/tokenizer/yomitan-kana-script.ts new file mode 100644 index 00000000..4e3c2de2 --- /dev/null +++ b/src/core/services/tokenizer/yomitan-kana-script.ts @@ -0,0 +1,45 @@ +// Kana classification and normalization for the injected scan runtime: the +// code-point ranges the walk tests every character against, and the folds that +// let halfwidth and katakana spellings compare equal to their dictionary form. +import { HAN_CODE_POINT_RANGES } from '../../text/han-code-points'; + +export const YOMITAN_KANA_HELPERS = String.raw` + const HIRAGANA_CONVERSION_RANGE = [0x3041, 0x3096]; + const KATAKANA_CONVERSION_RANGE = [0x30a1, 0x30f6]; + const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc; + const KATAKANA_SMALL_KA_CODE_POINT = 0x30f5; + const KATAKANA_SMALL_KE_CODE_POINT = 0x30f6; + const KANA_RANGES = [[0x3040, 0x309f], [0x30a0, 0x30ff], [0xff66, 0xff9f]]; + const HALFWIDTH_KATAKANA_RANGE = [0xff66, 0xff9d]; + const HALFWIDTH_KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0xff70; + // Folded one code point to one, so every index into a normalized string + // still lines up with the original text — the name-candidate prefilter + // and the furigana stem matching both index back into it. The standalone + // voiced marks (ﾞ ﾟ) have no one-character equivalent and stay as they are. + const HALFWIDTH_KATAKANA_TO_HIRAGANA = "をぁぃぅぇぉゃゅょっーあいうえおかきくけこさしすせそたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわん"; + function convertHalfwidthKanaCodePointToHiragana(codePoint) { + if (codePoint < HALFWIDTH_KATAKANA_RANGE[0] || codePoint > HALFWIDTH_KATAKANA_RANGE[1]) { return null; } + return HALFWIDTH_KATAKANA_TO_HIRAGANA[codePoint - HALFWIDTH_KATAKANA_RANGE[0]] || null; + } + // Halfwidth katakana is kana here but not to the rest of the pipeline + // (known-word matching and frequency lookups only fold fullwidth), so a + // reading taken from halfwidth text is written the way the fullwidth + // katakana path already writes it. NFKC rather than the per-code-point + // table: this is the one place where nothing indexes back into the + // result, so a voiced pair (ｶ + ﾞ) can compose into the single ガ it + // means instead of leaving a stray combining mark in the reading. Scoped + // to the halfwidth runs, because NFKC over everything else rewrites + // characters that have nothing to do with kana (① → 1, ㍑ → リットル). + function convertHalfwidthKanaToKatakana(text) { + return text.replace(/[ｦ-ﾟ]+/g, (run) => run.normalize("NFKC")); + } + // Han ranges come from the shared table so the scan walk and the character + // dictionary agree on what a kanji is (supplementary planes included). + // Halfwidth katakana counts as Japanese text: a name written that way has + // to reach the greedy pre-pass, which has its own handling for it. + const JAPANESE_RANGES = [[0x3040, 0x30ff], [0xff66, 0xff9f], ...${JSON.stringify(HAN_CODE_POINT_RANGES)}]; + function isCodePointInRange(codePoint, range) { return codePoint >= range[0] && codePoint <= range[1]; } + function isCodePointInRanges(codePoint, ranges) { return ranges.some((range) => isCodePointInRange(codePoint, range)); } + function isCodePointKana(codePoint) { return isCodePointInRanges(codePoint, KANA_RANGES); } + function isCodePointJapanese(codePoint) { return isCodePointInRanges(codePoint, JAPANESE_RANGES); } +`; diff --git a/src/core/services/tokenizer/yomitan-match-selection-script.ts b/src/core/services/tokenizer/yomitan-match-selection-script.ts new file mode 100644 index 00000000..cda9121a --- /dev/null +++ b/src/core/services/tokenizer/yomitan-match-selection-script.ts @@ -0,0 +1,81 @@ +// Match selection for the injected scan runtime: picks the headword a position +// tokenizes to, and the longest name or generic match in a window, which is how +// the greedy name pre-pass decides what to reserve. + +export const YOMITAN_MATCH_SELECTION_HELPERS = String.raw` + function findLongestNameMatch(dictionaryEntries, textWindow) { + let best = null; + for (const dictionaryEntry of dictionaryEntries || []) { + if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; } + const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : []; + for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) { + const headword = headwords[headwordIndex]; + for (const src of headword?.sources || []) { + if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; } + const originalText = typeof src.originalText === 'string' ? src.originalText : ''; + if (!originalText || !textWindow.startsWith(originalText)) { continue; } + if (best === null || originalText.length > best.sourceLength) { + best = { dictionaryEntry, headword, headwordIndex, sourceLength: originalText.length }; + } + } + } + } + return best; + } + function findLongestGenericMatchLength(dictionaryEntries, textWindow) { + let best = 0; + for (const dictionaryEntry of dictionaryEntries || []) { + if (isNameDictionaryEntry(dictionaryEntry)) { continue; } + const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : []; + for (const headword of headwords) { + for (const src of headword?.sources || []) { + if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; } + const originalText = typeof src.originalText === 'string' ? src.originalText : ''; + if (!originalText || !textWindow.startsWith(originalText)) { continue; } + if (originalText.length > best) { best = originalText.length; } + } + } + } + return best; + } + function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) { + const currentMediaDictionaryEntries = + currentCharacterDictionaryMediaId === null + ? (dictionaryEntries || []) + : (dictionaryEntries || []).filter((entry) => { + if (!isNameDictionaryEntry(entry)) { return true; } + return isCurrentMediaNameDictionaryEntry(entry); + }); + const exactPrimaryMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, true); + let matchedNameDictionary = false; + if (includeNameMatchMetadata) { + for (const dictionaryEntry of currentMediaDictionaryEntries || []) { + if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; } + for (const match of exactPrimaryMatches) { + if (match.dictionaryEntry !== dictionaryEntry) { continue; } + matchedNameDictionary = true; + break; + } + if (matchedNameDictionary) { break; } + } + } + const preferredMatch = exactPrimaryMatches[0]; + if (preferredMatch) { + const exactFrequencyMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, false) + .filter((match) => sameHeadword(match, preferredMatch)); + return { + term: preferredMatch.headword.term, + reading: preferredMatch.headword.reading, + wordClasses: normalizeWordClasses(preferredMatch.headword), + isNameMatch: + matchedNameDictionary || isCurrentMediaNameDictionaryEntry(preferredMatch.dictionaryEntry), + frequencyRank: getBestFrequencyRankForMatches( + exactFrequencyMatches.length > 0 ? exactFrequencyMatches : exactPrimaryMatches, + dictionaryPriorityByName, + dictionaryFrequencyModeByName + ) + }; + } + return null; + } +`; diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts index 9bffda78..ee7d8fa2 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts @@ -3,7 +3,12 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import test from 'node:test'; -import * as vm from 'node:vm'; +import { + countTermsFindLookups, + createDeps, + createScanDeps, + runInjectedYomitanScript, +} from './yomitan-scan-test-harness'; import { addYomitanNoteViaSearch, clearYomitanParserCachesForWindow, @@ -18,330 +23,6 @@ import { upsertYomitanDictionarySettings, } from './yomitan-parser-runtime'; -function createDeps( - executeJavaScript: (script: string) => Promise, - options?: { - createYomitanExtensionWindow?: (pageName: string) => Promise; - }, -) { - const parserWindow = { - isDestroyed: () => false, - webContents: { - executeJavaScript: async (script: string) => await executeJavaScript(script), - }, - }; - - return { - getYomitanExt: () => ({ id: 'ext-id' }) as never, - getYomitanParserWindow: () => parserWindow as never, - setYomitanParserWindow: () => undefined, - getYomitanParserReadyPromise: () => null, - setYomitanParserReadyPromise: () => undefined, - getYomitanParserInitPromise: () => null, - setYomitanParserInitPromise: () => undefined, - createYomitanExtensionWindow: options?.createYomitanExtensionWindow as never, - }; -} - -function createYomitanScriptSandbox(handler: (action: string, params: unknown) => unknown) { - return { - chrome: { - runtime: { - lastError: null, - sendMessage: ( - payload: { action?: string; params?: unknown }, - callback: (response: { result?: unknown; error?: { message?: string } }) => void, - ) => { - try { - callback({ result: handler(payload.action ?? '', payload.params) }); - } catch (error) { - callback({ error: { message: (error as Error).message } }); - } - }, - }, - }, - Array, - Error, - JSON, - Map, - Math, - Number, - Object, - Promise, - RegExp, - Set, - String, - }; -} - -async function runInjectedYomitanScript( - script: string, - handler: (action: string, params: unknown) => unknown, -): Promise { - return await vm.runInNewContext(script, createYomitanScriptSandbox(handler)); -} - -// Persistent page context shared across executeJavaScript calls, matching the -// real parser window: the scan runtime is installed once via -// globalThis.__subminerYomitanScan and per-line calls reuse it (and its -// cross-line termsFind cache). -function createPersistentYomitanScriptRunner( - handler: (action: string, params: unknown) => unknown, -): (script: string) => Promise { - const context = vm.createContext(createYomitanScriptSandbox(handler)); - return async (script: string) => await vm.runInContext(script, context); -} - -// Deps whose parser window executes every injected script (profile metadata, -// scan runtime install, per-line scan calls, parseText fallback) inside one -// persistent vm context, dispatching backend actions to `handler`. -function createScanDeps( - handler: (action: string, params: unknown) => unknown, - options?: { onScript?: (script: string) => void }, -) { - const runScript = createPersistentYomitanScriptRunner(handler); - return createDeps(async (script) => { - options?.onScript?.(script); - return await runScript(script); - }); -} - -function countTermsFindLookups(lookups: string[], prefix: string): number { - return lookups.filter((lookupText) => lookupText.startsWith(prefix)).length; -} - -// Backend stub for the greedy name pre-pass: one character name (ミナト) in a -// line of ordinary words, with the SubMiner character dictionary enabled. -const NAME_SCAN_WORDS: Array<[string, string, string, boolean]> = [ - ['ミナト', 'ミナト', 'みなと', true], - ['は', 'は', 'は', false], - ['まだ', 'まだ', 'まだ', false], - ['学校', '学校', 'がっこう', false], - ['に', 'に', 'に', false], - ['いない', 'いる', 'いる', false], -]; - -function createNameScanDeps( - lookups: string[], - words: Array<[string, string, string, boolean]> = NAME_SCAN_WORDS, -) { - return createScanDeps((action, params) => { - if (action === 'optionsGetFull') { - return { - profileCurrent: 0, - profiles: [ - { - options: { - scanning: { length: 40 }, - dictionaries: [ - { name: 'JMdict', enabled: true, id: 0 }, - { - name: 'SubMiner Character Dictionary (AniList 1)', - enabled: true, - id: 1, - }, - ], - }, - }, - ], - }; - } - if (action === 'getDictionaryInfo') { - return []; - } - if (action !== 'termsFind') { - throw new Error(`unexpected action: ${action}`); - } - const text = (params as { text?: string } | undefined)?.text ?? ''; - lookups.push(text); - for (const [surface, term, reading, isName] of words) { - if (text.startsWith(surface)) { - return { - originalTextLength: surface.length, - dictionaryEntries: [ - { - headwords: [ - { - term, - reading, - sources: [{ originalText: surface, isPrimary: true, matchType: 'exact' }], - }, - ], - definitions: [ - { dictionary: isName ? 'SubMiner Character Dictionary (AniList 1)' : 'JMdict' }, - ], - }, - ], - }; - } - } - return { originalTextLength: 0, dictionaryEntries: [] }; - }); -} - -const NAME_SCAN_LINE = 'ミナトはまだ学校にいない'; - -test('requestYomitanScanTokens skips name pre-pass lookups where no candidate name can start', async () => { - const exhaustiveLookups: string[] = []; - const exhaustive = await requestYomitanScanTokens( - NAME_SCAN_LINE, - createNameScanDeps(exhaustiveLookups), - { error: () => undefined }, - { includeNameMatchMetadata: true }, - ); - - const prefilteredLookups: string[] = []; - const prefiltered = await requestYomitanScanTokens( - NAME_SCAN_LINE, - createNameScanDeps(prefilteredLookups), - { error: () => undefined }, - { - includeNameMatchMetadata: true, - currentCharacterDictionaryMediaId: 1, - // Terms and readings the generated dictionary exposes for this media. - nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] }, - }, - ); - - // Same tokenization, including the name match, with fewer round trips. - assert.deepEqual(prefiltered, exhaustive); - assert.equal(prefiltered?.[0]?.surface, 'ミナト'); - assert.equal(prefiltered?.[0]?.isNameMatch, true); - assert.ok( - prefilteredLookups.length < exhaustiveLookups.length, - `expected fewer lookups with candidates (${prefilteredLookups.length} vs ${exhaustiveLookups.length})`, - ); - // Mid-token positions are exactly what the pre-pass used to probe (a name can - // start mid-token); with candidates they cost nothing, while the main walk's - // own token-start lookups are unaffected. - assert.ok(countTermsFindLookups(exhaustiveLookups, '校に') > 0); - assert.equal(countTermsFindLookups(prefilteredLookups, '校に'), 0); -}); - -test('requestYomitanScanTokens matches a katakana name from its kana-normalized candidate form', async () => { - const lookups: string[] = []; - const result = await requestYomitanScanTokens( - NAME_SCAN_LINE, - createNameScanDeps(lookups), - { error: () => undefined }, - { - includeNameMatchMetadata: true, - currentCharacterDictionaryMediaId: 1, - // Only the hiragana reading is listed; the katakana surface in the line - // must still be found through kana normalization. - nameCandidates: { key: 'media-1', forms: ['みなと'] }, - }, - ); - - assert.equal(result?.[0]?.surface, 'ミナト'); - assert.equal(result?.[0]?.isNameMatch, true); -}); - -// Kana normalization does not fold halfwidth katakana, so a name written that -// way can never prefix-match a candidate form; the pre-pass has a bypass for -// those positions, which only runs if they count as Japanese in the first place. -// The generic word here reaches into the name, so only a pre-pass reservation -// can keep the name whole. -const HALFWIDTH_NAME_SCAN_WORDS: Array<[string, string, string, boolean]> = [ - ['まだﾐ', 'まだミ', 'まだみ', false], - ['まだ', 'まだ', 'まだ', false], - ['ﾐﾅﾄ', 'ミナト', 'みなと', true], -]; - -test('requestYomitanScanTokens probes halfwidth katakana positions during the name pre-pass', async () => { - const lookups: string[] = []; - const result = await requestYomitanScanTokens( - 'まだﾐﾅﾄ', - createNameScanDeps(lookups, HALFWIDTH_NAME_SCAN_WORDS), - { error: () => undefined }, - { - includeNameMatchMetadata: true, - currentCharacterDictionaryMediaId: 1, - // Fullwidth forms only, as the generated dictionary stores them. - nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] }, - }, - ); - - assert.equal(countTermsFindLookups(lookups, 'ﾐﾅﾄ'), 1); - assert.deepEqual( - result?.map((token) => token.surface), - ['まだ', 'ﾐﾅﾄ'], - ); - assert.equal(result?.[1]?.isNameMatch, true); - // The reading is written the way the fullwidth katakana path writes it - // (surface spelling, fullwidth): halfwidth kana is not kana to the known-word - // and frequency code downstream, and an empty reading there disables the - // reading fallback entirely. - assert.equal(result?.[1]?.reading, 'ミナト'); - assert.equal(result?.[1]?.headwordReading, 'みなと'); -}); - -test('halfwidth voiced kana compose into the reading instead of leaving a stray mark', async () => { - const lookups: string[] = []; - const result = await requestYomitanScanTokens( - 'ｶﾞｸ ﾊﾟﾝ', - createNameScanDeps(lookups, [ - ['ｶﾞｸ', 'ガク', 'がく', false], - ['ﾊﾟﾝ', 'パン', 'ぱん', false], - ]), - { error: () => undefined }, - { includeNameMatchMetadata: true }, - ); - - const readings = (result ?? []) - .filter((token) => token.isUnparsedRun !== true) - .map((token) => [token.surface, token.reading]); - assert.deepEqual(readings, [ - ['ｶﾞｸ', 'ガク'], - ['ﾊﾟﾝ', 'パン'], - ]); -}); - -test('requestYomitanScanTokens falls back to the exhaustive name scan without candidates', async () => { - const withoutLookups: string[] = []; - const withoutCandidates = await requestYomitanScanTokens( - NAME_SCAN_LINE, - createNameScanDeps(withoutLookups), - { error: () => undefined }, - { includeNameMatchMetadata: true, currentCharacterDictionaryMediaId: 1, nameCandidates: null }, - ); - - assert.equal(withoutCandidates?.[0]?.isNameMatch, true); - // No candidate list means every Japanese position is probed, as before. - assert.ok(countTermsFindLookups(withoutLookups, '校に') > 0); -}); - -test('requestYomitanScanTokens reinstalls name candidates when the media changes', async () => { - const lookups: string[] = []; - const deps = createNameScanDeps(lookups); - - // First media's candidates cannot match this line's name. - const otherMedia = await requestYomitanScanTokens( - NAME_SCAN_LINE, - deps, - { error: () => undefined }, - { - includeNameMatchMetadata: true, - currentCharacterDictionaryMediaId: 2, - nameCandidates: { key: 'media-2', forms: ['カズマ'] }, - }, - ); - assert.equal(otherMedia?.[0]?.isNameMatch, undefined); - - const correctMedia = await requestYomitanScanTokens( - NAME_SCAN_LINE, - deps, - { error: () => undefined }, - { - includeNameMatchMetadata: true, - currentCharacterDictionaryMediaId: 1, - nameCandidates: { key: 'media-1', forms: ['ミナト'] }, - }, - ); - assert.equal(correctMedia?.[0]?.surface, 'ミナト'); - assert.equal(correctMedia?.[0]?.isNameMatch, true); -}); - test('syncYomitanDefaultAnkiServer updates default profile server when script reports update', async () => { let scriptValue = ''; const deps = createDeps(async (script) => { diff --git a/src/core/services/tokenizer/yomitan-scan-runtime-script.ts b/src/core/services/tokenizer/yomitan-scan-runtime-script.ts index e3d12450..64a91a9e 100644 --- a/src/core/services/tokenizer/yomitan-scan-runtime-script.ts +++ b/src/core/services/tokenizer/yomitan-scan-runtime-script.ts @@ -2,7 +2,8 @@ // parser window as globalThis.__subminerYomitanScan, plus the tiny per-line // call script. Kept separate from the host runtime module so the injected // script text (which is data, not executed here) does not dominate that file; -// the helper bundle it embeds lives in yomitan-scanning-helpers-script.ts. +// the helper bundle it embeds is composed in yomitan-scanning-helpers-script.ts +// from the yomitan-*-script.ts fragments. import { YOMITAN_SCANNING_HELPERS } from './yomitan-scanning-helpers-script'; export { CHARACTER_DICTIONARY_TITLE_PREFIX } from './yomitan-scanning-helpers-script'; @@ -11,7 +12,7 @@ export type YomitanFrequencyMode = 'occurrence-based' | 'rank-based'; // Bump whenever the install script below changes so already-loaded parser // windows re-install the new scan runtime instead of running the stale one. -export const YOMITAN_SCAN_RUNTIME_VERSION = 7; +export const YOMITAN_SCAN_RUNTIME_VERSION = 10; export const YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL = '__subminer-yomitan-scan-runtime-missing__'; export interface YomitanScanRequestParams { @@ -307,24 +308,39 @@ ${YOMITAN_SCANNING_HELPERS} } return { token: buildScanToken(position, source, preferredHeadword), matchedLength: originalTextLength }; } - // Halfwidth katakana survives kana normalization unchanged, so a name - // written that way would not prefix-match a candidate form. Those - // positions bypass the prefilter rather than risk a missed name. - function isHalfwidthKatakanaCodePoint(codePoint) { - return codePoint >= 0xff66 && codePoint <= 0xff9f; + // Kana normalization folds halfwidth katakana one code point to one, so an + // unvoiced halfwidth spelling prefix-matches a candidate form like any + // other. What it cannot fold is a voiced pair: ｶ + ﾞ stays two characters + // where the candidate form carries the single が, so the comparison fails + // at that character. That break can sit anywhere inside the name, not + // just at its first character (山ｶﾞｸ starts on a kanji), so the bypass is + // keyed on the region a candidate could cover, not on how it starts. + function isHalfwidthKanaVoicedMarkCodePoint(codePoint) { + return codePoint === 0xff9e || codePoint === 0xff9f; + } + function hasHalfwidthVoicedMarkInCandidateRegion(position, regionLength) { + const end = Math.min(text.length, position + regionLength); + for (let index = position; index < end; index += 1) { + if (isHalfwidthKanaVoicedMarkCodePoint(text.charCodeAt(index))) { return true; } + } + return false; } // Build (once per candidate list) a first-character bucket index of the // normalized name forms, so the pre-pass can reject a position with a // single map hit instead of a backend round trip. if (rawNameCandidates && nameCandidateIndex?.key !== rawNameCandidates.key) { const byFirstChar = new Map(); + let longestFormLength = 0; for (const form of rawNameCandidates.forms) { const normalized = typeof form === "string" ? convertKatakanaToHiragana(form.trim()) : ""; if (!normalized) { continue; } + if (normalized.length > longestFormLength) { longestFormLength = normalized.length; } const bucket = byFirstChar.get(normalized[0]); if (bucket) { bucket.push(normalized); } else { byFirstChar.set(normalized[0], [normalized]); } } - nameCandidateIndex = byFirstChar.size > 0 ? { key: rawNameCandidates.key, byFirstChar } : null; + nameCandidateIndex = byFirstChar.size > 0 + ? { key: rawNameCandidates.key, byFirstChar, longestFormLength } + : null; } else if (!rawNameCandidates) { nameCandidateIndex = null; } @@ -355,15 +371,26 @@ ${YOMITAN_SCANNING_HELPERS} } return true; } + // Doubled because matching may skip emphatic characters as it goes, so a + // form can span more text than it has characters; capped at the window a + // name lookup covers anyway. + const candidateRegionLength = activeNameCandidateIndex + ? Math.min(scanLength, activeNameCandidateIndex.longestFormLength * 2) + : 0; function couldNameStartAt(position, codePoint) { + // Nothing starts with a combining voiced mark, whether or not the + // prefilter is active. + if (isHalfwidthKanaVoicedMarkCodePoint(codePoint)) { return false; } if (!activeNameCandidateIndex) { return true; } - if (isHalfwidthKatakanaCodePoint(codePoint)) { return true; } const bucket = activeNameCandidateIndex.byFirstChar.get(normalizedText[position]); - if (!bucket) { return false; } - for (const form of bucket) { - if (matchesCandidateFormAt(form, position)) { return true; } + if (bucket) { + for (const form of bucket) { + if (matchesCandidateFormAt(form, position)) { return true; } + } } - return false; + // No match, but an unfoldable voiced pair in reach means the comparison + // above could not have seen one: probe rather than drop the name. + return hasHalfwidthVoicedMarkInCandidateRegion(position, candidateRegionLength); } // Greedy name pre-pass: character-name matches claim their spans before // the left-to-right walk, so a longer generic match starting earlier diff --git a/src/core/services/tokenizer/yomitan-scan-runtime.test.ts b/src/core/services/tokenizer/yomitan-scan-runtime.test.ts new file mode 100644 index 00000000..3c61f830 --- /dev/null +++ b/src/core/services/tokenizer/yomitan-scan-runtime.test.ts @@ -0,0 +1,237 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { requestYomitanScanTokens } from './yomitan-parser-runtime'; +import { + countTermsFindLookups, + createNameScanDeps, + NAME_SCAN_WORDS, +} from './yomitan-scan-test-harness'; + +// Behaviour of the in-page scan runtime around character names and kana: +// which positions the greedy pre-pass probes, and what the walk makes of +// halfwidth spellings. Driven end to end through requestYomitanScanTokens +// because the runtime only exists inside the parser window. + +const NAME_SCAN_LINE = 'ミナトはまだ学校にいない'; + +test('requestYomitanScanTokens skips name pre-pass lookups where no candidate name can start', async () => { + const exhaustiveLookups: string[] = []; + const exhaustive = await requestYomitanScanTokens( + NAME_SCAN_LINE, + createNameScanDeps(exhaustiveLookups), + { error: () => undefined }, + { includeNameMatchMetadata: true }, + ); + + const prefilteredLookups: string[] = []; + const prefiltered = await requestYomitanScanTokens( + NAME_SCAN_LINE, + createNameScanDeps(prefilteredLookups), + { error: () => undefined }, + { + includeNameMatchMetadata: true, + currentCharacterDictionaryMediaId: 1, + // Terms and readings the generated dictionary exposes for this media. + nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] }, + }, + ); + + // Same tokenization, including the name match, with fewer round trips. + assert.deepEqual(prefiltered, exhaustive); + assert.equal(prefiltered?.[0]?.surface, 'ミナト'); + assert.equal(prefiltered?.[0]?.isNameMatch, true); + assert.ok( + prefilteredLookups.length < exhaustiveLookups.length, + `expected fewer lookups with candidates (${prefilteredLookups.length} vs ${exhaustiveLookups.length})`, + ); + // Mid-token positions are exactly what the pre-pass used to probe (a name can + // start mid-token); with candidates they cost nothing, while the main walk's + // own token-start lookups are unaffected. + assert.ok(countTermsFindLookups(exhaustiveLookups, '校に') > 0); + assert.equal(countTermsFindLookups(prefilteredLookups, '校に'), 0); +}); + +test('requestYomitanScanTokens matches a katakana name from its kana-normalized candidate form', async () => { + const lookups: string[] = []; + const result = await requestYomitanScanTokens( + NAME_SCAN_LINE, + createNameScanDeps(lookups), + { error: () => undefined }, + { + includeNameMatchMetadata: true, + currentCharacterDictionaryMediaId: 1, + // Only the hiragana reading is listed; the katakana surface in the line + // must still be found through kana normalization. + nameCandidates: { key: 'media-1', forms: ['みなと'] }, + }, + ); + + assert.equal(result?.[0]?.surface, 'ミナト'); + assert.equal(result?.[0]?.isNameMatch, true); +}); + +// Kana normalization folds halfwidth katakana, so a name written that way does +// prefix-match a candidate form — but only if the position counts as Japanese +// in the first place. The generic word here reaches into the name, so only a +// pre-pass reservation can keep the name whole. +const HALFWIDTH_NAME_SCAN_WORDS: Array<[string, string, string, boolean]> = [ + ['ﾈｺ', 'ネコ', 'ねこ', false], + ['まだﾐ', 'まだミ', 'まだみ', false], + ['まだ', 'まだ', 'まだ', false], + ['ﾐﾅﾄ', 'ミナト', 'みなと', true], +]; + +test('requestYomitanScanTokens probes halfwidth katakana positions during the name pre-pass', async () => { + const lookups: string[] = []; + const result = await requestYomitanScanTokens( + 'ﾈｺまだﾐﾅﾄ', + createNameScanDeps(lookups, HALFWIDTH_NAME_SCAN_WORDS), + { error: () => undefined }, + { + includeNameMatchMetadata: true, + currentCharacterDictionaryMediaId: 1, + // Fullwidth forms only, as the generated dictionary stores them. + nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] }, + }, + ); + + assert.equal(countTermsFindLookups(lookups, 'ﾐﾅﾄ'), 1); + // ｺ is mid-token, so only the pre-pass would ever look it up, and it matches + // no candidate: folding halfwidth made those positions indexable, so they no + // longer cost a round trip apiece. + assert.equal(countTermsFindLookups(lookups, 'ｺ'), 0); + assert.deepEqual( + result?.map((token) => token.surface), + ['ﾈｺ', 'まだ', 'ﾐﾅﾄ'], + ); + assert.equal(result?.[2]?.isNameMatch, true); + // The reading is written the way the fullwidth katakana path writes it + // (surface spelling, fullwidth): halfwidth kana is not kana to the known-word + // and frequency code downstream, and an empty reading there disables the + // reading fallback entirely. + assert.equal(result?.[2]?.reading, 'ミナト'); + assert.equal(result?.[2]?.headwordReading, 'みなと'); +}); + +test('a voiced halfwidth name still bypasses the candidate prefilter', async () => { + const lookups: string[] = []; + const result = await requestYomitanScanTokens( + 'まだｶﾞｸ', + createNameScanDeps(lookups, [ + ['まだｶ', 'まだカ', 'まだか', false], + ['まだ', 'まだ', 'まだ', false], + ['ｶﾞｸ', 'ガク', 'がく', true], + ]), + { error: () => undefined }, + { + includeNameMatchMetadata: true, + currentCharacterDictionaryMediaId: 1, + nameCandidates: { key: 'media-1', forms: ['ガク', 'がく'] }, + }, + ); + + // ｶ + ﾞ folds to か + ﾞ, which cannot prefix-match が, so the prefilter would + // drop this position; the voiced-mark bypass is what keeps the name. + assert.deepEqual( + result?.map((token) => token.surface), + ['まだ', 'ｶﾞｸ'], + ); + assert.equal(result?.[1]?.isNameMatch, true); +}); + +test('a mixed-width voiced name survives the candidate prefilter', async () => { + const lookups: string[] = []; + const result = await requestYomitanScanTokens( + 'まだ山ｶﾞｸ', + createNameScanDeps(lookups, [ + ['まだ山', 'まだ山', 'まだやま', false], + ['まだ', 'まだ', 'まだ', false], + ['山ｶﾞｸ', '山ガク', 'やまがく', true], + ]), + { error: () => undefined }, + { + includeNameMatchMetadata: true, + currentCharacterDictionaryMediaId: 1, + nameCandidates: { key: 'media-1', forms: ['山ガク', 'やまがく'] }, + }, + ); + + // The name starts on a kanji, so a bypass keyed on the first character misses + // it: 山ｶﾞｸ normalizes to 山かﾞく, which cannot match the candidate 山がく, and + // the generic word starting earlier then swallows the 山. + assert.deepEqual( + result?.map((token) => token.surface), + ['まだ', '山ｶﾞｸ'], + ); + assert.equal(result?.[1]?.isNameMatch, true); +}); + +test('halfwidth voiced kana compose into the reading instead of leaving a stray mark', async () => { + const lookups: string[] = []; + const result = await requestYomitanScanTokens( + 'ｶﾞｸ ﾊﾟﾝ', + createNameScanDeps(lookups, [ + ['ｶﾞｸ', 'ガク', 'がく', false], + ['ﾊﾟﾝ', 'パン', 'ぱん', false], + ]), + { error: () => undefined }, + { includeNameMatchMetadata: true }, + ); + + // The name pre-pass runs over every position here (no candidate list), but a + // standalone voiced mark can never start a name, so it costs no lookup. + assert.equal(countTermsFindLookups(lookups, 'ﾞ'), 0); + assert.equal(countTermsFindLookups(lookups, 'ﾟ'), 0); + const readings = (result ?? []) + .filter((token) => token.isUnparsedRun !== true) + .map((token) => [token.surface, token.reading]); + assert.deepEqual(readings, [ + ['ｶﾞｸ', 'ガク'], + ['ﾊﾟﾝ', 'パン'], + ]); +}); + +test('requestYomitanScanTokens falls back to the exhaustive name scan without candidates', async () => { + const withoutLookups: string[] = []; + const withoutCandidates = await requestYomitanScanTokens( + NAME_SCAN_LINE, + createNameScanDeps(withoutLookups), + { error: () => undefined }, + { includeNameMatchMetadata: true, currentCharacterDictionaryMediaId: 1, nameCandidates: null }, + ); + + assert.equal(withoutCandidates?.[0]?.isNameMatch, true); + // No candidate list means every Japanese position is probed, as before. + assert.ok(countTermsFindLookups(withoutLookups, '校に') > 0); +}); + +test('requestYomitanScanTokens reinstalls name candidates when the media changes', async () => { + const lookups: string[] = []; + const deps = createNameScanDeps(lookups); + + // First media's candidates cannot match this line's name. + const otherMedia = await requestYomitanScanTokens( + NAME_SCAN_LINE, + deps, + { error: () => undefined }, + { + includeNameMatchMetadata: true, + currentCharacterDictionaryMediaId: 2, + nameCandidates: { key: 'media-2', forms: ['カズマ'] }, + }, + ); + assert.equal(otherMedia?.[0]?.isNameMatch, undefined); + + const correctMedia = await requestYomitanScanTokens( + NAME_SCAN_LINE, + deps, + { error: () => undefined }, + { + includeNameMatchMetadata: true, + currentCharacterDictionaryMediaId: 1, + nameCandidates: { key: 'media-1', forms: ['ミナト'] }, + }, + ); + assert.equal(correctMedia?.[0]?.surface, 'ミナト'); + assert.equal(correctMedia?.[0]?.isNameMatch, true); +}); diff --git a/src/core/services/tokenizer/yomitan-scan-test-harness.ts b/src/core/services/tokenizer/yomitan-scan-test-harness.ts new file mode 100644 index 00000000..01745a96 --- /dev/null +++ b/src/core/services/tokenizer/yomitan-scan-test-harness.ts @@ -0,0 +1,166 @@ +// Shared harness for the Yomitan parser-runtime and scan-runtime tests: fake +// parser-window deps whose injected scripts run in a vm context, plus the +// backend stubs the scanner tests drive them with. Kept out of the test files +// so the runtime tests and the in-page scanner tests can share one setup. +import * as vm from 'node:vm'; + +export function createDeps( + executeJavaScript: (script: string) => Promise, + options?: { + createYomitanExtensionWindow?: (pageName: string) => Promise; + }, +) { + const parserWindow = { + isDestroyed: () => false, + webContents: { + executeJavaScript: async (script: string) => await executeJavaScript(script), + }, + }; + + return { + getYomitanExt: () => ({ id: 'ext-id' }) as never, + getYomitanParserWindow: () => parserWindow as never, + setYomitanParserWindow: () => undefined, + getYomitanParserReadyPromise: () => null, + setYomitanParserReadyPromise: () => undefined, + getYomitanParserInitPromise: () => null, + setYomitanParserInitPromise: () => undefined, + createYomitanExtensionWindow: options?.createYomitanExtensionWindow as never, + }; +} + +function createYomitanScriptSandbox(handler: (action: string, params: unknown) => unknown) { + return { + chrome: { + runtime: { + lastError: null, + sendMessage: ( + payload: { action?: string; params?: unknown }, + callback: (response: { result?: unknown; error?: { message?: string } }) => void, + ) => { + try { + callback({ result: handler(payload.action ?? '', payload.params) }); + } catch (error) { + callback({ error: { message: (error as Error).message } }); + } + }, + }, + }, + Array, + Error, + JSON, + Map, + Math, + Number, + Object, + Promise, + RegExp, + Set, + String, + }; +} + +export async function runInjectedYomitanScript( + script: string, + handler: (action: string, params: unknown) => unknown, +): Promise { + return await vm.runInNewContext(script, createYomitanScriptSandbox(handler)); +} + +// Persistent page context shared across executeJavaScript calls, matching the +// real parser window: the scan runtime is installed once via +// globalThis.__subminerYomitanScan and per-line calls reuse it (and its +// cross-line termsFind cache). +function createPersistentYomitanScriptRunner( + handler: (action: string, params: unknown) => unknown, +): (script: string) => Promise { + const context = vm.createContext(createYomitanScriptSandbox(handler)); + return async (script: string) => await vm.runInContext(script, context); +} + +// Deps whose parser window executes every injected script (profile metadata, +// scan runtime install, per-line scan calls, parseText fallback) inside one +// persistent vm context, dispatching backend actions to `handler`. +export function createScanDeps( + handler: (action: string, params: unknown) => unknown, + options?: { onScript?: (script: string) => void }, +) { + const runScript = createPersistentYomitanScriptRunner(handler); + return createDeps(async (script) => { + options?.onScript?.(script); + return await runScript(script); + }); +} + +export function countTermsFindLookups(lookups: string[], prefix: string): number { + return lookups.filter((lookupText) => lookupText.startsWith(prefix)).length; +} + +// Backend stub for the greedy name pre-pass: one character name (ミナト) in a +// line of ordinary words, with the SubMiner character dictionary enabled. +export const NAME_SCAN_WORDS: Array<[string, string, string, boolean]> = [ + ['ミナト', 'ミナト', 'みなと', true], + ['は', 'は', 'は', false], + ['まだ', 'まだ', 'まだ', false], + ['学校', '学校', 'がっこう', false], + ['に', 'に', 'に', false], + ['いない', 'いる', 'いる', false], +]; + +export function createNameScanDeps( + lookups: string[], + words: Array<[string, string, string, boolean]> = NAME_SCAN_WORDS, +) { + return createScanDeps((action, params) => { + if (action === 'optionsGetFull') { + return { + profileCurrent: 0, + profiles: [ + { + options: { + scanning: { length: 40 }, + dictionaries: [ + { name: 'JMdict', enabled: true, id: 0 }, + { + name: 'SubMiner Character Dictionary (AniList 1)', + enabled: true, + id: 1, + }, + ], + }, + }, + ], + }; + } + if (action === 'getDictionaryInfo') { + return []; + } + if (action !== 'termsFind') { + throw new Error(`unexpected action: ${action}`); + } + const text = (params as { text?: string } | undefined)?.text ?? ''; + lookups.push(text); + for (const [surface, term, reading, isName] of words) { + if (text.startsWith(surface)) { + return { + originalTextLength: surface.length, + dictionaryEntries: [ + { + headwords: [ + { + term, + reading, + sources: [{ originalText: surface, isPrimary: true, matchType: 'exact' }], + }, + ], + definitions: [ + { dictionary: isName ? 'SubMiner Character Dictionary (AniList 1)' : 'JMdict' }, + ], + }, + ], + }; + } + } + return { originalTextLength: 0, dictionaryEntries: [] }; + }); +} diff --git a/src/core/services/tokenizer/yomitan-scanning-helpers-script.ts b/src/core/services/tokenizer/yomitan-scanning-helpers-script.ts index f3c6e8d8..a467d334 100644 --- a/src/core/services/tokenizer/yomitan-scanning-helpers-script.ts +++ b/src/core/services/tokenizer/yomitan-scanning-helpers-script.ts @@ -1,548 +1,21 @@ -// Helper bundle for the in-page Yomitan scan runtime: kana/furigana handling, -// headword preference, and frequency-rank resolution. Injected as text into the -// parser window by yomitan-scan-runtime-script.ts, so it is data here, not code -// this process runs. -import { HAN_CODE_POINT_RANGES } from '../../text/han-code-points'; +// Helper bundle for the in-page Yomitan scan runtime, composed from the +// fragments below. Injected as text into the parser window by +// yomitan-scan-runtime-script.ts, so it is data here, not code this process +// runs. The fragments are concatenated into a single function body and share +// one lexical scope: every function in them is hoisted, but the constants are +// not, so kana stays first — the later fragments read its ranges as they run. +import { YOMITAN_DICTIONARY_CLASSIFICATION_HELPERS } from './yomitan-dictionary-classification-script'; +import { YOMITAN_FREQUENCY_HELPERS } from './yomitan-frequency-script'; +import { YOMITAN_FURIGANA_HELPERS } from './yomitan-furigana-script'; +import { YOMITAN_KANA_HELPERS } from './yomitan-kana-script'; +import { YOMITAN_MATCH_SELECTION_HELPERS } from './yomitan-match-selection-script'; -export const CHARACTER_DICTIONARY_TITLE_PREFIX = 'SubMiner Character Dictionary'; +export { CHARACTER_DICTIONARY_TITLE_PREFIX } from './character-dictionary-title'; -export const YOMITAN_SCANNING_HELPERS = String.raw` - const HIRAGANA_CONVERSION_RANGE = [0x3041, 0x3096]; - const KATAKANA_CONVERSION_RANGE = [0x30a1, 0x30f6]; - const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc; - const KATAKANA_SMALL_KA_CODE_POINT = 0x30f5; - const KATAKANA_SMALL_KE_CODE_POINT = 0x30f6; - const KANA_RANGES = [[0x3040, 0x309f], [0x30a0, 0x30ff], [0xff66, 0xff9f]]; - const HALFWIDTH_KATAKANA_RANGE = [0xff66, 0xff9d]; - const HALFWIDTH_KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0xff70; - // Folded one code point to one, so every index into a normalized string - // still lines up with the original text — the name-candidate prefilter - // and the furigana stem matching both index back into it. The standalone - // voiced marks (ﾞ ﾟ) have no one-character equivalent and stay as they are. - const HALFWIDTH_KATAKANA_TO_HIRAGANA = "をぁぃぅぇぉゃゅょっーあいうえおかきくけこさしすせそたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわん"; - function convertHalfwidthKanaCodePointToHiragana(codePoint) { - if (codePoint < HALFWIDTH_KATAKANA_RANGE[0] || codePoint > HALFWIDTH_KATAKANA_RANGE[1]) { return null; } - return HALFWIDTH_KATAKANA_TO_HIRAGANA[codePoint - HALFWIDTH_KATAKANA_RANGE[0]] || null; - } - // Halfwidth katakana is kana here but not to the rest of the pipeline - // (known-word matching and frequency lookups only fold fullwidth), so a - // reading taken from halfwidth text is written the way the fullwidth - // katakana path already writes it. NFKC rather than the per-code-point - // table: this is the one place where nothing indexes back into the - // result, so a voiced pair (ｶ + ﾞ) can compose into the single ガ it - // means instead of leaving a stray combining mark in the reading. Scoped - // to the halfwidth runs, because NFKC over everything else rewrites - // characters that have nothing to do with kana (① → 1, ㍑ → リットル). - function convertHalfwidthKanaToKatakana(text) { - return text.replace(/[ｦ-ﾟ]+/g, (run) => run.normalize("NFKC")); - } - // Han ranges come from the shared table so the scan walk and the character - // dictionary agree on what a kanji is (supplementary planes included). - // Halfwidth katakana counts as Japanese text: a name written that way has - // to reach the greedy pre-pass, which has its own handling for it. - const JAPANESE_RANGES = [[0x3040, 0x30ff], [0xff66, 0xff9f], ...${JSON.stringify(HAN_CODE_POINT_RANGES)}]; - function isCodePointInRange(codePoint, range) { return codePoint >= range[0] && codePoint <= range[1]; } - function isCodePointInRanges(codePoint, ranges) { return ranges.some((range) => isCodePointInRange(codePoint, range)); } - function isCodePointKana(codePoint) { return isCodePointInRanges(codePoint, KANA_RANGES); } - function isCodePointJapanese(codePoint) { return isCodePointInRanges(codePoint, JAPANESE_RANGES); } - function createFuriganaSegment(text, reading) { return {text, reading}; } - function getSegmentReadingContribution(segment) { - if (typeof segment.reading === "string" && segment.reading.length > 0) { return segment.reading; } - const segmentText = typeof segment.text === "string" ? segment.text : ""; - const isKanaOnly = segmentText.length > 0 && [...segmentText].every((char) => isCodePointKana(char.codePointAt(0))); - return isKanaOnly ? convertHalfwidthKanaToKatakana(segmentText) : ""; - } - function getProlongedHiragana(previousCharacter) { - switch (previousCharacter) { - case "あ": case "か": case "が": case "さ": case "ざ": case "た": case "だ": case "な": case "は": case "ば": case "ぱ": case "ま": case "や": case "ら": case "わ": case "ぁ": case "ゃ": case "ゎ": return "あ"; - case "い": case "き": case "ぎ": case "し": case "じ": case "ち": case "ぢ": case "に": case "ひ": case "び": case "ぴ": case "み": case "り": case "ぃ": return "い"; - case "う": case "く": case "ぐ": case "す": case "ず": case "つ": case "づ": case "ぬ": case "ふ": case "ぶ": case "ぷ": case "む": case "ゆ": case "る": case "ぅ": case "ゅ": return "う"; - case "え": case "け": case "げ": case "せ": case "ぜ": case "て": case "で": case "ね": case "へ": case "べ": case "ぺ": case "め": case "れ": case "ぇ": return "え"; - case "お": case "こ": case "ご": case "そ": case "ぞ": case "と": case "ど": case "の": case "ほ": case "ぼ": case "ぽ": case "も": case "よ": case "ろ": case "を": case "ぉ": case "ょ": return "う"; - default: return null; - } - } - function getFuriganaKanaSegments(text, reading) { - const newSegments = []; - let start = 0; - let state = (reading[0] === text[0]); - for (let i = 1; i < text.length; ++i) { - const newState = (reading[i] === text[i]); - if (state === newState) { continue; } - newSegments.push(createFuriganaSegment(text.substring(start, i), state ? '' : reading.substring(start, i))); - state = newState; - start = i; - } - newSegments.push(createFuriganaSegment(text.substring(start), state ? '' : reading.substring(start))); - return newSegments; - } - function convertKatakanaToHiragana(text, keepProlongedSoundMarks = false) { - let result = ''; - const offset = (HIRAGANA_CONVERSION_RANGE[0] - KATAKANA_CONVERSION_RANGE[0]); - for (let char of text) { - const codePoint = char.codePointAt(0); - switch (codePoint) { - case KATAKANA_SMALL_KA_CODE_POINT: - case KATAKANA_SMALL_KE_CODE_POINT: - break; - case KANA_PROLONGED_SOUND_MARK_CODE_POINT: - case HALFWIDTH_KANA_PROLONGED_SOUND_MARK_CODE_POINT: - char = "ー"; - if (!keepProlongedSoundMarks && result.length > 0) { - const char2 = getProlongedHiragana(result[result.length - 1]); - if (char2 !== null) { char = char2; } - } - break; - default: - if (isCodePointInRange(codePoint, KATAKANA_CONVERSION_RANGE)) { - char = String.fromCodePoint(codePoint + offset); - break; - } - // Halfwidth katakana folds too, or a name written that way would - // match neither a candidate form nor its own reading. - const halfwidthHiragana = convertHalfwidthKanaCodePointToHiragana(codePoint); - if (halfwidthHiragana !== null) { char = halfwidthHiragana; } - break; - } - result += char; - } - return result; - } - function segmentizeFurigana(reading, readingNormalized, groups, groupsStart) { - const groupCount = groups.length - groupsStart; - if (groupCount <= 0) { return reading.length === 0 ? [] : null; } - const group = groups[groupsStart]; - const {isKana, text} = group; - if (isKana) { - if (group.textNormalized !== null && readingNormalized.startsWith(group.textNormalized)) { - const segments = segmentizeFurigana(reading.substring(text.length), readingNormalized.substring(text.length), groups, groupsStart + 1); - if (segments !== null) { - if (reading.startsWith(text)) { segments.unshift(createFuriganaSegment(text, '')); } - else { segments.unshift(...getFuriganaKanaSegments(text, reading)); } - return segments; - } - } - return null; - } - let result = null; - for (let i = reading.length; i >= text.length; --i) { - const segments = segmentizeFurigana(reading.substring(i), readingNormalized.substring(i), groups, groupsStart + 1); - if (segments !== null) { - if (result !== null) { return null; } - segments.unshift(createFuriganaSegment(text, reading.substring(0, i))); - result = segments; - } - if (groupCount === 1) { break; } - } - return result; - } - function distributeFurigana(term, reading) { - if (reading === term) { return [createFuriganaSegment(term, '')]; } - const groups = []; - let groupPre = null; - let isKanaPre = null; - for (const c of term) { - const isKana = isCodePointKana(c.codePointAt(0)); - if (isKana === isKanaPre) { groupPre.text += c; } - else { - groupPre = {isKana, text: c, textNormalized: null}; - groups.push(groupPre); - isKanaPre = isKana; - } - } - for (const group of groups) { - if (group.isKana) { group.textNormalized = convertKatakanaToHiragana(group.text); } - } - const segments = segmentizeFurigana(reading, convertKatakanaToHiragana(reading), groups, 0); - return segments !== null ? segments : [createFuriganaSegment(term, reading)]; - } - function getStemLength(text1, text2) { - const minLength = Math.min(text1.length, text2.length); - if (minLength === 0) { return 0; } - let i = 0; - while (true) { - const char1 = text1.codePointAt(i); - const char2 = text2.codePointAt(i); - if (char1 !== char2) { break; } - const charLength = String.fromCodePoint(char1).length; - i += charLength; - if (i >= minLength) { - if (i > minLength) { i -= charLength; } - break; - } - } - return i; - } - function distributeFuriganaInflected(term, reading, source) { - const termNormalized = convertKatakanaToHiragana(term); - const readingNormalized = convertKatakanaToHiragana(reading); - const sourceNormalized = convertKatakanaToHiragana(source); - let mainText = term; - let stemLength = getStemLength(termNormalized, sourceNormalized); - const readingStemLength = getStemLength(readingNormalized, sourceNormalized); - if (readingStemLength > 0 && readingStemLength >= stemLength) { - mainText = reading; - stemLength = readingStemLength; - reading = source.substring(0, stemLength) + reading.substring(stemLength); - } - const segments = []; - if (stemLength > 0) { - mainText = source.substring(0, stemLength) + mainText.substring(stemLength); - const segments2 = distributeFurigana(mainText, reading); - let consumed = 0; - for (const segment of segments2) { - const start = consumed; - consumed += segment.text.length; - if (consumed < stemLength) { segments.push(segment); } - else if (consumed === stemLength) { segments.push(segment); break; } - else { - if (start < stemLength) { segments.push(createFuriganaSegment(mainText.substring(start, stemLength), '')); } - break; - } - } - } - if (stemLength < source.length) { - const remainder = source.substring(stemLength); - const last = segments[segments.length - 1]; - if (last && last.reading.length === 0) { last.text += remainder; } - else { segments.push(createFuriganaSegment(remainder, '')); } - } - return segments; - } - function parsePositiveFrequencyNumber(value) { - if (typeof value === 'number' && Number.isFinite(value) && value > 0) { - return Math.max(1, Math.floor(value)); - } - if (typeof value === 'string') { - const numericMatch = value.trim().match(/[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?/)?.[0]; - if (!numericMatch) { return null; } - const parsed = Number.parseFloat(numericMatch); - if (!Number.isFinite(parsed) || parsed <= 0) { return null; } - return Math.max(1, Math.floor(parsed)); - } - if (Array.isArray(value)) { - for (const item of value) { - const parsed = parsePositiveFrequencyNumber(item); - if (parsed !== null) { return parsed; } - } - } - return null; - } - function parseDisplayFrequencyNumber(value) { - if (typeof value === 'string') { - const leadingDigits = value.trim().match(/^\d+/)?.[0]; - if (!leadingDigits) { return null; } - const parsed = Number.parseInt(leadingDigits, 10); - return Number.isFinite(parsed) && parsed > 0 ? parsed : null; - } - return parsePositiveFrequencyNumber(value); - } - function getFrequencyDictionaryName(frequency) { - const candidates = [ - frequency?.dictionary, - frequency?.dictionaryName, - frequency?.name, - frequency?.title, - frequency?.dictionaryTitle, - frequency?.dictionaryAlias - ]; - for (const candidate of candidates) { - if (typeof candidate === 'string' && candidate.trim().length > 0) { - return candidate.trim(); - } - } - return null; - } - function getBestFrequencyRank(dictionaryEntry, headwordIndex, dictionaryPriorityByName, dictionaryFrequencyModeByName) { - let best = null; - const headwordCount = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords.length : 0; - for (const frequency of dictionaryEntry?.frequencies || []) { - if (!frequency || typeof frequency !== 'object') { continue; } - const frequencyHeadwordIndex = frequency.headwordIndex; - if (typeof frequencyHeadwordIndex === 'number') { - if (frequencyHeadwordIndex !== headwordIndex) { continue; } - } else if (headwordCount > 1) { - continue; - } - const dictionary = getFrequencyDictionaryName(frequency); - if (!dictionary) { continue; } - if (dictionaryFrequencyModeByName[dictionary] === 'occurrence-based') { continue; } - const rank = - parseDisplayFrequencyNumber(frequency.displayValue) ?? - parsePositiveFrequencyNumber(frequency.frequency); - if (rank === null) { continue; } - const priorityRaw = dictionaryPriorityByName[dictionary]; - const fallbackPriority = - typeof frequency.dictionaryIndex === 'number' && Number.isFinite(frequency.dictionaryIndex) - ? Math.max(0, Math.floor(frequency.dictionaryIndex)) - : Number.MAX_SAFE_INTEGER; - const priority = - typeof priorityRaw === 'number' && Number.isFinite(priorityRaw) - ? Math.max(0, Math.floor(priorityRaw)) - : fallbackPriority; - if (best === null || priority < best.priority || (priority === best.priority && rank < best.rank)) { - best = { priority, rank }; - } - } - return best?.rank ?? null; - } - function hasExactSource(headword, token, requirePrimary) { - for (const src of headword.sources || []) { - if (src.originalText !== token) { continue; } - if (requirePrimary && !src.isPrimary) { continue; } - if (src.matchType !== 'exact') { continue; } - return true; - } - return false; - } - function collectExactHeadwordMatches(dictionaryEntries, token, requirePrimary) { - const matches = []; - for (const dictionaryEntry of dictionaryEntries || []) { - const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : []; - for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) { - const headword = headwords[headwordIndex]; - if (!hasExactSource(headword, token, requirePrimary)) { continue; } - matches.push({ dictionaryEntry, headword, headwordIndex }); - } - } - return matches; - } - function sameHeadword(match, preferredMatch) { - if (!match || !preferredMatch) { - return false; - } - if (match.headword?.term !== preferredMatch.headword?.term) { - return false; - } - const matchReading = typeof match.headword?.reading === 'string' ? match.headword.reading : ''; - const preferredReading = - typeof preferredMatch.headword?.reading === 'string' ? preferredMatch.headword.reading : ''; - if (!matchReading || !preferredReading) { - return true; - } - return matchReading === preferredReading; - } - function getBestFrequencyRankForMatches(matches, dictionaryPriorityByName, dictionaryFrequencyModeByName) { - let best = null; - for (const match of matches) { - const rank = getBestFrequencyRank( - match.dictionaryEntry, - match.headwordIndex, - dictionaryPriorityByName, - dictionaryFrequencyModeByName - ); - if (rank === null) { continue; } - if (best === null || rank < best) { - best = rank; - } - } - return best; - } - function normalizeWordClasses(headword) { - if (!Array.isArray(headword?.wordClasses)) { return undefined; } - const classes = headword.wordClasses.filter((wordClass) => typeof wordClass === "string" && wordClass.trim().length > 0); - return classes.length > 0 ? classes : undefined; - } - function appendDictionaryNames(target, value) { - if (!value || typeof value !== 'object') { - return; - } - const candidates = [ - value.dictionary, - value.dictionaryName, - value.name, - value.title, - value.dictionaryTitle, - value.dictionaryAlias - ]; - for (const candidate of candidates) { - if (typeof candidate === 'string' && candidate.trim().length > 0) { - target.push(candidate.trim()); - } - } - } - // Memoized on the entry object: termsFind results are cached across - // lines, so the same entries come back for every repeated lookup, and - // each one is classified several times per scan (name pre-pass, - // headword preference, every retry window). - function getDictionaryEntryNames(entry) { - if (!entry || typeof entry !== 'object') { return []; } - const cached = dictionaryEntryNamesCache.get(entry); - if (cached !== undefined) { return cached; } - const names = []; - appendDictionaryNames(names, entry); - for (const definition of entry?.definitions || []) { - appendDictionaryNames(names, definition); - } - for (const frequency of entry?.frequencies || []) { - appendDictionaryNames(names, frequency); - } - for (const pronunciation of entry?.pronunciations || []) { - appendDictionaryNames(names, pronunciation); - } - dictionaryEntryNamesCache.set(entry, names); - return names; - } - // Cached per scan rather than per runtime: the answer depends on - // includeNameMatchMetadata, which is a per-call parameter. - const nameDictionaryEntryCache = new WeakMap(); - function isNameDictionaryEntry(entry) { - if (!includeNameMatchMetadata || !entry || typeof entry !== 'object') { - return false; - } - const cached = nameDictionaryEntryCache.get(entry); - if (cached !== undefined) { return cached; } - const isName = getDictionaryEntryNames(entry).some((name) => name.startsWith(${JSON.stringify(CHARACTER_DICTIONARY_TITLE_PREFIX)})); - nameDictionaryEntryCache.set(entry, isName); - return isName; - } - function parseSubMinerMediaIdFromString(value) { - const imageMatch = value.match(/\bimg\/m(\d+)-/i); - if (imageMatch) { - const parsed = Number.parseInt(imageMatch[1], 10); - if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; } - } - const titleMatch = value.match(/${CHARACTER_DICTIONARY_TITLE_PREFIX}[^\d]*(?:AniList\s*)?(\d+)/i); - if (titleMatch) { - const parsed = Number.parseInt(titleMatch[1], 10); - if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; } - } - return null; - } - function parseSubMinerMediaIdCandidate(value) { - if (typeof value === 'number' && Number.isSafeInteger(value) && value > 0) { - return value; - } - if (typeof value === 'string' && /^\d+$/.test(value.trim())) { - const parsed = Number.parseInt(value.trim(), 10); - if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; } - } - return null; - } - function collectSubMinerMediaIds(value, target) { - if (typeof value === 'string') { - const parsed = parseSubMinerMediaIdFromString(value); - if (parsed !== null) { target.add(parsed); } - return; - } - if (!value || typeof value !== 'object') { - return; - } - if (Array.isArray(value)) { - for (const item of value) { collectSubMinerMediaIds(item, target); } - return; - } - const mediaIdCandidates = [ - value.subminerMediaId, - value.subMinerMediaId, - value.characterDictionaryMediaId, - value.data?.subminerMediaId, - value.data?.subMinerMediaId, - value.data?.characterDictionaryMediaId - ]; - for (const candidate of mediaIdCandidates) { - const parsed = parseSubMinerMediaIdCandidate(candidate); - if (parsed !== null) { target.add(parsed); } - } - for (const child of Object.values(value)) { - collectSubMinerMediaIds(child, target); - } - } - // Walking an entry collects media ids from every nested value, so this - // is the most expensive classification step; memoized on the entry for - // the same reason as the dictionary names above. - function getSubMinerMediaIds(entry) { - if (!entry || typeof entry !== 'object') { return EMPTY_MEDIA_ID_SET; } - const cached = subMinerMediaIdsCache.get(entry); - if (cached !== undefined) { return cached; } - const mediaIds = new Set(); - collectSubMinerMediaIds(entry, mediaIds); - subMinerMediaIdsCache.set(entry, mediaIds); - return mediaIds; - } - function isCurrentMediaNameDictionaryEntry(entry) { - if (!isNameDictionaryEntry(entry)) { - return false; - } - if (currentCharacterDictionaryMediaId === null) { - return true; - } - const mediaIds = getSubMinerMediaIds(entry); - return mediaIds.size === 0 || mediaIds.has(currentCharacterDictionaryMediaId); - } - function findLongestNameMatch(dictionaryEntries, textWindow) { - let best = null; - for (const dictionaryEntry of dictionaryEntries || []) { - if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; } - const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : []; - for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) { - const headword = headwords[headwordIndex]; - for (const src of headword?.sources || []) { - if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; } - const originalText = typeof src.originalText === 'string' ? src.originalText : ''; - if (!originalText || !textWindow.startsWith(originalText)) { continue; } - if (best === null || originalText.length > best.sourceLength) { - best = { dictionaryEntry, headword, headwordIndex, sourceLength: originalText.length }; - } - } - } - } - return best; - } - function findLongestGenericMatchLength(dictionaryEntries, textWindow) { - let best = 0; - for (const dictionaryEntry of dictionaryEntries || []) { - if (isNameDictionaryEntry(dictionaryEntry)) { continue; } - const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : []; - for (const headword of headwords) { - for (const src of headword?.sources || []) { - if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; } - const originalText = typeof src.originalText === 'string' ? src.originalText : ''; - if (!originalText || !textWindow.startsWith(originalText)) { continue; } - if (originalText.length > best) { best = originalText.length; } - } - } - } - return best; - } - function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) { - const currentMediaDictionaryEntries = - currentCharacterDictionaryMediaId === null - ? (dictionaryEntries || []) - : (dictionaryEntries || []).filter((entry) => { - if (!isNameDictionaryEntry(entry)) { return true; } - return isCurrentMediaNameDictionaryEntry(entry); - }); - const exactPrimaryMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, true); - let matchedNameDictionary = false; - if (includeNameMatchMetadata) { - for (const dictionaryEntry of currentMediaDictionaryEntries || []) { - if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; } - for (const match of exactPrimaryMatches) { - if (match.dictionaryEntry !== dictionaryEntry) { continue; } - matchedNameDictionary = true; - break; - } - if (matchedNameDictionary) { break; } - } - } - const preferredMatch = exactPrimaryMatches[0]; - if (preferredMatch) { - const exactFrequencyMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, false) - .filter((match) => sameHeadword(match, preferredMatch)); - return { - term: preferredMatch.headword.term, - reading: preferredMatch.headword.reading, - wordClasses: normalizeWordClasses(preferredMatch.headword), - isNameMatch: - matchedNameDictionary || isCurrentMediaNameDictionaryEntry(preferredMatch.dictionaryEntry), - frequencyRank: getBestFrequencyRankForMatches( - exactFrequencyMatches.length > 0 ? exactFrequencyMatches : exactPrimaryMatches, - dictionaryPriorityByName, - dictionaryFrequencyModeByName - ) - }; - } - return null; - } -`; +export const YOMITAN_SCANNING_HELPERS = [ + YOMITAN_KANA_HELPERS, + YOMITAN_FURIGANA_HELPERS, + YOMITAN_FREQUENCY_HELPERS, + YOMITAN_DICTIONARY_CLASSIFICATION_HELPERS, + YOMITAN_MATCH_SELECTION_HELPERS, +].join('\n');