mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-04 19:21:33 -07:00
sudacode
b0a2ce6e8a
The greedy name pre-pass asked the Yomitan backend at every Japanese position, because a character name can begin mid-token. With the character dictionary enabled that roughly doubled the round trips per line (measured 10 -> 21 on a 23-char line). SubMiner generates the character dictionary, so the cached snapshots already list every form a character entry can be matched by (term and reading). Those forms are installed into the scan runtime once per media and the pre-pass now probes only positions where one of them starts, compared after kana normalization so a katakana name still matches a hiragana reading form. The overhead drops to zero (21 -> 10, the same as with the dictionary disabled). Fail-safe: with no candidate list (no media id, no cached snapshot, failed install) the pre-pass keeps its exhaustive behavior, so stale character data costs speed rather than a missing name. Halfwidth katakana positions bypass the filter since kana normalization does not fold them. The candidate lookup is consulted per subtitle line, so it caches its snapshot directory signature for 5s; dictionary writes still call invalidate().
889 lines
40 KiB
TypeScript
889 lines
40 KiB
TypeScript
// In-page Yomitan scan runtime: the helper bundle and scan walk that get
|
||
// installed once per parser window as globalThis.__subminerYomitanScan, plus
|
||
// the tiny per-line call script. Kept separate from the host runtime module so
|
||
// the injected-script text (which is data, not executed here) does not
|
||
// dominate that file.
|
||
|
||
export type YomitanFrequencyMode = 'occurrence-based' | 'rank-based';
|
||
|
||
export const CHARACTER_DICTIONARY_TITLE_PREFIX = 'SubMiner Character Dictionary';
|
||
|
||
const YOMITAN_SCANNING_HELPERS = String.raw`
|
||
const HIRAGANA_CONVERSION_RANGE = [0x3041, 0x3096];
|
||
const KATAKANA_CONVERSION_RANGE = [0x30a1, 0x30f6];
|
||
const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc;
|
||
const KATAKANA_SMALL_KA_CODE_POINT = 0x30f5;
|
||
const KATAKANA_SMALL_KE_CODE_POINT = 0x30f6;
|
||
const KANA_RANGES = [[0x3040, 0x309f], [0x30a0, 0x30ff]];
|
||
const JAPANESE_RANGES = [[0x3040, 0x30ff], [0x3400, 0x9fff]];
|
||
function isCodePointInRange(codePoint, range) { return codePoint >= range[0] && codePoint <= range[1]; }
|
||
function isCodePointInRanges(codePoint, ranges) { return ranges.some((range) => isCodePointInRange(codePoint, range)); }
|
||
function isCodePointKana(codePoint) { return isCodePointInRanges(codePoint, KANA_RANGES); }
|
||
function isCodePointJapanese(codePoint) { return isCodePointInRanges(codePoint, JAPANESE_RANGES); }
|
||
function createFuriganaSegment(text, reading) { return {text, reading}; }
|
||
function getSegmentReadingContribution(segment) {
|
||
if (typeof segment.reading === "string" && segment.reading.length > 0) { return segment.reading; }
|
||
const segmentText = typeof segment.text === "string" ? segment.text : "";
|
||
const isKanaOnly = segmentText.length > 0 && [...segmentText].every((char) => isCodePointKana(char.codePointAt(0)));
|
||
return isKanaOnly ? segmentText : "";
|
||
}
|
||
function getProlongedHiragana(previousCharacter) {
|
||
switch (previousCharacter) {
|
||
case "あ": case "か": case "が": case "さ": case "ざ": case "た": case "だ": case "な": case "は": case "ば": case "ぱ": case "ま": case "や": case "ら": case "わ": case "ぁ": case "ゃ": case "ゎ": return "あ";
|
||
case "い": case "き": case "ぎ": case "し": case "じ": case "ち": case "ぢ": case "に": case "ひ": case "び": case "ぴ": case "み": case "り": case "ぃ": return "い";
|
||
case "う": case "く": case "ぐ": case "す": case "ず": case "つ": case "づ": case "ぬ": case "ふ": case "ぶ": case "ぷ": case "む": case "ゆ": case "る": case "ぅ": case "ゅ": return "う";
|
||
case "え": case "け": case "げ": case "せ": case "ぜ": case "て": case "で": case "ね": case "へ": case "べ": case "ぺ": case "め": case "れ": case "ぇ": return "え";
|
||
case "お": case "こ": case "ご": case "そ": case "ぞ": case "と": case "ど": case "の": case "ほ": case "ぼ": case "ぽ": case "も": case "よ": case "ろ": case "を": case "ぉ": case "ょ": return "う";
|
||
default: return null;
|
||
}
|
||
}
|
||
function getFuriganaKanaSegments(text, reading) {
|
||
const newSegments = [];
|
||
let start = 0;
|
||
let state = (reading[0] === text[0]);
|
||
for (let i = 1; i < text.length; ++i) {
|
||
const newState = (reading[i] === text[i]);
|
||
if (state === newState) { continue; }
|
||
newSegments.push(createFuriganaSegment(text.substring(start, i), state ? '' : reading.substring(start, i)));
|
||
state = newState;
|
||
start = i;
|
||
}
|
||
newSegments.push(createFuriganaSegment(text.substring(start), state ? '' : reading.substring(start)));
|
||
return newSegments;
|
||
}
|
||
function convertKatakanaToHiragana(text, keepProlongedSoundMarks = false) {
|
||
let result = '';
|
||
const offset = (HIRAGANA_CONVERSION_RANGE[0] - KATAKANA_CONVERSION_RANGE[0]);
|
||
for (let char of text) {
|
||
const codePoint = char.codePointAt(0);
|
||
switch (codePoint) {
|
||
case KATAKANA_SMALL_KA_CODE_POINT:
|
||
case KATAKANA_SMALL_KE_CODE_POINT:
|
||
break;
|
||
case KANA_PROLONGED_SOUND_MARK_CODE_POINT:
|
||
if (!keepProlongedSoundMarks && result.length > 0) {
|
||
const char2 = getProlongedHiragana(result[result.length - 1]);
|
||
if (char2 !== null) { char = char2; }
|
||
}
|
||
break;
|
||
default:
|
||
if (isCodePointInRange(codePoint, KATAKANA_CONVERSION_RANGE)) {
|
||
char = String.fromCodePoint(codePoint + offset);
|
||
}
|
||
break;
|
||
}
|
||
result += char;
|
||
}
|
||
return result;
|
||
}
|
||
function segmentizeFurigana(reading, readingNormalized, groups, groupsStart) {
|
||
const groupCount = groups.length - groupsStart;
|
||
if (groupCount <= 0) { return reading.length === 0 ? [] : null; }
|
||
const group = groups[groupsStart];
|
||
const {isKana, text} = group;
|
||
if (isKana) {
|
||
if (group.textNormalized !== null && readingNormalized.startsWith(group.textNormalized)) {
|
||
const segments = segmentizeFurigana(reading.substring(text.length), readingNormalized.substring(text.length), groups, groupsStart + 1);
|
||
if (segments !== null) {
|
||
if (reading.startsWith(text)) { segments.unshift(createFuriganaSegment(text, '')); }
|
||
else { segments.unshift(...getFuriganaKanaSegments(text, reading)); }
|
||
return segments;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
let result = null;
|
||
for (let i = reading.length; i >= text.length; --i) {
|
||
const segments = segmentizeFurigana(reading.substring(i), readingNormalized.substring(i), groups, groupsStart + 1);
|
||
if (segments !== null) {
|
||
if (result !== null) { return null; }
|
||
segments.unshift(createFuriganaSegment(text, reading.substring(0, i)));
|
||
result = segments;
|
||
}
|
||
if (groupCount === 1) { break; }
|
||
}
|
||
return result;
|
||
}
|
||
function distributeFurigana(term, reading) {
|
||
if (reading === term) { return [createFuriganaSegment(term, '')]; }
|
||
const groups = [];
|
||
let groupPre = null;
|
||
let isKanaPre = null;
|
||
for (const c of term) {
|
||
const isKana = isCodePointKana(c.codePointAt(0));
|
||
if (isKana === isKanaPre) { groupPre.text += c; }
|
||
else {
|
||
groupPre = {isKana, text: c, textNormalized: null};
|
||
groups.push(groupPre);
|
||
isKanaPre = isKana;
|
||
}
|
||
}
|
||
for (const group of groups) {
|
||
if (group.isKana) { group.textNormalized = convertKatakanaToHiragana(group.text); }
|
||
}
|
||
const segments = segmentizeFurigana(reading, convertKatakanaToHiragana(reading), groups, 0);
|
||
return segments !== null ? segments : [createFuriganaSegment(term, reading)];
|
||
}
|
||
function getStemLength(text1, text2) {
|
||
const minLength = Math.min(text1.length, text2.length);
|
||
if (minLength === 0) { return 0; }
|
||
let i = 0;
|
||
while (true) {
|
||
const char1 = text1.codePointAt(i);
|
||
const char2 = text2.codePointAt(i);
|
||
if (char1 !== char2) { break; }
|
||
const charLength = String.fromCodePoint(char1).length;
|
||
i += charLength;
|
||
if (i >= minLength) {
|
||
if (i > minLength) { i -= charLength; }
|
||
break;
|
||
}
|
||
}
|
||
return i;
|
||
}
|
||
function distributeFuriganaInflected(term, reading, source) {
|
||
const termNormalized = convertKatakanaToHiragana(term);
|
||
const readingNormalized = convertKatakanaToHiragana(reading);
|
||
const sourceNormalized = convertKatakanaToHiragana(source);
|
||
let mainText = term;
|
||
let stemLength = getStemLength(termNormalized, sourceNormalized);
|
||
const readingStemLength = getStemLength(readingNormalized, sourceNormalized);
|
||
if (readingStemLength > 0 && readingStemLength >= stemLength) {
|
||
mainText = reading;
|
||
stemLength = readingStemLength;
|
||
reading = source.substring(0, stemLength) + reading.substring(stemLength);
|
||
}
|
||
const segments = [];
|
||
if (stemLength > 0) {
|
||
mainText = source.substring(0, stemLength) + mainText.substring(stemLength);
|
||
const segments2 = distributeFurigana(mainText, reading);
|
||
let consumed = 0;
|
||
for (const segment of segments2) {
|
||
const start = consumed;
|
||
consumed += segment.text.length;
|
||
if (consumed < stemLength) { segments.push(segment); }
|
||
else if (consumed === stemLength) { segments.push(segment); break; }
|
||
else {
|
||
if (start < stemLength) { segments.push(createFuriganaSegment(mainText.substring(start, stemLength), '')); }
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
if (stemLength < source.length) {
|
||
const remainder = source.substring(stemLength);
|
||
const last = segments[segments.length - 1];
|
||
if (last && last.reading.length === 0) { last.text += remainder; }
|
||
else { segments.push(createFuriganaSegment(remainder, '')); }
|
||
}
|
||
return segments;
|
||
}
|
||
function parsePositiveFrequencyNumber(value) {
|
||
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
|
||
return Math.max(1, Math.floor(value));
|
||
}
|
||
if (typeof value === 'string') {
|
||
const numericMatch = value.trim().match(/[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?/)?.[0];
|
||
if (!numericMatch) { return null; }
|
||
const parsed = Number.parseFloat(numericMatch);
|
||
if (!Number.isFinite(parsed) || parsed <= 0) { return null; }
|
||
return Math.max(1, Math.floor(parsed));
|
||
}
|
||
if (Array.isArray(value)) {
|
||
for (const item of value) {
|
||
const parsed = parsePositiveFrequencyNumber(item);
|
||
if (parsed !== null) { return parsed; }
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
function parseDisplayFrequencyNumber(value) {
|
||
if (typeof value === 'string') {
|
||
const leadingDigits = value.trim().match(/^\d+/)?.[0];
|
||
if (!leadingDigits) { return null; }
|
||
const parsed = Number.parseInt(leadingDigits, 10);
|
||
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
|
||
}
|
||
return parsePositiveFrequencyNumber(value);
|
||
}
|
||
function getFrequencyDictionaryName(frequency) {
|
||
const candidates = [
|
||
frequency?.dictionary,
|
||
frequency?.dictionaryName,
|
||
frequency?.name,
|
||
frequency?.title,
|
||
frequency?.dictionaryTitle,
|
||
frequency?.dictionaryAlias
|
||
];
|
||
for (const candidate of candidates) {
|
||
if (typeof candidate === 'string' && candidate.trim().length > 0) {
|
||
return candidate.trim();
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
function getBestFrequencyRank(dictionaryEntry, headwordIndex, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
|
||
let best = null;
|
||
const headwordCount = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords.length : 0;
|
||
for (const frequency of dictionaryEntry?.frequencies || []) {
|
||
if (!frequency || typeof frequency !== 'object') { continue; }
|
||
const frequencyHeadwordIndex = frequency.headwordIndex;
|
||
if (typeof frequencyHeadwordIndex === 'number') {
|
||
if (frequencyHeadwordIndex !== headwordIndex) { continue; }
|
||
} else if (headwordCount > 1) {
|
||
continue;
|
||
}
|
||
const dictionary = getFrequencyDictionaryName(frequency);
|
||
if (!dictionary) { continue; }
|
||
if (dictionaryFrequencyModeByName[dictionary] === 'occurrence-based') { continue; }
|
||
const rank =
|
||
parseDisplayFrequencyNumber(frequency.displayValue) ??
|
||
parsePositiveFrequencyNumber(frequency.frequency);
|
||
if (rank === null) { continue; }
|
||
const priorityRaw = dictionaryPriorityByName[dictionary];
|
||
const fallbackPriority =
|
||
typeof frequency.dictionaryIndex === 'number' && Number.isFinite(frequency.dictionaryIndex)
|
||
? Math.max(0, Math.floor(frequency.dictionaryIndex))
|
||
: Number.MAX_SAFE_INTEGER;
|
||
const priority =
|
||
typeof priorityRaw === 'number' && Number.isFinite(priorityRaw)
|
||
? Math.max(0, Math.floor(priorityRaw))
|
||
: fallbackPriority;
|
||
if (best === null || priority < best.priority || (priority === best.priority && rank < best.rank)) {
|
||
best = { priority, rank };
|
||
}
|
||
}
|
||
return best?.rank ?? null;
|
||
}
|
||
function hasExactSource(headword, token, requirePrimary) {
|
||
for (const src of headword.sources || []) {
|
||
if (src.originalText !== token) { continue; }
|
||
if (requirePrimary && !src.isPrimary) { continue; }
|
||
if (src.matchType !== 'exact') { continue; }
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
function collectExactHeadwordMatches(dictionaryEntries, token, requirePrimary) {
|
||
const matches = [];
|
||
for (const dictionaryEntry of dictionaryEntries || []) {
|
||
const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
|
||
for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
|
||
const headword = headwords[headwordIndex];
|
||
if (!hasExactSource(headword, token, requirePrimary)) { continue; }
|
||
matches.push({ dictionaryEntry, headword, headwordIndex });
|
||
}
|
||
}
|
||
return matches;
|
||
}
|
||
function sameHeadword(match, preferredMatch) {
|
||
if (!match || !preferredMatch) {
|
||
return false;
|
||
}
|
||
if (match.headword?.term !== preferredMatch.headword?.term) {
|
||
return false;
|
||
}
|
||
const matchReading = typeof match.headword?.reading === 'string' ? match.headword.reading : '';
|
||
const preferredReading =
|
||
typeof preferredMatch.headword?.reading === 'string' ? preferredMatch.headword.reading : '';
|
||
if (!matchReading || !preferredReading) {
|
||
return true;
|
||
}
|
||
return matchReading === preferredReading;
|
||
}
|
||
function getBestFrequencyRankForMatches(matches, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
|
||
let best = null;
|
||
for (const match of matches) {
|
||
const rank = getBestFrequencyRank(
|
||
match.dictionaryEntry,
|
||
match.headwordIndex,
|
||
dictionaryPriorityByName,
|
||
dictionaryFrequencyModeByName
|
||
);
|
||
if (rank === null) { continue; }
|
||
if (best === null || rank < best) {
|
||
best = rank;
|
||
}
|
||
}
|
||
return best;
|
||
}
|
||
function normalizeWordClasses(headword) {
|
||
if (!Array.isArray(headword?.wordClasses)) { return undefined; }
|
||
const classes = headword.wordClasses.filter((wordClass) => typeof wordClass === "string" && wordClass.trim().length > 0);
|
||
return classes.length > 0 ? classes : undefined;
|
||
}
|
||
function appendDictionaryNames(target, value) {
|
||
if (!value || typeof value !== 'object') {
|
||
return;
|
||
}
|
||
const candidates = [
|
||
value.dictionary,
|
||
value.dictionaryName,
|
||
value.name,
|
||
value.title,
|
||
value.dictionaryTitle,
|
||
value.dictionaryAlias
|
||
];
|
||
for (const candidate of candidates) {
|
||
if (typeof candidate === 'string' && candidate.trim().length > 0) {
|
||
target.push(candidate.trim());
|
||
}
|
||
}
|
||
}
|
||
function getDictionaryEntryNames(entry) {
|
||
const names = [];
|
||
appendDictionaryNames(names, entry);
|
||
for (const definition of entry?.definitions || []) {
|
||
appendDictionaryNames(names, definition);
|
||
}
|
||
for (const frequency of entry?.frequencies || []) {
|
||
appendDictionaryNames(names, frequency);
|
||
}
|
||
for (const pronunciation of entry?.pronunciations || []) {
|
||
appendDictionaryNames(names, pronunciation);
|
||
}
|
||
return names;
|
||
}
|
||
function isNameDictionaryEntry(entry) {
|
||
if (!includeNameMatchMetadata || !entry || typeof entry !== 'object') {
|
||
return false;
|
||
}
|
||
return getDictionaryEntryNames(entry).some((name) => name.startsWith(${JSON.stringify(CHARACTER_DICTIONARY_TITLE_PREFIX)}));
|
||
}
|
||
function parseSubMinerMediaIdFromString(value) {
|
||
const imageMatch = value.match(/\bimg\/m(\d+)-/i);
|
||
if (imageMatch) {
|
||
const parsed = Number.parseInt(imageMatch[1], 10);
|
||
if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
|
||
}
|
||
const titleMatch = value.match(/${CHARACTER_DICTIONARY_TITLE_PREFIX}[^\d]*(?:AniList\s*)?(\d+)/i);
|
||
if (titleMatch) {
|
||
const parsed = Number.parseInt(titleMatch[1], 10);
|
||
if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
|
||
}
|
||
return null;
|
||
}
|
||
function parseSubMinerMediaIdCandidate(value) {
|
||
if (typeof value === 'number' && Number.isSafeInteger(value) && value > 0) {
|
||
return value;
|
||
}
|
||
if (typeof value === 'string' && /^\d+$/.test(value.trim())) {
|
||
const parsed = Number.parseInt(value.trim(), 10);
|
||
if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; }
|
||
}
|
||
return null;
|
||
}
|
||
function collectSubMinerMediaIds(value, target) {
|
||
if (typeof value === 'string') {
|
||
const parsed = parseSubMinerMediaIdFromString(value);
|
||
if (parsed !== null) { target.add(parsed); }
|
||
return;
|
||
}
|
||
if (!value || typeof value !== 'object') {
|
||
return;
|
||
}
|
||
if (Array.isArray(value)) {
|
||
for (const item of value) { collectSubMinerMediaIds(item, target); }
|
||
return;
|
||
}
|
||
const mediaIdCandidates = [
|
||
value.subminerMediaId,
|
||
value.subMinerMediaId,
|
||
value.characterDictionaryMediaId,
|
||
value.data?.subminerMediaId,
|
||
value.data?.subMinerMediaId,
|
||
value.data?.characterDictionaryMediaId
|
||
];
|
||
for (const candidate of mediaIdCandidates) {
|
||
const parsed = parseSubMinerMediaIdCandidate(candidate);
|
||
if (parsed !== null) { target.add(parsed); }
|
||
}
|
||
for (const child of Object.values(value)) {
|
||
collectSubMinerMediaIds(child, target);
|
||
}
|
||
}
|
||
function getSubMinerMediaIds(entry) {
|
||
const mediaIds = new Set();
|
||
collectSubMinerMediaIds(entry, mediaIds);
|
||
return mediaIds;
|
||
}
|
||
function isCurrentMediaNameDictionaryEntry(entry) {
|
||
if (!isNameDictionaryEntry(entry)) {
|
||
return false;
|
||
}
|
||
if (currentCharacterDictionaryMediaId === null) {
|
||
return true;
|
||
}
|
||
const mediaIds = getSubMinerMediaIds(entry);
|
||
return mediaIds.size === 0 || mediaIds.has(currentCharacterDictionaryMediaId);
|
||
}
|
||
function findLongestNameMatch(dictionaryEntries, textWindow) {
|
||
let best = null;
|
||
for (const dictionaryEntry of dictionaryEntries || []) {
|
||
if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; }
|
||
const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
|
||
for (let headwordIndex = 0; headwordIndex < headwords.length; headwordIndex += 1) {
|
||
const headword = headwords[headwordIndex];
|
||
for (const src of headword?.sources || []) {
|
||
if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
|
||
const originalText = typeof src.originalText === 'string' ? src.originalText : '';
|
||
if (!originalText || !textWindow.startsWith(originalText)) { continue; }
|
||
if (best === null || originalText.length > best.sourceLength) {
|
||
best = { dictionaryEntry, headword, headwordIndex, sourceLength: originalText.length };
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return best;
|
||
}
|
||
function findLongestGenericMatchLength(dictionaryEntries, textWindow) {
|
||
let best = 0;
|
||
for (const dictionaryEntry of dictionaryEntries || []) {
|
||
if (isNameDictionaryEntry(dictionaryEntry)) { continue; }
|
||
const headwords = Array.isArray(dictionaryEntry?.headwords) ? dictionaryEntry.headwords : [];
|
||
for (const headword of headwords) {
|
||
for (const src of headword?.sources || []) {
|
||
if (src.matchType !== 'exact' || src.isPrimary !== true) { continue; }
|
||
const originalText = typeof src.originalText === 'string' ? src.originalText : '';
|
||
if (!originalText || !textWindow.startsWith(originalText)) { continue; }
|
||
if (originalText.length > best) { best = originalText.length; }
|
||
}
|
||
}
|
||
}
|
||
return best;
|
||
}
|
||
function getPreferredHeadword(dictionaryEntries, token, dictionaryPriorityByName, dictionaryFrequencyModeByName) {
|
||
const currentMediaDictionaryEntries =
|
||
currentCharacterDictionaryMediaId === null
|
||
? (dictionaryEntries || [])
|
||
: (dictionaryEntries || []).filter((entry) => {
|
||
if (!isNameDictionaryEntry(entry)) { return true; }
|
||
return isCurrentMediaNameDictionaryEntry(entry);
|
||
});
|
||
const exactPrimaryMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, true);
|
||
let matchedNameDictionary = false;
|
||
if (includeNameMatchMetadata) {
|
||
for (const dictionaryEntry of currentMediaDictionaryEntries || []) {
|
||
if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; }
|
||
for (const match of exactPrimaryMatches) {
|
||
if (match.dictionaryEntry !== dictionaryEntry) { continue; }
|
||
matchedNameDictionary = true;
|
||
break;
|
||
}
|
||
if (matchedNameDictionary) { break; }
|
||
}
|
||
}
|
||
const preferredMatch = exactPrimaryMatches[0];
|
||
if (preferredMatch) {
|
||
const exactFrequencyMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, false)
|
||
.filter((match) => sameHeadword(match, preferredMatch));
|
||
return {
|
||
term: preferredMatch.headword.term,
|
||
reading: preferredMatch.headword.reading,
|
||
wordClasses: normalizeWordClasses(preferredMatch.headword),
|
||
isNameMatch:
|
||
matchedNameDictionary || isCurrentMediaNameDictionaryEntry(preferredMatch.dictionaryEntry),
|
||
frequencyRank: getBestFrequencyRankForMatches(
|
||
exactFrequencyMatches.length > 0 ? exactFrequencyMatches : exactPrimaryMatches,
|
||
dictionaryPriorityByName,
|
||
dictionaryFrequencyModeByName
|
||
)
|
||
};
|
||
}
|
||
return null;
|
||
}
|
||
`;
|
||
|
||
// Bump whenever the install script below changes so already-loaded parser
|
||
// windows re-install the new scan runtime instead of running the stale one.
|
||
export const YOMITAN_SCAN_RUNTIME_VERSION = 3;
|
||
export const YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL = '__subminer-yomitan-scan-runtime-missing__';
|
||
|
||
export interface YomitanScanRequestParams {
|
||
text: string;
|
||
profileIndex: number;
|
||
scanLength: number;
|
||
includeNameMatchMetadata: boolean;
|
||
greedyNameScanEnabled: boolean;
|
||
currentCharacterDictionaryMediaId: number | null;
|
||
dictionaryPriorityByName: Record<string, number>;
|
||
dictionaryFrequencyModeByName: Partial<Record<string, YomitanFrequencyMode>>;
|
||
cacheEpoch: number;
|
||
/**
|
||
* Key of the character-name candidate list installed for the current media,
|
||
* or null to scan every Japanese position (see the pre-pass prefilter).
|
||
*/
|
||
nameCandidateKey: string | null;
|
||
}
|
||
|
||
// Installed once per parser window (and re-installed after in-page reloads):
|
||
// keeps V8 from re-parsing the helper bundle on every subtitle line, and hosts
|
||
// the cross-line termsFind cache. Each subtitle line then only evaluates a tiny
|
||
// call into globalThis.__subminerYomitanScan.
|
||
export const YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT = String.raw`
|
||
(() => {
|
||
if (globalThis.__subminerYomitanScanVersion === ${YOMITAN_SCAN_RUNTIME_VERSION}) {
|
||
return true;
|
||
}
|
||
const invoke = (action, params) =>
|
||
new Promise((resolve, reject) => {
|
||
chrome.runtime.sendMessage({ action, params }, (response) => {
|
||
if (chrome.runtime.lastError) {
|
||
reject(new Error(chrome.runtime.lastError.message));
|
||
return;
|
||
}
|
||
if (!response || typeof response !== "object") {
|
||
reject(new Error("Invalid response from Yomitan backend"));
|
||
return;
|
||
}
|
||
if (response.error) {
|
||
reject(new Error(response.error.message || "Yomitan backend error"));
|
||
return;
|
||
}
|
||
resolve(response.result);
|
||
});
|
||
});
|
||
// Cross-line termsFind LRU keyed by profile + substring: subtitle lines
|
||
// repeat particles and inflections constantly, so most lookups hit here.
|
||
// Entries hold in-flight promises so concurrent identical lookups dedupe.
|
||
const termsFindCache = new Map();
|
||
const TERMS_FIND_CACHE_LIMIT = 2000;
|
||
let termsFindCacheEpoch = -1;
|
||
const MAX_SHRINKING_WINDOW_RETRY_LOOKUPS = 4;
|
||
// Character-name candidate forms for the current media, installed
|
||
// separately from the per-line scan call so the per-line script stays tiny.
|
||
// Stored raw here; the normalized lookup index is built inside the scan,
|
||
// where the kana-normalization helper is in scope, and reused by key.
|
||
let rawNameCandidates = null;
|
||
let nameCandidateIndex = null;
|
||
globalThis.__subminerYomitanScanSetNameCandidates = (key, forms) => {
|
||
if (!key || !Array.isArray(forms) || forms.length === 0) {
|
||
rawNameCandidates = null;
|
||
nameCandidateIndex = null;
|
||
return false;
|
||
}
|
||
rawNameCandidates = { key, forms };
|
||
nameCandidateIndex = null;
|
||
return true;
|
||
};
|
||
globalThis.__subminerYomitanScanVersion = ${YOMITAN_SCAN_RUNTIME_VERSION};
|
||
globalThis.__subminerYomitanScan = async (scanParams) => {
|
||
const {
|
||
text,
|
||
profileIndex,
|
||
scanLength,
|
||
includeNameMatchMetadata,
|
||
greedyNameScanEnabled,
|
||
currentCharacterDictionaryMediaId,
|
||
dictionaryPriorityByName,
|
||
dictionaryFrequencyModeByName,
|
||
cacheEpoch,
|
||
nameCandidateKey
|
||
} = scanParams;
|
||
if (cacheEpoch !== termsFindCacheEpoch) {
|
||
termsFindCache.clear();
|
||
termsFindCacheEpoch = cacheEpoch;
|
||
}
|
||
${YOMITAN_SCANNING_HELPERS}
|
||
const CAPTION_OPENING_BRACKETS = new Set(["(", "（", "[", "［", "{", "｛", "「", "『", "【", "〈", "《", "≪", "＜", "<"]);
|
||
function shouldEmitUnparsedRunAsToken(runText) {
|
||
if (!/[\p{L}\p{N}]/u.test(runText)) { return false; }
|
||
const firstChar = Array.from(runText.trim())[0];
|
||
return firstChar !== undefined && !CAPTION_OPENING_BRACKETS.has(firstChar);
|
||
}
|
||
function isLookupWorthyCodePoint(codePoint) {
|
||
if (isCodePointJapanese(codePoint)) { return true; }
|
||
return /[\p{L}\p{N}]/u.test(String.fromCodePoint(codePoint));
|
||
}
|
||
function isKanaOnlyRunText(runText) {
|
||
const chars = Array.from(runText);
|
||
return chars.length > 0 && chars.every((char) => isCodePointKana(char.codePointAt(0)));
|
||
}
|
||
const details = {matchType: "exact", deinflect: true};
|
||
const tokens = [];
|
||
async function termsFindAt(position, windowLength) {
|
||
const substring = text.substring(position, position + windowLength);
|
||
const cacheKey = profileIndex + " " + substring;
|
||
const cached = termsFindCache.get(cacheKey);
|
||
if (cached !== undefined) {
|
||
termsFindCache.delete(cacheKey);
|
||
termsFindCache.set(cacheKey, cached);
|
||
return await cached;
|
||
}
|
||
const pending = invoke("termsFind", { text: substring, details, optionsContext: { index: profileIndex } });
|
||
termsFindCache.set(cacheKey, pending);
|
||
while (termsFindCache.size > TERMS_FIND_CACHE_LIMIT) {
|
||
const oldestKey = termsFindCache.keys().next().value;
|
||
if (oldestKey === undefined) { break; }
|
||
termsFindCache.delete(oldestKey);
|
||
}
|
||
try {
|
||
return await pending;
|
||
} catch (error) {
|
||
termsFindCache.delete(cacheKey);
|
||
throw error;
|
||
}
|
||
}
|
||
// Text the walk skips accumulates into unparsed runs, mirroring the
|
||
// filler chunks the parseText segmentation used to provide: runs stay
|
||
// hoverable (flagged isUnparsedRun) unless they are punctuation-only or
|
||
// caption-style asides, and kana continuations of a longer headword
|
||
// extend the previous token instead.
|
||
function flushUnparsedRun(runStart, runEnd) {
|
||
if (runStart === null || runEnd <= runStart) { return; }
|
||
const runText = text.substring(runStart, runEnd);
|
||
const previousToken = tokens[tokens.length - 1];
|
||
if (
|
||
previousToken &&
|
||
previousToken.endPos === runStart &&
|
||
isKanaOnlyRunText(runText) &&
|
||
typeof previousToken.headword === "string" &&
|
||
previousToken.headword.length > previousToken.surface.length &&
|
||
previousToken.headword.startsWith(previousToken.surface + runText)
|
||
) {
|
||
previousToken.surface += runText;
|
||
// The run is kana-only, so its reading is itself: append it or the
|
||
// reading stops covering the surface, which disables the known-word
|
||
// reading fallback (isCompleteReadingForSurface) downstream.
|
||
previousToken.reading += runText;
|
||
// The run is kana-only, so its reading is itself: append it or the
|
||
// reading stops covering the surface, which disables the known-word
|
||
// reading fallback (isCompleteReadingForSurface) downstream.
|
||
previousToken.endPos = runEnd;
|
||
return;
|
||
}
|
||
if (!shouldEmitUnparsedRunAsToken(runText)) { return; }
|
||
tokens.push({
|
||
surface: runText,
|
||
reading: "",
|
||
headword: runText,
|
||
startPos: runStart,
|
||
endPos: runEnd,
|
||
isUnparsedRun: true
|
||
});
|
||
}
|
||
function buildScanToken(position, source, preferredHeadword) {
|
||
const reading = typeof preferredHeadword.reading === "string" ? preferredHeadword.reading : "";
|
||
const segments = distributeFuriganaInflected(preferredHeadword.term, reading, source);
|
||
const tokenPayload = {
|
||
surface: segments.map((segment) => segment.text).join("") || source,
|
||
reading: segments.map(getSegmentReadingContribution).join(""),
|
||
headword: preferredHeadword.term,
|
||
headwordReading: reading || undefined,
|
||
startPos: position,
|
||
endPos: position + source.length,
|
||
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
|
||
frequencyRank:
|
||
typeof preferredHeadword.frequencyRank === "number" && Number.isFinite(preferredHeadword.frequencyRank)
|
||
? Math.max(1, Math.floor(preferredHeadword.frequencyRank))
|
||
: undefined,
|
||
};
|
||
if (Array.isArray(preferredHeadword.wordClasses) && preferredHeadword.wordClasses.length > 0) {
|
||
tokenPayload.wordClasses = preferredHeadword.wordClasses;
|
||
}
|
||
return tokenPayload;
|
||
}
|
||
async function findTokenAt(position, windowLength) {
|
||
const codePoint = text.codePointAt(position);
|
||
const character = String.fromCodePoint(codePoint);
|
||
const result = await termsFindAt(position, windowLength);
|
||
const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
|
||
const originalTextLength = typeof result?.originalTextLength === "number" ? result.originalTextLength : 0;
|
||
if (dictionaryEntries.length === 0 || originalTextLength <= 0 || (originalTextLength === character.length && !isCodePointJapanese(codePoint))) {
|
||
return { token: null, matchedLength: 0 };
|
||
}
|
||
const source = text.substring(position, position + originalTextLength);
|
||
const preferredHeadword = getPreferredHeadword(
|
||
dictionaryEntries,
|
||
source,
|
||
dictionaryPriorityByName,
|
||
dictionaryFrequencyModeByName
|
||
);
|
||
if (!preferredHeadword || typeof preferredHeadword.term !== "string") {
|
||
return { token: null, matchedLength: originalTextLength };
|
||
}
|
||
return { token: buildScanToken(position, source, preferredHeadword), matchedLength: originalTextLength };
|
||
}
|
||
// Halfwidth katakana survives kana normalization unchanged, so a name
|
||
// written that way would not prefix-match a candidate form. Those
|
||
// positions bypass the prefilter rather than risk a missed name.
|
||
function isHalfwidthKatakanaCodePoint(codePoint) {
|
||
return codePoint >= 0xff66 && codePoint <= 0xff9f;
|
||
}
|
||
// Build (once per candidate list) a first-character bucket index of the
|
||
// normalized name forms, so the pre-pass can reject a position with a
|
||
// single map hit instead of a backend round trip.
|
||
if (rawNameCandidates && nameCandidateIndex?.key !== rawNameCandidates.key) {
|
||
const byFirstChar = new Map();
|
||
for (const form of rawNameCandidates.forms) {
|
||
const normalized = typeof form === "string" ? convertKatakanaToHiragana(form.trim()) : "";
|
||
if (!normalized) { continue; }
|
||
const bucket = byFirstChar.get(normalized[0]);
|
||
if (bucket) { bucket.push(normalized); } else { byFirstChar.set(normalized[0], [normalized]); }
|
||
}
|
||
nameCandidateIndex = byFirstChar.size > 0 ? { key: rawNameCandidates.key, byFirstChar } : null;
|
||
} else if (!rawNameCandidates) {
|
||
nameCandidateIndex = null;
|
||
}
|
||
// Only meaningful when the installed list matches the media this scan is
|
||
// for; otherwise fall back to scanning every position.
|
||
const activeNameCandidateIndex =
|
||
nameCandidateKey !== null && nameCandidateIndex?.key === nameCandidateKey
|
||
? nameCandidateIndex
|
||
: null;
|
||
const normalizedText = activeNameCandidateIndex ? convertKatakanaToHiragana(text) : "";
|
||
function couldNameStartAt(position, codePoint) {
|
||
if (!activeNameCandidateIndex) { return true; }
|
||
if (isHalfwidthKatakanaCodePoint(codePoint)) { return true; }
|
||
const bucket = activeNameCandidateIndex.byFirstChar.get(normalizedText[position]);
|
||
if (!bucket) { return false; }
|
||
for (const form of bucket) {
|
||
if (normalizedText.startsWith(form, position)) { return true; }
|
||
}
|
||
return false;
|
||
}
|
||
// Greedy name pre-pass: character-name matches claim their spans before
|
||
// the left-to-right walk, so a longer generic match starting earlier
|
||
// (e.g. とヨー → 渡洋) cannot swallow the start of a name (ヨータ).
|
||
const nameTokens = [];
|
||
if (greedyNameScanEnabled) {
|
||
let namePos = 0;
|
||
while (namePos < text.length) {
|
||
const codePoint = text.codePointAt(namePos);
|
||
if (!isCodePointJapanese(codePoint) || !couldNameStartAt(namePos, codePoint)) {
|
||
namePos += String.fromCodePoint(codePoint).length;
|
||
continue;
|
||
}
|
||
const result = await termsFindAt(namePos, scanLength);
|
||
const dictionaryEntries = Array.isArray(result?.dictionaryEntries) ? result.dictionaryEntries : [];
|
||
const textWindow = text.substring(namePos, namePos + scanLength);
|
||
const nameMatch = findLongestNameMatch(dictionaryEntries, textWindow);
|
||
// A name only claims its span when no strictly longer generic word
|
||
// starts at the same position (a character named 空 must not split
|
||
// 空気). Ties go to the name. Generic matches that start earlier and
|
||
// overlap the name are still blocked by the reservation.
|
||
if (
|
||
!nameMatch ||
|
||
findLongestGenericMatchLength(dictionaryEntries, textWindow) > nameMatch.sourceLength
|
||
) {
|
||
namePos += String.fromCodePoint(codePoint).length;
|
||
continue;
|
||
}
|
||
const source = text.substring(namePos, namePos + nameMatch.sourceLength);
|
||
nameTokens.push(buildScanToken(namePos, source, {
|
||
term: nameMatch.headword.term,
|
||
reading: nameMatch.headword.reading,
|
||
wordClasses: normalizeWordClasses(nameMatch.headword),
|
||
isNameMatch: true,
|
||
frequencyRank: getBestFrequencyRank(
|
||
nameMatch.dictionaryEntry,
|
||
nameMatch.headwordIndex,
|
||
dictionaryPriorityByName,
|
||
dictionaryFrequencyModeByName
|
||
)
|
||
}));
|
||
namePos += nameMatch.sourceLength;
|
||
}
|
||
}
|
||
let i = 0;
|
||
let nameIndex = 0;
|
||
let unparsedRunStart = null;
|
||
while (i < text.length) {
|
||
while (nameIndex < nameTokens.length && nameTokens[nameIndex].startPos < i) { nameIndex += 1; }
|
||
const nextNameToken = nameIndex < nameTokens.length ? nameTokens[nameIndex] : null;
|
||
if (nextNameToken && nextNameToken.startPos === i) {
|
||
flushUnparsedRun(unparsedRunStart, i);
|
||
unparsedRunStart = null;
|
||
tokens.push(nextNameToken);
|
||
i = nextNameToken.endPos;
|
||
nameIndex += 1;
|
||
continue;
|
||
}
|
||
const codePoint = text.codePointAt(i);
|
||
// Punctuation and whitespace can never start a token: skip the backend
|
||
// round trip entirely. Latin letters and digits stay lookup-worthy
|
||
// (terms like Tシャツ start on an ASCII letter).
|
||
if (!isLookupWorthyCodePoint(codePoint)) {
|
||
if (unparsedRunStart === null) { unparsedRunStart = i; }
|
||
i += String.fromCodePoint(codePoint).length;
|
||
continue;
|
||
}
|
||
// Cap the window at the next reserved name span so a generic match
|
||
// cannot consume into it.
|
||
const windowLength = nextNameToken ? Math.min(scanLength, nextNameToken.startPos - i) : scanLength;
|
||
let attempt = await findTokenAt(i, windowLength);
|
||
// Yomitan text normalization can consume characters (whitespace,
|
||
// punctuation) beyond the matched term, leaving no headword whose
|
||
// source equals the consumed text. Retry with shorter windows so a
|
||
// valid prefix term (e.g. a character name before a paren) still
|
||
// tokenizes instead of the position being skipped. The ladder is
|
||
// capped: without a cap it degrades to O(scanLength) lookups at a
|
||
// single position.
|
||
let retryLength = Math.min(attempt.matchedLength, windowLength) - 1;
|
||
let retryLookupsRemaining = MAX_SHRINKING_WINDOW_RETRY_LOOKUPS;
|
||
while (!attempt.token && retryLength >= 1 && retryLookupsRemaining > 0) {
|
||
retryLookupsRemaining -= 1;
|
||
const retry = await findTokenAt(i, retryLength);
|
||
if (retry.token) {
|
||
attempt = retry;
|
||
break;
|
||
}
|
||
retryLength = Math.min(retryLength - 1, retry.matchedLength - 1);
|
||
}
|
||
if (attempt.token) {
|
||
flushUnparsedRun(unparsedRunStart, i);
|
||
unparsedRunStart = null;
|
||
tokens.push(attempt.token);
|
||
i += attempt.matchedLength;
|
||
continue;
|
||
}
|
||
if (unparsedRunStart === null) { unparsedRunStart = i; }
|
||
i += String.fromCodePoint(text.codePointAt(i)).length;
|
||
}
|
||
flushUnparsedRun(unparsedRunStart, text.length);
|
||
return tokens;
|
||
};
|
||
return true;
|
||
})();
|
||
`;
|
||
|
||
// Installs (or clears) the character-name candidate forms for the current
|
||
// media. Runs only when the list changes, not per line. Passing null restores
|
||
// the exhaustive every-position pre-pass.
|
||
export function buildYomitanScanNameCandidatesScript(
|
||
nameCandidates: { key: string; forms: string[] } | null,
|
||
): string {
|
||
if (!nameCandidates) {
|
||
return `
|
||
(() => {
|
||
if (typeof globalThis.__subminerYomitanScanSetNameCandidates !== "function") {
|
||
return false;
|
||
}
|
||
return globalThis.__subminerYomitanScanSetNameCandidates(null, null);
|
||
})();
|
||
`;
|
||
}
|
||
|
||
return `
|
||
(() => {
|
||
if (typeof globalThis.__subminerYomitanScanSetNameCandidates !== "function") {
|
||
return false;
|
||
}
|
||
return globalThis.__subminerYomitanScanSetNameCandidates(
|
||
${JSON.stringify(nameCandidates.key)},
|
||
${JSON.stringify(nameCandidates.forms)}
|
||
);
|
||
})();
|
||
`;
|
||
}
|
||
|
||
export function buildYomitanScanCallScript(params: YomitanScanRequestParams): string {
|
||
return `
|
||
(async () => {
|
||
if (typeof globalThis.__subminerYomitanScan !== "function") {
|
||
return ${JSON.stringify(YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL)};
|
||
}
|
||
return await globalThis.__subminerYomitanScan(${JSON.stringify(params)});
|
||
})();
|
||
`;
|
||
}
|