From 64853330fb4f5621c0de63c09aae721917ef52d8 Mon Sep 17 00:00:00 2001 From: sudacode Date: Wed, 5 Aug 2026 22:56:42 -0700 Subject: [PATCH] fix(tokenizer): widen halfwidth voiced-mark bypass to full scan window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Probe for an unfoldable halfwidth voiced mark (ｶﾞ) across the whole name-lookup window instead of a region sized off the longest candidate form, so emphatic-padded names (山ーーーーーーｶﾞｸ) still survive the prefilter - Escape the character-dictionary title prefix before interpolating it into the generated media-id regex - Guard hasExactSource against an undefined headword - Simplify name-dictionary match classification to check each match's own entry instead of re-scanning all entries per match --- changes/subtitle-tokenization-performance.md | 2 +- ...omitan-dictionary-classification-script.ts | 11 ++- .../tokenizer/yomitan-frequency-script.ts | 2 +- .../yomitan-match-selection-script.ts | 14 ++-- .../tokenizer/yomitan-scan-runtime-script.ts | 67 ++++++++++------- .../tokenizer/yomitan-scan-runtime.test.ts | 73 ++++++++++++++++++- 6 files changed, 129 insertions(+), 40 deletions(-) diff --git a/changes/subtitle-tokenization-performance.md b/changes/subtitle-tokenization-performance.md index 40ae971b..5eeb38de 100644 --- a/changes/subtitle-tokenization-performance.md +++ b/changes/subtitle-tokenization-performance.md @@ -15,6 +15,6 @@ area: subtitles - Character name annotations no longer cost a dictionary lookup at every position in a line. The scanner now knows which name forms the current title's character dictionary actually contains and only checks where one can start, which removes the whole overhead of having the character dictionary enabled (measured: 21 lookups per line down to 10, the same as with it disabled). Titles with no cached character data keep the previous exhaustive scan, so a missing snapshot costs speed rather than a missing name. - The cross-line termsFind cache is now bounded by the number of retained dictionary entries as well as by key count, so a run of lookups that each carry hundreds of entries with full glossaries cannot grow the parser window's memory without limit. The budget is re-checked when a lookup resolves, so a single oversized response is dropped rather than parked in the cache and reused. - The unnamed-mob disambiguator filter (Girl A / Girl B) now only drops a single letter or digit split off a name, instead of every one-character term: a name that is genuinely one character keeps its terms whatever the script (𠮷, あ, 별 김, ア・ベ). The character dictionary and the scanner's name pre-pass also share one Han code-point table now, so a name the dictionary accepts is a name the scanner will look for. -- A character name written in halfwidth katakana takes part in the greedy name pre-pass again, so a longer generic word can no longer swallow the start of it, and it now carries a reading (it used to come out blank, which disables known-word matching and frequency lookups for the token). Voiced halfwidth kana compose properly, so ｶﾞｸ reads ガク rather than カﾞク, and kana normalization folds halfwidth throughout so those tokens compare equal to the same word written fullwidth. Because the fold makes halfwidth text indexable, the character-name prefilter now judges halfwidth spellings like any other, and only a position whose candidate-sized region holds an unfoldable voiced mark bypasses it. That also covers a name that starts on a kanji and turns halfwidth later (山ｶﾞｸ), which the earlier first-character rule dropped. +- A character name written in halfwidth katakana takes part in the greedy name pre-pass again, so a longer generic word can no longer swallow the start of it, and it now carries a reading (it used to come out blank, which disables known-word matching and frequency lookups for the token). Voiced halfwidth kana compose properly, so ｶﾞｸ reads ガク rather than カﾞク, and kana normalization folds halfwidth throughout so those tokens compare equal to the same word written fullwidth. Because the fold makes halfwidth text indexable, the character-name prefilter now judges halfwidth spellings like any other, and a position only bypasses it when an unfoldable voiced mark sits inside the lookup window. That covers a name that starts on a kanji and turns halfwidth later (山ｶﾞｸ), and one stretched out with emphatic characters in between (山ーーーーーーｶﾞｸ). - Dictionary-entry classification (source dictionaries, character-dictionary media ids) is memoized per entry object for as long as the entry is cached, instead of being recomputed for every headword comparison and every retry window. - Autoplay priming no longer broadcasts the plain subtitle twice: it tells the processing controller the line has already been painted, so the controller goes straight to the annotated payload. diff --git a/src/core/services/tokenizer/yomitan-dictionary-classification-script.ts b/src/core/services/tokenizer/yomitan-dictionary-classification-script.ts index 68c9823b..4123159a 100644 --- a/src/core/services/tokenizer/yomitan-dictionary-classification-script.ts +++ b/src/core/services/tokenizer/yomitan-dictionary-classification-script.ts @@ -5,6 +5,14 @@ import { CHARACTER_DICTIONARY_TITLE_PREFIX } from './character-dictionary-title'; +// The prefix is interpolated into generated regex source, so metacharacters in +// it would change what the pattern matches (or fail to compile). +const ESCAPED_TITLE_PREFIX = CHARACTER_DICTIONARY_TITLE_PREFIX.replace( + /[.*+?^${}()|[\]\\]/g, + '\\$&', +); +const TITLE_MEDIA_ID_PATTERN = ESCAPED_TITLE_PREFIX + String.raw`[^\d]*(?:AniList\s*)?(\d+)`; + export const YOMITAN_DICTIONARY_CLASSIFICATION_HELPERS = String.raw` function normalizeWordClasses(headword) { if (!Array.isArray(headword?.wordClasses)) { return undefined; } @@ -64,13 +72,14 @@ export const YOMITAN_DICTIONARY_CLASSIFICATION_HELPERS = String.raw` nameDictionaryEntryCache.set(entry, isName); return isName; } + const TITLE_MEDIA_ID_REGEX = new RegExp(${JSON.stringify(TITLE_MEDIA_ID_PATTERN)}, 'i'); function parseSubMinerMediaIdFromString(value) { const imageMatch = value.match(/\bimg\/m(\d+)-/i); if (imageMatch) { const parsed = Number.parseInt(imageMatch[1], 10); if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; } } - const titleMatch = value.match(/${CHARACTER_DICTIONARY_TITLE_PREFIX}[^\d]*(?:AniList\s*)?(\d+)/i); + const titleMatch = value.match(TITLE_MEDIA_ID_REGEX); if (titleMatch) { const parsed = Number.parseInt(titleMatch[1], 10); if (Number.isSafeInteger(parsed) && parsed > 0) { return parsed; } diff --git a/src/core/services/tokenizer/yomitan-frequency-script.ts b/src/core/services/tokenizer/yomitan-frequency-script.ts index 90955fb4..a73a47d5 100644 --- a/src/core/services/tokenizer/yomitan-frequency-script.ts +++ b/src/core/services/tokenizer/yomitan-frequency-script.ts @@ -81,7 +81,7 @@ export const YOMITAN_FREQUENCY_HELPERS = String.raw` return best?.rank ?? null; } function hasExactSource(headword, token, requirePrimary) { - for (const src of headword.sources || []) { + for (const src of headword?.sources || []) { if (src.originalText !== token) { continue; } if (requirePrimary && !src.isPrimary) { continue; } if (src.matchType !== 'exact') { continue; } diff --git a/src/core/services/tokenizer/yomitan-match-selection-script.ts b/src/core/services/tokenizer/yomitan-match-selection-script.ts index cda9121a..79dc67ce 100644 --- a/src/core/services/tokenizer/yomitan-match-selection-script.ts +++ b/src/core/services/tokenizer/yomitan-match-selection-script.ts @@ -49,14 +49,12 @@ export const YOMITAN_MATCH_SELECTION_HELPERS = String.raw` const exactPrimaryMatches = collectExactHeadwordMatches(currentMediaDictionaryEntries, token, true); let matchedNameDictionary = false; if (includeNameMatchMetadata) { - for (const dictionaryEntry of currentMediaDictionaryEntries || []) { - if (!isCurrentMediaNameDictionaryEntry(dictionaryEntry)) { continue; } - for (const match of exactPrimaryMatches) { - if (match.dictionaryEntry !== dictionaryEntry) { continue; } - matchedNameDictionary = true; - break; - } - if (matchedNameDictionary) { break; } + // Every match already comes from currentMediaDictionaryEntries, so + // classifying its own entry is enough. + for (const match of exactPrimaryMatches) { + if (!isCurrentMediaNameDictionaryEntry(match.dictionaryEntry)) { continue; } + matchedNameDictionary = true; + break; } } const preferredMatch = exactPrimaryMatches[0]; diff --git a/src/core/services/tokenizer/yomitan-scan-runtime-script.ts b/src/core/services/tokenizer/yomitan-scan-runtime-script.ts index 64a91a9e..f6c9ffeb 100644 --- a/src/core/services/tokenizer/yomitan-scan-runtime-script.ts +++ b/src/core/services/tokenizer/yomitan-scan-runtime-script.ts @@ -12,7 +12,7 @@ export type YomitanFrequencyMode = 'occurrence-based' | 'rank-based'; // Bump whenever the install script below changes so already-loaded parser // windows re-install the new scan runtime instead of running the stale one. -export const YOMITAN_SCAN_RUNTIME_VERSION = 10; +export const YOMITAN_SCAN_RUNTIME_VERSION = 12; export const YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL = '__subminer-yomitan-scan-runtime-missing__'; export interface YomitanScanRequestParams { @@ -318,29 +318,18 @@ ${YOMITAN_SCANNING_HELPERS} function isHalfwidthKanaVoicedMarkCodePoint(codePoint) { return codePoint === 0xff9e || codePoint === 0xff9f; } - function hasHalfwidthVoicedMarkInCandidateRegion(position, regionLength) { - const end = Math.min(text.length, position + regionLength); - for (let index = position; index < end; index += 1) { - if (isHalfwidthKanaVoicedMarkCodePoint(text.charCodeAt(index))) { return true; } - } - return false; - } // Build (once per candidate list) a first-character bucket index of the // normalized name forms, so the pre-pass can reject a position with a // single map hit instead of a backend round trip. if (rawNameCandidates && nameCandidateIndex?.key !== rawNameCandidates.key) { const byFirstChar = new Map(); - let longestFormLength = 0; for (const form of rawNameCandidates.forms) { const normalized = typeof form === "string" ? convertKatakanaToHiragana(form.trim()) : ""; if (!normalized) { continue; } - if (normalized.length > longestFormLength) { longestFormLength = normalized.length; } const bucket = byFirstChar.get(normalized[0]); if (bucket) { bucket.push(normalized); } else { byFirstChar.set(normalized[0], [normalized]); } } - nameCandidateIndex = byFirstChar.size > 0 - ? { key: rawNameCandidates.key, byFirstChar, longestFormLength } - : null; + nameCandidateIndex = byFirstChar.size > 0 ? { key: rawNameCandidates.key, byFirstChar } : null; } else if (!rawNameCandidates) { nameCandidateIndex = null; } @@ -371,26 +360,52 @@ ${YOMITAN_SCANNING_HELPERS} } return true; } - // Doubled because matching may skip emphatic characters as it goes, so a - // form can span more text than it has characters; capped at the window a - // name lookup covers anyway. - const candidateRegionLength = activeNameCandidateIndex - ? Math.min(scanLength, activeNameCandidateIndex.longestFormLength * 2) - : 0; + // Where the folding gives up, listed once per line. Matching may skip any + // number of emphatic characters on its way through a form (山ーーーーーーｶﾞｸ), + // so there is no shorter honest bound than the window a name lookup + // covers: scanLength. The list is almost always empty, which is what + // keeps the check below free on ordinary lines. + const halfwidthVoicedMarkPositions = []; + if (activeNameCandidateIndex) { + for (let index = 0; index < text.length; index += 1) { + if (isHalfwidthKanaVoicedMarkCodePoint(text.charCodeAt(index))) { + halfwidthVoicedMarkPositions.push(index); + } + } + } + function hasHalfwidthVoicedMarkInScanWindow(position) { + const end = position + scanLength; + for (const markPosition of halfwidthVoicedMarkPositions) { + if (markPosition >= position && markPosition < end) { return true; } + } + return false; + } + // A name written ｶﾞ... folds to か + ﾞ, so its first character never leads + // to the が bucket the candidate form is filed under. Nothing else can + // find it, so such a position is always worth a probe. + function startsHalfwidthVoicedPair(position, codePoint) { + if (codePoint < 0xff66 || codePoint > 0xff9d) { return false; } + return isHalfwidthKanaVoicedMarkCodePoint(text.charCodeAt(position + 1)); + } function couldNameStartAt(position, codePoint) { // Nothing starts with a combining voiced mark, whether or not the // prefilter is active. if (isHalfwidthKanaVoicedMarkCodePoint(codePoint)) { return false; } if (!activeNameCandidateIndex) { return true; } const bucket = activeNameCandidateIndex.byFirstChar.get(normalizedText[position]); - if (bucket) { - for (const form of bucket) { - if (matchesCandidateFormAt(form, position)) { return true; } - } + if (!bucket) { + // No candidate begins with this character, and the window search + // below would only ever say yes to positions like this one, so an + // unrelated ｶﾞ elsewhere in the line must not drag them in. + return startsHalfwidthVoicedPair(position, codePoint); } - // No match, but an unfoldable voiced pair in reach means the comparison - // above could not have seen one: probe rather than drop the name. - return hasHalfwidthVoicedMarkInCandidateRegion(position, candidateRegionLength); + for (const form of bucket) { + if (matchesCandidateFormAt(form, position)) { return true; } + } + // A candidate does start here but did not match: an unfoldable voiced + // pair anywhere in the window is a reason the comparison could not see + // it (山ｶﾞｸ, 山ーーーーーーｶﾞｸ), so probe rather than drop the name. + return hasHalfwidthVoicedMarkInScanWindow(position); } // Greedy name pre-pass: character-name matches claim their spans before // the left-to-right walk, so a longer generic match starting earlier diff --git a/src/core/services/tokenizer/yomitan-scan-runtime.test.ts b/src/core/services/tokenizer/yomitan-scan-runtime.test.ts index 3c61f830..26713464 100644 --- a/src/core/services/tokenizer/yomitan-scan-runtime.test.ts +++ b/src/core/services/tokenizer/yomitan-scan-runtime.test.ts @@ -139,6 +139,45 @@ test('a voiced halfwidth name still bypasses the candidate prefilter', async () assert.equal(result?.[1]?.isNameMatch, true); }); +test('an unrelated halfwidth voiced word does not restore the exhaustive pre-pass', async () => { + const baseline: string[] = []; + await requestYomitanScanTokens( + NAME_SCAN_LINE, + createNameScanDeps(baseline), + { error: () => undefined }, + { + includeNameMatchMetadata: true, + currentCharacterDictionaryMediaId: 1, + nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] }, + }, + ); + + const withVoicedTail: string[] = []; + await requestYomitanScanTokens( + `${NAME_SCAN_LINE}ｶﾞ`, + createNameScanDeps(withVoicedTail), + { error: () => undefined }, + { + includeNameMatchMetadata: true, + currentCharacterDictionaryMediaId: 1, + nameCandidates: { key: 'media-1', forms: ['ミナト', 'みなと'] }, + }, + ); + + // Mid-token positions are the ones only the pre-pass would ever probe. A ｶﾞ + // anywhere in the line used to drag every position within scanLength of it + // back in; now only the voiced pair itself, which the fold cannot index, is + // added to what the line already looked up. + for (const midTokenPrefix of ['ナト', 'だ学', '校に', 'ない']) { + assert.equal(countTermsFindLookups(baseline, midTokenPrefix), 0, midTokenPrefix); + assert.equal(countTermsFindLookups(withVoicedTail, midTokenPrefix), 0, midTokenPrefix); + } + assert.ok( + withVoicedTail.length - baseline.length <= 3, + `expected the ｶﾞ tail to add only its own lookups, saw ${JSON.stringify(withVoicedTail)}`, + ); +}); + test('a mixed-width voiced name survives the candidate prefilter', async () => { const lookups: string[] = []; const result = await requestYomitanScanTokens( @@ -156,9 +195,10 @@ test('a mixed-width voiced name survives the candidate prefilter', async () => { }, ); - // The name starts on a kanji, so a bypass keyed on the first character misses - // it: 山ｶﾞｸ normalizes to 山かﾞく, which cannot match the candidate 山がく, and - // the generic word starting earlier then swallows the 山. + // The name starts on a kanji, so the fold only breaks mid-name: 山ｶﾞｸ + // normalizes to 山かﾞく, which still cannot match the candidate 山がく. The + // bypass is keyed on the scan window rather than the first character, so the + // position is still probed and the generic まだ山 cannot swallow the 山. assert.deepEqual( result?.map((token) => token.surface), ['まだ', '山ｶﾞｸ'], @@ -166,6 +206,33 @@ test('a mixed-width voiced name survives the candidate prefilter', async () => { assert.equal(result?.[1]?.isNameMatch, true); }); +test('a stretched mixed-width voiced name survives the candidate prefilter', async () => { + const lookups: string[] = []; + const result = await requestYomitanScanTokens( + 'まだ山ーーーーーーｶﾞｸ', + createNameScanDeps(lookups, [ + ['まだ山', 'まだ山', 'まだやま', false], + ['まだ', 'まだ', 'まだ', false], + ['山ーーーーーーｶﾞｸ', '山ガク', 'やまがく', true], + ]), + { error: () => undefined }, + { + includeNameMatchMetadata: true, + currentCharacterDictionaryMediaId: 1, + nameCandidates: { key: 'media-1', forms: ['山ガク', 'やまがく'] }, + }, + ); + + // Matching skips any number of emphatic characters, so the voiced mark that + // defeats the fold can sit arbitrarily far into the name: the search for it + // has to cover the whole lookup window, not a multiple of the form length. + assert.deepEqual( + result?.map((token) => token.surface), + ['まだ', '山ーーーーーーｶﾞｸ'], + ); + assert.equal(result?.[1]?.isNameMatch, true); +}); + test('halfwidth voiced kana compose into the reading instead of leaving a stray mark', async () => { const lookups: string[] = []; const result = await requestYomitanScanTokens(