diff --git a/changes/subtitle-tokenization-performance.md b/changes/subtitle-tokenization-performance.md index b32b22a3..41ef3f0e 100644 --- a/changes/subtitle-tokenization-performance.md +++ b/changes/subtitle-tokenization-performance.md @@ -14,5 +14,7 @@ area: subtitles - A subtitle that was on screen when its annotations were invalidated (by mining a card, for example) is now re-annotated instead of staying plain for the rest of the line. - Character name annotations no longer cost a dictionary lookup at every position in a line. The scanner now knows which name forms the current title's character dictionary actually contains and only checks where one can start, which removes the whole overhead of having the character dictionary enabled (measured: 21 lookups per line down to 10, the same as with it disabled). Titles with no cached character data keep the previous exhaustive scan, so a missing snapshot costs speed rather than a missing name. - The cross-line termsFind cache is now bounded by the number of retained dictionary entries as well as by key count, so a run of lookups that each carry hundreds of entries with full glossaries cannot grow the parser window's memory without limit. The budget is re-checked when a lookup resolves, so a single oversized response is dropped rather than parked in the cache and reused. -- The unnamed-mob disambiguator filter (Girl A / Girl B) now targets the letters those labels are split into, instead of every one-character term: a character whose name really is one character (𠮷, or a single kana) keeps it. The character dictionary and the scanner's name pre-pass also share one Han code-point table now, so a name the dictionary accepts is a name the scanner will look for. -- A character name written in halfwidth katakana takes part in the greedy name pre-pass again, so a longer generic word can no longer swallow the start of it. +- The unnamed-mob disambiguator filter (Girl A / Girl B) now only drops a single letter or digit split off a name, instead of every one-character term: a name that is genuinely one character keeps its terms whatever the script (𠮷, あ, 별 김, ア・ベ). The character dictionary and the scanner's name pre-pass also share one Han code-point table now, so a name the dictionary accepts is a name the scanner will look for. +- A character name written in halfwidth katakana takes part in the greedy name pre-pass again, so a longer generic word can no longer swallow the start of it, and it now carries a reading (it used to come out blank, which disables known-word matching and frequency lookups for the token). Voiced halfwidth kana compose properly, so ｶﾞｸ reads ガク rather than カﾞク, and kana normalization folds halfwidth throughout so those tokens compare equal to the same word written fullwidth. +- Dictionary-entry classification (source dictionaries, character-dictionary media ids) is memoized per entry object for as long as the entry is cached, instead of being recomputed for every headword comparison and every retry window. +- Autoplay priming no longer broadcasts the plain subtitle twice: it tells the processing controller the line has already been painted, so the controller goes straight to the annotated payload. diff --git a/docs/architecture/subtitle-overlay-priming.md b/docs/architecture/subtitle-overlay-priming.md index de8001ef..8e06612a 100644 --- a/docs/architecture/subtitle-overlay-priming.md +++ b/docs/architecture/subtitle-overlay-priming.md @@ -56,6 +56,10 @@ background prefetch work. Prefetch is not re-centered here: restarting the run p (`onSeek`) discarded the in-flight tokenization every time the subtitle changed, so only real seeks restart it (see `onTimePosUpdate` in `src/main.ts`). +On an uncached autoplay prime the raw payload is emitted here and reported to the controller with +`notePlainSubtitleEmitted`, so the controller skips its own plain emit for that line and the +overlay receives one plain payload followed by the annotated one. + The pause is released by the controller's `onProcessingSettled` callback, which fires once it has no work left. Emits do not release it: the first emit for an uncached line is the plain payload that precedes tokenization, and a run can finish without emitting at all (a suppressed duplicate, diff --git a/src/core/services/subtitle-processing-controller.test.ts b/src/core/services/subtitle-processing-controller.test.ts index eee10561..b2847979 100644 --- a/src/core/services/subtitle-processing-controller.test.ts +++ b/src/core/services/subtitle-processing-controller.test.ts @@ -635,6 +635,21 @@ test('onProcessingSettled fires once after the queue drains, including runs that assert.deepEqual(events, ['settled']); }); +test('notePlainSubtitleEmitted suppresses the controller repeat of a payload already shown', async () => { + const emitted: SubtitleData[] = []; + const controller = createSubtitleProcessingController({ + tokenizeSubtitle: async (text) => ({ text, tokens: [] }), + emitSubtitle: (payload) => emitted.push(payload), + }); + + // Autoplay priming paints the plain line itself, then asks for tokenization. + controller.notePlainSubtitleEmitted('字幕'); + controller.refreshCurrentSubtitle('字幕'); + await flushMicrotasks(); + + assert.deepEqual(emitted, [{ text: '字幕', tokens: [] }]); +}); + test('refreshCurrentSubtitle reports no emit for empty text when nothing is running', async () => { const emitted: SubtitleData[] = []; const controller = createSubtitleProcessingController({ diff --git a/src/core/services/subtitle-processing-controller.ts b/src/core/services/subtitle-processing-controller.ts index 1743617c..488691e7 100644 --- a/src/core/services/subtitle-processing-controller.ts +++ b/src/core/services/subtitle-processing-controller.ts @@ -34,6 +34,13 @@ export interface SubtitleProcessingController { onSubtitleChange: (text: string) => boolean; /** Same contract as onSubtitleChange: whether processing is pending. */ refreshCurrentSubtitle: (textOverride?: string) => boolean; + /** + * Records that this exact text has already been shown plain by someone else + * (autoplay priming paints its first frame before scheduling tokenization), + * so the controller does not repeat that payload on its way to the tokenized + * one. + */ + notePlainSubtitleEmitted: (text: string) => void; invalidateTokenizationCache: () => void; preCacheTokenization: (text: string, data: SubtitleData) => void; consumeCachedSubtitle: (text: string) => SubtitleData | null; @@ -224,6 +231,9 @@ export function createSubtitleProcessingController( processLatest(); return true; }, + notePlainSubtitleEmitted: (text: string) => { + lastPlainEmittedText = text; + }, invalidateTokenizationCache: () => { tokenizationCache.clear(); cacheGeneration += 1; diff --git a/src/core/services/tokenizer/token-classification.test.ts b/src/core/services/tokenizer/token-classification.test.ts index 6d5c699b..fe4d3c64 100644 --- a/src/core/services/tokenizer/token-classification.test.ts +++ b/src/core/services/tokenizer/token-classification.test.ts @@ -8,6 +8,7 @@ import { isKanaChar, isKanaOnlyText, isTokenPos2Excluded, + normalizeKana, } from './token-classification'; const POS1_EXCLUSIONS = new Set(['助詞']); @@ -29,6 +30,26 @@ function makeNoun(surface: string): MergedToken { }; } +test('kana normalization folds halfwidth kana, composing the voiced pairs', () => { + // ｶ + ﾞ is two code points for one character: without composing them, a + // halfwidth word counts as longer than the reading that spells it, which + // disqualifies the reading from known-word matching. + assert.equal(normalizeKana('ｶﾞｸ'), normalizeKana('ガク')); + assert.equal(normalizeKana('ﾊﾟﾝ'), normalizeKana('パン')); + assert.equal(normalizeKana('ﾐﾅﾄ'), 'みなと'); + assert.ok(isKanaOnlyText('ｶﾞｸ')); +}); + +test('kana normalization leaves characters other than halfwidth kana alone', () => { + // The composition is scoped to the halfwidth runs: applied to the whole + // string, NFKC would also rewrite these into something the dictionary, the + // known-word list, and the frequency data were never keyed on. + assert.equal(normalizeKana('①ｶﾞ'), '①が'); + assert.equal(normalizeKana('Ａｶﾞ'), 'Ａが'); + assert.equal(normalizeKana('㍑ｶﾞ'), '㍑が'); + assert.equal(normalizeKana('fiｶﾞ'), 'fiが'); +}); + test('kana classification excludes the katakana-hiragana double hyphen', () => { assert.equal(isKanaChar('゠'), false); assert.equal(isKanaOnlyText('゠'), false); diff --git a/src/core/services/tokenizer/token-classification.ts b/src/core/services/tokenizer/token-classification.ts index d5c46cf8..ba0632c8 100644 --- a/src/core/services/tokenizer/token-classification.ts +++ b/src/core/services/tokenizer/token-classification.ts @@ -4,8 +4,20 @@ const KATAKANA_TO_HIRAGANA_OFFSET = 0x60; const KATAKANA_CODEPOINT_START = 0x30a1; const KATAKANA_CODEPOINT_END = 0x30f6; +// No `u` flag: the range is entirely BMP so it changes nothing here, and +// Bun's unicode-mode matcher mis-handles this class next to certain ligatures. +const HALFWIDTH_KANA_RUN = /[\uff66-\uff9f]+/g; + +// NFKC over the halfwidth kana only, never the whole string: it composes the +// voiced pairs (ｶ + ﾞ) into single characters so ｶﾞｸ compares equal to ガク +// instead of counting one character longer than the word it spells, but run +// over everything it would also rewrite unrelated text (① → 1, ㍑ → リットル). +function composeHalfwidthKana(text: string): string { + return text.replace(HALFWIDTH_KANA_RUN, (run) => run.normalize('NFKC')); +} + export function normalizeKana(text: string): string { - const raw = text.trim(); + const raw = composeHalfwidthKana(text).trim(); if (!raw) { return ''; } diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts index 0628fbf2..9bffda78 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts @@ -268,6 +268,33 @@ test('requestYomitanScanTokens probes halfwidth katakana positions during the na ['まだ', 'ﾐﾅﾄ'], ); assert.equal(result?.[1]?.isNameMatch, true); + // The reading is written the way the fullwidth katakana path writes it + // (surface spelling, fullwidth): halfwidth kana is not kana to the known-word + // and frequency code downstream, and an empty reading there disables the + // reading fallback entirely. + assert.equal(result?.[1]?.reading, 'ミナト'); + assert.equal(result?.[1]?.headwordReading, 'みなと'); +}); + +test('halfwidth voiced kana compose into the reading instead of leaving a stray mark', async () => { + const lookups: string[] = []; + const result = await requestYomitanScanTokens( + 'ｶﾞｸ ﾊﾟﾝ', + createNameScanDeps(lookups, [ + ['ｶﾞｸ', 'ガク', 'がく', false], + ['ﾊﾟﾝ', 'パン', 'ぱん', false], + ]), + { error: () => undefined }, + { includeNameMatchMetadata: true }, + ); + + const readings = (result ?? []) + .filter((token) => token.isUnparsedRun !== true) + .map((token) => [token.surface, token.reading]); + assert.deepEqual(readings, [ + ['ｶﾞｸ', 'ガク'], + ['ﾊﾟﾝ', 'パン'], + ]); }); test('requestYomitanScanTokens falls back to the exhaustive name scan without candidates', async () => { diff --git a/src/core/services/tokenizer/yomitan-scan-runtime-script.ts b/src/core/services/tokenizer/yomitan-scan-runtime-script.ts index 6dfda1df..e3d12450 100644 --- a/src/core/services/tokenizer/yomitan-scan-runtime-script.ts +++ b/src/core/services/tokenizer/yomitan-scan-runtime-script.ts @@ -11,7 +11,7 @@ export type YomitanFrequencyMode = 'occurrence-based' | 'rank-based'; // Bump whenever the install script below changes so already-loaded parser // windows re-install the new scan runtime instead of running the stale one. -export const YOMITAN_SCAN_RUNTIME_VERSION = 6; +export const YOMITAN_SCAN_RUNTIME_VERSION = 7; export const YOMITAN_SCAN_RUNTIME_MISSING_SENTINEL = '__subminer-yomitan-scan-runtime-missing__'; export interface YomitanScanRequestParams { @@ -88,6 +88,14 @@ export const YOMITAN_SCAN_RUNTIME_INSTALL_SCRIPT = String.raw` dropCachedTermsFind(oldest[0], oldest[1]); } } + // Classification of a dictionary entry (which dictionaries it came from, + // which media ids it mentions) depends only on the entry object, so it is + // memoized for as long as that object lives. Entries are shared with the + // termsFind cache above, which is what makes this worth keeping: the same + // objects come back for every repeated lookup, on every line. + const dictionaryEntryNamesCache = new WeakMap(); + const subMinerMediaIdsCache = new WeakMap(); + const EMPTY_MEDIA_ID_SET = new Set(); // Only blind ladder steps are capped (see the retry loop): those are the // ones that would otherwise degrade into O(scanLength) lookups at a single // position. Steps the backend guides by reporting a shorter consumed length diff --git a/src/core/services/tokenizer/yomitan-scanning-helpers-script.ts b/src/core/services/tokenizer/yomitan-scanning-helpers-script.ts index 77c43aa4..f3c6e8d8 100644 --- a/src/core/services/tokenizer/yomitan-scanning-helpers-script.ts +++ b/src/core/services/tokenizer/yomitan-scanning-helpers-script.ts @@ -12,7 +12,30 @@ export const YOMITAN_SCANNING_HELPERS = String.raw` const KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0x30fc; const KATAKANA_SMALL_KA_CODE_POINT = 0x30f5; const KATAKANA_SMALL_KE_CODE_POINT = 0x30f6; - const KANA_RANGES = [[0x3040, 0x309f], [0x30a0, 0x30ff]]; + const KANA_RANGES = [[0x3040, 0x309f], [0x30a0, 0x30ff], [0xff66, 0xff9f]]; + const HALFWIDTH_KATAKANA_RANGE = [0xff66, 0xff9d]; + const HALFWIDTH_KANA_PROLONGED_SOUND_MARK_CODE_POINT = 0xff70; + // Folded one code point to one, so every index into a normalized string + // still lines up with the original text — the name-candidate prefilter + // and the furigana stem matching both index back into it. The standalone + // voiced marks (ﾞ ﾟ) have no one-character equivalent and stay as they are. + const HALFWIDTH_KATAKANA_TO_HIRAGANA = "をぁぃぅぇぉゃゅょっーあいうえおかきくけこさしすせそたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわん"; + function convertHalfwidthKanaCodePointToHiragana(codePoint) { + if (codePoint < HALFWIDTH_KATAKANA_RANGE[0] || codePoint > HALFWIDTH_KATAKANA_RANGE[1]) { return null; } + return HALFWIDTH_KATAKANA_TO_HIRAGANA[codePoint - HALFWIDTH_KATAKANA_RANGE[0]] || null; + } + // Halfwidth katakana is kana here but not to the rest of the pipeline + // (known-word matching and frequency lookups only fold fullwidth), so a + // reading taken from halfwidth text is written the way the fullwidth + // katakana path already writes it. NFKC rather than the per-code-point + // table: this is the one place where nothing indexes back into the + // result, so a voiced pair (ｶ + ﾞ) can compose into the single ガ it + // means instead of leaving a stray combining mark in the reading. Scoped + // to the halfwidth runs, because NFKC over everything else rewrites + // characters that have nothing to do with kana (① → 1, ㍑ → リットル). + function convertHalfwidthKanaToKatakana(text) { + return text.replace(/[ｦ-ﾟ]+/g, (run) => run.normalize("NFKC")); + } // Han ranges come from the shared table so the scan walk and the character // dictionary agree on what a kanji is (supplementary planes included). // Halfwidth katakana counts as Japanese text: a name written that way has @@ -27,7 +50,7 @@ export const YOMITAN_SCANNING_HELPERS = String.raw` if (typeof segment.reading === "string" && segment.reading.length > 0) { return segment.reading; } const segmentText = typeof segment.text === "string" ? segment.text : ""; const isKanaOnly = segmentText.length > 0 && [...segmentText].every((char) => isCodePointKana(char.codePointAt(0))); - return isKanaOnly ? segmentText : ""; + return isKanaOnly ? convertHalfwidthKanaToKatakana(segmentText) : ""; } function getProlongedHiragana(previousCharacter) { switch (previousCharacter) { @@ -63,6 +86,8 @@ export const YOMITAN_SCANNING_HELPERS = String.raw` case KATAKANA_SMALL_KE_CODE_POINT: break; case KANA_PROLONGED_SOUND_MARK_CODE_POINT: + case HALFWIDTH_KANA_PROLONGED_SOUND_MARK_CODE_POINT: + char = "ー"; if (!keepProlongedSoundMarks && result.length > 0) { const char2 = getProlongedHiragana(result[result.length - 1]); if (char2 !== null) { char = char2; } @@ -71,7 +96,12 @@ export const YOMITAN_SCANNING_HELPERS = String.raw` default: if (isCodePointInRange(codePoint, KATAKANA_CONVERSION_RANGE)) { char = String.fromCodePoint(codePoint + offset); + break; } + // Halfwidth katakana folds too, or a name written that way would + // match neither a candidate form nor its own reading. + const halfwidthHiragana = convertHalfwidthKanaCodePointToHiragana(codePoint); + if (halfwidthHiragana !== null) { char = halfwidthHiragana; } break; } result += char; @@ -331,7 +361,14 @@ export const YOMITAN_SCANNING_HELPERS = String.raw` } } } + // Memoized on the entry object: termsFind results are cached across + // lines, so the same entries come back for every repeated lookup, and + // each one is classified several times per scan (name pre-pass, + // headword preference, every retry window). function getDictionaryEntryNames(entry) { + if (!entry || typeof entry !== 'object') { return []; } + const cached = dictionaryEntryNamesCache.get(entry); + if (cached !== undefined) { return cached; } const names = []; appendDictionaryNames(names, entry); for (const definition of entry?.definitions || []) { @@ -343,13 +380,21 @@ export const YOMITAN_SCANNING_HELPERS = String.raw` for (const pronunciation of entry?.pronunciations || []) { appendDictionaryNames(names, pronunciation); } + dictionaryEntryNamesCache.set(entry, names); return names; } + // Cached per scan rather than per runtime: the answer depends on + // includeNameMatchMetadata, which is a per-call parameter. + const nameDictionaryEntryCache = new WeakMap(); function isNameDictionaryEntry(entry) { if (!includeNameMatchMetadata || !entry || typeof entry !== 'object') { return false; } - return getDictionaryEntryNames(entry).some((name) => name.startsWith(${JSON.stringify(CHARACTER_DICTIONARY_TITLE_PREFIX)})); + const cached = nameDictionaryEntryCache.get(entry); + if (cached !== undefined) { return cached; } + const isName = getDictionaryEntryNames(entry).some((name) => name.startsWith(${JSON.stringify(CHARACTER_DICTIONARY_TITLE_PREFIX)})); + nameDictionaryEntryCache.set(entry, isName); + return isName; } function parseSubMinerMediaIdFromString(value) { const imageMatch = value.match(/\bimg\/m(\d+)-/i); @@ -403,9 +448,16 @@ export const YOMITAN_SCANNING_HELPERS = String.raw` collectSubMinerMediaIds(child, target); } } + // Walking an entry collects media ids from every nested value, so this + // is the most expensive classification step; memoized on the entry for + // the same reason as the dictionary names above. function getSubMinerMediaIds(entry) { + if (!entry || typeof entry !== 'object') { return EMPTY_MEDIA_ID_SET; } + const cached = subMinerMediaIdsCache.get(entry); + if (cached !== undefined) { return cached; } const mediaIds = new Set(); collectSubMinerMediaIds(entry, mediaIds); + subMinerMediaIdsCache.set(entry, mediaIds); return mediaIds; } function isCurrentMediaNameDictionaryEntry(entry) { diff --git a/src/main/character-dictionary-runtime/term-building.test.ts b/src/main/character-dictionary-runtime/term-building.test.ts index fe4fde4a..6b8d12e3 100644 --- a/src/main/character-dictionary-runtime/term-building.test.ts +++ b/src/main/character-dictionary-runtime/term-building.test.ts @@ -65,16 +65,75 @@ test('buildNameTerms keeps a character whose whole name is one kana', () => { }), ); - // The mob-disambiguator filter targets letters split off a longer name; an - // explicit one-character name is the character's actual name. + // The mob-label rule only judges parts a name was split into; a name the + // source gives us whole is the character's actual name. assert.ok(terms.includes('あ')); assert.ok(terms.includes('あさん')); - // The romanized "A" is still a label, so it contributes neither itself nor - // its single-kana alias. + // Romanized forms are never lookup targets (the subtitles are Japanese), and + // the single-kana alias "A" transliterates to is dropped as a collision. assert.ok(!terms.includes('A')); assert.ok(!terms.includes('ア')); }); +test('buildNameTerms keeps a one-character name written in another script', () => { + const terms = buildNameTerms( + characterRecord({ + firstNameHint: '', + lastNameHint: '', + fullName: 'Byeol', + nativeName: '별', + alternativeNames: ['Я'], + }), + ); + + assert.ok(terms.includes('별')); + assert.ok(terms.includes('별さん')); + assert.ok(terms.includes('Я')); +}); + +test('buildNameTerms yields nothing for a character whose only name is a bare letter', () => { + // Documented policy rather than an oversight: a romanized name is never a + // term on its own (the subtitles are Japanese), and the single kana a bare + // letter transliterates to would match every あ〜 in the line. + assert.deepEqual( + buildNameTerms( + characterRecord({ + firstNameHint: '', + lastNameHint: '', + fullName: 'A', + nativeName: '', + }), + ), + [], + ); +}); + +test('buildNameTerms keeps one-character split parts that are not mob labels', () => { + const hangul = buildNameTerms( + characterRecord({ + firstNameHint: '', + lastNameHint: '', + fullName: 'Byeol Kim', + nativeName: '별 김', + }), + ); + + assert.ok(hangul.includes('별')); + assert.ok(hangul.includes('김')); + + const middleDot = buildNameTerms( + characterRecord({ + firstNameHint: '', + lastNameHint: '', + fullName: 'A Be', + nativeName: 'ア・ベ', + }), + ); + + assert.ok(middleDot.includes('ア')); + assert.ok(middleDot.includes('ベ')); +}); + test('buildNameTerms keeps a single-kanji name part', () => { // The name is an alias, not the native name, so the parts come from the // space split rather than from the native-name split. diff --git a/src/main/character-dictionary-runtime/term-building.ts b/src/main/character-dictionary-runtime/term-building.ts index 8fdeb00a..30d9b8ee 100644 --- a/src/main/character-dictionary-runtime/term-building.ts +++ b/src/main/character-dictionary-runtime/term-building.ts @@ -43,22 +43,17 @@ export function expandRawNameVariants(rawName: string): string[] { return [...variants]; } -// Kana, halfwidth included: one of these can stand alone as a name, where a -// latin letter or a digit cannot. -const SINGLE_KANA_CHARACTER = /^[\u3040-\u30ff\u31f0-\u31ff\uff66-\uff9f]$/u; +// The label AniList appends to unnamed mob characters: one letter or digit, +// halfwidth or fullwidth (女子A / "Joshi A" / 女子１). Nothing else qualifies — +// a one-character part in any script is a real name part (별 김, ア・ベ, 山田 空). +const SINGLE_LABEL_CHARACTER = /^[0-9A-Za-z\uff10-\uff19\uff21-\uff3a\uff41-\uff5a]$/u; -// AniList disambiguates unnamed mob characters with a trailing letter (女子A / -// "Joshi A"), and a lone letter romanizes into a single-kana alias (A → ア) -// that collides with interjections (あ〜 matching ア). That letter is a label, -// not a name, so it is dropped where a name splits into it and before it can -// become a kana alias. A name that is genuinely one character, a character -// actually called あ or a single kanji, is a real lookup target and is kept. -function isNameDisambiguatorLetter(name: string): boolean { - return [...name].length === 1 && !containsKanji(name) && !SINGLE_KANA_CHARACTER.test(name); -} - -function isUsableNameTerm(name: string): boolean { - return !isNameDisambiguatorLetter(name); +// Judged on split parts only: a name the source gives us whole in a script the +// subtitles can contain is kept whatever it looks like, because a character +// really can be called あ or 별. (A romanized name is a separate matter: it is +// never a term on its own, only a source of kana aliases. See below.) +function isUsableNameSplitPart(part: string): boolean { + return !SINGLE_LABEL_CHARACTER.test(part); } // Kana, Han (shared ranges), and the marks that only ever appear inside a @@ -122,7 +117,7 @@ export function buildNameTerms( const split = name.split(/[\s\u3000]+/).filter((part) => part.trim().length > 0); if (split.length === 2) { for (const part of split) { - if (isUsableNameTerm(part)) { + if (isUsableNameSplitPart(part)) { target.add(part); } } @@ -134,7 +129,7 @@ export function buildNameTerms( .filter((part) => part.length > 0); if (splitByMiddleDot.length >= 2) { for (const part of splitByMiddleDot) { - if (isUsableNameTerm(part)) { + if (isUsableNameSplitPart(part)) { target.add(part); } } @@ -146,10 +141,15 @@ export function buildNameTerms( } } - // Romanized forms that are a bare letter would become a single-kana alias. - for (const alias of addRomanizedKanaAliases( - [...romanizedBase].filter((entry) => !isNameDisambiguatorLetter(entry)), - )) { + // Romanized names never become terms themselves — the subtitles are Japanese, + // so "Joshi A" would never appear in one — they only contribute the kana a + // Japanese writer would spell them with. + for (const alias of addRomanizedKanaAliases(romanizedBase)) { + // Except when the whole name is one letter: it transliterates to a single + // kana (A → ア) that matches every あ〜 in the subtitles. A character whose + // only recorded name is a bare letter therefore yields no terms at all, + // which is the intended outcome: those are unnamed mob characters. + if ([...alias].length === 1) continue; base.add(alias); } @@ -168,9 +168,6 @@ export function buildNameTerms( const withHonorifics = new Set(); for (const entry of base) { - // Only labels split off a longer name are filtered (see above); an explicit - // one-character name reaches this point intact. - if (isNameDisambiguatorLetter(entry)) continue; withHonorifics.add(entry); for (const suffix of HONORIFIC_SUFFIXES) { withHonorifics.add(`${entry}${suffix.term}`); diff --git a/src/main/runtime/autoplay-subtitle-priming-runtime.test.ts b/src/main/runtime/autoplay-subtitle-priming-runtime.test.ts index a3969c68..25f0ef52 100644 --- a/src/main/runtime/autoplay-subtitle-priming-runtime.test.ts +++ b/src/main/runtime/autoplay-subtitle-priming-runtime.test.ts @@ -46,6 +46,7 @@ test('scheduleSubtitlePrefetchRefresh logs refresh failures from timer callback' consumeCachedSubtitle: () => null, onSubtitleChange: () => true, refreshCurrentSubtitle: () => true, + notePlainSubtitleEmitted: () => {}, }, emitSubtitlePayload: () => {}, getSubtitlePrefetchService: () => null, @@ -103,6 +104,7 @@ test('primeCurrentSubtitleForAutoplay refreshes active subtitle cues when mpv su calls.push(`refresh:${text ?? ''}`); return true; }, + notePlainSubtitleEmitted: () => {}, }, emitSubtitlePayload: (payload, options) => calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`), @@ -174,6 +176,7 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before calls.push(`refresh:${text ?? ''}`); return true; }, + notePlainSubtitleEmitted: () => {}, }, emitSubtitlePayload: (payload, options) => calls.push(`emit:${payload.text}:resume=${options?.resumePrefetch !== false}`), @@ -395,7 +398,7 @@ test('prefetch stays paused until tokenization of an uncached line completes', a const tokenizationGate = new Promise((resolve) => { finishTokenization = resolve; }); - const { subtitleProcessingController } = createPrimingRuntimeWithRealController({ + const { runtime, mediaPath } = createPrimingRuntimeWithRealController({ text, calls, onTokenize: () => {}, @@ -405,17 +408,23 @@ test('prefetch stays paused until tokenization of an uncached line completes', a }, }); - subtitleProcessingController.onSubtitleChange(text); + // Driven through the priming path, which is what takes the pause out. + await runtime.primeCurrentSubtitleForAutoplay(mediaPath); await new Promise((resolve) => setTimeout(resolve, 0)); - // The provisional plain emit must not release the pause: the expensive scan - // is still ahead of it and would compete with prefetching for the parser. - assert.deepEqual(calls, [`emit:${text}:tokens=none`]); + // Neither the priming emit nor the controller's provisional plain emit may + // release the pause: the expensive scan is still ahead of them and would + // compete with prefetching for the parser. + // One plain payload, not two: priming paints it and tells the controller, so + // the controller goes straight for the tokenized one. And it does not resume + // prefetch, because the expensive scan is still ahead of it. + assert.deepEqual(calls, ['prefetch:pause', `emit-raw:${text}`]); finishTokenization(); await new Promise((resolve) => setTimeout(resolve, 0)); assert.deepEqual(calls, [ - `emit:${text}:tokens=none`, + 'prefetch:pause', + `emit-raw:${text}`, `emit:${text}:tokens=yes`, 'prefetch:resume', ]); diff --git a/src/main/runtime/autoplay-subtitle-priming-runtime.ts b/src/main/runtime/autoplay-subtitle-priming-runtime.ts index 244c40df..708fd259 100644 --- a/src/main/runtime/autoplay-subtitle-priming-runtime.ts +++ b/src/main/runtime/autoplay-subtitle-priming-runtime.ts @@ -33,6 +33,7 @@ export interface AutoplaySubtitlePrimingRuntimeDeps { // Both report whether processing is pending; see pausePrefetchUntilProcessed. onSubtitleChange: (text: string) => boolean; refreshCurrentSubtitle: (text: string) => boolean; + notePlainSubtitleEmitted: (text: string) => void; }; emitSubtitlePayload: (payload: SubtitleData, options?: { resumePrefetch?: boolean }) => void; getSubtitlePrefetchService: () => AutoplaySubtitlePrimingPrefetchService | null; @@ -124,8 +125,10 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi } // Provisional raw emit: keep prefetch paused until the processing - // controller is done with this line. + // controller is done with this line, and tell it this line has already been + // painted plain so it does not broadcast the same payload again. emitSubtitlePayload({ text, tokens: null }, { resumePrefetch: false }); + subtitleProcessingController.notePlainSubtitleEmitted(text); // refreshCurrentSubtitle, not onSubtitleChange: the cache miss above can be // an invalidation (mining a card) on text the controller still holds, and // onSubtitleChange treats unchanged text as nothing to do, which would