diff --git a/changes/unparsed-token-hover-lookup.md b/changes/unparsed-token-hover-lookup.md new file mode 100644 index 00000000..49217f8c --- /dev/null +++ b/changes/unparsed-token-hover-lookup.md @@ -0,0 +1,4 @@ +type: fixed +area: overlay + +- Fixed subtitle text that Yomitan's parser cannot match (e.g. the truncated volitional in とこ戻ろ…) being rendered as plain, non-interactive text: it was invisible to hover/lookup and excluded from the n+1 word count, which could wrongly mark a sentence as n+1. Unparsed runs are now kept as hoverable tokens matching Yomitan's own segmentation; bracketed SFX/speaker captions and punctuation-only runs are still skipped. diff --git a/src/core/services/tokenizer.test.ts b/src/core/services/tokenizer.test.ts index 98bb8398..7e01a54c 100644 --- a/src/core/services/tokenizer.test.ts +++ b/src/core/services/tokenizer.test.ts @@ -1821,7 +1821,7 @@ test('tokenizeSubtitle returns null tokens when mecab throws', async () => { assert.deepEqual(result, { text: '猫です', tokens: null }); }); -test('tokenizeSubtitle uses Yomitan parser result when available and drops no-headword groups', async () => { +test('tokenizeSubtitle uses Yomitan parser result and keeps no-headword groups as surface tokens', async () => { const parserWindow = { isDestroyed: () => false, webContents: { @@ -1859,10 +1859,12 @@ test('tokenizeSubtitle uses Yomitan parser result when available and drops no-he ); assert.equal(result.text, '猫です'); - assert.equal(result.tokens?.length, 1); + assert.equal(result.tokens?.length, 2); assert.equal(result.tokens?.[0]?.surface, '猫'); assert.equal(result.tokens?.[0]?.reading, 'ねこ'); assert.equal(result.tokens?.[0]?.isKnown, false); + assert.equal(result.tokens?.[1]?.surface, 'です'); + assert.equal(result.tokens?.[1]?.headword, 'です'); }); test('tokenizeSubtitle logs selected Yomitan groups when debug toggle is enabled', async () => { diff --git a/src/core/services/tokenizer/parser-selection-stage.test.ts b/src/core/services/tokenizer/parser-selection-stage.test.ts index 5a58afa3..f190fbe6 100644 --- a/src/core/services/tokenizer/parser-selection-stage.test.ts +++ b/src/core/services/tokenizer/parser-selection-stage.test.ts @@ -128,6 +128,61 @@ test('drops scanning parser tokens which have no dictionary headword', () => { ); }); +// Regression: 「…とこ戻ろ…」 — Yomitan cannot deinflect the truncated volitional 戻ろ +// (the bare ろ rule is ichidan-only, 戻る is godan), so 戻 comes back with no headword +// while ろ matches an unrelated term (櫓). Dropping 戻 made it a plain text node: +// unhoverable, unannotated, and invisible to the n+1 candidate count. +test('emits unparsed non-caption text as a token with surface headword', () => { + const parseResults = [ + makeParseItem('scanning-parser', [ + [{ text: 'みんな', reading: 'みんな', headword: '皆' }], + [{ text: 'の', reading: 'の', headword: 'の' }], + [{ text: 'とこ', reading: 'とこ', headword: '所' }], + [{ text: '戻', reading: '' }], + [{ text: 'ろ', reading: 'ろ', headword: '櫓' }], + [{ text: '…', reading: '' }], + ]), + ]; + + const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword'); + assert.deepEqual( + tokens?.map((token) => ({ surface: token.surface, headword: token.headword })), + [ + { surface: 'みんな', headword: '皆' }, + { surface: 'の', headword: 'の' }, + { surface: 'とこ', headword: '所' }, + { surface: '戻', headword: '戻' }, + { surface: 'ろ', headword: '櫓' }, + ], + ); +}); + +test('still drops punctuation-only and whitespace-only unparsed runs', () => { + const parseResults = [ + makeParseItem('scanning-parser', [ + [{ text: '猫', reading: 'ねこ', headword: '猫' }], + [{ text: '…', reading: '' }], + [{ text: ' ', reading: '' }], + [{ text: '犬', reading: 'いぬ', headword: '犬' }], + ]), + ]; + + const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword'); + assert.equal(tokens?.map((token) => token.surface).join(','), '猫,犬'); +}); + +test('candidate with only unparsed tokens still yields no dictionary match', () => { + const parseResults = [ + makeParseItem('scanning-parser', [ + [{ text: '戻', reading: '' }], + [{ text: '轟', reading: '' }], + ]), + ]; + + const tokens = selectYomitanParseTokens(parseResults, () => false, 'headword'); + assert.equal(tokens, null); +}); + test('prefers the longest dictionary headword across merged segments', () => { const parseResults = [ makeParseItem('scanning-parser', [ diff --git a/src/core/services/tokenizer/parser-selection-stage.ts b/src/core/services/tokenizer/parser-selection-stage.ts index 1f0e9eda..0b5fee79 100644 --- a/src/core/services/tokenizer/parser-selection-stage.ts +++ b/src/core/services/tokenizer/parser-selection-stage.ts @@ -151,6 +151,39 @@ function isStandaloneGrammarEndingSegment(segment: YomitanParseSegment): boolean ); } +// Caption-style asides (SFX/speaker labels) start with a bracket and should stay +// non-interactive; dropping punctuation-only runs also keeps the source-text gap that +// sentence-boundary detection relies on (e.g. a dropped 「…」). +const CAPTION_OPENING_BRACKETS = new Set([ + '(', + '(', + '[', + '[', + '{', + '{', + '「', + '『', + '【', + '〈', + '《', + '≪', + '<', + '<', +]); + +function hasLookupWorthyText(text: string): boolean { + return /[\p{L}\p{N}]/u.test(text); +} + +function isCaptionLikeUnparsedText(text: string): boolean { + const firstChar = Array.from(text.trim())[0]; + return firstChar !== undefined && CAPTION_OPENING_BRACKETS.has(firstChar); +} + +function shouldEmitUnparsedRunAsToken(text: string): boolean { + return hasLookupWorthyText(text) && !isCaptionLikeUnparsedText(text); +} + function shouldMergeKanaContinuation( previousToken: MergedToken | undefined, continuationSurface: string, @@ -241,6 +274,11 @@ export function mapYomitanParseResultItemToMergedTokens( previousToken.surface += combinedSurface; previousToken.reading += combinedReading; previousToken.endPos = end; + } else if (shouldEmitUnparsedRunAsToken(combinedSurface)) { + // Yomitan couldn't parse this run (e.g. 戻ろ… truncated volitional). Keep it + // as a token with its surface as headword so it stays hoverable and counts in + // the n+1 math — matching what the embedded Yomitan actually returns. + pushToken(combinedSurface, combinedReading, combinedSurface, combinedStart, end); } } else { hasDictionaryMatch = true;