fix(subtitles): drop ASS furigana from recorded cues (#233)

changes/subtitle-recorders-drop-ass-furigana.md

@@ -0,0 +1,4 @@
+type: fixed
+area: subtitles
+
+- Copying the current subtitle, Anki sentence mining from recent lines, and immersion stats no longer include the separate furigana lines that broadcast-caption ASS files place above a word; recorders now use the same furigana-free text the overlay displays.

docs/architecture/subtitle-overlay-priming.md

@@ -131,7 +131,9 @@ coming and prefetching would otherwise idle for the rest of the cue.
   authored source order when no usable position exists.
 - Half-size kana positioned directly above a same-timed kanji caption is treated as ASS
   furigana. The parser omits it from published cues but retains hidden matching metadata so
-  mpv's raw live text can be reconciled without displaying or mining the reading.
+  mpv's raw live text can be reconciled without displaying or mining the reading. The
+  timing tracker (clipboard copy, recent-line mining) and immersion recorders run the same
+  reconciliation on the `sub-start`/`sub-end` sample, so they record what the overlay shows.
 - Fragment-only ASS karaoke is reconstructed per style before publication. Explicit spaces
   survive concatenation. Latin fragment typesetting with no literal spaces also recovers word
   boundaries represented only by materially larger horizontal `\pos` or `\move` gaps within that

src/main/runtime/mpv-main-event-main-deps.test.ts

@@ -1,5 +1,6 @@
 import assert from 'node:assert/strict';
 import test from 'node:test';
+import { parseAssCues } from '../../core/services/subtitle-cue-parser';
 import { createBuildBindMpvMainEventHandlersMainDepsHandler } from './mpv-main-event-main-deps';
 
 test('mpv main event main deps map app state updates and delegate callbacks', async () => {
@@ -582,3 +583,156 @@ test('subtitle-track changes stop stale canonical cues from substituting immedia
   assert.equal(appState.activeParsedSubtitleSource, null);
   assert.equal(handlers.resolveSubtitleText?.('今\n手にある'), '今\n手にある');
 });
+
+test('subtitle recorders drop ASS furigana events the same way the display does', () => {
+  // Broadcast-caption ASS (Caption2Ass style): furigana are separate half-scale events
+  // positioned above their base line, and mpv lists them as their own live lines.
+  const cues = parseAssCues(
+    [
+      '[Script Info]',
+      'PlayResX: 960',
+      'PlayResY: 540',
+      '',
+      '[V4+ Styles]',
+      'Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding',
+      'Style: Default,Yu Gothic,46,&H00FFFFFF,&H000000FF,&H00000000,&H7F000000,1,0,0,0,100,100,4,0,1,2,2,1,0,0,0,1',
+      '',
+      '[Events]',
+      'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
+      'Dialogue: 0,0:04:56.26,0:04:59.63,Default,,0000,0000,0000,,{\\pos(472,443)\\fscx50\\fscy50}あくむ',
+      'Dialogue: 0,0:04:56.26,0:04:59.63,Default,,0000,0000,0000,,{\\pos(172,497)}こんな短時間で{\\fscx50}　{\\fscx100}悪夢{\\fscx50}　{\\fscx100}見んなよ{\\fscx50}。',
+      'Dialogue: 0,0:04:59.63,0:05:03.54,Default,,0000,0000,0000,,{\\pos(172,407)\\fscx50}（{\\fscx100}平{\\fscx50}）{\\fscx100}暗記教科は{\\fscx50}　{\\fscx100}もう',
+      'Dialogue: 0,0:04:59.63,0:05:03.54,Default,,0000,0000,0000,,{\\pos(332,443)\\fscx50\\fscy50}かた',
+      'Dialogue: 0,0:04:59.63,0:05:03.54,Default,,0000,0000,0000,,{\\pos(412,443)\\fscx50\\fscy50}ぱし',
+      'Dialogue: 0,0:04:59.63,0:05:03.54,Default,,0000,0000,0000,,{\\pos(172,497)}とにかく片っ端から覚えるんだよ{\\fscx50}。',
+    ].join('\n'),
+  );
+  assert.deepEqual(
+    cues.map((cue) => cue.text),
+    [
+      'こんな短時間で　悪夢　見んなよ。',
+      '（平）暗記教科は　もう',
+      'とにかく片っ端から覚えるんだよ。',
+    ],
+  );
+
+  const immersion: string[] = [];
+  const timing: string[] = [];
+  const handlers = createBuildBindMpvMainEventHandlersMainDepsHandler({
+    appState: {
+      initialArgs: null,
+      overlayRuntimeInitialized: true,
+      mpvClient: { currentTimePos: 299.7 },
+      immersionTracker: { recordSubtitleLine: (text: string) => immersion.push(text) },
+      subtitleTimingTracker: { recordSubtitle: (text: string) => timing.push(text) },
+      activeParsedSubtitleCues: cues,
+      currentSubText: '',
+      currentSubAssText: '',
+      playbackPaused: null,
+      previousSecondarySubVisibility: false,
+    },
+    getQuitOnDisconnectArmed: () => false,
+    scheduleQuitCheck: () => {},
+    quitApp: () => {},
+    reportJellyfinRemoteStopped: () => {},
+    syncOverlayMpvSubtitleSuppression: () => {},
+    maybeRunAnilistPostWatchUpdate: async () => {},
+    logSubtitleTimingError: () => {},
+    broadcastToOverlayWindows: () => {},
+    onSubtitleChange: () => {},
+    ensureImmersionTrackerInitialized: () => {},
+    updateCurrentMediaPath: () => {},
+    restoreMpvSubVisibility: () => {},
+    getCurrentAnilistMediaKey: () => null,
+    resetAnilistMediaTracking: () => {},
+    maybeProbeAnilistDuration: () => {},
+    ensureAnilistMediaGuess: () => {},
+    syncImmersionMediaState: () => {},
+    updateCurrentMediaTitle: () => {},
+    resetAnilistMediaGuessState: () => {},
+    reportJellyfinRemoteProgress: () => {},
+    updateSubtitleRenderMetrics: () => {},
+    refreshDiscordPresence: () => {},
+  })();
+
+  const liveText = '（平）暗記教科は　もう\nかた\nぱし\nとにかく片っ端から覚えるんだよ。';
+  const expected = '（平）暗記教科は　もう\n\nとにかく片っ端から覚えるんだよ。';
+  assert.equal(handlers.resolveSubtitleText?.(liveText), expected);
+  handlers.recordImmersionSubtitleLine(liveText, 299.63, 303.54);
+  handlers.recordSubtitleTiming(liveText, 299.63, 303.54);
+
+  assert.deepEqual(immersion, [expected]);
+  assert.deepEqual(timing, [expected]);
+});
+
+test('a resolved line survives recording while a fragment grid is on screen', () => {
+  // Fragment stripping drops every line it can trace back to a cue, and returns nothing
+  // at all when a fragment grid is nearby. Text the parsed cues already resolved is a
+  // complete line, not raw mpv output, so it must not be fed through that path.
+  const immersion: string[] = [];
+  const timing: string[] = [];
+  const handlers = createBuildBindMpvMainEventHandlersMainDepsHandler({
+    appState: {
+      initialArgs: null,
+      overlayRuntimeInitialized: true,
+      mpvClient: { currentTimePos: 3.2 },
+      immersionTracker: { recordSubtitleLine: (text: string) => immersion.push(text) },
+      subtitleTimingTracker: { recordSubtitle: (text: string) => timing.push(text) },
+      activeParsedSubtitleCues: [
+        { startTime: 3, endTime: 6, text: '飛び越えてみたくて', source: 'canonical-ass' },
+        {
+          startTime: 3,
+          endTime: 6,
+          text: 'MaidCafeMaidCafe',
+          source: 'reconstructed-ass',
+          assLayout: { kind: 'fragment-grid', sourceOrder: 2 },
+        },
+      ],
+      currentSubText: '',
+      currentSubAssText: '',
+      playbackPaused: null,
+      previousSecondarySubVisibility: false,
+    },
+    getQuitOnDisconnectArmed: () => false,
+    scheduleQuitCheck: () => {},
+    quitApp: () => {},
+    reportJellyfinRemoteStopped: () => {},
+    syncOverlayMpvSubtitleSuppression: () => {},
+    maybeRunAnilistPostWatchUpdate: async () => {},
+    logSubtitleTimingError: () => {},
+    broadcastToOverlayWindows: () => {},
+    onSubtitleChange: () => {},
+    ensureImmersionTrackerInitialized: () => {},
+    updateCurrentMediaPath: () => {},
+    restoreMpvSubVisibility: () => {},
+    getCurrentAnilistMediaKey: () => null,
+    resetAnilistMediaTracking: () => {},
+    maybeProbeAnilistDuration: () => {},
+    ensureAnilistMediaGuess: () => {},
+    syncImmersionMediaState: () => {},
+    updateCurrentMediaTitle: () => {},
+    resetAnilistMediaGuessState: () => {},
+    reportJellyfinRemoteProgress: () => {},
+    updateSubtitleRenderMetrics: () => {},
+    refreshDiscordPresence: () => {},
+  })();
+
+  // The grid fragment beside the lyric keeps canonical substitution from applying, so
+  // recording falls to the parsed view -- which is where the whole line is recovered.
+  const liveText = '飛び越え\nMaid';
+  assert.equal(handlers.resolveSubtitleText?.(liveText), '飛び越えてみたくて');
+  handlers.recordImmersionSubtitleLine(liveText, 3, 6);
+  handlers.recordSubtitleTiming(liveText, 3, 6);
+
+  assert.deepEqual(immersion, ['飛び越えてみたくて']);
+  assert.deepEqual(timing, ['飛び越えてみたくて']);
+
+  // A spacer event left as literal control debris is not a subtitle line. The display
+  // drops it, so no recorder may keep it either.
+  assert.equal(handlers.resolveSubtitleText?.('\\'), '');
+  handlers.recordImmersionSubtitleLine('\\', 3, 6);
+  handlers.recordSubtitleTiming('\\', 3, 6);
+
+  assert.equal(immersion.length, 1);
+  assert.equal(timing.length, 1);
+});

src/main/runtime/mpv-main-event-main-deps.ts

@@ -3,7 +3,7 @@ import type { MergedToken, SubtitleCue, SubtitleData } from '../../types';
 import {
   resolveCanonicalPrimarySubtitle,
   resolvePrimarySubtitleText,
-  stripCanonicalFragmentLines,
+  resolveRecordedPrimarySubtitleText,
 } from './primary-subtitle-text';
 
 type AnilistPostWatchRunOptions = {
@@ -131,10 +131,12 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
       currentTimeSec: startSec,
       cues: deps.appState.activeParsedSubtitleCues,
     });
-  // When substitution declined because dialogue shares the screen with a song, record
-  // the dialogue alone rather than the combined dialogue-plus-fragments stack.
-  const stripFragmentsForRecording = (liveText: string, startSec: number) =>
-    stripCanonicalFragmentLines({
+  // Recorders see the same text the overlay displays: mpv's live `sub-text` lists every
+  // simultaneously active ASS event, including furigana events the parser folded into
+  // their base line. Cues the caller already resolved canonically are recorded one by
+  // one above; everything else goes through the shared recording resolution.
+  const resolveTextForRecording = (liveText: string, startSec: number): string =>
+    resolveRecordedPrimarySubtitleText({
       liveText,
       currentTimeSec: startSec,
       cues: deps.appState.activeParsedSubtitleCues,
@@ -218,7 +220,7 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
         }
         return;
       }
-      text = stripFragmentsForRecording(text, start);
+      text = resolveTextForRecording(text, start);
       if (!text.trim()) {
         return;
       }
@@ -232,7 +234,7 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
       const secondaryText = deps.appState.mpvClient?.currentSecondarySubText || undefined;
       const canonical = resolveCanonicalSample(text, start);
       if (!canonical) {
-        const recordableText = stripFragmentsForRecording(text, start);
+        const recordableText = resolveTextForRecording(text, start);
         if (!recordableText.trim()) {
           return;
         }

src/main/runtime/primary-subtitle-text.ts

@@ -26,6 +26,13 @@ function cuesUseAssSyntax(cues: readonly SubtitleCue[] | null | undefined): bool
   );
 }
 
+function decodedLiveText(
+  liveText: string,
+  cues: readonly SubtitleCue[] | null | undefined,
+): string {
+  return cuesUseAssSyntax(cues) ? removeAssControlDebrisLines(liveText) : liveText;
+}
+
 function animationSpan(cue: SubtitleCue): { start: number; end: number } {
   return {
     start: cue.animationStartTime ?? cue.startTime,
@@ -290,14 +297,34 @@ export function stripCanonicalFragmentLines(options: {
   return removeLiveGlyphFragmentLines(options.liveText);
 }
 
+/**
+ * Recording text for a live sample. Callers substitute canonical cues themselves and
+ * record those cue by cue, so what is resolved here is the parsed view -- the one that
+ * folds ASS furigana events back into their base line. Its text is already a complete
+ * line, while fragment stripping takes raw mpv text and would discard a resolved line
+ * whole while a fragment grid is on screen, so only one of the two ever runs.
+ */
+export function resolveRecordedPrimarySubtitleText(options: {
+  liveText: string;
+  currentTimeSec: number;
+  cues: readonly SubtitleCue[] | null | undefined;
+}): string {
+  const liveText = decodedLiveText(options.liveText, options.cues);
+  if (!liveText.trim()) {
+    return liveText;
+  }
+  return (
+    resolveActiveParsedPrimarySubtitle({ ...options, liveText })?.text ??
+    stripCanonicalFragmentLines({ ...options, liveText })
+  );
+}
+
 export function resolvePrimarySubtitleText(options: {
   liveText: string;
   currentTimeSec: number;
   cues: readonly SubtitleCue[] | null | undefined;
 }): string {
-  const liveText = cuesUseAssSyntax(options.cues)
-    ? removeAssControlDebrisLines(options.liveText)
-    : options.liveText;
+  const liveText = decodedLiveText(options.liveText, options.cues);
   if (!liveText.trim()) {
     return liveText;
   }