fix(subtitles): follow mpv sub-delay in mining, sidebar, and navigation (#302)

src/core/services/index.ts

@@ -55,6 +55,7 @@ export {
   subtitleCueListSeekTime,
   subtitleCueSeekTime,
 } from './subtitle-cue-navigation';
+export { shiftSubtitleRangeToVideoClock, toSubtitleClock, toVideoClock } from './subtitle-clock';
 export { createFrequencyDictionaryLookup } from './frequency-dictionary';
 export { createJlptVocabularyLookup } from './jlpt-vocab';
 export {

src/core/services/ipc.test.ts

@@ -89,6 +89,7 @@ function createSubtitleSidebarSnapshotFixture(): SubtitleSidebarSnapshot {
     cuesRevision: 1,
     cues: [],
     currentTimeSec: null,
+    subDelay: 0,
     currentSubtitle: { text: '', startTime: null, endTime: null },
     config: {
       enabled: false,

src/core/services/mining.test.ts

@@ -77,6 +77,7 @@ test('mineSentenceCard handles missing integration and disconnected mpv', async
         currentSubText: 'line',
         currentSubStart: 1,
         currentSubEnd: 2,
+        currentSubDelay: 0,
       },
       showMpvOsd: (text) => osd.push(text),
     }),
@@ -109,6 +110,7 @@ test('mineSentenceCard creates sentence card from mpv subtitle state', async ()
       currentSubText: 'subtitle line',
       currentSubStart: 10,
       currentSubEnd: 12,
+      currentSubDelay: 0,
       currentSecondarySubText: 'secondary line',
     },
     showMpvOsd: () => {},
@@ -148,6 +150,7 @@ test('mineSentenceCard prefers a canonical primary subtitle snapshot', async ()
       currentSubText: '今今今手手手',
       currentSubStart: 11.4,
       currentSubEnd: 11.8,
+      currentSubDelay: 0,
       currentSecondarySubText: 'English subtitle',
     },
     primarySubtitle: {
@@ -187,6 +190,7 @@ test('mineSentenceCard uses normalized secondary subtitle state instead of raw m
       currentSubText: '日本語字幕',
       currentSubStart: 10,
       currentSubEnd: 12,
+      currentSubDelay: 0,
       currentSecondarySubText: 'Your\nmosaic',
       requestProperty: async () => {
         requestedRawSecondaryText = true;
@@ -218,6 +222,7 @@ test('mineSentenceCard omits normalized secondary text that matches the primary
       currentSubText: '日本語字幕',
       currentSubStart: 10,
       currentSubEnd: 12,
+      currentSubDelay: 0,
       currentSecondarySubText: '日本語字幕',
     },
     showMpvOsd: () => {},
@@ -396,7 +401,7 @@ test('handleMineSentenceDigit joins per-entry secondary subtitles when available
 
 test('captureLiveSubtitleMiningContext snapshots the current line and timings', () => {
   const context = captureLiveSubtitleMiningContext(
-    { currentSubText: ' 食べる ', currentSubStart: 12.5, currentSubEnd: 15.25 },
+    { currentSubText: ' 食べる ', currentSubStart: 12.5, currentSubEnd: 15.25, currentSubDelay: 0 },
     () => 1234,
   );
 
@@ -409,6 +414,16 @@ test('captureLiveSubtitleMiningContext snapshots the current line and timings',
   });
 });
 
+test('captureLiveSubtitleMiningContext moves sub-delay retimed lines onto the video clock', () => {
+  const context = captureLiveSubtitleMiningContext(
+    { currentSubText: '食べる', currentSubStart: 5, currentSubEnd: 8, currentSubDelay: -1.5 },
+    () => 1234,
+  );
+
+  assert.equal(context?.startTime, 3.5);
+  assert.equal(context?.endTime, 6.5);
+});
+
 test('captureLiveSubtitleMiningContext rejects missing client, empty text and bad timings', () => {
   assert.equal(captureLiveSubtitleMiningContext(null), null);
   assert.equal(
@@ -416,6 +431,7 @@ test('captureLiveSubtitleMiningContext rejects missing client, empty text and ba
       currentSubText: '  ',
       currentSubStart: 1,
       currentSubEnd: 2,
+      currentSubDelay: 0,
     }),
     null,
   );
@@ -424,6 +440,7 @@ test('captureLiveSubtitleMiningContext rejects missing client, empty text and ba
       currentSubText: 'line',
       currentSubStart: 5,
       currentSubEnd: 5,
+      currentSubDelay: 0,
     }),
     null,
   );
@@ -432,6 +449,7 @@ test('captureLiveSubtitleMiningContext rejects missing client, empty text and ba
       currentSubText: 'line',
       currentSubStart: NaN,
       currentSubEnd: 2,
+      currentSubDelay: 0,
     }),
     null,
   );

src/core/services/mining.ts

@@ -1,5 +1,6 @@
 import type { SubtitleTimingBlock } from '../../subtitle-timing-tracker';
 import type { SubtitleMiningContext } from '../../types/subtitle';
+import { shiftSubtitleRangeToVideoClock } from './subtitle-clock';
 
 interface SubtitleTimingTrackerLike {
   getRecentBlocks: (count: number) => string[];
@@ -25,6 +26,7 @@ interface MpvClientLike {
   currentSubText: string;
   currentSubStart: number;
   currentSubEnd: number;
+  currentSubDelay: number;
   currentSecondarySubText?: string;
   requestProperty?: (name: string) => Promise<unknown>;
 }
@@ -62,7 +64,10 @@ export function handleMultiCopyDigit(
  * line is on screen by then; callers capture here at mining time instead.
  */
 export function captureLiveSubtitleMiningContext(
-  client: Pick<MpvClientLike, 'currentSubText' | 'currentSubStart' | 'currentSubEnd'> | null,
+  client: Pick<
+    MpvClientLike,
+    'currentSubText' | 'currentSubStart' | 'currentSubEnd' | 'currentSubDelay'
+  > | null,
   now: () => number = Date.now,
 ): SubtitleMiningContext | null {
   if (!client) {
@@ -73,7 +78,10 @@ export function captureLiveSubtitleMiningContext(
   if (!text || !Number.isFinite(startTime) || !Number.isFinite(endTime) || endTime <= startTime) {
     return null;
   }
-  return { source: 'overlay', text, startTime, endTime, capturedAtMs: now() };
+  return shiftSubtitleRangeToVideoClock<SubtitleMiningContext>(
+    { source: 'overlay', text, startTime, endTime, capturedAtMs: now() },
+    client.currentSubDelay,
+  );
 }
 
 export function copyCurrentSubtitle(deps: {
@@ -167,6 +175,7 @@ export async function markLastCardAsAudioCard(deps: {
 export async function mineSentenceCard(deps: {
   ankiIntegration: AnkiIntegrationLike | null;
   mpvClient: MpvClientLike | null;
+  /** Already on the video clock, like every `SubtitleMiningContext`. */
   primarySubtitle?: Pick<SubtitleMiningContext, 'text' | 'startTime' | 'endTime'>;
   showMpvOsd: (text: string) => void;
 }): Promise<boolean> {
@@ -185,10 +194,16 @@ export async function mineSentenceCard(deps: {
   }
 
   const secondarySubText = getCurrentSecondarySubTextForSentenceCard(mpvClient, primaryText);
+  const range =
+    deps.primarySubtitle ??
+    shiftSubtitleRangeToVideoClock(
+      { startTime: mpvClient.currentSubStart, endTime: mpvClient.currentSubEnd },
+      mpvClient.currentSubDelay,
+    );
   return await anki.createSentenceCard(
     primaryText,
-    deps.primarySubtitle?.startTime ?? mpvClient.currentSubStart,
-    deps.primarySubtitle?.endTime ?? mpvClient.currentSubEnd,
+    range.startTime,
+    range.endTime,
     secondarySubText,
   );
 }

src/core/services/mpv-properties.ts

@@ -38,6 +38,7 @@ const MPV_SUBTITLE_PROPERTY_OBSERVATIONS: string[] = [
   'path',
   'sub-start',
   'sub-end',
+  'sub-delay',
   'time-pos',
   'secondary-sub-text',
   'aid',

src/core/services/mpv-protocol.test.ts

@@ -73,6 +73,8 @@ function createDeps(overrides: Partial<MpvProtocolHandleMessageDeps> = {}): {
       getCurrentSubStart: () => 0,
       setCurrentSubEnd: () => {},
       getCurrentSubEnd: () => 0,
+      setCurrentSubDelay: () => {},
+      getCurrentSubDelay: () => 0,
       emitMediaPathChange: (payload) => {
         state.mediaPath = payload.path;
       },
@@ -131,6 +133,22 @@ test('dispatchMpvProtocolMessage emits subtitle text on property change', async
   assert.deepEqual(state.events, [{ text: '字幕', isOverlayVisible: false }]);
 });
 
+test('dispatchMpvProtocolMessage tracks sub-delay retiming', async () => {
+  const delays: number[] = [];
+  const { deps } = createDeps({ setCurrentSubDelay: (value) => delays.push(value) });
+
+  await dispatchMpvProtocolMessage(
+    { event: 'property-change', name: 'sub-delay', data: 1.5 },
+    deps,
+  );
+  await dispatchMpvProtocolMessage(
+    { event: 'property-change', name: 'sub-delay', data: null },
+    deps,
+  );
+
+  assert.deepEqual(delays, [1.5, 0]);
+});
+
 test('dispatchMpvProtocolMessage widens halfwidth katakana in subtitle text', async () => {
   const { deps, state } = createDeps();
 
@@ -379,6 +397,29 @@ test('dispatchMpvProtocolMessage pauses on sub-end when pendingPauseAtSubEnd is
   });
 });
 
+test('dispatchMpvProtocolMessage pauses at the sub-delay retimed sub-end', async () => {
+  let pauseAtTime: number | null = null;
+  let subDelay = 1.5;
+  const { deps } = createDeps({
+    getPendingPauseAtSubEnd: () => true,
+    getCurrentSubDelay: () => subDelay,
+    setCurrentSubDelay: (next) => {
+      subDelay = next;
+    },
+    getPauseAtTime: () => pauseAtTime,
+    setPauseAtTime: (next) => {
+      pauseAtTime = next;
+    },
+  });
+
+  await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sub-end', data: 42 }, deps);
+  assert.equal(pauseAtTime, 43.5);
+
+  // Retiming before the replay reaches its end moves the pause with the line.
+  await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sub-delay', data: 2 }, deps);
+  assert.equal(pauseAtTime, 44);
+});
+
 test('dispatchMpvProtocolMessage updates current time before emitting time-pos change', async () => {
   const calls: string[] = [];
   let currentTimePos = 0;

src/core/services/mpv-protocol.ts

@@ -1,6 +1,7 @@
 import { MpvSubtitleRenderMetrics } from '../../types';
 import { sanitizeMediaTitle } from '../../shared/media-identity';
 import { widenHalfwidthKana } from '../utils/halfwidth-kana';
+import { toVideoClock } from './subtitle-clock';
 
 export type MpvMessage = {
   event?: string;
@@ -64,6 +65,8 @@ export interface MpvProtocolHandleMessageDeps {
   getCurrentSubStart: () => number;
   setCurrentSubEnd: (value: number) => void;
   getCurrentSubEnd: () => number;
+  setCurrentSubDelay: (value: number) => void;
+  getCurrentSubDelay: () => number;
   emitMediaPathChange: (payload: { path: string }) => void;
   emitMediaTitleChange: (payload: { title: string | null }) => void;
   emitTimePosChange: (payload: { time: number }) => void;
@@ -263,7 +266,8 @@ export async function dispatchMpvProtocolMessage(
       const subEnd = (msg.data as number) || 0;
       deps.setCurrentSubEnd(subEnd);
       if (deps.getPendingPauseAtSubEnd() && subEnd > 0) {
-        deps.setPauseAtTime(subEnd);
+        // `sub-end` is on the subtitle clock; the pause check compares it with `time-pos`.
+        deps.setPauseAtTime(toVideoClock(subEnd, deps.getCurrentSubDelay()));
         deps.setPendingPauseAtSubEnd(false);
         deps.sendCommand({ command: ['set_property', 'pause', false] });
       }
@@ -272,6 +276,16 @@ export async function dispatchMpvProtocolMessage(
         start: deps.getCurrentSubStart(),
         end: deps.getCurrentSubEnd(),
       });
+    } else if (msg.name === 'sub-delay') {
+      const parsed = typeof msg.data === 'number' ? msg.data : Number(msg.data);
+      const delay = Number.isFinite(parsed) ? parsed : 0;
+      // A scheduled replay pause sits at the line's end on the video clock; retiming
+      // mid-replay moves that end, so the pause moves with it.
+      const pauseAtTime = deps.getPauseAtTime();
+      if (pauseAtTime !== null) {
+        deps.setPauseAtTime(pauseAtTime + delay - deps.getCurrentSubDelay());
+      }
+      deps.setCurrentSubDelay(delay);
     } else if (msg.name === 'secondary-sub-text') {
       const nextSubText = (msg.data as string) || '';
       deps.setCurrentSecondarySubText(nextSubText);

src/core/services/mpv.ts

@@ -160,6 +160,8 @@ export class MpvIpcClient implements MpvClient {
   public currentTimePos = 0;
   public currentSubStart = 0;
   public currentSubEnd = 0;
+  /** mpv `sub-delay`: seconds each primary line plays after its `sub-start`/`sub-end`. */
+  public currentSubDelay = 0;
   public currentSubText = '';
   public currentSecondarySubText = '';
   public currentAudioStreamIndex: number | null = null;
@@ -459,6 +461,10 @@ export class MpvIpcClient implements MpvClient {
         this.currentSubEnd = value;
       },
       getCurrentSubEnd: () => this.currentSubEnd,
+      setCurrentSubDelay: (value: number) => {
+        this.currentSubDelay = value;
+      },
+      getCurrentSubDelay: () => this.currentSubDelay,
       emitMediaPathChange: (payload) => {
         this.emit('media-path-change', payload);
       },

src/core/services/subtitle-clock.ts

@@ -0,0 +1,32 @@
+/**
+ * mpv's `sub-start`/`sub-end` and parsed cues carry the subtitle file's own timestamps
+ * (the subtitle clock), but mpv plays each line `sub-delay` seconds later (z/x retiming,
+ * `sub-step`). Playback position, seeks, and media cuts are on the video clock, so any
+ * value crossing between the two converts here.
+ */
+
+function finiteDelay(subDelay: number): number {
+  return Number.isFinite(subDelay) ? subDelay : 0;
+}
+
+/** Playback position (`time-pos`) on the subtitle clock, comparable with cue times. */
+export function toSubtitleClock(videoTimeSec: number, subDelay: number): number {
+  return videoTimeSec - finiteDelay(subDelay);
+}
+
+/** Subtitle-clock seconds on the video clock, for seeks. */
+export function toVideoClock(subtitleTimeSec: number, subDelay: number): number {
+  return subtitleTimeSec + finiteDelay(subDelay);
+}
+
+/** A subtitle-clock range on the video clock, for mining and media generation. */
+export function shiftSubtitleRangeToVideoClock<T extends { startTime: number; endTime: number }>(
+  range: T,
+  subDelay: number,
+): T {
+  const delay = finiteDelay(subDelay);
+  if (delay === 0) {
+    return range;
+  }
+  return { ...range, startTime: range.startTime + delay, endTime: range.endTime + delay };
+}

src/core/services/subtitle-cue-navigation.test.ts

@@ -26,7 +26,7 @@ test('next subtitle navigation skips generated ASS events and seeks to the next
     },
   ];
 
-  assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.2), [
+  assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.2, 0), [
     'seek',
     13.08,
     'absolute+exact',
@@ -41,7 +41,7 @@ test('next subtitle navigation treats simultaneous sanitized cues as one line bo
     { startTime: 13.02, endTime: 16, text: 'Next English' },
   ];
 
-  assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.1), [
+  assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.1, 0), [
     'seek',
     13.08,
     'absolute+exact',
@@ -55,7 +55,7 @@ test('next subtitle navigation advances past the latest overlapping lyric', () =
     { startTime: 16, endTime: 19, text: 'next lyric' },
   ];
 
-  assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 13.2), [
+  assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 13.2, 0), [
     'seek',
     16.08,
     'absolute+exact',
@@ -68,7 +68,7 @@ test('previous subtitle navigation leaves the current cue and seeks to the prior
     { startTime: 13, endTime: 16, text: 'current line' },
   ];
 
-  assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', -1], cues, 14.5), [
+  assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', -1], cues, 14.5, 0), [
     'seek',
     10.08,
     'absolute+exact',
@@ -78,8 +78,22 @@ test('previous subtitle navigation leaves the current cue and seeks to the prior
 test('subtitle navigation falls back when no sanitized destination exists', () => {
   const cues = [{ startTime: 10, endTime: 13, text: 'only line' }];
 
-  assert.equal(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.2), null);
-  assert.equal(resolveSanitizedSubtitleSeekCommand(['seek', 5], cues, 10.2), null);
+  assert.equal(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.2, 0), null);
+  assert.equal(resolveSanitizedSubtitleSeekCommand(['seek', 5], cues, 10.2, 0), null);
+});
+
+test('subtitle cue navigation finds and seeks cues on the sub-delay retimed timeline', () => {
+  const cues = [
+    { startTime: 10, endTime: 13, text: 'one' },
+    { startTime: 13, endTime: 16, text: 'two' },
+  ];
+
+  // At 14.5 with a 2s delay, mpv is showing "one" (12-15 on the video clock).
+  assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 14.5, 2), [
+    'seek',
+    15.08,
+    'absolute+exact',
+  ]);
 });
 
 test('sidebar cue seeks share the boundary-safe sanitized cue timestamp', () => {

src/core/services/subtitle-cue-navigation.ts

@@ -1,3 +1,4 @@
+import { toSubtitleClock, toVideoClock } from './subtitle-clock';
 import type { SubtitleCue } from './subtitle-cue-parser';
 
 const CUE_START_GROUP_TOLERANCE_SECONDS = 0.05;
@@ -80,8 +81,10 @@ export function subtitleCueListSeekTime(
 export function resolveSanitizedSubtitleSeekCommand(
   command: readonly (string | number)[],
   cues: readonly SubtitleCue[],
-  currentTimeSec: number,
+  videoTimeSec: number,
+  subDelay: number,
 ): (string | number)[] | null {
+  const currentTimeSec = toSubtitleClock(videoTimeSec, subDelay);
   if (
     command.length < 2 ||
     command[0] !== 'sub-seek' ||
@@ -124,5 +127,5 @@ export function resolveSanitizedSubtitleSeekCommand(
   if (!destination) {
     return null;
   }
-  return ['seek', subtitleCueSeekTime(destination.cue), 'absolute+exact'];
+  return ['seek', toVideoClock(subtitleCueSeekTime(destination.cue), subDelay), 'absolute+exact'];
 }