Compare commits

..
Author SHA1 Message Date
sudacode d06fe8b572 fix(mpv): move a pending replay pause when sub-delay changes 2026-10-25 22:21:59 -07:00
sudacode 8a2da51e41 test(e2e): mirror AnkiConnect notesInfo and updateNoteFields in fake
notesInfo now returns one entry per requested id ({} for unknown notes),
and updateNoteFields writes through so later reads see current fields.
2026-10-25 21:53:52 -07:00
sudacode 4d27e068c6 fix(subtitles): follow sub-delay in sidebar, navigation, and overlay matching
Anything comparing time-pos with parsed cues now converts through a
shared subtitle-clock helper: the sidebar highlight and click-to-seek,
the sub-seek cue sanitizer, overlay/primer line resolution, and the
replay pause point after sub-seek.
2026-10-25 21:40:02 -07:00
sudacode 929e78b28c fix(anki): clip mined media on the sub-delay retimed timeline
mpv's sub-start/sub-end and parsed cues carry the subtitle file's own
timestamps; sub-delay (z/x, sub-step) only moves where mpv plays them.
Mining now shifts subtitle timings onto the video clock before media
generation, review, and MiscInfo timestamps.
2026-10-25 21:32:59 -07:00
27 changed files with 392 additions and 51 deletions

No files matched your search

+1 -1
View File
@@ -25,7 +25,7 @@ Look up words with Yomitan or Hachidori, mine them to Anki, and track your immer
Hover over any word in the subtitles to open the full Yomitan popup with definitions, pitch accent, and frequency data. SubMiner bundles its own Yomitan, separate from any browser install.
Yomitan remains the default. Select the bundled [Hachidori](https://github.com/bee-san/hachidori) backend with `dictionaryBackend: "hachidori"` and restart SubMiner. The tray opens the selected backend's settings. See [dictionary setup](https://docs.subminer.moe/usage#hachidori-setup) for importing dictionaries, linking an external Hachidori host, and configuring Anki.
Yomitan remains the default. Select the bundled Hachidori backend with `dictionaryBackend: "hachidori"` and restart SubMiner. The tray opens the selected backend's settings. See [dictionary setup](https://docs.subminer.moe/usage#hachidori-setup) for importing dictionaries, linking an external Hachidori host, and configuring Anki.
<div align="center">
<img src="docs-site/public/screenshots/yomitan-lookup.png" width="800" alt="Yomitan dictionary popup over annotated subtitles in mpv">
+4
View File
@@ -0,0 +1,4 @@
type: fixed
area: subtitles
- mpv's manual subtitle retiming (`sub-delay`, z/x and sub-step) now carries through SubMiner: mined card audio, images, and timestamps clip the retimed line, and the subtitle sidebar highlight, click-to-seek, subtitle navigation, overlay line matching, and replay pause point all follow the retimed timing.
+1 -1
View File
@@ -22,7 +22,7 @@ The bundled Yomitan is separate from any Yomitan in your browser. It has its own
### Hachidori setup
[Hachidori](https://github.com/bee-san/hachidori) is an alternative lookup backend. Set `dictionaryBackend` to `"hachidori"` in settings or `config.jsonc`, then restart SubMiner. Set it back to `"yomitan"` and restart to switch back.
Hachidori is an alternative lookup backend. Set `dictionaryBackend` to `"hachidori"` in settings or `config.jsonc`, then restart SubMiner. Set it back to `"yomitan"` and restart to switch back.
Open Hachidori settings with `Alt+Shift+Y`, the tray's **Open Hachidori Settings**, or `subminer app --hachidori`. Import dictionary ZIPs or use Hachidori's recommended dictionary installer, then set up its Anki template (SubMiner [fills in what it can](/anki-integration#hachidori-settings-from-subminer)). Yomitan and Hachidori keep separate dictionaries and settings. Yomitan profiles, custom Handlebars templates, and `yomitan.externalProfilePath` do not carry over. Subtitle annotations and lookups wait while Hachidori imports a dictionary, so large imports pause them until the import finishes.
+35 -5
View File
@@ -37,10 +37,12 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
// A recording AnkiConnect stand-in: an empty collection with one deck and one
// Lapis-shaped note type. Scenarios assert on `requests`, not on stored notes.
// A recording AnkiConnect stand-in: one deck and one Lapis-shaped note type.
// Added notes (with later `updateNoteFields` writes) are echoed back by `notesInfo`
// so the app can resolve their fields; scenarios still assert on `requests`.
export async function startFakeAnki(): Promise<FakeAnki> {
const requests: AnkiRequest[] = [];
const addedNotes = new Map<number, Record<string, unknown>>();
let nextNoteId = 1000;
const handle = (body: unknown): unknown => {
@@ -59,10 +61,38 @@ export async function startFakeAnki(): Promise<FakeAnki> {
return [FAKE_ANKI_MODEL];
case 'modelFieldNames':
return MODEL_FIELDS;
case 'addNote':
return nextNoteId++;
case 'addNote': {
const noteId = nextNoteId++;
const note = isRecord(params.note) ? params.note : {};
addedNotes.set(noteId, isRecord(note.fields) ? { ...note.fields } : {});
return noteId;
}
case 'updateNoteFields': {
const note = isRecord(params.note) ? params.note : {};
const fields = typeof note.id === 'number' ? addedNotes.get(note.id) : undefined;
if (fields && isRecord(note.fields)) Object.assign(fields, note.fields);
return null;
}
case 'notesInfo': {
// Like AnkiConnect: one entry per requested id, `{}` for unknown notes.
const ids = Array.isArray(params.notes) ? params.notes : [];
return ids.map((id) => {
const fields = typeof id === 'number' ? addedNotes.get(id) : undefined;
if (!fields) return {};
return {
noteId: id,
modelName: FAKE_ANKI_MODEL,
tags: [],
fields: Object.fromEntries(
MODEL_FIELDS.map((name, order) => [
name,
{ value: String(fields[name] ?? ''), order },
]),
),
};
});
}
case 'findNotes':
case 'notesInfo':
return [];
case 'storeMediaFile':
return params.filename ?? null;
+87 -4
View File
@@ -91,6 +91,14 @@ async function showCue(cue: (typeof FIXTURE_CUES)[number]): Promise<RenderedToke
});
}
// Card-kind flags are written before media, so wait for the update carrying audio.
function findMediaUpdate(session: E2eSession) {
return session.anki.requests.find(
(request) =>
request.action === 'updateNoteFields' && JSON.stringify(request.params).includes('[sound:'),
);
}
function hasToken(tokens: RenderedToken[], surface: string, headword: string): boolean {
return tokens.some((token) => token.surface === surface && token.headword === headword);
}
@@ -126,10 +134,10 @@ test(
session.anki.requests.length = 0;
await session.runAppCommand('--mine-sentence');
const update = await waitUntil(
() => session.anki.requests.find((request) => request.action === 'updateNoteFields'),
{ description: 'media fields to be written to the mined note', timeoutMs: 30_000 },
);
const update = await waitUntil(() => findMediaUpdate(session), {
description: 'media fields to be written to the mined note',
timeoutMs: 30_000,
});
const added = session.anki.requests.filter((request) => request.action === 'addNote');
assert.equal(added.length, 1);
@@ -156,6 +164,38 @@ test(
},
);
test(
'mining a line retimed with sub-delay clips it where mpv plays it',
{ timeout: TEST_TIMEOUT_MS },
async () => {
const { session, overlay } = await useSession();
const cue = FIXTURE_CUES[1];
const delay = 2;
await session.mpv.command('set_property', 'sub-delay', delay);
try {
// Past the cue's own end, so the line is only on screen because of the delay.
await session.mpv.command('seek', cue.end + delay / 2, 'absolute+exact');
await overlay.waitFor(
`document.getElementById('subtitleRoot').textContent === ${JSON.stringify(cue.text)}`,
{ description: 'overlay to show the delayed line' },
);
session.anki.requests.length = 0;
await session.runAppCommand('--mine-sentence');
await waitUntil(() => findMediaUpdate(session), {
description: 'media fields to be written to the mined note',
timeoutMs: 30_000,
});
// MiscInfo stamps the clip start: the cue's start on the video clock.
const written = JSON.stringify(session.anki.requests);
assert.ok(written.includes(`(00:00:0${cue.start + delay})`), written);
} finally {
await session.mpv.command('set_property', 'sub-delay', 0);
}
},
);
test(
'open subtitle sidebar lists the parsed cues and follows playback',
{ timeout: TEST_TIMEOUT_MS },
@@ -184,6 +224,49 @@ test(
},
);
test(
'subtitle sidebar follows and seeks a sub-delay retimed track',
{ timeout: TEST_TIMEOUT_MS },
async () => {
const { session, overlay } = await useSession();
const delay = 2;
await session.mpv.command('set_property', 'sub-delay', delay);
try {
// At 9s cue 2 starts on the file's clock, but mpv is still showing cue 1.
await session.mpv.command('seek', FIXTURE_CUES[2].start, 'absolute+exact');
await overlay.waitFor(
`document.getElementById('subtitleRoot').textContent === ${JSON.stringify(FIXTURE_CUES[1].text)}`,
{ description: 'overlay to show the delayed line' },
);
await session.runAppCommand('--toggle-subtitle-sidebar');
await overlay.waitFor(
`document.querySelector('#subtitleSidebarList .subtitle-sidebar-item.active')?.dataset.index === '1'`,
{ description: 'sidebar to highlight the delayed line' },
);
await overlay.evaluate(
`document.querySelector('#subtitleSidebarList .subtitle-sidebar-item[data-index="2"]').click()`,
);
const target = FIXTURE_CUES[2].start + delay;
await waitUntil(
async () => {
const time = Number(await session.mpv.command('get_property', 'time-pos'));
return time >= target && time < target + 0.5;
},
{ description: `click to seek to cue 2 at ${target}s on the video clock` },
);
await session.runAppCommand('--toggle-subtitle-sidebar');
await overlay.waitFor(
`document.getElementById('subtitleSidebarModal').classList.contains('hidden')`,
{ description: 'sidebar to close' },
);
} finally {
await session.mpv.command('set_property', 'sub-delay', 0);
}
},
);
// Runs last: it leaves the lookup popup open over the overlay.
test(
'holding Shift over a word opens the Yomitan popup with its dictionary entry',
+7 -6
View File
@@ -45,6 +45,7 @@ import {
} from './anki-field-config';
import { createLogger } from './logger';
import { captureLiveSubtitleMiningContext } from './core/services/mining';
import { shiftSubtitleRangeToVideoClock } from './core/services/subtitle-clock';
import {
createUiFeedbackState,
beginUpdateProgress,
@@ -1029,10 +1030,10 @@ export class AnkiIntegration {
Number.isFinite(this.mpvClient.currentSubEnd) &&
this.mpvClient.currentSubEnd > this.mpvClient.currentSubStart
) {
return {
startTime: this.mpvClient.currentSubStart,
endTime: this.mpvClient.currentSubEnd,
};
return shiftSubtitleRangeToVideoClock(
{ startTime: this.mpvClient.currentSubStart, endTime: this.mpvClient.currentSubEnd },
this.mpvClient.currentSubDelay,
);
}
const currentTime = this.mpvClient.currentTimePos || 0;
@@ -1672,7 +1673,7 @@ export class AnkiIntegration {
if (this.config.fields?.miscInfo) {
result.miscInfoValue = this.formatMiscInfoPattern(
audioFilename,
this.mpvClient.currentSubStart,
this.mpvClient.currentSubStart + this.mpvClient.currentSubDelay,
);
}
}
@@ -1695,7 +1696,7 @@ export class AnkiIntegration {
if (this.config.fields?.miscInfo && !result.miscInfoValue) {
result.miscInfoValue = this.formatMiscInfoPattern(
imageFilename,
this.mpvClient.currentSubStart,
this.mpvClient.currentSubStart + this.mpvClient.currentSubDelay,
);
}
}
+6 -3
View File
@@ -10,6 +10,7 @@ import {
type MediaTimingReviewRequest,
type WordCardKind,
} from '../types/anki';
import { shiftSubtitleRangeToVideoClock } from '../core/services/subtitle-clock';
import { createLogger } from '../logger';
import type { MediaInput } from '../media-input';
import { SubtitleTimingTracker } from '../subtitle-timing-tracker';
@@ -428,10 +429,12 @@ export class CardCreationService {
return;
}
let startTime = mpvClient.currentSubStart;
let endTime = mpvClient.currentSubEnd;
let { startTime, endTime } = shiftSubtitleRangeToVideoClock(
{ startTime: mpvClient.currentSubStart, endTime: mpvClient.currentSubEnd },
mpvClient.currentSubDelay,
);
if (startTime === undefined || endTime === undefined) {
if (mpvClient.currentSubStart === undefined || mpvClient.currentSubEnd === undefined) {
const currentTime = mpvClient.currentTimePos || 0;
const fallback = this.deps.getFallbackDurationSeconds() / 2;
startTime = currentTime - fallback;
+1
View File
@@ -55,6 +55,7 @@ export {
subtitleCueListSeekTime,
subtitleCueSeekTime,
} from './subtitle-cue-navigation';
export { shiftSubtitleRangeToVideoClock, toSubtitleClock, toVideoClock } from './subtitle-clock';
export { createFrequencyDictionaryLookup } from './frequency-dictionary';
export { createJlptVocabularyLookup } from './jlpt-vocab';
export {
+1
View File
@@ -89,6 +89,7 @@ function createSubtitleSidebarSnapshotFixture(): SubtitleSidebarSnapshot {
cuesRevision: 1,
cues: [],
currentTimeSec: null,
subDelay: 0,
currentSubtitle: { text: '', startTime: null, endTime: null },
config: {
enabled: false,
+19 -1
View File
@@ -77,6 +77,7 @@ test('mineSentenceCard handles missing integration and disconnected mpv', async
currentSubText: 'line',
currentSubStart: 1,
currentSubEnd: 2,
currentSubDelay: 0,
},
showMpvOsd: (text) => osd.push(text),
}),
@@ -109,6 +110,7 @@ test('mineSentenceCard creates sentence card from mpv subtitle state', async ()
currentSubText: 'subtitle line',
currentSubStart: 10,
currentSubEnd: 12,
currentSubDelay: 0,
currentSecondarySubText: 'secondary line',
},
showMpvOsd: () => {},
@@ -148,6 +150,7 @@ test('mineSentenceCard prefers a canonical primary subtitle snapshot', async ()
currentSubText: '今今今手手手',
currentSubStart: 11.4,
currentSubEnd: 11.8,
currentSubDelay: 0,
currentSecondarySubText: 'English subtitle',
},
primarySubtitle: {
@@ -187,6 +190,7 @@ test('mineSentenceCard uses normalized secondary subtitle state instead of raw m
currentSubText: '日本語字幕',
currentSubStart: 10,
currentSubEnd: 12,
currentSubDelay: 0,
currentSecondarySubText: 'Your\nmosaic',
requestProperty: async () => {
requestedRawSecondaryText = true;
@@ -218,6 +222,7 @@ test('mineSentenceCard omits normalized secondary text that matches the primary
currentSubText: '日本語字幕',
currentSubStart: 10,
currentSubEnd: 12,
currentSubDelay: 0,
currentSecondarySubText: '日本語字幕',
},
showMpvOsd: () => {},
@@ -396,7 +401,7 @@ test('handleMineSentenceDigit joins per-entry secondary subtitles when available
test('captureLiveSubtitleMiningContext snapshots the current line and timings', () => {
const context = captureLiveSubtitleMiningContext(
{ currentSubText: ' 食べる ', currentSubStart: 12.5, currentSubEnd: 15.25 },
{ currentSubText: ' 食べる ', currentSubStart: 12.5, currentSubEnd: 15.25, currentSubDelay: 0 },
() => 1234,
);
@@ -409,6 +414,16 @@ test('captureLiveSubtitleMiningContext snapshots the current line and timings',
});
});
test('captureLiveSubtitleMiningContext moves sub-delay retimed lines onto the video clock', () => {
const context = captureLiveSubtitleMiningContext(
{ currentSubText: '食べる', currentSubStart: 5, currentSubEnd: 8, currentSubDelay: -1.5 },
() => 1234,
);
assert.equal(context?.startTime, 3.5);
assert.equal(context?.endTime, 6.5);
});
test('captureLiveSubtitleMiningContext rejects missing client, empty text and bad timings', () => {
assert.equal(captureLiveSubtitleMiningContext(null), null);
assert.equal(
@@ -416,6 +431,7 @@ test('captureLiveSubtitleMiningContext rejects missing client, empty text and ba
currentSubText: ' ',
currentSubStart: 1,
currentSubEnd: 2,
currentSubDelay: 0,
}),
null,
);
@@ -424,6 +440,7 @@ test('captureLiveSubtitleMiningContext rejects missing client, empty text and ba
currentSubText: 'line',
currentSubStart: 5,
currentSubEnd: 5,
currentSubDelay: 0,
}),
null,
);
@@ -432,6 +449,7 @@ test('captureLiveSubtitleMiningContext rejects missing client, empty text and ba
currentSubText: 'line',
currentSubStart: NaN,
currentSubEnd: 2,
currentSubDelay: 0,
}),
null,
);
+19 -4
View File
@@ -1,5 +1,6 @@
import type { SubtitleTimingBlock } from '../../subtitle-timing-tracker';
import type { SubtitleMiningContext } from '../../types/subtitle';
import { shiftSubtitleRangeToVideoClock } from './subtitle-clock';
interface SubtitleTimingTrackerLike {
getRecentBlocks: (count: number) => string[];
@@ -25,6 +26,7 @@ interface MpvClientLike {
currentSubText: string;
currentSubStart: number;
currentSubEnd: number;
currentSubDelay: number;
currentSecondarySubText?: string;
requestProperty?: (name: string) => Promise<unknown>;
}
@@ -62,7 +64,10 @@ export function handleMultiCopyDigit(
* line is on screen by then; callers capture here at mining time instead.
*/
export function captureLiveSubtitleMiningContext(
client: Pick<MpvClientLike, 'currentSubText' | 'currentSubStart' | 'currentSubEnd'> | null,
client: Pick<
MpvClientLike,
'currentSubText' | 'currentSubStart' | 'currentSubEnd' | 'currentSubDelay'
> | null,
now: () => number = Date.now,
): SubtitleMiningContext | null {
if (!client) {
@@ -73,7 +78,10 @@ export function captureLiveSubtitleMiningContext(
if (!text || !Number.isFinite(startTime) || !Number.isFinite(endTime) || endTime <= startTime) {
return null;
}
return { source: 'overlay', text, startTime, endTime, capturedAtMs: now() };
return shiftSubtitleRangeToVideoClock<SubtitleMiningContext>(
{ source: 'overlay', text, startTime, endTime, capturedAtMs: now() },
client.currentSubDelay,
);
}
export function copyCurrentSubtitle(deps: {
@@ -167,6 +175,7 @@ export async function markLastCardAsAudioCard(deps: {
export async function mineSentenceCard(deps: {
ankiIntegration: AnkiIntegrationLike | null;
mpvClient: MpvClientLike | null;
/** Already on the video clock, like every `SubtitleMiningContext`. */
primarySubtitle?: Pick<SubtitleMiningContext, 'text' | 'startTime' | 'endTime'>;
showMpvOsd: (text: string) => void;
}): Promise<boolean> {
@@ -185,10 +194,16 @@ export async function mineSentenceCard(deps: {
}
const secondarySubText = getCurrentSecondarySubTextForSentenceCard(mpvClient, primaryText);
const range =
deps.primarySubtitle ??
shiftSubtitleRangeToVideoClock(
{ startTime: mpvClient.currentSubStart, endTime: mpvClient.currentSubEnd },
mpvClient.currentSubDelay,
);
return await anki.createSentenceCard(
primaryText,
deps.primarySubtitle?.startTime ?? mpvClient.currentSubStart,
deps.primarySubtitle?.endTime ?? mpvClient.currentSubEnd,
range.startTime,
range.endTime,
secondarySubText,
);
}
+1
View File
@@ -38,6 +38,7 @@ const MPV_SUBTITLE_PROPERTY_OBSERVATIONS: string[] = [
'path',
'sub-start',
'sub-end',
'sub-delay',
'time-pos',
'secondary-sub-text',
'aid',
+41
View File
@@ -73,6 +73,8 @@ function createDeps(overrides: Partial<MpvProtocolHandleMessageDeps> = {}): {
getCurrentSubStart: () => 0,
setCurrentSubEnd: () => {},
getCurrentSubEnd: () => 0,
setCurrentSubDelay: () => {},
getCurrentSubDelay: () => 0,
emitMediaPathChange: (payload) => {
state.mediaPath = payload.path;
},
@@ -131,6 +133,22 @@ test('dispatchMpvProtocolMessage emits subtitle text on property change', async
assert.deepEqual(state.events, [{ text: '字幕', isOverlayVisible: false }]);
});
test('dispatchMpvProtocolMessage tracks sub-delay retiming', async () => {
const delays: number[] = [];
const { deps } = createDeps({ setCurrentSubDelay: (value) => delays.push(value) });
await dispatchMpvProtocolMessage(
{ event: 'property-change', name: 'sub-delay', data: 1.5 },
deps,
);
await dispatchMpvProtocolMessage(
{ event: 'property-change', name: 'sub-delay', data: null },
deps,
);
assert.deepEqual(delays, [1.5, 0]);
});
test('dispatchMpvProtocolMessage widens halfwidth katakana in subtitle text', async () => {
const { deps, state } = createDeps();
@@ -379,6 +397,29 @@ test('dispatchMpvProtocolMessage pauses on sub-end when pendingPauseAtSubEnd is
});
});
test('dispatchMpvProtocolMessage pauses at the sub-delay retimed sub-end', async () => {
let pauseAtTime: number | null = null;
let subDelay = 1.5;
const { deps } = createDeps({
getPendingPauseAtSubEnd: () => true,
getCurrentSubDelay: () => subDelay,
setCurrentSubDelay: (next) => {
subDelay = next;
},
getPauseAtTime: () => pauseAtTime,
setPauseAtTime: (next) => {
pauseAtTime = next;
},
});
await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sub-end', data: 42 }, deps);
assert.equal(pauseAtTime, 43.5);
// Retiming before the replay reaches its end moves the pause with the line.
await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sub-delay', data: 2 }, deps);
assert.equal(pauseAtTime, 44);
});
test('dispatchMpvProtocolMessage updates current time before emitting time-pos change', async () => {
const calls: string[] = [];
let currentTimePos = 0;
+15 -1
View File
@@ -1,6 +1,7 @@
import { MpvSubtitleRenderMetrics } from '../../types';
import { sanitizeMediaTitle } from '../../shared/media-identity';
import { widenHalfwidthKana } from '../utils/halfwidth-kana';
import { toVideoClock } from './subtitle-clock';
export type MpvMessage = {
event?: string;
@@ -64,6 +65,8 @@ export interface MpvProtocolHandleMessageDeps {
getCurrentSubStart: () => number;
setCurrentSubEnd: (value: number) => void;
getCurrentSubEnd: () => number;
setCurrentSubDelay: (value: number) => void;
getCurrentSubDelay: () => number;
emitMediaPathChange: (payload: { path: string }) => void;
emitMediaTitleChange: (payload: { title: string | null }) => void;
emitTimePosChange: (payload: { time: number }) => void;
@@ -263,7 +266,8 @@ export async function dispatchMpvProtocolMessage(
const subEnd = (msg.data as number) || 0;
deps.setCurrentSubEnd(subEnd);
if (deps.getPendingPauseAtSubEnd() && subEnd > 0) {
deps.setPauseAtTime(subEnd);
// `sub-end` is on the subtitle clock; the pause check compares it with `time-pos`.
deps.setPauseAtTime(toVideoClock(subEnd, deps.getCurrentSubDelay()));
deps.setPendingPauseAtSubEnd(false);
deps.sendCommand({ command: ['set_property', 'pause', false] });
}
@@ -272,6 +276,16 @@ export async function dispatchMpvProtocolMessage(
start: deps.getCurrentSubStart(),
end: deps.getCurrentSubEnd(),
});
} else if (msg.name === 'sub-delay') {
const parsed = typeof msg.data === 'number' ? msg.data : Number(msg.data);
const delay = Number.isFinite(parsed) ? parsed : 0;
// A scheduled replay pause sits at the line's end on the video clock; retiming
// mid-replay moves that end, so the pause moves with it.
const pauseAtTime = deps.getPauseAtTime();
if (pauseAtTime !== null) {
deps.setPauseAtTime(pauseAtTime + delay - deps.getCurrentSubDelay());
}
deps.setCurrentSubDelay(delay);
} else if (msg.name === 'secondary-sub-text') {
const nextSubText = (msg.data as string) || '';
deps.setCurrentSecondarySubText(nextSubText);
+6
View File
@@ -160,6 +160,8 @@ export class MpvIpcClient implements MpvClient {
public currentTimePos = 0;
public currentSubStart = 0;
public currentSubEnd = 0;
/** mpv `sub-delay`: seconds each primary line plays after its `sub-start`/`sub-end`. */
public currentSubDelay = 0;
public currentSubText = '';
public currentSecondarySubText = '';
public currentAudioStreamIndex: number | null = null;
@@ -459,6 +461,10 @@ export class MpvIpcClient implements MpvClient {
this.currentSubEnd = value;
},
getCurrentSubEnd: () => this.currentSubEnd,
setCurrentSubDelay: (value: number) => {
this.currentSubDelay = value;
},
getCurrentSubDelay: () => this.currentSubDelay,
emitMediaPathChange: (payload) => {
this.emit('media-path-change', payload);
},
+32
View File
@@ -0,0 +1,32 @@
/**
* mpv's `sub-start`/`sub-end` and parsed cues carry the subtitle file's own timestamps
* (the subtitle clock), but mpv plays each line `sub-delay` seconds later (z/x retiming,
* `sub-step`). Playback position, seeks, and media cuts are on the video clock, so any
* value crossing between the two converts here.
*/
function finiteDelay(subDelay: number): number {
return Number.isFinite(subDelay) ? subDelay : 0;
}
/** Playback position (`time-pos`) on the subtitle clock, comparable with cue times. */
export function toSubtitleClock(videoTimeSec: number, subDelay: number): number {
return videoTimeSec - finiteDelay(subDelay);
}
/** Subtitle-clock seconds on the video clock, for seeks. */
export function toVideoClock(subtitleTimeSec: number, subDelay: number): number {
return subtitleTimeSec + finiteDelay(subDelay);
}
/** A subtitle-clock range on the video clock, for mining and media generation. */
export function shiftSubtitleRangeToVideoClock<T extends { startTime: number; endTime: number }>(
range: T,
subDelay: number,
): T {
const delay = finiteDelay(subDelay);
if (delay === 0) {
return range;
}
return { ...range, startTime: range.startTime + delay, endTime: range.endTime + delay };
}
@@ -26,7 +26,7 @@ test('next subtitle navigation skips generated ASS events and seeks to the next
},
];
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.2), [
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.2, 0), [
'seek',
13.08,
'absolute+exact',
@@ -41,7 +41,7 @@ test('next subtitle navigation treats simultaneous sanitized cues as one line bo
{ startTime: 13.02, endTime: 16, text: 'Next English' },
];
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.1), [
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.1, 0), [
'seek',
13.08,
'absolute+exact',
@@ -55,7 +55,7 @@ test('next subtitle navigation advances past the latest overlapping lyric', () =
{ startTime: 16, endTime: 19, text: 'next lyric' },
];
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 13.2), [
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 13.2, 0), [
'seek',
16.08,
'absolute+exact',
@@ -68,7 +68,7 @@ test('previous subtitle navigation leaves the current cue and seeks to the prior
{ startTime: 13, endTime: 16, text: 'current line' },
];
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', -1], cues, 14.5), [
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', -1], cues, 14.5, 0), [
'seek',
10.08,
'absolute+exact',
@@ -78,8 +78,22 @@ test('previous subtitle navigation leaves the current cue and seeks to the prior
test('subtitle navigation falls back when no sanitized destination exists', () => {
const cues = [{ startTime: 10, endTime: 13, text: 'only line' }];
assert.equal(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.2), null);
assert.equal(resolveSanitizedSubtitleSeekCommand(['seek', 5], cues, 10.2), null);
assert.equal(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.2, 0), null);
assert.equal(resolveSanitizedSubtitleSeekCommand(['seek', 5], cues, 10.2, 0), null);
});
test('subtitle cue navigation finds and seeks cues on the sub-delay retimed timeline', () => {
const cues = [
{ startTime: 10, endTime: 13, text: 'one' },
{ startTime: 13, endTime: 16, text: 'two' },
];
// At 14.5 with a 2s delay, mpv is showing "one" (12-15 on the video clock).
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 14.5, 2), [
'seek',
15.08,
'absolute+exact',
]);
});
test('sidebar cue seeks share the boundary-safe sanitized cue timestamp', () => {
+5 -2
View File
@@ -1,3 +1,4 @@
import { toSubtitleClock, toVideoClock } from './subtitle-clock';
import type { SubtitleCue } from './subtitle-cue-parser';
const CUE_START_GROUP_TOLERANCE_SECONDS = 0.05;
@@ -80,8 +81,10 @@ export function subtitleCueListSeekTime(
export function resolveSanitizedSubtitleSeekCommand(
command: readonly (string | number)[],
cues: readonly SubtitleCue[],
currentTimeSec: number,
videoTimeSec: number,
subDelay: number,
): (string | number)[] | null {
const currentTimeSec = toSubtitleClock(videoTimeSec, subDelay);
if (
command.length < 2 ||
command[0] !== 'sub-seek' ||
@@ -124,5 +127,5 @@ export function resolveSanitizedSubtitleSeekCommand(
if (!destination) {
return null;
}
return ['seek', subtitleCueSeekTime(destination.cue), 'absolute+exact'];
return ['seek', toVideoClock(subtitleCueSeekTime(destination.cue), subDelay), 'absolute+exact'];
}
+30 -7
View File
@@ -328,9 +328,11 @@ import {
setMpvSubVisibilityRuntime,
setOverlayDebugVisualizationEnabledRuntime,
setVisibleOverlayVisible as setVisibleOverlayVisibleCore,
shiftSubtitleRangeToVideoClock,
showMpvOsdRuntime,
JellyfinPlaybackReporter,
tokenizeSubtitle as tokenizeSubtitleCore,
toSubtitleClock,
triggerFieldGrouping as triggerFieldGroupingCore,
upsertYomitanDictionarySettings,
updateLastCardFromClipboard as updateLastCardFromClipboardCore,
@@ -1937,7 +1939,10 @@ let tokenizeSubtitleDeferred: ((text: string) => Promise<SubtitleData>) | null =
function withCurrentSubtitleTiming(payload: SubtitleData): SubtitleData {
const canonical = resolveCanonicalPrimarySubtitle({
liveText: payload.text,
currentTimeSec: Number(appState.mpvClient?.currentTimePos),
currentTimeSec: toSubtitleClock(
Number(appState.mpvClient?.currentTimePos),
appState.mpvClient?.currentSubDelay ?? 0,
),
cues: appState.activeParsedSubtitleCues,
});
return {
@@ -1951,9 +1956,11 @@ function captureCurrentPrimarySubtitleMiningContext(): SubtitleMiningContext | n
// Mine what the overlay shows, not raw mpv `sub-text`: the raw text lists every active
// event, so a finished caption row lingering beside a fresh line would end up on the
// card. The parsed view also carries the cue's own timings for the clip range.
// Cues sit on the subtitle clock, so the lookup undoes `sub-delay` first.
const subDelay = appState.mpvClient?.currentSubDelay ?? 0;
const resolved = resolvePrimarySubtitle({
liveText: appState.mpvClient?.currentSubText ?? '',
currentTimeSec: Number(appState.mpvClient?.currentTimePos),
currentTimeSec: toSubtitleClock(Number(appState.mpvClient?.currentTimePos), subDelay),
cues: appState.activeParsedSubtitleCues,
});
// Same validity bar as the live capture path: an unusable resolved span must fall
@@ -1968,13 +1975,16 @@ function captureCurrentPrimarySubtitleMiningContext(): SubtitleMiningContext | n
) {
return captureLiveSubtitleMiningContext(appState.mpvClient);
}
return {
return shiftSubtitleRangeToVideoClock<SubtitleMiningContext>(
{
source: 'overlay',
text: resolvedText,
startTime: resolved.startTime,
endTime: resolved.endTime,
capturedAtMs: Date.now(),
};
},
subDelay,
);
}
function emitSubtitlePayload(payload: SubtitleData, options?: { resumePrefetch?: boolean }): void {
const timedPayload = withCurrentSubtitleTiming(payload);
@@ -2116,13 +2126,18 @@ function getSubtitleSidebarPlaybackState(): SubtitleSidebarPlaybackState {
subtitleSidebarCuesRef = appState.activeParsedSubtitleCues;
subtitleSidebarCuesRevision += 1;
}
const subDelay = appState.mpvClient?.currentSubDelay ?? 0;
return {
sourceKey: JSON.stringify([
appState.activeParsedSubtitleMediaPath,
appState.activeParsedSubtitleSource,
]),
cuesRevision: subtitleSidebarCuesRevision,
currentTimeSec: appState.mpvClient?.currentTimePos ?? null,
currentTimeSec:
appState.mpvClient?.currentTimePos === undefined
? null
: toSubtitleClock(appState.mpvClient.currentTimePos, subDelay),
subDelay,
currentSubtitle: {
text: appState.currentSubText,
startTime: appState.mpvClient?.currentSubStart ?? null,
@@ -2161,6 +2176,7 @@ function sendRendererMpvCommand(rawCommand: (string | number)[]): void {
rawCommand,
appState.activeParsedSubtitleCues,
appState.mpvClient?.currentTimePos ?? Number.NaN,
appState.mpvClient?.currentSubDelay ?? 0,
) ?? rawCommand;
if (isExplicitMpvSeekCommand(command)) {
explicitSeekIntentExpiresAtMs = Date.now() + EXPLICIT_SEEK_INTENT_TTL_MS;
@@ -3115,7 +3131,10 @@ const mediaTimingReviewRuntime = createMediaTimingReviewRuntime({
acquireMediaWindow: (source, range) => getSharedRemoteMediaWindowCache().acquire(source, range),
getSubtitleContextLines: (range) =>
collectMediaTimingContextLines({
cues: appState.activeParsedSubtitleCues,
// Review ranges and tracker entries are on the video clock; parsed cues are not.
cues: appState.activeParsedSubtitleCues.map((cue) =>
shiftSubtitleRangeToVideoClock(cue, appState.mpvClient?.currentSubDelay ?? 0),
),
fallbackPrevious: appState.subtitleTimingTracker?.getRecentEntries(40) ?? [],
startTime: range.startTime,
endTime: range.endTime,
@@ -6028,7 +6047,11 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
// live mpv sub timings at lookup time so media generation clips the mined line even
// when extraction finishes long after playback has moved on.
recordSubtitleMiningContext: (context) =>
recordSubtitleMiningContext(context ?? captureCurrentPrimarySubtitleMiningContext()),
recordSubtitleMiningContext(
context
? shiftSubtitleRangeToVideoClock(context, appState.mpvClient?.currentSubDelay ?? 0)
: captureCurrentPrimarySubtitleMiningContext(),
),
tokenizeCurrentSubtitle: async () => {
const tokenizeSubtitleForCurrent = tokenizeSubtitleDeferred;
return resolveCurrentSubtitleForRenderer({
@@ -1,3 +1,4 @@
import { toSubtitleClock } from '../../core/services/subtitle-clock';
import type { SubtitleCue, SubtitleData } from '../../types';
import { selectAutoplayStartupCue } from './autoplay-subtitle-primer';
import { primeVisibleOverlaySubtitleFromMpv } from './current-subtitle-snapshot';
@@ -12,6 +13,7 @@ type AutoplaySubtitlePrimingMpvClient = {
requestProperty: (name: string) => Promise<unknown>;
currentVideoPath?: string;
currentTimePos?: number;
currentSubDelay?: number;
currentSubText?: string;
currentSecondarySubText?: string;
setCurrentSecondarySubText?: (text: string) => void;
@@ -156,7 +158,7 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
const currentTimeSec = Number(client?.currentTimePos ?? deps.getLastObservedTimePos());
return resolvePrimarySubtitleText({
liveText: text,
currentTimeSec,
currentTimeSec: toSubtitleClock(currentTimeSec, client?.currentSubDelay ?? 0),
cues: deps.getActiveParsedSubtitleCues(),
});
}
@@ -270,7 +272,10 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
const currentTimeSeconds = Number(
timePosRaw ?? client?.currentTimePos ?? deps.getLastObservedTimePos() ?? 0,
);
const resolvedTimeSeconds = Number.isFinite(currentTimeSeconds) ? currentTimeSeconds : 0;
const resolvedTimeSeconds = toSubtitleClock(
Number.isFinite(currentTimeSeconds) ? currentTimeSeconds : 0,
client?.currentSubDelay ?? 0,
);
const cue = selectAutoplayStartupCue(
cues,
resolvedTimeSeconds,
@@ -176,6 +176,21 @@ test('subtitle-track transitions ignore stale parsed cues until replacement cues
assert.deepEqual(recordedStarts.slice(-1), [20]);
});
test('subtitle timing history records lines on the video clock under sub-delay', () => {
const timing: Array<{ text: string; start: number; end: number }> = [];
const { handlers } = buildMainDeps({
mpvClient: { currentSubDelay: 2 },
subtitleTimingTracker: {
recordSubtitle: (text: string, start: number, end: number) =>
timing.push({ text, start, end }),
},
});
handlers.recordSubtitleTiming('食べる', 5, 8);
assert.deepEqual(timing, [{ text: '食べる', start: 7, end: 10 }]);
});
test('canonical ASS cues replace live glyph spam for display, history, and immersion', () => {
const immersion: Array<{ text: string; start: number; end: number }> = [];
const timing: Array<{ text: string; start: number; end: number }> = [];
+13 -5
View File
@@ -1,3 +1,4 @@
import { toSubtitleClock } from '../../core/services/subtitle-clock';
import { createSubtitleLineDedupGate } from '../../core/services/subtitle-line-dedup-gate';
import type { MergedToken, SubtitleCue, SubtitleData } from '../../types';
import {
@@ -23,6 +24,7 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
currentSubText?: string;
currentSecondarySubText?: string;
currentTimePos?: number;
currentSubDelay?: number;
requestProperty?: (name: string) => Promise<unknown>;
} | null;
immersionTracker: {
@@ -161,7 +163,10 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
resolveSubtitleText: (liveText: string) =>
resolvePrimarySubtitleText({
liveText,
currentTimeSec: Number(deps.appState.mpvClient?.currentTimePos),
currentTimeSec: toSubtitleClock(
Number(deps.appState.mpvClient?.currentTimePos),
deps.appState.mpvClient?.currentSubDelay ?? 0,
),
cues: deps.appState.activeParsedSubtitleCues,
}),
getCurrentLiveSubtitleText: () => deps.appState.mpvClient?.currentSubText ?? '',
@@ -231,6 +236,9 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
hasSubtitleTimingTracker: () => Boolean(deps.appState.subtitleTimingTracker),
recordSubtitleTiming: (text: string, start: number, end: number) => {
const secondaryText = deps.appState.mpvClient?.currentSecondarySubText || undefined;
// The tracker feeds multi-line mining, so it stores video-clock times: the subtitle
// timings plus the `sub-delay` in effect while the line was on screen.
const subDelay = deps.appState.mpvClient?.currentSubDelay ?? 0;
const canonical = resolveCanonicalSample(text, start);
if (!canonical) {
const recordableText = resolveTextForRecording(text, start);
@@ -239,8 +247,8 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
}
deps.appState.subtitleTimingTracker?.recordSubtitle?.(
recordableText,
start,
end,
start + subDelay,
end + subDelay,
secondaryText,
);
return;
@@ -253,8 +261,8 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
recordedTimingCanonicalKeys.add(key);
deps.appState.subtitleTimingTracker?.recordSubtitle?.(
cue.text,
cue.startTime,
cue.endTime,
cue.startTime + subDelay,
cue.endTime + subDelay,
secondaryText,
);
}
@@ -28,6 +28,7 @@ export async function setup() {
],
currentSubtitle: { text: '最初の台詞', startTime: 0, endTime: 1 },
currentTimeSec: 0,
subDelay: 0,
config: {
...SUBTITLE_DEFAULT_CONFIG.subtitleSidebar,
enabled: true,
+13 -2
View File
@@ -199,6 +199,7 @@ function createSidebarHarness(options: SidebarHarnessOptions = {}) {
cues: options.cues ?? [{ startTime: 1, endTime: 2, text: 'first' }],
currentSubtitle: options.currentSubtitle ?? { text: 'first', startTime: 1, endTime: 2 },
currentTimeSec: 'currentTimeSec' in options ? (options.currentTimeSec ?? null) : 1.1,
subDelay: 0,
config: { ...DEFAULT_SIDEBAR_CONFIG, ...options.config },
};
const requests = { snapshot: 0, playback: 0 };
@@ -226,8 +227,8 @@ function createSidebarHarness(options: SidebarHarnessOptions = {}) {
},
getSubtitleSidebarPlaybackState: async (): Promise<SubtitleSidebarPlaybackState> => {
requests.playback += 1;
const { sourceKey, cuesRevision, currentTimeSec, currentSubtitle } = snapshot;
return { sourceKey, cuesRevision, currentTimeSec, currentSubtitle };
const { sourceKey, cuesRevision, currentTimeSec, subDelay, currentSubtitle } = snapshot;
return { sourceKey, cuesRevision, currentTimeSec, subDelay, currentSubtitle };
},
getPlaybackPaused: async () => false,
sendMpvCommand: (command: MpvCommand) => {
@@ -528,6 +529,16 @@ test('subtitle sidebar seeks to the selected cue start using the overlap-aware s
assert.deepEqual(h.mpvCommands.at(-1), ['seek', 3.48, 'absolute+exact']);
});
test('subtitle sidebar seeks to where mpv plays a sub-delay retimed cue', async () => {
const h = createSidebarHarness();
h.setSnapshot({ subDelay: 2 });
await h.modal.openSubtitleSidebarModal();
h.modal.seekToCue({ startTime: 1, endTime: 2, text: 'first' });
assert.deepEqual(h.mpvCommands.at(-1), ['seek', 3.08, 'absolute+exact']);
});
test('subtitle sidebar close reports hidden visibility and notifies the modal close', async () => {
const h = createSidebarHarness();
await h.modal.openSubtitleSidebarModal();
+6 -1
View File
@@ -4,6 +4,7 @@ import type {
SubtitleMiningContext,
SubtitleSidebarSnapshot,
} from '../../types';
import { toVideoClock } from '../../core/services/subtitle-clock.js';
import { subtitleCueListSeekTime } from '../../core/services/subtitle-cue-navigation.js';
import type { ModalStateReader, RendererContext } from '../context';
import { syncOverlayMouseIgnoreState } from '../overlay-mouse-ignore.js';
@@ -231,6 +232,8 @@ export function createSubtitleSidebarModal(
let lastSubtitleSidebarLookupCueIndex = -1;
let subtitleSourceKey: string | null = null;
let subtitleCuesRevision: number | null = null;
// Latest mpv `sub-delay` from main; cue times plus this are where mpv plays the cue.
let subtitleSubDelay = 0;
function restoreEmbeddedSidebarPassthrough(): void {
syncOverlayMouseIgnoreState(ctx);
@@ -416,7 +419,7 @@ export function createSubtitleSidebarModal(
function seekToCue(cue: SubtitleCue): void {
window.electronAPI.sendMpvCommand([
'seek',
subtitleCueListSeekTime(ctx.state.subtitleSidebarCues, cue),
toVideoClock(subtitleCueListSeekTime(ctx.state.subtitleSidebarCues, cue), subtitleSubDelay),
'absolute+exact',
]);
}
@@ -591,6 +594,7 @@ export function createSubtitleSidebarModal(
const snapshot = await window.electronAPI.getSubtitleSidebarSnapshot();
pollsSinceFullSnapshot = 0;
subtitleCuesRevision = snapshot.cuesRevision;
subtitleSubDelay = snapshot.subDelay;
if (snapshot.sourceKey !== subtitleSourceKey) {
clearSubtitleSidebarSelection(ctx.dom.subtitleSidebarList);
lastSubtitleSidebarLookupCueIndex = -1;
@@ -640,6 +644,7 @@ export function createSubtitleSidebarModal(
await refreshSnapshot();
return;
}
subtitleSubDelay = playback.subDelay;
updateActiveCue(playback.currentSubtitle, playback.currentTimeSec);
}
+2
View File
@@ -90,6 +90,8 @@ export interface MpvClient {
currentTimePos: number;
currentSubStart: number;
currentSubEnd: number;
/** mpv `sub-delay`; `sub-start`/`sub-end` do not include it. */
currentSubDelay: number;
currentAudioStreamIndex: number | null;
requestProperty?: (name: string) => Promise<unknown>;
send(command: { command: unknown[]; request_id?: number }): boolean;
+4
View File
@@ -235,7 +235,10 @@ export interface SubtitleSidebarPlaybackState {
sourceKey: string | null;
/** Bumps whenever main swaps in a different parsed cue array. */
cuesRevision: number;
/** Playback position on the subtitle clock (`time-pos` minus `sub-delay`), like cue times. */
currentTimeSec: number | null;
/** mpv `sub-delay`: cue times plus this give the video-clock seek target. */
subDelay: number;
currentSubtitle: {
text: string;
startTime: number | null;
@@ -251,6 +254,7 @@ export interface SubtitleSidebarSnapshot extends SubtitleSidebarPlaybackState {
export interface SubtitleMiningContext {
source: 'subtitle-sidebar' | 'overlay';
text: string;
/** Video-clock seconds: subtitle timings plus mpv `sub-delay` at capture time. */
startTime: number;
endTime: number;
capturedAtMs?: number;