fix(subtitles): collapse layered ASS lyrics across startup and seeks

- Reconstruct fragment-only karaoke while preserving canonical animated signs
- Reprocess live subtitles after initial playback and seek updates
- Add a development launcher for playable media files
This commit is contained in:
2026-08-19 22:55:10 -07:00
parent 695613b1e7
commit c01bcd9d0f
16 changed files with 830 additions and 55 deletions
+3 -3
View File
@@ -149,8 +149,8 @@ function collectRepeatedPhaseRuns(cues: AnnotatedSubtitleCue[]): RepeatedPhaseRu
const isFlush =
Math.abs(next.startTime - current.endTime) <= DUPLICATE_CUE_GAP_TOLERANCE_SECONDS;
if (
first.source === 'canonical-ass' ||
next.source === 'canonical-ass' ||
first.source !== undefined ||
next.source !== undefined ||
next.text !== first.text ||
assStyleKey(next) !== styleKey ||
!isFlush
@@ -223,7 +223,7 @@ function countFramesShorterThan(run: AnnotatedSubtitleCue[], maxSeconds: number)
* anything wrapped in `\t(...)`), an animated `Effect` column, or a value that actually
* changes from event to event, which is how per-frame typesetting is authored.
*/
export function hasAssAnimationEvidence(run: AnnotatedSubtitleCue[]): boolean {
export function hasAssAnimationEvidence(run: readonly AnnotatedSubtitleCue[]): boolean {
if (run.every((cue) => hasAssTemporalOverride(cue.overrides))) {
return true;
}
@@ -570,6 +570,63 @@ test('parseSubtitleCues does not promote a short animated fragment as a complete
);
});
test('parseSubtitleCues keeps short animated English dialogue as separate cues', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:02.00,English Dialogue,,0,0,0,,{\\t(0,100,\\fscx110)}Hi',
'Dialogue: 0,0:00:02.00,0:00:03.00,English Dialogue,,0,0,0,,{\\t(0,100,\\fscx110)}No',
].join('\n');
assert.deepEqual(parseSubtitleCues(content, 'test.ass'), [
{ startTime: 1, endTime: 2, text: 'Hi' },
{ startTime: 2, endTime: 3, text: 'No' },
]);
});
test('parseSubtitleCues does not reconstruct an already canonical English cue', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Comment: 0,0:00:01.00,0:00:03.00,OP English,,0,0,0,,{\\move(100,100,120,100)}POOF',
'Dialogue: 0,0:00:01.00,0:00:01.04,OP English,,0,0,0,,{\\pos(100,100)\\clip(m 1 1)}POOF',
'Dialogue: 0,0:00:01.04,0:00:01.08,OP English,,0,0,0,,{\\pos(100,100)\\clip(m 2 2)}POOF',
'Dialogue: 0,0:00:01.08,0:00:03.00,OP English,,0,0,0,,{\\pos(100,100)\\clip(m 3 3)}POOF',
].join('\n');
assert.deepEqual(parseSubtitleCues(content, 'test.ass'), [
{
startTime: 1,
endTime: 3,
text: 'POOF',
source: 'canonical-ass',
animationStartTime: 1,
animationEndTime: 3,
},
]);
});
test('parseSubtitleCues reconstructs a short positioned fragment without a lyric style name', () => {
const content = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:03.00,Karaoke,,0,0,0,,{\\pos(100,100)\\t(0,100,\\fscx110)}Oh',
'Dialogue: 1,0:00:01.00,0:00:03.00,Karaoke,,0,0,0,,{\\pos(100,100)\\t(0,100,\\fscx110)}Oh',
].join('\n');
assert.deepEqual(parseSubtitleCues(content, 'test.ass'), [
{
startTime: 1,
endTime: 3,
text: 'Oh',
source: 'reconstructed-ass',
animationStartTime: 1,
animationEndTime: 3,
assStyle: 'Karaoke',
},
]);
});
test('parseSubtitleCues ignores timed comments without a matching animated dialogue cluster', () => {
const content = [
'[Events]',
+304 -12
View File
@@ -8,19 +8,26 @@ import {
} from './ass-text';
import { hasAssAnimationEvidence, mergeDuplicateCues } from './subtitle-cue-dedup';
export type AssCueLayout =
| { kind: 'positioned'; sourceOrder: number; y: number }
| { kind: 'source-order'; sourceOrder: number };
export interface SubtitleCue {
startTime: number;
endTime: number;
text: string;
/** A complete authored line recovered from matching generated ASS animation events. */
source?: 'canonical-ass';
/** How a complete line was recovered from generated ASS animation events. */
source?: 'canonical-ass' | 'reconstructed-ass';
/**
* Full span of the generated animation events a canonical cue replaced. Entrance and
* exit frames routinely run past the authored `startTime`/`endTime`, so live-text
* matching must use this envelope while display and history keep the authored timing.
* Full span of the generated animation events a recovered cue replaced. Entrance and
* exit frames can run past canonical authored timing.
*/
animationStartTime?: number;
animationEndTime?: number;
/** ASS style retained only for fragment-reconstructed lines. */
assStyle?: string;
/** Authored ASS ordering metadata used when flattening simultaneous positioned cues. */
assLayout?: AssCueLayout;
}
/**
@@ -29,7 +36,7 @@ export interface SubtitleCue {
* override commands it carries, whether the `Effect` column was set -- to tell a karaoke
* burst apart from two characters saying the same word in turn. None of it is meaningful
* outside the parser, so the public API exposes only timing, text, and the optional
* canonical-source marker used by live subtitle consumers.
* recovery marker used by live subtitle consumers.
*/
export interface AnnotatedSubtitleCue extends SubtitleCue {
/** Text exactly as authored, override blocks and all. */
@@ -75,15 +82,55 @@ function parseTimestamp(
* line breaks, matching what mpv hands over for the same line played live. No layer
* downstream decodes ASS again.
*/
function decodeSubtitleCueText(text: string): string {
return assToPlainText(text, '\n').replace(HTML_SUBTITLE_TAG_PATTERN, '');
}
function sanitizeSubtitleCueText(text: string): string {
return assToPlainText(text, '\n').replace(HTML_SUBTITLE_TAG_PATTERN, '').trim();
return decodeSubtitleCueText(text).trim();
}
function attachAssLayout<T extends SubtitleCue>(cue: T, assLayout: AssCueLayout | undefined): T {
if (assLayout) {
Object.defineProperty(cue, 'assLayout', { value: assLayout, enumerable: false });
}
return cue;
}
function toPublicCues(cues: AnnotatedSubtitleCue[]): SubtitleCue[] {
return cues.map(({ startTime, endTime, text, source, animationStartTime, animationEndTime }) =>
source
? { startTime, endTime, text, source, animationStartTime, animationEndTime }
: { startTime, endTime, text },
return cues.map(
({
startTime,
endTime,
text,
source,
animationStartTime,
animationEndTime,
style,
assLayout,
}) => {
const common = {
startTime,
endTime,
text,
};
if (source === 'reconstructed-ass') {
return attachAssLayout(
{
...common,
source,
animationStartTime,
animationEndTime,
assStyle: style,
},
assLayout,
);
}
return attachAssLayout(
source ? { ...common, source, animationStartTime, animationEndTime } : common,
assLayout,
);
},
);
}
@@ -159,6 +206,10 @@ const MIN_CANONICAL_ANIMATION_EVENTS = 3;
// A tiny animated fragment can itself be composed from still smaller glyph events. It is
// not enough evidence that the fragment represents an authored line boundary.
const MIN_CANONICAL_DIALOGUE_TEXT_LENGTH = 4;
const MIN_FRAGMENT_LINE_EVENTS = 8;
const MIN_FRAGMENT_LINE_PARTS = 4;
const MAX_FRAGMENT_MEDIAN_LENGTH = 4;
const MAX_FRAGMENT_LINE_TIMING_VARIANCE_SECONDS = 2;
function parseAssTimestamp(raw: string): number | null {
const match = ASS_TIMING_PATTERN.exec(raw.trim());
@@ -296,6 +347,215 @@ function isRepeatedFragmentCopy(
);
}
function hasRelaxedAssFragmentEvidence(events: readonly AnnotatedSubtitleCue[]): boolean {
if (events.length < 2 || !events.every((event) => fragmentPlacementAnchors(event).size > 0)) {
return false;
}
const latestStart = events.reduce(
(latest, event) => Math.max(latest, event.startTime),
-Infinity,
);
const earliestEnd = events.reduce(
(earliest, event) => Math.min(earliest, event.endTime),
Infinity,
);
if (latestStart >= earliestEnd) {
return false;
}
const first = events[0]!;
const hasChangingOverrides = events.some(
(event) => event.overrideSignature !== first.overrideSignature,
);
const hasPositionedLayerCopy = events.some((event, index) =>
events
.slice(0, index)
.some(
(previous) =>
compactCueMatchText(previous) === compactCueMatchText(event) &&
isRepeatedFragmentCopy(previous, event),
),
);
return hasChangingOverrides || hasPositionedLayerCopy;
}
interface AssFragmentPart {
cue: AnnotatedSubtitleCue;
text: string;
}
interface AssFragmentTimingCluster {
events: AnnotatedSubtitleCue[];
minStartTime: number;
maxStartTime: number;
minEndTime: number;
maxEndTime: number;
}
function addToFragmentTimingCluster(
cluster: AssFragmentTimingCluster,
cue: AnnotatedSubtitleCue,
): void {
cluster.events.push(cue);
cluster.minStartTime = Math.min(cluster.minStartTime, cue.startTime);
cluster.maxStartTime = Math.max(cluster.maxStartTime, cue.startTime);
cluster.minEndTime = Math.min(cluster.minEndTime, cue.endTime);
cluster.maxEndTime = Math.max(cluster.maxEndTime, cue.endTime);
}
function fragmentTimingDistance(
cluster: AssFragmentTimingCluster,
cue: AnnotatedSubtitleCue,
): number {
const nextMinStart = Math.min(cluster.minStartTime, cue.startTime);
const nextMaxStart = Math.max(cluster.maxStartTime, cue.startTime);
const nextMinEnd = Math.min(cluster.minEndTime, cue.endTime);
const nextMaxEnd = Math.max(cluster.maxEndTime, cue.endTime);
if (
nextMaxStart - nextMinStart > MAX_FRAGMENT_LINE_TIMING_VARIANCE_SECONDS ||
nextMaxEnd - nextMinEnd > MAX_FRAGMENT_LINE_TIMING_VARIANCE_SECONDS
) {
return Infinity;
}
return (
Math.abs(cue.startTime - (cluster.minStartTime + cluster.maxStartTime) / 2) +
Math.abs(cue.endTime - (cluster.minEndTime + cluster.maxEndTime) / 2)
);
}
function clusterAssFragmentEvents(
events: readonly AnnotatedSubtitleCue[],
): AssFragmentTimingCluster[] {
const clusters: AssFragmentTimingCluster[] = [];
for (const cue of events) {
let nearest: AssFragmentTimingCluster | null = null;
let nearestDistance = Infinity;
for (const cluster of clusters) {
const distance = fragmentTimingDistance(cluster, cue);
if (distance < nearestDistance) {
nearest = cluster;
nearestDistance = distance;
}
}
if (nearest) {
addToFragmentTimingCluster(nearest, cue);
} else {
clusters.push({
events: [cue],
minStartTime: cue.startTime,
maxStartTime: cue.startTime,
minEndTime: cue.endTime,
maxEndTime: cue.endTime,
});
}
}
return clusters;
}
function decodeSingleAssFragment(cue: AnnotatedSubtitleCue): string | null {
const visibleLines = decodeSubtitleCueText(cue.rawText)
.split('\n')
.filter((line) => line.trim().length > 0);
return visibleLines.length === 1 ? visibleLines[0]! : null;
}
function reconstructAssFragmentLine(
events: readonly AnnotatedSubtitleCue[],
): AnnotatedSubtitleCue | null {
const hasRelaxedEvidence = hasRelaxedAssFragmentEvidence(events);
const minimumEvents = hasRelaxedEvidence ? 2 : MIN_FRAGMENT_LINE_EVENTS;
if (events.length < minimumEvents || !hasAssAnimationEvidence(events)) {
return null;
}
const parts: AssFragmentPart[] = [];
for (const cue of events) {
const text = decodeSingleAssFragment(cue);
if (text === null) {
return null;
}
const compactText = compactAssMatchText(text);
const isLayerCopy = parts.some(
(part) =>
compactAssMatchText(part.text) === compactText && isRepeatedFragmentCopy(part.cue, cue),
);
if (!isLayerCopy) {
parts.push({ cue, text });
}
}
const minimumParts = hasRelaxedEvidence ? 1 : MIN_FRAGMENT_LINE_PARTS;
if (parts.length < minimumParts || (!hasRelaxedEvidence && parts.length === events.length)) {
return null;
}
const lengths = parts
.map((part) => compactAssMatchText(part.text).length)
.sort((left, right) => left - right);
if ((lengths[Math.floor(lengths.length / 2)] ?? Infinity) > MAX_FRAGMENT_MEDIAN_LENGTH) {
return null;
}
const text = parts
.map((part) => part.text)
.join('')
.trim();
if (!text) {
return null;
}
const owner = parts[0]!.cue;
const animationStartTime = earliestStartTime(events);
const animationEndTime = latestEndTime(events);
return {
...owner,
startTime: animationStartTime,
endTime: animationEndTime,
text,
rawText: text,
source: 'reconstructed-ass',
animationStartTime,
animationEndTime,
overrides: [],
overrideSignature: '',
};
}
function recoverFragmentOnlyAssLines(dialogue: AnnotatedSubtitleCue[]): AnnotatedSubtitleCue[] {
const groups = new Map<string, AnnotatedSubtitleCue[]>();
for (const cue of dialogue) {
if (cue.source !== undefined) {
continue;
}
const key = assEventGroupKey(cue);
const group = groups.get(key);
if (group) {
group.push(cue);
} else {
groups.set(key, [cue]);
}
}
const recovered: AnnotatedSubtitleCue[] = [];
const suppressed = new Set<AnnotatedSubtitleCue>();
for (const events of groups.values()) {
for (const cluster of clusterAssFragmentEvents(events)) {
const line = reconstructAssFragmentLine(cluster.events);
if (!line) {
continue;
}
recovered.push(line);
cluster.events.forEach((event) => suppressed.add(event));
}
}
if (recovered.length === 0) {
return dialogue;
}
return [...dialogue.filter((cue) => !suppressed.has(cue)), ...recovered].sort(
(left, right) =>
left.startTime - right.startTime || left.endTime - right.endTime || left.order - right.order,
);
}
function groupConsecutiveAssFragments(events: readonly AnnotatedSubtitleCue[]): FragmentGroup[] {
const groups: FragmentGroup[] = [];
for (const event of events) {
@@ -507,6 +767,37 @@ function recoverCanonicalAssEvents({
);
}
function parseAssCoordinate(value: string | undefined): number | null {
if (!value?.trim()) return null;
const coordinate = Number(value.trim());
return Number.isFinite(coordinate) ? coordinate : null;
}
function buildAssCueLayout(
overrides: readonly AssOverrideCommand[],
sourceOrder: number,
): AssCueLayout {
let y: number | null = null;
for (const command of overrides) {
if (command.animated) continue;
const name = command.name.toLowerCase();
const args = command.args.split(',');
if (name === 'pos') {
y = parseAssCoordinate(args[1]) ?? y;
continue;
}
if (name !== 'move') continue;
const startY = parseAssCoordinate(args[1]);
const endY = parseAssCoordinate(args[3]);
if (startY !== null && endY !== null) {
y = (startY + endY) / 2;
}
}
return y === null
? { kind: 'source-order', sourceOrder }
: { kind: 'positioned', sourceOrder, y };
}
function parseAnnotatedAssEvents(content: string): ParsedAssEvents {
const cues: AnnotatedSubtitleCue[] = [];
const comments: AnnotatedSubtitleCue[] = [];
@@ -615,6 +906,7 @@ function parseAnnotatedAssEvents(content: string): ParsedAssEvents {
overrides,
overrideSignature: assOverrideSignature(overrides),
order: eventOrder,
assLayout: buildAssCueLayout(overrides, eventOrder),
};
eventOrder += 1;
if (eventPrefix === ASS_COMMENT_PREFIX) {
@@ -628,7 +920,7 @@ function parseAnnotatedAssEvents(content: string): ParsedAssEvents {
}
function parseAnnotatedAssCues(content: string): AnnotatedSubtitleCue[] {
return recoverCanonicalAssEvents(parseAnnotatedAssEvents(content));
return recoverFragmentOnlyAssLines(recoverCanonicalAssEvents(parseAnnotatedAssEvents(content)));
}
export function parseAssCues(content: string): SubtitleCue[] {
@@ -1,5 +1,6 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { parseSubtitleCues } from '../../core/services/subtitle-cue-parser';
import { createSubtitleProcessingController } from '../../core/services/subtitle-processing-controller';
import type { SubtitleData } from '../../types';
import {
@@ -211,6 +212,69 @@ test('primeCurrentSubtitleForAutoplay emits raw first paint on cache miss before
]);
});
test('parsed cues replace a duplicate raw autoplay subtitle that was already primed', async () => {
const rawText = 'ジグザグな道を抜け\nジグザグな道を抜け';
const correctedText = 'ジグザグな道を抜け';
const mediaPath = '/media/video.mkv';
let currentSubText = '';
const emitted: string[] = [];
const client = {
connected: true,
currentVideoPath: mediaPath,
currentTimePos: 90,
currentSubText: rawText,
requestProperty: async (name: string) => {
if (name === 'sub-text') return rawText;
if (name === 'time-pos') return 90;
return null;
},
};
const cues = parseSubtitleCues(
[
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
`Dialogue: 1,0:01:29.00,0:01:32.00,EDJP,,0,0,0,,${correctedText}`,
`Dialogue: 0,0:01:29.00,0:01:32.00,EDJP,,0,0,0,,${correctedText}`,
].join('\n'),
'startup-ending.ass',
);
let activeCues = cues.slice(0, 0);
const runtime = createAutoplaySubtitlePrimingRuntime({
getCurrentMediaPath: () => mediaPath,
getMpvClient: () => client,
setCurrentSubText: (text) => {
currentSubText = text;
},
getCurrentSubText: () => currentSubText,
getCurrentSubtitleData: () => null,
getActiveParsedSubtitleCues: () => activeCues,
setActiveParsedSubtitleMediaPath: () => {},
subtitleProcessingController: {
consumeCachedSubtitle: () => null,
onSubtitleChange: () => true,
refreshCurrentSubtitle: () => true,
notePlainSubtitleEmitted: () => {},
},
emitSubtitlePayload: (payload) => emitted.push(payload.text),
getSubtitlePrefetchService: () => null,
getLastObservedTimePos: () => 90,
getVisibleOverlayVisible: () => true,
emitSecondarySubtitle: () => {},
initSubtitlePrefetch: async () => {},
refreshSubtitlePrefetchFromActiveTrack: async () => {},
logDebug: () => {},
});
await runtime.primeCurrentSubtitleForAutoplay(mediaPath);
assert.equal(currentSubText, rawText);
activeCues = cues;
await runtime.primeAutoplaySubtitleFromParsedCues(mediaPath, cues);
assert.equal(currentSubText, correctedText);
assert.deepEqual(emitted, [rawText, correctedText]);
});
// Driven by the real processing controller rather than a stub: the failure this
// covers is a disagreement between the priming path and the controller's own
// staleness rules, which a hand-written stub cannot reproduce.
@@ -12,6 +12,7 @@ type AutoplaySubtitlePrimingMpvClient = {
requestProperty: (name: string) => Promise<unknown>;
currentVideoPath?: string;
currentTimePos?: number;
currentSubText?: string;
currentSecondarySubText?: string;
setCurrentSecondarySubText?: (text: string) => void;
};
@@ -107,11 +108,19 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
autoplaySubtitlePrimedMediaPath = null;
}
function emitAutoplayPrimedSubtitle(mediaPath: string, text: string): boolean {
function emitAutoplayPrimedSubtitle(
mediaPath: string,
text: string,
options: { replaceExisting?: boolean } = {},
): boolean {
if (!text.trim() || !isCurrentAutoplayMediaPath(mediaPath)) {
return false;
}
if (!markAutoplaySubtitlePrimeConsumed(mediaPath)) {
if (autoplaySubtitlePrimedMediaPath === mediaPath) {
if (!options.replaceExisting || deps.getCurrentSubText() === text) {
return false;
}
} else if (!markAutoplaySubtitlePrimeConsumed(mediaPath)) {
return false;
}
@@ -252,11 +261,7 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
mediaPath: string,
cues: SubtitleCue[],
): Promise<void> {
if (
cues.length === 0 ||
autoplaySubtitlePrimedMediaPath === mediaPath ||
!isCurrentAutoplayMediaPath(mediaPath)
) {
if (cues.length === 0 || !isCurrentAutoplayMediaPath(mediaPath)) {
return;
}
@@ -265,16 +270,21 @@ export function createAutoplaySubtitlePrimingRuntime(deps: AutoplaySubtitlePrimi
const currentTimeSeconds = Number(
timePosRaw ?? client?.currentTimePos ?? deps.getLastObservedTimePos() ?? 0,
);
const resolvedTimeSeconds = Number.isFinite(currentTimeSeconds) ? currentTimeSeconds : 0;
const cue = selectAutoplayStartupCue(
cues,
Number.isFinite(currentTimeSeconds) ? currentTimeSeconds : 0,
resolvedTimeSeconds,
AUTOPLAY_SUBTITLE_PRIME_LOOKAHEAD_SECONDS,
);
if (!cue) {
const liveText = client?.currentSubText ?? '';
const text = liveText.trim()
? resolvePrimarySubtitleText({ liveText, currentTimeSec: resolvedTimeSeconds, cues })
: (cue?.text ?? '');
if (!text) {
return;
}
emitAutoplayPrimedSubtitle(mediaPath, cue.text);
emitAutoplayPrimedSubtitle(mediaPath, text, { replaceExisting: true });
}
function clearScheduledSubtitlePrefetchRefresh(): void {
+11 -3
View File
@@ -4,6 +4,8 @@ type AnilistPostWatchRunOptions = {
watchedSeconds?: number;
};
type TimePosUpdateKind = 'initial' | 'playback' | 'seek';
/** Jump size that marks a time-pos change as a seek rather than normal playback. */
export const SEEK_LIKE_TIME_DELTA_SECONDS = 2.5;
@@ -138,12 +140,18 @@ export function createHandleMpvTimePosChangeHandler(deps: {
refreshDiscordPresence: () => void;
maybeRunAnilistPostWatchUpdate?: (options?: AnilistPostWatchRunOptions) => Promise<void>;
logError?: (message: string, error: unknown) => void;
onTimePosUpdate?: (time: number) => void;
onTimePosUpdate?: (time: number, kind: TimePosUpdateKind) => void;
}) {
let lastObservedTime: number | null = null;
return ({ time }: { time: number }): void => {
const forceImmediate = isSeekLikeTimeChange(lastObservedTime, time);
const updateKind: TimePosUpdateKind =
lastObservedTime === null
? 'initial'
: isSeekLikeTimeChange(lastObservedTime, time)
? 'seek'
: 'playback';
const forceImmediate = updateKind === 'seek';
if (Number.isFinite(time)) {
lastObservedTime = time;
}
@@ -153,7 +161,7 @@ export function createHandleMpvTimePosChangeHandler(deps: {
void deps.maybeRunAnilistPostWatchUpdate?.({ watchedSeconds: time }).catch((error) => {
deps.logError?.('AniList post-watch update failed unexpectedly', error);
});
deps.onTimePosUpdate?.(time);
deps.onTimePosUpdate?.(time, updateKind);
};
}
@@ -1,10 +1,23 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { parseSubtitleCues } from '../../core/services/subtitle-cue-parser';
import { createBindMpvMainEventHandlersHandler } from './mpv-main-event-bindings';
import { resolvePrimarySubtitleText } from './primary-subtitle-text';
test('main mpv event binder wires callbacks through to runtime deps', () => {
const handlers = new Map<string, (payload: unknown) => void>();
const calls: string[] = [];
let currentTime = 0;
const seekLiveText = '少しだけ好きになる\n少しだけ好きになる';
const seekCues = parseSubtitleCues(
[
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 1,0:01:29.00,0:01:32.00,EDJP,,0,0,0,,少しだけ好きになる',
'Dialogue: 0,0:01:29.00,0:01:32.00,EDJP,,0,0,0,,少しだけ好きになる',
].join('\n'),
'seek-ending.ass',
);
const bind = createBindMpvMainEventHandlersHandler({
reportJellyfinRemoteStopped: () => calls.push('remote-stopped'),
@@ -27,6 +40,9 @@ test('main mpv event binder wires callbacks through to runtime deps', () => {
calls.push(`post-watch:${options?.watchedSeconds ?? 'none'}`);
},
logSubtitleTimingError: () => calls.push('subtitle-error'),
resolveSubtitleText: (liveText) =>
resolvePrimarySubtitleText({ liveText, currentTimeSec: currentTime, cues: seekCues }),
getCurrentLiveSubtitleText: () => seekLiveText,
setCurrentSubText: (text) => calls.push(`set-sub:${text}`),
getImmediateSubtitlePayload: (text) => ({ text, tokens: [] }),
broadcastSubtitle: (payload) => calls.push(`broadcast-sub:${payload.text}`),
@@ -60,6 +76,9 @@ test('main mpv event binder wires callbacks through to runtime deps', () => {
recordMediaDuration: (duration) => calls.push(`duration:${duration}`),
reportJellyfinRemoteProgress: (forceImmediate) =>
calls.push(`progress:${forceImmediate ? 'force' : 'normal'}`),
onTimePosUpdate: (time) => {
currentTime = time;
},
recordPauseState: (paused) => calls.push(`pause:${paused ? 'yes' : 'no'}`),
updateSubtitleRenderMetrics: () => calls.push('subtitle-metrics'),
@@ -83,7 +102,13 @@ test('main mpv event binder wires callbacks through to runtime deps', () => {
handlers.get('media-path-change')?.({ path: '' });
handlers.get('media-title-change')?.({ title: 'Episode 1' });
handlers.get('subtitle-timing')?.({ text: 'timed line', start: 899, end: 901 });
handlers.get('subtitle-change')?.({ text: seekLiveText });
handlers.get('time-pos-change')?.({ time: 90 });
assert.ok(calls.includes('set-sub:少しだけ好きになる'));
handlers.get('time-pos-change')?.({ time: 2.5 });
handlers.get('subtitle-change')?.({ text: seekLiveText });
handlers.get('time-pos-change')?.({ time: 90 });
handlers.get('pause-change')?.({ paused: true });
assert.ok(calls.includes('set-sub:line'));
+9 -1
View File
@@ -44,6 +44,7 @@ export function createBindMpvMainEventHandlersHandler(deps: {
setCurrentSubText: (text: string) => void;
resolveSubtitleText?: (text: string) => string;
getCurrentLiveSubtitleText?: () => string;
getImmediateSubtitlePayload?: (text: string) => SubtitleData | null;
emitImmediateSubtitle?: (payload: SubtitleData) => void;
broadcastSubtitle: (payload: SubtitleData) => void;
@@ -171,7 +172,14 @@ export function createBindMpvMainEventHandlersHandler(deps: {
refreshDiscordPresence: () => deps.refreshDiscordPresence(),
maybeRunAnilistPostWatchUpdate: (options) => deps.maybeRunAnilistPostWatchUpdate(options),
logError: (message, error) => deps.logSubtitleTimingError(message, error),
onTimePosUpdate: (time) => deps.onTimePosUpdate?.(time),
onTimePosUpdate: (time, updateKind) => {
deps.onTimePosUpdate?.(time);
if (updateKind === 'playback') return;
const liveText = deps.getCurrentLiveSubtitleText?.();
if (liveText !== undefined) {
handleMpvSubtitleChange({ text: liveText });
}
},
});
const handleMpvPauseChange = createHandleMpvPauseChangeHandler({
recordPauseState: (paused) => deps.recordPauseState(paused),
@@ -21,6 +21,7 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
overlayRuntimeInitialized: boolean;
mpvClient: {
connected?: boolean;
currentSubText?: string;
currentSecondarySubText?: string;
currentTimePos?: number;
requestProperty?: (name: string) => Promise<unknown>;
@@ -160,6 +161,7 @@ export function createBuildBindMpvMainEventHandlersMainDepsHandler(deps: {
currentTimeSec: Number(deps.appState.mpvClient?.currentTimePos),
cues: deps.appState.activeParsedSubtitleCues,
}),
getCurrentLiveSubtitleText: () => deps.appState.mpvClient?.currentSubText ?? '',
recordImmersionSubtitleLine: (text: string, start: number, end: number) => {
deps.ensureImmersionTrackerInitialized();
const tracker = deps.appState.immersionTracker;
@@ -64,6 +64,30 @@ test('resolvePrimarySubtitleText combines unique simultaneous parsed cues', () =
);
});
test('resolvePrimarySubtitleText collapses whitespace variants of one ASS lyric', () => {
const ass = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 2,0:00:01.00,0:00:03.00,EDJP,,0,0,0,,少しだけ好きになる',
'Dialogue: 1,0:00:01.00,0:00:03.00,EDJP,,0,0,0,,少しだけ\\h好きになる',
'Dialogue: 0,0:00:01.00,0:00:03.00,EDJP,,0,0,0,,少しだけ 好きになる',
].join('\n');
const cues = parseSubtitleCues(ass, 'polar-opposites-s01e10.ass');
assert.deepEqual(
cues.map((cue) => cue.text),
['少しだけ好きになる', '少しだけ 好きになる', '少しだけ 好きになる'],
);
assert.equal(
resolvePrimarySubtitleText({
liveText: ['少しだけ好きになる', '少しだけ 好きになる', '少しだけ 好きになる'].join('\n'),
currentTimeSec: 2,
cues,
}),
'少しだけ好きになる',
);
});
test('resolvePrimarySubtitleText tolerates stale time-pos at a parsed cue edge', () => {
assert.equal(
resolvePrimarySubtitleText({
@@ -184,6 +208,20 @@ test('resolvePrimarySubtitleText combines simultaneous canonical cues in source
assert.equal(text, 'first\nsecond');
});
test('resolvePrimarySubtitleText collapses whitespace variants of a canonical lyric', () => {
assert.equal(
resolvePrimarySubtitleText({
liveText: '少しだけ好きになる\n少しだけ 好きになる',
currentTimeSec: 2,
cues: [
{ startTime: 1, endTime: 3, text: '少しだけ好きになる', source: 'canonical-ass' },
{ startTime: 1, endTime: 3, text: '少しだけ 好きになる', source: 'canonical-ass' },
],
}),
'少しだけ好きになる',
);
});
test('resolveCanonicalPrimarySubtitle covers a nearby generated animation edge', () => {
const cue = {
startTime: 1.2,
+18 -17
View File
@@ -25,7 +25,7 @@ function nearbyCanonicalCues(
currentTimeSec: number,
): SubtitleCue[] {
return (cues ?? []).filter((cue) => {
if (cue.source !== 'canonical-ass') {
if (cue.source !== 'canonical-ass' && cue.source !== 'reconstructed-ass') {
return false;
}
const span = animationSpan(cue);
@@ -40,6 +40,21 @@ function compactWhitespace(text: string): string {
return text.replace(/\s+/gu, '');
}
// ASS layers can encode the same visible spacing with ordinary, hard, or
// ideographic spaces. Matching and emission must use the same identity or each
// layer reappears as a copy.
function uniqueCueTexts(cues: readonly SubtitleCue[]): string[] {
const texts: string[] = [];
const seen = new Set<string>();
for (const cue of cues) {
const compactText = compactWhitespace(cue.text);
if (seen.has(compactText)) continue;
seen.add(compactText);
texts.push(cue.text);
}
return texts;
}
function compactLineSegments(text: string): string[] {
return text.split('\n').map(compactWhitespace).filter(Boolean);
}
@@ -83,14 +98,7 @@ function resolveActiveParsedPrimarySubtitle(options: {
return null;
}
const texts: string[] = [];
const seen = new Set<string>();
for (const cue of selected) {
if (!seen.has(cue.text)) {
seen.add(cue.text);
texts.push(cue.text);
}
}
const texts = uniqueCueTexts(selected);
return {
text: texts.join('\n'),
startTime: Math.min(...selected.map((cue) => cue.startTime)),
@@ -159,14 +167,7 @@ export function resolveCanonicalPrimarySubtitle(options: {
return null;
}
const texts: string[] = [];
const seen = new Set<string>();
for (const cue of selected) {
if (!seen.has(cue.text)) {
seen.add(cue.text);
texts.push(cue.text);
}
}
const texts = uniqueCueTexts(selected);
return {
text: texts.join('\n'),
startTime: Math.min(...selected.map((cue) => cue.startTime)),
@@ -20,6 +20,156 @@ test('findActiveSubtitleText combines unique simultaneous parsed cues', () => {
);
});
test('findActiveSubtitleText collapses whitespace variants of one ASS lyric', () => {
assert.equal(
findActiveSubtitleText(
[
{ startTime: 1, endTime: 3, text: '少しだけ好きになる' },
{ startTime: 1, endTime: 3, text: '少しだけ 好きになる' },
{ startTime: 1, endTime: 3, text: '少しだけ 好きになる' },
],
2,
),
'少しだけ好きになる',
);
});
test('parsed secondary lyrics keep explicit ASS vertical order when durations alternate', () => {
const lyric = (options: { start: string; end: string; style: string; y: number; text: string }) =>
`Dialogue: 0,0:00:${options.start},0:00:${options.end},${options.style},,0,0,0,fx,{\\move(100,${options.y},120,${options.y})\\t(0,200,\\fscx110)}${options.text}\\N{\\p1}m 0 0 l 0 5`;
const ass = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
lyric({
start: '01.00',
end: '02.20',
style: 'ed_romaji',
y: 66,
text: 'ima wo kakusarechau mae ni',
}),
lyric({
start: '01.00',
end: '02.00',
style: 'ed_english',
y: 1020,
text: 'Before the present moment gets hidden away.',
}),
lyric({
start: '03.00',
end: '04.00',
style: 'ed_romaji',
y: 66,
text: 'ame mitai ni hikatteru',
}),
lyric({
start: '03.00',
end: '04.20',
style: 'ed_english',
y: 1020,
text: 'Is shining like rain.',
}),
].join('\n');
const cues = parseSubtitleCues(ass, 'polar-opposites-s01e08.ass');
assert.equal(
findActiveSubtitleText(cues, 1.5),
'ima wo kakusarechau mae ni\nBefore the present moment gets hidden away.',
);
assert.equal(findActiveSubtitleText(cues, 3.5), 'ame mitai ni hikatteru\nIs shining like rain.');
});
test('unpositioned secondary lyrics fall back to ASS source order', () => {
const ass = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
'Dialogue: 0,0:00:01.00,0:00:02.20,ED Romaji,,0,0,0,,ima wo kakusarechau mae ni',
'Dialogue: 0,0:00:01.00,0:00:02.00,ED English,,0,0,0,,Before the present moment gets hidden away.',
].join('\n');
assert.equal(
findActiveSubtitleText(parseSubtitleCues(ass, 'ending.ass'), 1.5),
'ima wo kakusarechau mae ni\nBefore the present moment gets hidden away.',
);
});
test('findActiveSubtitleText keeps a canonical ASS cue for its generated animation span', () => {
const poof = {
startTime: 1110.67,
endTime: 1110.71,
text: 'POOF',
source: 'canonical-ass' as const,
animationStartTime: 1110.67,
animationEndTime: 1111.59,
};
assert.equal(findActiveSubtitleText([poof], 1111.58), 'POOF');
assert.equal(findActiveSubtitleText([poof], 1111.59), '');
});
test('ASS fragment karaoke stays separated by style with authored word spacing', () => {
const lineEvents = (
style: string,
fragments: readonly string[],
y: number,
baseTime = 1,
): string[] => {
const events: string[] = [];
for (const layer of [0, 1]) {
fragments.forEach((fragment, index) => {
const x = 100 + index * 40;
const start = (baseTime + index * 0.25).toFixed(2).padStart(5, '0');
const end = (baseTime + 3 + index * 0.2).toFixed(2).padStart(5, '0');
events.push(
`Dialogue: ${layer},0:00:${start},0:00:${end},${style},,0,0,0,,{\\pos(${x},${y})\\t(0,200,\\fscx110)}${fragment}\\N{\\p1}m 0 0 l 0 10`,
);
});
}
return events;
};
const ass = [
'[Events]',
'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text',
...lineEvents('ed_romaji', ['ji', 'gu', 'za', 'gu ', 'na', 'mi'], 70),
...lineEvents('ed_english', ['Pas', 'si', 'ng ', 'thro', 'u', 'gh '], 110),
...lineEvents('op_english', ['I', 'want', 'to', 'go'], 110, 7),
].join('\n');
assert.equal(
findActiveSubtitleText(parseSubtitleCues(ass, 'ending.ass'), 2.5),
'jiguzagu nami\nPassing through',
);
// Some generated scripts discard spaces and retain only positioned chunks. Joining
// without invented separators avoids turning one word into spaced syllables.
assert.equal(findActiveSubtitleText(parseSubtitleCues(ass, 'ending.ass'), 8.5), 'Iwanttogo');
});
test('findActiveSubtitleText keeps a complete reconstructed line over entrance fragments', () => {
const current = {
startTime: 1,
endTime: 4,
text: 'Complete current line',
source: 'reconstructed-ass' as const,
assStyle: 'op_english',
};
const nextEntrance = {
startTime: 3.8,
endTime: 4.2,
text: 'Ne',
source: 'reconstructed-ass' as const,
assStyle: 'op_english',
};
const nextLine = {
startTime: 4,
endTime: 7,
text: 'Next complete line',
source: 'reconstructed-ass' as const,
assStyle: 'op_english',
};
assert.equal(findActiveSubtitleText([current, nextEntrance], 3.9), current.text);
assert.equal(findActiveSubtitleText([current, nextEntrance, nextLine], 4.1), nextLine.text);
});
test('secondary track controller parses the selected ASS file before publishing', async () => {
const broadcasts: string[] = [];
let currentText = '';
+90 -4
View File
@@ -58,16 +58,102 @@ function buildSelectedTrackIdentity(
]);
}
type IndexedSubtitleCue = { cue: SubtitleCue; index: number };
function compareAuthoredSubtitleOrder(left: IndexedSubtitleCue, right: IndexedSubtitleCue): number {
const leftLayout = left.cue.assLayout;
const rightLayout = right.cue.assLayout;
if (leftLayout?.kind === 'positioned' && rightLayout?.kind === 'positioned') {
const verticalOrder = leftLayout.y - rightLayout.y;
if (verticalOrder !== 0) return verticalOrder;
}
if (leftLayout && rightLayout) {
const sourceOrder = leftLayout.sourceOrder - rightLayout.sourceOrder;
if (sourceOrder !== 0) return sourceOrder;
}
return left.index - right.index;
}
export function findActiveSubtitleText(cues: readonly SubtitleCue[], timeSeconds: number): string {
if (!Number.isFinite(timeSeconds)) return '';
const authoredCanonical = cues.filter(
(cue) =>
cue.source === 'canonical-ass' && cue.startTime <= timeSeconds && cue.endTime > timeSeconds,
);
const selectedCanonical = new Set<SubtitleCue>(authoredCanonical);
if (selectedCanonical.size === 0) {
const animatedCanonical = cues.filter(
(cue) =>
cue.source === 'canonical-ass' &&
(cue.animationStartTime ?? cue.startTime) <= timeSeconds &&
(cue.animationEndTime ?? cue.endTime) > timeSeconds,
);
const nearestDistance = animatedCanonical.reduce((nearest, cue) => {
const distance =
timeSeconds < cue.startTime
? cue.startTime - timeSeconds
: Math.max(0, timeSeconds - cue.endTime);
return Math.min(nearest, distance);
}, Infinity);
for (const cue of animatedCanonical) {
const distance =
timeSeconds < cue.startTime
? cue.startTime - timeSeconds
: Math.max(0, timeSeconds - cue.endTime);
if (distance === nearestDistance) {
selectedCanonical.add(cue);
}
}
}
const activeReconstructed = cues.filter(
(cue) =>
cue.source === 'reconstructed-ass' &&
cue.startTime <= timeSeconds &&
cue.endTime > timeSeconds,
);
const reconstructedByStyle = new Map<string, SubtitleCue>();
for (const cue of activeReconstructed) {
const style = cue.assStyle ?? '';
const existing = reconstructedByStyle.get(style);
if (!existing) {
reconstructedByStyle.set(style, cue);
continue;
}
const duration = cue.endTime - cue.startTime;
const existingDuration = existing.endTime - existing.startTime;
if (
duration > existingDuration ||
(duration === existingDuration && cue.text.length > existing.text.length) ||
(duration === existingDuration &&
cue.text.length === existing.text.length &&
cue.startTime > existing.startTime)
) {
reconstructedByStyle.set(style, cue);
}
}
const selectedReconstructed = new Set(reconstructedByStyle.values());
const seen = new Set<string>();
const activeText: string[] = [];
for (const cue of cues) {
if (cue.startTime > timeSeconds || cue.endTime <= timeSeconds) continue;
const activeCues: IndexedSubtitleCue[] = [];
cues.forEach((cue, index) => {
const active =
cue.source === 'canonical-ass'
? selectedCanonical.has(cue)
: cue.source === 'reconstructed-ass'
? selectedReconstructed.has(cue)
: cue.startTime <= timeSeconds && cue.endTime > timeSeconds;
if (active) activeCues.push({ cue, index });
});
activeCues.sort(compareAuthoredSubtitleOrder);
for (const { cue } of activeCues) {
const text = cue.text.trim();
if (!text || seen.has(text)) continue;
seen.add(text);
const compactText = text.replace(/\s+/gu, '');
if (!compactText || seen.has(compactText)) continue;
seen.add(compactText);
activeText.push(text);
}
return activeText.join('\n');