mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-30 00:15:27 -07:00
fix(youtube): keep auto captions on screen for their full span (#219)
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
type: fixed
|
||||
area: youtube
|
||||
|
||||
- YouTube auto-generated captions now follow their intended timing and two-row roll-up layout: long speech is paged instead of covering the video with a wall of text, while explicitly timed sound cues such as `[音楽]` no longer cover later dialogue.
|
||||
@@ -39,6 +39,118 @@ test('convertYoutubeTimedTextToVtt does not swallow text after zero-length overl
|
||||
);
|
||||
});
|
||||
|
||||
test('convertYoutubeTimedTextToVtt extends rolling captions to the next window event', () => {
|
||||
// Real-world shape of YouTube's sentence-level auto captions: window-append
|
||||
// filler rows (a="1", sometimes without d) mark the display timeline, while
|
||||
// long text rows carry a placeholder d="3000" far shorter than the speech.
|
||||
const result = convertYoutubeTimedTextToVtt(
|
||||
[
|
||||
'<timedtext><body>',
|
||||
'<p t="98550" d="3010" w="1" a="1">\n</p>',
|
||||
'<p t="98560" d="3000" w="1"><s ac="0">ありがとうって言えないよね。こんなんじゃ。</s></p>',
|
||||
'<p t="106950" w="1" a="1">\n</p>',
|
||||
'<p t="106960" d="3799" w="1"><s ac="0">私だったら無理だよ。</s></p>',
|
||||
'</body></timedtext>',
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
result,
|
||||
[
|
||||
'WEBVTT',
|
||||
'',
|
||||
'00:01:38.560 --> 00:01:46.950',
|
||||
'ありがとうって言えないよね。こんなんじゃ。',
|
||||
'',
|
||||
'00:01:46.960 --> 00:01:50.759',
|
||||
'私だったら無理だよ。',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
});
|
||||
|
||||
test('convertYoutubeTimedTextToVtt pages oversized two-row rolling captions', () => {
|
||||
const text =
|
||||
'あの西に結構こう山田がスーパーアプローチしてるんだけど西気づかないからちょっとこっちも気づかない感じでこう接してあげようかなて思ってんだけどあの唇巻き込んじゃうしあの思ってることも全部縁に出ちゃって自分であちゃったって言っちゃうタイプなんで結構なんかこうドライなんだけどそこがおもろいよねみたいな';
|
||||
const result = convertYoutubeTimedTextToVtt(
|
||||
[
|
||||
'<timedtext format="3">',
|
||||
'<head>',
|
||||
'<ws id="1" mh="2" ju="0" sd="3"/>',
|
||||
'<wp id="1" ap="6" ah="20" av="100" rc="2" cc="40"/>',
|
||||
'</head>',
|
||||
'<body>',
|
||||
'<w t="0" id="1" wp="1" ws="1"/>',
|
||||
`<p t="60440" d="3000" w="1"><s ac="0">${text}</s></p>`,
|
||||
'<p t="72695" w="1" a="1">\n</p>',
|
||||
'</body>',
|
||||
'</timedtext>',
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
const cues = result
|
||||
.trim()
|
||||
.split(/\n\n/)
|
||||
.filter((block) => block.includes('-->'));
|
||||
const cueText = cues.map((cue) => cue.split('\n').slice(1).join('\n'));
|
||||
|
||||
assert.equal(cues.length, 2);
|
||||
assert.deepEqual(
|
||||
cues.map((cue) => cue.split('\n')[0]),
|
||||
['00:01:00.440 --> 00:01:07.064', '00:01:07.064 --> 00:01:12.695'],
|
||||
);
|
||||
assert.ok(cueText.every((page) => [...page].length <= 80));
|
||||
assert.equal(cueText.join(''), text);
|
||||
});
|
||||
|
||||
test('convertYoutubeTimedTextToVtt leaves pop-on captions intact', () => {
|
||||
const result = convertYoutubeTimedTextToVtt(
|
||||
[
|
||||
'<timedtext format="3">',
|
||||
'<head>',
|
||||
'<ws id="1" mh="0"/>',
|
||||
'<wp id="1" rc="2" cc="4"/>',
|
||||
'</head>',
|
||||
'<body>',
|
||||
'<w t="0" id="1" wp="1" ws="1"/>',
|
||||
'<p t="1000" d="3000" w="1">abcdefghijklmnopqrst</p>',
|
||||
'</body>',
|
||||
'</timedtext>',
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
result,
|
||||
['WEBVTT', '', '00:00:01.000 --> 00:00:04.000', 'abcdefghijklmnopqrst', ''].join('\n'),
|
||||
);
|
||||
});
|
||||
|
||||
test('convertYoutubeTimedTextToVtt keeps explicit 3000ms sound-cue durations in rolling documents', () => {
|
||||
const result = convertYoutubeTimedTextToVtt(
|
||||
[
|
||||
'<timedtext><body>',
|
||||
'<p t="20305" d="3000" w="1">[音楽]</p>',
|
||||
'<p t="26269" w="1" a="1">\n</p>',
|
||||
'<p t="26279" d="3000" w="1"><s ac="0">じゃあ、君からお願いします。</s></p>',
|
||||
'</body></timedtext>',
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
result,
|
||||
[
|
||||
'WEBVTT',
|
||||
'',
|
||||
'00:00:20.305 --> 00:00:23.305',
|
||||
'[音楽]',
|
||||
'',
|
||||
'00:00:26.279 --> 00:00:29.279',
|
||||
'じゃあ、君からお願いします。',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
});
|
||||
|
||||
test('normalizeYoutubeAutoVtt strips cumulative rolling-caption prefixes', () => {
|
||||
const result = normalizeYoutubeAutoVtt(
|
||||
[
|
||||
|
||||
@@ -2,9 +2,31 @@ interface YoutubeTimedTextRow {
|
||||
startMs: number;
|
||||
durationMs: number;
|
||||
text: string;
|
||||
isGenerated: boolean;
|
||||
rollingWindow: YoutubeRollingWindow | null;
|
||||
}
|
||||
|
||||
interface YoutubeRollingWindow {
|
||||
rowCount: number;
|
||||
columnCount: number;
|
||||
}
|
||||
|
||||
interface YoutubeTimedTextWindowDefinitions {
|
||||
rollingStyleIds: Set<string>;
|
||||
positions: Map<string, YoutubeRollingWindow>;
|
||||
windows: Map<string, YoutubeRollingWindow>;
|
||||
}
|
||||
|
||||
interface YoutubeTimedTextDocument {
|
||||
rows: YoutubeTimedTextRow[];
|
||||
// Start times of every <p> event, including empty window-append fillers.
|
||||
// Rolling speech rows with a 3000ms placeholder display until the next event.
|
||||
eventStartsMs: number[];
|
||||
hasRollingWindowEvents: boolean;
|
||||
}
|
||||
|
||||
const YOUTUBE_TIMEDTEXT_EXTENSIONS = new Set(['srv1', 'srv2', 'srv3', 'ytsrv3']);
|
||||
const YOUTUBE_ROLLING_PLACEHOLDER_DURATION_MS = 3_000;
|
||||
|
||||
function decodeNumericEntity(match: string, codePoint: number): string {
|
||||
if (
|
||||
@@ -39,27 +61,129 @@ function parseAttributeMap(raw: string): Map<string, string> {
|
||||
return attrs;
|
||||
}
|
||||
|
||||
function extractYoutubeTimedTextRows(xml: string): YoutubeTimedTextRow[] {
|
||||
function parsePositiveInteger(value: string | undefined): number | null {
|
||||
if (value === undefined) {
|
||||
return null;
|
||||
}
|
||||
const parsed = Number(value);
|
||||
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : null;
|
||||
}
|
||||
|
||||
function extractYoutubeTimedTextWindowDefinitions(xml: string): YoutubeTimedTextWindowDefinitions {
|
||||
const rollingStyleIds = new Set<string>();
|
||||
for (const match of xml.matchAll(/<ws\b([^>]*)\/?\s*>/g)) {
|
||||
const attrs = parseAttributeMap(match[1] ?? '');
|
||||
const id = attrs.get('id');
|
||||
if (id !== undefined && attrs.get('mh') === '2') {
|
||||
rollingStyleIds.add(id);
|
||||
}
|
||||
}
|
||||
|
||||
const positions = new Map<string, YoutubeRollingWindow>();
|
||||
for (const match of xml.matchAll(/<wp\b([^>]*)\/?\s*>/g)) {
|
||||
const attrs = parseAttributeMap(match[1] ?? '');
|
||||
const id = attrs.get('id');
|
||||
const rowCount = parsePositiveInteger(attrs.get('rc'));
|
||||
const columnCount = parsePositiveInteger(attrs.get('cc'));
|
||||
if (id !== undefined && rowCount !== null && columnCount !== null) {
|
||||
positions.set(id, { rowCount, columnCount });
|
||||
}
|
||||
}
|
||||
|
||||
const windows = new Map<string, YoutubeRollingWindow>();
|
||||
for (const match of xml.matchAll(/<w\b([^>]*)\/?\s*>/g)) {
|
||||
const attrs = parseAttributeMap(match[1] ?? '');
|
||||
const id = attrs.get('id');
|
||||
const styleId = attrs.get('ws');
|
||||
const positionId = attrs.get('wp');
|
||||
const position = positionId === undefined ? undefined : positions.get(positionId);
|
||||
if (
|
||||
id !== undefined &&
|
||||
styleId !== undefined &&
|
||||
rollingStyleIds.has(styleId) &&
|
||||
position !== undefined
|
||||
) {
|
||||
windows.set(id, position);
|
||||
}
|
||||
}
|
||||
|
||||
return { rollingStyleIds, positions, windows };
|
||||
}
|
||||
|
||||
function resolveRollingWindow(
|
||||
attrs: Map<string, string>,
|
||||
definitions: YoutubeTimedTextWindowDefinitions,
|
||||
): YoutubeRollingWindow | null {
|
||||
const windowId = attrs.get('w');
|
||||
if (windowId !== undefined) {
|
||||
return definitions.windows.get(windowId) ?? null;
|
||||
}
|
||||
|
||||
const styleId = attrs.get('ws');
|
||||
const positionId = attrs.get('wp');
|
||||
if (
|
||||
styleId === undefined ||
|
||||
positionId === undefined ||
|
||||
!definitions.rollingStyleIds.has(styleId)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return definitions.positions.get(positionId) ?? null;
|
||||
}
|
||||
|
||||
function extractYoutubeTimedTextDocument(xml: string): YoutubeTimedTextDocument {
|
||||
const rows: YoutubeTimedTextRow[] = [];
|
||||
const eventStartsMs: number[] = [];
|
||||
let hasRollingWindowEvents = false;
|
||||
const windowDefinitions = extractYoutubeTimedTextWindowDefinitions(xml);
|
||||
|
||||
for (const match of xml.matchAll(/<p\b([^>]*)>([\s\S]*?)<\/p>/g)) {
|
||||
const attrs = parseAttributeMap(match[1] ?? '');
|
||||
const startMs = Number(attrs.get('t'));
|
||||
if (!Number.isFinite(startMs)) {
|
||||
continue;
|
||||
}
|
||||
eventStartsMs.push(startMs);
|
||||
if (attrs.get('a') === '1') {
|
||||
hasRollingWindowEvents = true;
|
||||
}
|
||||
|
||||
const durationMs = Number(attrs.get('d'));
|
||||
if (!Number.isFinite(startMs) || !Number.isFinite(durationMs)) {
|
||||
if (!Number.isFinite(durationMs)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const inner = (match[2] ?? '').replace(/<br\s*\/?>/gi, '\n').replace(/<[^>]+>/g, '');
|
||||
const rawInner = match[2] ?? '';
|
||||
const inner = rawInner.replace(/<br\s*\/?>/gi, '\n').replace(/<[^>]+>/g, '');
|
||||
const text = decodeHtmlEntities(inner).trim();
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
|
||||
rows.push({ startMs, durationMs, text });
|
||||
rows.push({
|
||||
startMs,
|
||||
durationMs,
|
||||
text,
|
||||
isGenerated: /<s\b/.test(rawInner),
|
||||
rollingWindow: resolveRollingWindow(attrs, windowDefinitions),
|
||||
});
|
||||
}
|
||||
|
||||
return rows;
|
||||
eventStartsMs.sort((a, b) => a - b);
|
||||
return { rows, eventStartsMs, hasRollingWindowEvents };
|
||||
}
|
||||
|
||||
function findNextEventStartMs(eventStartsMs: number[], afterMs: number): number | undefined {
|
||||
for (const startMs of eventStartsMs) {
|
||||
if (startMs > afterMs) {
|
||||
return startMs;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isGeneratedRollingCue(row: YoutubeTimedTextRow, hasRollingWindowEvents: boolean): boolean {
|
||||
return row.isGenerated && (row.rollingWindow !== null || hasRollingWindowEvents);
|
||||
}
|
||||
|
||||
function formatVttTimestamp(ms: number): string {
|
||||
@@ -71,6 +195,79 @@ function formatVttTimestamp(ms: number): string {
|
||||
return `${String(hours).padStart(2, '0')}:${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')}.${String(millis).padStart(3, '0')}`;
|
||||
}
|
||||
|
||||
const ROLLING_PAGE_BREAK_PATTERN = /[\s、。！？!?]/u;
|
||||
|
||||
// VTT cannot carry SRV3's row and column limits. Page only roll-up windows so
|
||||
// the overlay keeps their bounded presentation without changing authored cues.
|
||||
function splitRollingCaptionIntoPages(text: string, rollingWindow: YoutubeRollingWindow): string[] {
|
||||
const pageCapacity = rollingWindow.rowCount * rollingWindow.columnCount;
|
||||
const characters = [...text];
|
||||
if (
|
||||
!Number.isSafeInteger(pageCapacity) ||
|
||||
pageCapacity <= 0 ||
|
||||
characters.length <= pageCapacity
|
||||
) {
|
||||
return [text];
|
||||
}
|
||||
|
||||
const pages: string[] = [];
|
||||
let pageStart = 0;
|
||||
while (pageStart < characters.length) {
|
||||
let pageEnd = Math.min(pageStart + pageCapacity, characters.length);
|
||||
if (pageEnd < characters.length) {
|
||||
const earliestNaturalBreak = pageStart + Math.ceil(pageCapacity * 0.6);
|
||||
for (let index = pageEnd - 1; index >= earliestNaturalBreak; index -= 1) {
|
||||
if (ROLLING_PAGE_BREAK_PATTERN.test(characters[index]!)) {
|
||||
pageEnd = index + 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
pages.push(characters.slice(pageStart, pageEnd).join(''));
|
||||
pageStart = pageEnd;
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
|
||||
interface TimedCaptionPage {
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
text: string;
|
||||
}
|
||||
|
||||
function timeCaptionPages(input: {
|
||||
text: string;
|
||||
pages: string[];
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
}): TimedCaptionPage[] {
|
||||
const durationMs = input.endMs - input.startMs;
|
||||
if (input.pages.length === 1 || durationMs < input.pages.length) {
|
||||
return [{ startMs: input.startMs, endMs: input.endMs, text: input.text }];
|
||||
}
|
||||
|
||||
const totalCharacters = [...input.text].length;
|
||||
const timedPages: TimedCaptionPage[] = [];
|
||||
let consumedCharacters = 0;
|
||||
let pageStartMs = input.startMs;
|
||||
// Automatic captions often omit span offsets, so distribute the known cue
|
||||
// duration by page length while guaranteeing every page at least one ms.
|
||||
for (let index = 0; index < input.pages.length; index += 1) {
|
||||
const page = input.pages[index]!;
|
||||
consumedCharacters += [...page].length;
|
||||
const remainingPages = input.pages.length - index - 1;
|
||||
const proportionalEndMs =
|
||||
input.startMs + Math.round((durationMs * consumedCharacters) / totalCharacters);
|
||||
const pageEndMs =
|
||||
remainingPages === 0
|
||||
? input.endMs
|
||||
: Math.min(Math.max(proportionalEndMs, pageStartMs + 1), input.endMs - remainingPages);
|
||||
timedPages.push({ startMs: pageStartMs, endMs: pageEndMs, text: page });
|
||||
pageStartMs = pageEndMs;
|
||||
}
|
||||
return timedPages;
|
||||
}
|
||||
|
||||
export function isYoutubeTimedTextExtension(value: string | undefined): boolean {
|
||||
if (!value) {
|
||||
return false;
|
||||
@@ -79,7 +276,7 @@ export function isYoutubeTimedTextExtension(value: string | undefined): boolean
|
||||
}
|
||||
|
||||
export function convertYoutubeTimedTextToVtt(xml: string): string {
|
||||
const rows = extractYoutubeTimedTextRows(xml);
|
||||
const { rows, eventStartsMs, hasRollingWindowEvents } = extractYoutubeTimedTextDocument(xml);
|
||||
if (rows.length === 0) {
|
||||
return 'WEBVTT\n';
|
||||
}
|
||||
@@ -90,10 +287,19 @@ export function convertYoutubeTimedTextToVtt(xml: string): string {
|
||||
const row = rows[index]!;
|
||||
const nextRow = rows[index + 1];
|
||||
const unclampedEnd = row.startMs + row.durationMs;
|
||||
// YouTube uses exactly 3000ms as a placeholder for generated rolling speech.
|
||||
// Plain-text cues can explicitly use the same duration and must keep it.
|
||||
const nextEventStart =
|
||||
isGeneratedRollingCue(row, hasRollingWindowEvents) &&
|
||||
row.durationMs === YOUTUBE_ROLLING_PLACEHOLDER_DURATION_MS
|
||||
? findNextEventStartMs(eventStartsMs, row.startMs)
|
||||
: undefined;
|
||||
const clampedEnd =
|
||||
nextRow && unclampedEnd > nextRow.startMs
|
||||
? Math.max(row.startMs, nextRow.startMs - 1)
|
||||
: unclampedEnd;
|
||||
nextEventStart !== undefined
|
||||
? nextEventStart
|
||||
: nextRow && unclampedEnd > nextRow.startMs
|
||||
? Math.max(row.startMs, nextRow.startMs - 1)
|
||||
: unclampedEnd;
|
||||
if (clampedEnd <= row.startMs) {
|
||||
continue;
|
||||
}
|
||||
@@ -106,9 +312,19 @@ export function convertYoutubeTimedTextToVtt(xml: string): string {
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
blocks.push(
|
||||
`${formatVttTimestamp(row.startMs)} --> ${formatVttTimestamp(clampedEnd)}\n${text}`,
|
||||
);
|
||||
const pages = row.rollingWindow
|
||||
? splitRollingCaptionIntoPages(text, row.rollingWindow)
|
||||
: [text];
|
||||
for (const page of timeCaptionPages({
|
||||
text,
|
||||
pages,
|
||||
startMs: row.startMs,
|
||||
endMs: clampedEnd,
|
||||
})) {
|
||||
blocks.push(
|
||||
`${formatVttTimestamp(page.startMs)} --> ${formatVttTimestamp(page.endMs)}\n${page.text}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return `WEBVTT\n\n${blocks.join('\n\n')}\n`;
|
||||
|
||||
Reference in New Issue
Block a user