From c85db5e07e7aa43b5f8be811fe1a697c81979d94 Mon Sep 17 00:00:00 2001 From: sudacode Date: Wed, 2 Sep 2026 18:22:03 -0700 Subject: [PATCH] fix(mining): ignore stale captions and improve waveform contrast - Use active parsed cue text and timings for overlay mining - Normalize waveform levels against the clip's noise floor --- changes/media-timing-review.md | 2 +- changes/mining-stale-caption-row.md | 4 ++ docs-site/anki-integration.md | 2 +- .../services/media-timing-waveform.test.ts | 34 ++++++++++++-- src/core/services/media-timing-waveform.ts | 44 ++++++++++++++----- src/main.ts | 30 ++++++++----- .../runtime/primary-subtitle-text.test.ts | 35 +++++++++++++++ src/main/runtime/primary-subtitle-text.ts | 30 +++++++++---- 8 files changed, 145 insertions(+), 36 deletions(-) create mode 100644 changes/mining-stale-caption-row.md diff --git a/changes/media-timing-review.md b/changes/media-timing-review.md index 1082627f..27d49a0b 100644 --- a/changes/media-timing-review.md +++ b/changes/media-timing-review.md @@ -1,5 +1,5 @@ type: added area: mining -- Added optional pre-generation timing review for word, sentence, and audio cards with a compact speech-weighted waveform, clearly labeled mined-line boundaries, drag and keyboard adjustments, audio preview with a sweeping playhead, exact screenshot and AVIF timing, cancellation choices that include keeping a card without media, and a session-only runtime toggle. +- Added optional pre-generation timing review for word, sentence, and audio cards with a compact speech-weighted waveform that flattens steady background noise so dialogue edges are easy to see, clearly labeled mined-line boundaries, drag and keyboard adjustments, audio preview with a sweeping playhead, exact screenshot and AVIF timing, cancellation choices that include keeping a card without media, and a session-only runtime toggle. - The timing review can pull any number of previous and next subtitle lines onto the card: `P`/`N` (or the Prev/Next steppers) add lines one at a time, Shift removes them, the sentence preview highlights exactly what the card will contain, and the clip range and line boundary markers on the waveform follow the added lines automatically. diff --git a/changes/mining-stale-caption-row.md b/changes/mining-stale-caption-row.md new file mode 100644 index 00000000..caa9bfa6 --- /dev/null +++ b/changes/mining-stale-caption-row.md @@ -0,0 +1,4 @@ +type: fixed +area: mining + +- Mining from the overlay no longer copies a lingering row of the previous broadcast caption onto the card sentence. The mined line and its clip timing now come from the parsed cue the overlay is showing, falling back to mpv's live text only when the cue list cannot explain it. diff --git a/docs-site/anki-integration.md b/docs-site/anki-integration.md index f573a81a..3641df71 100644 --- a/docs-site/anki-integration.md +++ b/docs-site/anki-integration.md @@ -183,7 +183,7 @@ Output format: MP3 at 44100 Hz. If the video has multiple audio streams, SubMine The audio is uploaded to Anki's media folder and inserted as `[sound:audio_.mp3]`. -Set `media.reviewTiming` to `true` to pause playback and review each word, sentence, or audio card before its media is generated. The review opens with the subtitle range plus configured audio padding. Drag either edge of the clip to trim it, drag the middle to slide it without changing its length, or press anywhere else on the waveform to snap the nearer edge there. A focused edge also moves with the arrow keys, by 100 ms alone or 500 ms with Shift, and the 100 ms buttons do the same. Space previews the selection with a playhead that sweeps the clip, Enter confirms, and Escape cancels. The Earlier and Later buttons reveal another two seconds of available timeline without moving the selected clip. A speech-weighted waveform shows the mined subtitle as a tinted band with labeled line-start and line-end rails, making adjacent dialogue easier to distinguish. SubMiner uses a center channel when one carries dialogue, then falls back to a speech-band mono mix. Waveform analysis failure leaves the timing controls available. The confirmed range is exact: SubMiner does not apply audio padding a second time. Static screenshots use its midpoint, and animated AVIF clips use the full confirmed range. +Set `media.reviewTiming` to `true` to pause playback and review each word, sentence, or audio card before its media is generated. The review opens with the subtitle range plus configured audio padding. Drag either edge of the clip to trim it, drag the middle to slide it without changing its length, or press anywhere else on the waveform to snap the nearer edge there. A focused edge also moves with the arrow keys, by 100 ms alone or 500 ms with Shift, and the 100 ms buttons do the same. Space previews the selection with a playhead that sweeps the clip, Enter confirms, and Escape cancels. The Earlier and Later buttons reveal another two seconds of available timeline without moving the selected clip. A speech-weighted waveform shows the mined subtitle as a tinted band with labeled line-start and line-end rails, making adjacent dialogue easier to distinguish. SubMiner uses a center channel when one carries dialogue, then falls back to a mono mix, keeps only the 250 to 3500 Hz speech band, and draws each slice's loudness relative to the clip's own noise floor, so steady background music or ambience reads as a flat line while dialogue stands out. Waveform analysis failure leaves the timing controls available. The confirmed range is exact: SubMiner does not apply audio padding a second time. Static screenshots use its midpoint, and animated AVIF clips use the full confirmed range. The review can also pull adjacent subtitle lines onto the card. Press `P` or `N` (or use the Prev and Next steppers above the sentence preview) to add the previous or next line, as many times as lines are available; Shift+`P` and Shift+`N` remove them again. The sentence preview lists every included line with the mined line highlighted, so the card's sentence field is always visible before you confirm, and the clip start or end, along with the line-start and line-end rails on the waveform, follows the outermost added line, keeping the review's audio padding. Confirming writes the combined lines to the sentence field; the Reset button drops the added lines along with any timing changes. Adjacent lines come from the parsed subtitle track when one is loaded; otherwise only lines that already played are offered, and a clip capped by `media.maxMediaDuration` keeps the full combined sentence even when the audio cannot cover every added line. diff --git a/src/core/services/media-timing-waveform.test.ts b/src/core/services/media-timing-waveform.test.ts index bdeff0e0..3b555eb9 100644 --- a/src/core/services/media-timing-waveform.test.ts +++ b/src/core/services/media-timing-waveform.test.ts @@ -58,16 +58,44 @@ test('speech waveform seeks cached windows by source timestamps', () => { assert.equal(args.includes('-map'), false); }); -test('waveform peaks are normalized without flattening quieter sections', () => { +test('waveform levels rise with loudness and top out at the reference level', () => { const peaks = computeWaveformPeaks(pcm([0, 1_000, -2_000, 4_000, -8_000, 16_000]), 3); assert.equal(peaks.length, 3); - assert.ok((peaks[0] ?? 0) > 0); - assert.ok((peaks[0] ?? 0) < (peaks[1] ?? 0)); + assert.equal(peaks[0], 0); + assert.ok((peaks[1] ?? 0) > 0); assert.ok((peaks[1] ?? 0) < (peaks[2] ?? 0)); assert.equal(peaks[2], 1); }); +test('waveform flattens steady background noise and keeps speech bursts tall', () => { + // 20 slices of steady noise at a fixed level with an 18 dB louder "speech" burst in the middle. + const noise = 1_000; + const samples: number[] = []; + for (let slice = 0; slice < 20; slice += 1) { + const level = slice >= 8 && slice < 12 ? noise * 8 : noise; + for (let sample = 0; sample < 50; sample += 1) { + samples.push(sample % 2 === 0 ? level : -level); + } + } + + const peaks = computeWaveformPeaks(pcm(samples), 20); + + for (const [index, peak] of peaks.entries()) { + if (index >= 8 && index < 12) assert.equal(peak, 1); + else assert.equal(peak, 0); + } +}); + +test('waveform stays flat when the whole range is a single steady level', () => { + const peaks = computeWaveformPeaks( + pcm(Array.from({ length: 400 }, (_, i) => (i % 2 ? 900 : -900))), + 40, + ); + + assert.ok(peaks.every((peak) => peak === 0)); +}); + test('speech waveform uses a mono downmix when the source has no center activity', async () => { const calls: string[][] = []; const peaks = await generateSpeechWaveform( diff --git a/src/core/services/media-timing-waveform.ts b/src/core/services/media-timing-waveform.ts index 157a76fd..59aa30ad 100644 --- a/src/core/services/media-timing-waveform.ts +++ b/src/core/services/media-timing-waveform.ts @@ -5,7 +5,13 @@ const WAVEFORM_SAMPLE_RATE = 8_000; const WAVEFORM_POINT_COUNT = 480; const WAVEFORM_TIMEOUT_MS = 15_000; const MAX_WAVEFORM_BYTES = 16 * 1024 * 1024; -const SPEECH_FILTER = 'highpass=f=120,lowpass=f=4000'; +// Keep the band where speech intelligibility lives; bass, drums, and hum sit below it. +const SPEECH_FILTER = 'highpass=f=250,lowpass=f=3500'; +const NOISE_FLOOR_PERCENTILE = 0.2; +const REFERENCE_PERCENTILE = 0.95; +const NOISE_GATE_DB = 3; +const MIN_DISPLAY_RANGE_DB = 12; +const SILENCE_DB = -100; const CENTER_CHANNEL_FILTER = `pan=mono|c0=FC,${SPEECH_FILTER}`; const DOWNMIX_FILTER = `aformat=channel_layouts=mono,${SPEECH_FILTER}`; @@ -111,11 +117,22 @@ function runFfmpeg(args: string[]): Promise { }); } +function percentile(sortedValues: number[], fraction: number): number { + const index = Math.min(sortedValues.length - 1, Math.floor(sortedValues.length * fraction)); + return sortedValues[index] ?? SILENCE_DB; +} + +/** + * Turns mono PCM into 0..1 display heights. Each point is the RMS level of its slice in + * dB, measured against the clip's own noise floor (a low percentile of the slices), so + * constant background noise draws flat and sustained speech stands out. Peak sampling + * would instead follow music transients and lift the floor to nearly speech height. + */ export function computeWaveformPeaks(pcm: Buffer, pointCount = WAVEFORM_POINT_COUNT): number[] { const sampleCount = Math.floor(pcm.byteLength / 2); if (sampleCount === 0 || pointCount <= 0) return []; const resolvedPointCount = Math.min(pointCount, sampleCount); - const peaks = Array.from({ length: resolvedPointCount }, () => 0); + const levelsDb = Array.from({ length: resolvedPointCount }, () => SILENCE_DB); for (let point = 0; point < resolvedPointCount; point += 1) { const sampleStart = Math.floor((point * sampleCount) / resolvedPointCount); @@ -123,18 +140,25 @@ export function computeWaveformPeaks(pcm: Buffer, pointCount = WAVEFORM_POINT_CO sampleStart + 1, Math.floor(((point + 1) * sampleCount) / resolvedPointCount), ); - let peak = 0; + let energy = 0; for (let sample = sampleStart; sample < sampleEnd; sample += 1) { - peak = Math.max(peak, Math.abs(pcm.readInt16LE(sample * 2)) / 32_768); + const value = pcm.readInt16LE(sample * 2) / 32_768; + energy += value * value; } - peaks[point] = peak; + const rms = Math.sqrt(energy / (sampleEnd - sampleStart)); + levelsDb[point] = rms > 0 ? Math.max(SILENCE_DB, 20 * Math.log10(rms)) : SILENCE_DB; } - const sortedPeaks = [...peaks].sort((left, right) => left - right); - const referenceIndex = Math.min(sortedPeaks.length - 1, Math.floor(sortedPeaks.length * 0.95)); - const referencePeak = Math.max(sortedPeaks[referenceIndex] ?? 0, 0.01); - return peaks.map( - (peak) => Math.round(Math.sqrt(Math.min(1, peak / referencePeak)) * 1_000) / 1_000, + const sortedLevels = [...levelsDb].sort((left, right) => left - right); + const floorDb = percentile(sortedLevels, NOISE_FLOOR_PERCENTILE) + NOISE_GATE_DB; + const referenceDb = Math.max( + percentile(sortedLevels, REFERENCE_PERCENTILE), + floorDb + MIN_DISPLAY_RANGE_DB, + ); + return levelsDb.map( + (levelDb) => + Math.round(Math.min(1, Math.max(0, (levelDb - floorDb) / (referenceDb - floorDb))) * 1_000) / + 1_000, ); } diff --git a/src/main.ts b/src/main.ts index e51a0924..ba512865 100644 --- a/src/main.ts +++ b/src/main.ts @@ -235,7 +235,10 @@ import { createCycleSecondarySubModeRuntimeHandler, } from './main/runtime/domains/mpv'; import { buildSubtitleTrackDiagnostics } from './main/runtime/mpv-track-diagnostics'; -import { resolveCanonicalPrimarySubtitle } from './main/runtime/primary-subtitle-text'; +import { + resolveCanonicalPrimarySubtitle, + resolvePrimarySubtitle, +} from './main/runtime/primary-subtitle-text'; import { createBuildCopyCurrentSubtitleMainDepsHandler, createBuildHandleMineSentenceDigitMainDepsHandler, @@ -1829,28 +1832,31 @@ function withCurrentSubtitleTiming(payload: SubtitleData): SubtitleData { } function captureCurrentPrimarySubtitleMiningContext(): SubtitleMiningContext | null { - const canonical = resolveCanonicalPrimarySubtitle({ + // Mine what the overlay shows, not raw mpv `sub-text`: the raw text lists every active + // event, so a finished caption row lingering beside a fresh line would end up on the + // card. The parsed view also carries the cue's own timings for the clip range. + const resolved = resolvePrimarySubtitle({ liveText: appState.mpvClient?.currentSubText ?? '', currentTimeSec: Number(appState.mpvClient?.currentTimePos), cues: appState.activeParsedSubtitleCues, }); - // Same validity bar as the live capture path: an unusable canonical span must fall + // Same validity bar as the live capture path: an unusable resolved span must fall // back rather than hand mining an empty line or an inverted range. - const canonicalText = canonical?.text.trim(); + const resolvedText = resolved?.text.replace(/\n{2,}/g, '\n').trim(); if ( - !canonical || - !canonicalText || - !Number.isFinite(canonical.startTime) || - !Number.isFinite(canonical.endTime) || - canonical.endTime <= canonical.startTime + !resolved || + !resolvedText || + !Number.isFinite(resolved.startTime) || + !Number.isFinite(resolved.endTime) || + resolved.endTime <= resolved.startTime ) { return captureLiveSubtitleMiningContext(appState.mpvClient); } return { source: 'overlay', - text: canonicalText, - startTime: canonical.startTime, - endTime: canonical.endTime, + text: resolvedText, + startTime: resolved.startTime, + endTime: resolved.endTime, capturedAtMs: Date.now(), }; } diff --git a/src/main/runtime/primary-subtitle-text.test.ts b/src/main/runtime/primary-subtitle-text.test.ts index e42398c3..c9ff6db0 100644 --- a/src/main/runtime/primary-subtitle-text.test.ts +++ b/src/main/runtime/primary-subtitle-text.test.ts @@ -3,6 +3,7 @@ import test from 'node:test'; import { parseSubtitleCues } from '../../core/services/subtitle-cue-parser'; import { resolveCanonicalPrimarySubtitle, + resolvePrimarySubtitle, resolvePrimarySubtitleText, stripCanonicalFragmentLines, } from './primary-subtitle-text'; @@ -702,3 +703,37 @@ test('resolvePrimarySubtitleText publishes a wrapped caption sentence as one cue '（東）≪好きだと\n自覚してしまったものの➡', ); }); + +test('resolvePrimarySubtitle drops a finished caption row lingering beside a fresh line', () => { + // Broadcast captions give each row its own event, and a row of the previous line can + // outlive its siblings by a frame. mpv's sub-text still lists it, so the mined line + // must come from the parsed cue that is actually running, with that cue's timings. + const ass = [ + '[Script Info]', + 'PlayResY: 540', + '', + '[Events]', + 'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text', + 'Dialogue: 0,0:14:30.00,0:14:33.00,Default,,0,0,0,,{\\pos(232,437)\\fscx50}（{\\fscx100}東{\\fscx50}）{\\fscx100}ずっと　言えなかっ', + 'Dialogue: 0,0:14:30.00,0:14:33.02,Default,,0,0,0,,{\\pos(232,497)}たが', + 'Dialogue: 0,0:14:33.00,0:14:36.00,Default,,0,0,0,,{\\pos(232,437)}⸨もし　お互い', + 'Dialogue: 0,0:14:33.00,0:14:36.00,Default,,0,0,0,,{\\pos(232,497)}本命　受かったら　大学　近いし➡', + ].join('\n'); + const cues = parseSubtitleCues(ass, 'polar-opposites-s02e09.ass'); + + const resolved = resolvePrimarySubtitle({ + liveText: 'たが\n⸨もし　お互い\n本命　受かったら　大学　近いし➡', + currentTimeSec: 14 * 60 + 33.05, + cues, + }); + + assert.deepEqual( + { ...resolved, cues: resolved?.cues.map((cue) => cue.text) }, + { + text: '⸨もし　お互い\n本命　受かったら　大学　近いし➡', + startTime: 14 * 60 + 33, + endTime: 14 * 60 + 36, + cues: ['⸨もし　お互い\n本命　受かったら　大学　近いし➡'], + }, + ); +}); diff --git a/src/main/runtime/primary-subtitle-text.ts b/src/main/runtime/primary-subtitle-text.ts index 4d0b7a30..afce7a18 100644 --- a/src/main/runtime/primary-subtitle-text.ts +++ b/src/main/runtime/primary-subtitle-text.ts @@ -319,6 +319,26 @@ export function resolveRecordedPrimarySubtitleText(options: { ); } +/** + * The parsed view of the live text with its cue timings: a canonical animation when one + * explains the live lines, otherwise the active parsed cues. Null when the parsed cues + * cannot account for every live line, in which case callers keep the raw mpv text. + */ +export function resolvePrimarySubtitle(options: { + liveText: string; + currentTimeSec: number; + cues: readonly SubtitleCue[] | null | undefined; +}): ResolvedPrimarySubtitle | null { + const liveText = decodedLiveText(options.liveText, options.cues); + if (!liveText.trim()) { + return null; + } + return ( + resolveCanonicalPrimarySubtitle({ ...options, liveText }) ?? + resolveActiveParsedPrimarySubtitle({ ...options, liveText }) + ); +} + export function resolvePrimarySubtitleText(options: { liveText: string; currentTimeSec: number; @@ -328,13 +348,5 @@ export function resolvePrimarySubtitleText(options: { if (!liveText.trim()) { return liveText; } - return ( - resolveCanonicalPrimarySubtitle({ - liveText, - currentTimeSec: options.currentTimeSec, - cues: options.cues, - })?.text ?? - resolveActiveParsedPrimarySubtitle({ ...options, liveText })?.text ?? - removeLiveGlyphFragmentLines(liveText) - ); + return resolvePrimarySubtitle(options)?.text ?? removeLiveGlyphFragmentLines(liveText); }