fix(anki): trim timing review clips to dialogue end

- Auto-adjust untouched clip ends after waveform analysis
- Document and test trailing-silence trimming behavior
This commit is contained in:
2026-09-02 23:43:34 -07:00
parent dcd05ee4ea
commit 8e47e0d761
4 changed files with 113 additions and 2 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
type: added
area: mining
- Added optional pre-generation timing review for word, sentence, and audio cards with a compact speech-weighted waveform that flattens steady background noise so dialogue edges are easy to see, clearly labeled mined-line boundaries, drag and keyboard adjustments, audio preview with a sweeping playhead that plays the clip to its true end even on high-latency outputs such as Bluetooth headphones, exact screenshot and AVIF timing, cancellation choices that include keeping a card without media, and a session-only runtime toggle.
- Added optional pre-generation timing review for word, sentence, and audio cards with a compact speech-weighted waveform that flattens steady background noise so dialogue edges are easy to see, clearly labeled mined-line boundaries, a clip end that moves back to where the line's dialogue actually ends once the waveform loads (subtitles usually linger past speech), drag and keyboard adjustments, audio preview with a sweeping playhead that plays the clip to its true end even on high-latency outputs such as Bluetooth headphones, exact screenshot and AVIF timing, cancellation choices that include keeping a card without media, and a session-only runtime toggle.
- The timing review can pull any number of previous and next subtitle lines onto the card: `P`/`N` (or the Prev/Next steppers) add lines one at a time, Shift removes them, the sentence preview highlights exactly what the card will contain, and the clip range and line boundary markers on the waveform follow the added lines automatically.
+1 -1
View File
@@ -183,7 +183,7 @@ Output format: MP3 at 44100 Hz. If the video has multiple audio streams, SubMine
The audio is uploaded to Anki's media folder and inserted as `[sound:audio_<timestamp>.mp3]`.
Set `media.reviewTiming` to `true` to pause playback and review each word, sentence, or audio card before its media is generated. The review opens with the subtitle range plus configured audio padding. Drag either edge of the clip to trim it, drag the middle to slide it without changing its length, or press anywhere else on the waveform to snap the nearer edge there. A focused edge also moves with the arrow keys, by 100 ms alone or 500 ms with Shift, and the 100 ms buttons do the same. Space previews the selection with a playhead that sweeps the clip; the preview ends when the hidden player has actually played the last sample, so output latency such as Bluetooth headphones does not cut the clip short. Enter confirms, and Escape cancels. The Earlier and Later buttons reveal another two seconds of available timeline without moving the selected clip. A speech-weighted waveform shows the mined subtitle as a tinted band with labeled line-start and line-end rails, making adjacent dialogue easier to distinguish. SubMiner uses a center channel when one carries dialogue, then falls back to a mono mix, keeps only the 250 to 3500 Hz speech band, and draws each slice's loudness relative to the clip's own noise floor, so steady background music or ambience reads as a flat line while dialogue stands out. Waveform analysis failure leaves the timing controls available. The confirmed range is exact: SubMiner does not apply audio padding a second time. Static screenshots use its midpoint, and animated AVIF clips use the full confirmed range.
Set `media.reviewTiming` to `true` to pause playback and review each word, sentence, or audio card before its media is generated. The review opens with the subtitle range plus configured audio padding. Subtitles usually linger past the dialogue, so once the waveform loads an untouched clip end moves back to just after the line's last speech (plus the configured padding); the Line end rail keeps marking the subtitle timing, Reset restores it, and a line whose speech runs through its end is left alone. Drag either edge of the clip to trim it, drag the middle to slide it without changing its length, or press anywhere else on the waveform to snap the nearer edge there. A focused edge also moves with the arrow keys, by 100 ms alone or 500 ms with Shift, and the 100 ms buttons do the same. Space previews the selection with a playhead that sweeps the clip; the preview ends when the hidden player has actually played the last sample, so output latency such as Bluetooth headphones does not cut the clip short. Enter confirms, and Escape cancels. The Earlier and Later buttons reveal another two seconds of available timeline without moving the selected clip. A speech-weighted waveform shows the mined subtitle as a tinted band with labeled line-start and line-end rails, making adjacent dialogue easier to distinguish. SubMiner uses a center channel when one carries dialogue, then falls back to a mono mix, keeps only the 250 to 3500 Hz speech band, and draws each slice's loudness relative to the clip's own noise floor, so steady background music or ambience reads as a flat line while dialogue stands out. Waveform analysis failure leaves the timing controls available. The confirmed range is exact: SubMiner does not apply audio padding a second time. Static screenshots use its midpoint, and animated AVIF clips use the full confirmed range.
The review can also pull adjacent subtitle lines onto the card. Press `P` or `N` (or use the Prev and Next steppers above the sentence preview) to add the previous or next line, as many times as lines are available; Shift+`P` and Shift+`N` remove them again. The sentence preview lists every included line with the mined line highlighted, so the card's sentence field is always visible before you confirm, and the clip start or end, along with the line-start and line-end rails on the waveform, follows the outermost added line, keeping the review's audio padding. Confirming writes the combined lines to the sentence field; the Reset button drops the added lines along with any timing changes. Adjacent lines come from the parsed subtitle track when one is loaded; otherwise only lines that already played are offered, and a clip capped by `media.maxMediaDuration` keeps the full combined sentence even when the audio cannot cover every added line.
@@ -8,6 +8,7 @@ import {
formatMediaTimingTimestamp,
mediaTimingTimeFromPointer,
slideMediaTimingSelection,
trimMediaTimingSelectionEnd,
} from './media-timing-review';
test('waveform path mirrors normalized peaks around its center line', () => {
@@ -122,3 +123,35 @@ test('preview request guard blocks overlap and invalidates stale responses', ()
guard.finish(second!);
assert.equal(guard.isInFlight(), false);
});
test('trailing silence trim follows the last speech slice and keeps cut-off or silent lines', () => {
// 10 slices over a 10 s timeline: one slice per second, speech in seconds 2-4 only.
const peaks = [0, 0, 0.9, 0.8, 0.7, 0.1, 0.2, 0, 0, 0];
const base = { peaks, timelineStart: 0, timelineEnd: 10, lineStart: 2, lineEnd: 8 };
const trimmed = trimMediaTimingSelectionEnd({ ...base, selectionEnd: 8, endPadSeconds: 0 });
assert.ok(trimmed !== null && Math.abs(trimmed - 5.15) < 1e-9);
const padded = trimMediaTimingSelectionEnd({ ...base, selectionEnd: 8.5, endPadSeconds: 0.5 });
assert.ok(padded !== null && Math.abs(padded - 5.65) < 1e-9);
// Speech running through the line end means the subtitle cuts the audio off; keep it.
assert.equal(
trimMediaTimingSelectionEnd({ ...base, lineEnd: 5, selectionEnd: 5, endPadSeconds: 0 }),
null,
);
// No speech inside the line at all: keep the subtitle timing rather than guess.
assert.equal(
trimMediaTimingSelectionEnd({ ...base, lineStart: 6, selectionEnd: 8, endPadSeconds: 0 }),
null,
);
// A saving below the minimum trim is not worth moving the handle for.
assert.equal(
trimMediaTimingSelectionEnd({ ...base, lineEnd: 5.2, selectionEnd: 5.2, endPadSeconds: 0 }),
null,
);
assert.equal(
trimMediaTimingSelectionEnd({ ...base, peaks: [1], selectionEnd: 8, endPadSeconds: 0 }),
null,
);
});
@@ -11,6 +11,14 @@ const FINE_ADJUST_SECONDS = 0.1;
const COARSE_ADJUST_SECONDS = 0.5;
const TIMELINE_EXPANSION_SECONDS = 2;
const LINE_REVEAL_MARGIN_SECONDS = 1;
/**
* Subtitles usually linger past the dialogue for readability, so an untouched clip end
* follows the last speech-weighted waveform slice above this level (0..1, relative to the
* clip's own noise floor) plus a short tail, when that saves at least the minimum trim.
*/
const SPEECH_LEVEL_THRESHOLD = 0.3;
const SPEECH_TAIL_SECONDS = 0.15;
const MINIMUM_TRAILING_TRIM_SECONDS = 0.1;
/** Slack past the clip length before the UI gives up waiting for mpv's end-of-clip signal. */
const PREVIEW_END_GRACE_MS = 2_500;
@@ -107,6 +115,48 @@ export function buildMediaTimingWaveformPath(peaks: number[]): string {
return `M ${upper.join(' L ')} L ${lower.join(' L ')} Z`;
}
/**
* Where an untouched clip should end once the waveform is known: just after the line's
* last speech slice, plus the configured end padding. Null keeps the subtitle timing when
* no speech shows inside the line, when speech runs through the line end (the subtitle is
* cutting the audio off, not lingering), or when the saving is too small to matter.
*/
export function trimMediaTimingSelectionEnd(options: {
peaks: readonly number[];
timelineStart: number;
timelineEnd: number;
lineStart: number;
lineEnd: number;
selectionEnd: number;
endPadSeconds: number;
}): number | null {
const pointCount = options.peaks.length;
const span = options.timelineEnd - options.timelineStart;
if (pointCount < 2 || span <= 0 || options.lineEnd <= options.lineStart) return null;
// Point i covers [i, i + 1) / pointCount of the timeline (see computeWaveformPeaks).
const sliceEnd = (index: number): number =>
options.timelineStart + ((index + 1) / pointCount) * span;
const firstIndex = Math.max(
0,
Math.floor(((options.lineStart - options.timelineStart) / span) * pointCount),
);
const lastIndex = Math.min(
pointCount - 1,
Math.ceil(((options.lineEnd - options.timelineStart) / span) * pointCount) - 1,
);
let lastSpeechIndex = -1;
for (let index = lastIndex; index >= firstIndex; index -= 1) {
if ((options.peaks[index] ?? 0) >= SPEECH_LEVEL_THRESHOLD) {
lastSpeechIndex = index;
break;
}
}
if (lastSpeechIndex === -1 || lastSpeechIndex >= lastIndex) return null;
const trimmedEnd = sliceEnd(lastSpeechIndex) + SPEECH_TAIL_SECONDS + options.endPadSeconds;
if (options.selectionEnd - trimmedEnd < MINIMUM_TRAILING_TRIM_SECONDS) return null;
return trimmedEnd;
}
export function constrainMediaTimingSelection(options: {
nextStart: number;
nextEnd: number;
@@ -176,6 +226,8 @@ export function createMediaTimingReviewModal(
let nextCount = 0;
let startPadSeconds = 0;
let endPadSeconds = 0;
/** True until the user moves the clip; the first waveform then trims trailing silence. */
let trailingTrimPending = false;
let resolveInFlight = false;
let previewPlaying = false;
let previewTimer: ReturnType<typeof setTimeout> | null = null;
@@ -342,6 +394,7 @@ export function createMediaTimingReviewModal(
}
ctx.dom.mediaTimingReviewWaveformPath.setAttribute('d', path);
setWaveformState('ready');
trimTrailingSilence(result.peaks ?? []);
} catch {
if (sequence === waveformSequence && payload?.reviewId === reviewId) {
setWaveformState('unavailable');
@@ -349,6 +402,29 @@ export function createMediaTimingReviewModal(
}
}
/**
* Moves an untouched clip end back to where the line's dialogue ends. The Line end
* rail keeps marking the subtitle timing, and Reset restores it.
*/
function trimTrailingSilence(peaks: readonly number[]): void {
if (!payload || !trailingTrimPending || previewPlaying || previewRequest.isInFlight()) {
return;
}
const lineRange = currentLineSelection();
const trimmedEnd = trimMediaTimingSelectionEnd({
peaks,
timelineStart,
timelineEnd,
lineStart: lineRange.rangeStart,
lineEnd: lineRange.rangeEnd,
selectionEnd,
endPadSeconds,
});
if (trimmedEnd === null) return;
updateSelection(selectionStart, trimmedEnd);
setStatus('Clip end moved to where the dialogue ends. Reset restores the subtitle timing.');
}
function queueWaveformLoad(delayMs = 0): void {
if (waveformTimer !== null) clearTimeout(waveformTimer);
waveformSequence += 1;
@@ -456,6 +532,7 @@ export function createMediaTimingReviewModal(
});
selectionStart = nextSelection.start;
selectionEnd = nextSelection.end;
trailingTrimPending = false;
if (previewPlaying || previewRequest.isInFlight()) stopPreview();
setStatus('');
renderSelection();
@@ -717,6 +794,7 @@ export function createMediaTimingReviewModal(
nextCount = 0;
startPadSeconds = Math.max(0, nextPayload.originalStartTime - nextPayload.selectionStartTime);
endPadSeconds = Math.max(0, nextPayload.selectionEndTime - nextPayload.originalEndTime);
trailingTrimPending = true;
resolveInFlight = false;
setPreviewPlaying(false);
ctx.dom.mediaTimingReviewKind.textContent =