fix(mining): ignore stale captions and improve waveform contrast

- Use active parsed cue text and timings for overlay mining
- Normalize waveform levels against the clip's noise floor
This commit is contained in:
2026-09-02 18:22:03 -07:00
parent f90ee78204
commit c85db5e07e
8 changed files with 145 additions and 36 deletions
+34 -10
View File
@@ -5,7 +5,13 @@ const WAVEFORM_SAMPLE_RATE = 8_000;
const WAVEFORM_POINT_COUNT = 480;
const WAVEFORM_TIMEOUT_MS = 15_000;
const MAX_WAVEFORM_BYTES = 16 * 1024 * 1024;
const SPEECH_FILTER = 'highpass=f=120,lowpass=f=4000';
// Keep the band where speech intelligibility lives; bass, drums, and hum sit below it.
const SPEECH_FILTER = 'highpass=f=250,lowpass=f=3500';
const NOISE_FLOOR_PERCENTILE = 0.2;
const REFERENCE_PERCENTILE = 0.95;
const NOISE_GATE_DB = 3;
const MIN_DISPLAY_RANGE_DB = 12;
const SILENCE_DB = -100;
const CENTER_CHANNEL_FILTER = `pan=mono|c0=FC,${SPEECH_FILTER}`;
const DOWNMIX_FILTER = `aformat=channel_layouts=mono,${SPEECH_FILTER}`;
@@ -111,11 +117,22 @@ function runFfmpeg(args: string[]): Promise<Buffer> {
});
}
function percentile(sortedValues: number[], fraction: number): number {
const index = Math.min(sortedValues.length - 1, Math.floor(sortedValues.length * fraction));
return sortedValues[index] ?? SILENCE_DB;
}
/**
* Turns mono PCM into 0..1 display heights. Each point is the RMS level of its slice in
* dB, measured against the clip's own noise floor (a low percentile of the slices), so
* constant background noise draws flat and sustained speech stands out. Peak sampling
* would instead follow music transients and lift the floor to nearly speech height.
*/
export function computeWaveformPeaks(pcm: Buffer, pointCount = WAVEFORM_POINT_COUNT): number[] {
const sampleCount = Math.floor(pcm.byteLength / 2);
if (sampleCount === 0 || pointCount <= 0) return [];
const resolvedPointCount = Math.min(pointCount, sampleCount);
const peaks = Array.from({ length: resolvedPointCount }, () => 0);
const levelsDb = Array.from({ length: resolvedPointCount }, () => SILENCE_DB);
for (let point = 0; point < resolvedPointCount; point += 1) {
const sampleStart = Math.floor((point * sampleCount) / resolvedPointCount);
@@ -123,18 +140,25 @@ export function computeWaveformPeaks(pcm: Buffer, pointCount = WAVEFORM_POINT_CO
sampleStart + 1,
Math.floor(((point + 1) * sampleCount) / resolvedPointCount),
);
let peak = 0;
let energy = 0;
for (let sample = sampleStart; sample < sampleEnd; sample += 1) {
peak = Math.max(peak, Math.abs(pcm.readInt16LE(sample * 2)) / 32_768);
const value = pcm.readInt16LE(sample * 2) / 32_768;
energy += value * value;
}
peaks[point] = peak;
const rms = Math.sqrt(energy / (sampleEnd - sampleStart));
levelsDb[point] = rms > 0 ? Math.max(SILENCE_DB, 20 * Math.log10(rms)) : SILENCE_DB;
}
const sortedPeaks = [...peaks].sort((left, right) => left - right);
const referenceIndex = Math.min(sortedPeaks.length - 1, Math.floor(sortedPeaks.length * 0.95));
const referencePeak = Math.max(sortedPeaks[referenceIndex] ?? 0, 0.01);
return peaks.map(
(peak) => Math.round(Math.sqrt(Math.min(1, peak / referencePeak)) * 1_000) / 1_000,
const sortedLevels = [...levelsDb].sort((left, right) => left - right);
const floorDb = percentile(sortedLevels, NOISE_FLOOR_PERCENTILE) + NOISE_GATE_DB;
const referenceDb = Math.max(
percentile(sortedLevels, REFERENCE_PERCENTILE),
floorDb + MIN_DISPLAY_RANGE_DB,
);
return levelsDb.map(
(levelDb) =>
Math.round(Math.min(1, Math.max(0, (levelDb - floorDb) / (referenceDb - floorDb))) * 1_000) /
1_000,
);
}