fix(stats): stop counting duplicate typeset subtitle lines

- Collapse animation-burst subtitle lines (karaoke OPs, animated signs) at ingest time using the same dedup rules the subtitle sidebar already applies, so repeated frames no longer flood "Top Repeated Words"
- Add retroactive cleanup for stats already affected: a "Duplicates" scanner/cleaner in the Vocabulary tab and `subminer stats cleanup --duplicate-lines` (`--dry-run`, `--lookback-days`) on the CLI
- Only subtitle lines and the vocabulary counts they feed are touched; watch time and lines-seen totals are left as recorded
This commit is contained in:
2026-08-10 23:18:43 -07:00
parent 7b0fbdf254
commit 684ab9eaff
31 changed files with 1586 additions and 30 deletions
@@ -88,6 +88,23 @@ export function parseExcludedWordsBody(body: unknown): StatsExcludedWord[] | nul
return words;
}
/**
* Read a duplicate-line cleanup request. An absent or unusable `lookbackDays` scans all
* history, which is what the CLI does; only a positive number narrows the window.
*/
export function parseDuplicateLineCleanupBody(body: unknown): {
dryRun: boolean;
lookbackDays: number | null;
} {
const source = body && typeof body === 'object' ? (body as Record<string, unknown>) : {};
const rawLookback = source.lookbackDays;
const lookbackDays =
typeof rawLookback === 'number' && Number.isFinite(rawLookback) && rawLookback > 0
? Math.floor(rawLookback)
: null;
return { dryRun: source.dryRun === true, lookbackDays };
}
export function loadKnownWordsSet(cachePath: string | undefined): Set<string> | null {
if (!cachePath || !existsSync(cachePath)) return null;
try {