fix(stats): stop counting duplicate typeset subtitle lines

- Collapse animation-burst subtitle lines (karaoke OPs, animated signs) at ingest time using the same dedup rules the subtitle sidebar already applies, so repeated frames no longer flood "Top Repeated Words"
- Add retroactive cleanup for stats already affected: a "Duplicates" scanner/cleaner in the Vocabulary tab and `subminer stats cleanup --duplicate-lines` (`--dry-run`, `--lookback-days`) on the CLI
- Only subtitle lines and the vocabulary counts they feed are touched; watch time and lines-seen totals are left as recorded
This commit is contained in:
2026-08-10 23:18:43 -07:00
parent 7b0fbdf254
commit 684ab9eaff
31 changed files with 1586 additions and 30 deletions
+6 -3
View File
@@ -1,4 +1,4 @@
import { useState, useEffect } from 'react';
import { useState, useEffect, useCallback } from 'react';
import { getStatsClient } from './useStatsApi';
import type { VocabularyEntry, KanjiEntry } from '../types/stats';
@@ -8,6 +8,9 @@ export function useVocabulary() {
const [knownWords, setKnownWords] = useState<Set<string>>(new Set());
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
// Bumped by `reload` after maintenance rewrites the vocabulary tables.
const [reloadToken, setReloadToken] = useState(0);
const reload = useCallback(() => setReloadToken((token) => token + 1), []);
useEffect(() => {
let cancelled = false;
@@ -46,7 +49,7 @@ export function useVocabulary() {
return () => {
cancelled = true;
};
}, []);
}, [reloadToken]);
return { words, kanji, knownWords, loading, error };
return { words, kanji, knownWords, loading, error, reload };
}