fix(stats): stop counting duplicate typeset subtitle lines

- Collapse animation-burst subtitle lines (karaoke OPs, animated signs) at ingest time using the same dedup rules the subtitle sidebar already applies, so repeated frames no longer flood "Top Repeated Words"
- Add retroactive cleanup for stats already affected: a "Duplicates" scanner/cleaner in the Vocabulary tab and `subminer stats cleanup --duplicate-lines` (`--dry-run`, `--lookback-days`) on the CLI
- Only subtitle lines and the vocabulary counts they feed are touched; watch time and lines-seen totals are left as recorded
This commit is contained in:
2026-08-10 23:18:43 -07:00
parent 7b0fbdf254
commit 684ab9eaff
31 changed files with 1586 additions and 30 deletions
+12
View File
@@ -18,6 +18,7 @@ import type {
SessionTimelinePoint,
StatsAnkiNoteInfo,
StatsCoverImagesData,
StatsDuplicateLineCleanupResult,
StatsExcludedWord,
StreakCalendarDay,
TrendsDashboardData,
@@ -31,6 +32,13 @@ export type StatsTrendRange = '7d' | '30d' | '90d' | '365d' | 'all';
export type StatsTrendGroupBy = 'day' | 'month';
export type StatsMineMode = 'word' | 'sentence' | 'audio';
/** Body of `POST /api/stats/maintenance/duplicate-lines`. */
export interface StatsDuplicateLineCleanupRequest {
dryRun?: boolean;
/** Null means every recorded line, whatever its age. */
lookbackDays?: number | null;
}
export interface StatsSessionKnownWordsTimelinePoint {
linesSeen: number;
knownWordsSeen: number;
@@ -125,6 +133,7 @@ export interface StatsJsonResponseMap {
vocabulary: VocabularyEntry[];
excludedWords: StatsExcludedWord[];
setExcludedWords: StatsOkResponse;
duplicateLineCleanup: StatsDuplicateLineCleanupResult;
wordOccurrences: VocabularyOccurrenceEntry[];
sentenceSearch: SentenceSearchResult[];
kanji: KanjiEntry[];
@@ -178,6 +187,9 @@ export interface StatsHttpClient {
getVocabulary: (limit?: number) => Promise<VocabularyEntry[]>;
getExcludedWords: () => Promise<StatsExcludedWord[]>;
setExcludedWords: (words: StatsExcludedWord[]) => Promise<void>;
cleanupDuplicateLines: (
options?: StatsDuplicateLineCleanupRequest,
) => Promise<StatsDuplicateLineCleanupResult>;
getWordOccurrences: (
headword: string,
word: string,