From d8c23154051b7908f39523ff43117049da9f42ed Mon Sep 17 00:00:00 2001 From: sudacode Date: Tue, 3 Nov 2026 22:04:01 -0800 Subject: [PATCH] perf(stats): speed up dashboard queries on large databases (#322) --- changes/stats-dashboard-query-speed.md | 8 + docs/architecture/stats-trends-data-flow.md | 5 +- .../services/__tests__/stats-server.test.ts | 2 +- .../immersion-tracker/__tests__/query.test.ts | 22 +- .../immersion-tracker/lexical-rollups.test.ts | 183 ++++++++++++- .../immersion-tracker/lexical-rollups.ts | 5 +- .../immersion-tracker/query-lexical.ts | 94 ++++--- .../immersion-tracker/query-library.ts | 67 +++-- .../immersion-tracker/query-maintenance.ts | 21 ++ .../immersion-tracker/query-sessions.ts | 248 ++++++++++-------- .../immersion-tracker/query-shared.ts | 92 +++++-- .../immersion-tracker/query-trends.ts | 137 +++++----- .../services/immersion-tracker/storage.ts | 19 +- src/types/stats-wire.ts | 3 +- stats/src/components/trends/TrendsTab.tsx | 34 ++- .../trends/anime-cumulative.test.ts | 36 +++ .../src/components/trends/anime-cumulative.ts | 41 +++ stats/src/lib/api-client.test.ts | 2 +- 18 files changed, 701 insertions(+), 318 deletions(-) create mode 100644 changes/stats-dashboard-query-speed.md create mode 100644 stats/src/components/trends/anime-cumulative.test.ts create mode 100644 stats/src/components/trends/anime-cumulative.ts diff --git a/changes/stats-dashboard-query-speed.md b/changes/stats-dashboard-query-speed.md new file mode 100644 index 00000000..605c99d2 --- /dev/null +++ b/changes/stats-dashboard-query-speed.md @@ -0,0 +1,8 @@ +type: fixed +area: stats + +- The stats dashboard loads much faster on large databases. Overview, media and library detail, and trends no longer re-count every recorded word in the whole database on each request, so these views only cost as much as the sessions they show. On a ~1 GB database the overview went from ~1.4 s to ~0.14 s, media detail from ~1.5 s to ~0.08 s, the 30-day trends from ~1.5 s to ~0.06 s, and all-time trends from ~1.6 s to ~0.2 s. +- The Vocabulary tab's summary and word list load several times faster (summary ~2 s to ~0.2 s, word list ~0.6 s to ~0.2 s). +- The Trends tab downloads far less data: the per-title library charts now arrive as daily values and are totalled in the dashboard, so the all-time response shrank from ~3.9 MB to ~0.2 MB. +- Sentence search with headword matching returns several times faster (~0.3 s to ~0.08 s per query). +- Vocabulary totals and new-word charts stay correct after running vocabulary cleanup. Words it repaired could be counted or hidden by mistake; the first launch after updating corrects them in the background. diff --git a/docs/architecture/stats-trends-data-flow.md b/docs/architecture/stats-trends-data-flow.md index d0a6624b..221d4212 100644 --- a/docs/architecture/stats-trends-data-flow.md +++ b/docs/architecture/stats-trends-data-flow.md @@ -17,11 +17,12 @@ Trend charts now consume one chart-oriented backend payload from `/api/stats/tre - rollup-backed: - activity charts - cumulative watch/cards/tokens/sessions trends - - per-anime watch/cards/tokens/episodes series + - per-anime watch/cards/tokens/episodes series (`animeDaily`: sparse per-title values for active days only; `TrendsTab` builds the running totals with `buildAnimeCumulative`, because a dense title x day payload grew to several MB) - session-metric-backed: - lookup trends - lookup rate trends - watch-time by day-of-week/hour +- rollup word counts and session metrics share one session query per request (`getTrendSessionRows`), so the per-session display-word count runs once instead of once per rollup query - vocabulary-backed: - new-words trend reads permanent daily lexical rollups - rollup rows count only vocabulary-visible tokens and normalize mixed legacy timestamp units @@ -39,6 +40,6 @@ Stats data crosses the process boundary over HTTP only. Shared JSON models live The stats preload bridge remains limited to native window behavior such as confirmation-dialog layering. Do not add stats data request channels back to Electron IPC. -The stats UI should treat the trends payload as chart-ready data. Presentation-only work in the client is fine, but rebuilding the main trend datasets from raw sessions should stay out of the render path. +The stats UI should treat the trends payload as chart-ready data. Presentation-only work in the client is fine (including the per-anime running totals, memoized per payload), but rebuilding the main trend datasets from raw sessions should stay out of the render path. For session detail timelines, omitting `limit` now means "return the full retained session telemetry/history". Explicit `limit` remains available for bounded callers, but the default stats UI path should not trim long sessions to the newest 200 samples. diff --git a/src/core/services/__tests__/stats-server.test.ts b/src/core/services/__tests__/stats-server.test.ts index d76f63b9..5c8c7329 100644 --- a/src/core/services/__tests__/stats-server.test.ts +++ b/src/core/services/__tests__/stats-server.test.ts @@ -189,7 +189,7 @@ const TRENDS_DASHBOARD = { lastWatched: 20_000, }, ], - animeCumulative: { + animeDaily: { watchTime: [{ epochDay: 20_000, animeTitle: 'Little Witch Academia', value: 25 }], episodes: [{ epochDay: 20_000, animeTitle: 'Little Witch Academia', value: 1 }], cards: [{ epochDay: 20_000, animeTitle: 'Little Witch Academia', value: 5 }], diff --git a/src/core/services/immersion-tracker/__tests__/query.test.ts b/src/core/services/immersion-tracker/__tests__/query.test.ts index b965200b..78e9cd44 100644 --- a/src/core/services/immersion-tracker/__tests__/query.test.ts +++ b/src/core/services/immersion-tracker/__tests__/query.test.ts @@ -745,7 +745,10 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => { assert.equal(dashboard.ratios.readingSpeed[0]?.value, +(120 / 30).toFixed(1)); assert.equal(dashboard.ratios.readingSpeed[1]?.value, +(140 / 45).toFixed(1)); assert.equal(dashboard.librarySummary[0]?.title, 'Trend Dashboard Anime'); - assert.equal(dashboard.animeCumulative.watchTime[1]?.value, 75); + assert.deepEqual( + dashboard.animeDaily.watchTime.map((point) => point.value).sort((a, b) => a - b), + [30, 45], + ); assert.equal( dashboard.patterns.watchTimeByDayOfWeek.reduce((sum, point) => sum + point.value, 0), 75, @@ -819,7 +822,7 @@ test('getTrendsDashboard redacts legacy Jellyfin stream titles', () => { const dashboard = getTrendsDashboard(db, 'all', 'day'); const titles = [ - ...dashboard.animeCumulative.watchTime.map((point) => point.animeTitle), + ...dashboard.animeDaily.watchTime.map((point) => point.animeTitle), ...dashboard.librarySummary.map((row) => row.title), ]; @@ -1521,6 +1524,8 @@ test('word-count read models use filtered persisted occurrences with raw fallbac const startedAtMs = 1_700_000_000_000; const withOccurrences = startSessionRecord(db, videoId, startedAtMs); const fallbackOnly = startSessionRecord(db, videoId, startedAtMs + 60_000); + // Still running, so its words stay out of the all-time total. + const stillActive = startSessionRecord(db, videoId, startedAtMs + 120_000); db.prepare( ` @@ -1543,6 +1548,14 @@ test('word-count read models use filtered persisted occurrences with raw fallbac occurrenceCount: 2, startedAtMs, }); + insertFilteredWordOccurrence(db, { + sessionId: stillActive.sessionId, + videoId, + lineIndex: 3, + occurrenceCount: 4, + startedAtMs: startedAtMs + 120_000, + headword: '犬', + }); insertFilteredWordOccurrence(db, { sessionId: withOccurrences.sessionId, videoId, @@ -5075,7 +5088,10 @@ test('getTrendsDashboard rollup word metrics keep persisted totals over partial assert.equal(dashboard.ratios.lookupsPerHundred[0]?.value, 200); assert.equal(dashboard.librarySummary[0]?.words, 30); assert.equal(dashboard.librarySummary[0]?.lookupsPerHundred, 33.3); - assert.equal(dashboard.animeCumulative.words.at(-1)?.value, 30); + assert.equal( + dashboard.animeDaily.words.reduce((sum, point) => sum + point.value, 0), + 30, + ); } finally { db.close(); cleanupDbPath(dbPath); diff --git a/src/core/services/immersion-tracker/lexical-rollups.test.ts b/src/core/services/immersion-tracker/lexical-rollups.test.ts index 897b7dfc..8e023f6f 100644 --- a/src/core/services/immersion-tracker/lexical-rollups.test.ts +++ b/src/core/services/immersion-tracker/lexical-rollups.test.ts @@ -14,9 +14,11 @@ import { getVocabularySummary, replaceStatsExcludedWords, } from './query-lexical'; +import { cleanupVocabularyStats } from './query-maintenance'; import { Database } from './sqlite'; import type { DatabaseSync } from './sqlite'; import { createTrackerPreparedStatements, ensureSchema, executeQueuedWrite } from './storage'; +import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility'; import { BASE_MS, createDb, seedEndedSession, seedVideo } from './__tests__/lifetime-test-fixtures'; function makeDbPath(): string { @@ -384,7 +386,7 @@ test('current lexical rollup readiness accepts legacy integer state storage', () state_value INTEGER NOT NULL ); INSERT INTO imm_rollup_state(state_key, state_value) - VALUES ('lexical_daily_rollups_version', 3); + VALUES ('lexical_daily_rollups_version', 4); `); assert.equal(areLexicalDailyRollupsReady(db), true); @@ -394,6 +396,185 @@ test('current lexical rollup readiness accepts legacy integer state storage', () } }); +function insertWord( + db: DatabaseSync, + word: { + headword: string; + word: string; + reading: string; + partOfSpeech: string; + pos1: string; + pos2: string; + }, + firstSeen: number, +): number { + const result = db + .prepare( + `INSERT INTO imm_words( + headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency + ) VALUES (?, ?, ?, ?, ?, ?, '', ?, ?, 1)`, + ) + .run( + word.headword, + word.word, + word.reading, + word.partOfSpeech, + word.pos1, + word.pos2, + firstSeen, + firstSeen, + ); + return Number(result.lastInsertRowid); +} + +function assertPersistedVisibilityMatchesPredicate(db: DatabaseSync): void { + const rows = db + .prepare( + `SELECT headword, word, reading, part_of_speech AS partOfSpeech, pos1, pos2, pos3, + frequency_rank AS frequencyRank, vocabulary_visible AS vocabularyVisible + FROM imm_words`, + ) + .all() as Array; + assert.ok(rows.length > 0); + assert.deepEqual( + rows.map((row) => [row.headword, row.vocabularyVisible === 1]), + rows.map((row) => [row.headword, isVocabularyStatsRowVisible(row)]), + ); +} + +test('cleanupVocabularyStats re-derives visibility for a repaired word', async () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const epochDay = 19_500; + const firstSeen = epochDay * 86_400 + 43_200; + const legacyWord = { + headword: '旧', + word: '旧', + reading: '', + partOfSpeech: 'particle', + pos1: '助詞', + pos2: '係助詞', + }; + const legacyId = insertWord(db, legacyWord, firstSeen); + rebuildLexicalDailyRollups(db); + assert.equal(getVocabularySummary(db, null).uniqueWords, 0); + + await cleanupVocabularyStats(db, { + resolveLegacyPos: async (row) => + row.id === legacyId + ? { + headword: '旧', + reading: 'きゅう', + partOfSpeech: 'noun', + pos1: '名詞', + pos2: '一般', + pos3: '', + } + : null, + }); + + assertPersistedVisibilityMatchesPredicate(db); + assert.equal(getVocabularySummary(db, null).uniqueWords, 1); + assert.equal(getLexicalDailyRollups(db)[0]?.wordCount, 1); + } finally { + db.close(); + fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }); + } +}); + +test('cleanupVocabularyStats re-derives visibility for a merged duplicate', async () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const firstSeen = 19_500 * 86_400 + 43_200; + // The merge fills these blank POS fields from the repair, which hides the word. + insertWord( + db, + { + headword: '知る', + word: '知っている', + reading: 'しっている', + partOfSpeech: 'noun', + pos1: '', + pos2: '', + }, + firstSeen, + ); + const legacyId = insertWord( + db, + { + headword: '知っている', + word: '知っている', + reading: '', + partOfSpeech: 'verb', + pos1: '動詞', + pos2: '自立', + }, + firstSeen, + ); + rebuildLexicalDailyRollups(db); + assert.equal(getVocabularySummary(db, null).uniqueWords, 2); + + await cleanupVocabularyStats(db, { + resolveLegacyPos: async (row) => + row.id === legacyId + ? { + headword: '知る', + reading: 'しっている', + partOfSpeech: 'particle', + pos1: '助詞', + pos2: '係助詞', + pos3: '', + } + : null, + }); + + assertPersistedVisibilityMatchesPredicate(db); + assert.equal(getVocabularySummary(db, null).uniqueWords, 0); + assert.deepEqual(getLexicalDailyRollups(db), []); + } finally { + db.close(); + fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }); + } +}); + +test('a queued word write persists visibility from the stored row', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const firstSeen = 19_500 * 86_400 + 43_200; + executeQueuedWrite( + { + kind: 'word', + headword: 'は', + word: 'は', + reading: 'は', + partOfSpeech: 'particle', + pos1: '助詞', + pos2: '係助詞', + pos3: '', + firstSeen, + lastSeen: firstSeen, + frequencyRank: null, + }, + createTrackerPreparedStatements(db), + ); + + assertPersistedVisibilityMatchesPredicate(db); + assert.equal(getVocabularySummary(db, null).uniqueWords, 0); + } finally { + db.close(); + fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }); + } +}); + test('imm_words persists vocabulary visibility for rollup maintenance', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); diff --git a/src/core/services/immersion-tracker/lexical-rollups.ts b/src/core/services/immersion-tracker/lexical-rollups.ts index f00b926d..7b2dd10f 100644 --- a/src/core/services/immersion-tracker/lexical-rollups.ts +++ b/src/core/services/immersion-tracker/lexical-rollups.ts @@ -19,8 +19,9 @@ const LOCAL_EPOCH_DAY_SQL = ` `; // v3 re-derives visibility for rows a re-seen word forced visible and repairs -// last_seen values left behind by session deletes. -const LEXICAL_DAILY_ROLLUP_VERSION = '3'; +// last_seen values left behind by session deletes. v4 re-derives visibility for +// rows vocabulary cleanup repaired without updating it. +const LEXICAL_DAILY_ROLLUP_VERSION = '4'; const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version'; const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000; diff --git a/src/core/services/immersion-tracker/query-lexical.ts b/src/core/services/immersion-tracker/query-lexical.ts index 67bc922b..866c7229 100644 --- a/src/core/services/immersion-tracker/query-lexical.ts +++ b/src/core/services/immersion-tracker/query-lexical.ts @@ -16,7 +16,7 @@ import type { WordDetailRow, WordOccurrenceRow, } from './types'; -import { fromDbTimestamp, toDbTimestamp } from './query-shared'; +import { forEachIdChunk, fromDbTimestamp, makePlaceholders, toDbTimestamp } from './query-shared'; import { nowMs } from './time'; import { narrowFullwidthKana, widenHalfwidthKana } from '../../utils/halfwidth-kana'; import { @@ -106,42 +106,58 @@ export function getVocabularyStats( const whereClause = hasExclude ? `WHERE (part_of_speech IS NULL OR part_of_speech NOT IN (${placeholders}))` : ''; - // The page is selected before the join so `animeCount` is only computed for the - // rows being returned. Aggregating first made every request walk each word's - // entire occurrence history — seconds of blocked event loop on a large library, - // because only the ordering, not the aggregate, decides which rows survive. + // Rows are paged and filtered before `animeCount` is computed, and only the + // rows being returned are counted. The count walks each word's entire + // occurrence history, and the most frequent words hold most occurrences, so + // counting the oversampled page or aggregating first cost seconds of blocked + // event loop on a large library. const stmt = db.prepare(` - WITH page AS ( - SELECT id, headword, word, reading, part_of_speech, pos1, pos2, pos3, - frequency, frequency_rank, first_seen, last_seen - FROM imm_words - ${whereClause} - ORDER BY frequency DESC, id - LIMIT ? OFFSET ? - ) - SELECT p.id AS wordId, p.headword, p.word, p.reading, - p.part_of_speech AS partOfSpeech, p.pos1, p.pos2, p.pos3, - p.frequency, p.frequency_rank AS frequencyRank, - p.first_seen AS firstSeen, p.last_seen AS lastSeen, - COUNT(DISTINCT sl.anime_id) AS animeCount - FROM page p - LEFT JOIN imm_word_line_occurrences o ON o.word_id = p.id - LEFT JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id AND sl.anime_id IS NOT NULL - GROUP BY p.id - ORDER BY p.frequency DESC, p.id + SELECT id AS wordId, headword, word, reading, + part_of_speech AS partOfSpeech, pos1, pos2, pos3, + frequency, frequency_rank AS frequencyRank, + first_seen AS firstSeen, last_seen AS lastSeen + FROM imm_words + ${whereClause} + ORDER BY frequency DESC, id + LIMIT ? OFFSET ? `); - const visibleRows: VocabularyStatsRow[] = []; + const visibleRows: Array> = []; let offset = 0; while (visibleRows.length < limit) { const params = hasExclude ? [...excludePos, pageSize, offset] : [pageSize, offset]; - const page = stmt.all(...params) as VocabularyStatsRow[]; + const page = stmt.all(...params) as Array>; if (page.length === 0) break; visibleRows.push(...page.filter(isVocabularyStatsRowVisible)); offset += page.length; } - return visibleRows.slice(0, limit); + const rows = visibleRows.slice(0, limit); + const animeCounts = getWordAnimeCounts( + db, + rows.map((row) => row.wordId), + ); + return rows.map((row) => ({ ...row, animeCount: animeCounts.get(row.wordId) ?? 0 })); +} + +/** Number of distinct anime each word appears in, keyed by word id. */ +function getWordAnimeCounts(db: DatabaseSync, wordIds: number[]): Map { + const counts = new Map(); + forEachIdChunk(wordIds, (chunk) => { + const rows = db + .prepare( + ` + SELECT o.word_id AS wordId, COUNT(DISTINCT sl.anime_id) AS animeCount + FROM imm_word_line_occurrences o + JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id + WHERE o.word_id IN (${makePlaceholders(chunk)}) AND sl.anime_id IS NOT NULL + GROUP BY o.word_id + `, + ) + .all(...chunk) as Array<{ wordId: number; animeCount: number }>; + for (const row of rows) counts.set(row.wordId, row.animeCount); + }); + return counts; } /** @@ -276,8 +292,12 @@ export function getVocabularySummary( nowMs: number = Date.now(), scanBatchSize: number = VOCABULARY_SUMMARY_SCAN_BATCH_SIZE, ): VocabularyStatsSummary { - // Visibility and exclusion rules live in JS, so rows are scanned in id-keyed - // batches to keep memory bounded on large vocabularies. + // Exclusion and known-word rules live in JS, so rows are scanned in id-keyed + // batches to keep memory bounded on large vocabularies. Once the lexical + // rollups are ready, `vocabulary_visible` holds the persisted result of + // isVocabularyStatsRowVisible (the same column the new-word charts count), so + // the scan skips re-running the subtitle filter rules for every word. + const persistedVisibility = areLexicalDailyRollupsReady(db); const scanStmt = db.prepare(` SELECT id AS wordId, headword, word, reading, part_of_speech AS partOfSpeech, pos1, pos2, pos3, @@ -285,10 +305,11 @@ export function getVocabularySummary( first_seen AS firstSeen, last_seen AS lastSeen, 0 AS animeCount FROM imm_words - WHERE id > ? + WHERE id > ?${persistedVisibility ? ' AND vocabulary_visible = 1' : ''} ORDER BY id LIMIT ? `); + const isVisible = persistedVisibility ? () => true : isVocabularyStatsRowVisible; const excludedAliases = new Set( getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)), ); @@ -311,7 +332,7 @@ export function getVocabularySummary( lastId = words[words.length - 1]!.wordId; for (const word of words) { if ( - !isVocabularyStatsRowVisible(word) || + !isVisible(word) || excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias)) ) { continue; @@ -489,15 +510,16 @@ export function searchSubtitleSentences( narrowFullwidthKana(widenedTerm), ]).map((variant) => `%${escapeLikeTerm(variant)}%`); const headwords = getHeadwordCandidatesForSentenceSearchTerm(term, options); + // Uncorrelated on purpose: SQLite resolves the headword's lines once through + // the word indexes. A per-line EXISTS probed every subtitle line (~0.2 s). const headwordClause = headwords.length > 0 ? ` - OR EXISTS ( - SELECT 1 - FROM imm_word_line_occurrences o - JOIN imm_words w ON w.id = o.word_id - WHERE o.line_id = l.line_id - AND w.headword IN (${headwords.map(() => '?').join(', ')}) + OR l.line_id IN ( + SELECT o.line_id + FROM imm_words w + JOIN imm_word_line_occurrences o ON o.word_id = w.id + WHERE w.headword IN (${headwords.map(() => '?').join(', ')}) ) ` : ''; diff --git a/src/core/services/immersion-tracker/query-library.ts b/src/core/services/immersion-tracker/query-library.ts index a496ff45..65d9626a 100644 --- a/src/core/services/immersion-tracker/query-library.ts +++ b/src/core/services/immersion-tracker/query-library.ts @@ -18,7 +18,6 @@ import type { } from './types'; import { ACTIVE_SESSION_METRICS_CTE, - SESSION_WORD_COUNTS_SELECT, fromDbTimestamp, resolvedCoverBlobExpr, sessionDisplayWordsExpr, @@ -123,7 +122,7 @@ export function getAnimeAnilistEntries(db: DatabaseSync, animeId: number): Anime } export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisodeRow[] { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); + const wordsExpr = sessionDisplayWordsExpr('s', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); const rows = db .prepare( ` @@ -184,7 +183,6 @@ export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisod FROM imm_videos v LEFT JOIN imm_sessions s ON s.video_id = v.video_id LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id WHERE v.anime_id = ? GROUP BY v.video_id ORDER BY @@ -250,7 +248,7 @@ export function getMediaLibrary(db: DatabaseSync): MediaLibraryRow[] { } export function getMediaDetail(db: DatabaseSync, videoId: number): MediaDetailRow | null { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); + const wordsExpr = sessionDisplayWordsExpr('s', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); return db .prepare( ` @@ -298,7 +296,6 @@ export function getMediaDetail(db: DatabaseSync, videoId: number): MediaDetailRo LEFT JOIN imm_youtube_videos yv ON yv.video_id = v.video_id LEFT JOIN imm_sessions s ON s.video_id = v.video_id LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id WHERE v.video_id = ? AND (lm.video_id IS NOT NULL OR s.session_id IS NOT NULL) GROUP BY v.video_id @@ -312,7 +309,7 @@ export function getMediaSessions( videoId: number, limit = 100, ): SessionSummaryQueryRow[] { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); + const wordsExpr = sessionDisplayWordsExpr('s', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); const rows = db .prepare( ` @@ -333,7 +330,6 @@ export function getMediaSessions( COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0) AS yomitanLookupCount FROM imm_sessions s LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id LEFT JOIN imm_videos v ON v.video_id = s.video_id WHERE s.video_id = ? ORDER BY s.started_at_ms DESC @@ -358,12 +354,15 @@ export function getMediaDailyRollups( videoId: number, limit = 90, ): ImmersionSessionRollupRow[] { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc'); return db .prepare( ` - WITH session_word_counts AS ( - ${SESSION_WORD_COUNTS_SELECT} + WITH recent_days AS ( + SELECT DISTINCT rollup_day + FROM imm_daily_rollups + WHERE video_id = ? + ORDER BY rollup_day DESC + LIMIT ? ), daily_word_counts AS ( SELECT @@ -372,18 +371,15 @@ export function getMediaDailyRollups( AS INTEGER ) AS rollupDay, s.video_id AS videoId, - SUM(${wordsExpr}) AS totalTokensSeen + SUM(${sessionDisplayWordsExpr('s')}) AS totalTokensSeen FROM imm_sessions s - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id WHERE s.ended_at_ms IS NOT NULL + AND s.video_id = ? + AND CAST( + julianday(CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 + AS INTEGER + ) >= (SELECT MIN(rollup_day) FROM recent_days) GROUP BY rollupDay, s.video_id - ), - recent_days AS ( - SELECT DISTINCT rollup_day - FROM imm_daily_rollups - WHERE video_id = ? - ORDER BY rollup_day DESC - LIMIT ? ) SELECT rollup_day AS rollupDayOrMonth, @@ -416,7 +412,7 @@ export function getMediaDailyRollups( ORDER BY rollup_day DESC, video_id DESC `, ) - .all(videoId, limit, videoId) as unknown as ImmersionSessionRollupRow[]; + .all(videoId, limit, videoId, videoId) as unknown as ImmersionSessionRollupRow[]; } export function getAnimeDailyRollups( @@ -424,12 +420,16 @@ export function getAnimeDailyRollups( animeId: number, limit = 90, ): ImmersionSessionRollupRow[] { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc'); return db .prepare( ` - WITH session_word_counts AS ( - ${SESSION_WORD_COUNTS_SELECT} + WITH recent_days AS ( + SELECT DISTINCT r.rollup_day + FROM imm_daily_rollups r + JOIN imm_videos v ON v.video_id = r.video_id + WHERE v.anime_id = ? + ORDER BY r.rollup_day DESC + LIMIT ? ), daily_word_counts AS ( SELECT @@ -438,19 +438,15 @@ export function getAnimeDailyRollups( AS INTEGER ) AS rollupDay, s.video_id AS videoId, - SUM(${wordsExpr}) AS totalTokensSeen + SUM(${sessionDisplayWordsExpr('s')}) AS totalTokensSeen FROM imm_sessions s - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id WHERE s.ended_at_ms IS NOT NULL + AND s.video_id IN (SELECT video_id FROM imm_videos WHERE anime_id = ?) + AND CAST( + julianday(CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 + AS INTEGER + ) >= (SELECT MIN(rollup_day) FROM recent_days) GROUP BY rollupDay, s.video_id - ), - recent_days AS ( - SELECT DISTINCT r.rollup_day - FROM imm_daily_rollups r - JOIN imm_videos v ON v.video_id = r.video_id - WHERE v.anime_id = ? - ORDER BY r.rollup_day DESC - LIMIT ? ) SELECT r.rollup_day AS rollupDayOrMonth, r.video_id AS videoId, r.total_sessions AS totalSessions, r.total_active_min AS totalActiveMin, @@ -481,7 +477,7 @@ export function getAnimeDailyRollups( ORDER BY r.rollup_day DESC, r.video_id DESC `, ) - .all(animeId, limit, animeId) as unknown as ImmersionSessionRollupRow[]; + .all(animeId, limit, animeId, animeId) as unknown as ImmersionSessionRollupRow[]; } export function getAnimeCoverArt(db: DatabaseSync, animeId: number): MediaArtRow | null { @@ -657,7 +653,7 @@ export function getEpisodeWords(db: DatabaseSync, videoId: number, limit = 50): } export function getEpisodeSessions(db: DatabaseSync, videoId: number): SessionSummaryQueryRow[] { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); + const wordsExpr = sessionDisplayWordsExpr('s', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); const rows = db .prepare( ` @@ -677,7 +673,6 @@ export function getEpisodeSessions(db: DatabaseSync, videoId: number): SessionSu FROM imm_sessions s JOIN imm_videos v ON v.video_id = s.video_id LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id WHERE s.video_id = ? ORDER BY s.started_at_ms DESC `, diff --git a/src/core/services/immersion-tracker/query-maintenance.ts b/src/core/services/immersion-tracker/query-maintenance.ts index 98dc43be..684d1b7b 100644 --- a/src/core/services/immersion-tracker/query-maintenance.ts +++ b/src/core/services/immersion-tracker/query-maintenance.ts @@ -13,6 +13,7 @@ import { deleteMaintenanceBatch } from './query-delete-maintenance'; import { PartOfSpeech, type MergedToken } from '../../../types'; import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage'; import { deriveStoredPartOfSpeech } from '../tokenizer/part-of-speech'; +import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility'; import { cleanupUnusedCoverArtBlobHash, findSharedCoverBlobHash, @@ -241,6 +242,24 @@ export async function cleanupVocabularyStats( const deleteOccurrencesStmt = db.prepare( 'DELETE FROM imm_word_line_occurrences WHERE word_id = ?', ); + const selectVisibilityInputsStmt = db.prepare( + `SELECT word, headword, reading, part_of_speech AS partOfSpeech, + pos1, pos2, pos3, frequency_rank AS frequencyRank + FROM imm_words + WHERE id = ?`, + ); + const updateVisibilityStmt = db.prepare( + `UPDATE imm_words SET vocabulary_visible = ? WHERE id = ? AND vocabulary_visible IS NOT ?`, + ); + // Repairs rewrite the visibility predicate's inputs, and the vocabulary summary + // and new-word rollups read the persisted result, so re-derive it after each one. + const syncWordVisibility = (wordId: number): void => { + // The scan awaits POS resolution, so another writer may have removed the word. + const stored = selectVisibilityInputsStmt.get(wordId) as VocabularyVisibilityRow | null; + if (!stored) return; + const visible = isVocabularyStatsRowVisible(stored) ? 1 : 0; + updateVisibilityStmt.run(visible, wordId, visible); + }; let kept = 0; let deleted = 0; let repaired = 0; @@ -279,6 +298,7 @@ export async function cleanupVocabularyStats( row.last_seen ?? duplicate.last_seen ?? 0, duplicate.id, ); + syncWordVisibility(duplicate.id); deleteStmt.run(row.id); repaired += 1; deleted += 1; @@ -294,6 +314,7 @@ export async function cleanupVocabularyStats( resolvedPos.pos3, row.id, ); + syncWordVisibility(row.id); repaired += 1; } diff --git a/src/core/services/immersion-tracker/query-sessions.ts b/src/core/services/immersion-tracker/query-sessions.ts index 41b1fe69..7351ed1b 100644 --- a/src/core/services/immersion-tracker/query-sessions.ts +++ b/src/core/services/immersion-tracker/query-sessions.ts @@ -6,9 +6,8 @@ import type { } from './types'; import { ACTIVE_SESSION_METRICS_CTE, - SESSION_WORD_COUNTS_CTE, - SESSION_WORD_COUNTS_SELECT, currentDbTimestamp, + ENDED_SESSIONS_DISPLAY_WORDS_SQL, fromDbTimestamp, getLocalEpochDay, getShiftedLocalDaySec, @@ -295,32 +294,17 @@ export function getQueryHints(db: DatabaseSync): { const totalCards = Number(lifetime?.totalCards ?? 0); const activeDays = Number(lifetime?.activeDays ?? 0); - const lookupWordsExpr = sessionDisplayWordsExpr( - 's', - 'swc', - 'COALESCE(t.tokens_seen, s.tokens_seen)', - ); + // Ended sessions are finalized from their last telemetry sample, so their + // own counters already hold the telemetry maxima. const lookupTotals = db .prepare( ` - ${SESSION_WORD_COUNTS_CTE} SELECT - COALESCE(SUM(${lookupWordsExpr}), 0) AS totalTokensSeen, - COALESCE(SUM(COALESCE(t.lookup_count, s.lookup_count, 0)), 0) AS totalLookupCount, - COALESCE(SUM(COALESCE(t.lookup_hits, s.lookup_hits, 0)), 0) AS totalLookupHits, - COALESCE(SUM(COALESCE(t.yomitan_lookup_count, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount + ${ENDED_SESSIONS_DISPLAY_WORDS_SQL} AS totalTokensSeen, + COALESCE(SUM(COALESCE(s.lookup_count, 0)), 0) AS totalLookupCount, + COALESCE(SUM(COALESCE(s.lookup_hits, 0)), 0) AS totalLookupHits, + COALESCE(SUM(COALESCE(s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount FROM imm_sessions s - LEFT JOIN ( - SELECT - session_id, - MAX(tokens_seen) AS tokens_seen, - MAX(lookup_count) AS lookup_count, - MAX(lookup_hits) AS lookup_hits, - MAX(yomitan_lookup_count) AS yomitan_lookup_count - FROM imm_session_telemetry - GROUP BY session_id - ) t ON t.session_id = s.session_id - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id WHERE s.ended_at_ms IS NOT NULL `, ) @@ -349,123 +333,157 @@ export function getQueryHints(db: DatabaseSync): { }; } -export function getDailyRollups(db: DatabaseSync, limit = 60): ImmersionSessionRollupRow[] { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc'); - const prepared = db.prepare(` - WITH session_word_counts AS ( - ${SESSION_WORD_COUNTS_SELECT} - ), - daily_word_counts AS ( - SELECT - CAST( - julianday(CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 - AS INTEGER - ) AS rollupDay, - s.video_id AS videoId, - SUM(${wordsExpr}) AS totalTokensSeen - FROM imm_sessions s - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id - WHERE s.ended_at_ms IS NOT NULL - GROUP BY rollupDay, s.video_id - ), - recent_days AS ( - SELECT DISTINCT rollup_day - FROM imm_daily_rollups - ORDER BY rollup_day DESC - LIMIT ? - ) +/** Local calendar day (epoch days) a session row `alias` started on, matching `imm_daily_rollups.rollup_day`. */ +export function sessionRollupDaySql(alias: string): string { + return `CAST(julianday(CAST(${alias}.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER)`; +} + +/** Local `YYYYMM` month a session row `alias` started in, matching `imm_monthly_rollups.rollup_month`. */ +export function sessionRollupMonthSql(alias: string): string { + return `CAST(strftime('%Y%m', CAST(${alias}.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER)`; +} + +/** + * Display-word sums of ended sessions, keyed by `rollupWordCountKey`. Rollup + * rows take these over their persisted token totals when they are larger. + */ +export type RollupWordCounts = Map; + +export function rollupWordCountKey(rollupDayOrMonth: number, videoId: number | null): string { + return `${rollupDayOrMonth}:${videoId ?? ''}`; +} + +export function addRollupWordCount( + counts: RollupWordCounts, + rollupDayOrMonth: number, + videoId: number | null, + words: number, +): void { + const key = rollupWordCountKey(rollupDayOrMonth, videoId); + counts.set(key, (counts.get(key) ?? 0) + words); +} + +type RawRollupRow = Omit; + +/** + * Rollup rows for the `limit` most recent days, without the display-word + * merge. Pair with `applyRollupWordCounts`; `getDailyRollups` does both. + */ +export function selectDailyRollupRows(db: DatabaseSync, limit: number): RawRollupRow[] { + return db + .prepare( + ` SELECT r.rollup_day AS rollupDayOrMonth, r.video_id AS videoId, r.total_sessions AS totalSessions, r.total_active_min AS totalActiveMin, r.total_lines_seen AS totalLinesSeen, - CASE - WHEN dwc.totalTokensSeen IS NOT NULL AND dwc.totalTokensSeen > r.total_tokens_seen THEN dwc.totalTokensSeen - ELSE r.total_tokens_seen - END AS totalTokensSeen, + r.total_tokens_seen AS totalTokensSeen, r.total_cards AS totalCards, r.cards_per_hour AS cardsPerHour, - CASE - WHEN r.total_active_min > 0 THEN ( - CASE - WHEN dwc.totalTokensSeen IS NOT NULL AND dwc.totalTokensSeen > r.total_tokens_seen THEN dwc.totalTokensSeen - ELSE r.total_tokens_seen - END - ) * 1.0 / r.total_active_min - ELSE NULL - END AS tokensPerMin, r.lookup_hit_rate AS lookupHitRate FROM imm_daily_rollups r - LEFT JOIN daily_word_counts dwc - ON dwc.rollupDay = r.rollup_day - AND ( - (dwc.videoId IS NULL AND r.video_id IS NULL) - OR dwc.videoId = r.video_id - ) - WHERE r.rollup_day IN (SELECT rollup_day FROM recent_days) + WHERE r.rollup_day IN ( + SELECT DISTINCT rollup_day FROM imm_daily_rollups ORDER BY rollup_day DESC LIMIT ? + ) ORDER BY r.rollup_day DESC, r.video_id DESC - `); - - return prepared.all(limit) as unknown as ImmersionSessionRollupRow[]; + `, + ) + .all(limit) as unknown as RawRollupRow[]; } -export function getMonthlyRollups(db: DatabaseSync, limit = 24): ImmersionSessionRollupRow[] { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc'); - const prepared = db.prepare(` - WITH session_word_counts AS ( - ${SESSION_WORD_COUNTS_SELECT} - ), - monthly_word_counts AS ( - SELECT - CAST(strftime('%Y%m', CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) AS rollupMonth, - s.video_id AS videoId, - SUM(${wordsExpr}) AS totalTokensSeen - FROM imm_sessions s - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id - WHERE s.ended_at_ms IS NOT NULL - GROUP BY rollupMonth, s.video_id - ), - recent_months AS ( - SELECT DISTINCT rollup_month - FROM imm_monthly_rollups - ORDER BY rollup_month DESC - LIMIT ? - ) +/** Monthly counterpart of `selectDailyRollupRows`. */ +export function selectMonthlyRollupRows(db: DatabaseSync, limit: number): RawRollupRow[] { + return db + .prepare( + ` SELECT r.rollup_month AS rollupDayOrMonth, r.video_id AS videoId, r.total_sessions AS totalSessions, r.total_active_min AS totalActiveMin, r.total_lines_seen AS totalLinesSeen, - CASE - WHEN mwc.totalTokensSeen IS NOT NULL AND mwc.totalTokensSeen > r.total_tokens_seen THEN mwc.totalTokensSeen - ELSE r.total_tokens_seen - END AS totalTokensSeen, + r.total_tokens_seen AS totalTokensSeen, r.total_cards AS totalCards, CASE WHEN r.total_active_min > 0 THEN (r.total_cards * 60.0) / r.total_active_min ELSE NULL END AS cardsPerHour, - CASE - WHEN r.total_active_min > 0 THEN ( - CASE - WHEN mwc.totalTokensSeen IS NOT NULL AND mwc.totalTokensSeen > r.total_tokens_seen THEN mwc.totalTokensSeen - ELSE r.total_tokens_seen - END - ) * 1.0 / r.total_active_min - ELSE NULL - END AS tokensPerMin, NULL AS lookupHitRate FROM imm_monthly_rollups r - LEFT JOIN monthly_word_counts mwc - ON mwc.rollupMonth = r.rollup_month - AND ( - (mwc.videoId IS NULL AND r.video_id IS NULL) - OR mwc.videoId = r.video_id - ) - WHERE r.rollup_month IN (SELECT rollup_month FROM recent_months) + WHERE r.rollup_month IN ( + SELECT DISTINCT rollup_month FROM imm_monthly_rollups ORDER BY rollup_month DESC LIMIT ? + ) ORDER BY r.rollup_month DESC, r.video_id DESC - `); - return prepared.all(limit) as unknown as ImmersionSessionRollupRow[]; + `, + ) + .all(limit) as unknown as RawRollupRow[]; +} + +/** + * Raises each rollup's token total to the display-word count of its sessions + * when that is larger (persisted totals can lag behind cleaned-up lines), and + * derives `tokensPerMin` from the result. + */ +export function applyRollupWordCounts( + rows: RawRollupRow[], + counts: RollupWordCounts, +): ImmersionSessionRollupRow[] { + return rows.map((row) => { + const words = counts.get(rollupWordCountKey(row.rollupDayOrMonth, row.videoId)); + const totalTokensSeen = + words !== undefined && words > row.totalTokensSeen ? words : row.totalTokensSeen; + return { + ...row, + totalTokensSeen, + tokensPerMin: row.totalActiveMin > 0 ? totalTokensSeen / row.totalActiveMin : null, + }; + }); +} + +function getEndedSessionWordCounts( + db: DatabaseSync, + bucketSql: string, + sinceBucket: number, +): RollupWordCounts { + const rows = db + .prepare( + ` + SELECT + ${bucketSql} AS bucket, + s.video_id AS videoId, + SUM(${sessionDisplayWordsExpr('s')}) AS words + FROM imm_sessions s + WHERE s.ended_at_ms IS NOT NULL + AND ${bucketSql} >= ? + GROUP BY bucket, s.video_id + `, + ) + .all(sinceBucket) as Array<{ bucket: number; videoId: number | null; words: number }>; + const counts: RollupWordCounts = new Map(); + for (const row of rows) { + addRollupWordCount(counts, row.bucket, row.videoId, row.words); + } + return counts; +} + +function withSessionWordCounts( + db: DatabaseSync, + rows: RawRollupRow[], + bucketSql: string, +): ImmersionSessionRollupRow[] { + if (rows.length === 0) { + return []; + } + const sinceBucket = Math.min(...rows.map((row) => row.rollupDayOrMonth)); + return applyRollupWordCounts(rows, getEndedSessionWordCounts(db, bucketSql, sinceBucket)); +} + +export function getDailyRollups(db: DatabaseSync, limit = 60): ImmersionSessionRollupRow[] { + return withSessionWordCounts(db, selectDailyRollupRows(db, limit), sessionRollupDaySql('s')); +} + +export function getMonthlyRollups(db: DatabaseSync, limit = 24): ImmersionSessionRollupRow[] { + return withSessionWordCounts(db, selectMonthlyRollupRows(db, limit), sessionRollupMonthSql('s')); } diff --git a/src/core/services/immersion-tracker/query-shared.ts b/src/core/services/immersion-tracker/query-shared.ts index a445bab8..014ea312 100644 --- a/src/core/services/immersion-tracker/query-shared.ts +++ b/src/core/services/immersion-tracker/query-shared.ts @@ -18,24 +18,10 @@ export function visibleWordSql(wordAlias: string): string { )`; } -export function filteredWordOccurrenceCountSql(occurrenceAlias: string, wordAlias: string): string { - return `CASE - WHEN ${occurrenceAlias}.word_id IS NOT NULL AND ${visibleWordSql(wordAlias)} - THEN ${occurrenceAlias}.occurrence_count - ELSE 0 - END`; -} - -export const SESSION_WORD_COUNTS_SELECT = ` - SELECT - sl.session_id AS sessionId, - COUNT(DISTINCT sl.line_id) AS persistedLineCount, - COALESCE(SUM(${filteredWordOccurrenceCountSql('wlo', 'w')}), 0) AS filteredWordsSeen - FROM imm_subtitle_lines sl - LEFT JOIN imm_word_line_occurrences wlo ON wlo.line_id = sl.line_id - LEFT JOIN imm_words w ON w.id = wlo.word_id - GROUP BY sl.session_id -`; +// Hidden words are a small set, so SQLite builds this list once per statement +// and checks occurrences against it instead of re-evaluating the term filter +// for every occurrence row. +const HIDDEN_WORD_IDS_SQL = `SELECT hidden_word.id FROM imm_words hidden_word WHERE NOT ${visibleWordSql('hidden_word')}`; export const ACTIVE_SESSION_METRICS_CTE = ` WITH active_session_metrics AS ( @@ -53,29 +39,77 @@ export const ACTIVE_SESSION_METRICS_CTE = ` JOIN imm_sessions s ON s.session_id = t.session_id WHERE s.ended_at_ms IS NULL GROUP BY t.session_id - ), - session_word_counts AS ( - ${SESSION_WORD_COUNTS_SELECT} - ) -`; - -export const SESSION_WORD_COUNTS_CTE = ` - WITH session_word_counts AS ( - ${SESSION_WORD_COUNTS_SELECT} ) `; +/** + * Display word count for the session row `sessionAlias`: visible persisted word + * occurrences when the session has persisted subtitle lines, otherwise the raw + * token counter `rawTokensExpr`. + * + * The subqueries are correlated on `session_id`, so a query only reads the + * lines of the sessions it actually returns. Do not replace this with a CTE + * aggregated over every subtitle line; that costs a full scan of the + * occurrence table on every stats request. + */ export function sessionDisplayWordsExpr( sessionAlias: string, - wordCountAlias: string, rawTokensExpr = `${sessionAlias}.tokens_seen`, ): string { return `CASE - WHEN COALESCE(${wordCountAlias}.persistedLineCount, 0) > 0 THEN COALESCE(${wordCountAlias}.filteredWordsSeen, 0) + WHEN EXISTS ( + SELECT 1 FROM imm_subtitle_lines display_line + WHERE display_line.session_id = ${sessionAlias}.session_id + ) THEN ( + SELECT COALESCE(SUM(display_occurrence.occurrence_count), 0) + FROM imm_subtitle_lines display_line + JOIN imm_word_line_occurrences display_occurrence + ON display_occurrence.line_id = display_line.line_id + WHERE display_line.session_id = ${sessionAlias}.session_id + AND display_occurrence.word_id NOT IN (${HIDDEN_WORD_IDS_SQL}) + ) ELSE COALESCE(${rawTokensExpr}, 0) END`; } +/** + * `SUM(sessionDisplayWordsExpr(s))` over every ended session, as one scalar + * expression that avoids walking each session's lines: all visible + * occurrences, minus those on lines outside ended sessions, plus the raw token + * counter of ended sessions without persisted lines. Relies on occurrences + * cascading with their line (foreign keys are on for every connection), so + * every occurrence belongs to an existing line. + */ +export const ENDED_SESSIONS_DISPLAY_WORDS_SQL = `( + (SELECT COALESCE(SUM(occurrence_count), 0) FROM imm_word_line_occurrences) + - ( + SELECT COALESCE(SUM(hidden_occurrence.occurrence_count), 0) + FROM imm_word_line_occurrences hidden_occurrence + WHERE hidden_occurrence.word_id IN (${HIDDEN_WORD_IDS_SQL}) + ) + - ( + SELECT COALESCE(SUM(open_occurrence.occurrence_count), 0) + FROM imm_subtitle_lines open_line + JOIN imm_word_line_occurrences open_occurrence ON open_occurrence.line_id = open_line.line_id + WHERE ( + open_line.session_id IS NULL + OR open_line.session_id NOT IN ( + SELECT ended.session_id FROM imm_sessions ended WHERE ended.ended_at_ms IS NOT NULL + ) + ) + AND open_occurrence.word_id NOT IN (${HIDDEN_WORD_IDS_SQL}) + ) + + ( + SELECT COALESCE(SUM(COALESCE(lineless.tokens_seen, 0)), 0) + FROM imm_sessions lineless + WHERE lineless.ended_at_ms IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM imm_subtitle_lines lineless_line + WHERE lineless_line.session_id = lineless.session_id + ) + ) +)`; + export function makePlaceholders(values: number[]): string { return values.map(() => '?').join(','); } diff --git a/src/core/services/immersion-tracker/query-trends.ts b/src/core/services/immersion-tracker/query-trends.ts index dfa3c3e2..a9ffecf6 100644 --- a/src/core/services/immersion-tracker/query-trends.ts +++ b/src/core/services/immersion-tracker/query-trends.ts @@ -3,16 +3,22 @@ import type { ImmersionSessionRollupRow } from './types'; import { ACTIVE_SESSION_METRICS_CTE, currentDbTimestamp, - getLocalDayOfWeek, getLocalEpochDay, - getLocalHourOfDay, getLocalMonthKey, getShiftedLocalDayTimestamp, makePlaceholders, sessionDisplayWordsExpr, toDbTimestamp, } from './query-shared'; -import { getDailyRollups, getMonthlyRollups } from './query-sessions'; +import { + addRollupWordCount, + applyRollupWordCounts, + selectDailyRollupRows, + selectMonthlyRollupRows, + sessionRollupDaySql, + sessionRollupMonthSql, + type RollupWordCounts, +} from './query-sessions'; import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups'; type TrendRange = '7d' | '30d' | '90d' | '365d' | 'all'; @@ -43,7 +49,6 @@ export interface LibrarySummaryRow { } interface TrendSessionMetricRow { - startedAtMs: number; epochDay: number; monthKey: number; dayOfWeek: number; @@ -78,7 +83,8 @@ export interface TrendsDashboardQueryResult { cardsPerHour: TrendChartPoint[]; readingSpeed: TrendChartPoint[]; }; - animeCumulative: { + /** Sparse per-title values for each active day; the stats UI builds the running totals. */ + animeDaily: { watchTime: TrendPerAnimePoint[]; episodes: TrendPerAnimePoint[]; cards: TrendPerAnimePoint[]; @@ -387,41 +393,6 @@ function buildLookupsPerHundredWords( return fillAxisPoints(axis, ratioByBucket); } -function buildCumulativePerAnime(points: TrendPerAnimePoint[]): TrendPerAnimePoint[] { - const byAnime = new Map>(); - const allDays = new Set(); - - for (const point of points) { - const dayMap = byAnime.get(point.animeTitle) ?? new Map(); - dayMap.set(point.epochDay, (dayMap.get(point.epochDay) ?? 0) + point.value); - byAnime.set(point.animeTitle, dayMap); - allDays.add(point.epochDay); - } - - const sortedDays = [...allDays].sort((left, right) => left - right); - if (sortedDays.length === 0) { - return []; - } - - const minDay = sortedDays[0]!; - const maxDay = sortedDays[sortedDays.length - 1]!; - const result: TrendPerAnimePoint[] = []; - - for (const [animeTitle, dayMap] of byAnime) { - const firstDay = Math.min(...dayMap.keys()); - let cumulative = 0; - for (let epochDay = minDay; epochDay <= maxDay; epochDay += 1) { - if (epochDay < firstDay) { - continue; - } - cumulative += dayMap.get(epochDay) ?? 0; - result.push({ epochDay, animeTitle, value: cumulative }); - } - } - - return result; -} - function buildLibrarySummary( rollups: ImmersionSessionRollupRow[], sessions: TrendSessionMetricRow[], @@ -616,44 +587,64 @@ function buildEpisodesSeriesFromRollups( return fillAxisPoints(axis, counts); } -function getTrendSessionMetrics( +interface TrendSessionRow extends TrendSessionMetricRow { + ended: number; + inTrendRange: number; +} + +/** + * One pass over the sessions the dashboard needs. Rows inside the trend range + * feed the session-metric charts; ended rows also supply the rollup word counts, + * so the per-session word count is computed once per request instead of once + * per rollup query. `minRollupDay` / `minRollupMonth` widen the scan to the + * oldest rollup bucket shown. + */ +function getTrendSessionRows( db: DatabaseSync, cutoffMs: string | null, -): TrendSessionMetricRow[] { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); - const whereClause = cutoffMs === null ? '' : 'WHERE s.started_at_ms >= ?'; + minRollupDay: number | null, + minRollupMonth: number | null, +): TrendSessionRow[] { + const wordsExpr = sessionDisplayWordsExpr('s', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); const cutoffValue = cutoffMs === null ? null : toDbTimestamp(cutoffMs); + const rangeSql = cutoffValue === null ? '1' : 's.started_at_ms >= ?'; + const dayValue = minRollupDay ?? Number.MAX_SAFE_INTEGER; + const monthValue = minRollupMonth ?? Number.MAX_SAFE_INTEGER; const prepared = db.prepare(` ${ACTIVE_SESSION_METRICS_CTE} SELECT - s.started_at_ms AS startedAtMs, s.video_id AS videoId, v.canonical_title AS canonicalTitle, a.canonical_title AS animeTitle, COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0) AS activeWatchedMs, ${wordsExpr} AS tokensSeen, COALESCE(asm.cardsMined, s.cards_mined, 0) AS cardsMined, - COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0) AS yomitanLookupCount + COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0) AS yomitanLookupCount, + CASE WHEN s.ended_at_ms IS NOT NULL THEN 1 ELSE 0 END AS ended, + CASE WHEN ${rangeSql} THEN 1 ELSE 0 END AS inTrendRange, + ${sessionRollupDaySql('s')} AS epochDay, + ${sessionRollupMonthSql('s')} AS monthKey, + CAST(strftime('%w', CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) AS dayOfWeek, + CAST(strftime('%H', CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) AS hourOfDay FROM imm_sessions s LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id LEFT JOIN imm_videos v ON v.video_id = s.video_id LEFT JOIN imm_anime a ON a.anime_id = v.anime_id - ${whereClause} + WHERE ${rangeSql} + OR (s.ended_at_ms IS NOT NULL AND ${sessionRollupDaySql('s')} >= ?) + OR (s.ended_at_ms IS NOT NULL AND ${sessionRollupMonthSql('s')} >= ?) ORDER BY s.started_at_ms ASC `); - const rows = (cutoffValue === null ? prepared.all() : prepared.all(cutoffValue)) as Array< - TrendSessionMetricRow & { startedAtMs: number | string } - >; - return rows.map((row) => ({ - ...row, - startedAtMs: 0, - epochDay: getLocalEpochDay(db, row.startedAtMs), - monthKey: getLocalMonthKey(db, row.startedAtMs), - dayOfWeek: getLocalDayOfWeek(db, row.startedAtMs), - hourOfDay: getLocalHourOfDay(db, row.startedAtMs), - })); + const params = + cutoffValue === null + ? [dayValue, monthValue] + : [cutoffValue, cutoffValue, dayValue, monthValue]; + return prepared.all(...params) as unknown as TrendSessionRow[]; +} + +function minRollupBucket(rows: Array<{ rollupDayOrMonth: number }>): number | null { + return rows.length === 0 ? null : Math.min(...rows.map((row) => row.rollupDayOrMonth)); } function buildNewWordsPerDay( @@ -750,8 +741,24 @@ export function getTrendsDashboard( const monthlyLimit = getTrendMonthlyLimit(db, range); const cutoffMs = getTrendCutoffMs(db, range); const useMonthlyBuckets = groupBy === 'month'; - const dailyRollups = getDailyRollups(db, dayLimit); - const monthlyRollups = getMonthlyRollups(db, monthlyLimit); + const rawDailyRollups = selectDailyRollupRows(db, dayLimit); + const rawMonthlyRollups = selectMonthlyRollupRows(db, monthlyLimit); + const sessionRows = getTrendSessionRows( + db, + cutoffMs, + minRollupBucket(rawDailyRollups), + minRollupBucket(rawMonthlyRollups), + ); + const dailyWordCounts: RollupWordCounts = new Map(); + const monthlyWordCounts: RollupWordCounts = new Map(); + for (const row of sessionRows) { + if (row.ended !== 1) continue; + addRollupWordCount(dailyWordCounts, row.epochDay, row.videoId, row.tokensSeen); + addRollupWordCount(monthlyWordCounts, row.monthKey, row.videoId, row.tokensSeen); + } + const dailyRollups = applyRollupWordCounts(rawDailyRollups, dailyWordCounts); + const monthlyRollups = applyRollupWordCounts(rawMonthlyRollups, monthlyWordCounts); + const sessions = sessionRows.filter((row) => row.inTrendRange === 1); // A null axis makes the builders fall back to only the buckets present in the // data; the contiguous axis zero-fills every calendar bucket in the window. const bucketAxis = fillEmptyBuckets @@ -759,7 +766,6 @@ export function getTrendsDashboard( : null; const chartRollups = useMonthlyBuckets ? monthlyRollups : dailyRollups; - const sessions = getTrendSessionMetrics(db, cutoffMs); const titlesByVideoId = getVideoAnimeTitleMap( db, dailyRollups.map((rollup) => rollup.videoId), @@ -820,12 +826,7 @@ export function getTrendsDashboard( cardsPerHour: efficiency.cardsPerHour, readingSpeed: efficiency.readingSpeed, }, - animeCumulative: { - watchTime: buildCumulativePerAnime(animePerDay.watchTime), - episodes: buildCumulativePerAnime(animePerDay.episodes), - cards: buildCumulativePerAnime(animePerDay.cards), - words: buildCumulativePerAnime(animePerDay.words), - }, + animeDaily: animePerDay, patterns: { watchTimeByDayOfWeek: buildWatchTimeByDayOfWeek(sessions), watchTimeByHour: buildWatchTimeByHour(sessions), diff --git a/src/core/services/immersion-tracker/storage.ts b/src/core/services/immersion-tracker/storage.ts index 493d945f..7946f7c8 100644 --- a/src/core/services/immersion-tracker/storage.ts +++ b/src/core/services/immersion-tracker/storage.ts @@ -1849,14 +1849,24 @@ function incrementWordAggregate( occurrence.frequencyRank ?? null, ); } - const row = stmts.wordSelectStmt.get(occurrence.headword, occurrence.word, occurrence.reading) as + return syncStoredWordVisibility(stmts, occurrence); +} + +/** + * Persists visibility for a word that was just upserted and returns its id. The + * upsert keeps the first-seen POS metadata, so visibility comes from the stored + * row (as the vocabulary summary expects) rather than the incoming token. + */ +function syncStoredWordVisibility( + stmts: TrackerPreparedStatements, + key: Pick, 'headword' | 'word' | 'reading'>, +): number { + const row = stmts.wordSelectStmt.get(key.headword, key.word, key.reading) as | (VocabularyVisibilityRow & { id: number; vocabularyVisible: number }) | null; if (!row?.id) { - throw new Error(`Failed to resolve imm_words id for ${occurrence.headword}`); + throw new Error(`Failed to resolve imm_words id for ${key.headword}`); } - // The upsert keeps the first-seen POS metadata, so derive visibility from the - // stored row (as the vocabulary summary does) rather than the incoming token. const visible = isVocabularyStatsRowVisible(row) ? 1 : 0; if (row.vocabularyVisible !== visible) { stmts.wordVisibilityUpdateStmt.run(visible, row.id); @@ -1939,6 +1949,7 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta write.lastSeen, write.frequencyRank ?? null, ); + syncStoredWordVisibility(stmts, write); return; } if (write.kind === 'kanji') { diff --git a/src/types/stats-wire.ts b/src/types/stats-wire.ts index 4875085f..e3b9e992 100644 --- a/src/types/stats-wire.ts +++ b/src/types/stats-wire.ts @@ -384,7 +384,8 @@ export interface TrendsDashboardData { readingSpeed: TrendChartPoint[]; }; librarySummary: LibrarySummaryRow[]; - animeCumulative: { + /** Sparse per-title values for each active day; the client accumulates them for charts. */ + animeDaily: { watchTime: TrendPerAnimePoint[]; episodes: TrendPerAnimePoint[]; cards: TrendPerAnimePoint[]; diff --git a/stats/src/components/trends/TrendsTab.tsx b/stats/src/components/trends/TrendsTab.tsx index bf7e629c..e3981d4a 100644 --- a/stats/src/components/trends/TrendsTab.tsx +++ b/stats/src/components/trends/TrendsTab.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useMemo, useState } from 'react'; import { useTrends, type TimeRange, type GroupBy } from '../../hooks/useTrends'; import { DateRangeSelector } from './DateRangeSelector'; import { TrendChart } from './TrendChart'; @@ -20,6 +20,7 @@ import { type MaxTitlesMode, } from './anime-visibility'; import { LibrarySummarySection } from './LibrarySummarySection'; +import { buildAnimeCumulative } from './anime-cumulative'; const MAX_TITLES_MODE_LABELS: Record = { recent: 'most recent', @@ -156,6 +157,10 @@ export function TrendsTab() { const [maxTitles, setMaxTitles] = useState(() => loadMaxTitles()); const [maxTitlesMode, setMaxTitlesMode] = useState(() => loadMaxTitlesMode()); const { data, loading, error } = useTrends(range, groupBy, showEmptyDays); + const animeCumulative = useMemo( + () => (data ? buildAnimeCumulative(data.animeDaily) : null), + [data], + ); const updateHiddenAnime = (next: Set) => { setHiddenAnime(next); @@ -187,30 +192,21 @@ export function TrendsTab() { if (loading) return
Loading...
; if (error) return
Error: {error}
; - if (!data) return null; + if (!data || !animeCumulative) return null; const animeTitles = buildAnimeVisibilityOptions([ - data.animeCumulative.episodes, - data.animeCumulative.cards, - data.animeCumulative.words, - data.animeCumulative.watchTime, + animeCumulative.episodes, + animeCumulative.cards, + animeCumulative.words, + animeCumulative.watchTime, ]); const activeHiddenAnime = pruneHiddenAnime(hiddenAnime, animeTitles); - const filteredAnimeProgress = filterHiddenAnimeData( - data.animeCumulative.episodes, - activeHiddenAnime, - ); - const filteredCardsProgress = filterHiddenAnimeData( - data.animeCumulative.cards, - activeHiddenAnime, - ); - const filteredWordsProgress = filterHiddenAnimeData( - data.animeCumulative.words, - activeHiddenAnime, - ); + const filteredAnimeProgress = filterHiddenAnimeData(animeCumulative.episodes, activeHiddenAnime); + const filteredCardsProgress = filterHiddenAnimeData(animeCumulative.cards, activeHiddenAnime); + const filteredWordsProgress = filterHiddenAnimeData(animeCumulative.words, activeHiddenAnime); const filteredWatchTimeProgress = filterHiddenAnimeData( - data.animeCumulative.watchTime, + animeCumulative.watchTime, activeHiddenAnime, ); diff --git a/stats/src/components/trends/anime-cumulative.test.ts b/stats/src/components/trends/anime-cumulative.test.ts new file mode 100644 index 00000000..32f4c5f6 --- /dev/null +++ b/stats/src/components/trends/anime-cumulative.test.ts @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { buildCumulativePerAnime } from './anime-cumulative'; + +test('buildCumulativePerAnime carries each title forward to the last active day', () => { + const result = buildCumulativePerAnime([ + { epochDay: 103, animeTitle: 'B', value: 4 }, + { epochDay: 100, animeTitle: 'A', value: 10 }, + { epochDay: 102, animeTitle: 'A', value: 5 }, + { epochDay: 102, animeTitle: 'A', value: 1 }, + ]); + + assert.deepEqual( + result + .filter((point) => point.animeTitle === 'A') + .map((point) => [point.epochDay, point.value]), + [ + [100, 10], + [101, 10], + [102, 16], + [103, 16], + ], + ); + // A title starts at its own first active day, not the earliest day overall. + assert.deepEqual( + result + .filter((point) => point.animeTitle === 'B') + .map((point) => [point.epochDay, point.value]), + [[103, 4]], + ); +}); + +test('buildCumulativePerAnime returns nothing for no points', () => { + assert.deepEqual(buildCumulativePerAnime([]), []); +}); diff --git a/stats/src/components/trends/anime-cumulative.ts b/stats/src/components/trends/anime-cumulative.ts new file mode 100644 index 00000000..d13a0661 --- /dev/null +++ b/stats/src/components/trends/anime-cumulative.ts @@ -0,0 +1,41 @@ +import type { TrendsDashboardData } from '../../types/stats'; +import type { PerAnimeDataPoint } from './StackedTrendChart'; + +type AnimeSeries = TrendsDashboardData['animeDaily']; + +/** + * Turns sparse per-title daily values into running totals: one point per + * title for every calendar day from its first active day to the last active + * day of any title, so the stacked area charts carry each total forward. + */ +export function buildCumulativePerAnime(points: PerAnimeDataPoint[]): PerAnimeDataPoint[] { + const byAnime = new Map>(); + let maxDay = Number.NEGATIVE_INFINITY; + + for (const point of points) { + const dayMap = byAnime.get(point.animeTitle) ?? new Map(); + dayMap.set(point.epochDay, (dayMap.get(point.epochDay) ?? 0) + point.value); + byAnime.set(point.animeTitle, dayMap); + maxDay = Math.max(maxDay, point.epochDay); + } + + const result: PerAnimeDataPoint[] = []; + for (const [animeTitle, dayMap] of byAnime) { + const firstDay = Math.min(...dayMap.keys()); + let cumulative = 0; + for (let epochDay = firstDay; epochDay <= maxDay; epochDay += 1) { + cumulative += dayMap.get(epochDay) ?? 0; + result.push({ epochDay, animeTitle, value: cumulative }); + } + } + return result; +} + +export function buildAnimeCumulative(daily: AnimeSeries): AnimeSeries { + return { + watchTime: buildCumulativePerAnime(daily.watchTime), + episodes: buildCumulativePerAnime(daily.episodes), + cards: buildCumulativePerAnime(daily.cards), + words: buildCumulativePerAnime(daily.words), + }; +} diff --git a/stats/src/lib/api-client.test.ts b/stats/src/lib/api-client.test.ts index 64069b06..b728a89b 100644 --- a/stats/src/lib/api-client.test.ts +++ b/stats/src/lib/api-client.test.ts @@ -105,7 +105,7 @@ test('getTrendsDashboard accepts 365d range and builds correct URL', async () => }, ratios: { lookupsPerHundred: [] }, librarySummary: [], - animeCumulative: { watchTime: [], episodes: [], cards: [], words: [] }, + animeDaily: { watchTime: [], episodes: [], cards: [], words: [] }, patterns: { watchTimeByDayOfWeek: [], watchTimeByHour: [] }, }), async (requests) => {