perf(stats): speed up dashboard queries on large databases (#322)

changes/stats-dashboard-query-speed.md

@@ -0,0 +1,8 @@
+type: fixed
+area: stats
+
+- The stats dashboard loads much faster on large databases. Overview, media and library detail, and trends no longer re-count every recorded word in the whole database on each request, so these views only cost as much as the sessions they show. On a ~1 GB database the overview went from ~1.4 s to ~0.14 s, media detail from ~1.5 s to ~0.08 s, the 30-day trends from ~1.5 s to ~0.06 s, and all-time trends from ~1.6 s to ~0.2 s.
+- The Vocabulary tab's summary and word list load several times faster (summary ~2 s to ~0.2 s, word list ~0.6 s to ~0.2 s).
+- The Trends tab downloads far less data: the per-title library charts now arrive as daily values and are totalled in the dashboard, so the all-time response shrank from ~3.9 MB to ~0.2 MB.
+- Sentence search with headword matching returns several times faster (~0.3 s to ~0.08 s per query).
+- Vocabulary totals and new-word charts stay correct after running vocabulary cleanup. Words it repaired could be counted or hidden by mistake; the first launch after updating corrects them in the background.

docs/architecture/stats-trends-data-flow.md

@@ -17,11 +17,12 @@ Trend charts now consume one chart-oriented backend payload from `/api/stats/tre
 - rollup-backed:
   - activity charts
   - cumulative watch/cards/tokens/sessions trends
-  - per-anime watch/cards/tokens/episodes series
+  - per-anime watch/cards/tokens/episodes series (`animeDaily`: sparse per-title values for active days only; `TrendsTab` builds the running totals with `buildAnimeCumulative`, because a dense title x day payload grew to several MB)
 - session-metric-backed:
   - lookup trends
   - lookup rate trends
   - watch-time by day-of-week/hour
+- rollup word counts and session metrics share one session query per request (`getTrendSessionRows`), so the per-session display-word count runs once instead of once per rollup query
 - vocabulary-backed:
   - new-words trend reads permanent daily lexical rollups
   - rollup rows count only vocabulary-visible tokens and normalize mixed legacy timestamp units
@@ -39,6 +40,6 @@ Stats data crosses the process boundary over HTTP only. Shared JSON models live
 
 The stats preload bridge remains limited to native window behavior such as confirmation-dialog layering. Do not add stats data request channels back to Electron IPC.
 
-The stats UI should treat the trends payload as chart-ready data. Presentation-only work in the client is fine, but rebuilding the main trend datasets from raw sessions should stay out of the render path.
+The stats UI should treat the trends payload as chart-ready data. Presentation-only work in the client is fine (including the per-anime running totals, memoized per payload), but rebuilding the main trend datasets from raw sessions should stay out of the render path.
 
 For session detail timelines, omitting `limit` now means "return the full retained session telemetry/history". Explicit `limit` remains available for bounded callers, but the default stats UI path should not trim long sessions to the newest 200 samples.

src/core/services/__tests__/stats-server.test.ts

@@ -189,7 +189,7 @@ const TRENDS_DASHBOARD = {
       lastWatched: 20_000,
     },
   ],
-  animeCumulative: {
+  animeDaily: {
     watchTime: [{ epochDay: 20_000, animeTitle: 'Little Witch Academia', value: 25 }],
     episodes: [{ epochDay: 20_000, animeTitle: 'Little Witch Academia', value: 1 }],
     cards: [{ epochDay: 20_000, animeTitle: 'Little Witch Academia', value: 5 }],

src/core/services/immersion-tracker/__tests__/query.test.ts

@@ -745,7 +745,10 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => {
     assert.equal(dashboard.ratios.readingSpeed[0]?.value, +(120 / 30).toFixed(1));
     assert.equal(dashboard.ratios.readingSpeed[1]?.value, +(140 / 45).toFixed(1));
     assert.equal(dashboard.librarySummary[0]?.title, 'Trend Dashboard Anime');
-    assert.equal(dashboard.animeCumulative.watchTime[1]?.value, 75);
+    assert.deepEqual(
+      dashboard.animeDaily.watchTime.map((point) => point.value).sort((a, b) => a - b),
+      [30, 45],
+    );
     assert.equal(
       dashboard.patterns.watchTimeByDayOfWeek.reduce((sum, point) => sum + point.value, 0),
       75,
@@ -819,7 +822,7 @@ test('getTrendsDashboard redacts legacy Jellyfin stream titles', () => {
 
     const dashboard = getTrendsDashboard(db, 'all', 'day');
     const titles = [
-      ...dashboard.animeCumulative.watchTime.map((point) => point.animeTitle),
+      ...dashboard.animeDaily.watchTime.map((point) => point.animeTitle),
       ...dashboard.librarySummary.map((row) => row.title),
     ];
 
@@ -1521,6 +1524,8 @@ test('word-count read models use filtered persisted occurrences with raw fallbac
     const startedAtMs = 1_700_000_000_000;
     const withOccurrences = startSessionRecord(db, videoId, startedAtMs);
     const fallbackOnly = startSessionRecord(db, videoId, startedAtMs + 60_000);
+    // Still running, so its words stay out of the all-time total.
+    const stillActive = startSessionRecord(db, videoId, startedAtMs + 120_000);
 
     db.prepare(
       `
@@ -1543,6 +1548,14 @@ test('word-count read models use filtered persisted occurrences with raw fallbac
       occurrenceCount: 2,
       startedAtMs,
     });
+    insertFilteredWordOccurrence(db, {
+      sessionId: stillActive.sessionId,
+      videoId,
+      lineIndex: 3,
+      occurrenceCount: 4,
+      startedAtMs: startedAtMs + 120_000,
+      headword: '犬',
+    });
     insertFilteredWordOccurrence(db, {
       sessionId: withOccurrences.sessionId,
       videoId,
@@ -5075,7 +5088,10 @@ test('getTrendsDashboard rollup word metrics keep persisted totals over partial
     assert.equal(dashboard.ratios.lookupsPerHundred[0]?.value, 200);
     assert.equal(dashboard.librarySummary[0]?.words, 30);
     assert.equal(dashboard.librarySummary[0]?.lookupsPerHundred, 33.3);
-    assert.equal(dashboard.animeCumulative.words.at(-1)?.value, 30);
+    assert.equal(
+      dashboard.animeDaily.words.reduce((sum, point) => sum + point.value, 0),
+      30,
+    );
   } finally {
     db.close();
     cleanupDbPath(dbPath);

src/core/services/immersion-tracker/lexical-rollups.test.ts

@@ -14,9 +14,11 @@ import {
   getVocabularySummary,
   replaceStatsExcludedWords,
 } from './query-lexical';
+import { cleanupVocabularyStats } from './query-maintenance';
 import { Database } from './sqlite';
 import type { DatabaseSync } from './sqlite';
 import { createTrackerPreparedStatements, ensureSchema, executeQueuedWrite } from './storage';
+import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility';
 import { BASE_MS, createDb, seedEndedSession, seedVideo } from './__tests__/lifetime-test-fixtures';
 
 function makeDbPath(): string {
@@ -384,7 +386,7 @@ test('current lexical rollup readiness accepts legacy integer state storage', ()
         state_value INTEGER NOT NULL
       );
       INSERT INTO imm_rollup_state(state_key, state_value)
-      VALUES ('lexical_daily_rollups_version', 3);
+      VALUES ('lexical_daily_rollups_version', 4);
     `);
 
     assert.equal(areLexicalDailyRollupsReady(db), true);
@@ -394,6 +396,185 @@ test('current lexical rollup readiness accepts legacy integer state storage', ()
   }
 });
 
+function insertWord(
+  db: DatabaseSync,
+  word: {
+    headword: string;
+    word: string;
+    reading: string;
+    partOfSpeech: string;
+    pos1: string;
+    pos2: string;
+  },
+  firstSeen: number,
+): number {
+  const result = db
+    .prepare(
+      `INSERT INTO imm_words(
+         headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency
+       ) VALUES (?, ?, ?, ?, ?, ?, '', ?, ?, 1)`,
+    )
+    .run(
+      word.headword,
+      word.word,
+      word.reading,
+      word.partOfSpeech,
+      word.pos1,
+      word.pos2,
+      firstSeen,
+      firstSeen,
+    );
+  return Number(result.lastInsertRowid);
+}
+
+function assertPersistedVisibilityMatchesPredicate(db: DatabaseSync): void {
+  const rows = db
+    .prepare(
+      `SELECT headword, word, reading, part_of_speech AS partOfSpeech, pos1, pos2, pos3,
+         frequency_rank AS frequencyRank, vocabulary_visible AS vocabularyVisible
+       FROM imm_words`,
+    )
+    .all() as Array<VocabularyVisibilityRow & { vocabularyVisible: number }>;
+  assert.ok(rows.length > 0);
+  assert.deepEqual(
+    rows.map((row) => [row.headword, row.vocabularyVisible === 1]),
+    rows.map((row) => [row.headword, isVocabularyStatsRowVisible(row)]),
+  );
+}
+
+test('cleanupVocabularyStats re-derives visibility for a repaired word', async () => {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    const epochDay = 19_500;
+    const firstSeen = epochDay * 86_400 + 43_200;
+    const legacyWord = {
+      headword: '旧',
+      word: '旧',
+      reading: '',
+      partOfSpeech: 'particle',
+      pos1: '助詞',
+      pos2: '係助詞',
+    };
+    const legacyId = insertWord(db, legacyWord, firstSeen);
+    rebuildLexicalDailyRollups(db);
+    assert.equal(getVocabularySummary(db, null).uniqueWords, 0);
+
+    await cleanupVocabularyStats(db, {
+      resolveLegacyPos: async (row) =>
+        row.id === legacyId
+          ? {
+              headword: '旧',
+              reading: 'きゅう',
+              partOfSpeech: 'noun',
+              pos1: '名詞',
+              pos2: '一般',
+              pos3: '',
+            }
+          : null,
+    });
+
+    assertPersistedVisibilityMatchesPredicate(db);
+    assert.equal(getVocabularySummary(db, null).uniqueWords, 1);
+    assert.equal(getLexicalDailyRollups(db)[0]?.wordCount, 1);
+  } finally {
+    db.close();
+    fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+  }
+});
+
+test('cleanupVocabularyStats re-derives visibility for a merged duplicate', async () => {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    const firstSeen = 19_500 * 86_400 + 43_200;
+    // The merge fills these blank POS fields from the repair, which hides the word.
+    insertWord(
+      db,
+      {
+        headword: '知る',
+        word: '知っている',
+        reading: 'しっている',
+        partOfSpeech: 'noun',
+        pos1: '',
+        pos2: '',
+      },
+      firstSeen,
+    );
+    const legacyId = insertWord(
+      db,
+      {
+        headword: '知っている',
+        word: '知っている',
+        reading: '',
+        partOfSpeech: 'verb',
+        pos1: '動詞',
+        pos2: '自立',
+      },
+      firstSeen,
+    );
+    rebuildLexicalDailyRollups(db);
+    assert.equal(getVocabularySummary(db, null).uniqueWords, 2);
+
+    await cleanupVocabularyStats(db, {
+      resolveLegacyPos: async (row) =>
+        row.id === legacyId
+          ? {
+              headword: '知る',
+              reading: 'しっている',
+              partOfSpeech: 'particle',
+              pos1: '助詞',
+              pos2: '係助詞',
+              pos3: '',
+            }
+          : null,
+    });
+
+    assertPersistedVisibilityMatchesPredicate(db);
+    assert.equal(getVocabularySummary(db, null).uniqueWords, 0);
+    assert.deepEqual(getLexicalDailyRollups(db), []);
+  } finally {
+    db.close();
+    fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+  }
+});
+
+test('a queued word write persists visibility from the stored row', () => {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    const firstSeen = 19_500 * 86_400 + 43_200;
+    executeQueuedWrite(
+      {
+        kind: 'word',
+        headword: 'は',
+        word: 'は',
+        reading: 'は',
+        partOfSpeech: 'particle',
+        pos1: '助詞',
+        pos2: '係助詞',
+        pos3: '',
+        firstSeen,
+        lastSeen: firstSeen,
+        frequencyRank: null,
+      },
+      createTrackerPreparedStatements(db),
+    );
+
+    assertPersistedVisibilityMatchesPredicate(db);
+    assert.equal(getVocabularySummary(db, null).uniqueWords, 0);
+  } finally {
+    db.close();
+    fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+  }
+});
+
 test('imm_words persists vocabulary visibility for rollup maintenance', () => {
   const dbPath = makeDbPath();
   const db = new Database(dbPath);

src/core/services/immersion-tracker/lexical-rollups.ts

@@ -19,8 +19,9 @@ const LOCAL_EPOCH_DAY_SQL = `
 `;
 
 // v3 re-derives visibility for rows a re-seen word forced visible and repairs
-// last_seen values left behind by session deletes.
-const LEXICAL_DAILY_ROLLUP_VERSION = '3';
+// last_seen values left behind by session deletes. v4 re-derives visibility for
+// rows vocabulary cleanup repaired without updating it.
+const LEXICAL_DAILY_ROLLUP_VERSION = '4';
 const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
 const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
 

src/core/services/immersion-tracker/query-lexical.ts

@@ -16,7 +16,7 @@ import type {
   WordDetailRow,
   WordOccurrenceRow,
 } from './types';
-import { fromDbTimestamp, toDbTimestamp } from './query-shared';
+import { forEachIdChunk, fromDbTimestamp, makePlaceholders, toDbTimestamp } from './query-shared';
 import { nowMs } from './time';
 import { narrowFullwidthKana, widenHalfwidthKana } from '../../utils/halfwidth-kana';
 import {
@@ -106,42 +106,58 @@ export function getVocabularyStats(
   const whereClause = hasExclude
     ? `WHERE (part_of_speech IS NULL OR part_of_speech NOT IN (${placeholders}))`
     : '';
-  // The page is selected before the join so `animeCount` is only computed for the
-  // rows being returned. Aggregating first made every request walk each word's
-  // entire occurrence history — seconds of blocked event loop on a large library,
-  // because only the ordering, not the aggregate, decides which rows survive.
+  // Rows are paged and filtered before `animeCount` is computed, and only the
+  // rows being returned are counted. The count walks each word's entire
+  // occurrence history, and the most frequent words hold most occurrences, so
+  // counting the oversampled page or aggregating first cost seconds of blocked
+  // event loop on a large library.
   const stmt = db.prepare(`
-    WITH page AS (
-      SELECT id, headword, word, reading, part_of_speech, pos1, pos2, pos3,
-             frequency, frequency_rank, first_seen, last_seen
-      FROM imm_words
-      ${whereClause}
-      ORDER BY frequency DESC, id
-      LIMIT ? OFFSET ?
-    )
-    SELECT p.id AS wordId, p.headword, p.word, p.reading,
-      p.part_of_speech AS partOfSpeech, p.pos1, p.pos2, p.pos3,
-      p.frequency, p.frequency_rank AS frequencyRank,
-      p.first_seen AS firstSeen, p.last_seen AS lastSeen,
-      COUNT(DISTINCT sl.anime_id) AS animeCount
-    FROM page p
-    LEFT JOIN imm_word_line_occurrences o ON o.word_id = p.id
-    LEFT JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id AND sl.anime_id IS NOT NULL
-    GROUP BY p.id
-    ORDER BY p.frequency DESC, p.id
+    SELECT id AS wordId, headword, word, reading,
+      part_of_speech AS partOfSpeech, pos1, pos2, pos3,
+      frequency, frequency_rank AS frequencyRank,
+      first_seen AS firstSeen, last_seen AS lastSeen
+    FROM imm_words
+    ${whereClause}
+    ORDER BY frequency DESC, id
+    LIMIT ? OFFSET ?
   `);
-  const visibleRows: VocabularyStatsRow[] = [];
+  const visibleRows: Array<Omit<VocabularyStatsRow, 'animeCount'>> = [];
   let offset = 0;
 
   while (visibleRows.length < limit) {
     const params = hasExclude ? [...excludePos, pageSize, offset] : [pageSize, offset];
-    const page = stmt.all(...params) as VocabularyStatsRow[];
+    const page = stmt.all(...params) as Array<Omit<VocabularyStatsRow, 'animeCount'>>;
     if (page.length === 0) break;
     visibleRows.push(...page.filter(isVocabularyStatsRowVisible));
     offset += page.length;
   }
 
-  return visibleRows.slice(0, limit);
+  const rows = visibleRows.slice(0, limit);
+  const animeCounts = getWordAnimeCounts(
+    db,
+    rows.map((row) => row.wordId),
+  );
+  return rows.map((row) => ({ ...row, animeCount: animeCounts.get(row.wordId) ?? 0 }));
+}
+
+/** Number of distinct anime each word appears in, keyed by word id. */
+function getWordAnimeCounts(db: DatabaseSync, wordIds: number[]): Map<number, number> {
+  const counts = new Map<number, number>();
+  forEachIdChunk(wordIds, (chunk) => {
+    const rows = db
+      .prepare(
+        `
+        SELECT o.word_id AS wordId, COUNT(DISTINCT sl.anime_id) AS animeCount
+        FROM imm_word_line_occurrences o
+        JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
+        WHERE o.word_id IN (${makePlaceholders(chunk)}) AND sl.anime_id IS NOT NULL
+        GROUP BY o.word_id
+      `,
+      )
+      .all(...chunk) as Array<{ wordId: number; animeCount: number }>;
+    for (const row of rows) counts.set(row.wordId, row.animeCount);
+  });
+  return counts;
 }
 
 /**
@@ -276,8 +292,12 @@ export function getVocabularySummary(
   nowMs: number = Date.now(),
   scanBatchSize: number = VOCABULARY_SUMMARY_SCAN_BATCH_SIZE,
 ): VocabularyStatsSummary {
-  // Visibility and exclusion rules live in JS, so rows are scanned in id-keyed
-  // batches to keep memory bounded on large vocabularies.
+  // Exclusion and known-word rules live in JS, so rows are scanned in id-keyed
+  // batches to keep memory bounded on large vocabularies. Once the lexical
+  // rollups are ready, `vocabulary_visible` holds the persisted result of
+  // isVocabularyStatsRowVisible (the same column the new-word charts count), so
+  // the scan skips re-running the subtitle filter rules for every word.
+  const persistedVisibility = areLexicalDailyRollupsReady(db);
   const scanStmt = db.prepare(`
     SELECT id AS wordId, headword, word, reading,
       part_of_speech AS partOfSpeech, pos1, pos2, pos3,
@@ -285,10 +305,11 @@ export function getVocabularySummary(
       first_seen AS firstSeen, last_seen AS lastSeen,
       0 AS animeCount
     FROM imm_words
-    WHERE id > ?
+    WHERE id > ?${persistedVisibility ? ' AND vocabulary_visible = 1' : ''}
     ORDER BY id
     LIMIT ?
   `);
+  const isVisible = persistedVisibility ? () => true : isVocabularyStatsRowVisible;
   const excludedAliases = new Set(
     getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
   );
@@ -311,7 +332,7 @@ export function getVocabularySummary(
     lastId = words[words.length - 1]!.wordId;
     for (const word of words) {
       if (
-        !isVocabularyStatsRowVisible(word) ||
+        !isVisible(word) ||
         excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias))
       ) {
         continue;
@@ -489,15 +510,16 @@ export function searchSubtitleSentences(
       narrowFullwidthKana(widenedTerm),
     ]).map((variant) => `%${escapeLikeTerm(variant)}%`);
     const headwords = getHeadwordCandidatesForSentenceSearchTerm(term, options);
+    // Uncorrelated on purpose: SQLite resolves the headword's lines once through
+    // the word indexes. A per-line EXISTS probed every subtitle line (~0.2 s).
     const headwordClause =
       headwords.length > 0
         ? `
-        OR EXISTS (
-          SELECT 1
-          FROM imm_word_line_occurrences o
-          JOIN imm_words w ON w.id = o.word_id
-          WHERE o.line_id = l.line_id
-            AND w.headword IN (${headwords.map(() => '?').join(', ')})
+        OR l.line_id IN (
+          SELECT o.line_id
+          FROM imm_words w
+          JOIN imm_word_line_occurrences o ON o.word_id = w.id
+          WHERE w.headword IN (${headwords.map(() => '?').join(', ')})
         )
       `
         : '';

src/core/services/immersion-tracker/query-library.ts

@@ -18,7 +18,6 @@ import type {
 } from './types';
 import {
   ACTIVE_SESSION_METRICS_CTE,
-  SESSION_WORD_COUNTS_SELECT,
   fromDbTimestamp,
   resolvedCoverBlobExpr,
   sessionDisplayWordsExpr,
@@ -123,7 +122,7 @@ export function getAnimeAnilistEntries(db: DatabaseSync, animeId: number): Anime
 }
 
 export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisodeRow[] {
-  const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)');
+  const wordsExpr = sessionDisplayWordsExpr('s', 'COALESCE(asm.tokensSeen, s.tokens_seen)');
   const rows = db
     .prepare(
       `
@@ -184,7 +183,6 @@ export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisod
     FROM imm_videos v
     LEFT JOIN imm_sessions s ON s.video_id = v.video_id
     LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
-    LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
     WHERE v.anime_id = ?
     GROUP BY v.video_id
     ORDER BY
@@ -250,7 +248,7 @@ export function getMediaLibrary(db: DatabaseSync): MediaLibraryRow[] {
 }
 
 export function getMediaDetail(db: DatabaseSync, videoId: number): MediaDetailRow | null {
-  const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)');
+  const wordsExpr = sessionDisplayWordsExpr('s', 'COALESCE(asm.tokensSeen, s.tokens_seen)');
   return db
     .prepare(
       `
@@ -298,7 +296,6 @@ export function getMediaDetail(db: DatabaseSync, videoId: number): MediaDetailRo
     LEFT JOIN imm_youtube_videos yv ON yv.video_id = v.video_id
     LEFT JOIN imm_sessions s ON s.video_id = v.video_id
     LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
-    LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
     WHERE v.video_id = ?
       AND (lm.video_id IS NOT NULL OR s.session_id IS NOT NULL)
     GROUP BY v.video_id
@@ -312,7 +309,7 @@ export function getMediaSessions(
   videoId: number,
   limit = 100,
 ): SessionSummaryQueryRow[] {
-  const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)');
+  const wordsExpr = sessionDisplayWordsExpr('s', 'COALESCE(asm.tokensSeen, s.tokens_seen)');
   const rows = db
     .prepare(
       `
@@ -333,7 +330,6 @@ export function getMediaSessions(
       COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0) AS yomitanLookupCount
     FROM imm_sessions s
     LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
-    LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
     LEFT JOIN imm_videos v ON v.video_id = s.video_id
     WHERE s.video_id = ?
     ORDER BY s.started_at_ms DESC
@@ -358,12 +354,15 @@ export function getMediaDailyRollups(
   videoId: number,
   limit = 90,
 ): ImmersionSessionRollupRow[] {
-  const wordsExpr = sessionDisplayWordsExpr('s', 'swc');
   return db
     .prepare(
       `
-    WITH session_word_counts AS (
-      ${SESSION_WORD_COUNTS_SELECT}
+    WITH recent_days AS (
+      SELECT DISTINCT rollup_day
+      FROM imm_daily_rollups
+      WHERE video_id = ?
+      ORDER BY rollup_day DESC
+      LIMIT ?
     ),
     daily_word_counts AS (
       SELECT
@@ -372,18 +371,15 @@ export function getMediaDailyRollups(
           AS INTEGER
         ) AS rollupDay,
         s.video_id AS videoId,
-        SUM(${wordsExpr}) AS totalTokensSeen
+        SUM(${sessionDisplayWordsExpr('s')}) AS totalTokensSeen
       FROM imm_sessions s
-      LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
       WHERE s.ended_at_ms IS NOT NULL
+        AND s.video_id = ?
+        AND CAST(
+            julianday(CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5
+            AS INTEGER
+          ) >= (SELECT MIN(rollup_day) FROM recent_days)
       GROUP BY rollupDay, s.video_id
-    ),
-    recent_days AS (
-      SELECT DISTINCT rollup_day
-      FROM imm_daily_rollups
-      WHERE video_id = ?
-      ORDER BY rollup_day DESC
-      LIMIT ?
     )
     SELECT
       rollup_day AS rollupDayOrMonth,
@@ -416,7 +412,7 @@ export function getMediaDailyRollups(
     ORDER BY rollup_day DESC, video_id DESC
     `,
     )
-    .all(videoId, limit, videoId) as unknown as ImmersionSessionRollupRow[];
+    .all(videoId, limit, videoId, videoId) as unknown as ImmersionSessionRollupRow[];
 }
 
 export function getAnimeDailyRollups(
@@ -424,12 +420,16 @@ export function getAnimeDailyRollups(
   animeId: number,
   limit = 90,
 ): ImmersionSessionRollupRow[] {
-  const wordsExpr = sessionDisplayWordsExpr('s', 'swc');
   return db
     .prepare(
       `
-    WITH session_word_counts AS (
-      ${SESSION_WORD_COUNTS_SELECT}
+    WITH recent_days AS (
+      SELECT DISTINCT r.rollup_day
+      FROM imm_daily_rollups r
+      JOIN imm_videos v ON v.video_id = r.video_id
+      WHERE v.anime_id = ?
+      ORDER BY r.rollup_day DESC
+      LIMIT ?
     ),
     daily_word_counts AS (
       SELECT
@@ -438,19 +438,15 @@ export function getAnimeDailyRollups(
           AS INTEGER
         ) AS rollupDay,
         s.video_id AS videoId,
-        SUM(${wordsExpr}) AS totalTokensSeen
+        SUM(${sessionDisplayWordsExpr('s')}) AS totalTokensSeen
       FROM imm_sessions s
-      LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
       WHERE s.ended_at_ms IS NOT NULL
+        AND s.video_id IN (SELECT video_id FROM imm_videos WHERE anime_id = ?)
+        AND CAST(
+            julianday(CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5
+            AS INTEGER
+          ) >= (SELECT MIN(rollup_day) FROM recent_days)
       GROUP BY rollupDay, s.video_id
-    ),
-    recent_days AS (
-      SELECT DISTINCT r.rollup_day
-      FROM imm_daily_rollups r
-      JOIN imm_videos v ON v.video_id = r.video_id
-      WHERE v.anime_id = ?
-      ORDER BY r.rollup_day DESC
-      LIMIT ?
     )
     SELECT r.rollup_day AS rollupDayOrMonth, r.video_id AS videoId,
            r.total_sessions AS totalSessions, r.total_active_min AS totalActiveMin,
@@ -481,7 +477,7 @@ export function getAnimeDailyRollups(
     ORDER BY r.rollup_day DESC, r.video_id DESC
     `,
     )
-    .all(animeId, limit, animeId) as unknown as ImmersionSessionRollupRow[];
+    .all(animeId, limit, animeId, animeId) as unknown as ImmersionSessionRollupRow[];
 }
 
 export function getAnimeCoverArt(db: DatabaseSync, animeId: number): MediaArtRow | null {
@@ -657,7 +653,7 @@ export function getEpisodeWords(db: DatabaseSync, videoId: number, limit = 50):
 }
 
 export function getEpisodeSessions(db: DatabaseSync, videoId: number): SessionSummaryQueryRow[] {
-  const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)');
+  const wordsExpr = sessionDisplayWordsExpr('s', 'COALESCE(asm.tokensSeen, s.tokens_seen)');
   const rows = db
     .prepare(
       `
@@ -677,7 +673,6 @@ export function getEpisodeSessions(db: DatabaseSync, videoId: number): SessionSu
     FROM imm_sessions s
     JOIN imm_videos v ON v.video_id = s.video_id
     LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
-    LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
     WHERE s.video_id = ?
     ORDER BY s.started_at_ms DESC
   `,

src/core/services/immersion-tracker/query-maintenance.ts

@@ -13,6 +13,7 @@ import { deleteMaintenanceBatch } from './query-delete-maintenance';
 import { PartOfSpeech, type MergedToken } from '../../../types';
 import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
 import { deriveStoredPartOfSpeech } from '../tokenizer/part-of-speech';
+import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility';
 import {
   cleanupUnusedCoverArtBlobHash,
   findSharedCoverBlobHash,
@@ -241,6 +242,24 @@ export async function cleanupVocabularyStats(
   const deleteOccurrencesStmt = db.prepare(
     'DELETE FROM imm_word_line_occurrences WHERE word_id = ?',
   );
+  const selectVisibilityInputsStmt = db.prepare(
+    `SELECT word, headword, reading, part_of_speech AS partOfSpeech,
+       pos1, pos2, pos3, frequency_rank AS frequencyRank
+     FROM imm_words
+     WHERE id = ?`,
+  );
+  const updateVisibilityStmt = db.prepare(
+    `UPDATE imm_words SET vocabulary_visible = ? WHERE id = ? AND vocabulary_visible IS NOT ?`,
+  );
+  // Repairs rewrite the visibility predicate's inputs, and the vocabulary summary
+  // and new-word rollups read the persisted result, so re-derive it after each one.
+  const syncWordVisibility = (wordId: number): void => {
+    // The scan awaits POS resolution, so another writer may have removed the word.
+    const stored = selectVisibilityInputsStmt.get(wordId) as VocabularyVisibilityRow | null;
+    if (!stored) return;
+    const visible = isVocabularyStatsRowVisible(stored) ? 1 : 0;
+    updateVisibilityStmt.run(visible, wordId, visible);
+  };
   let kept = 0;
   let deleted = 0;
   let repaired = 0;
@@ -279,6 +298,7 @@ export async function cleanupVocabularyStats(
           row.last_seen ?? duplicate.last_seen ?? 0,
           duplicate.id,
         );
+        syncWordVisibility(duplicate.id);
         deleteStmt.run(row.id);
         repaired += 1;
         deleted += 1;
@@ -294,6 +314,7 @@ export async function cleanupVocabularyStats(
         resolvedPos.pos3,
         row.id,
       );
+      syncWordVisibility(row.id);
       repaired += 1;
     }
 

src/core/services/immersion-tracker/query-sessions.ts

@@ -6,9 +6,8 @@ import type {
 } from './types';
 import {
   ACTIVE_SESSION_METRICS_CTE,
-  SESSION_WORD_COUNTS_CTE,
-  SESSION_WORD_COUNTS_SELECT,
   currentDbTimestamp,
+  ENDED_SESSIONS_DISPLAY_WORDS_SQL,
   fromDbTimestamp,
   getLocalEpochDay,
   getShiftedLocalDaySec,
@@ -295,32 +294,17 @@ export function getQueryHints(db: DatabaseSync): {
   const totalCards = Number(lifetime?.totalCards ?? 0);
   const activeDays = Number(lifetime?.activeDays ?? 0);
 
-  const lookupWordsExpr = sessionDisplayWordsExpr(
-    's',
-    'swc',
-    'COALESCE(t.tokens_seen, s.tokens_seen)',
-  );
+  // Ended sessions are finalized from their last telemetry sample, so their
+  // own counters already hold the telemetry maxima.
   const lookupTotals = db
     .prepare(
       `
-    ${SESSION_WORD_COUNTS_CTE}
     SELECT
-      COALESCE(SUM(${lookupWordsExpr}), 0) AS totalTokensSeen,
-      COALESCE(SUM(COALESCE(t.lookup_count, s.lookup_count, 0)), 0) AS totalLookupCount,
-      COALESCE(SUM(COALESCE(t.lookup_hits, s.lookup_hits, 0)), 0) AS totalLookupHits,
-      COALESCE(SUM(COALESCE(t.yomitan_lookup_count, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount
+      ${ENDED_SESSIONS_DISPLAY_WORDS_SQL} AS totalTokensSeen,
+      COALESCE(SUM(COALESCE(s.lookup_count, 0)), 0) AS totalLookupCount,
+      COALESCE(SUM(COALESCE(s.lookup_hits, 0)), 0) AS totalLookupHits,
+      COALESCE(SUM(COALESCE(s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount
     FROM imm_sessions s
-    LEFT JOIN (
-      SELECT
-        session_id,
-        MAX(tokens_seen) AS tokens_seen,
-        MAX(lookup_count) AS lookup_count,
-        MAX(lookup_hits) AS lookup_hits,
-        MAX(yomitan_lookup_count) AS yomitan_lookup_count
-      FROM imm_session_telemetry
-      GROUP BY session_id
-    ) t ON t.session_id = s.session_id
-    LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
     WHERE s.ended_at_ms IS NOT NULL
   `,
     )
@@ -349,123 +333,157 @@ export function getQueryHints(db: DatabaseSync): {
   };
 }
 
-export function getDailyRollups(db: DatabaseSync, limit = 60): ImmersionSessionRollupRow[] {
-  const wordsExpr = sessionDisplayWordsExpr('s', 'swc');
-  const prepared = db.prepare(`
-    WITH session_word_counts AS (
-      ${SESSION_WORD_COUNTS_SELECT}
-    ),
-    daily_word_counts AS (
-      SELECT
-        CAST(
-          julianday(CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5
-          AS INTEGER
-        ) AS rollupDay,
-        s.video_id AS videoId,
-        SUM(${wordsExpr}) AS totalTokensSeen
-      FROM imm_sessions s
-      LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
-      WHERE s.ended_at_ms IS NOT NULL
-      GROUP BY rollupDay, s.video_id
-    ),
-    recent_days AS (
-      SELECT DISTINCT rollup_day
-      FROM imm_daily_rollups
-      ORDER BY rollup_day DESC
-      LIMIT ?
-    )
+/** Local calendar day (epoch days) a session row `alias` started on, matching `imm_daily_rollups.rollup_day`. */
+export function sessionRollupDaySql(alias: string): string {
+  return `CAST(julianday(CAST(${alias}.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER)`;
+}
+
+/** Local `YYYYMM` month a session row `alias` started in, matching `imm_monthly_rollups.rollup_month`. */
+export function sessionRollupMonthSql(alias: string): string {
+  return `CAST(strftime('%Y%m', CAST(${alias}.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER)`;
+}
+
+/**
+ * Display-word sums of ended sessions, keyed by `rollupWordCountKey`. Rollup
+ * rows take these over their persisted token totals when they are larger.
+ */
+export type RollupWordCounts = Map<string, number>;
+
+export function rollupWordCountKey(rollupDayOrMonth: number, videoId: number | null): string {
+  return `${rollupDayOrMonth}:${videoId ?? ''}`;
+}
+
+export function addRollupWordCount(
+  counts: RollupWordCounts,
+  rollupDayOrMonth: number,
+  videoId: number | null,
+  words: number,
+): void {
+  const key = rollupWordCountKey(rollupDayOrMonth, videoId);
+  counts.set(key, (counts.get(key) ?? 0) + words);
+}
+
+type RawRollupRow = Omit<ImmersionSessionRollupRow, 'tokensPerMin'>;
+
+/**
+ * Rollup rows for the `limit` most recent days, without the display-word
+ * merge. Pair with `applyRollupWordCounts`; `getDailyRollups` does both.
+ */
+export function selectDailyRollupRows(db: DatabaseSync, limit: number): RawRollupRow[] {
+  return db
+    .prepare(
+      `
     SELECT
       r.rollup_day AS rollupDayOrMonth,
       r.video_id AS videoId,
       r.total_sessions AS totalSessions,
       r.total_active_min AS totalActiveMin,
       r.total_lines_seen AS totalLinesSeen,
-      CASE
-        WHEN dwc.totalTokensSeen IS NOT NULL AND dwc.totalTokensSeen > r.total_tokens_seen THEN dwc.totalTokensSeen
-        ELSE r.total_tokens_seen
-      END AS totalTokensSeen,
+      r.total_tokens_seen AS totalTokensSeen,
       r.total_cards AS totalCards,
       r.cards_per_hour AS cardsPerHour,
-      CASE
-        WHEN r.total_active_min > 0 THEN (
-          CASE
-            WHEN dwc.totalTokensSeen IS NOT NULL AND dwc.totalTokensSeen > r.total_tokens_seen THEN dwc.totalTokensSeen
-            ELSE r.total_tokens_seen
-          END
-        ) * 1.0 / r.total_active_min
-        ELSE NULL
-      END AS tokensPerMin,
       r.lookup_hit_rate AS lookupHitRate
     FROM imm_daily_rollups r
-    LEFT JOIN daily_word_counts dwc
-      ON dwc.rollupDay = r.rollup_day
-      AND (
-        (dwc.videoId IS NULL AND r.video_id IS NULL)
-        OR dwc.videoId = r.video_id
-      )
-    WHERE r.rollup_day IN (SELECT rollup_day FROM recent_days)
+    WHERE r.rollup_day IN (
+      SELECT DISTINCT rollup_day FROM imm_daily_rollups ORDER BY rollup_day DESC LIMIT ?
+    )
     ORDER BY r.rollup_day DESC, r.video_id DESC
-    `);
-
-  return prepared.all(limit) as unknown as ImmersionSessionRollupRow[];
+  `,
+    )
+    .all(limit) as unknown as RawRollupRow[];
 }
 
-export function getMonthlyRollups(db: DatabaseSync, limit = 24): ImmersionSessionRollupRow[] {
-  const wordsExpr = sessionDisplayWordsExpr('s', 'swc');
-  const prepared = db.prepare(`
-    WITH session_word_counts AS (
-      ${SESSION_WORD_COUNTS_SELECT}
-    ),
-    monthly_word_counts AS (
-      SELECT
-        CAST(strftime('%Y%m', CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) AS rollupMonth,
-        s.video_id AS videoId,
-        SUM(${wordsExpr}) AS totalTokensSeen
-      FROM imm_sessions s
-      LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
-      WHERE s.ended_at_ms IS NOT NULL
-      GROUP BY rollupMonth, s.video_id
-    ),
-    recent_months AS (
-      SELECT DISTINCT rollup_month
-      FROM imm_monthly_rollups
-      ORDER BY rollup_month DESC
-      LIMIT ?
-    )
+/** Monthly counterpart of `selectDailyRollupRows`. */
+export function selectMonthlyRollupRows(db: DatabaseSync, limit: number): RawRollupRow[] {
+  return db
+    .prepare(
+      `
     SELECT
       r.rollup_month AS rollupDayOrMonth,
       r.video_id AS videoId,
       r.total_sessions AS totalSessions,
       r.total_active_min AS totalActiveMin,
       r.total_lines_seen AS totalLinesSeen,
-      CASE
-        WHEN mwc.totalTokensSeen IS NOT NULL AND mwc.totalTokensSeen > r.total_tokens_seen THEN mwc.totalTokensSeen
-        ELSE r.total_tokens_seen
-      END AS totalTokensSeen,
+      r.total_tokens_seen AS totalTokensSeen,
       r.total_cards AS totalCards,
       CASE
         WHEN r.total_active_min > 0 THEN (r.total_cards * 60.0) / r.total_active_min
         ELSE NULL
       END AS cardsPerHour,
-      CASE
-        WHEN r.total_active_min > 0 THEN (
-          CASE
-            WHEN mwc.totalTokensSeen IS NOT NULL AND mwc.totalTokensSeen > r.total_tokens_seen THEN mwc.totalTokensSeen
-            ELSE r.total_tokens_seen
-          END
-        ) * 1.0 / r.total_active_min
-        ELSE NULL
-      END AS tokensPerMin,
       NULL AS lookupHitRate
     FROM imm_monthly_rollups r
-    LEFT JOIN monthly_word_counts mwc
-      ON mwc.rollupMonth = r.rollup_month
-      AND (
-        (mwc.videoId IS NULL AND r.video_id IS NULL)
-        OR mwc.videoId = r.video_id
-      )
-    WHERE r.rollup_month IN (SELECT rollup_month FROM recent_months)
+    WHERE r.rollup_month IN (
+      SELECT DISTINCT rollup_month FROM imm_monthly_rollups ORDER BY rollup_month DESC LIMIT ?
+    )
     ORDER BY r.rollup_month DESC, r.video_id DESC
-  `);
-  return prepared.all(limit) as unknown as ImmersionSessionRollupRow[];
+  `,
+    )
+    .all(limit) as unknown as RawRollupRow[];
+}
+
+/**
+ * Raises each rollup's token total to the display-word count of its sessions
+ * when that is larger (persisted totals can lag behind cleaned-up lines), and
+ * derives `tokensPerMin` from the result.
+ */
+export function applyRollupWordCounts(
+  rows: RawRollupRow[],
+  counts: RollupWordCounts,
+): ImmersionSessionRollupRow[] {
+  return rows.map((row) => {
+    const words = counts.get(rollupWordCountKey(row.rollupDayOrMonth, row.videoId));
+    const totalTokensSeen =
+      words !== undefined && words > row.totalTokensSeen ? words : row.totalTokensSeen;
+    return {
+      ...row,
+      totalTokensSeen,
+      tokensPerMin: row.totalActiveMin > 0 ? totalTokensSeen / row.totalActiveMin : null,
+    };
+  });
+}
+
+function getEndedSessionWordCounts(
+  db: DatabaseSync,
+  bucketSql: string,
+  sinceBucket: number,
+): RollupWordCounts {
+  const rows = db
+    .prepare(
+      `
+    SELECT
+      ${bucketSql} AS bucket,
+      s.video_id AS videoId,
+      SUM(${sessionDisplayWordsExpr('s')}) AS words
+    FROM imm_sessions s
+    WHERE s.ended_at_ms IS NOT NULL
+      AND ${bucketSql} >= ?
+    GROUP BY bucket, s.video_id
+  `,
+    )
+    .all(sinceBucket) as Array<{ bucket: number; videoId: number | null; words: number }>;
+  const counts: RollupWordCounts = new Map();
+  for (const row of rows) {
+    addRollupWordCount(counts, row.bucket, row.videoId, row.words);
+  }
+  return counts;
+}
+
+function withSessionWordCounts(
+  db: DatabaseSync,
+  rows: RawRollupRow[],
+  bucketSql: string,
+): ImmersionSessionRollupRow[] {
+  if (rows.length === 0) {
+    return [];
+  }
+  const sinceBucket = Math.min(...rows.map((row) => row.rollupDayOrMonth));
+  return applyRollupWordCounts(rows, getEndedSessionWordCounts(db, bucketSql, sinceBucket));
+}
+
+export function getDailyRollups(db: DatabaseSync, limit = 60): ImmersionSessionRollupRow[] {
+  return withSessionWordCounts(db, selectDailyRollupRows(db, limit), sessionRollupDaySql('s'));
+}
+
+export function getMonthlyRollups(db: DatabaseSync, limit = 24): ImmersionSessionRollupRow[] {
+  return withSessionWordCounts(db, selectMonthlyRollupRows(db, limit), sessionRollupMonthSql('s'));
 }

src/core/services/immersion-tracker/query-shared.ts

@@ -18,24 +18,10 @@ export function visibleWordSql(wordAlias: string): string {
   )`;
 }
 
-export function filteredWordOccurrenceCountSql(occurrenceAlias: string, wordAlias: string): string {
-  return `CASE
-    WHEN ${occurrenceAlias}.word_id IS NOT NULL AND ${visibleWordSql(wordAlias)}
-      THEN ${occurrenceAlias}.occurrence_count
-    ELSE 0
-  END`;
-}
-
-export const SESSION_WORD_COUNTS_SELECT = `
-  SELECT
-    sl.session_id AS sessionId,
-    COUNT(DISTINCT sl.line_id) AS persistedLineCount,
-    COALESCE(SUM(${filteredWordOccurrenceCountSql('wlo', 'w')}), 0) AS filteredWordsSeen
-  FROM imm_subtitle_lines sl
-  LEFT JOIN imm_word_line_occurrences wlo ON wlo.line_id = sl.line_id
-  LEFT JOIN imm_words w ON w.id = wlo.word_id
-  GROUP BY sl.session_id
-`;
+// Hidden words are a small set, so SQLite builds this list once per statement
+// and checks occurrences against it instead of re-evaluating the term filter
+// for every occurrence row.
+const HIDDEN_WORD_IDS_SQL = `SELECT hidden_word.id FROM imm_words hidden_word WHERE NOT ${visibleWordSql('hidden_word')}`;
 
 export const ACTIVE_SESSION_METRICS_CTE = `
   WITH active_session_metrics AS (
@@ -53,29 +39,77 @@ export const ACTIVE_SESSION_METRICS_CTE = `
     JOIN imm_sessions s ON s.session_id = t.session_id
     WHERE s.ended_at_ms IS NULL
     GROUP BY t.session_id
-  ),
-  session_word_counts AS (
-    ${SESSION_WORD_COUNTS_SELECT}
-  )
-`;
-
-export const SESSION_WORD_COUNTS_CTE = `
-  WITH session_word_counts AS (
-    ${SESSION_WORD_COUNTS_SELECT}
   )
 `;
 
+/**
+ * Display word count for the session row `sessionAlias`: visible persisted word
+ * occurrences when the session has persisted subtitle lines, otherwise the raw
+ * token counter `rawTokensExpr`.
+ *
+ * The subqueries are correlated on `session_id`, so a query only reads the
+ * lines of the sessions it actually returns. Do not replace this with a CTE
+ * aggregated over every subtitle line; that costs a full scan of the
+ * occurrence table on every stats request.
+ */
 export function sessionDisplayWordsExpr(
   sessionAlias: string,
-  wordCountAlias: string,
   rawTokensExpr = `${sessionAlias}.tokens_seen`,
 ): string {
   return `CASE
-    WHEN COALESCE(${wordCountAlias}.persistedLineCount, 0) > 0 THEN COALESCE(${wordCountAlias}.filteredWordsSeen, 0)
+    WHEN EXISTS (
+      SELECT 1 FROM imm_subtitle_lines display_line
+      WHERE display_line.session_id = ${sessionAlias}.session_id
+    ) THEN (
+      SELECT COALESCE(SUM(display_occurrence.occurrence_count), 0)
+      FROM imm_subtitle_lines display_line
+      JOIN imm_word_line_occurrences display_occurrence
+        ON display_occurrence.line_id = display_line.line_id
+      WHERE display_line.session_id = ${sessionAlias}.session_id
+        AND display_occurrence.word_id NOT IN (${HIDDEN_WORD_IDS_SQL})
+    )
     ELSE COALESCE(${rawTokensExpr}, 0)
   END`;
 }
 
+/**
+ * `SUM(sessionDisplayWordsExpr(s))` over every ended session, as one scalar
+ * expression that avoids walking each session's lines: all visible
+ * occurrences, minus those on lines outside ended sessions, plus the raw token
+ * counter of ended sessions without persisted lines. Relies on occurrences
+ * cascading with their line (foreign keys are on for every connection), so
+ * every occurrence belongs to an existing line.
+ */
+export const ENDED_SESSIONS_DISPLAY_WORDS_SQL = `(
+  (SELECT COALESCE(SUM(occurrence_count), 0) FROM imm_word_line_occurrences)
+  - (
+    SELECT COALESCE(SUM(hidden_occurrence.occurrence_count), 0)
+    FROM imm_word_line_occurrences hidden_occurrence
+    WHERE hidden_occurrence.word_id IN (${HIDDEN_WORD_IDS_SQL})
+  )
+  - (
+    SELECT COALESCE(SUM(open_occurrence.occurrence_count), 0)
+    FROM imm_subtitle_lines open_line
+    JOIN imm_word_line_occurrences open_occurrence ON open_occurrence.line_id = open_line.line_id
+    WHERE (
+        open_line.session_id IS NULL
+        OR open_line.session_id NOT IN (
+          SELECT ended.session_id FROM imm_sessions ended WHERE ended.ended_at_ms IS NOT NULL
+        )
+      )
+      AND open_occurrence.word_id NOT IN (${HIDDEN_WORD_IDS_SQL})
+  )
+  + (
+    SELECT COALESCE(SUM(COALESCE(lineless.tokens_seen, 0)), 0)
+    FROM imm_sessions lineless
+    WHERE lineless.ended_at_ms IS NOT NULL
+      AND NOT EXISTS (
+        SELECT 1 FROM imm_subtitle_lines lineless_line
+        WHERE lineless_line.session_id = lineless.session_id
+      )
+  )
+)`;
+
 export function makePlaceholders(values: number[]): string {
   return values.map(() => '?').join(',');
 }

src/core/services/immersion-tracker/query-trends.ts

@@ -3,16 +3,22 @@ import type { ImmersionSessionRollupRow } from './types';
 import {
   ACTIVE_SESSION_METRICS_CTE,
   currentDbTimestamp,
-  getLocalDayOfWeek,
   getLocalEpochDay,
-  getLocalHourOfDay,
   getLocalMonthKey,
   getShiftedLocalDayTimestamp,
   makePlaceholders,
   sessionDisplayWordsExpr,
   toDbTimestamp,
 } from './query-shared';
-import { getDailyRollups, getMonthlyRollups } from './query-sessions';
+import {
+  addRollupWordCount,
+  applyRollupWordCounts,
+  selectDailyRollupRows,
+  selectMonthlyRollupRows,
+  sessionRollupDaySql,
+  sessionRollupMonthSql,
+  type RollupWordCounts,
+} from './query-sessions';
 import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
 
 type TrendRange = '7d' | '30d' | '90d' | '365d' | 'all';
@@ -43,7 +49,6 @@ export interface LibrarySummaryRow {
 }
 
 interface TrendSessionMetricRow {
-  startedAtMs: number;
   epochDay: number;
   monthKey: number;
   dayOfWeek: number;
@@ -78,7 +83,8 @@ export interface TrendsDashboardQueryResult {
     cardsPerHour: TrendChartPoint[];
     readingSpeed: TrendChartPoint[];
   };
-  animeCumulative: {
+  /** Sparse per-title values for each active day; the stats UI builds the running totals. */
+  animeDaily: {
     watchTime: TrendPerAnimePoint[];
     episodes: TrendPerAnimePoint[];
     cards: TrendPerAnimePoint[];
@@ -387,41 +393,6 @@ function buildLookupsPerHundredWords(
   return fillAxisPoints(axis, ratioByBucket);
 }
 
-function buildCumulativePerAnime(points: TrendPerAnimePoint[]): TrendPerAnimePoint[] {
-  const byAnime = new Map<string, Map<number, number>>();
-  const allDays = new Set<number>();
-
-  for (const point of points) {
-    const dayMap = byAnime.get(point.animeTitle) ?? new Map();
-    dayMap.set(point.epochDay, (dayMap.get(point.epochDay) ?? 0) + point.value);
-    byAnime.set(point.animeTitle, dayMap);
-    allDays.add(point.epochDay);
-  }
-
-  const sortedDays = [...allDays].sort((left, right) => left - right);
-  if (sortedDays.length === 0) {
-    return [];
-  }
-
-  const minDay = sortedDays[0]!;
-  const maxDay = sortedDays[sortedDays.length - 1]!;
-  const result: TrendPerAnimePoint[] = [];
-
-  for (const [animeTitle, dayMap] of byAnime) {
-    const firstDay = Math.min(...dayMap.keys());
-    let cumulative = 0;
-    for (let epochDay = minDay; epochDay <= maxDay; epochDay += 1) {
-      if (epochDay < firstDay) {
-        continue;
-      }
-      cumulative += dayMap.get(epochDay) ?? 0;
-      result.push({ epochDay, animeTitle, value: cumulative });
-    }
-  }
-
-  return result;
-}
-
 function buildLibrarySummary(
   rollups: ImmersionSessionRollupRow[],
   sessions: TrendSessionMetricRow[],
@@ -616,44 +587,64 @@ function buildEpisodesSeriesFromRollups(
   return fillAxisPoints(axis, counts);
 }
 
-function getTrendSessionMetrics(
+interface TrendSessionRow extends TrendSessionMetricRow {
+  ended: number;
+  inTrendRange: number;
+}
+
+/**
+ * One pass over the sessions the dashboard needs. Rows inside the trend range
+ * feed the session-metric charts; ended rows also supply the rollup word counts,
+ * so the per-session word count is computed once per request instead of once
+ * per rollup query. `minRollupDay` / `minRollupMonth` widen the scan to the
+ * oldest rollup bucket shown.
+ */
+function getTrendSessionRows(
   db: DatabaseSync,
   cutoffMs: string | null,
-): TrendSessionMetricRow[] {
-  const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)');
-  const whereClause = cutoffMs === null ? '' : 'WHERE s.started_at_ms >= ?';
+  minRollupDay: number | null,
+  minRollupMonth: number | null,
+): TrendSessionRow[] {
+  const wordsExpr = sessionDisplayWordsExpr('s', 'COALESCE(asm.tokensSeen, s.tokens_seen)');
   const cutoffValue = cutoffMs === null ? null : toDbTimestamp(cutoffMs);
+  const rangeSql = cutoffValue === null ? '1' : 's.started_at_ms >= ?';
+  const dayValue = minRollupDay ?? Number.MAX_SAFE_INTEGER;
+  const monthValue = minRollupMonth ?? Number.MAX_SAFE_INTEGER;
   const prepared = db.prepare(`
     ${ACTIVE_SESSION_METRICS_CTE}
     SELECT
-      s.started_at_ms AS startedAtMs,
       s.video_id AS videoId,
       v.canonical_title AS canonicalTitle,
       a.canonical_title AS animeTitle,
       COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0) AS activeWatchedMs,
       ${wordsExpr} AS tokensSeen,
       COALESCE(asm.cardsMined, s.cards_mined, 0) AS cardsMined,
-      COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0) AS yomitanLookupCount
+      COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0) AS yomitanLookupCount,
+      CASE WHEN s.ended_at_ms IS NOT NULL THEN 1 ELSE 0 END AS ended,
+      CASE WHEN ${rangeSql} THEN 1 ELSE 0 END AS inTrendRange,
+      ${sessionRollupDaySql('s')} AS epochDay,
+      ${sessionRollupMonthSql('s')} AS monthKey,
+      CAST(strftime('%w', CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) AS dayOfWeek,
+      CAST(strftime('%H', CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) AS hourOfDay
     FROM imm_sessions s
     LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
-    LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
     LEFT JOIN imm_videos v ON v.video_id = s.video_id
     LEFT JOIN imm_anime a ON a.anime_id = v.anime_id
-    ${whereClause}
+    WHERE ${rangeSql}
+      OR (s.ended_at_ms IS NOT NULL AND ${sessionRollupDaySql('s')} >= ?)
+      OR (s.ended_at_ms IS NOT NULL AND ${sessionRollupMonthSql('s')} >= ?)
     ORDER BY s.started_at_ms ASC
   `);
 
-  const rows = (cutoffValue === null ? prepared.all() : prepared.all(cutoffValue)) as Array<
-    TrendSessionMetricRow & { startedAtMs: number | string }
-  >;
-  return rows.map((row) => ({
-    ...row,
-    startedAtMs: 0,
-    epochDay: getLocalEpochDay(db, row.startedAtMs),
-    monthKey: getLocalMonthKey(db, row.startedAtMs),
-    dayOfWeek: getLocalDayOfWeek(db, row.startedAtMs),
-    hourOfDay: getLocalHourOfDay(db, row.startedAtMs),
-  }));
+  const params =
+    cutoffValue === null
+      ? [dayValue, monthValue]
+      : [cutoffValue, cutoffValue, dayValue, monthValue];
+  return prepared.all(...params) as unknown as TrendSessionRow[];
+}
+
+function minRollupBucket(rows: Array<{ rollupDayOrMonth: number }>): number | null {
+  return rows.length === 0 ? null : Math.min(...rows.map((row) => row.rollupDayOrMonth));
 }
 
 function buildNewWordsPerDay(
@@ -750,8 +741,24 @@ export function getTrendsDashboard(
   const monthlyLimit = getTrendMonthlyLimit(db, range);
   const cutoffMs = getTrendCutoffMs(db, range);
   const useMonthlyBuckets = groupBy === 'month';
-  const dailyRollups = getDailyRollups(db, dayLimit);
-  const monthlyRollups = getMonthlyRollups(db, monthlyLimit);
+  const rawDailyRollups = selectDailyRollupRows(db, dayLimit);
+  const rawMonthlyRollups = selectMonthlyRollupRows(db, monthlyLimit);
+  const sessionRows = getTrendSessionRows(
+    db,
+    cutoffMs,
+    minRollupBucket(rawDailyRollups),
+    minRollupBucket(rawMonthlyRollups),
+  );
+  const dailyWordCounts: RollupWordCounts = new Map();
+  const monthlyWordCounts: RollupWordCounts = new Map();
+  for (const row of sessionRows) {
+    if (row.ended !== 1) continue;
+    addRollupWordCount(dailyWordCounts, row.epochDay, row.videoId, row.tokensSeen);
+    addRollupWordCount(monthlyWordCounts, row.monthKey, row.videoId, row.tokensSeen);
+  }
+  const dailyRollups = applyRollupWordCounts(rawDailyRollups, dailyWordCounts);
+  const monthlyRollups = applyRollupWordCounts(rawMonthlyRollups, monthlyWordCounts);
+  const sessions = sessionRows.filter((row) => row.inTrendRange === 1);
   // A null axis makes the builders fall back to only the buckets present in the
   // data; the contiguous axis zero-fills every calendar bucket in the window.
   const bucketAxis = fillEmptyBuckets
@@ -759,7 +766,6 @@ export function getTrendsDashboard(
     : null;
 
   const chartRollups = useMonthlyBuckets ? monthlyRollups : dailyRollups;
-  const sessions = getTrendSessionMetrics(db, cutoffMs);
   const titlesByVideoId = getVideoAnimeTitleMap(
     db,
     dailyRollups.map((rollup) => rollup.videoId),
@@ -820,12 +826,7 @@ export function getTrendsDashboard(
       cardsPerHour: efficiency.cardsPerHour,
       readingSpeed: efficiency.readingSpeed,
     },
-    animeCumulative: {
-      watchTime: buildCumulativePerAnime(animePerDay.watchTime),
-      episodes: buildCumulativePerAnime(animePerDay.episodes),
-      cards: buildCumulativePerAnime(animePerDay.cards),
-      words: buildCumulativePerAnime(animePerDay.words),
-    },
+    animeDaily: animePerDay,
     patterns: {
       watchTimeByDayOfWeek: buildWatchTimeByDayOfWeek(sessions),
       watchTimeByHour: buildWatchTimeByHour(sessions),

src/core/services/immersion-tracker/storage.ts

@@ -1849,14 +1849,24 @@ function incrementWordAggregate(
       occurrence.frequencyRank ?? null,
     );
   }
-  const row = stmts.wordSelectStmt.get(occurrence.headword, occurrence.word, occurrence.reading) as
+  return syncStoredWordVisibility(stmts, occurrence);
+}
+
+/**
+ * Persists visibility for a word that was just upserted and returns its id. The
+ * upsert keeps the first-seen POS metadata, so visibility comes from the stored
+ * row (as the vocabulary summary expects) rather than the incoming token.
+ */
+function syncStoredWordVisibility(
+  stmts: TrackerPreparedStatements,
+  key: Pick<Extract<QueuedWrite, { kind: 'word' }>, 'headword' | 'word' | 'reading'>,
+): number {
+  const row = stmts.wordSelectStmt.get(key.headword, key.word, key.reading) as
     | (VocabularyVisibilityRow & { id: number; vocabularyVisible: number })
     | null;
   if (!row?.id) {
-    throw new Error(`Failed to resolve imm_words id for ${occurrence.headword}`);
+    throw new Error(`Failed to resolve imm_words id for ${key.headword}`);
   }
-  // The upsert keeps the first-seen POS metadata, so derive visibility from the
-  // stored row (as the vocabulary summary does) rather than the incoming token.
   const visible = isVocabularyStatsRowVisible(row) ? 1 : 0;
   if (row.vocabularyVisible !== visible) {
     stmts.wordVisibilityUpdateStmt.run(visible, row.id);
@@ -1939,6 +1949,7 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta
       write.lastSeen,
       write.frequencyRank ?? null,
     );
+    syncStoredWordVisibility(stmts, write);
     return;
   }
   if (write.kind === 'kanji') {

src/types/stats-wire.ts

@@ -384,7 +384,8 @@ export interface TrendsDashboardData {
     readingSpeed: TrendChartPoint[];
   };
   librarySummary: LibrarySummaryRow[];
-  animeCumulative: {
+  /** Sparse per-title values for each active day; the client accumulates them for charts. */
+  animeDaily: {
     watchTime: TrendPerAnimePoint[];
     episodes: TrendPerAnimePoint[];
     cards: TrendPerAnimePoint[];

stats/src/components/trends/TrendsTab.tsx

@@ -1,4 +1,4 @@
-import { useState } from 'react';
+import { useMemo, useState } from 'react';
 import { useTrends, type TimeRange, type GroupBy } from '../../hooks/useTrends';
 import { DateRangeSelector } from './DateRangeSelector';
 import { TrendChart } from './TrendChart';
@@ -20,6 +20,7 @@ import {
   type MaxTitlesMode,
 } from './anime-visibility';
 import { LibrarySummarySection } from './LibrarySummarySection';
+import { buildAnimeCumulative } from './anime-cumulative';
 
 const MAX_TITLES_MODE_LABELS: Record<MaxTitlesMode, string> = {
   recent: 'most recent',
@@ -156,6 +157,10 @@ export function TrendsTab() {
   const [maxTitles, setMaxTitles] = useState<number | null>(() => loadMaxTitles());
   const [maxTitlesMode, setMaxTitlesMode] = useState<MaxTitlesMode>(() => loadMaxTitlesMode());
   const { data, loading, error } = useTrends(range, groupBy, showEmptyDays);
+  const animeCumulative = useMemo(
+    () => (data ? buildAnimeCumulative(data.animeDaily) : null),
+    [data],
+  );
 
   const updateHiddenAnime = (next: Set<string>) => {
     setHiddenAnime(next);
@@ -187,30 +192,21 @@ export function TrendsTab() {
 
   if (loading) return <div className="text-ctp-overlay2 p-4">Loading...</div>;
   if (error) return <div className="text-ctp-red p-4">Error: {error}</div>;
-  if (!data) return null;
+  if (!data || !animeCumulative) return null;
 
   const animeTitles = buildAnimeVisibilityOptions([
-    data.animeCumulative.episodes,
-    data.animeCumulative.cards,
-    data.animeCumulative.words,
-    data.animeCumulative.watchTime,
+    animeCumulative.episodes,
+    animeCumulative.cards,
+    animeCumulative.words,
+    animeCumulative.watchTime,
   ]);
   const activeHiddenAnime = pruneHiddenAnime(hiddenAnime, animeTitles);
 
-  const filteredAnimeProgress = filterHiddenAnimeData(
-    data.animeCumulative.episodes,
-    activeHiddenAnime,
-  );
-  const filteredCardsProgress = filterHiddenAnimeData(
-    data.animeCumulative.cards,
-    activeHiddenAnime,
-  );
-  const filteredWordsProgress = filterHiddenAnimeData(
-    data.animeCumulative.words,
-    activeHiddenAnime,
-  );
+  const filteredAnimeProgress = filterHiddenAnimeData(animeCumulative.episodes, activeHiddenAnime);
+  const filteredCardsProgress = filterHiddenAnimeData(animeCumulative.cards, activeHiddenAnime);
+  const filteredWordsProgress = filterHiddenAnimeData(animeCumulative.words, activeHiddenAnime);
   const filteredWatchTimeProgress = filterHiddenAnimeData(
-    data.animeCumulative.watchTime,
+    animeCumulative.watchTime,
     activeHiddenAnime,
   );
 

stats/src/components/trends/anime-cumulative.test.ts

@@ -0,0 +1,36 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { buildCumulativePerAnime } from './anime-cumulative';
+
+test('buildCumulativePerAnime carries each title forward to the last active day', () => {
+  const result = buildCumulativePerAnime([
+    { epochDay: 103, animeTitle: 'B', value: 4 },
+    { epochDay: 100, animeTitle: 'A', value: 10 },
+    { epochDay: 102, animeTitle: 'A', value: 5 },
+    { epochDay: 102, animeTitle: 'A', value: 1 },
+  ]);
+
+  assert.deepEqual(
+    result
+      .filter((point) => point.animeTitle === 'A')
+      .map((point) => [point.epochDay, point.value]),
+    [
+      [100, 10],
+      [101, 10],
+      [102, 16],
+      [103, 16],
+    ],
+  );
+  // A title starts at its own first active day, not the earliest day overall.
+  assert.deepEqual(
+    result
+      .filter((point) => point.animeTitle === 'B')
+      .map((point) => [point.epochDay, point.value]),
+    [[103, 4]],
+  );
+});
+
+test('buildCumulativePerAnime returns nothing for no points', () => {
+  assert.deepEqual(buildCumulativePerAnime([]), []);
+});

stats/src/components/trends/anime-cumulative.ts

@@ -0,0 +1,41 @@
+import type { TrendsDashboardData } from '../../types/stats';
+import type { PerAnimeDataPoint } from './StackedTrendChart';
+
+type AnimeSeries = TrendsDashboardData['animeDaily'];
+
+/**
+ * Turns sparse per-title daily values into running totals: one point per
+ * title for every calendar day from its first active day to the last active
+ * day of any title, so the stacked area charts carry each total forward.
+ */
+export function buildCumulativePerAnime(points: PerAnimeDataPoint[]): PerAnimeDataPoint[] {
+  const byAnime = new Map<string, Map<number, number>>();
+  let maxDay = Number.NEGATIVE_INFINITY;
+
+  for (const point of points) {
+    const dayMap = byAnime.get(point.animeTitle) ?? new Map<number, number>();
+    dayMap.set(point.epochDay, (dayMap.get(point.epochDay) ?? 0) + point.value);
+    byAnime.set(point.animeTitle, dayMap);
+    maxDay = Math.max(maxDay, point.epochDay);
+  }
+
+  const result: PerAnimeDataPoint[] = [];
+  for (const [animeTitle, dayMap] of byAnime) {
+    const firstDay = Math.min(...dayMap.keys());
+    let cumulative = 0;
+    for (let epochDay = firstDay; epochDay <= maxDay; epochDay += 1) {
+      cumulative += dayMap.get(epochDay) ?? 0;
+      result.push({ epochDay, animeTitle, value: cumulative });
+    }
+  }
+  return result;
+}
+
+export function buildAnimeCumulative(daily: AnimeSeries): AnimeSeries {
+  return {
+    watchTime: buildCumulativePerAnime(daily.watchTime),
+    episodes: buildCumulativePerAnime(daily.episodes),
+    cards: buildCumulativePerAnime(daily.cards),
+    words: buildCumulativePerAnime(daily.words),
+  };
+}

stats/src/lib/api-client.test.ts

@@ -105,7 +105,7 @@ test('getTrendsDashboard accepts 365d range and builds correct URL', async () =>
         },
         ratios: { lookupsPerHundred: [] },
         librarySummary: [],
-        animeCumulative: { watchTime: [], episodes: [], cards: [], words: [] },
+        animeDaily: { watchTime: [], episodes: [], cards: [], words: [] },
         patterns: { watchTimeByDayOfWeek: [], watchTimeByHour: [] },
       }),
     async (requests) => {