From db61ce358d788ced75c15244fc363b58f4e0511c Mon Sep 17 00:00:00 2001 From: sudacode Date: Tue, 18 Aug 2026 01:12:58 -0700 Subject: [PATCH] fix(stats): preserve session rollups across schema upgrades - Stop deleting imm_daily_rollups/imm_monthly_rollups on unrelated schema version bumps; their source session/telemetry rows may already be pruned, so deleted buckets could not be rebuilt - Run startup session-rollup maintenance before the lexical rollup backfill takes the write lock, so recovery no longer races playback writes - Update the vocabulary summary totals changelog fragment to reflect that watch-time, activity, efficiency, and library charts are no longer cleared during rebuilds --- changes/stats-vocabulary-summary-totals.md | 2 +- .../immersion-tracker-service.test.ts | 76 +++++++++++++++++++ .../services/immersion-tracker-service.ts | 2 +- .../immersion-tracker/storage-session.test.ts | 45 +++++++++++ .../services/immersion-tracker/storage.ts | 10 +-- 5 files changed, 126 insertions(+), 9 deletions(-) diff --git a/changes/stats-vocabulary-summary-totals.md b/changes/stats-vocabulary-summary-totals.md index bbd71508..4b6f1346 100644 --- a/changes/stats-vocabulary-summary-totals.md +++ b/changes/stats-vocabulary-summary-totals.md @@ -2,7 +2,7 @@ type: fixed area: stats - Fixed Vocabulary totals and charts counting only the first browsing page instead of all tracked vocabulary, without delaying the rest of the page. -- New-word history now uses permanent daily lexical rollups that apply the same vocabulary filters as the totals and normalize legacy second/millisecond timestamps; versioned background rebuilds repair existing history without dropping playback writes. +- New-word history now uses permanent daily lexical rollups that apply the same vocabulary filters as the totals and normalize legacy second/millisecond timestamps; versioned background rebuilds repair existing history without dropping playback writes or clearing watch-time, activity, efficiency, and library charts. - Calendar-day chart labels now preserve the recorded local date in time zones west of UTC. - Vocabulary summary cards and charts refresh automatically after the word exclusion list changes, and failed or unfinished loads use bounded retries before showing an inline error with a Retry control. - Rapid exclusion edits no longer race each other; writes are sent in order so a slower earlier save cannot overwrite a newer list. diff --git a/src/core/services/immersion-tracker-service.test.ts b/src/core/services/immersion-tracker-service.test.ts index 5dd31f4b..ae995172 100644 --- a/src/core/services/immersion-tracker-service.test.ts +++ b/src/core/services/immersion-tracker-service.test.ts @@ -617,6 +617,82 @@ test('tracker starts the injected lexical rollup backfill when it is pending', a } }); +test('tracker runs startup session-rollup maintenance before lexical backfill locks writes', async () => { + const dbPath = makeDbPath(); + let tracker: ImmersionTrackerService | null = null; + let releaseBackfill = (): void => {}; + const heldBackfill = new Promise((resolve) => { + releaseBackfill = resolve; + }); + + try { + const startedAtMs = trackerNowMs() - 60_000; + const endedAtMs = trackerNowMs(); + const setupDb = new Database(dbPath); + const { ensureSchema } = await import('./immersion-tracker/storage'); + ensureSchema(setupDb); + setupDb.exec(` + INSERT INTO imm_videos ( + video_id, video_key, canonical_title, source_type, duration_ms, CREATED_DATE, LAST_UPDATE_DATE + ) VALUES (1, 'local:/tmp/rollup-recovery.mkv', 'Rollup Recovery', 1, 0, '1', '1'); + INSERT INTO imm_sessions ( + session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status, + active_watched_ms, lines_seen, tokens_seen, cards_mined, CREATED_DATE, LAST_UPDATE_DATE + ) VALUES ( + 1, 'rollup-recovery', 1, '${startedAtMs}', '${endedAtMs}', 2, + 60000, 10, 20, 2, '${startedAtMs}', '${endedAtMs}' + ); + INSERT INTO imm_session_telemetry ( + session_id, sample_ms, total_watched_ms, active_watched_ms, lines_seen, + tokens_seen, cards_mined, lookup_count, lookup_hits, CREATED_DATE, LAST_UPDATE_DATE + ) VALUES ( + 1, '${endedAtMs}', 60000, 60000, 10, 20, 2, 0, 0, + '${endedAtMs}', '${endedAtMs}' + ); + DELETE FROM imm_daily_rollups; + DELETE FROM imm_monthly_rollups; + UPDATE imm_rollup_state SET state_value = '0'; + `); + setupDb.close(); + + const Ctor = await loadTrackerCtor(); + tracker = new Ctor({ dbPath }, { + runLexicalRollupBackfillTask: async () => heldBackfill, + } as never); + + const privateApi = tracker as unknown as { + db: DatabaseSync; + writeLock: { locked: boolean }; + }; + assert.equal(privateApi.writeLock.locked, true); + assert.equal( + ( + privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_daily_rollups').get() as { + total: number; + } + ).total, + 1, + ); + assert.equal( + ( + privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_monthly_rollups').get() as { + total: number; + } + ).total, + 1, + ); + } finally { + releaseBackfill(); + if (tracker) { + await waitForCondition( + () => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked, + ); + } + tracker?.destroy(); + cleanupDbPath(dbPath); + } +}); + test('tracker queues playback writes until lexical rollup backfill settles', async () => { const dbPath = makeDbPath(); let tracker: ImmersionTrackerService | null = null; diff --git a/src/core/services/immersion-tracker-service.ts b/src/core/services/immersion-tracker-service.ts index f9ed92c0..74f34bea 100644 --- a/src/core/services/immersion-tracker-service.ts +++ b/src/core/services/immersion-tracker-service.ts @@ -591,8 +591,8 @@ export class ImmersionTrackerService { } } this.preparedStatements = createTrackerPreparedStatements(this.db); - if (!areLexicalDailyRollupsReady(this.db)) this.startLexicalRollupBackfill(); this.scheduleMaintenance(); + if (!areLexicalDailyRollupsReady(this.db)) this.startLexicalRollupBackfill(); this.scheduleFlush(); } diff --git a/src/core/services/immersion-tracker/storage-session.test.ts b/src/core/services/immersion-tracker/storage-session.test.ts index 9bb718d4..07a21995 100644 --- a/src/core/services/immersion-tracker/storage-session.test.ts +++ b/src/core/services/immersion-tracker/storage-session.test.ts @@ -184,6 +184,51 @@ test('ensureSchema adds manual assignment locks when upgrading the previous sche } }); +test('ensureSchema preserves durable session rollups across unrelated schema upgrades', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + db.exec(` + INSERT INTO imm_videos ( + video_id, video_key, canonical_title, source_type, duration_ms, CREATED_DATE, LAST_UPDATE_DATE + ) VALUES (1, 'local:/tmp/preserved.mkv', 'Preserved', 1, 0, '1', '1'); + INSERT INTO imm_daily_rollups ( + rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, + total_tokens_seen, total_cards + ) VALUES (20000, 1, 2, 30, 40, 50, 3); + INSERT INTO imm_monthly_rollups ( + rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, + total_tokens_seen, total_cards + ) VALUES (202410, 1, 2, 30, 40, 50, 3); + UPDATE imm_rollup_state + SET state_value = '123' + WHERE state_key = 'last_rollup_sample_ms'; + UPDATE imm_schema_version SET schema_version = 21; + `); + + ensureSchema(db); + + const daily = db + .prepare('SELECT total_sessions AS totalSessions FROM imm_daily_rollups') + .get() as { totalSessions: number } | null; + const monthly = db + .prepare('SELECT total_sessions AS totalSessions FROM imm_monthly_rollups') + .get() as { totalSessions: number } | null; + const rollupState = db + .prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`) + .get('last_rollup_sample_ms') as { value: string } | null; + + assert.equal(daily?.totalSessions, 2); + assert.equal(monthly?.totalSessions, 2); + assert.equal(rollupState?.value, '123'); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('stats excluded words are replaced and read from sqlite storage', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); diff --git a/src/core/services/immersion-tracker/storage.ts b/src/core/services/immersion-tracker/storage.ts index c55de8dc..74286dff 100644 --- a/src/core/services/immersion-tracker/storage.ts +++ b/src/core/services/immersion-tracker/storage.ts @@ -1584,13 +1584,9 @@ export function ensureSchema(db: DatabaseSync): void { ON imm_youtube_videos(youtube_video_id) `); - if (currentVersion?.schema_version && currentVersion.schema_version < SCHEMA_VERSION) { - db.exec('DELETE FROM imm_daily_rollups'); - db.exec('DELETE FROM imm_monthly_rollups'); - db.exec( - `UPDATE imm_rollup_state SET state_value = 0 WHERE state_key = 'last_rollup_sample_ms'`, - ); - } + // Session rollups intentionally outlive raw session and telemetry retention. + // Preserve them across unrelated schema upgrades because deleted historical + // buckets cannot be rebuilt after their source rows have been pruned. db.exec(` INSERT INTO imm_schema_version(schema_version, applied_at_ms)