fix(stats): preserve session rollups across schema upgrades

- Stop deleting imm_daily_rollups/imm_monthly_rollups on unrelated schema version bumps; their source session/telemetry rows may already be pruned, so deleted buckets could not be rebuilt
- Run startup session-rollup maintenance before the lexical rollup backfill takes the write lock, so recovery no longer races playback writes
- Update the vocabulary summary totals changelog fragment to reflect that watch-time, activity, efficiency, and library charts are no longer cleared during rebuilds
This commit is contained in:
2026-08-18 01:12:58 -07:00
parent e9778a945a
commit db61ce358d
5 changed files with 126 additions and 9 deletions
+1 -1
View File
@@ -2,7 +2,7 @@ type: fixed
area: stats
- Fixed Vocabulary totals and charts counting only the first browsing page instead of all tracked vocabulary, without delaying the rest of the page.
- New-word history now uses permanent daily lexical rollups that apply the same vocabulary filters as the totals and normalize legacy second/millisecond timestamps; versioned background rebuilds repair existing history without dropping playback writes.
- New-word history now uses permanent daily lexical rollups that apply the same vocabulary filters as the totals and normalize legacy second/millisecond timestamps; versioned background rebuilds repair existing history without dropping playback writes or clearing watch-time, activity, efficiency, and library charts.
- Calendar-day chart labels now preserve the recorded local date in time zones west of UTC.
- Vocabulary summary cards and charts refresh automatically after the word exclusion list changes, and failed or unfinished loads use bounded retries before showing an inline error with a Retry control.
- Rapid exclusion edits no longer race each other; writes are sent in order so a slower earlier save cannot overwrite a newer list.
@@ -617,6 +617,82 @@ test('tracker starts the injected lexical rollup backfill when it is pending', a
}
});
test('tracker runs startup session-rollup maintenance before lexical backfill locks writes', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let releaseBackfill = (): void => {};
const heldBackfill = new Promise<void>((resolve) => {
releaseBackfill = resolve;
});
try {
const startedAtMs = trackerNowMs() - 60_000;
const endedAtMs = trackerNowMs();
const setupDb = new Database(dbPath);
const { ensureSchema } = await import('./immersion-tracker/storage');
ensureSchema(setupDb);
setupDb.exec(`
INSERT INTO imm_videos (
video_id, video_key, canonical_title, source_type, duration_ms, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (1, 'local:/tmp/rollup-recovery.mkv', 'Rollup Recovery', 1, 0, '1', '1');
INSERT INTO imm_sessions (
session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status,
active_watched_ms, lines_seen, tokens_seen, cards_mined, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (
1, 'rollup-recovery', 1, '${startedAtMs}', '${endedAtMs}', 2,
60000, 10, 20, 2, '${startedAtMs}', '${endedAtMs}'
);
INSERT INTO imm_session_telemetry (
session_id, sample_ms, total_watched_ms, active_watched_ms, lines_seen,
tokens_seen, cards_mined, lookup_count, lookup_hits, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (
1, '${endedAtMs}', 60000, 60000, 10, 20, 2, 0, 0,
'${endedAtMs}', '${endedAtMs}'
);
DELETE FROM imm_daily_rollups;
DELETE FROM imm_monthly_rollups;
UPDATE imm_rollup_state SET state_value = '0';
`);
setupDb.close();
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath }, {
runLexicalRollupBackfillTask: async () => heldBackfill,
} as never);
const privateApi = tracker as unknown as {
db: DatabaseSync;
writeLock: { locked: boolean };
};
assert.equal(privateApi.writeLock.locked, true);
assert.equal(
(
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_daily_rollups').get() as {
total: number;
}
).total,
1,
);
assert.equal(
(
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_monthly_rollups').get() as {
total: number;
}
).total,
1,
);
} finally {
releaseBackfill();
if (tracker) {
await waitForCondition(
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
);
}
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('tracker queues playback writes until lexical rollup backfill settles', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
@@ -591,8 +591,8 @@ export class ImmersionTrackerService {
}
}
this.preparedStatements = createTrackerPreparedStatements(this.db);
if (!areLexicalDailyRollupsReady(this.db)) this.startLexicalRollupBackfill();
this.scheduleMaintenance();
if (!areLexicalDailyRollupsReady(this.db)) this.startLexicalRollupBackfill();
this.scheduleFlush();
}
@@ -184,6 +184,51 @@ test('ensureSchema adds manual assignment locks when upgrading the previous sche
}
});
test('ensureSchema preserves durable session rollups across unrelated schema upgrades', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
db.exec(`
INSERT INTO imm_videos (
video_id, video_key, canonical_title, source_type, duration_ms, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (1, 'local:/tmp/preserved.mkv', 'Preserved', 1, 0, '1', '1');
INSERT INTO imm_daily_rollups (
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards
) VALUES (20000, 1, 2, 30, 40, 50, 3);
INSERT INTO imm_monthly_rollups (
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards
) VALUES (202410, 1, 2, 30, 40, 50, 3);
UPDATE imm_rollup_state
SET state_value = '123'
WHERE state_key = 'last_rollup_sample_ms';
UPDATE imm_schema_version SET schema_version = 21;
`);
ensureSchema(db);
const daily = db
.prepare('SELECT total_sessions AS totalSessions FROM imm_daily_rollups')
.get() as { totalSessions: number } | null;
const monthly = db
.prepare('SELECT total_sessions AS totalSessions FROM imm_monthly_rollups')
.get() as { totalSessions: number } | null;
const rollupState = db
.prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
.get('last_rollup_sample_ms') as { value: string } | null;
assert.equal(daily?.totalSessions, 2);
assert.equal(monthly?.totalSessions, 2);
assert.equal(rollupState?.value, '123');
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('stats excluded words are replaced and read from sqlite storage', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -1584,13 +1584,9 @@ export function ensureSchema(db: DatabaseSync): void {
ON imm_youtube_videos(youtube_video_id)
`);
if (currentVersion?.schema_version && currentVersion.schema_version < SCHEMA_VERSION) {
db.exec('DELETE FROM imm_daily_rollups');
db.exec('DELETE FROM imm_monthly_rollups');
db.exec(
`UPDATE imm_rollup_state SET state_value = 0 WHERE state_key = 'last_rollup_sample_ms'`,
);
}
// Session rollups intentionally outlive raw session and telemetry retention.
// Preserve them across unrelated schema upgrades because deleted historical
// buckets cannot be rebuilt after their source rows have been pruned.
db.exec(`
INSERT INTO imm_schema_version(schema_version, applied_at_ms)