mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-18 12:18:29 -07:00
fix(stats): preserve session rollups across schema upgrades
- Stop deleting imm_daily_rollups/imm_monthly_rollups on unrelated schema version bumps; their source session/telemetry rows may already be pruned, so deleted buckets could not be rebuilt - Run startup session-rollup maintenance before the lexical rollup backfill takes the write lock, so recovery no longer races playback writes - Update the vocabulary summary totals changelog fragment to reflect that watch-time, activity, efficiency, and library charts are no longer cleared during rebuilds
This commit is contained in:
@@ -2,7 +2,7 @@ type: fixed
|
||||
area: stats
|
||||
|
||||
- Fixed Vocabulary totals and charts counting only the first browsing page instead of all tracked vocabulary, without delaying the rest of the page.
|
||||
- New-word history now uses permanent daily lexical rollups that apply the same vocabulary filters as the totals and normalize legacy second/millisecond timestamps; versioned background rebuilds repair existing history without dropping playback writes.
|
||||
- New-word history now uses permanent daily lexical rollups that apply the same vocabulary filters as the totals and normalize legacy second/millisecond timestamps; versioned background rebuilds repair existing history without dropping playback writes or clearing watch-time, activity, efficiency, and library charts.
|
||||
- Calendar-day chart labels now preserve the recorded local date in time zones west of UTC.
|
||||
- Vocabulary summary cards and charts refresh automatically after the word exclusion list changes, and failed or unfinished loads use bounded retries before showing an inline error with a Retry control.
|
||||
- Rapid exclusion edits no longer race each other; writes are sent in order so a slower earlier save cannot overwrite a newer list.
|
||||
|
||||
@@ -617,6 +617,82 @@ test('tracker starts the injected lexical rollup backfill when it is pending', a
|
||||
}
|
||||
});
|
||||
|
||||
test('tracker runs startup session-rollup maintenance before lexical backfill locks writes', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let releaseBackfill = (): void => {};
|
||||
const heldBackfill = new Promise<void>((resolve) => {
|
||||
releaseBackfill = resolve;
|
||||
});
|
||||
|
||||
try {
|
||||
const startedAtMs = trackerNowMs() - 60_000;
|
||||
const endedAtMs = trackerNowMs();
|
||||
const setupDb = new Database(dbPath);
|
||||
const { ensureSchema } = await import('./immersion-tracker/storage');
|
||||
ensureSchema(setupDb);
|
||||
setupDb.exec(`
|
||||
INSERT INTO imm_videos (
|
||||
video_id, video_key, canonical_title, source_type, duration_ms, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (1, 'local:/tmp/rollup-recovery.mkv', 'Rollup Recovery', 1, 0, '1', '1');
|
||||
INSERT INTO imm_sessions (
|
||||
session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status,
|
||||
active_watched_ms, lines_seen, tokens_seen, cards_mined, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (
|
||||
1, 'rollup-recovery', 1, '${startedAtMs}', '${endedAtMs}', 2,
|
||||
60000, 10, 20, 2, '${startedAtMs}', '${endedAtMs}'
|
||||
);
|
||||
INSERT INTO imm_session_telemetry (
|
||||
session_id, sample_ms, total_watched_ms, active_watched_ms, lines_seen,
|
||||
tokens_seen, cards_mined, lookup_count, lookup_hits, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (
|
||||
1, '${endedAtMs}', 60000, 60000, 10, 20, 2, 0, 0,
|
||||
'${endedAtMs}', '${endedAtMs}'
|
||||
);
|
||||
DELETE FROM imm_daily_rollups;
|
||||
DELETE FROM imm_monthly_rollups;
|
||||
UPDATE imm_rollup_state SET state_value = '0';
|
||||
`);
|
||||
setupDb.close();
|
||||
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor({ dbPath }, {
|
||||
runLexicalRollupBackfillTask: async () => heldBackfill,
|
||||
} as never);
|
||||
|
||||
const privateApi = tracker as unknown as {
|
||||
db: DatabaseSync;
|
||||
writeLock: { locked: boolean };
|
||||
};
|
||||
assert.equal(privateApi.writeLock.locked, true);
|
||||
assert.equal(
|
||||
(
|
||||
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_daily_rollups').get() as {
|
||||
total: number;
|
||||
}
|
||||
).total,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_monthly_rollups').get() as {
|
||||
total: number;
|
||||
}
|
||||
).total,
|
||||
1,
|
||||
);
|
||||
} finally {
|
||||
releaseBackfill();
|
||||
if (tracker) {
|
||||
await waitForCondition(
|
||||
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
|
||||
);
|
||||
}
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('tracker queues playback writes until lexical rollup backfill settles', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
|
||||
@@ -591,8 +591,8 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
}
|
||||
this.preparedStatements = createTrackerPreparedStatements(this.db);
|
||||
if (!areLexicalDailyRollupsReady(this.db)) this.startLexicalRollupBackfill();
|
||||
this.scheduleMaintenance();
|
||||
if (!areLexicalDailyRollupsReady(this.db)) this.startLexicalRollupBackfill();
|
||||
this.scheduleFlush();
|
||||
}
|
||||
|
||||
|
||||
@@ -184,6 +184,51 @@ test('ensureSchema adds manual assignment locks when upgrading the previous sche
|
||||
}
|
||||
});
|
||||
|
||||
test('ensureSchema preserves durable session rollups across unrelated schema upgrades', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.exec(`
|
||||
INSERT INTO imm_videos (
|
||||
video_id, video_key, canonical_title, source_type, duration_ms, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (1, 'local:/tmp/preserved.mkv', 'Preserved', 1, 0, '1', '1');
|
||||
INSERT INTO imm_daily_rollups (
|
||||
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards
|
||||
) VALUES (20000, 1, 2, 30, 40, 50, 3);
|
||||
INSERT INTO imm_monthly_rollups (
|
||||
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards
|
||||
) VALUES (202410, 1, 2, 30, 40, 50, 3);
|
||||
UPDATE imm_rollup_state
|
||||
SET state_value = '123'
|
||||
WHERE state_key = 'last_rollup_sample_ms';
|
||||
UPDATE imm_schema_version SET schema_version = 21;
|
||||
`);
|
||||
|
||||
ensureSchema(db);
|
||||
|
||||
const daily = db
|
||||
.prepare('SELECT total_sessions AS totalSessions FROM imm_daily_rollups')
|
||||
.get() as { totalSessions: number } | null;
|
||||
const monthly = db
|
||||
.prepare('SELECT total_sessions AS totalSessions FROM imm_monthly_rollups')
|
||||
.get() as { totalSessions: number } | null;
|
||||
const rollupState = db
|
||||
.prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
|
||||
.get('last_rollup_sample_ms') as { value: string } | null;
|
||||
|
||||
assert.equal(daily?.totalSessions, 2);
|
||||
assert.equal(monthly?.totalSessions, 2);
|
||||
assert.equal(rollupState?.value, '123');
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('stats excluded words are replaced and read from sqlite storage', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
@@ -1584,13 +1584,9 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
ON imm_youtube_videos(youtube_video_id)
|
||||
`);
|
||||
|
||||
if (currentVersion?.schema_version && currentVersion.schema_version < SCHEMA_VERSION) {
|
||||
db.exec('DELETE FROM imm_daily_rollups');
|
||||
db.exec('DELETE FROM imm_monthly_rollups');
|
||||
db.exec(
|
||||
`UPDATE imm_rollup_state SET state_value = 0 WHERE state_key = 'last_rollup_sample_ms'`,
|
||||
);
|
||||
}
|
||||
// Session rollups intentionally outlive raw session and telemetry retention.
|
||||
// Preserve them across unrelated schema upgrades because deleted historical
|
||||
// buckets cannot be rebuilt after their source rows have been pruned.
|
||||
|
||||
db.exec(`
|
||||
INSERT INTO imm_schema_version(schema_version, applied_at_ms)
|
||||
|
||||
Reference in New Issue
Block a user