fix(stats): count mined cards once and keep lexical stats exact after deletes (#280)

This commit is contained in:
sudacode authored and GitHub committed 2026-10-03 21:16:12 -07:00
1 parent 121b688afb
commit e77aa5c3b3
24 files changed
+736 -153

No files matched your search

@@ -18,7 +18,9 @@ const LOCAL_EPOCH_DAY_SQL = `
) - 2440587.5 AS INTEGER)
`;
const LEXICAL_DAILY_ROLLUP_VERSION = '2';
// v3 re-derives visibility for rows a re-seen word forced visible and repairs
// last_seen values left behind by session deletes.
const LEXICAL_DAILY_ROLLUP_VERSION = '3';
const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
@@ -26,6 +28,32 @@ export function localEpochDaySql(value: string): string {
return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
}
// Session deletes once rounded removed timestamps to whole seconds, missing
// fractional last_seen extremes that pointed at deleted history. New seen_ms
// values match observation time; legacy values were stamped later at flush.
// Retention prunes the oldest sessions first, so healthy rows never exceed
// their newest surviving occurrence. Rows with undated occurrences are skipped.
//
// first_seen is deliberately not repaired: a stale bound is indistinguishable
// from real history whose occurrences retention pruned, and frequency does not
// reliably tell them apart.
function repairStaleLastSeen(db: DatabaseSync): void {
for (const [table, occurrenceTable, col] of [
['imm_words', 'imm_word_line_occurrences', 'word_id'],
['imm_kanji', 'imm_kanji_line_occurrences', 'kanji_id'],
] as const) {
const newestSeen = `(SELECT MAX(o.seen_ms) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id) / 1000.0`;
db.exec(`
UPDATE ${table}
SET last_seen = ${newestSeen}
WHERE last_seen > ${newestSeen}
AND NOT EXISTS (
SELECT 1 FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id AND o.seen_ms IS NULL
)
`);
}
}
function createWordRollupTriggers(db: DatabaseSync): void {
const dayForNew = localEpochDaySql('NEW.first_seen');
const dayForOld = localEpochDaySql('OLD.first_seen');
@@ -186,6 +214,7 @@ export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
lastId = vocabularyRows[vocabularyRows.length - 1]!.id;
if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break;
}
repairStaleLastSeen(db);
db.exec('DELETE FROM imm_lexical_daily_rollups');
db.exec(`
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)