mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-10-21 05:01:26 -07:00
fix(stats): count mined cards once and keep lexical stats exact after deletes (#280)
This commit is contained in:
1 parent
121b688afb
commit
e77aa5c3b3
24 files changed
+736
-153
No files matched your search
@@ -18,7 +18,9 @@ const LOCAL_EPOCH_DAY_SQL = `
|
||||
) - 2440587.5 AS INTEGER)
|
||||
`;
|
||||
|
||||
const LEXICAL_DAILY_ROLLUP_VERSION = '2';
|
||||
// v3 re-derives visibility for rows a re-seen word forced visible and repairs
|
||||
// last_seen values left behind by session deletes.
|
||||
const LEXICAL_DAILY_ROLLUP_VERSION = '3';
|
||||
const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
|
||||
const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
|
||||
|
||||
@@ -26,6 +28,32 @@ export function localEpochDaySql(value: string): string {
|
||||
return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
|
||||
}
|
||||
|
||||
// Session deletes once rounded removed timestamps to whole seconds, missing
|
||||
// fractional last_seen extremes that pointed at deleted history. New seen_ms
|
||||
// values match observation time; legacy values were stamped later at flush.
|
||||
// Retention prunes the oldest sessions first, so healthy rows never exceed
|
||||
// their newest surviving occurrence. Rows with undated occurrences are skipped.
|
||||
//
|
||||
// first_seen is deliberately not repaired: a stale bound is indistinguishable
|
||||
// from real history whose occurrences retention pruned, and frequency does not
|
||||
// reliably tell them apart.
|
||||
function repairStaleLastSeen(db: DatabaseSync): void {
|
||||
for (const [table, occurrenceTable, col] of [
|
||||
['imm_words', 'imm_word_line_occurrences', 'word_id'],
|
||||
['imm_kanji', 'imm_kanji_line_occurrences', 'kanji_id'],
|
||||
] as const) {
|
||||
const newestSeen = `(SELECT MAX(o.seen_ms) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id) / 1000.0`;
|
||||
db.exec(`
|
||||
UPDATE ${table}
|
||||
SET last_seen = ${newestSeen}
|
||||
WHERE last_seen > ${newestSeen}
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id AND o.seen_ms IS NULL
|
||||
)
|
||||
`);
|
||||
}
|
||||
}
|
||||
|
||||
function createWordRollupTriggers(db: DatabaseSync): void {
|
||||
const dayForNew = localEpochDaySql('NEW.first_seen');
|
||||
const dayForOld = localEpochDaySql('OLD.first_seen');
|
||||
@@ -186,6 +214,7 @@ export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
|
||||
lastId = vocabularyRows[vocabularyRows.length - 1]!.id;
|
||||
if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break;
|
||||
}
|
||||
repairStaleLastSeen(db);
|
||||
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||
db.exec(`
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
|
||||
Reference in new issue
Block a user