fix(stats): move first_seen correctly when deleting earliest session

- Decide first_seen movement against the surviving min seen_ms, since
  stored bounds are stamped at queue time and may predate seen_ms
- Extend v3 rollup repair to fix stale first_seen as well as last_seen,
  allowing slack for queue-to-flush lag
This commit is contained in:
sudacode committed 2026-10-01 23:12:09 -07:00
1 parent 4677f5624b
commit ce680b746a
4 files changed
+68 -26

No files matched your search

@@ -232,6 +232,26 @@ for (const legacyRows of [false, true]) {
});
}
test('deleting the earliest session moves first_seen even when the stored bound predates seen_ms', () => {
const { db, dbPath } = createDb([
{ session: 1, wordId: 23, dayOffset: 0, msOffset: 400 },
{ session: 2, wordId: 23, dayOffset: 5, msOffset: 400 },
]);
try {
// first_seen is stamped when the line is queued (and older refreshes rounded
// it down), so it sits slightly before the occurrence's flush-time seen_ms.
db.prepare('UPDATE imm_words SET first_seen = ? WHERE id = 23').run(BASE_MS / 1000);
deleteSession(db, 1);
assert.equal(readWord(db, 23)?.firstSeen, (BASE_MS + 5 * DAY_MS + 400) / 1000);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('deleting an interior occurrence leaves the surrounding extremes untouched', () => {
// Session 2 carries the middle occurrence; sessions bracket it in time.
const { db, dbPath } = createDb([
@@ -182,7 +182,7 @@ test('re-seeing a filtered legacy word keeps charts consistent with the summary'
}
});
test('lexical rollup rebuild repairs last_seen left pointing at deleted history', () => {
test('lexical rollup rebuild repairs seen bounds left pointing at deleted history', () => {
const db = createDb();
try {
@@ -197,14 +197,15 @@ test('lexical rollup rebuild repairs last_seen left pointing at deleted history'
.run(sessionId, videoId, BASE_MS, BASE_MS).lastInsertRowid,
);
const seen = (BASE_MS + 251) / 1000;
const stale = seen + 86_400.631;
const staleFirst = seen - 86_400.5;
const staleLast = seen + 86_400.631;
db.prepare(
`INSERT INTO imm_words(id, headword, word, reading, first_seen, last_seen, frequency)
VALUES (1, '猫', '猫', 'ねこ', ?, ?, 1), (2, '犬', '犬', 'いぬ', ?, ?, 1)`,
).run(seen, stale, seen - 0.1, seen - 0.1);
).run(staleFirst, staleLast, seen - 0.1, seen - 0.1);
db.prepare(
`INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency) VALUES (1, '猫', ?, ?, 1)`,
).run(seen, stale);
).run(staleFirst, staleLast);
db.prepare(
`INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms)
VALUES (?, 1, 1, ?), (?, 2, 1, ?)`,
@@ -216,14 +217,24 @@ test('lexical rollup rebuild repairs last_seen left pointing at deleted history'
rebuildLexicalDailyRollups(db);
const lastSeen = (table: string): number[] =>
const bounds = (table: string): Array<[number, number]> =>
(
db.prepare(`SELECT last_seen AS value FROM ${table} ORDER BY id`).all() as Array<{
value: number;
db
.prepare(`SELECT first_seen AS first, last_seen AS last FROM ${table} ORDER BY id`)
.all() as Array<{
first: number;
last: number;
}>
).map((row) => row.value);
assert.deepEqual(lastSeen('imm_words'), [seen, seen - 0.1], 'healthy rows are untouched');
assert.deepEqual(lastSeen('imm_kanji'), [seen]);
).map((row) => [row.first, row.last]);
assert.deepEqual(
bounds('imm_words'),
[
[seen, seen],
[seen - 0.1, seen - 0.1],
],
'healthy rows within flush lag are untouched',
);
assert.deepEqual(bounds('imm_kanji'), [[seen, seen]]);
} finally {
db.close();
}
@@ -19,32 +19,39 @@ const LOCAL_EPOCH_DAY_SQL = `
`;
// v3 re-derives visibility for rows a re-seen word forced visible and repairs
// last_seen values left behind by session deletes.
// first_seen/last_seen values left behind by session deletes.
const LEXICAL_DAILY_ROLLUP_VERSION = '3';
const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
// Healthy first_seen trails its oldest seen_ms only by queue-to-flush lag.
const FIRST_SEEN_FLUSH_LAG_SLACK_SEC = 60;
export function localEpochDaySql(value: string): string {
return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
}
// Session deletes once rounded removed timestamps to whole seconds, missing
// fractional last_seen extremes that pointed at deleted history. Healthy rows
// never exceed their newest dated occurrence (seen_ms is stamped at flush, after
// last_seen), so only stale rows match. Rows with undated occurrences are skipped.
function repairStaleLastSeen(db: DatabaseSync): void {
// Session deletes once compared rounded timestamps against stored bounds and
// could leave first_seen/last_seen pointing at deleted history. Bounds are
// stamped at queue time, before seen_ms is stamped at flush, so healthy rows
// never exceed their newest occurrence and only trail their oldest by flush lag.
// Rows with undated occurrences are skipped.
function repairStaleSeenBounds(db: DatabaseSync): void {
for (const [table, occurrenceTable, col] of [
['imm_words', 'imm_word_line_occurrences', 'word_id'],
['imm_kanji', 'imm_kanji_line_occurrences', 'kanji_id'],
] as const) {
const newestSeen = `(SELECT MAX(o.seen_ms) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id) / 1000.0`;
const seenSec = (fn: 'MIN' | 'MAX'): string =>
`(SELECT ${fn}(o.seen_ms) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id) / 1000.0`;
const hasNoUndated = `NOT EXISTS (
SELECT 1 FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id AND o.seen_ms IS NULL
)`;
db.exec(`
UPDATE ${table}
SET last_seen = ${newestSeen}
WHERE last_seen > ${newestSeen}
AND NOT EXISTS (
SELECT 1 FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id AND o.seen_ms IS NULL
)
SET first_seen = ${seenSec('MIN')}
WHERE first_seen < ${seenSec('MIN')} - ${FIRST_SEEN_FLUSH_LAG_SLACK_SEC} AND ${hasNoUndated};
UPDATE ${table}
SET last_seen = ${seenSec('MAX')}
WHERE last_seen > ${seenSec('MAX')} AND ${hasNoUndated};
`);
}
}
@@ -209,7 +216,7 @@ export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
lastId = vocabularyRows[vocabularyRows.length - 1]!.id;
if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break;
}
repairStaleLastSeen(db);
repairStaleSeenBounds(db);
db.exec('DELETE FROM imm_lexical_daily_rollups');
db.exec(`
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
@@ -382,11 +382,16 @@ function applyRemovalsForEntity(
continue;
}
const removedFirstSeen = toStoredSeenSeconds(removal.removedFirstSeenMs);
// Stored bounds are stamped when a line is queued, before its seen_ms is
// stamped at flush, and older refreshes rounded them down to whole seconds.
// So first_seen can sit just before the occurrence that produced it; decide
// against the surviving minimum instead. last_seen errs the safe way.
const minSeenMs = (minSeenStmt.get(removal.id) as { value: number | null }).value;
const removedLastSeen = toStoredSeenSeconds(removal.removedLastSeenMs);
const firstSeenMayHaveMoved =
current.firstSeen === null ||
(removedFirstSeen !== null && removedFirstSeen <= current.firstSeen);
(removal.removedFirstSeenMs !== null &&
(minSeenMs === null || Number(removal.removedFirstSeenMs) <= Number(minSeenMs)));
const lastSeenMayHaveMoved =
current.lastSeen === null ||
(removedLastSeen !== null && removedLastSeen >= current.lastSeen);
@@ -397,7 +402,6 @@ function applyRemovalsForEntity(
needsExactRefresh.push(removal.id);
continue;
}
const minSeenMs = (minSeenStmt.get(removal.id) as { value: number | null }).value;
const maxSeenMs = (maxSeenStmt.get(removal.id) as { value: number | null }).value;
if (minSeenMs === null || maxSeenMs === null) {
// Frequency says occurrences remain but none exist: stale row, let the