diff --git a/src/core/services/immersion-tracker/lexical-rollups.test.ts b/src/core/services/immersion-tracker/lexical-rollups.test.ts index c26c59a2..2a5c35b6 100644 --- a/src/core/services/immersion-tracker/lexical-rollups.test.ts +++ b/src/core/services/immersion-tracker/lexical-rollups.test.ts @@ -201,15 +201,17 @@ test('lexical rollup rebuild repairs seen bounds left pointing at deleted histor const staleLast = seen + 86_400.631; db.prepare( `INSERT INTO imm_words(id, headword, word, reading, first_seen, last_seen, frequency) - VALUES (1, '猫', '猫', 'ねこ', ?, ?, 1), (2, '犬', '犬', 'いぬ', ?, ?, 1)`, - ).run(staleFirst, staleLast, seen - 0.1, seen - 0.1); + VALUES (1, '猫', '猫', 'ねこ', ?, ?, 1), (2, '犬', '犬', 'いぬ', ?, ?, 1), + (3, '鳥', '鳥', 'とり', ?, ?, 5)`, + // Word 3's earlier history was pruned by retention: frequency still counts it. + ).run(staleFirst, staleLast, seen - 0.1, seen - 0.1, staleFirst, seen); db.prepare( `INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency) VALUES (1, '猫', ?, ?, 1)`, ).run(staleFirst, staleLast); db.prepare( `INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms) - VALUES (?, 1, 1, ?), (?, 2, 1, ?)`, - ).run(lineId, BASE_MS + 251, lineId, BASE_MS + 251); + VALUES (?, 1, 1, ?), (?, 2, 1, ?), (?, 3, 1, ?)`, + ).run(lineId, BASE_MS + 251, lineId, BASE_MS + 251, lineId, BASE_MS + 251); db.prepare( `INSERT INTO imm_kanji_line_occurrences(line_id, kanji_id, occurrence_count, seen_ms) VALUES (?, 1, 1, ?)`, @@ -231,8 +233,9 @@ test('lexical rollup rebuild repairs seen bounds left pointing at deleted histor [ [seen, seen], [seen - 0.1, seen - 0.1], + [staleFirst, seen], ], - 'healthy rows within flush lag are untouched', + 'flush-lag and retention-pruned rows keep their first_seen', ); assert.deepEqual(bounds('imm_kanji'), [[seen, seen]]); } finally { diff --git a/src/core/services/immersion-tracker/lexical-rollups.ts b/src/core/services/immersion-tracker/lexical-rollups.ts index 79e421a5..fec8a8af 100644 --- a/src/core/services/immersion-tracker/lexical-rollups.ts +++ b/src/core/services/immersion-tracker/lexical-rollups.ts @@ -35,6 +35,12 @@ export function localEpochDaySql(value: string): string { // stamped at queue time, before seen_ms is stamped at flush, so healthy rows // never exceed their newest occurrence and only trail their oldest by flush lag. // Rows with undated occurrences are skipped. +// +// first_seen can also legitimately predate every surviving occurrence: retention +// pruning cascades away old lines, and early history predates the occurrence +// tables, while both keep the lifetime frequency. Deletes subtract frequency +// exactly, so only rows whose frequency matches their surviving occurrences have +// their whole history on hand and a provably stale first_seen. function repairStaleSeenBounds(db: DatabaseSync): void { for (const [table, occurrenceTable, col] of [ ['imm_words', 'imm_word_line_occurrences', 'word_id'], @@ -48,7 +54,11 @@ function repairStaleSeenBounds(db: DatabaseSync): void { db.exec(` UPDATE ${table} SET first_seen = ${seenSec('MIN')} - WHERE first_seen < ${seenSec('MIN')} - ${FIRST_SEEN_FLUSH_LAG_SLACK_SEC} AND ${hasNoUndated}; + WHERE first_seen < ${seenSec('MIN')} - ${FIRST_SEEN_FLUSH_LAG_SLACK_SEC} + AND ${hasNoUndated} + AND frequency = ( + SELECT SUM(o.occurrence_count) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id + ); UPDATE ${table} SET last_seen = ${seenSec('MAX')} WHERE last_seen > ${seenSec('MAX')} AND ${hasNoUndated};