diff --git a/src/core/services/immersion-tracker/lexical-rollups.test.ts b/src/core/services/immersion-tracker/lexical-rollups.test.ts index 2a5c35b6..897b7dfc 100644 --- a/src/core/services/immersion-tracker/lexical-rollups.test.ts +++ b/src/core/services/immersion-tracker/lexical-rollups.test.ts @@ -182,7 +182,7 @@ test('re-seeing a filtered legacy word keeps charts consistent with the summary' } }); -test('lexical rollup rebuild repairs seen bounds left pointing at deleted history', () => { +test('lexical rollup rebuild repairs last_seen left pointing at deleted history', () => { const db = createDb(); try { @@ -197,21 +197,20 @@ test('lexical rollup rebuild repairs seen bounds left pointing at deleted histor .run(sessionId, videoId, BASE_MS, BASE_MS).lastInsertRowid, ); const seen = (BASE_MS + 251) / 1000; - const staleFirst = seen - 86_400.5; + // first_seen this early may be real history pruned by retention; never touched. + const earlyFirst = seen - 86_400.5; const staleLast = seen + 86_400.631; db.prepare( `INSERT INTO imm_words(id, headword, word, reading, first_seen, last_seen, frequency) - VALUES (1, '猫', '猫', 'ねこ', ?, ?, 1), (2, '犬', '犬', 'いぬ', ?, ?, 1), - (3, '鳥', '鳥', 'とり', ?, ?, 5)`, - // Word 3's earlier history was pruned by retention: frequency still counts it. - ).run(staleFirst, staleLast, seen - 0.1, seen - 0.1, staleFirst, seen); + VALUES (1, '猫', '猫', 'ねこ', ?, ?, 1), (2, '犬', '犬', 'いぬ', ?, ?, 1)`, + ).run(earlyFirst, staleLast, seen - 0.1, seen - 0.1); db.prepare( `INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency) VALUES (1, '猫', ?, ?, 1)`, - ).run(staleFirst, staleLast); + ).run(earlyFirst, staleLast); db.prepare( `INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms) - VALUES (?, 1, 1, ?), (?, 2, 1, ?), (?, 3, 1, ?)`, - ).run(lineId, BASE_MS + 251, lineId, BASE_MS + 251, lineId, BASE_MS + 251); + VALUES (?, 1, 1, ?), (?, 2, 1, ?)`, + ).run(lineId, BASE_MS + 251, lineId, BASE_MS + 251); db.prepare( `INSERT INTO imm_kanji_line_occurrences(line_id, kanji_id, occurrence_count, seen_ms) VALUES (?, 1, 1, ?)`, @@ -231,13 +230,12 @@ test('lexical rollup rebuild repairs seen bounds left pointing at deleted histor assert.deepEqual( bounds('imm_words'), [ - [seen, seen], + [earlyFirst, seen], [seen - 0.1, seen - 0.1], - [staleFirst, seen], ], - 'flush-lag and retention-pruned rows keep their first_seen', + 'healthy rows are untouched', ); - assert.deepEqual(bounds('imm_kanji'), [[seen, seen]]); + assert.deepEqual(bounds('imm_kanji'), [[earlyFirst, seen]]); } finally { db.close(); } diff --git a/src/core/services/immersion-tracker/lexical-rollups.ts b/src/core/services/immersion-tracker/lexical-rollups.ts index fec8a8af..073a630d 100644 --- a/src/core/services/immersion-tracker/lexical-rollups.ts +++ b/src/core/services/immersion-tracker/lexical-rollups.ts @@ -19,49 +19,37 @@ const LOCAL_EPOCH_DAY_SQL = ` `; // v3 re-derives visibility for rows a re-seen word forced visible and repairs -// first_seen/last_seen values left behind by session deletes. +// last_seen values left behind by session deletes. const LEXICAL_DAILY_ROLLUP_VERSION = '3'; const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version'; const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000; -// Healthy first_seen trails its oldest seen_ms only by queue-to-flush lag. -const FIRST_SEEN_FLUSH_LAG_SLACK_SEC = 60; export function localEpochDaySql(value: string): string { return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value); } -// Session deletes once compared rounded timestamps against stored bounds and -// could leave first_seen/last_seen pointing at deleted history. Bounds are -// stamped at queue time, before seen_ms is stamped at flush, so healthy rows -// never exceed their newest occurrence and only trail their oldest by flush lag. -// Rows with undated occurrences are skipped. +// Session deletes once rounded removed timestamps to whole seconds, missing +// fractional last_seen extremes that pointed at deleted history. last_seen is +// stamped at queue time, before seen_ms is stamped at flush, and retention +// prunes the oldest sessions first, so healthy rows never exceed their newest +// surviving occurrence. Rows with undated occurrences are skipped. // -// first_seen can also legitimately predate every surviving occurrence: retention -// pruning cascades away old lines, and early history predates the occurrence -// tables, while both keep the lifetime frequency. Deletes subtract frequency -// exactly, so only rows whose frequency matches their surviving occurrences have -// their whole history on hand and a provably stale first_seen. -function repairStaleSeenBounds(db: DatabaseSync): void { +// first_seen is deliberately not repaired: a stale bound is indistinguishable +// from real history whose occurrences retention pruned, and frequency does not +// reliably tell them apart. +function repairStaleLastSeen(db: DatabaseSync): void { for (const [table, occurrenceTable, col] of [ ['imm_words', 'imm_word_line_occurrences', 'word_id'], ['imm_kanji', 'imm_kanji_line_occurrences', 'kanji_id'], ] as const) { - const seenSec = (fn: 'MIN' | 'MAX'): string => - `(SELECT ${fn}(o.seen_ms) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id) / 1000.0`; - const hasNoUndated = `NOT EXISTS ( - SELECT 1 FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id AND o.seen_ms IS NULL - )`; + const newestSeen = `(SELECT MAX(o.seen_ms) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id) / 1000.0`; db.exec(` UPDATE ${table} - SET first_seen = ${seenSec('MIN')} - WHERE first_seen < ${seenSec('MIN')} - ${FIRST_SEEN_FLUSH_LAG_SLACK_SEC} - AND ${hasNoUndated} - AND frequency = ( - SELECT SUM(o.occurrence_count) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id - ); - UPDATE ${table} - SET last_seen = ${seenSec('MAX')} - WHERE last_seen > ${seenSec('MAX')} AND ${hasNoUndated}; + SET last_seen = ${newestSeen} + WHERE last_seen > ${newestSeen} + AND NOT EXISTS ( + SELECT 1 FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id AND o.seen_ms IS NULL + ) `); } } @@ -226,7 +214,7 @@ export function rebuildLexicalDailyRollups(db: DatabaseSync): void { lastId = vocabularyRows[vocabularyRows.length - 1]!.id; if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break; } - repairStaleSeenBounds(db); + repairStaleLastSeen(db); db.exec('DELETE FROM imm_lexical_daily_rollups'); db.exec(` INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)