diff --git a/src/core/services/immersion-tracker/__tests__/lexical-removal.test.ts b/src/core/services/immersion-tracker/__tests__/lexical-removal.test.ts index 33de22b2..7b927dc7 100644 --- a/src/core/services/immersion-tracker/__tests__/lexical-removal.test.ts +++ b/src/core/services/immersion-tracker/__tests__/lexical-removal.test.ts @@ -232,6 +232,26 @@ for (const legacyRows of [false, true]) { }); } +test('deleting the earliest session moves first_seen even when the stored bound predates seen_ms', () => { + const { db, dbPath } = createDb([ + { session: 1, wordId: 23, dayOffset: 0, msOffset: 400 }, + { session: 2, wordId: 23, dayOffset: 5, msOffset: 400 }, + ]); + + try { + // first_seen is stamped when the line is queued (and older refreshes rounded + // it down), so it sits slightly before the occurrence's flush-time seen_ms. + db.prepare('UPDATE imm_words SET first_seen = ? WHERE id = 23').run(BASE_MS / 1000); + + deleteSession(db, 1); + + assert.equal(readWord(db, 23)?.firstSeen, (BASE_MS + 5 * DAY_MS + 400) / 1000); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('deleting an interior occurrence leaves the surrounding extremes untouched', () => { // Session 2 carries the middle occurrence; sessions bracket it in time. const { db, dbPath } = createDb([ diff --git a/src/core/services/immersion-tracker/lexical-rollups.test.ts b/src/core/services/immersion-tracker/lexical-rollups.test.ts index 179544a3..c26c59a2 100644 --- a/src/core/services/immersion-tracker/lexical-rollups.test.ts +++ b/src/core/services/immersion-tracker/lexical-rollups.test.ts @@ -182,7 +182,7 @@ test('re-seeing a filtered legacy word keeps charts consistent with the summary' } }); -test('lexical rollup rebuild repairs last_seen left pointing at deleted history', () => { +test('lexical rollup rebuild repairs seen bounds left pointing at deleted history', () => { const db = createDb(); try { @@ -197,14 +197,15 @@ test('lexical rollup rebuild repairs last_seen left pointing at deleted history' .run(sessionId, videoId, BASE_MS, BASE_MS).lastInsertRowid, ); const seen = (BASE_MS + 251) / 1000; - const stale = seen + 86_400.631; + const staleFirst = seen - 86_400.5; + const staleLast = seen + 86_400.631; db.prepare( `INSERT INTO imm_words(id, headword, word, reading, first_seen, last_seen, frequency) VALUES (1, '猫', '猫', 'ねこ', ?, ?, 1), (2, '犬', '犬', 'いぬ', ?, ?, 1)`, - ).run(seen, stale, seen - 0.1, seen - 0.1); + ).run(staleFirst, staleLast, seen - 0.1, seen - 0.1); db.prepare( `INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency) VALUES (1, '猫', ?, ?, 1)`, - ).run(seen, stale); + ).run(staleFirst, staleLast); db.prepare( `INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms) VALUES (?, 1, 1, ?), (?, 2, 1, ?)`, @@ -216,14 +217,24 @@ test('lexical rollup rebuild repairs last_seen left pointing at deleted history' rebuildLexicalDailyRollups(db); - const lastSeen = (table: string): number[] => + const bounds = (table: string): Array<[number, number]> => ( - db.prepare(`SELECT last_seen AS value FROM ${table} ORDER BY id`).all() as Array<{ - value: number; + db + .prepare(`SELECT first_seen AS first, last_seen AS last FROM ${table} ORDER BY id`) + .all() as Array<{ + first: number; + last: number; }> - ).map((row) => row.value); - assert.deepEqual(lastSeen('imm_words'), [seen, seen - 0.1], 'healthy rows are untouched'); - assert.deepEqual(lastSeen('imm_kanji'), [seen]); + ).map((row) => [row.first, row.last]); + assert.deepEqual( + bounds('imm_words'), + [ + [seen, seen], + [seen - 0.1, seen - 0.1], + ], + 'healthy rows within flush lag are untouched', + ); + assert.deepEqual(bounds('imm_kanji'), [[seen, seen]]); } finally { db.close(); } diff --git a/src/core/services/immersion-tracker/lexical-rollups.ts b/src/core/services/immersion-tracker/lexical-rollups.ts index a09f5383..79e421a5 100644 --- a/src/core/services/immersion-tracker/lexical-rollups.ts +++ b/src/core/services/immersion-tracker/lexical-rollups.ts @@ -19,32 +19,39 @@ const LOCAL_EPOCH_DAY_SQL = ` `; // v3 re-derives visibility for rows a re-seen word forced visible and repairs -// last_seen values left behind by session deletes. +// first_seen/last_seen values left behind by session deletes. const LEXICAL_DAILY_ROLLUP_VERSION = '3'; const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version'; const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000; +// Healthy first_seen trails its oldest seen_ms only by queue-to-flush lag. +const FIRST_SEEN_FLUSH_LAG_SLACK_SEC = 60; export function localEpochDaySql(value: string): string { return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value); } -// Session deletes once rounded removed timestamps to whole seconds, missing -// fractional last_seen extremes that pointed at deleted history. Healthy rows -// never exceed their newest dated occurrence (seen_ms is stamped at flush, after -// last_seen), so only stale rows match. Rows with undated occurrences are skipped. -function repairStaleLastSeen(db: DatabaseSync): void { +// Session deletes once compared rounded timestamps against stored bounds and +// could leave first_seen/last_seen pointing at deleted history. Bounds are +// stamped at queue time, before seen_ms is stamped at flush, so healthy rows +// never exceed their newest occurrence and only trail their oldest by flush lag. +// Rows with undated occurrences are skipped. +function repairStaleSeenBounds(db: DatabaseSync): void { for (const [table, occurrenceTable, col] of [ ['imm_words', 'imm_word_line_occurrences', 'word_id'], ['imm_kanji', 'imm_kanji_line_occurrences', 'kanji_id'], ] as const) { - const newestSeen = `(SELECT MAX(o.seen_ms) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id) / 1000.0`; + const seenSec = (fn: 'MIN' | 'MAX'): string => + `(SELECT ${fn}(o.seen_ms) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id) / 1000.0`; + const hasNoUndated = `NOT EXISTS ( + SELECT 1 FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id AND o.seen_ms IS NULL + )`; db.exec(` UPDATE ${table} - SET last_seen = ${newestSeen} - WHERE last_seen > ${newestSeen} - AND NOT EXISTS ( - SELECT 1 FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id AND o.seen_ms IS NULL - ) + SET first_seen = ${seenSec('MIN')} + WHERE first_seen < ${seenSec('MIN')} - ${FIRST_SEEN_FLUSH_LAG_SLACK_SEC} AND ${hasNoUndated}; + UPDATE ${table} + SET last_seen = ${seenSec('MAX')} + WHERE last_seen > ${seenSec('MAX')} AND ${hasNoUndated}; `); } } @@ -209,7 +216,7 @@ export function rebuildLexicalDailyRollups(db: DatabaseSync): void { lastId = vocabularyRows[vocabularyRows.length - 1]!.id; if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break; } - repairStaleLastSeen(db); + repairStaleSeenBounds(db); db.exec('DELETE FROM imm_lexical_daily_rollups'); db.exec(` INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) diff --git a/src/core/services/immersion-tracker/query-shared.ts b/src/core/services/immersion-tracker/query-shared.ts index 78ecda16..8335f57f 100644 --- a/src/core/services/immersion-tracker/query-shared.ts +++ b/src/core/services/immersion-tracker/query-shared.ts @@ -382,11 +382,16 @@ function applyRemovalsForEntity( continue; } - const removedFirstSeen = toStoredSeenSeconds(removal.removedFirstSeenMs); + // Stored bounds are stamped when a line is queued, before its seen_ms is + // stamped at flush, and older refreshes rounded them down to whole seconds. + // So first_seen can sit just before the occurrence that produced it; decide + // against the surviving minimum instead. last_seen errs the safe way. + const minSeenMs = (minSeenStmt.get(removal.id) as { value: number | null }).value; const removedLastSeen = toStoredSeenSeconds(removal.removedLastSeenMs); const firstSeenMayHaveMoved = current.firstSeen === null || - (removedFirstSeen !== null && removedFirstSeen <= current.firstSeen); + (removal.removedFirstSeenMs !== null && + (minSeenMs === null || Number(removal.removedFirstSeenMs) <= Number(minSeenMs))); const lastSeenMayHaveMoved = current.lastSeen === null || (removedLastSeen !== null && removedLastSeen >= current.lastSeen); @@ -397,7 +402,6 @@ function applyRemovalsForEntity( needsExactRefresh.push(removal.id); continue; } - const minSeenMs = (minSeenStmt.get(removal.id) as { value: number | null }).value; const maxSeenMs = (maxSeenStmt.get(removal.id) as { value: number | null }).value; if (minSeenMs === null || maxSeenMs === null) { // Frequency says occurrences remain but none exist: stale row, let the