fix(stats): preserve pruned first_seen and stamp seen_ms at observation

- Record occurrence seen_ms from the observation time instead of flush time
- On session delete, only move first_seen when removed timestamps prove it was deleted; keep earlier dates from retention-pruned or legacy history
- Exact refresh fallback keeps subtraction-exact frequency and preserved first_seen
- Add tests for pruned history, legacy rows, and delayed flushes; update docs and change fragments

changes/stats-mining-and-delete-accuracy.md

@@ -2,6 +2,6 @@ type: fixed
 area: stats
 
 - Fixed sentence mining (single and multi-line) counting each created card twice in session and lifetime stats.
-- Fixed deleting a session leaving word and kanji "last seen" dates pointing at the deleted session; affected rows are repaired once in the background on next launch.
+- Fixed word and kanji dates after session deletion. Historical "first seen" dates and counts survive retention pruning; new occurrences record observation time so deletion can update dates exactly. Legacy earlier dates are preserved when the remaining history cannot prove their origin. Stale "last seen" dates are repaired once in the background on next launch.
 - Fixed re-seen words that the vocabulary filter hides still being counted in the new-words charts, so charts and the vocabulary summary now agree. Existing mismatches are repaired by the same background pass.
 - Fixed Overview totals, charts, activity calendar, and vocabulary summary staying stale after deleting sessions. Failed refreshes keep the dashboard visible and offer a retry.

changes/stats-overview-refresh-docs.md

@@ -2,3 +2,4 @@ type: docs
 area: stats
 
 - Documented the automatic Overview refresh after deleting sessions or session groups and retrying failed refreshes.
+- Documented discovery-date preservation after retention pruning and the handling of older write-time timestamps.

docs-site/immersion-tracking.md

@@ -124,4 +124,6 @@ Leave out `--lookback-days` to scan all history. Word and kanji counts are corre
 
 By default SubMiner keeps everything. To limit history, set `immersionTracking.retentionPreset` to `minimal`, `balanced`, or `deep-history`, or set the `immersionTracking.retention.*Days` values yourself (`0` keeps all). Lifetime totals and vocabulary counts are stored separately and stay exact when old sessions are pruned.
 
+Deleting retained sessions preserves earlier word and kanji discovery dates from pruned history. New occurrences record observation time so deletion can update their dates exactly. Older history used write time, so an earlier discovery date is preserved when its origin cannot be determined.
+
 See [Immersion tracking](/configuration#immersion-tracking) and [Stats dashboard](/configuration#stats-dashboard) in the config reference for every option and default.

src/core/services/immersion-tracker/__tests__/lexical-removal.test.ts

@@ -5,8 +5,9 @@ import path from 'node:path';
 import test from 'node:test';
 import { Database } from '../sqlite.js';
 import type { DatabaseSync } from '../sqlite.js';
-import { ensureSchema } from '../storage.js';
+import { createTrackerPreparedStatements, ensureSchema, executeQueuedWrite } from '../storage.js';
 import { deleteSession, deleteSessions, deleteVideo } from '../query-maintenance.js';
+import { pruneRawRetention } from '../maintenance.js';
 
 const DAY_MS = 86_400_000;
 const BASE_MS = 1_700_000_000_000;
@@ -232,20 +233,135 @@ for (const legacyRows of [false, true]) {
   });
 }
 
-test('deleting the earliest session moves first_seen even when the stored bound predates seen_ms', () => {
+test('deleting legacy history preserves first_seen when it predates the flush timestamp', () => {
   const { db, dbPath } = createDb([
     { session: 1, wordId: 23, dayOffset: 0, msOffset: 400 },
     { session: 2, wordId: 23, dayOffset: 5, msOffset: 400 },
   ]);
 
   try {
-    // first_seen is stamped when the line is queued (and older refreshes rounded
+    // A queue delay and retention-pruned history cannot be distinguished from
-    // it down), so it sits slightly before the occurrence's flush-time seen_ms.
+    // the old flush-time timestamp. Preserve the earlier date in either case.
     db.prepare('UPDATE imm_words SET first_seen = ? WHERE id = 23').run(BASE_MS / 1000);
 
     deleteSession(db, 1);
 
-    assert.equal(readWord(db, 23)?.firstSeen, (BASE_MS + 5 * DAY_MS + 400) / 1000);
+    assert.equal(readWord(db, 23)?.firstSeen, BASE_MS / 1000);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+for (const legacyRows of [false, true]) {
+  for (const deletedSession of [2, 3]) {
+    test(`deleting retained session ${deletedSession} preserves pruned first_seen (legacyRows=${legacyRows})`, () => {
+      const { db, dbPath } = createDb(
+        [
+          { session: 1, wordId: 24, dayOffset: 0 },
+          { session: 2, wordId: 24, dayOffset: 1 },
+        ],
+        { legacyRows },
+      );
+
+      try {
+        const latestMs = BASE_MS + 2 * DAY_MS;
+        db.exec(`
+          INSERT INTO imm_sessions(session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status, CREATED_DATE, LAST_UPDATE_DATE)
+            VALUES (3, 's3', 2, '${latestMs}', '${latestMs + 1000}', 2, ${latestMs}, ${latestMs});
+          INSERT INTO imm_subtitle_lines(line_id, session_id, video_id, line_index, text, CREATED_DATE, LAST_UPDATE_DATE)
+            VALUES (3, 3, 2, 3, '猫', ${latestMs}, ${latestMs});
+          INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms)
+            VALUES (3, 24, 1, ${legacyRows ? 'NULL' : latestMs});
+          UPDATE imm_words SET frequency = 3, last_seen = ${latestMs / 1000} WHERE id = 24;
+          INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency)
+            SELECT 24, '猫', first_seen, last_seen, frequency FROM imm_words WHERE id = 24;
+          INSERT INTO imm_kanji_line_occurrences(line_id, kanji_id, occurrence_count, seen_ms)
+            SELECT line_id, 24, occurrence_count, seen_ms FROM imm_word_line_occurrences WHERE word_id = 24;
+        `);
+        pruneRawRetention(db, latestMs, {
+          eventsRetentionMs: Infinity,
+          telemetryRetentionMs: Infinity,
+          sessionsRetentionMs: 1.5 * DAY_MS,
+        });
+
+        deleteSession(db, deletedSession);
+
+        for (const table of ['imm_words', 'imm_kanji']) {
+          const row = db
+            .prepare(
+              `SELECT frequency, first_seen AS firstSeen, last_seen AS lastSeen FROM ${table} WHERE id = 24`,
+            )
+            .get() as { frequency: number; firstSeen: number; lastSeen: number };
+          assert.equal(row.frequency, 2);
+          assert.equal(row.firstSeen, BASE_MS / 1000, `${table} keeps the pruned discovery date`);
+          assert.equal(row.lastSeen, (BASE_MS + (deletedSession === 2 ? 2 : 1) * DAY_MS) / 1000);
+        }
+      } finally {
+        db.close();
+        cleanupDbPath(dbPath);
+      }
+    });
+  }
+}
+
+test('deleting a queued subtitle uses observation time despite a delayed flush', () => {
+  const { db, dbPath } = createDb([]);
+
+  try {
+    const stmts = createTrackerPreparedStatements(db);
+    for (const sessionId of [1, 2]) {
+      const seenMs = BASE_MS + (sessionId - 1) * DAY_MS + 400;
+      executeQueuedWrite(
+        {
+          kind: 'subtitleLine',
+          sessionId,
+          videoId: sessionId,
+          lineIndex: 1,
+          segmentStartMs: null,
+          segmentEndMs: null,
+          text: '猫',
+          wordOccurrences: [
+            {
+              headword: '猫',
+              word: '猫',
+              reading: 'ねこ',
+              partOfSpeech: 'noun',
+              pos1: '名詞',
+              pos2: '一般',
+              pos3: '',
+              occurrenceCount: 1,
+              frequencyRank: null,
+            },
+          ],
+          kanjiOccurrences: [{ kanji: '猫', occurrenceCount: 1 }],
+          firstSeen: seenMs / 1000,
+          lastSeen: seenMs / 1000,
+        },
+        stmts,
+      );
+      for (const table of ['imm_word_line_occurrences', 'imm_kanji_line_occurrences']) {
+        const row = db
+          .prepare(`SELECT seen_ms AS seenMs FROM ${table} WHERE line_id = ?`)
+          .get(sessionId) as { seenMs: number };
+        assert.equal(row.seenMs, seenMs);
+      }
+    }
+
+    deleteSession(db, 1);
+
+    for (const table of ['imm_words', 'imm_kanji']) {
+      const row = db
+        .prepare(`SELECT frequency, first_seen AS firstSeen, last_seen AS lastSeen FROM ${table}`)
+        .get() as {
+        frequency: number;
+        firstSeen: number;
+        lastSeen: number;
+      };
+      assert.equal(row.frequency, 1);
+      assert.equal(row.firstSeen, (BASE_MS + DAY_MS + 400) / 1000);
+      assert.equal(row.lastSeen, (BASE_MS + DAY_MS + 400) / 1000);
+    }
   } finally {
     db.close();
     cleanupDbPath(dbPath);

src/core/services/immersion-tracker/lexical-rollups.ts

@@ -29,10 +29,10 @@ export function localEpochDaySql(value: string): string {
 }
 
 // Session deletes once rounded removed timestamps to whole seconds, missing
-// fractional last_seen extremes that pointed at deleted history. last_seen is
+// fractional last_seen extremes that pointed at deleted history. New seen_ms
-// stamped at queue time, before seen_ms is stamped at flush, and retention
+// values match observation time; legacy values were stamped later at flush.
-// prunes the oldest sessions first, so healthy rows never exceed their newest
+// Retention prunes the oldest sessions first, so healthy rows never exceed
-// surviving occurrence. Rows with undated occurrences are skipped.
+// their newest surviving occurrence. Rows with undated occurrences are skipped.
 //
 // first_seen is deliberately not repaired: a stale bound is indistinguishable
 // from real history whose occurrences retention pruned, and frequency does not

src/core/services/immersion-tracker/query-shared.ts

@@ -322,6 +322,13 @@ export function applyLexicalRemovals(db: DatabaseSync, plan: LexicalRemovalPlan)
   applyRemovalsForEntity(db, 'kanji', plan.kanji);
 }
 
+// A timestamp repair must keep subtraction-exact lifetime counts and any
+// first_seen date that the deleted occurrences cannot account for.
+type LexicalAggregateOverride = {
+  frequency: number | null;
+  firstSeen: number | null;
+};
+
 function applyRemovalsForEntity(
   db: DatabaseSync,
   entity: LexicalEntity,
@@ -360,7 +367,7 @@ function applyRemovalsForEntity(
     `UPDATE ${entityTable} SET frequency = ?, first_seen = ?, last_seen = ? WHERE id = ?`,
   );
 
-  const needsExactRefresh: number[] = [];
+  const needsExactRefresh = new Map<number, LexicalAggregateOverride>();
 
   for (const removal of removals) {
     const current = selectStmt.get(removal.id) as {
@@ -371,47 +378,50 @@ function applyRemovalsForEntity(
     if (!current) continue;
 
     const nextFrequency = (current.frequency ?? 0) - removal.removedFrequency;
+    // Legacy occurrences use flush time, which can follow the actual observation.
+    // An earlier stored date may also belong to retention-pruned history. Only
+    // move it when the removed timestamps prove that its observation was deleted.
+    const removedFirstSeen = toStoredSeenSeconds(removal.removedFirstSeenMs);
+    const removedLastSeen = toStoredSeenSeconds(removal.removedLastSeenMs);
+    const firstSeenMayHaveMoved =
+      current.firstSeen === null ||
+      (removedFirstSeen !== null && removedFirstSeen <= current.firstSeen);
+    const lastSeenMayHaveMoved =
+      current.lastSeen === null ||
+      (removedLastSeen !== null && removedLastSeen >= current.lastSeen);
+    const exactRefreshOverride = {
+      frequency: nextFrequency > 0 ? nextFrequency : null,
+      firstSeen: firstSeenMayHaveMoved ? null : current.firstSeen,
+    };
     if (nextFrequency <= 0) {
       // The rows in scope are already gone by now, so anything still pointing at
       // this entity means the stored frequency was stale rather than exhausted.
       if (hasOccurrencesStmt.get(removal.id)) {
-        needsExactRefresh.push(removal.id);
+        needsExactRefresh.set(removal.id, exactRefreshOverride);
       } else {
         deleteStmt.run(removal.id);
       }
       continue;
     }
 
-    // Stored bounds are stamped when a line is queued, before its seen_ms is
-    // stamped at flush, and older refreshes rounded them down to whole seconds.
-    // So first_seen can sit just before the occurrence that produced it; decide
-    // against the surviving minimum instead. last_seen errs the safe way.
-    const minSeenMs = (minSeenStmt.get(removal.id) as { value: number | null }).value;
-    const removedLastSeen = toStoredSeenSeconds(removal.removedLastSeenMs);
-    const firstSeenMayHaveMoved =
-      current.firstSeen === null ||
-      (removal.removedFirstSeenMs !== null &&
-        (minSeenMs === null || Number(removal.removedFirstSeenMs) <= Number(minSeenMs)));
-    const lastSeenMayHaveMoved =
-      current.lastSeen === null ||
-      (removedLastSeen !== null && removedLastSeen >= current.lastSeen);
     if (firstSeenMayHaveMoved || lastSeenMayHaveMoved) {
       // Undated pre-migration occurrences are invisible to the seeks below;
-      // fall back to the full re-aggregate that resolves their dates.
+      // resolve their dates without discarding retention-pruned aggregates.
       if (hasUndatedOccurrenceStmt.get(removal.id)) {
-        needsExactRefresh.push(removal.id);
+        needsExactRefresh.set(removal.id, exactRefreshOverride);
         continue;
       }
+      const minSeenMs = (minSeenStmt.get(removal.id) as { value: number | null }).value;
       const maxSeenMs = (maxSeenStmt.get(removal.id) as { value: number | null }).value;
       if (minSeenMs === null || maxSeenMs === null) {
         // Frequency says occurrences remain but none exist: stale row, let the
         // exact refresh reconcile (it deletes rows with nothing left).
-        needsExactRefresh.push(removal.id);
+        needsExactRefresh.set(removal.id, exactRefreshOverride);
         continue;
       }
       updateAggregatesStmt.run(
         nextFrequency,
-        toStoredSeenSeconds(Number(minSeenMs)),
+        firstSeenMayHaveMoved ? toStoredSeenSeconds(Number(minSeenMs)) : current.firstSeen,
         toStoredSeenSeconds(Number(maxSeenMs)),
         removal.id,
       );
@@ -422,13 +432,17 @@ function applyRemovalsForEntity(
   }
 
   if (entity === 'word') {
-    refreshWordAggregates(db, needsExactRefresh);
+    refreshWordAggregates(db, [...needsExactRefresh.keys()], needsExactRefresh);
   } else {
-    refreshKanjiAggregates(db, needsExactRefresh);
+    refreshKanjiAggregates(db, [...needsExactRefresh.keys()], needsExactRefresh);
   }
 }
 
-function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void {
+function refreshWordAggregates(
+  db: DatabaseSync,
+  wordIds: number[],
+  overrides?: ReadonlyMap<number, LexicalAggregateOverride>,
+): void {
   if (wordIds.length === 0) {
     return;
   }
@@ -476,15 +490,19 @@ function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void {
       continue;
     }
     updateStmt.run(
-      row.frequency,
+      overrides?.get(row.wordId)?.frequency ?? row.frequency,
-      toStoredSeenSeconds(row.firstSeen),
+      overrides?.get(row.wordId)?.firstSeen ?? toStoredSeenSeconds(row.firstSeen),
       toStoredSeenSeconds(row.lastSeen),
       row.wordId,
     );
   }
 }
 
-function refreshKanjiAggregates(db: DatabaseSync, kanjiIds: number[]): void {
+function refreshKanjiAggregates(
+  db: DatabaseSync,
+  kanjiIds: number[],
+  overrides?: ReadonlyMap<number, LexicalAggregateOverride>,
+): void {
   if (kanjiIds.length === 0) {
     return;
   }
@@ -532,8 +550,8 @@ function refreshKanjiAggregates(db: DatabaseSync, kanjiIds: number[]): void {
       continue;
     }
     updateStmt.run(
-      row.frequency,
+      overrides?.get(row.kanjiId)?.frequency ?? row.frequency,
-      toStoredSeenSeconds(row.firstSeen),
+      overrides?.get(row.kanjiId)?.firstSeen ?? toStoredSeenSeconds(row.firstSeen),
       toStoredSeenSeconds(row.lastSeen),
       row.kanjiId,
     );

src/core/services/immersion-tracker/storage.ts

@@ -1963,18 +1963,16 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta
       currentMs,
     );
     const lineId = Number(lineResult.lastInsertRowid);
+    // Match aggregate bounds to the observation, even if this write was queued
+    // for a while. Flush time cannot identify which occurrence set first_seen.
+    const seenMs = toDbTimestamp(Math.round(write.firstSeen * 1000));
     for (const occurrence of write.wordOccurrences) {
       const wordId = incrementWordAggregate(stmts, occurrence, write.firstSeen, write.lastSeen);
-      stmts.wordLineOccurrenceUpsertStmt.run(lineId, wordId, occurrence.occurrenceCount, currentMs);
+      stmts.wordLineOccurrenceUpsertStmt.run(lineId, wordId, occurrence.occurrenceCount, seenMs);
     }
     for (const occurrence of write.kanjiOccurrences) {
       const kanjiId = incrementKanjiAggregate(stmts, occurrence, write.firstSeen, write.lastSeen);
-      stmts.kanjiLineOccurrenceUpsertStmt.run(
+      stmts.kanjiLineOccurrenceUpsertStmt.run(lineId, kanjiId, occurrence.occurrenceCount, seenMs);
-        lineId,
-        kanjiId,
-        occurrence.occurrenceCount,
-        currentMs,
-      );
     }
     return;
   }