fix(stats): count mined cards once and keep lexical stats exact after deletes (#280)

src/core/services/immersion-tracker/storage.ts

@@ -6,6 +6,7 @@ import { normalizeTitleIdentity } from '../../utils/title-normalization';
 import type { DatabaseSync } from './sqlite';
 import { nowMs } from './time';
 import { ensureLexicalDailyRollupTables, markLexicalDailyRollupsReady } from './lexical-rollups';
+import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility';
 import { SCHEMA_VERSION } from './types';
 import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types';
 import { toDbMs, toDbTimestamp } from './query-shared';
@@ -17,7 +18,8 @@ export interface TrackerPreparedStatements {
   wordUpsertStmt: ReturnType<DatabaseSync['prepare']>;
   kanjiUpsertStmt: ReturnType<DatabaseSync['prepare']>;
   subtitleLineInsertStmt: ReturnType<DatabaseSync['prepare']>;
-  wordIdSelectStmt: ReturnType<DatabaseSync['prepare']>;
+  wordSelectStmt: ReturnType<DatabaseSync['prepare']>;
+  wordVisibilityUpdateStmt: ReturnType<DatabaseSync['prepare']>;
   kanjiIdSelectStmt: ReturnType<DatabaseSync['prepare']>;
   wordLineOccurrenceUpsertStmt: ReturnType<DatabaseSync['prepare']>;
   kanjiLineOccurrenceUpsertStmt: ReturnType<DatabaseSync['prepare']>;
@@ -1759,7 +1761,6 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
         pos1 = COALESCE(NULLIF(imm_words.pos1, ''), excluded.pos1),
         pos2 = COALESCE(NULLIF(imm_words.pos2, ''), excluded.pos2),
         pos3 = COALESCE(NULLIF(imm_words.pos3, ''), excluded.pos3),
-        vocabulary_visible = 1,
         first_seen = MIN(COALESCE(first_seen, excluded.first_seen), excluded.first_seen),
         last_seen = MAX(COALESCE(last_seen, excluded.last_seen), excluded.last_seen),
         frequency_rank = CASE
@@ -1787,10 +1788,16 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
         ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
       )
     `),
-    wordIdSelectStmt: db.prepare(`
-      SELECT id FROM imm_words
+    wordSelectStmt: db.prepare(`
+      SELECT id, word, headword, reading, part_of_speech AS partOfSpeech,
+        pos1, pos2, pos3, frequency_rank AS frequencyRank,
+        vocabulary_visible AS vocabularyVisible
+      FROM imm_words
       WHERE headword = ? AND word = ? AND reading = ?
     `),
+    wordVisibilityUpdateStmt: db.prepare(`
+      UPDATE imm_words SET vocabulary_visible = ? WHERE id = ?
+    `),
     kanjiIdSelectStmt: db.prepare(`
       SELECT id FROM imm_kanji
       WHERE kanji = ?
@@ -1842,14 +1849,18 @@ function incrementWordAggregate(
       occurrence.frequencyRank ?? null,
     );
   }
-  const row = stmts.wordIdSelectStmt.get(
-    occurrence.headword,
-    occurrence.word,
-    occurrence.reading,
-  ) as { id: number } | null;
+  const row = stmts.wordSelectStmt.get(occurrence.headword, occurrence.word, occurrence.reading) as
+    | (VocabularyVisibilityRow & { id: number; vocabularyVisible: number })
+    | null;
   if (!row?.id) {
     throw new Error(`Failed to resolve imm_words id for ${occurrence.headword}`);
   }
+  // The upsert keeps the first-seen POS metadata, so derive visibility from the
+  // stored row (as the vocabulary summary does) rather than the incoming token.
+  const visible = isVocabularyStatsRowVisible(row) ? 1 : 0;
+  if (row.vocabularyVisible !== visible) {
+    stmts.wordVisibilityUpdateStmt.run(visible, row.id);
+  }
   return row.id;
 }
 
@@ -1952,18 +1963,16 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta
       currentMs,
     );
     const lineId = Number(lineResult.lastInsertRowid);
+    // Match aggregate bounds to the observation, even if this write was queued
+    // for a while. Flush time cannot identify which occurrence set first_seen.
+    const seenMs = toDbTimestamp(Math.round(write.firstSeen * 1000));
     for (const occurrence of write.wordOccurrences) {
       const wordId = incrementWordAggregate(stmts, occurrence, write.firstSeen, write.lastSeen);
-      stmts.wordLineOccurrenceUpsertStmt.run(lineId, wordId, occurrence.occurrenceCount, currentMs);
+      stmts.wordLineOccurrenceUpsertStmt.run(lineId, wordId, occurrence.occurrenceCount, seenMs);
     }
     for (const occurrence of write.kanjiOccurrences) {
       const kanjiId = incrementKanjiAggregate(stmts, occurrence, write.firstSeen, write.lastSeen);
-      stmts.kanjiLineOccurrenceUpsertStmt.run(
-        lineId,
-        kanjiId,
-        occurrence.occurrenceCount,
-        currentMs,
-      );
+      stmts.kanjiLineOccurrenceUpsertStmt.run(lineId, kanjiId, occurrence.occurrenceCount, seenMs);
     }
     return;
   }