fix(stats): version lexical rollups and preserve backfill writes

- Persist vocabulary_visible on imm_words and filter rollups/triggers by it, matching the totals' exclusion rules
- Normalize legacy second/millisecond timestamps when bucketing rollup rows into local days
- Replace the boolean rollup-ready flag with a version key so schema changes trigger an atomic rebuild
- Preserve queued playback writes during rollup backfill instead of dropping them under queue pressure
- Time out the lexical rollup worker instead of hanging forever on no response
- Key concurrent vocabulary summary requests by their known-words snapshot so they no longer share stale state
- Bound stats dashboard retries for unfinished vocabulary loads before showing the inline Retry control

src/core/services/__tests__/stats-server.test.ts

@@ -744,6 +744,21 @@ describe('stats server API routes', () => {
     });
   });
 
+  it('GET /api/stats/vocabulary/charts returns complete chart datasets', async () => {
+    const app = createStatsApp(createMockTracker());
+
+    const res = await app.request('/api/stats/vocabulary/charts');
+
+    assert.equal(res.status, 200);
+    assert.deepEqual(await res.json(), {
+      ready: true,
+      topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
+      topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
+      newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
+      newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
+    });
+  });
+
   it('GET /api/stats/kanji returns kanji frequency data', async () => {
     const app = createStatsApp(createMockTracker());
     const res = await app.request('/api/stats/kanji');

src/core/services/immersion-tracker-service.test.ts

@@ -590,7 +590,7 @@ test('tracker starts the injected lexical rollup backfill when it is pending', a
     ensureSchema(setupDb);
     setupDb
       .prepare(
-        `UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_ready'`,
+        `UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
       )
       .run();
     setupDb.close();
@@ -603,6 +603,14 @@ test('tracker starts the injected lexical rollup backfill when it is pending', a
     } as never);
 
     assert.equal(backfillRuns, 1);
+    await waitForCondition(
+      () => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
+    );
+    assert.equal(
+      (tracker as unknown as { preserveWriteQueueUntilDrained: boolean })
+        .preserveWriteQueueUntilDrained,
+      false,
+    );
   } finally {
     tracker?.destroy();
     cleanupDbPath(dbPath);
@@ -631,14 +639,14 @@ test('tracker queues playback writes until lexical rollup backfill settles', asy
     ensureSchema(setupDb);
     setupDb
       .prepare(
-        `UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_ready'`,
+        `UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
       )
       .run();
     setupDb.close();
 
     const Ctor = await loadTrackerCtor();
     tracker = new Ctor(
-      { dbPath },
+      { dbPath, policy: { queueCap: 100 } },
       {
         runLexicalRollupBackfillTask: async (workerDbPath) => {
           await backfillStartGate;
@@ -664,18 +672,20 @@ test('tracker queues playback writes until lexical rollup backfill settles', asy
     tracker.handleMediaChange('https://example.com/backfill-test.mp4', 'Backfill Test');
     startBackfill();
     await backfillStarted;
-    tracker.recordCardsMined(1);
+    for (let index = 0; index < 125; index += 1) tracker.recordCardsMined(1);
 
     const privateApi = tracker as unknown as {
       db: DatabaseSync;
       queue: unknown[];
+      droppedWriteCount: number;
       flushNow: () => void;
       writeLock: { locked: boolean };
     };
     assert.equal(privateApi.writeLock.locked, true);
     privateApi.flushNow();
 
-    assert.ok(privateApi.queue.length > 0);
+    assert.ok(privateApi.queue.length > 100, 'the protected queue may grow past its normal cap');
+    assert.equal(privateApi.droppedWriteCount, 0, 'backfill must not discard playback writes');
     assert.equal(
       (
         privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
@@ -686,17 +696,23 @@ test('tracker queues playback writes until lexical rollup backfill settles', asy
     );
 
     releaseBackfill();
-    await waitForCondition(() => privateApi.queue.length === 0);
+    await waitForCondition(() => privateApi.queue.length === 0, 5_000);
     assert.equal(
       (
         privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
           total: number;
         }
       ).total,
-      1,
+      125,
     );
   } finally {
     releaseBackfill();
+    if (tracker) {
+      await waitForCondition(
+        () => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
+        5_000,
+      );
+    }
     tracker?.destroy();
     cleanupDbPath(dbPath);
   }
@@ -5086,8 +5102,9 @@ test('getVocabularySummary coalesces concurrent requests into one worker task',
       },
     );
 
-    const first = tracker.getVocabularySummary(null);
-    const second = tracker.getVocabularySummary(new Set(['猫']));
+    const knownWordsSnapshot = new Set(['猫']);
+    const first = tracker.getVocabularySummary(knownWordsSnapshot);
+    const second = tracker.getVocabularySummary(knownWordsSnapshot);
     await waitForCondition(() => releaseTask !== null);
     let release = releaseTask as (() => void) | null;
     assert.ok(release);
@@ -5095,9 +5112,7 @@ test('getVocabularySummary coalesces concurrent requests into one worker task',
     assert.deepEqual(await first, summary);
     assert.equal(await second, await first);
     assert.equal(taskRuns, 1);
-    // The coalesced caller's known-words set must not replace the snapshot the
-    // in-flight scan already started with.
-    assert.deepEqual(seenKnownWords, [null]);
+    assert.deepEqual(seenKnownWords, [knownWordsSnapshot]);
 
     releaseTask = null;
     const third = tracker.getVocabularySummary(null);
@@ -5112,3 +5127,42 @@ test('getVocabularySummary coalesces concurrent requests into one worker task',
     cleanupDbPath(dbPath);
   }
 });
+
+test('getVocabularySummary keeps different known-word snapshots independent', async () => {
+  const dbPath = makeDbPath();
+  let tracker: ImmersionTrackerService | null = null;
+  const releases: Array<() => void> = [];
+
+  try {
+    const Ctor = await loadTrackerCtor();
+    tracker = new Ctor(
+      { dbPath },
+      {
+        runVocabularySummaryTask: async (_dbPath, knownWords) => {
+          await new Promise<void>((resolve) => releases.push(resolve));
+          return {
+            uniqueWords: 1,
+            uniqueWordsWithoutNames: 1,
+            uniqueKanji: 0,
+            newThisWeek: 0,
+            newThisWeekWithoutNames: 0,
+            knownWordCount: knownWords?.size ?? null,
+            knownWordCountWithoutNames: knownWords?.size ?? null,
+          };
+        },
+        destroyVocabularySummaryRunner: () => {},
+      },
+    );
+
+    const withoutKnownWords = tracker.getVocabularySummary(null);
+    const withKnownWords = tracker.getVocabularySummary(new Set(['猫']));
+    await waitForCondition(() => releases.length === 2);
+    for (const release of releases) release();
+
+    assert.equal((await withoutKnownWords).knownWordCount, null);
+    assert.equal((await withKnownWords).knownWordCount, 1);
+  } finally {
+    tracker?.destroy();
+    cleanupDbPath(dbPath);
+  }
+});

src/core/services/immersion-tracker-service.ts

@@ -421,7 +421,10 @@ export class ImmersionTrackerService {
   private readonly runVocabularySummaryTask: (
     knownWords: ReadonlySet<string> | null,
   ) => Promise<VocabularyStatsSummary>;
-  private vocabularySummaryInFlight: Promise<VocabularyStatsSummary> | null = null;
+  private readonly vocabularySummariesInFlight = new Map<
+    ReadonlySet<string> | null,
+    Promise<VocabularyStatsSummary>
+  >();
   private readonly destroyVocabularySummaryRunner: () => void;
   private readonly runLexicalRollupBackfillTask: () => Promise<void>;
   private readonly destroyLexicalRollupBackfillRunner: () => void;
@@ -430,6 +433,7 @@ export class ImmersionTrackerService {
   private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
   private flushScheduled = false;
   private droppedWriteCount = 0;
+  private preserveWriteQueueUntilDrained = false;
   private lastVacuumMs = 0;
   private isDestroyed = false;
   private sessionState: SessionState | null = null;
@@ -681,16 +685,16 @@ export class ImmersionTrackerService {
   }
 
   async getVocabularySummary(knownWords: ReadonlySet<string> | null) {
-    // Concurrent dashboard refreshes share one worker scan; the coalesced
-    // callers accept the first caller's known-words snapshot.
-    const inFlight = this.vocabularySummaryInFlight;
+    const inFlight = this.vocabularySummariesInFlight.get(knownWords);
     if (inFlight) return inFlight;
     const task = this.runVocabularySummaryTask(knownWords);
-    this.vocabularySummaryInFlight = task;
+    this.vocabularySummariesInFlight.set(knownWords, task);
     try {
       return await task;
     } finally {
-      if (this.vocabularySummaryInFlight === task) this.vocabularySummaryInFlight = null;
+      if (this.vocabularySummariesInFlight.get(knownWords) === task) {
+        this.vocabularySummariesInFlight.delete(knownWords);
+      }
     }
   }
 
@@ -985,6 +989,7 @@ export class ImmersionTrackerService {
 
   private startLexicalRollupBackfill(): void {
     this.requireWriteQueueDrained('lexical rollup backfill');
+    this.preserveWriteQueueUntilDrained = true;
     this.setWriteLock('lexical-rollup-backfill', true);
     void this.runLexicalRollupBackfillTask()
       .catch((error: unknown) => {
@@ -995,7 +1000,8 @@ export class ImmersionTrackerService {
       })
       .finally(() => {
         this.setWriteLock('lexical-rollup-backfill', false);
-        if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
+        if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
+        else if (!this.isDestroyed) this.scheduleFlush(0);
       });
   }
 
@@ -1995,7 +2001,12 @@ export class ImmersionTrackerService {
 
   private recordWrite(write: QueuedWrite): void {
     if (this.isDestroyed) return;
-    const { dropped } = enqueueWrite(this.queue, write, this.queueCap);
+    // A lexical migration owns the database write lock, so dropping the oldest
+    // entry cannot relieve pressure: nothing can flush until the worker exits.
+    // Preserve that finite startup burst and drain it as soon as the lock lifts.
+    const { dropped } = this.preserveWriteQueueUntilDrained
+      ? (this.queue.push(write), { dropped: 0 })
+      : enqueueWrite(this.queue, write, this.queueCap);
     if (dropped > 0) {
       this.droppedWriteCount += dropped;
       this.logger.warn(`Immersion tracker queue overflow; dropped ${dropped} oldest writes`);
@@ -2043,6 +2054,7 @@ export class ImmersionTrackerService {
   private flushNow(): void {
     if (this.writeLock.locked || this.isDestroyed) return;
     if (this.queue.length === 0) {
+      this.preserveWriteQueueUntilDrained = false;
       this.flushScheduled = false;
       return;
     }
@@ -2068,6 +2080,7 @@ export class ImmersionTrackerService {
     } finally {
       this.setWriteLock('flush', false);
       this.flushScheduled = false;
+      if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
       if (this.queue.length > 0) {
         this.scheduleFlush(this.flushIntervalMs);
       }

src/core/services/immersion-tracker/lexical-rollup-worker-runtime.test.ts

@@ -26,7 +26,7 @@ test('lexical rollup worker backfills without using the tracker connection', asy
     ).run();
     db.exec('DELETE FROM imm_lexical_daily_rollups');
     db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
-      'lexical_daily_rollups_ready',
+      'lexical_daily_rollups_version',
     );
     db.close();
 
@@ -101,3 +101,36 @@ test('lexical rollup worker absorbs termination failures after settling', async
     runtime.destroy();
   }
 });
+
+test('lexical rollup worker times out when it never responds', async () => {
+  let terminated = false;
+  const runtime = new LexicalRollupWorkerRuntime({
+    resolveWorkerPath: () => '/tmp/fake-worker.js',
+    createWorker: async () => ({
+      once() {
+        return this;
+      },
+      terminate: async () => {
+        terminated = true;
+        return 0;
+      },
+    }),
+    timeoutMs: 1,
+    warn: () => {},
+  } as never);
+
+  try {
+    const outcome = await Promise.race([
+      runtime.run('/tmp/not-used.sqlite').then(
+        () => 'resolved',
+        (error: unknown) => String(error),
+      ),
+      new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
+    ]);
+
+    assert.match(outcome, /timed out/);
+    assert.equal(terminated, true);
+  } finally {
+    runtime.destroy();
+  }
+});

src/core/services/immersion-tracker/lexical-rollup-worker-runtime.ts

@@ -17,10 +17,12 @@ interface WorkerHandle {
 interface LexicalRollupWorkerRuntimeOptions {
   resolveWorkerPath?: () => string | null;
   createWorker?: (workerPath: string, workerData: { dbPath: string }) => Promise<WorkerHandle>;
+  timeoutMs?: number;
   warn?: (message: string, ...meta: unknown[]) => void;
 }
 
 const logger = createLogger('main:immersion-tracker:lexical-rollup-worker');
+const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
 
 export function resolveLexicalRollupWorkerPath(): string | null {
   const fileName = __filename.endsWith('.ts')
@@ -65,15 +67,21 @@ export class LexicalRollupWorkerRuntime {
 
     return new Promise<void>((resolve, reject) => {
       let settled = false;
+      let timeout: ReturnType<typeof setTimeout> | null = null;
       this.activeWorkers.add(worker);
       const settle = (error?: Error) => {
         if (settled) return;
         settled = true;
+        if (timeout) clearTimeout(timeout);
         this.activeWorkers.delete(worker);
         void worker.terminate().catch(() => undefined);
         if (error) reject(error);
         else resolve();
       };
+      timeout = setTimeout(
+        () => settle(new Error('Lexical rollup worker timed out')),
+        this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
+      );
       worker.once('message', (message) => {
         if (message.ok) settle();
         else

src/core/services/immersion-tracker/lexical-rollup-worker.test.ts

@@ -21,7 +21,7 @@ test('lexical rollup backfill materializes pre-existing vocabulary off the calle
     ).run('犬', '犬', 'いぬ', 1_700_000_000, 1_700_000_000);
     db.exec('DELETE FROM imm_lexical_daily_rollups');
     db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
-      'lexical_daily_rollups_ready',
+      'lexical_daily_rollups_version',
     );
 
     executeLexicalRollupBackfillTask(dbPath);

src/core/services/immersion-tracker/lexical-rollups.test.ts

@@ -3,9 +3,17 @@ import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import test from 'node:test';
-import { getLexicalDailyRollups, rebuildLexicalDailyRollups } from './lexical-rollups';
+import {
+  areLexicalDailyRollupsReady,
+  getLexicalDailyRollups,
+  rebuildLexicalDailyRollups,
+} from './lexical-rollups';
 import { getTrendsDashboard } from './query-trends';
-import { getVocabularyChartData, replaceStatsExcludedWords } from './query-lexical';
+import {
+  getVocabularyChartData,
+  getVocabularySummary,
+  replaceStatsExcludedWords,
+} from './query-lexical';
 import { Database } from './sqlite';
 import type { DatabaseSync } from './sqlite';
 import { ensureSchema } from './storage';
@@ -51,6 +59,169 @@ test('lexical daily rollups follow first-seen corrections and deletions', () =>
   }
 });
 
+test('lexical daily rollups normalize second and millisecond timestamps', () => {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    const epochDay = 19_500;
+    const timestampSeconds = epochDay * 86_400 + 43_200;
+    const timestampMilliseconds = timestampSeconds * 1_000;
+
+    db.prepare(
+      `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
+       VALUES (?, ?, ?, ?, ?, 1)`,
+    ).run('猫', '猫', 'ねこ', timestampSeconds, timestampSeconds);
+    db.prepare(
+      `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
+       VALUES (?, ?, ?, ?, ?, 1)`,
+    ).run('犬', '犬', 'いぬ', timestampMilliseconds, timestampMilliseconds);
+    db.prepare(
+      `INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
+       VALUES (?, ?, ?, 1)`,
+    ).run('猫', timestampSeconds, timestampSeconds);
+    db.prepare(
+      `INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
+       VALUES (?, ?, ?, 1)`,
+    ).run('犬', timestampMilliseconds, timestampMilliseconds);
+
+    assert.deepEqual(getLexicalDailyRollups(db), [
+      { epochDay, wordCount: 2, wordCountWithoutNames: 2, kanjiCount: 2 },
+    ]);
+  } finally {
+    db.close();
+    fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+  }
+});
+
+test('lexical rollup rebuild excludes rows hidden by vocabulary persistence rules', () => {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    const epochDay = 19_500;
+    const firstSeen = epochDay * 86_400 + 43_200;
+    db.prepare(
+      `INSERT INTO imm_words(
+         headword, word, reading, part_of_speech, first_seen, last_seen, frequency
+       ) VALUES (?, ?, ?, ?, ?, ?, 1)`,
+    ).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
+    db.prepare(
+      `INSERT INTO imm_words(
+         headword, word, reading, part_of_speech, first_seen, last_seen, frequency
+       ) VALUES (?, ?, ?, ?, ?, ?, 1)`,
+    ).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
+
+    rebuildLexicalDailyRollups(db);
+
+    assert.deepEqual(getLexicalDailyRollups(db), [
+      { epochDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
+    ]);
+  } finally {
+    db.close();
+    fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+  }
+});
+
+test('lexical rollup rebuild tolerates nullable legacy vocabulary text', () => {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    db.prepare(
+      `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
+       VALUES (NULL, NULL, NULL, 1700000000, 1700000000, 1)`,
+    ).run();
+    db.prepare(
+      `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
+       VALUES (NULL, '猫', 'ねこ', 1700000000, 1700000000, 1)`,
+    ).run();
+
+    assert.doesNotThrow(() => rebuildLexicalDailyRollups(db));
+    assert.equal(areLexicalDailyRollupsReady(db), true);
+    assert.equal(getVocabularySummary(db, null).uniqueWords, 1);
+    assert.equal(getVocabularySummary(db, new Set(['猫'])).knownWordCount, 1);
+    assert.equal(getVocabularyChartData(db).topWords[0]?.headword, '猫');
+  } finally {
+    db.close();
+    fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+  }
+});
+
+test('chart exclusions do not subtract vocabulary rows already hidden from the rollup', () => {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    const epochDay = 19_500;
+    const firstSeen = epochDay * 86_400 + 43_200;
+    db.prepare(
+      `INSERT INTO imm_words(
+         headword, word, reading, part_of_speech, first_seen, last_seen, frequency
+       ) VALUES (?, ?, ?, ?, ?, ?, 1)`,
+    ).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
+    db.prepare(
+      `INSERT INTO imm_words(
+         headword, word, reading, part_of_speech, first_seen, last_seen, frequency
+       ) VALUES (?, ?, ?, ?, ?, ?, 1)`,
+    ).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
+    rebuildLexicalDailyRollups(db);
+    replaceStatsExcludedWords(db, [{ headword: 'は', word: 'は', reading: 'は' }]);
+
+    const charts = getVocabularyChartData(db);
+
+    assert.deepEqual(charts.newWordsTimeline, [{ epochDay, wordCount: 1 }]);
+    assert.deepEqual(charts.newWordsTimelineWithoutNames, [{ epochDay, wordCount: 1 }]);
+  } finally {
+    db.close();
+    fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+  }
+});
+
+test('legacy lexical rollup readiness does not satisfy the current rollup version', () => {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    db.prepare(
+      `INSERT INTO imm_rollup_state(state_key, state_value)
+       VALUES ('lexical_daily_rollups_ready', '1')
+       ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
+    ).run();
+    db.prepare(
+      `DELETE FROM imm_rollup_state WHERE state_key = 'lexical_daily_rollups_version'`,
+    ).run();
+
+    assert.equal(areLexicalDailyRollupsReady(db), false);
+  } finally {
+    db.close();
+    fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+  }
+});
+
+test('imm_words persists vocabulary visibility for rollup maintenance', () => {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    const columns = db.prepare(`PRAGMA table_info(imm_words)`).all() as Array<{ name: string }>;
+
+    assert.equal(
+      columns.some((column) => column.name === 'vocabulary_visible'),
+      true,
+    );
+  } finally {
+    db.close();
+    fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+  }
+});
+
 test('vocabulary charts use complete top-word and lexical rollup data', () => {
   const dbPath = makeDbPath();
   const db = new Database(dbPath);
@@ -146,6 +317,9 @@ test('lexical rollup rebuild preserves the original error when rollback also fai
       if (sql === 'ROLLBACK') throw new Error('rollback failed');
       throw originalError;
     },
+    prepare() {
+      return { all: () => [], run: () => undefined };
+    },
   } as unknown as DatabaseSync;
 
   assert.throws(() => rebuildLexicalDailyRollups(db), originalError);

src/core/services/immersion-tracker/lexical-rollups.ts

@@ -1,4 +1,5 @@
 import type { DatabaseSync } from './sqlite';
+import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility';
 
 export interface LexicalDailyRollup {
   epochDay: number;
@@ -8,11 +9,20 @@ export interface LexicalDailyRollup {
 }
 
 const LOCAL_EPOCH_DAY_SQL = `
-  CAST(julianday(CAST(%VALUE% AS REAL), 'unixepoch', 'localtime') - 2440587.5 AS INTEGER)
+  CAST(julianday(
+    CASE
+      WHEN ABS(CAST(%VALUE% AS REAL)) >= 10000000000 THEN CAST(%VALUE% AS REAL) / 1000
+      ELSE CAST(%VALUE% AS REAL)
+    END,
+    'unixepoch', 'localtime'
+  ) - 2440587.5 AS INTEGER)
 `;
 
+const LEXICAL_DAILY_ROLLUP_VERSION = '2';
+const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
+
 export function localEpochDaySql(value: string): string {
-  return LOCAL_EPOCH_DAY_SQL.replace('%VALUE%', value);
+  return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
 }
 
 function createWordRollupTriggers(db: DatabaseSync): void {
@@ -20,9 +30,13 @@ function createWordRollupTriggers(db: DatabaseSync): void {
   const dayForOld = localEpochDaySql('OLD.first_seen');
 
   db.exec(`
-    CREATE TRIGGER IF NOT EXISTS imm_words_lexical_rollup_insert
+    DROP TRIGGER IF EXISTS imm_words_lexical_rollup_insert;
+    DROP TRIGGER IF EXISTS imm_words_lexical_rollup_delete;
+    DROP TRIGGER IF EXISTS imm_words_lexical_rollup_first_seen_update;
+
+    CREATE TRIGGER imm_words_lexical_rollup_insert
     AFTER INSERT ON imm_words
-    WHEN NEW.first_seen IS NOT NULL
+    WHEN NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
     BEGIN
       INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
       VALUES (${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0)
@@ -31,9 +45,9 @@ function createWordRollupTriggers(db: DatabaseSync): void {
         word_count_without_names = word_count_without_names + excluded.word_count_without_names;
     END;
 
-    CREATE TRIGGER IF NOT EXISTS imm_words_lexical_rollup_delete
+    CREATE TRIGGER imm_words_lexical_rollup_delete
     AFTER DELETE ON imm_words
-    WHEN OLD.first_seen IS NOT NULL
+    WHEN OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
     BEGIN
       INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
       VALUES (${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0)
@@ -44,19 +58,21 @@ function createWordRollupTriggers(db: DatabaseSync): void {
       WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
     END;
 
-    CREATE TRIGGER IF NOT EXISTS imm_words_lexical_rollup_first_seen_update
-    AFTER UPDATE OF first_seen, pos2 ON imm_words
-    WHEN OLD.first_seen IS NOT NEW.first_seen OR OLD.pos2 IS NOT NEW.pos2
+    CREATE TRIGGER imm_words_lexical_rollup_first_seen_update
+    AFTER UPDATE OF first_seen, pos2, vocabulary_visible ON imm_words
+    WHEN OLD.first_seen IS NOT NEW.first_seen
+      OR OLD.pos2 IS NOT NEW.pos2
+      OR OLD.vocabulary_visible IS NOT NEW.vocabulary_visible
     BEGIN
       INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
       SELECT ${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0
-      WHERE OLD.first_seen IS NOT NULL
+      WHERE OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
       ON CONFLICT(epoch_day) DO UPDATE SET
         word_count = word_count - 1,
         word_count_without_names = word_count_without_names + excluded.word_count_without_names;
       INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
       SELECT ${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0
-      WHERE NEW.first_seen IS NOT NULL
+      WHERE NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
       ON CONFLICT(epoch_day) DO UPDATE SET
         word_count = word_count + 1,
         word_count_without_names = word_count_without_names + excluded.word_count_without_names;
@@ -70,14 +86,18 @@ function createKanjiRollupTriggers(db: DatabaseSync): void {
   const dayForNew = localEpochDaySql('NEW.first_seen');
   const dayForOld = localEpochDaySql('OLD.first_seen');
   db.exec(`
-    CREATE TRIGGER IF NOT EXISTS imm_kanji_lexical_rollup_insert
+    DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_insert;
+    DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_delete;
+    DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_first_seen_update;
+
+    CREATE TRIGGER imm_kanji_lexical_rollup_insert
     AFTER INSERT ON imm_kanji WHEN NEW.first_seen IS NOT NULL
     BEGIN
       INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
       VALUES (${dayForNew}, 0, 0, 1)
       ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
     END;
-    CREATE TRIGGER IF NOT EXISTS imm_kanji_lexical_rollup_delete
+    CREATE TRIGGER imm_kanji_lexical_rollup_delete
     AFTER DELETE ON imm_kanji WHEN OLD.first_seen IS NOT NULL
     BEGIN
       INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
@@ -86,7 +106,7 @@ function createKanjiRollupTriggers(db: DatabaseSync): void {
       DELETE FROM imm_lexical_daily_rollups
       WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
     END;
-    CREATE TRIGGER IF NOT EXISTS imm_kanji_lexical_rollup_first_seen_update
+    CREATE TRIGGER imm_kanji_lexical_rollup_first_seen_update
     AFTER UPDATE OF first_seen ON imm_kanji WHEN OLD.first_seen IS NOT NEW.first_seen
     BEGIN
       INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
@@ -109,7 +129,7 @@ export function ensureLexicalDailyRollupTables(db: DatabaseSync): void {
       kanji_count INTEGER NOT NULL DEFAULT 0
     );
     INSERT INTO imm_rollup_state(state_key, state_value)
-    VALUES ('lexical_daily_rollups_ready', '0')
+    VALUES ('${LEXICAL_DAILY_ROLLUP_VERSION_KEY}', '0')
     ON CONFLICT(state_key) DO NOTHING;
   `);
   createWordRollupTriggers(db);
@@ -119,14 +139,16 @@ export function ensureLexicalDailyRollupTables(db: DatabaseSync): void {
 export function areLexicalDailyRollupsReady(db: DatabaseSync): boolean {
   const row = db
     .prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
-    .get('lexical_daily_rollups_ready') as { value: string } | null;
-  return row?.value === '1';
+    .get(LEXICAL_DAILY_ROLLUP_VERSION_KEY) as { value: string } | null;
+  return row?.value === LEXICAL_DAILY_ROLLUP_VERSION;
 }
 
 export function markLexicalDailyRollupsReady(db: DatabaseSync): void {
-  db.prepare(`UPDATE imm_rollup_state SET state_value = '1' WHERE state_key = ?`).run(
-    'lexical_daily_rollups_ready',
-  );
+  db.prepare(
+    `INSERT INTO imm_rollup_state(state_key, state_value)
+     VALUES (?, ?)
+     ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
+  ).run(LEXICAL_DAILY_ROLLUP_VERSION_KEY, LEXICAL_DAILY_ROLLUP_VERSION);
 }
 
 /** Rebuild from the first-seen source of truth; run off the UI/main DB thread. */
@@ -135,13 +157,27 @@ export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
   try {
     db.exec('BEGIN IMMEDIATE');
     transactionStarted = true;
+    const vocabularyRows = db
+      .prepare(
+        `SELECT id, word, headword, reading, part_of_speech AS partOfSpeech,
+           pos1, pos2, pos3, frequency_rank AS frequencyRank
+         FROM imm_words`,
+      )
+      .all() as Array<VocabularyVisibilityRow & { id: number }>;
+    const updateVisibility = db.prepare(
+      `UPDATE imm_words SET vocabulary_visible = ? WHERE id = ? AND vocabulary_visible IS NOT ?`,
+    );
+    for (const row of vocabularyRows) {
+      const visible = isVocabularyStatsRowVisible(row) ? 1 : 0;
+      updateVisibility.run(visible, row.id, visible);
+    }
     db.exec('DELETE FROM imm_lexical_daily_rollups');
     db.exec(`
       INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
       SELECT ${localEpochDaySql('first_seen')}, COUNT(*),
         SUM(CASE WHEN pos2 = '固有名詞' THEN 0 ELSE 1 END), 0
       FROM imm_words
-      WHERE first_seen IS NOT NULL
+      WHERE first_seen IS NOT NULL AND vocabulary_visible = 1
       GROUP BY ${localEpochDaySql('first_seen')};
       INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
       SELECT ${localEpochDaySql('first_seen')}, 0, 0, COUNT(*)

src/core/services/immersion-tracker/query-lexical.ts

@@ -1,6 +1,4 @@
 import type { DatabaseSync } from './sqlite';
-import { PartOfSpeech, type MergedToken } from '../../../types';
-import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
 import type {
   KanjiAnimeAppearanceRow,
   KanjiDetailRow,
@@ -25,6 +23,7 @@ import {
   getLexicalDailyRollups,
   localEpochDaySql,
 } from './lexical-rollups';
+import { isVocabularyStatsRowVisible } from './vocabulary-visibility';
 
 const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
 const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
@@ -91,33 +90,6 @@ function uniqueKanji(text: string): string[] {
   return Array.from(new Set(text.match(KANJI_PATTERN) ?? []));
 }
 
-function toVocabularyToken(row: VocabularyStatsRow): MergedToken {
-  const partOfSpeech =
-    row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
-      ? (row.partOfSpeech as PartOfSpeech)
-      : PartOfSpeech.other;
-
-  return {
-    surface: row.word,
-    reading: row.reading ?? '',
-    headword: row.headword,
-    startPos: 0,
-    endPos: row.word.length,
-    partOfSpeech,
-    pos1: row.pos1 ?? '',
-    pos2: row.pos2 ?? '',
-    pos3: row.pos3 ?? '',
-    frequencyRank: row.frequencyRank ?? undefined,
-    isMerged: false,
-    isKnown: false,
-    isNPlusOneTarget: false,
-  };
-}
-
-function isVocabularyStatsRowVisible(row: VocabularyStatsRow): boolean {
-  return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
-}
-
 export function getVocabularyStats(
   db: DatabaseSync,
   limit = 100,
@@ -204,7 +176,8 @@ export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData {
             SELECT id AS wordId, headword, word, reading, pos2,
               ${localEpochDaySql('first_seen')} AS epochDay
             FROM imm_words
-            WHERE headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders})
+            WHERE vocabulary_visible = 1
+              AND (headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}))
           `,
         )
         .all(...batch, ...batch, ...batch) as Array<
@@ -227,12 +200,12 @@ export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData {
     ready,
     topWords: topWords.all.map((word) => ({
       wordId: word.wordId,
-      headword: word.headword,
+      headword: vocabularyDisplayHeadword(word),
       frequency: word.frequency,
     })),
     topWordsWithoutNames: topWords.withoutNames.map((word) => ({
       wordId: word.wordId,
-      headword: word.headword,
+      headword: vocabularyDisplayHeadword(word),
       frequency: word.frequency,
     })),
     newWordsTimeline: [...timeline.values()]
@@ -281,11 +254,17 @@ function getTopVocabularyChartWords(
 function excludedVocabularyAliases(
   word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
 ): string[] {
-  const aliases = [word.headword.trim(), word.word.trim()].filter(Boolean);
-  if (aliases.length === 0) aliases.push(word.reading.trim());
+  const aliases = [word.headword?.trim() ?? '', word.word?.trim() ?? ''].filter(Boolean);
+  if (aliases.length === 0) aliases.push(word.reading?.trim() ?? '');
   return [...new Set(aliases)];
 }
 
+function vocabularyDisplayHeadword(
+  word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
+): string {
+  return word.headword?.trim() || word.word?.trim() || word.reading?.trim() || '';
+}
+
 function timestampSeconds(timestamp: number): number {
   return timestamp < 10_000_000_000 ? timestamp : Math.floor(timestamp / 1000);
 }
@@ -338,7 +317,7 @@ export function getVocabularySummary(
       }
       const isName = word.pos2 === '固有名詞';
       const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec;
-      const isKnown = knownWords?.has(word.headword) ?? false;
+      const isKnown = knownWords?.has(vocabularyDisplayHeadword(word)) ?? false;
       summary.uniqueWords += 1;
       if (!isName) summary.uniqueWordsWithoutNames += 1;
       if (isNewThisWeek) {

src/core/services/immersion-tracker/storage.ts

@@ -1069,6 +1069,7 @@ export function ensureSchema(db: DatabaseSync): void {
       last_seen REAL,
       frequency INTEGER,
       frequency_rank INTEGER,
+      vocabulary_visible INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1)),
       UNIQUE(headword, word, reading)
     );
   `);
@@ -1452,6 +1453,15 @@ export function ensureSchema(db: DatabaseSync): void {
     addColumnIfMissing(db, 'imm_sessions', 'ended_media_ms', 'INTEGER');
   }
 
+  if (currentVersion?.schema_version && currentVersion.schema_version < 23) {
+    addColumnIfMissing(
+      db,
+      'imm_words',
+      'vocabulary_visible',
+      'INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1))',
+    );
+  }
+
   migrateSessionEventTimestampsToText(db);
 
   ensureLexicalDailyRollupTables(db);
@@ -1625,9 +1635,10 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
     `),
     wordUpsertStmt: db.prepare(`
       INSERT INTO imm_words (
-        headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency, frequency_rank
+        headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen,
+        frequency, frequency_rank, vocabulary_visible
       ) VALUES (
-        ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?
+        ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, 1
       )
       ON CONFLICT(headword, word, reading) DO UPDATE SET
         frequency = COALESCE(frequency, 0) + 1,
@@ -1640,6 +1651,7 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
         pos1 = COALESCE(NULLIF(imm_words.pos1, ''), excluded.pos1),
         pos2 = COALESCE(NULLIF(imm_words.pos2, ''), excluded.pos2),
         pos3 = COALESCE(NULLIF(imm_words.pos3, ''), excluded.pos3),
+        vocabulary_visible = 1,
         first_seen = MIN(COALESCE(first_seen, excluded.first_seen), excluded.first_seen),
         last_seen = MAX(COALESCE(last_seen, excluded.last_seen), excluded.last_seen),
         frequency_rank = CASE

src/core/services/immersion-tracker/types.ts

@@ -1,4 +1,4 @@
-export const SCHEMA_VERSION = 22;
+export const SCHEMA_VERSION = 23;
 export const DEFAULT_QUEUE_CAP = 1_000;
 export const DEFAULT_BATCH_SIZE = 25;
 export const DEFAULT_FLUSH_INTERVAL_MS = 500;

src/core/services/immersion-tracker/vocabulary-visibility.ts

@@ -0,0 +1,43 @@
+import { PartOfSpeech, type MergedToken } from '../../../types';
+import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
+
+export interface VocabularyVisibilityRow {
+  word: string | null;
+  headword: string | null;
+  reading?: string | null;
+  partOfSpeech?: string | null;
+  pos1?: string | null;
+  pos2?: string | null;
+  pos3?: string | null;
+  frequencyRank?: number | null;
+}
+
+function toVocabularyToken(row: VocabularyVisibilityRow): MergedToken {
+  const word = row.word ?? '';
+  const headword = row.headword ?? word;
+  const partOfSpeech =
+    row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
+      ? (row.partOfSpeech as PartOfSpeech)
+      : PartOfSpeech.other;
+
+  return {
+    surface: word,
+    reading: row.reading ?? '',
+    headword,
+    startPos: 0,
+    endPos: word.length,
+    partOfSpeech,
+    pos1: row.pos1 ?? '',
+    pos2: row.pos2 ?? '',
+    pos3: row.pos3 ?? '',
+    frequencyRank: row.frequencyRank ?? undefined,
+    isMerged: false,
+    isKnown: false,
+    isNPlusOneTarget: false,
+  };
+}
+
+export function isVocabularyStatsRowVisible(row: VocabularyVisibilityRow): boolean {
+  if (!(row.word?.trim() || row.headword?.trim())) return false;
+  return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
+}

src/core/services/stats-sync/merge-catalog.ts

@@ -89,6 +89,7 @@ const WORD_COPY_COLUMNS = [
   'last_seen',
   'frequency',
   'frequency_rank',
+  'vocabulary_visible',
 ] as const;
 
 export function mergeAnime(