perf(stats): roll up lexical chart history

src/core/services/__tests__/stats-server.test.ts

@@ -293,6 +293,13 @@ function createMockTracker(
       knownWordCount: 250,
       knownWordCountWithoutNames: 249,
     }),
+    getVocabularyChartData: async () => ({
+      ready: true,
+      topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
+      topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
+      newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
+      newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
+    }),
     getStatsExcludedWords: async () => [],
     replaceStatsExcludedWords: async () => {},
     getKanjiStats: async () => KANJI_STATS,

src/core/services/immersion-tracker-service.ts

@@ -58,6 +58,7 @@ import {
   getSessionEvents,
   getSimilarWords,
   getStatsExcludedWords,
+  getVocabularyChartData,
   getVocabularyStats,
   replaceStatsExcludedWords,
   searchSubtitleSentences,
@@ -100,6 +101,7 @@ import {
   VocabularySummaryWorkerRuntime,
   type RunVocabularySummaryTask,
 } from './immersion-tracker/vocabulary-summary-worker-runtime';
+import { LexicalRollupWorkerRuntime } from './immersion-tracker/lexical-rollup-worker-runtime';
 import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
 import {
   cleanupDuplicateSubtitleLines,
@@ -416,6 +418,7 @@ export class ImmersionTrackerService {
     knownWords: ReadonlySet<string> | null,
   ) => Promise<VocabularyStatsSummary>;
   private readonly destroyVocabularySummaryRunner: () => void;
+  private readonly destroyLexicalRollupBackfillRunner: () => void;
   private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
   private flushTimer: ReturnType<typeof setTimeout> | null = null;
   private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
@@ -482,6 +485,8 @@ export class ImmersionTrackerService {
         vocabularySummaryRuntime.run(this.dbPath, knownWords);
       this.destroyVocabularySummaryRunner = () => vocabularySummaryRuntime.destroy();
     }
+    const lexicalRollupRuntime = new LexicalRollupWorkerRuntime();
+    this.destroyLexicalRollupBackfillRunner = () => lexicalRollupRuntime.destroy();
     const parentDir = path.dirname(this.dbPath);
     if (!fs.existsSync(parentDir)) {
       fs.mkdirSync(parentDir, { recursive: true });
@@ -541,6 +546,12 @@ export class ImmersionTrackerService {
     this.db = new Database(this.dbPath);
     applyPragmas(this.db);
     ensureSchema(this.db);
+    void lexicalRollupRuntime.run(this.dbPath).catch((error: unknown) => {
+      this.logger.warn(
+        'Lexical daily rollup backfill failed; it will retry on next startup',
+        error,
+      );
+    });
     const reconciledSessions = reconcileStaleActiveSessions(this.db);
     if (reconciledSessions > 0) {
       this.logger.info(
@@ -588,6 +599,7 @@ export class ImmersionTrackerService {
     this.deleteMaintenanceScheduler.destroy();
     this.destroyDeleteMaintenanceRunner();
     this.destroyVocabularySummaryRunner();
+    this.destroyLexicalRollupBackfillRunner();
     this.db.close();
   }
 
@@ -661,6 +673,10 @@ export class ImmersionTrackerService {
     return this.runVocabularySummaryTask(knownWords);
   }
 
+  async getVocabularyChartData() {
+    return getVocabularyChartData(this.db);
+  }
+
   async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {
     return getStatsExcludedWords(this.db);
   }

src/core/services/immersion-tracker/lexical-rollup-worker-runtime.test.ts

@@ -0,0 +1,56 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+import {
+  LexicalRollupWorkerRuntime,
+  resolveLexicalRollupWorkerPath,
+} from './lexical-rollup-worker-runtime';
+import { areLexicalDailyRollupsReady } from './lexical-rollups';
+import { Database } from './sqlite';
+import { applyPragmas, ensureSchema } from './storage';
+
+test('lexical rollup worker backfills without using the tracker connection', async () => {
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-runtime-'));
+  const dbPath = path.join(directory, 'immersion.sqlite');
+  const runtime = new LexicalRollupWorkerRuntime();
+  const db = new Database(dbPath);
+
+  try {
+    applyPragmas(db);
+    ensureSchema(db);
+    db.prepare(
+      `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
+       VALUES ('鳥', '鳥', 'とり', 1700000000, 1700000000, 1)`,
+    ).run();
+    db.exec('DELETE FROM imm_lexical_daily_rollups');
+    db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
+      'lexical_daily_rollups_ready',
+    );
+    db.close();
+
+    await runtime.run(dbPath);
+
+    const checkDb = new Database(dbPath);
+    try {
+      assert.equal(areLexicalDailyRollupsReady(checkDb), true);
+    } finally {
+      checkDb.close();
+    }
+  } finally {
+    runtime.destroy();
+    try {
+      db.close();
+    } catch {
+      // Closed before the worker starts.
+    }
+    fs.rmSync(directory, { recursive: true, force: true });
+  }
+});
+
+test('lexical rollup worker module resolves in the current layout', () => {
+  const workerPath = resolveLexicalRollupWorkerPath();
+  assert.ok(workerPath, 'expected the lexical rollup worker module to resolve');
+  assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
+});

src/core/services/immersion-tracker/lexical-rollup-worker-runtime.ts

@@ -0,0 +1,96 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { createLogger } from '../../../logger';
+import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
+
+interface WorkerResponse {
+  ok?: boolean;
+  error?: unknown;
+}
+
+interface WorkerHandle {
+  once(event: 'message', listener: (message: WorkerResponse) => void): this;
+  once(event: 'error', listener: (error: Error) => void): this;
+  once(event: 'exit', listener: (code: number) => void): this;
+  terminate(): Promise<number>;
+}
+
+const logger = createLogger('main:immersion-tracker:lexical-rollup-worker');
+
+export function resolveLexicalRollupWorkerPath(): string | null {
+  const fileName = __filename.endsWith('.ts')
+    ? 'lexical-rollup-worker-thread.ts'
+    : 'lexical-rollup-worker-thread.js';
+  const workerPath = path.join(__dirname, fileName);
+  return fs.existsSync(workerPath) ? workerPath : null;
+}
+
+export class LexicalRollupWorkerRuntime {
+  private readonly activeWorkers = new Set<WorkerHandle>();
+  private destroyed = false;
+
+  async run(dbPath: string): Promise<void> {
+    if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
+    let worker: WorkerHandle;
+    try {
+      const workerPath = resolveLexicalRollupWorkerPath();
+      if (!workerPath) throw new Error('Emitted lexical rollup worker module was not found');
+      const { Worker } = await import('node:worker_threads');
+      worker = new Worker(workerPath, { workerData: { dbPath } });
+    } catch (error) {
+      if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
+      logger.warn(
+        'Lexical rollup worker unavailable; running backfill on the current thread',
+        error,
+      );
+      executeLexicalRollupBackfillTask(dbPath);
+      return;
+    }
+
+    if (this.destroyed) {
+      await worker.terminate().catch(() => undefined);
+      throw new Error('Lexical rollup worker is shut down');
+    }
+
+    return new Promise<void>((resolve, reject) => {
+      let settled = false;
+      this.activeWorkers.add(worker);
+      const settle = (error?: Error) => {
+        if (settled) return;
+        settled = true;
+        this.activeWorkers.delete(worker);
+        void worker.terminate();
+        if (error) reject(error);
+        else resolve();
+      };
+      worker.once('message', (message) => {
+        if (message.ok) settle();
+        else
+          settle(
+            new Error(
+              `Lexical rollup backfill failed: ${String(message.error ?? 'unknown error')}`,
+            ),
+          );
+      });
+      worker.once('error', (error) => settle(error));
+      worker.once('exit', (code) => {
+        if (!settled) {
+          settle(
+            new Error(
+              code === 0
+                ? 'Lexical rollup worker exited without a response'
+                : `Lexical rollup worker exited with code ${code}`,
+            ),
+          );
+        }
+      });
+    });
+  }
+
+  destroy(): void {
+    if (this.destroyed) return;
+    this.destroyed = true;
+    for (const worker of this.activeWorkers) void worker.terminate();
+    this.activeWorkers.clear();
+  }
+}

src/core/services/immersion-tracker/lexical-rollup-worker-thread.ts

@@ -0,0 +1,11 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
+
+if (!parentPort) throw new Error('lexical rollup worker missing parent port');
+
+try {
+  executeLexicalRollupBackfillTask((workerData as { dbPath: string }).dbPath);
+  parentPort.postMessage({ ok: true });
+} catch (error) {
+  parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
+}

src/core/services/immersion-tracker/lexical-rollup-worker.test.ts

@@ -0,0 +1,35 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
+import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
+import { Database } from './sqlite';
+import { ensureSchema } from './storage';
+
+test('lexical rollup backfill materializes pre-existing vocabulary off the caller DB connection', () => {
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-worker-'));
+  const dbPath = path.join(directory, 'immersion.sqlite');
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    db.prepare(
+      `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
+       VALUES (?, ?, ?, ?, ?, 1)`,
+    ).run('犬', '犬', 'いぬ', 1_700_000_000, 1_700_000_000);
+    db.exec('DELETE FROM imm_lexical_daily_rollups');
+    db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
+      'lexical_daily_rollups_ready',
+    );
+
+    executeLexicalRollupBackfillTask(dbPath);
+
+    assert.equal(areLexicalDailyRollupsReady(db), true);
+    assert.equal(getLexicalDailyRollups(db)[0]?.wordCount, 1);
+  } finally {
+    db.close();
+    fs.rmSync(directory, { recursive: true, force: true });
+  }
+});

src/core/services/immersion-tracker/lexical-rollup-worker.ts

@@ -0,0 +1,15 @@
+import { areLexicalDailyRollupsReady, rebuildLexicalDailyRollups } from './lexical-rollups';
+import { Database } from './sqlite';
+import { applyPragmas } from './storage';
+
+export function executeLexicalRollupBackfillTask(dbPath: string): void {
+  const db = new Database(dbPath);
+  try {
+    applyPragmas(db);
+    if (!areLexicalDailyRollupsReady(db)) {
+      rebuildLexicalDailyRollups(db);
+    }
+  } finally {
+    db.close();
+  }
+}

src/core/services/immersion-tracker/lexical-rollups.test.ts

@@ -0,0 +1,97 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+import { getLexicalDailyRollups } from './lexical-rollups';
+import { getTrendsDashboard } from './query-trends';
+import { getVocabularyChartData } from './query-lexical';
+import { Database } from './sqlite';
+import { ensureSchema } from './storage';
+
+function makeDbPath(): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollups-'));
+  return path.join(dir, 'immersion.sqlite');
+}
+
+test('lexical daily rollups follow first-seen corrections and deletions', () => {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    const firstDay = 19_500;
+    const correctedDay = firstDay + 2;
+    const firstSeen = firstDay * 86_400 + 43_200;
+    const correctedSeen = correctedDay * 86_400 + 43_200;
+
+    db.prepare(
+      `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
+       VALUES (?, ?, ?, ?, ?, 1)`,
+    ).run('猫', '猫', 'ねこ', firstSeen, firstSeen);
+    db.prepare(
+      `INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
+       VALUES (?, ?, ?, 1)`,
+    ).run('猫', firstSeen, firstSeen);
+
+    assert.deepEqual(getLexicalDailyRollups(db), [
+      { epochDay: firstDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 1 },
+    ]);
+
+    db.prepare(`UPDATE imm_words SET first_seen = ? WHERE headword = ?`).run(correctedSeen, '猫');
+    db.prepare(`DELETE FROM imm_kanji WHERE kanji = ?`).run('猫');
+
+    assert.deepEqual(getLexicalDailyRollups(db), [
+      { epochDay: correctedDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
+    ]);
+  } finally {
+    db.close();
+    fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+  }
+});
+
+test('vocabulary charts use complete top-word and lexical rollup data', () => {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    const insertWord = db.prepare(
+      `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
+       VALUES (?, ?, '', 1700000000, 1700000000, ?)`,
+    );
+    for (let index = 0; index < 501; index += 1) {
+      insertWord.run(`語${index}`, `語${index}`, index === 500 ? 10_000 : 1);
+    }
+
+    const charts = getVocabularyChartData(db);
+
+    assert.equal(charts.topWords[0]?.headword, '語500');
+    assert.equal(charts.topWords[0]?.frequency, 10_000);
+    assert.equal(charts.newWordsTimeline[0]?.wordCount, 501);
+  } finally {
+    db.close();
+    fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+  }
+});
+
+test('trends read historical new-word buckets from lexical rollups', () => {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+
+  try {
+    ensureSchema(db);
+    db.prepare(
+      `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
+       VALUES ('海', '海', 'うみ', 1700000000, 1700000000, 1)`,
+    ).run();
+    db.prepare(`UPDATE imm_lexical_daily_rollups SET word_count = 9`).run();
+
+    const dashboard = getTrendsDashboard(db, 'all', 'day', false);
+
+    assert.equal(dashboard.progress.newWords[0]?.value, 9);
+  } finally {
+    db.close();
+    fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
+  }
+});

src/core/services/immersion-tracker/lexical-rollups.ts

@@ -0,0 +1,170 @@
+import type { DatabaseSync } from './sqlite';
+
+export interface LexicalDailyRollup {
+  epochDay: number;
+  wordCount: number;
+  wordCountWithoutNames: number;
+  kanjiCount: number;
+}
+
+const LOCAL_EPOCH_DAY_SQL = `
+  CAST(julianday(CAST(%VALUE% AS REAL), 'unixepoch', 'localtime') - 2440587.5 AS INTEGER)
+`;
+
+function localEpochDaySql(value: string): string {
+  return LOCAL_EPOCH_DAY_SQL.replace('%VALUE%', value);
+}
+
+function createWordRollupTriggers(db: DatabaseSync): void {
+  const dayForNew = localEpochDaySql('NEW.first_seen');
+  const dayForOld = localEpochDaySql('OLD.first_seen');
+
+  db.exec(`
+    CREATE TRIGGER IF NOT EXISTS imm_words_lexical_rollup_insert
+    AFTER INSERT ON imm_words
+    WHEN NEW.first_seen IS NOT NULL
+    BEGIN
+      INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
+      VALUES (${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0)
+      ON CONFLICT(epoch_day) DO UPDATE SET
+        word_count = word_count + 1,
+        word_count_without_names = word_count_without_names + excluded.word_count_without_names;
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS imm_words_lexical_rollup_delete
+    AFTER DELETE ON imm_words
+    WHEN OLD.first_seen IS NOT NULL
+    BEGIN
+      INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
+      VALUES (${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0)
+      ON CONFLICT(epoch_day) DO UPDATE SET
+        word_count = word_count - 1,
+        word_count_without_names = word_count_without_names + excluded.word_count_without_names;
+      DELETE FROM imm_lexical_daily_rollups
+      WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS imm_words_lexical_rollup_first_seen_update
+    AFTER UPDATE OF first_seen, pos2 ON imm_words
+    WHEN OLD.first_seen IS NOT NEW.first_seen OR OLD.pos2 IS NOT NEW.pos2
+    BEGIN
+      INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
+      SELECT ${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0
+      WHERE OLD.first_seen IS NOT NULL
+      ON CONFLICT(epoch_day) DO UPDATE SET
+        word_count = word_count - 1,
+        word_count_without_names = word_count_without_names + excluded.word_count_without_names;
+      INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
+      SELECT ${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0
+      WHERE NEW.first_seen IS NOT NULL
+      ON CONFLICT(epoch_day) DO UPDATE SET
+        word_count = word_count + 1,
+        word_count_without_names = word_count_without_names + excluded.word_count_without_names;
+      DELETE FROM imm_lexical_daily_rollups
+      WHERE word_count = 0 AND kanji_count = 0;
+    END;
+  `);
+}
+
+function createKanjiRollupTriggers(db: DatabaseSync): void {
+  const dayForNew = localEpochDaySql('NEW.first_seen');
+  const dayForOld = localEpochDaySql('OLD.first_seen');
+  db.exec(`
+    CREATE TRIGGER IF NOT EXISTS imm_kanji_lexical_rollup_insert
+    AFTER INSERT ON imm_kanji WHEN NEW.first_seen IS NOT NULL
+    BEGIN
+      INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
+      VALUES (${dayForNew}, 0, 0, 1)
+      ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
+    END;
+    CREATE TRIGGER IF NOT EXISTS imm_kanji_lexical_rollup_delete
+    AFTER DELETE ON imm_kanji WHEN OLD.first_seen IS NOT NULL
+    BEGIN
+      INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
+      VALUES (${dayForOld}, 0, 0, -1)
+      ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
+      DELETE FROM imm_lexical_daily_rollups
+      WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
+    END;
+    CREATE TRIGGER IF NOT EXISTS imm_kanji_lexical_rollup_first_seen_update
+    AFTER UPDATE OF first_seen ON imm_kanji WHEN OLD.first_seen IS NOT NEW.first_seen
+    BEGIN
+      INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
+      SELECT ${dayForOld}, 0, 0, -1 WHERE OLD.first_seen IS NOT NULL
+      ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
+      INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
+      SELECT ${dayForNew}, 0, 0, 1 WHERE NEW.first_seen IS NOT NULL
+      ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
+      DELETE FROM imm_lexical_daily_rollups WHERE word_count = 0 AND kanji_count = 0;
+    END;
+  `);
+}
+
+export function ensureLexicalDailyRollupTables(db: DatabaseSync): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS imm_lexical_daily_rollups(
+      epoch_day INTEGER PRIMARY KEY,
+      word_count INTEGER NOT NULL DEFAULT 0,
+      word_count_without_names INTEGER NOT NULL DEFAULT 0,
+      kanji_count INTEGER NOT NULL DEFAULT 0
+    );
+    INSERT INTO imm_rollup_state(state_key, state_value)
+    VALUES ('lexical_daily_rollups_ready', '0')
+    ON CONFLICT(state_key) DO NOTHING;
+  `);
+  createWordRollupTriggers(db);
+  createKanjiRollupTriggers(db);
+}
+
+export function areLexicalDailyRollupsReady(db: DatabaseSync): boolean {
+  const row = db
+    .prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
+    .get('lexical_daily_rollups_ready') as { value: string } | null;
+  return row?.value === '1';
+}
+
+export function markLexicalDailyRollupsReady(db: DatabaseSync): void {
+  db.prepare(`UPDATE imm_rollup_state SET state_value = '1' WHERE state_key = ?`).run(
+    'lexical_daily_rollups_ready',
+  );
+}
+
+/** Rebuild from the first-seen source of truth; run off the UI/main DB thread. */
+export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
+  db.exec('BEGIN IMMEDIATE');
+  try {
+    db.exec('DELETE FROM imm_lexical_daily_rollups');
+    db.exec(`
+      INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
+      SELECT ${localEpochDaySql('first_seen')}, COUNT(*),
+        SUM(CASE WHEN pos2 = '固有名詞' THEN 0 ELSE 1 END), 0
+      FROM imm_words
+      WHERE first_seen IS NOT NULL
+      GROUP BY ${localEpochDaySql('first_seen')};
+      INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
+      SELECT ${localEpochDaySql('first_seen')}, 0, 0, COUNT(*)
+      FROM imm_kanji
+      WHERE first_seen IS NOT NULL
+      GROUP BY ${localEpochDaySql('first_seen')}
+      ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + excluded.kanji_count;
+    `);
+    markLexicalDailyRollupsReady(db);
+    db.exec('COMMIT');
+  } catch (error) {
+    db.exec('ROLLBACK');
+    throw error;
+  }
+}
+
+export function getLexicalDailyRollups(db: DatabaseSync): LexicalDailyRollup[] {
+  return db
+    .prepare(
+      `
+        SELECT epoch_day AS epochDay, word_count AS wordCount,
+          word_count_without_names AS wordCountWithoutNames, kanji_count AS kanjiCount
+        FROM imm_lexical_daily_rollups
+        ORDER BY epoch_day ASC
+      `,
+    )
+    .all() as LexicalDailyRollup[];
+}

src/core/services/immersion-tracker/query-lexical.ts

@@ -20,6 +20,7 @@ import type {
 } from './types';
 import { fromDbTimestamp, toDbTimestamp } from './query-shared';
 import { nowMs } from './time';
+import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
 
 const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
 const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
@@ -27,6 +28,14 @@ const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
 const SENTENCE_SEARCH_MAX_LIMIT = 100;
 const KANJI_PATTERN = /\p{Script=Han}/gu;
 
+export interface VocabularyChartData {
+  ready: boolean;
+  topWords: Array<{ wordId: number; headword: string; frequency: number }>;
+  topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>;
+  newWordsTimeline: Array<{ epochDay: number; wordCount: number }>;
+  newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>;
+}
+
 function resolveSentenceSearchLimit(limit: number): number {
   if (!Number.isFinite(limit)) return SENTENCE_SEARCH_DEFAULT_LIMIT;
   const normalized = Math.floor(limit);
@@ -154,6 +163,64 @@ export function getVocabularyStats(
   return visibleRows.slice(0, limit);
 }
 
+/**
+ * Chart data is intentionally independent of the paginated vocabulary tables.
+ * Top words use the frequency index; new-word history reads permanent daily
+ * lexical rollups rather than loading every vocabulary row into the dashboard.
+ */
+export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData {
+  const ready = areLexicalDailyRollupsReady(db);
+  const excludedAliases = new Set(
+    getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
+  );
+  const isExcluded = (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>): boolean =>
+    excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias));
+  const topWords = getVocabularyStats(db, 48).filter((word) => !isExcluded(word));
+  const rollups = ready ? getLexicalDailyRollups(db) : [];
+  const timeline = new Map(rollups.map((row) => [row.epochDay, { ...row }]));
+  if (excludedAliases.size > 0 && ready) {
+    const aliases = [...excludedAliases];
+    const placeholders = aliases.map(() => '?').join(', ');
+    const excludedRows = db
+      .prepare(
+        `
+          SELECT headword, word, reading, pos2,
+            CAST(julianday(CAST(first_seen AS REAL), 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) AS epochDay
+          FROM imm_words
+          WHERE headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders})
+        `,
+      )
+      .all(...aliases, ...aliases, ...aliases) as Array<
+      Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & { epochDay: number }
+    >;
+    for (const word of excludedRows) {
+      if (!isExcluded(word)) continue;
+      const rollup = timeline.get(word.epochDay);
+      if (!rollup) continue;
+      rollup.wordCount -= 1;
+      if (word.pos2 !== '固有名詞') rollup.wordCountWithoutNames -= 1;
+    }
+  }
+  return {
+    ready,
+    topWords: topWords.slice(0, 12).map((word) => ({
+      wordId: word.wordId,
+      headword: word.headword,
+      frequency: word.frequency,
+    })),
+    topWordsWithoutNames: topWords
+      .filter((word) => word.pos2 !== '固有名詞')
+      .slice(0, 12)
+      .map((word) => ({ wordId: word.wordId, headword: word.headword, frequency: word.frequency })),
+    newWordsTimeline: [...timeline.values()]
+      .filter((row) => row.wordCount > 0)
+      .map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCount })),
+    newWordsTimelineWithoutNames: [...timeline.values()]
+      .filter((row) => row.wordCountWithoutNames > 0)
+      .map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCountWithoutNames })),
+  };
+}
+
 function excludedVocabularyAliases(
   word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
 ): string[] {

src/core/services/immersion-tracker/query-trends.ts

@@ -13,6 +13,7 @@ import {
   toDbTimestamp,
 } from './query-shared';
 import { getDailyRollups, getMonthlyRollups } from './query-sessions';
+import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
 
 type TrendRange = '7d' | '30d' | '90d' | '365d' | 'all';
 type TrendGroupBy = 'day' | 'month';
@@ -660,6 +661,14 @@ function buildNewWordsPerDay(
   cutoffMs: string | null,
   axis: number[] | null,
 ): TrendChartPoint[] {
+  if (areLexicalDailyRollupsReady(db)) {
+    const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
+    const rows = getLexicalDailyRollups(db).filter(
+      (row) => cutoffDay === null || row.epochDay >= cutoffDay,
+    );
+    return fillAxisPoints(axis, new Map(rows.map((row) => [row.epochDay, row.wordCount])));
+  }
+
   const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
   const prepared = db.prepare(`
     SELECT
@@ -691,6 +700,18 @@ function buildNewWordsPerMonth(
   cutoffMs: string | null,
   axis: number[] | null,
 ): TrendChartPoint[] {
+  if (areLexicalDailyRollupsReady(db)) {
+    const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
+    const byMonth = new Map<number, number>();
+    for (const row of getLexicalDailyRollups(db)) {
+      if (cutoffDay !== null && row.epochDay < cutoffDay) continue;
+      const { year, month } = dayPartsFromEpochDay(row.epochDay);
+      const monthKey = year * 100 + month;
+      byMonth.set(monthKey, (byMonth.get(monthKey) ?? 0) + row.wordCount);
+    }
+    return fillAxisPoints(axis, byMonth);
+  }
+
   const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
   const prepared = db.prepare(`
     SELECT

src/core/services/immersion-tracker/storage.ts

@@ -4,6 +4,7 @@ import { parseMediaInfo } from '../../../jimaku/utils';
 import { normalizeTitleIdentity } from '../../utils/title-normalization';
 import type { DatabaseSync } from './sqlite';
 import { nowMs } from './time';
+import { ensureLexicalDailyRollupTables, markLexicalDailyRollupsReady } from './lexical-rollups';
 import { SCHEMA_VERSION } from './types';
 import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types';
 import { toDbMs, toDbTimestamp } from './query-shared';
@@ -890,11 +891,11 @@ export function ensureSchema(db: DatabaseSync): void {
     VALUES ('last_rollup_sample_ms', 0)
     ON CONFLICT(state_key) DO NOTHING
   `);
-
   const currentVersion = db
     .prepare('SELECT schema_version FROM imm_schema_version ORDER BY schema_version DESC LIMIT 1')
     .get() as { schema_version: number } | null;
   if (currentVersion?.schema_version === SCHEMA_VERSION) {
+    ensureLexicalDailyRollupTables(db);
     ensureLifetimeSummaryTables(db);
     ensureStatsExcludedWordsTable(db);
     ensureAnimeMergeTables(db);
@@ -1453,6 +1454,7 @@ export function ensureSchema(db: DatabaseSync): void {
 
   migrateSessionEventTimestampsToText(db);
 
+  ensureLexicalDailyRollupTables(db);
   ensureLifetimeSummaryTables(db);
   ensureStatsExcludedWordsTable(db);
 
@@ -1585,6 +1587,12 @@ export function ensureSchema(db: DatabaseSync): void {
     VALUES (${SCHEMA_VERSION}, ${toDbTimestamp(nowMs())})
     ON CONFLICT DO NOTHING
   `);
+
+  // A new database has no history to materialize. Upgrades are populated by the
+  // background worker so startup never scans the existing vocabulary table.
+  if (!currentVersion) {
+    markLexicalDailyRollupsReady(db);
+  }
 }
 
 export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPreparedStatements {

src/core/services/immersion-tracker/types.ts

@@ -1,4 +1,4 @@
-export const SCHEMA_VERSION = 21;
+export const SCHEMA_VERSION = 22;
 export const DEFAULT_QUEUE_CAP = 1_000;
 export const DEFAULT_BATCH_SIZE = 25;
 export const DEFAULT_FLUSH_INTERVAL_MS = 500;

src/core/services/stats-server/library-routes.ts

@@ -39,6 +39,10 @@ export function registerStatsLibraryRoutes(
     return c.json(statsJson('vocabularySummary', summary));
   });
 
+  app.get('/api/stats/vocabulary/charts', async (c) => {
+    return c.json(statsJson('vocabularyCharts', await tracker.getVocabularyChartData()));
+  });
+
   app.get('/api/stats/excluded-words', async (c) => {
     return c.json(statsJson('excludedWords', await tracker.getStatsExcludedWords()));
   });

src/types/stats-http-contract.ts

@@ -60,6 +60,14 @@ export interface StatsVocabularySummary {
   knownWordCountWithoutNames: number | null;
 }
 
+export interface StatsVocabularyCharts {
+  ready: boolean;
+  topWords: Array<{ wordId: number; headword: string; frequency: number }>;
+  topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>;
+  newWordsTimeline: Array<{ epochDay: number; wordCount: number }>;
+  newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>;
+}
+
 export interface StatsAnilistSearchResult {
   id: number;
   episodes: number | null;
@@ -175,6 +183,7 @@ export interface StatsJsonResponseMap {
   sessionKnownWordsTimeline: StatsSessionKnownWordsTimelinePoint[];
   vocabulary: VocabularyEntry[];
   vocabularySummary: StatsVocabularySummary;
+  vocabularyCharts: StatsVocabularyCharts;
   excludedWords: StatsExcludedWord[];
   setExcludedWords: StatsOkResponse;
   duplicateLineCleanup: StatsDuplicateLineCleanupResult;
@@ -234,6 +243,7 @@ export interface StatsHttpClient {
   getSessionKnownWordsTimeline: (id: number) => Promise<StatsSessionKnownWordsTimelinePoint[]>;
   getVocabulary: (limit?: number) => Promise<VocabularyEntry[]>;
   getVocabularySummary: () => Promise<StatsVocabularySummary>;
+  getVocabularyCharts: () => Promise<StatsVocabularyCharts>;
   getExcludedWords: () => Promise<StatsExcludedWord[]>;
   setExcludedWords: (words: StatsExcludedWord[]) => Promise<void>;
   cleanupDuplicateLines: (