perf(stats): calculate vocabulary totals off main thread

changes/stats-vocabulary-summary-totals.md

@@ -1,4 +1,4 @@
 type: fixed
 area: stats
 
-- Fixed Vocabulary summary cards counting only the first page of frequency-ranked words and kanji instead of all tracked vocabulary.
+- Fixed Vocabulary summary cards counting only the first page of frequency-ranked words and kanji instead of all tracked vocabulary, without delaying the rest of the page.

docs-site/immersion-tracking.md

@@ -82,7 +82,7 @@ Expandable session history with new-word activity, cumulative totals, and pause/
 
 #### Vocabulary
 
-The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The rest of the tab includes top repeated words (click a bar to open the word), new-word timeline, cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons.
+The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The word and kanji tables load first while those complete totals calculate separately. The rest of the tab includes top repeated words (click a bar to open the word), new-word timeline, cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons.
 
 ![Stats Vocabulary](/screenshots/stats-vocabulary.png)
 

src/core/services/immersion-tracker-service.ts

@@ -59,7 +59,6 @@ import {
   getSimilarWords,
   getStatsExcludedWords,
   getVocabularyStats,
-  getVocabularySummary,
   replaceStatsExcludedWords,
   searchSubtitleSentences,
   getWordAnimeAppearances,
@@ -97,6 +96,10 @@ import {
   DeleteMaintenanceWorkerRuntime,
   type RunDeleteMaintenanceTask,
 } from './immersion-tracker/delete-maintenance-worker-runtime';
+import {
+  VocabularySummaryWorkerRuntime,
+  type RunVocabularySummaryTask,
+} from './immersion-tracker/vocabulary-summary-worker-runtime';
 import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
 import {
   cleanupDuplicateSubtitleLines,
@@ -186,6 +189,7 @@ import {
   type StatsExcludedWordRow,
   type StreakCalendarRow,
   type VocabularyCleanupSummary,
+  type VocabularyStatsSummary,
   type WatchTimePerAnimeRow,
   type WordAnimeAppearanceRow,
   type WordDetailRow,
@@ -408,6 +412,10 @@ export class ImmersionTrackerService {
   private readonly dbPath: string;
   private readonly writeLock = { locked: false };
   private readonly destroyDeleteMaintenanceRunner: () => void;
+  private readonly runVocabularySummaryTask: (
+    knownWords: ReadonlySet<string> | null,
+  ) => Promise<VocabularyStatsSummary>;
+  private readonly destroyVocabularySummaryRunner: () => void;
   private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
   private flushTimer: ReturnType<typeof setTimeout> | null = null;
   private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
@@ -435,6 +443,8 @@ export class ImmersionTrackerService {
     dependencies: {
       runDeleteMaintenanceTask?: RunDeleteMaintenanceTask;
       destroyDeleteMaintenanceRunner?: () => void;
+      runVocabularySummaryTask?: RunVocabularySummaryTask;
+      destroyVocabularySummaryRunner?: () => void;
     } = {},
   ) {
     this.dbPath = options.dbPath;
@@ -461,6 +471,17 @@ export class ImmersionTrackerService {
         if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
       },
     });
+    if (dependencies.runVocabularySummaryTask) {
+      this.runVocabularySummaryTask = (knownWords) =>
+        dependencies.runVocabularySummaryTask!(this.dbPath, knownWords);
+      this.destroyVocabularySummaryRunner =
+        dependencies.destroyVocabularySummaryRunner ?? (() => {});
+    } else {
+      const vocabularySummaryRuntime = new VocabularySummaryWorkerRuntime();
+      this.runVocabularySummaryTask = (knownWords) =>
+        vocabularySummaryRuntime.run(this.dbPath, knownWords);
+      this.destroyVocabularySummaryRunner = () => vocabularySummaryRuntime.destroy();
+    }
     const parentDir = path.dirname(this.dbPath);
     if (!fs.existsSync(parentDir)) {
       fs.mkdirSync(parentDir, { recursive: true });
@@ -566,6 +587,7 @@ export class ImmersionTrackerService {
     this.isDestroyed = true;
     this.deleteMaintenanceScheduler.destroy();
     this.destroyDeleteMaintenanceRunner();
+    this.destroyVocabularySummaryRunner();
     this.db.close();
   }
 
@@ -636,7 +658,7 @@ export class ImmersionTrackerService {
   }
 
   async getVocabularySummary(knownWords: ReadonlySet<string> | null) {
-    return getVocabularySummary(this.db, knownWords);
+    return this.runVocabularySummaryTask(knownWords);
   }
 
   async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {

src/core/services/immersion-tracker/vocabulary-summary-worker-runtime.test.ts

@@ -0,0 +1,51 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+import {
+  resolveVocabularySummaryWorkerPath,
+  VocabularySummaryWorkerRuntime,
+} from './vocabulary-summary-worker-runtime';
+import { Database } from './sqlite';
+import { applyPragmas, ensureSchema } from './storage';
+
+test('vocabulary summary worker reads the database from a separate connection', async () => {
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-vocabulary-summary-worker-'));
+  const dbPath = path.join(tempDir, 'immersion.sqlite');
+  const runtime = new VocabularySummaryWorkerRuntime();
+  const db = new Database(dbPath);
+
+  try {
+    applyPragmas(db);
+    ensureSchema(db);
+    db.prepare(
+      `
+      INSERT INTO imm_words (
+        headword, word, reading, part_of_speech, pos1, pos2, pos3,
+        first_seen, last_seen, frequency
+      ) VALUES ('猫', '猫', 'ねこ', 'noun', '名詞', '一般', '', 1, 1, 1)
+    `,
+    ).run();
+    db.close();
+
+    const summary = await runtime.run(dbPath, new Set(['猫']));
+
+    assert.equal(summary.uniqueWords, 1);
+    assert.equal(summary.knownWordCount, 1);
+  } finally {
+    runtime.destroy();
+    try {
+      db.close();
+    } catch {
+      // The worker needs the setup connection closed before it starts.
+    }
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  }
+});
+
+test('vocabulary summary worker module resolves in the current layout', () => {
+  const workerPath = resolveVocabularySummaryWorkerPath();
+  assert.ok(workerPath, 'expected the vocabulary summary worker module to resolve');
+  assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
+});

src/core/services/immersion-tracker/vocabulary-summary-worker-runtime.ts

@@ -0,0 +1,128 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { createLogger } from '../../../logger';
+import type { VocabularyStatsSummary } from './types';
+import { executeVocabularySummaryTask } from './vocabulary-summary-worker';
+
+interface VocabularySummaryWorkerResponse {
+  summary?: VocabularyStatsSummary;
+  error?: unknown;
+}
+
+interface VocabularySummaryWorkerHandle {
+  once(event: 'message', listener: (message: VocabularySummaryWorkerResponse) => void): this;
+  once(event: 'error', listener: (error: Error) => void): this;
+  once(event: 'exit', listener: (code: number) => void): this;
+  terminate(): Promise<number>;
+}
+
+interface VocabularySummaryWorkerRuntimeOptions {
+  resolveWorkerPath?: () => string | null;
+  createWorker?: (
+    workerPath: string,
+    workerData: { dbPath: string; knownWords: string[] | null },
+  ) => Promise<VocabularySummaryWorkerHandle>;
+  executeFallback?: typeof executeVocabularySummaryTask;
+  warn?: (message: string, ...meta: unknown[]) => void;
+}
+
+export type RunVocabularySummaryTask = (
+  dbPath: string,
+  knownWords: ReadonlySet<string> | null,
+) => Promise<VocabularyStatsSummary>;
+
+export function resolveVocabularySummaryWorkerPath(): string | null {
+  const fileName = __filename.endsWith('.ts')
+    ? 'vocabulary-summary-worker-thread.ts'
+    : 'vocabulary-summary-worker-thread.js';
+  const workerPath = path.join(__dirname, fileName);
+  return fs.existsSync(workerPath) ? workerPath : null;
+}
+
+const logger = createLogger('main:immersion-tracker:vocabulary-summary-worker');
+
+export class VocabularySummaryWorkerRuntime {
+  private readonly activeWorkers = new Set<VocabularySummaryWorkerHandle>();
+  private destroyed = false;
+
+  constructor(private readonly options: VocabularySummaryWorkerRuntimeOptions = {}) {}
+
+  async run(
+    dbPath: string,
+    knownWords: ReadonlySet<string> | null,
+  ): Promise<VocabularyStatsSummary> {
+    if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
+    const workerData = { dbPath, knownWords: knownWords ? [...knownWords] : null };
+    let worker: VocabularySummaryWorkerHandle;
+    try {
+      const workerPath = (this.options.resolveWorkerPath ?? resolveVocabularySummaryWorkerPath)();
+      if (!workerPath) throw new Error('Emitted vocabulary summary worker module was not found');
+      const createWorker =
+        this.options.createWorker ??
+        (async (resolvedPath, data) => {
+          const { Worker } = await import('node:worker_threads');
+          return new Worker(resolvedPath, { workerData: data });
+        });
+      worker = await createWorker(workerPath, workerData);
+    } catch (error) {
+      if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
+      (this.options.warn ?? logger.warn)(
+        'Vocabulary summary worker unavailable; running summary on the current thread',
+        error,
+      );
+      return (this.options.executeFallback ?? executeVocabularySummaryTask)(
+        dbPath,
+        workerData.knownWords,
+      );
+    }
+
+    if (this.destroyed) {
+      await worker.terminate().catch(() => undefined);
+      throw new Error('Vocabulary summary worker is shut down');
+    }
+
+    return new Promise<VocabularyStatsSummary>((resolve, reject) => {
+      let settled = false;
+      this.activeWorkers.add(worker);
+      const settle = (result: VocabularyStatsSummary | Error) => {
+        if (settled) return;
+        settled = true;
+        this.activeWorkers.delete(worker);
+        void worker.terminate();
+        if (result instanceof Error) reject(result);
+        else resolve(result);
+      };
+
+      worker.once('message', (message) => {
+        if (message.summary) {
+          settle(message.summary);
+          return;
+        }
+        settle(
+          new Error(
+            `Vocabulary summary failed: ${String(message.error ?? 'unknown worker error')}`,
+          ),
+        );
+      });
+      worker.once('error', (error) => settle(error));
+      worker.once('exit', (code) => {
+        if (!settled) {
+          settle(
+            new Error(
+              code === 0
+                ? 'Vocabulary summary worker exited without a response'
+                : `Vocabulary summary worker exited with code ${code}`,
+            ),
+          );
+        }
+      });
+    });
+  }
+
+  destroy(): void {
+    if (this.destroyed) return;
+    this.destroyed = true;
+    for (const worker of this.activeWorkers) void worker.terminate();
+    this.activeWorkers.clear();
+  }
+}

src/core/services/immersion-tracker/vocabulary-summary-worker-thread.ts

@@ -0,0 +1,19 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { executeVocabularySummaryTask } from './vocabulary-summary-worker';
+
+interface VocabularySummaryWorkerData {
+  dbPath: string;
+  knownWords: string[] | null;
+}
+
+if (!parentPort) throw new Error('vocabulary summary worker missing parent port');
+
+const request = workerData as VocabularySummaryWorkerData;
+
+try {
+  parentPort.postMessage({
+    summary: executeVocabularySummaryTask(request.dbPath, request.knownWords),
+  });
+} catch (error) {
+  parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
+}

src/core/services/immersion-tracker/vocabulary-summary-worker.ts

@@ -0,0 +1,17 @@
+import { getVocabularySummary } from './query-lexical';
+import { Database } from './sqlite';
+import { applyPragmas } from './storage';
+import type { VocabularyStatsSummary } from './types';
+
+export function executeVocabularySummaryTask(
+  dbPath: string,
+  knownWords: string[] | null,
+): VocabularyStatsSummary {
+  const db = new Database(dbPath);
+  try {
+    applyPragmas(db);
+    return getVocabularySummary(db, knownWords ? new Set(knownWords) : null);
+  } finally {
+    db.close();
+  }
+}

stats/src/components/vocabulary/VocabularyTab.tsx

@@ -98,24 +98,26 @@ export function VocabularyTab({
       <div className="grid grid-cols-2 xl:grid-cols-4 gap-3">
         <StatCard
           label="Unique Words"
-          value={formatNumber(displayedSummary.uniqueWords)}
+          value={summary ? formatNumber(displayedSummary.uniqueWords) : '…'}
           color="text-ctp-blue"
         />
-        {displayedSummary.knownWordCount !== null && (
+        {displayedSummary.knownWordCount !== null ? (
           <StatCard
             label="Known Words"
             value={`${formatNumber(displayedSummary.knownWordCount)} (${displayedSummary.uniqueWords > 0 ? Math.round((displayedSummary.knownWordCount / displayedSummary.uniqueWords) * 100) : 0}%)`}
             color="text-ctp-green"
           />
-        )}
+        ) : knownWords.size > 0 ? (
+          <StatCard label="Known Words" value="…" color="text-ctp-green" />
+        ) : null}
         <StatCard
           label="Unique Kanji"
-          value={formatNumber(summary?.uniqueKanji ?? 0)}
+          value={summary ? formatNumber(summary.uniqueKanji) : '…'}
           color="text-ctp-teal"
         />
         <StatCard
           label="New This Week"
-          value={`+${formatNumber(displayedSummary.newThisWeek)}`}
+          value={summary ? `+${formatNumber(displayedSummary.newThisWeek)}` : '…'}
           color="text-ctp-mauve"
         />
       </div>

stats/src/hooks/useVocabulary.ts

@@ -17,14 +17,10 @@ export function useVocabulary() {
     let cancelled = false;
     setLoading(true);
     setError(null);
+    setSummary(null);
     const client = getStatsClient();
-    Promise.allSettled([
-      client.getVocabulary(500),
-      client.getKanji(200),
-      client.getKnownWords(),
-      client.getVocabularySummary(),
-    ])
-      .then(([wordsResult, kanjiResult, knownResult, summaryResult]) => {
+    Promise.allSettled([client.getVocabulary(500), client.getKanji(200), client.getKnownWords()])
+      .then(([wordsResult, kanjiResult, knownResult]) => {
         if (cancelled) return;
         const errors: string[] = [];
 
@@ -44,12 +40,6 @@ export function useVocabulary() {
           setKnownWords(new Set(knownResult.value));
         }
 
-        if (summaryResult.status === 'fulfilled') {
-          setSummary(summaryResult.value);
-        } else {
-          errors.push(summaryResult.reason.message);
-        }
-
         if (errors.length > 0) {
           setError(errors.join('; '));
         }
@@ -58,6 +48,14 @@ export function useVocabulary() {
         if (cancelled) return;
         setLoading(false);
       });
+    void client
+      .getVocabularySummary()
+      .then((nextSummary) => {
+        if (!cancelled) setSummary(nextSummary);
+      })
+      .catch((summaryError: unknown) => {
+        console.error('Failed to load vocabulary summary', summaryError);
+      });
     return () => {
       cancelled = true;
     };

stats/src/lib/vocabulary-tab.test.ts

@@ -6,6 +6,7 @@ import { fileURLToPath } from 'node:url';
 const VOCABULARY_TAB_PATH = fileURLToPath(
   new URL('../components/vocabulary/VocabularyTab.tsx', import.meta.url),
 );
+const VOCABULARY_HOOK_PATH = fileURLToPath(new URL('../hooks/useVocabulary.ts', import.meta.url));
 
 test('VocabularyTab declares all hooks before loading and error early returns', () => {
   const source = fs.readFileSync(VOCABULARY_TAB_PATH, 'utf8');
@@ -33,5 +34,15 @@ test('VocabularyTab uses database-wide summary totals for its stat cards', () =>
   );
   assert.match(source, /uniqueWords: summary\?\.uniqueWordsWithoutNames \?\? 0/);
   assert.match(source, /uniqueWords: summary\?\.uniqueWords \?\? 0/);
-  assert.match(source, /value=\{formatNumber\(summary\?\.uniqueKanji \?\? 0\)\}/);
+  assert.match(source, /value=\{summary \? formatNumber\(summary\.uniqueKanji\) : '…'\}/);
+});
+
+test('useVocabulary loads exact card totals without holding up the vocabulary tables', () => {
+  const source = fs.readFileSync(VOCABULARY_HOOK_PATH, 'utf8');
+
+  assert.match(
+    source,
+    /Promise\.allSettled\(\[\s*client\.getVocabulary\(500\),\s*client\.getKanji\(200\),\s*client\.getKnownWords\(\),?\s*\]\)/,
+  );
+  assert.match(source, /void client\s*\.getVocabularySummary\(\)\s*\.then\(/);
 });