perf(stats): calculate vocabulary totals off main thread

src/core/services/immersion-tracker-service.ts

@@ -59,7 +59,6 @@ import {
   getSimilarWords,
   getStatsExcludedWords,
   getVocabularyStats,
-  getVocabularySummary,
   replaceStatsExcludedWords,
   searchSubtitleSentences,
   getWordAnimeAppearances,
@@ -97,6 +96,10 @@ import {
   DeleteMaintenanceWorkerRuntime,
   type RunDeleteMaintenanceTask,
 } from './immersion-tracker/delete-maintenance-worker-runtime';
+import {
+  VocabularySummaryWorkerRuntime,
+  type RunVocabularySummaryTask,
+} from './immersion-tracker/vocabulary-summary-worker-runtime';
 import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
 import {
   cleanupDuplicateSubtitleLines,
@@ -186,6 +189,7 @@ import {
   type StatsExcludedWordRow,
   type StreakCalendarRow,
   type VocabularyCleanupSummary,
+  type VocabularyStatsSummary,
   type WatchTimePerAnimeRow,
   type WordAnimeAppearanceRow,
   type WordDetailRow,
@@ -408,6 +412,10 @@ export class ImmersionTrackerService {
   private readonly dbPath: string;
   private readonly writeLock = { locked: false };
   private readonly destroyDeleteMaintenanceRunner: () => void;
+  private readonly runVocabularySummaryTask: (
+    knownWords: ReadonlySet<string> | null,
+  ) => Promise<VocabularyStatsSummary>;
+  private readonly destroyVocabularySummaryRunner: () => void;
   private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
   private flushTimer: ReturnType<typeof setTimeout> | null = null;
   private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
@@ -435,6 +443,8 @@ export class ImmersionTrackerService {
     dependencies: {
       runDeleteMaintenanceTask?: RunDeleteMaintenanceTask;
       destroyDeleteMaintenanceRunner?: () => void;
+      runVocabularySummaryTask?: RunVocabularySummaryTask;
+      destroyVocabularySummaryRunner?: () => void;
     } = {},
   ) {
     this.dbPath = options.dbPath;
@@ -461,6 +471,17 @@ export class ImmersionTrackerService {
         if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
       },
     });
+    if (dependencies.runVocabularySummaryTask) {
+      this.runVocabularySummaryTask = (knownWords) =>
+        dependencies.runVocabularySummaryTask!(this.dbPath, knownWords);
+      this.destroyVocabularySummaryRunner =
+        dependencies.destroyVocabularySummaryRunner ?? (() => {});
+    } else {
+      const vocabularySummaryRuntime = new VocabularySummaryWorkerRuntime();
+      this.runVocabularySummaryTask = (knownWords) =>
+        vocabularySummaryRuntime.run(this.dbPath, knownWords);
+      this.destroyVocabularySummaryRunner = () => vocabularySummaryRuntime.destroy();
+    }
     const parentDir = path.dirname(this.dbPath);
     if (!fs.existsSync(parentDir)) {
       fs.mkdirSync(parentDir, { recursive: true });
@@ -566,6 +587,7 @@ export class ImmersionTrackerService {
     this.isDestroyed = true;
     this.deleteMaintenanceScheduler.destroy();
     this.destroyDeleteMaintenanceRunner();
+    this.destroyVocabularySummaryRunner();
     this.db.close();
   }
 
@@ -636,7 +658,7 @@ export class ImmersionTrackerService {
   }
 
   async getVocabularySummary(knownWords: ReadonlySet<string> | null) {
-    return getVocabularySummary(this.db, knownWords);
+    return this.runVocabularySummaryTask(knownWords);
   }
 
   async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {

src/core/services/immersion-tracker/vocabulary-summary-worker-runtime.test.ts

@@ -0,0 +1,51 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+import {
+  resolveVocabularySummaryWorkerPath,
+  VocabularySummaryWorkerRuntime,
+} from './vocabulary-summary-worker-runtime';
+import { Database } from './sqlite';
+import { applyPragmas, ensureSchema } from './storage';
+
+test('vocabulary summary worker reads the database from a separate connection', async () => {
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-vocabulary-summary-worker-'));
+  const dbPath = path.join(tempDir, 'immersion.sqlite');
+  const runtime = new VocabularySummaryWorkerRuntime();
+  const db = new Database(dbPath);
+
+  try {
+    applyPragmas(db);
+    ensureSchema(db);
+    db.prepare(
+      `
+      INSERT INTO imm_words (
+        headword, word, reading, part_of_speech, pos1, pos2, pos3,
+        first_seen, last_seen, frequency
+      ) VALUES ('猫', '猫', 'ねこ', 'noun', '名詞', '一般', '', 1, 1, 1)
+    `,
+    ).run();
+    db.close();
+
+    const summary = await runtime.run(dbPath, new Set(['猫']));
+
+    assert.equal(summary.uniqueWords, 1);
+    assert.equal(summary.knownWordCount, 1);
+  } finally {
+    runtime.destroy();
+    try {
+      db.close();
+    } catch {
+      // The worker needs the setup connection closed before it starts.
+    }
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  }
+});
+
+test('vocabulary summary worker module resolves in the current layout', () => {
+  const workerPath = resolveVocabularySummaryWorkerPath();
+  assert.ok(workerPath, 'expected the vocabulary summary worker module to resolve');
+  assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
+});

src/core/services/immersion-tracker/vocabulary-summary-worker-runtime.ts

@@ -0,0 +1,128 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { createLogger } from '../../../logger';
+import type { VocabularyStatsSummary } from './types';
+import { executeVocabularySummaryTask } from './vocabulary-summary-worker';
+
+interface VocabularySummaryWorkerResponse {
+  summary?: VocabularyStatsSummary;
+  error?: unknown;
+}
+
+interface VocabularySummaryWorkerHandle {
+  once(event: 'message', listener: (message: VocabularySummaryWorkerResponse) => void): this;
+  once(event: 'error', listener: (error: Error) => void): this;
+  once(event: 'exit', listener: (code: number) => void): this;
+  terminate(): Promise<number>;
+}
+
+interface VocabularySummaryWorkerRuntimeOptions {
+  resolveWorkerPath?: () => string | null;
+  createWorker?: (
+    workerPath: string,
+    workerData: { dbPath: string; knownWords: string[] | null },
+  ) => Promise<VocabularySummaryWorkerHandle>;
+  executeFallback?: typeof executeVocabularySummaryTask;
+  warn?: (message: string, ...meta: unknown[]) => void;
+}
+
+export type RunVocabularySummaryTask = (
+  dbPath: string,
+  knownWords: ReadonlySet<string> | null,
+) => Promise<VocabularyStatsSummary>;
+
+export function resolveVocabularySummaryWorkerPath(): string | null {
+  const fileName = __filename.endsWith('.ts')
+    ? 'vocabulary-summary-worker-thread.ts'
+    : 'vocabulary-summary-worker-thread.js';
+  const workerPath = path.join(__dirname, fileName);
+  return fs.existsSync(workerPath) ? workerPath : null;
+}
+
+const logger = createLogger('main:immersion-tracker:vocabulary-summary-worker');
+
+export class VocabularySummaryWorkerRuntime {
+  private readonly activeWorkers = new Set<VocabularySummaryWorkerHandle>();
+  private destroyed = false;
+
+  constructor(private readonly options: VocabularySummaryWorkerRuntimeOptions = {}) {}
+
+  async run(
+    dbPath: string,
+    knownWords: ReadonlySet<string> | null,
+  ): Promise<VocabularyStatsSummary> {
+    if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
+    const workerData = { dbPath, knownWords: knownWords ? [...knownWords] : null };
+    let worker: VocabularySummaryWorkerHandle;
+    try {
+      const workerPath = (this.options.resolveWorkerPath ?? resolveVocabularySummaryWorkerPath)();
+      if (!workerPath) throw new Error('Emitted vocabulary summary worker module was not found');
+      const createWorker =
+        this.options.createWorker ??
+        (async (resolvedPath, data) => {
+          const { Worker } = await import('node:worker_threads');
+          return new Worker(resolvedPath, { workerData: data });
+        });
+      worker = await createWorker(workerPath, workerData);
+    } catch (error) {
+      if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
+      (this.options.warn ?? logger.warn)(
+        'Vocabulary summary worker unavailable; running summary on the current thread',
+        error,
+      );
+      return (this.options.executeFallback ?? executeVocabularySummaryTask)(
+        dbPath,
+        workerData.knownWords,
+      );
+    }
+
+    if (this.destroyed) {
+      await worker.terminate().catch(() => undefined);
+      throw new Error('Vocabulary summary worker is shut down');
+    }
+
+    return new Promise<VocabularyStatsSummary>((resolve, reject) => {
+      let settled = false;
+      this.activeWorkers.add(worker);
+      const settle = (result: VocabularyStatsSummary | Error) => {
+        if (settled) return;
+        settled = true;
+        this.activeWorkers.delete(worker);
+        void worker.terminate();
+        if (result instanceof Error) reject(result);
+        else resolve(result);
+      };
+
+      worker.once('message', (message) => {
+        if (message.summary) {
+          settle(message.summary);
+          return;
+        }
+        settle(
+          new Error(
+            `Vocabulary summary failed: ${String(message.error ?? 'unknown worker error')}`,
+          ),
+        );
+      });
+      worker.once('error', (error) => settle(error));
+      worker.once('exit', (code) => {
+        if (!settled) {
+          settle(
+            new Error(
+              code === 0
+                ? 'Vocabulary summary worker exited without a response'
+                : `Vocabulary summary worker exited with code ${code}`,
+            ),
+          );
+        }
+      });
+    });
+  }
+
+  destroy(): void {
+    if (this.destroyed) return;
+    this.destroyed = true;
+    for (const worker of this.activeWorkers) void worker.terminate();
+    this.activeWorkers.clear();
+  }
+}

src/core/services/immersion-tracker/vocabulary-summary-worker-thread.ts

@@ -0,0 +1,19 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { executeVocabularySummaryTask } from './vocabulary-summary-worker';
+
+interface VocabularySummaryWorkerData {
+  dbPath: string;
+  knownWords: string[] | null;
+}
+
+if (!parentPort) throw new Error('vocabulary summary worker missing parent port');
+
+const request = workerData as VocabularySummaryWorkerData;
+
+try {
+  parentPort.postMessage({
+    summary: executeVocabularySummaryTask(request.dbPath, request.knownWords),
+  });
+} catch (error) {
+  parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
+}

src/core/services/immersion-tracker/vocabulary-summary-worker.ts

@@ -0,0 +1,17 @@
+import { getVocabularySummary } from './query-lexical';
+import { Database } from './sqlite';
+import { applyPragmas } from './storage';
+import type { VocabularyStatsSummary } from './types';
+
+export function executeVocabularySummaryTask(
+  dbPath: string,
+  knownWords: string[] | null,
+): VocabularyStatsSummary {
+  const db = new Database(dbPath);
+  try {
+    applyPragmas(db);
+    return getVocabularySummary(db, knownWords ? new Set(knownWords) : null);
+  } finally {
+    db.close();
+  }
+}