diff --git a/changes/stats-vocabulary-summary-totals.md b/changes/stats-vocabulary-summary-totals.md index 660313e0..69908410 100644 --- a/changes/stats-vocabulary-summary-totals.md +++ b/changes/stats-vocabulary-summary-totals.md @@ -1,4 +1,4 @@ type: fixed area: stats -- Fixed Vocabulary summary cards counting only the first page of frequency-ranked words and kanji instead of all tracked vocabulary. +- Fixed Vocabulary summary cards counting only the first page of frequency-ranked words and kanji instead of all tracked vocabulary, without delaying the rest of the page. diff --git a/docs-site/immersion-tracking.md b/docs-site/immersion-tracking.md index 5e7ad11b..edf65408 100644 --- a/docs-site/immersion-tracking.md +++ b/docs-site/immersion-tracking.md @@ -82,7 +82,7 @@ Expandable session history with new-word activity, cumulative totals, and pause/ #### Vocabulary -The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The rest of the tab includes top repeated words (click a bar to open the word), new-word timeline, cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons. +The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The word and kanji tables load first while those complete totals calculate separately. The rest of the tab includes top repeated words (click a bar to open the word), new-word timeline, cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons. ![Stats Vocabulary](/screenshots/stats-vocabulary.png) diff --git a/src/core/services/immersion-tracker-service.ts b/src/core/services/immersion-tracker-service.ts index 686dc88d..b315e655 100644 --- a/src/core/services/immersion-tracker-service.ts +++ b/src/core/services/immersion-tracker-service.ts @@ -59,7 +59,6 @@ import { getSimilarWords, getStatsExcludedWords, getVocabularyStats, - getVocabularySummary, replaceStatsExcludedWords, searchSubtitleSentences, getWordAnimeAppearances, @@ -97,6 +96,10 @@ import { DeleteMaintenanceWorkerRuntime, type RunDeleteMaintenanceTask, } from './immersion-tracker/delete-maintenance-worker-runtime'; +import { + VocabularySummaryWorkerRuntime, + type RunVocabularySummaryTask, +} from './immersion-tracker/vocabulary-summary-worker-runtime'; import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler'; import { cleanupDuplicateSubtitleLines, @@ -186,6 +189,7 @@ import { type StatsExcludedWordRow, type StreakCalendarRow, type VocabularyCleanupSummary, + type VocabularyStatsSummary, type WatchTimePerAnimeRow, type WordAnimeAppearanceRow, type WordDetailRow, @@ -408,6 +412,10 @@ export class ImmersionTrackerService { private readonly dbPath: string; private readonly writeLock = { locked: false }; private readonly destroyDeleteMaintenanceRunner: () => void; + private readonly runVocabularySummaryTask: ( + knownWords: ReadonlySet | null, + ) => Promise; + private readonly destroyVocabularySummaryRunner: () => void; private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler; private flushTimer: ReturnType | null = null; private maintenanceTimer: ReturnType | null = null; @@ -435,6 +443,8 @@ export class ImmersionTrackerService { dependencies: { runDeleteMaintenanceTask?: RunDeleteMaintenanceTask; destroyDeleteMaintenanceRunner?: () => void; + runVocabularySummaryTask?: RunVocabularySummaryTask; + destroyVocabularySummaryRunner?: () => void; } = {}, ) { this.dbPath = options.dbPath; @@ -461,6 +471,17 @@ export class ImmersionTrackerService { if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0); }, }); + if (dependencies.runVocabularySummaryTask) { + this.runVocabularySummaryTask = (knownWords) => + dependencies.runVocabularySummaryTask!(this.dbPath, knownWords); + this.destroyVocabularySummaryRunner = + dependencies.destroyVocabularySummaryRunner ?? (() => {}); + } else { + const vocabularySummaryRuntime = new VocabularySummaryWorkerRuntime(); + this.runVocabularySummaryTask = (knownWords) => + vocabularySummaryRuntime.run(this.dbPath, knownWords); + this.destroyVocabularySummaryRunner = () => vocabularySummaryRuntime.destroy(); + } const parentDir = path.dirname(this.dbPath); if (!fs.existsSync(parentDir)) { fs.mkdirSync(parentDir, { recursive: true }); @@ -566,6 +587,7 @@ export class ImmersionTrackerService { this.isDestroyed = true; this.deleteMaintenanceScheduler.destroy(); this.destroyDeleteMaintenanceRunner(); + this.destroyVocabularySummaryRunner(); this.db.close(); } @@ -636,7 +658,7 @@ export class ImmersionTrackerService { } async getVocabularySummary(knownWords: ReadonlySet | null) { - return getVocabularySummary(this.db, knownWords); + return this.runVocabularySummaryTask(knownWords); } async getStatsExcludedWords(): Promise { diff --git a/src/core/services/immersion-tracker/vocabulary-summary-worker-runtime.test.ts b/src/core/services/immersion-tracker/vocabulary-summary-worker-runtime.test.ts new file mode 100644 index 00000000..8e72c2e6 --- /dev/null +++ b/src/core/services/immersion-tracker/vocabulary-summary-worker-runtime.test.ts @@ -0,0 +1,51 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { + resolveVocabularySummaryWorkerPath, + VocabularySummaryWorkerRuntime, +} from './vocabulary-summary-worker-runtime'; +import { Database } from './sqlite'; +import { applyPragmas, ensureSchema } from './storage'; + +test('vocabulary summary worker reads the database from a separate connection', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-vocabulary-summary-worker-')); + const dbPath = path.join(tempDir, 'immersion.sqlite'); + const runtime = new VocabularySummaryWorkerRuntime(); + const db = new Database(dbPath); + + try { + applyPragmas(db); + ensureSchema(db); + db.prepare( + ` + INSERT INTO imm_words ( + headword, word, reading, part_of_speech, pos1, pos2, pos3, + first_seen, last_seen, frequency + ) VALUES ('猫', '猫', 'ねこ', 'noun', '名詞', '一般', '', 1, 1, 1) + `, + ).run(); + db.close(); + + const summary = await runtime.run(dbPath, new Set(['猫'])); + + assert.equal(summary.uniqueWords, 1); + assert.equal(summary.knownWordCount, 1); + } finally { + runtime.destroy(); + try { + db.close(); + } catch { + // The worker needs the setup connection closed before it starts. + } + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('vocabulary summary worker module resolves in the current layout', () => { + const workerPath = resolveVocabularySummaryWorkerPath(); + assert.ok(workerPath, 'expected the vocabulary summary worker module to resolve'); + assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js')); +}); diff --git a/src/core/services/immersion-tracker/vocabulary-summary-worker-runtime.ts b/src/core/services/immersion-tracker/vocabulary-summary-worker-runtime.ts new file mode 100644 index 00000000..c7dc9f7d --- /dev/null +++ b/src/core/services/immersion-tracker/vocabulary-summary-worker-runtime.ts @@ -0,0 +1,128 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { createLogger } from '../../../logger'; +import type { VocabularyStatsSummary } from './types'; +import { executeVocabularySummaryTask } from './vocabulary-summary-worker'; + +interface VocabularySummaryWorkerResponse { + summary?: VocabularyStatsSummary; + error?: unknown; +} + +interface VocabularySummaryWorkerHandle { + once(event: 'message', listener: (message: VocabularySummaryWorkerResponse) => void): this; + once(event: 'error', listener: (error: Error) => void): this; + once(event: 'exit', listener: (code: number) => void): this; + terminate(): Promise; +} + +interface VocabularySummaryWorkerRuntimeOptions { + resolveWorkerPath?: () => string | null; + createWorker?: ( + workerPath: string, + workerData: { dbPath: string; knownWords: string[] | null }, + ) => Promise; + executeFallback?: typeof executeVocabularySummaryTask; + warn?: (message: string, ...meta: unknown[]) => void; +} + +export type RunVocabularySummaryTask = ( + dbPath: string, + knownWords: ReadonlySet | null, +) => Promise; + +export function resolveVocabularySummaryWorkerPath(): string | null { + const fileName = __filename.endsWith('.ts') + ? 'vocabulary-summary-worker-thread.ts' + : 'vocabulary-summary-worker-thread.js'; + const workerPath = path.join(__dirname, fileName); + return fs.existsSync(workerPath) ? workerPath : null; +} + +const logger = createLogger('main:immersion-tracker:vocabulary-summary-worker'); + +export class VocabularySummaryWorkerRuntime { + private readonly activeWorkers = new Set(); + private destroyed = false; + + constructor(private readonly options: VocabularySummaryWorkerRuntimeOptions = {}) {} + + async run( + dbPath: string, + knownWords: ReadonlySet | null, + ): Promise { + if (this.destroyed) throw new Error('Vocabulary summary worker is shut down'); + const workerData = { dbPath, knownWords: knownWords ? [...knownWords] : null }; + let worker: VocabularySummaryWorkerHandle; + try { + const workerPath = (this.options.resolveWorkerPath ?? resolveVocabularySummaryWorkerPath)(); + if (!workerPath) throw new Error('Emitted vocabulary summary worker module was not found'); + const createWorker = + this.options.createWorker ?? + (async (resolvedPath, data) => { + const { Worker } = await import('node:worker_threads'); + return new Worker(resolvedPath, { workerData: data }); + }); + worker = await createWorker(workerPath, workerData); + } catch (error) { + if (this.destroyed) throw new Error('Vocabulary summary worker is shut down'); + (this.options.warn ?? logger.warn)( + 'Vocabulary summary worker unavailable; running summary on the current thread', + error, + ); + return (this.options.executeFallback ?? executeVocabularySummaryTask)( + dbPath, + workerData.knownWords, + ); + } + + if (this.destroyed) { + await worker.terminate().catch(() => undefined); + throw new Error('Vocabulary summary worker is shut down'); + } + + return new Promise((resolve, reject) => { + let settled = false; + this.activeWorkers.add(worker); + const settle = (result: VocabularyStatsSummary | Error) => { + if (settled) return; + settled = true; + this.activeWorkers.delete(worker); + void worker.terminate(); + if (result instanceof Error) reject(result); + else resolve(result); + }; + + worker.once('message', (message) => { + if (message.summary) { + settle(message.summary); + return; + } + settle( + new Error( + `Vocabulary summary failed: ${String(message.error ?? 'unknown worker error')}`, + ), + ); + }); + worker.once('error', (error) => settle(error)); + worker.once('exit', (code) => { + if (!settled) { + settle( + new Error( + code === 0 + ? 'Vocabulary summary worker exited without a response' + : `Vocabulary summary worker exited with code ${code}`, + ), + ); + } + }); + }); + } + + destroy(): void { + if (this.destroyed) return; + this.destroyed = true; + for (const worker of this.activeWorkers) void worker.terminate(); + this.activeWorkers.clear(); + } +} diff --git a/src/core/services/immersion-tracker/vocabulary-summary-worker-thread.ts b/src/core/services/immersion-tracker/vocabulary-summary-worker-thread.ts new file mode 100644 index 00000000..abd10481 --- /dev/null +++ b/src/core/services/immersion-tracker/vocabulary-summary-worker-thread.ts @@ -0,0 +1,19 @@ +import { parentPort, workerData } from 'node:worker_threads'; +import { executeVocabularySummaryTask } from './vocabulary-summary-worker'; + +interface VocabularySummaryWorkerData { + dbPath: string; + knownWords: string[] | null; +} + +if (!parentPort) throw new Error('vocabulary summary worker missing parent port'); + +const request = workerData as VocabularySummaryWorkerData; + +try { + parentPort.postMessage({ + summary: executeVocabularySummaryTask(request.dbPath, request.knownWords), + }); +} catch (error) { + parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) }); +} diff --git a/src/core/services/immersion-tracker/vocabulary-summary-worker.ts b/src/core/services/immersion-tracker/vocabulary-summary-worker.ts new file mode 100644 index 00000000..55e34c9f --- /dev/null +++ b/src/core/services/immersion-tracker/vocabulary-summary-worker.ts @@ -0,0 +1,17 @@ +import { getVocabularySummary } from './query-lexical'; +import { Database } from './sqlite'; +import { applyPragmas } from './storage'; +import type { VocabularyStatsSummary } from './types'; + +export function executeVocabularySummaryTask( + dbPath: string, + knownWords: string[] | null, +): VocabularyStatsSummary { + const db = new Database(dbPath); + try { + applyPragmas(db); + return getVocabularySummary(db, knownWords ? new Set(knownWords) : null); + } finally { + db.close(); + } +} diff --git a/stats/src/components/vocabulary/VocabularyTab.tsx b/stats/src/components/vocabulary/VocabularyTab.tsx index d89595ee..bf01b5ee 100644 --- a/stats/src/components/vocabulary/VocabularyTab.tsx +++ b/stats/src/components/vocabulary/VocabularyTab.tsx @@ -98,24 +98,26 @@ export function VocabularyTab({
- {displayedSummary.knownWordCount !== null && ( + {displayedSummary.knownWordCount !== null ? ( 0 ? Math.round((displayedSummary.knownWordCount / displayedSummary.uniqueWords) * 100) : 0}%)`} color="text-ctp-green" /> - )} + ) : knownWords.size > 0 ? ( + + ) : null}
diff --git a/stats/src/hooks/useVocabulary.ts b/stats/src/hooks/useVocabulary.ts index eedbe821..9d9e3194 100644 --- a/stats/src/hooks/useVocabulary.ts +++ b/stats/src/hooks/useVocabulary.ts @@ -17,14 +17,10 @@ export function useVocabulary() { let cancelled = false; setLoading(true); setError(null); + setSummary(null); const client = getStatsClient(); - Promise.allSettled([ - client.getVocabulary(500), - client.getKanji(200), - client.getKnownWords(), - client.getVocabularySummary(), - ]) - .then(([wordsResult, kanjiResult, knownResult, summaryResult]) => { + Promise.allSettled([client.getVocabulary(500), client.getKanji(200), client.getKnownWords()]) + .then(([wordsResult, kanjiResult, knownResult]) => { if (cancelled) return; const errors: string[] = []; @@ -44,12 +40,6 @@ export function useVocabulary() { setKnownWords(new Set(knownResult.value)); } - if (summaryResult.status === 'fulfilled') { - setSummary(summaryResult.value); - } else { - errors.push(summaryResult.reason.message); - } - if (errors.length > 0) { setError(errors.join('; ')); } @@ -58,6 +48,14 @@ export function useVocabulary() { if (cancelled) return; setLoading(false); }); + void client + .getVocabularySummary() + .then((nextSummary) => { + if (!cancelled) setSummary(nextSummary); + }) + .catch((summaryError: unknown) => { + console.error('Failed to load vocabulary summary', summaryError); + }); return () => { cancelled = true; }; diff --git a/stats/src/lib/vocabulary-tab.test.ts b/stats/src/lib/vocabulary-tab.test.ts index f2da5a51..856581ac 100644 --- a/stats/src/lib/vocabulary-tab.test.ts +++ b/stats/src/lib/vocabulary-tab.test.ts @@ -6,6 +6,7 @@ import { fileURLToPath } from 'node:url'; const VOCABULARY_TAB_PATH = fileURLToPath( new URL('../components/vocabulary/VocabularyTab.tsx', import.meta.url), ); +const VOCABULARY_HOOK_PATH = fileURLToPath(new URL('../hooks/useVocabulary.ts', import.meta.url)); test('VocabularyTab declares all hooks before loading and error early returns', () => { const source = fs.readFileSync(VOCABULARY_TAB_PATH, 'utf8'); @@ -33,5 +34,15 @@ test('VocabularyTab uses database-wide summary totals for its stat cards', () => ); assert.match(source, /uniqueWords: summary\?\.uniqueWordsWithoutNames \?\? 0/); assert.match(source, /uniqueWords: summary\?\.uniqueWords \?\? 0/); - assert.match(source, /value=\{formatNumber\(summary\?\.uniqueKanji \?\? 0\)\}/); + assert.match(source, /value=\{summary \? formatNumber\(summary\.uniqueKanji\) : '…'\}/); +}); + +test('useVocabulary loads exact card totals without holding up the vocabulary tables', () => { + const source = fs.readFileSync(VOCABULARY_HOOK_PATH, 'utf8'); + + assert.match( + source, + /Promise\.allSettled\(\[\s*client\.getVocabulary\(500\),\s*client\.getKanji\(200\),\s*client\.getKnownWords\(\),?\s*\]\)/, + ); + assert.match(source, /void client\s*\.getVocabularySummary\(\)\s*\.then\(/); });