From e53f6d61ff38b0d1eedd2f1b16ef8644c047267e Mon Sep 17 00:00:00 2001 From: sudacode Date: Sat, 15 Aug 2026 22:29:16 -0700 Subject: [PATCH] perf(stats): roll up lexical chart history --- changes/stats-vocabulary-summary-totals.md | 3 +- docs-site/immersion-tracking.md | 4 +- .../services/__tests__/stats-server.test.ts | 7 + .../services/immersion-tracker-service.ts | 16 ++ .../lexical-rollup-worker-runtime.test.ts | 56 ++++++ .../lexical-rollup-worker-runtime.ts | 96 ++++++++++ .../lexical-rollup-worker-thread.ts | 11 ++ .../lexical-rollup-worker.test.ts | 35 ++++ .../lexical-rollup-worker.ts | 15 ++ .../immersion-tracker/lexical-rollups.test.ts | 97 ++++++++++ .../immersion-tracker/lexical-rollups.ts | 170 ++++++++++++++++++ .../immersion-tracker/query-lexical.ts | 67 +++++++ .../immersion-tracker/query-trends.ts | 21 +++ .../services/immersion-tracker/storage.ts | 10 +- src/core/services/immersion-tracker/types.ts | 2 +- .../services/stats-server/library-routes.ts | 4 + src/types/stats-http-contract.ts | 10 ++ .../components/vocabulary/VocabularyTab.tsx | 39 +++- stats/src/hooks/useVocabulary.ts | 28 ++- stats/src/lib/api-client.ts | 1 + stats/src/lib/vocabulary-tab.test.ts | 12 +- 21 files changed, 683 insertions(+), 21 deletions(-) create mode 100644 src/core/services/immersion-tracker/lexical-rollup-worker-runtime.test.ts create mode 100644 src/core/services/immersion-tracker/lexical-rollup-worker-runtime.ts create mode 100644 src/core/services/immersion-tracker/lexical-rollup-worker-thread.ts create mode 100644 src/core/services/immersion-tracker/lexical-rollup-worker.test.ts create mode 100644 src/core/services/immersion-tracker/lexical-rollup-worker.ts create mode 100644 src/core/services/immersion-tracker/lexical-rollups.test.ts create mode 100644 src/core/services/immersion-tracker/lexical-rollups.ts diff --git a/changes/stats-vocabulary-summary-totals.md b/changes/stats-vocabulary-summary-totals.md index 69908410..616290e7 100644 --- a/changes/stats-vocabulary-summary-totals.md +++ b/changes/stats-vocabulary-summary-totals.md @@ -1,4 +1,5 @@ type: fixed area: stats -- Fixed Vocabulary summary cards counting only the first page of frequency-ranked words and kanji instead of all tracked vocabulary, without delaying the rest of the page. +- Fixed Vocabulary totals and charts counting only the first browsing page instead of all tracked vocabulary, without delaying the rest of the page. +- New-word history now uses permanent daily lexical rollups, backfilled in the background and repaired when tracked material is removed or reprocessed. diff --git a/docs-site/immersion-tracking.md b/docs-site/immersion-tracking.md index edf65408..e1fb3aaa 100644 --- a/docs-site/immersion-tracking.md +++ b/docs-site/immersion-tracking.md @@ -82,7 +82,7 @@ Expandable session history with new-word activity, cumulative totals, and pause/ #### Vocabulary -The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The word and kanji tables load first while those complete totals calculate separately. The rest of the tab includes top repeated words (click a bar to open the word), new-word timeline, cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons. +The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The word and kanji tables load first while those complete totals calculate separately. Top Repeated Words and New Words by Day use complete tracking history rather than the table's browsing page; new-word history is maintained as a permanent daily lexical rollup, including retroactive corrections when tracked material is removed or reprocessed. On the first launch after upgrading, that history is built in the background and the chart refreshes when it is ready. The rest of the tab includes cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons. ![Stats Vocabulary](/screenshots/stats-vocabulary.png) @@ -180,6 +180,7 @@ In practice: - Anime and episode pages keep lifetime totals from summary tables while session drill-down still reads retained sessions directly. With the current defaults, both are kept forever. - Trends can read the full available history because daily/monthly rollups are also kept forever by default. - Vocabulary and kanji totals are cumulative and not bounded by the raw session retention knobs. +- New-word charts use their own permanent lexical daily rollups, which are not pruned by activity-rollup retention. ## Storage / Performance Model @@ -349,6 +350,7 @@ Rollup tables: - `imm_daily_rollups` - `imm_monthly_rollups` +- `imm_lexical_daily_rollups` - permanent first-discovery counts for vocabulary and kanji chart history - `imm_rollup_state` - incremental rollup progress bookkeeping Vocabulary tables: diff --git a/src/core/services/__tests__/stats-server.test.ts b/src/core/services/__tests__/stats-server.test.ts index aaa43b8a..0dff5209 100644 --- a/src/core/services/__tests__/stats-server.test.ts +++ b/src/core/services/__tests__/stats-server.test.ts @@ -293,6 +293,13 @@ function createMockTracker( knownWordCount: 250, knownWordCountWithoutNames: 249, }), + getVocabularyChartData: async () => ({ + ready: true, + topWords: [{ wordId: 1, headword: 'する', frequency: 50 }], + topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }], + newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }], + newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }], + }), getStatsExcludedWords: async () => [], replaceStatsExcludedWords: async () => {}, getKanjiStats: async () => KANJI_STATS, diff --git a/src/core/services/immersion-tracker-service.ts b/src/core/services/immersion-tracker-service.ts index b315e655..3e21f5ac 100644 --- a/src/core/services/immersion-tracker-service.ts +++ b/src/core/services/immersion-tracker-service.ts @@ -58,6 +58,7 @@ import { getSessionEvents, getSimilarWords, getStatsExcludedWords, + getVocabularyChartData, getVocabularyStats, replaceStatsExcludedWords, searchSubtitleSentences, @@ -100,6 +101,7 @@ import { VocabularySummaryWorkerRuntime, type RunVocabularySummaryTask, } from './immersion-tracker/vocabulary-summary-worker-runtime'; +import { LexicalRollupWorkerRuntime } from './immersion-tracker/lexical-rollup-worker-runtime'; import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler'; import { cleanupDuplicateSubtitleLines, @@ -416,6 +418,7 @@ export class ImmersionTrackerService { knownWords: ReadonlySet | null, ) => Promise; private readonly destroyVocabularySummaryRunner: () => void; + private readonly destroyLexicalRollupBackfillRunner: () => void; private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler; private flushTimer: ReturnType | null = null; private maintenanceTimer: ReturnType | null = null; @@ -482,6 +485,8 @@ export class ImmersionTrackerService { vocabularySummaryRuntime.run(this.dbPath, knownWords); this.destroyVocabularySummaryRunner = () => vocabularySummaryRuntime.destroy(); } + const lexicalRollupRuntime = new LexicalRollupWorkerRuntime(); + this.destroyLexicalRollupBackfillRunner = () => lexicalRollupRuntime.destroy(); const parentDir = path.dirname(this.dbPath); if (!fs.existsSync(parentDir)) { fs.mkdirSync(parentDir, { recursive: true }); @@ -541,6 +546,12 @@ export class ImmersionTrackerService { this.db = new Database(this.dbPath); applyPragmas(this.db); ensureSchema(this.db); + void lexicalRollupRuntime.run(this.dbPath).catch((error: unknown) => { + this.logger.warn( + 'Lexical daily rollup backfill failed; it will retry on next startup', + error, + ); + }); const reconciledSessions = reconcileStaleActiveSessions(this.db); if (reconciledSessions > 0) { this.logger.info( @@ -588,6 +599,7 @@ export class ImmersionTrackerService { this.deleteMaintenanceScheduler.destroy(); this.destroyDeleteMaintenanceRunner(); this.destroyVocabularySummaryRunner(); + this.destroyLexicalRollupBackfillRunner(); this.db.close(); } @@ -661,6 +673,10 @@ export class ImmersionTrackerService { return this.runVocabularySummaryTask(knownWords); } + async getVocabularyChartData() { + return getVocabularyChartData(this.db); + } + async getStatsExcludedWords(): Promise { return getStatsExcludedWords(this.db); } diff --git a/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.test.ts b/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.test.ts new file mode 100644 index 00000000..802df029 --- /dev/null +++ b/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.test.ts @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { + LexicalRollupWorkerRuntime, + resolveLexicalRollupWorkerPath, +} from './lexical-rollup-worker-runtime'; +import { areLexicalDailyRollupsReady } from './lexical-rollups'; +import { Database } from './sqlite'; +import { applyPragmas, ensureSchema } from './storage'; + +test('lexical rollup worker backfills without using the tracker connection', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-runtime-')); + const dbPath = path.join(directory, 'immersion.sqlite'); + const runtime = new LexicalRollupWorkerRuntime(); + const db = new Database(dbPath); + + try { + applyPragmas(db); + ensureSchema(db); + db.prepare( + `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency) + VALUES ('鳥', '鳥', 'とり', 1700000000, 1700000000, 1)`, + ).run(); + db.exec('DELETE FROM imm_lexical_daily_rollups'); + db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run( + 'lexical_daily_rollups_ready', + ); + db.close(); + + await runtime.run(dbPath); + + const checkDb = new Database(dbPath); + try { + assert.equal(areLexicalDailyRollupsReady(checkDb), true); + } finally { + checkDb.close(); + } + } finally { + runtime.destroy(); + try { + db.close(); + } catch { + // Closed before the worker starts. + } + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +test('lexical rollup worker module resolves in the current layout', () => { + const workerPath = resolveLexicalRollupWorkerPath(); + assert.ok(workerPath, 'expected the lexical rollup worker module to resolve'); + assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js')); +}); diff --git a/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.ts b/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.ts new file mode 100644 index 00000000..5c320873 --- /dev/null +++ b/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.ts @@ -0,0 +1,96 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { createLogger } from '../../../logger'; +import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker'; + +interface WorkerResponse { + ok?: boolean; + error?: unknown; +} + +interface WorkerHandle { + once(event: 'message', listener: (message: WorkerResponse) => void): this; + once(event: 'error', listener: (error: Error) => void): this; + once(event: 'exit', listener: (code: number) => void): this; + terminate(): Promise; +} + +const logger = createLogger('main:immersion-tracker:lexical-rollup-worker'); + +export function resolveLexicalRollupWorkerPath(): string | null { + const fileName = __filename.endsWith('.ts') + ? 'lexical-rollup-worker-thread.ts' + : 'lexical-rollup-worker-thread.js'; + const workerPath = path.join(__dirname, fileName); + return fs.existsSync(workerPath) ? workerPath : null; +} + +export class LexicalRollupWorkerRuntime { + private readonly activeWorkers = new Set(); + private destroyed = false; + + async run(dbPath: string): Promise { + if (this.destroyed) throw new Error('Lexical rollup worker is shut down'); + let worker: WorkerHandle; + try { + const workerPath = resolveLexicalRollupWorkerPath(); + if (!workerPath) throw new Error('Emitted lexical rollup worker module was not found'); + const { Worker } = await import('node:worker_threads'); + worker = new Worker(workerPath, { workerData: { dbPath } }); + } catch (error) { + if (this.destroyed) throw new Error('Lexical rollup worker is shut down'); + logger.warn( + 'Lexical rollup worker unavailable; running backfill on the current thread', + error, + ); + executeLexicalRollupBackfillTask(dbPath); + return; + } + + if (this.destroyed) { + await worker.terminate().catch(() => undefined); + throw new Error('Lexical rollup worker is shut down'); + } + + return new Promise((resolve, reject) => { + let settled = false; + this.activeWorkers.add(worker); + const settle = (error?: Error) => { + if (settled) return; + settled = true; + this.activeWorkers.delete(worker); + void worker.terminate(); + if (error) reject(error); + else resolve(); + }; + worker.once('message', (message) => { + if (message.ok) settle(); + else + settle( + new Error( + `Lexical rollup backfill failed: ${String(message.error ?? 'unknown error')}`, + ), + ); + }); + worker.once('error', (error) => settle(error)); + worker.once('exit', (code) => { + if (!settled) { + settle( + new Error( + code === 0 + ? 'Lexical rollup worker exited without a response' + : `Lexical rollup worker exited with code ${code}`, + ), + ); + } + }); + }); + } + + destroy(): void { + if (this.destroyed) return; + this.destroyed = true; + for (const worker of this.activeWorkers) void worker.terminate(); + this.activeWorkers.clear(); + } +} diff --git a/src/core/services/immersion-tracker/lexical-rollup-worker-thread.ts b/src/core/services/immersion-tracker/lexical-rollup-worker-thread.ts new file mode 100644 index 00000000..9e7cb104 --- /dev/null +++ b/src/core/services/immersion-tracker/lexical-rollup-worker-thread.ts @@ -0,0 +1,11 @@ +import { parentPort, workerData } from 'node:worker_threads'; +import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker'; + +if (!parentPort) throw new Error('lexical rollup worker missing parent port'); + +try { + executeLexicalRollupBackfillTask((workerData as { dbPath: string }).dbPath); + parentPort.postMessage({ ok: true }); +} catch (error) { + parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) }); +} diff --git a/src/core/services/immersion-tracker/lexical-rollup-worker.test.ts b/src/core/services/immersion-tracker/lexical-rollup-worker.test.ts new file mode 100644 index 00000000..b3bdcb15 --- /dev/null +++ b/src/core/services/immersion-tracker/lexical-rollup-worker.test.ts @@ -0,0 +1,35 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups'; +import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker'; +import { Database } from './sqlite'; +import { ensureSchema } from './storage'; + +test('lexical rollup backfill materializes pre-existing vocabulary off the caller DB connection', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-worker-')); + const dbPath = path.join(directory, 'immersion.sqlite'); + const db = new Database(dbPath); + + try { + ensureSchema(db); + db.prepare( + `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency) + VALUES (?, ?, ?, ?, ?, 1)`, + ).run('犬', '犬', 'いぬ', 1_700_000_000, 1_700_000_000); + db.exec('DELETE FROM imm_lexical_daily_rollups'); + db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run( + 'lexical_daily_rollups_ready', + ); + + executeLexicalRollupBackfillTask(dbPath); + + assert.equal(areLexicalDailyRollupsReady(db), true); + assert.equal(getLexicalDailyRollups(db)[0]?.wordCount, 1); + } finally { + db.close(); + fs.rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/src/core/services/immersion-tracker/lexical-rollup-worker.ts b/src/core/services/immersion-tracker/lexical-rollup-worker.ts new file mode 100644 index 00000000..582019ae --- /dev/null +++ b/src/core/services/immersion-tracker/lexical-rollup-worker.ts @@ -0,0 +1,15 @@ +import { areLexicalDailyRollupsReady, rebuildLexicalDailyRollups } from './lexical-rollups'; +import { Database } from './sqlite'; +import { applyPragmas } from './storage'; + +export function executeLexicalRollupBackfillTask(dbPath: string): void { + const db = new Database(dbPath); + try { + applyPragmas(db); + if (!areLexicalDailyRollupsReady(db)) { + rebuildLexicalDailyRollups(db); + } + } finally { + db.close(); + } +} diff --git a/src/core/services/immersion-tracker/lexical-rollups.test.ts b/src/core/services/immersion-tracker/lexical-rollups.test.ts new file mode 100644 index 00000000..6f8dcfc0 --- /dev/null +++ b/src/core/services/immersion-tracker/lexical-rollups.test.ts @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { getLexicalDailyRollups } from './lexical-rollups'; +import { getTrendsDashboard } from './query-trends'; +import { getVocabularyChartData } from './query-lexical'; +import { Database } from './sqlite'; +import { ensureSchema } from './storage'; + +function makeDbPath(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollups-')); + return path.join(dir, 'immersion.sqlite'); +} + +test('lexical daily rollups follow first-seen corrections and deletions', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const firstDay = 19_500; + const correctedDay = firstDay + 2; + const firstSeen = firstDay * 86_400 + 43_200; + const correctedSeen = correctedDay * 86_400 + 43_200; + + db.prepare( + `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency) + VALUES (?, ?, ?, ?, ?, 1)`, + ).run('猫', '猫', 'ねこ', firstSeen, firstSeen); + db.prepare( + `INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency) + VALUES (?, ?, ?, 1)`, + ).run('猫', firstSeen, firstSeen); + + assert.deepEqual(getLexicalDailyRollups(db), [ + { epochDay: firstDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 1 }, + ]); + + db.prepare(`UPDATE imm_words SET first_seen = ? WHERE headword = ?`).run(correctedSeen, '猫'); + db.prepare(`DELETE FROM imm_kanji WHERE kanji = ?`).run('猫'); + + assert.deepEqual(getLexicalDailyRollups(db), [ + { epochDay: correctedDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 }, + ]); + } finally { + db.close(); + fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }); + } +}); + +test('vocabulary charts use complete top-word and lexical rollup data', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const insertWord = db.prepare( + `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency) + VALUES (?, ?, '', 1700000000, 1700000000, ?)`, + ); + for (let index = 0; index < 501; index += 1) { + insertWord.run(`語${index}`, `語${index}`, index === 500 ? 10_000 : 1); + } + + const charts = getVocabularyChartData(db); + + assert.equal(charts.topWords[0]?.headword, '語500'); + assert.equal(charts.topWords[0]?.frequency, 10_000); + assert.equal(charts.newWordsTimeline[0]?.wordCount, 501); + } finally { + db.close(); + fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }); + } +}); + +test('trends read historical new-word buckets from lexical rollups', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + db.prepare( + `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency) + VALUES ('海', '海', 'うみ', 1700000000, 1700000000, 1)`, + ).run(); + db.prepare(`UPDATE imm_lexical_daily_rollups SET word_count = 9`).run(); + + const dashboard = getTrendsDashboard(db, 'all', 'day', false); + + assert.equal(dashboard.progress.newWords[0]?.value, 9); + } finally { + db.close(); + fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }); + } +}); diff --git a/src/core/services/immersion-tracker/lexical-rollups.ts b/src/core/services/immersion-tracker/lexical-rollups.ts new file mode 100644 index 00000000..7052df36 --- /dev/null +++ b/src/core/services/immersion-tracker/lexical-rollups.ts @@ -0,0 +1,170 @@ +import type { DatabaseSync } from './sqlite'; + +export interface LexicalDailyRollup { + epochDay: number; + wordCount: number; + wordCountWithoutNames: number; + kanjiCount: number; +} + +const LOCAL_EPOCH_DAY_SQL = ` + CAST(julianday(CAST(%VALUE% AS REAL), 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) +`; + +function localEpochDaySql(value: string): string { + return LOCAL_EPOCH_DAY_SQL.replace('%VALUE%', value); +} + +function createWordRollupTriggers(db: DatabaseSync): void { + const dayForNew = localEpochDaySql('NEW.first_seen'); + const dayForOld = localEpochDaySql('OLD.first_seen'); + + db.exec(` + CREATE TRIGGER IF NOT EXISTS imm_words_lexical_rollup_insert + AFTER INSERT ON imm_words + WHEN NEW.first_seen IS NOT NULL + BEGIN + INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) + VALUES (${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0) + ON CONFLICT(epoch_day) DO UPDATE SET + word_count = word_count + 1, + word_count_without_names = word_count_without_names + excluded.word_count_without_names; + END; + + CREATE TRIGGER IF NOT EXISTS imm_words_lexical_rollup_delete + AFTER DELETE ON imm_words + WHEN OLD.first_seen IS NOT NULL + BEGIN + INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) + VALUES (${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0) + ON CONFLICT(epoch_day) DO UPDATE SET + word_count = word_count - 1, + word_count_without_names = word_count_without_names + excluded.word_count_without_names; + DELETE FROM imm_lexical_daily_rollups + WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0; + END; + + CREATE TRIGGER IF NOT EXISTS imm_words_lexical_rollup_first_seen_update + AFTER UPDATE OF first_seen, pos2 ON imm_words + WHEN OLD.first_seen IS NOT NEW.first_seen OR OLD.pos2 IS NOT NEW.pos2 + BEGIN + INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) + SELECT ${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0 + WHERE OLD.first_seen IS NOT NULL + ON CONFLICT(epoch_day) DO UPDATE SET + word_count = word_count - 1, + word_count_without_names = word_count_without_names + excluded.word_count_without_names; + INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) + SELECT ${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0 + WHERE NEW.first_seen IS NOT NULL + ON CONFLICT(epoch_day) DO UPDATE SET + word_count = word_count + 1, + word_count_without_names = word_count_without_names + excluded.word_count_without_names; + DELETE FROM imm_lexical_daily_rollups + WHERE word_count = 0 AND kanji_count = 0; + END; + `); +} + +function createKanjiRollupTriggers(db: DatabaseSync): void { + const dayForNew = localEpochDaySql('NEW.first_seen'); + const dayForOld = localEpochDaySql('OLD.first_seen'); + db.exec(` + CREATE TRIGGER IF NOT EXISTS imm_kanji_lexical_rollup_insert + AFTER INSERT ON imm_kanji WHEN NEW.first_seen IS NOT NULL + BEGIN + INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) + VALUES (${dayForNew}, 0, 0, 1) + ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1; + END; + CREATE TRIGGER IF NOT EXISTS imm_kanji_lexical_rollup_delete + AFTER DELETE ON imm_kanji WHEN OLD.first_seen IS NOT NULL + BEGIN + INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) + VALUES (${dayForOld}, 0, 0, -1) + ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1; + DELETE FROM imm_lexical_daily_rollups + WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0; + END; + CREATE TRIGGER IF NOT EXISTS imm_kanji_lexical_rollup_first_seen_update + AFTER UPDATE OF first_seen ON imm_kanji WHEN OLD.first_seen IS NOT NEW.first_seen + BEGIN + INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) + SELECT ${dayForOld}, 0, 0, -1 WHERE OLD.first_seen IS NOT NULL + ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1; + INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) + SELECT ${dayForNew}, 0, 0, 1 WHERE NEW.first_seen IS NOT NULL + ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1; + DELETE FROM imm_lexical_daily_rollups WHERE word_count = 0 AND kanji_count = 0; + END; + `); +} + +export function ensureLexicalDailyRollupTables(db: DatabaseSync): void { + db.exec(` + CREATE TABLE IF NOT EXISTS imm_lexical_daily_rollups( + epoch_day INTEGER PRIMARY KEY, + word_count INTEGER NOT NULL DEFAULT 0, + word_count_without_names INTEGER NOT NULL DEFAULT 0, + kanji_count INTEGER NOT NULL DEFAULT 0 + ); + INSERT INTO imm_rollup_state(state_key, state_value) + VALUES ('lexical_daily_rollups_ready', '0') + ON CONFLICT(state_key) DO NOTHING; + `); + createWordRollupTriggers(db); + createKanjiRollupTriggers(db); +} + +export function areLexicalDailyRollupsReady(db: DatabaseSync): boolean { + const row = db + .prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`) + .get('lexical_daily_rollups_ready') as { value: string } | null; + return row?.value === '1'; +} + +export function markLexicalDailyRollupsReady(db: DatabaseSync): void { + db.prepare(`UPDATE imm_rollup_state SET state_value = '1' WHERE state_key = ?`).run( + 'lexical_daily_rollups_ready', + ); +} + +/** Rebuild from the first-seen source of truth; run off the UI/main DB thread. */ +export function rebuildLexicalDailyRollups(db: DatabaseSync): void { + db.exec('BEGIN IMMEDIATE'); + try { + db.exec('DELETE FROM imm_lexical_daily_rollups'); + db.exec(` + INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) + SELECT ${localEpochDaySql('first_seen')}, COUNT(*), + SUM(CASE WHEN pos2 = '固有名詞' THEN 0 ELSE 1 END), 0 + FROM imm_words + WHERE first_seen IS NOT NULL + GROUP BY ${localEpochDaySql('first_seen')}; + INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) + SELECT ${localEpochDaySql('first_seen')}, 0, 0, COUNT(*) + FROM imm_kanji + WHERE first_seen IS NOT NULL + GROUP BY ${localEpochDaySql('first_seen')} + ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + excluded.kanji_count; + `); + markLexicalDailyRollupsReady(db); + db.exec('COMMIT'); + } catch (error) { + db.exec('ROLLBACK'); + throw error; + } +} + +export function getLexicalDailyRollups(db: DatabaseSync): LexicalDailyRollup[] { + return db + .prepare( + ` + SELECT epoch_day AS epochDay, word_count AS wordCount, + word_count_without_names AS wordCountWithoutNames, kanji_count AS kanjiCount + FROM imm_lexical_daily_rollups + ORDER BY epoch_day ASC + `, + ) + .all() as LexicalDailyRollup[]; +} diff --git a/src/core/services/immersion-tracker/query-lexical.ts b/src/core/services/immersion-tracker/query-lexical.ts index 28e0d689..f31e96fd 100644 --- a/src/core/services/immersion-tracker/query-lexical.ts +++ b/src/core/services/immersion-tracker/query-lexical.ts @@ -20,6 +20,7 @@ import type { } from './types'; import { fromDbTimestamp, toDbTimestamp } from './query-shared'; import { nowMs } from './time'; +import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups'; const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4; const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100; @@ -27,6 +28,14 @@ const SENTENCE_SEARCH_DEFAULT_LIMIT = 50; const SENTENCE_SEARCH_MAX_LIMIT = 100; const KANJI_PATTERN = /\p{Script=Han}/gu; +export interface VocabularyChartData { + ready: boolean; + topWords: Array<{ wordId: number; headword: string; frequency: number }>; + topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>; + newWordsTimeline: Array<{ epochDay: number; wordCount: number }>; + newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>; +} + function resolveSentenceSearchLimit(limit: number): number { if (!Number.isFinite(limit)) return SENTENCE_SEARCH_DEFAULT_LIMIT; const normalized = Math.floor(limit); @@ -154,6 +163,64 @@ export function getVocabularyStats( return visibleRows.slice(0, limit); } +/** + * Chart data is intentionally independent of the paginated vocabulary tables. + * Top words use the frequency index; new-word history reads permanent daily + * lexical rollups rather than loading every vocabulary row into the dashboard. + */ +export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData { + const ready = areLexicalDailyRollupsReady(db); + const excludedAliases = new Set( + getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)), + ); + const isExcluded = (word: Pick): boolean => + excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias)); + const topWords = getVocabularyStats(db, 48).filter((word) => !isExcluded(word)); + const rollups = ready ? getLexicalDailyRollups(db) : []; + const timeline = new Map(rollups.map((row) => [row.epochDay, { ...row }])); + if (excludedAliases.size > 0 && ready) { + const aliases = [...excludedAliases]; + const placeholders = aliases.map(() => '?').join(', '); + const excludedRows = db + .prepare( + ` + SELECT headword, word, reading, pos2, + CAST(julianday(CAST(first_seen AS REAL), 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) AS epochDay + FROM imm_words + WHERE headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}) + `, + ) + .all(...aliases, ...aliases, ...aliases) as Array< + Pick & { epochDay: number } + >; + for (const word of excludedRows) { + if (!isExcluded(word)) continue; + const rollup = timeline.get(word.epochDay); + if (!rollup) continue; + rollup.wordCount -= 1; + if (word.pos2 !== '固有名詞') rollup.wordCountWithoutNames -= 1; + } + } + return { + ready, + topWords: topWords.slice(0, 12).map((word) => ({ + wordId: word.wordId, + headword: word.headword, + frequency: word.frequency, + })), + topWordsWithoutNames: topWords + .filter((word) => word.pos2 !== '固有名詞') + .slice(0, 12) + .map((word) => ({ wordId: word.wordId, headword: word.headword, frequency: word.frequency })), + newWordsTimeline: [...timeline.values()] + .filter((row) => row.wordCount > 0) + .map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCount })), + newWordsTimelineWithoutNames: [...timeline.values()] + .filter((row) => row.wordCountWithoutNames > 0) + .map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCountWithoutNames })), + }; +} + function excludedVocabularyAliases( word: Pick, ): string[] { diff --git a/src/core/services/immersion-tracker/query-trends.ts b/src/core/services/immersion-tracker/query-trends.ts index b0cf0ef5..0ff7fbdc 100644 --- a/src/core/services/immersion-tracker/query-trends.ts +++ b/src/core/services/immersion-tracker/query-trends.ts @@ -13,6 +13,7 @@ import { toDbTimestamp, } from './query-shared'; import { getDailyRollups, getMonthlyRollups } from './query-sessions'; +import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups'; type TrendRange = '7d' | '30d' | '90d' | '365d' | 'all'; type TrendGroupBy = 'day' | 'month'; @@ -660,6 +661,14 @@ function buildNewWordsPerDay( cutoffMs: string | null, axis: number[] | null, ): TrendChartPoint[] { + if (areLexicalDailyRollupsReady(db)) { + const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs); + const rows = getLexicalDailyRollups(db).filter( + (row) => cutoffDay === null || row.epochDay >= cutoffDay, + ); + return fillAxisPoints(axis, new Map(rows.map((row) => [row.epochDay, row.wordCount]))); + } + const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?'; const prepared = db.prepare(` SELECT @@ -691,6 +700,18 @@ function buildNewWordsPerMonth( cutoffMs: string | null, axis: number[] | null, ): TrendChartPoint[] { + if (areLexicalDailyRollupsReady(db)) { + const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs); + const byMonth = new Map(); + for (const row of getLexicalDailyRollups(db)) { + if (cutoffDay !== null && row.epochDay < cutoffDay) continue; + const { year, month } = dayPartsFromEpochDay(row.epochDay); + const monthKey = year * 100 + month; + byMonth.set(monthKey, (byMonth.get(monthKey) ?? 0) + row.wordCount); + } + return fillAxisPoints(axis, byMonth); + } + const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?'; const prepared = db.prepare(` SELECT diff --git a/src/core/services/immersion-tracker/storage.ts b/src/core/services/immersion-tracker/storage.ts index 77a33986..1d11baf7 100644 --- a/src/core/services/immersion-tracker/storage.ts +++ b/src/core/services/immersion-tracker/storage.ts @@ -4,6 +4,7 @@ import { parseMediaInfo } from '../../../jimaku/utils'; import { normalizeTitleIdentity } from '../../utils/title-normalization'; import type { DatabaseSync } from './sqlite'; import { nowMs } from './time'; +import { ensureLexicalDailyRollupTables, markLexicalDailyRollupsReady } from './lexical-rollups'; import { SCHEMA_VERSION } from './types'; import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types'; import { toDbMs, toDbTimestamp } from './query-shared'; @@ -890,11 +891,11 @@ export function ensureSchema(db: DatabaseSync): void { VALUES ('last_rollup_sample_ms', 0) ON CONFLICT(state_key) DO NOTHING `); - const currentVersion = db .prepare('SELECT schema_version FROM imm_schema_version ORDER BY schema_version DESC LIMIT 1') .get() as { schema_version: number } | null; if (currentVersion?.schema_version === SCHEMA_VERSION) { + ensureLexicalDailyRollupTables(db); ensureLifetimeSummaryTables(db); ensureStatsExcludedWordsTable(db); ensureAnimeMergeTables(db); @@ -1453,6 +1454,7 @@ export function ensureSchema(db: DatabaseSync): void { migrateSessionEventTimestampsToText(db); + ensureLexicalDailyRollupTables(db); ensureLifetimeSummaryTables(db); ensureStatsExcludedWordsTable(db); @@ -1585,6 +1587,12 @@ export function ensureSchema(db: DatabaseSync): void { VALUES (${SCHEMA_VERSION}, ${toDbTimestamp(nowMs())}) ON CONFLICT DO NOTHING `); + + // A new database has no history to materialize. Upgrades are populated by the + // background worker so startup never scans the existing vocabulary table. + if (!currentVersion) { + markLexicalDailyRollupsReady(db); + } } export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPreparedStatements { diff --git a/src/core/services/immersion-tracker/types.ts b/src/core/services/immersion-tracker/types.ts index 50a13f66..e28caa37 100644 --- a/src/core/services/immersion-tracker/types.ts +++ b/src/core/services/immersion-tracker/types.ts @@ -1,4 +1,4 @@ -export const SCHEMA_VERSION = 21; +export const SCHEMA_VERSION = 22; export const DEFAULT_QUEUE_CAP = 1_000; export const DEFAULT_BATCH_SIZE = 25; export const DEFAULT_FLUSH_INTERVAL_MS = 500; diff --git a/src/core/services/stats-server/library-routes.ts b/src/core/services/stats-server/library-routes.ts index 6255d155..f705f1ba 100644 --- a/src/core/services/stats-server/library-routes.ts +++ b/src/core/services/stats-server/library-routes.ts @@ -39,6 +39,10 @@ export function registerStatsLibraryRoutes( return c.json(statsJson('vocabularySummary', summary)); }); + app.get('/api/stats/vocabulary/charts', async (c) => { + return c.json(statsJson('vocabularyCharts', await tracker.getVocabularyChartData())); + }); + app.get('/api/stats/excluded-words', async (c) => { return c.json(statsJson('excludedWords', await tracker.getStatsExcludedWords())); }); diff --git a/src/types/stats-http-contract.ts b/src/types/stats-http-contract.ts index cea7556f..a920761a 100644 --- a/src/types/stats-http-contract.ts +++ b/src/types/stats-http-contract.ts @@ -60,6 +60,14 @@ export interface StatsVocabularySummary { knownWordCountWithoutNames: number | null; } +export interface StatsVocabularyCharts { + ready: boolean; + topWords: Array<{ wordId: number; headword: string; frequency: number }>; + topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>; + newWordsTimeline: Array<{ epochDay: number; wordCount: number }>; + newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>; +} + export interface StatsAnilistSearchResult { id: number; episodes: number | null; @@ -175,6 +183,7 @@ export interface StatsJsonResponseMap { sessionKnownWordsTimeline: StatsSessionKnownWordsTimelinePoint[]; vocabulary: VocabularyEntry[]; vocabularySummary: StatsVocabularySummary; + vocabularyCharts: StatsVocabularyCharts; excludedWords: StatsExcludedWord[]; setExcludedWords: StatsOkResponse; duplicateLineCleanup: StatsDuplicateLineCleanupResult; @@ -234,6 +243,7 @@ export interface StatsHttpClient { getSessionKnownWordsTimeline: (id: number) => Promise; getVocabulary: (limit?: number) => Promise; getVocabularySummary: () => Promise; + getVocabularyCharts: () => Promise; getExcludedWords: () => Promise; setExcludedWords: (words: StatsExcludedWord[]) => Promise; cleanupDuplicateLines: ( diff --git a/stats/src/components/vocabulary/VocabularyTab.tsx b/stats/src/components/vocabulary/VocabularyTab.tsx index bf01b5ee..2798d784 100644 --- a/stats/src/components/vocabulary/VocabularyTab.tsx +++ b/stats/src/components/vocabulary/VocabularyTab.tsx @@ -6,11 +6,10 @@ import { KanjiBreakdown } from './KanjiBreakdown'; import { KanjiDetailPanel } from './KanjiDetailPanel'; import { ExclusionManager } from './ExclusionManager'; import { DuplicateLineCleanup } from './DuplicateLineCleanup'; -import { formatNumber } from '../../lib/formatters'; +import { epochDayToDate, formatNumber } from '../../lib/formatters'; import { TrendChart } from '../trends/TrendChart'; import { FrequencyRankTable } from './FrequencyRankTable'; import { CrossAnimeWordsTable } from './CrossAnimeWordsTable'; -import { buildVocabularySummary } from '../../lib/dashboard-data'; import type { ExcludedWord } from '../../hooks/useExcludedWords'; import type { KanjiEntry, VocabularyEntry } from '../../types/stats'; @@ -35,7 +34,7 @@ export function VocabularyTab({ onRemoveExclusion, onClearExclusions, }: VocabularyTabProps) { - const { words, kanji, knownWords, summary, loading, error, reload } = useVocabulary(); + const { words, kanji, knownWords, summary, charts, loading, error, reload } = useVocabulary(); const [selectedKanjiId, setSelectedKanjiId] = useState(null); const [hideNames, setHideNames] = useState(false); const [showExclusionManager, setShowExclusionManager] = useState(false); @@ -48,9 +47,23 @@ export function VocabularyTab({ if (excluded.length > 0) result = result.filter((w) => !isExcluded(w)); return result; }, [words, hideNames, excluded, isExcluded]); - const chartSummary = useMemo( - () => buildVocabularySummary(filteredWords, kanji), - [filteredWords, kanji], + const chartData = useMemo( + () => ({ + topWords: + ((hideNames ? charts?.topWordsWithoutNames : charts?.topWords) ?? []).map((word) => ({ + label: word.headword, + value: word.frequency, + })) ?? [], + newWordsTimeline: + ((hideNames ? charts?.newWordsTimelineWithoutNames : charts?.newWordsTimeline) ?? []).map((point) => ({ + label: epochDayToDate(point.epochDay).toLocaleDateString(undefined, { + month: 'short', + day: 'numeric', + }), + value: point.wordCount, + })) ?? [], + }), + [charts, hideNames], ); if (loading) { @@ -73,7 +86,9 @@ export function VocabularyTab({ }; const handleBarClick = (headword: string): void => { - const match = filteredWords.find((w) => w.headword === headword); + const match = (hideNames ? charts?.topWordsWithoutNames : charts?.topWords)?.find( + (word) => word.headword === headword, + ); if (match) onOpenWordDetail?.(match.wordId); }; @@ -159,19 +174,25 @@ export function VocabularyTab({
+ {charts && !charts.ready && ( +

+ Building vocabulary history in the background… +

+ )} + ([]); const [kanji, setKanji] = useState([]); const [knownWords, setKnownWords] = useState>(new Set()); const [summary, setSummary] = useState(null); + const [charts, setCharts] = useState(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); // Bumped by `reload` after maintenance rewrites the vocabulary tables. @@ -18,6 +24,7 @@ export function useVocabulary() { setLoading(true); setError(null); setSummary(null); + setCharts(null); const client = getStatsClient(); Promise.allSettled([client.getVocabulary(500), client.getKanji(200), client.getKnownWords()]) .then(([wordsResult, kanjiResult, knownResult]) => { @@ -56,10 +63,27 @@ export function useVocabulary() { .catch((summaryError: unknown) => { console.error('Failed to load vocabulary summary', summaryError); }); + let chartRetryTimer: ReturnType | null = null; + const loadCharts = (): void => { + void client + .getVocabularyCharts() + .then((nextCharts) => { + if (cancelled) return; + setCharts(nextCharts); + if (!nextCharts.ready) { + chartRetryTimer = setTimeout(loadCharts, 1_000); + } + }) + .catch((chartError: unknown) => { + console.error('Failed to load vocabulary charts', chartError); + }); + }; + loadCharts(); return () => { cancelled = true; + if (chartRetryTimer) clearTimeout(chartRetryTimer); }; }, [reloadToken]); - return { words, kanji, knownWords, summary, loading, error, reload }; + return { words, kanji, knownWords, summary, charts, loading, error, reload }; } diff --git a/stats/src/lib/api-client.ts b/stats/src/lib/api-client.ts index 35a808f7..addf6ef6 100644 --- a/stats/src/lib/api-client.ts +++ b/stats/src/lib/api-client.ts @@ -101,6 +101,7 @@ export const apiClient = { fetchJson('sessionKnownWordsTimeline', `/api/stats/sessions/${id}/known-words-timeline`), getVocabulary: (limit = 100) => fetchJson('vocabulary', `/api/stats/vocabulary?limit=${limit}`), getVocabularySummary: () => fetchJson('vocabularySummary', '/api/stats/vocabulary/summary'), + getVocabularyCharts: () => fetchJson('vocabularyCharts', '/api/stats/vocabulary/charts'), getExcludedWords: () => fetchJson('excludedWords', '/api/stats/excluded-words'), setExcludedWords: async (words: StatsExcludedWord[]): Promise => { await fetchResponse('/api/stats/excluded-words', { diff --git a/stats/src/lib/vocabulary-tab.test.ts b/stats/src/lib/vocabulary-tab.test.ts index 856581ac..6a7a9c3f 100644 --- a/stats/src/lib/vocabulary-tab.test.ts +++ b/stats/src/lib/vocabulary-tab.test.ts @@ -21,17 +21,16 @@ test('VocabularyTab declares all hooks before loading and error early returns', assert.deepEqual(hooksAfterLoadingGuard ?? [], []); }); -test('VocabularyTab uses database-wide summary totals for its stat cards', () => { +test('VocabularyTab uses uncapped server-side data for its charts and card totals', () => { const source = fs.readFileSync(VOCABULARY_TAB_PATH, 'utf8'); assert.match( source, - /const chartSummary = useMemo\([\s\S]*buildVocabularySummary\(filteredWords, kanji\)[\s\S]*\[filteredWords, kanji\][\s\S]*\);/, - ); - assert.match( - source, - /const \{ words, kanji, knownWords, summary, loading, error, reload \} = useVocabulary\(\);/, + /const \{ words, kanji, knownWords, summary, charts, loading, error, reload \} = useVocabulary\(\);/, ); + assert.match(source, /charts\?\.topWordsWithoutNames/); + assert.match(source, /charts\?\.newWordsTimelineWithoutNames/); + assert.doesNotMatch(source, /buildVocabularySummary\(/); assert.match(source, /uniqueWords: summary\?\.uniqueWordsWithoutNames \?\? 0/); assert.match(source, /uniqueWords: summary\?\.uniqueWords \?\? 0/); assert.match(source, /value=\{summary \? formatNumber\(summary\.uniqueKanji\) : '…'\}/); @@ -45,4 +44,5 @@ test('useVocabulary loads exact card totals without holding up the vocabulary ta /Promise\.allSettled\(\[\s*client\.getVocabulary\(500\),\s*client\.getKanji\(200\),\s*client\.getKnownWords\(\),?\s*\]\)/, ); assert.match(source, /void client\s*\.getVocabularySummary\(\)\s*\.then\(/); + assert.match(source, /client\s*\.getVocabularyCharts\(\)/); });