fix(stats): report complete vocabulary summary totals

changes/stats-vocabulary-summary-totals.md

@@ -0,0 +1,4 @@
+type: fixed
+area: stats
+
+- Fixed Vocabulary summary cards counting only the first page of frequency-ranked words and kanji instead of all tracked vocabulary.

docs-site/immersion-tracking.md

@@ -82,7 +82,7 @@ Expandable session history with new-word activity, cumulative totals, and pause/
 
 #### Vocabulary
 
-Top repeated words (click a bar to open the word), new-word timeline, cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons.
+The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The rest of the tab includes top repeated words (click a bar to open the word), new-word timeline, cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons.
 
 ![Stats Vocabulary](/screenshots/stats-vocabulary.png)
 

src/core/services/__tests__/stats-server.test.ts

@@ -284,6 +284,15 @@ function createMockTracker(
     getSessionTimeline: async () => [],
     getSessionEvents: async () => [],
     getVocabularyStats: async () => VOCABULARY_STATS,
+    getVocabularySummary: async () => ({
+      uniqueWords: 501,
+      uniqueWordsWithoutNames: 500,
+      uniqueKanji: 201,
+      newThisWeek: 7,
+      newThisWeekWithoutNames: 6,
+      knownWordCount: 250,
+      knownWordCountWithoutNames: 249,
+    }),
     getStatsExcludedWords: async () => [],
     replaceStatsExcludedWords: async () => {},
     getKanjiStats: async () => KANJI_STATS,
@@ -711,6 +720,23 @@ describe('stats server API routes', () => {
     assert.equal(body[0].headword, 'する');
   });
 
+  it('GET /api/stats/vocabulary/summary returns database-wide card totals', async () => {
+    const app = createStatsApp(createMockTracker());
+
+    const res = await app.request('/api/stats/vocabulary/summary');
+
+    assert.equal(res.status, 200);
+    assert.deepEqual(await res.json(), {
+      uniqueWords: 501,
+      uniqueWordsWithoutNames: 500,
+      uniqueKanji: 201,
+      newThisWeek: 7,
+      newThisWeekWithoutNames: 6,
+      knownWordCount: 250,
+      knownWordCountWithoutNames: 249,
+    });
+  });
+
   it('GET /api/stats/kanji returns kanji frequency data', async () => {
     const app = createStatsApp(createMockTracker());
     const res = await app.request('/api/stats/kanji');

src/core/services/immersion-tracker-service.ts

@@ -59,6 +59,7 @@ import {
   getSimilarWords,
   getStatsExcludedWords,
   getVocabularyStats,
+  getVocabularySummary,
   replaceStatsExcludedWords,
   searchSubtitleSentences,
   getWordAnimeAppearances,
@@ -634,6 +635,10 @@ export class ImmersionTrackerService {
     return getVocabularyStats(this.db, limit, excludePos);
   }
 
+  async getVocabularySummary(knownWords: ReadonlySet<string> | null) {
+    return getVocabularySummary(this.db, knownWords);
+  }
+
   async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {
     return getStatsExcludedWords(this.db);
   }

src/core/services/immersion-tracker/__tests__/query.test.ts

@@ -31,6 +31,7 @@ import {
   getKanjiOccurrences,
   getSessionSummaries,
   getVocabularyStats,
+  getVocabularySummary,
   getKanjiStats,
   getSessionEvents,
   getSessionTimeline,
@@ -1875,6 +1876,88 @@ test('getVocabularyStats returns rows ordered by frequency descending', () => {
   }
 });
 
+test('getVocabularySummary counts every tracked vocabulary row instead of a display page', () => {
+  const dbPath = makeDbPath();
+  const db = openTestDb(dbPath);
+
+  try {
+    ensureSchema(db);
+    const nowSec = Math.floor(Date.now() / 1000);
+    const insertWord = db.prepare(`
+      INSERT INTO imm_words (
+        headword, word, reading, part_of_speech, pos1, pos2, pos3,
+        first_seen, last_seen, frequency
+      ) VALUES (?, ?, '', 'noun', '名詞', '一般', '', ?, ?, 1)
+    `);
+    const insertKanji = db.prepare(`
+      INSERT INTO imm_kanji (kanji, first_seen, last_seen, frequency)
+      VALUES (?, ?, ?, 1)
+    `);
+
+    for (let index = 0; index < 501; index += 1) {
+      insertWord.run(`単語${index}`, `単語${index}`, nowSec - 8 * 86_400, nowSec - 8 * 86_400);
+    }
+    for (let index = 0; index < 201; index += 1) {
+      insertKanji.run(
+        String.fromCodePoint(0x4e00 + index),
+        nowSec - 8 * 86_400,
+        nowSec - 8 * 86_400,
+      );
+    }
+    insertWord.run('今週', '今週', nowSec - 86_400, nowSec - 86_400);
+
+    assert.deepEqual(getVocabularySummary(db, new Set(['単語0', '今週']), nowSec * 1000), {
+      uniqueWords: 502,
+      uniqueWordsWithoutNames: 502,
+      uniqueKanji: 201,
+      newThisWeek: 1,
+      newThisWeekWithoutNames: 1,
+      knownWordCount: 2,
+      knownWordCountWithoutNames: 2,
+    });
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('getVocabularySummary applies vocabulary exclusions and Hide Names totals', () => {
+  const dbPath = makeDbPath();
+  const db = openTestDb(dbPath);
+
+  try {
+    ensureSchema(db);
+    const insertWord = db.prepare(`
+      INSERT INTO imm_words (
+        headword, word, reading, part_of_speech, pos1, pos2, pos3,
+        first_seen, last_seen, frequency
+      ) VALUES (?, ?, '', 'noun', '名詞', ?, '', 1, 1, 1)
+    `);
+    insertWord.run('猫', '猫', '一般');
+    insertWord.run('太郎', '太郎', '固有名詞');
+    insertWord.run('東京', '東京都', '一般');
+    db.prepare(
+      `
+      INSERT INTO imm_stats_excluded_words (headword, word, reading)
+      VALUES ('東京', '東京', '')
+    `,
+    ).run();
+
+    assert.deepEqual(getVocabularySummary(db, new Set(['猫', '太郎', '東京']), 9 * 86_400_000), {
+      uniqueWords: 2,
+      uniqueWordsWithoutNames: 1,
+      uniqueKanji: 0,
+      newThisWeek: 0,
+      newThisWeekWithoutNames: 0,
+      knownWordCount: 2,
+      knownWordCountWithoutNames: 1,
+    });
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
 test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () => {
   const dbPath = makeDbPath();
   const db = openTestDb(dbPath);

src/core/services/immersion-tracker/query-lexical.ts

@@ -13,6 +13,7 @@ import type {
   SimilarWordRow,
   StatsExcludedWordRow,
   VocabularyStatsRow,
+  VocabularyStatsSummary,
   WordAnimeAppearanceRow,
   WordDetailRow,
   WordOccurrenceRow,
@@ -153,6 +154,75 @@ export function getVocabularyStats(
   return visibleRows.slice(0, limit);
 }
 
+function excludedVocabularyAliases(
+  word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
+): string[] {
+  const aliases = [word.headword.trim(), word.word.trim()].filter(Boolean);
+  if (aliases.length === 0) aliases.push(word.reading.trim());
+  return [...new Set(aliases)];
+}
+
+function timestampSeconds(timestamp: number): number {
+  return timestamp < 10_000_000_000 ? timestamp : Math.floor(timestamp / 1000);
+}
+
+export function getVocabularySummary(
+  db: DatabaseSync,
+  knownWords: ReadonlySet<string> | null,
+  nowMs: number = Date.now(),
+): VocabularyStatsSummary {
+  const words = db
+    .prepare(
+      `
+      SELECT id AS wordId, headword, word, reading,
+        part_of_speech AS partOfSpeech, pos1, pos2, pos3,
+        frequency, frequency_rank AS frequencyRank,
+        first_seen AS firstSeen, last_seen AS lastSeen,
+        0 AS animeCount
+      FROM imm_words
+    `,
+    )
+    .all() as VocabularyStatsRow[];
+  const excludedAliases = new Set(
+    getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
+  );
+  const weekAgoSec = nowMs / 1000 - 7 * 86_400;
+  const summary: VocabularyStatsSummary = {
+    uniqueWords: 0,
+    uniqueWordsWithoutNames: 0,
+    uniqueKanji: (db.prepare('SELECT COUNT(*) AS count FROM imm_kanji').get() as { count: number })
+      .count,
+    newThisWeek: 0,
+    newThisWeekWithoutNames: 0,
+    knownWordCount: knownWords ? 0 : null,
+    knownWordCountWithoutNames: knownWords ? 0 : null,
+  };
+
+  for (const word of words) {
+    if (
+      !isVocabularyStatsRowVisible(word) ||
+      excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias))
+    ) {
+      continue;
+    }
+    const isName = word.pos2 === '固有名詞';
+    const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec;
+    const isKnown = knownWords?.has(word.headword) ?? false;
+    summary.uniqueWords += 1;
+    if (!isName) summary.uniqueWordsWithoutNames += 1;
+    if (isNewThisWeek) {
+      summary.newThisWeek += 1;
+      if (!isName) summary.newThisWeekWithoutNames += 1;
+    }
+    if (isKnown) {
+      summary.knownWordCount! += 1;
+      if (!isName) summary.knownWordCountWithoutNames! += 1;
+    }
+  }
+
+  return summary;
+}
+
 export function getStatsExcludedWords(db: DatabaseSync): StatsExcludedWordRow[] {
   return db
     .prepare(

src/core/services/immersion-tracker/types.ts

@@ -306,6 +306,16 @@ export interface VocabularyStatsRow {
   lastSeen: number;
 }
 
+export interface VocabularyStatsSummary {
+  uniqueWords: number;
+  uniqueWordsWithoutNames: number;
+  uniqueKanji: number;
+  newThisWeek: number;
+  newThisWeekWithoutNames: number;
+  knownWordCount: number | null;
+  knownWordCountWithoutNames: number | null;
+}
+
 export interface StatsExcludedWordRow {
   headword: string;
   word: string;

src/core/services/stats-server/library-routes.ts

@@ -10,6 +10,7 @@ import {
   parseExcludedWordsBody,
   parseIntQuery,
   parsePositiveIdList,
+  loadKnownWordsSet,
 } from './route-support.js';
 
 export function registerStatsLibraryRoutes(
@@ -31,6 +32,13 @@ export function registerStatsLibraryRoutes(
     return c.json(statsJson('vocabulary', vocab));
   });
 
+  app.get('/api/stats/vocabulary/summary', async (c) => {
+    const summary = await tracker.getVocabularySummary(
+      loadKnownWordsSet(options?.knownWordCachePath),
+    );
+    return c.json(statsJson('vocabularySummary', summary));
+  });
+
   app.get('/api/stats/excluded-words', async (c) => {
     return c.json(statsJson('excludedWords', await tracker.getStatsExcludedWords()));
   });

src/types/stats-http-contract.ts

@@ -50,6 +50,16 @@ export interface StatsKnownWordsSummary {
   knownWordCount: number;
 }
 
+export interface StatsVocabularySummary {
+  uniqueWords: number;
+  uniqueWordsWithoutNames: number;
+  uniqueKanji: number;
+  newThisWeek: number;
+  newThisWeekWithoutNames: number;
+  knownWordCount: number | null;
+  knownWordCountWithoutNames: number | null;
+}
+
 export interface StatsAnilistSearchResult {
   id: number;
   episodes: number | null;
@@ -164,6 +174,7 @@ export interface StatsJsonResponseMap {
   sessionEvents: SessionEvent[];
   sessionKnownWordsTimeline: StatsSessionKnownWordsTimelinePoint[];
   vocabulary: VocabularyEntry[];
+  vocabularySummary: StatsVocabularySummary;
   excludedWords: StatsExcludedWord[];
   setExcludedWords: StatsOkResponse;
   duplicateLineCleanup: StatsDuplicateLineCleanupResult;
@@ -222,6 +233,7 @@ export interface StatsHttpClient {
   getSessionEvents: (id: number, limit?: number, eventTypes?: number[]) => Promise<SessionEvent[]>;
   getSessionKnownWordsTimeline: (id: number) => Promise<StatsSessionKnownWordsTimelinePoint[]>;
   getVocabulary: (limit?: number) => Promise<VocabularyEntry[]>;
+  getVocabularySummary: () => Promise<StatsVocabularySummary>;
   getExcludedWords: () => Promise<StatsExcludedWord[]>;
   setExcludedWords: (words: StatsExcludedWord[]) => Promise<void>;
   cleanupDuplicateLines: (

stats/src/components/vocabulary/VocabularyTab.tsx

@@ -35,7 +35,7 @@ export function VocabularyTab({
   onRemoveExclusion,
   onClearExclusions,
 }: VocabularyTabProps) {
-  const { words, kanji, knownWords, loading, error, reload } = useVocabulary();
+  const { words, kanji, knownWords, summary, loading, error, reload } = useVocabulary();
   const [selectedKanjiId, setSelectedKanjiId] = useState<number | null>(null);
   const [hideNames, setHideNames] = useState(false);
   const [showExclusionManager, setShowExclusionManager] = useState(false);
@@ -48,19 +48,10 @@ export function VocabularyTab({
     if (excluded.length > 0) result = result.filter((w) => !isExcluded(w));
     return result;
   }, [words, hideNames, excluded, isExcluded]);
-  const summary = useMemo(
+  const chartSummary = useMemo(
     () => buildVocabularySummary(filteredWords, kanji),
     [filteredWords, kanji],
   );
-  const knownWordCount = useMemo(() => {
-    if (knownWords.size === 0) return 0;
-
-    let count = 0;
-    for (const w of filteredWords) {
-      if (knownWords.has(w.headword)) count += 1;
-    }
-    return count;
-  }, [filteredWords, knownWords]);
 
   if (loading) {
     return (
@@ -90,29 +81,41 @@ export function VocabularyTab({
     setSelectedKanjiId(entry.kanjiId);
   };
 
+  const displayedSummary = hideNames
+    ? {
+        uniqueWords: summary?.uniqueWordsWithoutNames ?? 0,
+        newThisWeek: summary?.newThisWeekWithoutNames ?? 0,
+        knownWordCount: summary?.knownWordCountWithoutNames ?? null,
+      }
+    : {
+        uniqueWords: summary?.uniqueWords ?? 0,
+        newThisWeek: summary?.newThisWeek ?? 0,
+        knownWordCount: summary?.knownWordCount ?? null,
+      };
+
   return (
     <div className="space-y-4">
       <div className="grid grid-cols-2 xl:grid-cols-4 gap-3">
         <StatCard
           label="Unique Words"
-          value={formatNumber(summary.uniqueWords)}
+          value={formatNumber(displayedSummary.uniqueWords)}
           color="text-ctp-blue"
         />
-        {knownWords.size > 0 && (
+        {displayedSummary.knownWordCount !== null && (
           <StatCard
             label="Known Words"
-            value={`${formatNumber(knownWordCount)} (${summary.uniqueWords > 0 ? Math.round((knownWordCount / summary.uniqueWords) * 100) : 0}%)`}
+            value={`${formatNumber(displayedSummary.knownWordCount)} (${displayedSummary.uniqueWords > 0 ? Math.round((displayedSummary.knownWordCount / displayedSummary.uniqueWords) * 100) : 0}%)`}
             color="text-ctp-green"
           />
         )}
         <StatCard
           label="Unique Kanji"
-          value={formatNumber(summary.uniqueKanji)}
+          value={formatNumber(summary?.uniqueKanji ?? 0)}
           color="text-ctp-teal"
         />
         <StatCard
           label="New This Week"
-          value={`+${formatNumber(summary.newThisWeek)}`}
+          value={`+${formatNumber(displayedSummary.newThisWeek)}`}
           color="text-ctp-mauve"
         />
       </div>
@@ -154,14 +157,14 @@ export function VocabularyTab({
       <div className="grid grid-cols-1 xl:grid-cols-2 gap-4">
         <TrendChart
           title="Top Repeated Words"
-          data={summary.topWords}
+          data={chartSummary.topWords}
           color="#8aadf4"
           type="bar"
           onBarClick={handleBarClick}
         />
         <TrendChart
           title="New Words by Day"
-          data={summary.newWordsTimeline}
+          data={chartSummary.newWordsTimeline}
           color="#c6a0f6"
           type="line"
         />

stats/src/hooks/useVocabulary.ts

@@ -1,11 +1,12 @@
 import { useState, useEffect, useCallback } from 'react';
 import { getStatsClient } from './useStatsApi';
-import type { VocabularyEntry, KanjiEntry } from '../types/stats';
+import type { VocabularyEntry, KanjiEntry, StatsVocabularySummary } from '../types/stats';
 
 export function useVocabulary() {
   const [words, setWords] = useState<VocabularyEntry[]>([]);
   const [kanji, setKanji] = useState<KanjiEntry[]>([]);
   const [knownWords, setKnownWords] = useState<Set<string>>(new Set());
+  const [summary, setSummary] = useState<StatsVocabularySummary | null>(null);
   const [loading, setLoading] = useState(true);
   const [error, setError] = useState<string | null>(null);
   // Bumped by `reload` after maintenance rewrites the vocabulary tables.
@@ -17,8 +18,13 @@ export function useVocabulary() {
     setLoading(true);
     setError(null);
     const client = getStatsClient();
-    Promise.allSettled([client.getVocabulary(500), client.getKanji(200), client.getKnownWords()])
-      .then(([wordsResult, kanjiResult, knownResult]) => {
+    Promise.allSettled([
+      client.getVocabulary(500),
+      client.getKanji(200),
+      client.getKnownWords(),
+      client.getVocabularySummary(),
+    ])
+      .then(([wordsResult, kanjiResult, knownResult, summaryResult]) => {
         if (cancelled) return;
         const errors: string[] = [];
 
@@ -38,6 +44,12 @@ export function useVocabulary() {
           setKnownWords(new Set(knownResult.value));
         }
 
+        if (summaryResult.status === 'fulfilled') {
+          setSummary(summaryResult.value);
+        } else {
+          errors.push(summaryResult.reason.message);
+        }
+
         if (errors.length > 0) {
           setError(errors.join('; '));
         }
@@ -51,5 +63,5 @@ export function useVocabulary() {
     };
   }, [reloadToken]);
 
-  return { words, kanji, knownWords, loading, error, reload };
+  return { words, kanji, knownWords, summary, loading, error, reload };
 }

stats/src/lib/api-client.ts

@@ -100,6 +100,7 @@ export const apiClient = {
   getSessionKnownWordsTimeline: (id: number) =>
     fetchJson('sessionKnownWordsTimeline', `/api/stats/sessions/${id}/known-words-timeline`),
   getVocabulary: (limit = 100) => fetchJson('vocabulary', `/api/stats/vocabulary?limit=${limit}`),
+  getVocabularySummary: () => fetchJson('vocabularySummary', '/api/stats/vocabulary/summary'),
   getExcludedWords: () => fetchJson('excludedWords', '/api/stats/excluded-words'),
   setExcludedWords: async (words: StatsExcludedWord[]): Promise<void> => {
     await fetchResponse('/api/stats/excluded-words', {

stats/src/lib/vocabulary-tab.test.ts

@@ -20,15 +20,18 @@ test('VocabularyTab declares all hooks before loading and error early returns',
   assert.deepEqual(hooksAfterLoadingGuard ?? [], []);
 });
 
-test('VocabularyTab memoizes summary and known-word aggregate calculations', () => {
+test('VocabularyTab uses database-wide summary totals for its stat cards', () => {
   const source = fs.readFileSync(VOCABULARY_TAB_PATH, 'utf8');
 
   assert.match(
     source,
-    /const summary = useMemo\([\s\S]*buildVocabularySummary\(filteredWords, kanji\)[\s\S]*\[filteredWords, kanji\][\s\S]*\);/,
+    /const chartSummary = useMemo\([\s\S]*buildVocabularySummary\(filteredWords, kanji\)[\s\S]*\[filteredWords, kanji\][\s\S]*\);/,
   );
   assert.match(
     source,
-    /const knownWordCount = useMemo\(\(\) => \{[\s\S]*for \(const w of filteredWords\) \{[\s\S]*knownWords\.has\(w\.headword\)[\s\S]*\}\s*return count;\s*\}, \[filteredWords, knownWords\]\);/,
+    /const \{ words, kanji, knownWords, summary, loading, error, reload \} = useVocabulary\(\);/,
   );
+  assert.match(source, /uniqueWords: summary\?\.uniqueWordsWithoutNames \?\? 0/);
+  assert.match(source, /uniqueWords: summary\?\.uniqueWords \?\? 0/);
+  assert.match(source, /value=\{formatNumber\(summary\?\.uniqueKanji \?\? 0\)\}/);
 });