From 9ae303af7d2526585997952bf65de611dd31d504 Mon Sep 17 00:00:00 2001 From: sudacode Date: Sat, 15 Aug 2026 21:50:23 -0700 Subject: [PATCH] fix(stats): report complete vocabulary summary totals --- changes/stats-vocabulary-summary-totals.md | 4 + docs-site/immersion-tracking.md | 2 +- .../services/__tests__/stats-server.test.ts | 26 ++++++ .../services/immersion-tracker-service.ts | 5 ++ .../immersion-tracker/__tests__/query.test.ts | 83 +++++++++++++++++++ .../immersion-tracker/query-lexical.ts | 70 ++++++++++++++++ src/core/services/immersion-tracker/types.ts | 10 +++ .../services/stats-server/library-routes.ts | 8 ++ src/types/stats-http-contract.ts | 12 +++ .../components/vocabulary/VocabularyTab.tsx | 39 +++++---- stats/src/hooks/useVocabulary.ts | 20 ++++- stats/src/lib/api-client.ts | 1 + stats/src/lib/vocabulary-tab.test.ts | 9 +- 13 files changed, 263 insertions(+), 26 deletions(-) create mode 100644 changes/stats-vocabulary-summary-totals.md diff --git a/changes/stats-vocabulary-summary-totals.md b/changes/stats-vocabulary-summary-totals.md new file mode 100644 index 00000000..660313e0 --- /dev/null +++ b/changes/stats-vocabulary-summary-totals.md @@ -0,0 +1,4 @@ +type: fixed +area: stats + +- Fixed Vocabulary summary cards counting only the first page of frequency-ranked words and kanji instead of all tracked vocabulary. diff --git a/docs-site/immersion-tracking.md b/docs-site/immersion-tracking.md index 6147d8d7..5e7ad11b 100644 --- a/docs-site/immersion-tracking.md +++ b/docs-site/immersion-tracking.md @@ -82,7 +82,7 @@ Expandable session history with new-word activity, cumulative totals, and pause/ #### Vocabulary -Top repeated words (click a bar to open the word), new-word timeline, cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons. +The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The rest of the tab includes top repeated words (click a bar to open the word), new-word timeline, cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons. ![Stats Vocabulary](/screenshots/stats-vocabulary.png) diff --git a/src/core/services/__tests__/stats-server.test.ts b/src/core/services/__tests__/stats-server.test.ts index a7c2a5dd..aaa43b8a 100644 --- a/src/core/services/__tests__/stats-server.test.ts +++ b/src/core/services/__tests__/stats-server.test.ts @@ -284,6 +284,15 @@ function createMockTracker( getSessionTimeline: async () => [], getSessionEvents: async () => [], getVocabularyStats: async () => VOCABULARY_STATS, + getVocabularySummary: async () => ({ + uniqueWords: 501, + uniqueWordsWithoutNames: 500, + uniqueKanji: 201, + newThisWeek: 7, + newThisWeekWithoutNames: 6, + knownWordCount: 250, + knownWordCountWithoutNames: 249, + }), getStatsExcludedWords: async () => [], replaceStatsExcludedWords: async () => {}, getKanjiStats: async () => KANJI_STATS, @@ -711,6 +720,23 @@ describe('stats server API routes', () => { assert.equal(body[0].headword, 'する'); }); + it('GET /api/stats/vocabulary/summary returns database-wide card totals', async () => { + const app = createStatsApp(createMockTracker()); + + const res = await app.request('/api/stats/vocabulary/summary'); + + assert.equal(res.status, 200); + assert.deepEqual(await res.json(), { + uniqueWords: 501, + uniqueWordsWithoutNames: 500, + uniqueKanji: 201, + newThisWeek: 7, + newThisWeekWithoutNames: 6, + knownWordCount: 250, + knownWordCountWithoutNames: 249, + }); + }); + it('GET /api/stats/kanji returns kanji frequency data', async () => { const app = createStatsApp(createMockTracker()); const res = await app.request('/api/stats/kanji'); diff --git a/src/core/services/immersion-tracker-service.ts b/src/core/services/immersion-tracker-service.ts index d4efbdd0..686dc88d 100644 --- a/src/core/services/immersion-tracker-service.ts +++ b/src/core/services/immersion-tracker-service.ts @@ -59,6 +59,7 @@ import { getSimilarWords, getStatsExcludedWords, getVocabularyStats, + getVocabularySummary, replaceStatsExcludedWords, searchSubtitleSentences, getWordAnimeAppearances, @@ -634,6 +635,10 @@ export class ImmersionTrackerService { return getVocabularyStats(this.db, limit, excludePos); } + async getVocabularySummary(knownWords: ReadonlySet | null) { + return getVocabularySummary(this.db, knownWords); + } + async getStatsExcludedWords(): Promise { return getStatsExcludedWords(this.db); } diff --git a/src/core/services/immersion-tracker/__tests__/query.test.ts b/src/core/services/immersion-tracker/__tests__/query.test.ts index f1eddcdd..2c6b7ec7 100644 --- a/src/core/services/immersion-tracker/__tests__/query.test.ts +++ b/src/core/services/immersion-tracker/__tests__/query.test.ts @@ -31,6 +31,7 @@ import { getKanjiOccurrences, getSessionSummaries, getVocabularyStats, + getVocabularySummary, getKanjiStats, getSessionEvents, getSessionTimeline, @@ -1875,6 +1876,88 @@ test('getVocabularyStats returns rows ordered by frequency descending', () => { } }); +test('getVocabularySummary counts every tracked vocabulary row instead of a display page', () => { + const dbPath = makeDbPath(); + const db = openTestDb(dbPath); + + try { + ensureSchema(db); + const nowSec = Math.floor(Date.now() / 1000); + const insertWord = db.prepare(` + INSERT INTO imm_words ( + headword, word, reading, part_of_speech, pos1, pos2, pos3, + first_seen, last_seen, frequency + ) VALUES (?, ?, '', 'noun', '名詞', '一般', '', ?, ?, 1) + `); + const insertKanji = db.prepare(` + INSERT INTO imm_kanji (kanji, first_seen, last_seen, frequency) + VALUES (?, ?, ?, 1) + `); + + for (let index = 0; index < 501; index += 1) { + insertWord.run(`単語${index}`, `単語${index}`, nowSec - 8 * 86_400, nowSec - 8 * 86_400); + } + for (let index = 0; index < 201; index += 1) { + insertKanji.run( + String.fromCodePoint(0x4e00 + index), + nowSec - 8 * 86_400, + nowSec - 8 * 86_400, + ); + } + insertWord.run('今週', '今週', nowSec - 86_400, nowSec - 86_400); + + assert.deepEqual(getVocabularySummary(db, new Set(['単語0', '今週']), nowSec * 1000), { + uniqueWords: 502, + uniqueWordsWithoutNames: 502, + uniqueKanji: 201, + newThisWeek: 1, + newThisWeekWithoutNames: 1, + knownWordCount: 2, + knownWordCountWithoutNames: 2, + }); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + +test('getVocabularySummary applies vocabulary exclusions and Hide Names totals', () => { + const dbPath = makeDbPath(); + const db = openTestDb(dbPath); + + try { + ensureSchema(db); + const insertWord = db.prepare(` + INSERT INTO imm_words ( + headword, word, reading, part_of_speech, pos1, pos2, pos3, + first_seen, last_seen, frequency + ) VALUES (?, ?, '', 'noun', '名詞', ?, '', 1, 1, 1) + `); + insertWord.run('猫', '猫', '一般'); + insertWord.run('太郎', '太郎', '固有名詞'); + insertWord.run('東京', '東京都', '一般'); + db.prepare( + ` + INSERT INTO imm_stats_excluded_words (headword, word, reading) + VALUES ('東京', '東京', '') + `, + ).run(); + + assert.deepEqual(getVocabularySummary(db, new Set(['猫', '太郎', '東京']), 9 * 86_400_000), { + uniqueWords: 2, + uniqueWordsWithoutNames: 1, + uniqueKanji: 0, + newThisWeek: 0, + newThisWeekWithoutNames: 0, + knownWordCount: 2, + knownWordCountWithoutNames: 1, + }); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () => { const dbPath = makeDbPath(); const db = openTestDb(dbPath); diff --git a/src/core/services/immersion-tracker/query-lexical.ts b/src/core/services/immersion-tracker/query-lexical.ts index 13533158..28e0d689 100644 --- a/src/core/services/immersion-tracker/query-lexical.ts +++ b/src/core/services/immersion-tracker/query-lexical.ts @@ -13,6 +13,7 @@ import type { SimilarWordRow, StatsExcludedWordRow, VocabularyStatsRow, + VocabularyStatsSummary, WordAnimeAppearanceRow, WordDetailRow, WordOccurrenceRow, @@ -153,6 +154,75 @@ export function getVocabularyStats( return visibleRows.slice(0, limit); } +function excludedVocabularyAliases( + word: Pick, +): string[] { + const aliases = [word.headword.trim(), word.word.trim()].filter(Boolean); + if (aliases.length === 0) aliases.push(word.reading.trim()); + return [...new Set(aliases)]; +} + +function timestampSeconds(timestamp: number): number { + return timestamp < 10_000_000_000 ? timestamp : Math.floor(timestamp / 1000); +} + +export function getVocabularySummary( + db: DatabaseSync, + knownWords: ReadonlySet | null, + nowMs: number = Date.now(), +): VocabularyStatsSummary { + const words = db + .prepare( + ` + SELECT id AS wordId, headword, word, reading, + part_of_speech AS partOfSpeech, pos1, pos2, pos3, + frequency, frequency_rank AS frequencyRank, + first_seen AS firstSeen, last_seen AS lastSeen, + 0 AS animeCount + FROM imm_words + `, + ) + .all() as VocabularyStatsRow[]; + const excludedAliases = new Set( + getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)), + ); + const weekAgoSec = nowMs / 1000 - 7 * 86_400; + const summary: VocabularyStatsSummary = { + uniqueWords: 0, + uniqueWordsWithoutNames: 0, + uniqueKanji: (db.prepare('SELECT COUNT(*) AS count FROM imm_kanji').get() as { count: number }) + .count, + newThisWeek: 0, + newThisWeekWithoutNames: 0, + knownWordCount: knownWords ? 0 : null, + knownWordCountWithoutNames: knownWords ? 0 : null, + }; + + for (const word of words) { + if ( + !isVocabularyStatsRowVisible(word) || + excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias)) + ) { + continue; + } + const isName = word.pos2 === '固有名詞'; + const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec; + const isKnown = knownWords?.has(word.headword) ?? false; + summary.uniqueWords += 1; + if (!isName) summary.uniqueWordsWithoutNames += 1; + if (isNewThisWeek) { + summary.newThisWeek += 1; + if (!isName) summary.newThisWeekWithoutNames += 1; + } + if (isKnown) { + summary.knownWordCount! += 1; + if (!isName) summary.knownWordCountWithoutNames! += 1; + } + } + + return summary; +} + export function getStatsExcludedWords(db: DatabaseSync): StatsExcludedWordRow[] { return db .prepare( diff --git a/src/core/services/immersion-tracker/types.ts b/src/core/services/immersion-tracker/types.ts index faabb4d4..50a13f66 100644 --- a/src/core/services/immersion-tracker/types.ts +++ b/src/core/services/immersion-tracker/types.ts @@ -306,6 +306,16 @@ export interface VocabularyStatsRow { lastSeen: number; } +export interface VocabularyStatsSummary { + uniqueWords: number; + uniqueWordsWithoutNames: number; + uniqueKanji: number; + newThisWeek: number; + newThisWeekWithoutNames: number; + knownWordCount: number | null; + knownWordCountWithoutNames: number | null; +} + export interface StatsExcludedWordRow { headword: string; word: string; diff --git a/src/core/services/stats-server/library-routes.ts b/src/core/services/stats-server/library-routes.ts index 89866d9b..6255d155 100644 --- a/src/core/services/stats-server/library-routes.ts +++ b/src/core/services/stats-server/library-routes.ts @@ -10,6 +10,7 @@ import { parseExcludedWordsBody, parseIntQuery, parsePositiveIdList, + loadKnownWordsSet, } from './route-support.js'; export function registerStatsLibraryRoutes( @@ -31,6 +32,13 @@ export function registerStatsLibraryRoutes( return c.json(statsJson('vocabulary', vocab)); }); + app.get('/api/stats/vocabulary/summary', async (c) => { + const summary = await tracker.getVocabularySummary( + loadKnownWordsSet(options?.knownWordCachePath), + ); + return c.json(statsJson('vocabularySummary', summary)); + }); + app.get('/api/stats/excluded-words', async (c) => { return c.json(statsJson('excludedWords', await tracker.getStatsExcludedWords())); }); diff --git a/src/types/stats-http-contract.ts b/src/types/stats-http-contract.ts index 874e9b76..cea7556f 100644 --- a/src/types/stats-http-contract.ts +++ b/src/types/stats-http-contract.ts @@ -50,6 +50,16 @@ export interface StatsKnownWordsSummary { knownWordCount: number; } +export interface StatsVocabularySummary { + uniqueWords: number; + uniqueWordsWithoutNames: number; + uniqueKanji: number; + newThisWeek: number; + newThisWeekWithoutNames: number; + knownWordCount: number | null; + knownWordCountWithoutNames: number | null; +} + export interface StatsAnilistSearchResult { id: number; episodes: number | null; @@ -164,6 +174,7 @@ export interface StatsJsonResponseMap { sessionEvents: SessionEvent[]; sessionKnownWordsTimeline: StatsSessionKnownWordsTimelinePoint[]; vocabulary: VocabularyEntry[]; + vocabularySummary: StatsVocabularySummary; excludedWords: StatsExcludedWord[]; setExcludedWords: StatsOkResponse; duplicateLineCleanup: StatsDuplicateLineCleanupResult; @@ -222,6 +233,7 @@ export interface StatsHttpClient { getSessionEvents: (id: number, limit?: number, eventTypes?: number[]) => Promise; getSessionKnownWordsTimeline: (id: number) => Promise; getVocabulary: (limit?: number) => Promise; + getVocabularySummary: () => Promise; getExcludedWords: () => Promise; setExcludedWords: (words: StatsExcludedWord[]) => Promise; cleanupDuplicateLines: ( diff --git a/stats/src/components/vocabulary/VocabularyTab.tsx b/stats/src/components/vocabulary/VocabularyTab.tsx index 70dbf2f2..d89595ee 100644 --- a/stats/src/components/vocabulary/VocabularyTab.tsx +++ b/stats/src/components/vocabulary/VocabularyTab.tsx @@ -35,7 +35,7 @@ export function VocabularyTab({ onRemoveExclusion, onClearExclusions, }: VocabularyTabProps) { - const { words, kanji, knownWords, loading, error, reload } = useVocabulary(); + const { words, kanji, knownWords, summary, loading, error, reload } = useVocabulary(); const [selectedKanjiId, setSelectedKanjiId] = useState(null); const [hideNames, setHideNames] = useState(false); const [showExclusionManager, setShowExclusionManager] = useState(false); @@ -48,19 +48,10 @@ export function VocabularyTab({ if (excluded.length > 0) result = result.filter((w) => !isExcluded(w)); return result; }, [words, hideNames, excluded, isExcluded]); - const summary = useMemo( + const chartSummary = useMemo( () => buildVocabularySummary(filteredWords, kanji), [filteredWords, kanji], ); - const knownWordCount = useMemo(() => { - if (knownWords.size === 0) return 0; - - let count = 0; - for (const w of filteredWords) { - if (knownWords.has(w.headword)) count += 1; - } - return count; - }, [filteredWords, knownWords]); if (loading) { return ( @@ -90,29 +81,41 @@ export function VocabularyTab({ setSelectedKanjiId(entry.kanjiId); }; + const displayedSummary = hideNames + ? { + uniqueWords: summary?.uniqueWordsWithoutNames ?? 0, + newThisWeek: summary?.newThisWeekWithoutNames ?? 0, + knownWordCount: summary?.knownWordCountWithoutNames ?? null, + } + : { + uniqueWords: summary?.uniqueWords ?? 0, + newThisWeek: summary?.newThisWeek ?? 0, + knownWordCount: summary?.knownWordCount ?? null, + }; + return (
- {knownWords.size > 0 && ( + {displayedSummary.knownWordCount !== null && ( 0 ? Math.round((knownWordCount / summary.uniqueWords) * 100) : 0}%)`} + value={`${formatNumber(displayedSummary.knownWordCount)} (${displayedSummary.uniqueWords > 0 ? Math.round((displayedSummary.knownWordCount / displayedSummary.uniqueWords) * 100) : 0}%)`} color="text-ctp-green" /> )}
@@ -154,14 +157,14 @@ export function VocabularyTab({
diff --git a/stats/src/hooks/useVocabulary.ts b/stats/src/hooks/useVocabulary.ts index 943cd752..eedbe821 100644 --- a/stats/src/hooks/useVocabulary.ts +++ b/stats/src/hooks/useVocabulary.ts @@ -1,11 +1,12 @@ import { useState, useEffect, useCallback } from 'react'; import { getStatsClient } from './useStatsApi'; -import type { VocabularyEntry, KanjiEntry } from '../types/stats'; +import type { VocabularyEntry, KanjiEntry, StatsVocabularySummary } from '../types/stats'; export function useVocabulary() { const [words, setWords] = useState([]); const [kanji, setKanji] = useState([]); const [knownWords, setKnownWords] = useState>(new Set()); + const [summary, setSummary] = useState(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); // Bumped by `reload` after maintenance rewrites the vocabulary tables. @@ -17,8 +18,13 @@ export function useVocabulary() { setLoading(true); setError(null); const client = getStatsClient(); - Promise.allSettled([client.getVocabulary(500), client.getKanji(200), client.getKnownWords()]) - .then(([wordsResult, kanjiResult, knownResult]) => { + Promise.allSettled([ + client.getVocabulary(500), + client.getKanji(200), + client.getKnownWords(), + client.getVocabularySummary(), + ]) + .then(([wordsResult, kanjiResult, knownResult, summaryResult]) => { if (cancelled) return; const errors: string[] = []; @@ -38,6 +44,12 @@ export function useVocabulary() { setKnownWords(new Set(knownResult.value)); } + if (summaryResult.status === 'fulfilled') { + setSummary(summaryResult.value); + } else { + errors.push(summaryResult.reason.message); + } + if (errors.length > 0) { setError(errors.join('; ')); } @@ -51,5 +63,5 @@ export function useVocabulary() { }; }, [reloadToken]); - return { words, kanji, knownWords, loading, error, reload }; + return { words, kanji, knownWords, summary, loading, error, reload }; } diff --git a/stats/src/lib/api-client.ts b/stats/src/lib/api-client.ts index d3aa11f5..35a808f7 100644 --- a/stats/src/lib/api-client.ts +++ b/stats/src/lib/api-client.ts @@ -100,6 +100,7 @@ export const apiClient = { getSessionKnownWordsTimeline: (id: number) => fetchJson('sessionKnownWordsTimeline', `/api/stats/sessions/${id}/known-words-timeline`), getVocabulary: (limit = 100) => fetchJson('vocabulary', `/api/stats/vocabulary?limit=${limit}`), + getVocabularySummary: () => fetchJson('vocabularySummary', '/api/stats/vocabulary/summary'), getExcludedWords: () => fetchJson('excludedWords', '/api/stats/excluded-words'), setExcludedWords: async (words: StatsExcludedWord[]): Promise => { await fetchResponse('/api/stats/excluded-words', { diff --git a/stats/src/lib/vocabulary-tab.test.ts b/stats/src/lib/vocabulary-tab.test.ts index a5dc52cf..f2da5a51 100644 --- a/stats/src/lib/vocabulary-tab.test.ts +++ b/stats/src/lib/vocabulary-tab.test.ts @@ -20,15 +20,18 @@ test('VocabularyTab declares all hooks before loading and error early returns', assert.deepEqual(hooksAfterLoadingGuard ?? [], []); }); -test('VocabularyTab memoizes summary and known-word aggregate calculations', () => { +test('VocabularyTab uses database-wide summary totals for its stat cards', () => { const source = fs.readFileSync(VOCABULARY_TAB_PATH, 'utf8'); assert.match( source, - /const summary = useMemo\([\s\S]*buildVocabularySummary\(filteredWords, kanji\)[\s\S]*\[filteredWords, kanji\][\s\S]*\);/, + /const chartSummary = useMemo\([\s\S]*buildVocabularySummary\(filteredWords, kanji\)[\s\S]*\[filteredWords, kanji\][\s\S]*\);/, ); assert.match( source, - /const knownWordCount = useMemo\(\(\) => \{[\s\S]*for \(const w of filteredWords\) \{[\s\S]*knownWords\.has\(w\.headword\)[\s\S]*\}\s*return count;\s*\}, \[filteredWords, knownWords\]\);/, + /const \{ words, kanji, knownWords, summary, loading, error, reload \} = useVocabulary\(\);/, ); + assert.match(source, /uniqueWords: summary\?\.uniqueWordsWithoutNames \?\? 0/); + assert.match(source, /uniqueWords: summary\?\.uniqueWords \?\? 0/); + assert.match(source, /value=\{formatNumber\(summary\?\.uniqueKanji \?\? 0\)\}/); });