fix(stats): report complete vocabulary summary totals

This commit is contained in:
2026-08-15 21:50:23 -07:00
parent 297aa382c6
commit 9ae303af7d
13 changed files with 263 additions and 26 deletions
@@ -284,6 +284,15 @@ function createMockTracker(
getSessionTimeline: async () => [],
getSessionEvents: async () => [],
getVocabularyStats: async () => VOCABULARY_STATS,
getVocabularySummary: async () => ({
uniqueWords: 501,
uniqueWordsWithoutNames: 500,
uniqueKanji: 201,
newThisWeek: 7,
newThisWeekWithoutNames: 6,
knownWordCount: 250,
knownWordCountWithoutNames: 249,
}),
getStatsExcludedWords: async () => [],
replaceStatsExcludedWords: async () => {},
getKanjiStats: async () => KANJI_STATS,
@@ -711,6 +720,23 @@ describe('stats server API routes', () => {
assert.equal(body[0].headword, 'する');
});
it('GET /api/stats/vocabulary/summary returns database-wide card totals', async () => {
const app = createStatsApp(createMockTracker());
const res = await app.request('/api/stats/vocabulary/summary');
assert.equal(res.status, 200);
assert.deepEqual(await res.json(), {
uniqueWords: 501,
uniqueWordsWithoutNames: 500,
uniqueKanji: 201,
newThisWeek: 7,
newThisWeekWithoutNames: 6,
knownWordCount: 250,
knownWordCountWithoutNames: 249,
});
});
it('GET /api/stats/kanji returns kanji frequency data', async () => {
const app = createStatsApp(createMockTracker());
const res = await app.request('/api/stats/kanji');
@@ -59,6 +59,7 @@ import {
getSimilarWords,
getStatsExcludedWords,
getVocabularyStats,
getVocabularySummary,
replaceStatsExcludedWords,
searchSubtitleSentences,
getWordAnimeAppearances,
@@ -634,6 +635,10 @@ export class ImmersionTrackerService {
return getVocabularyStats(this.db, limit, excludePos);
}
async getVocabularySummary(knownWords: ReadonlySet<string> | null) {
return getVocabularySummary(this.db, knownWords);
}
async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {
return getStatsExcludedWords(this.db);
}
@@ -31,6 +31,7 @@ import {
getKanjiOccurrences,
getSessionSummaries,
getVocabularyStats,
getVocabularySummary,
getKanjiStats,
getSessionEvents,
getSessionTimeline,
@@ -1875,6 +1876,88 @@ test('getVocabularyStats returns rows ordered by frequency descending', () => {
}
});
test('getVocabularySummary counts every tracked vocabulary row instead of a display page', () => {
const dbPath = makeDbPath();
const db = openTestDb(dbPath);
try {
ensureSchema(db);
const nowSec = Math.floor(Date.now() / 1000);
const insertWord = db.prepare(`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3,
first_seen, last_seen, frequency
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', ?, ?, 1)
`);
const insertKanji = db.prepare(`
INSERT INTO imm_kanji (kanji, first_seen, last_seen, frequency)
VALUES (?, ?, ?, 1)
`);
for (let index = 0; index < 501; index += 1) {
insertWord.run(`単語${index}`, `単語${index}`, nowSec - 8 * 86_400, nowSec - 8 * 86_400);
}
for (let index = 0; index < 201; index += 1) {
insertKanji.run(
String.fromCodePoint(0x4e00 + index),
nowSec - 8 * 86_400,
nowSec - 8 * 86_400,
);
}
insertWord.run('今週', '今週', nowSec - 86_400, nowSec - 86_400);
assert.deepEqual(getVocabularySummary(db, new Set(['単語0', '今週']), nowSec * 1000), {
uniqueWords: 502,
uniqueWordsWithoutNames: 502,
uniqueKanji: 201,
newThisWeek: 1,
newThisWeekWithoutNames: 1,
knownWordCount: 2,
knownWordCountWithoutNames: 2,
});
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getVocabularySummary applies vocabulary exclusions and Hide Names totals', () => {
const dbPath = makeDbPath();
const db = openTestDb(dbPath);
try {
ensureSchema(db);
const insertWord = db.prepare(`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3,
first_seen, last_seen, frequency
) VALUES (?, ?, '', 'noun', '名詞', ?, '', 1, 1, 1)
`);
insertWord.run('猫', '猫', '一般');
insertWord.run('太郎', '太郎', '固有名詞');
insertWord.run('東京', '東京都', '一般');
db.prepare(
`
INSERT INTO imm_stats_excluded_words (headword, word, reading)
VALUES ('東京', '東京', '')
`,
).run();
assert.deepEqual(getVocabularySummary(db, new Set(['猫', '太郎', '東京']), 9 * 86_400_000), {
uniqueWords: 2,
uniqueWordsWithoutNames: 1,
uniqueKanji: 0,
newThisWeek: 0,
newThisWeekWithoutNames: 0,
knownWordCount: 2,
knownWordCountWithoutNames: 1,
});
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () => {
const dbPath = makeDbPath();
const db = openTestDb(dbPath);
@@ -13,6 +13,7 @@ import type {
SimilarWordRow,
StatsExcludedWordRow,
VocabularyStatsRow,
VocabularyStatsSummary,
WordAnimeAppearanceRow,
WordDetailRow,
WordOccurrenceRow,
@@ -153,6 +154,75 @@ export function getVocabularyStats(
return visibleRows.slice(0, limit);
}
function excludedVocabularyAliases(
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
): string[] {
const aliases = [word.headword.trim(), word.word.trim()].filter(Boolean);
if (aliases.length === 0) aliases.push(word.reading.trim());
return [...new Set(aliases)];
}
function timestampSeconds(timestamp: number): number {
return timestamp < 10_000_000_000 ? timestamp : Math.floor(timestamp / 1000);
}
export function getVocabularySummary(
db: DatabaseSync,
knownWords: ReadonlySet<string> | null,
nowMs: number = Date.now(),
): VocabularyStatsSummary {
const words = db
.prepare(
`
SELECT id AS wordId, headword, word, reading,
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
frequency, frequency_rank AS frequencyRank,
first_seen AS firstSeen, last_seen AS lastSeen,
0 AS animeCount
FROM imm_words
`,
)
.all() as VocabularyStatsRow[];
const excludedAliases = new Set(
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
);
const weekAgoSec = nowMs / 1000 - 7 * 86_400;
const summary: VocabularyStatsSummary = {
uniqueWords: 0,
uniqueWordsWithoutNames: 0,
uniqueKanji: (db.prepare('SELECT COUNT(*) AS count FROM imm_kanji').get() as { count: number })
.count,
newThisWeek: 0,
newThisWeekWithoutNames: 0,
knownWordCount: knownWords ? 0 : null,
knownWordCountWithoutNames: knownWords ? 0 : null,
};
for (const word of words) {
if (
!isVocabularyStatsRowVisible(word) ||
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias))
) {
continue;
}
const isName = word.pos2 === '固有名詞';
const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec;
const isKnown = knownWords?.has(word.headword) ?? false;
summary.uniqueWords += 1;
if (!isName) summary.uniqueWordsWithoutNames += 1;
if (isNewThisWeek) {
summary.newThisWeek += 1;
if (!isName) summary.newThisWeekWithoutNames += 1;
}
if (isKnown) {
summary.knownWordCount! += 1;
if (!isName) summary.knownWordCountWithoutNames! += 1;
}
}
return summary;
}
export function getStatsExcludedWords(db: DatabaseSync): StatsExcludedWordRow[] {
return db
.prepare(
@@ -306,6 +306,16 @@ export interface VocabularyStatsRow {
lastSeen: number;
}
export interface VocabularyStatsSummary {
uniqueWords: number;
uniqueWordsWithoutNames: number;
uniqueKanji: number;
newThisWeek: number;
newThisWeekWithoutNames: number;
knownWordCount: number | null;
knownWordCountWithoutNames: number | null;
}
export interface StatsExcludedWordRow {
headword: string;
word: string;
@@ -10,6 +10,7 @@ import {
parseExcludedWordsBody,
parseIntQuery,
parsePositiveIdList,
loadKnownWordsSet,
} from './route-support.js';
export function registerStatsLibraryRoutes(
@@ -31,6 +32,13 @@ export function registerStatsLibraryRoutes(
return c.json(statsJson('vocabulary', vocab));
});
app.get('/api/stats/vocabulary/summary', async (c) => {
const summary = await tracker.getVocabularySummary(
loadKnownWordsSet(options?.knownWordCachePath),
);
return c.json(statsJson('vocabularySummary', summary));
});
app.get('/api/stats/excluded-words', async (c) => {
return c.json(statsJson('excludedWords', await tracker.getStatsExcludedWords()));
});
+12
View File
@@ -50,6 +50,16 @@ export interface StatsKnownWordsSummary {
knownWordCount: number;
}
export interface StatsVocabularySummary {
uniqueWords: number;
uniqueWordsWithoutNames: number;
uniqueKanji: number;
newThisWeek: number;
newThisWeekWithoutNames: number;
knownWordCount: number | null;
knownWordCountWithoutNames: number | null;
}
export interface StatsAnilistSearchResult {
id: number;
episodes: number | null;
@@ -164,6 +174,7 @@ export interface StatsJsonResponseMap {
sessionEvents: SessionEvent[];
sessionKnownWordsTimeline: StatsSessionKnownWordsTimelinePoint[];
vocabulary: VocabularyEntry[];
vocabularySummary: StatsVocabularySummary;
excludedWords: StatsExcludedWord[];
setExcludedWords: StatsOkResponse;
duplicateLineCleanup: StatsDuplicateLineCleanupResult;
@@ -222,6 +233,7 @@ export interface StatsHttpClient {
getSessionEvents: (id: number, limit?: number, eventTypes?: number[]) => Promise<SessionEvent[]>;
getSessionKnownWordsTimeline: (id: number) => Promise<StatsSessionKnownWordsTimelinePoint[]>;
getVocabulary: (limit?: number) => Promise<VocabularyEntry[]>;
getVocabularySummary: () => Promise<StatsVocabularySummary>;
getExcludedWords: () => Promise<StatsExcludedWord[]>;
setExcludedWords: (words: StatsExcludedWord[]) => Promise<void>;
cleanupDuplicateLines: (