mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-16 01:55:51 -07:00
fix(stats): report complete vocabulary summary totals
This commit is contained in:
@@ -284,6 +284,15 @@ function createMockTracker(
|
||||
getSessionTimeline: async () => [],
|
||||
getSessionEvents: async () => [],
|
||||
getVocabularyStats: async () => VOCABULARY_STATS,
|
||||
getVocabularySummary: async () => ({
|
||||
uniqueWords: 501,
|
||||
uniqueWordsWithoutNames: 500,
|
||||
uniqueKanji: 201,
|
||||
newThisWeek: 7,
|
||||
newThisWeekWithoutNames: 6,
|
||||
knownWordCount: 250,
|
||||
knownWordCountWithoutNames: 249,
|
||||
}),
|
||||
getStatsExcludedWords: async () => [],
|
||||
replaceStatsExcludedWords: async () => {},
|
||||
getKanjiStats: async () => KANJI_STATS,
|
||||
@@ -711,6 +720,23 @@ describe('stats server API routes', () => {
|
||||
assert.equal(body[0].headword, 'する');
|
||||
});
|
||||
|
||||
it('GET /api/stats/vocabulary/summary returns database-wide card totals', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
|
||||
const res = await app.request('/api/stats/vocabulary/summary');
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(await res.json(), {
|
||||
uniqueWords: 501,
|
||||
uniqueWordsWithoutNames: 500,
|
||||
uniqueKanji: 201,
|
||||
newThisWeek: 7,
|
||||
newThisWeekWithoutNames: 6,
|
||||
knownWordCount: 250,
|
||||
knownWordCountWithoutNames: 249,
|
||||
});
|
||||
});
|
||||
|
||||
it('GET /api/stats/kanji returns kanji frequency data', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
const res = await app.request('/api/stats/kanji');
|
||||
|
||||
@@ -59,6 +59,7 @@ import {
|
||||
getSimilarWords,
|
||||
getStatsExcludedWords,
|
||||
getVocabularyStats,
|
||||
getVocabularySummary,
|
||||
replaceStatsExcludedWords,
|
||||
searchSubtitleSentences,
|
||||
getWordAnimeAppearances,
|
||||
@@ -634,6 +635,10 @@ export class ImmersionTrackerService {
|
||||
return getVocabularyStats(this.db, limit, excludePos);
|
||||
}
|
||||
|
||||
async getVocabularySummary(knownWords: ReadonlySet<string> | null) {
|
||||
return getVocabularySummary(this.db, knownWords);
|
||||
}
|
||||
|
||||
async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {
|
||||
return getStatsExcludedWords(this.db);
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
getKanjiOccurrences,
|
||||
getSessionSummaries,
|
||||
getVocabularyStats,
|
||||
getVocabularySummary,
|
||||
getKanjiStats,
|
||||
getSessionEvents,
|
||||
getSessionTimeline,
|
||||
@@ -1875,6 +1876,88 @@ test('getVocabularyStats returns rows ordered by frequency descending', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary counts every tracked vocabulary row instead of a display page', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const nowSec = Math.floor(Date.now() / 1000);
|
||||
const insertWord = db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', ?, ?, 1)
|
||||
`);
|
||||
const insertKanji = db.prepare(`
|
||||
INSERT INTO imm_kanji (kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)
|
||||
`);
|
||||
|
||||
for (let index = 0; index < 501; index += 1) {
|
||||
insertWord.run(`単語${index}`, `単語${index}`, nowSec - 8 * 86_400, nowSec - 8 * 86_400);
|
||||
}
|
||||
for (let index = 0; index < 201; index += 1) {
|
||||
insertKanji.run(
|
||||
String.fromCodePoint(0x4e00 + index),
|
||||
nowSec - 8 * 86_400,
|
||||
nowSec - 8 * 86_400,
|
||||
);
|
||||
}
|
||||
insertWord.run('今週', '今週', nowSec - 86_400, nowSec - 86_400);
|
||||
|
||||
assert.deepEqual(getVocabularySummary(db, new Set(['単語0', '今週']), nowSec * 1000), {
|
||||
uniqueWords: 502,
|
||||
uniqueWordsWithoutNames: 502,
|
||||
uniqueKanji: 201,
|
||||
newThisWeek: 1,
|
||||
newThisWeekWithoutNames: 1,
|
||||
knownWordCount: 2,
|
||||
knownWordCountWithoutNames: 2,
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary applies vocabulary exclusions and Hide Names totals', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, '', 'noun', '名詞', ?, '', 1, 1, 1)
|
||||
`);
|
||||
insertWord.run('猫', '猫', '一般');
|
||||
insertWord.run('太郎', '太郎', '固有名詞');
|
||||
insertWord.run('東京', '東京都', '一般');
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_stats_excluded_words (headword, word, reading)
|
||||
VALUES ('東京', '東京', '')
|
||||
`,
|
||||
).run();
|
||||
|
||||
assert.deepEqual(getVocabularySummary(db, new Set(['猫', '太郎', '東京']), 9 * 86_400_000), {
|
||||
uniqueWords: 2,
|
||||
uniqueWordsWithoutNames: 1,
|
||||
uniqueKanji: 0,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: 2,
|
||||
knownWordCountWithoutNames: 1,
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
@@ -13,6 +13,7 @@ import type {
|
||||
SimilarWordRow,
|
||||
StatsExcludedWordRow,
|
||||
VocabularyStatsRow,
|
||||
VocabularyStatsSummary,
|
||||
WordAnimeAppearanceRow,
|
||||
WordDetailRow,
|
||||
WordOccurrenceRow,
|
||||
@@ -153,6 +154,75 @@ export function getVocabularyStats(
|
||||
return visibleRows.slice(0, limit);
|
||||
}
|
||||
|
||||
function excludedVocabularyAliases(
|
||||
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
|
||||
): string[] {
|
||||
const aliases = [word.headword.trim(), word.word.trim()].filter(Boolean);
|
||||
if (aliases.length === 0) aliases.push(word.reading.trim());
|
||||
return [...new Set(aliases)];
|
||||
}
|
||||
|
||||
function timestampSeconds(timestamp: number): number {
|
||||
return timestamp < 10_000_000_000 ? timestamp : Math.floor(timestamp / 1000);
|
||||
}
|
||||
|
||||
export function getVocabularySummary(
|
||||
db: DatabaseSync,
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
nowMs: number = Date.now(),
|
||||
): VocabularyStatsSummary {
|
||||
const words = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id AS wordId, headword, word, reading,
|
||||
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
|
||||
frequency, frequency_rank AS frequencyRank,
|
||||
first_seen AS firstSeen, last_seen AS lastSeen,
|
||||
0 AS animeCount
|
||||
FROM imm_words
|
||||
`,
|
||||
)
|
||||
.all() as VocabularyStatsRow[];
|
||||
const excludedAliases = new Set(
|
||||
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
|
||||
);
|
||||
const weekAgoSec = nowMs / 1000 - 7 * 86_400;
|
||||
const summary: VocabularyStatsSummary = {
|
||||
uniqueWords: 0,
|
||||
uniqueWordsWithoutNames: 0,
|
||||
uniqueKanji: (db.prepare('SELECT COUNT(*) AS count FROM imm_kanji').get() as { count: number })
|
||||
.count,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: knownWords ? 0 : null,
|
||||
knownWordCountWithoutNames: knownWords ? 0 : null,
|
||||
};
|
||||
|
||||
for (const word of words) {
|
||||
if (
|
||||
!isVocabularyStatsRowVisible(word) ||
|
||||
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const isName = word.pos2 === '固有名詞';
|
||||
const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec;
|
||||
const isKnown = knownWords?.has(word.headword) ?? false;
|
||||
summary.uniqueWords += 1;
|
||||
if (!isName) summary.uniqueWordsWithoutNames += 1;
|
||||
if (isNewThisWeek) {
|
||||
summary.newThisWeek += 1;
|
||||
if (!isName) summary.newThisWeekWithoutNames += 1;
|
||||
}
|
||||
if (isKnown) {
|
||||
summary.knownWordCount! += 1;
|
||||
if (!isName) summary.knownWordCountWithoutNames! += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
export function getStatsExcludedWords(db: DatabaseSync): StatsExcludedWordRow[] {
|
||||
return db
|
||||
.prepare(
|
||||
|
||||
@@ -306,6 +306,16 @@ export interface VocabularyStatsRow {
|
||||
lastSeen: number;
|
||||
}
|
||||
|
||||
export interface VocabularyStatsSummary {
|
||||
uniqueWords: number;
|
||||
uniqueWordsWithoutNames: number;
|
||||
uniqueKanji: number;
|
||||
newThisWeek: number;
|
||||
newThisWeekWithoutNames: number;
|
||||
knownWordCount: number | null;
|
||||
knownWordCountWithoutNames: number | null;
|
||||
}
|
||||
|
||||
export interface StatsExcludedWordRow {
|
||||
headword: string;
|
||||
word: string;
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
parseExcludedWordsBody,
|
||||
parseIntQuery,
|
||||
parsePositiveIdList,
|
||||
loadKnownWordsSet,
|
||||
} from './route-support.js';
|
||||
|
||||
export function registerStatsLibraryRoutes(
|
||||
@@ -31,6 +32,13 @@ export function registerStatsLibraryRoutes(
|
||||
return c.json(statsJson('vocabulary', vocab));
|
||||
});
|
||||
|
||||
app.get('/api/stats/vocabulary/summary', async (c) => {
|
||||
const summary = await tracker.getVocabularySummary(
|
||||
loadKnownWordsSet(options?.knownWordCachePath),
|
||||
);
|
||||
return c.json(statsJson('vocabularySummary', summary));
|
||||
});
|
||||
|
||||
app.get('/api/stats/excluded-words', async (c) => {
|
||||
return c.json(statsJson('excludedWords', await tracker.getStatsExcludedWords()));
|
||||
});
|
||||
|
||||
@@ -50,6 +50,16 @@ export interface StatsKnownWordsSummary {
|
||||
knownWordCount: number;
|
||||
}
|
||||
|
||||
export interface StatsVocabularySummary {
|
||||
uniqueWords: number;
|
||||
uniqueWordsWithoutNames: number;
|
||||
uniqueKanji: number;
|
||||
newThisWeek: number;
|
||||
newThisWeekWithoutNames: number;
|
||||
knownWordCount: number | null;
|
||||
knownWordCountWithoutNames: number | null;
|
||||
}
|
||||
|
||||
export interface StatsAnilistSearchResult {
|
||||
id: number;
|
||||
episodes: number | null;
|
||||
@@ -164,6 +174,7 @@ export interface StatsJsonResponseMap {
|
||||
sessionEvents: SessionEvent[];
|
||||
sessionKnownWordsTimeline: StatsSessionKnownWordsTimelinePoint[];
|
||||
vocabulary: VocabularyEntry[];
|
||||
vocabularySummary: StatsVocabularySummary;
|
||||
excludedWords: StatsExcludedWord[];
|
||||
setExcludedWords: StatsOkResponse;
|
||||
duplicateLineCleanup: StatsDuplicateLineCleanupResult;
|
||||
@@ -222,6 +233,7 @@ export interface StatsHttpClient {
|
||||
getSessionEvents: (id: number, limit?: number, eventTypes?: number[]) => Promise<SessionEvent[]>;
|
||||
getSessionKnownWordsTimeline: (id: number) => Promise<StatsSessionKnownWordsTimelinePoint[]>;
|
||||
getVocabulary: (limit?: number) => Promise<VocabularyEntry[]>;
|
||||
getVocabularySummary: () => Promise<StatsVocabularySummary>;
|
||||
getExcludedWords: () => Promise<StatsExcludedWord[]>;
|
||||
setExcludedWords: (words: StatsExcludedWord[]) => Promise<void>;
|
||||
cleanupDuplicateLines: (
|
||||
|
||||
Reference in New Issue
Block a user