diff --git a/changes/stats-vocabulary-summary-totals.md b/changes/stats-vocabulary-summary-totals.md index af8f4023..3168cd17 100644 --- a/changes/stats-vocabulary-summary-totals.md +++ b/changes/stats-vocabulary-summary-totals.md @@ -4,3 +4,4 @@ area: stats - Fixed Vocabulary totals and charts counting only the first browsing page instead of all tracked vocabulary, without delaying the rest of the page. - New-word history now uses permanent daily lexical rollups, backfilled in the background and repaired when tracked material is removed or reprocessed; playback writes queue safely during the one-time rebuild and resume afterward. - Calendar-day chart labels now preserve the recorded local date in time zones west of UTC. +- Vocabulary summary cards and charts refresh automatically after the word exclusion list changes, and failed loads retry with backoff before showing an inline error with a Retry control. diff --git a/docs-site/immersion-tracking.md b/docs-site/immersion-tracking.md index e1fb3aaa..291f4617 100644 --- a/docs-site/immersion-tracking.md +++ b/docs-site/immersion-tracking.md @@ -82,7 +82,7 @@ Expandable session history with new-word activity, cumulative totals, and pause/ #### Vocabulary -The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The word and kanji tables load first while those complete totals calculate separately. Top Repeated Words and New Words by Day use complete tracking history rather than the table's browsing page; new-word history is maintained as a permanent daily lexical rollup, including retroactive corrections when tracked material is removed or reprocessed. On the first launch after upgrading, that history is built in the background and the chart refreshes when it is ready. The rest of the tab includes cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons. +The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The word and kanji tables load first while those complete totals calculate separately. Top Repeated Words and New Words by Day use complete tracking history rather than the table's browsing page; new-word history is maintained as a permanent daily lexical rollup, including retroactive corrections when tracked material is removed or reprocessed. On the first launch after upgrading, that history is built in the background and the chart refreshes when it is ready. The cards and charts also refresh automatically after the word exclusion list changes. The rest of the tab includes cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons. ![Stats Vocabulary](/screenshots/stats-vocabulary.png) diff --git a/src/core/services/immersion-tracker-service.test.ts b/src/core/services/immersion-tracker-service.test.ts index 6e4e6a68..f2df4bd8 100644 --- a/src/core/services/immersion-tracker-service.test.ts +++ b/src/core/services/immersion-tracker-service.test.ts @@ -5052,3 +5052,58 @@ test('ensureAnimeCoverArt fetches art via the latest video of the anime', async cleanupDbPath(dbPath); } }); + +test('getVocabularySummary coalesces concurrent requests into one worker task', async () => { + const dbPath = makeDbPath(); + let tracker: ImmersionTrackerService | null = null; + let taskRuns = 0; + let releaseTask: (() => void) | null = null; + const summary = { + uniqueWords: 1, + uniqueWordsWithoutNames: 1, + uniqueKanji: 0, + newThisWeek: 0, + newThisWeekWithoutNames: 0, + knownWordCount: null, + knownWordCountWithoutNames: null, + }; + + try { + const Ctor = await loadTrackerCtor(); + tracker = new Ctor( + { dbPath }, + { + runVocabularySummaryTask: async () => { + taskRuns += 1; + await new Promise((resolve) => { + releaseTask = resolve; + }); + return summary; + }, + destroyVocabularySummaryRunner: () => {}, + }, + ); + + const first = tracker.getVocabularySummary(null); + const second = tracker.getVocabularySummary(new Set(['猫'])); + await waitForCondition(() => releaseTask !== null); + let release = releaseTask as (() => void) | null; + assert.ok(release); + release(); + assert.deepEqual(await first, summary); + assert.equal(await second, await first); + assert.equal(taskRuns, 1); + + releaseTask = null; + const third = tracker.getVocabularySummary(null); + await waitForCondition(() => releaseTask !== null); + release = releaseTask as (() => void) | null; + assert.ok(release); + release(); + assert.deepEqual(await third, summary); + assert.equal(taskRuns, 2); + } finally { + tracker?.destroy(); + cleanupDbPath(dbPath); + } +}); diff --git a/src/core/services/immersion-tracker-service.ts b/src/core/services/immersion-tracker-service.ts index 0cca853c..60d54ae1 100644 --- a/src/core/services/immersion-tracker-service.ts +++ b/src/core/services/immersion-tracker-service.ts @@ -421,6 +421,7 @@ export class ImmersionTrackerService { private readonly runVocabularySummaryTask: ( knownWords: ReadonlySet | null, ) => Promise; + private vocabularySummaryInFlight: Promise | null = null; private readonly destroyVocabularySummaryRunner: () => void; private readonly runLexicalRollupBackfillTask: () => Promise; private readonly destroyLexicalRollupBackfillRunner: () => void; @@ -680,7 +681,17 @@ export class ImmersionTrackerService { } async getVocabularySummary(knownWords: ReadonlySet | null) { - return this.runVocabularySummaryTask(knownWords); + // Concurrent dashboard refreshes share one worker scan; the coalesced + // callers accept the first caller's known-words snapshot. + const inFlight = this.vocabularySummaryInFlight; + if (inFlight) return inFlight; + const task = this.runVocabularySummaryTask(knownWords); + this.vocabularySummaryInFlight = task; + try { + return await task; + } finally { + if (this.vocabularySummaryInFlight === task) this.vocabularySummaryInFlight = null; + } } async getVocabularyChartData() { diff --git a/src/core/services/immersion-tracker/__tests__/query.test.ts b/src/core/services/immersion-tracker/__tests__/query.test.ts index 2c6b7ec7..d7fc64e5 100644 --- a/src/core/services/immersion-tracker/__tests__/query.test.ts +++ b/src/core/services/immersion-tracker/__tests__/query.test.ts @@ -1958,6 +1958,33 @@ test('getVocabularySummary applies vocabulary exclusions and Hide Names totals', } }); +test('getVocabularySummary counts identically across id-keyed scan batches', () => { + const dbPath = makeDbPath(); + const db = openTestDb(dbPath); + + try { + ensureSchema(db); + const insertWord = db.prepare(` + INSERT INTO imm_words ( + headword, word, reading, part_of_speech, pos1, pos2, pos3, + first_seen, last_seen, frequency + ) VALUES (?, ?, '', 'noun', '名詞', '一般', '', 1, 1, 1) + `); + for (let index = 0; index < 5; index += 1) { + insertWord.run(`単語${index}`, `単語${index}`); + } + + const fullScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000); + const batchedScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000, 2); + + assert.equal(fullScan.uniqueWords, 5); + assert.deepEqual(batchedScan, fullScan); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () => { const dbPath = makeDbPath(); const db = openTestDb(dbPath); diff --git a/src/core/services/immersion-tracker/query-lexical.ts b/src/core/services/immersion-tracker/query-lexical.ts index 7388a3e4..9eacb333 100644 --- a/src/core/services/immersion-tracker/query-lexical.ts +++ b/src/core/services/immersion-tracker/query-lexical.ts @@ -31,6 +31,7 @@ const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100; const VOCABULARY_CHART_LIMIT = 12; const VOCABULARY_CHART_PAGE_SIZE = 100; const EXCLUSION_ALIAS_BATCH_SIZE = 300; +const VOCABULARY_SUMMARY_SCAN_BATCH_SIZE = 5_000; const SENTENCE_SEARCH_DEFAULT_LIMIT = 50; const SENTENCE_SEARCH_MAX_LIMIT = 100; const KANJI_PATTERN = /\p{Script=Han}/gu; @@ -293,19 +294,21 @@ export function getVocabularySummary( db: DatabaseSync, knownWords: ReadonlySet | null, nowMs: number = Date.now(), + scanBatchSize: number = VOCABULARY_SUMMARY_SCAN_BATCH_SIZE, ): VocabularyStatsSummary { - const words = db - .prepare( - ` - SELECT id AS wordId, headword, word, reading, - part_of_speech AS partOfSpeech, pos1, pos2, pos3, - frequency, frequency_rank AS frequencyRank, - first_seen AS firstSeen, last_seen AS lastSeen, - 0 AS animeCount - FROM imm_words - `, - ) - .all() as VocabularyStatsRow[]; + // Visibility and exclusion rules live in JS, so rows are scanned in id-keyed + // batches to keep memory bounded on large vocabularies. + const scanStmt = db.prepare(` + SELECT id AS wordId, headword, word, reading, + part_of_speech AS partOfSpeech, pos1, pos2, pos3, + frequency, frequency_rank AS frequencyRank, + first_seen AS firstSeen, last_seen AS lastSeen, + 0 AS animeCount + FROM imm_words + WHERE id > ? + ORDER BY id + LIMIT ? + `); const excludedAliases = new Set( getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)), ); @@ -321,26 +324,33 @@ export function getVocabularySummary( knownWordCountWithoutNames: knownWords ? 0 : null, }; - for (const word of words) { - if ( - !isVocabularyStatsRowVisible(word) || - excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias)) - ) { - continue; - } - const isName = word.pos2 === '固有名詞'; - const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec; - const isKnown = knownWords?.has(word.headword) ?? false; - summary.uniqueWords += 1; - if (!isName) summary.uniqueWordsWithoutNames += 1; - if (isNewThisWeek) { - summary.newThisWeek += 1; - if (!isName) summary.newThisWeekWithoutNames += 1; - } - if (isKnown) { - summary.knownWordCount! += 1; - if (!isName) summary.knownWordCountWithoutNames! += 1; + let lastId = Number.MIN_SAFE_INTEGER; + for (;;) { + const words = scanStmt.all(lastId, scanBatchSize) as VocabularyStatsRow[]; + if (words.length === 0) break; + lastId = words[words.length - 1]!.wordId; + for (const word of words) { + if ( + !isVocabularyStatsRowVisible(word) || + excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias)) + ) { + continue; + } + const isName = word.pos2 === '固有名詞'; + const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec; + const isKnown = knownWords?.has(word.headword) ?? false; + summary.uniqueWords += 1; + if (!isName) summary.uniqueWordsWithoutNames += 1; + if (isNewThisWeek) { + summary.newThisWeek += 1; + if (!isName) summary.newThisWeekWithoutNames += 1; + } + if (isKnown) { + summary.knownWordCount! += 1; + if (!isName) summary.knownWordCountWithoutNames! += 1; + } } + if (words.length < scanBatchSize) break; } return summary; diff --git a/stats/src/components/vocabulary/VocabularyTab.tsx b/stats/src/components/vocabulary/VocabularyTab.tsx index b1fb1a96..0eefcf06 100644 --- a/stats/src/components/vocabulary/VocabularyTab.tsx +++ b/stats/src/components/vocabulary/VocabularyTab.tsx @@ -34,7 +34,18 @@ export function VocabularyTab({ onRemoveExclusion, onClearExclusions, }: VocabularyTabProps) { - const { words, kanji, knownWords, summary, charts, loading, error, reload } = useVocabulary(); + const { + words, + kanji, + knownWords, + summary, + charts, + loading, + error, + aggregatesError, + refreshAggregates, + reload, + } = useVocabulary(); const [selectedKanjiId, setSelectedKanjiId] = useState(null); const [hideNames, setHideNames] = useState(false); const [showExclusionManager, setShowExclusionManager] = useState(false); @@ -49,21 +60,21 @@ export function VocabularyTab({ }, [words, hideNames, excluded, isExcluded]); const chartData = useMemo( () => ({ - topWords: - ((hideNames ? charts?.topWordsWithoutNames : charts?.topWords) ?? []).map((word) => ({ + topWords: ((hideNames ? charts?.topWordsWithoutNames : charts?.topWords) ?? []).map( + (word) => ({ label: word.headword, value: word.frequency, - })) ?? [], - newWordsTimeline: - ((hideNames ? charts?.newWordsTimelineWithoutNames : charts?.newWordsTimeline) ?? []).map( - (point) => ({ - label: epochDayToDate(point.epochDay).toLocaleDateString(undefined, { - month: 'short', - day: 'numeric', - }), - value: point.wordCount, - }), - ) ?? [], + }), + ), + newWordsTimeline: ( + (hideNames ? charts?.newWordsTimelineWithoutNames : charts?.newWordsTimeline) ?? [] + ).map((point) => ({ + label: epochDayToDate(point.epochDay).toLocaleDateString(undefined, { + month: 'short', + day: 'numeric', + }), + value: point.wordCount, + })), }), [charts, hideNames], ); @@ -139,6 +150,19 @@ export function VocabularyTab({ /> + {aggregatesError && ( +

+ {aggregatesError}{' '} + +

+ )} +
{hasNames && (