diff --git a/changes/stats-mining-and-delete-accuracy.md b/changes/stats-mining-and-delete-accuracy.md new file mode 100644 index 00000000..72fa739f --- /dev/null +++ b/changes/stats-mining-and-delete-accuracy.md @@ -0,0 +1,7 @@ +type: fixed +area: stats + +- Fixed sentence mining (single and multi-line) counting each created card twice in session and lifetime stats. +- Fixed word and kanji dates after session deletion. Historical "first seen" dates and counts survive retention pruning; new occurrences record observation time so deletion can update dates exactly. Legacy earlier dates are preserved when the remaining history cannot prove their origin. Stale "last seen" dates are repaired once in the background on next launch. +- Fixed re-seen words that the vocabulary filter hides still being counted in the new-words charts, so charts and the vocabulary summary now agree. Existing mismatches are repaired by the same background pass. +- Fixed Overview totals, charts, activity calendar, and vocabulary summary staying stale after deleting sessions. Failed refreshes keep the dashboard visible and offer a retry. diff --git a/changes/stats-overview-refresh-docs.md b/changes/stats-overview-refresh-docs.md new file mode 100644 index 00000000..f03b9bd5 --- /dev/null +++ b/changes/stats-overview-refresh-docs.md @@ -0,0 +1,5 @@ +type: docs +area: stats + +- Documented the automatic Overview refresh after deleting sessions or session groups and retrying failed refreshes. +- Documented discovery-date preservation after retention pruning and the handling of older write-time timestamps. diff --git a/docs-site/immersion-tracking.md b/docs-site/immersion-tracking.md index 5aa0624a..ad9c117b 100644 --- a/docs-site/immersion-tracking.md +++ b/docs-site/immersion-tracking.md @@ -42,6 +42,8 @@ To share stats and watch history between machines, use [`subminer sync `]( Recent sessions, a streak calendar, watch-time history, and totals for completed episodes and shows. +Deleting a session or session group refreshes the Overview totals, charts, calendar, and vocabulary summary automatically. If a refresh fails, Overview keeps the previous totals visible and offers **Retry**. + ![Stats Overview](/screenshots/stats-overview.png) ### Library @@ -122,4 +124,6 @@ Leave out `--lookback-days` to scan all history. Word and kanji counts are corre By default SubMiner keeps everything. To limit history, set `immersionTracking.retentionPreset` to `minimal`, `balanced`, or `deep-history`, or set the `immersionTracking.retention.*Days` values yourself (`0` keeps all). Lifetime totals and vocabulary counts are stored separately and stay exact when old sessions are pruned. +Deleting retained sessions preserves earlier word and kanji discovery dates from pruned history. New occurrences record observation time so deletion can update their dates exactly. Older history used write time, so an earlier discovery date is preserved when its origin cannot be determined. + See [Immersion tracking](/configuration#immersion-tracking) and [Stats dashboard](/configuration#stats-dashboard) in the config reference for every option and default. diff --git a/src/core/services/immersion-tracker/__tests__/lexical-removal.test.ts b/src/core/services/immersion-tracker/__tests__/lexical-removal.test.ts index 50e1ec8c..95d40c6a 100644 --- a/src/core/services/immersion-tracker/__tests__/lexical-removal.test.ts +++ b/src/core/services/immersion-tracker/__tests__/lexical-removal.test.ts @@ -5,8 +5,9 @@ import path from 'node:path'; import test from 'node:test'; import { Database } from '../sqlite.js'; import type { DatabaseSync } from '../sqlite.js'; -import { ensureSchema } from '../storage.js'; +import { createTrackerPreparedStatements, ensureSchema, executeQueuedWrite } from '../storage.js'; import { deleteSession, deleteSessions, deleteVideo } from '../query-maintenance.js'; +import { pruneRawRetention } from '../maintenance.js'; const DAY_MS = 86_400_000; const BASE_MS = 1_700_000_000_000; @@ -30,7 +31,13 @@ function cleanupDbPath(dbPath: string): void { */ function seed( db: DatabaseSync, - lines: Array<{ session: 1 | 2; wordId: number; dayOffset: number; count?: number }>, + lines: Array<{ + session: 1 | 2; + wordId: number; + dayOffset: number; + msOffset?: number; + count?: number; + }>, options: { legacyRows?: boolean } = {}, ): void { db.exec(` @@ -65,7 +72,7 @@ function seed( let lineId = 0; for (const line of lines) { lineId += 1; - const seenMs = BASE_MS + line.dayOffset * DAY_MS; + const seenMs = BASE_MS + line.dayOffset * DAY_MS + (line.msOffset ?? 0); insertLine.run(lineId, line.session, line.session, lineId, `line ${lineId}`, seenMs, seenMs); insertWord.run(line.wordId, `語${line.wordId}`, `語${line.wordId}`); if (options.legacyRows) { @@ -83,13 +90,13 @@ function seed( FROM imm_word_line_occurrences o WHERE o.word_id = imm_words.id ), first_seen = ( - SELECT MIN(sl.CREATED_DATE) / 1000 + SELECT MIN(sl.CREATED_DATE) / 1000.0 FROM imm_word_line_occurrences o JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id WHERE o.word_id = imm_words.id ), last_seen = ( - SELECT MAX(sl.LAST_UPDATE_DATE) / 1000 + SELECT MAX(sl.LAST_UPDATE_DATE) / 1000.0 FROM imm_word_line_occurrences o JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id WHERE o.word_id = imm_words.id @@ -200,6 +207,167 @@ test('deleting the latest session moves last_seen back to the surviving line', ( } }); +for (const legacyRows of [false, true]) { + test(`deleting the latest session keeps fractional seen timestamps exact (legacyRows=${legacyRows})`, () => { + // Stored seen values are fractional seconds; whole-second rounding used to miss + // the removed extreme (or truncate the refreshed one). + const { db, dbPath } = createDb( + [ + { session: 1, wordId: 22, dayOffset: 0, msOffset: 251 }, + { session: 2, wordId: 22, dayOffset: 5, msOffset: 882 }, + ], + { legacyRows }, + ); + + try { + deleteSession(db, 2); + + const word = readWord(db, 22); + assert.equal(word?.frequency, 1); + assert.equal(word?.firstSeen, (BASE_MS + 251) / 1000); + assert.equal(word?.lastSeen, (BASE_MS + 251) / 1000); + } finally { + db.close(); + cleanupDbPath(dbPath); + } + }); +} + +test('deleting legacy history preserves first_seen when it predates the flush timestamp', () => { + const { db, dbPath } = createDb([ + { session: 1, wordId: 23, dayOffset: 0, msOffset: 400 }, + { session: 2, wordId: 23, dayOffset: 5, msOffset: 400 }, + ]); + + try { + // A queue delay and retention-pruned history cannot be distinguished from + // the old flush-time timestamp. Preserve the earlier date in either case. + db.prepare('UPDATE imm_words SET first_seen = ? WHERE id = 23').run(BASE_MS / 1000); + + deleteSession(db, 1); + + assert.equal(readWord(db, 23)?.firstSeen, BASE_MS / 1000); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + +for (const legacyRows of [false, true]) { + for (const deletedSession of [2, 3]) { + test(`deleting retained session ${deletedSession} preserves pruned first_seen (legacyRows=${legacyRows})`, () => { + const { db, dbPath } = createDb( + [ + { session: 1, wordId: 24, dayOffset: 0 }, + { session: 2, wordId: 24, dayOffset: 1 }, + ], + { legacyRows }, + ); + + try { + const latestMs = BASE_MS + 2 * DAY_MS; + db.exec(` + INSERT INTO imm_sessions(session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status, CREATED_DATE, LAST_UPDATE_DATE) + VALUES (3, 's3', 2, '${latestMs}', '${latestMs + 1000}', 2, ${latestMs}, ${latestMs}); + INSERT INTO imm_subtitle_lines(line_id, session_id, video_id, line_index, text, CREATED_DATE, LAST_UPDATE_DATE) + VALUES (3, 3, 2, 3, '猫', ${latestMs}, ${latestMs}); + INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms) + VALUES (3, 24, 1, ${legacyRows ? 'NULL' : latestMs}); + UPDATE imm_words SET frequency = 3, last_seen = ${latestMs / 1000} WHERE id = 24; + INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency) + SELECT 24, '猫', first_seen, last_seen, frequency FROM imm_words WHERE id = 24; + INSERT INTO imm_kanji_line_occurrences(line_id, kanji_id, occurrence_count, seen_ms) + SELECT line_id, 24, occurrence_count, seen_ms FROM imm_word_line_occurrences WHERE word_id = 24; + `); + pruneRawRetention(db, latestMs, { + eventsRetentionMs: Infinity, + telemetryRetentionMs: Infinity, + sessionsRetentionMs: 1.5 * DAY_MS, + }); + + deleteSession(db, deletedSession); + + for (const table of ['imm_words', 'imm_kanji']) { + const row = db + .prepare( + `SELECT frequency, first_seen AS firstSeen, last_seen AS lastSeen FROM ${table} WHERE id = 24`, + ) + .get() as { frequency: number; firstSeen: number; lastSeen: number }; + assert.equal(row.frequency, 2); + assert.equal(row.firstSeen, BASE_MS / 1000, `${table} keeps the pruned discovery date`); + assert.equal(row.lastSeen, (BASE_MS + (deletedSession === 2 ? 2 : 1) * DAY_MS) / 1000); + } + } finally { + db.close(); + cleanupDbPath(dbPath); + } + }); + } +} + +test('deleting a queued subtitle uses observation time despite a delayed flush', () => { + const { db, dbPath } = createDb([]); + + try { + const stmts = createTrackerPreparedStatements(db); + for (const sessionId of [1, 2]) { + const seenMs = BASE_MS + (sessionId - 1) * DAY_MS + 400; + executeQueuedWrite( + { + kind: 'subtitleLine', + sessionId, + videoId: sessionId, + lineIndex: 1, + segmentStartMs: null, + segmentEndMs: null, + text: '猫', + wordOccurrences: [ + { + headword: '猫', + word: '猫', + reading: 'ねこ', + partOfSpeech: 'noun', + pos1: '名詞', + pos2: '一般', + pos3: '', + occurrenceCount: 1, + frequencyRank: null, + }, + ], + kanjiOccurrences: [{ kanji: '猫', occurrenceCount: 1 }], + firstSeen: seenMs / 1000, + lastSeen: seenMs / 1000, + }, + stmts, + ); + for (const table of ['imm_word_line_occurrences', 'imm_kanji_line_occurrences']) { + const row = db + .prepare(`SELECT seen_ms AS seenMs FROM ${table} WHERE line_id = ?`) + .get(sessionId) as { seenMs: number }; + assert.equal(row.seenMs, seenMs); + } + } + + deleteSession(db, 1); + + for (const table of ['imm_words', 'imm_kanji']) { + const row = db + .prepare(`SELECT frequency, first_seen AS firstSeen, last_seen AS lastSeen FROM ${table}`) + .get() as { + frequency: number; + firstSeen: number; + lastSeen: number; + }; + assert.equal(row.frequency, 1); + assert.equal(row.firstSeen, (BASE_MS + DAY_MS + 400) / 1000); + assert.equal(row.lastSeen, (BASE_MS + DAY_MS + 400) / 1000); + } + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('deleting an interior occurrence leaves the surrounding extremes untouched', () => { // Session 2 carries the middle occurrence; sessions bracket it in time. const { db, dbPath } = createDb([ diff --git a/src/core/services/immersion-tracker/__tests__/query.test.ts b/src/core/services/immersion-tracker/__tests__/query.test.ts index d7fc64e5..36b9c3fe 100644 --- a/src/core/services/immersion-tracker/__tests__/query.test.ts +++ b/src/core/services/immersion-tracker/__tests__/query.test.ts @@ -4567,13 +4567,13 @@ test('deleteSession rebuilds word and kanji aggregates from retained subtitle li assert.ok(sharedWordRow); assert.equal(sharedWordRow.frequency, 1); - assert.equal(sharedWordRow.first_seen, Math.floor(keptTs / 1000)); - assert.equal(sharedWordRow.last_seen, Math.floor(keptTs / 1000)); + assert.equal(sharedWordRow.first_seen, keptTs / 1000); + assert.equal(sharedWordRow.last_seen, keptTs / 1000); assert.equal(deletedOnlyWordRow ?? null, null); assert.ok(sharedKanjiRow); assert.equal(sharedKanjiRow.frequency, 1); - assert.equal(sharedKanjiRow.first_seen, Math.floor(keptTs / 1000)); - assert.equal(sharedKanjiRow.last_seen, Math.floor(keptTs / 1000)); + assert.equal(sharedKanjiRow.first_seen, keptTs / 1000); + assert.equal(sharedKanjiRow.last_seen, keptTs / 1000); assert.equal(deletedOnlyKanjiRow ?? null, null); } finally { db.close(); diff --git a/src/core/services/immersion-tracker/lexical-rollups.test.ts b/src/core/services/immersion-tracker/lexical-rollups.test.ts index 60ce4daf..897b7dfc 100644 --- a/src/core/services/immersion-tracker/lexical-rollups.test.ts +++ b/src/core/services/immersion-tracker/lexical-rollups.test.ts @@ -16,7 +16,8 @@ import { } from './query-lexical'; import { Database } from './sqlite'; import type { DatabaseSync } from './sqlite'; -import { ensureSchema } from './storage'; +import { createTrackerPreparedStatements, ensureSchema, executeQueuedWrite } from './storage'; +import { BASE_MS, createDb, seedEndedSession, seedVideo } from './__tests__/lifetime-test-fixtures'; function makeDbPath(): string { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollups-')); @@ -125,6 +126,121 @@ test('lexical rollup rebuild excludes rows hidden by vocabulary persistence rule } }); +test('re-seeing a filtered legacy word keeps charts consistent with the summary', () => { + const db = createDb(); + + try { + const videoId = seedVideo(db, null, 'episode'); + const sessionId = seedEndedSession(db, videoId, BASE_MS, { activeMs: 1_000 }); + const firstSeen = BASE_MS / 1000; + // Legacy row whose stored merged-token POS fails the persistence filter. + db.prepare( + `INSERT INTO imm_words( + headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency + ) VALUES ('二死', 'にし', 'にし', 'other', '助詞|動詞', '格助詞|自立', '一般|*', ?, ?, 1)`, + ).run(firstSeen, firstSeen); + rebuildLexicalDailyRollups(db); + + // The incoming token passes the filter, but the upsert keeps the stored POS. + executeQueuedWrite( + { + kind: 'subtitleLine', + sessionId, + videoId, + lineIndex: 1, + segmentStartMs: null, + segmentEndMs: null, + text: 'にし', + wordOccurrences: [ + { + headword: '二死', + word: 'にし', + reading: 'にし', + partOfSpeech: 'verb', + pos1: '動詞', + pos2: '自立', + pos3: '*', + frequencyRank: null, + occurrenceCount: 1, + }, + ], + kanjiOccurrences: [], + firstSeen: firstSeen + 60, + lastSeen: firstSeen + 60, + }, + createTrackerPreparedStatements(db), + ); + + const plotted = getVocabularyChartData(db).newWordsTimeline.reduce( + (sum, day) => sum + day.wordCount, + 0, + ); + assert.equal(getVocabularySummary(db, null).uniqueWords, 0); + assert.equal(plotted, 0); + } finally { + db.close(); + } +}); + +test('lexical rollup rebuild repairs last_seen left pointing at deleted history', () => { + const db = createDb(); + + try { + const videoId = seedVideo(db, null, 'episode'); + const sessionId = seedEndedSession(db, videoId, BASE_MS, { activeMs: 1_000 }); + const lineId = Number( + db + .prepare( + `INSERT INTO imm_subtitle_lines(session_id, video_id, line_index, text, CREATED_DATE, LAST_UPDATE_DATE) + VALUES (?, ?, 1, '猫', ?, ?)`, + ) + .run(sessionId, videoId, BASE_MS, BASE_MS).lastInsertRowid, + ); + const seen = (BASE_MS + 251) / 1000; + // first_seen this early may be real history pruned by retention; never touched. + const earlyFirst = seen - 86_400.5; + const staleLast = seen + 86_400.631; + db.prepare( + `INSERT INTO imm_words(id, headword, word, reading, first_seen, last_seen, frequency) + VALUES (1, '猫', '猫', 'ねこ', ?, ?, 1), (2, '犬', '犬', 'いぬ', ?, ?, 1)`, + ).run(earlyFirst, staleLast, seen - 0.1, seen - 0.1); + db.prepare( + `INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency) VALUES (1, '猫', ?, ?, 1)`, + ).run(earlyFirst, staleLast); + db.prepare( + `INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms) + VALUES (?, 1, 1, ?), (?, 2, 1, ?)`, + ).run(lineId, BASE_MS + 251, lineId, BASE_MS + 251); + db.prepare( + `INSERT INTO imm_kanji_line_occurrences(line_id, kanji_id, occurrence_count, seen_ms) + VALUES (?, 1, 1, ?)`, + ).run(lineId, BASE_MS + 251); + + rebuildLexicalDailyRollups(db); + + const bounds = (table: string): Array<[number, number]> => + ( + db + .prepare(`SELECT first_seen AS first, last_seen AS last FROM ${table} ORDER BY id`) + .all() as Array<{ + first: number; + last: number; + }> + ).map((row) => [row.first, row.last]); + assert.deepEqual( + bounds('imm_words'), + [ + [earlyFirst, seen], + [seen - 0.1, seen - 0.1], + ], + 'healthy rows are untouched', + ); + assert.deepEqual(bounds('imm_kanji'), [[earlyFirst, seen]]); + } finally { + db.close(); + } +}); + test('lexical rollup rebuild tolerates nullable legacy vocabulary text', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -268,7 +384,7 @@ test('current lexical rollup readiness accepts legacy integer state storage', () state_value INTEGER NOT NULL ); INSERT INTO imm_rollup_state(state_key, state_value) - VALUES ('lexical_daily_rollups_version', 2); + VALUES ('lexical_daily_rollups_version', 3); `); assert.equal(areLexicalDailyRollupsReady(db), true); diff --git a/src/core/services/immersion-tracker/lexical-rollups.ts b/src/core/services/immersion-tracker/lexical-rollups.ts index ea5480a9..f00b926d 100644 --- a/src/core/services/immersion-tracker/lexical-rollups.ts +++ b/src/core/services/immersion-tracker/lexical-rollups.ts @@ -18,7 +18,9 @@ const LOCAL_EPOCH_DAY_SQL = ` ) - 2440587.5 AS INTEGER) `; -const LEXICAL_DAILY_ROLLUP_VERSION = '2'; +// v3 re-derives visibility for rows a re-seen word forced visible and repairs +// last_seen values left behind by session deletes. +const LEXICAL_DAILY_ROLLUP_VERSION = '3'; const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version'; const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000; @@ -26,6 +28,32 @@ export function localEpochDaySql(value: string): string { return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value); } +// Session deletes once rounded removed timestamps to whole seconds, missing +// fractional last_seen extremes that pointed at deleted history. New seen_ms +// values match observation time; legacy values were stamped later at flush. +// Retention prunes the oldest sessions first, so healthy rows never exceed +// their newest surviving occurrence. Rows with undated occurrences are skipped. +// +// first_seen is deliberately not repaired: a stale bound is indistinguishable +// from real history whose occurrences retention pruned, and frequency does not +// reliably tell them apart. +function repairStaleLastSeen(db: DatabaseSync): void { + for (const [table, occurrenceTable, col] of [ + ['imm_words', 'imm_word_line_occurrences', 'word_id'], + ['imm_kanji', 'imm_kanji_line_occurrences', 'kanji_id'], + ] as const) { + const newestSeen = `(SELECT MAX(o.seen_ms) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id) / 1000.0`; + db.exec(` + UPDATE ${table} + SET last_seen = ${newestSeen} + WHERE last_seen > ${newestSeen} + AND NOT EXISTS ( + SELECT 1 FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id AND o.seen_ms IS NULL + ) + `); + } +} + function createWordRollupTriggers(db: DatabaseSync): void { const dayForNew = localEpochDaySql('NEW.first_seen'); const dayForOld = localEpochDaySql('OLD.first_seen'); @@ -186,6 +214,7 @@ export function rebuildLexicalDailyRollups(db: DatabaseSync): void { lastId = vocabularyRows[vocabularyRows.length - 1]!.id; if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break; } + repairStaleLastSeen(db); db.exec('DELETE FROM imm_lexical_daily_rollups'); db.exec(` INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) diff --git a/src/core/services/immersion-tracker/query-shared.ts b/src/core/services/immersion-tracker/query-shared.ts index 5d09a19b..a445bab8 100644 --- a/src/core/services/immersion-tracker/query-shared.ts +++ b/src/core/services/immersion-tracker/query-shared.ts @@ -298,9 +298,11 @@ export function planLexicalRemovalsForVideos( return planLexicalRemovals(db, `sl.video_id IN (${makePlaceholders(videoIds)})`, videoIds); } +// first_seen/last_seen are REAL seconds that keep millisecond fractions; never round +// them, or comparisons against stored extremes miss and refreshes drift. function toStoredSeenSeconds(ms: number | null): number | null { if (ms === null || !Number.isFinite(ms)) return null; - return Math.floor(ms / 1000); + return ms / 1000; } /** @@ -320,6 +322,13 @@ export function applyLexicalRemovals(db: DatabaseSync, plan: LexicalRemovalPlan) applyRemovalsForEntity(db, 'kanji', plan.kanji); } +// A timestamp repair must keep subtraction-exact lifetime counts and any +// first_seen date that the deleted occurrences cannot account for. +type LexicalAggregateOverride = { + frequency: number | null; + firstSeen: number | null; +}; + function applyRemovalsForEntity( db: DatabaseSync, entity: LexicalEntity, @@ -358,7 +367,7 @@ function applyRemovalsForEntity( `UPDATE ${entityTable} SET frequency = ?, first_seen = ?, last_seen = ? WHERE id = ?`, ); - const needsExactRefresh: number[] = []; + const needsExactRefresh = new Map(); for (const removal of removals) { const current = selectStmt.get(removal.id) as { @@ -369,17 +378,9 @@ function applyRemovalsForEntity( if (!current) continue; const nextFrequency = (current.frequency ?? 0) - removal.removedFrequency; - if (nextFrequency <= 0) { - // The rows in scope are already gone by now, so anything still pointing at - // this entity means the stored frequency was stale rather than exhausted. - if (hasOccurrencesStmt.get(removal.id)) { - needsExactRefresh.push(removal.id); - } else { - deleteStmt.run(removal.id); - } - continue; - } - + // Legacy occurrences use flush time, which can follow the actual observation. + // An earlier stored date may also belong to retention-pruned history. Only + // move it when the removed timestamps prove that its observation was deleted. const removedFirstSeen = toStoredSeenSeconds(removal.removedFirstSeenMs); const removedLastSeen = toStoredSeenSeconds(removal.removedLastSeenMs); const firstSeenMayHaveMoved = @@ -388,11 +389,26 @@ function applyRemovalsForEntity( const lastSeenMayHaveMoved = current.lastSeen === null || (removedLastSeen !== null && removedLastSeen >= current.lastSeen); + const exactRefreshOverride = { + frequency: nextFrequency > 0 ? nextFrequency : null, + firstSeen: firstSeenMayHaveMoved ? null : current.firstSeen, + }; + if (nextFrequency <= 0) { + // The rows in scope are already gone by now, so anything still pointing at + // this entity means the stored frequency was stale rather than exhausted. + if (hasOccurrencesStmt.get(removal.id)) { + needsExactRefresh.set(removal.id, exactRefreshOverride); + } else { + deleteStmt.run(removal.id); + } + continue; + } + if (firstSeenMayHaveMoved || lastSeenMayHaveMoved) { // Undated pre-migration occurrences are invisible to the seeks below; - // fall back to the full re-aggregate that resolves their dates. + // resolve their dates without discarding retention-pruned aggregates. if (hasUndatedOccurrenceStmt.get(removal.id)) { - needsExactRefresh.push(removal.id); + needsExactRefresh.set(removal.id, exactRefreshOverride); continue; } const minSeenMs = (minSeenStmt.get(removal.id) as { value: number | null }).value; @@ -400,13 +416,13 @@ function applyRemovalsForEntity( if (minSeenMs === null || maxSeenMs === null) { // Frequency says occurrences remain but none exist: stale row, let the // exact refresh reconcile (it deletes rows with nothing left). - needsExactRefresh.push(removal.id); + needsExactRefresh.set(removal.id, exactRefreshOverride); continue; } updateAggregatesStmt.run( nextFrequency, - Math.floor(Number(minSeenMs) / 1000), - Math.floor(Number(maxSeenMs) / 1000), + firstSeenMayHaveMoved ? toStoredSeenSeconds(Number(minSeenMs)) : current.firstSeen, + toStoredSeenSeconds(Number(maxSeenMs)), removal.id, ); continue; @@ -416,13 +432,17 @@ function applyRemovalsForEntity( } if (entity === 'word') { - refreshWordAggregates(db, needsExactRefresh); + refreshWordAggregates(db, [...needsExactRefresh.keys()], needsExactRefresh); } else { - refreshKanjiAggregates(db, needsExactRefresh); + refreshKanjiAggregates(db, [...needsExactRefresh.keys()], needsExactRefresh); } } -function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void { +function refreshWordAggregates( + db: DatabaseSync, + wordIds: number[], + overrides?: ReadonlyMap, +): void { if (wordIds.length === 0) { return; } @@ -470,15 +490,19 @@ function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void { continue; } updateStmt.run( - row.frequency, - Math.floor(row.firstSeen / 1000), - Math.floor(row.lastSeen / 1000), + overrides?.get(row.wordId)?.frequency ?? row.frequency, + overrides?.get(row.wordId)?.firstSeen ?? toStoredSeenSeconds(row.firstSeen), + toStoredSeenSeconds(row.lastSeen), row.wordId, ); } } -function refreshKanjiAggregates(db: DatabaseSync, kanjiIds: number[]): void { +function refreshKanjiAggregates( + db: DatabaseSync, + kanjiIds: number[], + overrides?: ReadonlyMap, +): void { if (kanjiIds.length === 0) { return; } @@ -526,9 +550,9 @@ function refreshKanjiAggregates(db: DatabaseSync, kanjiIds: number[]): void { continue; } updateStmt.run( - row.frequency, - Math.floor(row.firstSeen / 1000), - Math.floor(row.lastSeen / 1000), + overrides?.get(row.kanjiId)?.frequency ?? row.frequency, + overrides?.get(row.kanjiId)?.firstSeen ?? toStoredSeenSeconds(row.firstSeen), + toStoredSeenSeconds(row.lastSeen), row.kanjiId, ); } diff --git a/src/core/services/immersion-tracker/storage.ts b/src/core/services/immersion-tracker/storage.ts index 533ec88d..4651731d 100644 --- a/src/core/services/immersion-tracker/storage.ts +++ b/src/core/services/immersion-tracker/storage.ts @@ -6,6 +6,7 @@ import { normalizeTitleIdentity } from '../../utils/title-normalization'; import type { DatabaseSync } from './sqlite'; import { nowMs } from './time'; import { ensureLexicalDailyRollupTables, markLexicalDailyRollupsReady } from './lexical-rollups'; +import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility'; import { SCHEMA_VERSION } from './types'; import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types'; import { toDbMs, toDbTimestamp } from './query-shared'; @@ -17,7 +18,8 @@ export interface TrackerPreparedStatements { wordUpsertStmt: ReturnType; kanjiUpsertStmt: ReturnType; subtitleLineInsertStmt: ReturnType; - wordIdSelectStmt: ReturnType; + wordSelectStmt: ReturnType; + wordVisibilityUpdateStmt: ReturnType; kanjiIdSelectStmt: ReturnType; wordLineOccurrenceUpsertStmt: ReturnType; kanjiLineOccurrenceUpsertStmt: ReturnType; @@ -1759,7 +1761,6 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar pos1 = COALESCE(NULLIF(imm_words.pos1, ''), excluded.pos1), pos2 = COALESCE(NULLIF(imm_words.pos2, ''), excluded.pos2), pos3 = COALESCE(NULLIF(imm_words.pos3, ''), excluded.pos3), - vocabulary_visible = 1, first_seen = MIN(COALESCE(first_seen, excluded.first_seen), excluded.first_seen), last_seen = MAX(COALESCE(last_seen, excluded.last_seen), excluded.last_seen), frequency_rank = CASE @@ -1787,10 +1788,16 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) `), - wordIdSelectStmt: db.prepare(` - SELECT id FROM imm_words + wordSelectStmt: db.prepare(` + SELECT id, word, headword, reading, part_of_speech AS partOfSpeech, + pos1, pos2, pos3, frequency_rank AS frequencyRank, + vocabulary_visible AS vocabularyVisible + FROM imm_words WHERE headword = ? AND word = ? AND reading = ? `), + wordVisibilityUpdateStmt: db.prepare(` + UPDATE imm_words SET vocabulary_visible = ? WHERE id = ? + `), kanjiIdSelectStmt: db.prepare(` SELECT id FROM imm_kanji WHERE kanji = ? @@ -1842,14 +1849,18 @@ function incrementWordAggregate( occurrence.frequencyRank ?? null, ); } - const row = stmts.wordIdSelectStmt.get( - occurrence.headword, - occurrence.word, - occurrence.reading, - ) as { id: number } | null; + const row = stmts.wordSelectStmt.get(occurrence.headword, occurrence.word, occurrence.reading) as + | (VocabularyVisibilityRow & { id: number; vocabularyVisible: number }) + | null; if (!row?.id) { throw new Error(`Failed to resolve imm_words id for ${occurrence.headword}`); } + // The upsert keeps the first-seen POS metadata, so derive visibility from the + // stored row (as the vocabulary summary does) rather than the incoming token. + const visible = isVocabularyStatsRowVisible(row) ? 1 : 0; + if (row.vocabularyVisible !== visible) { + stmts.wordVisibilityUpdateStmt.run(visible, row.id); + } return row.id; } @@ -1952,18 +1963,16 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta currentMs, ); const lineId = Number(lineResult.lastInsertRowid); + // Match aggregate bounds to the observation, even if this write was queued + // for a while. Flush time cannot identify which occurrence set first_seen. + const seenMs = toDbTimestamp(Math.round(write.firstSeen * 1000)); for (const occurrence of write.wordOccurrences) { const wordId = incrementWordAggregate(stmts, occurrence, write.firstSeen, write.lastSeen); - stmts.wordLineOccurrenceUpsertStmt.run(lineId, wordId, occurrence.occurrenceCount, currentMs); + stmts.wordLineOccurrenceUpsertStmt.run(lineId, wordId, occurrence.occurrenceCount, seenMs); } for (const occurrence of write.kanjiOccurrences) { const kanjiId = incrementKanjiAggregate(stmts, occurrence, write.firstSeen, write.lastSeen); - stmts.kanjiLineOccurrenceUpsertStmt.run( - lineId, - kanjiId, - occurrence.occurrenceCount, - currentMs, - ); + stmts.kanjiLineOccurrenceUpsertStmt.run(lineId, kanjiId, occurrence.occurrenceCount, seenMs); } return; } diff --git a/src/core/services/mining.test.ts b/src/core/services/mining.test.ts index 5e6ad438..772f1f94 100644 --- a/src/core/services/mining.test.ts +++ b/src/core/services/mining.test.ts @@ -276,7 +276,6 @@ test('handleMultiCopyDigit copies backward from the current subtitle after a bac test('handleMineSentenceDigit reports async create failures', async () => { const osd: string[] = []; const logs: Array<{ message: string; err: unknown }> = []; - let cardsMined = 0; handleMineSentenceDigit(2, { subtitleTimingTracker: { @@ -296,9 +295,6 @@ test('handleMineSentenceDigit reports async create failures', async () => { getCurrentSecondarySubText: () => 'sub2', showMpvOsd: (text) => osd.push(text), logError: (message, err) => logs.push({ message, err }), - onCardsMined: (count) => { - cardsMined += count; - }, }); await new Promise((resolve) => setImmediate(resolve)); @@ -307,37 +303,6 @@ test('handleMineSentenceDigit reports async create failures', async () => { assert.equal(logs[0]?.message, 'mineSentenceMultiple failed:'); assert.equal((logs[0]?.err as Error).message, 'mine boom'); assert.ok(osd.some((entry) => entry.includes('Mine sentence failed: mine boom'))); - assert.equal(cardsMined, 0); -}); - -test('handleMineSentenceDigit increments successful card count', async () => { - const osd: string[] = []; - let cardsMined = 0; - - handleMineSentenceDigit(2, { - subtitleTimingTracker: { - getRecentBlocks: () => ['one', 'two'], - getCurrentSubtitle: () => null, - findTiming: (text) => - text === 'one' ? { startTime: 1, endTime: 3 } : { startTime: 4, endTime: 7 }, - }, - ankiIntegration: { - updateLastAddedFromClipboard: async () => {}, - triggerFieldGroupingForLastAddedCard: async () => {}, - markLastCardAsAudioCard: async () => {}, - createSentenceCard: async () => true, - }, - getCurrentSecondarySubText: () => 'sub2', - showMpvOsd: (text) => osd.push(text), - logError: () => {}, - onCardsMined: (count) => { - cardsMined += count; - }, - }); - - await new Promise((resolve) => setImmediate(resolve)); - - assert.equal(cardsMined, 1); }); test('handleMineSentenceDigit keeps per-entry timings when subtitle text repeats', async () => { diff --git a/src/core/services/mining.ts b/src/core/services/mining.ts index 62b199e5..b6aa59e7 100644 --- a/src/core/services/mining.ts +++ b/src/core/services/mining.ts @@ -201,7 +201,6 @@ export function handleMineSentenceDigit( getCurrentSecondarySubText: () => string | undefined; showMpvOsd: (text: string) => void; logError: (message: string, err: unknown) => void; - onCardsMined?: (count: number) => void; }, ): void { if (!deps.subtitleTimingTracker || !deps.ankiIntegration) return; @@ -233,14 +232,9 @@ export function handleMineSentenceDigit( entries, deps.getCurrentSecondarySubText, ); - const cardsToMine = 1; + // Card creation records the mined card (with its note id) itself. deps.ankiIntegration .createSentenceCard(sentence, rangeStart, rangeEnd, secondarySubText) - .then((created) => { - if (created) { - deps.onCardsMined?.(cardsToMine); - } - }) .catch((err) => { deps.logError('mineSentenceMultiple failed:', err); deps.showMpvOsd(`Mine sentence failed: ${(err as Error).message}`); diff --git a/src/main.ts b/src/main.ts index bdd93afe..e288e2bc 100644 --- a/src/main.ts +++ b/src/main.ts @@ -5497,10 +5497,6 @@ const buildMineSentenceCardMainDepsHandler = createBuildMineSentenceCardMainDeps getPrimarySubtitle: () => captureCurrentPrimarySubtitleMiningContext(), showMpvOsd: (text) => overlayNotificationsRuntime.showConfiguredStatusNotification(text), mineSentenceCardCore, - recordCardsMined: (count, noteIds) => { - ensureImmersionTrackerStarted(); - appState.immersionTracker?.recordCardsMined(count, noteIds); - }, }); const mineSentenceCardHandler = createMineSentenceCardHandler( buildMineSentenceCardMainDepsHandler(), @@ -5533,10 +5529,6 @@ const buildHandleMineSentenceDigitMainDepsHandler = logError: (message, err) => { logger.error(message, err); }, - onCardsMined: (cards) => { - ensureImmersionTrackerStarted(); - appState.immersionTracker?.recordCardsMined(cards); - }, handleMineSentenceDigitCore, }); const handleMineSentenceDigitMainDeps = buildHandleMineSentenceDigitMainDepsHandler(); diff --git a/src/main/runtime/anki-actions-main-deps.test.ts b/src/main/runtime/anki-actions-main-deps.test.ts index d436f022..a75489ad 100644 --- a/src/main/runtime/anki-actions-main-deps.test.ts +++ b/src/main/runtime/anki-actions-main-deps.test.ts @@ -67,7 +67,6 @@ test('anki action main deps builders map callbacks', async () => { getPrimarySubtitle: () => ({ text: '正式な字幕', startTime: 1, endTime: 3 }), showMpvOsd: (text) => calls.push(`mine:${text}`), mineSentenceCardCore: async () => true, - recordCardsMined: (count) => calls.push(`cards:${count}`), })(); assert.deepEqual(mine.getMpvClient(), { connected: true }); assert.deepEqual(mine.getPrimarySubtitle?.(), { @@ -81,16 +80,6 @@ test('anki action main deps builders map callbacks', async () => { mpvClient: { connected: true }, showMpvOsd: () => {}, }); - mine.recordCardsMined(1); - assert.deepEqual(calls, [ - 'osd:x', - 'update', - 'fg:fg', - 'trigger', - 'audio:a', - 'mark', - 'mine:m', - 'cards:1', - ]); + assert.deepEqual(calls, ['osd:x', 'update', 'fg:fg', 'trigger', 'audio:a', 'mark', 'mine:m']); }); diff --git a/src/main/runtime/anki-actions-main-deps.ts b/src/main/runtime/anki-actions-main-deps.ts index 5bc8c35b..948bc40e 100644 --- a/src/main/runtime/anki-actions-main-deps.ts +++ b/src/main/runtime/anki-actions-main-deps.ts @@ -80,7 +80,6 @@ export function createBuildMineSentenceCardMainDepsHandler(deps: { primarySubtitle?: PrimarySubtitle; showMpvOsd: (text: string) => void; }) => Promise; - recordCardsMined: (count: number, noteIds?: number[]) => void; }) { return () => ({ getAnkiIntegration: () => deps.getAnkiIntegration(), @@ -95,6 +94,5 @@ export function createBuildMineSentenceCardMainDepsHandler(deps: { primarySubtitle?: PrimarySubtitle; showMpvOsd: (text: string) => void; }) => deps.mineSentenceCardCore(options), - recordCardsMined: (count: number, noteIds?: number[]) => deps.recordCardsMined(count, noteIds), }); } diff --git a/src/main/runtime/anki-actions.test.ts b/src/main/runtime/anki-actions.test.ts index ff40e22d..2be35a88 100644 --- a/src/main/runtime/anki-actions.test.ts +++ b/src/main/runtime/anki-actions.test.ts @@ -63,7 +63,7 @@ test('trigger and mark handlers delegate to core services', async () => { assert.deepEqual(calls, ['osd:group', 'group-core', 'osd:mark', 'mark-core']); }); -test('mine sentence handler records mined cards only when core returns true', async () => { +test('mine sentence handler leaves card counting to card creation', async () => { const calls: string[] = []; const integration = {}; const mpvClient = {}; @@ -78,14 +78,13 @@ test('mine sentence handler records mined cards only when core returns true', as options.showMpvOsd('mine'); return created; }, - recordCardsMined: (count) => calls.push(`cards:${count}`), }); created = false; await mineSentenceCard(); created = true; await mineSentenceCard(); - assert.deepEqual(calls, ['osd:mine', 'osd:mine', 'cards:1']); + assert.deepEqual(calls, ['osd:mine', 'osd:mine']); }); test('mine sentence handler forwards the canonical primary subtitle snapshot', async () => { @@ -99,7 +98,6 @@ test('mine sentence handler forwards the canonical primary subtitle snapshot', a assert.equal(options.primarySubtitle, primarySubtitle); return true; }, - recordCardsMined: () => {}, }); await mineSentenceCard(); diff --git a/src/main/runtime/anki-actions.ts b/src/main/runtime/anki-actions.ts index 2c5c18c2..e77c2e79 100644 --- a/src/main/runtime/anki-actions.ts +++ b/src/main/runtime/anki-actions.ts @@ -83,18 +83,15 @@ export function createMineSentenceCardHandler(deps: { primarySubtitle?: PrimarySubtitle; showMpvOsd: (text: string) => void; }) => Promise; - recordCardsMined: (count: number, noteIds?: number[]) => void; }) { + // Card creation records the mined card (with its note id) itself. return async (): Promise => { const primarySubtitle = deps.getPrimarySubtitle?.(); - const created = await deps.mineSentenceCardCore({ + await deps.mineSentenceCardCore({ ankiIntegration: deps.getAnkiIntegration(), mpvClient: deps.getMpvClient(), ...(primarySubtitle ? { primarySubtitle } : {}), showMpvOsd: deps.showMpvOsd, }); - if (created) { - deps.recordCardsMined(1); - } }; } diff --git a/src/main/runtime/mining-actions-main-deps.test.ts b/src/main/runtime/mining-actions-main-deps.test.ts index 389c3f5e..87680597 100644 --- a/src/main/runtime/mining-actions-main-deps.test.ts +++ b/src/main/runtime/mining-actions-main-deps.test.ts @@ -45,20 +45,17 @@ test('mining action main deps builders map callbacks', () => { getCurrentSecondarySubText: () => 'sub', showMpvOsd: (text) => calls.push(`mine-osd:${text}`), logError: (message) => calls.push(`err:${message}`), - onCardsMined: (count) => calls.push(`cards:${count}`), handleMineSentenceDigitCore: () => calls.push('mine-digit'), })(); assert.equal(mineDigit.getCurrentSecondarySubText(), 'sub'); mineDigit.showMpvOsd('done'); mineDigit.logError('bad', null); - mineDigit.onCardsMined(2); mineDigit.handleMineSentenceDigitCore(2, { subtitleTimingTracker: { track: true }, ankiIntegration: { enabled: true }, getCurrentSecondarySubText: () => 'sub', showMpvOsd: () => {}, logError: () => {}, - onCardsMined: () => {}, }); assert.deepEqual(calls, [ @@ -70,7 +67,6 @@ test('mining action main deps builders map callbacks', () => { 'copy-current', 'mine-osd:done', 'err:bad', - 'cards:2', 'mine-digit', ]); }); diff --git a/src/main/runtime/mining-actions-main-deps.ts b/src/main/runtime/mining-actions-main-deps.ts index e79fdbd8..4281eeba 100644 --- a/src/main/runtime/mining-actions-main-deps.ts +++ b/src/main/runtime/mining-actions-main-deps.ts @@ -57,7 +57,6 @@ export function createBuildHandleMineSentenceDigitMainDepsHandler< getCurrentSecondarySubText: () => string | undefined; showMpvOsd: (text: string) => void; logError: (message: string, err: unknown) => void; - onCardsMined: (count: number) => void; handleMineSentenceDigitCore: ( count: number, options: { @@ -66,7 +65,6 @@ export function createBuildHandleMineSentenceDigitMainDepsHandler< getCurrentSecondarySubText: () => string | undefined; showMpvOsd: (text: string) => void; logError: (message: string, err: unknown) => void; - onCardsMined: (count: number) => void; }, ) => void; }) { @@ -76,7 +74,6 @@ export function createBuildHandleMineSentenceDigitMainDepsHandler< getCurrentSecondarySubText: () => deps.getCurrentSecondarySubText(), showMpvOsd: (text: string) => deps.showMpvOsd(text), logError: (message: string, err: unknown) => deps.logError(message, err), - onCardsMined: (count: number) => deps.onCardsMined(count), handleMineSentenceDigitCore: ( count: number, options: { @@ -85,7 +82,6 @@ export function createBuildHandleMineSentenceDigitMainDepsHandler< getCurrentSecondarySubText: () => string | undefined; showMpvOsd: (text: string) => void; logError: (message: string, err: unknown) => void; - onCardsMined: (count: number) => void; }, ) => deps.handleMineSentenceDigitCore(count, options), }); diff --git a/src/main/runtime/mining-actions.test.ts b/src/main/runtime/mining-actions.test.ts index 9b388843..0557998b 100644 --- a/src/main/runtime/mining-actions.test.ts +++ b/src/main/runtime/mining-actions.test.ts @@ -53,7 +53,6 @@ test('mine sentence digit handler forwards all dependencies', () => { getCurrentSecondarySubText: () => 'secondary', showMpvOsd: (text) => calls.push(`osd:${text}`), logError: (message) => calls.push(`err:${message}`), - onCardsMined: (count) => calls.push(`cards:${count}`), handleMineSentenceDigitCore: (count, options) => { assert.equal(count, 4); assert.equal(options.subtitleTimingTracker, tracker); @@ -61,10 +60,9 @@ test('mine sentence digit handler forwards all dependencies', () => { assert.equal(options.getCurrentSecondarySubText(), 'secondary'); options.showMpvOsd('mine'); options.logError('boom', new Error('x')); - options.onCardsMined(2); }, }); handleMineSentenceDigit(4); - assert.deepEqual(calls, ['osd:mine', 'err:boom', 'cards:2']); + assert.deepEqual(calls, ['osd:mine', 'err:boom']); }); diff --git a/src/main/runtime/mining-actions.ts b/src/main/runtime/mining-actions.ts index 9ee1935f..797fe5ee 100644 --- a/src/main/runtime/mining-actions.ts +++ b/src/main/runtime/mining-actions.ts @@ -48,7 +48,6 @@ export function createHandleMineSentenceDigitHandler< getCurrentSecondarySubText: () => string | undefined; showMpvOsd: (text: string) => void; logError: (message: string, err: unknown) => void; - onCardsMined: (count: number) => void; handleMineSentenceDigitCore: ( count: number, options: { @@ -57,7 +56,6 @@ export function createHandleMineSentenceDigitHandler< getCurrentSecondarySubText: () => string | undefined; showMpvOsd: (text: string) => void; logError: (message: string, err: unknown) => void; - onCardsMined: (count: number) => void; }, ) => void; }) { @@ -68,7 +66,6 @@ export function createHandleMineSentenceDigitHandler< getCurrentSecondarySubText: deps.getCurrentSecondarySubText, showMpvOsd: deps.showMpvOsd, logError: deps.logError, - onCardsMined: deps.onCardsMined, }); }; } diff --git a/stats/src/components/overview/OverviewTab.test.tsx b/stats/src/components/overview/OverviewTab.test.tsx new file mode 100644 index 00000000..9316f039 --- /dev/null +++ b/stats/src/components/overview/OverviewTab.test.tsx @@ -0,0 +1,271 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { Window } from 'happy-dom'; +import { act } from 'react'; +import { createRoot } from 'react-dom/client'; +import { apiClient } from '../../lib/api-client'; +import { setDeleteConfirmPresenter } from '../../lib/delete-confirm'; +import { localDayFromMs } from '../../lib/formatters'; +import type { OverviewData, SessionSummary } from '../../types/stats'; +import { OverviewTab } from './OverviewTab'; + +function installDom() { + const window = new Window(); + const properties = { + window, + document: window.document, + HTMLElement: window.HTMLElement, + ResizeObserver: window.ResizeObserver, + IS_REACT_ACT_ENVIRONMENT: true, + }; + const previous = Object.keys(properties).map( + (key) => [key, Object.getOwnPropertyDescriptor(globalThis, key)] as const, + ); + for (const [key, value] of Object.entries(properties)) { + Object.defineProperty(globalThis, key, { value, configurable: true, writable: true }); + } + return () => { + for (const [key, descriptor] of previous) { + if (descriptor) Object.defineProperty(globalThis, key, descriptor); + else Reflect.deleteProperty(globalThis, key); + } + window.happyDOM.abort(); + }; +} + +function session(sessionId: number): SessionSummary { + return { + sessionId, + canonicalTitle: `Test episode ${sessionId}`, + videoId: null, + animeId: null, + animeTitle: 'Test anime', + startedAtMs: Date.now(), + endedAtMs: Date.now(), + totalWatchedMs: 60_000, + activeWatchedMs: 60_000, + linesSeen: 1, + tokensSeen: 10, + cardsMined: 2, + lookupCount: 0, + lookupHits: 0, + yomitanLookupCount: 0, + knownWordsSeen: 1, + knownWordRate: 0.1, + }; +} + +function overview(sessions: SessionSummary[]): OverviewData { + const count = sessions.length; + return { + sessions, + rollups: count + ? [ + { + rollupDayOrMonth: localDayFromMs(Date.now()), + videoId: null, + totalSessions: count, + totalActiveMin: count, + totalLinesSeen: count, + totalTokensSeen: count * 10, + totalCards: count * 2, + cardsPerHour: null, + tokensPerMin: null, + lookupHitRate: null, + }, + ] + : [], + hints: { + totalSessions: count, + activeSessions: 0, + episodesToday: count, + activeAnimeCount: count ? 1 : 0, + totalEpisodesWatched: count, + totalAnimeCompleted: 0, + totalActiveMin: count, + activeDays: count ? 1 : 0, + totalCards: count * 2, + totalTokensSeen: count * 10, + totalLookupCount: 0, + totalLookupHits: 0, + totalYomitanLookupCount: 0, + newWordsToday: count, + newWordsThisWeek: count, + }, + }; +} + +function metric(container: HTMLElement, label: string): string { + const element = [...container.querySelectorAll('div')].find( + (element) => element.textContent === label, + ); + assert.ok(element?.parentElement, `expected metric ${label}`); + return (element.parentElement.textContent ?? '').replace(label, '').trim(); +} + +for (const mode of ['session', 'day', 'anime', 'failed'] as const) { + test(`Overview refreshes all dependent data after ${mode} deletion`, async () => { + const uninstallDom = installDom(); + const original = { ...apiClient }; + const restoreConfirm = setDeleteConfirmPresenter(() => true); + let sessions = + mode === 'session' || mode === 'failed' + ? [session(1)] + : [ + { ...session(1), animeId: 7 }, + { ...session(2), animeId: 7 }, + ]; + const reads = { overview: 0, sessions: 0, calendar: 0, words: 0 }; + const deletedIds: number[] = []; + apiClient.getOverview = async () => { + reads.overview++; + return overview(sessions); + }; + apiClient.getSessions = async () => { + reads.sessions++; + return sessions; + }; + apiClient.getStreakCalendar = async () => { + reads.calendar++; + return [{ epochDay: localDayFromMs(Date.now()), totalActiveMin: sessions.length }]; + }; + apiClient.getKnownWordsSummary = async () => { + reads.words++; + return { totalUniqueWords: 10 + sessions.length, knownWordCount: 5 }; + }; + apiClient.getCoverImages = async () => ({ anime: {}, media: {} }); + async function deleteSessions(ids: number[]) { + if (mode === 'failed') throw new Error('Deletion failed'); + deletedIds.push(...ids); + sessions = sessions.filter((session) => !ids.includes(session.sessionId)); + } + apiClient.deleteSession = async (id) => deleteSessions([id]); + apiClient.deleteSessions = deleteSessions; + const container = document.createElement('div'); + document.body.append(container); + const root = createRoot(container); + try { + await act(async () => + root.render( + {}} onNavigateToSession={() => {}} />, + ), + ); + const count = sessions.length; + assert.match(metric(container, 'Cards Mined Today'), new RegExp(`^${count * 2}`)); + assert.match(metric(container, 'Known Words'), new RegExp(`/ ${10 + count}`)); + assert.equal(container.querySelectorAll('.bg-ctp-green\\/30').length, 1); + const prefix = + mode === 'day' + ? 'Delete all sessions from ' + : mode === 'anime' + ? 'Delete all sessions for ' + : 'Delete session '; + const button = [...container.querySelectorAll('button')].find((button) => + button.getAttribute('aria-label')?.startsWith(prefix), + ); + assert.ok(button, `expected ${mode} delete button`); + await act(async () => button.click()); + if (mode === 'failed') { + assert.match(container.textContent ?? '', /Deletion failed/); + assert.match(metric(container, 'Cards Mined Today'), /^2/); + assert.deepEqual(reads, { overview: 1, sessions: 1, calendar: 1, words: 1 }); + assert.deepEqual(deletedIds, []); + } else { + assert.deepEqual(deletedIds, count === 1 ? [1] : [1, 2]); + for (const label of [ + 'Cards Mined Today', + 'Episodes Today', + 'Sessions', + 'Cards Mined', + 'Words Today', + 'New Words Today', + ]) { + assert.equal(metric(container, label), '0', `${label} should reflect the deletion`); + } + assert.match(metric(container, 'Known Words'), /5\s*\/ 10/); + assert.equal(container.querySelectorAll('.bg-ctp-green\\/30').length, 0); + assert.match(container.textContent ?? '', /No sessions yet/); + assert.deepEqual(reads, { overview: 2, sessions: 2, calendar: 2, words: 2 }); + } + } finally { + await act(async () => root.unmount()); + restoreConfirm(); + Object.assign(apiClient, original); + uninstallDom(); + } + }); +} + +for (const failedRead of ['overview', 'sessions', 'calendar'] as const) { + test(`Overview stays available during a failed ${failedRead} refresh and can retry`, async () => { + const uninstallDom = installDom(); + const original = { ...apiClient }; + const restoreConfirm = setDeleteConfirmPresenter(() => true); + let sessions = [session(1)]; + let refreshing = false; + let failRefresh = true; + let rejectRefresh = (_error: Error) => {}; + const pendingRefresh = new Promise((_resolve, reject) => { + rejectRefresh = reject; + }); + apiClient.getOverview = async () => { + if (refreshing && failRefresh && failedRead === 'overview') return pendingRefresh; + return overview(sessions); + }; + apiClient.getSessions = async () => { + if (refreshing && failRefresh && failedRead === 'sessions') return pendingRefresh; + return sessions; + }; + apiClient.getStreakCalendar = async () => { + if (refreshing && failRefresh && failedRead === 'calendar') return pendingRefresh; + return []; + }; + apiClient.getKnownWordsSummary = async () => ({ + totalUniqueWords: 0, + knownWordCount: 0, + }); + apiClient.deleteSession = async () => { + sessions = []; + refreshing = true; + }; + // A calendar-only failure still lets the overview totals refresh. + const staleCards = failedRead === 'calendar' ? '0' : '2'; + const container = document.createElement('div'); + document.body.append(container); + const root = createRoot(container); + try { + await act(async () => + root.render( + {}} onNavigateToSession={() => {}} />, + ), + ); + const deleteButton = [...container.querySelectorAll('button')].find((button) => + button.getAttribute('aria-label')?.startsWith('Delete session '), + ); + assert.ok(deleteButton); + await act(async () => deleteButton.click()); + assert.match(container.textContent ?? '', /Tracking Snapshot/); + assert.equal(metric(container, 'Cards Mined Today'), staleCards); + assert.match(container.textContent ?? '', /No sessions yet/); + + await act(async () => rejectRefresh(new Error(`${failedRead} unavailable`))); + assert.match(container.textContent ?? '', /Tracking Snapshot/); + assert.equal(metric(container, 'Cards Mined Today'), staleCards); + assert.match(container.querySelector('[role="alert"]')?.textContent ?? '', /unavailable/); + + const retryButton = [...container.querySelectorAll('button')].find( + (button) => button.textContent === 'Retry', + ); + assert.ok(retryButton, 'failed refresh should offer a retry'); + failRefresh = false; + await act(async () => retryButton.click()); + assert.equal(metric(container, 'Cards Mined Today'), '0'); + assert.equal(container.querySelector('[role="alert"]'), null); + } finally { + await act(async () => root.unmount()); + restoreConfirm(); + Object.assign(apiClient, original); + uninstallDom(); + } + }); +} diff --git a/stats/src/components/overview/OverviewTab.tsx b/stats/src/components/overview/OverviewTab.tsx index 767aa3fa..efd47855 100644 --- a/stats/src/components/overview/OverviewTab.tsx +++ b/stats/src/components/overview/OverviewTab.tsx @@ -27,8 +27,9 @@ export function OverviewTab({ onNavigateToSession, isActive = true, }: OverviewTabProps) { - const { data, sessions, setSessions, loading, error } = useOverview(); - const { calendar, loading: calLoading } = useStreakCalendar(90); + const [refreshKey, setRefreshKey] = useState(0); + const { data, sessions, setSessions, loading, error } = useOverview(refreshKey); + const { calendar, loading: calLoading, error: calendarError } = useStreakCalendar(90, refreshKey); const [deleteError, setDeleteError] = useState(null); const [deletingIds, setDeletingIds] = useState>(new Set()); const [knownWordsSummary, setKnownWordsSummary] = useState<{ @@ -49,7 +50,7 @@ export function OverviewTab({ return () => { cancelled = true; }; - }, []); + }, [refreshKey]); const handleDeleteSession = async (session: SessionSummary) => { if (!(await confirmSessionDelete())) return; @@ -58,6 +59,7 @@ export function OverviewTab({ try { await apiClient.deleteSession(session.sessionId); setSessions((prev) => prev.filter((s) => s.sessionId !== session.sessionId)); + setRefreshKey((prev) => prev + 1); } catch (err) { setDeleteError(err instanceof Error ? err.message : 'Failed to delete session.'); } finally { @@ -82,6 +84,7 @@ export function OverviewTab({ await apiClient.deleteSessions(ids); const idSet = new Set(ids); setSessions((prev) => prev.filter((s) => !idSet.has(s.sessionId))); + setRefreshKey((prev) => prev + 1); } catch (err) { setDeleteError(err instanceof Error ? err.message : 'Failed to delete sessions.'); } finally { @@ -108,6 +111,7 @@ export function OverviewTab({ await apiClient.deleteSessions(ids); const idSet = new Set(ids); setSessions((prev) => prev.filter((s) => !idSet.has(s.sessionId))); + setRefreshKey((prev) => prev + 1); } catch (err) { setDeleteError(err instanceof Error ? err.message : 'Failed to delete sessions.'); } finally { @@ -119,9 +123,32 @@ export function OverviewTab({ } }; - if (loading) return
Loading...
; - if (error) return
Error: {error}
; - if (!data) return null; + const refreshError = error ?? calendarError; + const errorNotice = refreshError ? ( +
+ + {!data + ? 'Could not load Overview.' + : error + ? 'Could not refresh Overview. Displayed totals may be out of date.' + : 'Could not refresh the activity calendar. It may be out of date.'}{' '} + {refreshError} + + +
+ ) : null; + + if (loading && !data) return
Loading...
; + if (!data) return errorNotice; const summary = buildOverviewSummary(data); const streakData = buildStreakCalendar(calendar); @@ -129,6 +156,7 @@ export function OverviewTab({ return (
+ {errorNotice}
diff --git a/stats/src/hooks/useOverview.ts b/stats/src/hooks/useOverview.ts index ac73cd49..a5174277 100644 --- a/stats/src/hooks/useOverview.ts +++ b/stats/src/hooks/useOverview.ts @@ -2,7 +2,7 @@ import { useState, useEffect } from 'react'; import { getStatsClient } from './useStatsApi'; import type { OverviewData, SessionSummary } from '../types/stats'; -export function useOverview() { +export function useOverview(refreshKey = 0) { const [data, setData] = useState(null); const [sessions, setSessions] = useState([]); const [loading, setLoading] = useState(true); @@ -30,7 +30,7 @@ export function useOverview() { return () => { cancelled = true; }; - }, []); + }, [refreshKey]); return { data, sessions, setSessions, loading, error }; } diff --git a/stats/src/hooks/useStreakCalendar.ts b/stats/src/hooks/useStreakCalendar.ts index 02a930c3..cdb7c47a 100644 --- a/stats/src/hooks/useStreakCalendar.ts +++ b/stats/src/hooks/useStreakCalendar.ts @@ -2,13 +2,15 @@ import { useState, useEffect } from 'react'; import { getStatsClient } from './useStatsApi'; import type { StreakCalendarDay } from '../types/stats'; -export function useStreakCalendar(days = 90) { +export function useStreakCalendar(days = 90, refreshKey = 0) { const [calendar, setCalendar] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); useEffect(() => { let cancelled = false; + setLoading(true); + setError(null); getStatsClient() .getStreakCalendar(days) .then((data) => { @@ -23,7 +25,7 @@ export function useStreakCalendar(days = 90) { return () => { cancelled = true; }; - }, [days]); + }, [days, refreshKey]); return { calendar, loading, error }; }