diff --git a/backlog/tasks/task-330 - Fix-PR-60-CI-failures-and-CodeRabbit-feedback.md b/backlog/tasks/task-330 - Fix-PR-60-CI-failures-and-CodeRabbit-feedback.md new file mode 100644 index 00000000..505ab427 --- /dev/null +++ b/backlog/tasks/task-330 - Fix-PR-60-CI-failures-and-CodeRabbit-feedback.md @@ -0,0 +1,70 @@ +--- +id: TASK-330 +title: Fix PR 60 CI failures and CodeRabbit feedback +status: Done +assignee: + - codex +created_date: '2026-05-04 02:50' +updated_date: '2026-05-04 02:59' +labels: + - ci + - pr-review +dependencies: [] +references: + - 'https://github.com/ksyasuda/SubMiner/pull/60' +priority: high +--- + +## Description + + +Resolve failing GitHub Actions checks and actionable unresolved CodeRabbit review feedback on PR #60 (Persist stats exclusions in DB and fix word metrics filtering). Keep fixes scoped to the PR behavior and preserve existing project patterns. + + +## Acceptance Criteria + +- [x] #1 Failing GitHub Actions checks for PR #60 have an identified root cause and local fix. +- [x] #2 All actionable unresolved CodeRabbit review comments on PR #60 are addressed locally or explicitly documented as non-actionable. +- [x] #3 Relevant local verification passes for the changed code paths. +- [x] #4 Task notes summarize CI failure context, review-comment handling, and any residual verification gaps. + + +## Implementation Plan + + +1. Resolve PR #60 context and inspect GitHub Actions failures with the gh-fix-ci workflow. +2. Fetch unresolved review threads with the gh-address-comments workflow, focusing on CodeRabbit actionable comments. +3. Read the touched files/tests around the failing paths and comments; identify root cause before edits. +4. Apply minimal fixes with regression coverage where appropriate. +5. Run targeted verification first, then broader repo gates as time permits. +6. Update Backlog notes/acceptance criteria with CI/comment outcomes and residual risks. + + +## Implementation Notes + + +Resolved PR #60 CI failure by restoring raw `tokensSeen` for session summaries while keeping filtered persisted word counts in aggregate/known-word paths. Addressed CodeRabbit feedback: fixed missing `headword` test fixture binding; paged vocabulary stats past filtered rows; preserved lifetime/rollup totals when retained-session recomputation is partial; emitted flat known-word timeline points for zero-visible-word line gaps; restored localStorage mocks; added rollback/retry behavior for excluded-word store persistence/initialization. + + +## Final Summary + + +Fixed the PR #60 CI failure and addressed actionable CodeRabbit feedback. + +Key changes: +- Restored exact Yomitan token counts for session summary metrics while leaving filtered word counts for aggregate and known-word calculations. +- Fixed malformed query test fixtures by binding `headword` into `imm_words` inserts. +- Updated vocabulary stats to page until enough visible rows are collected after post-query filtering. +- Made library/detail/rollup read models preserve lifetime or stored rollup totals when retained-session recomputation is partial, including dashboard rollup-derived word metrics. +- Kept known-word timeline line positions stable by emitting flat points for missing line indexes. +- Made excluded-word persistence rollback on failed writes, allow initialization retries after transient load failures, and restored mocked `localStorage` in tests. + +Verification passed: +- `bun run typecheck` +- `bun run test:fast` +- `bun run test:env` +- `bun run build` +- `bun run test:smoke:dist` +- `bun run format:check:src` +- `git diff --check` + diff --git a/src/core/services/__tests__/stats-server.test.ts b/src/core/services/__tests__/stats-server.test.ts index d88bdfa7..b7e1666f 100644 --- a/src/core/services/__tests__/stats-server.test.ts +++ b/src/core/services/__tests__/stats-server.test.ts @@ -463,7 +463,9 @@ describe('stats server API routes', () => { const res = await app.request('/api/stats/sessions/1/known-words-timeline'); assert.equal(res.status, 200); assert.deepEqual(await res.json(), [ + { linesSeen: 0, knownWordsSeen: 0, totalWordsSeen: 0 }, { linesSeen: 1, knownWordsSeen: 2, totalWordsSeen: 2 }, + { linesSeen: 2, knownWordsSeen: 2, totalWordsSeen: 2 }, { linesSeen: 3, knownWordsSeen: 3, totalWordsSeen: 7 }, ]); }); diff --git a/src/core/services/immersion-tracker/__tests__/query.test.ts b/src/core/services/immersion-tracker/__tests__/query.test.ts index bafe6bfa..bf21759e 100644 --- a/src/core/services/immersion-tracker/__tests__/query.test.ts +++ b/src/core/services/immersion-tracker/__tests__/query.test.ts @@ -139,6 +139,7 @@ function insertFilteredWordOccurrence( RETURNING id`, ) .get( + headword, word, options.reading ?? '', options.pos1 ?? '名詞', @@ -1371,7 +1372,7 @@ test('word-count read models use filtered persisted occurrences with raw fallbac const summaries = getSessionSummaries(db, 10); assert.equal( summaries.find((session) => session.sessionId === withOccurrences.sessionId)?.tokensSeen, - 2, + 5, ); assert.equal( summaries.find((session) => session.sessionId === fallbackOnly.sessionId)?.tokensSeen, @@ -1382,8 +1383,69 @@ test('word-count read models use filtered persisted occurrences with raw fallbac assert.equal(hints.totalTokensSeen, 9); const rollup = getDailyRollups(db, 1)[0]!; - assert.equal(rollup.totalTokensSeen, 9); - assert.equal(rollup.tokensPerMin, 9); + assert.equal(rollup.totalTokensSeen, 12); + assert.equal(rollup.tokensPerMin, 12); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + +test('rollups keep persisted totals when retained-session word counts are partial', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const videoId = getOrCreateVideoRecord(db, 'local:/tmp/partial-rollup.mkv', { + canonicalTitle: 'Partial Rollup', + sourcePath: '/tmp/partial-rollup.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + + const startedAtMs = 1_700_000_000_000; + const { sessionId } = startSessionRecord(db, videoId, startedAtMs); + db.prepare( + ` + UPDATE imm_sessions + SET ended_at_ms = ?, status = 2, active_watched_ms = ?, tokens_seen = ? + WHERE session_id = ? + `, + ).run(startedAtMs + 60_000, 60_000, 4, sessionId); + + insertFilteredWordOccurrence(db, { + sessionId, + videoId, + occurrenceCount: 4, + startedAtMs, + }); + + const rollupDay = Math.floor(startedAtMs / 86_400_000); + db.prepare( + ` + INSERT INTO imm_daily_rollups ( + rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, + total_tokens_seen, total_cards + ) VALUES (?, ?, ?, ?, ?, ?, ?) + `, + ).run(rollupDay, videoId, 2, 2, 8, 12, 0); + db.prepare( + ` + INSERT INTO imm_monthly_rollups ( + rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, + total_tokens_seen, total_cards + ) VALUES (?, ?, ?, ?, ?, ?, ?) + `, + ).run(202311, videoId, 2, 3, 12, 18, 0); + + const daily = getDailyRollups(db, 1)[0]!; + assert.equal(daily.totalTokensSeen, 12); + assert.equal(daily.tokensPerMin, 6); + + const monthly = getMonthlyRollups(db, 1)[0]!; + assert.equal(monthly.totalTokensSeen, 18); + assert.equal(monthly.tokensPerMin, 6); } finally { db.close(); cleanupDbPath(dbPath); @@ -1639,6 +1701,41 @@ test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () } }); +test('getVocabularyStats pages past hidden rows until enough visible rows are collected', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const stmts = createTrackerPreparedStatements(db); + + for (let i = 0; i < 105; i += 1) { + stmts.wordUpsertStmt.run( + `助詞${i}`, + `助詞${i}`, + `じょし${i}`, + 'particle', + '助詞', + '格助詞', + '', + 10_000 - i, + 1_000, + ); + } + stmts.wordUpsertStmt.run('猫', '猫', 'ねこ', 'noun', '名詞', '一般', '', 1, 1_000); + + const rows = getVocabularyStats(db, 1); + + assert.deepEqual( + rows.map((row) => row.headword), + ['猫'], + ); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('getVocabularyStats returns empty array when no words exist', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -2863,6 +2960,96 @@ test('anime library and detail still return lifetime rows without retained sessi } }); +test('anime and media detail prefer lifetime totals over partial retained sessions', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + + const animeId = getOrCreateAnimeRecord(db, { + parsedTitle: 'Partial History Anime', + canonicalTitle: 'Partial History Anime', + anilistId: null, + titleRomaji: null, + titleEnglish: null, + titleNative: null, + metadataJson: null, + }); + const videoId = getOrCreateVideoRecord(db, 'local:/tmp/partial-history.mkv', { + canonicalTitle: 'Partial History Episode', + sourcePath: '/tmp/partial-history.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + linkVideoToAnimeRecord(db, videoId, { + animeId, + parsedBasename: 'Partial History Episode', + parsedTitle: 'Partial History Anime', + parsedSeason: 1, + parsedEpisode: 1, + parserSource: 'fallback', + parserConfidence: 1, + parseMetadataJson: null, + }); + + const startedAtMs = 1_700_000_000_000; + const { sessionId } = startSessionRecord(db, videoId, startedAtMs); + db.prepare( + ` + UPDATE imm_sessions + SET ended_at_ms = ?, status = 2, active_watched_ms = ?, tokens_seen = ? + WHERE session_id = ? + `, + ).run(startedAtMs + 30_000, 30_000, 10, sessionId); + + const now = Date.now(); + db.prepare( + ` + INSERT INTO imm_lifetime_anime ( + anime_id, + total_sessions, + total_active_ms, + total_cards, + total_lines_seen, + total_tokens_seen, + episodes_started, + episodes_completed, + first_watched_ms, + last_watched_ms, + CREATED_DATE, + LAST_UPDATE_DATE + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + ).run(animeId, 3, 90_000, 1, 12, 100, 1, 0, startedAtMs, startedAtMs, now, now); + db.prepare( + ` + INSERT INTO imm_lifetime_media ( + video_id, + total_sessions, + total_active_ms, + total_cards, + total_lines_seen, + total_tokens_seen, + completed, + first_watched_ms, + last_watched_ms, + CREATED_DATE, + LAST_UPDATE_DATE + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + ).run(videoId, 3, 90_000, 1, 12, 100, 0, startedAtMs, startedAtMs, now, now); + + assert.equal(getAnimeLibrary(db)[0]?.totalTokensSeen, 100); + assert.equal(getAnimeDetail(db, animeId)?.totalTokensSeen, 100); + assert.equal(getMediaLibrary(db)[0]?.totalTokensSeen, 100); + assert.equal(getMediaDetail(db, videoId)?.totalTokensSeen, 100); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('media library and detail queries read lifetime totals', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -4209,7 +4396,7 @@ test('getTrendsDashboard librarySummary returns null lookupsPerHundred when word } }); -test('getTrendsDashboard word metrics use filtered persisted occurrences', () => { +test('getTrendsDashboard rollup word metrics keep persisted totals over partial session counts', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -4308,16 +4495,16 @@ test('getTrendsDashboard word metrics use filtered persisted occurrences', () => const dashboard = getTrendsDashboard(db, 'all', 'day'); assert.deepEqual( dashboard.activity.words.map((point) => point.value), - [2, 3], + [10, 20], ); assert.deepEqual( dashboard.progress.words.map((point) => point.value), - [2, 5], + [10, 30], ); assert.equal(dashboard.ratios.lookupsPerHundred[0]?.value, 200); - assert.equal(dashboard.librarySummary[0]?.words, 5); - assert.equal(dashboard.librarySummary[0]?.lookupsPerHundred, 200); - assert.equal(dashboard.animeCumulative.words.at(-1)?.value, 5); + assert.equal(dashboard.librarySummary[0]?.words, 30); + assert.equal(dashboard.librarySummary[0]?.lookupsPerHundred, 33.3); + assert.equal(dashboard.animeCumulative.words.at(-1)?.value, 30); } finally { db.close(); cleanupDbPath(dbPath); diff --git a/src/core/services/immersion-tracker/query-lexical.ts b/src/core/services/immersion-tracker/query-lexical.ts index 6774e294..fc8ea5d3 100644 --- a/src/core/services/immersion-tracker/query-lexical.ts +++ b/src/core/services/immersion-tracker/query-lexical.ts @@ -53,7 +53,7 @@ export function getVocabularyStats( limit = 100, excludePos?: string[], ): VocabularyStatsRow[] { - const queryLimit = Math.max( + const pageSize = Math.max( limit, limit * VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR, limit + VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN, @@ -74,12 +74,20 @@ export function getVocabularyStats( LEFT JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id AND sl.anime_id IS NOT NULL ${whereClause ? whereClause.replace('part_of_speech', 'w.part_of_speech') : ''} GROUP BY w.id - ORDER BY w.frequency DESC LIMIT ? + ORDER BY w.frequency DESC LIMIT ? OFFSET ? `); - const params = hasExclude ? [...excludePos, queryLimit] : [queryLimit]; - return (stmt.all(...params) as VocabularyStatsRow[]) - .filter(isVocabularyStatsRowVisible) - .slice(0, limit); + const visibleRows: VocabularyStatsRow[] = []; + let offset = 0; + + while (visibleRows.length < limit) { + const params = hasExclude ? [...excludePos, pageSize, offset] : [pageSize, offset]; + const page = stmt.all(...params) as VocabularyStatsRow[]; + if (page.length === 0) break; + visibleRows.push(...page.filter(isVocabularyStatsRowVisible)); + offset += page.length; + } + + return visibleRows.slice(0, limit); } export function getStatsExcludedWords(db: DatabaseSync): StatsExcludedWordRow[] { diff --git a/src/core/services/immersion-tracker/query-library.ts b/src/core/services/immersion-tracker/query-library.ts index a650abc4..1202c5a5 100644 --- a/src/core/services/immersion-tracker/query-library.ts +++ b/src/core/services/immersion-tracker/query-library.ts @@ -27,20 +27,9 @@ import { } from './query-shared'; export function getAnimeLibrary(db: DatabaseSync): AnimeLibraryRow[] { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc'); const rows = db .prepare( ` - ${SESSION_WORD_COUNTS_CTE}, - anime_word_counts AS ( - SELECT v.anime_id AS animeId, SUM(${wordsExpr}) AS totalTokensSeen - FROM imm_sessions s - JOIN imm_videos v ON v.video_id = s.video_id - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id - WHERE s.ended_at_ms IS NOT NULL - AND v.anime_id IS NOT NULL - GROUP BY v.anime_id - ) SELECT a.anime_id AS animeId, a.canonical_title AS canonicalTitle, @@ -48,14 +37,13 @@ export function getAnimeLibrary(db: DatabaseSync): AnimeLibraryRow[] { COALESCE(lm.total_sessions, 0) AS totalSessions, COALESCE(lm.total_active_ms, 0) AS totalActiveMs, COALESCE(lm.total_cards, 0) AS totalCards, - COALESCE(awc.totalTokensSeen, lm.total_tokens_seen, 0) AS totalTokensSeen, + COALESCE(lm.total_tokens_seen, 0) AS totalTokensSeen, COUNT(DISTINCT v.video_id) AS episodeCount, a.episodes_total AS episodesTotal, COALESCE(lm.last_watched_ms, 0) AS lastWatchedMs FROM imm_anime a JOIN imm_lifetime_anime lm ON lm.anime_id = a.anime_id JOIN imm_videos v ON v.anime_id = a.anime_id - LEFT JOIN anime_word_counts awc ON awc.animeId = a.anime_id GROUP BY a.anime_id ORDER BY totalActiveMs DESC, lm.last_watched_ms DESC, canonicalTitle ASC `, @@ -68,7 +56,6 @@ export function getAnimeLibrary(db: DatabaseSync): AnimeLibraryRow[] { } export function getAnimeDetail(db: DatabaseSync, animeId: number): AnimeDetailRow | null { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); const row = db .prepare( ` @@ -84,10 +71,7 @@ export function getAnimeDetail(db: DatabaseSync, animeId: number): AnimeDetailRo COALESCE(lm.total_sessions, 0) AS totalSessions, COALESCE(lm.total_active_ms, 0) AS totalActiveMs, COALESCE(lm.total_cards, 0) AS totalCards, - CASE - WHEN COUNT(s.session_id) > 0 THEN COALESCE(SUM(${wordsExpr}), 0) - ELSE COALESCE(lm.total_tokens_seen, 0) - END AS totalTokensSeen, + COALESCE(lm.total_tokens_seen, 0) AS totalTokensSeen, COALESCE(lm.total_lines_seen, 0) AS totalLinesSeen, COALESCE(SUM(COALESCE(asm.lookupCount, s.lookup_count, 0)), 0) AS totalLookupCount, COALESCE(SUM(COALESCE(asm.lookupHits, s.lookup_hits, 0)), 0) AS totalLookupHits, @@ -99,7 +83,6 @@ export function getAnimeDetail(db: DatabaseSync, animeId: number): AnimeDetailRo JOIN imm_videos v ON v.anime_id = a.anime_id LEFT JOIN imm_sessions s ON s.video_id = v.video_id LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id WHERE a.anime_id = ? GROUP BY a.anime_id `, @@ -219,25 +202,16 @@ export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisod } export function getMediaLibrary(db: DatabaseSync): MediaLibraryRow[] { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc'); const rows = db .prepare( ` - ${SESSION_WORD_COUNTS_CTE}, - media_word_counts AS ( - SELECT s.video_id AS videoId, SUM(${wordsExpr}) AS totalTokensSeen - FROM imm_sessions s - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id - WHERE s.ended_at_ms IS NOT NULL - GROUP BY s.video_id - ) SELECT v.video_id AS videoId, v.canonical_title AS canonicalTitle, COALESCE(lm.total_sessions, 0) AS totalSessions, COALESCE(lm.total_active_ms, 0) AS totalActiveMs, COALESCE(lm.total_cards, 0) AS totalCards, - COALESCE(mwc.totalTokensSeen, lm.total_tokens_seen, 0) AS totalTokensSeen, + COALESCE(lm.total_tokens_seen, 0) AS totalTokensSeen, COALESCE(lm.last_watched_ms, 0) AS lastWatchedMs, yv.youtube_video_id AS youtubeVideoId, yv.video_url AS videoUrl, @@ -256,7 +230,6 @@ export function getMediaLibrary(db: DatabaseSync): MediaLibraryRow[] { END AS hasCoverArt FROM imm_videos v JOIN imm_lifetime_media lm ON lm.video_id = v.video_id - LEFT JOIN media_word_counts mwc ON mwc.videoId = v.video_id LEFT JOIN imm_media_art ma ON ma.video_id = v.video_id LEFT JOIN imm_youtube_videos yv ON yv.video_id = v.video_id ORDER BY lm.last_watched_ms DESC @@ -270,7 +243,6 @@ export function getMediaLibrary(db: DatabaseSync): MediaLibraryRow[] { } export function getMediaDetail(db: DatabaseSync, videoId: number): MediaDetailRow | null { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); return db .prepare( ` @@ -282,10 +254,7 @@ export function getMediaDetail(db: DatabaseSync, videoId: number): MediaDetailRo COALESCE(lm.total_sessions, 0) AS totalSessions, COALESCE(lm.total_active_ms, 0) AS totalActiveMs, COALESCE(lm.total_cards, 0) AS totalCards, - CASE - WHEN COUNT(s.session_id) > 0 THEN COALESCE(SUM(${wordsExpr}), 0) - ELSE COALESCE(lm.total_tokens_seen, 0) - END AS totalTokensSeen, + COALESCE(lm.total_tokens_seen, 0) AS totalTokensSeen, COALESCE(lm.total_lines_seen, 0) AS totalLinesSeen, COALESCE(SUM(COALESCE(asm.lookupCount, s.lookup_count, 0)), 0) AS totalLookupCount, COALESCE(SUM(COALESCE(asm.lookupHits, s.lookup_hits, 0)), 0) AS totalLookupHits, @@ -306,7 +275,6 @@ export function getMediaDetail(db: DatabaseSync, videoId: number): MediaDetailRo LEFT JOIN imm_youtube_videos yv ON yv.video_id = v.video_id LEFT JOIN imm_sessions s ON s.video_id = v.video_id LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id WHERE v.video_id = ? GROUP BY v.video_id `, @@ -398,11 +366,19 @@ export function getMediaDailyRollups( total_sessions AS totalSessions, total_active_min AS totalActiveMin, total_lines_seen AS totalLinesSeen, - COALESCE(dwc.totalTokensSeen, total_tokens_seen) AS totalTokensSeen, + CASE + WHEN dwc.totalTokensSeen IS NOT NULL AND dwc.totalTokensSeen > total_tokens_seen THEN dwc.totalTokensSeen + ELSE total_tokens_seen + END AS totalTokensSeen, total_cards AS totalCards, cards_per_hour AS cardsPerHour, CASE - WHEN total_active_min > 0 THEN COALESCE(dwc.totalTokensSeen, total_tokens_seen) * 1.0 / total_active_min + WHEN total_active_min > 0 THEN ( + CASE + WHEN dwc.totalTokensSeen IS NOT NULL AND dwc.totalTokensSeen > total_tokens_seen THEN dwc.totalTokensSeen + ELSE total_tokens_seen + END + ) * 1.0 / total_active_min ELSE NULL END AS tokensPerMin, lookup_hit_rate AS lookupHitRate @@ -454,11 +430,19 @@ export function getAnimeDailyRollups( SELECT r.rollup_day AS rollupDayOrMonth, r.video_id AS videoId, r.total_sessions AS totalSessions, r.total_active_min AS totalActiveMin, r.total_lines_seen AS totalLinesSeen, - COALESCE(dwc.totalTokensSeen, r.total_tokens_seen) AS totalTokensSeen, + CASE + WHEN dwc.totalTokensSeen IS NOT NULL AND dwc.totalTokensSeen > r.total_tokens_seen THEN dwc.totalTokensSeen + ELSE r.total_tokens_seen + END AS totalTokensSeen, r.total_cards AS totalCards, r.cards_per_hour AS cardsPerHour, CASE - WHEN r.total_active_min > 0 THEN COALESCE(dwc.totalTokensSeen, r.total_tokens_seen) * 1.0 / r.total_active_min + WHEN r.total_active_min > 0 THEN ( + CASE + WHEN dwc.totalTokensSeen IS NOT NULL AND dwc.totalTokensSeen > r.total_tokens_seen THEN dwc.totalTokensSeen + ELSE r.total_tokens_seen + END + ) * 1.0 / r.total_active_min ELSE NULL END AS tokensPerMin, r.lookup_hit_rate AS lookupHitRate diff --git a/src/core/services/immersion-tracker/query-sessions.ts b/src/core/services/immersion-tracker/query-sessions.ts index a135f226..41b1fe69 100644 --- a/src/core/services/immersion-tracker/query-sessions.ts +++ b/src/core/services/immersion-tracker/query-sessions.ts @@ -17,7 +17,6 @@ import { } from './query-shared'; export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummaryQueryRow[] { - const wordsExpr = sessionDisplayWordsExpr('s', 'swc', 'COALESCE(asm.tokensSeen, s.tokens_seen)'); const prepared = db.prepare(` ${ACTIVE_SESSION_METRICS_CTE} SELECT @@ -31,14 +30,13 @@ export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummar COALESCE(asm.totalWatchedMs, s.total_watched_ms, 0) AS totalWatchedMs, COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0) AS activeWatchedMs, COALESCE(asm.linesSeen, s.lines_seen, 0) AS linesSeen, - ${wordsExpr} AS tokensSeen, + COALESCE(asm.tokensSeen, s.tokens_seen, 0) AS tokensSeen, COALESCE(asm.cardsMined, s.cards_mined, 0) AS cardsMined, COALESCE(asm.lookupCount, s.lookup_count, 0) AS lookupCount, COALESCE(asm.lookupHits, s.lookup_hits, 0) AS lookupHits, COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0) AS yomitanLookupCount FROM imm_sessions s LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id - LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id LEFT JOIN imm_videos v ON v.video_id = s.video_id LEFT JOIN imm_anime a ON a.anime_id = v.anime_id ORDER BY s.started_at_ms DESC @@ -382,11 +380,19 @@ export function getDailyRollups(db: DatabaseSync, limit = 60): ImmersionSessionR r.total_sessions AS totalSessions, r.total_active_min AS totalActiveMin, r.total_lines_seen AS totalLinesSeen, - COALESCE(dwc.totalTokensSeen, r.total_tokens_seen) AS totalTokensSeen, + CASE + WHEN dwc.totalTokensSeen IS NOT NULL AND dwc.totalTokensSeen > r.total_tokens_seen THEN dwc.totalTokensSeen + ELSE r.total_tokens_seen + END AS totalTokensSeen, r.total_cards AS totalCards, r.cards_per_hour AS cardsPerHour, CASE - WHEN r.total_active_min > 0 THEN COALESCE(dwc.totalTokensSeen, r.total_tokens_seen) * 1.0 / r.total_active_min + WHEN r.total_active_min > 0 THEN ( + CASE + WHEN dwc.totalTokensSeen IS NOT NULL AND dwc.totalTokensSeen > r.total_tokens_seen THEN dwc.totalTokensSeen + ELSE r.total_tokens_seen + END + ) * 1.0 / r.total_active_min ELSE NULL END AS tokensPerMin, r.lookup_hit_rate AS lookupHitRate @@ -432,14 +438,22 @@ export function getMonthlyRollups(db: DatabaseSync, limit = 24): ImmersionSessio r.total_sessions AS totalSessions, r.total_active_min AS totalActiveMin, r.total_lines_seen AS totalLinesSeen, - COALESCE(mwc.totalTokensSeen, r.total_tokens_seen) AS totalTokensSeen, + CASE + WHEN mwc.totalTokensSeen IS NOT NULL AND mwc.totalTokensSeen > r.total_tokens_seen THEN mwc.totalTokensSeen + ELSE r.total_tokens_seen + END AS totalTokensSeen, r.total_cards AS totalCards, CASE WHEN r.total_active_min > 0 THEN (r.total_cards * 60.0) / r.total_active_min ELSE NULL END AS cardsPerHour, CASE - WHEN r.total_active_min > 0 THEN COALESCE(mwc.totalTokensSeen, r.total_tokens_seen) * 1.0 / r.total_active_min + WHEN r.total_active_min > 0 THEN ( + CASE + WHEN mwc.totalTokensSeen IS NOT NULL AND mwc.totalTokensSeen > r.total_tokens_seen THEN mwc.totalTokensSeen + ELSE r.total_tokens_seen + END + ) * 1.0 / r.total_active_min ELSE NULL END AS tokensPerMin, NULL AS lookupHitRate diff --git a/src/core/services/immersion-tracker/storage-session.test.ts b/src/core/services/immersion-tracker/storage-session.test.ts index 650542b9..3476173c 100644 --- a/src/core/services/immersion-tracker/storage-session.test.ts +++ b/src/core/services/immersion-tracker/storage-session.test.ts @@ -172,9 +172,7 @@ test('stats excluded words are replaced and read from sqlite storage', () => { ]); replaceStatsExcludedWords(db, [{ headword: '犬', word: '犬', reading: 'いぬ' }]); - assert.deepEqual(getStatsExcludedWords(db), [ - { headword: '犬', word: '犬', reading: 'いぬ' }, - ]); + assert.deepEqual(getStatsExcludedWords(db), [{ headword: '犬', word: '犬', reading: 'いぬ' }]); } finally { db.close(); cleanupDbPath(dbPath); diff --git a/src/core/services/stats-server.ts b/src/core/services/stats-server.ts index ad97c242..d36b1515 100644 --- a/src/core/services/stats-server.ts +++ b/src/core/services/stats-server.ts @@ -452,7 +452,7 @@ export function createStatsApp( } } - const sortedLineIndices = [...totalLineGroups.keys()].sort((a, b) => a - b); + const maxLineIndex = Math.max(...totalLineGroups.keys(), ...knownLineGroups.keys(), -1); let knownWordsSeen = 0; let totalWordsSeen = 0; const knownByLinesSeen: Array<{ @@ -461,9 +461,9 @@ export function createStatsApp( totalWordsSeen: number; }> = []; - for (const lineIdx of sortedLineIndices) { + for (let lineIdx = 0; lineIdx <= maxLineIndex; lineIdx += 1) { knownWordsSeen += knownLineGroups.get(lineIdx) ?? 0; - totalWordsSeen += totalLineGroups.get(lineIdx)!; + totalWordsSeen += totalLineGroups.get(lineIdx) ?? 0; knownByLinesSeen.push({ linesSeen: lineIdx, knownWordsSeen, diff --git a/stats/src/hooks/useExcludedWords.test.ts b/stats/src/hooks/useExcludedWords.test.ts index 83d81a12..9c96e34b 100644 --- a/stats/src/hooks/useExcludedWords.test.ts +++ b/stats/src/hooks/useExcludedWords.test.ts @@ -11,6 +11,7 @@ import { BASE_URL } from '../lib/api-client'; const STORAGE_KEY = 'subminer-excluded-words'; function installLocalStorage(initial: Record = {}) { + const previous = Object.getOwnPropertyDescriptor(globalThis, 'localStorage'); const values = new Map(Object.entries(initial)); Object.defineProperty(globalThis, 'localStorage', { configurable: true, @@ -20,13 +21,24 @@ function installLocalStorage(initial: Record = {}) { removeItem: (key: string) => values.delete(key), }, }); - return values; + return { + values, + restore: () => { + if (previous) { + Object.defineProperty(globalThis, 'localStorage', previous); + } else { + delete (globalThis as { localStorage?: unknown }).localStorage; + } + }, + }; } test('initializeExcludedWordsStore seeds empty database exclusions from localStorage', async () => { resetExcludedWordsStoreForTests(); const localRows = [{ headword: '猫', word: '猫', reading: 'ねこ' }]; - const storage = installLocalStorage({ [STORAGE_KEY]: JSON.stringify(localRows) }); + const { values: storage, restore } = installLocalStorage({ + [STORAGE_KEY]: JSON.stringify(localRows), + }); const originalFetch = globalThis.fetch; const requests: Array<{ url: string; method: string; body: string }> = []; globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { @@ -59,13 +71,14 @@ test('initializeExcludedWordsStore seeds empty database exclusions from localSto assert.equal(storage.get(STORAGE_KEY), JSON.stringify(localRows)); } finally { globalThis.fetch = originalFetch; + restore(); resetExcludedWordsStoreForTests(); } }); test('setExcludedWords updates the database-backed exclusion list', async () => { resetExcludedWordsStoreForTests(); - const storage = installLocalStorage(); + const { values: storage, restore } = installLocalStorage(); const originalFetch = globalThis.fetch; let seenBody = ''; globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { @@ -82,6 +95,66 @@ test('setExcludedWords updates the database-backed exclusion list', async () => assert.equal(storage.get(STORAGE_KEY), JSON.stringify(rows)); } finally { globalThis.fetch = originalFetch; + restore(); + resetExcludedWordsStoreForTests(); + } +}); + +test('setExcludedWords rolls back local state when persistence fails', async () => { + resetExcludedWordsStoreForTests(); + const previousRows = [{ headword: '猫', word: '猫', reading: 'ねこ' }]; + const nextRows = [{ headword: 'する', word: 'する', reading: 'する' }]; + const { values: storage, restore } = installLocalStorage({ + [STORAGE_KEY]: JSON.stringify(previousRows), + }); + const originalFetch = globalThis.fetch; + const originalConsoleError = console.error; + console.error = () => {}; + globalThis.fetch = (async () => { + return new Response('failed', { status: 500 }); + }) as typeof globalThis.fetch; + + try { + await assert.rejects(() => setExcludedWords(nextRows), /Stats API error: 500/); + + assert.deepEqual(getExcludedWordsSnapshot(), previousRows); + assert.equal(storage.get(STORAGE_KEY), JSON.stringify(previousRows)); + } finally { + globalThis.fetch = originalFetch; + console.error = originalConsoleError; + restore(); + resetExcludedWordsStoreForTests(); + } +}); + +test('initializeExcludedWordsStore retries after transient database load failures', async () => { + resetExcludedWordsStoreForTests(); + const { restore } = installLocalStorage(); + const originalFetch = globalThis.fetch; + const originalConsoleError = console.error; + console.error = () => {}; + let calls = 0; + globalThis.fetch = (async () => { + calls += 1; + if (calls === 1) { + return new Response('failed', { status: 500 }); + } + return new Response(JSON.stringify([{ headword: '猫', word: '猫', reading: 'ねこ' }]), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof globalThis.fetch; + + try { + await initializeExcludedWordsStore(); + await initializeExcludedWordsStore(); + + assert.equal(calls, 2); + assert.deepEqual(getExcludedWordsSnapshot(), [{ headword: '猫', word: '猫', reading: 'ねこ' }]); + } finally { + globalThis.fetch = originalFetch; + console.error = originalConsoleError; + restore(); resetExcludedWordsStoreForTests(); } }); diff --git a/stats/src/hooks/useExcludedWords.ts b/stats/src/hooks/useExcludedWords.ts index 97ffb7b6..4af0ca3c 100644 --- a/stats/src/hooks/useExcludedWords.ts +++ b/stats/src/hooks/useExcludedWords.ts @@ -64,12 +64,20 @@ export function getExcludedWordsSnapshot(): ExcludedWord[] { } export async function setExcludedWords(words: ExcludedWord[]): Promise { - revision += 1; + const previousWords = [...load()]; + const previousRevision = revision; + const writeRevision = previousRevision + 1; + revision = writeRevision; applyWords(words); try { await apiClient.setExcludedWords(words); } catch (error) { + if (revision === writeRevision) { + revision = previousRevision; + applyWords(previousWords); + } console.error('Failed to persist excluded words to stats database', error); + throw error; } } @@ -82,17 +90,25 @@ export function initializeExcludedWordsStore(): Promise { try { dbWords = await apiClient.getExcludedWords(); } catch (error) { + initialized = null; console.error('Failed to load excluded words from stats database', error); return; } - if (revision !== startRevision) return; + if (revision !== startRevision) { + initialized = null; + return; + } if (dbWords.length > 0) { applyWords(dbWords); return; } if (localWords.length > 0) { - await setExcludedWords(localWords); + try { + await setExcludedWords(localWords); + } catch { + initialized = null; + } return; } applyWords([]);