From 8f39416ff5ea38b546521ccccb8e7f48c4767650 Mon Sep 17 00:00:00 2001 From: sudacode Date: Tue, 17 Mar 2026 22:33:08 -0700 Subject: [PATCH] fix(stats): use yomitan tokens for subtitle counts --- ...s-word-counts-with-Yomitan-token-counts.md | 56 +++ .../2026-03-18-stats-yomitan-token-counts.md | 5 + docs/architecture/stats-trends-data-flow.md | 12 +- .../services/__tests__/stats-server.test.ts | 51 ++- .../immersion-tracker-service.test.ts | 134 +++++-- .../services/immersion-tracker-service.ts | 38 +- .../immersion-tracker/__tests__/query.test.ts | 290 +++++++++----- .../services/immersion-tracker/lifetime.ts | 23 +- .../immersion-tracker/maintenance.test.ts | 8 +- .../services/immersion-tracker/maintenance.ts | 18 +- src/core/services/immersion-tracker/query.ts | 75 ++-- .../services/immersion-tracker/reducer.ts | 11 - .../services/immersion-tracker/session.ts | 2 - .../immersion-tracker/storage-session.test.ts | 3 +- .../services/immersion-tracker/storage.ts | 27 +- src/core/services/immersion-tracker/types.ts | 23 +- .../components/anime/AnimeOverviewStats.tsx | 26 +- stats/src/components/anime/EpisodeDetail.tsx | 6 +- stats/src/components/anime/EpisodeList.tsx | 4 +- .../components/library/MediaDetailView.tsx | 5 +- stats/src/components/library/MediaHeader.tsx | 17 +- stats/src/components/overview/OverviewTab.tsx | 15 +- .../components/overview/RecentSessions.tsx | 27 +- .../src/components/sessions/SessionDetail.tsx | 353 +++++++++++++----- stats/src/components/sessions/SessionRow.tsx | 4 +- stats/src/components/trends/TrendsTab.tsx | 103 ++++- stats/src/lib/dashboard-data.test.ts | 58 ++- stats/src/lib/dashboard-data.ts | 8 +- stats/src/lib/media-session-list.test.tsx | 5 +- stats/src/lib/session-detail.test.tsx | 32 +- stats/src/lib/session-word-count.ts | 3 +- stats/src/lib/stats-ui-navigation.test.tsx | 2 +- stats/src/lib/yomitan-lookup.test.tsx | 29 +- stats/src/lib/yomitan-lookup.ts | 10 +- stats/src/types/stats.ts | 15 +- 35 files changed, 991 insertions(+), 507 deletions(-) create mode 100644 backlog/tasks/task-189 - Replace-stats-word-counts-with-Yomitan-token-counts.md create mode 100644 changes/2026-03-18-stats-yomitan-token-counts.md diff --git a/backlog/tasks/task-189 - Replace-stats-word-counts-with-Yomitan-token-counts.md b/backlog/tasks/task-189 - Replace-stats-word-counts-with-Yomitan-token-counts.md new file mode 100644 index 0000000..3d42d90 --- /dev/null +++ b/backlog/tasks/task-189 - Replace-stats-word-counts-with-Yomitan-token-counts.md @@ -0,0 +1,56 @@ +--- +id: TASK-189 +title: Replace stats word counts with Yomitan token counts +status: Done +assignee: + - codex +created_date: '2026-03-18 01:35' +updated_date: '2026-03-18 05:28' +labels: + - stats + - tokenizer + - bug +milestone: m-1 +dependencies: [] +references: + - src/core/services/immersion-tracker-service.ts + - src/core/services/immersion-tracker/reducer.ts + - src/core/services/immersion-tracker/storage.ts + - src/core/services/immersion-tracker/query.ts + - src/core/services/immersion-tracker/lifetime.ts + - stats/src/components + - stats/src/lib/yomitan-lookup.ts +priority: medium +ordinal: 100500 +--- + +## Description + + +Replace heuristic immersion stats word counting with Yomitan token counts. Session/media/anime stats should use the exact merged Yomitan token stream as the denominator and display metric, with no whitespace/CJK-character fallback and no active `wordsSeen` concept in the runtime, storage, API, or stats UI. + + +## Acceptance Criteria + +- [x] #1 `recordSubtitleLine` derives session count deltas from Yomitan token arrays instead of `calculateTextMetrics`. +- [x] #2 Active immersion tracking/storage/query code no longer depends on `wordsSeen` / `totalWordsSeen` fields for stats behavior. +- [x] #3 Stats UI labels and lookup-rate copy refer to tokens instead of words where those counts are shown to users. +- [x] #4 Regression tests cover token-count sourcing, zero-count behavior when tokenization payload is absent, and updated stats copy. +- [x] #5 A changelog fragment documents the user-visible stats denominator change. + + +## Implementation Plan + + +1. Add failing tracker tests proving subtitle count metrics come from Yomitan token arrays and stay zero when tokenization is absent. +2. Add failing stats UI tests for token-based copy and token-count display helpers. +3. Remove `wordsSeen` from active tracker/session/query/type paths and use `tokensSeen` as the single stats count field. +4. Update stats UI labels and lookup-rate copy from words to tokens. +5. Run targeted verification, then add the changelog fragment and any needed docs update. + + +## Outcome + + +Completed. Stats subtitle counts now come directly from Yomitan merged-token counts, `wordsSeen` is removed from the active tracker/storage/query/UI path, token-facing copy is updated, and focused regression coverage plus `bun run typecheck` are green. + diff --git a/changes/2026-03-18-stats-yomitan-token-counts.md b/changes/2026-03-18-stats-yomitan-token-counts.md new file mode 100644 index 0000000..01be369 --- /dev/null +++ b/changes/2026-03-18-stats-yomitan-token-counts.md @@ -0,0 +1,5 @@ +type: fixed +area: stats + +- Replaced heuristic stats word counts with Yomitan token counts, so session, media, anime, and trend subtitle totals now come directly from parsed subtitle tokens. +- Updated stats UI labels and lookup-rate copy to refer to tokens instead of words where those counts are shown. diff --git a/docs/architecture/stats-trends-data-flow.md b/docs/architecture/stats-trends-data-flow.md index 06cf736..93edc58 100644 --- a/docs/architecture/stats-trends-data-flow.md +++ b/docs/architecture/stats-trends-data-flow.md @@ -16,8 +16,8 @@ Trend charts now consume one chart-oriented backend payload from `/api/stats/tre - rollup-backed: - activity charts - - cumulative watch/cards/words/sessions trends - - per-anime watch/cards/words/episodes series + - cumulative watch/cards/tokens/sessions trends + - per-anime watch/cards/tokens/episodes series - session-metric-backed: - lookup trends - lookup rate trends @@ -25,6 +25,14 @@ Trend charts now consume one chart-oriented backend payload from `/api/stats/tre - vocabulary-backed: - new-words trend +## Metric Semantics + +- subtitle-count stats now use Yomitan merged-token counts as the source of truth +- `tokensSeen` is the only active subtitle-count metric in tracker/session/rollup/query paths +- no whitespace/CJK-character fallback remains in the live stats path + ## Contract The stats UI should treat the trends payload as chart-ready data. Presentation-only work in the client is fine, but rebuilding the main trend datasets from raw sessions should stay out of the render path. + +For session detail timelines, omitting `limit` now means "return the full retained session telemetry/history". Explicit `limit` remains available for bounded callers, but the default stats UI path should not trim long sessions to the newest 200 samples. diff --git a/src/core/services/__tests__/stats-server.test.ts b/src/core/services/__tests__/stats-server.test.ts index d49dcaa..807ce9b 100644 --- a/src/core/services/__tests__/stats-server.test.ts +++ b/src/core/services/__tests__/stats-server.test.ts @@ -18,7 +18,6 @@ const SESSION_SUMMARIES = [ totalWatchedMs: 60_000, activeWatchedMs: 50_000, linesSeen: 10, - wordsSeen: 100, tokensSeen: 80, cardsMined: 2, lookupCount: 5, @@ -34,11 +33,10 @@ const DAILY_ROLLUPS = [ totalSessions: 1, totalActiveMin: 10, totalLinesSeen: 10, - totalWordsSeen: 100, totalTokensSeen: 80, totalCards: 2, cardsPerHour: 12, - wordsPerMin: 10, + tokensPerMin: 10, lookupHitRate: 0.8, }, ]; @@ -96,7 +94,7 @@ const ANIME_LIBRARY = [ totalSessions: 3, totalActiveMs: 180_000, totalCards: 5, - totalWordsSeen: 300, + totalTokensSeen: 300, episodeCount: 2, episodesTotal: 25, lastWatchedMs: Date.now(), @@ -113,7 +111,7 @@ const ANIME_DETAIL = { totalSessions: 3, totalActiveMs: 180_000, totalCards: 5, - totalWordsSeen: 300, + totalTokensSeen: 300, totalLinesSeen: 50, totalLookupCount: 20, totalLookupHits: 15, @@ -198,7 +196,7 @@ const ANIME_EPISODES = [ totalSessions: 1, totalActiveMs: 90_000, totalCards: 3, - totalWordsSeen: 150, + totalTokensSeen: 150, lastWatchedMs: Date.now(), }, ]; @@ -349,6 +347,47 @@ describe('stats server API routes', () => { assert.ok(Array.isArray(body)); }); + it('GET /api/stats/sessions/:id/events forwards event type filters to the tracker', async () => { + let seenSessionId = 0; + let seenLimit = 0; + let seenTypes: number[] | undefined; + const app = createStatsApp( + createMockTracker({ + getSessionEvents: async (sessionId: number, limit?: number, eventTypes?: number[]) => { + seenSessionId = sessionId; + seenLimit = limit ?? 0; + seenTypes = eventTypes; + return []; + }, + }), + ); + + const res = await app.request('/api/stats/sessions/7/events?limit=12&types=4,5,9'); + assert.equal(res.status, 200); + assert.equal(seenSessionId, 7); + assert.equal(seenLimit, 12); + assert.deepEqual(seenTypes, [4, 5, 9]); + }); + + it('GET /api/stats/sessions/:id/timeline requests the full session when no limit is provided', async () => { + let seenSessionId = 0; + let seenLimit: number | undefined; + const app = createStatsApp( + createMockTracker({ + getSessionTimeline: async (sessionId: number, limit?: number) => { + seenSessionId = sessionId; + seenLimit = limit; + return []; + }, + }), + ); + + const res = await app.request('/api/stats/sessions/7/timeline'); + assert.equal(res.status, 200); + assert.equal(seenSessionId, 7); + assert.equal(seenLimit, undefined); + }); + it('GET /api/stats/sessions/:id/known-words-timeline preserves line positions and counts known occurrences', async () => { await withTempDir(async (dir) => { const cachePath = path.join(dir, 'known-words.json'); diff --git a/src/core/services/immersion-tracker-service.test.ts b/src/core/services/immersion-tracker-service.test.ts index ae6e048..e22fa93 100644 --- a/src/core/services/immersion-tracker-service.test.ts +++ b/src/core/services/immersion-tracker-service.test.ts @@ -218,13 +218,13 @@ test('finalize updates lifetime summary rows from final session metrics', async } | null; const mediaRow = db .prepare( - 'SELECT total_sessions, total_cards, total_active_ms, total_words_seen, total_lines_seen FROM imm_lifetime_media WHERE video_id = ?', + 'SELECT total_sessions, total_cards, total_active_ms, total_tokens_seen, total_lines_seen FROM imm_lifetime_media WHERE video_id = ?', ) .get(videoId) as { total_sessions: number; total_cards: number; total_active_ms: number; - total_words_seen: number; + total_tokens_seen: number; total_lines_seen: number; } | null; const animeIdRow = db @@ -675,7 +675,6 @@ test('startup finalizes stale active sessions and applies lifetime summaries', a total_watched_ms, active_watched_ms, lines_seen, - words_seen, tokens_seen, cards_mined, lookup_count, @@ -691,7 +690,6 @@ test('startup finalizes stale active sessions and applies lifetime summaries', a 5000, 4000, 12, - 90, 120, 2, 5, @@ -711,7 +709,7 @@ test('startup finalizes stale active sessions and applies lifetime summaries', a const sessionRow = restartedApi.db .prepare( ` - SELECT ended_at_ms, status, active_watched_ms, words_seen, cards_mined + SELECT ended_at_ms, status, active_watched_ms, tokens_seen, cards_mined FROM imm_sessions WHERE session_id = 1 `, @@ -720,7 +718,7 @@ test('startup finalizes stale active sessions and applies lifetime summaries', a ended_at_ms: number | null; status: number; active_watched_ms: number; - words_seen: number; + tokens_seen: number; cards_mined: number; } | null; const globalRow = restartedApi.db @@ -754,7 +752,7 @@ test('startup finalizes stale active sessions and applies lifetime summaries', a assert.ok(Number(sessionRow?.ended_at_ms ?? 0) >= sampleMs); assert.equal(sessionRow?.status, 2); assert.equal(sessionRow?.active_watched_ms, 4000); - assert.equal(sessionRow?.words_seen, 90); + assert.equal(sessionRow?.tokens_seen, 120); assert.equal(sessionRow?.cards_mined, 2); assert.ok(globalRow); @@ -782,7 +780,18 @@ test('persists and retrieves minimum immersion tracking fields', async () => { tracker = new Ctor({ dbPath }); tracker.handleMediaChange('/tmp/episode-3.mkv', 'Episode 3'); - tracker.recordSubtitleLine('alpha beta', 0, 1.2); + tracker.recordSubtitleLine('alpha beta', 0, 1.2, [ + makeMergedToken({ + surface: 'alpha', + headword: 'alpha', + reading: 'alpha', + }), + makeMergedToken({ + surface: 'beta', + headword: 'beta', + reading: 'beta', + }), + ]); tracker.recordCardsMined(2); tracker.recordLookup(true); tracker.recordPlaybackPosition(12.5); @@ -811,14 +820,13 @@ test('persists and retrieves minimum immersion tracking fields', async () => { } | null; const telemetryRow = db .prepare( - `SELECT lines_seen, words_seen, tokens_seen, cards_mined + `SELECT lines_seen, tokens_seen, cards_mined FROM imm_session_telemetry ORDER BY sample_ms DESC, telemetry_id DESC LIMIT 1`, ) .get() as { lines_seen: number; - words_seen: number; tokens_seen: number; cards_mined: number; } | null; @@ -831,7 +839,6 @@ test('persists and retrieves minimum immersion tracking fields', async () => { assert.ok(telemetryRow); assert.ok(Number(telemetryRow?.lines_seen ?? 0) >= 1); - assert.ok(Number(telemetryRow?.words_seen ?? 0) >= 2); assert.ok(Number(telemetryRow?.tokens_seen ?? 0) >= 2); assert.ok(Number(telemetryRow?.cards_mined ?? 0) >= 2); } finally { @@ -1062,6 +1069,87 @@ test('recordSubtitleLine persists counted allowed tokenized vocabulary rows and } }); +test('recordSubtitleLine counts exact Yomitan tokens for session metrics', async () => { + const dbPath = makeDbPath(); + let tracker: ImmersionTrackerService | null = null; + + try { + const Ctor = await loadTrackerCtor(); + tracker = new Ctor({ dbPath }); + + tracker.handleMediaChange('/tmp/token-counting.mkv', 'Token Counting'); + tracker.recordSubtitleLine('猫 猫 日 日 は 知っている', 0, 1, [ + makeMergedToken({ + surface: '猫', + headword: '猫', + reading: 'ねこ', + partOfSpeech: PartOfSpeech.noun, + pos1: '名詞', + }), + makeMergedToken({ + surface: '猫', + headword: '猫', + reading: 'ねこ', + partOfSpeech: PartOfSpeech.noun, + pos1: '名詞', + }), + makeMergedToken({ + surface: 'は', + headword: 'は', + reading: 'は', + partOfSpeech: PartOfSpeech.particle, + pos1: '助詞', + }), + makeMergedToken({ + surface: '知っている', + headword: '知る', + reading: 'しっている', + partOfSpeech: PartOfSpeech.other, + pos1: '動詞', + }), + ]); + + const privateApi = tracker as unknown as { + flushTelemetry: (force?: boolean) => void; + flushNow: () => void; + }; + privateApi.flushTelemetry(true); + privateApi.flushNow(); + + const summaries = await tracker.getSessionSummaries(10); + assert.equal(summaries[0]?.tokensSeen, 4); + } finally { + tracker?.destroy(); + cleanupDbPath(dbPath); + } +}); + +test('recordSubtitleLine leaves session token counts at zero when tokenization is unavailable', async () => { + const dbPath = makeDbPath(); + let tracker: ImmersionTrackerService | null = null; + + try { + const Ctor = await loadTrackerCtor(); + tracker = new Ctor({ dbPath }); + + tracker.handleMediaChange('/tmp/no-tokenization.mkv', 'No Tokenization'); + tracker.recordSubtitleLine('alpha beta gamma', 0, 1.2, null); + + const privateApi = tracker as unknown as { + flushTelemetry: (force?: boolean) => void; + flushNow: () => void; + }; + privateApi.flushTelemetry(true); + privateApi.flushNow(); + + const summaries = await tracker.getSessionSummaries(10); + assert.equal(summaries[0]?.tokensSeen, 0); + } finally { + tracker?.destroy(); + cleanupDbPath(dbPath); + } +}); + test('subtitle-line event payload omits duplicated subtitle text', async () => { const dbPath = makeDbPath(); let tracker: ImmersionTrackerService | null = null; @@ -1094,11 +1182,11 @@ test('subtitle-line event payload omits duplicated subtitle text', async () => { assert.ok(row?.payloadJson); const parsed = JSON.parse(row?.payloadJson ?? '{}') as { event?: string; - words?: number; + tokens?: number; text?: string; }; assert.equal(parsed.event, 'subtitle-line'); - assert.equal(typeof parsed.words, 'number'); + assert.equal(typeof parsed.tokens, 'number'); assert.equal('text' in parsed, false); } finally { tracker?.destroy(); @@ -1548,12 +1636,11 @@ test('zero retention days disables prune checks while preserving rollups', async total_sessions, total_active_min, total_lines_seen, - total_words_seen, total_tokens_seen, total_cards ) VALUES - (${insertedDailyRollupKeys[0]}, 1, 1, 1, 1, 1, 1, 1), - (${insertedDailyRollupKeys[1]}, 1, 1, 1, 1, 1, 1, 1) + (${insertedDailyRollupKeys[0]}, 1, 1, 1, 1, 1, 1), + (${insertedDailyRollupKeys[1]}, 1, 1, 1, 1, 1, 1) `); privateApi.db.exec(` INSERT INTO imm_monthly_rollups ( @@ -1562,14 +1649,13 @@ test('zero retention days disables prune checks while preserving rollups', async total_sessions, total_active_min, total_lines_seen, - total_words_seen, total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE ) VALUES - (${insertedMonthlyRollupKeys[0]}, 1, 1, 1, 1, 1, 1, 1, ${olderMs}, ${olderMs}), - (${insertedMonthlyRollupKeys[1]}, 1, 1, 1, 1, 1, 1, 1, ${oldMs}, ${oldMs}) + (${insertedMonthlyRollupKeys[0]}, 1, 1, 1, 1, 1, 1, ${olderMs}, ${olderMs}), + (${insertedMonthlyRollupKeys[1]}, 1, 1, 1, 1, 1, 1, ${oldMs}, ${oldMs}) `); privateApi.runMaintenance(); @@ -1668,7 +1754,6 @@ test('monthly rollups are grouped by calendar month', async () => { total_watched_ms, active_watched_ms, lines_seen, - words_seen, tokens_seen, cards_mined, lookup_count, @@ -1685,7 +1770,6 @@ test('monthly rollups are grouped by calendar month', async () => { 5000, 1, 2, - 2, 0, 0, 0, @@ -1725,7 +1809,6 @@ test('monthly rollups are grouped by calendar month', async () => { total_watched_ms, active_watched_ms, lines_seen, - words_seen, tokens_seen, cards_mined, lookup_count, @@ -1742,7 +1825,6 @@ test('monthly rollups are grouped by calendar month', async () => { 4000, 2, 3, - 3, 1, 1, 1, @@ -1786,13 +1868,12 @@ test('flushSingle reuses cached prepared statements', async () => { lineIndex?: number | null; segmentStartMs?: number | null; segmentEndMs?: number | null; - wordsDelta?: number; + tokensDelta?: number; cardsDelta?: number; payloadJson?: string | null; totalWatchedMs?: number; activeWatchedMs?: number; linesSeen?: number; - wordsSeen?: number; tokensSeen?: number; cardsMined?: number; lookupCount?: number; @@ -1862,7 +1943,6 @@ test('flushSingle reuses cached prepared statements', async () => { totalWatchedMs: 1000, activeWatchedMs: 1000, linesSeen: 1, - wordsSeen: 2, tokensSeen: 2, cardsMined: 0, lookupCount: 0, @@ -1882,7 +1962,7 @@ test('flushSingle reuses cached prepared statements', async () => { lineIndex: 1, segmentStartMs: 0, segmentEndMs: 1000, - wordsDelta: 2, + tokensDelta: 2, cardsDelta: 0, payloadJson: '{"event":"subtitle-line"}', }); diff --git a/src/core/services/immersion-tracker-service.ts b/src/core/services/immersion-tracker-service.ts index 15c7b6e..613e5e5 100644 --- a/src/core/services/immersion-tracker-service.ts +++ b/src/core/services/immersion-tracker-service.ts @@ -80,7 +80,6 @@ import { } from './immersion-tracker/query'; import { buildVideoKey, - calculateTextMetrics, deriveCanonicalTitle, isKanji, isRemoteSource, @@ -334,7 +333,7 @@ export class ImmersionTrackerService { return getSessionSummaries(this.db, limit); } - async getSessionTimeline(sessionId: number, limit = 200): Promise { + async getSessionTimeline(sessionId: number, limit?: number): Promise { return getSessionTimeline(this.db, sessionId, limit); } @@ -419,8 +418,12 @@ export class ImmersionTrackerService { return getKanjiOccurrences(this.db, kanji, limit, offset); } - async getSessionEvents(sessionId: number, limit = 500): Promise { - return getSessionEvents(this.db, sessionId, limit); + async getSessionEvents( + sessionId: number, + limit = 500, + eventTypes?: number[], + ): Promise { + return getSessionEvents(this.db, sessionId, limit, eventTypes); } async getMediaLibrary(): Promise { @@ -747,11 +750,10 @@ export class ImmersionTrackerService { const nowMs = Date.now(); const nowSec = nowMs / 1000; - const metrics = calculateTextMetrics(cleaned); + const tokenCount = tokens?.length ?? 0; this.sessionState.currentLineIndex += 1; this.sessionState.linesSeen += 1; - this.sessionState.wordsSeen += metrics.words; - this.sessionState.tokensSeen += metrics.tokens; + this.sessionState.tokensSeen += tokenCount; this.sessionState.pendingTelemetry = true; const wordOccurrences = new Map(); @@ -821,13 +823,13 @@ export class ImmersionTrackerService { lineIndex: this.sessionState.currentLineIndex, segmentStartMs: secToMs(startSec), segmentEndMs: secToMs(endSec), - wordsDelta: metrics.words, + tokensDelta: tokenCount, cardsDelta: 0, eventType: EVENT_SUBTITLE_LINE, payloadJson: sanitizePayload( { event: 'subtitle-line', - words: metrics.words, + tokens: tokenCount, }, this.maxPayloadBytes, ), @@ -876,7 +878,7 @@ export class ImmersionTrackerService { sessionId: this.sessionState.sessionId, sampleMs: nowMs, eventType: EVENT_SEEK_FORWARD, - wordsDelta: 0, + tokensDelta: 0, cardsDelta: 0, segmentStartMs: this.sessionState.lastMediaMs, segmentEndMs: mediaMs, @@ -896,7 +898,7 @@ export class ImmersionTrackerService { sessionId: this.sessionState.sessionId, sampleMs: nowMs, eventType: EVENT_SEEK_BACKWARD, - wordsDelta: 0, + tokensDelta: 0, cardsDelta: 0, segmentStartMs: this.sessionState.lastMediaMs, segmentEndMs: mediaMs, @@ -940,7 +942,7 @@ export class ImmersionTrackerService { sampleMs: nowMs, eventType: EVENT_PAUSE_START, cardsDelta: 0, - wordsDelta: 0, + tokensDelta: 0, payloadJson: sanitizePayload({ paused: true }, this.maxPayloadBytes), }); } else { @@ -955,7 +957,7 @@ export class ImmersionTrackerService { sampleMs: nowMs, eventType: EVENT_PAUSE_END, cardsDelta: 0, - wordsDelta: 0, + tokensDelta: 0, payloadJson: sanitizePayload({ paused: false }, this.maxPayloadBytes), }); } @@ -976,7 +978,7 @@ export class ImmersionTrackerService { sampleMs: Date.now(), eventType: EVENT_LOOKUP, cardsDelta: 0, - wordsDelta: 0, + tokensDelta: 0, payloadJson: sanitizePayload( { hit, @@ -996,7 +998,7 @@ export class ImmersionTrackerService { sampleMs: Date.now(), eventType: EVENT_YOMITAN_LOOKUP, cardsDelta: 0, - wordsDelta: 0, + tokensDelta: 0, payloadJson: null, }); } @@ -1010,7 +1012,7 @@ export class ImmersionTrackerService { sessionId: this.sessionState.sessionId, sampleMs: Date.now(), eventType: EVENT_CARD_MINED, - wordsDelta: 0, + tokensDelta: 0, cardsDelta: count, payloadJson: sanitizePayload( { cardsMined: count, ...(noteIds?.length ? { noteIds } : {}) }, @@ -1029,7 +1031,7 @@ export class ImmersionTrackerService { sampleMs: Date.now(), eventType: EVENT_MEDIA_BUFFER, cardsDelta: 0, - wordsDelta: 0, + tokensDelta: 0, payloadJson: sanitizePayload( { buffer: true, @@ -1062,7 +1064,6 @@ export class ImmersionTrackerService { totalWatchedMs: this.sessionState.totalWatchedMs, activeWatchedMs: this.sessionState.activeWatchedMs, linesSeen: this.sessionState.linesSeen, - wordsSeen: this.sessionState.wordsSeen, tokensSeen: this.sessionState.tokensSeen, cardsMined: this.sessionState.cardsMined, lookupCount: this.sessionState.lookupCount, @@ -1191,7 +1192,6 @@ export class ImmersionTrackerService { totalWatchedMs: 0, activeWatchedMs: 0, linesSeen: 0, - wordsSeen: 0, tokensSeen: 0, cardsMined: 0, lookupCount: 0, diff --git a/src/core/services/immersion-tracker/__tests__/query.test.ts b/src/core/services/immersion-tracker/__tests__/query.test.ts index b7bfbe2..e8c767a 100644 --- a/src/core/services/immersion-tracker/__tests__/query.test.ts +++ b/src/core/services/immersion-tracker/__tests__/query.test.ts @@ -32,11 +32,17 @@ import { getVocabularyStats, getKanjiStats, getSessionEvents, + getSessionTimeline, getSessionWordsByLine, getWordOccurrences, upsertCoverArt, } from '../query.js'; -import { SOURCE_TYPE_LOCAL, EVENT_SUBTITLE_LINE } from '../types.js'; +import { + SOURCE_TYPE_LOCAL, + EVENT_CARD_MINED, + EVENT_SUBTITLE_LINE, + EVENT_YOMITAN_LOOKUP, +} from '../types.js'; function makeDbPath(): string { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-imm-query-test-')); @@ -99,7 +105,6 @@ test('getSessionSummaries returns sessionId and canonicalTitle', () => { 2_500, 5, 10, - 10, 1, 2, 1, @@ -124,7 +129,6 @@ test('getSessionSummaries returns sessionId and canonicalTitle', () => { assert.equal(row.linesSeen, 5); assert.equal(row.totalWatchedMs, 3_000); assert.equal(row.activeWatchedMs, 2_500); - assert.equal(row.wordsSeen, 10); assert.equal(row.tokensSeen, 10); assert.equal(row.lookupCount, 2); assert.equal(row.lookupHits, 1); @@ -135,6 +139,57 @@ test('getSessionSummaries returns sessionId and canonicalTitle', () => { } }); +test('getSessionTimeline returns the full session when no limit is provided', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const stmts = createTrackerPreparedStatements(db); + + const videoId = getOrCreateVideoRecord(db, 'local:/tmp/full-timeline-test.mkv', { + canonicalTitle: 'Full Timeline Test', + sourcePath: '/tmp/full-timeline-test.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + + const startedAtMs = 2_000_000; + const { sessionId } = startSessionRecord(db, videoId, startedAtMs); + + for (let sample = 0; sample < 205; sample += 1) { + const sampleMs = startedAtMs + sample * 500; + stmts.telemetryInsertStmt.run( + sessionId, + sampleMs, + sample * 500, + sample * 450, + sample, + sample * 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + sampleMs, + sampleMs, + ); + } + + const rows = getSessionTimeline(db, sessionId); + + assert.equal(rows.length, 205); + assert.equal(rows[0]?.linesSeen, 204); + assert.equal(rows.at(-1)?.linesSeen, 0); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('getDailyRollups limits by distinct days (not rows)', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -146,15 +201,15 @@ test('getDailyRollups limits by distinct days (not rows)', () => { ` INSERT INTO imm_daily_rollups ( rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, - total_words_seen, total_tokens_seen, total_cards - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + total_tokens_seen, total_cards + ) VALUES (?, ?, ?, ?, ?, ?, ?) `, ); - insert.run(10, 1, 1, 1, 0, 0, 0, 2); - insert.run(10, 2, 1, 1, 0, 0, 0, 3); - insert.run(9, 1, 1, 1, 0, 0, 0, 1); - insert.run(8, 1, 1, 1, 0, 0, 0, 1); + insert.run(10, 1, 1, 1, 0, 0, 2); + insert.run(10, 2, 1, 1, 0, 0, 3); + insert.run(9, 1, 1, 1, 0, 0, 1); + insert.run(8, 1, 1, 1, 0, 0, 1); const rows = getDailyRollups(db, 2); assert.equal(rows.length, 3); @@ -213,12 +268,11 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => { startedAtMs, activeWatchedMs, cardsMined, - wordsSeen, tokensSeen, yomitanLookupCount, ] of [ - [sessionOne.sessionId, dayOneStart, 30 * 60_000, 2, 100, 120, 8], - [sessionTwo.sessionId, dayTwoStart, 45 * 60_000, 3, 120, 140, 10], + [sessionOne.sessionId, dayOneStart, 30 * 60_000, 2, 120, 8], + [sessionTwo.sessionId, dayTwoStart, 45 * 60_000, 3, 140, 10], ] as const) { stmts.telemetryInsertStmt.run( sessionId, @@ -226,7 +280,6 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => { activeWatchedMs, activeWatchedMs, 10, - wordsSeen, tokensSeen, cardsMined, 0, @@ -248,7 +301,6 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => { total_watched_ms = ?, active_watched_ms = ?, lines_seen = ?, - words_seen = ?, tokens_seen = ?, cards_mined = ?, yomitan_lookup_count = ? @@ -259,7 +311,6 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => { activeWatchedMs, activeWatchedMs, 10, - wordsSeen, tokensSeen, cardsMined, yomitanLookupCount, @@ -271,19 +322,19 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => { ` INSERT INTO imm_daily_rollups ( rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, - total_words_seen, total_tokens_seen, total_cards - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + total_tokens_seen, total_cards + ) VALUES (?, ?, ?, ?, ?, ?, ?) `, - ).run(Math.floor(dayOneStart / 86_400_000), videoId, 1, 30, 10, 100, 120, 2); + ).run(Math.floor(dayOneStart / 86_400_000), videoId, 1, 30, 10, 120, 2); db.prepare( ` INSERT INTO imm_daily_rollups ( rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, - total_words_seen, total_tokens_seen, total_cards - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + total_tokens_seen, total_cards + ) VALUES (?, ?, ?, ?, ?, ?, ?) `, - ).run(Math.floor(dayTwoStart / 86_400_000), videoId, 1, 45, 10, 120, 140, 3); + ).run(Math.floor(dayTwoStart / 86_400_000), videoId, 1, 45, 10, 140, 3); db.prepare( ` @@ -349,14 +400,14 @@ test('getQueryHints reads all-time totals from lifetime summary', () => { ` INSERT INTO imm_daily_rollups ( rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, - total_words_seen, total_tokens_seen, total_cards - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + total_tokens_seen, total_cards + ) VALUES (?, ?, ?, ?, ?, ?, ?) `, ); - insert.run(10, 1, 1, 12, 0, 0, 0, 2); - insert.run(10, 2, 1, 11, 0, 0, 0, 3); - insert.run(9, 1, 1, 10, 0, 0, 0, 1); + insert.run(10, 1, 1, 12, 0, 0, 2); + insert.run(10, 2, 1, 11, 0, 0, 3); + insert.run(9, 1, 1, 10, 0, 0, 1); const hints = getQueryHints(db); assert.equal(hints.totalSessions, 4); @@ -394,7 +445,6 @@ test('getSessionSummaries with no telemetry returns zero aggregates', () => { assert.equal(row.totalWatchedMs, 0); assert.equal(row.activeWatchedMs, 0); assert.equal(row.linesSeen, 0); - assert.equal(row.wordsSeen, 0); assert.equal(row.tokensSeen, 0); assert.equal(row.lookupCount, 0); assert.equal(row.lookupHits, 0); @@ -432,7 +482,6 @@ test('getSessionSummaries uses denormalized session metrics for ended sessions w total_watched_ms = ?, active_watched_ms = ?, lines_seen = ?, - words_seen = ?, tokens_seen = ?, cards_mined = ?, lookup_count = ?, @@ -440,7 +489,7 @@ test('getSessionSummaries uses denormalized session metrics for ended sessions w LAST_UPDATE_DATE = ? WHERE session_id = ? `, - ).run(endedAtMs, 8_000, 7_000, 12, 34, 34, 5, 9, 6, endedAtMs, sessionId); + ).run(endedAtMs, 8_000, 7_000, 12, 34, 5, 9, 6, endedAtMs, sessionId); const rows = getSessionSummaries(db, 10); const row = rows.find((r) => r.sessionId === sessionId); @@ -448,7 +497,6 @@ test('getSessionSummaries uses denormalized session metrics for ended sessions w assert.equal(row.totalWatchedMs, 8_000); assert.equal(row.activeWatchedMs, 7_000); assert.equal(row.linesSeen, 12); - assert.equal(row.wordsSeen, 34); assert.equal(row.tokensSeen, 34); assert.equal(row.cardsMined, 5); assert.equal(row.lookupCount, 9); @@ -639,15 +687,15 @@ test('getDailyRollups returns all rows for the most recent rollup days', () => { const insertRollup = db.prepare( ` INSERT INTO imm_daily_rollups ( - rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_words_seen, - total_tokens_seen, total_cards, cards_per_hour, words_per_min, lookup_hit_rate - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, + total_tokens_seen, total_cards, cards_per_hour, tokens_per_min, lookup_hit_rate + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, ); - insertRollup.run(3_000, 1, 1, 10, 20, 30, 40, 2, 0.1, 0.2, 0.3); - insertRollup.run(3_000, 2, 2, 10, 20, 30, 40, 3, 0.1, 0.2, 0.3); - insertRollup.run(2_999, 3, 1, 5, 10, 15, 20, 1, 0.1, 0.2, 0.3); - insertRollup.run(2_998, 4, 1, 5, 10, 15, 20, 1, 0.1, 0.2, 0.3); + insertRollup.run(3_000, 1, 1, 10, 20, 40, 2, 0.1, 0.2, 0.3); + insertRollup.run(3_000, 2, 2, 10, 20, 40, 3, 0.1, 0.2, 0.3); + insertRollup.run(2_999, 3, 1, 5, 10, 20, 1, 0.1, 0.2, 0.3); + insertRollup.run(2_998, 4, 1, 5, 10, 20, 1, 0.1, 0.2, 0.3); const rows = getDailyRollups(db, 1); assert.equal(rows.length, 2); @@ -675,16 +723,16 @@ test('getMonthlyRollups returns all rows for the most recent rollup months', () const insertRollup = db.prepare( ` INSERT INTO imm_monthly_rollups ( - rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, total_words_seen, + rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) `, ); const nowMs = Date.now(); - insertRollup.run(202602, 1, 1, 10, 20, 30, 40, 5, nowMs, nowMs); - insertRollup.run(202602, 2, 1, 10, 20, 30, 40, 6, nowMs, nowMs); - insertRollup.run(202601, 3, 1, 5, 10, 15, 20, 2, nowMs, nowMs); - insertRollup.run(202600, 4, 1, 5, 10, 15, 20, 2, nowMs, nowMs); + insertRollup.run(202602, 1, 1, 10, 20, 40, 5, nowMs, nowMs); + insertRollup.run(202602, 2, 1, 10, 20, 40, 6, nowMs, nowMs); + insertRollup.run(202601, 3, 1, 5, 10, 20, 2, nowMs, nowMs); + insertRollup.run(202600, 4, 1, 5, 10, 20, 2, nowMs, nowMs); const rows = getMonthlyRollups(db, 1); assert.equal(rows.length, 2); @@ -706,9 +754,9 @@ test('getAnimeDailyRollups returns all rows for the most recent rollup days', () const insertRollup = db.prepare( ` INSERT INTO imm_daily_rollups ( - rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_words_seen, - total_tokens_seen, total_cards, cards_per_hour, words_per_min, lookup_hit_rate - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, + total_tokens_seen, total_cards, cards_per_hour, tokens_per_min, lookup_hit_rate + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, ); const animeId = getOrCreateAnimeRecord(db, { @@ -738,9 +786,9 @@ test('getAnimeDailyRollups returns all rows for the most recent rollup days', () video2, ); - insertRollup.run(4_000, video1, 1, 10, 20, 30, 40, 2, 0.1, 0.2, 0.3); - insertRollup.run(4_000, video2, 1, 10, 20, 30, 40, 2, 0.1, 0.2, 0.3); - insertRollup.run(3_999, video1, 1, 10, 20, 30, 40, 2, 0.1, 0.2, 0.3); + insertRollup.run(4_000, video1, 1, 10, 20, 40, 2, 0.1, 0.2, 0.3); + insertRollup.run(4_000, video2, 1, 10, 20, 40, 2, 0.1, 0.2, 0.3); + insertRollup.run(3_999, video1, 1, 10, 20, 40, 2, 0.1, 0.2, 0.3); const rows = getAnimeDailyRollups(db, animeId, 1); assert.equal(rows.length, 2); @@ -1112,6 +1160,78 @@ test('getSessionEvents respects limit parameter', () => { } }); +test('getSessionEvents filters by event type before applying limit', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const stmts = createTrackerPreparedStatements(db); + + const videoId = getOrCreateVideoRecord(db, 'local:/tmp/events-type-filter.mkv', { + canonicalTitle: 'Events Type Filter', + sourcePath: '/tmp/events-type-filter.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + + const startedAtMs = 7_500_000; + const { sessionId } = startSessionRecord(db, videoId, startedAtMs); + + for (let i = 0; i < 5; i += 1) { + stmts.eventInsertStmt.run( + sessionId, + startedAtMs + i * 1_000, + EVENT_SUBTITLE_LINE, + i, + 0, + 500, + 1, + 0, + `{"line":"subtitle-${i}"}`, + startedAtMs + i * 1_000, + startedAtMs + i * 1_000, + ); + } + + stmts.eventInsertStmt.run( + sessionId, + startedAtMs + 10_000, + EVENT_CARD_MINED, + null, + null, + null, + 0, + 1, + '{"cardsMined":1}', + startedAtMs + 10_000, + startedAtMs + 10_000, + ); + + stmts.eventInsertStmt.run( + sessionId, + startedAtMs + 11_000, + EVENT_YOMITAN_LOOKUP, + null, + null, + null, + 0, + 0, + null, + startedAtMs + 11_000, + startedAtMs + 11_000, + ); + + const filtered = getSessionEvents(db, sessionId, 1, [EVENT_CARD_MINED]); + assert.equal(filtered.length, 1); + assert.equal(filtered[0]?.eventType, EVENT_CARD_MINED); + assert.equal(filtered[0]?.payload, '{"cardsMined":1}'); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('getSessionWordsByLine joins word occurrences through imm_words.id', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -1251,7 +1371,6 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () 3_000, 10, 25, - 25, 1, 3, 2, @@ -1270,7 +1389,6 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () 4_000, 11, 27, - 27, 2, 4, 2, @@ -1289,7 +1407,6 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () 5_000, 12, 28, - 28, 3, 5, 4, @@ -1308,7 +1425,6 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () 3_500, 8, 20, - 20, 1, 2, 1, @@ -1329,7 +1445,6 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () total_sessions, total_active_ms, total_cards, - total_words_seen, total_lines_seen, total_tokens_seen, episodes_started, @@ -1338,9 +1453,9 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () last_watched_ms, CREATED_DATE, LAST_UPDATE_DATE - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, - ).run(lwaAnimeId, 3, 12_000, 6, 80, 33, 0, 2, 1, 1_000_000, 1_021_000, now, now); + ).run(lwaAnimeId, 3, 12_000, 6, 33, 80, 2, 1, 1_000_000, 1_021_000, now, now); db.prepare( ` INSERT INTO imm_lifetime_anime ( @@ -1348,7 +1463,6 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () total_sessions, total_active_ms, total_cards, - total_words_seen, total_lines_seen, total_tokens_seen, episodes_started, @@ -1357,9 +1471,9 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () last_watched_ms, CREATED_DATE, LAST_UPDATE_DATE - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, - ).run(frierenAnimeId, 1, 3_500, 1, 20, 8, 0, 1, 1, 1_030_000, 1_030_000, now, now); + ).run(frierenAnimeId, 1, 3_500, 1, 8, 20, 1, 1, 1_030_000, 1_030_000, now, now); const animeLibrary = getAnimeLibrary(db); assert.equal(animeLibrary.length, 2); @@ -1400,7 +1514,7 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () assert.equal(animeDetail?.totalSessions, 3); assert.equal(animeDetail?.totalActiveMs, 12_000); assert.equal(animeDetail?.totalCards, 6); - assert.equal(animeDetail?.totalWordsSeen, 80); + assert.equal(animeDetail?.totalTokensSeen, 80); assert.equal(animeDetail?.totalLinesSeen, 33); assert.equal(animeDetail?.totalLookupCount, 12); assert.equal(animeDetail?.totalLookupHits, 8); @@ -1416,7 +1530,7 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () totalSessions: row.totalSessions, totalActiveMs: row.totalActiveMs, totalCards: row.totalCards, - totalWordsSeen: row.totalWordsSeen, + totalTokensSeen: row.totalTokensSeen, totalYomitanLookupCount: row.totalYomitanLookupCount, })), [ @@ -1427,7 +1541,7 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () totalSessions: 2, totalActiveMs: 7_000, totalCards: 3, - totalWordsSeen: 52, + totalTokensSeen: 52, totalYomitanLookupCount: 0, }, { @@ -1437,7 +1551,7 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () totalSessions: 1, totalActiveMs: 5_000, totalCards: 3, - totalWordsSeen: 28, + totalTokensSeen: 28, totalYomitanLookupCount: 0, }, ], @@ -1506,7 +1620,6 @@ test('anime library and detail still return lifetime rows without retained sessi total_sessions, total_active_ms, total_cards, - total_words_seen, total_lines_seen, total_tokens_seen, episodes_started, @@ -1515,9 +1628,9 @@ test('anime library and detail still return lifetime rows without retained sessi last_watched_ms, CREATED_DATE, LAST_UPDATE_DATE - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, - ).run(animeId, 12, 4_500, 9, 200, 80, 15, 2, 2, 1_000_000, now, now, now); + ).run(animeId, 12, 4_500, 9, 80, 200, 2, 2, 1_000_000, now, now, now); const library = getAnimeLibrary(db); assert.equal(library.length, 1); @@ -1535,7 +1648,7 @@ test('anime library and detail still return lifetime rows without retained sessi assert.equal(detail?.totalSessions, 12); assert.equal(detail?.totalActiveMs, 4_500); assert.equal(detail?.totalCards, 9); - assert.equal(detail?.totalWordsSeen, 200); + assert.equal(detail?.totalTokensSeen, 200); assert.equal(detail?.totalLinesSeen, 80); assert.equal(detail?.episodeCount, 2); assert.equal(detail?.totalLookupCount, 0); @@ -1573,7 +1686,6 @@ test('media library and detail queries read lifetime totals', () => { total_sessions, total_active_ms, total_cards, - total_words_seen, total_lines_seen, total_tokens_seen, completed, @@ -1581,13 +1693,13 @@ test('media library and detail queries read lifetime totals', () => { last_watched_ms, CREATED_DATE, LAST_UPDATE_DATE - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, ); const now = Date.now(); const older = now - 10_000; - insertLifetime.run(mediaOne, 3, 12_000, 4, 180, 10, 20, 1, 1_000, now, now, now, now); - insertLifetime.run(mediaTwo, 1, 2_000, 2, 40, 4, 6, 0, 900, older, now, now); + insertLifetime.run(mediaOne, 3, 12_000, 4, 10, 180, 1, 1_000, now, now, now); + insertLifetime.run(mediaTwo, 1, 2_000, 2, 4, 40, 0, 900, older, now, now); const library = getMediaLibrary(db); assert.equal(library.length, 2); @@ -1598,7 +1710,7 @@ test('media library and detail queries read lifetime totals', () => { totalSessions: row.totalSessions, totalActiveMs: row.totalActiveMs, totalCards: row.totalCards, - totalWordsSeen: row.totalWordsSeen, + totalTokensSeen: row.totalTokensSeen, lastWatchedMs: row.lastWatchedMs, hasCoverArt: row.hasCoverArt, })), @@ -1609,7 +1721,7 @@ test('media library and detail queries read lifetime totals', () => { totalSessions: 3, totalActiveMs: 12_000, totalCards: 4, - totalWordsSeen: 180, + totalTokensSeen: 180, lastWatchedMs: now, hasCoverArt: 0, }, @@ -1619,7 +1731,7 @@ test('media library and detail queries read lifetime totals', () => { totalSessions: 1, totalActiveMs: 2_000, totalCards: 2, - totalWordsSeen: 40, + totalTokensSeen: 40, lastWatchedMs: older, hasCoverArt: 0, }, @@ -1631,7 +1743,7 @@ test('media library and detail queries read lifetime totals', () => { assert.equal(detail.totalSessions, 3); assert.equal(detail.totalActiveMs, 12_000); assert.equal(detail.totalCards, 4); - assert.equal(detail.totalWordsSeen, 180); + assert.equal(detail.totalTokensSeen, 180); assert.equal(detail.totalLinesSeen, 10); } finally { db.close(); @@ -1697,7 +1809,6 @@ test('cover art queries reuse a shared blob across duplicate anime art rows', () total_sessions, total_active_ms, total_cards, - total_words_seen, total_lines_seen, total_tokens_seen, completed, @@ -1705,7 +1816,7 @@ test('cover art queries reuse a shared blob across duplicate anime art rows', () last_watched_ms, CREATED_DATE, LAST_UPDATE_DATE - ) VALUES (?, 1, 1000, 0, 0, 0, 0, 0, ?, ?, ?, ?) + ) VALUES (?, 1, 1000, 0, 0, 0, 0, ?, ?, ?, ?) `, ).run(videoOne, now, now, now, now); db.prepare( @@ -1715,7 +1826,6 @@ test('cover art queries reuse a shared blob across duplicate anime art rows', () total_sessions, total_active_ms, total_cards, - total_words_seen, total_lines_seen, total_tokens_seen, completed, @@ -1723,7 +1833,7 @@ test('cover art queries reuse a shared blob across duplicate anime art rows', () last_watched_ms, CREATED_DATE, LAST_UPDATE_DATE - ) VALUES (?, 1, 1000, 0, 0, 0, 0, 0, ?, ?, ?, ?) + ) VALUES (?, 1, 1000, 0, 0, 0, 0, ?, ?, ?, ?) `, ).run(videoTwo, now, now - 1, now, now); @@ -1823,20 +1933,20 @@ test('anime/media detail and episode queries use ended-session metrics when tele db.prepare( ` INSERT INTO imm_lifetime_anime ( - anime_id, total_sessions, total_active_ms, total_cards, total_words_seen, total_lines_seen, + anime_id, total_sessions, total_active_ms, total_cards, total_lines_seen, total_tokens_seen, episodes_started, episodes_completed, first_watched_ms, last_watched_ms, CREATED_DATE, LAST_UPDATE_DATE - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, - ).run(animeId, 3, 12_000, 6, 60, 24, 60, 2, 2, 1_000_000, 1_020_000, now, now); + ).run(animeId, 3, 12_000, 6, 24, 60, 2, 2, 1_000_000, 1_020_000, now, now); db.prepare( ` INSERT INTO imm_lifetime_media ( - video_id, total_sessions, total_active_ms, total_cards, total_words_seen, total_lines_seen, + video_id, total_sessions, total_active_ms, total_cards, total_lines_seen, total_tokens_seen, completed, first_watched_ms, last_watched_ms, CREATED_DATE, LAST_UPDATE_DATE - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, - ).run(episodeOne, 2, 7_000, 3, 30, 12, 30, 1, 1_000_000, 1_010_000, now, now); + ).run(episodeOne, 2, 7_000, 3, 12, 30, 1, 1_000_000, 1_010_000, now, now); const s1 = startSessionRecord(db, episodeOne, 1_000_000).sessionId; const s2 = startSessionRecord(db, episodeOne, 1_010_000).sessionId; @@ -1849,7 +1959,7 @@ test('anime/media detail and episode queries use ended-session metrics when tele status = 2, active_watched_ms = ?, cards_mined = ?, - words_seen = ?, + tokens_seen = ?, lookup_count = ?, lookup_hits = ?, LAST_UPDATE_DATE = ? @@ -1872,7 +1982,7 @@ test('anime/media detail and episode queries use ended-session metrics when tele totalSessions: row.totalSessions, totalActiveMs: row.totalActiveMs, totalCards: row.totalCards, - totalWordsSeen: row.totalWordsSeen, + totalTokensSeen: row.totalTokensSeen, })), [ { @@ -1880,14 +1990,14 @@ test('anime/media detail and episode queries use ended-session metrics when tele totalSessions: 2, totalActiveMs: 7_000, totalCards: 3, - totalWordsSeen: 30, + totalTokensSeen: 30, }, { videoId: episodeTwo, totalSessions: 1, totalActiveMs: 5_000, totalCards: 3, - totalWordsSeen: 30, + totalTokensSeen: 30, }, ], ); @@ -1897,7 +2007,7 @@ test('anime/media detail and episode queries use ended-session metrics when tele assert.equal(mediaDetail?.totalSessions, 2); assert.equal(mediaDetail?.totalActiveMs, 7_000); assert.equal(mediaDetail?.totalCards, 3); - assert.equal(mediaDetail?.totalWordsSeen, 30); + assert.equal(mediaDetail?.totalTokensSeen, 30); assert.equal(mediaDetail?.totalLookupCount, 9); assert.equal(mediaDetail?.totalLookupHits, 7); assert.equal(mediaDetail?.totalYomitanLookupCount, 0); diff --git a/src/core/services/immersion-tracker/lifetime.ts b/src/core/services/immersion-tracker/lifetime.ts index 9673dad..7f11ca2 100644 --- a/src/core/services/immersion-tracker/lifetime.ts +++ b/src/core/services/immersion-tracker/lifetime.ts @@ -7,7 +7,6 @@ interface TelemetryRow { cards_mined: number | null; lines_seen: number | null; tokens_seen: number | null; - words_seen: number | null; } interface VideoRow { @@ -46,7 +45,6 @@ interface RetainedSessionRow { totalWatchedMs: number; activeWatchedMs: number; linesSeen: number; - wordsSeen: number; tokensSeen: number; cardsMined: number; lookupCount: number; @@ -150,7 +148,6 @@ function toRebuildSessionState(row: RetainedSessionRow): SessionState { totalWatchedMs: Math.max(0, row.totalWatchedMs), activeWatchedMs: Math.max(0, row.activeWatchedMs), linesSeen: Math.max(0, row.linesSeen), - wordsSeen: Math.max(0, row.wordsSeen), tokensSeen: Math.max(0, row.tokensSeen), cardsMined: Math.max(0, row.cardsMined), lookupCount: Math.max(0, row.lookupCount), @@ -176,7 +173,6 @@ function getRetainedStaleActiveSessions(db: DatabaseSync): RetainedSessionRow[] COALESCE(t.total_watched_ms, s.total_watched_ms, 0) AS totalWatchedMs, COALESCE(t.active_watched_ms, s.active_watched_ms, 0) AS activeWatchedMs, COALESCE(t.lines_seen, s.lines_seen, 0) AS linesSeen, - COALESCE(t.words_seen, s.words_seen, 0) AS wordsSeen, COALESCE(t.tokens_seen, s.tokens_seen, 0) AS tokensSeen, COALESCE(t.cards_mined, s.cards_mined, 0) AS cardsMined, COALESCE(t.lookup_count, s.lookup_count, 0) AS lookupCount, @@ -209,7 +205,6 @@ function upsertLifetimeMedia( nowMs: number, activeMs: number, cardsMined: number, - wordsSeen: number, linesSeen: number, tokensSeen: number, completed: number, @@ -223,7 +218,6 @@ function upsertLifetimeMedia( total_sessions, total_active_ms, total_cards, - total_words_seen, total_lines_seen, total_tokens_seen, completed, @@ -232,12 +226,11 @@ function upsertLifetimeMedia( CREATED_DATE, LAST_UPDATE_DATE ) - VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(video_id) DO UPDATE SET total_sessions = total_sessions + 1, total_active_ms = total_active_ms + excluded.total_active_ms, total_cards = total_cards + excluded.total_cards, - total_words_seen = total_words_seen + excluded.total_words_seen, total_lines_seen = total_lines_seen + excluded.total_lines_seen, total_tokens_seen = total_tokens_seen + excluded.total_tokens_seen, completed = MAX(completed, excluded.completed), @@ -259,7 +252,6 @@ function upsertLifetimeMedia( videoId, activeMs, cardsMined, - wordsSeen, linesSeen, tokensSeen, completed, @@ -276,7 +268,6 @@ function upsertLifetimeAnime( nowMs: number, activeMs: number, cardsMined: number, - wordsSeen: number, linesSeen: number, tokensSeen: number, episodesStartedDelta: number, @@ -291,7 +282,6 @@ function upsertLifetimeAnime( total_sessions, total_active_ms, total_cards, - total_words_seen, total_lines_seen, total_tokens_seen, episodes_started, @@ -301,12 +291,11 @@ function upsertLifetimeAnime( CREATED_DATE, LAST_UPDATE_DATE ) - VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(anime_id) DO UPDATE SET total_sessions = total_sessions + 1, total_active_ms = total_active_ms + excluded.total_active_ms, total_cards = total_cards + excluded.total_cards, - total_words_seen = total_words_seen + excluded.total_words_seen, total_lines_seen = total_lines_seen + excluded.total_lines_seen, total_tokens_seen = total_tokens_seen + excluded.total_tokens_seen, episodes_started = episodes_started + excluded.episodes_started, @@ -329,7 +318,6 @@ function upsertLifetimeAnime( animeId, activeMs, cardsMined, - wordsSeen, linesSeen, tokensSeen, episodesStartedDelta, @@ -372,7 +360,6 @@ export function applySessionLifetimeSummary( SELECT active_watched_ms, cards_mined, - words_seen, lines_seen, tokens_seen FROM imm_session_telemetry @@ -407,9 +394,6 @@ export function applySessionLifetimeSummary( const cardsMined = telemetry ? asPositiveNumber(telemetry.cards_mined, session.cardsMined) : session.cardsMined; - const wordsSeen = telemetry - ? asPositiveNumber(telemetry.words_seen, session.wordsSeen) - : session.wordsSeen; const linesSeen = telemetry ? asPositiveNumber(telemetry.lines_seen, session.linesSeen) : session.linesSeen; @@ -470,7 +454,6 @@ export function applySessionLifetimeSummary( nowMs, activeMs, cardsMined, - wordsSeen, linesSeen, tokensSeen, watched > 0 ? 1 : 0, @@ -485,7 +468,6 @@ export function applySessionLifetimeSummary( nowMs, activeMs, cardsMined, - wordsSeen, linesSeen, tokensSeen, isFirstSessionForVideoRun ? 1 : 0, @@ -509,7 +491,6 @@ export function rebuildLifetimeSummaries(db: DatabaseSync): LifetimeRebuildSumma total_watched_ms AS totalWatchedMs, active_watched_ms AS activeWatchedMs, lines_seen AS linesSeen, - words_seen AS wordsSeen, tokens_seen AS tokensSeen, cards_mined AS cardsMined, lookup_count AS lookupCount, diff --git a/src/core/services/immersion-tracker/maintenance.test.ts b/src/core/services/immersion-tracker/maintenance.test.ts index 9ff738b..0b27a2a 100644 --- a/src/core/services/immersion-tracker/maintenance.test.ts +++ b/src/core/services/immersion-tracker/maintenance.test.ts @@ -109,16 +109,16 @@ test('raw retention keeps rollups and rollup retention prunes them separately', 1, ${nowMs - 90 * 86_400_000}, 0, 0, ${nowMs}, ${nowMs} ); INSERT INTO imm_daily_rollups ( - rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_words_seen, + rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards ) VALUES ( - ${oldDay}, 1, 1, 10, 1, 1, 1, 1 + ${oldDay}, 1, 1, 10, 1, 1, 1 ); INSERT INTO imm_monthly_rollups ( - rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, total_words_seen, + rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE ) VALUES ( - ${oldMonth}, 1, 1, 10, 1, 1, 1, 1, ${nowMs}, ${nowMs} + ${oldMonth}, 1, 1, 10, 1, 1, 1, ${nowMs}, ${nowMs} ); `); diff --git a/src/core/services/immersion-tracker/maintenance.ts b/src/core/services/immersion-tracker/maintenance.ts index 9393737..0f767bf 100644 --- a/src/core/services/immersion-tracker/maintenance.ts +++ b/src/core/services/immersion-tracker/maintenance.ts @@ -125,8 +125,8 @@ function upsertDailyRollupsForGroups( const upsertStmt = db.prepare(` INSERT INTO imm_daily_rollups ( rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, - total_words_seen, total_tokens_seen, total_cards, cards_per_hour, - words_per_min, lookup_hit_rate, CREATED_DATE, LAST_UPDATE_DATE + total_tokens_seen, total_cards, cards_per_hour, + tokens_per_min, lookup_hit_rate, CREATED_DATE, LAST_UPDATE_DATE ) SELECT CAST(julianday(s.started_at_ms / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) AS rollup_day, @@ -134,7 +134,6 @@ function upsertDailyRollupsForGroups( COUNT(DISTINCT s.session_id) AS total_sessions, COALESCE(SUM(sm.max_active_ms), 0) / 60000.0 AS total_active_min, COALESCE(SUM(sm.max_lines), 0) AS total_lines_seen, - COALESCE(SUM(sm.max_words), 0) AS total_words_seen, COALESCE(SUM(sm.max_tokens), 0) AS total_tokens_seen, COALESCE(SUM(sm.max_cards), 0) AS total_cards, CASE @@ -144,9 +143,9 @@ function upsertDailyRollupsForGroups( END AS cards_per_hour, CASE WHEN COALESCE(SUM(sm.max_active_ms), 0) > 0 - THEN COALESCE(SUM(sm.max_words), 0) / (COALESCE(SUM(sm.max_active_ms), 0) / 60000.0) + THEN COALESCE(SUM(sm.max_tokens), 0) / (COALESCE(SUM(sm.max_active_ms), 0) / 60000.0) ELSE NULL - END AS words_per_min, + END AS tokens_per_min, CASE WHEN COALESCE(SUM(sm.max_lookups), 0) > 0 THEN CAST(COALESCE(SUM(sm.max_hits), 0) AS REAL) / CAST(SUM(sm.max_lookups) AS REAL) @@ -160,7 +159,6 @@ function upsertDailyRollupsForGroups( t.session_id, MAX(t.active_watched_ms) AS max_active_ms, MAX(t.lines_seen) AS max_lines, - MAX(t.words_seen) AS max_words, MAX(t.tokens_seen) AS max_tokens, MAX(t.cards_mined) AS max_cards, MAX(t.lookup_count) AS max_lookups, @@ -174,11 +172,10 @@ function upsertDailyRollupsForGroups( total_sessions = excluded.total_sessions, total_active_min = excluded.total_active_min, total_lines_seen = excluded.total_lines_seen, - total_words_seen = excluded.total_words_seen, total_tokens_seen = excluded.total_tokens_seen, total_cards = excluded.total_cards, cards_per_hour = excluded.cards_per_hour, - words_per_min = excluded.words_per_min, + tokens_per_min = excluded.tokens_per_min, lookup_hit_rate = excluded.lookup_hit_rate, CREATED_DATE = COALESCE(imm_daily_rollups.CREATED_DATE, excluded.CREATED_DATE), LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE @@ -201,7 +198,7 @@ function upsertMonthlyRollupsForGroups( const upsertStmt = db.prepare(` INSERT INTO imm_monthly_rollups ( rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, - total_words_seen, total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE + total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE ) SELECT CAST(strftime('%Y%m', s.started_at_ms / 1000, 'unixepoch', 'localtime') AS INTEGER) AS rollup_month, @@ -209,7 +206,6 @@ function upsertMonthlyRollupsForGroups( COUNT(DISTINCT s.session_id) AS total_sessions, COALESCE(SUM(sm.max_active_ms), 0) / 60000.0 AS total_active_min, COALESCE(SUM(sm.max_lines), 0) AS total_lines_seen, - COALESCE(SUM(sm.max_words), 0) AS total_words_seen, COALESCE(SUM(sm.max_tokens), 0) AS total_tokens_seen, COALESCE(SUM(sm.max_cards), 0) AS total_cards, ? AS CREATED_DATE, @@ -220,7 +216,6 @@ function upsertMonthlyRollupsForGroups( t.session_id, MAX(t.active_watched_ms) AS max_active_ms, MAX(t.lines_seen) AS max_lines, - MAX(t.words_seen) AS max_words, MAX(t.tokens_seen) AS max_tokens, MAX(t.cards_mined) AS max_cards FROM imm_session_telemetry t @@ -232,7 +227,6 @@ function upsertMonthlyRollupsForGroups( total_sessions = excluded.total_sessions, total_active_min = excluded.total_active_min, total_lines_seen = excluded.total_lines_seen, - total_words_seen = excluded.total_words_seen, total_tokens_seen = excluded.total_tokens_seen, total_cards = excluded.total_cards, CREATED_DATE = COALESCE(imm_monthly_rollups.CREATED_DATE, excluded.CREATED_DATE), diff --git a/src/core/services/immersion-tracker/query.ts b/src/core/services/immersion-tracker/query.ts index 2d5b243..ec19bc0 100644 --- a/src/core/services/immersion-tracker/query.ts +++ b/src/core/services/immersion-tracker/query.ts @@ -77,7 +77,6 @@ const ACTIVE_SESSION_METRICS_CTE = ` MAX(t.total_watched_ms) AS totalWatchedMs, MAX(t.active_watched_ms) AS activeWatchedMs, MAX(t.lines_seen) AS linesSeen, - MAX(t.words_seen) AS wordsSeen, MAX(t.tokens_seen) AS tokensSeen, MAX(t.cards_mined) AS cardsMined, MAX(t.lookup_count) AS lookupCount, @@ -353,7 +352,6 @@ export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummar COALESCE(asm.totalWatchedMs, s.total_watched_ms, 0) AS totalWatchedMs, COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0) AS activeWatchedMs, COALESCE(asm.linesSeen, s.lines_seen, 0) AS linesSeen, - COALESCE(asm.wordsSeen, s.words_seen, 0) AS wordsSeen, COALESCE(asm.tokensSeen, s.tokens_seen, 0) AS tokensSeen, COALESCE(asm.cardsMined, s.cards_mined, 0) AS cardsMined, COALESCE(asm.lookupCount, s.lookup_count, 0) AS lookupCount, @@ -372,15 +370,30 @@ export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummar export function getSessionTimeline( db: DatabaseSync, sessionId: number, - limit = 200, + limit?: number, ): SessionTimelineRow[] { + if (limit === undefined) { + const prepared = db.prepare(` + SELECT + sample_ms AS sampleMs, + total_watched_ms AS totalWatchedMs, + active_watched_ms AS activeWatchedMs, + lines_seen AS linesSeen, + tokens_seen AS tokensSeen, + cards_mined AS cardsMined + FROM imm_session_telemetry + WHERE session_id = ? + ORDER BY sample_ms DESC, telemetry_id DESC + `); + return prepared.all(sessionId) as unknown as SessionTimelineRow[]; + } + const prepared = db.prepare(` SELECT sample_ms AS sampleMs, total_watched_ms AS totalWatchedMs, active_watched_ms AS activeWatchedMs, lines_seen AS linesSeen, - words_seen AS wordsSeen, tokens_seen AS tokensSeen, cards_mined AS cardsMined FROM imm_session_telemetry @@ -609,11 +622,10 @@ export function getDailyRollups(db: DatabaseSync, limit = 60): ImmersionSessionR r.total_sessions AS totalSessions, r.total_active_min AS totalActiveMin, r.total_lines_seen AS totalLinesSeen, - r.total_words_seen AS totalWordsSeen, r.total_tokens_seen AS totalTokensSeen, r.total_cards AS totalCards, r.cards_per_hour AS cardsPerHour, - r.words_per_min AS wordsPerMin, + r.tokens_per_min AS tokensPerMin, r.lookup_hit_rate AS lookupHitRate FROM imm_daily_rollups r WHERE r.rollup_day IN (SELECT rollup_day FROM recent_days) @@ -637,11 +649,10 @@ export function getMonthlyRollups(db: DatabaseSync, limit = 24): ImmersionSessio total_sessions AS totalSessions, total_active_min AS totalActiveMin, total_lines_seen AS totalLinesSeen, - total_words_seen AS totalWordsSeen, total_tokens_seen AS totalTokensSeen, total_cards AS totalCards, 0 AS cardsPerHour, - 0 AS wordsPerMin, + 0 AS tokensPerMin, 0 AS lookupHitRate FROM imm_monthly_rollups WHERE rollup_month IN (SELECT rollup_month FROM recent_months) @@ -670,7 +681,6 @@ interface TrendSessionMetricRow { canonicalTitle: string | null; animeTitle: string | null; activeWatchedMs: number; - wordsSeen: number; tokensSeen: number; cardsMined: number; yomitanLookupCount: number; @@ -760,10 +770,8 @@ function makeTrendLabel(value: number): string { }); } -function getTrendSessionWordCount( - session: Pick, -): number { - return session.tokensSeen > 0 ? session.tokensSeen : session.wordsSeen; +function getTrendSessionWordCount(session: Pick): number { + return session.tokensSeen; } function resolveTrendAnimeTitle(value: { @@ -796,7 +804,7 @@ function buildAggregatedTrendRows(rollups: ImmersionSessionRollupRow[]) { }; existing.activeMin += Math.round(rollup.totalActiveMin); existing.cards += rollup.totalCards; - existing.words += rollup.totalWordsSeen; + existing.words += rollup.totalTokensSeen; existing.sessions += rollup.totalSessions; byKey.set(rollup.rollupDayOrMonth, existing); } @@ -1087,7 +1095,6 @@ function getTrendSessionMetrics( v.canonical_title AS canonicalTitle, a.canonical_title AS animeTitle, COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0) AS activeWatchedMs, - COALESCE(asm.wordsSeen, s.words_seen, 0) AS wordsSeen, COALESCE(asm.tokensSeen, s.tokens_seen, 0) AS tokensSeen, COALESCE(asm.cardsMined, s.cards_mined, 0) AS cardsMined, COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0) AS yomitanLookupCount @@ -1167,7 +1174,7 @@ export function getTrendsDashboard( words: buildPerAnimeFromDailyRollups( dailyRollups, titlesByVideoId, - (rollup) => rollup.totalWordsSeen, + (rollup) => rollup.totalTokensSeen, ), lookups: buildPerAnimeFromSessions( sessions, @@ -1595,12 +1602,25 @@ export function getSessionEvents( db: DatabaseSync, sessionId: number, limit = 500, + eventTypes?: number[], ): SessionEventRow[] { + if (!eventTypes || eventTypes.length === 0) { + const stmt = db.prepare(` + SELECT event_type AS eventType, ts_ms AS tsMs, payload_json AS payload + FROM imm_session_events WHERE session_id = ? ORDER BY ts_ms ASC LIMIT ? + `); + return stmt.all(sessionId, limit) as SessionEventRow[]; + } + + const placeholders = eventTypes.map(() => '?').join(', '); const stmt = db.prepare(` SELECT event_type AS eventType, ts_ms AS tsMs, payload_json AS payload - FROM imm_session_events WHERE session_id = ? ORDER BY ts_ms ASC LIMIT ? + FROM imm_session_events + WHERE session_id = ? AND event_type IN (${placeholders}) + ORDER BY ts_ms ASC + LIMIT ? `); - return stmt.all(sessionId, limit) as SessionEventRow[]; + return stmt.all(sessionId, ...eventTypes, limit) as SessionEventRow[]; } export function getAnimeLibrary(db: DatabaseSync): AnimeLibraryRow[] { @@ -1614,7 +1634,7 @@ export function getAnimeLibrary(db: DatabaseSync): AnimeLibraryRow[] { COALESCE(lm.total_sessions, 0) AS totalSessions, COALESCE(lm.total_active_ms, 0) AS totalActiveMs, COALESCE(lm.total_cards, 0) AS totalCards, - COALESCE(lm.total_words_seen, 0) AS totalWordsSeen, + COALESCE(lm.total_tokens_seen, 0) AS totalTokensSeen, COUNT(DISTINCT v.video_id) AS episodeCount, a.episodes_total AS episodesTotal, COALESCE(lm.last_watched_ms, 0) AS lastWatchedMs @@ -1644,7 +1664,7 @@ export function getAnimeDetail(db: DatabaseSync, animeId: number): AnimeDetailRo COALESCE(lm.total_sessions, 0) AS totalSessions, COALESCE(lm.total_active_ms, 0) AS totalActiveMs, COALESCE(lm.total_cards, 0) AS totalCards, - COALESCE(lm.total_words_seen, 0) AS totalWordsSeen, + COALESCE(lm.total_tokens_seen, 0) AS totalTokensSeen, COALESCE(lm.total_lines_seen, 0) AS totalLinesSeen, COALESCE(SUM(COALESCE(asm.lookupCount, s.lookup_count, 0)), 0) AS totalLookupCount, COALESCE(SUM(COALESCE(asm.lookupHits, s.lookup_hits, 0)), 0) AS totalLookupHits, @@ -1699,7 +1719,7 @@ export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisod COUNT(DISTINCT s.session_id) AS totalSessions, COALESCE(SUM(COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0)), 0) AS totalActiveMs, COALESCE(SUM(COALESCE(asm.cardsMined, s.cards_mined, 0)), 0) AS totalCards, - COALESCE(SUM(COALESCE(asm.wordsSeen, s.words_seen, 0)), 0) AS totalWordsSeen, + COALESCE(SUM(COALESCE(asm.tokensSeen, s.tokens_seen, 0)), 0) AS totalTokensSeen, COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount, MAX(s.started_at_ms) AS lastWatchedMs FROM imm_videos v @@ -1728,7 +1748,7 @@ export function getMediaLibrary(db: DatabaseSync): MediaLibraryRow[] { COALESCE(lm.total_sessions, 0) AS totalSessions, COALESCE(lm.total_active_ms, 0) AS totalActiveMs, COALESCE(lm.total_cards, 0) AS totalCards, - COALESCE(lm.total_words_seen, 0) AS totalWordsSeen, + COALESCE(lm.total_tokens_seen, 0) AS totalTokensSeen, COALESCE(lm.last_watched_ms, 0) AS lastWatchedMs, CASE WHEN ma.cover_blob_hash IS NOT NULL OR ma.cover_blob IS NOT NULL THEN 1 @@ -1754,7 +1774,7 @@ export function getMediaDetail(db: DatabaseSync, videoId: number): MediaDetailRo COALESCE(lm.total_sessions, 0) AS totalSessions, COALESCE(lm.total_active_ms, 0) AS totalActiveMs, COALESCE(lm.total_cards, 0) AS totalCards, - COALESCE(lm.total_words_seen, 0) AS totalWordsSeen, + COALESCE(lm.total_tokens_seen, 0) AS totalTokensSeen, COALESCE(lm.total_lines_seen, 0) AS totalLinesSeen, COALESCE(SUM(COALESCE(asm.lookupCount, s.lookup_count, 0)), 0) AS totalLookupCount, COALESCE(SUM(COALESCE(asm.lookupHits, s.lookup_hits, 0)), 0) AS totalLookupHits, @@ -1788,7 +1808,6 @@ export function getMediaSessions( COALESCE(asm.totalWatchedMs, s.total_watched_ms, 0) AS totalWatchedMs, COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0) AS activeWatchedMs, COALESCE(asm.linesSeen, s.lines_seen, 0) AS linesSeen, - COALESCE(asm.wordsSeen, s.words_seen, 0) AS wordsSeen, COALESCE(asm.tokensSeen, s.tokens_seen, 0) AS tokensSeen, COALESCE(asm.cardsMined, s.cards_mined, 0) AS cardsMined, COALESCE(asm.lookupCount, s.lookup_count, 0) AS lookupCount, @@ -1826,11 +1845,10 @@ export function getMediaDailyRollups( total_sessions AS totalSessions, total_active_min AS totalActiveMin, total_lines_seen AS totalLinesSeen, - total_words_seen AS totalWordsSeen, total_tokens_seen AS totalTokensSeen, total_cards AS totalCards, cards_per_hour AS cardsPerHour, - words_per_min AS wordsPerMin, + tokens_per_min AS tokensPerMin, lookup_hit_rate AS lookupHitRate FROM imm_daily_rollups WHERE video_id = ? @@ -1859,9 +1877,9 @@ export function getAnimeDailyRollups( ) SELECT r.rollup_day AS rollupDayOrMonth, r.video_id AS videoId, r.total_sessions AS totalSessions, r.total_active_min AS totalActiveMin, - r.total_lines_seen AS totalLinesSeen, r.total_words_seen AS totalWordsSeen, + r.total_lines_seen AS totalLinesSeen, r.total_tokens_seen AS totalTokensSeen, r.total_cards AS totalCards, - r.cards_per_hour AS cardsPerHour, r.words_per_min AS wordsPerMin, + r.cards_per_hour AS cardsPerHour, r.tokens_per_min AS tokensPerMin, r.lookup_hit_rate AS lookupHitRate FROM imm_daily_rollups r JOIN imm_videos v ON v.video_id = r.video_id @@ -2153,7 +2171,6 @@ export function getEpisodeSessions(db: DatabaseSync, videoId: number): SessionSu COALESCE(asm.totalWatchedMs, s.total_watched_ms, 0) AS totalWatchedMs, COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0) AS activeWatchedMs, COALESCE(asm.linesSeen, s.lines_seen, 0) AS linesSeen, - COALESCE(asm.wordsSeen, s.words_seen, 0) AS wordsSeen, COALESCE(asm.tokensSeen, s.tokens_seen, 0) AS tokensSeen, COALESCE(asm.cardsMined, s.cards_mined, 0) AS cardsMined, COALESCE(asm.lookupCount, s.lookup_count, 0) AS lookupCount, diff --git a/src/core/services/immersion-tracker/reducer.ts b/src/core/services/immersion-tracker/reducer.ts index 7db2b57..17549f5 100644 --- a/src/core/services/immersion-tracker/reducer.ts +++ b/src/core/services/immersion-tracker/reducer.ts @@ -15,7 +15,6 @@ export function createInitialSessionState( totalWatchedMs: 0, activeWatchedMs: 0, linesSeen: 0, - wordsSeen: 0, tokensSeen: 0, cardsMined: 0, lookupCount: 0, @@ -52,16 +51,6 @@ export function sanitizePayload(payload: Record, maxPayloadByte return json.length <= maxPayloadBytes ? json : JSON.stringify({ truncated: true }); } -export function calculateTextMetrics(value: string): { - words: number; - tokens: number; -} { - const words = value.split(/\s+/).filter(Boolean).length; - const cjkCount = value.match(/[\u3040-\u30ff\u4e00-\u9fff]/g)?.length ?? 0; - const tokens = Math.max(words, cjkCount); - return { words, tokens }; -} - export function secToMs(seconds: number): number { const coerced = Number(seconds); if (!Number.isFinite(coerced)) return 0; diff --git a/src/core/services/immersion-tracker/session.ts b/src/core/services/immersion-tracker/session.ts index 50f6dbd..3461c8c 100644 --- a/src/core/services/immersion-tracker/session.ts +++ b/src/core/services/immersion-tracker/session.ts @@ -42,7 +42,6 @@ export function finalizeSessionRecord( total_watched_ms = ?, active_watched_ms = ?, lines_seen = ?, - words_seen = ?, tokens_seen = ?, cards_mined = ?, lookup_count = ?, @@ -62,7 +61,6 @@ export function finalizeSessionRecord( sessionState.totalWatchedMs, sessionState.activeWatchedMs, sessionState.linesSeen, - sessionState.wordsSeen, sessionState.tokensSeen, sessionState.cardsMined, sessionState.lookupCount, diff --git a/src/core/services/immersion-tracker/storage-session.test.ts b/src/core/services/immersion-tracker/storage-session.test.ts index 3b69407..0b63488 100644 --- a/src/core/services/immersion-tracker/storage-session.test.ts +++ b/src/core/services/immersion-tracker/storage-session.test.ts @@ -763,7 +763,6 @@ test('executeQueuedWrite inserts event and telemetry rows', () => { totalWatchedMs: 1_000, activeWatchedMs: 900, linesSeen: 3, - wordsSeen: 6, tokensSeen: 6, cardsMined: 1, lookupCount: 2, @@ -786,7 +785,7 @@ test('executeQueuedWrite inserts event and telemetry rows', () => { lineIndex: 1, segmentStartMs: 0, segmentEndMs: 800, - wordsDelta: 2, + tokensDelta: 2, cardsDelta: 0, payloadJson: '{"event":"subtitle-line"}', }, diff --git a/src/core/services/immersion-tracker/storage.ts b/src/core/services/immersion-tracker/storage.ts index 701e724..05bd18f 100644 --- a/src/core/services/immersion-tracker/storage.ts +++ b/src/core/services/immersion-tracker/storage.ts @@ -290,7 +290,6 @@ function ensureLifetimeSummaryTables(db: DatabaseSync): void { total_sessions INTEGER NOT NULL DEFAULT 0, total_active_ms INTEGER NOT NULL DEFAULT 0, total_cards INTEGER NOT NULL DEFAULT 0, - total_words_seen INTEGER NOT NULL DEFAULT 0, total_lines_seen INTEGER NOT NULL DEFAULT 0, total_tokens_seen INTEGER NOT NULL DEFAULT 0, episodes_started INTEGER NOT NULL DEFAULT 0, @@ -309,7 +308,6 @@ function ensureLifetimeSummaryTables(db: DatabaseSync): void { total_sessions INTEGER NOT NULL DEFAULT 0, total_active_ms INTEGER NOT NULL DEFAULT 0, total_cards INTEGER NOT NULL DEFAULT 0, - total_words_seen INTEGER NOT NULL DEFAULT 0, total_lines_seen INTEGER NOT NULL DEFAULT 0, total_tokens_seen INTEGER NOT NULL DEFAULT 0, completed INTEGER NOT NULL DEFAULT 0, @@ -574,7 +572,6 @@ export function ensureSchema(db: DatabaseSync): void { total_watched_ms INTEGER NOT NULL DEFAULT 0, active_watched_ms INTEGER NOT NULL DEFAULT 0, lines_seen INTEGER NOT NULL DEFAULT 0, - words_seen INTEGER NOT NULL DEFAULT 0, tokens_seen INTEGER NOT NULL DEFAULT 0, cards_mined INTEGER NOT NULL DEFAULT 0, lookup_count INTEGER NOT NULL DEFAULT 0, @@ -598,7 +595,6 @@ export function ensureSchema(db: DatabaseSync): void { total_watched_ms INTEGER NOT NULL DEFAULT 0, active_watched_ms INTEGER NOT NULL DEFAULT 0, lines_seen INTEGER NOT NULL DEFAULT 0, - words_seen INTEGER NOT NULL DEFAULT 0, tokens_seen INTEGER NOT NULL DEFAULT 0, cards_mined INTEGER NOT NULL DEFAULT 0, lookup_count INTEGER NOT NULL DEFAULT 0, @@ -623,7 +619,7 @@ export function ensureSchema(db: DatabaseSync): void { line_index INTEGER, segment_start_ms INTEGER, segment_end_ms INTEGER, - words_delta INTEGER NOT NULL DEFAULT 0, + tokens_delta INTEGER NOT NULL DEFAULT 0, cards_delta INTEGER NOT NULL DEFAULT 0, payload_json TEXT, CREATED_DATE INTEGER, @@ -638,11 +634,10 @@ export function ensureSchema(db: DatabaseSync): void { total_sessions INTEGER NOT NULL DEFAULT 0, total_active_min REAL NOT NULL DEFAULT 0, total_lines_seen INTEGER NOT NULL DEFAULT 0, - total_words_seen INTEGER NOT NULL DEFAULT 0, total_tokens_seen INTEGER NOT NULL DEFAULT 0, total_cards INTEGER NOT NULL DEFAULT 0, cards_per_hour REAL, - words_per_min REAL, + tokens_per_min REAL, lookup_hit_rate REAL, CREATED_DATE INTEGER, LAST_UPDATE_DATE INTEGER, @@ -656,7 +651,6 @@ export function ensureSchema(db: DatabaseSync): void { total_sessions INTEGER NOT NULL DEFAULT 0, total_active_min REAL NOT NULL DEFAULT 0, total_lines_seen INTEGER NOT NULL DEFAULT 0, - total_words_seen INTEGER NOT NULL DEFAULT 0, total_tokens_seen INTEGER NOT NULL DEFAULT 0, total_cards INTEGER NOT NULL DEFAULT 0, CREATED_DATE INTEGER, @@ -895,7 +889,6 @@ export function ensureSchema(db: DatabaseSync): void { addColumnIfMissing(db, 'imm_sessions', 'total_watched_ms', 'INTEGER NOT NULL DEFAULT 0'); addColumnIfMissing(db, 'imm_sessions', 'active_watched_ms', 'INTEGER NOT NULL DEFAULT 0'); addColumnIfMissing(db, 'imm_sessions', 'lines_seen', 'INTEGER NOT NULL DEFAULT 0'); - addColumnIfMissing(db, 'imm_sessions', 'words_seen', 'INTEGER NOT NULL DEFAULT 0'); addColumnIfMissing(db, 'imm_sessions', 'tokens_seen', 'INTEGER NOT NULL DEFAULT 0'); addColumnIfMissing(db, 'imm_sessions', 'cards_mined', 'INTEGER NOT NULL DEFAULT 0'); addColumnIfMissing(db, 'imm_sessions', 'lookup_count', 'INTEGER NOT NULL DEFAULT 0'); @@ -930,13 +923,6 @@ export function ensureSchema(db: DatabaseSync): void { ORDER BY t.sample_ms DESC, t.telemetry_id DESC LIMIT 1 ), lines_seen), - words_seen = COALESCE(( - SELECT t.words_seen - FROM imm_session_telemetry t - WHERE t.session_id = imm_sessions.session_id - ORDER BY t.sample_ms DESC, t.telemetry_id DESC - LIMIT 1 - ), words_seen), tokens_seen = COALESCE(( SELECT t.tokens_seen FROM imm_session_telemetry t @@ -1163,17 +1149,17 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar telemetryInsertStmt: db.prepare(` INSERT INTO imm_session_telemetry ( session_id, sample_ms, total_watched_ms, active_watched_ms, - lines_seen, words_seen, tokens_seen, cards_mined, lookup_count, + lines_seen, tokens_seen, cards_mined, lookup_count, lookup_hits, yomitan_lookup_count, pause_count, pause_ms, seek_forward_count, seek_backward_count, media_buffer_events, CREATED_DATE, LAST_UPDATE_DATE ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) `), eventInsertStmt: db.prepare(` INSERT INTO imm_session_events ( session_id, ts_ms, event_type, line_index, segment_start_ms, segment_end_ms, - words_delta, cards_delta, payload_json, CREATED_DATE, LAST_UPDATE_DATE + tokens_delta, cards_delta, payload_json, CREATED_DATE, LAST_UPDATE_DATE ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) @@ -1310,7 +1296,6 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta write.totalWatchedMs!, write.activeWatchedMs!, write.linesSeen!, - write.wordsSeen!, write.tokensSeen!, write.cardsMined!, write.lookupCount!, @@ -1381,7 +1366,7 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta write.lineIndex ?? null, write.segmentStartMs ?? null, write.segmentEndMs ?? null, - write.wordsDelta ?? 0, + write.tokensDelta ?? 0, write.cardsDelta ?? 0, write.payloadJson ?? null, Date.now(), diff --git a/src/core/services/immersion-tracker/types.ts b/src/core/services/immersion-tracker/types.ts index f44c7c9..70d5081 100644 --- a/src/core/services/immersion-tracker/types.ts +++ b/src/core/services/immersion-tracker/types.ts @@ -56,7 +56,6 @@ export interface TelemetryAccumulator { totalWatchedMs: number; activeWatchedMs: number; linesSeen: number; - wordsSeen: number; tokensSeen: number; cardsMined: number; lookupCount: number; @@ -89,7 +88,6 @@ interface QueuedTelemetryWrite { totalWatchedMs?: number; activeWatchedMs?: number; linesSeen?: number; - wordsSeen?: number; tokensSeen?: number; cardsMined?: number; lookupCount?: number; @@ -104,7 +102,7 @@ interface QueuedTelemetryWrite { lineIndex?: number | null; segmentStartMs?: number | null; segmentEndMs?: number | null; - wordsDelta?: number; + tokensDelta?: number; cardsDelta?: number; payloadJson?: string | null; } @@ -117,7 +115,7 @@ interface QueuedEventWrite { lineIndex?: number | null; segmentStartMs?: number | null; segmentEndMs?: number | null; - wordsDelta?: number; + tokensDelta?: number; cardsDelta?: number; payloadJson?: string | null; } @@ -231,7 +229,6 @@ export interface SessionSummaryQueryRow { totalWatchedMs: number; activeWatchedMs: number; linesSeen: number; - wordsSeen: number; tokensSeen: number; cardsMined: number; lookupCount: number; @@ -255,7 +252,6 @@ export interface LifetimeAnimeRow { totalSessions: number; totalActiveMs: number; totalCards: number; - totalWordsSeen: number; totalLinesSeen: number; totalTokensSeen: number; episodesStarted: number; @@ -269,7 +265,6 @@ export interface LifetimeMediaRow { totalSessions: number; totalActiveMs: number; totalCards: number; - totalWordsSeen: number; totalLinesSeen: number; totalTokensSeen: number; completed: number; @@ -374,7 +369,6 @@ export interface SessionTimelineRow { totalWatchedMs: number; activeWatchedMs: number; linesSeen: number; - wordsSeen: number; tokensSeen: number; cardsMined: number; } @@ -385,11 +379,10 @@ export interface ImmersionSessionRollupRow { totalSessions: number; totalActiveMin: number; totalLinesSeen: number; - totalWordsSeen: number; totalTokensSeen: number; totalCards: number; cardsPerHour: number | null; - wordsPerMin: number | null; + tokensPerMin: number | null; lookupHitRate: number | null; } @@ -421,7 +414,7 @@ export interface MediaLibraryRow { totalSessions: number; totalActiveMs: number; totalCards: number; - totalWordsSeen: number; + totalTokensSeen: number; lastWatchedMs: number; hasCoverArt: number; } @@ -432,7 +425,7 @@ export interface MediaDetailRow { totalSessions: number; totalActiveMs: number; totalCards: number; - totalWordsSeen: number; + totalTokensSeen: number; totalLinesSeen: number; totalLookupCount: number; totalLookupHits: number; @@ -446,7 +439,7 @@ export interface AnimeLibraryRow { totalSessions: number; totalActiveMs: number; totalCards: number; - totalWordsSeen: number; + totalTokensSeen: number; episodeCount: number; episodesTotal: number | null; lastWatchedMs: number; @@ -463,7 +456,7 @@ export interface AnimeDetailRow { totalSessions: number; totalActiveMs: number; totalCards: number; - totalWordsSeen: number; + totalTokensSeen: number; totalLinesSeen: number; totalLookupCount: number; totalLookupHits: number; @@ -491,7 +484,7 @@ export interface AnimeEpisodeRow { totalSessions: number; totalActiveMs: number; totalCards: number; - totalWordsSeen: number; + totalTokensSeen: number; totalYomitanLookupCount: number; lastWatchedMs: number; } diff --git a/stats/src/components/anime/AnimeOverviewStats.tsx b/stats/src/components/anime/AnimeOverviewStats.tsx index a6aab4f..f4c8fc9 100644 --- a/stats/src/components/anime/AnimeOverviewStats.tsx +++ b/stats/src/components/anime/AnimeOverviewStats.tsx @@ -37,20 +37,12 @@ function Metric({ label, value, unit, color, tooltip, sub }: MetricProps) { ); } -export function AnimeOverviewStats({ - detail, - knownWordsSummary, -}: AnimeOverviewStatsProps) { - const lookupRate = buildLookupRateDisplay( - detail.totalYomitanLookupCount, - detail.totalWordsSeen, - ); +export function AnimeOverviewStats({ detail, knownWordsSummary }: AnimeOverviewStatsProps) { + const lookupRate = buildLookupRateDisplay(detail.totalYomitanLookupCount, detail.totalTokensSeen); const knownPct = knownWordsSummary && knownWordsSummary.totalUniqueWords > 0 - ? Math.round( - (knownWordsSummary.knownWordCount / knownWordsSummary.totalUniqueWords) * 100, - ) + ? Math.round((knownWordsSummary.knownWordCount / knownWordsSummary.totalUniqueWords) * 100) : null; return ( @@ -76,10 +68,10 @@ export function AnimeOverviewStats({ tooltip="Number of completed episodes for this anime" /> @@ -88,7 +80,7 @@ export function AnimeOverviewStats({ ) : ( )} diff --git a/stats/src/components/anime/EpisodeDetail.tsx b/stats/src/components/anime/EpisodeDetail.tsx index 678316f..9502db9 100644 --- a/stats/src/components/anime/EpisodeDetail.tsx +++ b/stats/src/components/anime/EpisodeDetail.tsx @@ -89,9 +89,9 @@ export function EpisodeDetail({ videoId, onSessionDeleted }: EpisodeDetailProps) {s.startedAtMs > 0 ? formatRelativeDate(s.startedAtMs) : '\u2014'} {formatDuration(s.activeWatchedMs)} - {formatNumber(s.cardsMined)} cards + {formatNumber(s.cardsMined)} cards - {formatNumber(getSessionDisplayWordCount(s))} words + {formatNumber(getSessionDisplayWordCount(s))} tokens @@ -245,18 +245,18 @@ function AnimeGroupRow({ {group.sessions.length} sessions · {formatDuration(group.totalActiveMs)} active -
-
-
- {formatNumber(group.totalCards)} -
-
cards
+
+
+
+ {formatNumber(group.totalCards)}
+
cards
+
{formatNumber(group.totalWords)}
-
words
+
tokens
{ if (navigationTarget.type === 'media-detail') { - onNavigateToMediaDetail( - navigationTarget.videoId, - navigationTarget.sessionId, - ); + onNavigateToMediaDetail(navigationTarget.videoId, navigationTarget.sessionId); return; } onNavigateToSession(navigationTarget.sessionId); @@ -319,7 +316,7 @@ function AnimeGroupRow({
-
+
{formatNumber(s.cardsMined)}
cards
@@ -328,7 +325,7 @@ function AnimeGroupRow({
{formatNumber(getSessionDisplayWordCount(s))}
-
words
+
tokens
diff --git a/stats/src/components/sessions/SessionDetail.tsx b/stats/src/components/sessions/SessionDetail.tsx index 5c20b1b..16025f5 100644 --- a/stats/src/components/sessions/SessionDetail.tsx +++ b/stats/src/components/sessions/SessionDetail.tsx @@ -1,3 +1,4 @@ +import { useEffect, useMemo, useRef, useState } from 'react'; import { AreaChart, Area, @@ -12,12 +13,19 @@ import { CartesianGrid, } from 'recharts'; import { useSessionDetail } from '../../hooks/useSessions'; +import { getStatsClient } from '../../hooks/useStatsApi'; import type { KnownWordsTimelinePoint } from '../../hooks/useSessions'; import { CHART_THEME } from '../../lib/chart-theme'; -import { buildLookupRateDisplay, getYomitanLookupEvents } from '../../lib/yomitan-lookup'; +import { + buildSessionChartEvents, + type SessionChartMarker, + type SessionEventNoteInfo, +} from '../../lib/session-events'; +import { buildLookupRateDisplay } from '../../lib/yomitan-lookup'; import { getSessionDisplayWordCount } from '../../lib/session-word-count'; import { EventType } from '../../types/stats'; import type { SessionEvent, SessionSummary } from '../../types/stats'; +import { SessionEventOverlay } from './SessionEventOverlay'; interface SessionDetailProps { session: SessionSummary; @@ -40,9 +48,7 @@ function formatTime(ms: number): string { } /** Build a lookup: linesSeen → knownWordsSeen */ -function buildKnownWordsLookup( - knownWordsTimeline: KnownWordsTimelinePoint[], -): Map { +function buildKnownWordsLookup(knownWordsTimeline: KnownWordsTimelinePoint[]): Map { const map = new Map(); for (const pt of knownWordsTimeline) { map.set(pt.linesSeen, pt.knownWordsSeen); @@ -63,24 +69,17 @@ function lookupKnownWords(map: Map, linesSeen: number): number { return best > 0 ? map.get(best)! : 0; } -interface PauseRegion { - startMs: number; - endMs: number; -} - -function buildPauseRegions(events: SessionEvent[]): PauseRegion[] { - const regions: PauseRegion[] = []; - const starts = events.filter((e) => e.eventType === EventType.PAUSE_START); - const ends = events.filter((e) => e.eventType === EventType.PAUSE_END); - - for (const start of starts) { - const end = ends.find((e) => e.tsMs > start.tsMs); - regions.push({ - startMs: start.tsMs, - endMs: end ? end.tsMs : start.tsMs + 2000, - }); - } - return regions; +function extractNoteExpression(note: { + noteId: number; + fields: Record; +}): SessionEventNoteInfo { + const expression = + note.fields?.Expression?.value ?? + note.fields?.expression?.value ?? + note.fields?.Word?.value ?? + note.fields?.word?.value ?? + ''; + return { noteId: note.noteId, expression }; } interface RatioChartPoint { @@ -100,7 +99,6 @@ interface FallbackChartPoint { type TimelineEntry = { sampleMs: number; linesSeen: number; - wordsSeen: number; tokensSeen: number; }; @@ -108,19 +106,17 @@ export function SessionDetail({ session }: SessionDetailProps) { const { timeline, events, knownWordsTimeline, loading, error } = useSessionDetail( session.sessionId, ); - - if (loading) return
Loading timeline...
; - if (error) return
Error: {error}
; + const [activeMarkerKey, setActiveMarkerKey] = useState(null); + const [noteInfos, setNoteInfos] = useState>(new Map()); + const [loadingNoteIds, setLoadingNoteIds] = useState>(new Set()); + const requestedNoteIdsRef = useRef>(new Set()); const sorted = [...timeline].reverse(); const knownWordsMap = buildKnownWordsLookup(knownWordsTimeline); const hasKnownWords = knownWordsMap.size > 0; - const cardEvents = events.filter((e) => e.eventType === EventType.CARD_MINED); - const seekEvents = events.filter( - (e) => e.eventType === EventType.SEEK_FORWARD || e.eventType === EventType.SEEK_BACKWARD, - ); - const yomitanLookupEvents = getYomitanLookupEvents(events); + const { cardEvents, seekEvents, yomitanLookupEvents, pauseRegions, markers } = + buildSessionChartEvents(events); const lookupRate = buildLookupRateDisplay( session.yomitanLookupCount, getSessionDisplayWordCount(session), @@ -128,7 +124,76 @@ export function SessionDetail({ session }: SessionDetailProps) { const pauseCount = events.filter((e) => e.eventType === EventType.PAUSE_START).length; const seekCount = seekEvents.length; const cardEventCount = cardEvents.length; - const pauseRegions = buildPauseRegions(events); + const activeMarker = useMemo( + () => markers.find((marker) => marker.key === activeMarkerKey) ?? null, + [markers, activeMarkerKey], + ); + + useEffect(() => { + if (!activeMarker || activeMarker.kind !== 'card' || activeMarker.noteIds.length === 0) { + return; + } + + const missingNoteIds = activeMarker.noteIds.filter( + (noteId) => !requestedNoteIdsRef.current.has(noteId) && !noteInfos.has(noteId), + ); + if (missingNoteIds.length === 0) { + return; + } + + for (const noteId of missingNoteIds) { + requestedNoteIdsRef.current.add(noteId); + } + + let cancelled = false; + setLoadingNoteIds((prev) => { + const next = new Set(prev); + for (const noteId of missingNoteIds) { + next.add(noteId); + } + return next; + }); + + getStatsClient() + .ankiNotesInfo(missingNoteIds) + .then((notes) => { + if (cancelled) return; + setNoteInfos((prev) => { + const next = new Map(prev); + for (const note of notes) { + const info = extractNoteExpression(note); + next.set(info.noteId, info); + } + return next; + }); + }) + .catch((err) => { + if (!cancelled) { + console.warn('Failed to fetch session event Anki note info:', err); + } + }) + .finally(() => { + if (cancelled) return; + setLoadingNoteIds((prev) => { + const next = new Set(prev); + for (const noteId of missingNoteIds) { + next.delete(noteId); + } + return next; + }); + }); + + return () => { + cancelled = true; + }; + }, [activeMarker, noteInfos]); + + const handleOpenNote = (noteId: number) => { + void getStatsClient().ankiBrowse(noteId); + }; + + if (loading) return
Loading timeline...
; + if (error) return
Error: {error}
; if (hasKnownWords) { return ( @@ -136,8 +201,15 @@ export function SessionDetail({ session }: SessionDetailProps) { sorted={sorted} knownWordsMap={knownWordsMap} cardEvents={cardEvents} + seekEvents={seekEvents} yomitanLookupEvents={yomitanLookupEvents} pauseRegions={pauseRegions} + markers={markers} + activeMarkerKey={activeMarkerKey} + onActiveMarkerChange={setActiveMarkerKey} + noteInfos={noteInfos} + loadingNoteIds={loadingNoteIds} + onOpenNote={handleOpenNote} pauseCount={pauseCount} seekCount={seekCount} cardEventCount={cardEventCount} @@ -151,8 +223,15 @@ export function SessionDetail({ session }: SessionDetailProps) { ; cardEvents: SessionEvent[]; + seekEvents: SessionEvent[]; yomitanLookupEvents: SessionEvent[]; - pauseRegions: PauseRegion[]; + pauseRegions: Array<{ startMs: number; endMs: number }>; + markers: SessionChartMarker[]; + activeMarkerKey: string | null; + onActiveMarkerChange: (markerKey: string | null) => void; + noteInfos: Map; + loadingNoteIds: Set; + onOpenNote: (noteId: number) => void; pauseCount: number; seekCount: number; cardEventCount: number; @@ -205,7 +298,7 @@ function RatioView({ } if (chartData.length === 0) { - return
No word data for this session.
; + return
No token data for this session.
; } const tsMin = chartData[0]!.tsMs; @@ -217,8 +310,9 @@ function RatioView({ return (
{/* ── Top: Percentage area chart ── */} - - +
+ + @@ -297,36 +391,45 @@ function RatioView({ ))} {/* Card mine markers */} - {cardEvents.map((e, i) => ( - - ))} + {cardEvents.map((e, i) => ( + + ))} + + {seekEvents.map((e, i) => { + const isBackward = e.eventType === EventType.SEEK_BACKWARD; + const stroke = isBackward ? '#f5bde6' : '#8bd5ca'; + return ( + + ); + })} {/* Yomitan lookup markers */} - {yomitanLookupEvents.map((e, i) => ( - - ))} + {yomitanLookupEvents.map((e, i) => ( + + ))} - - + + + +
- {/* ── Bottom: Word accumulation sparkline ── */} + {/* ── Bottom: Token accumulation sparkline ── */}
- total words + total tokens
@@ -398,8 +512,15 @@ function RatioView({ function FallbackView({ sorted, cardEvents, + seekEvents, yomitanLookupEvents, pauseRegions, + markers, + activeMarkerKey, + onActiveMarkerChange, + noteInfos, + loadingNoteIds, + onOpenNote, pauseCount, seekCount, cardEventCount, @@ -408,8 +529,15 @@ function FallbackView({ }: { sorted: TimelineEntry[]; cardEvents: SessionEvent[]; + seekEvents: SessionEvent[]; yomitanLookupEvents: SessionEvent[]; - pauseRegions: PauseRegion[]; + pauseRegions: Array<{ startMs: number; endMs: number }>; + markers: SessionChartMarker[]; + activeMarkerKey: string | null; + onActiveMarkerChange: (markerKey: string | null) => void; + noteInfos: Map; + loadingNoteIds: Set; + onOpenNote: (noteId: number) => void; pauseCount: number; seekCount: number; cardEventCount: number; @@ -424,7 +552,7 @@ function FallbackView({ } if (chartData.length === 0) { - return
No word data for this session.
; + return
No token data for this session.
; } const tsMin = chartData[0]!.tsMs; @@ -432,8 +560,9 @@ function FallbackView({ return (
- - +
+ + [`${value.toLocaleString()}`, 'Total words']} + formatter={(value: number) => [`${value.toLocaleString()}`, 'Total tokens']} /> {pauseRegions.map((r, i) => ( @@ -471,32 +600,39 @@ function FallbackView({ /> ))} - {cardEvents.map((e, i) => ( - - ))} - {yomitanLookupEvents.map((e, i) => ( - - ))} + {cardEvents.map((e, i) => ( + + ))} + {seekEvents.map((e, i) => { + const isBackward = e.eventType === EventType.SEEK_BACKWARD; + const stroke = isBackward ? '#f5bde6' : '#8bd5ca'; + return ( + + ); + })} + {yomitanLookupEvents.map((e, i) => ( + + ))} - - + + + +
{'\u26CF'} - + {Math.max(cardEventCount, session.cardsMined)} card {Math.max(cardEventCount, session.cardsMined) !== 1 ? 's' : ''} mined diff --git a/stats/src/components/sessions/SessionRow.tsx b/stats/src/components/sessions/SessionRow.tsx index 4e70132..db8e776 100644 --- a/stats/src/components/sessions/SessionRow.tsx +++ b/stats/src/components/sessions/SessionRow.tsx @@ -84,7 +84,7 @@ export function SessionRow({
-
+
{formatNumber(session.cardsMined)}
cards
@@ -93,7 +93,7 @@ export function SessionRow({
{formatNumber(displayWordCount)}
-
words
+
tokens
('day'); const [hiddenAnime, setHiddenAnime] = useState>(() => new Set()); const { data, loading, error } = useTrends(range, groupBy); + const cardsMinedColor = 'var(--color-ctp-cards-mined)'; + const cardsMinedStackedColors = [ + cardsMinedColor, + '#8aadf4', + '#c6a0f6', + '#f5a97f', + '#f5bde6', + '#91d7e3', + '#ee99a0', + '#f4dbd6', + ]; if (loading) return
Loading...
; if (error) return
Error: {error}
; @@ -115,19 +126,40 @@ export function TrendsTab() { ]); const activeHiddenAnime = pruneHiddenAnime(hiddenAnime, animeTitles); - const filteredEpisodesPerAnime = filterHiddenAnimeData(data.animePerDay.episodes, activeHiddenAnime); - const filteredWatchTimePerAnime = filterHiddenAnimeData(data.animePerDay.watchTime, activeHiddenAnime); + const filteredEpisodesPerAnime = filterHiddenAnimeData( + data.animePerDay.episodes, + activeHiddenAnime, + ); + const filteredWatchTimePerAnime = filterHiddenAnimeData( + data.animePerDay.watchTime, + activeHiddenAnime, + ); const filteredCardsPerAnime = filterHiddenAnimeData(data.animePerDay.cards, activeHiddenAnime); const filteredWordsPerAnime = filterHiddenAnimeData(data.animePerDay.words, activeHiddenAnime); - const filteredLookupsPerAnime = filterHiddenAnimeData(data.animePerDay.lookups, activeHiddenAnime); + const filteredLookupsPerAnime = filterHiddenAnimeData( + data.animePerDay.lookups, + activeHiddenAnime, + ); const filteredLookupsPerHundredPerAnime = filterHiddenAnimeData( data.animePerDay.lookupsPerHundred, activeHiddenAnime, ); - const filteredAnimeProgress = filterHiddenAnimeData(data.animeCumulative.episodes, activeHiddenAnime); - const filteredCardsProgress = filterHiddenAnimeData(data.animeCumulative.cards, activeHiddenAnime); - const filteredWordsProgress = filterHiddenAnimeData(data.animeCumulative.words, activeHiddenAnime); - const filteredWatchTimeProgress = filterHiddenAnimeData(data.animeCumulative.watchTime, activeHiddenAnime); + const filteredAnimeProgress = filterHiddenAnimeData( + data.animeCumulative.episodes, + activeHiddenAnime, + ); + const filteredCardsProgress = filterHiddenAnimeData( + data.animeCumulative.cards, + activeHiddenAnime, + ); + const filteredWordsProgress = filterHiddenAnimeData( + data.animeCumulative.words, + activeHiddenAnime, + ); + const filteredWatchTimeProgress = filterHiddenAnimeData( + data.animeCumulative.watchTime, + activeHiddenAnime, + ); return (
@@ -145,19 +177,39 @@ export function TrendsTab() { color="#8aadf4" type="bar" /> - - + + Period Trends - + - - - - + + + + - + Anime — Per Day - - + + - + Anime — Cumulative - - + + Patterns totalWatchedMs: 3_600_000, activeWatchedMs: 3_000_000, linesSeen: 20, - wordsSeen: 100, tokensSeen: 80, cardsMined: 2, lookupCount: 10, @@ -45,33 +44,32 @@ test('buildOverviewSummary aggregates tracked totals and recent windows', () => totalSessions: 1, totalActiveMin: 50, totalLinesSeen: 20, - totalWordsSeen: 100, totalTokensSeen: 80, totalCards: 2, cardsPerHour: 2.4, - wordsPerMin: 2, + tokensPerMin: 2, lookupHitRate: 0.8, }, ]; - const overview: OverviewData = { - sessions, - rollups, - hints: { - totalSessions: 15, - activeSessions: 0, - episodesToday: 2, - activeAnimeCount: 3, - totalEpisodesWatched: 5, - totalAnimeCompleted: 1, - totalActiveMin: 50, - activeDays: 2, - totalCards: 9, - totalLookupCount: 100, - totalLookupHits: 80, - newWordsToday: 5, - newWordsThisWeek: 20, - }, - }; + const overview: OverviewData = { + sessions, + rollups, + hints: { + totalSessions: 15, + activeSessions: 0, + episodesToday: 2, + activeAnimeCount: 3, + totalEpisodesWatched: 5, + totalAnimeCompleted: 1, + totalActiveMin: 50, + activeDays: 2, + totalCards: 9, + totalLookupCount: 100, + totalLookupHits: 80, + newWordsToday: 5, + newWordsThisWeek: 20, + }, + }; const summary = buildOverviewSummary(overview, now); assert.equal(summary.todayCards, 2); @@ -88,8 +86,8 @@ test('buildOverviewSummary aggregates tracked totals and recent windows', () => test('buildOverviewSummary prefers lifetime totals from hints when provided', () => { const now = Date.UTC(2026, 2, 13, 12); const today = Math.floor(now / 86_400_000); - const overview: OverviewData = { - sessions: [ + const overview: OverviewData = { + sessions: [ { sessionId: 2, canonicalTitle: 'B', @@ -101,7 +99,6 @@ test('buildOverviewSummary prefers lifetime totals from hints when provided', () totalWatchedMs: 60_000, activeWatchedMs: 60_000, linesSeen: 10, - wordsSeen: 10, tokensSeen: 10, cardsMined: 10, lookupCount: 1, @@ -116,11 +113,10 @@ test('buildOverviewSummary prefers lifetime totals from hints when provided', () totalSessions: 1, totalActiveMin: 1, totalLinesSeen: 10, - totalWordsSeen: 10, totalTokensSeen: 10, totalCards: 10, cardsPerHour: 600, - wordsPerMin: 10, + tokensPerMin: 10, lookupHitRate: 1, }, ], @@ -182,11 +178,10 @@ test('buildTrendDashboard derives dense chart series', () => { totalSessions: 2, totalActiveMin: 60, totalLinesSeen: 30, - totalWordsSeen: 120, totalTokensSeen: 100, totalCards: 3, cardsPerHour: 3, - wordsPerMin: 2, + tokensPerMin: 2, lookupHitRate: 0.5, }, { @@ -195,18 +190,17 @@ test('buildTrendDashboard derives dense chart series', () => { totalSessions: 1, totalActiveMin: 30, totalLinesSeen: 10, - totalWordsSeen: 40, totalTokensSeen: 30, totalCards: 1, cardsPerHour: 2, - wordsPerMin: 1.33, + tokensPerMin: 1.33, lookupHitRate: 0.75, }, ]; const dashboard = buildTrendDashboard(rollups); assert.equal(dashboard.watchTime.length, 2); - assert.equal(dashboard.words[1]?.value, 40); + assert.equal(dashboard.words[1]?.value, 30); assert.equal(dashboard.sessions[0]?.value, 2); }); diff --git a/stats/src/lib/dashboard-data.ts b/stats/src/lib/dashboard-data.ts index 9cdddf3..84c14e9 100644 --- a/stats/src/lib/dashboard-data.ts +++ b/stats/src/lib/dashboard-data.ts @@ -26,7 +26,7 @@ export interface OverviewSummary { activeDays: number; totalSessions: number; lookupRate: number | null; - todayWords: number; + todayTokens: number; newWordsToday: number; newWordsThisWeek: number; recentWatchTime: ChartPoint[]; @@ -100,7 +100,7 @@ function buildAggregatedDailyRows(rollups: DailyRollup[]) { existing.activeMin += rollup.totalActiveMin; existing.cards += rollup.totalCards; - existing.words += rollup.totalWordsSeen; + existing.words += rollup.totalTokensSeen; existing.sessions += rollup.totalSessions; if (rollup.lookupHitRate != null) { const weight = Math.max(rollup.totalSessions, 1); @@ -185,9 +185,9 @@ export function buildOverviewSummary( overview.hints.totalLookupCount > 0 ? Math.round((overview.hints.totalLookupHits / overview.hints.totalLookupCount) * 100) : null, - todayWords: Math.max( + todayTokens: Math.max( todayRow?.words ?? 0, - sumBy(todaySessions, (session) => session.wordsSeen), + sumBy(todaySessions, (session) => session.tokensSeen), ), newWordsToday: overview.hints.newWordsToday ?? 0, newWordsThisWeek: overview.hints.newWordsThisWeek ?? 0, diff --git a/stats/src/lib/media-session-list.test.tsx b/stats/src/lib/media-session-list.test.tsx index 3f37494..908cf4e 100644 --- a/stats/src/lib/media-session-list.test.tsx +++ b/stats/src/lib/media-session-list.test.tsx @@ -18,7 +18,6 @@ test('MediaSessionList renders expandable session rows with delete affordance', totalWatchedMs: 1_000, activeWatchedMs: 900, linesSeen: 12, - wordsSeen: 24, tokensSeen: 24, cardsMined: 2, lookupCount: 3, @@ -34,6 +33,6 @@ test('MediaSessionList renders expandable session rows with delete affordance', assert.match(markup, /Session History/); assert.match(markup, /aria-expanded="true"/); assert.match(markup, /Delete session Episode 7/); - assert.match(markup, /Total words/); - assert.match(markup, /1 Yomitan lookup/); + assert.match(markup, /tokens/); + assert.match(markup, /No token data for this session/); }); diff --git a/stats/src/lib/session-detail.test.tsx b/stats/src/lib/session-detail.test.tsx index 7e5e4fb..4fe5f1a 100644 --- a/stats/src/lib/session-detail.test.tsx +++ b/stats/src/lib/session-detail.test.tsx @@ -2,6 +2,8 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { renderToStaticMarkup } from 'react-dom/server'; import { SessionDetail } from '../components/sessions/SessionDetail'; +import { buildSessionChartEvents } from './session-events'; +import { EventType } from '../types/stats'; test('SessionDetail omits the misleading new words metric', () => { const markup = renderToStaticMarkup( @@ -17,7 +19,6 @@ test('SessionDetail omits the misleading new words metric', () => { totalWatchedMs: 0, activeWatchedMs: 0, linesSeen: 12, - wordsSeen: 24, tokensSeen: 24, cardsMined: 0, lookupCount: 0, @@ -27,6 +28,33 @@ test('SessionDetail omits the misleading new words metric', () => { />, ); - assert.match(markup, /Total words/); + assert.match(markup, /No token data/); assert.doesNotMatch(markup, /New words/); }); + +test('buildSessionChartEvents keeps only chart-relevant events and pairs pause ranges', () => { + const chartEvents = buildSessionChartEvents([ + { eventType: EventType.SUBTITLE_LINE, tsMs: 1_000, payload: '{"line":"ignored"}' }, + { eventType: EventType.PAUSE_START, tsMs: 2_000, payload: null }, + { eventType: EventType.SEEK_FORWARD, tsMs: 3_000, payload: null }, + { eventType: EventType.PAUSE_END, tsMs: 4_000, payload: null }, + { eventType: EventType.CARD_MINED, tsMs: 5_000, payload: '{"cardsMined":1}' }, + { eventType: EventType.YOMITAN_LOOKUP, tsMs: 6_000, payload: null }, + { eventType: EventType.SEEK_BACKWARD, tsMs: 7_000, payload: null }, + { eventType: EventType.LOOKUP, tsMs: 8_000, payload: '{"hit":true}' }, + ]); + + assert.deepEqual( + chartEvents.seekEvents.map((event) => event.eventType), + [EventType.SEEK_FORWARD, EventType.SEEK_BACKWARD], + ); + assert.deepEqual( + chartEvents.cardEvents.map((event) => event.tsMs), + [5_000], + ); + assert.deepEqual( + chartEvents.yomitanLookupEvents.map((event) => event.tsMs), + [6_000], + ); + assert.deepEqual(chartEvents.pauseRegions, [{ startMs: 2_000, endMs: 4_000 }]); +}); diff --git a/stats/src/lib/session-word-count.ts b/stats/src/lib/session-word-count.ts index 74740c6..a63c816 100644 --- a/stats/src/lib/session-word-count.ts +++ b/stats/src/lib/session-word-count.ts @@ -1,8 +1,7 @@ type SessionWordCountLike = { - wordsSeen: number; tokensSeen: number; }; export function getSessionDisplayWordCount(value: SessionWordCountLike): number { - return value.tokensSeen > 0 ? value.tokensSeen : value.wordsSeen; + return value.tokensSeen; } diff --git a/stats/src/lib/stats-ui-navigation.test.tsx b/stats/src/lib/stats-ui-navigation.test.tsx index e8f6b23..489b88e 100644 --- a/stats/src/lib/stats-ui-navigation.test.tsx +++ b/stats/src/lib/stats-ui-navigation.test.tsx @@ -26,7 +26,7 @@ test('EpisodeList renders explicit episode detail button alongside quick peek ro totalSessions: 1, totalActiveMs: 1, totalCards: 1, - totalWordsSeen: 350, + totalTokensSeen: 350, totalYomitanLookupCount: 7, lastWatchedMs: 0, }, diff --git a/stats/src/lib/yomitan-lookup.test.tsx b/stats/src/lib/yomitan-lookup.test.tsx index 779364f..38de357 100644 --- a/stats/src/lib/yomitan-lookup.test.tsx +++ b/stats/src/lib/yomitan-lookup.test.tsx @@ -8,10 +8,10 @@ import { SessionRow } from '../components/sessions/SessionRow'; import { EventType, type SessionEvent } from '../types/stats'; import { buildLookupRateDisplay, getYomitanLookupEvents } from './yomitan-lookup'; -test('buildLookupRateDisplay formats lookups per 100 words in short and long forms', () => { +test('buildLookupRateDisplay formats lookups per 100 tokens in short and long forms', () => { assert.deepEqual(buildLookupRateDisplay(23, 1000), { - shortValue: '2.3 / 100 words', - longValue: '2.3 lookups per 100 words', + shortValue: '2.3 / 100 tokens', + longValue: '2.3 lookups per 100 tokens', }); assert.equal(buildLookupRateDisplay(0, 0), null); }); @@ -38,7 +38,7 @@ test('MediaHeader renders Yomitan lookup count and lookup rate copy', () => { totalSessions: 4, totalActiveMs: 90_000, totalCards: 12, - totalWordsSeen: 1000, + totalTokensSeen: 1000, totalLinesSeen: 120, totalLookupCount: 30, totalLookupHits: 21, @@ -48,11 +48,11 @@ test('MediaHeader renders Yomitan lookup count and lookup rate copy', () => { ); assert.match(markup, /23/); - assert.match(markup, /2\.3 \/ 100 words/); - assert.match(markup, /2\.3 lookups per 100 words/); + assert.match(markup, /2\.3 \/ 100 tokens/); + assert.match(markup, /2\.3 lookups per 100 tokens/); }); -test('MediaHeader distinguishes word occurrences from known unique words', () => { +test('MediaHeader distinguishes token occurrences from known unique words', () => { const markup = renderToStaticMarkup( totalSessions: 4, totalActiveMs: 90_000, totalCards: 12, - totalWordsSeen: 30, + totalTokensSeen: 30, totalLinesSeen: 120, totalLookupCount: 30, totalLookupHits: 21, @@ -74,7 +74,7 @@ test('MediaHeader distinguishes word occurrences from known unique words', () => />, ); - assert.match(markup, /word occurrences/); + assert.match(markup, /token occurrences/); assert.match(markup, /known unique words \(50%\)/); assert.match(markup, /17 \/ 34/); }); @@ -93,7 +93,7 @@ test('EpisodeList renders per-episode Yomitan lookup rate', () => { totalSessions: 1, totalActiveMs: 1, totalCards: 1, - totalWordsSeen: 350, + totalTokensSeen: 350, totalYomitanLookupCount: 7, lastWatchedMs: 0, }, @@ -102,7 +102,7 @@ test('EpisodeList renders per-episode Yomitan lookup rate', () => { ); assert.match(markup, /Lookup Rate/); - assert.match(markup, /2\.0 \/ 100 words/); + assert.match(markup, /2\.0 \/ 100 tokens/); }); test('AnimeOverviewStats renders aggregate Yomitan lookup metrics', () => { @@ -119,7 +119,7 @@ test('AnimeOverviewStats renders aggregate Yomitan lookup metrics', () => { totalSessions: 5, totalActiveMs: 100_000, totalCards: 8, - totalWordsSeen: 800, + totalTokensSeen: 800, totalLinesSeen: 100, totalLookupCount: 50, totalLookupHits: 30, @@ -134,8 +134,8 @@ test('AnimeOverviewStats renders aggregate Yomitan lookup metrics', () => { assert.match(markup, /Lookups/); assert.match(markup, /16/); - assert.match(markup, /2\.0 \/ 100 words/); - assert.match(markup, /2\.0 lookups per 100 words/); + assert.match(markup, /2\.0 \/ 100 tokens/); + assert.match(markup, /Yomitan lookups per 100 tokens seen/); }); test('SessionRow prefers token-based word count when available', () => { @@ -152,7 +152,6 @@ test('SessionRow prefers token-based word count when available', () => { totalWatchedMs: 0, activeWatchedMs: 0, linesSeen: 12, - wordsSeen: 12, tokensSeen: 42, cardsMined: 0, lookupCount: 0, diff --git a/stats/src/lib/yomitan-lookup.ts b/stats/src/lib/yomitan-lookup.ts index 839ca2b..db8e769 100644 --- a/stats/src/lib/yomitan-lookup.ts +++ b/stats/src/lib/yomitan-lookup.ts @@ -8,15 +8,15 @@ export interface LookupRateDisplay { export function buildLookupRateDisplay( yomitanLookupCount: number, - wordsSeen: number, + tokensSeen: number, ): LookupRateDisplay | null { - if (!Number.isFinite(yomitanLookupCount) || !Number.isFinite(wordsSeen) || wordsSeen <= 0) { + if (!Number.isFinite(yomitanLookupCount) || !Number.isFinite(tokensSeen) || tokensSeen <= 0) { return null; } - const per100 = ((Math.max(0, yomitanLookupCount) / wordsSeen) * 100).toFixed(1); + const per100 = ((Math.max(0, yomitanLookupCount) / tokensSeen) * 100).toFixed(1); return { - shortValue: `${per100} / 100 words`, - longValue: `${per100} lookups per 100 words`, + shortValue: `${per100} / 100 tokens`, + longValue: `${per100} lookups per 100 tokens`, }; } diff --git a/stats/src/types/stats.ts b/stats/src/types/stats.ts index e1c7d4b..47671b4 100644 --- a/stats/src/types/stats.ts +++ b/stats/src/types/stats.ts @@ -9,7 +9,6 @@ export interface SessionSummary { totalWatchedMs: number; activeWatchedMs: number; linesSeen: number; - wordsSeen: number; tokensSeen: number; cardsMined: number; lookupCount: number; @@ -23,11 +22,10 @@ export interface DailyRollup { totalSessions: number; totalActiveMin: number; totalLinesSeen: number; - totalWordsSeen: number; totalTokensSeen: number; totalCards: number; cardsPerHour: number | null; - wordsPerMin: number | null; + tokensPerMin: number | null; lookupHitRate: number | null; } @@ -38,7 +36,6 @@ export interface SessionTimelinePoint { totalWatchedMs: number; activeWatchedMs: number; linesSeen: number; - wordsSeen: number; tokensSeen: number; cardsMined: number; } @@ -114,7 +111,7 @@ export interface MediaLibraryItem { totalSessions: number; totalActiveMs: number; totalCards: number; - totalWordsSeen: number; + totalTokensSeen: number; lastWatchedMs: number; hasCoverArt: number; } @@ -126,7 +123,7 @@ export interface MediaDetailData { totalSessions: number; totalActiveMs: number; totalCards: number; - totalWordsSeen: number; + totalTokensSeen: number; totalLinesSeen: number; totalLookupCount: number; totalLookupHits: number; @@ -157,7 +154,7 @@ export interface AnimeLibraryItem { totalSessions: number; totalActiveMs: number; totalCards: number; - totalWordsSeen: number; + totalTokensSeen: number; episodeCount: number; episodesTotal: number | null; lastWatchedMs: number; @@ -182,7 +179,7 @@ export interface AnimeDetailData { totalSessions: number; totalActiveMs: number; totalCards: number; - totalWordsSeen: number; + totalTokensSeen: number; totalLinesSeen: number; totalLookupCount: number; totalLookupHits: number; @@ -204,7 +201,7 @@ export interface AnimeEpisode { totalSessions: number; totalActiveMs: number; totalCards: number; - totalWordsSeen: number; + totalTokensSeen: number; totalYomitanLookupCount: number; lastWatchedMs: number; }