From 55ee12e87f4d5cd012f2783f2a47d922e2a38601 Mon Sep 17 00:00:00 2001 From: sudacode Date: Tue, 17 Mar 2026 19:52:59 -0700 Subject: [PATCH] feat: refactor immersion tracker queries and session word tracking Add comprehensive query helpers for session deletion with word aggregate refresh, known-words-per-session timeline, anime-level word summaries, and trends dashboard aggregation. Track yomitanLookupCount in session metrics and support bulk session operations. --- src/config/resolve/immersion-tracking.ts | 12 +- .../immersion-tracker-service.test.ts | 233 +++++ .../services/immersion-tracker-service.ts | 95 +- .../immersion-tracker/__tests__/query.test.ts | 376 +++++++ .../services/immersion-tracker/lifetime.ts | 4 + .../immersion-tracker/maintenance.test.ts | 33 +- .../services/immersion-tracker/maintenance.ts | 4 + src/core/services/immersion-tracker/query.ts | 930 +++++++++++++++++- .../services/immersion-tracker/reducer.ts | 1 + .../services/immersion-tracker/session.ts | 2 + .../immersion-tracker/storage-session.test.ts | 43 +- .../services/immersion-tracker/storage.ts | 32 +- src/core/services/immersion-tracker/types.ts | 9 +- 13 files changed, 1735 insertions(+), 39 deletions(-) diff --git a/src/config/resolve/immersion-tracking.ts b/src/config/resolve/immersion-tracking.ts index ebe8597..c3cf1e8 100644 --- a/src/config/resolve/immersion-tracking.ts +++ b/src/config/resolve/immersion-tracking.ts @@ -242,11 +242,7 @@ export function applyImmersionTrackingConfig(context: ResolveContext): void { } const dailyRollupsDays = asNumber(src.immersionTracking.retention.dailyRollupsDays); - if ( - dailyRollupsDays !== undefined && - dailyRollupsDays >= 0 && - dailyRollupsDays <= 36500 - ) { + if (dailyRollupsDays !== undefined && dailyRollupsDays >= 0 && dailyRollupsDays <= 36500) { retention.dailyRollupsDays = Math.floor(dailyRollupsDays); } else if (src.immersionTracking.retention.dailyRollupsDays !== undefined) { warn( @@ -274,7 +270,11 @@ export function applyImmersionTrackingConfig(context: ResolveContext): void { } const vacuumIntervalDays = asNumber(src.immersionTracking.retention.vacuumIntervalDays); - if (vacuumIntervalDays !== undefined && vacuumIntervalDays >= 0 && vacuumIntervalDays <= 3650) { + if ( + vacuumIntervalDays !== undefined && + vacuumIntervalDays >= 0 && + vacuumIntervalDays <= 3650 + ) { retention.vacuumIntervalDays = Math.floor(vacuumIntervalDays); } else if (src.immersionTracking.retention.vacuumIntervalDays !== undefined) { warn( diff --git a/src/core/services/immersion-tracker-service.test.ts b/src/core/services/immersion-tracker-service.test.ts index 143e6f7..ae6e048 100644 --- a/src/core/services/immersion-tracker-service.test.ts +++ b/src/core/services/immersion-tracker-service.test.ts @@ -840,6 +840,59 @@ test('persists and retrieves minimum immersion tracking fields', async () => { } }); +test('recordYomitanLookup persists a dedicated lookup counter without changing annotation lookup metrics', async () => { + const dbPath = makeDbPath(); + let tracker: ImmersionTrackerService | null = null; + + try { + const Ctor = await loadTrackerCtor(); + tracker = new Ctor({ dbPath }); + + tracker.handleMediaChange('/tmp/episode-yomitan.mkv', 'Episode Yomitan'); + tracker.recordSubtitleLine('alpha beta gamma', 0, 1.2); + tracker.recordLookup(true); + tracker.recordYomitanLookup(); + + const privateApi = tracker as unknown as { + flushTelemetry: (force?: boolean) => void; + flushNow: () => void; + }; + privateApi.flushTelemetry(true); + privateApi.flushNow(); + + const summaries = await tracker.getSessionSummaries(10); + assert.ok(summaries.length >= 1); + assert.equal(summaries[0]?.lookupCount, 1); + assert.equal(summaries[0]?.lookupHits, 1); + assert.equal(summaries[0]?.yomitanLookupCount, 1); + + tracker.destroy(); + + const db = new Database(dbPath); + const sessionRow = db + .prepare('SELECT lookup_count, lookup_hits, yomitan_lookup_count FROM imm_sessions LIMIT 1') + .get() as { + lookup_count: number; + lookup_hits: number; + yomitan_lookup_count: number; + } | null; + const eventRow = db + .prepare( + 'SELECT event_type FROM imm_session_events WHERE event_type = ? ORDER BY ts_ms DESC LIMIT 1', + ) + .get(9) as { event_type: number } | null; + db.close(); + + assert.equal(sessionRow?.lookup_count, 1); + assert.equal(sessionRow?.lookup_hits, 1); + assert.equal(sessionRow?.yomitan_lookup_count, 1); + assert.equal(eventRow?.event_type, 9); + } finally { + tracker?.destroy(); + cleanupDbPath(dbPath); + } +}); + test('recordSubtitleLine persists counted allowed tokenized vocabulary rows and subtitle-line occurrences', async () => { const dbPath = makeDbPath(); let tracker: ImmersionTrackerService | null = null; @@ -1053,6 +1106,140 @@ test('subtitle-line event payload omits duplicated subtitle text', async () => { } }); +test('recordPlaybackPosition marks watched at 85% completion', async () => { + const dbPath = makeDbPath(); + let tracker: ImmersionTrackerService | null = null; + + try { + const Ctor = await loadTrackerCtor(); + tracker = new Ctor({ dbPath }); + + tracker.handleMediaChange('/tmp/episode-85.mkv', 'Episode 85'); + tracker.recordMediaDuration(100); + await waitForPendingAnimeMetadata(tracker); + + const privateApi = tracker as unknown as { + db: DatabaseSync; + sessionState: { videoId: number } | null; + }; + const videoId = privateApi.sessionState?.videoId; + assert.ok(videoId); + + tracker.recordPlaybackPosition(84); + let row = privateApi.db + .prepare('SELECT watched FROM imm_videos WHERE video_id = ?') + .get(videoId) as { watched: number } | null; + assert.equal(row?.watched, 0); + + tracker.recordPlaybackPosition(85); + row = privateApi.db + .prepare('SELECT watched FROM imm_videos WHERE video_id = ?') + .get(videoId) as { watched: number } | null; + assert.equal(row?.watched, 1); + } finally { + tracker?.destroy(); + cleanupDbPath(dbPath); + } +}); + +test('deleteSession ignores the currently active session and keeps new writes flushable', async () => { + const dbPath = makeDbPath(); + let tracker: ImmersionTrackerService | null = null; + + try { + const Ctor = await loadTrackerCtor(); + tracker = new Ctor({ dbPath }); + tracker.handleMediaChange('/tmp/active-delete-test.mkv', 'Active Delete Test'); + + const privateApi = tracker as unknown as { + sessionState: { sessionId: number } | null; + flushTelemetry: (force?: boolean) => void; + flushNow: () => void; + queue: unknown[]; + }; + const sessionId = privateApi.sessionState?.sessionId; + assert.ok(sessionId); + + tracker.recordSubtitleLine('before delete', 0, 1); + privateApi.flushTelemetry(true); + privateApi.flushNow(); + + await tracker.deleteSession(sessionId); + + tracker.recordSubtitleLine('after delete', 1, 2); + privateApi.flushTelemetry(true); + privateApi.flushNow(); + + const db = new Database(dbPath); + const sessionCountRow = db + .prepare('SELECT COUNT(*) AS total FROM imm_sessions WHERE session_id = ?') + .get(sessionId) as { total: number }; + const subtitleLineCountRow = db + .prepare('SELECT COUNT(*) AS total FROM imm_subtitle_lines WHERE session_id = ?') + .get(sessionId) as { total: number }; + db.close(); + + assert.equal(sessionCountRow.total, 1); + assert.equal(subtitleLineCountRow.total, 2); + assert.equal(privateApi.queue.length, 0); + } finally { + tracker?.destroy(); + cleanupDbPath(dbPath); + } +}); + +test('deleteVideo ignores the currently active video and keeps new writes flushable', async () => { + const dbPath = makeDbPath(); + let tracker: ImmersionTrackerService | null = null; + + try { + const Ctor = await loadTrackerCtor(); + tracker = new Ctor({ dbPath }); + tracker.handleMediaChange('/tmp/active-video-delete-test.mkv', 'Active Video Delete Test'); + + const privateApi = tracker as unknown as { + sessionState: { sessionId: number; videoId: number } | null; + flushTelemetry: (force?: boolean) => void; + flushNow: () => void; + queue: unknown[]; + }; + const sessionId = privateApi.sessionState?.sessionId; + const videoId = privateApi.sessionState?.videoId; + assert.ok(sessionId); + assert.ok(videoId); + + tracker.recordSubtitleLine('before video delete', 0, 1); + privateApi.flushTelemetry(true); + privateApi.flushNow(); + + await tracker.deleteVideo(videoId); + + tracker.recordSubtitleLine('after video delete', 1, 2); + privateApi.flushTelemetry(true); + privateApi.flushNow(); + + const db = new Database(dbPath); + const sessionCountRow = db + .prepare('SELECT COUNT(*) AS total FROM imm_sessions WHERE session_id = ?') + .get(sessionId) as { total: number }; + const videoCountRow = db + .prepare('SELECT COUNT(*) AS total FROM imm_videos WHERE video_id = ?') + .get(videoId) as { total: number }; + const subtitleLineCountRow = db + .prepare('SELECT COUNT(*) AS total FROM imm_subtitle_lines WHERE session_id = ?') + .get(sessionId) as { total: number }; + db.close(); + + assert.equal(sessionCountRow.total, 1); + assert.equal(videoCountRow.total, 1); + assert.equal(subtitleLineCountRow.total, 2); + assert.equal(privateApi.queue.length, 0); + } finally { + tracker?.destroy(); + cleanupDbPath(dbPath); + } +}); + test('handleMediaChange links parsed anime metadata on the active video row', async () => { const dbPath = makeDbPath(); let tracker: ImmersionTrackerService | null = null; @@ -1821,3 +2008,49 @@ test('reassignAnimeAnilist deduplicates cover blobs and getCoverArt remains comp cleanupDbPath(dbPath); } }); + +test('markActiveVideoWatched marks current session video as watched', async () => { + const dbPath = makeDbPath(); + let tracker: ImmersionTrackerService | null = null; + + try { + const Ctor = await loadTrackerCtor(); + tracker = new Ctor({ dbPath }); + tracker.handleMediaChange('/tmp/test-mark-active.mkv', 'Test Mark Active'); + await waitForPendingAnimeMetadata(tracker); + + const privateApi = tracker as unknown as { + db: DatabaseSync; + sessionState: { videoId: number; markedWatched: boolean } | null; + }; + const videoId = privateApi.sessionState?.videoId; + assert.ok(videoId); + + const result = await tracker.markActiveVideoWatched(); + assert.equal(result, true); + assert.equal(privateApi.sessionState?.markedWatched, true); + + const row = privateApi.db + .prepare('SELECT watched FROM imm_videos WHERE video_id = ?') + .get(videoId) as { watched: number } | null; + assert.equal(row?.watched, 1); + } finally { + tracker?.destroy(); + cleanupDbPath(dbPath); + } +}); + +test('markActiveVideoWatched returns false when no active session', async () => { + const dbPath = makeDbPath(); + let tracker: ImmersionTrackerService | null = null; + + try { + const Ctor = await loadTrackerCtor(); + tracker = new Ctor({ dbPath }); + const result = await tracker.markActiveVideoWatched(); + assert.equal(result, false); + } finally { + tracker?.destroy(); + cleanupDbPath(dbPath); + } +}); diff --git a/src/core/services/immersion-tracker-service.ts b/src/core/services/immersion-tracker-service.ts index bb99dbb..15c7b6e 100644 --- a/src/core/services/immersion-tracker-service.ts +++ b/src/core/services/immersion-tracker-service.ts @@ -6,6 +6,7 @@ import { getLocalVideoMetadata, guessAnimeVideoMetadata } from './immersion-trac import { pruneRawRetention, pruneRollupRetention, + runOptimizeMaintenance, runRollupMaintenance, } from './immersion-tracker/maintenance'; import { Database, type DatabaseSync } from './immersion-tracker/sqlite'; @@ -60,6 +61,11 @@ import { getSessionEvents, getSessionSummaries, getSessionTimeline, + getSessionWordsByLine, + getTrendsDashboard, + getAllDistinctHeadwords, + getAnimeDistinctHeadwords, + getMediaDistinctHeadwords, getVocabularyStats, getWatchTimePerAnime, getWordAnimeAppearances, @@ -69,6 +75,7 @@ import { upsertCoverArt, markVideoWatched, deleteSession as deleteSessionQuery, + deleteSessions as deleteSessionsQuery, deleteVideo as deleteVideoQuery, } from './immersion-tracker/query'; import { @@ -83,6 +90,7 @@ import { sanitizePayload, secToMs, } from './immersion-tracker/reducer'; +import { DEFAULT_MIN_WATCH_RATIO } from '../../shared/watch-threshold'; import { enqueueWrite } from './immersion-tracker/queue'; import { DEFAULT_BATCH_SIZE, @@ -104,6 +112,7 @@ import { EVENT_SEEK_BACKWARD, EVENT_SEEK_FORWARD, EVENT_SUBTITLE_LINE, + EVENT_YOMITAN_LOOKUP, SOURCE_TYPE_LOCAL, SOURCE_TYPE_REMOTE, type ImmersionSessionRollupRow, @@ -244,13 +253,21 @@ export class ImmersionTrackerService { ); const retention = policy.retention ?? {}; - const daysToRetentionMs = (value: number | undefined, fallbackMs: number, maxDays: number): number => { + const daysToRetentionMs = ( + value: number | undefined, + fallbackMs: number, + maxDays: number, + ): number => { const fallbackDays = Math.floor(fallbackMs / 86_400_000); const resolvedDays = resolveBoundedInt(value, fallbackDays, 0, maxDays); return resolvedDays === 0 ? Number.POSITIVE_INFINITY : resolvedDays * 86_400_000; }; - this.eventsRetentionMs = daysToRetentionMs(retention.eventsDays, DEFAULT_EVENTS_RETENTION_MS, 3650); + this.eventsRetentionMs = daysToRetentionMs( + retention.eventsDays, + DEFAULT_EVENTS_RETENTION_MS, + 3650, + ); this.telemetryRetentionMs = daysToRetentionMs( retention.telemetryDays, DEFAULT_TELEMETRY_RETENTION_MS, @@ -321,6 +338,24 @@ export class ImmersionTrackerService { return getSessionTimeline(this.db, sessionId, limit); } + async getSessionWordsByLine( + sessionId: number, + ): Promise> { + return getSessionWordsByLine(this.db, sessionId); + } + + async getAllDistinctHeadwords(): Promise { + return getAllDistinctHeadwords(this.db); + } + + async getAnimeDistinctHeadwords(animeId: number): Promise { + return getAnimeDistinctHeadwords(this.db, animeId); + } + + async getMediaDistinctHeadwords(videoId: number): Promise { + return getMediaDistinctHeadwords(this.db, videoId); + } + async getQueryHints(): Promise<{ totalSessions: number; activeSessions: number; @@ -343,6 +378,13 @@ export class ImmersionTrackerService { return getMonthlyRollups(this.db, limit); } + async getTrendsDashboard( + range: '7d' | '30d' | '90d' | 'all' = '30d', + groupBy: 'day' | 'month' = 'day', + ): Promise { + return getTrendsDashboard(this.db, range, groupBy); + } + async getVocabularyStats(limit = 100, excludePos?: string[]): Promise { return getVocabularyStats(this.db, limit, excludePos); } @@ -437,11 +479,40 @@ export class ImmersionTrackerService { markVideoWatched(this.db, videoId, watched); } + async markActiveVideoWatched(): Promise { + if (!this.sessionState) return false; + markVideoWatched(this.db, this.sessionState.videoId, true); + this.sessionState.markedWatched = true; + return true; + } + async deleteSession(sessionId: number): Promise { + if (this.sessionState?.sessionId === sessionId) { + this.logger.warn(`Ignoring delete request for active immersion session ${sessionId}`); + return; + } deleteSessionQuery(this.db, sessionId); } + async deleteSessions(sessionIds: number[]): Promise { + const activeSessionId = this.sessionState?.sessionId; + const deletableSessionIds = + activeSessionId === undefined + ? sessionIds + : sessionIds.filter((sessionId) => sessionId !== activeSessionId); + if (deletableSessionIds.length !== sessionIds.length) { + this.logger.warn( + `Ignoring bulk delete request for active immersion session ${activeSessionId}`, + ); + } + deleteSessionsQuery(this.db, deletableSessionIds); + } + async deleteVideo(videoId: number): Promise { + if (this.sessionState?.videoId === videoId) { + this.logger.warn(`Ignoring delete request for active immersion video ${videoId}`); + return; + } deleteVideoQuery(this.db, videoId); } @@ -847,7 +918,7 @@ export class ImmersionTrackerService { if (!this.sessionState.markedWatched) { const durationMs = getVideoDurationMs(this.db, this.sessionState.videoId); - if (durationMs > 0 && mediaMs >= durationMs * 0.98) { + if (durationMs > 0 && mediaMs >= durationMs * DEFAULT_MIN_WATCH_RATIO) { markVideoWatched(this.db, this.sessionState.videoId, true); this.sessionState.markedWatched = true; } @@ -915,6 +986,21 @@ export class ImmersionTrackerService { }); } + recordYomitanLookup(): void { + if (!this.sessionState) return; + this.sessionState.yomitanLookupCount += 1; + this.sessionState.pendingTelemetry = true; + this.recordWrite({ + kind: 'event', + sessionId: this.sessionState.sessionId, + sampleMs: Date.now(), + eventType: EVENT_YOMITAN_LOOKUP, + cardsDelta: 0, + wordsDelta: 0, + payloadJson: null, + }); + } + recordCardsMined(count = 1, noteIds?: number[]): void { if (!this.sessionState) return; this.sessionState.cardsMined += count; @@ -981,6 +1067,7 @@ export class ImmersionTrackerService { cardsMined: this.sessionState.cardsMined, lookupCount: this.sessionState.lookupCount, lookupHits: this.sessionState.lookupHits, + yomitanLookupCount: this.sessionState.yomitanLookupCount, pauseCount: this.sessionState.pauseCount, pauseMs: this.sessionState.pauseMs, seekForwardCount: this.sessionState.seekForwardCount, @@ -1080,6 +1167,7 @@ export class ImmersionTrackerService { this.db.exec('VACUUM'); this.lastVacuumMs = nowMs; } + runOptimizeMaintenance(this.db); } catch (error) { this.logger.warn( 'Immersion tracker maintenance failed, will retry later', @@ -1108,6 +1196,7 @@ export class ImmersionTrackerService { cardsMined: 0, lookupCount: 0, lookupHits: 0, + yomitanLookupCount: 0, pauseCount: 0, pauseMs: 0, seekForwardCount: 0, diff --git a/src/core/services/immersion-tracker/__tests__/query.test.ts b/src/core/services/immersion-tracker/__tests__/query.test.ts index 1d646c9..b7bfbe2 100644 --- a/src/core/services/immersion-tracker/__tests__/query.test.ts +++ b/src/core/services/immersion-tracker/__tests__/query.test.ts @@ -17,6 +17,7 @@ import { cleanupVocabularyStats, deleteSession, getDailyRollups, + getTrendsDashboard, getQueryHints, getMonthlyRollups, getAnimeDetail, @@ -31,6 +32,7 @@ import { getVocabularyStats, getKanjiStats, getSessionEvents, + getSessionWordsByLine, getWordOccurrences, upsertCoverArt, } from '../query.js'; @@ -126,6 +128,7 @@ test('getSessionSummaries returns sessionId and canonicalTitle', () => { assert.equal(row.tokensSeen, 10); assert.equal(row.lookupCount, 2); assert.equal(row.lookupHits, 1); + assert.equal(row.yomitanLookupCount, 0); } finally { db.close(); cleanupDbPath(dbPath); @@ -165,6 +168,163 @@ test('getDailyRollups limits by distinct days (not rows)', () => { } }); +test('getTrendsDashboard returns chart-ready aggregated series', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const stmts = createTrackerPreparedStatements(db); + + const videoId = getOrCreateVideoRecord(db, 'local:/tmp/trends-dashboard-test.mkv', { + canonicalTitle: 'Trend Dashboard Test', + sourcePath: '/tmp/trends-dashboard-test.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + const animeId = getOrCreateAnimeRecord(db, { + parsedTitle: 'Trend Dashboard Anime', + canonicalTitle: 'Trend Dashboard Anime', + anilistId: null, + titleRomaji: null, + titleEnglish: null, + titleNative: null, + metadataJson: null, + }); + linkVideoToAnimeRecord(db, videoId, { + animeId, + parsedBasename: 'trends-dashboard-test.mkv', + parsedTitle: 'Trend Dashboard Anime', + parsedSeason: 1, + parsedEpisode: 1, + parserSource: 'test', + parserConfidence: 1, + parseMetadataJson: null, + }); + + const dayOneStart = new Date(2026, 2, 15, 12, 0, 0, 0).getTime(); + const dayTwoStart = new Date(2026, 2, 16, 18, 0, 0, 0).getTime(); + + const sessionOne = startSessionRecord(db, videoId, dayOneStart); + const sessionTwo = startSessionRecord(db, videoId, dayTwoStart); + + for (const [ + sessionId, + startedAtMs, + activeWatchedMs, + cardsMined, + wordsSeen, + tokensSeen, + yomitanLookupCount, + ] of [ + [sessionOne.sessionId, dayOneStart, 30 * 60_000, 2, 100, 120, 8], + [sessionTwo.sessionId, dayTwoStart, 45 * 60_000, 3, 120, 140, 10], + ] as const) { + stmts.telemetryInsertStmt.run( + sessionId, + startedAtMs + 60_000, + activeWatchedMs, + activeWatchedMs, + 10, + wordsSeen, + tokensSeen, + cardsMined, + 0, + 0, + yomitanLookupCount, + 0, + 0, + 0, + 0, + startedAtMs + 60_000, + startedAtMs + 60_000, + ); + + db.prepare( + ` + UPDATE imm_sessions + SET + ended_at_ms = ?, + total_watched_ms = ?, + active_watched_ms = ?, + lines_seen = ?, + words_seen = ?, + tokens_seen = ?, + cards_mined = ?, + yomitan_lookup_count = ? + WHERE session_id = ? + `, + ).run( + startedAtMs + activeWatchedMs, + activeWatchedMs, + activeWatchedMs, + 10, + wordsSeen, + tokensSeen, + cardsMined, + yomitanLookupCount, + sessionId, + ); + } + + db.prepare( + ` + INSERT INTO imm_daily_rollups ( + rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, + total_words_seen, total_tokens_seen, total_cards + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, + ).run(Math.floor(dayOneStart / 86_400_000), videoId, 1, 30, 10, 100, 120, 2); + + db.prepare( + ` + INSERT INTO imm_daily_rollups ( + rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, + total_words_seen, total_tokens_seen, total_cards + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, + ).run(Math.floor(dayTwoStart / 86_400_000), videoId, 1, 45, 10, 120, 140, 3); + + db.prepare( + ` + INSERT INTO imm_words ( + headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + ).run( + '勉強', + '勉強', + 'べんきょう', + 'noun', + '名詞', + null, + null, + Math.floor(dayOneStart / 1000), + Math.floor(dayTwoStart / 1000), + ); + + const dashboard = getTrendsDashboard(db, 'all', 'day'); + + assert.equal(dashboard.activity.watchTime.length, 2); + assert.equal(dashboard.activity.watchTime[0]?.value, 30); + assert.equal(dashboard.progress.watchTime[1]?.value, 75); + assert.equal(dashboard.progress.lookups[1]?.value, 18); + assert.equal( + dashboard.ratios.lookupsPerHundred[0]?.value, + +((8 / 120) * 100).toFixed(1), + ); + assert.equal(dashboard.animePerDay.watchTime[0]?.animeTitle, 'Trend Dashboard Anime'); + assert.equal(dashboard.animeCumulative.watchTime[1]?.value, 75); + assert.equal( + dashboard.patterns.watchTimeByDayOfWeek.reduce((sum, point) => sum + point.value, 0), + 75, + ); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('getQueryHints reads all-time totals from lifetime summary', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -238,6 +398,7 @@ test('getSessionSummaries with no telemetry returns zero aggregates', () => { assert.equal(row.tokensSeen, 0); assert.equal(row.lookupCount, 0); assert.equal(row.lookupHits, 0); + assert.equal(row.yomitanLookupCount, 0); assert.equal(row.cardsMined, 0); } finally { db.close(); @@ -292,6 +453,7 @@ test('getSessionSummaries uses denormalized session metrics for ended sessions w assert.equal(row.cardsMined, 5); assert.equal(row.lookupCount, 9); assert.equal(row.lookupHits, 6); + assert.equal(row.yomitanLookupCount, 0); } finally { db.close(); cleanupDbPath(dbPath); @@ -950,6 +1112,56 @@ test('getSessionEvents respects limit parameter', () => { } }); +test('getSessionWordsByLine joins word occurrences through imm_words.id', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const stmts = createTrackerPreparedStatements(db); + const startedAtMs = Date.UTC(2025, 0, 1, 12, 0, 0); + const videoId = getOrCreateVideoRecord(db, '/tmp/session-words-by-line.mkv', { + canonicalTitle: 'Episode', + sourcePath: '/tmp/session-words-by-line.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + const { sessionId } = startSessionRecord(db, videoId, startedAtMs); + const lineId = Number( + db + .prepare( + `INSERT INTO imm_subtitle_lines ( + session_id, event_id, video_id, anime_id, line_index, segment_start_ms, segment_end_ms, text, CREATED_DATE, LAST_UPDATE_DATE + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run(sessionId, null, videoId, null, 0, 0, 1000, '猫を見た', startedAtMs, startedAtMs) + .lastInsertRowid, + ); + const wordId = Number( + db + .prepare( + `INSERT INTO imm_words ( + headword, word, reading, pos1, pos2, pos3, part_of_speech, first_seen, last_seen, frequency + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run('猫', '猫', 'ねこ', null, null, null, null, startedAtMs, startedAtMs, 1) + .lastInsertRowid, + ); + + db.prepare( + `INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count) + VALUES (?, ?, ?)`, + ).run(lineId, wordId, 1); + + assert.deepEqual(getSessionWordsByLine(db, sessionId), [ + { lineIndex: 0, headword: '猫', occurrenceCount: 1 }, + ]); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('anime-level queries group by anime_id and preserve episode-level rows', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -1192,6 +1404,7 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () assert.equal(animeDetail?.totalLinesSeen, 33); assert.equal(animeDetail?.totalLookupCount, 12); assert.equal(animeDetail?.totalLookupHits, 8); + assert.equal(animeDetail?.totalYomitanLookupCount, 0); assert.equal(animeDetail?.episodeCount, 2); const episodes = getAnimeEpisodes(db, lwaAnimeId); @@ -1203,6 +1416,8 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () totalSessions: row.totalSessions, totalActiveMs: row.totalActiveMs, totalCards: row.totalCards, + totalWordsSeen: row.totalWordsSeen, + totalYomitanLookupCount: row.totalYomitanLookupCount, })), [ { @@ -1212,6 +1427,8 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () totalSessions: 2, totalActiveMs: 7_000, totalCards: 3, + totalWordsSeen: 52, + totalYomitanLookupCount: 0, }, { videoId: lwaEpisode6, @@ -1220,6 +1437,8 @@ test('anime-level queries group by anime_id and preserve episode-level rows', () totalSessions: 1, totalActiveMs: 5_000, totalCards: 3, + totalWordsSeen: 28, + totalYomitanLookupCount: 0, }, ], ); @@ -1681,6 +1900,7 @@ test('anime/media detail and episode queries use ended-session metrics when tele assert.equal(mediaDetail?.totalWordsSeen, 30); assert.equal(mediaDetail?.totalLookupCount, 9); assert.equal(mediaDetail?.totalLookupHits, 7); + assert.equal(mediaDetail?.totalYomitanLookupCount, 0); } finally { db.close(); cleanupDbPath(dbPath); @@ -1984,3 +2204,159 @@ test('deleteSession removes the session and all associated session-scoped rows', cleanupDbPath(dbPath); } }); + +test('deleteSession rebuilds word and kanji aggregates from retained subtitle lines', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const stmts = createTrackerPreparedStatements(db); + + const videoId = getOrCreateVideoRecord(db, 'local:/tmp/delete-session-aggregates.mkv', { + canonicalTitle: 'Delete Session Aggregates Test', + sourcePath: '/tmp/delete-session-aggregates.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + + const deletedSession = startSessionRecord(db, videoId, 7_000_000); + const keptSession = startSessionRecord(db, videoId, 8_000_000); + const deletedTs = 7_000_500; + const keptTs = 8_000_500; + + const sharedWordResult = db + .prepare( + `INSERT INTO imm_words ( + headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run('共有', '共有', 'きょうゆう', 'noun', '名詞', '一般', '', deletedTs, keptTs, 3); + const deletedOnlyWordResult = db + .prepare( + `INSERT INTO imm_words ( + headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + '削除専用', + '削除専用', + 'さくじょせんよう', + 'noun', + '名詞', + '一般', + '', + deletedTs, + deletedTs, + 1, + ); + const sharedKanjiResult = db + .prepare( + `INSERT INTO imm_kanji ( + kanji, first_seen, last_seen, frequency + ) VALUES (?, ?, ?, ?)`, + ) + .run('共', deletedTs, keptTs, 3); + const deletedOnlyKanjiResult = db + .prepare( + `INSERT INTO imm_kanji ( + kanji, first_seen, last_seen, frequency + ) VALUES (?, ?, ?, ?)`, + ) + .run('削', deletedTs, deletedTs, 1); + + const deletedLineResult = stmts.subtitleLineInsertStmt.run( + deletedSession.sessionId, + null, + videoId, + null, + 0, + 0, + 800, + 'delete me', + deletedTs, + deletedTs, + ); + const keptLineResult = stmts.subtitleLineInsertStmt.run( + keptSession.sessionId, + null, + videoId, + null, + 0, + 1_000, + 1_800, + 'keep me', + keptTs, + keptTs, + ); + + const deletedLineId = Number(deletedLineResult.lastInsertRowid); + const keptLineId = Number(keptLineResult.lastInsertRowid); + const sharedWordId = Number(sharedWordResult.lastInsertRowid); + const deletedOnlyWordId = Number(deletedOnlyWordResult.lastInsertRowid); + const sharedKanjiId = Number(sharedKanjiResult.lastInsertRowid); + const deletedOnlyKanjiId = Number(deletedOnlyKanjiResult.lastInsertRowid); + + db.prepare( + `INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count) + VALUES (?, ?, ?)`, + ).run(deletedLineId, sharedWordId, 2); + db.prepare( + `INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count) + VALUES (?, ?, ?)`, + ).run(deletedLineId, deletedOnlyWordId, 1); + db.prepare( + `INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count) + VALUES (?, ?, ?)`, + ).run(keptLineId, sharedWordId, 1); + db.prepare( + `INSERT INTO imm_kanji_line_occurrences (line_id, kanji_id, occurrence_count) + VALUES (?, ?, ?)`, + ).run(deletedLineId, sharedKanjiId, 2); + db.prepare( + `INSERT INTO imm_kanji_line_occurrences (line_id, kanji_id, occurrence_count) + VALUES (?, ?, ?)`, + ).run(deletedLineId, deletedOnlyKanjiId, 1); + db.prepare( + `INSERT INTO imm_kanji_line_occurrences (line_id, kanji_id, occurrence_count) + VALUES (?, ?, ?)`, + ).run(keptLineId, sharedKanjiId, 1); + + deleteSession(db, deletedSession.sessionId); + + const sharedWordRow = db + .prepare('SELECT frequency, first_seen, last_seen FROM imm_words WHERE id = ?') + .get(sharedWordId) as { + frequency: number; + first_seen: number; + last_seen: number; + } | null; + const deletedOnlyWordRow = db + .prepare('SELECT id FROM imm_words WHERE id = ?') + .get(deletedOnlyWordId) as { id: number } | null; + const sharedKanjiRow = db + .prepare('SELECT frequency, first_seen, last_seen FROM imm_kanji WHERE id = ?') + .get(sharedKanjiId) as { + frequency: number; + first_seen: number; + last_seen: number; + } | null; + const deletedOnlyKanjiRow = db + .prepare('SELECT id FROM imm_kanji WHERE id = ?') + .get(deletedOnlyKanjiId) as { id: number } | null; + + assert.ok(sharedWordRow); + assert.equal(sharedWordRow.frequency, 1); + assert.equal(sharedWordRow.first_seen, keptTs); + assert.equal(sharedWordRow.last_seen, keptTs); + assert.equal(deletedOnlyWordRow ?? null, null); + assert.ok(sharedKanjiRow); + assert.equal(sharedKanjiRow.frequency, 1); + assert.equal(sharedKanjiRow.first_seen, keptTs); + assert.equal(sharedKanjiRow.last_seen, keptTs); + assert.equal(deletedOnlyKanjiRow ?? null, null); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); diff --git a/src/core/services/immersion-tracker/lifetime.ts b/src/core/services/immersion-tracker/lifetime.ts index 8a28a56..9673dad 100644 --- a/src/core/services/immersion-tracker/lifetime.ts +++ b/src/core/services/immersion-tracker/lifetime.ts @@ -51,6 +51,7 @@ interface RetainedSessionRow { cardsMined: number; lookupCount: number; lookupHits: number; + yomitanLookupCount: number; pauseCount: number; pauseMs: number; seekForwardCount: number; @@ -154,6 +155,7 @@ function toRebuildSessionState(row: RetainedSessionRow): SessionState { cardsMined: Math.max(0, row.cardsMined), lookupCount: Math.max(0, row.lookupCount), lookupHits: Math.max(0, row.lookupHits), + yomitanLookupCount: Math.max(0, row.yomitanLookupCount), pauseCount: Math.max(0, row.pauseCount), pauseMs: Math.max(0, row.pauseMs), seekForwardCount: Math.max(0, row.seekForwardCount), @@ -179,6 +181,7 @@ function getRetainedStaleActiveSessions(db: DatabaseSync): RetainedSessionRow[] COALESCE(t.cards_mined, s.cards_mined, 0) AS cardsMined, COALESCE(t.lookup_count, s.lookup_count, 0) AS lookupCount, COALESCE(t.lookup_hits, s.lookup_hits, 0) AS lookupHits, + COALESCE(t.yomitan_lookup_count, s.yomitan_lookup_count, 0) AS yomitanLookupCount, COALESCE(t.pause_count, s.pause_count, 0) AS pauseCount, COALESCE(t.pause_ms, s.pause_ms, 0) AS pauseMs, COALESCE(t.seek_forward_count, s.seek_forward_count, 0) AS seekForwardCount, @@ -511,6 +514,7 @@ export function rebuildLifetimeSummaries(db: DatabaseSync): LifetimeRebuildSumma cards_mined AS cardsMined, lookup_count AS lookupCount, lookup_hits AS lookupHits, + yomitan_lookup_count AS yomitanLookupCount, pause_count AS pauseCount, pause_ms AS pauseMs, seek_forward_count AS seekForwardCount, diff --git a/src/core/services/immersion-tracker/maintenance.test.ts b/src/core/services/immersion-tracker/maintenance.test.ts index e99ceb0..9ff738b 100644 --- a/src/core/services/immersion-tracker/maintenance.test.ts +++ b/src/core/services/immersion-tracker/maintenance.test.ts @@ -4,7 +4,12 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { Database } from './sqlite'; -import { pruneRawRetention, pruneRollupRetention, toMonthKey } from './maintenance'; +import { + pruneRawRetention, + pruneRollupRetention, + runOptimizeMaintenance, + toMonthKey, +} from './maintenance'; import { ensureSchema } from './storage'; function makeDbPath(): string { @@ -161,15 +166,15 @@ test('ensureSchema adds sample_ms index for telemetry rollup scans', () => { try { ensureSchema(db); - const indexes = db - .prepare("PRAGMA index_list('imm_session_telemetry')") - .all() as Array<{ name: string }>; + const indexes = db.prepare("PRAGMA index_list('imm_session_telemetry')").all() as Array<{ + name: string; + }>; const hasSampleMsIndex = indexes.some((row) => row.name === 'idx_telemetry_sample_ms'); assert.equal(hasSampleMsIndex, true); - const indexColumns = db - .prepare("PRAGMA index_info('idx_telemetry_sample_ms')") - .all() as Array<{ name: string }>; + const indexColumns = db.prepare("PRAGMA index_info('idx_telemetry_sample_ms')").all() as Array<{ + name: string; + }>; assert.deepEqual( indexColumns.map((column) => column.name), ['sample_ms'], @@ -179,3 +184,17 @@ test('ensureSchema adds sample_ms index for telemetry rollup scans', () => { cleanupDbPath(dbPath); } }); + +test('runOptimizeMaintenance executes PRAGMA optimize', () => { + const executedSql: string[] = []; + const db = { + exec(source: string) { + executedSql.push(source); + return this; + }, + } as unknown as Parameters[0]; + + runOptimizeMaintenance(db); + + assert.deepEqual(executedSql, ['PRAGMA optimize']); +}); diff --git a/src/core/services/immersion-tracker/maintenance.ts b/src/core/services/immersion-tracker/maintenance.ts index 98412a9..9393737 100644 --- a/src/core/services/immersion-tracker/maintenance.ts +++ b/src/core/services/immersion-tracker/maintenance.ts @@ -329,3 +329,7 @@ export function runRollupMaintenance(db: DatabaseSync, forceRebuild = false): vo throw error; } } + +export function runOptimizeMaintenance(db: DatabaseSync): void { + db.exec('PRAGMA optimize'); +} diff --git a/src/core/services/immersion-tracker/query.ts b/src/core/services/immersion-tracker/query.ts index ad9d306..2d5b243 100644 --- a/src/core/services/immersion-tracker/query.ts +++ b/src/core/services/immersion-tracker/query.ts @@ -81,7 +81,8 @@ const ACTIVE_SESSION_METRICS_CTE = ` MAX(t.tokens_seen) AS tokensSeen, MAX(t.cards_mined) AS cardsMined, MAX(t.lookup_count) AS lookupCount, - MAX(t.lookup_hits) AS lookupHits + MAX(t.lookup_hits) AS lookupHits, + MAX(t.yomitan_lookup_count) AS yomitanLookupCount FROM imm_session_telemetry t JOIN imm_sessions s ON s.session_id = t.session_id WHERE s.ended_at_ms IS NULL @@ -155,6 +156,189 @@ function findSharedCoverBlobHash( return null; } +function makePlaceholders(values: number[]): string { + return values.map(() => '?').join(','); +} + +function getAffectedWordIdsForSessions(db: DatabaseSync, sessionIds: number[]): number[] { + if (sessionIds.length === 0) { + return []; + } + + return ( + db + .prepare( + ` + SELECT DISTINCT o.word_id AS wordId + FROM imm_word_line_occurrences o + JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id + WHERE sl.session_id IN (${makePlaceholders(sessionIds)}) + `, + ) + .all(...sessionIds) as Array<{ wordId: number }> + ).map((row) => row.wordId); +} + +function getAffectedKanjiIdsForSessions(db: DatabaseSync, sessionIds: number[]): number[] { + if (sessionIds.length === 0) { + return []; + } + + return ( + db + .prepare( + ` + SELECT DISTINCT o.kanji_id AS kanjiId + FROM imm_kanji_line_occurrences o + JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id + WHERE sl.session_id IN (${makePlaceholders(sessionIds)}) + `, + ) + .all(...sessionIds) as Array<{ kanjiId: number }> + ).map((row) => row.kanjiId); +} + +function getAffectedWordIdsForVideo(db: DatabaseSync, videoId: number): number[] { + return ( + db + .prepare( + ` + SELECT DISTINCT o.word_id AS wordId + FROM imm_word_line_occurrences o + JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id + WHERE sl.video_id = ? + `, + ) + .all(videoId) as Array<{ wordId: number }> + ).map((row) => row.wordId); +} + +function getAffectedKanjiIdsForVideo(db: DatabaseSync, videoId: number): number[] { + return ( + db + .prepare( + ` + SELECT DISTINCT o.kanji_id AS kanjiId + FROM imm_kanji_line_occurrences o + JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id + WHERE sl.video_id = ? + `, + ) + .all(videoId) as Array<{ kanjiId: number }> + ).map((row) => row.kanjiId); +} + +function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void { + if (wordIds.length === 0) { + return; + } + + const rows = db + .prepare( + ` + SELECT + w.id AS wordId, + COALESCE(SUM(o.occurrence_count), 0) AS frequency, + MIN(COALESCE(sl.CREATED_DATE, sl.LAST_UPDATE_DATE)) AS firstSeen, + MAX(COALESCE(sl.LAST_UPDATE_DATE, sl.CREATED_DATE)) AS lastSeen + FROM imm_words w + LEFT JOIN imm_word_line_occurrences o ON o.word_id = w.id + LEFT JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id + WHERE w.id IN (${makePlaceholders(wordIds)}) + GROUP BY w.id + `, + ) + .all(...wordIds) as Array<{ + wordId: number; + frequency: number; + firstSeen: number | null; + lastSeen: number | null; + }>; + const updateStmt = db.prepare( + ` + UPDATE imm_words + SET frequency = ?, first_seen = ?, last_seen = ? + WHERE id = ? + `, + ); + const deleteStmt = db.prepare('DELETE FROM imm_words WHERE id = ?'); + + for (const row of rows) { + if (row.frequency <= 0 || row.firstSeen === null || row.lastSeen === null) { + deleteStmt.run(row.wordId); + continue; + } + updateStmt.run(row.frequency, row.firstSeen, row.lastSeen, row.wordId); + } +} + +function refreshKanjiAggregates(db: DatabaseSync, kanjiIds: number[]): void { + if (kanjiIds.length === 0) { + return; + } + + const rows = db + .prepare( + ` + SELECT + k.id AS kanjiId, + COALESCE(SUM(o.occurrence_count), 0) AS frequency, + MIN(COALESCE(sl.CREATED_DATE, sl.LAST_UPDATE_DATE)) AS firstSeen, + MAX(COALESCE(sl.LAST_UPDATE_DATE, sl.CREATED_DATE)) AS lastSeen + FROM imm_kanji k + LEFT JOIN imm_kanji_line_occurrences o ON o.kanji_id = k.id + LEFT JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id + WHERE k.id IN (${makePlaceholders(kanjiIds)}) + GROUP BY k.id + `, + ) + .all(...kanjiIds) as Array<{ + kanjiId: number; + frequency: number; + firstSeen: number | null; + lastSeen: number | null; + }>; + const updateStmt = db.prepare( + ` + UPDATE imm_kanji + SET frequency = ?, first_seen = ?, last_seen = ? + WHERE id = ? + `, + ); + const deleteStmt = db.prepare('DELETE FROM imm_kanji WHERE id = ?'); + + for (const row of rows) { + if (row.frequency <= 0 || row.firstSeen === null || row.lastSeen === null) { + deleteStmt.run(row.kanjiId); + continue; + } + updateStmt.run(row.frequency, row.firstSeen, row.lastSeen, row.kanjiId); + } +} + +function refreshLexicalAggregates(db: DatabaseSync, wordIds: number[], kanjiIds: number[]): void { + refreshWordAggregates(db, [...new Set(wordIds)]); + refreshKanjiAggregates(db, [...new Set(kanjiIds)]); +} + +function deleteSessionsByIds(db: DatabaseSync, sessionIds: number[]): void { + if (sessionIds.length === 0) { + return; + } + + const placeholders = makePlaceholders(sessionIds); + db.prepare(`DELETE FROM imm_subtitle_lines WHERE session_id IN (${placeholders})`).run( + ...sessionIds, + ); + db.prepare(`DELETE FROM imm_session_telemetry WHERE session_id IN (${placeholders})`).run( + ...sessionIds, + ); + db.prepare(`DELETE FROM imm_session_events WHERE session_id IN (${placeholders})`).run( + ...sessionIds, + ); + db.prepare(`DELETE FROM imm_sessions WHERE session_id IN (${placeholders})`).run(...sessionIds); +} + export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummaryQueryRow[] { const prepared = db.prepare(` ${ACTIVE_SESSION_METRICS_CTE} @@ -173,7 +357,8 @@ export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummar COALESCE(asm.tokensSeen, s.tokens_seen, 0) AS tokensSeen, COALESCE(asm.cardsMined, s.cards_mined, 0) AS cardsMined, COALESCE(asm.lookupCount, s.lookup_count, 0) AS lookupCount, - COALESCE(asm.lookupHits, s.lookup_hits, 0) AS lookupHits + COALESCE(asm.lookupHits, s.lookup_hits, 0) AS lookupHits, + COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0) AS yomitanLookupCount FROM imm_sessions s LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id LEFT JOIN imm_videos v ON v.video_id = s.video_id @@ -206,6 +391,72 @@ export function getSessionTimeline( return prepared.all(sessionId, limit) as unknown as SessionTimelineRow[]; } +/** Returns all distinct headwords in the vocabulary table (global). */ +export function getAllDistinctHeadwords(db: DatabaseSync): string[] { + const rows = db.prepare('SELECT DISTINCT headword FROM imm_words').all() as Array<{ + headword: string; + }>; + return rows.map((r) => r.headword); +} + +/** Returns distinct headwords seen for a specific anime. */ +export function getAnimeDistinctHeadwords(db: DatabaseSync, animeId: number): string[] { + const rows = db + .prepare( + ` + SELECT DISTINCT w.headword + FROM imm_word_line_occurrences o + JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id + JOIN imm_words w ON w.id = o.word_id + WHERE sl.anime_id = ? + `, + ) + .all(animeId) as Array<{ headword: string }>; + return rows.map((r) => r.headword); +} + +/** Returns distinct headwords seen for a specific video/media. */ +export function getMediaDistinctHeadwords(db: DatabaseSync, videoId: number): string[] { + const rows = db + .prepare( + ` + SELECT DISTINCT w.headword + FROM imm_word_line_occurrences o + JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id + JOIN imm_words w ON w.id = o.word_id + WHERE sl.video_id = ? + `, + ) + .all(videoId) as Array<{ headword: string }>; + return rows.map((r) => r.headword); +} + +/** + * Returns the headword for each word seen in a session, grouped by line_index. + * Used to compute cumulative known-words counts for the session timeline chart. + */ +export function getSessionWordsByLine( + db: DatabaseSync, + sessionId: number, +): Array<{ lineIndex: number; headword: string; occurrenceCount: number }> { + const stmt = db.prepare(` + SELECT + sl.line_index AS lineIndex, + w.headword AS headword, + wlo.occurrence_count AS occurrenceCount + FROM imm_subtitle_lines sl + JOIN imm_word_line_occurrences wlo ON wlo.line_id = sl.line_id + JOIN imm_words w ON w.id = wlo.word_id + WHERE sl.session_id = ? + ORDER BY sl.line_index ASC + `); + return stmt.all(sessionId) as Array<{ + lineIndex: number; + headword: string; + occurrenceCount: number; + }>; +} + export function getQueryHints(db: DatabaseSync): { totalSessions: number; activeSessions: number; @@ -216,6 +467,10 @@ export function getQueryHints(db: DatabaseSync): { totalActiveMin: number; totalCards: number; activeDays: number; + totalLookupCount: number; + totalLookupHits: number; + newWordsToday: number; + newWordsThisWeek: number; } { const active = db.prepare('SELECT COUNT(*) AS total FROM imm_sessions WHERE ended_at_ms IS NULL'); const activeSessions = Number((active.get() as { total?: number } | null)?.total ?? 0); @@ -284,6 +539,23 @@ export function getQueryHints(db: DatabaseSync): { const totalCards = Number(lifetime?.totalCards ?? 0); const activeDays = Number(lifetime?.activeDays ?? 0); + const lookupTotals = db + .prepare( + ` + SELECT + COALESCE(SUM(COALESCE(t.lookup_count, s.lookup_count, 0)), 0) AS totalLookupCount, + COALESCE(SUM(COALESCE(t.lookup_hits, s.lookup_hits, 0)), 0) AS totalLookupHits + FROM imm_sessions s + LEFT JOIN ( + SELECT session_id, MAX(lookup_count) AS lookup_count, MAX(lookup_hits) AS lookup_hits + FROM imm_session_telemetry + GROUP BY session_id + ) t ON t.session_id = s.session_id + WHERE s.ended_at_ms IS NOT NULL + `, + ) + .get() as { totalLookupCount: number; totalLookupHits: number } | null; + return { totalSessions, activeSessions, @@ -294,6 +566,32 @@ export function getQueryHints(db: DatabaseSync): { totalActiveMin, totalCards, activeDays, + totalLookupCount: Number(lookupTotals?.totalLookupCount ?? 0), + totalLookupHits: Number(lookupTotals?.totalLookupHits ?? 0), + ...getNewWordCounts(db), + }; +} + +function getNewWordCounts(db: DatabaseSync): { newWordsToday: number; newWordsThisWeek: number } { + const now = new Date(); + const todayStartSec = new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime() / 1000; + const weekAgoSec = todayStartSec - 7 * 86_400; + + const row = db + .prepare( + ` + SELECT + COALESCE(SUM(CASE WHEN first_seen >= ? THEN 1 ELSE 0 END), 0) AS today, + COALESCE(SUM(CASE WHEN first_seen >= ? THEN 1 ELSE 0 END), 0) AS week + FROM imm_words + WHERE first_seen IS NOT NULL + `, + ) + .get(todayStartSec, weekAgoSec) as { today: number; week: number } | null; + + return { + newWordsToday: Number(row?.today ?? 0), + newWordsThisWeek: Number(row?.week ?? 0), }; } @@ -352,6 +650,562 @@ export function getMonthlyRollups(db: DatabaseSync, limit = 24): ImmersionSessio return prepared.all(limit) as unknown as ImmersionSessionRollupRow[]; } +type TrendRange = '7d' | '30d' | '90d' | 'all'; +type TrendGroupBy = 'day' | 'month'; + +interface TrendChartPoint { + label: string; + value: number; +} + +interface TrendPerAnimePoint { + epochDay: number; + animeTitle: string; + value: number; +} + +interface TrendSessionMetricRow { + startedAtMs: number; + videoId: number | null; + canonicalTitle: string | null; + animeTitle: string | null; + activeWatchedMs: number; + wordsSeen: number; + tokensSeen: number; + cardsMined: number; + yomitanLookupCount: number; +} + +export interface TrendsDashboardQueryResult { + activity: { + watchTime: TrendChartPoint[]; + cards: TrendChartPoint[]; + words: TrendChartPoint[]; + sessions: TrendChartPoint[]; + }; + progress: { + watchTime: TrendChartPoint[]; + sessions: TrendChartPoint[]; + words: TrendChartPoint[]; + newWords: TrendChartPoint[]; + cards: TrendChartPoint[]; + episodes: TrendChartPoint[]; + lookups: TrendChartPoint[]; + }; + ratios: { + lookupsPerHundred: TrendChartPoint[]; + }; + animePerDay: { + episodes: TrendPerAnimePoint[]; + watchTime: TrendPerAnimePoint[]; + cards: TrendPerAnimePoint[]; + words: TrendPerAnimePoint[]; + lookups: TrendPerAnimePoint[]; + lookupsPerHundred: TrendPerAnimePoint[]; + }; + animeCumulative: { + watchTime: TrendPerAnimePoint[]; + episodes: TrendPerAnimePoint[]; + cards: TrendPerAnimePoint[]; + words: TrendPerAnimePoint[]; + }; + patterns: { + watchTimeByDayOfWeek: TrendChartPoint[]; + watchTimeByHour: TrendChartPoint[]; + }; +} + +const TREND_DAY_LIMITS: Record, number> = { + '7d': 7, + '30d': 30, + '90d': 90, +}; + +const DAY_NAMES = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat']; + +function getTrendDayLimit(range: TrendRange): number { + return range === 'all' ? 365 : TREND_DAY_LIMITS[range]; +} + +function getTrendMonthlyLimit(range: TrendRange): number { + if (range === 'all') { + return 120; + } + return Math.max(1, Math.ceil(TREND_DAY_LIMITS[range] / 30)); +} + +function getTrendCutoffMs(range: TrendRange): number | null { + if (range === 'all') { + return null; + } + const dayLimit = getTrendDayLimit(range); + const now = new Date(); + const localMidnight = new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime(); + return localMidnight - (dayLimit - 1) * 86_400_000; +} + +function makeTrendLabel(value: number): string { + if (value > 100_000) { + const year = Math.floor(value / 100); + const month = value % 100; + return new Date(Date.UTC(year, month - 1, 1)).toLocaleDateString(undefined, { + month: 'short', + year: '2-digit', + }); + } + + return new Date(value * 86_400_000).toLocaleDateString(undefined, { + month: 'short', + day: 'numeric', + }); +} + +function getTrendSessionWordCount( + session: Pick, +): number { + return session.tokensSeen > 0 ? session.tokensSeen : session.wordsSeen; +} + +function resolveTrendAnimeTitle(value: { + animeTitle: string | null; + canonicalTitle: string | null; +}): string { + return value.animeTitle ?? value.canonicalTitle ?? 'Unknown'; +} + +function accumulatePoints(points: TrendChartPoint[]): TrendChartPoint[] { + let sum = 0; + return points.map((point) => { + sum += point.value; + return { + label: point.label, + value: sum, + }; + }); +} + +function buildAggregatedTrendRows(rollups: ImmersionSessionRollupRow[]) { + const byKey = new Map(); + + for (const rollup of rollups) { + const existing = byKey.get(rollup.rollupDayOrMonth) ?? { + activeMin: 0, + cards: 0, + words: 0, + sessions: 0, + }; + existing.activeMin += Math.round(rollup.totalActiveMin); + existing.cards += rollup.totalCards; + existing.words += rollup.totalWordsSeen; + existing.sessions += rollup.totalSessions; + byKey.set(rollup.rollupDayOrMonth, existing); + } + + return Array.from(byKey.entries()) + .sort(([left], [right]) => left - right) + .map(([key, value]) => ({ + label: makeTrendLabel(key), + activeMin: value.activeMin, + cards: value.cards, + words: value.words, + sessions: value.sessions, + })); +} + +function buildWatchTimeByDayOfWeek(sessions: TrendSessionMetricRow[]): TrendChartPoint[] { + const totals = new Array(7).fill(0); + for (const session of sessions) { + totals[new Date(session.startedAtMs).getDay()] += session.activeWatchedMs; + } + return DAY_NAMES.map((name, index) => ({ + label: name, + value: Math.round(totals[index] / 60_000), + })); +} + +function buildWatchTimeByHour(sessions: TrendSessionMetricRow[]): TrendChartPoint[] { + const totals = new Array(24).fill(0); + for (const session of sessions) { + totals[new Date(session.startedAtMs).getHours()] += session.activeWatchedMs; + } + return totals.map((ms, index) => ({ + label: `${String(index).padStart(2, '0')}:00`, + value: Math.round(ms / 60_000), + })); +} + +function dayLabel(epochDay: number): string { + return new Date(epochDay * 86_400_000).toLocaleDateString(undefined, { + month: 'short', + day: 'numeric', + }); +} + +function buildSessionSeriesByDay( + sessions: TrendSessionMetricRow[], + getValue: (session: TrendSessionMetricRow) => number, +): TrendChartPoint[] { + const byDay = new Map(); + for (const session of sessions) { + const epochDay = Math.floor(session.startedAtMs / 86_400_000); + byDay.set(epochDay, (byDay.get(epochDay) ?? 0) + getValue(session)); + } + return Array.from(byDay.entries()) + .sort(([left], [right]) => left - right) + .map(([epochDay, value]) => ({ label: dayLabel(epochDay), value })); +} + +function buildLookupsPerHundredWords(sessions: TrendSessionMetricRow[]): TrendChartPoint[] { + const lookupsByDay = new Map(); + const wordsByDay = new Map(); + + for (const session of sessions) { + const epochDay = Math.floor(session.startedAtMs / 86_400_000); + lookupsByDay.set( + epochDay, + (lookupsByDay.get(epochDay) ?? 0) + session.yomitanLookupCount, + ); + wordsByDay.set( + epochDay, + (wordsByDay.get(epochDay) ?? 0) + getTrendSessionWordCount(session), + ); + } + + return Array.from(lookupsByDay.entries()) + .sort(([left], [right]) => left - right) + .map(([epochDay, lookups]) => { + const words = wordsByDay.get(epochDay) ?? 0; + return { + label: dayLabel(epochDay), + value: words > 0 ? +((lookups / words) * 100).toFixed(1) : 0, + }; + }); +} + +function buildPerAnimeFromSessions( + sessions: TrendSessionMetricRow[], + getValue: (session: TrendSessionMetricRow) => number, +): TrendPerAnimePoint[] { + const byAnime = new Map>(); + + for (const session of sessions) { + const animeTitle = resolveTrendAnimeTitle(session); + const epochDay = Math.floor(session.startedAtMs / 86_400_000); + const dayMap = byAnime.get(animeTitle) ?? new Map(); + dayMap.set(epochDay, (dayMap.get(epochDay) ?? 0) + getValue(session)); + byAnime.set(animeTitle, dayMap); + } + + const result: TrendPerAnimePoint[] = []; + for (const [animeTitle, dayMap] of byAnime) { + for (const [epochDay, value] of dayMap) { + result.push({ epochDay, animeTitle, value }); + } + } + return result; +} + +function buildLookupsPerHundredPerAnime(sessions: TrendSessionMetricRow[]): TrendPerAnimePoint[] { + const lookups = new Map>(); + const words = new Map>(); + + for (const session of sessions) { + const animeTitle = resolveTrendAnimeTitle(session); + const epochDay = Math.floor(session.startedAtMs / 86_400_000); + + const lookupMap = lookups.get(animeTitle) ?? new Map(); + lookupMap.set(epochDay, (lookupMap.get(epochDay) ?? 0) + session.yomitanLookupCount); + lookups.set(animeTitle, lookupMap); + + const wordMap = words.get(animeTitle) ?? new Map(); + wordMap.set(epochDay, (wordMap.get(epochDay) ?? 0) + getTrendSessionWordCount(session)); + words.set(animeTitle, wordMap); + } + + const result: TrendPerAnimePoint[] = []; + for (const [animeTitle, dayMap] of lookups) { + const wordMap = words.get(animeTitle) ?? new Map(); + for (const [epochDay, lookupCount] of dayMap) { + const wordCount = wordMap.get(epochDay) ?? 0; + result.push({ + epochDay, + animeTitle, + value: wordCount > 0 ? +((lookupCount / wordCount) * 100).toFixed(1) : 0, + }); + } + } + return result; +} + +function buildCumulativePerAnime(points: TrendPerAnimePoint[]): TrendPerAnimePoint[] { + const byAnime = new Map>(); + const allDays = new Set(); + + for (const point of points) { + const dayMap = byAnime.get(point.animeTitle) ?? new Map(); + dayMap.set(point.epochDay, (dayMap.get(point.epochDay) ?? 0) + point.value); + byAnime.set(point.animeTitle, dayMap); + allDays.add(point.epochDay); + } + + const sortedDays = [...allDays].sort((left, right) => left - right); + if (sortedDays.length === 0) { + return []; + } + + const minDay = sortedDays[0]!; + const maxDay = sortedDays[sortedDays.length - 1]!; + const result: TrendPerAnimePoint[] = []; + + for (const [animeTitle, dayMap] of byAnime) { + const firstDay = Math.min(...dayMap.keys()); + let cumulative = 0; + for (let epochDay = minDay; epochDay <= maxDay; epochDay += 1) { + if (epochDay < firstDay) { + continue; + } + cumulative += dayMap.get(epochDay) ?? 0; + result.push({ epochDay, animeTitle, value: cumulative }); + } + } + + return result; +} + +function getVideoAnimeTitleMap(db: DatabaseSync, videoIds: Array): Map { + const uniqueIds = [...new Set(videoIds.filter((value): value is number => typeof value === 'number'))]; + if (uniqueIds.length === 0) { + return new Map(); + } + + const rows = db + .prepare( + ` + SELECT + v.video_id AS videoId, + COALESCE(a.canonical_title, v.canonical_title, 'Unknown') AS animeTitle + FROM imm_videos v + LEFT JOIN imm_anime a ON a.anime_id = v.anime_id + WHERE v.video_id IN (${makePlaceholders(uniqueIds)}) + `, + ) + .all(...uniqueIds) as Array<{ videoId: number; animeTitle: string }>; + + return new Map(rows.map((row) => [row.videoId, row.animeTitle])); +} + +function resolveVideoAnimeTitle(videoId: number | null, titlesByVideoId: Map): string { + if (videoId === null) { + return 'Unknown'; + } + return titlesByVideoId.get(videoId) ?? 'Unknown'; +} + +function buildPerAnimeFromDailyRollups( + rollups: ImmersionSessionRollupRow[], + titlesByVideoId: Map, + getValue: (rollup: ImmersionSessionRollupRow) => number, +): TrendPerAnimePoint[] { + const byAnime = new Map>(); + + for (const rollup of rollups) { + const animeTitle = resolveVideoAnimeTitle(rollup.videoId, titlesByVideoId); + const dayMap = byAnime.get(animeTitle) ?? new Map(); + dayMap.set( + rollup.rollupDayOrMonth, + (dayMap.get(rollup.rollupDayOrMonth) ?? 0) + getValue(rollup), + ); + byAnime.set(animeTitle, dayMap); + } + + const result: TrendPerAnimePoint[] = []; + for (const [animeTitle, dayMap] of byAnime) { + for (const [epochDay, value] of dayMap) { + result.push({ epochDay, animeTitle, value }); + } + } + return result; +} + +function buildEpisodesPerAnimeFromDailyRollups( + rollups: ImmersionSessionRollupRow[], + titlesByVideoId: Map, +): TrendPerAnimePoint[] { + const byAnime = new Map>>(); + + for (const rollup of rollups) { + if (rollup.videoId === null) { + continue; + } + const animeTitle = resolveVideoAnimeTitle(rollup.videoId, titlesByVideoId); + const dayMap = byAnime.get(animeTitle) ?? new Map(); + const videoIds = dayMap.get(rollup.rollupDayOrMonth) ?? new Set(); + videoIds.add(rollup.videoId); + dayMap.set(rollup.rollupDayOrMonth, videoIds); + byAnime.set(animeTitle, dayMap); + } + + const result: TrendPerAnimePoint[] = []; + for (const [animeTitle, dayMap] of byAnime) { + for (const [epochDay, videoIds] of dayMap) { + result.push({ epochDay, animeTitle, value: videoIds.size }); + } + } + return result; +} + +function buildEpisodesPerDayFromDailyRollups(rollups: ImmersionSessionRollupRow[]): TrendChartPoint[] { + const byDay = new Map>(); + + for (const rollup of rollups) { + if (rollup.videoId === null) { + continue; + } + const videoIds = byDay.get(rollup.rollupDayOrMonth) ?? new Set(); + videoIds.add(rollup.videoId); + byDay.set(rollup.rollupDayOrMonth, videoIds); + } + + return Array.from(byDay.entries()) + .sort(([left], [right]) => left - right) + .map(([epochDay, videoIds]) => ({ + label: dayLabel(epochDay), + value: videoIds.size, + })); +} + +function getTrendSessionMetrics( + db: DatabaseSync, + cutoffMs: number | null, +): TrendSessionMetricRow[] { + const whereClause = cutoffMs === null ? '' : 'WHERE s.started_at_ms >= ?'; + const prepared = db.prepare(` + ${ACTIVE_SESSION_METRICS_CTE} + SELECT + s.started_at_ms AS startedAtMs, + s.video_id AS videoId, + v.canonical_title AS canonicalTitle, + a.canonical_title AS animeTitle, + COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0) AS activeWatchedMs, + COALESCE(asm.wordsSeen, s.words_seen, 0) AS wordsSeen, + COALESCE(asm.tokensSeen, s.tokens_seen, 0) AS tokensSeen, + COALESCE(asm.cardsMined, s.cards_mined, 0) AS cardsMined, + COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0) AS yomitanLookupCount + FROM imm_sessions s + LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id + LEFT JOIN imm_videos v ON v.video_id = s.video_id + LEFT JOIN imm_anime a ON a.anime_id = v.anime_id + ${whereClause} + ORDER BY s.started_at_ms ASC + `); + + return (cutoffMs === null ? prepared.all() : prepared.all(cutoffMs)) as TrendSessionMetricRow[]; +} + +function buildNewWordsPerDay(db: DatabaseSync, cutoffMs: number | null): TrendChartPoint[] { + const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?'; + const prepared = db.prepare(` + SELECT + CAST(first_seen / 86400 AS INTEGER) AS epochDay, + COUNT(*) AS wordCount + FROM imm_words + WHERE first_seen IS NOT NULL + ${whereClause} + GROUP BY epochDay + ORDER BY epochDay ASC + `); + + const rows = (cutoffMs === null ? prepared.all() : prepared.all(Math.floor(cutoffMs / 1000))) as Array<{ + epochDay: number; + wordCount: number; + }>; + + return rows.map((row) => ({ + label: dayLabel(row.epochDay), + value: row.wordCount, + })); +} + +export function getTrendsDashboard( + db: DatabaseSync, + range: TrendRange = '30d', + groupBy: TrendGroupBy = 'day', +): TrendsDashboardQueryResult { + const dayLimit = getTrendDayLimit(range); + const monthlyLimit = getTrendMonthlyLimit(range); + const cutoffMs = getTrendCutoffMs(range); + + const chartRollups = + groupBy === 'month' ? getMonthlyRollups(db, monthlyLimit) : getDailyRollups(db, dayLimit); + const dailyRollups = getDailyRollups(db, dayLimit); + const sessions = getTrendSessionMetrics(db, cutoffMs); + const titlesByVideoId = getVideoAnimeTitleMap( + db, + dailyRollups.map((rollup) => rollup.videoId), + ); + + const aggregatedRows = buildAggregatedTrendRows(chartRollups); + const activity = { + watchTime: aggregatedRows.map((row) => ({ label: row.label, value: row.activeMin })), + cards: aggregatedRows.map((row) => ({ label: row.label, value: row.cards })), + words: aggregatedRows.map((row) => ({ label: row.label, value: row.words })), + sessions: aggregatedRows.map((row) => ({ label: row.label, value: row.sessions })), + }; + + const animePerDay = { + episodes: buildEpisodesPerAnimeFromDailyRollups(dailyRollups, titlesByVideoId), + watchTime: buildPerAnimeFromDailyRollups( + dailyRollups, + titlesByVideoId, + (rollup) => Math.round(rollup.totalActiveMin), + ), + cards: buildPerAnimeFromDailyRollups( + dailyRollups, + titlesByVideoId, + (rollup) => rollup.totalCards, + ), + words: buildPerAnimeFromDailyRollups( + dailyRollups, + titlesByVideoId, + (rollup) => rollup.totalWordsSeen, + ), + lookups: buildPerAnimeFromSessions( + sessions, + (session) => session.yomitanLookupCount, + ), + lookupsPerHundred: buildLookupsPerHundredPerAnime(sessions), + }; + + return { + activity, + progress: { + watchTime: accumulatePoints(activity.watchTime), + sessions: accumulatePoints(activity.sessions), + words: accumulatePoints(activity.words), + newWords: accumulatePoints(buildNewWordsPerDay(db, cutoffMs)), + cards: accumulatePoints(activity.cards), + episodes: accumulatePoints(buildEpisodesPerDayFromDailyRollups(dailyRollups)), + lookups: accumulatePoints( + buildSessionSeriesByDay(sessions, (session) => session.yomitanLookupCount), + ), + }, + ratios: { + lookupsPerHundred: buildLookupsPerHundredWords(sessions), + }, + animePerDay, + animeCumulative: { + watchTime: buildCumulativePerAnime(animePerDay.watchTime), + episodes: buildCumulativePerAnime(animePerDay.episodes), + cards: buildCumulativePerAnime(animePerDay.cards), + words: buildCumulativePerAnime(animePerDay.words), + }, + patterns: { + watchTimeByDayOfWeek: buildWatchTimeByDayOfWeek(sessions), + watchTimeByHour: buildWatchTimeByHour(sessions), + }, + }; +} + export function getVocabularyStats( db: DatabaseSync, limit = 100, @@ -794,6 +1648,7 @@ export function getAnimeDetail(db: DatabaseSync, animeId: number): AnimeDetailRo COALESCE(lm.total_lines_seen, 0) AS totalLinesSeen, COALESCE(SUM(COALESCE(asm.lookupCount, s.lookup_count, 0)), 0) AS totalLookupCount, COALESCE(SUM(COALESCE(asm.lookupHits, s.lookup_hits, 0)), 0) AS totalLookupHits, + COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount, COUNT(DISTINCT v.video_id) AS episodeCount, COALESCE(lm.last_watched_ms, 0) AS lastWatchedMs FROM imm_anime a @@ -845,6 +1700,7 @@ export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisod COALESCE(SUM(COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0)), 0) AS totalActiveMs, COALESCE(SUM(COALESCE(asm.cardsMined, s.cards_mined, 0)), 0) AS totalCards, COALESCE(SUM(COALESCE(asm.wordsSeen, s.words_seen, 0)), 0) AS totalWordsSeen, + COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount, MAX(s.started_at_ms) AS lastWatchedMs FROM imm_videos v JOIN imm_sessions s ON s.video_id = v.video_id @@ -901,7 +1757,8 @@ export function getMediaDetail(db: DatabaseSync, videoId: number): MediaDetailRo COALESCE(lm.total_words_seen, 0) AS totalWordsSeen, COALESCE(lm.total_lines_seen, 0) AS totalLinesSeen, COALESCE(SUM(COALESCE(asm.lookupCount, s.lookup_count, 0)), 0) AS totalLookupCount, - COALESCE(SUM(COALESCE(asm.lookupHits, s.lookup_hits, 0)), 0) AS totalLookupHits + COALESCE(SUM(COALESCE(asm.lookupHits, s.lookup_hits, 0)), 0) AS totalLookupHits, + COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount FROM imm_videos v JOIN imm_lifetime_media lm ON lm.video_id = v.video_id LEFT JOIN imm_sessions s ON s.video_id = v.video_id @@ -935,7 +1792,8 @@ export function getMediaSessions( COALESCE(asm.tokensSeen, s.tokens_seen, 0) AS tokensSeen, COALESCE(asm.cardsMined, s.cards_mined, 0) AS cardsMined, COALESCE(asm.lookupCount, s.lookup_count, 0) AS lookupCount, - COALESCE(asm.lookupHits, s.lookup_hits, 0) AS lookupHits + COALESCE(asm.lookupHits, s.lookup_hits, 0) AS lookupHits, + COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0) AS yomitanLookupCount FROM imm_sessions s LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id LEFT JOIN imm_videos v ON v.video_id = s.video_id @@ -1299,7 +2157,8 @@ export function getEpisodeSessions(db: DatabaseSync, videoId: number): SessionSu COALESCE(asm.tokensSeen, s.tokens_seen, 0) AS tokensSeen, COALESCE(asm.cardsMined, s.cards_mined, 0) AS cardsMined, COALESCE(asm.lookupCount, s.lookup_count, 0) AS lookupCount, - COALESCE(asm.lookupHits, s.lookup_hits, 0) AS lookupHits + COALESCE(asm.lookupHits, s.lookup_hits, 0) AS lookupHits, + COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0) AS yomitanLookupCount FROM imm_sessions s JOIN imm_videos v ON v.video_id = s.video_id LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id @@ -1488,10 +2347,35 @@ export function isVideoWatched(db: DatabaseSync, videoId: number): boolean { } export function deleteSession(db: DatabaseSync, sessionId: number): void { - db.prepare('DELETE FROM imm_subtitle_lines WHERE session_id = ?').run(sessionId); - db.prepare('DELETE FROM imm_session_telemetry WHERE session_id = ?').run(sessionId); - db.prepare('DELETE FROM imm_session_events WHERE session_id = ?').run(sessionId); - db.prepare('DELETE FROM imm_sessions WHERE session_id = ?').run(sessionId); + const sessionIds = [sessionId]; + const affectedWordIds = getAffectedWordIdsForSessions(db, sessionIds); + const affectedKanjiIds = getAffectedKanjiIdsForSessions(db, sessionIds); + + db.exec('BEGIN IMMEDIATE'); + try { + deleteSessionsByIds(db, sessionIds); + refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds); + db.exec('COMMIT'); + } catch (error) { + db.exec('ROLLBACK'); + throw error; + } +} + +export function deleteSessions(db: DatabaseSync, sessionIds: number[]): void { + if (sessionIds.length === 0) return; + const affectedWordIds = getAffectedWordIdsForSessions(db, sessionIds); + const affectedKanjiIds = getAffectedKanjiIdsForSessions(db, sessionIds); + + db.exec('BEGIN IMMEDIATE'); + try { + deleteSessionsByIds(db, sessionIds); + refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds); + db.exec('COMMIT'); + } catch (error) { + db.exec('ROLLBACK'); + throw error; + } } export function deleteVideo(db: DatabaseSync, videoId: number): void { @@ -1504,16 +2388,28 @@ export function deleteVideo(db: DatabaseSync, videoId: number): void { `, ) .get(videoId) as { coverBlobHash: string | null } | undefined; + const affectedWordIds = getAffectedWordIdsForVideo(db, videoId); + const affectedKanjiIds = getAffectedKanjiIdsForVideo(db, videoId); const sessions = db .prepare('SELECT session_id FROM imm_sessions WHERE video_id = ?') .all(videoId) as Array<{ session_id: number }>; - for (const s of sessions) { - deleteSession(db, s.session_id); + + db.exec('BEGIN IMMEDIATE'); + try { + deleteSessionsByIds( + db, + sessions.map((session) => session.session_id), + ); + db.prepare('DELETE FROM imm_subtitle_lines WHERE video_id = ?').run(videoId); + db.prepare('DELETE FROM imm_daily_rollups WHERE video_id = ?').run(videoId); + db.prepare('DELETE FROM imm_monthly_rollups WHERE video_id = ?').run(videoId); + db.prepare('DELETE FROM imm_media_art WHERE video_id = ?').run(videoId); + cleanupUnusedCoverArtBlobHash(db, artRow?.coverBlobHash ?? null); + db.prepare('DELETE FROM imm_videos WHERE video_id = ?').run(videoId); + refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds); + db.exec('COMMIT'); + } catch (error) { + db.exec('ROLLBACK'); + throw error; } - db.prepare('DELETE FROM imm_subtitle_lines WHERE video_id = ?').run(videoId); - db.prepare('DELETE FROM imm_daily_rollups WHERE video_id = ?').run(videoId); - db.prepare('DELETE FROM imm_monthly_rollups WHERE video_id = ?').run(videoId); - db.prepare('DELETE FROM imm_media_art WHERE video_id = ?').run(videoId); - cleanupUnusedCoverArtBlobHash(db, artRow?.coverBlobHash ?? null); - db.prepare('DELETE FROM imm_videos WHERE video_id = ?').run(videoId); } diff --git a/src/core/services/immersion-tracker/reducer.ts b/src/core/services/immersion-tracker/reducer.ts index 5f4fa58..7db2b57 100644 --- a/src/core/services/immersion-tracker/reducer.ts +++ b/src/core/services/immersion-tracker/reducer.ts @@ -20,6 +20,7 @@ export function createInitialSessionState( cardsMined: 0, lookupCount: 0, lookupHits: 0, + yomitanLookupCount: 0, pauseCount: 0, pauseMs: 0, seekForwardCount: 0, diff --git a/src/core/services/immersion-tracker/session.ts b/src/core/services/immersion-tracker/session.ts index 90ac8d1..50f6dbd 100644 --- a/src/core/services/immersion-tracker/session.ts +++ b/src/core/services/immersion-tracker/session.ts @@ -47,6 +47,7 @@ export function finalizeSessionRecord( cards_mined = ?, lookup_count = ?, lookup_hits = ?, + yomitan_lookup_count = ?, pause_count = ?, pause_ms = ?, seek_forward_count = ?, @@ -66,6 +67,7 @@ export function finalizeSessionRecord( sessionState.cardsMined, sessionState.lookupCount, sessionState.lookupHits, + sessionState.yomitanLookupCount, sessionState.pauseCount, sessionState.pauseMs, sessionState.seekForwardCount, diff --git a/src/core/services/immersion-tracker/storage-session.test.ts b/src/core/services/immersion-tracker/storage-session.test.ts index 74c286f..3b69407 100644 --- a/src/core/services/immersion-tracker/storage-session.test.ts +++ b/src/core/services/immersion-tracker/storage-session.test.ts @@ -6,6 +6,7 @@ import test from 'node:test'; import { Database } from './sqlite'; import { finalizeSessionRecord, startSessionRecord } from './session'; import { + applyPragmas, createTrackerPreparedStatements, ensureSchema, executeQueuedWrite, @@ -50,6 +51,34 @@ function cleanupDbPath(dbPath: string): void { // libsql keeps Windows file handles alive after close when prepared statements were used. } +test('applyPragmas sets the SQLite tuning defaults used by immersion tracking', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + applyPragmas(db); + + const journalModeRow = db.prepare('PRAGMA journal_mode').get() as { + journal_mode: string; + }; + const synchronousRow = db.prepare('PRAGMA synchronous').get() as { synchronous: number }; + const foreignKeysRow = db.prepare('PRAGMA foreign_keys').get() as { foreign_keys: number }; + const busyTimeoutRow = db.prepare('PRAGMA busy_timeout').get() as { timeout: number }; + const journalSizeLimitRow = db.prepare('PRAGMA journal_size_limit').get() as { + journal_size_limit: number; + }; + + assert.equal(journalModeRow.journal_mode, 'wal'); + assert.equal(synchronousRow.synchronous, 1); + assert.equal(foreignKeysRow.foreign_keys, 1); + assert.equal(busyTimeoutRow.timeout, 2500); + assert.equal(journalSizeLimitRow.journal_size_limit, 67_108_864); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('ensureSchema creates immersion core tables', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -125,7 +154,9 @@ test('ensureSchema creates large-history performance indexes', () => { ensureSchema(db); const indexNames = new Set( ( - db.prepare(`SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_%'`).all() as Array<{ + db + .prepare(`SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_%'`) + .all() as Array<{ name: string; }> ).map((row) => row.name), @@ -516,7 +547,9 @@ test('ensureSchema migrates legacy cover art blobs into the shared blob store', assert.doesNotThrow(() => ensureSchema(db)); const mediaArtRow = db - .prepare('SELECT cover_blob AS coverBlob, cover_blob_hash AS coverBlobHash FROM imm_media_art') + .prepare( + 'SELECT cover_blob AS coverBlob, cover_blob_hash AS coverBlobHash FROM imm_media_art', + ) .get() as { coverBlob: ArrayBuffer | Uint8Array | Buffer | null; coverBlobHash: string | null; @@ -524,7 +557,10 @@ test('ensureSchema migrates legacy cover art blobs into the shared blob store', assert.ok(mediaArtRow); assert.ok(mediaArtRow?.coverBlobHash); - assert.equal(parseCoverBlobReference(normalizeCoverBlobBytes(mediaArtRow?.coverBlob)), mediaArtRow?.coverBlobHash); + assert.equal( + parseCoverBlobReference(normalizeCoverBlobBytes(mediaArtRow?.coverBlob)), + mediaArtRow?.coverBlobHash, + ); const sharedBlobRow = db .prepare('SELECT cover_blob AS coverBlob FROM imm_cover_art_blobs WHERE blob_hash = ?') @@ -732,6 +768,7 @@ test('executeQueuedWrite inserts event and telemetry rows', () => { cardsMined: 1, lookupCount: 2, lookupHits: 1, + yomitanLookupCount: 0, pauseCount: 1, pauseMs: 50, seekForwardCount: 0, diff --git a/src/core/services/immersion-tracker/storage.ts b/src/core/services/immersion-tracker/storage.ts index fe2826d..701e724 100644 --- a/src/core/services/immersion-tracker/storage.ts +++ b/src/core/services/immersion-tracker/storage.ts @@ -39,6 +39,7 @@ export interface VideoAnimeLinkInput { } const COVER_BLOB_REFERENCE_PREFIX = '__subminer_cover_blob_ref__:'; +const WAL_JOURNAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024; export type CoverBlobBytes = ArrayBuffer | Uint8Array | Buffer; @@ -153,6 +154,7 @@ export function applyPragmas(db: DatabaseSync): void { db.exec('PRAGMA synchronous = NORMAL'); db.exec('PRAGMA foreign_keys = ON'); db.exec('PRAGMA busy_timeout = 2500'); + db.exec(`PRAGMA journal_size_limit = ${WAL_JOURNAL_SIZE_LIMIT_BYTES}`); } export function normalizeAnimeIdentityKey(title: string): string { @@ -577,6 +579,7 @@ export function ensureSchema(db: DatabaseSync): void { cards_mined INTEGER NOT NULL DEFAULT 0, lookup_count INTEGER NOT NULL DEFAULT 0, lookup_hits INTEGER NOT NULL DEFAULT 0, + yomitan_lookup_count INTEGER NOT NULL DEFAULT 0, pause_count INTEGER NOT NULL DEFAULT 0, pause_ms INTEGER NOT NULL DEFAULT 0, seek_forward_count INTEGER NOT NULL DEFAULT 0, @@ -600,6 +603,7 @@ export function ensureSchema(db: DatabaseSync): void { cards_mined INTEGER NOT NULL DEFAULT 0, lookup_count INTEGER NOT NULL DEFAULT 0, lookup_hits INTEGER NOT NULL DEFAULT 0, + yomitan_lookup_count INTEGER NOT NULL DEFAULT 0, pause_count INTEGER NOT NULL DEFAULT 0, pause_ms INTEGER NOT NULL DEFAULT 0, seek_forward_count INTEGER NOT NULL DEFAULT 0, @@ -1013,6 +1017,29 @@ export function ensureSchema(db: DatabaseSync): void { deduplicateExistingCoverArtRows(db); } + if (currentVersion?.schema_version && currentVersion.schema_version < 14) { + addColumnIfMissing(db, 'imm_sessions', 'yomitan_lookup_count', 'INTEGER NOT NULL DEFAULT 0'); + addColumnIfMissing( + db, + 'imm_session_telemetry', + 'yomitan_lookup_count', + 'INTEGER NOT NULL DEFAULT 0', + ); + + db.exec(` + UPDATE imm_sessions + SET + yomitan_lookup_count = COALESCE(( + SELECT t.yomitan_lookup_count + FROM imm_session_telemetry t + WHERE t.session_id = imm_sessions.session_id + ORDER BY t.sample_ms DESC, t.telemetry_id DESC + LIMIT 1 + ), yomitan_lookup_count) + WHERE ended_at_ms IS NOT NULL + `); + } + ensureLifetimeSummaryTables(db); db.exec(` @@ -1137,10 +1164,10 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar INSERT INTO imm_session_telemetry ( session_id, sample_ms, total_watched_ms, active_watched_ms, lines_seen, words_seen, tokens_seen, cards_mined, lookup_count, - lookup_hits, pause_count, pause_ms, seek_forward_count, + lookup_hits, yomitan_lookup_count, pause_count, pause_ms, seek_forward_count, seek_backward_count, media_buffer_events, CREATED_DATE, LAST_UPDATE_DATE ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) `), eventInsertStmt: db.prepare(` @@ -1288,6 +1315,7 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta write.cardsMined!, write.lookupCount!, write.lookupHits!, + write.yomitanLookupCount ?? 0, write.pauseCount!, write.pauseMs!, write.seekForwardCount!, diff --git a/src/core/services/immersion-tracker/types.ts b/src/core/services/immersion-tracker/types.ts index f137d2a..f44c7c9 100644 --- a/src/core/services/immersion-tracker/types.ts +++ b/src/core/services/immersion-tracker/types.ts @@ -1,4 +1,4 @@ -export const SCHEMA_VERSION = 13; +export const SCHEMA_VERSION = 14; export const DEFAULT_QUEUE_CAP = 1_000; export const DEFAULT_BATCH_SIZE = 25; export const DEFAULT_FLUSH_INTERVAL_MS = 500; @@ -26,6 +26,7 @@ export const EVENT_SEEK_FORWARD = 5; export const EVENT_SEEK_BACKWARD = 6; export const EVENT_PAUSE_START = 7; export const EVENT_PAUSE_END = 8; +export const EVENT_YOMITAN_LOOKUP = 9; export interface ImmersionTrackerOptions { dbPath: string; @@ -60,6 +61,7 @@ export interface TelemetryAccumulator { cardsMined: number; lookupCount: number; lookupHits: number; + yomitanLookupCount: number; pauseCount: number; pauseMs: number; seekForwardCount: number; @@ -92,6 +94,7 @@ interface QueuedTelemetryWrite { cardsMined?: number; lookupCount?: number; lookupHits?: number; + yomitanLookupCount?: number; pauseCount?: number; pauseMs?: number; seekForwardCount?: number; @@ -233,6 +236,7 @@ export interface SessionSummaryQueryRow { cardsMined: number; lookupCount: number; lookupHits: number; + yomitanLookupCount: number; } export interface LifetimeGlobalRow { @@ -432,6 +436,7 @@ export interface MediaDetailRow { totalLinesSeen: number; totalLookupCount: number; totalLookupHits: number; + totalYomitanLookupCount: number; } export interface AnimeLibraryRow { @@ -462,6 +467,7 @@ export interface AnimeDetailRow { totalLinesSeen: number; totalLookupCount: number; totalLookupHits: number; + totalYomitanLookupCount: number; episodeCount: number; lastWatchedMs: number; } @@ -486,6 +492,7 @@ export interface AnimeEpisodeRow { totalActiveMs: number; totalCards: number; totalWordsSeen: number; + totalYomitanLookupCount: number; lastWatchedMs: number; }