feat: refactor immersion tracker queries and session word tracking

Add comprehensive query helpers for session deletion with word aggregate
refresh, known-words-per-session timeline, anime-level word summaries,
and trends dashboard aggregation. Track yomitanLookupCount in session
metrics and support bulk session operations.
This commit is contained in:
2026-03-17 19:52:59 -07:00
parent a5a6426fe1
commit 55ee12e87f
13 changed files with 1735 additions and 39 deletions

View File

@@ -242,11 +242,7 @@ export function applyImmersionTrackingConfig(context: ResolveContext): void {
}
const dailyRollupsDays = asNumber(src.immersionTracking.retention.dailyRollupsDays);
if (
dailyRollupsDays !== undefined &&
dailyRollupsDays >= 0 &&
dailyRollupsDays <= 36500
) {
if (dailyRollupsDays !== undefined && dailyRollupsDays >= 0 && dailyRollupsDays <= 36500) {
retention.dailyRollupsDays = Math.floor(dailyRollupsDays);
} else if (src.immersionTracking.retention.dailyRollupsDays !== undefined) {
warn(
@@ -274,7 +270,11 @@ export function applyImmersionTrackingConfig(context: ResolveContext): void {
}
const vacuumIntervalDays = asNumber(src.immersionTracking.retention.vacuumIntervalDays);
if (vacuumIntervalDays !== undefined && vacuumIntervalDays >= 0 && vacuumIntervalDays <= 3650) {
if (
vacuumIntervalDays !== undefined &&
vacuumIntervalDays >= 0 &&
vacuumIntervalDays <= 3650
) {
retention.vacuumIntervalDays = Math.floor(vacuumIntervalDays);
} else if (src.immersionTracking.retention.vacuumIntervalDays !== undefined) {
warn(

View File

@@ -840,6 +840,59 @@ test('persists and retrieves minimum immersion tracking fields', async () => {
}
});
test('recordYomitanLookup persists a dedicated lookup counter without changing annotation lookup metrics', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath });
tracker.handleMediaChange('/tmp/episode-yomitan.mkv', 'Episode Yomitan');
tracker.recordSubtitleLine('alpha beta gamma', 0, 1.2);
tracker.recordLookup(true);
tracker.recordYomitanLookup();
const privateApi = tracker as unknown as {
flushTelemetry: (force?: boolean) => void;
flushNow: () => void;
};
privateApi.flushTelemetry(true);
privateApi.flushNow();
const summaries = await tracker.getSessionSummaries(10);
assert.ok(summaries.length >= 1);
assert.equal(summaries[0]?.lookupCount, 1);
assert.equal(summaries[0]?.lookupHits, 1);
assert.equal(summaries[0]?.yomitanLookupCount, 1);
tracker.destroy();
const db = new Database(dbPath);
const sessionRow = db
.prepare('SELECT lookup_count, lookup_hits, yomitan_lookup_count FROM imm_sessions LIMIT 1')
.get() as {
lookup_count: number;
lookup_hits: number;
yomitan_lookup_count: number;
} | null;
const eventRow = db
.prepare(
'SELECT event_type FROM imm_session_events WHERE event_type = ? ORDER BY ts_ms DESC LIMIT 1',
)
.get(9) as { event_type: number } | null;
db.close();
assert.equal(sessionRow?.lookup_count, 1);
assert.equal(sessionRow?.lookup_hits, 1);
assert.equal(sessionRow?.yomitan_lookup_count, 1);
assert.equal(eventRow?.event_type, 9);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('recordSubtitleLine persists counted allowed tokenized vocabulary rows and subtitle-line occurrences', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
@@ -1053,6 +1106,140 @@ test('subtitle-line event payload omits duplicated subtitle text', async () => {
}
});
test('recordPlaybackPosition marks watched at 85% completion', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath });
tracker.handleMediaChange('/tmp/episode-85.mkv', 'Episode 85');
tracker.recordMediaDuration(100);
await waitForPendingAnimeMetadata(tracker);
const privateApi = tracker as unknown as {
db: DatabaseSync;
sessionState: { videoId: number } | null;
};
const videoId = privateApi.sessionState?.videoId;
assert.ok(videoId);
tracker.recordPlaybackPosition(84);
let row = privateApi.db
.prepare('SELECT watched FROM imm_videos WHERE video_id = ?')
.get(videoId) as { watched: number } | null;
assert.equal(row?.watched, 0);
tracker.recordPlaybackPosition(85);
row = privateApi.db
.prepare('SELECT watched FROM imm_videos WHERE video_id = ?')
.get(videoId) as { watched: number } | null;
assert.equal(row?.watched, 1);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('deleteSession ignores the currently active session and keeps new writes flushable', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath });
tracker.handleMediaChange('/tmp/active-delete-test.mkv', 'Active Delete Test');
const privateApi = tracker as unknown as {
sessionState: { sessionId: number } | null;
flushTelemetry: (force?: boolean) => void;
flushNow: () => void;
queue: unknown[];
};
const sessionId = privateApi.sessionState?.sessionId;
assert.ok(sessionId);
tracker.recordSubtitleLine('before delete', 0, 1);
privateApi.flushTelemetry(true);
privateApi.flushNow();
await tracker.deleteSession(sessionId);
tracker.recordSubtitleLine('after delete', 1, 2);
privateApi.flushTelemetry(true);
privateApi.flushNow();
const db = new Database(dbPath);
const sessionCountRow = db
.prepare('SELECT COUNT(*) AS total FROM imm_sessions WHERE session_id = ?')
.get(sessionId) as { total: number };
const subtitleLineCountRow = db
.prepare('SELECT COUNT(*) AS total FROM imm_subtitle_lines WHERE session_id = ?')
.get(sessionId) as { total: number };
db.close();
assert.equal(sessionCountRow.total, 1);
assert.equal(subtitleLineCountRow.total, 2);
assert.equal(privateApi.queue.length, 0);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('deleteVideo ignores the currently active video and keeps new writes flushable', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath });
tracker.handleMediaChange('/tmp/active-video-delete-test.mkv', 'Active Video Delete Test');
const privateApi = tracker as unknown as {
sessionState: { sessionId: number; videoId: number } | null;
flushTelemetry: (force?: boolean) => void;
flushNow: () => void;
queue: unknown[];
};
const sessionId = privateApi.sessionState?.sessionId;
const videoId = privateApi.sessionState?.videoId;
assert.ok(sessionId);
assert.ok(videoId);
tracker.recordSubtitleLine('before video delete', 0, 1);
privateApi.flushTelemetry(true);
privateApi.flushNow();
await tracker.deleteVideo(videoId);
tracker.recordSubtitleLine('after video delete', 1, 2);
privateApi.flushTelemetry(true);
privateApi.flushNow();
const db = new Database(dbPath);
const sessionCountRow = db
.prepare('SELECT COUNT(*) AS total FROM imm_sessions WHERE session_id = ?')
.get(sessionId) as { total: number };
const videoCountRow = db
.prepare('SELECT COUNT(*) AS total FROM imm_videos WHERE video_id = ?')
.get(videoId) as { total: number };
const subtitleLineCountRow = db
.prepare('SELECT COUNT(*) AS total FROM imm_subtitle_lines WHERE session_id = ?')
.get(sessionId) as { total: number };
db.close();
assert.equal(sessionCountRow.total, 1);
assert.equal(videoCountRow.total, 1);
assert.equal(subtitleLineCountRow.total, 2);
assert.equal(privateApi.queue.length, 0);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('handleMediaChange links parsed anime metadata on the active video row', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
@@ -1821,3 +2008,49 @@ test('reassignAnimeAnilist deduplicates cover blobs and getCoverArt remains comp
cleanupDbPath(dbPath);
}
});
test('markActiveVideoWatched marks current session video as watched', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath });
tracker.handleMediaChange('/tmp/test-mark-active.mkv', 'Test Mark Active');
await waitForPendingAnimeMetadata(tracker);
const privateApi = tracker as unknown as {
db: DatabaseSync;
sessionState: { videoId: number; markedWatched: boolean } | null;
};
const videoId = privateApi.sessionState?.videoId;
assert.ok(videoId);
const result = await tracker.markActiveVideoWatched();
assert.equal(result, true);
assert.equal(privateApi.sessionState?.markedWatched, true);
const row = privateApi.db
.prepare('SELECT watched FROM imm_videos WHERE video_id = ?')
.get(videoId) as { watched: number } | null;
assert.equal(row?.watched, 1);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('markActiveVideoWatched returns false when no active session', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath });
const result = await tracker.markActiveVideoWatched();
assert.equal(result, false);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});

View File

@@ -6,6 +6,7 @@ import { getLocalVideoMetadata, guessAnimeVideoMetadata } from './immersion-trac
import {
pruneRawRetention,
pruneRollupRetention,
runOptimizeMaintenance,
runRollupMaintenance,
} from './immersion-tracker/maintenance';
import { Database, type DatabaseSync } from './immersion-tracker/sqlite';
@@ -60,6 +61,11 @@ import {
getSessionEvents,
getSessionSummaries,
getSessionTimeline,
getSessionWordsByLine,
getTrendsDashboard,
getAllDistinctHeadwords,
getAnimeDistinctHeadwords,
getMediaDistinctHeadwords,
getVocabularyStats,
getWatchTimePerAnime,
getWordAnimeAppearances,
@@ -69,6 +75,7 @@ import {
upsertCoverArt,
markVideoWatched,
deleteSession as deleteSessionQuery,
deleteSessions as deleteSessionsQuery,
deleteVideo as deleteVideoQuery,
} from './immersion-tracker/query';
import {
@@ -83,6 +90,7 @@ import {
sanitizePayload,
secToMs,
} from './immersion-tracker/reducer';
import { DEFAULT_MIN_WATCH_RATIO } from '../../shared/watch-threshold';
import { enqueueWrite } from './immersion-tracker/queue';
import {
DEFAULT_BATCH_SIZE,
@@ -104,6 +112,7 @@ import {
EVENT_SEEK_BACKWARD,
EVENT_SEEK_FORWARD,
EVENT_SUBTITLE_LINE,
EVENT_YOMITAN_LOOKUP,
SOURCE_TYPE_LOCAL,
SOURCE_TYPE_REMOTE,
type ImmersionSessionRollupRow,
@@ -244,13 +253,21 @@ export class ImmersionTrackerService {
);
const retention = policy.retention ?? {};
const daysToRetentionMs = (value: number | undefined, fallbackMs: number, maxDays: number): number => {
const daysToRetentionMs = (
value: number | undefined,
fallbackMs: number,
maxDays: number,
): number => {
const fallbackDays = Math.floor(fallbackMs / 86_400_000);
const resolvedDays = resolveBoundedInt(value, fallbackDays, 0, maxDays);
return resolvedDays === 0 ? Number.POSITIVE_INFINITY : resolvedDays * 86_400_000;
};
this.eventsRetentionMs = daysToRetentionMs(retention.eventsDays, DEFAULT_EVENTS_RETENTION_MS, 3650);
this.eventsRetentionMs = daysToRetentionMs(
retention.eventsDays,
DEFAULT_EVENTS_RETENTION_MS,
3650,
);
this.telemetryRetentionMs = daysToRetentionMs(
retention.telemetryDays,
DEFAULT_TELEMETRY_RETENTION_MS,
@@ -321,6 +338,24 @@ export class ImmersionTrackerService {
return getSessionTimeline(this.db, sessionId, limit);
}
async getSessionWordsByLine(
sessionId: number,
): Promise<Array<{ lineIndex: number; headword: string; occurrenceCount: number }>> {
return getSessionWordsByLine(this.db, sessionId);
}
async getAllDistinctHeadwords(): Promise<string[]> {
return getAllDistinctHeadwords(this.db);
}
async getAnimeDistinctHeadwords(animeId: number): Promise<string[]> {
return getAnimeDistinctHeadwords(this.db, animeId);
}
async getMediaDistinctHeadwords(videoId: number): Promise<string[]> {
return getMediaDistinctHeadwords(this.db, videoId);
}
async getQueryHints(): Promise<{
totalSessions: number;
activeSessions: number;
@@ -343,6 +378,13 @@ export class ImmersionTrackerService {
return getMonthlyRollups(this.db, limit);
}
async getTrendsDashboard(
range: '7d' | '30d' | '90d' | 'all' = '30d',
groupBy: 'day' | 'month' = 'day',
): Promise<unknown> {
return getTrendsDashboard(this.db, range, groupBy);
}
async getVocabularyStats(limit = 100, excludePos?: string[]): Promise<VocabularyStatsRow[]> {
return getVocabularyStats(this.db, limit, excludePos);
}
@@ -437,11 +479,40 @@ export class ImmersionTrackerService {
markVideoWatched(this.db, videoId, watched);
}
async markActiveVideoWatched(): Promise<boolean> {
if (!this.sessionState) return false;
markVideoWatched(this.db, this.sessionState.videoId, true);
this.sessionState.markedWatched = true;
return true;
}
async deleteSession(sessionId: number): Promise<void> {
if (this.sessionState?.sessionId === sessionId) {
this.logger.warn(`Ignoring delete request for active immersion session ${sessionId}`);
return;
}
deleteSessionQuery(this.db, sessionId);
}
async deleteSessions(sessionIds: number[]): Promise<void> {
const activeSessionId = this.sessionState?.sessionId;
const deletableSessionIds =
activeSessionId === undefined
? sessionIds
: sessionIds.filter((sessionId) => sessionId !== activeSessionId);
if (deletableSessionIds.length !== sessionIds.length) {
this.logger.warn(
`Ignoring bulk delete request for active immersion session ${activeSessionId}`,
);
}
deleteSessionsQuery(this.db, deletableSessionIds);
}
async deleteVideo(videoId: number): Promise<void> {
if (this.sessionState?.videoId === videoId) {
this.logger.warn(`Ignoring delete request for active immersion video ${videoId}`);
return;
}
deleteVideoQuery(this.db, videoId);
}
@@ -847,7 +918,7 @@ export class ImmersionTrackerService {
if (!this.sessionState.markedWatched) {
const durationMs = getVideoDurationMs(this.db, this.sessionState.videoId);
if (durationMs > 0 && mediaMs >= durationMs * 0.98) {
if (durationMs > 0 && mediaMs >= durationMs * DEFAULT_MIN_WATCH_RATIO) {
markVideoWatched(this.db, this.sessionState.videoId, true);
this.sessionState.markedWatched = true;
}
@@ -915,6 +986,21 @@ export class ImmersionTrackerService {
});
}
recordYomitanLookup(): void {
if (!this.sessionState) return;
this.sessionState.yomitanLookupCount += 1;
this.sessionState.pendingTelemetry = true;
this.recordWrite({
kind: 'event',
sessionId: this.sessionState.sessionId,
sampleMs: Date.now(),
eventType: EVENT_YOMITAN_LOOKUP,
cardsDelta: 0,
wordsDelta: 0,
payloadJson: null,
});
}
recordCardsMined(count = 1, noteIds?: number[]): void {
if (!this.sessionState) return;
this.sessionState.cardsMined += count;
@@ -981,6 +1067,7 @@ export class ImmersionTrackerService {
cardsMined: this.sessionState.cardsMined,
lookupCount: this.sessionState.lookupCount,
lookupHits: this.sessionState.lookupHits,
yomitanLookupCount: this.sessionState.yomitanLookupCount,
pauseCount: this.sessionState.pauseCount,
pauseMs: this.sessionState.pauseMs,
seekForwardCount: this.sessionState.seekForwardCount,
@@ -1080,6 +1167,7 @@ export class ImmersionTrackerService {
this.db.exec('VACUUM');
this.lastVacuumMs = nowMs;
}
runOptimizeMaintenance(this.db);
} catch (error) {
this.logger.warn(
'Immersion tracker maintenance failed, will retry later',
@@ -1108,6 +1196,7 @@ export class ImmersionTrackerService {
cardsMined: 0,
lookupCount: 0,
lookupHits: 0,
yomitanLookupCount: 0,
pauseCount: 0,
pauseMs: 0,
seekForwardCount: 0,

View File

@@ -17,6 +17,7 @@ import {
cleanupVocabularyStats,
deleteSession,
getDailyRollups,
getTrendsDashboard,
getQueryHints,
getMonthlyRollups,
getAnimeDetail,
@@ -31,6 +32,7 @@ import {
getVocabularyStats,
getKanjiStats,
getSessionEvents,
getSessionWordsByLine,
getWordOccurrences,
upsertCoverArt,
} from '../query.js';
@@ -126,6 +128,7 @@ test('getSessionSummaries returns sessionId and canonicalTitle', () => {
assert.equal(row.tokensSeen, 10);
assert.equal(row.lookupCount, 2);
assert.equal(row.lookupHits, 1);
assert.equal(row.yomitanLookupCount, 0);
} finally {
db.close();
cleanupDbPath(dbPath);
@@ -165,6 +168,163 @@ test('getDailyRollups limits by distinct days (not rows)', () => {
}
});
test('getTrendsDashboard returns chart-ready aggregated series', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const stmts = createTrackerPreparedStatements(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/trends-dashboard-test.mkv', {
canonicalTitle: 'Trend Dashboard Test',
sourcePath: '/tmp/trends-dashboard-test.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const animeId = getOrCreateAnimeRecord(db, {
parsedTitle: 'Trend Dashboard Anime',
canonicalTitle: 'Trend Dashboard Anime',
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
linkVideoToAnimeRecord(db, videoId, {
animeId,
parsedBasename: 'trends-dashboard-test.mkv',
parsedTitle: 'Trend Dashboard Anime',
parsedSeason: 1,
parsedEpisode: 1,
parserSource: 'test',
parserConfidence: 1,
parseMetadataJson: null,
});
const dayOneStart = new Date(2026, 2, 15, 12, 0, 0, 0).getTime();
const dayTwoStart = new Date(2026, 2, 16, 18, 0, 0, 0).getTime();
const sessionOne = startSessionRecord(db, videoId, dayOneStart);
const sessionTwo = startSessionRecord(db, videoId, dayTwoStart);
for (const [
sessionId,
startedAtMs,
activeWatchedMs,
cardsMined,
wordsSeen,
tokensSeen,
yomitanLookupCount,
] of [
[sessionOne.sessionId, dayOneStart, 30 * 60_000, 2, 100, 120, 8],
[sessionTwo.sessionId, dayTwoStart, 45 * 60_000, 3, 120, 140, 10],
] as const) {
stmts.telemetryInsertStmt.run(
sessionId,
startedAtMs + 60_000,
activeWatchedMs,
activeWatchedMs,
10,
wordsSeen,
tokensSeen,
cardsMined,
0,
0,
yomitanLookupCount,
0,
0,
0,
0,
startedAtMs + 60_000,
startedAtMs + 60_000,
);
db.prepare(
`
UPDATE imm_sessions
SET
ended_at_ms = ?,
total_watched_ms = ?,
active_watched_ms = ?,
lines_seen = ?,
words_seen = ?,
tokens_seen = ?,
cards_mined = ?,
yomitan_lookup_count = ?
WHERE session_id = ?
`,
).run(
startedAtMs + activeWatchedMs,
activeWatchedMs,
activeWatchedMs,
10,
wordsSeen,
tokensSeen,
cardsMined,
yomitanLookupCount,
sessionId,
);
}
db.prepare(
`
INSERT INTO imm_daily_rollups (
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
total_words_seen, total_tokens_seen, total_cards
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(Math.floor(dayOneStart / 86_400_000), videoId, 1, 30, 10, 100, 120, 2);
db.prepare(
`
INSERT INTO imm_daily_rollups (
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
total_words_seen, total_tokens_seen, total_cards
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(Math.floor(dayTwoStart / 86_400_000), videoId, 1, 45, 10, 120, 140, 3);
db.prepare(
`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(
'勉強',
'勉強',
'べんきょう',
'noun',
'名詞',
null,
null,
Math.floor(dayOneStart / 1000),
Math.floor(dayTwoStart / 1000),
);
const dashboard = getTrendsDashboard(db, 'all', 'day');
assert.equal(dashboard.activity.watchTime.length, 2);
assert.equal(dashboard.activity.watchTime[0]?.value, 30);
assert.equal(dashboard.progress.watchTime[1]?.value, 75);
assert.equal(dashboard.progress.lookups[1]?.value, 18);
assert.equal(
dashboard.ratios.lookupsPerHundred[0]?.value,
+((8 / 120) * 100).toFixed(1),
);
assert.equal(dashboard.animePerDay.watchTime[0]?.animeTitle, 'Trend Dashboard Anime');
assert.equal(dashboard.animeCumulative.watchTime[1]?.value, 75);
assert.equal(
dashboard.patterns.watchTimeByDayOfWeek.reduce((sum, point) => sum + point.value, 0),
75,
);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getQueryHints reads all-time totals from lifetime summary', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -238,6 +398,7 @@ test('getSessionSummaries with no telemetry returns zero aggregates', () => {
assert.equal(row.tokensSeen, 0);
assert.equal(row.lookupCount, 0);
assert.equal(row.lookupHits, 0);
assert.equal(row.yomitanLookupCount, 0);
assert.equal(row.cardsMined, 0);
} finally {
db.close();
@@ -292,6 +453,7 @@ test('getSessionSummaries uses denormalized session metrics for ended sessions w
assert.equal(row.cardsMined, 5);
assert.equal(row.lookupCount, 9);
assert.equal(row.lookupHits, 6);
assert.equal(row.yomitanLookupCount, 0);
} finally {
db.close();
cleanupDbPath(dbPath);
@@ -950,6 +1112,56 @@ test('getSessionEvents respects limit parameter', () => {
}
});
test('getSessionWordsByLine joins word occurrences through imm_words.id', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const stmts = createTrackerPreparedStatements(db);
const startedAtMs = Date.UTC(2025, 0, 1, 12, 0, 0);
const videoId = getOrCreateVideoRecord(db, '/tmp/session-words-by-line.mkv', {
canonicalTitle: 'Episode',
sourcePath: '/tmp/session-words-by-line.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const { sessionId } = startSessionRecord(db, videoId, startedAtMs);
const lineId = Number(
db
.prepare(
`INSERT INTO imm_subtitle_lines (
session_id, event_id, video_id, anime_id, line_index, segment_start_ms, segment_end_ms, text, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
)
.run(sessionId, null, videoId, null, 0, 0, 1000, '猫を見た', startedAtMs, startedAtMs)
.lastInsertRowid,
);
const wordId = Number(
db
.prepare(
`INSERT INTO imm_words (
headword, word, reading, pos1, pos2, pos3, part_of_speech, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
)
.run('猫', '猫', 'ねこ', null, null, null, null, startedAtMs, startedAtMs, 1)
.lastInsertRowid,
);
db.prepare(
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count)
VALUES (?, ?, ?)`,
).run(lineId, wordId, 1);
assert.deepEqual(getSessionWordsByLine(db, sessionId), [
{ lineIndex: 0, headword: '猫', occurrenceCount: 1 },
]);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('anime-level queries group by anime_id and preserve episode-level rows', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -1192,6 +1404,7 @@ test('anime-level queries group by anime_id and preserve episode-level rows', ()
assert.equal(animeDetail?.totalLinesSeen, 33);
assert.equal(animeDetail?.totalLookupCount, 12);
assert.equal(animeDetail?.totalLookupHits, 8);
assert.equal(animeDetail?.totalYomitanLookupCount, 0);
assert.equal(animeDetail?.episodeCount, 2);
const episodes = getAnimeEpisodes(db, lwaAnimeId);
@@ -1203,6 +1416,8 @@ test('anime-level queries group by anime_id and preserve episode-level rows', ()
totalSessions: row.totalSessions,
totalActiveMs: row.totalActiveMs,
totalCards: row.totalCards,
totalWordsSeen: row.totalWordsSeen,
totalYomitanLookupCount: row.totalYomitanLookupCount,
})),
[
{
@@ -1212,6 +1427,8 @@ test('anime-level queries group by anime_id and preserve episode-level rows', ()
totalSessions: 2,
totalActiveMs: 7_000,
totalCards: 3,
totalWordsSeen: 52,
totalYomitanLookupCount: 0,
},
{
videoId: lwaEpisode6,
@@ -1220,6 +1437,8 @@ test('anime-level queries group by anime_id and preserve episode-level rows', ()
totalSessions: 1,
totalActiveMs: 5_000,
totalCards: 3,
totalWordsSeen: 28,
totalYomitanLookupCount: 0,
},
],
);
@@ -1681,6 +1900,7 @@ test('anime/media detail and episode queries use ended-session metrics when tele
assert.equal(mediaDetail?.totalWordsSeen, 30);
assert.equal(mediaDetail?.totalLookupCount, 9);
assert.equal(mediaDetail?.totalLookupHits, 7);
assert.equal(mediaDetail?.totalYomitanLookupCount, 0);
} finally {
db.close();
cleanupDbPath(dbPath);
@@ -1984,3 +2204,159 @@ test('deleteSession removes the session and all associated session-scoped rows',
cleanupDbPath(dbPath);
}
});
test('deleteSession rebuilds word and kanji aggregates from retained subtitle lines', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const stmts = createTrackerPreparedStatements(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/delete-session-aggregates.mkv', {
canonicalTitle: 'Delete Session Aggregates Test',
sourcePath: '/tmp/delete-session-aggregates.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const deletedSession = startSessionRecord(db, videoId, 7_000_000);
const keptSession = startSessionRecord(db, videoId, 8_000_000);
const deletedTs = 7_000_500;
const keptTs = 8_000_500;
const sharedWordResult = db
.prepare(
`INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
)
.run('共有', '共有', 'きょうゆう', 'noun', '名詞', '一般', '', deletedTs, keptTs, 3);
const deletedOnlyWordResult = db
.prepare(
`INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
)
.run(
'削除専用',
'削除専用',
'さくじょせんよう',
'noun',
'名詞',
'一般',
'',
deletedTs,
deletedTs,
1,
);
const sharedKanjiResult = db
.prepare(
`INSERT INTO imm_kanji (
kanji, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?)`,
)
.run('共', deletedTs, keptTs, 3);
const deletedOnlyKanjiResult = db
.prepare(
`INSERT INTO imm_kanji (
kanji, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?)`,
)
.run('削', deletedTs, deletedTs, 1);
const deletedLineResult = stmts.subtitleLineInsertStmt.run(
deletedSession.sessionId,
null,
videoId,
null,
0,
0,
800,
'delete me',
deletedTs,
deletedTs,
);
const keptLineResult = stmts.subtitleLineInsertStmt.run(
keptSession.sessionId,
null,
videoId,
null,
0,
1_000,
1_800,
'keep me',
keptTs,
keptTs,
);
const deletedLineId = Number(deletedLineResult.lastInsertRowid);
const keptLineId = Number(keptLineResult.lastInsertRowid);
const sharedWordId = Number(sharedWordResult.lastInsertRowid);
const deletedOnlyWordId = Number(deletedOnlyWordResult.lastInsertRowid);
const sharedKanjiId = Number(sharedKanjiResult.lastInsertRowid);
const deletedOnlyKanjiId = Number(deletedOnlyKanjiResult.lastInsertRowid);
db.prepare(
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count)
VALUES (?, ?, ?)`,
).run(deletedLineId, sharedWordId, 2);
db.prepare(
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count)
VALUES (?, ?, ?)`,
).run(deletedLineId, deletedOnlyWordId, 1);
db.prepare(
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count)
VALUES (?, ?, ?)`,
).run(keptLineId, sharedWordId, 1);
db.prepare(
`INSERT INTO imm_kanji_line_occurrences (line_id, kanji_id, occurrence_count)
VALUES (?, ?, ?)`,
).run(deletedLineId, sharedKanjiId, 2);
db.prepare(
`INSERT INTO imm_kanji_line_occurrences (line_id, kanji_id, occurrence_count)
VALUES (?, ?, ?)`,
).run(deletedLineId, deletedOnlyKanjiId, 1);
db.prepare(
`INSERT INTO imm_kanji_line_occurrences (line_id, kanji_id, occurrence_count)
VALUES (?, ?, ?)`,
).run(keptLineId, sharedKanjiId, 1);
deleteSession(db, deletedSession.sessionId);
const sharedWordRow = db
.prepare('SELECT frequency, first_seen, last_seen FROM imm_words WHERE id = ?')
.get(sharedWordId) as {
frequency: number;
first_seen: number;
last_seen: number;
} | null;
const deletedOnlyWordRow = db
.prepare('SELECT id FROM imm_words WHERE id = ?')
.get(deletedOnlyWordId) as { id: number } | null;
const sharedKanjiRow = db
.prepare('SELECT frequency, first_seen, last_seen FROM imm_kanji WHERE id = ?')
.get(sharedKanjiId) as {
frequency: number;
first_seen: number;
last_seen: number;
} | null;
const deletedOnlyKanjiRow = db
.prepare('SELECT id FROM imm_kanji WHERE id = ?')
.get(deletedOnlyKanjiId) as { id: number } | null;
assert.ok(sharedWordRow);
assert.equal(sharedWordRow.frequency, 1);
assert.equal(sharedWordRow.first_seen, keptTs);
assert.equal(sharedWordRow.last_seen, keptTs);
assert.equal(deletedOnlyWordRow ?? null, null);
assert.ok(sharedKanjiRow);
assert.equal(sharedKanjiRow.frequency, 1);
assert.equal(sharedKanjiRow.first_seen, keptTs);
assert.equal(sharedKanjiRow.last_seen, keptTs);
assert.equal(deletedOnlyKanjiRow ?? null, null);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});

View File

@@ -51,6 +51,7 @@ interface RetainedSessionRow {
cardsMined: number;
lookupCount: number;
lookupHits: number;
yomitanLookupCount: number;
pauseCount: number;
pauseMs: number;
seekForwardCount: number;
@@ -154,6 +155,7 @@ function toRebuildSessionState(row: RetainedSessionRow): SessionState {
cardsMined: Math.max(0, row.cardsMined),
lookupCount: Math.max(0, row.lookupCount),
lookupHits: Math.max(0, row.lookupHits),
yomitanLookupCount: Math.max(0, row.yomitanLookupCount),
pauseCount: Math.max(0, row.pauseCount),
pauseMs: Math.max(0, row.pauseMs),
seekForwardCount: Math.max(0, row.seekForwardCount),
@@ -179,6 +181,7 @@ function getRetainedStaleActiveSessions(db: DatabaseSync): RetainedSessionRow[]
COALESCE(t.cards_mined, s.cards_mined, 0) AS cardsMined,
COALESCE(t.lookup_count, s.lookup_count, 0) AS lookupCount,
COALESCE(t.lookup_hits, s.lookup_hits, 0) AS lookupHits,
COALESCE(t.yomitan_lookup_count, s.yomitan_lookup_count, 0) AS yomitanLookupCount,
COALESCE(t.pause_count, s.pause_count, 0) AS pauseCount,
COALESCE(t.pause_ms, s.pause_ms, 0) AS pauseMs,
COALESCE(t.seek_forward_count, s.seek_forward_count, 0) AS seekForwardCount,
@@ -511,6 +514,7 @@ export function rebuildLifetimeSummaries(db: DatabaseSync): LifetimeRebuildSumma
cards_mined AS cardsMined,
lookup_count AS lookupCount,
lookup_hits AS lookupHits,
yomitan_lookup_count AS yomitanLookupCount,
pause_count AS pauseCount,
pause_ms AS pauseMs,
seek_forward_count AS seekForwardCount,

View File

@@ -4,7 +4,12 @@ import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { Database } from './sqlite';
import { pruneRawRetention, pruneRollupRetention, toMonthKey } from './maintenance';
import {
pruneRawRetention,
pruneRollupRetention,
runOptimizeMaintenance,
toMonthKey,
} from './maintenance';
import { ensureSchema } from './storage';
function makeDbPath(): string {
@@ -161,15 +166,15 @@ test('ensureSchema adds sample_ms index for telemetry rollup scans', () => {
try {
ensureSchema(db);
const indexes = db
.prepare("PRAGMA index_list('imm_session_telemetry')")
.all() as Array<{ name: string }>;
const indexes = db.prepare("PRAGMA index_list('imm_session_telemetry')").all() as Array<{
name: string;
}>;
const hasSampleMsIndex = indexes.some((row) => row.name === 'idx_telemetry_sample_ms');
assert.equal(hasSampleMsIndex, true);
const indexColumns = db
.prepare("PRAGMA index_info('idx_telemetry_sample_ms')")
.all() as Array<{ name: string }>;
const indexColumns = db.prepare("PRAGMA index_info('idx_telemetry_sample_ms')").all() as Array<{
name: string;
}>;
assert.deepEqual(
indexColumns.map((column) => column.name),
['sample_ms'],
@@ -179,3 +184,17 @@ test('ensureSchema adds sample_ms index for telemetry rollup scans', () => {
cleanupDbPath(dbPath);
}
});
test('runOptimizeMaintenance executes PRAGMA optimize', () => {
const executedSql: string[] = [];
const db = {
exec(source: string) {
executedSql.push(source);
return this;
},
} as unknown as Parameters<typeof runOptimizeMaintenance>[0];
runOptimizeMaintenance(db);
assert.deepEqual(executedSql, ['PRAGMA optimize']);
});

View File

@@ -329,3 +329,7 @@ export function runRollupMaintenance(db: DatabaseSync, forceRebuild = false): vo
throw error;
}
}
export function runOptimizeMaintenance(db: DatabaseSync): void {
db.exec('PRAGMA optimize');
}

File diff suppressed because it is too large Load Diff

View File

@@ -20,6 +20,7 @@ export function createInitialSessionState(
cardsMined: 0,
lookupCount: 0,
lookupHits: 0,
yomitanLookupCount: 0,
pauseCount: 0,
pauseMs: 0,
seekForwardCount: 0,

View File

@@ -47,6 +47,7 @@ export function finalizeSessionRecord(
cards_mined = ?,
lookup_count = ?,
lookup_hits = ?,
yomitan_lookup_count = ?,
pause_count = ?,
pause_ms = ?,
seek_forward_count = ?,
@@ -66,6 +67,7 @@ export function finalizeSessionRecord(
sessionState.cardsMined,
sessionState.lookupCount,
sessionState.lookupHits,
sessionState.yomitanLookupCount,
sessionState.pauseCount,
sessionState.pauseMs,
sessionState.seekForwardCount,

View File

@@ -6,6 +6,7 @@ import test from 'node:test';
import { Database } from './sqlite';
import { finalizeSessionRecord, startSessionRecord } from './session';
import {
applyPragmas,
createTrackerPreparedStatements,
ensureSchema,
executeQueuedWrite,
@@ -50,6 +51,34 @@ function cleanupDbPath(dbPath: string): void {
// libsql keeps Windows file handles alive after close when prepared statements were used.
}
test('applyPragmas sets the SQLite tuning defaults used by immersion tracking', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
applyPragmas(db);
const journalModeRow = db.prepare('PRAGMA journal_mode').get() as {
journal_mode: string;
};
const synchronousRow = db.prepare('PRAGMA synchronous').get() as { synchronous: number };
const foreignKeysRow = db.prepare('PRAGMA foreign_keys').get() as { foreign_keys: number };
const busyTimeoutRow = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };
const journalSizeLimitRow = db.prepare('PRAGMA journal_size_limit').get() as {
journal_size_limit: number;
};
assert.equal(journalModeRow.journal_mode, 'wal');
assert.equal(synchronousRow.synchronous, 1);
assert.equal(foreignKeysRow.foreign_keys, 1);
assert.equal(busyTimeoutRow.timeout, 2500);
assert.equal(journalSizeLimitRow.journal_size_limit, 67_108_864);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('ensureSchema creates immersion core tables', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -125,7 +154,9 @@ test('ensureSchema creates large-history performance indexes', () => {
ensureSchema(db);
const indexNames = new Set(
(
db.prepare(`SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_%'`).all() as Array<{
db
.prepare(`SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_%'`)
.all() as Array<{
name: string;
}>
).map((row) => row.name),
@@ -516,7 +547,9 @@ test('ensureSchema migrates legacy cover art blobs into the shared blob store',
assert.doesNotThrow(() => ensureSchema(db));
const mediaArtRow = db
.prepare('SELECT cover_blob AS coverBlob, cover_blob_hash AS coverBlobHash FROM imm_media_art')
.prepare(
'SELECT cover_blob AS coverBlob, cover_blob_hash AS coverBlobHash FROM imm_media_art',
)
.get() as {
coverBlob: ArrayBuffer | Uint8Array | Buffer | null;
coverBlobHash: string | null;
@@ -524,7 +557,10 @@ test('ensureSchema migrates legacy cover art blobs into the shared blob store',
assert.ok(mediaArtRow);
assert.ok(mediaArtRow?.coverBlobHash);
assert.equal(parseCoverBlobReference(normalizeCoverBlobBytes(mediaArtRow?.coverBlob)), mediaArtRow?.coverBlobHash);
assert.equal(
parseCoverBlobReference(normalizeCoverBlobBytes(mediaArtRow?.coverBlob)),
mediaArtRow?.coverBlobHash,
);
const sharedBlobRow = db
.prepare('SELECT cover_blob AS coverBlob FROM imm_cover_art_blobs WHERE blob_hash = ?')
@@ -732,6 +768,7 @@ test('executeQueuedWrite inserts event and telemetry rows', () => {
cardsMined: 1,
lookupCount: 2,
lookupHits: 1,
yomitanLookupCount: 0,
pauseCount: 1,
pauseMs: 50,
seekForwardCount: 0,

View File

@@ -39,6 +39,7 @@ export interface VideoAnimeLinkInput {
}
const COVER_BLOB_REFERENCE_PREFIX = '__subminer_cover_blob_ref__:';
const WAL_JOURNAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024;
export type CoverBlobBytes = ArrayBuffer | Uint8Array | Buffer;
@@ -153,6 +154,7 @@ export function applyPragmas(db: DatabaseSync): void {
db.exec('PRAGMA synchronous = NORMAL');
db.exec('PRAGMA foreign_keys = ON');
db.exec('PRAGMA busy_timeout = 2500');
db.exec(`PRAGMA journal_size_limit = ${WAL_JOURNAL_SIZE_LIMIT_BYTES}`);
}
export function normalizeAnimeIdentityKey(title: string): string {
@@ -577,6 +579,7 @@ export function ensureSchema(db: DatabaseSync): void {
cards_mined INTEGER NOT NULL DEFAULT 0,
lookup_count INTEGER NOT NULL DEFAULT 0,
lookup_hits INTEGER NOT NULL DEFAULT 0,
yomitan_lookup_count INTEGER NOT NULL DEFAULT 0,
pause_count INTEGER NOT NULL DEFAULT 0,
pause_ms INTEGER NOT NULL DEFAULT 0,
seek_forward_count INTEGER NOT NULL DEFAULT 0,
@@ -600,6 +603,7 @@ export function ensureSchema(db: DatabaseSync): void {
cards_mined INTEGER NOT NULL DEFAULT 0,
lookup_count INTEGER NOT NULL DEFAULT 0,
lookup_hits INTEGER NOT NULL DEFAULT 0,
yomitan_lookup_count INTEGER NOT NULL DEFAULT 0,
pause_count INTEGER NOT NULL DEFAULT 0,
pause_ms INTEGER NOT NULL DEFAULT 0,
seek_forward_count INTEGER NOT NULL DEFAULT 0,
@@ -1013,6 +1017,29 @@ export function ensureSchema(db: DatabaseSync): void {
deduplicateExistingCoverArtRows(db);
}
if (currentVersion?.schema_version && currentVersion.schema_version < 14) {
addColumnIfMissing(db, 'imm_sessions', 'yomitan_lookup_count', 'INTEGER NOT NULL DEFAULT 0');
addColumnIfMissing(
db,
'imm_session_telemetry',
'yomitan_lookup_count',
'INTEGER NOT NULL DEFAULT 0',
);
db.exec(`
UPDATE imm_sessions
SET
yomitan_lookup_count = COALESCE((
SELECT t.yomitan_lookup_count
FROM imm_session_telemetry t
WHERE t.session_id = imm_sessions.session_id
ORDER BY t.sample_ms DESC, t.telemetry_id DESC
LIMIT 1
), yomitan_lookup_count)
WHERE ended_at_ms IS NOT NULL
`);
}
ensureLifetimeSummaryTables(db);
db.exec(`
@@ -1137,10 +1164,10 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
INSERT INTO imm_session_telemetry (
session_id, sample_ms, total_watched_ms, active_watched_ms,
lines_seen, words_seen, tokens_seen, cards_mined, lookup_count,
lookup_hits, pause_count, pause_ms, seek_forward_count,
lookup_hits, yomitan_lookup_count, pause_count, pause_ms, seek_forward_count,
seek_backward_count, media_buffer_events, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
`),
eventInsertStmt: db.prepare(`
@@ -1288,6 +1315,7 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta
write.cardsMined!,
write.lookupCount!,
write.lookupHits!,
write.yomitanLookupCount ?? 0,
write.pauseCount!,
write.pauseMs!,
write.seekForwardCount!,

View File

@@ -1,4 +1,4 @@
export const SCHEMA_VERSION = 13;
export const SCHEMA_VERSION = 14;
export const DEFAULT_QUEUE_CAP = 1_000;
export const DEFAULT_BATCH_SIZE = 25;
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
@@ -26,6 +26,7 @@ export const EVENT_SEEK_FORWARD = 5;
export const EVENT_SEEK_BACKWARD = 6;
export const EVENT_PAUSE_START = 7;
export const EVENT_PAUSE_END = 8;
export const EVENT_YOMITAN_LOOKUP = 9;
export interface ImmersionTrackerOptions {
dbPath: string;
@@ -60,6 +61,7 @@ export interface TelemetryAccumulator {
cardsMined: number;
lookupCount: number;
lookupHits: number;
yomitanLookupCount: number;
pauseCount: number;
pauseMs: number;
seekForwardCount: number;
@@ -92,6 +94,7 @@ interface QueuedTelemetryWrite {
cardsMined?: number;
lookupCount?: number;
lookupHits?: number;
yomitanLookupCount?: number;
pauseCount?: number;
pauseMs?: number;
seekForwardCount?: number;
@@ -233,6 +236,7 @@ export interface SessionSummaryQueryRow {
cardsMined: number;
lookupCount: number;
lookupHits: number;
yomitanLookupCount: number;
}
export interface LifetimeGlobalRow {
@@ -432,6 +436,7 @@ export interface MediaDetailRow {
totalLinesSeen: number;
totalLookupCount: number;
totalLookupHits: number;
totalYomitanLookupCount: number;
}
export interface AnimeLibraryRow {
@@ -462,6 +467,7 @@ export interface AnimeDetailRow {
totalLinesSeen: number;
totalLookupCount: number;
totalLookupHits: number;
totalYomitanLookupCount: number;
episodeCount: number;
lastWatchedMs: number;
}
@@ -486,6 +492,7 @@ export interface AnimeEpisodeRow {
totalActiveMs: number;
totalCards: number;
totalWordsSeen: number;
totalYomitanLookupCount: number;
lastWatchedMs: number;
}