fix(stats): subtract lifetime totals incrementally on delete (#196)

This commit is contained in:
2026-08-14 22:13:59 -07:00
committed by GitHub
parent 6fb3d2eb13
commit d963def1a9
20 changed files with 1446 additions and 370 deletions
@@ -4015,6 +4015,8 @@ test('reassignAnimeAnilist redistributes conflicting legacy combined row before
(3, 1, 3000, 0, '5000', '6000', 5000, 6000);
`);
await tracker.rebuildLifetimeSummaries();
await tracker.reassignAnimeAnilist(2, {
anilistId: 21202,
titleRomaji: 'Kono Subarashii Sekai ni Shukufuku wo!',
+41 -7
View File
@@ -30,9 +30,11 @@ import {
} from './immersion-tracker/storage';
import {
applySessionLifetimeSummary,
recomputeLifetimeAnimeAggregates,
reconcileStaleActiveSessions,
rebuildLifetimeSummaries as rebuildLifetimeSummaryTables,
recomputeLifetimeAnimeFromMedia,
recomputeLifetimeGlobalFromSummaries,
repairLifetimeSummariesFromMedia,
shouldBackfillLifetimeSummaries,
} from './immersion-tracker/lifetime';
import {
@@ -534,7 +536,7 @@ export class ImmersionTrackerService {
this.logger.info(
`Repaired season-scoped stats links on startup: scanned=${seasonRepair.scanned} movedVideos=${seasonRepair.movedVideos} deletedAnimeRows=${seasonRepair.deletedAnimeRows}`,
);
recomputeLifetimeAnimeAggregates(this.db);
repairLifetimeSummariesFromMedia(this.db);
}
if (shouldBackfillLifetimeSummaries(this.db)) {
const result = rebuildLifetimeSummaryTables(this.db);
@@ -660,7 +662,17 @@ export class ImmersionTrackerService {
async rebuildLifetimeSummaries(): Promise<LifetimeRebuildSummary> {
this.requireWriteQueueDrained('rebuilding lifetime summaries');
return rebuildLifetimeSummaryTables(this.db);
// Non-destructive: recomputes from the media ledger (or bootstraps empty
// lifetime tables), so history older than session retention is never reset.
const repaired = repairLifetimeSummariesFromMedia(this.db);
// Sessions currently tracked in the applied-sessions ledger, not sessions
// processed by this call — the repair recomputes summaries instead of
// re-applying sessions. Retention prunes these rows (FK cascade), so on an
// old database this reads lower than the history the totals still include.
const appliedRow = this.db
.prepare('SELECT COUNT(*) AS count FROM imm_lifetime_applied_sessions')
.get() as { count: number };
return { appliedSessions: Number(appliedRow.count), rebuiltAtMs: repaired.repairedAtMs };
}
async getKanjiStats(limit = 100): Promise<KanjiStatsRow[]> {
@@ -943,8 +955,16 @@ export class ImmersionTrackerService {
nowMs(),
animeId,
);
if (repair.movedVideos > 0 || repair.deletedAnimeRows > 0) {
recomputeLifetimeAnimeAggregates(this.db);
// Empty lifetime tables still need the retained-session bootstrap. Once a
// media ledger exists, only the redistributed and explicitly edited anime
// can have changed.
if (shouldBackfillLifetimeSummaries(this.db)) {
repairLifetimeSummariesFromMedia(this.db);
} else {
const affectedAnimeIds = new Set(repair.affectedAnimeIds);
affectedAnimeIds.add(animeId);
recomputeLifetimeAnimeFromMedia(this.db, [...affectedAnimeIds]);
recomputeLifetimeGlobalFromSummaries(this.db);
}
// Update cover art for all videos in this anime
@@ -1393,7 +1413,7 @@ export class ImmersionTrackerService {
metadataJson: candidate.metadataJson,
});
}
recomputeLifetimeAnimeAggregates(this.db);
repairLifetimeSummariesFromMedia(this.db);
}
recordJellyfinPlaybackMetadata(metadata: JellyfinPlaybackMetadataInput): void {
@@ -1468,7 +1488,21 @@ export class ImmersionTrackerService {
this.db.prepare('SELECT 1 FROM imm_lifetime_media WHERE video_id = ?').get(videoId),
);
if (hasLifetimeMedia || (previousLink && previousLink.animeId !== animeId)) {
recomputeLifetimeAnimeAggregates(this.db);
// Playback-time relink: only the old and new anime are affected, so
// recompute just those from the media ledger instead of a full repair.
const affectedAnimeIds = new Set<number>([animeId]);
if (previousLink?.animeId) affectedAnimeIds.add(previousLink.animeId);
let transactionStarted = false;
try {
this.db.exec('BEGIN IMMEDIATE');
transactionStarted = true;
recomputeLifetimeAnimeFromMedia(this.db, [...affectedAnimeIds]);
recomputeLifetimeGlobalFromSummaries(this.db);
this.db.exec('COMMIT');
} catch (error) {
if (transactionStarted) this.db.exec('ROLLBACK');
throw error;
}
}
}
@@ -329,3 +329,28 @@ test('upgrading an older database backfills seen_ms from the subtitle lines', ()
cleanupDbPath(dbPath);
}
});
test('an extreme-moving delete keeps subtraction-exact frequency instead of re-summing', () => {
const { db, dbPath } = createDb([
{ session: 1, wordId: 7, dayOffset: 0, count: 2 },
{ session: 2, wordId: 7, dayOffset: 3, count: 1 },
]);
try {
// Simulate drift: the stored total is higher than the occurrences justify.
// The extremes move via index seeks while the count stays a pure
// subtraction; drifted counts reconcile only at the zero-crossing repair
// or via the cleanup command, never by rescanning every occurrence here.
db.prepare('UPDATE imm_words SET frequency = 10 WHERE id = 7').run();
deleteSession(db, 1);
const word = readWord(db, 7);
assert.equal(word?.frequency, 10 - 2);
assert.equal(word?.firstSeen, Math.floor((BASE_MS + 3 * DAY_MS) / 1000));
assert.equal(word?.lastSeen, Math.floor((BASE_MS + 3 * DAY_MS) / 1000));
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
@@ -0,0 +1,303 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { startSessionRecord } from '../session.js';
import { applySessionLifetimeSummary, rebuildLifetimeSummaries } from '../lifetime.js';
import { deleteMaintenanceBatch } from '../query-delete-maintenance.js';
import { toDbTimestamp } from '../query-shared.js';
import {
BASE_MS,
DAY_MS,
cleanRow,
createDb,
seedAnime,
seedEndedSession,
seedVideo,
snapshotAnime,
snapshotGlobal,
snapshotMedia,
} from './lifetime-test-fixtures.js';
test('fractional lifetime metrics stay normalized across apply, rebuild, and delete', () => {
const db = createDb();
try {
const videoId = seedVideo(db, null, 'fractional-metrics');
const seedFractionalSession = (
startedAtMs: number,
metrics: { activeMs: number; cards: number; lines: number; tokens: number },
) => {
const { state } = startSessionRecord(db, videoId, startedAtMs);
state.activeWatchedMs = metrics.activeMs;
state.cardsMined = metrics.cards;
state.linesSeen = metrics.lines;
state.tokensSeen = metrics.tokens;
const endedAtMs = startedAtMs + 2_000;
db.prepare(
`UPDATE imm_sessions SET
ended_at_ms = ?,
active_watched_ms = ?,
cards_mined = ?,
lines_seen = ?,
tokens_seen = ?
WHERE session_id = ?`,
).run(
toDbTimestamp(endedAtMs),
metrics.activeMs,
metrics.cards,
metrics.lines,
metrics.tokens,
state.sessionId,
);
return { state, endedAtMs };
};
const readMediaMetrics = () =>
cleanRow<{
total_sessions: number;
total_active_ms: number;
total_cards: number;
total_lines_seen: number;
total_tokens_seen: number;
}>(
db
.prepare(
`SELECT total_sessions, total_active_ms, total_cards,
total_lines_seen, total_tokens_seen
FROM imm_lifetime_media WHERE video_id = ?`,
)
.get(videoId),
);
const withoutTelemetry = seedFractionalSession(BASE_MS, {
activeMs: 1_234.9,
cards: 2.8,
lines: 3.7,
tokens: 4.6,
});
applySessionLifetimeSummary(db, withoutTelemetry.state, withoutTelemetry.endedAtMs);
assert.deepEqual(readMediaMetrics(), {
total_sessions: 1,
total_active_ms: 1_234,
total_cards: 2,
total_lines_seen: 3,
total_tokens_seen: 4,
});
const withTelemetry = seedFractionalSession(BASE_MS + DAY_MS, {
activeMs: 9_999.9,
cards: 9.9,
lines: 9.9,
tokens: 9.9,
});
db.prepare(
`INSERT INTO imm_session_telemetry (
session_id, sample_ms, active_watched_ms, cards_mined, lines_seen, tokens_seen
) VALUES (?, ?, ?, ?, ?, ?)`,
).run(withTelemetry.state.sessionId, withTelemetry.endedAtMs, 2_345.9, 5.8, 6.7, 7.6);
applySessionLifetimeSummary(db, withTelemetry.state, withTelemetry.endedAtMs);
assert.deepEqual(readMediaMetrics(), {
total_sessions: 2,
total_active_ms: 3_579,
total_cards: 7,
total_lines_seen: 9,
total_tokens_seen: 11,
});
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: withTelemetry.state.sessionId }]);
const retainedMetrics = {
total_sessions: 1,
total_active_ms: 1_234,
total_cards: 2,
total_lines_seen: 3,
total_tokens_seen: 4,
};
assert.deepEqual(readMediaMetrics(), retainedMetrics, 'delete subtracts floored telemetry');
rebuildLifetimeSummaries(db);
assert.deepEqual(
readMediaMetrics(),
retainedMetrics,
'rebuild floors session-row fallback values',
);
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: withoutTelemetry.state.sessionId }]);
assert.deepEqual(snapshotMedia(db), [], 'delete subtracts the normalized metrics exactly');
assert.deepEqual(snapshotGlobal(db), {
total_sessions: 0,
total_active_ms: 0,
total_cards: 0,
active_days: 0,
episodes_started: 0,
episodes_completed: 0,
anime_completed: 0,
});
} finally {
db.close();
}
});
test('incremental delete maintenance matches a full rebuild when no history is pruned', () => {
const db = createDb();
try {
const animeA = seedAnime(db, 'Anime A', 2);
const animeB = seedAnime(db, 'Anime B', 1);
const videoA1 = seedVideo(db, animeA, 'anime-a-ep1', { watched: true });
const videoA2 = seedVideo(db, animeA, 'anime-a-ep2', { watched: true });
const videoB1 = seedVideo(db, animeB, 'anime-b-ep1', { watched: true });
const videoLoose = seedVideo(db, null, 'loose-video');
seedEndedSession(db, videoA1, BASE_MS, { activeMs: 60_000, cards: 2, lines: 30, tokens: 200 });
const deletedSessionId = seedEndedSession(db, videoA1, BASE_MS + DAY_MS, {
activeMs: 45_000,
cards: 1,
lines: 20,
tokens: 100,
});
seedEndedSession(db, videoA2, BASE_MS + 2 * DAY_MS, { activeMs: 90_000, cards: 3 });
seedEndedSession(db, videoB1, BASE_MS + 3 * DAY_MS, { activeMs: 30_000 });
seedEndedSession(db, videoLoose, BASE_MS + 4 * DAY_MS, { activeMs: 15_000 });
rebuildLifetimeSummaries(db);
deleteMaintenanceBatch(db, [
{ kind: 'session', sessionId: deletedSessionId },
{ kind: 'video', videoId: videoLoose },
{ kind: 'anime', animeId: animeB },
]);
const incrementalGlobal = snapshotGlobal(db);
const incrementalMedia = snapshotMedia(db);
const incrementalAnime = snapshotAnime(db);
// With every session still retained, subtracting must land on exactly the
// state a from-scratch rebuild computes.
rebuildLifetimeSummaries(db);
assert.deepEqual(incrementalGlobal, snapshotGlobal(db));
assert.deepEqual(incrementalMedia, snapshotMedia(db));
assert.deepEqual(incrementalAnime, snapshotAnime(db));
} finally {
db.close();
}
});
test('deleting a retained session preserves lifetime history from pruned sessions', () => {
const db = createDb();
try {
const animeId = seedAnime(db, 'Pruned Anime', null);
const videoId = seedVideo(db, animeId, 'pruned-ep1');
const prunedSessionId = seedEndedSession(db, videoId, BASE_MS, {
activeMs: 120_000,
cards: 4,
lines: 50,
tokens: 400,
});
const retainedSessionId = seedEndedSession(db, videoId, BASE_MS + DAY_MS, {
activeMs: 30_000,
cards: 1,
lines: 10,
tokens: 80,
});
rebuildLifetimeSummaries(db);
// Simulate raw-session retention pruning the older session. Lifetime
// summaries intentionally keep its contribution.
db.prepare('DELETE FROM imm_sessions WHERE session_id = ?').run(prunedSessionId);
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: retainedSessionId }]);
const globalRow = snapshotGlobal(db);
assert.equal(globalRow.total_sessions, 1, 'pruned session contribution survives the delete');
assert.equal(globalRow.total_active_ms, 120_000);
assert.equal(globalRow.total_cards, 4);
assert.equal(globalRow.episodes_started, 1);
// The pruned session's day stays counted (pruning never subtracts); only
// the deleted retained session's day is dropped.
assert.equal(globalRow.active_days, 1);
const mediaRow = db
.prepare(
'SELECT total_sessions, total_active_ms, total_cards FROM imm_lifetime_media WHERE video_id = ?',
)
.get(videoId);
assert.deepEqual(
cleanRow<{ total_sessions: number; total_active_ms: number; total_cards: number }>(mediaRow),
{ total_sessions: 1, total_active_ms: 120_000, total_cards: 4 },
);
} finally {
db.close();
}
});
test('active_days only drops when the last session of a local day is deleted', () => {
const db = createDb();
try {
const videoId = seedVideo(db, null, 'same-day');
const firstSessionId = seedEndedSession(db, videoId, BASE_MS, { activeMs: 10_000 });
const secondSessionId = seedEndedSession(db, videoId, BASE_MS + 3_600_000, {
activeMs: 20_000,
});
rebuildLifetimeSummaries(db);
assert.equal(snapshotGlobal(db).active_days, 1);
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: firstSessionId }]);
assert.equal(snapshotGlobal(db).active_days, 1, 'day still has a session');
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: secondSessionId }]);
assert.equal(snapshotGlobal(db).active_days, 0, 'day lost its last session');
} finally {
db.close();
}
});
test('deleting a video updates anime and global rollups without a rebuild', () => {
const db = createDb();
try {
const animeId = seedAnime(db, 'Two Episode Anime', 2);
const videoEp1 = seedVideo(db, animeId, 'two-ep-1', { watched: true });
const videoEp2 = seedVideo(db, animeId, 'two-ep-2', { watched: true });
seedEndedSession(db, videoEp1, BASE_MS, { activeMs: 60_000, cards: 2 });
seedEndedSession(db, videoEp2, BASE_MS + DAY_MS, { activeMs: 40_000, cards: 1 });
rebuildLifetimeSummaries(db);
assert.equal(snapshotGlobal(db).anime_completed, 1);
deleteMaintenanceBatch(db, [{ kind: 'video', videoId: videoEp2 }]);
const globalRow = snapshotGlobal(db);
assert.equal(globalRow.total_sessions, 1);
assert.equal(globalRow.total_active_ms, 60_000);
assert.equal(globalRow.episodes_started, 1);
assert.equal(globalRow.episodes_completed, 1);
assert.equal(globalRow.anime_completed, 0, 'anime no longer has all episodes completed');
const animeRow = db
.prepare(
'SELECT total_sessions, episodes_started, episodes_completed FROM imm_lifetime_anime WHERE anime_id = ?',
)
.get(animeId);
assert.deepEqual(
cleanRow<{
total_sessions: number;
episodes_started: number;
episodes_completed: number;
}>(animeRow),
{ total_sessions: 1, episodes_started: 1, episodes_completed: 1 },
);
deleteMaintenanceBatch(db, [{ kind: 'video', videoId: videoEp1 }]);
assert.equal(
(db.prepare('SELECT COUNT(*) AS total FROM imm_lifetime_anime').get() as { total: number })
.total,
0,
'anime lifetime row is dropped once no episodes remain',
);
assert.deepEqual(snapshotGlobal(db), {
total_sessions: 0,
total_active_ms: 0,
total_cards: 0,
active_days: 0,
episodes_started: 0,
episodes_completed: 0,
anime_completed: 0,
});
} finally {
db.close();
}
});
@@ -0,0 +1,94 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { rebuildLifetimeSummaries, repairLifetimeSummariesFromMedia } from '../lifetime.js';
import {
BASE_MS,
DAY_MS,
cleanRow,
createDb,
seedAnime,
seedEndedSession,
seedVideo,
snapshotAnime,
snapshotGlobal,
snapshotMedia,
} from './lifetime-test-fixtures.js';
test('repair after a video moves between anime matches a full rebuild', () => {
const db = createDb();
try {
const animeA = seedAnime(db, 'Move Source', 2);
const animeB = seedAnime(db, 'Move Target', 2);
const movedVideo = seedVideo(db, animeA, 'moved-ep', { watched: true });
const stayingVideo = seedVideo(db, animeA, 'staying-ep');
const targetVideo = seedVideo(db, animeB, 'target-ep', { watched: true });
seedEndedSession(db, movedVideo, BASE_MS, { activeMs: 60_000, cards: 2 });
seedEndedSession(db, stayingVideo, BASE_MS + DAY_MS, { activeMs: 30_000 });
seedEndedSession(db, targetVideo, BASE_MS + 2 * DAY_MS, { activeMs: 45_000, cards: 1 });
rebuildLifetimeSummaries(db);
// Simulate a library merge reassigning the episode to the other anime.
db.prepare('UPDATE imm_videos SET anime_id = ? WHERE video_id = ?').run(animeB, movedVideo);
repairLifetimeSummariesFromMedia(db);
const repairedGlobal = snapshotGlobal(db);
const repairedMedia = snapshotMedia(db);
const repairedAnime = snapshotAnime(db);
rebuildLifetimeSummaries(db);
assert.deepEqual(repairedGlobal, snapshotGlobal(db));
assert.deepEqual(repairedMedia, snapshotMedia(db));
assert.deepEqual(repairedAnime, snapshotAnime(db));
} finally {
db.close();
}
});
test('repair preserves lifetime history from pruned sessions where a rebuild would not', () => {
const db = createDb();
try {
const animeId = seedAnime(db, 'Repair Anime', null);
const videoId = seedVideo(db, animeId, 'repair-ep');
const prunedSessionId = seedEndedSession(db, videoId, BASE_MS, {
activeMs: 90_000,
cards: 3,
});
seedEndedSession(db, videoId, BASE_MS + DAY_MS, { activeMs: 30_000, cards: 1 });
rebuildLifetimeSummaries(db);
db.prepare('DELETE FROM imm_sessions WHERE session_id = ?').run(prunedSessionId);
repairLifetimeSummariesFromMedia(db);
const globalRow = snapshotGlobal(db);
assert.equal(globalRow.total_sessions, 2, 'repair keeps the pruned session contribution');
assert.equal(globalRow.total_active_ms, 120_000);
assert.equal(globalRow.total_cards, 4);
assert.equal(globalRow.active_days, 2, 'repair never subtracts active days');
const animeRow = db
.prepare('SELECT total_sessions FROM imm_lifetime_anime WHERE anime_id = ?')
.get(animeId);
assert.equal(cleanRow<{ total_sessions: number }>(animeRow).total_sessions, 2);
} finally {
db.close();
}
});
test('repair leaves a caller-owned transaction intact when its begin fails', () => {
const db = createDb();
try {
db.exec('BEGIN');
const animeId = seedAnime(db, 'Caller Transaction', null);
assert.throws(() => repairLifetimeSummariesFromMedia(db), /transaction/i);
assert.ok(
db.prepare('SELECT 1 FROM imm_anime WHERE anime_id = ?').get(animeId),
'the repair did not roll back the caller transaction',
);
db.exec('ROLLBACK');
assert.equal(db.prepare('SELECT 1 FROM imm_anime WHERE anime_id = ?').get(animeId), undefined);
} finally {
db.close();
}
});
@@ -0,0 +1,157 @@
import { Database } from '../sqlite.js';
import type { DatabaseSync } from '../sqlite.js';
import {
applyPragmas,
ensureSchema,
getOrCreateAnimeRecord,
getOrCreateVideoRecord,
linkVideoToAnimeRecord,
} from '../storage.js';
import { startSessionRecord } from '../session.js';
import { toDbTimestamp } from '../query-shared.js';
const SOURCE_TYPE_LOCAL = 1;
export const DAY_MS = 86_400_000;
// Noon UTC keeps every seeded timestamp on the same local day regardless of
// the timezone the test host runs in.
export const BASE_MS = Date.UTC(2026, 0, 5, 12, 0, 0);
export function createDb(): DatabaseSync {
const db = new Database(':memory:');
applyPragmas(db);
ensureSchema(db);
return db;
}
export function seedAnime(db: DatabaseSync, title: string, episodesTotal: number | null): number {
const animeId = getOrCreateAnimeRecord(db, {
parsedTitle: title,
canonicalTitle: title,
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
if (episodesTotal !== null) {
db.prepare('UPDATE imm_anime SET episodes_total = ? WHERE anime_id = ?').run(
episodesTotal,
animeId,
);
}
return animeId;
}
export function seedVideo(
db: DatabaseSync,
animeId: number | null,
name: string,
options: { watched?: boolean } = {},
): number {
const videoId = getOrCreateVideoRecord(db, `local:/tmp/${name}.mkv`, {
canonicalTitle: name,
sourcePath: `/tmp/${name}.mkv`,
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
if (animeId !== null) {
linkVideoToAnimeRecord(db, videoId, {
animeId,
parsedBasename: `${name}.mkv`,
parsedTitle: name,
parsedSeason: 1,
parsedEpisode: 1,
parserSource: 'test',
parserConfidence: 1,
parseMetadataJson: null,
});
}
if (options.watched) {
db.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?').run(videoId);
}
return videoId;
}
export function seedEndedSession(
db: DatabaseSync,
videoId: number,
startedAtMs: number,
metrics: { activeMs: number; cards?: number; lines?: number; tokens?: number },
): number {
const sessionId = startSessionRecord(db, videoId, startedAtMs).sessionId;
db.prepare(
`
UPDATE imm_sessions SET
ended_at_ms = ?,
active_watched_ms = ?,
total_watched_ms = ?,
cards_mined = ?,
lines_seen = ?,
tokens_seen = ?
WHERE session_id = ?
`,
).run(
toDbTimestamp(startedAtMs + metrics.activeMs),
metrics.activeMs,
metrics.activeMs,
metrics.cards ?? 0,
metrics.lines ?? 0,
metrics.tokens ?? 0,
sessionId,
);
return sessionId;
}
// libsql attaches a per-query `_metadata` property to result rows; strip it so
// row snapshots can be compared with deepEqual.
export function cleanRow<T>(row: unknown): T {
const { _metadata: _ignored, ...rest } = row as Record<string, unknown>;
return rest as T;
}
export interface GlobalSnapshot {
total_sessions: number;
total_active_ms: number;
total_cards: number;
active_days: number;
episodes_started: number;
episodes_completed: number;
anime_completed: number;
}
export function snapshotGlobal(db: DatabaseSync): GlobalSnapshot {
const row = db
.prepare(
`SELECT total_sessions, total_active_ms, total_cards, active_days,
episodes_started, episodes_completed, anime_completed
FROM imm_lifetime_global WHERE global_id = 1`,
)
.get();
return cleanRow<GlobalSnapshot>(row);
}
export function snapshotMedia(db: DatabaseSync): unknown[] {
return db
.prepare(
`SELECT video_id, total_sessions, total_active_ms, total_cards,
total_lines_seen, total_tokens_seen, completed,
CAST(first_watched_ms AS REAL) AS first_watched,
CAST(last_watched_ms AS REAL) AS last_watched
FROM imm_lifetime_media ORDER BY video_id`,
)
.all()
.map((row) => cleanRow(row));
}
export function snapshotAnime(db: DatabaseSync): unknown[] {
return db
.prepare(
`SELECT anime_id, total_sessions, total_active_ms, total_cards,
total_lines_seen, total_tokens_seen, episodes_started, episodes_completed,
CAST(first_watched_ms AS REAL) AS first_watched,
CAST(last_watched_ms AS REAL) AS last_watched
FROM imm_lifetime_anime ORDER BY anime_id`,
)
.all()
.map((row) => cleanRow(row));
}
@@ -29,6 +29,7 @@ export interface AnimeSeasonRepairSummary {
mergeRecommended: boolean;
/** True when automatic metadata must not assign the colliding AniList id. */
anilistAssignmentBlocked: boolean;
affectedAnimeIds: number[];
}
export interface AnimeAnilistConflictOptions extends AnimeConflictRecommendationOptions {
@@ -79,6 +80,7 @@ function emptySummary(scanned = 0): AnimeSeasonRepairSummary {
survivingAnimeId: null,
mergeRecommended: false,
anilistAssignmentBlocked: false,
affectedAnimeIds: [],
};
}
@@ -93,6 +95,7 @@ function mergeSummary(
target.survivingAnimeId = source.survivingAnimeId ?? target.survivingAnimeId;
target.mergeRecommended ||= source.mergeRecommended;
target.anilistAssignmentBlocked ||= source.anilistAssignmentBlocked;
target.affectedAnimeIds = [...new Set([...target.affectedAnimeIds, ...source.affectedAnimeIds])];
return target;
}
@@ -231,6 +234,7 @@ function redistributeAnimeRowByParsedSeasonsInTransaction(
const videos = getParsedVideos(db, animeId);
const summary = emptySummary(1);
summary.affectedAnimeIds.push(animeId);
const updatedAt = toDbTimestamp(nowMs());
const targetBySeason = new Map<number, number>();
@@ -283,6 +287,9 @@ function redistributeAnimeRowByParsedSeasonsInTransaction(
if (videoUpdate.changes > 0 || lineUpdate.changes > 0) {
summary.movedVideos += 1;
if (!summary.affectedAnimeIds.includes(targetAnimeId)) {
summary.affectedAnimeIds.push(targetAnimeId);
}
}
}
@@ -425,6 +432,7 @@ export function resolveAnimeAnilistConflict(
// Only reported once a row really absorbed the other, so callers never
// follow this to an anime id that was never written.
summary.survivingAnimeId = survivingAnimeId;
summary.affectedAnimeIds.push(survivingAnimeId, absorbedAnimeId);
}
// Lifetime summaries are rebuilt by the caller off this summary, the same
// as the redistribution path below.
@@ -32,7 +32,7 @@ function createFakeWorker() {
type FakeWorker = ReturnType<typeof createFakeWorker>['worker'];
test('a delete batch rebuilds lifetime summaries once', () => {
test('a delete batch never runs a full lifetime rebuild', () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-delete-batch-test-'));
const dbPath = path.join(tempDir, 'immersion.sqlite');
let db = new Database(dbPath);
@@ -87,8 +87,8 @@ test('a delete batch rebuilds lifetime summaries once', () => {
assert.equal(deletedVideo, undefined);
assert.equal(
audit.total,
2,
'one rebuild performs exactly its reset and final global summary writes',
0,
'delete maintenance subtracts incrementally instead of rewriting last_rebuilt_ms',
);
} finally {
try {
@@ -100,6 +100,14 @@ test('a delete batch rebuilds lifetime summaries once', () => {
}
});
test('delete worker module resolves in the current layout', () => {
// If this resolves to null, every delete silently runs on the serving thread
// and blocks the stats API for the whole maintenance run.
const workerPath = resolveDeleteMaintenanceWorkerPath();
assert.ok(workerPath, 'delete-maintenance worker module must resolve');
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
});
test(
'compiled delete worker removes data through its separate database connection',
{ skip: resolveDeleteMaintenanceWorkerPath() === null },
@@ -178,19 +186,72 @@ test('worker runtime terminates a worker after successful settlement', async ()
assert.equal(terminationState.calls, 1);
});
test('worker runtime terminates a worker after failed settlement', async () => {
test('worker runtime falls back to the current thread when the worker crashes', async () => {
const { worker, listeners, terminationState } = createFakeWorker();
const fallbackTasks: unknown[] = [];
const warnings: string[] = [];
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => '/tmp/delete-worker.js',
createWorker: async () => worker,
executeFallback: (dbPath, task) => {
fallbackTasks.push({ dbPath, task });
},
warn: (message) => {
warnings.push(message);
},
});
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
await new Promise<void>((resolve) => setTimeout(resolve, 0));
listeners.get('error')?.(new Error('worker failed') as never);
await assert.rejects(result, /worker failed/);
await result;
assert.equal(terminationState.calls, 1);
assert.deepEqual(fallbackTasks, [
{ dbPath: '/tmp/test.sqlite', task: { kind: 'session', sessionId: 1 } },
]);
assert.equal(warnings.length, 1);
});
test('worker runtime falls back when a worker exits cleanly without a response', async () => {
const { worker, listeners, terminationState } = createFakeWorker();
const fallbackTasks: unknown[] = [];
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => '/tmp/delete-worker.js',
createWorker: async () => worker,
executeFallback: (dbPath, task) => {
fallbackTasks.push({ dbPath, task });
},
});
const task = { kind: 'session' as const, sessionId: 1 };
const result = runtime.run('/tmp/test.sqlite', task);
await new Promise<void>((resolve) => setTimeout(resolve, 0));
listeners.get('exit')?.(0 as never);
await result;
assert.equal(terminationState.calls, 1);
assert.deepEqual(fallbackTasks, [{ dbPath: '/tmp/test.sqlite', task }]);
});
test('worker runtime surfaces a task failure without rerunning it', async () => {
const { worker, listeners, terminationState } = createFakeWorker();
const fallbackTasks: unknown[] = [];
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => '/tmp/delete-worker.js',
createWorker: async () => worker,
executeFallback: () => {
fallbackTasks.push('ran');
},
});
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
await new Promise<void>((resolve) => setTimeout(resolve, 0));
listeners.get('message')?.({ ok: false, error: 'constraint violated' } as never);
await assert.rejects(result, /constraint violated/);
assert.equal(terminationState.calls, 1);
assert.equal(fallbackTasks.length, 0);
});
test('worker runtime terminates a worker created after shutdown begins', async () => {
@@ -31,7 +31,13 @@ interface DeleteMaintenanceWorkerRuntimeOptions {
}
export function resolveDeleteMaintenanceWorkerPath(): string | null {
const workerPath = path.join(__dirname, 'delete-maintenance-worker-thread.js');
// When the process runs TypeScript directly (Bun from source), the emitted
// .js sibling doesn't exist — spawn the .ts module instead, which such
// runtimes transpile for workers too. Compiled layouts keep using the .js.
const fileName = __filename.endsWith('.ts')
? 'delete-maintenance-worker-thread.ts'
: 'delete-maintenance-worker-thread.js';
const workerPath = path.join(__dirname, fileName);
return fs.existsSync(workerPath) ? workerPath : null;
}
@@ -76,38 +82,61 @@ export class DeleteMaintenanceWorkerRuntime {
throw new Error('Delete maintenance worker is shut down');
}
await new Promise<void>((resolve, reject) => {
type WorkerOutcome =
| { kind: 'ok' }
| { kind: 'task-error'; detail: string }
| { kind: 'worker-failure'; error: Error };
const outcome = await new Promise<WorkerOutcome>((resolve) => {
let settled = false;
this.activeWorkers.add(worker);
const settle = (error?: Error) => {
const settle = (result: WorkerOutcome) => {
if (settled) return;
settled = true;
this.activeWorkers.delete(worker);
if (error) reject(error);
else resolve();
resolve(result);
void worker.terminate();
};
worker.once('message', (message: DeleteMaintenanceWorkerResponse) => {
if (message.ok === true) {
settle();
settle({ kind: 'ok' });
return;
}
const detail = typeof message.error === 'string' ? message.error : 'unknown worker error';
settle(new Error(`Delete maintenance failed: ${detail}`));
settle({ kind: 'task-error', detail });
});
worker.once('error', (error) => settle(error));
worker.once('error', (error) => settle({ kind: 'worker-failure', error }));
worker.once('exit', (code) => {
settle(
new Error(
settle({
kind: 'worker-failure',
error: new Error(
code === 0
? 'Delete maintenance worker exited without a response'
: `Delete maintenance worker exited with code ${code}`,
),
);
});
});
});
if (outcome.kind === 'ok') return;
// The maintenance itself failed inside the worker — rerunning it on this
// thread would hit the same error, so surface it instead.
if (outcome.kind === 'task-error') {
throw new Error(`Delete maintenance failed: ${outcome.detail}`);
}
if (this.destroyed) {
throw new Error('Delete maintenance worker is shut down');
}
// The worker died without reporting a result (failed to load, crashed).
// Its transaction rolled back with its connection, and a rerun re-plans
// against the current rows, so falling back on this thread is safe.
(this.options.warn ?? logger.warn)(
'Delete maintenance worker failed; running maintenance on the current thread',
outcome.error,
);
(this.options.executeFallback ?? executeDeleteMaintenanceTask)(dbPath, task);
}
destroy(): void {
@@ -1,6 +1,5 @@
import { Database } from './sqlite';
import { applyPragmas } from './storage';
import { deleteAnime, deleteSession, deleteSessions, deleteVideo } from './query-maintenance';
import {
deleteMaintenanceBatch,
type DeleteMaintenanceOperation,
@@ -12,35 +11,11 @@ export type DeleteMaintenanceTask =
| DeleteMaintenanceOperation
| { kind: 'batch'; tasks: DeleteMaintenanceOperation[] };
function executeDeleteMaintenanceOperation(
db: InstanceType<typeof Database>,
task: DeleteMaintenanceOperation,
): void {
switch (task.kind) {
case 'session':
deleteSession(db, task.sessionId);
return;
case 'sessions':
deleteSessions(db, task.sessionIds);
return;
case 'video':
deleteVideo(db, task.videoId);
return;
case 'anime':
deleteAnime(db, task.animeId);
return;
}
}
export function executeDeleteMaintenanceTask(dbPath: string, task: DeleteMaintenanceTask): void {
const db = new Database(dbPath);
try {
applyPragmas(db);
if (task.kind === 'batch') {
deleteMaintenanceBatch(db, task.tasks);
return;
}
executeDeleteMaintenanceOperation(db, task);
deleteMaintenanceBatch(db, task.kind === 'batch' ? task.tasks : [task]);
} finally {
db.close();
}
+476 -70
View File
@@ -1,7 +1,7 @@
import type { DatabaseSync } from './sqlite';
import { finalizeSessionRecord } from './session';
import { nowMs } from './time';
import { toDbTimestamp } from './query-shared';
import { forEachIdChunk, makePlaceholders, toDbTimestamp } from './query-shared';
import type { LifetimeRebuildSummary, SessionState } from './types';
interface TelemetryRow {
@@ -21,10 +21,8 @@ interface AnimeRow {
}
function asPositiveNumber(value: number | null, fallback: number): number {
if (value === null || !Number.isFinite(value)) {
return fallback;
}
return Math.max(0, Math.floor(value));
const resolved = value !== null && Number.isFinite(value) ? value : fallback;
return Number.isFinite(resolved) ? Math.floor(Math.max(resolved, 0)) : 0;
}
interface ExistenceRow {
@@ -68,10 +66,10 @@ const RETAINED_SESSION_METRICS_CTE = `
v.anime_id,
s.started_at_ms,
s.ended_at_ms,
MAX(COALESCE(t.active_watched_ms, s.active_watched_ms, 0), 0) AS active_ms,
MAX(COALESCE(t.cards_mined, s.cards_mined, 0), 0) AS cards_mined,
MAX(COALESCE(t.lines_seen, s.lines_seen, 0), 0) AS lines_seen,
MAX(COALESCE(t.tokens_seen, s.tokens_seen, 0), 0) AS tokens_seen,
CAST(MAX(COALESCE(t.active_watched_ms, s.active_watched_ms, 0), 0) AS INTEGER) AS active_ms,
CAST(MAX(COALESCE(t.cards_mined, s.cards_mined, 0), 0) AS INTEGER) AS cards_mined,
CAST(MAX(COALESCE(t.lines_seen, s.lines_seen, 0), 0) AS INTEGER) AS lines_seen,
CAST(MAX(COALESCE(t.tokens_seen, s.tokens_seen, 0), 0) AS INTEGER) AS tokens_seen,
CASE WHEN v.watched > 0 THEN 1 ELSE 0 END AS completed
FROM imm_sessions s
JOIN imm_videos v
@@ -599,18 +597,10 @@ export function applySessionLifetimeSummary(
.get(video.anime_id) as AnimeRow | null | undefined) ?? null)
: null;
const activeMs = telemetry
? asPositiveNumber(telemetry.active_watched_ms, session.activeWatchedMs)
: session.activeWatchedMs;
const cardsMined = telemetry
? asPositiveNumber(telemetry.cards_mined, session.cardsMined)
: session.cardsMined;
const linesSeen = telemetry
? asPositiveNumber(telemetry.lines_seen, session.linesSeen)
: session.linesSeen;
const tokensSeen = telemetry
? asPositiveNumber(telemetry.tokens_seen, session.tokensSeen)
: session.tokensSeen;
const activeMs = asPositiveNumber(telemetry?.active_watched_ms ?? null, session.activeWatchedMs);
const cardsMined = asPositiveNumber(telemetry?.cards_mined ?? null, session.cardsMined);
const linesSeen = asPositiveNumber(telemetry?.lines_seen ?? null, session.linesSeen);
const tokensSeen = asPositiveNumber(telemetry?.tokens_seen ?? null, session.tokensSeen);
const watched = video?.watched ?? 0;
const isFirstSessionForVideoRun =
mediaLifetime === null &&
@@ -708,26 +698,284 @@ export function rebuildLifetimeSummariesInTransaction(
return rebuildLifetimeSummariesInternal(db, rebuiltAtMs);
}
const LOCAL_DAY_EXPR = `CAST(
julianday(CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5
AS INTEGER
)`;
interface LifetimeMediaRemoval {
videoId: number;
sessions: number;
activeMs: number;
cards: number;
linesSeen: number;
tokensSeen: number;
}
/**
* Re-derive every per-anime lifetime row from the per-video summaries after
* episodes changed owners (merge, move, season repair).
* What a pending delete removes from the lifetime summary tables.
*
* Deliberately NOT a full rebuild: {@link rebuildLifetimeSummariesInTransaction}
* recomputes from raw sessions, which are pruned after the retention window, so
* it silently truncates lifetime history. `imm_lifetime_media` is keyed by
* video and survives repointing, so aggregating it preserves all-time totals;
* `imm_lifetime_global` only needs `anime_completed` refreshed because moving
* attribution between entries cannot change the global counters.
*
* Assumes the caller holds a write transaction; use
* {@link recomputeLifetimeAnimeAggregates} otherwise.
* Lifetime totals intentionally outlive raw-session retention, so they can
* never be rebuilt from `imm_sessions` without collapsing history to the
* retention window. Deletes instead subtract exactly what the deleted rows
* contributed: this plan is measured before the rows are removed and applied
* after.
*/
export function recomputeLifetimeAnimeAggregatesInTransaction(db: DatabaseSync): void {
const updatedAt = toDbTimestamp(nowMs());
db.exec('DELETE FROM imm_lifetime_anime');
db.prepare(
export interface LifetimeRemovalPlan {
/** Per surviving video: summed metrics of its deleted, lifetime-applied sessions. */
mediaRemovals: LifetimeMediaRemoval[];
/** Surviving anime whose lifetime rows must be recomputed from their media rows. */
affectedAnimeIds: number[];
/** Local-day keys touched by deleted applied sessions, for active_days upkeep. */
affectedDayKeys: number[];
}
export function planLifetimeRemovals(
db: DatabaseSync,
args: {
/** Every session being deleted, including ones expanded from video/anime deletes. */
deletedSessionIds: number[];
/** Deleted sessions whose video survives the delete. */
sessionIdsOnSurvivingVideos: number[];
deletedVideoIds: number[];
deletedAnimeIds: number[];
},
): LifetimeRemovalPlan {
const mediaRemovalsByVideo = new Map<number, LifetimeMediaRemoval>();
forEachIdChunk(args.sessionIdsOnSurvivingVideos, (chunk) => {
const rows = db
.prepare(
`
SELECT
s.video_id AS videoId,
COUNT(*) AS sessions,
COALESCE(SUM(CAST(MAX(COALESCE(t.active_watched_ms, s.active_watched_ms, 0), 0) AS INTEGER)), 0) AS activeMs,
COALESCE(SUM(CAST(MAX(COALESCE(t.cards_mined, s.cards_mined, 0), 0) AS INTEGER)), 0) AS cards,
COALESCE(SUM(CAST(MAX(COALESCE(t.lines_seen, s.lines_seen, 0), 0) AS INTEGER)), 0) AS linesSeen,
COALESCE(SUM(CAST(MAX(COALESCE(t.tokens_seen, s.tokens_seen, 0), 0) AS INTEGER)), 0) AS tokensSeen
FROM imm_sessions s
JOIN imm_lifetime_applied_sessions a ON a.session_id = s.session_id
LEFT JOIN imm_session_telemetry t
ON t.telemetry_id = (
SELECT telemetry_id
FROM imm_session_telemetry
WHERE session_id = s.session_id
ORDER BY sample_ms DESC, telemetry_id DESC
LIMIT 1
)
WHERE s.session_id IN (${makePlaceholders(chunk)})
GROUP BY s.video_id
`,
)
.all(...chunk) as LifetimeMediaRemoval[];
for (const row of rows) {
const existing = mediaRemovalsByVideo.get(row.videoId);
if (!existing) {
mediaRemovalsByVideo.set(row.videoId, { ...row });
continue;
}
existing.sessions += row.sessions;
existing.activeMs += row.activeMs;
existing.cards += row.cards;
existing.linesSeen += row.linesSeen;
existing.tokensSeen += row.tokensSeen;
}
});
const deletedAnimeIds = new Set(args.deletedAnimeIds);
const affectedAnimeIds = new Set<number>();
forEachIdChunk(args.deletedVideoIds, (chunk) => {
const rows = db
.prepare(
`SELECT DISTINCT anime_id AS animeId FROM imm_videos
WHERE video_id IN (${makePlaceholders(chunk)}) AND anime_id IS NOT NULL`,
)
.all(...chunk) as Array<{ animeId: number }>;
for (const row of rows) affectedAnimeIds.add(row.animeId);
});
forEachIdChunk(args.sessionIdsOnSurvivingVideos, (chunk) => {
const rows = db
.prepare(
`SELECT DISTINCT v.anime_id AS animeId
FROM imm_sessions s
JOIN imm_videos v ON v.video_id = s.video_id
WHERE s.session_id IN (${makePlaceholders(chunk)}) AND v.anime_id IS NOT NULL`,
)
.all(...chunk) as Array<{ animeId: number }>;
for (const row of rows) affectedAnimeIds.add(row.animeId);
});
for (const animeId of deletedAnimeIds) affectedAnimeIds.delete(animeId);
const affectedDayKeys = new Set<number>();
forEachIdChunk(args.deletedSessionIds, (chunk) => {
const rows = db
.prepare(
`SELECT DISTINCT ${LOCAL_DAY_EXPR} AS dayKey
FROM imm_sessions s
JOIN imm_lifetime_applied_sessions a ON a.session_id = s.session_id
WHERE s.session_id IN (${makePlaceholders(chunk)})`,
)
.all(...chunk) as Array<{ dayKey: number }>;
for (const row of rows) affectedDayKeys.add(row.dayKey);
});
return {
mediaRemovals: [...mediaRemovalsByVideo.values()],
affectedAnimeIds: [...affectedAnimeIds],
affectedDayKeys: [...affectedDayKeys],
};
}
/**
* Apply a removal plan after the underlying rows are gone.
*
* Media rows are adjusted by subtraction (pruned-session history stays intact),
* affected anime rows are recomputed from their surviving media rows, and the
* global row is re-derived from the media/anime tables. `active_days` is the
* one metric that can't be derived, so a touched day is only decremented when
* no ended session remains on that local day; days whose sessions were pruned
* by retention keep their count because pruning never subtracts.
*/
export function applyLifetimeRemovals(db: DatabaseSync, plan: LifetimeRemovalPlan): void {
const updatedAtMs = toDbTimestamp(nowMs());
const subtractMediaStmt = db.prepare(
`
INSERT INTO imm_lifetime_anime (
UPDATE imm_lifetime_media SET
total_sessions = MAX(total_sessions - ?, 0),
total_active_ms = MAX(total_active_ms - ?, 0),
total_cards = MAX(total_cards - ?, 0),
total_lines_seen = MAX(total_lines_seen - ?, 0),
total_tokens_seen = MAX(total_tokens_seen - ?, 0),
LAST_UPDATE_DATE = ?
WHERE video_id = ?
`,
);
const dropEmptyMediaStmt = db.prepare(
'DELETE FROM imm_lifetime_media WHERE video_id = ? AND total_sessions <= 0',
);
const remainingSessionRangeStmt = db.prepare(
`
SELECT
MIN(CAST(started_at_ms AS REAL)) AS minStartedMs,
MAX(CAST(ended_at_ms AS REAL)) AS maxEndedMs
FROM imm_sessions
WHERE video_id = ? AND ended_at_ms IS NOT NULL
`,
);
const storedMediaRangeStmt = db.prepare(
`
SELECT CAST(first_watched_ms AS REAL) AS firstWatchedMs
FROM imm_lifetime_media
WHERE video_id = ?
`,
);
const refreshMediaRangeStmt = db.prepare(
`
UPDATE imm_lifetime_media SET
first_watched_ms = ?,
last_watched_ms = ?
WHERE video_id = ?
`,
);
for (const removal of plan.mediaRemovals) {
subtractMediaStmt.run(
removal.sessions,
removal.activeMs,
removal.cards,
removal.linesSeen,
removal.tokensSeen,
updatedAtMs,
removal.videoId,
);
dropEmptyMediaStmt.run(removal.videoId);
const stored = storedMediaRangeStmt.get(removal.videoId) as {
firstWatchedMs: number | null;
} | null;
if (!stored) continue;
const range = remainingSessionRangeStmt.get(removal.videoId) as {
minStartedMs: number | null;
maxEndedMs: number | null;
} | null;
// Retained sessions are always newer than pruned ones, so the surviving
// range is authoritative for last_watched while first_watched can only
// keep or extend the stored (possibly pruned-history) minimum. When no
// session survives, the stored values are all that's left.
if (range && range.minStartedMs !== null && range.maxEndedMs !== null) {
const firstWatchedMs =
stored.firstWatchedMs === null
? range.minStartedMs
: Math.min(stored.firstWatchedMs, range.minStartedMs);
refreshMediaRangeStmt.run(
toDbTimestamp(firstWatchedMs),
toDbTimestamp(range.maxEndedMs),
removal.videoId,
);
}
}
recomputeLifetimeAnimeFromMedia(db, plan.affectedAnimeIds, updatedAtMs);
// One pass over the sessions rather than a probe per affected day: the local
// day is a computed expression with no index, so each probe would be a table
// scan — and the miss case (the day we need to count) is the full-scan one.
let removedDays = 0;
if (plan.affectedDayKeys.length > 0) {
const survivingDayKeys = new Set(
(
db
.prepare(
`SELECT DISTINCT ${LOCAL_DAY_EXPR} AS dayKey
FROM imm_sessions
WHERE ended_at_ms IS NOT NULL`,
)
.all() as Array<{ dayKey: number }>
).map((row) => row.dayKey),
);
for (const dayKey of plan.affectedDayKeys) {
if (!survivingDayKeys.has(dayKey)) removedDays += 1;
}
}
recomputeLifetimeGlobalFromSummaries(db, { removedActiveDays: removedDays, updatedAtMs });
}
/**
* Recompute lifetime anime rows exactly from their surviving media rows.
*
* Media rows are the durable per-video ledger (they outlive session pruning and
* follow a video when it moves between anime), so this is the correct refresh
* after merges, moves, and deletes. Anime with no media rows left are dropped.
*/
export function recomputeLifetimeAnimeFromMedia(
db: DatabaseSync,
animeIds: number[],
updatedAtMs = toDbTimestamp(nowMs()),
): void {
if (animeIds.length === 0) return;
const animeSummaryStmt = db.prepare(
`
SELECT
COUNT(*) AS episodeRows,
COALESCE(SUM(m.total_sessions), 0) AS totalSessions,
COALESCE(SUM(m.total_active_ms), 0) AS totalActiveMs,
COALESCE(SUM(m.total_cards), 0) AS totalCards,
COALESCE(SUM(m.total_lines_seen), 0) AS totalLinesSeen,
COALESCE(SUM(m.total_tokens_seen), 0) AS totalTokensSeen,
COALESCE(SUM(CASE WHEN m.completed > 0 THEN 1 ELSE 0 END), 0) AS episodesCompleted,
MIN(CAST(m.first_watched_ms AS REAL)) AS firstWatchedMs,
MAX(CAST(m.last_watched_ms AS REAL)) AS lastWatchedMs
FROM imm_lifetime_media m
JOIN imm_videos v ON v.video_id = m.video_id
WHERE v.anime_id = ?
`,
);
const dropAnimeStmt = db.prepare('DELETE FROM imm_lifetime_anime WHERE anime_id = ?');
const upsertAnimeStmt = db.prepare(
`
INSERT INTO imm_lifetime_anime(
anime_id,
total_sessions,
total_active_ms,
@@ -741,50 +989,208 @@ export function recomputeLifetimeAnimeAggregatesInTransaction(db: DatabaseSync):
CREATED_DATE,
LAST_UPDATE_DATE
)
SELECT
v.anime_id,
COALESCE(SUM(m.total_sessions), 0),
COALESCE(SUM(m.total_active_ms), 0),
COALESCE(SUM(m.total_cards), 0),
COALESCE(SUM(m.total_lines_seen), 0),
COALESCE(SUM(m.total_tokens_seen), 0),
COUNT(*),
COUNT(CASE WHEN m.completed > 0 THEN 1 END),
MIN(m.first_watched_ms),
MAX(m.last_watched_ms),
?,
?
FROM imm_lifetime_media m
JOIN imm_videos v ON v.video_id = m.video_id
WHERE v.anime_id IS NOT NULL
GROUP BY v.anime_id
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(anime_id) DO UPDATE SET
total_sessions = excluded.total_sessions,
total_active_ms = excluded.total_active_ms,
total_cards = excluded.total_cards,
total_lines_seen = excluded.total_lines_seen,
total_tokens_seen = excluded.total_tokens_seen,
episodes_started = excluded.episodes_started,
episodes_completed = excluded.episodes_completed,
first_watched_ms = excluded.first_watched_ms,
last_watched_ms = excluded.last_watched_ms,
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
`,
).run(updatedAt, updatedAt);
);
for (const animeId of animeIds) {
const summary = animeSummaryStmt.get(animeId) as {
episodeRows: number;
totalSessions: number;
totalActiveMs: number;
totalCards: number;
totalLinesSeen: number;
totalTokensSeen: number;
episodesCompleted: number;
firstWatchedMs: number | null;
lastWatchedMs: number | null;
};
if (Number(summary.episodeRows) === 0) {
dropAnimeStmt.run(animeId);
continue;
}
upsertAnimeStmt.run(
animeId,
summary.totalSessions,
summary.totalActiveMs,
summary.totalCards,
summary.totalLinesSeen,
summary.totalTokensSeen,
summary.episodeRows,
summary.episodesCompleted,
summary.firstWatchedMs === null ? null : toDbTimestamp(summary.firstWatchedMs),
summary.lastWatchedMs === null ? null : toDbTimestamp(summary.lastWatchedMs),
updatedAtMs,
updatedAtMs,
);
}
}
/**
* Re-derive the global lifetime row from the media/anime summary tables.
*
* Every global metric except active_days is a pure aggregate of those tables;
* active_days can't be derived, so callers pass how many day slots their change
* removed (0 for moves/merges, which never touch sessions).
*/
export function recomputeLifetimeGlobalFromSummaries(
db: DatabaseSync,
options: { removedActiveDays?: number; updatedAtMs?: string } = {},
): void {
const updatedAtMs = options.updatedAtMs ?? toDbTimestamp(nowMs());
const mediaTotals = db
.prepare(
`
SELECT
COUNT(*) AS episodesStarted,
COALESCE(SUM(total_sessions), 0) AS totalSessions,
COALESCE(SUM(total_active_ms), 0) AS totalActiveMs,
COALESCE(SUM(total_cards), 0) AS totalCards,
COALESCE(SUM(CASE WHEN completed > 0 THEN 1 ELSE 0 END), 0) AS episodesCompleted
FROM imm_lifetime_media
`,
)
.get() as {
episodesStarted: number;
totalSessions: number;
totalActiveMs: number;
totalCards: number;
episodesCompleted: number;
};
const animeCompletedRow = db
.prepare(
`
SELECT COUNT(*) AS animeCompleted
FROM imm_lifetime_anime la
JOIN imm_anime a ON a.anime_id = la.anime_id
WHERE a.episodes_total IS NOT NULL
AND a.episodes_total > 0
AND la.episodes_completed >= a.episodes_total
`,
)
.get() as { animeCompleted: number };
db.prepare(
`
UPDATE imm_lifetime_global
SET
anime_completed = (
SELECT COUNT(*)
FROM imm_lifetime_anime la
JOIN imm_anime a ON a.anime_id = la.anime_id
WHERE a.episodes_total IS NOT NULL
AND a.episodes_total > 0
AND la.episodes_completed >= a.episodes_total
),
UPDATE imm_lifetime_global SET
total_sessions = ?,
total_active_ms = ?,
total_cards = ?,
episodes_started = ?,
episodes_completed = ?,
anime_completed = ?,
active_days = MAX(active_days - ?, 0),
LAST_UPDATE_DATE = ?
WHERE global_id = 1
`,
).run(updatedAt);
).run(
mediaTotals.totalSessions,
mediaTotals.totalActiveMs,
mediaTotals.totalCards,
mediaTotals.episodesStarted,
mediaTotals.episodesCompleted,
animeCompletedRow.animeCompleted,
options.removedActiveDays ?? 0,
updatedAtMs,
);
}
export function recomputeLifetimeAnimeAggregates(db: DatabaseSync): void {
db.exec('BEGIN IMMEDIATE');
export interface LifetimeRepairSummary {
recomputedAnime: number;
repairedAtMs: number;
}
/**
* Refresh every anime aggregate from the durable media ledger while the caller
* holds a write transaction. This keeps merge and move operations atomic.
*/
export function recomputeLifetimeAnimeAggregatesInTransaction(db: DatabaseSync): void {
const animeIds = new Set<number>();
for (const row of db
.prepare('SELECT DISTINCT anime_id AS animeId FROM imm_videos WHERE anime_id IS NOT NULL')
.all() as Array<{ animeId: number }>) {
animeIds.add(row.animeId);
}
for (const row of db
.prepare('SELECT anime_id AS animeId FROM imm_lifetime_anime')
.all() as Array<{ animeId: number }>) {
animeIds.add(row.animeId);
}
const updatedAtMs = toDbTimestamp(nowMs());
recomputeLifetimeAnimeFromMedia(db, [...animeIds], updatedAtMs);
// Moving attribution cannot change global totals. Recompute only the one
// global metric that depends on per-anime ownership, preserving history that
// may outlive the raw sessions and media rows available for aggregation.
db.prepare(
`
UPDATE imm_lifetime_global
SET anime_completed = (
SELECT COUNT(*)
FROM imm_lifetime_anime la
JOIN imm_anime a ON a.anime_id = la.anime_id
WHERE a.episodes_total IS NOT NULL
AND a.episodes_total > 0
AND la.episodes_completed >= a.episodes_total
), LAST_UPDATE_DATE = ?
WHERE global_id = 1
`,
).run(updatedAtMs);
}
/**
* Non-destructive lifetime repair: recompute every anime row and the global row
* from the per-video media ledger.
*
* Unlike {@link rebuildLifetimeSummaries}, this never resets the tables from
* retained sessions, so lifetime history older than the session retention
* window survives. The one exception is a database whose lifetime tables were
* never populated — there is no ledger to repair from, so it bootstraps with
* the full rebuild instead.
*/
export function repairLifetimeSummariesFromMedia(db: DatabaseSync): LifetimeRepairSummary {
const repairedAtMs = nowMs();
let transactionStarted = false;
try {
recomputeLifetimeAnimeAggregatesInTransaction(db);
db.exec('BEGIN IMMEDIATE');
transactionStarted = true;
if (shouldBackfillLifetimeSummaries(db)) {
const rebuilt = rebuildLifetimeSummariesInTransaction(db, repairedAtMs);
const animeRow = db
.prepare('SELECT COUNT(*) AS count FROM imm_lifetime_anime')
.get() as ExistenceRow;
db.exec('COMMIT');
return { recomputedAnime: Number(animeRow.count), repairedAtMs: rebuilt.rebuiltAtMs };
}
const animeIds = new Set<number>();
for (const row of db
.prepare('SELECT DISTINCT anime_id AS animeId FROM imm_videos WHERE anime_id IS NOT NULL')
.all() as Array<{ animeId: number }>) {
animeIds.add(row.animeId);
}
for (const row of db
.prepare('SELECT anime_id AS animeId FROM imm_lifetime_anime')
.all() as Array<{ animeId: number }>) {
animeIds.add(row.animeId);
}
const updatedAtMs = toDbTimestamp(repairedAtMs);
recomputeLifetimeAnimeFromMedia(db, [...animeIds], updatedAtMs);
recomputeLifetimeGlobalFromSummaries(db, { updatedAtMs });
db.exec('COMMIT');
return { recomputedAnime: animeIds.size, repairedAtMs };
} catch (error) {
db.exec('ROLLBACK');
if (transactionStarted) db.exec('ROLLBACK');
throw error;
}
}
@@ -1,5 +1,5 @@
import type { DatabaseSync } from './sqlite';
import { rebuildLifetimeSummariesInTransaction } from './lifetime';
import { applyLifetimeRemovals, planLifetimeRemovals } from './lifetime';
import { getRollupGroupsForSessions, refreshRollupsForGroupsInTransaction } from './maintenance';
import {
applyLexicalRemovals,
@@ -8,9 +8,10 @@ import {
forEachIdChunk,
makePlaceholders,
planLexicalRemovalsForSessions,
SQLITE_ID_CHUNK_SIZE,
planLexicalRemovalsForVideos,
type LexicalRemovalPlan,
} from './query-shared';
import type { RollupGroup } from './maintenance';
export type DeleteMaintenanceOperation =
| { kind: 'session'; sessionId: number }
@@ -59,40 +60,72 @@ function selectIds(
return ids;
}
function planLexicalRemovalsInChunks(db: DatabaseSync, sessionIds: number[]): LexicalRemovalPlan {
const combined: LexicalRemovalPlan = { words: [], kanji: [] };
const merge = (target: LexicalRemovalPlan['words'], source: LexicalRemovalPlan['words']) => {
const byId = new Map(target.map((entry) => [entry.id, entry]));
for (const entry of source) {
const existing = byId.get(entry.id);
if (!existing) {
const added = { ...entry };
target.push(added);
byId.set(entry.id, added);
continue;
}
existing.removedFrequency += entry.removedFrequency;
if (
entry.removedFirstSeenMs !== null &&
(existing.removedFirstSeenMs === null ||
entry.removedFirstSeenMs < existing.removedFirstSeenMs)
) {
existing.removedFirstSeenMs = entry.removedFirstSeenMs;
}
if (
entry.removedLastSeenMs !== null &&
(existing.removedLastSeenMs === null ||
entry.removedLastSeenMs > existing.removedLastSeenMs)
) {
existing.removedLastSeenMs = entry.removedLastSeenMs;
}
function mergeLexicalPlanEntries(
target: LexicalRemovalPlan['words'],
byId: Map<number, LexicalRemovalPlan['words'][number]>,
source: LexicalRemovalPlan['words'],
): void {
for (const entry of source) {
const existing = byId.get(entry.id);
if (!existing) {
const added = { ...entry };
target.push(added);
byId.set(entry.id, added);
continue;
}
};
existing.removedFrequency += entry.removedFrequency;
if (
entry.removedFirstSeenMs !== null &&
(existing.removedFirstSeenMs === null ||
entry.removedFirstSeenMs < existing.removedFirstSeenMs)
) {
existing.removedFirstSeenMs = entry.removedFirstSeenMs;
}
if (
entry.removedLastSeenMs !== null &&
(existing.removedLastSeenMs === null || entry.removedLastSeenMs > existing.removedLastSeenMs)
) {
existing.removedLastSeenMs = entry.removedLastSeenMs;
}
}
}
forEachIdChunk(sessionIds, (chunk) => {
const plan = planLexicalRemovalsForSessions(db, chunk);
merge(combined.words, plan.words);
merge(combined.kanji, plan.kanji);
interface LexicalPlanEntryMaps {
words: Map<number, LexicalRemovalPlan['words'][number]>;
kanji: Map<number, LexicalRemovalPlan['kanji'][number]>;
}
function mergeLexicalPlans(
target: LexicalRemovalPlan,
byId: LexicalPlanEntryMaps,
source: LexicalRemovalPlan,
): void {
mergeLexicalPlanEntries(target.words, byId.words, source.words);
mergeLexicalPlanEntries(target.kanji, byId.kanji, source.kanji);
}
/**
* Plan what the delete removes from imm_words/imm_kanji.
*
* Deleted videos are planned by video so orphaned subtitle lines (whose session
* is already gone) still get subtracted; sessions on surviving videos are
* planned by session. The two scopes are disjoint, so nothing is counted twice.
*/
function planLexicalRemovalsForDelete(
db: DatabaseSync,
sessionIdsOnSurvivingVideos: number[],
videoIds: number[],
): LexicalRemovalPlan {
const combined: LexicalRemovalPlan = { words: [], kanji: [] };
const byId: LexicalPlanEntryMaps = {
words: new Map(),
kanji: new Map(),
};
forEachIdChunk(sessionIdsOnSurvivingVideos, (chunk) => {
mergeLexicalPlans(combined, byId, planLexicalRemovalsForSessions(db, chunk));
});
forEachIdChunk(videoIds, (chunk) => {
mergeLexicalPlans(combined, byId, planLexicalRemovalsForVideos(db, chunk));
});
return combined;
}
@@ -121,24 +154,37 @@ export function deleteMaintenanceBatch(
}
const videoIdList = [...videoIds];
for (const sessionId of selectIds(
db,
(placeholders) => `SELECT session_id FROM imm_sessions WHERE video_id IN (${placeholders})`,
videoIdList,
'session_id',
)) {
sessionIds.add(sessionId);
}
const sessionIdsOnDeletedVideos = new Set(
selectIds(
db,
(placeholders) => `SELECT session_id FROM imm_sessions WHERE video_id IN (${placeholders})`,
videoIdList,
'session_id',
),
);
for (const sessionId of sessionIdsOnDeletedVideos) sessionIds.add(sessionId);
const sessionIdList = [...sessionIds];
const lexicalRemovals = planLexicalRemovalsInChunks(db, sessionIdList);
const affectedRollupGroups = sessionIdList
.flatMap((_, index) =>
index % SQLITE_ID_CHUNK_SIZE === 0
? getRollupGroupsForSessions(db, sessionIdList.slice(index, index + SQLITE_ID_CHUNK_SIZE))
: [],
)
.filter((group) => !videoIds.has(group.videoId));
const sessionIdsOnSurvivingVideos = sessionIdList.filter(
(sessionId) => !sessionIdsOnDeletedVideos.has(sessionId),
);
// Both plans must be measured before any rows are removed.
const lexicalRemovals = planLexicalRemovalsForDelete(
db,
sessionIdsOnSurvivingVideos,
videoIdList,
);
const lifetimeRemovals = planLifetimeRemovals(db, {
deletedSessionIds: sessionIdList,
sessionIdsOnSurvivingVideos,
deletedVideoIds: videoIdList,
deletedAnimeIds: animeIdList,
});
const affectedRollupGroups: RollupGroup[] = [];
forEachIdChunk(sessionIdsOnSurvivingVideos, (chunk) => {
affectedRollupGroups.push(...getRollupGroupsForSessions(db, chunk));
});
const coverBlobHashes = new Set<string>();
if (videoIdList.length > 0) {
forEachIdChunk(videoIdList, (chunk) => {
@@ -152,26 +198,31 @@ export function deleteMaintenanceBatch(
.all(...chunk) as Array<{ coverBlobHash: string }>;
for (const row of artRows) coverBlobHashes.add(row.coverBlobHash);
});
deleteSessionsByIds(db, sessionIdList);
forEachIdChunk(videoIdList, (chunk) => {
const placeholders = makePlaceholders(chunk);
db.prepare(`DELETE FROM imm_subtitle_lines WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_daily_rollups WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_monthly_rollups WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_media_art WHERE video_id IN (${placeholders})`).run(...chunk);
db.prepare(`DELETE FROM imm_videos WHERE video_id IN (${placeholders})`).run(...chunk);
});
} else {
deleteSessionsByIds(db, sessionIdList);
}
deleteSessionsByIds(db, sessionIdList);
forEachIdChunk(sessionIdList, (chunk) => {
const placeholders = makePlaceholders(chunk);
db.prepare(
`DELETE FROM imm_lifetime_applied_sessions WHERE session_id IN (${placeholders})`,
).run(...chunk);
});
forEachIdChunk(videoIdList, (chunk) => {
const placeholders = makePlaceholders(chunk);
db.prepare(`DELETE FROM imm_subtitle_lines WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_daily_rollups WHERE video_id IN (${placeholders})`).run(...chunk);
db.prepare(`DELETE FROM imm_monthly_rollups WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_media_art WHERE video_id IN (${placeholders})`).run(...chunk);
db.prepare(`DELETE FROM imm_lifetime_media WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_videos WHERE video_id IN (${placeholders})`).run(...chunk);
});
for (const coverBlobHash of coverBlobHashes) {
cleanupUnusedCoverArtBlobHash(db, coverBlobHash);
}
@@ -186,7 +237,7 @@ export function deleteMaintenanceBatch(
}
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
applyLifetimeRemovals(db, lifetimeRemovals);
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
db.exec('COMMIT');
} catch (error) {
@@ -2,22 +2,20 @@ import { createHash } from 'node:crypto';
import type { DatabaseSync } from './sqlite';
import { buildCoverBlobReference, normalizeCoverBlobBytes } from './storage';
import {
recomputeLifetimeAnimeAggregates,
rebuildLifetimeSummariesInTransaction,
recomputeLifetimeAnimeFromMedia,
recomputeLifetimeGlobalFromSummaries,
repairLifetimeSummariesFromMedia,
shouldBackfillLifetimeSummaries,
} from './lifetime';
import { getRollupGroupsForSessions, refreshRollupsForGroupsInTransaction } from './maintenance';
import { nowMs } from './time';
import { resolveAnimeAnilistConflict } from './anime-season-repair';
import { deleteMaintenanceBatch } from './query-delete-maintenance';
import { PartOfSpeech, type MergedToken } from '../../../types';
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
import { deriveStoredPartOfSpeech } from '../tokenizer/part-of-speech';
import {
applyLexicalRemovals,
cleanupUnusedCoverArtBlobHash,
deleteSessionsByIds,
findSharedCoverBlobHash,
planLexicalRemovalsForSessions,
planLexicalRemovalsForVideos,
toDbMs,
toDbTimestamp,
} from './query-shared';
@@ -462,8 +460,14 @@ export function updateAnimeAnilistInfo(
toDbTimestamp(nowMs()),
targetRow.anime_id,
);
if (repair.movedVideos > 0 || repair.deletedAnimeRows > 0) {
recomputeLifetimeAnimeAggregates(db);
if (shouldBackfillLifetimeSummaries(db)) {
repairLifetimeSummariesFromMedia(db);
} else {
const affectedAnimeIds = new Set(repair.affectedAnimeIds);
affectedAnimeIds.add(row.anime_id);
affectedAnimeIds.add(targetRow.anime_id);
recomputeLifetimeAnimeFromMedia(db, [...affectedAnimeIds]);
recomputeLifetimeGlobalFromSummaries(db);
}
}
@@ -490,136 +494,22 @@ export function isVideoWatched(db: DatabaseSync, videoId: number): boolean {
}
export function deleteSession(db: DatabaseSync, sessionId: number): void {
const sessionIds = [sessionId];
db.exec('BEGIN IMMEDIATE');
try {
// Measured inside the write lock: the plan records what the delete removes,
// and applying a plan taken against a different snapshot would subtract the
// wrong totals from imm_words/imm_kanji.
const lexicalRemovals = planLexicalRemovalsForSessions(db, sessionIds);
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
deleteSessionsByIds(db, sessionIds);
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId }]);
}
export function deleteSessions(db: DatabaseSync, sessionIds: number[]): void {
if (sessionIds.length === 0) return;
db.exec('BEGIN IMMEDIATE');
try {
const lexicalRemovals = planLexicalRemovalsForSessions(db, sessionIds);
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
deleteSessionsByIds(db, sessionIds);
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
deleteMaintenanceBatch(db, [{ kind: 'sessions', sessionIds }]);
}
/**
* Delete an entire library entry: every episode of the anime, all of their
* sessions and derived stats, and the anime row itself.
*
* Mirrors {@link deleteVideo} per episode, but batches the lexical refresh and
* lifetime rebuild into a single transaction so a multi-episode title doesn't
* pay for one full rebuild per episode.
*/
export function deleteAnime(db: DatabaseSync, animeId: number): void {
db.exec('BEGIN IMMEDIATE');
try {
const videoIds = (
db.prepare('SELECT video_id FROM imm_videos WHERE anime_id = ?').all(animeId) as Array<{
video_id: number;
}>
).map((row) => row.video_id);
const lexicalRemovals = planLexicalRemovalsForVideos(db, videoIds);
const coverBlobHashes: string[] = [];
const sessionIds: number[] = [];
for (const videoId of videoIds) {
const artRow = db
.prepare('SELECT cover_blob_hash AS coverBlobHash FROM imm_media_art WHERE video_id = ?')
.get(videoId) as { coverBlobHash: string | null } | undefined;
if (artRow?.coverBlobHash) {
coverBlobHashes.push(artRow.coverBlobHash);
}
const sessions = db
.prepare('SELECT session_id FROM imm_sessions WHERE video_id = ?')
.all(videoId) as Array<{ session_id: number }>;
sessionIds.push(...sessions.map((session) => session.session_id));
}
deleteSessionsByIds(db, sessionIds);
const deleteLinesStmt = db.prepare('DELETE FROM imm_subtitle_lines WHERE video_id = ?');
const deleteDailyStmt = db.prepare('DELETE FROM imm_daily_rollups WHERE video_id = ?');
const deleteMonthlyStmt = db.prepare('DELETE FROM imm_monthly_rollups WHERE video_id = ?');
const deleteArtStmt = db.prepare('DELETE FROM imm_media_art WHERE video_id = ?');
const deleteVideoStmt = db.prepare('DELETE FROM imm_videos WHERE video_id = ?');
for (const videoId of videoIds) {
deleteLinesStmt.run(videoId);
deleteDailyStmt.run(videoId);
deleteMonthlyStmt.run(videoId);
deleteArtStmt.run(videoId);
deleteVideoStmt.run(videoId);
}
for (const coverBlobHash of new Set(coverBlobHashes)) {
cleanupUnusedCoverArtBlobHash(db, coverBlobHash);
}
db.prepare('DELETE FROM imm_lifetime_anime WHERE anime_id = ?').run(animeId);
db.prepare('DELETE FROM imm_anime WHERE anime_id = ?').run(animeId);
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
deleteMaintenanceBatch(db, [{ kind: 'anime', animeId }]);
}
export function deleteVideo(db: DatabaseSync, videoId: number): void {
db.exec('BEGIN IMMEDIATE');
try {
const artRow = db
.prepare(
`
SELECT cover_blob_hash AS coverBlobHash
FROM imm_media_art
WHERE video_id = ?
`,
)
.get(videoId) as { coverBlobHash: string | null } | undefined;
const lexicalRemovals = planLexicalRemovalsForVideos(db, [videoId]);
const sessions = db
.prepare('SELECT session_id FROM imm_sessions WHERE video_id = ?')
.all(videoId) as Array<{ session_id: number }>;
deleteSessionsByIds(
db,
sessions.map((session) => session.session_id),
);
db.prepare('DELETE FROM imm_subtitle_lines WHERE video_id = ?').run(videoId);
db.prepare('DELETE FROM imm_daily_rollups WHERE video_id = ?').run(videoId);
db.prepare('DELETE FROM imm_monthly_rollups WHERE video_id = ?').run(videoId);
db.prepare('DELETE FROM imm_media_art WHERE video_id = ?').run(videoId);
cleanupUnusedCoverArtBlobHash(db, artRow?.coverBlobHash ?? null);
db.prepare('DELETE FROM imm_videos WHERE video_id = ?').run(videoId);
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
deleteMaintenanceBatch(db, [{ kind: 'video', videoId }]);
}
@@ -307,10 +307,13 @@ function toStoredSeenSeconds(ms: number | null): number | null {
* Apply a removal plan to the vocabulary aggregates.
*
* Frequencies are adjusted by subtraction, which is exact and touches only the
* affected rows. `first_seen`/`last_seen` only need a rescan when the removed
* lines held the current extreme, and rows whose frequency reaches zero are
* verified against the surviving occurrences before deletion — so stored counts
* that have drifted still converge on the truth instead of dropping a live row.
* affected rows. When the removed lines held a `first_seen`/`last_seen`
* extreme, the new extremes come from MIN/MAX index-endpoint seeks on the
* occurrence covering index — never a re-aggregation of every occurrence, which
* for common particles means scanning the whole library. The full re-aggregate
* survives only as the repair path: rows whose stored frequency reaches zero
* while occurrences remain (drift), and rows with undated pre-migration
* occurrences the seeks would skip.
*/
export function applyLexicalRemovals(db: DatabaseSync, plan: LexicalRemovalPlan): void {
applyRemovalsForEntity(db, 'word', plan.words);
@@ -339,6 +342,21 @@ function applyRemovalsForEntity(
`SELECT 1 AS found FROM ${occurrenceTable} WHERE ${col} = ? LIMIT 1`,
);
const deleteStmt = db.prepare(`DELETE FROM ${entityTable} WHERE id = ?`);
// Seeks to the front of this entity's index range, where NULL seen_ms sorts.
const hasUndatedOccurrenceStmt = db.prepare(
`SELECT 1 AS found FROM ${occurrenceTable} WHERE ${col} = ? AND seen_ms IS NULL LIMIT 1`,
);
// Kept as separate single-aggregate statements so SQLite's min/max
// optimization turns each into an index-endpoint seek instead of a scan.
const minSeenStmt = db.prepare(
`SELECT MIN(seen_ms) AS value FROM ${occurrenceTable} WHERE ${col} = ?`,
);
const maxSeenStmt = db.prepare(
`SELECT MAX(seen_ms) AS value FROM ${occurrenceTable} WHERE ${col} = ?`,
);
const updateAggregatesStmt = db.prepare(
`UPDATE ${entityTable} SET frequency = ?, first_seen = ?, last_seen = ? WHERE id = ?`,
);
const needsExactRefresh: number[] = [];
@@ -371,7 +389,26 @@ function applyRemovalsForEntity(
current.lastSeen === null ||
(removedLastSeen !== null && removedLastSeen >= current.lastSeen);
if (firstSeenMayHaveMoved || lastSeenMayHaveMoved) {
needsExactRefresh.push(removal.id);
// Undated pre-migration occurrences are invisible to the seeks below;
// fall back to the full re-aggregate that resolves their dates.
if (hasUndatedOccurrenceStmt.get(removal.id)) {
needsExactRefresh.push(removal.id);
continue;
}
const minSeenMs = (minSeenStmt.get(removal.id) as { value: number | null }).value;
const maxSeenMs = (maxSeenStmt.get(removal.id) as { value: number | null }).value;
if (minSeenMs === null || maxSeenMs === null) {
// Frequency says occurrences remain but none exist: stale row, let the
// exact refresh reconcile (it deletes rows with nothing left).
needsExactRefresh.push(removal.id);
continue;
}
updateAggregatesStmt.run(
nextFrequency,
Math.floor(Number(minSeenMs) / 1000),
Math.floor(Number(maxSeenMs) / 1000),
removal.id,
);
continue;
}
@@ -715,6 +715,37 @@ test('ensureSchema creates large-history performance indexes', () => {
}
});
test('ensureSchema adds the subtitle event index to current-version databases', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
db.exec('DROP INDEX idx_subtitle_lines_event_id');
ensureSchema(db);
const indexes = db.prepare("PRAGMA index_list('imm_subtitle_lines')").all() as Array<{
name: string;
partial: number;
}>;
const eventIndex = indexes.find(({ name }) => name === 'idx_subtitle_lines_event_id');
assert.ok(eventIndex);
assert.equal(eventIndex.partial, 1);
const columns = db.prepare("PRAGMA index_info('idx_subtitle_lines_event_id')").all() as Array<{
name: string;
}>;
assert.deepEqual(
columns.map(({ name }) => name),
['event_id'],
);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('ensureSchema migrates legacy videos and backfills anime metadata from filenames', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -555,6 +555,16 @@ function ensureAnimeMergeTables(db: DatabaseSync): void {
`);
}
function ensureSubtitleLineEventIndex(db: DatabaseSync): void {
// The event_id FK is ON DELETE SET NULL; without this index every deleted
// imm_session_events row full-scans imm_subtitle_lines looking for children,
// which made session deletes take minutes on large databases.
db.exec(`
CREATE INDEX IF NOT EXISTS idx_subtitle_lines_event_id
ON imm_subtitle_lines(event_id) WHERE event_id IS NOT NULL
`);
}
export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput): number {
const seasonScope = normalizeSeasonScope(input.seasonScope);
const identityTitle = buildSeasonScopedAnimeTitle(input.parsedTitle, seasonScope);
@@ -888,6 +898,7 @@ export function ensureSchema(db: DatabaseSync): void {
ensureLifetimeSummaryTables(db);
ensureStatsExcludedWordsTable(db);
ensureAnimeMergeTables(db);
ensureSubtitleLineEventIndex(db);
return;
}
@@ -1525,6 +1536,7 @@ export function ensureSchema(db: DatabaseSync): void {
CREATE INDEX IF NOT EXISTS idx_subtitle_lines_anime_line
ON imm_subtitle_lines(anime_id, line_index)
`);
ensureSubtitleLineEventIndex(db);
if (currentVersion?.schema_version && currentVersion.schema_version < 19) {
backfillLexicalOccurrenceSeenMs(db);
}