fix(stats): subtract lifetime totals incrementally on delete

- Preserve lifetime history beyond session retention during deletes and repairs
- Fall back to the current thread when the delete worker fails to load
This commit is contained in:
2026-08-14 00:27:57 -07:00
parent b98d4d65c7
commit de5b1b5d10
14 changed files with 1161 additions and 303 deletions
@@ -329,3 +329,28 @@ test('upgrading an older database backfills seen_ms from the subtitle lines', ()
cleanupDbPath(dbPath);
}
});
test('an extreme-moving delete keeps subtraction-exact frequency instead of re-summing', () => {
const { db, dbPath } = createDb([
{ session: 1, wordId: 7, dayOffset: 0, count: 2 },
{ session: 2, wordId: 7, dayOffset: 3, count: 1 },
]);
try {
// Simulate drift: the stored total is higher than the occurrences justify.
// The extremes move via index seeks while the count stays a pure
// subtraction; drifted counts reconcile only at the zero-crossing repair
// or via the cleanup command, never by rescanning every occurrence here.
db.prepare('UPDATE imm_words SET frequency = 10 WHERE id = 7').run();
deleteSession(db, 1);
const word = readWord(db, 7);
assert.equal(word?.frequency, 10 - 2);
assert.equal(word?.firstSeen, Math.floor((BASE_MS + 3 * DAY_MS) / 1000));
assert.equal(word?.lastSeen, Math.floor((BASE_MS + 3 * DAY_MS) / 1000));
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
@@ -0,0 +1,389 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { Database } from '../sqlite.js';
import type { DatabaseSync } from '../sqlite.js';
import {
applyPragmas,
ensureSchema,
getOrCreateAnimeRecord,
getOrCreateVideoRecord,
linkVideoToAnimeRecord,
} from '../storage.js';
import { startSessionRecord } from '../session.js';
import { rebuildLifetimeSummaries, repairLifetimeSummariesFromMedia } from '../lifetime.js';
import { deleteMaintenanceBatch } from '../query-delete-maintenance.js';
import { toDbTimestamp } from '../query-shared.js';
const SOURCE_TYPE_LOCAL = 1;
const DAY_MS = 86_400_000;
// Noon UTC keeps every seeded timestamp on the same local day regardless of
// the timezone the test host runs in.
const BASE_MS = Date.UTC(2026, 0, 5, 12, 0, 0);
function createDb(): DatabaseSync {
const db = new Database(':memory:');
applyPragmas(db);
ensureSchema(db);
return db;
}
function seedAnime(db: DatabaseSync, title: string, episodesTotal: number | null): number {
const animeId = getOrCreateAnimeRecord(db, {
parsedTitle: title,
canonicalTitle: title,
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
if (episodesTotal !== null) {
db.prepare('UPDATE imm_anime SET episodes_total = ? WHERE anime_id = ?').run(
episodesTotal,
animeId,
);
}
return animeId;
}
function seedVideo(
db: DatabaseSync,
animeId: number | null,
name: string,
options: { watched?: boolean } = {},
): number {
const videoId = getOrCreateVideoRecord(db, `local:/tmp/${name}.mkv`, {
canonicalTitle: name,
sourcePath: `/tmp/${name}.mkv`,
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
if (animeId !== null) {
linkVideoToAnimeRecord(db, videoId, {
animeId,
parsedBasename: `${name}.mkv`,
parsedTitle: name,
parsedSeason: 1,
parsedEpisode: 1,
parserSource: 'test',
parserConfidence: 1,
parseMetadataJson: null,
});
}
if (options.watched) {
db.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?').run(videoId);
}
return videoId;
}
function seedEndedSession(
db: DatabaseSync,
videoId: number,
startedAtMs: number,
metrics: { activeMs: number; cards?: number; lines?: number; tokens?: number },
): number {
const sessionId = startSessionRecord(db, videoId, startedAtMs).sessionId;
db.prepare(
`
UPDATE imm_sessions SET
ended_at_ms = ?,
active_watched_ms = ?,
total_watched_ms = ?,
cards_mined = ?,
lines_seen = ?,
tokens_seen = ?
WHERE session_id = ?
`,
).run(
toDbTimestamp(startedAtMs + metrics.activeMs),
metrics.activeMs,
metrics.activeMs,
metrics.cards ?? 0,
metrics.lines ?? 0,
metrics.tokens ?? 0,
sessionId,
);
return sessionId;
}
// libsql attaches a per-query `_metadata` property to result rows; strip it so
// row snapshots can be compared with deepEqual.
function cleanRow<T>(row: unknown): T {
const { _metadata: _ignored, ...rest } = row as Record<string, unknown>;
return rest as T;
}
interface GlobalSnapshot {
total_sessions: number;
total_active_ms: number;
total_cards: number;
active_days: number;
episodes_started: number;
episodes_completed: number;
anime_completed: number;
}
function snapshotGlobal(db: DatabaseSync): GlobalSnapshot {
const row = db
.prepare(
`SELECT total_sessions, total_active_ms, total_cards, active_days,
episodes_started, episodes_completed, anime_completed
FROM imm_lifetime_global WHERE global_id = 1`,
)
.get();
return cleanRow<GlobalSnapshot>(row);
}
function snapshotMedia(db: DatabaseSync): unknown[] {
return db
.prepare(
`SELECT video_id, total_sessions, total_active_ms, total_cards,
total_lines_seen, total_tokens_seen, completed,
CAST(first_watched_ms AS REAL) AS first_watched,
CAST(last_watched_ms AS REAL) AS last_watched
FROM imm_lifetime_media ORDER BY video_id`,
)
.all()
.map((row) => cleanRow(row));
}
function snapshotAnime(db: DatabaseSync): unknown[] {
return db
.prepare(
`SELECT anime_id, total_sessions, total_active_ms, total_cards,
total_lines_seen, total_tokens_seen, episodes_started, episodes_completed,
CAST(first_watched_ms AS REAL) AS first_watched,
CAST(last_watched_ms AS REAL) AS last_watched
FROM imm_lifetime_anime ORDER BY anime_id`,
)
.all()
.map((row) => cleanRow(row));
}
test('incremental delete maintenance matches a full rebuild when no history is pruned', () => {
const db = createDb();
try {
const animeA = seedAnime(db, 'Anime A', 2);
const animeB = seedAnime(db, 'Anime B', 1);
const videoA1 = seedVideo(db, animeA, 'anime-a-ep1', { watched: true });
const videoA2 = seedVideo(db, animeA, 'anime-a-ep2', { watched: true });
const videoB1 = seedVideo(db, animeB, 'anime-b-ep1', { watched: true });
const videoLoose = seedVideo(db, null, 'loose-video');
seedEndedSession(db, videoA1, BASE_MS, { activeMs: 60_000, cards: 2, lines: 30, tokens: 200 });
const deletedSessionId = seedEndedSession(db, videoA1, BASE_MS + DAY_MS, {
activeMs: 45_000,
cards: 1,
lines: 20,
tokens: 100,
});
seedEndedSession(db, videoA2, BASE_MS + 2 * DAY_MS, { activeMs: 90_000, cards: 3 });
seedEndedSession(db, videoB1, BASE_MS + 3 * DAY_MS, { activeMs: 30_000 });
seedEndedSession(db, videoLoose, BASE_MS + 4 * DAY_MS, { activeMs: 15_000 });
rebuildLifetimeSummaries(db);
deleteMaintenanceBatch(db, [
{ kind: 'session', sessionId: deletedSessionId },
{ kind: 'video', videoId: videoLoose },
{ kind: 'anime', animeId: animeB },
]);
const incrementalGlobal = snapshotGlobal(db);
const incrementalMedia = snapshotMedia(db);
const incrementalAnime = snapshotAnime(db);
// With every session still retained, subtracting must land on exactly the
// state a from-scratch rebuild computes.
rebuildLifetimeSummaries(db);
assert.deepEqual(incrementalGlobal, snapshotGlobal(db));
assert.deepEqual(incrementalMedia, snapshotMedia(db));
assert.deepEqual(incrementalAnime, snapshotAnime(db));
} finally {
db.close();
}
});
test('deleting a retained session preserves lifetime history from pruned sessions', () => {
const db = createDb();
try {
const animeId = seedAnime(db, 'Pruned Anime', null);
const videoId = seedVideo(db, animeId, 'pruned-ep1');
const prunedSessionId = seedEndedSession(db, videoId, BASE_MS, {
activeMs: 120_000,
cards: 4,
lines: 50,
tokens: 400,
});
const retainedSessionId = seedEndedSession(db, videoId, BASE_MS + DAY_MS, {
activeMs: 30_000,
cards: 1,
lines: 10,
tokens: 80,
});
rebuildLifetimeSummaries(db);
// Simulate raw-session retention pruning the older session. Lifetime
// summaries intentionally keep its contribution.
db.prepare('DELETE FROM imm_sessions WHERE session_id = ?').run(prunedSessionId);
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: retainedSessionId }]);
const globalRow = snapshotGlobal(db);
assert.equal(globalRow.total_sessions, 1, 'pruned session contribution survives the delete');
assert.equal(globalRow.total_active_ms, 120_000);
assert.equal(globalRow.total_cards, 4);
assert.equal(globalRow.episodes_started, 1);
// The pruned session's day stays counted (pruning never subtracts); only
// the deleted retained session's day is dropped.
assert.equal(globalRow.active_days, 1);
const mediaRow = db
.prepare(
'SELECT total_sessions, total_active_ms, total_cards FROM imm_lifetime_media WHERE video_id = ?',
)
.get(videoId);
assert.deepEqual(
cleanRow<{ total_sessions: number; total_active_ms: number; total_cards: number }>(mediaRow),
{ total_sessions: 1, total_active_ms: 120_000, total_cards: 4 },
);
} finally {
db.close();
}
});
test('active_days only drops when the last session of a local day is deleted', () => {
const db = createDb();
try {
const videoId = seedVideo(db, null, 'same-day');
const firstSessionId = seedEndedSession(db, videoId, BASE_MS, { activeMs: 10_000 });
const secondSessionId = seedEndedSession(db, videoId, BASE_MS + 3_600_000, {
activeMs: 20_000,
});
rebuildLifetimeSummaries(db);
assert.equal(snapshotGlobal(db).active_days, 1);
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: firstSessionId }]);
assert.equal(snapshotGlobal(db).active_days, 1, 'day still has a session');
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: secondSessionId }]);
assert.equal(snapshotGlobal(db).active_days, 0, 'day lost its last session');
} finally {
db.close();
}
});
test('deleting a video updates anime and global rollups without a rebuild', () => {
const db = createDb();
try {
const animeId = seedAnime(db, 'Two Episode Anime', 2);
const videoEp1 = seedVideo(db, animeId, 'two-ep-1', { watched: true });
const videoEp2 = seedVideo(db, animeId, 'two-ep-2', { watched: true });
seedEndedSession(db, videoEp1, BASE_MS, { activeMs: 60_000, cards: 2 });
seedEndedSession(db, videoEp2, BASE_MS + DAY_MS, { activeMs: 40_000, cards: 1 });
rebuildLifetimeSummaries(db);
assert.equal(snapshotGlobal(db).anime_completed, 1);
deleteMaintenanceBatch(db, [{ kind: 'video', videoId: videoEp2 }]);
const globalRow = snapshotGlobal(db);
assert.equal(globalRow.total_sessions, 1);
assert.equal(globalRow.total_active_ms, 60_000);
assert.equal(globalRow.episodes_started, 1);
assert.equal(globalRow.episodes_completed, 1);
assert.equal(globalRow.anime_completed, 0, 'anime no longer has all episodes completed');
const animeRow = db
.prepare(
'SELECT total_sessions, episodes_started, episodes_completed FROM imm_lifetime_anime WHERE anime_id = ?',
)
.get(animeId);
assert.deepEqual(
cleanRow<{
total_sessions: number;
episodes_started: number;
episodes_completed: number;
}>(animeRow),
{ total_sessions: 1, episodes_started: 1, episodes_completed: 1 },
);
deleteMaintenanceBatch(db, [{ kind: 'video', videoId: videoEp1 }]);
assert.equal(
(db.prepare('SELECT COUNT(*) AS total FROM imm_lifetime_anime').get() as { total: number })
.total,
0,
'anime lifetime row is dropped once no episodes remain',
);
assert.deepEqual(snapshotGlobal(db), {
total_sessions: 0,
total_active_ms: 0,
total_cards: 0,
active_days: 0,
episodes_started: 0,
episodes_completed: 0,
anime_completed: 0,
});
} finally {
db.close();
}
});
test('repair after a video moves between anime matches a full rebuild', () => {
const db = createDb();
try {
const animeA = seedAnime(db, 'Move Source', 2);
const animeB = seedAnime(db, 'Move Target', 2);
const movedVideo = seedVideo(db, animeA, 'moved-ep', { watched: true });
const stayingVideo = seedVideo(db, animeA, 'staying-ep');
const targetVideo = seedVideo(db, animeB, 'target-ep', { watched: true });
seedEndedSession(db, movedVideo, BASE_MS, { activeMs: 60_000, cards: 2 });
seedEndedSession(db, stayingVideo, BASE_MS + DAY_MS, { activeMs: 30_000 });
seedEndedSession(db, targetVideo, BASE_MS + 2 * DAY_MS, { activeMs: 45_000, cards: 1 });
rebuildLifetimeSummaries(db);
// Simulate a library merge reassigning the episode to the other anime.
db.prepare('UPDATE imm_videos SET anime_id = ? WHERE video_id = ?').run(animeB, movedVideo);
repairLifetimeSummariesFromMedia(db);
const repairedGlobal = snapshotGlobal(db);
const repairedMedia = snapshotMedia(db);
const repairedAnime = snapshotAnime(db);
rebuildLifetimeSummaries(db);
assert.deepEqual(repairedGlobal, snapshotGlobal(db));
assert.deepEqual(repairedMedia, snapshotMedia(db));
assert.deepEqual(repairedAnime, snapshotAnime(db));
} finally {
db.close();
}
});
test('repair preserves lifetime history from pruned sessions where a rebuild would not', () => {
const db = createDb();
try {
const animeId = seedAnime(db, 'Repair Anime', null);
const videoId = seedVideo(db, animeId, 'repair-ep');
const prunedSessionId = seedEndedSession(db, videoId, BASE_MS, {
activeMs: 90_000,
cards: 3,
});
seedEndedSession(db, videoId, BASE_MS + DAY_MS, { activeMs: 30_000, cards: 1 });
rebuildLifetimeSummaries(db);
db.prepare('DELETE FROM imm_sessions WHERE session_id = ?').run(prunedSessionId);
repairLifetimeSummariesFromMedia(db);
const globalRow = snapshotGlobal(db);
assert.equal(globalRow.total_sessions, 2, 'repair keeps the pruned session contribution');
assert.equal(globalRow.total_active_ms, 120_000);
assert.equal(globalRow.total_cards, 4);
assert.equal(globalRow.active_days, 2, 'repair never subtracts active days');
const animeRow = db
.prepare('SELECT total_sessions FROM imm_lifetime_anime WHERE anime_id = ?')
.get(animeId);
assert.equal(cleanRow<{ total_sessions: number }>(animeRow).total_sessions, 2);
} finally {
db.close();
}
});
@@ -32,7 +32,7 @@ function createFakeWorker() {
type FakeWorker = ReturnType<typeof createFakeWorker>['worker'];
test('a delete batch rebuilds lifetime summaries once', () => {
test('a delete batch never runs a full lifetime rebuild', () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-delete-batch-test-'));
const dbPath = path.join(tempDir, 'immersion.sqlite');
let db = new Database(dbPath);
@@ -87,8 +87,8 @@ test('a delete batch rebuilds lifetime summaries once', () => {
assert.equal(deletedVideo, undefined);
assert.equal(
audit.total,
2,
'one rebuild performs exactly its reset and final global summary writes',
0,
'delete maintenance subtracts incrementally instead of rewriting last_rebuilt_ms',
);
} finally {
try {
@@ -100,6 +100,14 @@ test('a delete batch rebuilds lifetime summaries once', () => {
}
});
test('delete worker module resolves in the current layout', () => {
// If this resolves to null, every delete silently runs on the serving thread
// and blocks the stats API for the whole maintenance run.
const workerPath = resolveDeleteMaintenanceWorkerPath();
assert.ok(workerPath, 'delete-maintenance worker module must resolve');
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
});
test(
'compiled delete worker removes data through its separate database connection',
{ skip: resolveDeleteMaintenanceWorkerPath() === null },
@@ -178,19 +186,51 @@ test('worker runtime terminates a worker after successful settlement', async ()
assert.equal(terminationState.calls, 1);
});
test('worker runtime terminates a worker after failed settlement', async () => {
test('worker runtime falls back to the current thread when the worker crashes', async () => {
const { worker, listeners, terminationState } = createFakeWorker();
const fallbackTasks: unknown[] = [];
const warnings: string[] = [];
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => '/tmp/delete-worker.js',
createWorker: async () => worker,
executeFallback: (dbPath, task) => {
fallbackTasks.push({ dbPath, task });
},
warn: (message) => {
warnings.push(message);
},
});
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
await new Promise<void>((resolve) => setTimeout(resolve, 0));
listeners.get('error')?.(new Error('worker failed') as never);
await assert.rejects(result, /worker failed/);
await result;
assert.equal(terminationState.calls, 1);
assert.deepEqual(fallbackTasks, [
{ dbPath: '/tmp/test.sqlite', task: { kind: 'session', sessionId: 1 } },
]);
assert.equal(warnings.length, 1);
});
test('worker runtime surfaces a task failure without rerunning it', async () => {
const { worker, listeners, terminationState } = createFakeWorker();
const fallbackTasks: unknown[] = [];
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => '/tmp/delete-worker.js',
createWorker: async () => worker,
executeFallback: () => {
fallbackTasks.push('ran');
},
});
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
await new Promise<void>((resolve) => setTimeout(resolve, 0));
listeners.get('message')?.({ ok: false, error: 'constraint violated' } as never);
await assert.rejects(result, /constraint violated/);
assert.equal(terminationState.calls, 1);
assert.equal(fallbackTasks.length, 0);
});
test('worker runtime terminates a worker created after shutdown begins', async () => {
@@ -31,7 +31,13 @@ interface DeleteMaintenanceWorkerRuntimeOptions {
}
export function resolveDeleteMaintenanceWorkerPath(): string | null {
const workerPath = path.join(__dirname, 'delete-maintenance-worker-thread.js');
// When the process runs TypeScript directly (Bun from source), the emitted
// .js sibling doesn't exist — spawn the .ts module instead, which such
// runtimes transpile for workers too. Compiled layouts keep using the .js.
const fileName = __filename.endsWith('.ts')
? 'delete-maintenance-worker-thread.ts'
: 'delete-maintenance-worker-thread.js';
const workerPath = path.join(__dirname, fileName);
return fs.existsSync(workerPath) ? workerPath : null;
}
@@ -76,38 +82,61 @@ export class DeleteMaintenanceWorkerRuntime {
throw new Error('Delete maintenance worker is shut down');
}
await new Promise<void>((resolve, reject) => {
type WorkerOutcome =
| { kind: 'ok' }
| { kind: 'task-error'; detail: string }
| { kind: 'worker-failure'; error: Error };
const outcome = await new Promise<WorkerOutcome>((resolve) => {
let settled = false;
this.activeWorkers.add(worker);
const settle = (error?: Error) => {
const settle = (result: WorkerOutcome) => {
if (settled) return;
settled = true;
this.activeWorkers.delete(worker);
if (error) reject(error);
else resolve();
resolve(result);
void worker.terminate();
};
worker.once('message', (message: DeleteMaintenanceWorkerResponse) => {
if (message.ok === true) {
settle();
settle({ kind: 'ok' });
return;
}
const detail = typeof message.error === 'string' ? message.error : 'unknown worker error';
settle(new Error(`Delete maintenance failed: ${detail}`));
settle({ kind: 'task-error', detail });
});
worker.once('error', (error) => settle(error));
worker.once('error', (error) => settle({ kind: 'worker-failure', error }));
worker.once('exit', (code) => {
settle(
new Error(
settle({
kind: 'worker-failure',
error: new Error(
code === 0
? 'Delete maintenance worker exited without a response'
: `Delete maintenance worker exited with code ${code}`,
),
);
});
});
});
if (outcome.kind === 'ok') return;
// The maintenance itself failed inside the worker — rerunning it on this
// thread would hit the same error, so surface it instead.
if (outcome.kind === 'task-error') {
throw new Error(`Delete maintenance failed: ${outcome.detail}`);
}
if (this.destroyed) {
throw new Error('Delete maintenance worker is shut down');
}
// The worker died without reporting a result (failed to load, crashed).
// Its transaction rolled back with its connection, and a rerun re-plans
// against the current rows, so falling back on this thread is safe.
(this.options.warn ?? logger.warn)(
'Delete maintenance worker failed; running maintenance on the current thread',
outcome.error,
);
(this.options.executeFallback ?? executeDeleteMaintenanceTask)(dbPath, task);
}
destroy(): void {
@@ -1,6 +1,5 @@
import { Database } from './sqlite';
import { applyPragmas } from './storage';
import { deleteAnime, deleteSession, deleteSessions, deleteVideo } from './query-maintenance';
import {
deleteMaintenanceBatch,
type DeleteMaintenanceOperation,
@@ -12,35 +11,11 @@ export type DeleteMaintenanceTask =
| DeleteMaintenanceOperation
| { kind: 'batch'; tasks: DeleteMaintenanceOperation[] };
function executeDeleteMaintenanceOperation(
db: InstanceType<typeof Database>,
task: DeleteMaintenanceOperation,
): void {
switch (task.kind) {
case 'session':
deleteSession(db, task.sessionId);
return;
case 'sessions':
deleteSessions(db, task.sessionIds);
return;
case 'video':
deleteVideo(db, task.videoId);
return;
case 'anime':
deleteAnime(db, task.animeId);
return;
}
}
export function executeDeleteMaintenanceTask(dbPath: string, task: DeleteMaintenanceTask): void {
const db = new Database(dbPath);
try {
applyPragmas(db);
if (task.kind === 'batch') {
deleteMaintenanceBatch(db, task.tasks);
return;
}
executeDeleteMaintenanceOperation(db, task);
deleteMaintenanceBatch(db, task.kind === 'batch' ? task.tasks : [task]);
} finally {
db.close();
}
+458 -1
View File
@@ -1,7 +1,7 @@
import type { DatabaseSync } from './sqlite';
import { finalizeSessionRecord } from './session';
import { nowMs } from './time';
import { toDbTimestamp } from './query-shared';
import { forEachIdChunk, makePlaceholders, toDbTimestamp } from './query-shared';
import type { LifetimeRebuildSummary, SessionState } from './types';
interface TelemetryRow {
@@ -708,6 +708,463 @@ export function rebuildLifetimeSummariesInTransaction(
return rebuildLifetimeSummariesInternal(db, rebuiltAtMs);
}
const LOCAL_DAY_EXPR = `CAST(
julianday(CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5
AS INTEGER
)`;
interface LifetimeMediaRemoval {
videoId: number;
sessions: number;
activeMs: number;
cards: number;
linesSeen: number;
tokensSeen: number;
}
/**
* What a pending delete removes from the lifetime summary tables.
*
* Lifetime totals intentionally outlive raw-session retention, so they can
* never be rebuilt from `imm_sessions` without collapsing history to the
* retention window. Deletes instead subtract exactly what the deleted rows
* contributed: this plan is measured before the rows are removed and applied
* after.
*/
export interface LifetimeRemovalPlan {
/** Per surviving video: summed metrics of its deleted, lifetime-applied sessions. */
mediaRemovals: LifetimeMediaRemoval[];
/** Surviving anime whose lifetime rows must be recomputed from their media rows. */
affectedAnimeIds: number[];
/** Local-day keys touched by deleted applied sessions, for active_days upkeep. */
affectedDayKeys: number[];
}
export function planLifetimeRemovals(
db: DatabaseSync,
args: {
/** Every session being deleted, including ones expanded from video/anime deletes. */
deletedSessionIds: number[];
/** Deleted sessions whose video survives the delete. */
sessionIdsOnSurvivingVideos: number[];
deletedVideoIds: number[];
deletedAnimeIds: number[];
},
): LifetimeRemovalPlan {
const mediaRemovalsByVideo = new Map<number, LifetimeMediaRemoval>();
forEachIdChunk(args.sessionIdsOnSurvivingVideos, (chunk) => {
const rows = db
.prepare(
`
SELECT
s.video_id AS videoId,
COUNT(*) AS sessions,
COALESCE(SUM(MAX(COALESCE(t.active_watched_ms, s.active_watched_ms, 0), 0)), 0) AS activeMs,
COALESCE(SUM(MAX(COALESCE(t.cards_mined, s.cards_mined, 0), 0)), 0) AS cards,
COALESCE(SUM(MAX(COALESCE(t.lines_seen, s.lines_seen, 0), 0)), 0) AS linesSeen,
COALESCE(SUM(MAX(COALESCE(t.tokens_seen, s.tokens_seen, 0), 0)), 0) AS tokensSeen
FROM imm_sessions s
JOIN imm_lifetime_applied_sessions a ON a.session_id = s.session_id
LEFT JOIN imm_session_telemetry t
ON t.telemetry_id = (
SELECT telemetry_id
FROM imm_session_telemetry
WHERE session_id = s.session_id
ORDER BY sample_ms DESC, telemetry_id DESC
LIMIT 1
)
WHERE s.session_id IN (${makePlaceholders(chunk)})
GROUP BY s.video_id
`,
)
.all(...chunk) as LifetimeMediaRemoval[];
for (const row of rows) {
const existing = mediaRemovalsByVideo.get(row.videoId);
if (!existing) {
mediaRemovalsByVideo.set(row.videoId, { ...row });
continue;
}
existing.sessions += row.sessions;
existing.activeMs += row.activeMs;
existing.cards += row.cards;
existing.linesSeen += row.linesSeen;
existing.tokensSeen += row.tokensSeen;
}
});
const deletedAnimeIds = new Set(args.deletedAnimeIds);
const affectedAnimeIds = new Set<number>();
forEachIdChunk(args.deletedVideoIds, (chunk) => {
const rows = db
.prepare(
`SELECT DISTINCT anime_id AS animeId FROM imm_videos
WHERE video_id IN (${makePlaceholders(chunk)}) AND anime_id IS NOT NULL`,
)
.all(...chunk) as Array<{ animeId: number }>;
for (const row of rows) affectedAnimeIds.add(row.animeId);
});
forEachIdChunk(args.sessionIdsOnSurvivingVideos, (chunk) => {
const rows = db
.prepare(
`SELECT DISTINCT v.anime_id AS animeId
FROM imm_sessions s
JOIN imm_videos v ON v.video_id = s.video_id
WHERE s.session_id IN (${makePlaceholders(chunk)}) AND v.anime_id IS NOT NULL`,
)
.all(...chunk) as Array<{ animeId: number }>;
for (const row of rows) affectedAnimeIds.add(row.animeId);
});
for (const animeId of deletedAnimeIds) affectedAnimeIds.delete(animeId);
const affectedDayKeys = new Set<number>();
forEachIdChunk(args.deletedSessionIds, (chunk) => {
const rows = db
.prepare(
`SELECT DISTINCT ${LOCAL_DAY_EXPR} AS dayKey
FROM imm_sessions s
JOIN imm_lifetime_applied_sessions a ON a.session_id = s.session_id
WHERE s.session_id IN (${makePlaceholders(chunk)})`,
)
.all(...chunk) as Array<{ dayKey: number }>;
for (const row of rows) affectedDayKeys.add(row.dayKey);
});
return {
mediaRemovals: [...mediaRemovalsByVideo.values()],
affectedAnimeIds: [...affectedAnimeIds],
affectedDayKeys: [...affectedDayKeys],
};
}
/**
* Apply a removal plan after the underlying rows are gone.
*
* Media rows are adjusted by subtraction (pruned-session history stays intact),
* affected anime rows are recomputed from their surviving media rows, and the
* global row is re-derived from the media/anime tables. `active_days` is the
* one metric that can't be derived, so a touched day is only decremented when
* no ended session remains on that local day; days whose sessions were pruned
* by retention keep their count because pruning never subtracts.
*/
export function applyLifetimeRemovals(db: DatabaseSync, plan: LifetimeRemovalPlan): void {
const updatedAtMs = toDbTimestamp(nowMs());
const subtractMediaStmt = db.prepare(
`
UPDATE imm_lifetime_media SET
total_sessions = MAX(total_sessions - ?, 0),
total_active_ms = MAX(total_active_ms - ?, 0),
total_cards = MAX(total_cards - ?, 0),
total_lines_seen = MAX(total_lines_seen - ?, 0),
total_tokens_seen = MAX(total_tokens_seen - ?, 0),
LAST_UPDATE_DATE = ?
WHERE video_id = ?
`,
);
const dropEmptyMediaStmt = db.prepare(
'DELETE FROM imm_lifetime_media WHERE video_id = ? AND total_sessions <= 0',
);
const remainingSessionRangeStmt = db.prepare(
`
SELECT
MIN(CAST(started_at_ms AS REAL)) AS minStartedMs,
MAX(CAST(ended_at_ms AS REAL)) AS maxEndedMs
FROM imm_sessions
WHERE video_id = ? AND ended_at_ms IS NOT NULL
`,
);
const storedMediaRangeStmt = db.prepare(
`
SELECT CAST(first_watched_ms AS REAL) AS firstWatchedMs
FROM imm_lifetime_media
WHERE video_id = ?
`,
);
const refreshMediaRangeStmt = db.prepare(
`
UPDATE imm_lifetime_media SET
first_watched_ms = ?,
last_watched_ms = ?
WHERE video_id = ?
`,
);
for (const removal of plan.mediaRemovals) {
subtractMediaStmt.run(
removal.sessions,
removal.activeMs,
removal.cards,
removal.linesSeen,
removal.tokensSeen,
updatedAtMs,
removal.videoId,
);
dropEmptyMediaStmt.run(removal.videoId);
const stored = storedMediaRangeStmt.get(removal.videoId) as {
firstWatchedMs: number | null;
} | null;
if (!stored) continue;
const range = remainingSessionRangeStmt.get(removal.videoId) as {
minStartedMs: number | null;
maxEndedMs: number | null;
} | null;
// Retained sessions are always newer than pruned ones, so the surviving
// range is authoritative for last_watched while first_watched can only
// keep or extend the stored (possibly pruned-history) minimum. When no
// session survives, the stored values are all that's left.
if (range && range.minStartedMs !== null && range.maxEndedMs !== null) {
const firstWatchedMs =
stored.firstWatchedMs === null
? range.minStartedMs
: Math.min(stored.firstWatchedMs, range.minStartedMs);
refreshMediaRangeStmt.run(
toDbTimestamp(firstWatchedMs),
toDbTimestamp(range.maxEndedMs),
removal.videoId,
);
}
}
recomputeLifetimeAnimeFromMedia(db, plan.affectedAnimeIds, updatedAtMs);
// One pass over the sessions rather than a probe per affected day: the local
// day is a computed expression with no index, so each probe would be a table
// scan — and the miss case (the day we need to count) is the full-scan one.
let removedDays = 0;
if (plan.affectedDayKeys.length > 0) {
const survivingDayKeys = new Set(
(
db
.prepare(
`SELECT DISTINCT ${LOCAL_DAY_EXPR} AS dayKey
FROM imm_sessions
WHERE ended_at_ms IS NOT NULL`,
)
.all() as Array<{ dayKey: number }>
).map((row) => row.dayKey),
);
for (const dayKey of plan.affectedDayKeys) {
if (!survivingDayKeys.has(dayKey)) removedDays += 1;
}
}
recomputeLifetimeGlobalFromSummaries(db, { removedActiveDays: removedDays, updatedAtMs });
}
/**
* Recompute lifetime anime rows exactly from their surviving media rows.
*
* Media rows are the durable per-video ledger (they outlive session pruning and
* follow a video when it moves between anime), so this is the correct refresh
* after merges, moves, and deletes. Anime with no media rows left are dropped.
*/
export function recomputeLifetimeAnimeFromMedia(
db: DatabaseSync,
animeIds: number[],
updatedAtMs = toDbTimestamp(nowMs()),
): void {
if (animeIds.length === 0) return;
const animeSummaryStmt = db.prepare(
`
SELECT
COUNT(*) AS episodeRows,
COALESCE(SUM(m.total_sessions), 0) AS totalSessions,
COALESCE(SUM(m.total_active_ms), 0) AS totalActiveMs,
COALESCE(SUM(m.total_cards), 0) AS totalCards,
COALESCE(SUM(m.total_lines_seen), 0) AS totalLinesSeen,
COALESCE(SUM(m.total_tokens_seen), 0) AS totalTokensSeen,
COALESCE(SUM(CASE WHEN m.completed > 0 THEN 1 ELSE 0 END), 0) AS episodesCompleted,
MIN(CAST(m.first_watched_ms AS REAL)) AS firstWatchedMs,
MAX(CAST(m.last_watched_ms AS REAL)) AS lastWatchedMs
FROM imm_lifetime_media m
JOIN imm_videos v ON v.video_id = m.video_id
WHERE v.anime_id = ?
`,
);
const dropAnimeStmt = db.prepare('DELETE FROM imm_lifetime_anime WHERE anime_id = ?');
const upsertAnimeStmt = db.prepare(
`
INSERT INTO imm_lifetime_anime(
anime_id,
total_sessions,
total_active_ms,
total_cards,
total_lines_seen,
total_tokens_seen,
episodes_started,
episodes_completed,
first_watched_ms,
last_watched_ms,
CREATED_DATE,
LAST_UPDATE_DATE
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(anime_id) DO UPDATE SET
total_sessions = excluded.total_sessions,
total_active_ms = excluded.total_active_ms,
total_cards = excluded.total_cards,
total_lines_seen = excluded.total_lines_seen,
total_tokens_seen = excluded.total_tokens_seen,
episodes_started = excluded.episodes_started,
episodes_completed = excluded.episodes_completed,
first_watched_ms = excluded.first_watched_ms,
last_watched_ms = excluded.last_watched_ms,
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
`,
);
for (const animeId of animeIds) {
const summary = animeSummaryStmt.get(animeId) as {
episodeRows: number;
totalSessions: number;
totalActiveMs: number;
totalCards: number;
totalLinesSeen: number;
totalTokensSeen: number;
episodesCompleted: number;
firstWatchedMs: number | null;
lastWatchedMs: number | null;
};
if (Number(summary.episodeRows) === 0) {
dropAnimeStmt.run(animeId);
continue;
}
upsertAnimeStmt.run(
animeId,
summary.totalSessions,
summary.totalActiveMs,
summary.totalCards,
summary.totalLinesSeen,
summary.totalTokensSeen,
summary.episodeRows,
summary.episodesCompleted,
summary.firstWatchedMs === null ? null : toDbTimestamp(summary.firstWatchedMs),
summary.lastWatchedMs === null ? null : toDbTimestamp(summary.lastWatchedMs),
updatedAtMs,
updatedAtMs,
);
}
}
/**
* Re-derive the global lifetime row from the media/anime summary tables.
*
* Every global metric except active_days is a pure aggregate of those tables;
* active_days can't be derived, so callers pass how many day slots their change
* removed (0 for moves/merges, which never touch sessions).
*/
export function recomputeLifetimeGlobalFromSummaries(
db: DatabaseSync,
options: { removedActiveDays?: number; updatedAtMs?: string } = {},
): void {
const updatedAtMs = options.updatedAtMs ?? toDbTimestamp(nowMs());
const mediaTotals = db
.prepare(
`
SELECT
COUNT(*) AS episodesStarted,
COALESCE(SUM(total_sessions), 0) AS totalSessions,
COALESCE(SUM(total_active_ms), 0) AS totalActiveMs,
COALESCE(SUM(total_cards), 0) AS totalCards,
COALESCE(SUM(CASE WHEN completed > 0 THEN 1 ELSE 0 END), 0) AS episodesCompleted
FROM imm_lifetime_media
`,
)
.get() as {
episodesStarted: number;
totalSessions: number;
totalActiveMs: number;
totalCards: number;
episodesCompleted: number;
};
const animeCompletedRow = db
.prepare(
`
SELECT COUNT(*) AS animeCompleted
FROM imm_lifetime_anime la
JOIN imm_anime a ON a.anime_id = la.anime_id
WHERE a.episodes_total IS NOT NULL
AND a.episodes_total > 0
AND la.episodes_completed >= a.episodes_total
`,
)
.get() as { animeCompleted: number };
db.prepare(
`
UPDATE imm_lifetime_global SET
total_sessions = ?,
total_active_ms = ?,
total_cards = ?,
episodes_started = ?,
episodes_completed = ?,
anime_completed = ?,
active_days = MAX(active_days - ?, 0),
LAST_UPDATE_DATE = ?
WHERE global_id = 1
`,
).run(
mediaTotals.totalSessions,
mediaTotals.totalActiveMs,
mediaTotals.totalCards,
mediaTotals.episodesStarted,
mediaTotals.episodesCompleted,
animeCompletedRow.animeCompleted,
options.removedActiveDays ?? 0,
updatedAtMs,
);
}
export interface LifetimeRepairSummary {
recomputedAnime: number;
repairedAtMs: number;
}
/**
* Non-destructive lifetime repair: recompute every anime row and the global row
* from the per-video media ledger.
*
* Unlike {@link rebuildLifetimeSummaries}, this never resets the tables from
* retained sessions, so lifetime history older than the session retention
* window survives. The one exception is a database whose lifetime tables were
* never populated — there is no ledger to repair from, so it bootstraps with
* the full rebuild instead.
*/
export function repairLifetimeSummariesFromMedia(db: DatabaseSync): LifetimeRepairSummary {
if (shouldBackfillLifetimeSummaries(db)) {
const rebuilt = rebuildLifetimeSummaries(db);
const animeRow = db
.prepare('SELECT COUNT(*) AS count FROM imm_lifetime_anime')
.get() as ExistenceRow;
return { recomputedAnime: Number(animeRow.count), repairedAtMs: rebuilt.rebuiltAtMs };
}
const repairedAtMs = nowMs();
db.exec('BEGIN');
try {
const animeIds = new Set<number>();
for (const row of db
.prepare('SELECT DISTINCT anime_id AS animeId FROM imm_videos WHERE anime_id IS NOT NULL')
.all() as Array<{ animeId: number }>) {
animeIds.add(row.animeId);
}
for (const row of db
.prepare('SELECT anime_id AS animeId FROM imm_lifetime_anime')
.all() as Array<{ animeId: number }>) {
animeIds.add(row.animeId);
}
const updatedAtMs = toDbTimestamp(repairedAtMs);
recomputeLifetimeAnimeFromMedia(db, [...animeIds], updatedAtMs);
recomputeLifetimeGlobalFromSummaries(db, { updatedAtMs });
db.exec('COMMIT');
return { recomputedAnime: animeIds.size, repairedAtMs };
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
}
export function reconcileStaleActiveSessions(db: DatabaseSync): number {
const sessions = getRetainedStaleActiveSessions(db);
if (sessions.length === 0) {
@@ -1,5 +1,5 @@
import type { DatabaseSync } from './sqlite';
import { rebuildLifetimeSummariesInTransaction } from './lifetime';
import { applyLifetimeRemovals, planLifetimeRemovals } from './lifetime';
import { getRollupGroupsForSessions, refreshRollupsForGroupsInTransaction } from './maintenance';
import {
applyLexicalRemovals,
@@ -8,9 +8,10 @@ import {
forEachIdChunk,
makePlaceholders,
planLexicalRemovalsForSessions,
SQLITE_ID_CHUNK_SIZE,
planLexicalRemovalsForVideos,
type LexicalRemovalPlan,
} from './query-shared';
import type { RollupGroup } from './maintenance';
export type DeleteMaintenanceOperation =
| { kind: 'session'; sessionId: number }
@@ -59,40 +60,59 @@ function selectIds(
return ids;
}
function planLexicalRemovalsInChunks(db: DatabaseSync, sessionIds: number[]): LexicalRemovalPlan {
const combined: LexicalRemovalPlan = { words: [], kanji: [] };
const merge = (target: LexicalRemovalPlan['words'], source: LexicalRemovalPlan['words']) => {
const byId = new Map(target.map((entry) => [entry.id, entry]));
for (const entry of source) {
const existing = byId.get(entry.id);
if (!existing) {
const added = { ...entry };
target.push(added);
byId.set(entry.id, added);
continue;
}
existing.removedFrequency += entry.removedFrequency;
if (
entry.removedFirstSeenMs !== null &&
(existing.removedFirstSeenMs === null ||
entry.removedFirstSeenMs < existing.removedFirstSeenMs)
) {
existing.removedFirstSeenMs = entry.removedFirstSeenMs;
}
if (
entry.removedLastSeenMs !== null &&
(existing.removedLastSeenMs === null ||
entry.removedLastSeenMs > existing.removedLastSeenMs)
) {
existing.removedLastSeenMs = entry.removedLastSeenMs;
}
function mergeLexicalPlanEntries(
target: LexicalRemovalPlan['words'],
source: LexicalRemovalPlan['words'],
): void {
const byId = new Map(target.map((entry) => [entry.id, entry]));
for (const entry of source) {
const existing = byId.get(entry.id);
if (!existing) {
const added = { ...entry };
target.push(added);
byId.set(entry.id, added);
continue;
}
};
existing.removedFrequency += entry.removedFrequency;
if (
entry.removedFirstSeenMs !== null &&
(existing.removedFirstSeenMs === null ||
entry.removedFirstSeenMs < existing.removedFirstSeenMs)
) {
existing.removedFirstSeenMs = entry.removedFirstSeenMs;
}
if (
entry.removedLastSeenMs !== null &&
(existing.removedLastSeenMs === null || entry.removedLastSeenMs > existing.removedLastSeenMs)
) {
existing.removedLastSeenMs = entry.removedLastSeenMs;
}
}
}
forEachIdChunk(sessionIds, (chunk) => {
const plan = planLexicalRemovalsForSessions(db, chunk);
merge(combined.words, plan.words);
merge(combined.kanji, plan.kanji);
function mergeLexicalPlans(target: LexicalRemovalPlan, source: LexicalRemovalPlan): void {
mergeLexicalPlanEntries(target.words, source.words);
mergeLexicalPlanEntries(target.kanji, source.kanji);
}
/**
* Plan what the delete removes from imm_words/imm_kanji.
*
* Deleted videos are planned by video so orphaned subtitle lines (whose session
* is already gone) still get subtracted; sessions on surviving videos are
* planned by session. The two scopes are disjoint, so nothing is counted twice.
*/
function planLexicalRemovalsForDelete(
db: DatabaseSync,
sessionIdsOnSurvivingVideos: number[],
videoIds: number[],
): LexicalRemovalPlan {
const combined: LexicalRemovalPlan = { words: [], kanji: [] };
forEachIdChunk(sessionIdsOnSurvivingVideos, (chunk) => {
mergeLexicalPlans(combined, planLexicalRemovalsForSessions(db, chunk));
});
forEachIdChunk(videoIds, (chunk) => {
mergeLexicalPlans(combined, planLexicalRemovalsForVideos(db, chunk));
});
return combined;
}
@@ -121,24 +141,37 @@ export function deleteMaintenanceBatch(
}
const videoIdList = [...videoIds];
for (const sessionId of selectIds(
db,
(placeholders) => `SELECT session_id FROM imm_sessions WHERE video_id IN (${placeholders})`,
videoIdList,
'session_id',
)) {
sessionIds.add(sessionId);
}
const sessionIdsOnDeletedVideos = new Set(
selectIds(
db,
(placeholders) => `SELECT session_id FROM imm_sessions WHERE video_id IN (${placeholders})`,
videoIdList,
'session_id',
),
);
for (const sessionId of sessionIdsOnDeletedVideos) sessionIds.add(sessionId);
const sessionIdList = [...sessionIds];
const lexicalRemovals = planLexicalRemovalsInChunks(db, sessionIdList);
const affectedRollupGroups = sessionIdList
.flatMap((_, index) =>
index % SQLITE_ID_CHUNK_SIZE === 0
? getRollupGroupsForSessions(db, sessionIdList.slice(index, index + SQLITE_ID_CHUNK_SIZE))
: [],
)
.filter((group) => !videoIds.has(group.videoId));
const sessionIdsOnSurvivingVideos = sessionIdList.filter(
(sessionId) => !sessionIdsOnDeletedVideos.has(sessionId),
);
// Both plans must be measured before any rows are removed.
const lexicalRemovals = planLexicalRemovalsForDelete(
db,
sessionIdsOnSurvivingVideos,
videoIdList,
);
const lifetimeRemovals = planLifetimeRemovals(db, {
deletedSessionIds: sessionIdList,
sessionIdsOnSurvivingVideos,
deletedVideoIds: videoIdList,
deletedAnimeIds: animeIdList,
});
const affectedRollupGroups: RollupGroup[] = [];
forEachIdChunk(sessionIdsOnSurvivingVideos, (chunk) => {
affectedRollupGroups.push(...getRollupGroupsForSessions(db, chunk));
});
const coverBlobHashes = new Set<string>();
if (videoIdList.length > 0) {
forEachIdChunk(videoIdList, (chunk) => {
@@ -152,26 +185,31 @@ export function deleteMaintenanceBatch(
.all(...chunk) as Array<{ coverBlobHash: string }>;
for (const row of artRows) coverBlobHashes.add(row.coverBlobHash);
});
deleteSessionsByIds(db, sessionIdList);
forEachIdChunk(videoIdList, (chunk) => {
const placeholders = makePlaceholders(chunk);
db.prepare(`DELETE FROM imm_subtitle_lines WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_daily_rollups WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_monthly_rollups WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_media_art WHERE video_id IN (${placeholders})`).run(...chunk);
db.prepare(`DELETE FROM imm_videos WHERE video_id IN (${placeholders})`).run(...chunk);
});
} else {
deleteSessionsByIds(db, sessionIdList);
}
deleteSessionsByIds(db, sessionIdList);
forEachIdChunk(sessionIdList, (chunk) => {
const placeholders = makePlaceholders(chunk);
db.prepare(
`DELETE FROM imm_lifetime_applied_sessions WHERE session_id IN (${placeholders})`,
).run(...chunk);
});
forEachIdChunk(videoIdList, (chunk) => {
const placeholders = makePlaceholders(chunk);
db.prepare(`DELETE FROM imm_subtitle_lines WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_daily_rollups WHERE video_id IN (${placeholders})`).run(...chunk);
db.prepare(`DELETE FROM imm_monthly_rollups WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_media_art WHERE video_id IN (${placeholders})`).run(...chunk);
db.prepare(`DELETE FROM imm_lifetime_media WHERE video_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_videos WHERE video_id IN (${placeholders})`).run(...chunk);
});
for (const coverBlobHash of coverBlobHashes) {
cleanupUnusedCoverArtBlobHash(db, coverBlobHash);
}
@@ -186,7 +224,7 @@ export function deleteMaintenanceBatch(
}
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
applyLifetimeRemovals(db, lifetimeRemovals);
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
db.exec('COMMIT');
} catch (error) {
@@ -1,20 +1,16 @@
import { createHash } from 'node:crypto';
import type { DatabaseSync } from './sqlite';
import { buildCoverBlobReference, normalizeCoverBlobBytes } from './storage';
import { rebuildLifetimeSummaries, rebuildLifetimeSummariesInTransaction } from './lifetime';
import { getRollupGroupsForSessions, refreshRollupsForGroupsInTransaction } from './maintenance';
import { repairLifetimeSummariesFromMedia } from './lifetime';
import { nowMs } from './time';
import { resolveAnimeAnilistConflict } from './anime-season-repair';
import { deleteMaintenanceBatch } from './query-delete-maintenance';
import { PartOfSpeech, type MergedToken } from '../../../types';
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
import { deriveStoredPartOfSpeech } from '../tokenizer/part-of-speech';
import {
applyLexicalRemovals,
cleanupUnusedCoverArtBlobHash,
deleteSessionsByIds,
findSharedCoverBlobHash,
planLexicalRemovalsForSessions,
planLexicalRemovalsForVideos,
toDbMs,
toDbTimestamp,
} from './query-shared';
@@ -425,7 +421,7 @@ export function updateAnimeAnilistInfo(
} | null;
if (!row?.anime_id) return;
const repair = resolveAnimeAnilistConflict(db, row.anime_id, info.anilistId);
resolveAnimeAnilistConflict(db, row.anime_id, info.anilistId);
const targetRow = db
.prepare('SELECT anime_id FROM imm_videos WHERE video_id = ?')
.get(videoId) as {
@@ -454,9 +450,10 @@ export function updateAnimeAnilistInfo(
toDbTimestamp(nowMs()),
targetRow.anime_id,
);
if (repair.movedVideos > 0 || repair.deletedAnimeRows > 0) {
rebuildLifetimeSummaries(db);
}
// Moves change which anime owns the media rows, and an episodes_total change
// can flip anime_completed even without a move — both are derivable from the
// summary tables, so a full (retention-lossy) rebuild is never needed here.
repairLifetimeSummariesFromMedia(db);
}
export function markVideoWatched(db: DatabaseSync, videoId: number, watched: boolean): void {
@@ -482,136 +479,22 @@ export function isVideoWatched(db: DatabaseSync, videoId: number): boolean {
}
export function deleteSession(db: DatabaseSync, sessionId: number): void {
const sessionIds = [sessionId];
db.exec('BEGIN IMMEDIATE');
try {
// Measured inside the write lock: the plan records what the delete removes,
// and applying a plan taken against a different snapshot would subtract the
// wrong totals from imm_words/imm_kanji.
const lexicalRemovals = planLexicalRemovalsForSessions(db, sessionIds);
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
deleteSessionsByIds(db, sessionIds);
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId }]);
}
export function deleteSessions(db: DatabaseSync, sessionIds: number[]): void {
if (sessionIds.length === 0) return;
db.exec('BEGIN IMMEDIATE');
try {
const lexicalRemovals = planLexicalRemovalsForSessions(db, sessionIds);
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
deleteSessionsByIds(db, sessionIds);
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
deleteMaintenanceBatch(db, [{ kind: 'sessions', sessionIds }]);
}
/**
* Delete an entire library entry: every episode of the anime, all of their
* sessions and derived stats, and the anime row itself.
*
* Mirrors {@link deleteVideo} per episode, but batches the lexical refresh and
* lifetime rebuild into a single transaction so a multi-episode title doesn't
* pay for one full rebuild per episode.
*/
export function deleteAnime(db: DatabaseSync, animeId: number): void {
db.exec('BEGIN IMMEDIATE');
try {
const videoIds = (
db.prepare('SELECT video_id FROM imm_videos WHERE anime_id = ?').all(animeId) as Array<{
video_id: number;
}>
).map((row) => row.video_id);
const lexicalRemovals = planLexicalRemovalsForVideos(db, videoIds);
const coverBlobHashes: string[] = [];
const sessionIds: number[] = [];
for (const videoId of videoIds) {
const artRow = db
.prepare('SELECT cover_blob_hash AS coverBlobHash FROM imm_media_art WHERE video_id = ?')
.get(videoId) as { coverBlobHash: string | null } | undefined;
if (artRow?.coverBlobHash) {
coverBlobHashes.push(artRow.coverBlobHash);
}
const sessions = db
.prepare('SELECT session_id FROM imm_sessions WHERE video_id = ?')
.all(videoId) as Array<{ session_id: number }>;
sessionIds.push(...sessions.map((session) => session.session_id));
}
deleteSessionsByIds(db, sessionIds);
const deleteLinesStmt = db.prepare('DELETE FROM imm_subtitle_lines WHERE video_id = ?');
const deleteDailyStmt = db.prepare('DELETE FROM imm_daily_rollups WHERE video_id = ?');
const deleteMonthlyStmt = db.prepare('DELETE FROM imm_monthly_rollups WHERE video_id = ?');
const deleteArtStmt = db.prepare('DELETE FROM imm_media_art WHERE video_id = ?');
const deleteVideoStmt = db.prepare('DELETE FROM imm_videos WHERE video_id = ?');
for (const videoId of videoIds) {
deleteLinesStmt.run(videoId);
deleteDailyStmt.run(videoId);
deleteMonthlyStmt.run(videoId);
deleteArtStmt.run(videoId);
deleteVideoStmt.run(videoId);
}
for (const coverBlobHash of new Set(coverBlobHashes)) {
cleanupUnusedCoverArtBlobHash(db, coverBlobHash);
}
db.prepare('DELETE FROM imm_lifetime_anime WHERE anime_id = ?').run(animeId);
db.prepare('DELETE FROM imm_anime WHERE anime_id = ?').run(animeId);
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
deleteMaintenanceBatch(db, [{ kind: 'anime', animeId }]);
}
export function deleteVideo(db: DatabaseSync, videoId: number): void {
db.exec('BEGIN IMMEDIATE');
try {
const artRow = db
.prepare(
`
SELECT cover_blob_hash AS coverBlobHash
FROM imm_media_art
WHERE video_id = ?
`,
)
.get(videoId) as { coverBlobHash: string | null } | undefined;
const lexicalRemovals = planLexicalRemovalsForVideos(db, [videoId]);
const sessions = db
.prepare('SELECT session_id FROM imm_sessions WHERE video_id = ?')
.all(videoId) as Array<{ session_id: number }>;
deleteSessionsByIds(
db,
sessions.map((session) => session.session_id),
);
db.prepare('DELETE FROM imm_subtitle_lines WHERE video_id = ?').run(videoId);
db.prepare('DELETE FROM imm_daily_rollups WHERE video_id = ?').run(videoId);
db.prepare('DELETE FROM imm_monthly_rollups WHERE video_id = ?').run(videoId);
db.prepare('DELETE FROM imm_media_art WHERE video_id = ?').run(videoId);
cleanupUnusedCoverArtBlobHash(db, artRow?.coverBlobHash ?? null);
db.prepare('DELETE FROM imm_videos WHERE video_id = ?').run(videoId);
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
deleteMaintenanceBatch(db, [{ kind: 'video', videoId }]);
}
@@ -307,10 +307,13 @@ function toStoredSeenSeconds(ms: number | null): number | null {
* Apply a removal plan to the vocabulary aggregates.
*
* Frequencies are adjusted by subtraction, which is exact and touches only the
* affected rows. `first_seen`/`last_seen` only need a rescan when the removed
* lines held the current extreme, and rows whose frequency reaches zero are
* verified against the surviving occurrences before deletion — so stored counts
* that have drifted still converge on the truth instead of dropping a live row.
* affected rows. When the removed lines held a `first_seen`/`last_seen`
* extreme, the new extremes come from MIN/MAX index-endpoint seeks on the
* occurrence covering index — never a re-aggregation of every occurrence, which
* for common particles means scanning the whole library. The full re-aggregate
* survives only as the repair path: rows whose stored frequency reaches zero
* while occurrences remain (drift), and rows with undated pre-migration
* occurrences the seeks would skip.
*/
export function applyLexicalRemovals(db: DatabaseSync, plan: LexicalRemovalPlan): void {
applyRemovalsForEntity(db, 'word', plan.words);
@@ -339,6 +342,21 @@ function applyRemovalsForEntity(
`SELECT 1 AS found FROM ${occurrenceTable} WHERE ${col} = ? LIMIT 1`,
);
const deleteStmt = db.prepare(`DELETE FROM ${entityTable} WHERE id = ?`);
// Seeks to the front of this entity's index range, where NULL seen_ms sorts.
const hasUndatedOccurrenceStmt = db.prepare(
`SELECT 1 AS found FROM ${occurrenceTable} WHERE ${col} = ? AND seen_ms IS NULL LIMIT 1`,
);
// Kept as separate single-aggregate statements so SQLite's min/max
// optimization turns each into an index-endpoint seek instead of a scan.
const minSeenStmt = db.prepare(
`SELECT MIN(seen_ms) AS value FROM ${occurrenceTable} WHERE ${col} = ?`,
);
const maxSeenStmt = db.prepare(
`SELECT MAX(seen_ms) AS value FROM ${occurrenceTable} WHERE ${col} = ?`,
);
const updateAggregatesStmt = db.prepare(
`UPDATE ${entityTable} SET frequency = ?, first_seen = ?, last_seen = ? WHERE id = ?`,
);
const needsExactRefresh: number[] = [];
@@ -371,7 +389,26 @@ function applyRemovalsForEntity(
current.lastSeen === null ||
(removedLastSeen !== null && removedLastSeen >= current.lastSeen);
if (firstSeenMayHaveMoved || lastSeenMayHaveMoved) {
needsExactRefresh.push(removal.id);
// Undated pre-migration occurrences are invisible to the seeks below;
// fall back to the full re-aggregate that resolves their dates.
if (hasUndatedOccurrenceStmt.get(removal.id)) {
needsExactRefresh.push(removal.id);
continue;
}
const minSeenMs = (minSeenStmt.get(removal.id) as { value: number | null }).value;
const maxSeenMs = (maxSeenStmt.get(removal.id) as { value: number | null }).value;
if (minSeenMs === null || maxSeenMs === null) {
// Frequency says occurrences remain but none exist: stale row, let the
// exact refresh reconcile (it deletes rows with nothing left).
needsExactRefresh.push(removal.id);
continue;
}
updateAggregatesStmt.run(
nextFrequency,
Math.floor(Number(minSeenMs) / 1000),
Math.floor(Number(maxSeenMs) / 1000),
removal.id,
);
continue;
}