mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-15 13:55:51 -07:00
fix(stats): subtract lifetime totals incrementally on delete
- Preserve lifetime history beyond session retention during deletes and repairs - Fall back to the current thread when the delete worker fails to load
This commit is contained in:
@@ -329,3 +329,28 @@ test('upgrading an older database backfills seen_ms from the subtitle lines', ()
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('an extreme-moving delete keeps subtraction-exact frequency instead of re-summing', () => {
|
||||
const { db, dbPath } = createDb([
|
||||
{ session: 1, wordId: 7, dayOffset: 0, count: 2 },
|
||||
{ session: 2, wordId: 7, dayOffset: 3, count: 1 },
|
||||
]);
|
||||
|
||||
try {
|
||||
// Simulate drift: the stored total is higher than the occurrences justify.
|
||||
// The extremes move via index seeks while the count stays a pure
|
||||
// subtraction; drifted counts reconcile only at the zero-crossing repair
|
||||
// or via the cleanup command, never by rescanning every occurrence here.
|
||||
db.prepare('UPDATE imm_words SET frequency = 10 WHERE id = 7').run();
|
||||
|
||||
deleteSession(db, 1);
|
||||
|
||||
const word = readWord(db, 7);
|
||||
assert.equal(word?.frequency, 10 - 2);
|
||||
assert.equal(word?.firstSeen, Math.floor((BASE_MS + 3 * DAY_MS) / 1000));
|
||||
assert.equal(word?.lastSeen, Math.floor((BASE_MS + 3 * DAY_MS) / 1000));
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { Database } from '../sqlite.js';
|
||||
import type { DatabaseSync } from '../sqlite.js';
|
||||
import {
|
||||
applyPragmas,
|
||||
ensureSchema,
|
||||
getOrCreateAnimeRecord,
|
||||
getOrCreateVideoRecord,
|
||||
linkVideoToAnimeRecord,
|
||||
} from '../storage.js';
|
||||
import { startSessionRecord } from '../session.js';
|
||||
import { rebuildLifetimeSummaries, repairLifetimeSummariesFromMedia } from '../lifetime.js';
|
||||
import { deleteMaintenanceBatch } from '../query-delete-maintenance.js';
|
||||
import { toDbTimestamp } from '../query-shared.js';
|
||||
|
||||
const SOURCE_TYPE_LOCAL = 1;
|
||||
const DAY_MS = 86_400_000;
|
||||
// Noon UTC keeps every seeded timestamp on the same local day regardless of
|
||||
// the timezone the test host runs in.
|
||||
const BASE_MS = Date.UTC(2026, 0, 5, 12, 0, 0);
|
||||
|
||||
function createDb(): DatabaseSync {
|
||||
const db = new Database(':memory:');
|
||||
applyPragmas(db);
|
||||
ensureSchema(db);
|
||||
return db;
|
||||
}
|
||||
|
||||
function seedAnime(db: DatabaseSync, title: string, episodesTotal: number | null): number {
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: title,
|
||||
canonicalTitle: title,
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: null,
|
||||
});
|
||||
if (episodesTotal !== null) {
|
||||
db.prepare('UPDATE imm_anime SET episodes_total = ? WHERE anime_id = ?').run(
|
||||
episodesTotal,
|
||||
animeId,
|
||||
);
|
||||
}
|
||||
return animeId;
|
||||
}
|
||||
|
||||
function seedVideo(
|
||||
db: DatabaseSync,
|
||||
animeId: number | null,
|
||||
name: string,
|
||||
options: { watched?: boolean } = {},
|
||||
): number {
|
||||
const videoId = getOrCreateVideoRecord(db, `local:/tmp/${name}.mkv`, {
|
||||
canonicalTitle: name,
|
||||
sourcePath: `/tmp/${name}.mkv`,
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
if (animeId !== null) {
|
||||
linkVideoToAnimeRecord(db, videoId, {
|
||||
animeId,
|
||||
parsedBasename: `${name}.mkv`,
|
||||
parsedTitle: name,
|
||||
parsedSeason: 1,
|
||||
parsedEpisode: 1,
|
||||
parserSource: 'test',
|
||||
parserConfidence: 1,
|
||||
parseMetadataJson: null,
|
||||
});
|
||||
}
|
||||
if (options.watched) {
|
||||
db.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?').run(videoId);
|
||||
}
|
||||
return videoId;
|
||||
}
|
||||
|
||||
function seedEndedSession(
|
||||
db: DatabaseSync,
|
||||
videoId: number,
|
||||
startedAtMs: number,
|
||||
metrics: { activeMs: number; cards?: number; lines?: number; tokens?: number },
|
||||
): number {
|
||||
const sessionId = startSessionRecord(db, videoId, startedAtMs).sessionId;
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE imm_sessions SET
|
||||
ended_at_ms = ?,
|
||||
active_watched_ms = ?,
|
||||
total_watched_ms = ?,
|
||||
cards_mined = ?,
|
||||
lines_seen = ?,
|
||||
tokens_seen = ?
|
||||
WHERE session_id = ?
|
||||
`,
|
||||
).run(
|
||||
toDbTimestamp(startedAtMs + metrics.activeMs),
|
||||
metrics.activeMs,
|
||||
metrics.activeMs,
|
||||
metrics.cards ?? 0,
|
||||
metrics.lines ?? 0,
|
||||
metrics.tokens ?? 0,
|
||||
sessionId,
|
||||
);
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
// libsql attaches a per-query `_metadata` property to result rows; strip it so
|
||||
// row snapshots can be compared with deepEqual.
|
||||
function cleanRow<T>(row: unknown): T {
|
||||
const { _metadata: _ignored, ...rest } = row as Record<string, unknown>;
|
||||
return rest as T;
|
||||
}
|
||||
|
||||
interface GlobalSnapshot {
|
||||
total_sessions: number;
|
||||
total_active_ms: number;
|
||||
total_cards: number;
|
||||
active_days: number;
|
||||
episodes_started: number;
|
||||
episodes_completed: number;
|
||||
anime_completed: number;
|
||||
}
|
||||
|
||||
function snapshotGlobal(db: DatabaseSync): GlobalSnapshot {
|
||||
const row = db
|
||||
.prepare(
|
||||
`SELECT total_sessions, total_active_ms, total_cards, active_days,
|
||||
episodes_started, episodes_completed, anime_completed
|
||||
FROM imm_lifetime_global WHERE global_id = 1`,
|
||||
)
|
||||
.get();
|
||||
return cleanRow<GlobalSnapshot>(row);
|
||||
}
|
||||
|
||||
function snapshotMedia(db: DatabaseSync): unknown[] {
|
||||
return db
|
||||
.prepare(
|
||||
`SELECT video_id, total_sessions, total_active_ms, total_cards,
|
||||
total_lines_seen, total_tokens_seen, completed,
|
||||
CAST(first_watched_ms AS REAL) AS first_watched,
|
||||
CAST(last_watched_ms AS REAL) AS last_watched
|
||||
FROM imm_lifetime_media ORDER BY video_id`,
|
||||
)
|
||||
.all()
|
||||
.map((row) => cleanRow(row));
|
||||
}
|
||||
|
||||
function snapshotAnime(db: DatabaseSync): unknown[] {
|
||||
return db
|
||||
.prepare(
|
||||
`SELECT anime_id, total_sessions, total_active_ms, total_cards,
|
||||
total_lines_seen, total_tokens_seen, episodes_started, episodes_completed,
|
||||
CAST(first_watched_ms AS REAL) AS first_watched,
|
||||
CAST(last_watched_ms AS REAL) AS last_watched
|
||||
FROM imm_lifetime_anime ORDER BY anime_id`,
|
||||
)
|
||||
.all()
|
||||
.map((row) => cleanRow(row));
|
||||
}
|
||||
|
||||
test('incremental delete maintenance matches a full rebuild when no history is pruned', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const animeA = seedAnime(db, 'Anime A', 2);
|
||||
const animeB = seedAnime(db, 'Anime B', 1);
|
||||
const videoA1 = seedVideo(db, animeA, 'anime-a-ep1', { watched: true });
|
||||
const videoA2 = seedVideo(db, animeA, 'anime-a-ep2', { watched: true });
|
||||
const videoB1 = seedVideo(db, animeB, 'anime-b-ep1', { watched: true });
|
||||
const videoLoose = seedVideo(db, null, 'loose-video');
|
||||
|
||||
seedEndedSession(db, videoA1, BASE_MS, { activeMs: 60_000, cards: 2, lines: 30, tokens: 200 });
|
||||
const deletedSessionId = seedEndedSession(db, videoA1, BASE_MS + DAY_MS, {
|
||||
activeMs: 45_000,
|
||||
cards: 1,
|
||||
lines: 20,
|
||||
tokens: 100,
|
||||
});
|
||||
seedEndedSession(db, videoA2, BASE_MS + 2 * DAY_MS, { activeMs: 90_000, cards: 3 });
|
||||
seedEndedSession(db, videoB1, BASE_MS + 3 * DAY_MS, { activeMs: 30_000 });
|
||||
seedEndedSession(db, videoLoose, BASE_MS + 4 * DAY_MS, { activeMs: 15_000 });
|
||||
|
||||
rebuildLifetimeSummaries(db);
|
||||
|
||||
deleteMaintenanceBatch(db, [
|
||||
{ kind: 'session', sessionId: deletedSessionId },
|
||||
{ kind: 'video', videoId: videoLoose },
|
||||
{ kind: 'anime', animeId: animeB },
|
||||
]);
|
||||
|
||||
const incrementalGlobal = snapshotGlobal(db);
|
||||
const incrementalMedia = snapshotMedia(db);
|
||||
const incrementalAnime = snapshotAnime(db);
|
||||
|
||||
// With every session still retained, subtracting must land on exactly the
|
||||
// state a from-scratch rebuild computes.
|
||||
rebuildLifetimeSummaries(db);
|
||||
assert.deepEqual(incrementalGlobal, snapshotGlobal(db));
|
||||
assert.deepEqual(incrementalMedia, snapshotMedia(db));
|
||||
assert.deepEqual(incrementalAnime, snapshotAnime(db));
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('deleting a retained session preserves lifetime history from pruned sessions', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const animeId = seedAnime(db, 'Pruned Anime', null);
|
||||
const videoId = seedVideo(db, animeId, 'pruned-ep1');
|
||||
const prunedSessionId = seedEndedSession(db, videoId, BASE_MS, {
|
||||
activeMs: 120_000,
|
||||
cards: 4,
|
||||
lines: 50,
|
||||
tokens: 400,
|
||||
});
|
||||
const retainedSessionId = seedEndedSession(db, videoId, BASE_MS + DAY_MS, {
|
||||
activeMs: 30_000,
|
||||
cards: 1,
|
||||
lines: 10,
|
||||
tokens: 80,
|
||||
});
|
||||
rebuildLifetimeSummaries(db);
|
||||
|
||||
// Simulate raw-session retention pruning the older session. Lifetime
|
||||
// summaries intentionally keep its contribution.
|
||||
db.prepare('DELETE FROM imm_sessions WHERE session_id = ?').run(prunedSessionId);
|
||||
|
||||
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: retainedSessionId }]);
|
||||
|
||||
const globalRow = snapshotGlobal(db);
|
||||
assert.equal(globalRow.total_sessions, 1, 'pruned session contribution survives the delete');
|
||||
assert.equal(globalRow.total_active_ms, 120_000);
|
||||
assert.equal(globalRow.total_cards, 4);
|
||||
assert.equal(globalRow.episodes_started, 1);
|
||||
// The pruned session's day stays counted (pruning never subtracts); only
|
||||
// the deleted retained session's day is dropped.
|
||||
assert.equal(globalRow.active_days, 1);
|
||||
|
||||
const mediaRow = db
|
||||
.prepare(
|
||||
'SELECT total_sessions, total_active_ms, total_cards FROM imm_lifetime_media WHERE video_id = ?',
|
||||
)
|
||||
.get(videoId);
|
||||
assert.deepEqual(
|
||||
cleanRow<{ total_sessions: number; total_active_ms: number; total_cards: number }>(mediaRow),
|
||||
{ total_sessions: 1, total_active_ms: 120_000, total_cards: 4 },
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('active_days only drops when the last session of a local day is deleted', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const videoId = seedVideo(db, null, 'same-day');
|
||||
const firstSessionId = seedEndedSession(db, videoId, BASE_MS, { activeMs: 10_000 });
|
||||
const secondSessionId = seedEndedSession(db, videoId, BASE_MS + 3_600_000, {
|
||||
activeMs: 20_000,
|
||||
});
|
||||
rebuildLifetimeSummaries(db);
|
||||
assert.equal(snapshotGlobal(db).active_days, 1);
|
||||
|
||||
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: firstSessionId }]);
|
||||
assert.equal(snapshotGlobal(db).active_days, 1, 'day still has a session');
|
||||
|
||||
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId: secondSessionId }]);
|
||||
assert.equal(snapshotGlobal(db).active_days, 0, 'day lost its last session');
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('deleting a video updates anime and global rollups without a rebuild', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const animeId = seedAnime(db, 'Two Episode Anime', 2);
|
||||
const videoEp1 = seedVideo(db, animeId, 'two-ep-1', { watched: true });
|
||||
const videoEp2 = seedVideo(db, animeId, 'two-ep-2', { watched: true });
|
||||
seedEndedSession(db, videoEp1, BASE_MS, { activeMs: 60_000, cards: 2 });
|
||||
seedEndedSession(db, videoEp2, BASE_MS + DAY_MS, { activeMs: 40_000, cards: 1 });
|
||||
rebuildLifetimeSummaries(db);
|
||||
assert.equal(snapshotGlobal(db).anime_completed, 1);
|
||||
|
||||
deleteMaintenanceBatch(db, [{ kind: 'video', videoId: videoEp2 }]);
|
||||
|
||||
const globalRow = snapshotGlobal(db);
|
||||
assert.equal(globalRow.total_sessions, 1);
|
||||
assert.equal(globalRow.total_active_ms, 60_000);
|
||||
assert.equal(globalRow.episodes_started, 1);
|
||||
assert.equal(globalRow.episodes_completed, 1);
|
||||
assert.equal(globalRow.anime_completed, 0, 'anime no longer has all episodes completed');
|
||||
|
||||
const animeRow = db
|
||||
.prepare(
|
||||
'SELECT total_sessions, episodes_started, episodes_completed FROM imm_lifetime_anime WHERE anime_id = ?',
|
||||
)
|
||||
.get(animeId);
|
||||
assert.deepEqual(
|
||||
cleanRow<{
|
||||
total_sessions: number;
|
||||
episodes_started: number;
|
||||
episodes_completed: number;
|
||||
}>(animeRow),
|
||||
{ total_sessions: 1, episodes_started: 1, episodes_completed: 1 },
|
||||
);
|
||||
|
||||
deleteMaintenanceBatch(db, [{ kind: 'video', videoId: videoEp1 }]);
|
||||
assert.equal(
|
||||
(db.prepare('SELECT COUNT(*) AS total FROM imm_lifetime_anime').get() as { total: number })
|
||||
.total,
|
||||
0,
|
||||
'anime lifetime row is dropped once no episodes remain',
|
||||
);
|
||||
assert.deepEqual(snapshotGlobal(db), {
|
||||
total_sessions: 0,
|
||||
total_active_ms: 0,
|
||||
total_cards: 0,
|
||||
active_days: 0,
|
||||
episodes_started: 0,
|
||||
episodes_completed: 0,
|
||||
anime_completed: 0,
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('repair after a video moves between anime matches a full rebuild', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const animeA = seedAnime(db, 'Move Source', 2);
|
||||
const animeB = seedAnime(db, 'Move Target', 2);
|
||||
const movedVideo = seedVideo(db, animeA, 'moved-ep', { watched: true });
|
||||
const stayingVideo = seedVideo(db, animeA, 'staying-ep');
|
||||
const targetVideo = seedVideo(db, animeB, 'target-ep', { watched: true });
|
||||
seedEndedSession(db, movedVideo, BASE_MS, { activeMs: 60_000, cards: 2 });
|
||||
seedEndedSession(db, stayingVideo, BASE_MS + DAY_MS, { activeMs: 30_000 });
|
||||
seedEndedSession(db, targetVideo, BASE_MS + 2 * DAY_MS, { activeMs: 45_000, cards: 1 });
|
||||
rebuildLifetimeSummaries(db);
|
||||
|
||||
// Simulate a library merge reassigning the episode to the other anime.
|
||||
db.prepare('UPDATE imm_videos SET anime_id = ? WHERE video_id = ?').run(animeB, movedVideo);
|
||||
repairLifetimeSummariesFromMedia(db);
|
||||
|
||||
const repairedGlobal = snapshotGlobal(db);
|
||||
const repairedMedia = snapshotMedia(db);
|
||||
const repairedAnime = snapshotAnime(db);
|
||||
|
||||
rebuildLifetimeSummaries(db);
|
||||
assert.deepEqual(repairedGlobal, snapshotGlobal(db));
|
||||
assert.deepEqual(repairedMedia, snapshotMedia(db));
|
||||
assert.deepEqual(repairedAnime, snapshotAnime(db));
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('repair preserves lifetime history from pruned sessions where a rebuild would not', () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
const animeId = seedAnime(db, 'Repair Anime', null);
|
||||
const videoId = seedVideo(db, animeId, 'repair-ep');
|
||||
const prunedSessionId = seedEndedSession(db, videoId, BASE_MS, {
|
||||
activeMs: 90_000,
|
||||
cards: 3,
|
||||
});
|
||||
seedEndedSession(db, videoId, BASE_MS + DAY_MS, { activeMs: 30_000, cards: 1 });
|
||||
rebuildLifetimeSummaries(db);
|
||||
|
||||
db.prepare('DELETE FROM imm_sessions WHERE session_id = ?').run(prunedSessionId);
|
||||
repairLifetimeSummariesFromMedia(db);
|
||||
|
||||
const globalRow = snapshotGlobal(db);
|
||||
assert.equal(globalRow.total_sessions, 2, 'repair keeps the pruned session contribution');
|
||||
assert.equal(globalRow.total_active_ms, 120_000);
|
||||
assert.equal(globalRow.total_cards, 4);
|
||||
assert.equal(globalRow.active_days, 2, 'repair never subtracts active days');
|
||||
|
||||
const animeRow = db
|
||||
.prepare('SELECT total_sessions FROM imm_lifetime_anime WHERE anime_id = ?')
|
||||
.get(animeId);
|
||||
assert.equal(cleanRow<{ total_sessions: number }>(animeRow).total_sessions, 2);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
@@ -32,7 +32,7 @@ function createFakeWorker() {
|
||||
|
||||
type FakeWorker = ReturnType<typeof createFakeWorker>['worker'];
|
||||
|
||||
test('a delete batch rebuilds lifetime summaries once', () => {
|
||||
test('a delete batch never runs a full lifetime rebuild', () => {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-delete-batch-test-'));
|
||||
const dbPath = path.join(tempDir, 'immersion.sqlite');
|
||||
let db = new Database(dbPath);
|
||||
@@ -87,8 +87,8 @@ test('a delete batch rebuilds lifetime summaries once', () => {
|
||||
assert.equal(deletedVideo, undefined);
|
||||
assert.equal(
|
||||
audit.total,
|
||||
2,
|
||||
'one rebuild performs exactly its reset and final global summary writes',
|
||||
0,
|
||||
'delete maintenance subtracts incrementally instead of rewriting last_rebuilt_ms',
|
||||
);
|
||||
} finally {
|
||||
try {
|
||||
@@ -100,6 +100,14 @@ test('a delete batch rebuilds lifetime summaries once', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('delete worker module resolves in the current layout', () => {
|
||||
// If this resolves to null, every delete silently runs on the serving thread
|
||||
// and blocks the stats API for the whole maintenance run.
|
||||
const workerPath = resolveDeleteMaintenanceWorkerPath();
|
||||
assert.ok(workerPath, 'delete-maintenance worker module must resolve');
|
||||
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
|
||||
});
|
||||
|
||||
test(
|
||||
'compiled delete worker removes data through its separate database connection',
|
||||
{ skip: resolveDeleteMaintenanceWorkerPath() === null },
|
||||
@@ -178,19 +186,51 @@ test('worker runtime terminates a worker after successful settlement', async ()
|
||||
assert.equal(terminationState.calls, 1);
|
||||
});
|
||||
|
||||
test('worker runtime terminates a worker after failed settlement', async () => {
|
||||
test('worker runtime falls back to the current thread when the worker crashes', async () => {
|
||||
const { worker, listeners, terminationState } = createFakeWorker();
|
||||
const fallbackTasks: unknown[] = [];
|
||||
const warnings: string[] = [];
|
||||
const runtime = new DeleteMaintenanceWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/delete-worker.js',
|
||||
createWorker: async () => worker,
|
||||
executeFallback: (dbPath, task) => {
|
||||
fallbackTasks.push({ dbPath, task });
|
||||
},
|
||||
warn: (message) => {
|
||||
warnings.push(message);
|
||||
},
|
||||
});
|
||||
|
||||
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
listeners.get('error')?.(new Error('worker failed') as never);
|
||||
|
||||
await assert.rejects(result, /worker failed/);
|
||||
await result;
|
||||
assert.equal(terminationState.calls, 1);
|
||||
assert.deepEqual(fallbackTasks, [
|
||||
{ dbPath: '/tmp/test.sqlite', task: { kind: 'session', sessionId: 1 } },
|
||||
]);
|
||||
assert.equal(warnings.length, 1);
|
||||
});
|
||||
|
||||
test('worker runtime surfaces a task failure without rerunning it', async () => {
|
||||
const { worker, listeners, terminationState } = createFakeWorker();
|
||||
const fallbackTasks: unknown[] = [];
|
||||
const runtime = new DeleteMaintenanceWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/delete-worker.js',
|
||||
createWorker: async () => worker,
|
||||
executeFallback: () => {
|
||||
fallbackTasks.push('ran');
|
||||
},
|
||||
});
|
||||
|
||||
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
listeners.get('message')?.({ ok: false, error: 'constraint violated' } as never);
|
||||
|
||||
await assert.rejects(result, /constraint violated/);
|
||||
assert.equal(terminationState.calls, 1);
|
||||
assert.equal(fallbackTasks.length, 0);
|
||||
});
|
||||
|
||||
test('worker runtime terminates a worker created after shutdown begins', async () => {
|
||||
|
||||
@@ -31,7 +31,13 @@ interface DeleteMaintenanceWorkerRuntimeOptions {
|
||||
}
|
||||
|
||||
export function resolveDeleteMaintenanceWorkerPath(): string | null {
|
||||
const workerPath = path.join(__dirname, 'delete-maintenance-worker-thread.js');
|
||||
// When the process runs TypeScript directly (Bun from source), the emitted
|
||||
// .js sibling doesn't exist — spawn the .ts module instead, which such
|
||||
// runtimes transpile for workers too. Compiled layouts keep using the .js.
|
||||
const fileName = __filename.endsWith('.ts')
|
||||
? 'delete-maintenance-worker-thread.ts'
|
||||
: 'delete-maintenance-worker-thread.js';
|
||||
const workerPath = path.join(__dirname, fileName);
|
||||
return fs.existsSync(workerPath) ? workerPath : null;
|
||||
}
|
||||
|
||||
@@ -76,38 +82,61 @@ export class DeleteMaintenanceWorkerRuntime {
|
||||
throw new Error('Delete maintenance worker is shut down');
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
type WorkerOutcome =
|
||||
| { kind: 'ok' }
|
||||
| { kind: 'task-error'; detail: string }
|
||||
| { kind: 'worker-failure'; error: Error };
|
||||
|
||||
const outcome = await new Promise<WorkerOutcome>((resolve) => {
|
||||
let settled = false;
|
||||
this.activeWorkers.add(worker);
|
||||
|
||||
const settle = (error?: Error) => {
|
||||
const settle = (result: WorkerOutcome) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
this.activeWorkers.delete(worker);
|
||||
if (error) reject(error);
|
||||
else resolve();
|
||||
resolve(result);
|
||||
void worker.terminate();
|
||||
};
|
||||
|
||||
worker.once('message', (message: DeleteMaintenanceWorkerResponse) => {
|
||||
if (message.ok === true) {
|
||||
settle();
|
||||
settle({ kind: 'ok' });
|
||||
return;
|
||||
}
|
||||
const detail = typeof message.error === 'string' ? message.error : 'unknown worker error';
|
||||
settle(new Error(`Delete maintenance failed: ${detail}`));
|
||||
settle({ kind: 'task-error', detail });
|
||||
});
|
||||
worker.once('error', (error) => settle(error));
|
||||
worker.once('error', (error) => settle({ kind: 'worker-failure', error }));
|
||||
worker.once('exit', (code) => {
|
||||
settle(
|
||||
new Error(
|
||||
settle({
|
||||
kind: 'worker-failure',
|
||||
error: new Error(
|
||||
code === 0
|
||||
? 'Delete maintenance worker exited without a response'
|
||||
: `Delete maintenance worker exited with code ${code}`,
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
if (outcome.kind === 'ok') return;
|
||||
// The maintenance itself failed inside the worker — rerunning it on this
|
||||
// thread would hit the same error, so surface it instead.
|
||||
if (outcome.kind === 'task-error') {
|
||||
throw new Error(`Delete maintenance failed: ${outcome.detail}`);
|
||||
}
|
||||
if (this.destroyed) {
|
||||
throw new Error('Delete maintenance worker is shut down');
|
||||
}
|
||||
// The worker died without reporting a result (failed to load, crashed).
|
||||
// Its transaction rolled back with its connection, and a rerun re-plans
|
||||
// against the current rows, so falling back on this thread is safe.
|
||||
(this.options.warn ?? logger.warn)(
|
||||
'Delete maintenance worker failed; running maintenance on the current thread',
|
||||
outcome.error,
|
||||
);
|
||||
(this.options.executeFallback ?? executeDeleteMaintenanceTask)(dbPath, task);
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas } from './storage';
|
||||
import { deleteAnime, deleteSession, deleteSessions, deleteVideo } from './query-maintenance';
|
||||
import {
|
||||
deleteMaintenanceBatch,
|
||||
type DeleteMaintenanceOperation,
|
||||
@@ -12,35 +11,11 @@ export type DeleteMaintenanceTask =
|
||||
| DeleteMaintenanceOperation
|
||||
| { kind: 'batch'; tasks: DeleteMaintenanceOperation[] };
|
||||
|
||||
function executeDeleteMaintenanceOperation(
|
||||
db: InstanceType<typeof Database>,
|
||||
task: DeleteMaintenanceOperation,
|
||||
): void {
|
||||
switch (task.kind) {
|
||||
case 'session':
|
||||
deleteSession(db, task.sessionId);
|
||||
return;
|
||||
case 'sessions':
|
||||
deleteSessions(db, task.sessionIds);
|
||||
return;
|
||||
case 'video':
|
||||
deleteVideo(db, task.videoId);
|
||||
return;
|
||||
case 'anime':
|
||||
deleteAnime(db, task.animeId);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
export function executeDeleteMaintenanceTask(dbPath: string, task: DeleteMaintenanceTask): void {
|
||||
const db = new Database(dbPath);
|
||||
try {
|
||||
applyPragmas(db);
|
||||
if (task.kind === 'batch') {
|
||||
deleteMaintenanceBatch(db, task.tasks);
|
||||
return;
|
||||
}
|
||||
executeDeleteMaintenanceOperation(db, task);
|
||||
deleteMaintenanceBatch(db, task.kind === 'batch' ? task.tasks : [task]);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { finalizeSessionRecord } from './session';
|
||||
import { nowMs } from './time';
|
||||
import { toDbTimestamp } from './query-shared';
|
||||
import { forEachIdChunk, makePlaceholders, toDbTimestamp } from './query-shared';
|
||||
import type { LifetimeRebuildSummary, SessionState } from './types';
|
||||
|
||||
interface TelemetryRow {
|
||||
@@ -708,6 +708,463 @@ export function rebuildLifetimeSummariesInTransaction(
|
||||
return rebuildLifetimeSummariesInternal(db, rebuiltAtMs);
|
||||
}
|
||||
|
||||
const LOCAL_DAY_EXPR = `CAST(
|
||||
julianday(CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5
|
||||
AS INTEGER
|
||||
)`;
|
||||
|
||||
interface LifetimeMediaRemoval {
|
||||
videoId: number;
|
||||
sessions: number;
|
||||
activeMs: number;
|
||||
cards: number;
|
||||
linesSeen: number;
|
||||
tokensSeen: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* What a pending delete removes from the lifetime summary tables.
|
||||
*
|
||||
* Lifetime totals intentionally outlive raw-session retention, so they can
|
||||
* never be rebuilt from `imm_sessions` without collapsing history to the
|
||||
* retention window. Deletes instead subtract exactly what the deleted rows
|
||||
* contributed: this plan is measured before the rows are removed and applied
|
||||
* after.
|
||||
*/
|
||||
export interface LifetimeRemovalPlan {
|
||||
/** Per surviving video: summed metrics of its deleted, lifetime-applied sessions. */
|
||||
mediaRemovals: LifetimeMediaRemoval[];
|
||||
/** Surviving anime whose lifetime rows must be recomputed from their media rows. */
|
||||
affectedAnimeIds: number[];
|
||||
/** Local-day keys touched by deleted applied sessions, for active_days upkeep. */
|
||||
affectedDayKeys: number[];
|
||||
}
|
||||
|
||||
export function planLifetimeRemovals(
|
||||
db: DatabaseSync,
|
||||
args: {
|
||||
/** Every session being deleted, including ones expanded from video/anime deletes. */
|
||||
deletedSessionIds: number[];
|
||||
/** Deleted sessions whose video survives the delete. */
|
||||
sessionIdsOnSurvivingVideos: number[];
|
||||
deletedVideoIds: number[];
|
||||
deletedAnimeIds: number[];
|
||||
},
|
||||
): LifetimeRemovalPlan {
|
||||
const mediaRemovalsByVideo = new Map<number, LifetimeMediaRemoval>();
|
||||
forEachIdChunk(args.sessionIdsOnSurvivingVideos, (chunk) => {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
s.video_id AS videoId,
|
||||
COUNT(*) AS sessions,
|
||||
COALESCE(SUM(MAX(COALESCE(t.active_watched_ms, s.active_watched_ms, 0), 0)), 0) AS activeMs,
|
||||
COALESCE(SUM(MAX(COALESCE(t.cards_mined, s.cards_mined, 0), 0)), 0) AS cards,
|
||||
COALESCE(SUM(MAX(COALESCE(t.lines_seen, s.lines_seen, 0), 0)), 0) AS linesSeen,
|
||||
COALESCE(SUM(MAX(COALESCE(t.tokens_seen, s.tokens_seen, 0), 0)), 0) AS tokensSeen
|
||||
FROM imm_sessions s
|
||||
JOIN imm_lifetime_applied_sessions a ON a.session_id = s.session_id
|
||||
LEFT JOIN imm_session_telemetry t
|
||||
ON t.telemetry_id = (
|
||||
SELECT telemetry_id
|
||||
FROM imm_session_telemetry
|
||||
WHERE session_id = s.session_id
|
||||
ORDER BY sample_ms DESC, telemetry_id DESC
|
||||
LIMIT 1
|
||||
)
|
||||
WHERE s.session_id IN (${makePlaceholders(chunk)})
|
||||
GROUP BY s.video_id
|
||||
`,
|
||||
)
|
||||
.all(...chunk) as LifetimeMediaRemoval[];
|
||||
for (const row of rows) {
|
||||
const existing = mediaRemovalsByVideo.get(row.videoId);
|
||||
if (!existing) {
|
||||
mediaRemovalsByVideo.set(row.videoId, { ...row });
|
||||
continue;
|
||||
}
|
||||
existing.sessions += row.sessions;
|
||||
existing.activeMs += row.activeMs;
|
||||
existing.cards += row.cards;
|
||||
existing.linesSeen += row.linesSeen;
|
||||
existing.tokensSeen += row.tokensSeen;
|
||||
}
|
||||
});
|
||||
|
||||
const deletedAnimeIds = new Set(args.deletedAnimeIds);
|
||||
const affectedAnimeIds = new Set<number>();
|
||||
forEachIdChunk(args.deletedVideoIds, (chunk) => {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT DISTINCT anime_id AS animeId FROM imm_videos
|
||||
WHERE video_id IN (${makePlaceholders(chunk)}) AND anime_id IS NOT NULL`,
|
||||
)
|
||||
.all(...chunk) as Array<{ animeId: number }>;
|
||||
for (const row of rows) affectedAnimeIds.add(row.animeId);
|
||||
});
|
||||
forEachIdChunk(args.sessionIdsOnSurvivingVideos, (chunk) => {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT DISTINCT v.anime_id AS animeId
|
||||
FROM imm_sessions s
|
||||
JOIN imm_videos v ON v.video_id = s.video_id
|
||||
WHERE s.session_id IN (${makePlaceholders(chunk)}) AND v.anime_id IS NOT NULL`,
|
||||
)
|
||||
.all(...chunk) as Array<{ animeId: number }>;
|
||||
for (const row of rows) affectedAnimeIds.add(row.animeId);
|
||||
});
|
||||
for (const animeId of deletedAnimeIds) affectedAnimeIds.delete(animeId);
|
||||
|
||||
const affectedDayKeys = new Set<number>();
|
||||
forEachIdChunk(args.deletedSessionIds, (chunk) => {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT DISTINCT ${LOCAL_DAY_EXPR} AS dayKey
|
||||
FROM imm_sessions s
|
||||
JOIN imm_lifetime_applied_sessions a ON a.session_id = s.session_id
|
||||
WHERE s.session_id IN (${makePlaceholders(chunk)})`,
|
||||
)
|
||||
.all(...chunk) as Array<{ dayKey: number }>;
|
||||
for (const row of rows) affectedDayKeys.add(row.dayKey);
|
||||
});
|
||||
|
||||
return {
|
||||
mediaRemovals: [...mediaRemovalsByVideo.values()],
|
||||
affectedAnimeIds: [...affectedAnimeIds],
|
||||
affectedDayKeys: [...affectedDayKeys],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a removal plan after the underlying rows are gone.
|
||||
*
|
||||
* Media rows are adjusted by subtraction (pruned-session history stays intact),
|
||||
* affected anime rows are recomputed from their surviving media rows, and the
|
||||
* global row is re-derived from the media/anime tables. `active_days` is the
|
||||
* one metric that can't be derived, so a touched day is only decremented when
|
||||
* no ended session remains on that local day; days whose sessions were pruned
|
||||
* by retention keep their count because pruning never subtracts.
|
||||
*/
|
||||
export function applyLifetimeRemovals(db: DatabaseSync, plan: LifetimeRemovalPlan): void {
|
||||
const updatedAtMs = toDbTimestamp(nowMs());
|
||||
|
||||
const subtractMediaStmt = db.prepare(
|
||||
`
|
||||
UPDATE imm_lifetime_media SET
|
||||
total_sessions = MAX(total_sessions - ?, 0),
|
||||
total_active_ms = MAX(total_active_ms - ?, 0),
|
||||
total_cards = MAX(total_cards - ?, 0),
|
||||
total_lines_seen = MAX(total_lines_seen - ?, 0),
|
||||
total_tokens_seen = MAX(total_tokens_seen - ?, 0),
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE video_id = ?
|
||||
`,
|
||||
);
|
||||
const dropEmptyMediaStmt = db.prepare(
|
||||
'DELETE FROM imm_lifetime_media WHERE video_id = ? AND total_sessions <= 0',
|
||||
);
|
||||
const remainingSessionRangeStmt = db.prepare(
|
||||
`
|
||||
SELECT
|
||||
MIN(CAST(started_at_ms AS REAL)) AS minStartedMs,
|
||||
MAX(CAST(ended_at_ms AS REAL)) AS maxEndedMs
|
||||
FROM imm_sessions
|
||||
WHERE video_id = ? AND ended_at_ms IS NOT NULL
|
||||
`,
|
||||
);
|
||||
const storedMediaRangeStmt = db.prepare(
|
||||
`
|
||||
SELECT CAST(first_watched_ms AS REAL) AS firstWatchedMs
|
||||
FROM imm_lifetime_media
|
||||
WHERE video_id = ?
|
||||
`,
|
||||
);
|
||||
const refreshMediaRangeStmt = db.prepare(
|
||||
`
|
||||
UPDATE imm_lifetime_media SET
|
||||
first_watched_ms = ?,
|
||||
last_watched_ms = ?
|
||||
WHERE video_id = ?
|
||||
`,
|
||||
);
|
||||
|
||||
for (const removal of plan.mediaRemovals) {
|
||||
subtractMediaStmt.run(
|
||||
removal.sessions,
|
||||
removal.activeMs,
|
||||
removal.cards,
|
||||
removal.linesSeen,
|
||||
removal.tokensSeen,
|
||||
updatedAtMs,
|
||||
removal.videoId,
|
||||
);
|
||||
dropEmptyMediaStmt.run(removal.videoId);
|
||||
const stored = storedMediaRangeStmt.get(removal.videoId) as {
|
||||
firstWatchedMs: number | null;
|
||||
} | null;
|
||||
if (!stored) continue;
|
||||
const range = remainingSessionRangeStmt.get(removal.videoId) as {
|
||||
minStartedMs: number | null;
|
||||
maxEndedMs: number | null;
|
||||
} | null;
|
||||
// Retained sessions are always newer than pruned ones, so the surviving
|
||||
// range is authoritative for last_watched while first_watched can only
|
||||
// keep or extend the stored (possibly pruned-history) minimum. When no
|
||||
// session survives, the stored values are all that's left.
|
||||
if (range && range.minStartedMs !== null && range.maxEndedMs !== null) {
|
||||
const firstWatchedMs =
|
||||
stored.firstWatchedMs === null
|
||||
? range.minStartedMs
|
||||
: Math.min(stored.firstWatchedMs, range.minStartedMs);
|
||||
refreshMediaRangeStmt.run(
|
||||
toDbTimestamp(firstWatchedMs),
|
||||
toDbTimestamp(range.maxEndedMs),
|
||||
removal.videoId,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
recomputeLifetimeAnimeFromMedia(db, plan.affectedAnimeIds, updatedAtMs);
|
||||
|
||||
// One pass over the sessions rather than a probe per affected day: the local
|
||||
// day is a computed expression with no index, so each probe would be a table
|
||||
// scan — and the miss case (the day we need to count) is the full-scan one.
|
||||
let removedDays = 0;
|
||||
if (plan.affectedDayKeys.length > 0) {
|
||||
const survivingDayKeys = new Set(
|
||||
(
|
||||
db
|
||||
.prepare(
|
||||
`SELECT DISTINCT ${LOCAL_DAY_EXPR} AS dayKey
|
||||
FROM imm_sessions
|
||||
WHERE ended_at_ms IS NOT NULL`,
|
||||
)
|
||||
.all() as Array<{ dayKey: number }>
|
||||
).map((row) => row.dayKey),
|
||||
);
|
||||
for (const dayKey of plan.affectedDayKeys) {
|
||||
if (!survivingDayKeys.has(dayKey)) removedDays += 1;
|
||||
}
|
||||
}
|
||||
|
||||
recomputeLifetimeGlobalFromSummaries(db, { removedActiveDays: removedDays, updatedAtMs });
|
||||
}
|
||||
|
||||
/**
|
||||
* Recompute lifetime anime rows exactly from their surviving media rows.
|
||||
*
|
||||
* Media rows are the durable per-video ledger (they outlive session pruning and
|
||||
* follow a video when it moves between anime), so this is the correct refresh
|
||||
* after merges, moves, and deletes. Anime with no media rows left are dropped.
|
||||
*/
|
||||
export function recomputeLifetimeAnimeFromMedia(
|
||||
db: DatabaseSync,
|
||||
animeIds: number[],
|
||||
updatedAtMs = toDbTimestamp(nowMs()),
|
||||
): void {
|
||||
if (animeIds.length === 0) return;
|
||||
|
||||
const animeSummaryStmt = db.prepare(
|
||||
`
|
||||
SELECT
|
||||
COUNT(*) AS episodeRows,
|
||||
COALESCE(SUM(m.total_sessions), 0) AS totalSessions,
|
||||
COALESCE(SUM(m.total_active_ms), 0) AS totalActiveMs,
|
||||
COALESCE(SUM(m.total_cards), 0) AS totalCards,
|
||||
COALESCE(SUM(m.total_lines_seen), 0) AS totalLinesSeen,
|
||||
COALESCE(SUM(m.total_tokens_seen), 0) AS totalTokensSeen,
|
||||
COALESCE(SUM(CASE WHEN m.completed > 0 THEN 1 ELSE 0 END), 0) AS episodesCompleted,
|
||||
MIN(CAST(m.first_watched_ms AS REAL)) AS firstWatchedMs,
|
||||
MAX(CAST(m.last_watched_ms AS REAL)) AS lastWatchedMs
|
||||
FROM imm_lifetime_media m
|
||||
JOIN imm_videos v ON v.video_id = m.video_id
|
||||
WHERE v.anime_id = ?
|
||||
`,
|
||||
);
|
||||
const dropAnimeStmt = db.prepare('DELETE FROM imm_lifetime_anime WHERE anime_id = ?');
|
||||
const upsertAnimeStmt = db.prepare(
|
||||
`
|
||||
INSERT INTO imm_lifetime_anime(
|
||||
anime_id,
|
||||
total_sessions,
|
||||
total_active_ms,
|
||||
total_cards,
|
||||
total_lines_seen,
|
||||
total_tokens_seen,
|
||||
episodes_started,
|
||||
episodes_completed,
|
||||
first_watched_ms,
|
||||
last_watched_ms,
|
||||
CREATED_DATE,
|
||||
LAST_UPDATE_DATE
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(anime_id) DO UPDATE SET
|
||||
total_sessions = excluded.total_sessions,
|
||||
total_active_ms = excluded.total_active_ms,
|
||||
total_cards = excluded.total_cards,
|
||||
total_lines_seen = excluded.total_lines_seen,
|
||||
total_tokens_seen = excluded.total_tokens_seen,
|
||||
episodes_started = excluded.episodes_started,
|
||||
episodes_completed = excluded.episodes_completed,
|
||||
first_watched_ms = excluded.first_watched_ms,
|
||||
last_watched_ms = excluded.last_watched_ms,
|
||||
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
|
||||
`,
|
||||
);
|
||||
|
||||
for (const animeId of animeIds) {
|
||||
const summary = animeSummaryStmt.get(animeId) as {
|
||||
episodeRows: number;
|
||||
totalSessions: number;
|
||||
totalActiveMs: number;
|
||||
totalCards: number;
|
||||
totalLinesSeen: number;
|
||||
totalTokensSeen: number;
|
||||
episodesCompleted: number;
|
||||
firstWatchedMs: number | null;
|
||||
lastWatchedMs: number | null;
|
||||
};
|
||||
if (Number(summary.episodeRows) === 0) {
|
||||
dropAnimeStmt.run(animeId);
|
||||
continue;
|
||||
}
|
||||
upsertAnimeStmt.run(
|
||||
animeId,
|
||||
summary.totalSessions,
|
||||
summary.totalActiveMs,
|
||||
summary.totalCards,
|
||||
summary.totalLinesSeen,
|
||||
summary.totalTokensSeen,
|
||||
summary.episodeRows,
|
||||
summary.episodesCompleted,
|
||||
summary.firstWatchedMs === null ? null : toDbTimestamp(summary.firstWatchedMs),
|
||||
summary.lastWatchedMs === null ? null : toDbTimestamp(summary.lastWatchedMs),
|
||||
updatedAtMs,
|
||||
updatedAtMs,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-derive the global lifetime row from the media/anime summary tables.
|
||||
*
|
||||
* Every global metric except active_days is a pure aggregate of those tables;
|
||||
* active_days can't be derived, so callers pass how many day slots their change
|
||||
* removed (0 for moves/merges, which never touch sessions).
|
||||
*/
|
||||
export function recomputeLifetimeGlobalFromSummaries(
|
||||
db: DatabaseSync,
|
||||
options: { removedActiveDays?: number; updatedAtMs?: string } = {},
|
||||
): void {
|
||||
const updatedAtMs = options.updatedAtMs ?? toDbTimestamp(nowMs());
|
||||
const mediaTotals = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
COUNT(*) AS episodesStarted,
|
||||
COALESCE(SUM(total_sessions), 0) AS totalSessions,
|
||||
COALESCE(SUM(total_active_ms), 0) AS totalActiveMs,
|
||||
COALESCE(SUM(total_cards), 0) AS totalCards,
|
||||
COALESCE(SUM(CASE WHEN completed > 0 THEN 1 ELSE 0 END), 0) AS episodesCompleted
|
||||
FROM imm_lifetime_media
|
||||
`,
|
||||
)
|
||||
.get() as {
|
||||
episodesStarted: number;
|
||||
totalSessions: number;
|
||||
totalActiveMs: number;
|
||||
totalCards: number;
|
||||
episodesCompleted: number;
|
||||
};
|
||||
const animeCompletedRow = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT COUNT(*) AS animeCompleted
|
||||
FROM imm_lifetime_anime la
|
||||
JOIN imm_anime a ON a.anime_id = la.anime_id
|
||||
WHERE a.episodes_total IS NOT NULL
|
||||
AND a.episodes_total > 0
|
||||
AND la.episodes_completed >= a.episodes_total
|
||||
`,
|
||||
)
|
||||
.get() as { animeCompleted: number };
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE imm_lifetime_global SET
|
||||
total_sessions = ?,
|
||||
total_active_ms = ?,
|
||||
total_cards = ?,
|
||||
episodes_started = ?,
|
||||
episodes_completed = ?,
|
||||
anime_completed = ?,
|
||||
active_days = MAX(active_days - ?, 0),
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE global_id = 1
|
||||
`,
|
||||
).run(
|
||||
mediaTotals.totalSessions,
|
||||
mediaTotals.totalActiveMs,
|
||||
mediaTotals.totalCards,
|
||||
mediaTotals.episodesStarted,
|
||||
mediaTotals.episodesCompleted,
|
||||
animeCompletedRow.animeCompleted,
|
||||
options.removedActiveDays ?? 0,
|
||||
updatedAtMs,
|
||||
);
|
||||
}
|
||||
|
||||
export interface LifetimeRepairSummary {
|
||||
recomputedAnime: number;
|
||||
repairedAtMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-destructive lifetime repair: recompute every anime row and the global row
|
||||
* from the per-video media ledger.
|
||||
*
|
||||
* Unlike {@link rebuildLifetimeSummaries}, this never resets the tables from
|
||||
* retained sessions, so lifetime history older than the session retention
|
||||
* window survives. The one exception is a database whose lifetime tables were
|
||||
* never populated — there is no ledger to repair from, so it bootstraps with
|
||||
* the full rebuild instead.
|
||||
*/
|
||||
export function repairLifetimeSummariesFromMedia(db: DatabaseSync): LifetimeRepairSummary {
|
||||
if (shouldBackfillLifetimeSummaries(db)) {
|
||||
const rebuilt = rebuildLifetimeSummaries(db);
|
||||
const animeRow = db
|
||||
.prepare('SELECT COUNT(*) AS count FROM imm_lifetime_anime')
|
||||
.get() as ExistenceRow;
|
||||
return { recomputedAnime: Number(animeRow.count), repairedAtMs: rebuilt.rebuiltAtMs };
|
||||
}
|
||||
|
||||
const repairedAtMs = nowMs();
|
||||
db.exec('BEGIN');
|
||||
try {
|
||||
const animeIds = new Set<number>();
|
||||
for (const row of db
|
||||
.prepare('SELECT DISTINCT anime_id AS animeId FROM imm_videos WHERE anime_id IS NOT NULL')
|
||||
.all() as Array<{ animeId: number }>) {
|
||||
animeIds.add(row.animeId);
|
||||
}
|
||||
for (const row of db
|
||||
.prepare('SELECT anime_id AS animeId FROM imm_lifetime_anime')
|
||||
.all() as Array<{ animeId: number }>) {
|
||||
animeIds.add(row.animeId);
|
||||
}
|
||||
const updatedAtMs = toDbTimestamp(repairedAtMs);
|
||||
recomputeLifetimeAnimeFromMedia(db, [...animeIds], updatedAtMs);
|
||||
recomputeLifetimeGlobalFromSummaries(db, { updatedAtMs });
|
||||
db.exec('COMMIT');
|
||||
return { recomputedAnime: animeIds.size, repairedAtMs };
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function reconcileStaleActiveSessions(db: DatabaseSync): number {
|
||||
const sessions = getRetainedStaleActiveSessions(db);
|
||||
if (sessions.length === 0) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { rebuildLifetimeSummariesInTransaction } from './lifetime';
|
||||
import { applyLifetimeRemovals, planLifetimeRemovals } from './lifetime';
|
||||
import { getRollupGroupsForSessions, refreshRollupsForGroupsInTransaction } from './maintenance';
|
||||
import {
|
||||
applyLexicalRemovals,
|
||||
@@ -8,9 +8,10 @@ import {
|
||||
forEachIdChunk,
|
||||
makePlaceholders,
|
||||
planLexicalRemovalsForSessions,
|
||||
SQLITE_ID_CHUNK_SIZE,
|
||||
planLexicalRemovalsForVideos,
|
||||
type LexicalRemovalPlan,
|
||||
} from './query-shared';
|
||||
import type { RollupGroup } from './maintenance';
|
||||
|
||||
export type DeleteMaintenanceOperation =
|
||||
| { kind: 'session'; sessionId: number }
|
||||
@@ -59,40 +60,59 @@ function selectIds(
|
||||
return ids;
|
||||
}
|
||||
|
||||
function planLexicalRemovalsInChunks(db: DatabaseSync, sessionIds: number[]): LexicalRemovalPlan {
|
||||
const combined: LexicalRemovalPlan = { words: [], kanji: [] };
|
||||
const merge = (target: LexicalRemovalPlan['words'], source: LexicalRemovalPlan['words']) => {
|
||||
const byId = new Map(target.map((entry) => [entry.id, entry]));
|
||||
for (const entry of source) {
|
||||
const existing = byId.get(entry.id);
|
||||
if (!existing) {
|
||||
const added = { ...entry };
|
||||
target.push(added);
|
||||
byId.set(entry.id, added);
|
||||
continue;
|
||||
}
|
||||
existing.removedFrequency += entry.removedFrequency;
|
||||
if (
|
||||
entry.removedFirstSeenMs !== null &&
|
||||
(existing.removedFirstSeenMs === null ||
|
||||
entry.removedFirstSeenMs < existing.removedFirstSeenMs)
|
||||
) {
|
||||
existing.removedFirstSeenMs = entry.removedFirstSeenMs;
|
||||
}
|
||||
if (
|
||||
entry.removedLastSeenMs !== null &&
|
||||
(existing.removedLastSeenMs === null ||
|
||||
entry.removedLastSeenMs > existing.removedLastSeenMs)
|
||||
) {
|
||||
existing.removedLastSeenMs = entry.removedLastSeenMs;
|
||||
}
|
||||
function mergeLexicalPlanEntries(
|
||||
target: LexicalRemovalPlan['words'],
|
||||
source: LexicalRemovalPlan['words'],
|
||||
): void {
|
||||
const byId = new Map(target.map((entry) => [entry.id, entry]));
|
||||
for (const entry of source) {
|
||||
const existing = byId.get(entry.id);
|
||||
if (!existing) {
|
||||
const added = { ...entry };
|
||||
target.push(added);
|
||||
byId.set(entry.id, added);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
existing.removedFrequency += entry.removedFrequency;
|
||||
if (
|
||||
entry.removedFirstSeenMs !== null &&
|
||||
(existing.removedFirstSeenMs === null ||
|
||||
entry.removedFirstSeenMs < existing.removedFirstSeenMs)
|
||||
) {
|
||||
existing.removedFirstSeenMs = entry.removedFirstSeenMs;
|
||||
}
|
||||
if (
|
||||
entry.removedLastSeenMs !== null &&
|
||||
(existing.removedLastSeenMs === null || entry.removedLastSeenMs > existing.removedLastSeenMs)
|
||||
) {
|
||||
existing.removedLastSeenMs = entry.removedLastSeenMs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
forEachIdChunk(sessionIds, (chunk) => {
|
||||
const plan = planLexicalRemovalsForSessions(db, chunk);
|
||||
merge(combined.words, plan.words);
|
||||
merge(combined.kanji, plan.kanji);
|
||||
function mergeLexicalPlans(target: LexicalRemovalPlan, source: LexicalRemovalPlan): void {
|
||||
mergeLexicalPlanEntries(target.words, source.words);
|
||||
mergeLexicalPlanEntries(target.kanji, source.kanji);
|
||||
}
|
||||
|
||||
/**
|
||||
* Plan what the delete removes from imm_words/imm_kanji.
|
||||
*
|
||||
* Deleted videos are planned by video so orphaned subtitle lines (whose session
|
||||
* is already gone) still get subtracted; sessions on surviving videos are
|
||||
* planned by session. The two scopes are disjoint, so nothing is counted twice.
|
||||
*/
|
||||
function planLexicalRemovalsForDelete(
|
||||
db: DatabaseSync,
|
||||
sessionIdsOnSurvivingVideos: number[],
|
||||
videoIds: number[],
|
||||
): LexicalRemovalPlan {
|
||||
const combined: LexicalRemovalPlan = { words: [], kanji: [] };
|
||||
forEachIdChunk(sessionIdsOnSurvivingVideos, (chunk) => {
|
||||
mergeLexicalPlans(combined, planLexicalRemovalsForSessions(db, chunk));
|
||||
});
|
||||
forEachIdChunk(videoIds, (chunk) => {
|
||||
mergeLexicalPlans(combined, planLexicalRemovalsForVideos(db, chunk));
|
||||
});
|
||||
return combined;
|
||||
}
|
||||
@@ -121,24 +141,37 @@ export function deleteMaintenanceBatch(
|
||||
}
|
||||
|
||||
const videoIdList = [...videoIds];
|
||||
for (const sessionId of selectIds(
|
||||
db,
|
||||
(placeholders) => `SELECT session_id FROM imm_sessions WHERE video_id IN (${placeholders})`,
|
||||
videoIdList,
|
||||
'session_id',
|
||||
)) {
|
||||
sessionIds.add(sessionId);
|
||||
}
|
||||
const sessionIdsOnDeletedVideos = new Set(
|
||||
selectIds(
|
||||
db,
|
||||
(placeholders) => `SELECT session_id FROM imm_sessions WHERE video_id IN (${placeholders})`,
|
||||
videoIdList,
|
||||
'session_id',
|
||||
),
|
||||
);
|
||||
for (const sessionId of sessionIdsOnDeletedVideos) sessionIds.add(sessionId);
|
||||
|
||||
const sessionIdList = [...sessionIds];
|
||||
const lexicalRemovals = planLexicalRemovalsInChunks(db, sessionIdList);
|
||||
const affectedRollupGroups = sessionIdList
|
||||
.flatMap((_, index) =>
|
||||
index % SQLITE_ID_CHUNK_SIZE === 0
|
||||
? getRollupGroupsForSessions(db, sessionIdList.slice(index, index + SQLITE_ID_CHUNK_SIZE))
|
||||
: [],
|
||||
)
|
||||
.filter((group) => !videoIds.has(group.videoId));
|
||||
const sessionIdsOnSurvivingVideos = sessionIdList.filter(
|
||||
(sessionId) => !sessionIdsOnDeletedVideos.has(sessionId),
|
||||
);
|
||||
|
||||
// Both plans must be measured before any rows are removed.
|
||||
const lexicalRemovals = planLexicalRemovalsForDelete(
|
||||
db,
|
||||
sessionIdsOnSurvivingVideos,
|
||||
videoIdList,
|
||||
);
|
||||
const lifetimeRemovals = planLifetimeRemovals(db, {
|
||||
deletedSessionIds: sessionIdList,
|
||||
sessionIdsOnSurvivingVideos,
|
||||
deletedVideoIds: videoIdList,
|
||||
deletedAnimeIds: animeIdList,
|
||||
});
|
||||
const affectedRollupGroups: RollupGroup[] = [];
|
||||
forEachIdChunk(sessionIdsOnSurvivingVideos, (chunk) => {
|
||||
affectedRollupGroups.push(...getRollupGroupsForSessions(db, chunk));
|
||||
});
|
||||
const coverBlobHashes = new Set<string>();
|
||||
if (videoIdList.length > 0) {
|
||||
forEachIdChunk(videoIdList, (chunk) => {
|
||||
@@ -152,26 +185,31 @@ export function deleteMaintenanceBatch(
|
||||
.all(...chunk) as Array<{ coverBlobHash: string }>;
|
||||
for (const row of artRows) coverBlobHashes.add(row.coverBlobHash);
|
||||
});
|
||||
|
||||
deleteSessionsByIds(db, sessionIdList);
|
||||
forEachIdChunk(videoIdList, (chunk) => {
|
||||
const placeholders = makePlaceholders(chunk);
|
||||
db.prepare(`DELETE FROM imm_subtitle_lines WHERE video_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_daily_rollups WHERE video_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_monthly_rollups WHERE video_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_media_art WHERE video_id IN (${placeholders})`).run(...chunk);
|
||||
db.prepare(`DELETE FROM imm_videos WHERE video_id IN (${placeholders})`).run(...chunk);
|
||||
});
|
||||
} else {
|
||||
deleteSessionsByIds(db, sessionIdList);
|
||||
}
|
||||
|
||||
deleteSessionsByIds(db, sessionIdList);
|
||||
forEachIdChunk(sessionIdList, (chunk) => {
|
||||
const placeholders = makePlaceholders(chunk);
|
||||
db.prepare(
|
||||
`DELETE FROM imm_lifetime_applied_sessions WHERE session_id IN (${placeholders})`,
|
||||
).run(...chunk);
|
||||
});
|
||||
forEachIdChunk(videoIdList, (chunk) => {
|
||||
const placeholders = makePlaceholders(chunk);
|
||||
db.prepare(`DELETE FROM imm_subtitle_lines WHERE video_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_daily_rollups WHERE video_id IN (${placeholders})`).run(...chunk);
|
||||
db.prepare(`DELETE FROM imm_monthly_rollups WHERE video_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_media_art WHERE video_id IN (${placeholders})`).run(...chunk);
|
||||
db.prepare(`DELETE FROM imm_lifetime_media WHERE video_id IN (${placeholders})`).run(
|
||||
...chunk,
|
||||
);
|
||||
db.prepare(`DELETE FROM imm_videos WHERE video_id IN (${placeholders})`).run(...chunk);
|
||||
});
|
||||
|
||||
for (const coverBlobHash of coverBlobHashes) {
|
||||
cleanupUnusedCoverArtBlobHash(db, coverBlobHash);
|
||||
}
|
||||
@@ -186,7 +224,7 @@ export function deleteMaintenanceBatch(
|
||||
}
|
||||
|
||||
applyLexicalRemovals(db, lexicalRemovals);
|
||||
rebuildLifetimeSummariesInTransaction(db);
|
||||
applyLifetimeRemovals(db, lifetimeRemovals);
|
||||
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
|
||||
@@ -1,20 +1,16 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { buildCoverBlobReference, normalizeCoverBlobBytes } from './storage';
|
||||
import { rebuildLifetimeSummaries, rebuildLifetimeSummariesInTransaction } from './lifetime';
|
||||
import { getRollupGroupsForSessions, refreshRollupsForGroupsInTransaction } from './maintenance';
|
||||
import { repairLifetimeSummariesFromMedia } from './lifetime';
|
||||
import { nowMs } from './time';
|
||||
import { resolveAnimeAnilistConflict } from './anime-season-repair';
|
||||
import { deleteMaintenanceBatch } from './query-delete-maintenance';
|
||||
import { PartOfSpeech, type MergedToken } from '../../../types';
|
||||
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
|
||||
import { deriveStoredPartOfSpeech } from '../tokenizer/part-of-speech';
|
||||
import {
|
||||
applyLexicalRemovals,
|
||||
cleanupUnusedCoverArtBlobHash,
|
||||
deleteSessionsByIds,
|
||||
findSharedCoverBlobHash,
|
||||
planLexicalRemovalsForSessions,
|
||||
planLexicalRemovalsForVideos,
|
||||
toDbMs,
|
||||
toDbTimestamp,
|
||||
} from './query-shared';
|
||||
@@ -425,7 +421,7 @@ export function updateAnimeAnilistInfo(
|
||||
} | null;
|
||||
if (!row?.anime_id) return;
|
||||
|
||||
const repair = resolveAnimeAnilistConflict(db, row.anime_id, info.anilistId);
|
||||
resolveAnimeAnilistConflict(db, row.anime_id, info.anilistId);
|
||||
const targetRow = db
|
||||
.prepare('SELECT anime_id FROM imm_videos WHERE video_id = ?')
|
||||
.get(videoId) as {
|
||||
@@ -454,9 +450,10 @@ export function updateAnimeAnilistInfo(
|
||||
toDbTimestamp(nowMs()),
|
||||
targetRow.anime_id,
|
||||
);
|
||||
if (repair.movedVideos > 0 || repair.deletedAnimeRows > 0) {
|
||||
rebuildLifetimeSummaries(db);
|
||||
}
|
||||
// Moves change which anime owns the media rows, and an episodes_total change
|
||||
// can flip anime_completed even without a move — both are derivable from the
|
||||
// summary tables, so a full (retention-lossy) rebuild is never needed here.
|
||||
repairLifetimeSummariesFromMedia(db);
|
||||
}
|
||||
|
||||
export function markVideoWatched(db: DatabaseSync, videoId: number, watched: boolean): void {
|
||||
@@ -482,136 +479,22 @@ export function isVideoWatched(db: DatabaseSync, videoId: number): boolean {
|
||||
}
|
||||
|
||||
export function deleteSession(db: DatabaseSync, sessionId: number): void {
|
||||
const sessionIds = [sessionId];
|
||||
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
// Measured inside the write lock: the plan records what the delete removes,
|
||||
// and applying a plan taken against a different snapshot would subtract the
|
||||
// wrong totals from imm_words/imm_kanji.
|
||||
const lexicalRemovals = planLexicalRemovalsForSessions(db, sessionIds);
|
||||
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
|
||||
deleteSessionsByIds(db, sessionIds);
|
||||
applyLexicalRemovals(db, lexicalRemovals);
|
||||
rebuildLifetimeSummariesInTransaction(db);
|
||||
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
deleteMaintenanceBatch(db, [{ kind: 'session', sessionId }]);
|
||||
}
|
||||
|
||||
export function deleteSessions(db: DatabaseSync, sessionIds: number[]): void {
|
||||
if (sessionIds.length === 0) return;
|
||||
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const lexicalRemovals = planLexicalRemovalsForSessions(db, sessionIds);
|
||||
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
|
||||
deleteSessionsByIds(db, sessionIds);
|
||||
applyLexicalRemovals(db, lexicalRemovals);
|
||||
rebuildLifetimeSummariesInTransaction(db);
|
||||
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
deleteMaintenanceBatch(db, [{ kind: 'sessions', sessionIds }]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an entire library entry: every episode of the anime, all of their
|
||||
* sessions and derived stats, and the anime row itself.
|
||||
*
|
||||
* Mirrors {@link deleteVideo} per episode, but batches the lexical refresh and
|
||||
* lifetime rebuild into a single transaction so a multi-episode title doesn't
|
||||
* pay for one full rebuild per episode.
|
||||
*/
|
||||
export function deleteAnime(db: DatabaseSync, animeId: number): void {
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const videoIds = (
|
||||
db.prepare('SELECT video_id FROM imm_videos WHERE anime_id = ?').all(animeId) as Array<{
|
||||
video_id: number;
|
||||
}>
|
||||
).map((row) => row.video_id);
|
||||
|
||||
const lexicalRemovals = planLexicalRemovalsForVideos(db, videoIds);
|
||||
const coverBlobHashes: string[] = [];
|
||||
const sessionIds: number[] = [];
|
||||
for (const videoId of videoIds) {
|
||||
const artRow = db
|
||||
.prepare('SELECT cover_blob_hash AS coverBlobHash FROM imm_media_art WHERE video_id = ?')
|
||||
.get(videoId) as { coverBlobHash: string | null } | undefined;
|
||||
if (artRow?.coverBlobHash) {
|
||||
coverBlobHashes.push(artRow.coverBlobHash);
|
||||
}
|
||||
const sessions = db
|
||||
.prepare('SELECT session_id FROM imm_sessions WHERE video_id = ?')
|
||||
.all(videoId) as Array<{ session_id: number }>;
|
||||
sessionIds.push(...sessions.map((session) => session.session_id));
|
||||
}
|
||||
|
||||
deleteSessionsByIds(db, sessionIds);
|
||||
const deleteLinesStmt = db.prepare('DELETE FROM imm_subtitle_lines WHERE video_id = ?');
|
||||
const deleteDailyStmt = db.prepare('DELETE FROM imm_daily_rollups WHERE video_id = ?');
|
||||
const deleteMonthlyStmt = db.prepare('DELETE FROM imm_monthly_rollups WHERE video_id = ?');
|
||||
const deleteArtStmt = db.prepare('DELETE FROM imm_media_art WHERE video_id = ?');
|
||||
const deleteVideoStmt = db.prepare('DELETE FROM imm_videos WHERE video_id = ?');
|
||||
for (const videoId of videoIds) {
|
||||
deleteLinesStmt.run(videoId);
|
||||
deleteDailyStmt.run(videoId);
|
||||
deleteMonthlyStmt.run(videoId);
|
||||
deleteArtStmt.run(videoId);
|
||||
deleteVideoStmt.run(videoId);
|
||||
}
|
||||
for (const coverBlobHash of new Set(coverBlobHashes)) {
|
||||
cleanupUnusedCoverArtBlobHash(db, coverBlobHash);
|
||||
}
|
||||
db.prepare('DELETE FROM imm_lifetime_anime WHERE anime_id = ?').run(animeId);
|
||||
db.prepare('DELETE FROM imm_anime WHERE anime_id = ?').run(animeId);
|
||||
applyLexicalRemovals(db, lexicalRemovals);
|
||||
rebuildLifetimeSummariesInTransaction(db);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
deleteMaintenanceBatch(db, [{ kind: 'anime', animeId }]);
|
||||
}
|
||||
|
||||
export function deleteVideo(db: DatabaseSync, videoId: number): void {
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const artRow = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT cover_blob_hash AS coverBlobHash
|
||||
FROM imm_media_art
|
||||
WHERE video_id = ?
|
||||
`,
|
||||
)
|
||||
.get(videoId) as { coverBlobHash: string | null } | undefined;
|
||||
const lexicalRemovals = planLexicalRemovalsForVideos(db, [videoId]);
|
||||
const sessions = db
|
||||
.prepare('SELECT session_id FROM imm_sessions WHERE video_id = ?')
|
||||
.all(videoId) as Array<{ session_id: number }>;
|
||||
|
||||
deleteSessionsByIds(
|
||||
db,
|
||||
sessions.map((session) => session.session_id),
|
||||
);
|
||||
db.prepare('DELETE FROM imm_subtitle_lines WHERE video_id = ?').run(videoId);
|
||||
db.prepare('DELETE FROM imm_daily_rollups WHERE video_id = ?').run(videoId);
|
||||
db.prepare('DELETE FROM imm_monthly_rollups WHERE video_id = ?').run(videoId);
|
||||
db.prepare('DELETE FROM imm_media_art WHERE video_id = ?').run(videoId);
|
||||
cleanupUnusedCoverArtBlobHash(db, artRow?.coverBlobHash ?? null);
|
||||
db.prepare('DELETE FROM imm_videos WHERE video_id = ?').run(videoId);
|
||||
applyLexicalRemovals(db, lexicalRemovals);
|
||||
rebuildLifetimeSummariesInTransaction(db);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
deleteMaintenanceBatch(db, [{ kind: 'video', videoId }]);
|
||||
}
|
||||
|
||||
@@ -307,10 +307,13 @@ function toStoredSeenSeconds(ms: number | null): number | null {
|
||||
* Apply a removal plan to the vocabulary aggregates.
|
||||
*
|
||||
* Frequencies are adjusted by subtraction, which is exact and touches only the
|
||||
* affected rows. `first_seen`/`last_seen` only need a rescan when the removed
|
||||
* lines held the current extreme, and rows whose frequency reaches zero are
|
||||
* verified against the surviving occurrences before deletion — so stored counts
|
||||
* that have drifted still converge on the truth instead of dropping a live row.
|
||||
* affected rows. When the removed lines held a `first_seen`/`last_seen`
|
||||
* extreme, the new extremes come from MIN/MAX index-endpoint seeks on the
|
||||
* occurrence covering index — never a re-aggregation of every occurrence, which
|
||||
* for common particles means scanning the whole library. The full re-aggregate
|
||||
* survives only as the repair path: rows whose stored frequency reaches zero
|
||||
* while occurrences remain (drift), and rows with undated pre-migration
|
||||
* occurrences the seeks would skip.
|
||||
*/
|
||||
export function applyLexicalRemovals(db: DatabaseSync, plan: LexicalRemovalPlan): void {
|
||||
applyRemovalsForEntity(db, 'word', plan.words);
|
||||
@@ -339,6 +342,21 @@ function applyRemovalsForEntity(
|
||||
`SELECT 1 AS found FROM ${occurrenceTable} WHERE ${col} = ? LIMIT 1`,
|
||||
);
|
||||
const deleteStmt = db.prepare(`DELETE FROM ${entityTable} WHERE id = ?`);
|
||||
// Seeks to the front of this entity's index range, where NULL seen_ms sorts.
|
||||
const hasUndatedOccurrenceStmt = db.prepare(
|
||||
`SELECT 1 AS found FROM ${occurrenceTable} WHERE ${col} = ? AND seen_ms IS NULL LIMIT 1`,
|
||||
);
|
||||
// Kept as separate single-aggregate statements so SQLite's min/max
|
||||
// optimization turns each into an index-endpoint seek instead of a scan.
|
||||
const minSeenStmt = db.prepare(
|
||||
`SELECT MIN(seen_ms) AS value FROM ${occurrenceTable} WHERE ${col} = ?`,
|
||||
);
|
||||
const maxSeenStmt = db.prepare(
|
||||
`SELECT MAX(seen_ms) AS value FROM ${occurrenceTable} WHERE ${col} = ?`,
|
||||
);
|
||||
const updateAggregatesStmt = db.prepare(
|
||||
`UPDATE ${entityTable} SET frequency = ?, first_seen = ?, last_seen = ? WHERE id = ?`,
|
||||
);
|
||||
|
||||
const needsExactRefresh: number[] = [];
|
||||
|
||||
@@ -371,7 +389,26 @@ function applyRemovalsForEntity(
|
||||
current.lastSeen === null ||
|
||||
(removedLastSeen !== null && removedLastSeen >= current.lastSeen);
|
||||
if (firstSeenMayHaveMoved || lastSeenMayHaveMoved) {
|
||||
needsExactRefresh.push(removal.id);
|
||||
// Undated pre-migration occurrences are invisible to the seeks below;
|
||||
// fall back to the full re-aggregate that resolves their dates.
|
||||
if (hasUndatedOccurrenceStmt.get(removal.id)) {
|
||||
needsExactRefresh.push(removal.id);
|
||||
continue;
|
||||
}
|
||||
const minSeenMs = (minSeenStmt.get(removal.id) as { value: number | null }).value;
|
||||
const maxSeenMs = (maxSeenStmt.get(removal.id) as { value: number | null }).value;
|
||||
if (minSeenMs === null || maxSeenMs === null) {
|
||||
// Frequency says occurrences remain but none exist: stale row, let the
|
||||
// exact refresh reconcile (it deletes rows with nothing left).
|
||||
needsExactRefresh.push(removal.id);
|
||||
continue;
|
||||
}
|
||||
updateAggregatesStmt.run(
|
||||
nextFrequency,
|
||||
Math.floor(Number(minSeenMs) / 1000),
|
||||
Math.floor(Number(maxSeenMs) / 1000),
|
||||
removal.id,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user