fix(stats): prevent delete maintenance from freezing the UI

- Serialize and coalesce delete requests through a dedicated scheduler
- Chunk large SQLite ID lists to stay below variable limits
This commit is contained in:
2026-08-11 23:11:45 -07:00
parent 320db591ae
commit 675cb33519
10 changed files with 525 additions and 155 deletions
+1 -1
View File
@@ -25,7 +25,7 @@ Read when: you need to find the owner module for a behavior or test surface
- Anki workflow: `src/anki-integration/`, `src/core/services/anki-jimaku*.ts` - Anki workflow: `src/anki-integration/`, `src/core/services/anki-jimaku*.ts`
- Immersion tracking: `src/core/services/immersion-tracker/` - Immersion tracking: `src/core/services/immersion-tracker/`
Includes stats storage/query schema such as `imm_videos`, `imm_media_art`, and `imm_youtube_videos` for per-video and YouTube-specific library metadata. Includes stats storage/query schema such as `imm_videos`, `imm_media_art`, and `imm_youtube_videos` for per-video and YouTube-specific library metadata.
Expensive stats deletion and summary rebuilds run in `delete-maintenance-worker-thread.ts`; the tracker queues playback writes until the serialized worker task finishes and coalesces concurrent requests into one transaction, lexical update, rollup refresh, and lifetime rebuild. `delete-maintenance-scheduler.ts` coalesces and serializes stats deletes; expensive deletion and summary rebuilds run in `delete-maintenance-worker-thread.ts` while the tracker queues playback writes. Each batch uses one transaction, lexical update, rollup refresh, and lifetime rebuild.
- AniList tracking + character dictionary: `src/core/services/anilist/`, `src/main/runtime/composers/anilist-*`, `src/main/character-dictionary-runtime.ts`, `src/main/character-dictionary-runtime/` - AniList tracking + character dictionary: `src/core/services/anilist/`, `src/main/runtime/composers/anilist-*`, `src/main/character-dictionary-runtime.ts`, `src/main/character-dictionary-runtime/`
- Jellyfin integration: `src/core/services/jellyfin*.ts`, `src/main/runtime/composers/jellyfin-*` - Jellyfin integration: `src/core/services/jellyfin*.ts`, `src/main/runtime/composers/jellyfin-*`
- Window trackers: `src/window-trackers/` - Window trackers: `src/window-trackers/`
@@ -1545,8 +1545,9 @@ test('concurrent delete requests share one maintenance worker batch', async () =
); );
const firstDelete = tracker.deleteSession(201); const firstDelete = tracker.deleteSession(201);
await new Promise<void>((resolve) => setTimeout(resolve, 0)); const secondDelete = tracker.deleteSessions([202, 203]);
await Promise.all([firstDelete, tracker.deleteSessions([202, 203]), tracker.deleteVideo(204)]); const thirdDelete = tracker.deleteVideo(204);
await Promise.all([firstDelete, secondDelete, thirdDelete]);
assert.equal(tasks.length, 1, 'concurrent deletes should use one maintenance pass'); assert.equal(tasks.length, 1, 'concurrent deletes should use one maintenance pass');
assert.deepEqual(tasks[0], { assert.deepEqual(tasks[0], {
@@ -1611,6 +1612,51 @@ test('destroy rejects delete requests waiting behind active maintenance', async
} }
}); });
test('delete requested after destroy rejects without running maintenance', async () => {
const dbPath = makeDbPath();
let maintenanceCalls = 0;
const Ctor = await loadTrackerCtor();
const tracker = new Ctor(
{ dbPath },
{
runDeleteMaintenanceTask: async () => {
maintenanceCalls += 1;
},
},
);
tracker.destroy();
await assert.rejects(tracker.deleteSession(303), /shutting down/);
assert.equal(maintenanceCalls, 0);
cleanupDbPath(dbPath);
});
test('deleteSessions skips maintenance when no sessions are deletable', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
const tasks: unknown[] = [];
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath },
{
runDeleteMaintenanceTask: async (_path, task) => {
tasks.push(task);
},
},
);
await tracker.deleteSessions([]);
assert.deepEqual(tasks, []);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('queued video delete is skipped when that video becomes active before dispatch', async () => { test('queued video delete is skipped when that video becomes active before dispatch', async () => {
const dbPath = makeDbPath(); const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null; let tracker: ImmersionTrackerService | null = null;
+24 -93
View File
@@ -87,14 +87,11 @@ import {
markVideoWatched, markVideoWatched,
upsertCoverArt, upsertCoverArt,
} from './immersion-tracker/query-maintenance'; } from './immersion-tracker/query-maintenance';
import type {
DeleteMaintenanceOperation,
DeleteMaintenanceTask,
} from './immersion-tracker/delete-maintenance';
import { import {
DeleteMaintenanceWorkerRuntime, DeleteMaintenanceWorkerRuntime,
type RunDeleteMaintenanceTask, type RunDeleteMaintenanceTask,
} from './immersion-tracker/delete-maintenance-worker-runtime'; } from './immersion-tracker/delete-maintenance-worker-runtime';
import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
import { repairJellyfinStreamVideoLinks } from './immersion-tracker/jellyfin-link-repair'; import { repairJellyfinStreamVideoLinks } from './immersion-tracker/jellyfin-link-repair';
import { import {
repairLegacySeasonlessAnimeRows, repairLegacySeasonlessAnimeRows,
@@ -390,19 +387,8 @@ export class ImmersionTrackerService {
private readonly vacuumIntervalMs: number; private readonly vacuumIntervalMs: number;
private readonly dbPath: string; private readonly dbPath: string;
private readonly writeLock = { locked: false }; private readonly writeLock = { locked: false };
private readonly runDeleteMaintenanceTask: RunDeleteMaintenanceTask;
private readonly destroyDeleteMaintenanceRunner: () => void; private readonly destroyDeleteMaintenanceRunner: () => void;
private readonly pendingDeleteRequests: Array<{ private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
resolveTask: () =>
| DeleteMaintenanceOperation
| null
| Promise<DeleteMaintenanceOperation | null>;
resolve: () => void;
reject: (error: unknown) => void;
}> = [];
private deleteTaskRunning = false;
private deleteDrainTimer: ReturnType<typeof setTimeout> | null = null;
private pendingDeleteTaskCount = 0;
private flushTimer: ReturnType<typeof setTimeout> | null = null; private flushTimer: ReturnType<typeof setTimeout> | null = null;
private maintenanceTimer: ReturnType<typeof setInterval> | null = null; private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
private flushScheduled = false; private flushScheduled = false;
@@ -433,15 +419,29 @@ export class ImmersionTrackerService {
) { ) {
this.dbPath = options.dbPath; this.dbPath = options.dbPath;
this.resolveLegacyVocabularyPos = options.resolveLegacyVocabularyPos; this.resolveLegacyVocabularyPos = options.resolveLegacyVocabularyPos;
let runDeleteMaintenanceTask: RunDeleteMaintenanceTask;
if (dependencies.runDeleteMaintenanceTask) { if (dependencies.runDeleteMaintenanceTask) {
this.runDeleteMaintenanceTask = dependencies.runDeleteMaintenanceTask; runDeleteMaintenanceTask = dependencies.runDeleteMaintenanceTask;
this.destroyDeleteMaintenanceRunner = this.destroyDeleteMaintenanceRunner =
dependencies.destroyDeleteMaintenanceRunner ?? (() => {}); dependencies.destroyDeleteMaintenanceRunner ?? (() => {});
} else { } else {
const deleteMaintenanceRuntime = new DeleteMaintenanceWorkerRuntime(); const deleteMaintenanceRuntime = new DeleteMaintenanceWorkerRuntime();
this.runDeleteMaintenanceTask = (dbPath, task) => deleteMaintenanceRuntime.run(dbPath, task); runDeleteMaintenanceTask = (dbPath, task) => deleteMaintenanceRuntime.run(dbPath, task);
this.destroyDeleteMaintenanceRunner = () => deleteMaintenanceRuntime.destroy(); this.destroyDeleteMaintenanceRunner = () => deleteMaintenanceRuntime.destroy();
} }
this.deleteMaintenanceScheduler = new DeleteMaintenanceScheduler({
batchWindowMs: DELETE_MAINTENANCE_BATCH_WINDOW_MS,
runTask: (task) => runDeleteMaintenanceTask(this.dbPath, task),
onBusy: () => {
this.flushTelemetry(true);
this.flushNow();
this.writeLock.locked = true;
},
onIdle: () => {
this.writeLock.locked = false;
if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
},
});
const parentDir = path.dirname(this.dbPath); const parentDir = path.dirname(this.dbPath);
if (!fs.existsSync(parentDir)) { if (!fs.existsSync(parentDir)) {
fs.mkdirSync(parentDir, { recursive: true }); fs.mkdirSync(parentDir, { recursive: true });
@@ -543,16 +543,9 @@ export class ImmersionTrackerService {
clearInterval(this.maintenanceTimer); clearInterval(this.maintenanceTimer);
this.maintenanceTimer = null; this.maintenanceTimer = null;
} }
if (this.deleteDrainTimer) {
clearTimeout(this.deleteDrainTimer);
this.deleteDrainTimer = null;
}
if (this.pendingDeleteRequests.length > 0) {
const error = new Error('Immersion tracker is shutting down');
for (const request of this.pendingDeleteRequests.splice(0)) request.reject(error);
}
this.finalizeActiveSession(); this.finalizeActiveSession();
this.isDestroyed = true; this.isDestroyed = true;
this.deleteMaintenanceScheduler.destroy();
this.destroyDeleteMaintenanceRunner(); this.destroyDeleteMaintenanceRunner();
this.db.close(); this.db.close();
} }
@@ -766,6 +759,7 @@ export class ImmersionTrackerService {
`Ignoring bulk delete request for active immersion session ${activeSessionId}`, `Ignoring bulk delete request for active immersion session ${activeSessionId}`,
); );
} }
if (deletableSessionIds.length === 0) return null;
return { kind: 'sessions', sessionIds: deletableSessionIds }; return { kind: 'sessions', sessionIds: deletableSessionIds };
}); });
} }
@@ -804,75 +798,12 @@ export class ImmersionTrackerService {
} }
private enqueueDeleteMaintenanceTask( private enqueueDeleteMaintenanceTask(
resolveTask: () => resolveTask: Parameters<DeleteMaintenanceScheduler['enqueue']>[0],
| DeleteMaintenanceOperation
| null
| Promise<DeleteMaintenanceOperation | null>,
): Promise<void> { ): Promise<void> {
if (!this.writeLock.locked) { if (this.isDestroyed) {
this.flushTelemetry(true); return Promise.reject(new Error('Immersion tracker is shutting down'));
this.flushNow();
} }
this.pendingDeleteTaskCount += 1; return this.deleteMaintenanceScheduler.enqueue(resolveTask);
this.writeLock.locked = true;
const result = new Promise<void>((resolve, reject) => {
this.pendingDeleteRequests.push({ resolveTask, resolve, reject });
this.scheduleDeleteMaintenanceDrain();
});
return result.finally(() => {
this.pendingDeleteTaskCount -= 1;
if (this.pendingDeleteTaskCount > 0) return;
this.writeLock.locked = false;
if (!this.isDestroyed && this.queue.length > 0) {
this.scheduleFlush(0);
}
});
}
private scheduleDeleteMaintenanceDrain(): void {
if (this.isDestroyed || this.deleteTaskRunning || this.deleteDrainTimer) return;
this.deleteDrainTimer = setTimeout(() => {
this.deleteDrainTimer = null;
void this.drainDeleteMaintenanceRequests();
}, DELETE_MAINTENANCE_BATCH_WINDOW_MS);
}
private async drainDeleteMaintenanceRequests(): Promise<void> {
if (this.deleteTaskRunning || this.pendingDeleteRequests.length === 0) return;
this.deleteTaskRunning = true;
const requests = this.pendingDeleteRequests.splice(0);
const runnable: Array<{
request: (typeof requests)[number];
task: DeleteMaintenanceOperation;
}> = [];
for (const request of requests) {
try {
const task = await request.resolveTask();
if (task) runnable.push({ request, task });
else request.resolve();
} catch (error) {
request.reject(error);
}
}
if (runnable.length > 0) {
const task: DeleteMaintenanceTask =
runnable.length === 1
? runnable[0]!.task
: { kind: 'batch', tasks: runnable.map((entry) => entry.task) };
try {
await this.runDeleteMaintenanceTask(this.dbPath, task);
for (const { request } of runnable) request.resolve();
} catch (error) {
for (const { request } of runnable) request.reject(error);
}
}
this.deleteTaskRunning = false;
this.scheduleDeleteMaintenanceDrain();
} }
async reassignAnimeAnilist( async reassignAnimeAnilist(
@@ -1087,7 +1087,13 @@ test('delete maintenance batch preserves retained data across overlapping sessio
} }
const rollupDay = getLocalEpochDay(db, startedAtMs); const rollupDay = getLocalEpochDay(db, startedAtMs);
const rollupMonth = 202311; const rollupMonth = (
db
.prepare(
`SELECT CAST(strftime('%Y%m', CAST(? AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) AS rollupMonth`,
)
.get(startedAtMs) as { rollupMonth: number }
).rollupMonth;
for (const videoId of [retainedVideoId, deletedVideoId, animeVideoId]) { for (const videoId of [retainedVideoId, deletedVideoId, animeVideoId]) {
db.prepare( db.prepare(
`INSERT INTO imm_daily_rollups ( `INSERT INTO imm_daily_rollups (
@@ -1155,3 +1161,22 @@ test('delete maintenance batch preserves retained data across overlapping sessio
cleanupDbPath(dbPath); cleanupDbPath(dbPath);
} }
}); });
test('delete maintenance batch chunks id lists below the SQLite variable limit', () => {
const { db, dbPath } = createDb();
try {
const ids = Array.from({ length: 32_767 }, (_, index) => index + 1);
assert.doesNotThrow(() => {
deleteMaintenanceBatch(db, [
{ kind: 'sessions', sessionIds: ids },
...ids.map((videoId) => ({ kind: 'video' as const, videoId })),
...ids.map((animeId) => ({ kind: 'anime' as const, animeId })),
]);
});
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
@@ -0,0 +1,83 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { DeleteMaintenanceScheduler } from './delete-maintenance-scheduler';
import type { DeleteMaintenanceTask } from './delete-maintenance';
test('scheduler batches same-turn requests and balances busy state', async () => {
const tasks: DeleteMaintenanceTask[] = [];
const states: string[] = [];
const scheduler = new DeleteMaintenanceScheduler({
batchWindowMs: 0,
runTask: async (task) => {
tasks.push(task);
},
onBusy: () => states.push('busy'),
onIdle: () => states.push('idle'),
});
const first = scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 }));
const second = scheduler.enqueue(() => ({ kind: 'sessions', sessionIds: [2, 3] }));
const third = scheduler.enqueue(() => null);
await Promise.all([first, second, third]);
assert.deepEqual(tasks, [
{
kind: 'batch',
tasks: [
{ kind: 'session', sessionId: 1 },
{ kind: 'sessions', sessionIds: [2, 3] },
],
},
]);
assert.deepEqual(states, ['busy', 'idle']);
});
test('scheduler rejects enqueue after destruction without entering busy state', async () => {
let busyCalls = 0;
let runCalls = 0;
const scheduler = new DeleteMaintenanceScheduler({
batchWindowMs: 0,
runTask: async () => {
runCalls += 1;
},
onBusy: () => {
busyCalls += 1;
},
onIdle: () => {},
});
scheduler.destroy();
await assert.rejects(
scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 })),
/shutting down/,
);
assert.equal(busyCalls, 0);
assert.equal(runCalls, 0);
});
test('scheduler serializes batches and rejects requests queued at destruction', async () => {
const releases: Array<() => void> = [];
let activeTasks = 0;
let maxActiveTasks = 0;
const scheduler = new DeleteMaintenanceScheduler({
batchWindowMs: 0,
runTask: async () => {
activeTasks += 1;
maxActiveTasks = Math.max(maxActiveTasks, activeTasks);
await new Promise<void>((resolve) => releases.push(resolve));
activeTasks -= 1;
},
onBusy: () => {},
onIdle: () => {},
});
const first = scheduler.enqueue(() => ({ kind: 'session', sessionId: 1 }));
while (releases.length === 0) await new Promise<void>((resolve) => setTimeout(resolve, 0));
const queued = scheduler.enqueue(() => ({ kind: 'session', sessionId: 2 }));
scheduler.destroy();
await assert.rejects(queued, /shutting down/);
releases[0]?.();
await first;
assert.equal(maxActiveTasks, 1);
});
@@ -0,0 +1,103 @@
import type { DeleteMaintenanceOperation, DeleteMaintenanceTask } from './delete-maintenance';
type ResolveDeleteMaintenanceOperation = () =>
| DeleteMaintenanceOperation
| null
| Promise<DeleteMaintenanceOperation | null>;
interface PendingDeleteMaintenanceRequest {
resolveTask: ResolveDeleteMaintenanceOperation;
resolve: () => void;
reject: (error: unknown) => void;
}
interface DeleteMaintenanceSchedulerOptions {
batchWindowMs: number;
runTask: (task: DeleteMaintenanceTask) => Promise<void>;
onBusy: () => void;
onIdle: () => void;
}
export class DeleteMaintenanceScheduler {
private readonly pendingRequests: PendingDeleteMaintenanceRequest[] = [];
private running = false;
private drainTimer: ReturnType<typeof setTimeout> | null = null;
private pendingTaskCount = 0;
private destroyed = false;
constructor(private readonly options: DeleteMaintenanceSchedulerOptions) {}
enqueue(resolveTask: ResolveDeleteMaintenanceOperation): Promise<void> {
if (this.destroyed) {
return Promise.reject(new Error('Immersion tracker is shutting down'));
}
if (this.pendingTaskCount === 0) this.options.onBusy();
this.pendingTaskCount += 1;
const result = new Promise<void>((resolve, reject) => {
this.pendingRequests.push({ resolveTask, resolve, reject });
this.scheduleDrain();
});
return result.finally(() => {
this.pendingTaskCount -= 1;
if (this.pendingTaskCount === 0) this.options.onIdle();
});
}
destroy(): void {
if (this.destroyed) return;
this.destroyed = true;
if (this.drainTimer) {
clearTimeout(this.drainTimer);
this.drainTimer = null;
}
const error = new Error('Immersion tracker is shutting down');
for (const request of this.pendingRequests.splice(0)) request.reject(error);
}
private scheduleDrain(): void {
if (this.destroyed || this.running || this.drainTimer) return;
this.drainTimer = setTimeout(() => {
this.drainTimer = null;
void this.drain();
}, this.options.batchWindowMs);
}
private async drain(): Promise<void> {
if (this.running || this.pendingRequests.length === 0) return;
this.running = true;
const requests = this.pendingRequests.splice(0);
const runnable: Array<{
request: PendingDeleteMaintenanceRequest;
task: DeleteMaintenanceOperation;
}> = [];
for (const request of requests) {
try {
const task = await request.resolveTask();
if (task) runnable.push({ request, task });
else request.resolve();
} catch (error) {
request.reject(error);
}
}
if (runnable.length > 0) {
const task: DeleteMaintenanceTask =
runnable.length === 1
? runnable[0]!.task
: { kind: 'batch', tasks: runnable.map((entry) => entry.task) };
try {
await this.options.runTask(task);
for (const { request } of runnable) request.resolve();
} catch (error) {
for (const { request } of runnable) request.reject(error);
}
}
this.running = false;
this.scheduleDrain();
}
}
@@ -3,7 +3,10 @@ import fs from 'node:fs';
import os from 'node:os'; import os from 'node:os';
import path from 'node:path'; import path from 'node:path';
import test from 'node:test'; import test from 'node:test';
import { DeleteMaintenanceWorkerRuntime } from './delete-maintenance-worker-runtime'; import {
DeleteMaintenanceWorkerRuntime,
resolveDeleteMaintenanceWorkerPath,
} from './delete-maintenance-worker-runtime';
import { executeDeleteMaintenanceTask } from './delete-maintenance'; import { executeDeleteMaintenanceTask } from './delete-maintenance';
import { startSessionRecord } from './session'; import { startSessionRecord } from './session';
import { Database } from './sqlite'; import { Database } from './sqlite';
@@ -79,7 +82,7 @@ test('a delete batch rebuilds lifetime summaries once', () => {
test( test(
'compiled delete worker removes data through its separate database connection', 'compiled delete worker removes data through its separate database connection',
{ skip: path.extname(__filename) !== '.js' }, { skip: resolveDeleteMaintenanceWorkerPath() === null },
async () => { async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-delete-worker-test-')); const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-delete-worker-test-'));
const dbPath = path.join(tempDir, 'immersion.sqlite'); const dbPath = path.join(tempDir, 'immersion.sqlite');
@@ -123,3 +126,73 @@ test(
} }
}, },
); );
test('worker runtime warns before falling back when no emitted worker is available', async () => {
const warnings: unknown[][] = [];
const fallbackTasks: unknown[] = [];
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => null,
warn: (...args) => warnings.push(args),
executeFallback: (_dbPath, task) => fallbackTasks.push(task),
});
await runtime.run('/tmp/fallback.sqlite', { kind: 'session', sessionId: 1 });
assert.equal(warnings.length, 1);
assert.match(String(warnings[0]?.[0]), /worker unavailable/i);
assert.deepEqual(fallbackTasks, [{ kind: 'session', sessionId: 1 }]);
});
test('worker runtime terminates a worker after successful settlement', async () => {
type Listener = (value: never) => void;
const listeners = new Map<string, Listener>();
let terminateCalls = 0;
const worker = {
once(event: string, listener: Listener) {
listeners.set(event, listener);
return this;
},
terminate: async () => {
terminateCalls += 1;
return 0;
},
};
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => '/tmp/delete-worker.js',
createWorker: async () => worker,
});
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
await new Promise<void>((resolve) => setTimeout(resolve, 0));
listeners.get('message')?.({ ok: true } as never);
await result;
assert.equal(terminateCalls, 1);
});
test('worker runtime terminates a worker after failed settlement', async () => {
type Listener = (value: never) => void;
const listeners = new Map<string, Listener>();
let terminateCalls = 0;
const worker = {
once(event: string, listener: Listener) {
listeners.set(event, listener);
return this;
},
terminate: async () => {
terminateCalls += 1;
return 0;
},
};
const runtime = new DeleteMaintenanceWorkerRuntime({
resolveWorkerPath: () => '/tmp/delete-worker.js',
createWorker: async () => worker,
});
const result = runtime.run('/tmp/test.sqlite', { kind: 'session', sessionId: 1 });
await new Promise<void>((resolve) => setTimeout(resolve, 0));
listeners.get('error')?.(new Error('worker failed') as never);
await assert.rejects(result, /worker failed/);
assert.equal(terminateCalls, 1);
});
@@ -1,3 +1,6 @@
import fs from 'node:fs';
import path from 'node:path';
import { createLogger } from '../../../logger';
import { executeDeleteMaintenanceTask, type DeleteMaintenanceTask } from './delete-maintenance'; import { executeDeleteMaintenanceTask, type DeleteMaintenanceTask } from './delete-maintenance';
interface DeleteMaintenanceWorkerResponse { interface DeleteMaintenanceWorkerResponse {
@@ -10,28 +13,63 @@ export type RunDeleteMaintenanceTask = (
task: DeleteMaintenanceTask, task: DeleteMaintenanceTask,
) => Promise<void>; ) => Promise<void>;
interface DeleteMaintenanceWorkerHandle {
once(event: 'message', listener: (message: DeleteMaintenanceWorkerResponse) => void): this;
once(event: 'error', listener: (error: Error) => void): this;
once(event: 'exit', listener: (code: number) => void): this;
terminate(): Promise<number>;
}
interface DeleteMaintenanceWorkerRuntimeOptions {
resolveWorkerPath?: () => string | null;
createWorker?: (
workerPath: string,
workerData: { dbPath: string; task: DeleteMaintenanceTask },
) => Promise<DeleteMaintenanceWorkerHandle>;
executeFallback?: typeof executeDeleteMaintenanceTask;
warn?: (message: string, ...meta: unknown[]) => void;
}
export function resolveDeleteMaintenanceWorkerPath(): string | null {
const workerPath = path.join(__dirname, 'delete-maintenance-worker-thread.js');
return fs.existsSync(workerPath) ? workerPath : null;
}
const logger = createLogger('main:immersion-tracker:delete-worker');
export class DeleteMaintenanceWorkerRuntime { export class DeleteMaintenanceWorkerRuntime {
private readonly activeWorkers = new Set<import('node:worker_threads').Worker>(); private readonly activeWorkers = new Set<DeleteMaintenanceWorkerHandle>();
private destroyed = false; private destroyed = false;
constructor(private readonly options: DeleteMaintenanceWorkerRuntimeOptions = {}) {}
async run(dbPath: string, task: DeleteMaintenanceTask): Promise<void> { async run(dbPath: string, task: DeleteMaintenanceTask): Promise<void> {
if (this.destroyed) { if (this.destroyed) {
throw new Error('Delete maintenance worker is shut down'); throw new Error('Delete maintenance worker is shut down');
} }
let workerThreads: typeof import('node:worker_threads'); let worker: DeleteMaintenanceWorkerHandle;
let workerPath: string;
try { try {
workerThreads = await import('node:worker_threads'); const workerPath = (this.options.resolveWorkerPath ?? resolveDeleteMaintenanceWorkerPath)();
workerPath = require.resolve('./delete-maintenance-worker-thread.js'); if (!workerPath) throw new Error('Emitted delete-maintenance worker module was not found');
} catch { const createWorker =
executeDeleteMaintenanceTask(dbPath, task); this.options.createWorker ??
(async (resolvedPath, workerData) => {
const { Worker } = await import('node:worker_threads');
return new Worker(resolvedPath, { workerData });
});
worker = await createWorker(workerPath, { dbPath, task });
} catch (error) {
(this.options.warn ?? logger.warn)(
'Delete maintenance worker unavailable; running maintenance on the current thread',
error,
);
(this.options.executeFallback ?? executeDeleteMaintenanceTask)(dbPath, task);
return; return;
} }
await new Promise<void>((resolve, reject) => { await new Promise<void>((resolve, reject) => {
let settled = false; let settled = false;
const worker = new workerThreads.Worker(workerPath, { workerData: { dbPath, task } });
this.activeWorkers.add(worker); this.activeWorkers.add(worker);
const settle = (error?: Error) => { const settle = (error?: Error) => {
@@ -40,6 +78,7 @@ export class DeleteMaintenanceWorkerRuntime {
this.activeWorkers.delete(worker); this.activeWorkers.delete(worker);
if (error) reject(error); if (error) reject(error);
else resolve(); else resolve();
void worker.terminate();
}; };
worker.once('message', (message: DeleteMaintenanceWorkerResponse) => { worker.once('message', (message: DeleteMaintenanceWorkerResponse) => {
@@ -7,8 +7,11 @@ import {
deleteSessionsByIds, deleteSessionsByIds,
makePlaceholders, makePlaceholders,
planLexicalRemovalsForSessions, planLexicalRemovalsForSessions,
type LexicalRemovalPlan,
} from './query-shared'; } from './query-shared';
const SQLITE_ID_CHUNK_SIZE = 1_000;
export type DeleteMaintenanceOperation = export type DeleteMaintenanceOperation =
| { kind: 'session'; sessionId: number } | { kind: 'session'; sessionId: number }
| { kind: 'sessions'; sessionIds: number[] } | { kind: 'sessions'; sessionIds: number[] }
@@ -39,11 +42,65 @@ function addOperationTargets(
} }
} }
function selectIds(db: DatabaseSync, sql: string, params: number[], column: string): number[] { function forEachIdChunk(ids: number[], callback: (chunk: number[]) => void): void {
for (let start = 0; start < ids.length; start += SQLITE_ID_CHUNK_SIZE) {
callback(ids.slice(start, start + SQLITE_ID_CHUNK_SIZE));
}
}
function selectIds(
db: DatabaseSync,
buildSql: (placeholders: string) => string,
params: number[],
column: string,
): number[] {
if (params.length === 0) return []; if (params.length === 0) return [];
return (db.prepare(sql).all(...params) as Array<Record<string, number>>).map( const ids: number[] = [];
(row) => row[column]!, forEachIdChunk(params, (chunk) => {
); const rows = db.prepare(buildSql(makePlaceholders(chunk))).all(...chunk) as Array<
Record<string, number>
>;
for (const row of rows) ids.push(row[column]!);
});
return ids;
}
function planLexicalRemovalsInChunks(db: DatabaseSync, sessionIds: number[]): LexicalRemovalPlan {
const combined: LexicalRemovalPlan = { words: [], kanji: [] };
const merge = (target: LexicalRemovalPlan['words'], source: LexicalRemovalPlan['words']) => {
const byId = new Map(target.map((entry) => [entry.id, entry]));
for (const entry of source) {
const existing = byId.get(entry.id);
if (!existing) {
const added = { ...entry };
target.push(added);
byId.set(entry.id, added);
continue;
}
existing.removedFrequency += entry.removedFrequency;
if (
entry.removedFirstSeenMs !== null &&
(existing.removedFirstSeenMs === null ||
entry.removedFirstSeenMs < existing.removedFirstSeenMs)
) {
existing.removedFirstSeenMs = entry.removedFirstSeenMs;
}
if (
entry.removedLastSeenMs !== null &&
(existing.removedLastSeenMs === null ||
entry.removedLastSeenMs > existing.removedLastSeenMs)
) {
existing.removedLastSeenMs = entry.removedLastSeenMs;
}
}
};
forEachIdChunk(sessionIds, (chunk) => {
const plan = planLexicalRemovalsForSessions(db, chunk);
merge(combined.words, plan.words);
merge(combined.kanji, plan.kanji);
});
return combined;
} }
export function deleteMaintenanceBatch( export function deleteMaintenanceBatch(
@@ -62,7 +119,7 @@ export function deleteMaintenanceBatch(
const animeIdList = [...animeIds]; const animeIdList = [...animeIds];
for (const videoId of selectIds( for (const videoId of selectIds(
db, db,
`SELECT video_id FROM imm_videos WHERE anime_id IN (${makePlaceholders(animeIdList)})`, (placeholders) => `SELECT video_id FROM imm_videos WHERE anime_id IN (${placeholders})`,
animeIdList, animeIdList,
'video_id', 'video_id',
)) { )) {
@@ -72,7 +129,7 @@ export function deleteMaintenanceBatch(
const videoIdList = [...videoIds]; const videoIdList = [...videoIds];
for (const sessionId of selectIds( for (const sessionId of selectIds(
db, db,
`SELECT session_id FROM imm_sessions WHERE video_id IN (${makePlaceholders(videoIdList)})`, (placeholders) => `SELECT session_id FROM imm_sessions WHERE video_id IN (${placeholders})`,
videoIdList, videoIdList,
'session_id', 'session_id',
)) { )) {
@@ -80,36 +137,43 @@ export function deleteMaintenanceBatch(
} }
const sessionIdList = [...sessionIds]; const sessionIdList = [...sessionIds];
const lexicalRemovals = planLexicalRemovalsForSessions(db, sessionIdList); const lexicalRemovals = planLexicalRemovalsInChunks(db, sessionIdList);
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIdList).filter( const affectedRollupGroups = sessionIdList
(group) => !videoIds.has(group.videoId), .flatMap((_, index) =>
); index % SQLITE_ID_CHUNK_SIZE === 0
? getRollupGroupsForSessions(db, sessionIdList.slice(index, index + SQLITE_ID_CHUNK_SIZE))
: [],
)
.filter((group) => !videoIds.has(group.videoId));
const coverBlobHashes = new Set<string>(); const coverBlobHashes = new Set<string>();
if (videoIdList.length > 0) { if (videoIdList.length > 0) {
const placeholders = makePlaceholders(videoIdList); forEachIdChunk(videoIdList, (chunk) => {
const artRows = db const placeholders = makePlaceholders(chunk);
.prepare( const artRows = db
`SELECT cover_blob_hash AS coverBlobHash .prepare(
FROM imm_media_art `SELECT cover_blob_hash AS coverBlobHash
WHERE video_id IN (${placeholders}) AND cover_blob_hash IS NOT NULL`, FROM imm_media_art
) WHERE video_id IN (${placeholders}) AND cover_blob_hash IS NOT NULL`,
.all(...videoIdList) as Array<{ coverBlobHash: string }>; )
for (const row of artRows) coverBlobHashes.add(row.coverBlobHash); .all(...chunk) as Array<{ coverBlobHash: string }>;
for (const row of artRows) coverBlobHashes.add(row.coverBlobHash);
});
deleteSessionsByIds(db, sessionIdList); deleteSessionsByIds(db, sessionIdList);
db.prepare(`DELETE FROM imm_subtitle_lines WHERE video_id IN (${placeholders})`).run( forEachIdChunk(videoIdList, (chunk) => {
...videoIdList, const placeholders = makePlaceholders(chunk);
); db.prepare(`DELETE FROM imm_subtitle_lines WHERE video_id IN (${placeholders})`).run(
db.prepare(`DELETE FROM imm_daily_rollups WHERE video_id IN (${placeholders})`).run( ...chunk,
...videoIdList, );
); db.prepare(`DELETE FROM imm_daily_rollups WHERE video_id IN (${placeholders})`).run(
db.prepare(`DELETE FROM imm_monthly_rollups WHERE video_id IN (${placeholders})`).run( ...chunk,
...videoIdList, );
); db.prepare(`DELETE FROM imm_monthly_rollups WHERE video_id IN (${placeholders})`).run(
db.prepare(`DELETE FROM imm_media_art WHERE video_id IN (${placeholders})`).run( ...chunk,
...videoIdList, );
); db.prepare(`DELETE FROM imm_media_art WHERE video_id IN (${placeholders})`).run(...chunk);
db.prepare(`DELETE FROM imm_videos WHERE video_id IN (${placeholders})`).run(...videoIdList); db.prepare(`DELETE FROM imm_videos WHERE video_id IN (${placeholders})`).run(...chunk);
});
} else { } else {
deleteSessionsByIds(db, sessionIdList); deleteSessionsByIds(db, sessionIdList);
} }
@@ -118,11 +182,13 @@ export function deleteMaintenanceBatch(
cleanupUnusedCoverArtBlobHash(db, coverBlobHash); cleanupUnusedCoverArtBlobHash(db, coverBlobHash);
} }
if (animeIdList.length > 0) { if (animeIdList.length > 0) {
const placeholders = makePlaceholders(animeIdList); forEachIdChunk(animeIdList, (chunk) => {
db.prepare(`DELETE FROM imm_lifetime_anime WHERE anime_id IN (${placeholders})`).run( const placeholders = makePlaceholders(chunk);
...animeIdList, db.prepare(`DELETE FROM imm_lifetime_anime WHERE anime_id IN (${placeholders})`).run(
); ...chunk,
db.prepare(`DELETE FROM imm_anime WHERE anime_id IN (${placeholders})`).run(...animeIdList); );
db.prepare(`DELETE FROM imm_anime WHERE anime_id IN (${placeholders})`).run(...chunk);
});
} }
applyLexicalRemovals(db, lexicalRemovals); applyLexicalRemovals(db, lexicalRemovals);
@@ -490,17 +490,21 @@ export function deleteSessionsByIds(db: DatabaseSync, sessionIds: number[]): voi
return; return;
} }
const placeholders = makePlaceholders(sessionIds); const chunkSize = 1_000;
db.prepare(`DELETE FROM imm_subtitle_lines WHERE session_id IN (${placeholders})`).run( for (let start = 0; start < sessionIds.length; start += chunkSize) {
...sessionIds, const chunk = sessionIds.slice(start, start + chunkSize);
); const placeholders = makePlaceholders(chunk);
db.prepare(`DELETE FROM imm_session_telemetry WHERE session_id IN (${placeholders})`).run( db.prepare(`DELETE FROM imm_subtitle_lines WHERE session_id IN (${placeholders})`).run(
...sessionIds, ...chunk,
); );
db.prepare(`DELETE FROM imm_session_events WHERE session_id IN (${placeholders})`).run( db.prepare(`DELETE FROM imm_session_telemetry WHERE session_id IN (${placeholders})`).run(
...sessionIds, ...chunk,
); );
db.prepare(`DELETE FROM imm_sessions WHERE session_id IN (${placeholders})`).run(...sessionIds); db.prepare(`DELETE FROM imm_session_events WHERE session_id IN (${placeholders})`).run(
...chunk,
);
db.prepare(`DELETE FROM imm_sessions WHERE session_id IN (${placeholders})`).run(...chunk);
}
} }
export function toDbMs(ms: number | bigint): bigint { export function toDbMs(ms: number | bigint): bigint {