From dfde19cc4de720fec394f086ef0b78d8be026ddf Mon Sep 17 00:00:00 2001 From: sudacode Date: Thu, 9 Jul 2026 00:30:30 -0700 Subject: [PATCH] fix(launcher): address sync review feedback --- changes/launcher-sync-command.md | 2 +- docs-site/usage.md | 4 + launcher/commands/sync-command.test.ts | 165 ++++++++++ launcher/commands/sync-command.ts | 171 ++++++++--- launcher/sync/merge-catalog.ts | 45 ++- launcher/sync/merge-rollups.ts | 17 +- launcher/sync/ssh.test.ts | 14 +- launcher/sync/ssh.ts | 31 +- launcher/sync/sync-db.test.ts | 290 ++++++++++++++---- launcher/test-support/immersion-db-fixture.ts | 32 +- .../test-support/immersion-db-schema.test.ts | 109 +++++++ 11 files changed, 747 insertions(+), 133 deletions(-) create mode 100644 launcher/commands/sync-command.test.ts create mode 100644 launcher/test-support/immersion-db-schema.test.ts diff --git a/changes/launcher-sync-command.md b/changes/launcher-sync-command.md index ddee2fdb..27c57fe6 100644 --- a/changes/launcher-sync-command.md +++ b/changes/launcher-sync-command.md @@ -1,4 +1,4 @@ type: added area: launcher -- Added `subminer sync ` to merge immersion stats and watch history between two machines over SSH. Each side snapshots its database (`VACUUM INTO`), snapshots are exchanged with `scp`, and each machine merges the other's data as an insert-only union keyed on session UUIDs / video keys / series title keys, so re-syncing is idempotent and nothing is double-counted. Lifetime totals and daily/monthly rollups are updated incrementally (history older than the session retention window is preserved); remote-only historical rollups are copied. `subminer sync --snapshot ` and `subminer sync --merge ` expose the underlying steps for manual transfers. The command refuses to run while the stats daemon or an mpv session is active (`--force` overrides) and aborts on stats schema version mismatches. +- Added `subminer sync ` to merge immersion stats and watch history between two machines over SSH. Each side snapshots its database (`VACUUM INTO`), snapshots are exchanged with `scp`, and each machine merges the other's data as an insert-only union keyed on session UUIDs / video keys / series title keys, so re-syncing is idempotent and nothing is double-counted. Lifetime totals and daily/monthly rollups are updated incrementally (history older than the session retention window is preserved); remote-only historical rollups are copied only when they do not conflict with retained local session history. `subminer sync --snapshot ` and `subminer sync --merge ` expose the underlying steps for manual transfers. The command refuses to run while the stats daemon or an mpv session is active (`--force` overrides), keeps that guard in place through local/remote merges, reports remote stderr on failures, and aborts on stats schema version mismatches. diff --git a/docs-site/usage.md b/docs-site/usage.md index 35196448..9938fb41 100644 --- a/docs-site/usage.md +++ b/docs-site/usage.md @@ -111,6 +111,9 @@ subminer config show # Print active config contents subminer mpv socket # Print active mpv socket path subminer mpv status # Exit 0 if socket is ready, else exit 1 subminer mpv idle # Launch detached idle mpv with SubMiner defaults +subminer sync media-box # Sync stats/watch history with an SSH host +subminer sync --snapshot ~/subminer-snapshot.sqlite # Write a local DB snapshot +subminer sync --merge ~/subminer-snapshot.sqlite # Merge a snapshot into the local DB subminer dictionary /path/to/file-or-directory # Generate character dictionary ZIP from target (manual Yomitan import) subminer dictionary --candidates /path/to/file.mkv subminer dictionary --select 21355 /path/to/file.mkv @@ -194,6 +197,7 @@ This flow requires `mpv.exe` to be discoverable. Leave `mpv.executablePath` blan - `subminer logs -e`: export a sanitized ZIP of today's local-date logs, or the most recent logs when no current-day log exists. The exported copy masks common PII and secrets; on-disk logs are unchanged. - `subminer config`: config file helpers (`path`, `show`). - `subminer mpv`: mpv helpers (`status`, `socket`, `idle`). +- `subminer sync `: sync immersion stats and watch history with another machine over SSH. The host is the SSH destination (`user@host` or an SSH config alias). Use `--snapshot ` to write a consistent local stats DB snapshot, `--merge ` to merge a snapshot into the local stats DB, and `--force` to skip the running stats/mpv safety check. Advanced options: `--db ` overrides the local stats DB path, and `--remote-cmd ` overrides the `subminer` command used on the remote host. - `subminer dictionary `: generates a Yomitan-importable character dictionary ZIP from a file/directory target. - Use `subminer dictionary --candidates ` and `subminer dictionary --select ` to correct AniList character-dictionary matches for a whole series. - `subminer texthooker`: texthooker-only shortcut (same behavior as `--texthooker`). A _texthooker_ is a web page that displays the current subtitle line as selectable text, so browser-based dictionary extensions and other tools can read along with playback. diff --git a/launcher/commands/sync-command.test.ts b/launcher/commands/sync-command.test.ts new file mode 100644 index 00000000..2f5e4c6d --- /dev/null +++ b/launcher/commands/sync-command.test.ts @@ -0,0 +1,165 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { Args } from '../types.js'; +import { createEmptyMergeSummary } from '../sync/sync-shared.js'; +import type { LauncherCommandContext } from './context.js'; +import { runSyncCommand, type SyncCommandDeps } from './sync-command.js'; + +function makeContext(overrides: Partial): LauncherCommandContext { + return { + args: { + sync: true, + syncHost: '', + syncSnapshotPath: '', + syncMergePath: '', + syncRemoteCmd: '', + syncDbPath: '', + syncForce: false, + logLevel: 'warn', + ...overrides, + } as Args, + scriptPath: '/tmp/subminer', + scriptName: 'subminer', + mpvSocketPath: '', + pluginRuntimeConfig: {}, + appPath: null, + launcherJellyfinConfig: {}, + processAdapter: process, + } as unknown as LauncherCommandContext; +} + +function ok(stdout = ''): { status: number; stdout: string; stderr: string } { + return { status: 0, stdout, stderr: '' }; +} + +test('runSyncCommand dispatches snapshot, merge, host, and missing-target modes', () => { + const calls: string[] = []; + const deps: Partial = { + createDbSnapshot: (dbPath: string, outPath: string) => { + calls.push(`snapshot:${dbPath}->${outPath}`); + }, + mergeSnapshotIntoDb: (dbPath: string, snapshotPath: string) => { + calls.push(`merge:${dbPath}<-${snapshotPath}`); + return createEmptyMergeSummary(); + }, + formatMergeSummary: () => 'summary', + ensureTrackerQuiescent: () => { + calls.push('quiescent'); + }, + assertSafeSshHost: (host: string) => { + calls.push(`host:${host}`); + }, + resolveRemoteSubminerCommand: () => 'subminer', + runSsh: (_host: string, command: string) => { + calls.push(`ssh:${command}`); + return command.startsWith('mktemp ') ? ok('/tmp/subminer-sync.remote\n') : ok(); + }, + runScp: (from: string, to: string) => { + calls.push(`scp:${from}->${to}`); + }, + fail: (message: string): never => { + throw new Error(message); + }, + }; + + assert.equal( + runSyncCommand( + makeContext({ syncDbPath: '/tmp/local.sqlite', syncSnapshotPath: '/tmp/out.sqlite' }), + deps, + ), + true, + ); + assert.ok(calls.includes('snapshot:/tmp/local.sqlite->/tmp/out.sqlite')); + + runSyncCommand( + makeContext({ syncDbPath: '/tmp/local.sqlite', syncMergePath: '/tmp/in.sqlite' }), + deps, + ); + assert.ok(calls.includes('quiescent')); + assert.ok(calls.includes('merge:/tmp/local.sqlite<-/tmp/in.sqlite')); + + runSyncCommand(makeContext({ syncDbPath: '/tmp/local.sqlite', syncHost: 'media-box' }), deps); + assert.ok(calls.includes('host:media-box')); + + assert.throws( + () => runSyncCommand(makeContext({ syncDbPath: '/tmp/local.sqlite' }), deps), + /sync requires a host, --snapshot , or --merge /, + ); +}); + +test('runHostSync keeps tracker quiescent through local and remote merge and cleans up after failure', () => { + const calls: string[] = []; + let localTmpDir = ''; + const deps: Partial = { + createDbSnapshot: (_dbPath: string, outPath: string) => { + calls.push(`snapshot:${outPath}`); + fs.writeFileSync(outPath, 'snapshot'); + }, + mergeSnapshotIntoDb: () => { + calls.push('local-merge'); + return createEmptyMergeSummary(); + }, + formatMergeSummary: () => 'summary', + ensureTrackerQuiescent: () => { + calls.push('quiescent'); + }, + assertSafeSshHost: () => {}, + resolveRemoteSubminerCommand: () => 'subminer', + mkdtempSync: ((prefix: string) => { + localTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), path.basename(prefix))); + return localTmpDir; + }) as typeof fs.mkdtempSync, + runSsh: (_host: string, command: string) => { + calls.push(`ssh:${command}`); + if (command.startsWith('mktemp ')) return ok('/tmp/subminer-sync.remote\n'); + if (command.includes(' sync --snapshot ')) return ok(); + if (command.includes(' sync --merge ')) { + return { status: 9, stdout: 'remote output', stderr: 'remote merge exploded' }; + } + return ok(); + }, + runScp: (from: string, to: string) => { + calls.push(`scp:${from}->${to}`); + if (!to.includes(':')) fs.writeFileSync(to, 'pulled'); + }, + }; + + assert.throws( + () => + runSyncCommand(makeContext({ syncDbPath: '/tmp/local.sqlite', syncHost: 'media-box' }), deps), + /Remote merge failed on media-box[\s\S]*remote merge exploded/, + ); + assert.equal(calls.filter((call) => call === 'quiescent').length, 3); + assert.ok(calls.indexOf('quiescent') < calls.findIndex((call) => call.startsWith('snapshot:'))); + assert.ok(calls.includes('local-merge')); + assert.ok(calls.some((call) => call.startsWith('ssh:rm -rf '))); + assert.equal(fs.existsSync(localTmpDir), false); +}); + +test('runHostSync includes remote snapshot stderr in failures', () => { + const deps: Partial = { + createDbSnapshot: (_dbPath: string, outPath: string) => { + fs.writeFileSync(outPath, 'snapshot'); + }, + ensureTrackerQuiescent: () => {}, + assertSafeSshHost: () => {}, + resolveRemoteSubminerCommand: () => 'subminer', + runSsh: (_host: string, command: string) => { + if (command.startsWith('mktemp ')) return ok('/tmp/subminer-sync.remote\n'); + if (command.includes(' sync --snapshot ')) { + return { status: 5, stdout: '', stderr: 'snapshot permission denied' }; + } + return ok(); + }, + runScp: () => {}, + }; + + assert.throws( + () => + runSyncCommand(makeContext({ syncDbPath: '/tmp/local.sqlite', syncHost: 'media-box' }), deps), + /Remote snapshot failed on media-box[\s\S]*snapshot permission denied/, + ); +}); diff --git a/launcher/commands/sync-command.ts b/launcher/commands/sync-command.ts index 45cc9b16..2a584bde 100644 --- a/launcher/commands/sync-command.ts +++ b/launcher/commands/sync-command.ts @@ -18,74 +18,144 @@ import { } from '../sync/ssh.js'; import { resolvePathMaybe } from '../util.js'; import type { LauncherCommandContext } from './context.js'; +import type { RemoteRunResult } from '../sync/ssh.js'; + +export interface SyncCommandDeps { + createDbSnapshot: typeof createDbSnapshot; + mergeSnapshotIntoDb: typeof mergeSnapshotIntoDb; + formatMergeSummary: typeof formatMergeSummary; + findLiveStatsDaemonPid: typeof findLiveStatsDaemonPid; + assertSafeSshHost: typeof assertSafeSshHost; + resolveRemoteSubminerCommand: typeof resolveRemoteSubminerCommand; + runScp: typeof runScp; + runSsh: typeof runSsh; + fail: typeof fail; + log: typeof log; + existsSync: typeof fs.existsSync; + realpathSync: typeof fs.realpathSync; + mkdtempSync: typeof fs.mkdtempSync; + rmSync: typeof fs.rmSync; + consoleLog: typeof console.log; + writeStdout: typeof process.stdout.write; + ensureTrackerQuiescent: (context: LauncherCommandContext, dbPath: string) => void; +} function resolveDbPath(context: LauncherCommandContext): string { const override = context.args.syncDbPath.trim(); return override ? resolvePathMaybe(override) : resolveImmersionDbPath(); } -function isTrackerDb(dbPath: string): boolean { +function isTrackerDb(dbPath: string, deps: SyncCommandDeps): boolean { const trackerDbPath = resolveImmersionDbPath(); try { - return fs.realpathSync(dbPath) === fs.realpathSync(trackerDbPath); + return deps.realpathSync(dbPath) === deps.realpathSync(trackerDbPath); } catch { return dbPath === trackerDbPath; } } -function ensureTrackerQuiescent(context: LauncherCommandContext, dbPath: string): void { +export function ensureTrackerQuiescent( + context: LauncherCommandContext, + dbPath: string, + inputDeps: Partial = {}, +): void { + const deps = resolveSyncCommandDeps(inputDeps); if (context.args.syncForce) return; // A running SubMiner only holds the tracker's own database; --db pointed // elsewhere needs no guard. - if (!isTrackerDb(dbPath)) return; - const daemonPid = findLiveStatsDaemonPid(dbPath); + if (!isTrackerDb(dbPath, deps)) return; + const daemonPid = deps.findLiveStatsDaemonPid(dbPath); if (daemonPid !== null) { - fail( + deps.fail( `The SubMiner stats server is running (pid ${daemonPid}). Stop it with "subminer stats -s" (or close SubMiner) before syncing, or pass --force.`, ); } - if (context.mpvSocketPath && fs.existsSync(context.mpvSocketPath)) { - fail( + if (context.mpvSocketPath && deps.existsSync(context.mpvSocketPath)) { + deps.fail( `An mpv/SubMiner session appears to be running (socket ${context.mpvSocketPath}). Close it before syncing, or pass --force.`, ); } } -function runSnapshotMode(context: LauncherCommandContext, dbPath: string): void { +const defaultSyncCommandDeps: SyncCommandDeps = { + createDbSnapshot, + mergeSnapshotIntoDb, + formatMergeSummary, + findLiveStatsDaemonPid, + assertSafeSshHost, + resolveRemoteSubminerCommand, + runScp, + runSsh, + fail, + log, + existsSync: fs.existsSync, + realpathSync: fs.realpathSync, + mkdtempSync: fs.mkdtempSync, + rmSync: fs.rmSync, + consoleLog: console.log, + writeStdout: process.stdout.write.bind(process.stdout), + ensureTrackerQuiescent: (context, dbPath) => ensureTrackerQuiescent(context, dbPath), +}; + +function resolveSyncCommandDeps(inputDeps: Partial = {}): SyncCommandDeps { + return { ...defaultSyncCommandDeps, ...inputDeps }; +} + +export function runSnapshotMode( + context: LauncherCommandContext, + dbPath: string, + inputDeps: Partial = {}, +): void { + const deps = resolveSyncCommandDeps(inputDeps); const outPath = resolvePathMaybe(context.args.syncSnapshotPath); - createDbSnapshot(dbPath, outPath); - console.log(outPath); + deps.createDbSnapshot(dbPath, outPath); + deps.consoleLog(outPath); } -function runMergeMode(context: LauncherCommandContext, dbPath: string): void { - ensureTrackerQuiescent(context, dbPath); +export function runMergeMode( + context: LauncherCommandContext, + dbPath: string, + inputDeps: Partial = {}, +): void { + const deps = resolveSyncCommandDeps(inputDeps); + deps.ensureTrackerQuiescent(context, dbPath); const snapshotPath = resolvePathMaybe(context.args.syncMergePath); - const summary = mergeSnapshotIntoDb(dbPath, snapshotPath); - console.log(formatMergeSummary(summary)); + const summary = deps.mergeSnapshotIntoDb(dbPath, snapshotPath); + deps.consoleLog(deps.formatMergeSummary(summary)); } -function cleanupRemote(host: string, remoteTmpDir: string): void { +function cleanupRemote(host: string, remoteTmpDir: string, deps: SyncCommandDeps): void { if (!remoteTmpDir.startsWith('/tmp/')) return; - runSsh(host, `rm -rf ${shellQuote(remoteTmpDir)}`); + deps.runSsh(host, `rm -rf ${shellQuote(remoteTmpDir)}`); } -function runHostSync(context: LauncherCommandContext, dbPath: string): void { +function formatRemoteRunError(message: string, run: RemoteRunResult): string { + const stderr = run.stderr.trim(); + return stderr ? `${message}\n${stderr}` : message; +} + +export function runHostSync( + context: LauncherCommandContext, + dbPath: string, + inputDeps: Partial = {}, +): void { + const deps = resolveSyncCommandDeps(inputDeps); const { args } = context; const host = args.syncHost; - assertSafeSshHost(host); + deps.assertSafeSshHost(host); - ensureTrackerQuiescent(context, dbPath); + deps.ensureTrackerQuiescent(context, dbPath); - const remoteCmd = resolveRemoteSubminerCommand(host, args.syncRemoteCmd || null); - log('debug', args.logLevel, `Remote subminer command: ${remoteCmd}`); + const remoteCmd = deps.resolveRemoteSubminerCommand(host, args.syncRemoteCmd || null); + deps.log('debug', args.logLevel, `Remote subminer command: ${remoteCmd}`); - const localTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-sync-')); + const localTmpDir = deps.mkdtempSync(path.join(os.tmpdir(), 'subminer-sync-')); let remoteTmpDir = ''; try { // Signal failures by throwing (not fail(), which exits synchronously and // would skip the finally cleanup, leaking temp dirs holding snapshot data). // main().catch() reports the message the same way fail() would. - const mktemp = runSsh(host, 'mktemp -d /tmp/subminer-sync.XXXXXX'); + const mktemp = deps.runSsh(host, 'mktemp -d /tmp/subminer-sync.XXXXXX'); remoteTmpDir = mktemp.stdout.trim(); if (mktemp.status !== 0 || !remoteTmpDir.startsWith('/tmp/')) { throw new Error(`Could not create a temporary directory on ${host}.`); @@ -93,47 +163,52 @@ function runHostSync(context: LauncherCommandContext, dbPath: string): void { const forceFlag = args.syncForce ? ' --force' : ''; - console.log(`Snapshotting local database (${dbPath})...`); + deps.consoleLog(`Snapshotting local database (${dbPath})...`); const localSnapshot = path.join(localTmpDir, 'local.sqlite'); - createDbSnapshot(dbPath, localSnapshot); + deps.createDbSnapshot(dbPath, localSnapshot); - console.log(`Snapshotting ${host}...`); + deps.consoleLog(`Snapshotting ${host}...`); const remoteSnapshot = `${remoteTmpDir}/snapshot.sqlite`; - const snapshotRun = runSsh( + const snapshotRun = deps.runSsh( host, `${remoteCmd} sync --snapshot ${shellQuote(remoteSnapshot)}${forceFlag}`, ); if (snapshotRun.status !== 0) { - throw new Error(`Remote snapshot failed on ${host}.`); + throw new Error(formatRemoteRunError(`Remote snapshot failed on ${host}.`, snapshotRun)); } const pulledSnapshot = path.join(localTmpDir, 'remote.sqlite'); - runScp(`${host}:${remoteSnapshot}`, pulledSnapshot); + deps.runScp(`${host}:${remoteSnapshot}`, pulledSnapshot); const incomingSnapshot = `${remoteTmpDir}/incoming.sqlite`; - runScp(localSnapshot, `${host}:${incomingSnapshot}`); + deps.runScp(localSnapshot, `${host}:${incomingSnapshot}`); - console.log(`\nMerging ${host} -> local:`); - const summary = mergeSnapshotIntoDb(dbPath, pulledSnapshot); - console.log(formatMergeSummary(summary)); + deps.consoleLog(`\nMerging ${host} -> local:`); + deps.ensureTrackerQuiescent(context, dbPath); + const summary = deps.mergeSnapshotIntoDb(dbPath, pulledSnapshot); + deps.consoleLog(deps.formatMergeSummary(summary)); - console.log(`\nMerging local -> ${host}:`); - const mergeRun = runSsh( + deps.consoleLog(`\nMerging local -> ${host}:`); + deps.ensureTrackerQuiescent(context, dbPath); + const mergeRun = deps.runSsh( host, `${remoteCmd} sync --merge ${shellQuote(incomingSnapshot)}${forceFlag}`, ); - process.stdout.write(mergeRun.stdout); + deps.writeStdout(mergeRun.stdout); if (mergeRun.status !== 0) { throw new Error( - `Remote merge failed on ${host}. The local database was updated; re-run "subminer sync ${host}" once the remote issue is fixed.`, + formatRemoteRunError( + `Remote merge failed on ${host}. The local database was updated; re-run "subminer sync ${host}" once the remote issue is fixed.`, + mergeRun, + ), ); } - console.log('\nSync complete.'); + deps.consoleLog('\nSync complete.'); } finally { - fs.rmSync(localTmpDir, { recursive: true, force: true }); + deps.rmSync(localTmpDir, { recursive: true, force: true }); if (remoteTmpDir) { try { - cleanupRemote(host, remoteTmpDir); + cleanupRemote(host, remoteTmpDir, deps); } catch { // best effort } @@ -141,19 +216,23 @@ function runHostSync(context: LauncherCommandContext, dbPath: string): void { } } -export function runSyncCommand(context: LauncherCommandContext): boolean { +export function runSyncCommand( + context: LauncherCommandContext, + inputDeps: Partial = {}, +): boolean { + const deps = resolveSyncCommandDeps(inputDeps); const { args } = context; if (!args.sync) return false; const dbPath = resolveDbPath(context); if (args.syncSnapshotPath) { - runSnapshotMode(context, dbPath); + runSnapshotMode(context, dbPath, deps); } else if (args.syncMergePath) { - runMergeMode(context, dbPath); + runMergeMode(context, dbPath, deps); } else if (args.syncHost) { - runHostSync(context, dbPath); + runHostSync(context, dbPath, deps); } else { - fail('sync requires a host, --snapshot , or --merge .'); + deps.fail('sync requires a host, --snapshot , or --merge .'); } return true; } diff --git a/launcher/sync/merge-catalog.ts b/launcher/sync/merge-catalog.ts index 56f22ad2..b3c9d60b 100644 --- a/launcher/sync/merge-catalog.ts +++ b/launcher/sync/merge-catalog.ts @@ -160,7 +160,9 @@ export function mergeVideos( ): VideoMergeResult { const videoIdMap = new Map(); const addedVideoIds = new Set(); - const byKey = local.prepare('SELECT video_id, watched FROM imm_videos WHERE video_key = ?'); + const byKey = local.prepare( + 'SELECT video_id, watched FROM imm_videos WHERE video_key = ?', + ); const setWatched = local.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?'); for (const row of selectAll( @@ -168,7 +170,8 @@ export function mergeVideos( `SELECT video_id, anime_id, ${VIDEO_COPY_COLUMNS.join(', ')} FROM imm_videos`, )) { const remoteId = Number(row.video_id); - const mappedAnimeId = row.anime_id === null ? null : (animeIdMap.get(Number(row.anime_id)) ?? null); + const mappedAnimeId = + row.anime_id === null ? null : (animeIdMap.get(Number(row.anime_id)) ?? null); const existing = byKey.get(row.video_key); if (existing) { const localId = Number(existing.video_id); @@ -194,9 +197,11 @@ export function mergeMediaMetadata( videoIdMap: Map, addedVideoIds: Set, ): void { - if (addedVideoIds.size === 0) return; + if (videoIdMap.size === 0) return; + const metadataVideoIds = new Set([...addedVideoIds, ...videoIdMap.keys()]); - const hasBlobStore = tableExists(local, 'imm_cover_art_blobs') && tableExists(remote, 'imm_cover_art_blobs'); + const hasBlobStore = + tableExists(local, 'imm_cover_art_blobs') && tableExists(remote, 'imm_cover_art_blobs'); const copyBlob = hasBlobStore ? local.prepare( `INSERT INTO imm_cover_art_blobs (blob_hash, cover_blob, CREATED_DATE, LAST_UPDATE_DATE) @@ -208,15 +213,19 @@ export function mergeMediaMetadata( ? remote.prepare('SELECT * FROM imm_cover_art_blobs WHERE blob_hash = ?') : null; - if (tableExists(remote, 'imm_media_art')) { - for (const remoteVideoId of addedVideoIds) { + if (tableExists(remote, 'imm_media_art') && tableExists(local, 'imm_media_art')) { + const localArtExists = local.prepare( + 'SELECT 1 FROM imm_media_art WHERE video_id = ? LIMIT 1', + ); + for (const remoteVideoId of metadataVideoIds) { + const localVideoId = videoIdMap.get(remoteVideoId)!; + if (localArtExists.get(localVideoId)) continue; const row = remote .query( `SELECT ${MEDIA_ART_COPY_COLUMNS.join(', ')} FROM imm_media_art WHERE video_id = ?`, ) .get(remoteVideoId); if (!row) continue; - const localVideoId = videoIdMap.get(remoteVideoId)!; if (row.cover_blob_hash && copyBlob && readBlob) { const blob = readBlob.get(row.cover_blob_hash); if (blob) { @@ -233,7 +242,12 @@ export function mergeMediaMetadata( } if (tableExists(remote, 'imm_youtube_videos') && tableExists(local, 'imm_youtube_videos')) { - for (const remoteVideoId of addedVideoIds) { + const localYoutubeExists = local.prepare( + 'SELECT 1 FROM imm_youtube_videos WHERE video_id = ? LIMIT 1', + ); + for (const remoteVideoId of metadataVideoIds) { + const localVideoId = videoIdMap.get(remoteVideoId)!; + if (localYoutubeExists.get(localVideoId)) continue; const row = remote .query( `SELECT ${YOUTUBE_COPY_COLUMNS.join(', ')} FROM imm_youtube_videos WHERE video_id = ?`, @@ -244,7 +258,7 @@ export function mergeMediaMetadata( local, 'imm_youtube_videos', ['video_id', ...YOUTUBE_COPY_COLUMNS], - [videoIdMap.get(remoteVideoId)!, ...YOUTUBE_COPY_COLUMNS.map((column) => row[column])], + [localVideoId, ...YOUTUBE_COPY_COLUMNS.map((column) => row[column])], ); } } @@ -255,7 +269,10 @@ export function mergeExcludedWords( remote: Database, summary: SyncMergeSummary, ): void { - if (!tableExists(remote, 'imm_stats_excluded_words') || !tableExists(local, 'imm_stats_excluded_words')) { + if ( + !tableExists(remote, 'imm_stats_excluded_words') || + !tableExists(local, 'imm_stats_excluded_words') + ) { return; } const insert = local.prepare( @@ -267,7 +284,13 @@ export function mergeExcludedWords( remote, 'SELECT headword, word, reading, CREATED_DATE, LAST_UPDATE_DATE FROM imm_stats_excluded_words', )) { - const result = insert.run(row.headword, row.word, row.reading, row.CREATED_DATE, row.LAST_UPDATE_DATE); + const result = insert.run( + row.headword, + row.word, + row.reading, + row.CREATED_DATE, + row.LAST_UPDATE_DATE, + ); summary.excludedWordsAdded += result.changes; } } diff --git a/launcher/sync/merge-rollups.ts b/launcher/sync/merge-rollups.ts index 8c4e7511..c4f6892a 100644 --- a/launcher/sync/merge-rollups.ts +++ b/launcher/sync/merge-rollups.ts @@ -149,7 +149,9 @@ export function refreshRollupsForNewSessions( } const stampMs = nowDbTimestamp(); - const deleteDaily = local.prepare('DELETE FROM imm_daily_rollups WHERE rollup_day = ? AND video_id = ?'); + const deleteDaily = local.prepare( + 'DELETE FROM imm_daily_rollups WHERE rollup_day = ? AND video_id = ?', + ); const deleteMonthly = local.prepare( 'DELETE FROM imm_monthly_rollups WHERE rollup_month = ? AND video_id = ?', ); @@ -192,6 +194,15 @@ export function copyRemoteOnlyRollups( const localDaySessions = local.prepare( `SELECT 1 FROM imm_sessions WHERE video_id = ? AND ${LOCAL_DAY_EXPR} = ? LIMIT 1`, ); + const localMonthSessions = local.prepare( + `SELECT 1 FROM imm_sessions WHERE video_id = ? AND ${LOCAL_MONTH_EXPR} = ? LIMIT 1`, + ); + const localMonthSessionsForDay = local.prepare( + `SELECT 1 FROM imm_sessions + WHERE video_id = ? + AND ${LOCAL_MONTH_EXPR} = CAST(strftime('%Y%m', CAST(? AS INTEGER) * 86400, 'unixepoch', 'localtime') AS INTEGER) + LIMIT 1`, + ); const insertDaily = local.prepare( `INSERT INTO imm_daily_rollups ( rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, @@ -205,6 +216,7 @@ export function copyRemoteOnlyRollups( if (localVideoId === undefined) continue; if (localDailyExists.get(row.rollup_day, localVideoId)) continue; if (localDaySessions.get(localVideoId, row.rollup_day)) continue; + if (localMonthSessionsForDay.get(localVideoId, row.rollup_day)) continue; insertDaily.run( row.rollup_day, localVideoId, @@ -225,9 +237,6 @@ export function copyRemoteOnlyRollups( const localMonthlyExists = local.prepare( 'SELECT 1 FROM imm_monthly_rollups WHERE rollup_month = ? AND video_id = ? LIMIT 1', ); - const localMonthSessions = local.prepare( - `SELECT 1 FROM imm_sessions WHERE video_id = ? AND ${LOCAL_MONTH_EXPR} = ? LIMIT 1`, - ); const insertMonthly = local.prepare( `INSERT INTO imm_monthly_rollups ( rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, diff --git a/launcher/sync/ssh.test.ts b/launcher/sync/ssh.test.ts index 44beb578..6bb4f4d3 100644 --- a/launcher/sync/ssh.test.ts +++ b/launcher/sync/ssh.test.ts @@ -1,6 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { assertSafeSshHost, shellQuote } from './ssh.js'; +import { assertSafeSshHost, runScp, shellQuote } from './ssh.js'; test('assertSafeSshHost rejects option-like hosts', () => { assert.throws(() => assertSafeSshHost('-oProxyCommand=touch pwned'), /looks like an option/); @@ -17,3 +17,15 @@ test('shellQuote escapes single quotes and wraps in quotes', () => { assert.equal(shellQuote('subminer'), `'subminer'`); assert.equal(shellQuote(`a'; rm -rf ~; '`), `'a'\\''; rm -rf ~; '\\'''`); }); + +test('runScp rejects option-like local endpoints before spawning scp', () => { + assert.throws(() => runScp('-oProxyCommand=sh', '/tmp/out.sqlite'), /looks like an option/); + assert.throws(() => runScp('/tmp/in.sqlite', '-bad-destination'), /looks like an option/); +}); + +test('runScp rejects option-like remote host components', () => { + assert.throws( + () => runScp('-oProxyCommand=sh:/tmp/in.sqlite', '/tmp/out.sqlite'), + /SSH host that looks like an option/, + ); +}); diff --git a/launcher/sync/ssh.ts b/launcher/sync/ssh.ts index ca7df226..7e033479 100644 --- a/launcher/sync/ssh.ts +++ b/launcher/sync/ssh.ts @@ -3,6 +3,7 @@ import { spawnSync } from 'node:child_process'; export interface RemoteRunResult { status: number; stdout: string; + stderr: string; } /** @@ -17,23 +18,43 @@ export function assertSafeSshHost(host: string): void { } /** - * Run a command on the SSH host. stdin/stderr stay attached to the terminal - * so key passphrase / password prompts and remote progress output work; - * stdout is captured for the caller. + * Run a command on the SSH host. stdin stays attached so interactive prompts + * can still read from the terminal; stdout/stderr are captured for callers + * that need actionable remote failure messages. */ export function runSsh(host: string, remoteCommand: string): RemoteRunResult { assertSafeSshHost(host); const result = spawnSync('ssh', [host, remoteCommand], { encoding: 'utf8', - stdio: ['inherit', 'pipe', 'inherit'], + stdio: ['inherit', 'pipe', 'pipe'], }); if (result.error) { throw new Error(`Failed to run ssh: ${(result.error as Error).message}`); } - return { status: result.status ?? 1, stdout: result.stdout ?? '' }; + return { status: result.status ?? 1, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }; +} + +function assertSafeScpEndpoint(endpoint: string): void { + const colon = endpoint.indexOf(':'); + const slash = endpoint.indexOf('/'); + if (colon <= 0 || (slash !== -1 && slash < colon)) { + if (endpoint.startsWith('-')) { + throw new Error(`Refusing to use scp endpoint that looks like an option: ${endpoint}`); + } + return; + } + + const host = endpoint.slice(0, colon); + const remotePath = endpoint.slice(colon + 1); + assertSafeSshHost(host); + if (remotePath.startsWith('-')) { + throw new Error(`Refusing to use scp remote path that looks like an option: ${remotePath}`); + } } export function runScp(from: string, to: string): void { + assertSafeScpEndpoint(from); + assertSafeScpEndpoint(to); const result = spawnSync('scp', ['-q', from, to], { encoding: 'utf8', stdio: ['inherit', 'inherit', 'inherit'], diff --git a/launcher/sync/sync-db.test.ts b/launcher/sync/sync-db.test.ts index 054ad8f3..6f72b190 100644 --- a/launcher/sync/sync-db.test.ts +++ b/launcher/sync/sync-db.test.ts @@ -26,7 +26,11 @@ function makeDbPair(): { dir: string; localPath: string; remotePath: string } { return { dir, localPath, remotePath }; } -function queryOne>(dbPath: string, sql: string, params: unknown[] = []): T | undefined { +function queryOne>( + dbPath: string, + sql: string, + params: unknown[] = [], +): T | undefined { const db = new Database(dbPath, { readonly: true }); try { return db.query(sql).get(...params) as T | undefined; @@ -39,6 +43,15 @@ function count(dbPath: string, sql: string, params: unknown[] = []): number { return Number(queryOne<{ n: number }>(dbPath, sql, params)?.n ?? 0); } +function withWritableDb(dbPath: string, fn: (db: Database) => T): T { + const db = new Database(dbPath, { readwrite: true }); + try { + return fn(db); + } finally { + db.close(); + } +} + test('merges remote-only sessions with catalog, lifetime, and rollups', () => { const { dir, localPath, remotePath } = makeDbPair(); try { @@ -73,7 +86,13 @@ test('merges remote-only sessions with catalog, lifetime, and rollups', () => { assert.equal(summary.telemetryRowsAdded, 1); assert.equal(count(localPath, 'SELECT COUNT(*) AS n FROM imm_sessions'), 2); - const global = queryOne<{ total_sessions: number; total_active_ms: number; total_cards: number; active_days: number; episodes_started: number }>( + const global = queryOne<{ + total_sessions: number; + total_active_ms: number; + total_cards: number; + active_days: number; + episodes_started: number; + }>( localPath, 'SELECT total_sessions, total_active_ms, total_cards, active_days, episodes_started FROM imm_lifetime_global WHERE global_id = 1', ); @@ -98,11 +117,15 @@ test('merges remote-only sessions with catalog, lifetime, and rollups', () => { // The merged session's rollup group was recomputed. assert.equal(summary.rollupGroupsRecomputed, 1); const mergedVideoId = Number( - queryOne<{ video_id: number }>(localPath, `SELECT video_id FROM imm_videos WHERE video_key = 'showb-e1'`) - ?.video_id, + queryOne<{ video_id: number }>( + localPath, + `SELECT video_id FROM imm_videos WHERE video_key = 'showb-e1'`, + )?.video_id, ); assert.equal( - count(localPath, 'SELECT COUNT(*) AS n FROM imm_daily_rollups WHERE video_id = ?', [mergedVideoId]), + count(localPath, 'SELECT COUNT(*) AS n FROM imm_daily_rollups WHERE video_id = ?', [ + mergedVideoId, + ]), 1, ); } finally { @@ -143,7 +166,12 @@ test('is idempotent: re-merging the same snapshot changes nothing', () => { { ...globalAfterFirst, LAST_UPDATE_DATE: null }, ); assert.equal( - Number(queryOne<{ frequency: number }>(localPath, `SELECT frequency FROM imm_words WHERE word = '見た'`)?.frequency), + Number( + queryOne<{ frequency: number }>( + localPath, + `SELECT frequency FROM imm_words WHERE word = '見た'`, + )?.frequency, + ), 4, ); } finally { @@ -161,9 +189,11 @@ test('preserves anilist_id when inserting a new anime from the snapshot', () => startedAtMs: BASE_MS, applyLifetime: true, }); - const remoteDb = new Database(remotePath, { readwrite: true }); - remoteDb.prepare(`UPDATE imm_anime SET anilist_id = 12345 WHERE normalized_title_key = 'showb'`).run(); - remoteDb.close(); + withWritableDb(remotePath, (remoteDb) => { + remoteDb + .prepare(`UPDATE imm_anime SET anilist_id = 12345 WHERE normalized_title_key = 'showb'`) + .run(); + }); const summary = mergeSnapshotIntoDb(localPath, remotePath); assert.equal(summary.animeAdded, 1); @@ -194,12 +224,16 @@ test('matches an existing local anime by anilist_id even when the title key diff startedAtMs: BASE_MS + DAY_MS, applyLifetime: true, }); - const localDb = new Database(localPath, { readwrite: true }); - localDb.prepare(`UPDATE imm_anime SET anilist_id = 999 WHERE normalized_title_key = 'show-romaji'`).run(); - localDb.close(); - const remoteDb = new Database(remotePath, { readwrite: true }); - remoteDb.prepare(`UPDATE imm_anime SET anilist_id = 999 WHERE normalized_title_key = 'show-native'`).run(); - remoteDb.close(); + withWritableDb(localPath, (localDb) => { + localDb + .prepare(`UPDATE imm_anime SET anilist_id = 999 WHERE normalized_title_key = 'show-romaji'`) + .run(); + }); + withWritableDb(remotePath, (remoteDb) => { + remoteDb + .prepare(`UPDATE imm_anime SET anilist_id = 999 WHERE normalized_title_key = 'show-native'`) + .run(); + }); const summary = mergeSnapshotIntoDb(localPath, remotePath); // Same anilist_id → one anime, not two. @@ -241,7 +275,11 @@ test('matches shared videos by video_key and merges the watched flag', () => { // Both sessions now credit the same video; episode was started once and // completed once (by the remote session that watched it to the end). - const global = queryOne<{ episodes_started: number; episodes_completed: number; total_sessions: number }>( + const global = queryOne<{ + episodes_started: number; + episodes_completed: number; + total_sessions: number; + }>( localPath, 'SELECT episodes_started, episodes_completed, total_sessions FROM imm_lifetime_global WHERE global_id = 1', ); @@ -253,6 +291,75 @@ test('matches shared videos by video_key and merges the watched flag', () => { } }); +test('backfills media metadata for matched videos when local metadata is absent', () => { + const { dir, localPath, remotePath } = makeDbPair(); + try { + insertFixtureSession(localPath, { + uuid: 'local-1', + videoKey: 'showa-e1', + animeTitleKey: 'showa', + startedAtMs: BASE_MS, + applyLifetime: true, + }); + insertFixtureSession(remotePath, { + uuid: 'remote-1', + videoKey: 'showa-e1', + animeTitleKey: 'showa', + startedAtMs: BASE_MS + DAY_MS, + applyLifetime: true, + }); + const remoteDb = new Database(remotePath, { readwrite: true }); + try { + const remoteVideoId = Number( + remoteDb + .query<{ + video_id: number; + }>(`SELECT video_id FROM imm_videos WHERE video_key = 'showa-e1'`) + .get()?.video_id, + ); + remoteDb + .prepare( + `INSERT INTO imm_media_art (video_id, anilist_id, cover_url, title_romaji, fetched_at_ms) + VALUES (?, 123, 'https://example.test/cover.jpg', 'Show A', ?)`, + ) + .run(remoteVideoId, String(BASE_MS)); + remoteDb + .prepare( + `INSERT INTO imm_youtube_videos (video_id, youtube_video_id, video_url, video_title, fetched_at_ms) + VALUES (?, 'yt-1', 'https://youtube.test/watch?v=yt-1', 'Remote Video', ?)`, + ) + .run(remoteVideoId, String(BASE_MS)); + } finally { + remoteDb.close(); + } + + const summary = mergeSnapshotIntoDb(localPath, remotePath); + assert.equal(summary.videosAdded, 0); + const localVideoId = Number( + queryOne<{ video_id: number }>( + localPath, + `SELECT video_id FROM imm_videos WHERE video_key = 'showa-e1'`, + )?.video_id, + ); + const art = queryOne<{ cover_url: string; title_romaji: string }>( + localPath, + 'SELECT cover_url, title_romaji FROM imm_media_art WHERE video_id = ?', + [localVideoId], + ); + assert.equal(art?.cover_url, 'https://example.test/cover.jpg'); + assert.equal(art?.title_romaji, 'Show A'); + const youtube = queryOne<{ youtube_video_id: string; video_title: string }>( + localPath, + 'SELECT youtube_video_id, video_title FROM imm_youtube_videos WHERE video_id = ?', + [localVideoId], + ); + assert.equal(youtube?.youtube_video_id, 'yt-1'); + assert.equal(youtube?.video_title, 'Remote Video'); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + test('does not double-count active_days when a merged session starts earlier on an already-credited day', () => { const { dir, localPath, remotePath } = makeDbPair(); try { @@ -310,18 +417,21 @@ test('copies remote-only historical rollups but never sums shared groups', () => startedAtMs: BASE_MS, applyLifetime: true, }); - const remoteDb = new Database(remotePath, { readwrite: true }); - const remoteVideoId = Number( - remoteDb.query<{ video_id: number }>(`SELECT video_id FROM imm_videos WHERE video_key = 'old-show-e1'`).get() - ?.video_id, - ); - remoteDb - .prepare( - `INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards) - VALUES (10000, ?, 4, 120.5, 400, 3200, 9)`, - ) - .run(remoteVideoId); - remoteDb.close(); + withWritableDb(remotePath, (remoteDb) => { + const remoteVideoId = Number( + remoteDb + .query<{ + video_id: number; + }>(`SELECT video_id FROM imm_videos WHERE video_key = 'old-show-e1'`) + .get()?.video_id, + ); + remoteDb + .prepare( + `INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards) + VALUES (10000, ?, 4, 120.5, 400, 3200, 9)`, + ) + .run(remoteVideoId); + }); // Local already has its own rollup row for a shared group. insertFixtureSession(localPath, { @@ -330,18 +440,22 @@ test('copies remote-only historical rollups but never sums shared groups', () => startedAtMs: BASE_MS, applyLifetime: true, }); - const localDb = new Database(localPath, { readwrite: true }); - const localVideoId = Number( - localDb.query<{ video_id: number }>(`SELECT video_id FROM imm_videos WHERE video_key = 'old-show-e1'`).get() - ?.video_id, - ); - localDb - .prepare( - `INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards) - VALUES (10000, ?, 2, 60.0, 200, 1600, 4)`, - ) - .run(localVideoId); - localDb.close(); + const localVideoId = withWritableDb(localPath, (localDb) => { + const videoId = Number( + localDb + .query<{ + video_id: number; + }>(`SELECT video_id FROM imm_videos WHERE video_key = 'old-show-e1'`) + .get()?.video_id, + ); + localDb + .prepare( + `INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards) + VALUES (10000, ?, 2, 60.0, 200, 1600, 4)`, + ) + .run(videoId); + return videoId; + }); const summary = mergeSnapshotIntoDb(localPath, remotePath); // Shared group (day 10000) untouched; the remote-only session's own group @@ -355,28 +469,30 @@ test('copies remote-only historical rollups but never sums shared groups', () => assert.equal(shared?.total_sessions, 2); // Now a rollup for a video with no local sessions at all gets copied. - const remoteDb2 = new Database(remotePath, { readwrite: true }); - const orphanVideoId = Number( + withWritableDb(remotePath, (remoteDb2) => { + const orphanVideoId = Number( + remoteDb2 + .prepare( + `INSERT INTO imm_videos (video_key, canonical_title, source_type, watched, duration_ms) + VALUES ('pruned-show-e1', 'pruned-show-e1', 1, 1, 1440000)`, + ) + .run().lastInsertRowid, + ); remoteDb2 .prepare( - `INSERT INTO imm_videos (video_key, canonical_title, source_type, watched, duration_ms) - VALUES ('pruned-show-e1', 'pruned-show-e1', 1, 1, 1440000)`, + `INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards) + VALUES (9000, ?, 3, 90.0, 300, 2400, 6)`, ) - .run().lastInsertRowid, - ); - remoteDb2 - .prepare( - `INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards) - VALUES (9000, ?, 3, 90.0, 300, 2400, 6)`, - ) - .run(orphanVideoId); - remoteDb2.close(); + .run(orphanVideoId); + }); const summary2 = mergeSnapshotIntoDb(localPath, remotePath); assert.equal(summary2.dailyRollupsCopied, 1); const copiedVideoId = Number( - queryOne<{ video_id: number }>(localPath, `SELECT video_id FROM imm_videos WHERE video_key = 'pruned-show-e1'`) - ?.video_id, + queryOne<{ video_id: number }>( + localPath, + `SELECT video_id FROM imm_videos WHERE video_key = 'pruned-show-e1'`, + )?.video_id, ); const copied = queryOne<{ total_sessions: number; total_active_min: number }>( localPath, @@ -390,12 +506,70 @@ test('copies remote-only historical rollups but never sums shared groups', () => } }); +test('does not copy remote-only daily rollups into months with local sessions', () => { + const { dir, localPath, remotePath } = makeDbPair(); + try { + insertFixtureSession(localPath, { + uuid: 'local-1', + videoKey: 'mixed-month-e1', + animeTitleKey: 'mixed-month', + startedAtMs: BASE_MS, + applyLifetime: true, + }); + const remoteDb = new Database(remotePath, { readwrite: true }); + try { + const remoteVideoId = Number( + remoteDb + .prepare( + `INSERT INTO imm_videos (video_key, canonical_title, source_type, watched, duration_ms) + VALUES ('mixed-month-e1', 'mixed-month-e1', 1, 1, 1440000)`, + ) + .run().lastInsertRowid, + ); + remoteDb + .prepare( + `INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards) + VALUES (20615, ?, 3, 90.0, 300, 2400, 6)`, + ) + .run(remoteVideoId); + remoteDb + .prepare( + `INSERT INTO imm_monthly_rollups (rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards) + VALUES (202606, ?, 3, 90.0, 300, 2400, 6)`, + ) + .run(remoteVideoId); + } finally { + remoteDb.close(); + } + + const summary = mergeSnapshotIntoDb(localPath, remotePath); + assert.equal(summary.dailyRollupsCopied, 0); + assert.equal(summary.monthlyRollupsCopied, 0); + const localVideoId = Number( + queryOne<{ video_id: number }>( + localPath, + `SELECT video_id FROM imm_videos WHERE video_key = 'mixed-month-e1'`, + )?.video_id, + ); + assert.equal( + count( + localPath, + 'SELECT COUNT(*) AS n FROM imm_daily_rollups WHERE rollup_day = 20615 AND video_id = ?', + [localVideoId], + ), + 0, + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + test('rejects snapshots at a different schema version', () => { const { dir, localPath, remotePath } = makeDbPair(); try { - const remoteDb = new Database(remotePath, { readwrite: true }); - remoteDb.prepare('UPDATE imm_schema_version SET schema_version = 17').run(); - remoteDb.close(); + withWritableDb(remotePath, (remoteDb) => { + remoteDb.prepare('UPDATE imm_schema_version SET schema_version = 17').run(); + }); assert.throws(() => mergeSnapshotIntoDb(localPath, remotePath), /schema version 17/); } finally { diff --git a/launcher/test-support/immersion-db-fixture.ts b/launcher/test-support/immersion-db-fixture.ts index d820ff1d..cc11a36e 100644 --- a/launcher/test-support/immersion-db-fixture.ts +++ b/launcher/test-support/immersion-db-fixture.ts @@ -5,6 +5,7 @@ import { IMMERSION_DB_FIXTURE_DDL } from './immersion-db-schema.js'; export function createImmersionDbFixture(dbPath: string): void { const db = new Database(dbPath, { create: true }); try { + db.run('PRAGMA foreign_keys = ON'); db.run('PRAGMA journal_mode = WAL'); for (const statement of IMMERSION_DB_FIXTURE_DDL.split(';')) { const sql = statement.trim(); @@ -14,7 +15,9 @@ export function createImmersionDbFixture(dbPath: string): void { SCHEMA_VERSION, String(Date.now()), ); - db.prepare(`INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('last_rollup_sample_ms', 0)`).run(); + db.prepare( + `INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('last_rollup_sample_ms', 0)`, + ).run(); db.prepare( `INSERT INTO imm_lifetime_global(global_id, CREATED_DATE, LAST_UPDATE_DATE) VALUES (1, ?, ?)`, ).run(String(Date.now()), String(Date.now())); @@ -53,7 +56,9 @@ export function insertFixtureSession(dbPath: string, input: FixtureSessionInput) let animeId: number | null = null; if (input.animeTitleKey) { const existing = db - .query<{ anime_id: number }>('SELECT anime_id FROM imm_anime WHERE normalized_title_key = ?') + .query<{ + anime_id: number; + }>('SELECT anime_id FROM imm_anime WHERE normalized_title_key = ?') .get(input.animeTitleKey); animeId = existing ? existing.anime_id @@ -98,7 +103,8 @@ export function insertFixtureSession(dbPath: string, input: FixtureSessionInput) db.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?').run(videoId); } - const endedAtMs = input.endedAtMs === undefined ? input.startedAtMs + 1_500_000 : input.endedAtMs; + const endedAtMs = + input.endedAtMs === undefined ? input.startedAtMs + 1_500_000 : input.endedAtMs; const activeWatchedMs = input.activeWatchedMs ?? 1_200_000; const cardsMined = input.cardsMined ?? 2; const linesSeen = input.linesSeen ?? 100; @@ -147,7 +153,9 @@ export function insertFixtureSession(dbPath: string, input: FixtureSessionInput) for (const word of input.words ?? []) { const existing = db - .query<{ id: number }>('SELECT id FROM imm_words WHERE headword = ? AND word = ? AND reading = ?') + .query<{ + id: number; + }>('SELECT id FROM imm_words WHERE headword = ? AND word = ? AND reading = ?') .get(word.headword, word.word, word.reading); const wordId = existing ? existing.id @@ -172,13 +180,23 @@ export function insertFixtureSession(dbPath: string, input: FixtureSessionInput) `INSERT INTO imm_subtitle_lines (session_id, video_id, anime_id, line_index, text, CREATED_DATE, LAST_UPDATE_DATE) VALUES (?, ?, ?, ?, ?, ?, ?)`, ) - .run(sessionId, videoId, animeId, uniqueCounter, `line ${word.word}`, input.startedAtMs, input.startedAtMs) - .lastInsertRowid, + .run( + sessionId, + videoId, + animeId, + uniqueCounter, + `line ${word.word}`, + input.startedAtMs, + input.startedAtMs, + ).lastInsertRowid, ); db.prepare( 'INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count) VALUES (?, ?, ?)', ).run(lineId, wordId, word.count); - db.prepare('UPDATE imm_words SET frequency = frequency + ? WHERE id = ?').run(word.count, wordId); + db.prepare('UPDATE imm_words SET frequency = frequency + ? WHERE id = ?').run( + word.count, + wordId, + ); } if (input.applyLifetime && endedAtMs !== null) { diff --git a/launcher/test-support/immersion-db-schema.test.ts b/launcher/test-support/immersion-db-schema.test.ts new file mode 100644 index 00000000..87d79877 --- /dev/null +++ b/launcher/test-support/immersion-db-schema.test.ts @@ -0,0 +1,109 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Database as BunDatabase } from 'bun:sqlite'; +import { ensureSchema } from '../../src/core/services/immersion-tracker/storage.js'; +import { createImmersionDbFixture } from './immersion-db-fixture.js'; + +type SchemaRow = { type: string; name: string; tbl_name: string; sql: string | null }; + +const SYNC_SCHEMA_OBJECTS = [ + 'imm_anime', + 'imm_videos', + 'imm_sessions', + 'imm_session_telemetry', + 'imm_session_events', + 'imm_daily_rollups', + 'imm_monthly_rollups', + 'imm_words', + 'imm_kanji', + 'imm_subtitle_lines', + 'imm_word_line_occurrences', + 'imm_kanji_line_occurrences', + 'imm_media_art', + 'imm_youtube_videos', + 'imm_cover_art_blobs', + 'imm_lifetime_global', + 'imm_lifetime_anime', + 'imm_lifetime_media', + 'imm_lifetime_applied_sessions', + 'imm_stats_excluded_words', + 'idx_anime_normalized_title', + 'idx_anime_anilist_id', + 'idx_videos_anime_id', + 'idx_sessions_video_started', + 'idx_sessions_status_started', + 'idx_sessions_started_at', + 'idx_sessions_ended_at', + 'idx_telemetry_session_sample', + 'idx_telemetry_sample_ms', + 'idx_events_session_ts', + 'idx_events_type_ts', + 'idx_rollups_day_video', + 'idx_rollups_month_video', + 'idx_words_headword_word_reading', + 'idx_words_frequency', + 'idx_kanji_kanji', + 'idx_kanji_frequency', + 'idx_subtitle_lines_session_line', + 'idx_subtitle_lines_video_line', + 'idx_subtitle_lines_anime_line', + 'idx_word_line_occurrences_word', + 'idx_kanji_line_occurrences_kanji', + 'idx_media_art_cover_blob_hash', + 'idx_media_art_anilist_id', + 'idx_media_art_cover_url', + 'idx_youtube_videos_channel_id', + 'idx_youtube_videos_youtube_video_id', +] as const; + +function normalizeSql(sql: string | null): string { + return (sql ?? '') + .replace(/\bIF NOT EXISTS\b/gi, '') + .replace(/\s+/g, ' ') + .replace(/\s+([(),])/g, '$1') + .replace(/,\s+/g, ', ') + .trim(); +} + +function readSchema(dbPath: string): Map { + const db = new BunDatabase(dbPath, { readonly: true }); + try { + const rows = db + .query( + `SELECT type, name, tbl_name, sql + FROM sqlite_schema + WHERE name IN (${SYNC_SCHEMA_OBJECTS.map(() => '?').join(',')}) + ORDER BY type, name`, + ) + .all(...SYNC_SCHEMA_OBJECTS); + return new Map(rows.map((row) => [row.name, normalizeSql(row.sql)])); + } finally { + db.close(); + } +} + +test('fixture schema stays aligned with production sync-touched tables and indexes', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-fixture-schema-')); + const fixturePath = path.join(dir, 'fixture.sqlite'); + const productionPath = path.join(dir, 'production.sqlite'); + const productionDb = new BunDatabase(productionPath, { create: true }); + try { + createImmersionDbFixture(fixturePath); + ensureSchema(productionDb as never); + productionDb.close(); + + const fixtureSchema = readSchema(fixturePath); + const productionSchema = readSchema(productionPath); + assert.deepEqual(fixtureSchema, productionSchema); + } finally { + try { + productionDb.close(); + } catch { + // already closed + } + fs.rmSync(dir, { recursive: true, force: true }); + } +});