fix(launcher): address sync review feedback

This commit is contained in:
2026-07-09 00:30:30 -07:00
parent 58d54311d8
commit dfde19cc4d
11 changed files with 747 additions and 133 deletions
+1 -1
View File
@@ -1,4 +1,4 @@
type: added type: added
area: launcher area: launcher
- Added `subminer sync <host>` to merge immersion stats and watch history between two machines over SSH. Each side snapshots its database (`VACUUM INTO`), snapshots are exchanged with `scp`, and each machine merges the other's data as an insert-only union keyed on session UUIDs / video keys / series title keys, so re-syncing is idempotent and nothing is double-counted. Lifetime totals and daily/monthly rollups are updated incrementally (history older than the session retention window is preserved); remote-only historical rollups are copied. `subminer sync --snapshot <file>` and `subminer sync --merge <file>` expose the underlying steps for manual transfers. The command refuses to run while the stats daemon or an mpv session is active (`--force` overrides) and aborts on stats schema version mismatches. - Added `subminer sync <host>` to merge immersion stats and watch history between two machines over SSH. Each side snapshots its database (`VACUUM INTO`), snapshots are exchanged with `scp`, and each machine merges the other's data as an insert-only union keyed on session UUIDs / video keys / series title keys, so re-syncing is idempotent and nothing is double-counted. Lifetime totals and daily/monthly rollups are updated incrementally (history older than the session retention window is preserved); remote-only historical rollups are copied only when they do not conflict with retained local session history. `subminer sync --snapshot <file>` and `subminer sync --merge <file>` expose the underlying steps for manual transfers. The command refuses to run while the stats daemon or an mpv session is active (`--force` overrides), keeps that guard in place through local/remote merges, reports remote stderr on failures, and aborts on stats schema version mismatches.
+4
View File
@@ -111,6 +111,9 @@ subminer config show # Print active config contents
subminer mpv socket # Print active mpv socket path subminer mpv socket # Print active mpv socket path
subminer mpv status # Exit 0 if socket is ready, else exit 1 subminer mpv status # Exit 0 if socket is ready, else exit 1
subminer mpv idle # Launch detached idle mpv with SubMiner defaults subminer mpv idle # Launch detached idle mpv with SubMiner defaults
subminer sync media-box # Sync stats/watch history with an SSH host
subminer sync --snapshot ~/subminer-snapshot.sqlite # Write a local DB snapshot
subminer sync --merge ~/subminer-snapshot.sqlite # Merge a snapshot into the local DB
subminer dictionary /path/to/file-or-directory # Generate character dictionary ZIP from target (manual Yomitan import) subminer dictionary /path/to/file-or-directory # Generate character dictionary ZIP from target (manual Yomitan import)
subminer dictionary --candidates /path/to/file.mkv subminer dictionary --candidates /path/to/file.mkv
subminer dictionary --select 21355 /path/to/file.mkv subminer dictionary --select 21355 /path/to/file.mkv
@@ -194,6 +197,7 @@ This flow requires `mpv.exe` to be discoverable. Leave `mpv.executablePath` blan
- `subminer logs -e`: export a sanitized ZIP of today's local-date logs, or the most recent logs when no current-day log exists. The exported copy masks common PII and secrets; on-disk logs are unchanged. - `subminer logs -e`: export a sanitized ZIP of today's local-date logs, or the most recent logs when no current-day log exists. The exported copy masks common PII and secrets; on-disk logs are unchanged.
- `subminer config`: config file helpers (`path`, `show`). - `subminer config`: config file helpers (`path`, `show`).
- `subminer mpv`: mpv helpers (`status`, `socket`, `idle`). - `subminer mpv`: mpv helpers (`status`, `socket`, `idle`).
- `subminer sync <host>`: sync immersion stats and watch history with another machine over SSH. The host is the SSH destination (`user@host` or an SSH config alias). Use `--snapshot <file>` to write a consistent local stats DB snapshot, `--merge <file>` to merge a snapshot into the local stats DB, and `--force` to skip the running stats/mpv safety check. Advanced options: `--db <file>` overrides the local stats DB path, and `--remote-cmd <cmd>` overrides the `subminer` command used on the remote host.
- `subminer dictionary <path>`: generates a Yomitan-importable character dictionary ZIP from a file/directory target. - `subminer dictionary <path>`: generates a Yomitan-importable character dictionary ZIP from a file/directory target.
- Use `subminer dictionary --candidates <path>` and `subminer dictionary --select <id> <path>` to correct AniList character-dictionary matches for a whole series. - Use `subminer dictionary --candidates <path>` and `subminer dictionary --select <id> <path>` to correct AniList character-dictionary matches for a whole series.
- `subminer texthooker`: texthooker-only shortcut (same behavior as `--texthooker`). A _texthooker_ is a web page that displays the current subtitle line as selectable text, so browser-based dictionary extensions and other tools can read along with playback. - `subminer texthooker`: texthooker-only shortcut (same behavior as `--texthooker`). A _texthooker_ is a web page that displays the current subtitle line as selectable text, so browser-based dictionary extensions and other tools can read along with playback.
+165
View File
@@ -0,0 +1,165 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import type { Args } from '../types.js';
import { createEmptyMergeSummary } from '../sync/sync-shared.js';
import type { LauncherCommandContext } from './context.js';
import { runSyncCommand, type SyncCommandDeps } from './sync-command.js';
function makeContext(overrides: Partial<Args>): LauncherCommandContext {
return {
args: {
sync: true,
syncHost: '',
syncSnapshotPath: '',
syncMergePath: '',
syncRemoteCmd: '',
syncDbPath: '',
syncForce: false,
logLevel: 'warn',
...overrides,
} as Args,
scriptPath: '/tmp/subminer',
scriptName: 'subminer',
mpvSocketPath: '',
pluginRuntimeConfig: {},
appPath: null,
launcherJellyfinConfig: {},
processAdapter: process,
} as unknown as LauncherCommandContext;
}
function ok(stdout = ''): { status: number; stdout: string; stderr: string } {
return { status: 0, stdout, stderr: '' };
}
test('runSyncCommand dispatches snapshot, merge, host, and missing-target modes', () => {
const calls: string[] = [];
const deps: Partial<SyncCommandDeps> = {
createDbSnapshot: (dbPath: string, outPath: string) => {
calls.push(`snapshot:${dbPath}->${outPath}`);
},
mergeSnapshotIntoDb: (dbPath: string, snapshotPath: string) => {
calls.push(`merge:${dbPath}<-${snapshotPath}`);
return createEmptyMergeSummary();
},
formatMergeSummary: () => 'summary',
ensureTrackerQuiescent: () => {
calls.push('quiescent');
},
assertSafeSshHost: (host: string) => {
calls.push(`host:${host}`);
},
resolveRemoteSubminerCommand: () => 'subminer',
runSsh: (_host: string, command: string) => {
calls.push(`ssh:${command}`);
return command.startsWith('mktemp ') ? ok('/tmp/subminer-sync.remote\n') : ok();
},
runScp: (from: string, to: string) => {
calls.push(`scp:${from}->${to}`);
},
fail: (message: string): never => {
throw new Error(message);
},
};
assert.equal(
runSyncCommand(
makeContext({ syncDbPath: '/tmp/local.sqlite', syncSnapshotPath: '/tmp/out.sqlite' }),
deps,
),
true,
);
assert.ok(calls.includes('snapshot:/tmp/local.sqlite->/tmp/out.sqlite'));
runSyncCommand(
makeContext({ syncDbPath: '/tmp/local.sqlite', syncMergePath: '/tmp/in.sqlite' }),
deps,
);
assert.ok(calls.includes('quiescent'));
assert.ok(calls.includes('merge:/tmp/local.sqlite<-/tmp/in.sqlite'));
runSyncCommand(makeContext({ syncDbPath: '/tmp/local.sqlite', syncHost: 'media-box' }), deps);
assert.ok(calls.includes('host:media-box'));
assert.throws(
() => runSyncCommand(makeContext({ syncDbPath: '/tmp/local.sqlite' }), deps),
/sync requires a host, --snapshot <file>, or --merge <file>/,
);
});
test('runHostSync keeps tracker quiescent through local and remote merge and cleans up after failure', () => {
const calls: string[] = [];
let localTmpDir = '';
const deps: Partial<SyncCommandDeps> = {
createDbSnapshot: (_dbPath: string, outPath: string) => {
calls.push(`snapshot:${outPath}`);
fs.writeFileSync(outPath, 'snapshot');
},
mergeSnapshotIntoDb: () => {
calls.push('local-merge');
return createEmptyMergeSummary();
},
formatMergeSummary: () => 'summary',
ensureTrackerQuiescent: () => {
calls.push('quiescent');
},
assertSafeSshHost: () => {},
resolveRemoteSubminerCommand: () => 'subminer',
mkdtempSync: ((prefix: string) => {
localTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), path.basename(prefix)));
return localTmpDir;
}) as typeof fs.mkdtempSync,
runSsh: (_host: string, command: string) => {
calls.push(`ssh:${command}`);
if (command.startsWith('mktemp ')) return ok('/tmp/subminer-sync.remote\n');
if (command.includes(' sync --snapshot ')) return ok();
if (command.includes(' sync --merge ')) {
return { status: 9, stdout: 'remote output', stderr: 'remote merge exploded' };
}
return ok();
},
runScp: (from: string, to: string) => {
calls.push(`scp:${from}->${to}`);
if (!to.includes(':')) fs.writeFileSync(to, 'pulled');
},
};
assert.throws(
() =>
runSyncCommand(makeContext({ syncDbPath: '/tmp/local.sqlite', syncHost: 'media-box' }), deps),
/Remote merge failed on media-box[\s\S]*remote merge exploded/,
);
assert.equal(calls.filter((call) => call === 'quiescent').length, 3);
assert.ok(calls.indexOf('quiescent') < calls.findIndex((call) => call.startsWith('snapshot:')));
assert.ok(calls.includes('local-merge'));
assert.ok(calls.some((call) => call.startsWith('ssh:rm -rf ')));
assert.equal(fs.existsSync(localTmpDir), false);
});
test('runHostSync includes remote snapshot stderr in failures', () => {
const deps: Partial<SyncCommandDeps> = {
createDbSnapshot: (_dbPath: string, outPath: string) => {
fs.writeFileSync(outPath, 'snapshot');
},
ensureTrackerQuiescent: () => {},
assertSafeSshHost: () => {},
resolveRemoteSubminerCommand: () => 'subminer',
runSsh: (_host: string, command: string) => {
if (command.startsWith('mktemp ')) return ok('/tmp/subminer-sync.remote\n');
if (command.includes(' sync --snapshot ')) {
return { status: 5, stdout: '', stderr: 'snapshot permission denied' };
}
return ok();
},
runScp: () => {},
};
assert.throws(
() =>
runSyncCommand(makeContext({ syncDbPath: '/tmp/local.sqlite', syncHost: 'media-box' }), deps),
/Remote snapshot failed on media-box[\s\S]*snapshot permission denied/,
);
});
+124 -45
View File
@@ -18,74 +18,144 @@ import {
} from '../sync/ssh.js'; } from '../sync/ssh.js';
import { resolvePathMaybe } from '../util.js'; import { resolvePathMaybe } from '../util.js';
import type { LauncherCommandContext } from './context.js'; import type { LauncherCommandContext } from './context.js';
import type { RemoteRunResult } from '../sync/ssh.js';
export interface SyncCommandDeps {
createDbSnapshot: typeof createDbSnapshot;
mergeSnapshotIntoDb: typeof mergeSnapshotIntoDb;
formatMergeSummary: typeof formatMergeSummary;
findLiveStatsDaemonPid: typeof findLiveStatsDaemonPid;
assertSafeSshHost: typeof assertSafeSshHost;
resolveRemoteSubminerCommand: typeof resolveRemoteSubminerCommand;
runScp: typeof runScp;
runSsh: typeof runSsh;
fail: typeof fail;
log: typeof log;
existsSync: typeof fs.existsSync;
realpathSync: typeof fs.realpathSync;
mkdtempSync: typeof fs.mkdtempSync;
rmSync: typeof fs.rmSync;
consoleLog: typeof console.log;
writeStdout: typeof process.stdout.write;
ensureTrackerQuiescent: (context: LauncherCommandContext, dbPath: string) => void;
}
function resolveDbPath(context: LauncherCommandContext): string { function resolveDbPath(context: LauncherCommandContext): string {
const override = context.args.syncDbPath.trim(); const override = context.args.syncDbPath.trim();
return override ? resolvePathMaybe(override) : resolveImmersionDbPath(); return override ? resolvePathMaybe(override) : resolveImmersionDbPath();
} }
function isTrackerDb(dbPath: string): boolean { function isTrackerDb(dbPath: string, deps: SyncCommandDeps): boolean {
const trackerDbPath = resolveImmersionDbPath(); const trackerDbPath = resolveImmersionDbPath();
try { try {
return fs.realpathSync(dbPath) === fs.realpathSync(trackerDbPath); return deps.realpathSync(dbPath) === deps.realpathSync(trackerDbPath);
} catch { } catch {
return dbPath === trackerDbPath; return dbPath === trackerDbPath;
} }
} }
function ensureTrackerQuiescent(context: LauncherCommandContext, dbPath: string): void { export function ensureTrackerQuiescent(
context: LauncherCommandContext,
dbPath: string,
inputDeps: Partial<SyncCommandDeps> = {},
): void {
const deps = resolveSyncCommandDeps(inputDeps);
if (context.args.syncForce) return; if (context.args.syncForce) return;
// A running SubMiner only holds the tracker's own database; --db pointed // A running SubMiner only holds the tracker's own database; --db pointed
// elsewhere needs no guard. // elsewhere needs no guard.
if (!isTrackerDb(dbPath)) return; if (!isTrackerDb(dbPath, deps)) return;
const daemonPid = findLiveStatsDaemonPid(dbPath); const daemonPid = deps.findLiveStatsDaemonPid(dbPath);
if (daemonPid !== null) { if (daemonPid !== null) {
fail( deps.fail(
`The SubMiner stats server is running (pid ${daemonPid}). Stop it with "subminer stats -s" (or close SubMiner) before syncing, or pass --force.`, `The SubMiner stats server is running (pid ${daemonPid}). Stop it with "subminer stats -s" (or close SubMiner) before syncing, or pass --force.`,
); );
} }
if (context.mpvSocketPath && fs.existsSync(context.mpvSocketPath)) { if (context.mpvSocketPath && deps.existsSync(context.mpvSocketPath)) {
fail( deps.fail(
`An mpv/SubMiner session appears to be running (socket ${context.mpvSocketPath}). Close it before syncing, or pass --force.`, `An mpv/SubMiner session appears to be running (socket ${context.mpvSocketPath}). Close it before syncing, or pass --force.`,
); );
} }
} }
function runSnapshotMode(context: LauncherCommandContext, dbPath: string): void { const defaultSyncCommandDeps: SyncCommandDeps = {
createDbSnapshot,
mergeSnapshotIntoDb,
formatMergeSummary,
findLiveStatsDaemonPid,
assertSafeSshHost,
resolveRemoteSubminerCommand,
runScp,
runSsh,
fail,
log,
existsSync: fs.existsSync,
realpathSync: fs.realpathSync,
mkdtempSync: fs.mkdtempSync,
rmSync: fs.rmSync,
consoleLog: console.log,
writeStdout: process.stdout.write.bind(process.stdout),
ensureTrackerQuiescent: (context, dbPath) => ensureTrackerQuiescent(context, dbPath),
};
function resolveSyncCommandDeps(inputDeps: Partial<SyncCommandDeps> = {}): SyncCommandDeps {
return { ...defaultSyncCommandDeps, ...inputDeps };
}
export function runSnapshotMode(
context: LauncherCommandContext,
dbPath: string,
inputDeps: Partial<SyncCommandDeps> = {},
): void {
const deps = resolveSyncCommandDeps(inputDeps);
const outPath = resolvePathMaybe(context.args.syncSnapshotPath); const outPath = resolvePathMaybe(context.args.syncSnapshotPath);
createDbSnapshot(dbPath, outPath); deps.createDbSnapshot(dbPath, outPath);
console.log(outPath); deps.consoleLog(outPath);
} }
function runMergeMode(context: LauncherCommandContext, dbPath: string): void { export function runMergeMode(
ensureTrackerQuiescent(context, dbPath); context: LauncherCommandContext,
dbPath: string,
inputDeps: Partial<SyncCommandDeps> = {},
): void {
const deps = resolveSyncCommandDeps(inputDeps);
deps.ensureTrackerQuiescent(context, dbPath);
const snapshotPath = resolvePathMaybe(context.args.syncMergePath); const snapshotPath = resolvePathMaybe(context.args.syncMergePath);
const summary = mergeSnapshotIntoDb(dbPath, snapshotPath); const summary = deps.mergeSnapshotIntoDb(dbPath, snapshotPath);
console.log(formatMergeSummary(summary)); deps.consoleLog(deps.formatMergeSummary(summary));
} }
function cleanupRemote(host: string, remoteTmpDir: string): void { function cleanupRemote(host: string, remoteTmpDir: string, deps: SyncCommandDeps): void {
if (!remoteTmpDir.startsWith('/tmp/')) return; if (!remoteTmpDir.startsWith('/tmp/')) return;
runSsh(host, `rm -rf ${shellQuote(remoteTmpDir)}`); deps.runSsh(host, `rm -rf ${shellQuote(remoteTmpDir)}`);
} }
function runHostSync(context: LauncherCommandContext, dbPath: string): void { function formatRemoteRunError(message: string, run: RemoteRunResult): string {
const stderr = run.stderr.trim();
return stderr ? `${message}\n${stderr}` : message;
}
export function runHostSync(
context: LauncherCommandContext,
dbPath: string,
inputDeps: Partial<SyncCommandDeps> = {},
): void {
const deps = resolveSyncCommandDeps(inputDeps);
const { args } = context; const { args } = context;
const host = args.syncHost; const host = args.syncHost;
assertSafeSshHost(host); deps.assertSafeSshHost(host);
ensureTrackerQuiescent(context, dbPath); deps.ensureTrackerQuiescent(context, dbPath);
const remoteCmd = resolveRemoteSubminerCommand(host, args.syncRemoteCmd || null); const remoteCmd = deps.resolveRemoteSubminerCommand(host, args.syncRemoteCmd || null);
log('debug', args.logLevel, `Remote subminer command: ${remoteCmd}`); deps.log('debug', args.logLevel, `Remote subminer command: ${remoteCmd}`);
const localTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-sync-')); const localTmpDir = deps.mkdtempSync(path.join(os.tmpdir(), 'subminer-sync-'));
let remoteTmpDir = ''; let remoteTmpDir = '';
try { try {
// Signal failures by throwing (not fail(), which exits synchronously and // Signal failures by throwing (not fail(), which exits synchronously and
// would skip the finally cleanup, leaking temp dirs holding snapshot data). // would skip the finally cleanup, leaking temp dirs holding snapshot data).
// main().catch() reports the message the same way fail() would. // main().catch() reports the message the same way fail() would.
const mktemp = runSsh(host, 'mktemp -d /tmp/subminer-sync.XXXXXX'); const mktemp = deps.runSsh(host, 'mktemp -d /tmp/subminer-sync.XXXXXX');
remoteTmpDir = mktemp.stdout.trim(); remoteTmpDir = mktemp.stdout.trim();
if (mktemp.status !== 0 || !remoteTmpDir.startsWith('/tmp/')) { if (mktemp.status !== 0 || !remoteTmpDir.startsWith('/tmp/')) {
throw new Error(`Could not create a temporary directory on ${host}.`); throw new Error(`Could not create a temporary directory on ${host}.`);
@@ -93,47 +163,52 @@ function runHostSync(context: LauncherCommandContext, dbPath: string): void {
const forceFlag = args.syncForce ? ' --force' : ''; const forceFlag = args.syncForce ? ' --force' : '';
console.log(`Snapshotting local database (${dbPath})...`); deps.consoleLog(`Snapshotting local database (${dbPath})...`);
const localSnapshot = path.join(localTmpDir, 'local.sqlite'); const localSnapshot = path.join(localTmpDir, 'local.sqlite');
createDbSnapshot(dbPath, localSnapshot); deps.createDbSnapshot(dbPath, localSnapshot);
console.log(`Snapshotting ${host}...`); deps.consoleLog(`Snapshotting ${host}...`);
const remoteSnapshot = `${remoteTmpDir}/snapshot.sqlite`; const remoteSnapshot = `${remoteTmpDir}/snapshot.sqlite`;
const snapshotRun = runSsh( const snapshotRun = deps.runSsh(
host, host,
`${remoteCmd} sync --snapshot ${shellQuote(remoteSnapshot)}${forceFlag}`, `${remoteCmd} sync --snapshot ${shellQuote(remoteSnapshot)}${forceFlag}`,
); );
if (snapshotRun.status !== 0) { if (snapshotRun.status !== 0) {
throw new Error(`Remote snapshot failed on ${host}.`); throw new Error(formatRemoteRunError(`Remote snapshot failed on ${host}.`, snapshotRun));
} }
const pulledSnapshot = path.join(localTmpDir, 'remote.sqlite'); const pulledSnapshot = path.join(localTmpDir, 'remote.sqlite');
runScp(`${host}:${remoteSnapshot}`, pulledSnapshot); deps.runScp(`${host}:${remoteSnapshot}`, pulledSnapshot);
const incomingSnapshot = `${remoteTmpDir}/incoming.sqlite`; const incomingSnapshot = `${remoteTmpDir}/incoming.sqlite`;
runScp(localSnapshot, `${host}:${incomingSnapshot}`); deps.runScp(localSnapshot, `${host}:${incomingSnapshot}`);
console.log(`\nMerging ${host} -> local:`); deps.consoleLog(`\nMerging ${host} -> local:`);
const summary = mergeSnapshotIntoDb(dbPath, pulledSnapshot); deps.ensureTrackerQuiescent(context, dbPath);
console.log(formatMergeSummary(summary)); const summary = deps.mergeSnapshotIntoDb(dbPath, pulledSnapshot);
deps.consoleLog(deps.formatMergeSummary(summary));
console.log(`\nMerging local -> ${host}:`); deps.consoleLog(`\nMerging local -> ${host}:`);
const mergeRun = runSsh( deps.ensureTrackerQuiescent(context, dbPath);
const mergeRun = deps.runSsh(
host, host,
`${remoteCmd} sync --merge ${shellQuote(incomingSnapshot)}${forceFlag}`, `${remoteCmd} sync --merge ${shellQuote(incomingSnapshot)}${forceFlag}`,
); );
process.stdout.write(mergeRun.stdout); deps.writeStdout(mergeRun.stdout);
if (mergeRun.status !== 0) { if (mergeRun.status !== 0) {
throw new Error( throw new Error(
formatRemoteRunError(
`Remote merge failed on ${host}. The local database was updated; re-run "subminer sync ${host}" once the remote issue is fixed.`, `Remote merge failed on ${host}. The local database was updated; re-run "subminer sync ${host}" once the remote issue is fixed.`,
mergeRun,
),
); );
} }
console.log('\nSync complete.'); deps.consoleLog('\nSync complete.');
} finally { } finally {
fs.rmSync(localTmpDir, { recursive: true, force: true }); deps.rmSync(localTmpDir, { recursive: true, force: true });
if (remoteTmpDir) { if (remoteTmpDir) {
try { try {
cleanupRemote(host, remoteTmpDir); cleanupRemote(host, remoteTmpDir, deps);
} catch { } catch {
// best effort // best effort
} }
@@ -141,19 +216,23 @@ function runHostSync(context: LauncherCommandContext, dbPath: string): void {
} }
} }
export function runSyncCommand(context: LauncherCommandContext): boolean { export function runSyncCommand(
context: LauncherCommandContext,
inputDeps: Partial<SyncCommandDeps> = {},
): boolean {
const deps = resolveSyncCommandDeps(inputDeps);
const { args } = context; const { args } = context;
if (!args.sync) return false; if (!args.sync) return false;
const dbPath = resolveDbPath(context); const dbPath = resolveDbPath(context);
if (args.syncSnapshotPath) { if (args.syncSnapshotPath) {
runSnapshotMode(context, dbPath); runSnapshotMode(context, dbPath, deps);
} else if (args.syncMergePath) { } else if (args.syncMergePath) {
runMergeMode(context, dbPath); runMergeMode(context, dbPath, deps);
} else if (args.syncHost) { } else if (args.syncHost) {
runHostSync(context, dbPath); runHostSync(context, dbPath, deps);
} else { } else {
fail('sync requires a host, --snapshot <file>, or --merge <file>.'); deps.fail('sync requires a host, --snapshot <file>, or --merge <file>.');
} }
return true; return true;
} }
+34 -11
View File
@@ -160,7 +160,9 @@ export function mergeVideos(
): VideoMergeResult { ): VideoMergeResult {
const videoIdMap = new Map<number, number>(); const videoIdMap = new Map<number, number>();
const addedVideoIds = new Set<number>(); const addedVideoIds = new Set<number>();
const byKey = local.prepare<SqlRow>('SELECT video_id, watched FROM imm_videos WHERE video_key = ?'); const byKey = local.prepare<SqlRow>(
'SELECT video_id, watched FROM imm_videos WHERE video_key = ?',
);
const setWatched = local.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?'); const setWatched = local.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?');
for (const row of selectAll( for (const row of selectAll(
@@ -168,7 +170,8 @@ export function mergeVideos(
`SELECT video_id, anime_id, ${VIDEO_COPY_COLUMNS.join(', ')} FROM imm_videos`, `SELECT video_id, anime_id, ${VIDEO_COPY_COLUMNS.join(', ')} FROM imm_videos`,
)) { )) {
const remoteId = Number(row.video_id); const remoteId = Number(row.video_id);
const mappedAnimeId = row.anime_id === null ? null : (animeIdMap.get(Number(row.anime_id)) ?? null); const mappedAnimeId =
row.anime_id === null ? null : (animeIdMap.get(Number(row.anime_id)) ?? null);
const existing = byKey.get(row.video_key); const existing = byKey.get(row.video_key);
if (existing) { if (existing) {
const localId = Number(existing.video_id); const localId = Number(existing.video_id);
@@ -194,9 +197,11 @@ export function mergeMediaMetadata(
videoIdMap: Map<number, number>, videoIdMap: Map<number, number>,
addedVideoIds: Set<number>, addedVideoIds: Set<number>,
): void { ): void {
if (addedVideoIds.size === 0) return; if (videoIdMap.size === 0) return;
const metadataVideoIds = new Set<number>([...addedVideoIds, ...videoIdMap.keys()]);
const hasBlobStore = tableExists(local, 'imm_cover_art_blobs') && tableExists(remote, 'imm_cover_art_blobs'); const hasBlobStore =
tableExists(local, 'imm_cover_art_blobs') && tableExists(remote, 'imm_cover_art_blobs');
const copyBlob = hasBlobStore const copyBlob = hasBlobStore
? local.prepare( ? local.prepare(
`INSERT INTO imm_cover_art_blobs (blob_hash, cover_blob, CREATED_DATE, LAST_UPDATE_DATE) `INSERT INTO imm_cover_art_blobs (blob_hash, cover_blob, CREATED_DATE, LAST_UPDATE_DATE)
@@ -208,15 +213,19 @@ export function mergeMediaMetadata(
? remote.prepare<SqlRow>('SELECT * FROM imm_cover_art_blobs WHERE blob_hash = ?') ? remote.prepare<SqlRow>('SELECT * FROM imm_cover_art_blobs WHERE blob_hash = ?')
: null; : null;
if (tableExists(remote, 'imm_media_art')) { if (tableExists(remote, 'imm_media_art') && tableExists(local, 'imm_media_art')) {
for (const remoteVideoId of addedVideoIds) { const localArtExists = local.prepare<SqlRow>(
'SELECT 1 FROM imm_media_art WHERE video_id = ? LIMIT 1',
);
for (const remoteVideoId of metadataVideoIds) {
const localVideoId = videoIdMap.get(remoteVideoId)!;
if (localArtExists.get(localVideoId)) continue;
const row = remote const row = remote
.query<SqlRow>( .query<SqlRow>(
`SELECT ${MEDIA_ART_COPY_COLUMNS.join(', ')} FROM imm_media_art WHERE video_id = ?`, `SELECT ${MEDIA_ART_COPY_COLUMNS.join(', ')} FROM imm_media_art WHERE video_id = ?`,
) )
.get(remoteVideoId); .get(remoteVideoId);
if (!row) continue; if (!row) continue;
const localVideoId = videoIdMap.get(remoteVideoId)!;
if (row.cover_blob_hash && copyBlob && readBlob) { if (row.cover_blob_hash && copyBlob && readBlob) {
const blob = readBlob.get(row.cover_blob_hash); const blob = readBlob.get(row.cover_blob_hash);
if (blob) { if (blob) {
@@ -233,7 +242,12 @@ export function mergeMediaMetadata(
} }
if (tableExists(remote, 'imm_youtube_videos') && tableExists(local, 'imm_youtube_videos')) { if (tableExists(remote, 'imm_youtube_videos') && tableExists(local, 'imm_youtube_videos')) {
for (const remoteVideoId of addedVideoIds) { const localYoutubeExists = local.prepare<SqlRow>(
'SELECT 1 FROM imm_youtube_videos WHERE video_id = ? LIMIT 1',
);
for (const remoteVideoId of metadataVideoIds) {
const localVideoId = videoIdMap.get(remoteVideoId)!;
if (localYoutubeExists.get(localVideoId)) continue;
const row = remote const row = remote
.query<SqlRow>( .query<SqlRow>(
`SELECT ${YOUTUBE_COPY_COLUMNS.join(', ')} FROM imm_youtube_videos WHERE video_id = ?`, `SELECT ${YOUTUBE_COPY_COLUMNS.join(', ')} FROM imm_youtube_videos WHERE video_id = ?`,
@@ -244,7 +258,7 @@ export function mergeMediaMetadata(
local, local,
'imm_youtube_videos', 'imm_youtube_videos',
['video_id', ...YOUTUBE_COPY_COLUMNS], ['video_id', ...YOUTUBE_COPY_COLUMNS],
[videoIdMap.get(remoteVideoId)!, ...YOUTUBE_COPY_COLUMNS.map((column) => row[column])], [localVideoId, ...YOUTUBE_COPY_COLUMNS.map((column) => row[column])],
); );
} }
} }
@@ -255,7 +269,10 @@ export function mergeExcludedWords(
remote: Database, remote: Database,
summary: SyncMergeSummary, summary: SyncMergeSummary,
): void { ): void {
if (!tableExists(remote, 'imm_stats_excluded_words') || !tableExists(local, 'imm_stats_excluded_words')) { if (
!tableExists(remote, 'imm_stats_excluded_words') ||
!tableExists(local, 'imm_stats_excluded_words')
) {
return; return;
} }
const insert = local.prepare( const insert = local.prepare(
@@ -267,7 +284,13 @@ export function mergeExcludedWords(
remote, remote,
'SELECT headword, word, reading, CREATED_DATE, LAST_UPDATE_DATE FROM imm_stats_excluded_words', 'SELECT headword, word, reading, CREATED_DATE, LAST_UPDATE_DATE FROM imm_stats_excluded_words',
)) { )) {
const result = insert.run(row.headword, row.word, row.reading, row.CREATED_DATE, row.LAST_UPDATE_DATE); const result = insert.run(
row.headword,
row.word,
row.reading,
row.CREATED_DATE,
row.LAST_UPDATE_DATE,
);
summary.excludedWordsAdded += result.changes; summary.excludedWordsAdded += result.changes;
} }
} }
+13 -4
View File
@@ -149,7 +149,9 @@ export function refreshRollupsForNewSessions(
} }
const stampMs = nowDbTimestamp(); const stampMs = nowDbTimestamp();
const deleteDaily = local.prepare('DELETE FROM imm_daily_rollups WHERE rollup_day = ? AND video_id = ?'); const deleteDaily = local.prepare(
'DELETE FROM imm_daily_rollups WHERE rollup_day = ? AND video_id = ?',
);
const deleteMonthly = local.prepare( const deleteMonthly = local.prepare(
'DELETE FROM imm_monthly_rollups WHERE rollup_month = ? AND video_id = ?', 'DELETE FROM imm_monthly_rollups WHERE rollup_month = ? AND video_id = ?',
); );
@@ -192,6 +194,15 @@ export function copyRemoteOnlyRollups(
const localDaySessions = local.prepare( const localDaySessions = local.prepare(
`SELECT 1 FROM imm_sessions WHERE video_id = ? AND ${LOCAL_DAY_EXPR} = ? LIMIT 1`, `SELECT 1 FROM imm_sessions WHERE video_id = ? AND ${LOCAL_DAY_EXPR} = ? LIMIT 1`,
); );
const localMonthSessions = local.prepare(
`SELECT 1 FROM imm_sessions WHERE video_id = ? AND ${LOCAL_MONTH_EXPR} = ? LIMIT 1`,
);
const localMonthSessionsForDay = local.prepare(
`SELECT 1 FROM imm_sessions
WHERE video_id = ?
AND ${LOCAL_MONTH_EXPR} = CAST(strftime('%Y%m', CAST(? AS INTEGER) * 86400, 'unixepoch', 'localtime') AS INTEGER)
LIMIT 1`,
);
const insertDaily = local.prepare( const insertDaily = local.prepare(
`INSERT INTO imm_daily_rollups ( `INSERT INTO imm_daily_rollups (
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
@@ -205,6 +216,7 @@ export function copyRemoteOnlyRollups(
if (localVideoId === undefined) continue; if (localVideoId === undefined) continue;
if (localDailyExists.get(row.rollup_day, localVideoId)) continue; if (localDailyExists.get(row.rollup_day, localVideoId)) continue;
if (localDaySessions.get(localVideoId, row.rollup_day)) continue; if (localDaySessions.get(localVideoId, row.rollup_day)) continue;
if (localMonthSessionsForDay.get(localVideoId, row.rollup_day)) continue;
insertDaily.run( insertDaily.run(
row.rollup_day, row.rollup_day,
localVideoId, localVideoId,
@@ -225,9 +237,6 @@ export function copyRemoteOnlyRollups(
const localMonthlyExists = local.prepare( const localMonthlyExists = local.prepare(
'SELECT 1 FROM imm_monthly_rollups WHERE rollup_month = ? AND video_id = ? LIMIT 1', 'SELECT 1 FROM imm_monthly_rollups WHERE rollup_month = ? AND video_id = ? LIMIT 1',
); );
const localMonthSessions = local.prepare(
`SELECT 1 FROM imm_sessions WHERE video_id = ? AND ${LOCAL_MONTH_EXPR} = ? LIMIT 1`,
);
const insertMonthly = local.prepare( const insertMonthly = local.prepare(
`INSERT INTO imm_monthly_rollups ( `INSERT INTO imm_monthly_rollups (
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
+13 -1
View File
@@ -1,6 +1,6 @@
import test from 'node:test'; import test from 'node:test';
import assert from 'node:assert/strict'; import assert from 'node:assert/strict';
import { assertSafeSshHost, shellQuote } from './ssh.js'; import { assertSafeSshHost, runScp, shellQuote } from './ssh.js';
test('assertSafeSshHost rejects option-like hosts', () => { test('assertSafeSshHost rejects option-like hosts', () => {
assert.throws(() => assertSafeSshHost('-oProxyCommand=touch pwned'), /looks like an option/); assert.throws(() => assertSafeSshHost('-oProxyCommand=touch pwned'), /looks like an option/);
@@ -17,3 +17,15 @@ test('shellQuote escapes single quotes and wraps in quotes', () => {
assert.equal(shellQuote('subminer'), `'subminer'`); assert.equal(shellQuote('subminer'), `'subminer'`);
assert.equal(shellQuote(`a'; rm -rf ~; '`), `'a'\\''; rm -rf ~; '\\'''`); assert.equal(shellQuote(`a'; rm -rf ~; '`), `'a'\\''; rm -rf ~; '\\'''`);
}); });
test('runScp rejects option-like local endpoints before spawning scp', () => {
assert.throws(() => runScp('-oProxyCommand=sh', '/tmp/out.sqlite'), /looks like an option/);
assert.throws(() => runScp('/tmp/in.sqlite', '-bad-destination'), /looks like an option/);
});
test('runScp rejects option-like remote host components', () => {
assert.throws(
() => runScp('-oProxyCommand=sh:/tmp/in.sqlite', '/tmp/out.sqlite'),
/SSH host that looks like an option/,
);
});
+26 -5
View File
@@ -3,6 +3,7 @@ import { spawnSync } from 'node:child_process';
export interface RemoteRunResult { export interface RemoteRunResult {
status: number; status: number;
stdout: string; stdout: string;
stderr: string;
} }
/** /**
@@ -17,23 +18,43 @@ export function assertSafeSshHost(host: string): void {
} }
/** /**
* Run a command on the SSH host. stdin/stderr stay attached to the terminal * Run a command on the SSH host. stdin stays attached so interactive prompts
* so key passphrase / password prompts and remote progress output work; * can still read from the terminal; stdout/stderr are captured for callers
* stdout is captured for the caller. * that need actionable remote failure messages.
*/ */
export function runSsh(host: string, remoteCommand: string): RemoteRunResult { export function runSsh(host: string, remoteCommand: string): RemoteRunResult {
assertSafeSshHost(host); assertSafeSshHost(host);
const result = spawnSync('ssh', [host, remoteCommand], { const result = spawnSync('ssh', [host, remoteCommand], {
encoding: 'utf8', encoding: 'utf8',
stdio: ['inherit', 'pipe', 'inherit'], stdio: ['inherit', 'pipe', 'pipe'],
}); });
if (result.error) { if (result.error) {
throw new Error(`Failed to run ssh: ${(result.error as Error).message}`); throw new Error(`Failed to run ssh: ${(result.error as Error).message}`);
} }
return { status: result.status ?? 1, stdout: result.stdout ?? '' }; return { status: result.status ?? 1, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
}
function assertSafeScpEndpoint(endpoint: string): void {
const colon = endpoint.indexOf(':');
const slash = endpoint.indexOf('/');
if (colon <= 0 || (slash !== -1 && slash < colon)) {
if (endpoint.startsWith('-')) {
throw new Error(`Refusing to use scp endpoint that looks like an option: ${endpoint}`);
}
return;
}
const host = endpoint.slice(0, colon);
const remotePath = endpoint.slice(colon + 1);
assertSafeSshHost(host);
if (remotePath.startsWith('-')) {
throw new Error(`Refusing to use scp remote path that looks like an option: ${remotePath}`);
}
} }
export function runScp(from: string, to: string): void { export function runScp(from: string, to: string): void {
assertSafeScpEndpoint(from);
assertSafeScpEndpoint(to);
const result = spawnSync('scp', ['-q', from, to], { const result = spawnSync('scp', ['-q', from, to], {
encoding: 'utf8', encoding: 'utf8',
stdio: ['inherit', 'inherit', 'inherit'], stdio: ['inherit', 'inherit', 'inherit'],
+206 -32
View File
@@ -26,7 +26,11 @@ function makeDbPair(): { dir: string; localPath: string; remotePath: string } {
return { dir, localPath, remotePath }; return { dir, localPath, remotePath };
} }
function queryOne<T extends Record<string, unknown>>(dbPath: string, sql: string, params: unknown[] = []): T | undefined { function queryOne<T extends Record<string, unknown>>(
dbPath: string,
sql: string,
params: unknown[] = [],
): T | undefined {
const db = new Database(dbPath, { readonly: true }); const db = new Database(dbPath, { readonly: true });
try { try {
return db.query<T>(sql).get(...params) as T | undefined; return db.query<T>(sql).get(...params) as T | undefined;
@@ -39,6 +43,15 @@ function count(dbPath: string, sql: string, params: unknown[] = []): number {
return Number(queryOne<{ n: number }>(dbPath, sql, params)?.n ?? 0); return Number(queryOne<{ n: number }>(dbPath, sql, params)?.n ?? 0);
} }
function withWritableDb<T>(dbPath: string, fn: (db: Database) => T): T {
const db = new Database(dbPath, { readwrite: true });
try {
return fn(db);
} finally {
db.close();
}
}
test('merges remote-only sessions with catalog, lifetime, and rollups', () => { test('merges remote-only sessions with catalog, lifetime, and rollups', () => {
const { dir, localPath, remotePath } = makeDbPair(); const { dir, localPath, remotePath } = makeDbPair();
try { try {
@@ -73,7 +86,13 @@ test('merges remote-only sessions with catalog, lifetime, and rollups', () => {
assert.equal(summary.telemetryRowsAdded, 1); assert.equal(summary.telemetryRowsAdded, 1);
assert.equal(count(localPath, 'SELECT COUNT(*) AS n FROM imm_sessions'), 2); assert.equal(count(localPath, 'SELECT COUNT(*) AS n FROM imm_sessions'), 2);
const global = queryOne<{ total_sessions: number; total_active_ms: number; total_cards: number; active_days: number; episodes_started: number }>( const global = queryOne<{
total_sessions: number;
total_active_ms: number;
total_cards: number;
active_days: number;
episodes_started: number;
}>(
localPath, localPath,
'SELECT total_sessions, total_active_ms, total_cards, active_days, episodes_started FROM imm_lifetime_global WHERE global_id = 1', 'SELECT total_sessions, total_active_ms, total_cards, active_days, episodes_started FROM imm_lifetime_global WHERE global_id = 1',
); );
@@ -98,11 +117,15 @@ test('merges remote-only sessions with catalog, lifetime, and rollups', () => {
// The merged session's rollup group was recomputed. // The merged session's rollup group was recomputed.
assert.equal(summary.rollupGroupsRecomputed, 1); assert.equal(summary.rollupGroupsRecomputed, 1);
const mergedVideoId = Number( const mergedVideoId = Number(
queryOne<{ video_id: number }>(localPath, `SELECT video_id FROM imm_videos WHERE video_key = 'showb-e1'`) queryOne<{ video_id: number }>(
?.video_id, localPath,
`SELECT video_id FROM imm_videos WHERE video_key = 'showb-e1'`,
)?.video_id,
); );
assert.equal( assert.equal(
count(localPath, 'SELECT COUNT(*) AS n FROM imm_daily_rollups WHERE video_id = ?', [mergedVideoId]), count(localPath, 'SELECT COUNT(*) AS n FROM imm_daily_rollups WHERE video_id = ?', [
mergedVideoId,
]),
1, 1,
); );
} finally { } finally {
@@ -143,7 +166,12 @@ test('is idempotent: re-merging the same snapshot changes nothing', () => {
{ ...globalAfterFirst, LAST_UPDATE_DATE: null }, { ...globalAfterFirst, LAST_UPDATE_DATE: null },
); );
assert.equal( assert.equal(
Number(queryOne<{ frequency: number }>(localPath, `SELECT frequency FROM imm_words WHERE word = '見た'`)?.frequency), Number(
queryOne<{ frequency: number }>(
localPath,
`SELECT frequency FROM imm_words WHERE word = '見た'`,
)?.frequency,
),
4, 4,
); );
} finally { } finally {
@@ -161,9 +189,11 @@ test('preserves anilist_id when inserting a new anime from the snapshot', () =>
startedAtMs: BASE_MS, startedAtMs: BASE_MS,
applyLifetime: true, applyLifetime: true,
}); });
const remoteDb = new Database(remotePath, { readwrite: true }); withWritableDb(remotePath, (remoteDb) => {
remoteDb.prepare(`UPDATE imm_anime SET anilist_id = 12345 WHERE normalized_title_key = 'showb'`).run(); remoteDb
remoteDb.close(); .prepare(`UPDATE imm_anime SET anilist_id = 12345 WHERE normalized_title_key = 'showb'`)
.run();
});
const summary = mergeSnapshotIntoDb(localPath, remotePath); const summary = mergeSnapshotIntoDb(localPath, remotePath);
assert.equal(summary.animeAdded, 1); assert.equal(summary.animeAdded, 1);
@@ -194,12 +224,16 @@ test('matches an existing local anime by anilist_id even when the title key diff
startedAtMs: BASE_MS + DAY_MS, startedAtMs: BASE_MS + DAY_MS,
applyLifetime: true, applyLifetime: true,
}); });
const localDb = new Database(localPath, { readwrite: true }); withWritableDb(localPath, (localDb) => {
localDb.prepare(`UPDATE imm_anime SET anilist_id = 999 WHERE normalized_title_key = 'show-romaji'`).run(); localDb
localDb.close(); .prepare(`UPDATE imm_anime SET anilist_id = 999 WHERE normalized_title_key = 'show-romaji'`)
const remoteDb = new Database(remotePath, { readwrite: true }); .run();
remoteDb.prepare(`UPDATE imm_anime SET anilist_id = 999 WHERE normalized_title_key = 'show-native'`).run(); });
remoteDb.close(); withWritableDb(remotePath, (remoteDb) => {
remoteDb
.prepare(`UPDATE imm_anime SET anilist_id = 999 WHERE normalized_title_key = 'show-native'`)
.run();
});
const summary = mergeSnapshotIntoDb(localPath, remotePath); const summary = mergeSnapshotIntoDb(localPath, remotePath);
// Same anilist_id → one anime, not two. // Same anilist_id → one anime, not two.
@@ -241,7 +275,11 @@ test('matches shared videos by video_key and merges the watched flag', () => {
// Both sessions now credit the same video; episode was started once and // Both sessions now credit the same video; episode was started once and
// completed once (by the remote session that watched it to the end). // completed once (by the remote session that watched it to the end).
const global = queryOne<{ episodes_started: number; episodes_completed: number; total_sessions: number }>( const global = queryOne<{
episodes_started: number;
episodes_completed: number;
total_sessions: number;
}>(
localPath, localPath,
'SELECT episodes_started, episodes_completed, total_sessions FROM imm_lifetime_global WHERE global_id = 1', 'SELECT episodes_started, episodes_completed, total_sessions FROM imm_lifetime_global WHERE global_id = 1',
); );
@@ -253,6 +291,75 @@ test('matches shared videos by video_key and merges the watched flag', () => {
} }
}); });
test('backfills media metadata for matched videos when local metadata is absent', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
insertFixtureSession(localPath, {
uuid: 'local-1',
videoKey: 'showa-e1',
animeTitleKey: 'showa',
startedAtMs: BASE_MS,
applyLifetime: true,
});
insertFixtureSession(remotePath, {
uuid: 'remote-1',
videoKey: 'showa-e1',
animeTitleKey: 'showa',
startedAtMs: BASE_MS + DAY_MS,
applyLifetime: true,
});
const remoteDb = new Database(remotePath, { readwrite: true });
try {
const remoteVideoId = Number(
remoteDb
.query<{
video_id: number;
}>(`SELECT video_id FROM imm_videos WHERE video_key = 'showa-e1'`)
.get()?.video_id,
);
remoteDb
.prepare(
`INSERT INTO imm_media_art (video_id, anilist_id, cover_url, title_romaji, fetched_at_ms)
VALUES (?, 123, 'https://example.test/cover.jpg', 'Show A', ?)`,
)
.run(remoteVideoId, String(BASE_MS));
remoteDb
.prepare(
`INSERT INTO imm_youtube_videos (video_id, youtube_video_id, video_url, video_title, fetched_at_ms)
VALUES (?, 'yt-1', 'https://youtube.test/watch?v=yt-1', 'Remote Video', ?)`,
)
.run(remoteVideoId, String(BASE_MS));
} finally {
remoteDb.close();
}
const summary = mergeSnapshotIntoDb(localPath, remotePath);
assert.equal(summary.videosAdded, 0);
const localVideoId = Number(
queryOne<{ video_id: number }>(
localPath,
`SELECT video_id FROM imm_videos WHERE video_key = 'showa-e1'`,
)?.video_id,
);
const art = queryOne<{ cover_url: string; title_romaji: string }>(
localPath,
'SELECT cover_url, title_romaji FROM imm_media_art WHERE video_id = ?',
[localVideoId],
);
assert.equal(art?.cover_url, 'https://example.test/cover.jpg');
assert.equal(art?.title_romaji, 'Show A');
const youtube = queryOne<{ youtube_video_id: string; video_title: string }>(
localPath,
'SELECT youtube_video_id, video_title FROM imm_youtube_videos WHERE video_id = ?',
[localVideoId],
);
assert.equal(youtube?.youtube_video_id, 'yt-1');
assert.equal(youtube?.video_title, 'Remote Video');
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('does not double-count active_days when a merged session starts earlier on an already-credited day', () => { test('does not double-count active_days when a merged session starts earlier on an already-credited day', () => {
const { dir, localPath, remotePath } = makeDbPair(); const { dir, localPath, remotePath } = makeDbPair();
try { try {
@@ -310,10 +417,13 @@ test('copies remote-only historical rollups but never sums shared groups', () =>
startedAtMs: BASE_MS, startedAtMs: BASE_MS,
applyLifetime: true, applyLifetime: true,
}); });
const remoteDb = new Database(remotePath, { readwrite: true }); withWritableDb(remotePath, (remoteDb) => {
const remoteVideoId = Number( const remoteVideoId = Number(
remoteDb.query<{ video_id: number }>(`SELECT video_id FROM imm_videos WHERE video_key = 'old-show-e1'`).get() remoteDb
?.video_id, .query<{
video_id: number;
}>(`SELECT video_id FROM imm_videos WHERE video_key = 'old-show-e1'`)
.get()?.video_id,
); );
remoteDb remoteDb
.prepare( .prepare(
@@ -321,7 +431,7 @@ test('copies remote-only historical rollups but never sums shared groups', () =>
VALUES (10000, ?, 4, 120.5, 400, 3200, 9)`, VALUES (10000, ?, 4, 120.5, 400, 3200, 9)`,
) )
.run(remoteVideoId); .run(remoteVideoId);
remoteDb.close(); });
// Local already has its own rollup row for a shared group. // Local already has its own rollup row for a shared group.
insertFixtureSession(localPath, { insertFixtureSession(localPath, {
@@ -330,18 +440,22 @@ test('copies remote-only historical rollups but never sums shared groups', () =>
startedAtMs: BASE_MS, startedAtMs: BASE_MS,
applyLifetime: true, applyLifetime: true,
}); });
const localDb = new Database(localPath, { readwrite: true }); const localVideoId = withWritableDb(localPath, (localDb) => {
const localVideoId = Number( const videoId = Number(
localDb.query<{ video_id: number }>(`SELECT video_id FROM imm_videos WHERE video_key = 'old-show-e1'`).get() localDb
?.video_id, .query<{
video_id: number;
}>(`SELECT video_id FROM imm_videos WHERE video_key = 'old-show-e1'`)
.get()?.video_id,
); );
localDb localDb
.prepare( .prepare(
`INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards) `INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards)
VALUES (10000, ?, 2, 60.0, 200, 1600, 4)`, VALUES (10000, ?, 2, 60.0, 200, 1600, 4)`,
) )
.run(localVideoId); .run(videoId);
localDb.close(); return videoId;
});
const summary = mergeSnapshotIntoDb(localPath, remotePath); const summary = mergeSnapshotIntoDb(localPath, remotePath);
// Shared group (day 10000) untouched; the remote-only session's own group // Shared group (day 10000) untouched; the remote-only session's own group
@@ -355,7 +469,7 @@ test('copies remote-only historical rollups but never sums shared groups', () =>
assert.equal(shared?.total_sessions, 2); assert.equal(shared?.total_sessions, 2);
// Now a rollup for a video with no local sessions at all gets copied. // Now a rollup for a video with no local sessions at all gets copied.
const remoteDb2 = new Database(remotePath, { readwrite: true }); withWritableDb(remotePath, (remoteDb2) => {
const orphanVideoId = Number( const orphanVideoId = Number(
remoteDb2 remoteDb2
.prepare( .prepare(
@@ -370,13 +484,15 @@ test('copies remote-only historical rollups but never sums shared groups', () =>
VALUES (9000, ?, 3, 90.0, 300, 2400, 6)`, VALUES (9000, ?, 3, 90.0, 300, 2400, 6)`,
) )
.run(orphanVideoId); .run(orphanVideoId);
remoteDb2.close(); });
const summary2 = mergeSnapshotIntoDb(localPath, remotePath); const summary2 = mergeSnapshotIntoDb(localPath, remotePath);
assert.equal(summary2.dailyRollupsCopied, 1); assert.equal(summary2.dailyRollupsCopied, 1);
const copiedVideoId = Number( const copiedVideoId = Number(
queryOne<{ video_id: number }>(localPath, `SELECT video_id FROM imm_videos WHERE video_key = 'pruned-show-e1'`) queryOne<{ video_id: number }>(
?.video_id, localPath,
`SELECT video_id FROM imm_videos WHERE video_key = 'pruned-show-e1'`,
)?.video_id,
); );
const copied = queryOne<{ total_sessions: number; total_active_min: number }>( const copied = queryOne<{ total_sessions: number; total_active_min: number }>(
localPath, localPath,
@@ -390,12 +506,70 @@ test('copies remote-only historical rollups but never sums shared groups', () =>
} }
}); });
test('does not copy remote-only daily rollups into months with local sessions', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
insertFixtureSession(localPath, {
uuid: 'local-1',
videoKey: 'mixed-month-e1',
animeTitleKey: 'mixed-month',
startedAtMs: BASE_MS,
applyLifetime: true,
});
const remoteDb = new Database(remotePath, { readwrite: true });
try {
const remoteVideoId = Number(
remoteDb
.prepare(
`INSERT INTO imm_videos (video_key, canonical_title, source_type, watched, duration_ms)
VALUES ('mixed-month-e1', 'mixed-month-e1', 1, 1, 1440000)`,
)
.run().lastInsertRowid,
);
remoteDb
.prepare(
`INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards)
VALUES (20615, ?, 3, 90.0, 300, 2400, 6)`,
)
.run(remoteVideoId);
remoteDb
.prepare(
`INSERT INTO imm_monthly_rollups (rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards)
VALUES (202606, ?, 3, 90.0, 300, 2400, 6)`,
)
.run(remoteVideoId);
} finally {
remoteDb.close();
}
const summary = mergeSnapshotIntoDb(localPath, remotePath);
assert.equal(summary.dailyRollupsCopied, 0);
assert.equal(summary.monthlyRollupsCopied, 0);
const localVideoId = Number(
queryOne<{ video_id: number }>(
localPath,
`SELECT video_id FROM imm_videos WHERE video_key = 'mixed-month-e1'`,
)?.video_id,
);
assert.equal(
count(
localPath,
'SELECT COUNT(*) AS n FROM imm_daily_rollups WHERE rollup_day = 20615 AND video_id = ?',
[localVideoId],
),
0,
);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('rejects snapshots at a different schema version', () => { test('rejects snapshots at a different schema version', () => {
const { dir, localPath, remotePath } = makeDbPair(); const { dir, localPath, remotePath } = makeDbPair();
try { try {
const remoteDb = new Database(remotePath, { readwrite: true }); withWritableDb(remotePath, (remoteDb) => {
remoteDb.prepare('UPDATE imm_schema_version SET schema_version = 17').run(); remoteDb.prepare('UPDATE imm_schema_version SET schema_version = 17').run();
remoteDb.close(); });
assert.throws(() => mergeSnapshotIntoDb(localPath, remotePath), /schema version 17/); assert.throws(() => mergeSnapshotIntoDb(localPath, remotePath), /schema version 17/);
} finally { } finally {
+25 -7
View File
@@ -5,6 +5,7 @@ import { IMMERSION_DB_FIXTURE_DDL } from './immersion-db-schema.js';
export function createImmersionDbFixture(dbPath: string): void { export function createImmersionDbFixture(dbPath: string): void {
const db = new Database(dbPath, { create: true }); const db = new Database(dbPath, { create: true });
try { try {
db.run('PRAGMA foreign_keys = ON');
db.run('PRAGMA journal_mode = WAL'); db.run('PRAGMA journal_mode = WAL');
for (const statement of IMMERSION_DB_FIXTURE_DDL.split(';')) { for (const statement of IMMERSION_DB_FIXTURE_DDL.split(';')) {
const sql = statement.trim(); const sql = statement.trim();
@@ -14,7 +15,9 @@ export function createImmersionDbFixture(dbPath: string): void {
SCHEMA_VERSION, SCHEMA_VERSION,
String(Date.now()), String(Date.now()),
); );
db.prepare(`INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('last_rollup_sample_ms', 0)`).run(); db.prepare(
`INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('last_rollup_sample_ms', 0)`,
).run();
db.prepare( db.prepare(
`INSERT INTO imm_lifetime_global(global_id, CREATED_DATE, LAST_UPDATE_DATE) VALUES (1, ?, ?)`, `INSERT INTO imm_lifetime_global(global_id, CREATED_DATE, LAST_UPDATE_DATE) VALUES (1, ?, ?)`,
).run(String(Date.now()), String(Date.now())); ).run(String(Date.now()), String(Date.now()));
@@ -53,7 +56,9 @@ export function insertFixtureSession(dbPath: string, input: FixtureSessionInput)
let animeId: number | null = null; let animeId: number | null = null;
if (input.animeTitleKey) { if (input.animeTitleKey) {
const existing = db const existing = db
.query<{ anime_id: number }>('SELECT anime_id FROM imm_anime WHERE normalized_title_key = ?') .query<{
anime_id: number;
}>('SELECT anime_id FROM imm_anime WHERE normalized_title_key = ?')
.get(input.animeTitleKey); .get(input.animeTitleKey);
animeId = existing animeId = existing
? existing.anime_id ? existing.anime_id
@@ -98,7 +103,8 @@ export function insertFixtureSession(dbPath: string, input: FixtureSessionInput)
db.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?').run(videoId); db.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?').run(videoId);
} }
const endedAtMs = input.endedAtMs === undefined ? input.startedAtMs + 1_500_000 : input.endedAtMs; const endedAtMs =
input.endedAtMs === undefined ? input.startedAtMs + 1_500_000 : input.endedAtMs;
const activeWatchedMs = input.activeWatchedMs ?? 1_200_000; const activeWatchedMs = input.activeWatchedMs ?? 1_200_000;
const cardsMined = input.cardsMined ?? 2; const cardsMined = input.cardsMined ?? 2;
const linesSeen = input.linesSeen ?? 100; const linesSeen = input.linesSeen ?? 100;
@@ -147,7 +153,9 @@ export function insertFixtureSession(dbPath: string, input: FixtureSessionInput)
for (const word of input.words ?? []) { for (const word of input.words ?? []) {
const existing = db const existing = db
.query<{ id: number }>('SELECT id FROM imm_words WHERE headword = ? AND word = ? AND reading = ?') .query<{
id: number;
}>('SELECT id FROM imm_words WHERE headword = ? AND word = ? AND reading = ?')
.get(word.headword, word.word, word.reading); .get(word.headword, word.word, word.reading);
const wordId = existing const wordId = existing
? existing.id ? existing.id
@@ -172,13 +180,23 @@ export function insertFixtureSession(dbPath: string, input: FixtureSessionInput)
`INSERT INTO imm_subtitle_lines (session_id, video_id, anime_id, line_index, text, CREATED_DATE, LAST_UPDATE_DATE) `INSERT INTO imm_subtitle_lines (session_id, video_id, anime_id, line_index, text, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (?, ?, ?, ?, ?, ?, ?)`, VALUES (?, ?, ?, ?, ?, ?, ?)`,
) )
.run(sessionId, videoId, animeId, uniqueCounter, `line ${word.word}`, input.startedAtMs, input.startedAtMs) .run(
.lastInsertRowid, sessionId,
videoId,
animeId,
uniqueCounter,
`line ${word.word}`,
input.startedAtMs,
input.startedAtMs,
).lastInsertRowid,
); );
db.prepare( db.prepare(
'INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count) VALUES (?, ?, ?)', 'INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count) VALUES (?, ?, ?)',
).run(lineId, wordId, word.count); ).run(lineId, wordId, word.count);
db.prepare('UPDATE imm_words SET frequency = frequency + ? WHERE id = ?').run(word.count, wordId); db.prepare('UPDATE imm_words SET frequency = frequency + ? WHERE id = ?').run(
word.count,
wordId,
);
} }
if (input.applyLifetime && endedAtMs !== null) { if (input.applyLifetime && endedAtMs !== null) {
@@ -0,0 +1,109 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { Database as BunDatabase } from 'bun:sqlite';
import { ensureSchema } from '../../src/core/services/immersion-tracker/storage.js';
import { createImmersionDbFixture } from './immersion-db-fixture.js';
type SchemaRow = { type: string; name: string; tbl_name: string; sql: string | null };
const SYNC_SCHEMA_OBJECTS = [
'imm_anime',
'imm_videos',
'imm_sessions',
'imm_session_telemetry',
'imm_session_events',
'imm_daily_rollups',
'imm_monthly_rollups',
'imm_words',
'imm_kanji',
'imm_subtitle_lines',
'imm_word_line_occurrences',
'imm_kanji_line_occurrences',
'imm_media_art',
'imm_youtube_videos',
'imm_cover_art_blobs',
'imm_lifetime_global',
'imm_lifetime_anime',
'imm_lifetime_media',
'imm_lifetime_applied_sessions',
'imm_stats_excluded_words',
'idx_anime_normalized_title',
'idx_anime_anilist_id',
'idx_videos_anime_id',
'idx_sessions_video_started',
'idx_sessions_status_started',
'idx_sessions_started_at',
'idx_sessions_ended_at',
'idx_telemetry_session_sample',
'idx_telemetry_sample_ms',
'idx_events_session_ts',
'idx_events_type_ts',
'idx_rollups_day_video',
'idx_rollups_month_video',
'idx_words_headword_word_reading',
'idx_words_frequency',
'idx_kanji_kanji',
'idx_kanji_frequency',
'idx_subtitle_lines_session_line',
'idx_subtitle_lines_video_line',
'idx_subtitle_lines_anime_line',
'idx_word_line_occurrences_word',
'idx_kanji_line_occurrences_kanji',
'idx_media_art_cover_blob_hash',
'idx_media_art_anilist_id',
'idx_media_art_cover_url',
'idx_youtube_videos_channel_id',
'idx_youtube_videos_youtube_video_id',
] as const;
function normalizeSql(sql: string | null): string {
return (sql ?? '')
.replace(/\bIF NOT EXISTS\b/gi, '')
.replace(/\s+/g, ' ')
.replace(/\s+([(),])/g, '$1')
.replace(/,\s+/g, ', ')
.trim();
}
function readSchema(dbPath: string): Map<string, string> {
const db = new BunDatabase(dbPath, { readonly: true });
try {
const rows = db
.query<SchemaRow>(
`SELECT type, name, tbl_name, sql
FROM sqlite_schema
WHERE name IN (${SYNC_SCHEMA_OBJECTS.map(() => '?').join(',')})
ORDER BY type, name`,
)
.all(...SYNC_SCHEMA_OBJECTS);
return new Map(rows.map((row) => [row.name, normalizeSql(row.sql)]));
} finally {
db.close();
}
}
test('fixture schema stays aligned with production sync-touched tables and indexes', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-fixture-schema-'));
const fixturePath = path.join(dir, 'fixture.sqlite');
const productionPath = path.join(dir, 'production.sqlite');
const productionDb = new BunDatabase(productionPath, { create: true });
try {
createImmersionDbFixture(fixturePath);
ensureSchema(productionDb as never);
productionDb.close();
const fixtureSchema = readSchema(fixturePath);
const productionSchema = readSchema(productionPath);
assert.deepEqual(fixtureSchema, productionSchema);
} finally {
try {
productionDb.close();
} catch {
// already closed
}
fs.rmSync(dir, { recursive: true, force: true });
}
});