mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-28 04:49:49 -07:00
Sync Stats & History window, headless --sync-cli, and Windows remote support (#160)
This commit is contained in:
GitHub
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
launchAppCommandDetached,
|
||||
launchAppBackgroundDetached,
|
||||
launchTexthookerOnly,
|
||||
runAppCommandWithInherit,
|
||||
@@ -7,6 +8,10 @@ import type { LauncherCommandContext } from './context.js';
|
||||
|
||||
type AppCommandDeps = {
|
||||
runAppCommandWithInherit: (appPath: string, appArgs: string[]) => void;
|
||||
launchSyncUiDetached: (
|
||||
appPath: string,
|
||||
logLevel: LauncherCommandContext['args']['logLevel'],
|
||||
) => void;
|
||||
launchAppBackgroundDetached: (
|
||||
appPath: string,
|
||||
logLevel: LauncherCommandContext['args']['logLevel'],
|
||||
@@ -15,6 +20,8 @@ type AppCommandDeps = {
|
||||
|
||||
const defaultAppCommandDeps: AppCommandDeps = {
|
||||
runAppCommandWithInherit,
|
||||
launchSyncUiDetached: (appPath, logLevel) =>
|
||||
launchAppCommandDetached(appPath, ['--sync-window'], logLevel, 'sync-ui'),
|
||||
launchAppBackgroundDetached,
|
||||
};
|
||||
|
||||
@@ -30,6 +37,10 @@ export function runAppPassthroughCommand(
|
||||
deps.runAppCommandWithInherit(appPath, ['--settings']);
|
||||
return true;
|
||||
}
|
||||
if (args.syncUi) {
|
||||
deps.launchSyncUiDetached(appPath, args.logLevel);
|
||||
return true;
|
||||
}
|
||||
if (!args.appPassthrough) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -206,6 +206,7 @@ test('app command starts default macOS background app detached from launcher', (
|
||||
runAppCommandWithInherit: () => {
|
||||
calls.push('attached');
|
||||
},
|
||||
launchSyncUiDetached: () => calls.push('sync-ui'),
|
||||
launchAppBackgroundDetached: (appPath, logLevel) => {
|
||||
calls.push(`detached:${appPath}:${logLevel}`);
|
||||
},
|
||||
@@ -225,6 +226,7 @@ test('app command starts default Linux background app detached from launcher', (
|
||||
runAppCommandWithInherit: () => {
|
||||
calls.push('attached');
|
||||
},
|
||||
launchSyncUiDetached: () => calls.push('sync-ui'),
|
||||
launchAppBackgroundDetached: (appPath, logLevel) => {
|
||||
calls.push(`detached:${appPath}:${logLevel}`);
|
||||
},
|
||||
@@ -245,6 +247,7 @@ test('app command keeps explicit passthrough args attached', () => {
|
||||
runAppCommandWithInherit: (_appPath, appArgs) => {
|
||||
forwarded.push(appArgs);
|
||||
},
|
||||
launchSyncUiDetached: () => detached.push('sync-ui'),
|
||||
launchAppBackgroundDetached: () => {
|
||||
detached.push('detached');
|
||||
},
|
||||
@@ -255,6 +258,21 @@ test('app command keeps explicit passthrough args attached', () => {
|
||||
assert.deepEqual(detached, []);
|
||||
});
|
||||
|
||||
test('sync UI command launches the app detached from the terminal', () => {
|
||||
const context = createContext();
|
||||
context.args.syncUi = true;
|
||||
const calls: string[] = [];
|
||||
|
||||
const handled = runAppPassthroughCommand(context, {
|
||||
runAppCommandWithInherit: () => calls.push('piped'),
|
||||
launchSyncUiDetached: (appPath, logLevel) => calls.push(`sync-ui:${appPath}:${logLevel}`),
|
||||
launchAppBackgroundDetached: () => calls.push('detached'),
|
||||
});
|
||||
|
||||
assert.equal(handled, true);
|
||||
assert.deepEqual(calls, ['sync-ui:/tmp/subminer.app:warn']);
|
||||
});
|
||||
|
||||
test('mpv pre-app command exits non-zero when socket is not ready', async () => {
|
||||
const context = createContext();
|
||||
context.args.mpvStatus = true;
|
||||
|
||||
@@ -37,13 +37,8 @@ function createContext(): LauncherCommandContext {
|
||||
useRofi: false,
|
||||
history: false,
|
||||
sync: false,
|
||||
syncHost: '',
|
||||
syncSnapshotPath: '',
|
||||
syncMergePath: '',
|
||||
syncDirection: 'both',
|
||||
syncRemoteCmd: '',
|
||||
syncDbPath: '',
|
||||
syncForce: false,
|
||||
syncCliTokens: [],
|
||||
syncUi: false,
|
||||
logLevel: 'info',
|
||||
logRotation: 7,
|
||||
passwordStore: '',
|
||||
|
||||
@@ -1,24 +1,17 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import type { Args } from '../types.js';
|
||||
import { createEmptyMergeSummary } from '../sync/sync-shared.js';
|
||||
import type { LauncherCommandContext } from './context.js';
|
||||
import { ensureTrackerQuiescent, runSyncCommand, type SyncCommandDeps } from './sync-command.js';
|
||||
import { runSyncCommand, type SyncCommandDeps } from './sync-command.js';
|
||||
|
||||
function makeContext(overrides: Partial<Args>): LauncherCommandContext {
|
||||
function makeContext(
|
||||
overrides: Partial<Args>,
|
||||
appPath: string | null = '/opt/SubMiner/subminer-app',
|
||||
): LauncherCommandContext {
|
||||
return {
|
||||
args: {
|
||||
sync: true,
|
||||
syncHost: '',
|
||||
syncSnapshotPath: '',
|
||||
syncMergePath: '',
|
||||
syncDirection: 'both',
|
||||
syncRemoteCmd: '',
|
||||
syncDbPath: '',
|
||||
syncForce: false,
|
||||
syncCliTokens: [],
|
||||
logLevel: 'warn',
|
||||
...overrides,
|
||||
} as Args,
|
||||
@@ -26,232 +19,90 @@ function makeContext(overrides: Partial<Args>): LauncherCommandContext {
|
||||
scriptName: 'subminer',
|
||||
mpvSocketPath: '',
|
||||
pluginRuntimeConfig: {},
|
||||
appPath: null,
|
||||
appPath,
|
||||
launcherJellyfinConfig: {},
|
||||
processAdapter: process,
|
||||
processAdapter: { platform: () => 'linux' },
|
||||
} as unknown as LauncherCommandContext;
|
||||
}
|
||||
|
||||
function ok(stdout = ''): { status: number; stdout: string; stderr: string } {
|
||||
return { status: 0, stdout, stderr: '' };
|
||||
}
|
||||
|
||||
test('ensureTrackerQuiescent ignores stale sockets but rejects live sockets', async () => {
|
||||
const context = makeContext({ syncDbPath: '/tmp/local.sqlite' });
|
||||
context.mpvSocketPath = '/tmp/subminer-socket';
|
||||
let socketConnectable = false;
|
||||
test('runSyncCommand proxies sync argv to the app in --sync-cli mode', async () => {
|
||||
const spawned: Array<{ appPath: string; appArgs: string[] }> = [];
|
||||
const deps: Partial<SyncCommandDeps> = {
|
||||
realpathSync: (() => '/tracker.sqlite') as unknown as typeof fs.realpathSync,
|
||||
findLiveStatsDaemonPid: () => null,
|
||||
canConnectUnixSocket: async () => socketConnectable,
|
||||
fail: (message: string): never => {
|
||||
throw new Error(message);
|
||||
},
|
||||
};
|
||||
|
||||
await ensureTrackerQuiescent(context, '/tmp/local.sqlite', deps);
|
||||
|
||||
socketConnectable = true;
|
||||
await assert.rejects(
|
||||
async () => ensureTrackerQuiescent(context, '/tmp/local.sqlite', deps),
|
||||
/mpv\/SubMiner session appears to be running/,
|
||||
);
|
||||
});
|
||||
|
||||
test('runSyncCommand dispatches snapshot, merge, host, and missing-target modes', async () => {
|
||||
const calls: string[] = [];
|
||||
const deps: Partial<SyncCommandDeps> = {
|
||||
createDbSnapshot: (dbPath: string, outPath: string) => {
|
||||
calls.push(`snapshot:${dbPath}->${outPath}`);
|
||||
},
|
||||
mergeSnapshotIntoDb: (dbPath: string, snapshotPath: string) => {
|
||||
calls.push(`merge:${dbPath}<-${snapshotPath}`);
|
||||
return createEmptyMergeSummary();
|
||||
},
|
||||
formatMergeSummary: () => 'summary',
|
||||
ensureTrackerQuiescent: async () => {
|
||||
calls.push('quiescent');
|
||||
},
|
||||
assertSafeSshHost: (host: string) => {
|
||||
calls.push(`host:${host}`);
|
||||
},
|
||||
resolveRemoteSubminerCommand: () => 'subminer',
|
||||
runSsh: (_host: string, command: string) => {
|
||||
calls.push(`ssh:${command}`);
|
||||
return command.startsWith('mktemp ') ? ok('/tmp/subminer-sync.remote\n') : ok();
|
||||
},
|
||||
runScp: (from: string, to: string) => {
|
||||
calls.push(`scp:${from}->${to}`);
|
||||
},
|
||||
fail: (message: string): never => {
|
||||
throw new Error(message);
|
||||
runAppCommand: (appPath, appArgs) => {
|
||||
spawned.push({ appPath, appArgs });
|
||||
},
|
||||
};
|
||||
|
||||
assert.equal(
|
||||
await runSyncCommand(
|
||||
makeContext({ syncDbPath: '/tmp/local.sqlite', syncSnapshotPath: '/tmp/out.sqlite' }),
|
||||
deps,
|
||||
),
|
||||
await runSyncCommand(makeContext({ syncCliTokens: ['media-box', '--json'] }), deps),
|
||||
true,
|
||||
);
|
||||
assert.ok(calls.includes('snapshot:/tmp/local.sqlite->/tmp/out.sqlite'));
|
||||
assert.deepEqual(spawned, [
|
||||
{
|
||||
appPath: '/opt/SubMiner/subminer-app',
|
||||
appArgs: ['--sync-cli', 'sync', 'media-box', '--json', '--log-level', 'warn'],
|
||||
},
|
||||
]);
|
||||
|
||||
await runSyncCommand(
|
||||
makeContext({ syncDbPath: '/tmp/local.sqlite', syncMergePath: '/tmp/in.sqlite' }),
|
||||
deps,
|
||||
);
|
||||
assert.ok(calls.includes('quiescent'));
|
||||
assert.ok(calls.includes('merge:/tmp/local.sqlite<-/tmp/in.sqlite'));
|
||||
|
||||
await runSyncCommand(
|
||||
makeContext({ syncDbPath: '/tmp/local.sqlite', syncHost: 'media-box' }),
|
||||
deps,
|
||||
);
|
||||
assert.ok(calls.includes('host:media-box'));
|
||||
|
||||
await assert.rejects(
|
||||
() => runSyncCommand(makeContext({ syncDbPath: '/tmp/local.sqlite' }), deps),
|
||||
/sync requires a host, --snapshot <file>, or --merge <file>/,
|
||||
);
|
||||
assert.equal(await runSyncCommand(makeContext({ sync: false }), deps), false);
|
||||
assert.equal(spawned.length, 1);
|
||||
});
|
||||
|
||||
test('runHostSync keeps tracker quiescent through local and remote merge and cleans up after failure', async () => {
|
||||
const calls: string[] = [];
|
||||
let localTmpDir = '';
|
||||
test('runSyncCommand forwards tokens verbatim and appends the effective log level', async () => {
|
||||
const spawned: string[][] = [];
|
||||
const deps: Partial<SyncCommandDeps> = {
|
||||
createDbSnapshot: (_dbPath: string, outPath: string) => {
|
||||
calls.push(`snapshot:${outPath}`);
|
||||
fs.writeFileSync(outPath, 'snapshot');
|
||||
},
|
||||
mergeSnapshotIntoDb: () => {
|
||||
calls.push('local-merge');
|
||||
return createEmptyMergeSummary();
|
||||
},
|
||||
formatMergeSummary: () => 'summary',
|
||||
ensureTrackerQuiescent: async () => {
|
||||
calls.push('quiescent');
|
||||
},
|
||||
assertSafeSshHost: () => {},
|
||||
resolveRemoteSubminerCommand: () => 'subminer',
|
||||
mkdtempSync: ((prefix: string) => {
|
||||
localTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), path.basename(prefix)));
|
||||
return localTmpDir;
|
||||
}) as typeof fs.mkdtempSync,
|
||||
runSsh: (_host: string, command: string) => {
|
||||
calls.push(`ssh:${command}`);
|
||||
if (command.startsWith('mktemp ')) return ok('/tmp/subminer-sync.remote\n');
|
||||
if (command.includes(' sync --snapshot ')) return ok();
|
||||
if (command.includes(' sync --merge ')) {
|
||||
return { status: 9, stdout: 'remote output', stderr: 'remote merge exploded' };
|
||||
}
|
||||
return ok();
|
||||
},
|
||||
runScp: (from: string, to: string) => {
|
||||
calls.push(`scp:${from}->${to}`);
|
||||
if (!to.includes(':')) fs.writeFileSync(to, 'pulled');
|
||||
runAppCommand: (_appPath, appArgs) => {
|
||||
spawned.push(appArgs);
|
||||
},
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
runSyncCommand(makeContext({ syncDbPath: '/tmp/local.sqlite', syncHost: 'media-box' }), deps),
|
||||
/Remote merge failed on media-box[\s\S]*remote merge exploded/,
|
||||
);
|
||||
assert.equal(calls.filter((call) => call === 'quiescent').length, 3);
|
||||
assert.ok(calls.indexOf('quiescent') < calls.findIndex((call) => call.startsWith('snapshot:')));
|
||||
assert.ok(calls.includes('local-merge'));
|
||||
assert.ok(calls.some((call) => call.startsWith('ssh:rm -rf ')));
|
||||
assert.equal(fs.existsSync(localTmpDir), false);
|
||||
});
|
||||
|
||||
test('runHostSync includes remote snapshot stderr in failures', async () => {
|
||||
const deps: Partial<SyncCommandDeps> = {
|
||||
createDbSnapshot: (_dbPath: string, outPath: string) => {
|
||||
fs.writeFileSync(outPath, 'snapshot');
|
||||
},
|
||||
ensureTrackerQuiescent: async () => {},
|
||||
assertSafeSshHost: () => {},
|
||||
resolveRemoteSubminerCommand: () => 'subminer',
|
||||
runSsh: (_host: string, command: string) => {
|
||||
if (command.startsWith('mktemp ')) return ok('/tmp/subminer-sync.remote\n');
|
||||
if (command.includes(' sync --snapshot ')) {
|
||||
return { status: 5, stdout: '', stderr: 'snapshot permission denied' };
|
||||
}
|
||||
return ok();
|
||||
},
|
||||
runScp: () => {},
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
runSyncCommand(makeContext({ syncDbPath: '/tmp/local.sqlite', syncHost: 'media-box' }), deps),
|
||||
/Remote snapshot failed on media-box[\s\S]*snapshot permission denied/,
|
||||
);
|
||||
});
|
||||
|
||||
function makeDirectionDeps(calls: string[]): Partial<SyncCommandDeps> {
|
||||
return {
|
||||
createDbSnapshot: (_dbPath: string, outPath: string) => {
|
||||
calls.push(`snapshot:${outPath}`);
|
||||
fs.writeFileSync(outPath, 'snapshot');
|
||||
},
|
||||
mergeSnapshotIntoDb: () => {
|
||||
calls.push('local-merge');
|
||||
return createEmptyMergeSummary();
|
||||
},
|
||||
formatMergeSummary: () => 'summary',
|
||||
ensureTrackerQuiescent: async () => {
|
||||
calls.push('quiescent');
|
||||
},
|
||||
assertSafeSshHost: () => {},
|
||||
resolveRemoteSubminerCommand: () => 'subminer',
|
||||
runSsh: (_host: string, command: string) => {
|
||||
calls.push(`ssh:${command}`);
|
||||
if (command.startsWith('mktemp ')) return ok('/tmp/subminer-sync.remote\n');
|
||||
return ok();
|
||||
},
|
||||
runScp: (from: string, to: string) => {
|
||||
calls.push(`scp:${from}->${to}`);
|
||||
if (!to.includes(':')) fs.writeFileSync(to, 'pulled');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('runHostSync push only snapshots locally and merges remotely', async () => {
|
||||
const calls: string[] = [];
|
||||
|
||||
await runSyncCommand(
|
||||
makeContext({
|
||||
syncDbPath: '/tmp/local.sqlite',
|
||||
syncHost: 'media-box',
|
||||
syncDirection: 'push',
|
||||
syncCliTokens: [
|
||||
'media-box',
|
||||
'--pull',
|
||||
'--remote-cmd',
|
||||
'/opt/SubMiner.AppImage',
|
||||
'--db',
|
||||
'/tmp/db.sqlite',
|
||||
'--force',
|
||||
'--json',
|
||||
],
|
||||
logLevel: 'debug',
|
||||
}),
|
||||
makeDirectionDeps(calls),
|
||||
deps,
|
||||
);
|
||||
|
||||
assert.ok(calls.some((call) => call.startsWith('snapshot:')));
|
||||
assert.ok(calls.some((call) => call.includes(' sync --merge ')));
|
||||
assert.ok(calls.some((call) => call.startsWith('scp:') && call.includes('->media-box:')));
|
||||
assert.ok(!calls.some((call) => call.includes(' sync --snapshot ')));
|
||||
assert.ok(!calls.includes('local-merge'));
|
||||
assert.deepEqual(spawned, [
|
||||
[
|
||||
'--sync-cli',
|
||||
'sync',
|
||||
'media-box',
|
||||
'--pull',
|
||||
'--remote-cmd',
|
||||
'/opt/SubMiner.AppImage',
|
||||
'--db',
|
||||
'/tmp/db.sqlite',
|
||||
'--force',
|
||||
'--json',
|
||||
'--log-level',
|
||||
'debug',
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
test('runHostSync pull only snapshots remotely and merges locally', async () => {
|
||||
const calls: string[] = [];
|
||||
test('runSyncCommand fails with a clear message when the app binary is missing', async () => {
|
||||
const deps: Partial<SyncCommandDeps> = {
|
||||
runAppCommand: () => {
|
||||
throw new Error('should not spawn');
|
||||
},
|
||||
fail: (message: string): never => {
|
||||
throw new Error(message);
|
||||
},
|
||||
};
|
||||
|
||||
await runSyncCommand(
|
||||
makeContext({
|
||||
syncDbPath: '/tmp/local.sqlite',
|
||||
syncHost: 'media-box',
|
||||
syncDirection: 'pull',
|
||||
}),
|
||||
makeDirectionDeps(calls),
|
||||
await assert.rejects(
|
||||
() => runSyncCommand(makeContext({ syncCliTokens: ['media-box'] }, null), deps),
|
||||
/SubMiner app binary not found \(sync runs inside the app\)/,
|
||||
);
|
||||
|
||||
assert.ok(calls.some((call) => call.includes(' sync --snapshot ')));
|
||||
assert.ok(calls.some((call) => call.startsWith('scp:media-box:')));
|
||||
assert.ok(calls.includes('local-merge'));
|
||||
assert.ok(!calls.some((call) => call.startsWith('snapshot:')));
|
||||
assert.ok(!calls.some((call) => call.includes(' sync --merge ')));
|
||||
});
|
||||
|
||||
@@ -1,253 +1,45 @@
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fail, log } from '../log.js';
|
||||
import { resolveImmersionDbPath } from '../history-db.js';
|
||||
import {
|
||||
createDbSnapshot,
|
||||
findLiveStatsDaemonPid,
|
||||
formatMergeSummary,
|
||||
mergeSnapshotIntoDb,
|
||||
} from '../sync/sync-db.js';
|
||||
import {
|
||||
assertSafeSshHost,
|
||||
resolveRemoteSubminerCommand,
|
||||
runScp,
|
||||
runSsh,
|
||||
shellQuote,
|
||||
} from '../sync/ssh.js';
|
||||
import { resolvePathMaybe } from '../util.js';
|
||||
import { canConnectUnixSocket } from '../mpv.js';
|
||||
import { SYNC_CLI_FLAG } from '../../src/core/services/stats-sync/cli-args.js';
|
||||
import { fail } from '../log.js';
|
||||
import { runAppCommandInteractive } from '../mpv.js';
|
||||
import type { LauncherCommandContext } from './context.js';
|
||||
import type { RemoteRunResult } from '../sync/ssh.js';
|
||||
|
||||
export interface SyncCommandDeps {
|
||||
createDbSnapshot: typeof createDbSnapshot;
|
||||
mergeSnapshotIntoDb: typeof mergeSnapshotIntoDb;
|
||||
formatMergeSummary: typeof formatMergeSummary;
|
||||
findLiveStatsDaemonPid: typeof findLiveStatsDaemonPid;
|
||||
assertSafeSshHost: typeof assertSafeSshHost;
|
||||
resolveRemoteSubminerCommand: typeof resolveRemoteSubminerCommand;
|
||||
runScp: typeof runScp;
|
||||
runSsh: typeof runSsh;
|
||||
fail: typeof fail;
|
||||
log: typeof log;
|
||||
canConnectUnixSocket: typeof canConnectUnixSocket;
|
||||
realpathSync: typeof fs.realpathSync;
|
||||
mkdtempSync: typeof fs.mkdtempSync;
|
||||
rmSync: typeof fs.rmSync;
|
||||
consoleLog: typeof console.log;
|
||||
writeStdout: typeof process.stdout.write;
|
||||
ensureTrackerQuiescent: (context: LauncherCommandContext, dbPath: string) => Promise<void>;
|
||||
}
|
||||
|
||||
function resolveDbPath(context: LauncherCommandContext): string {
|
||||
const override = context.args.syncDbPath.trim();
|
||||
return override ? resolvePathMaybe(override) : resolveImmersionDbPath();
|
||||
}
|
||||
|
||||
function isTrackerDb(dbPath: string, deps: SyncCommandDeps): boolean {
|
||||
const trackerDbPath = resolveImmersionDbPath();
|
||||
try {
|
||||
return deps.realpathSync(dbPath) === deps.realpathSync(trackerDbPath);
|
||||
} catch {
|
||||
return dbPath === trackerDbPath;
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureTrackerQuiescent(
|
||||
context: LauncherCommandContext,
|
||||
dbPath: string,
|
||||
inputDeps: Partial<SyncCommandDeps> = {},
|
||||
): Promise<void> {
|
||||
const deps = resolveSyncCommandDeps(inputDeps);
|
||||
if (context.args.syncForce) return;
|
||||
// A running SubMiner only holds the tracker's own database; --db pointed
|
||||
// elsewhere needs no guard.
|
||||
if (!isTrackerDb(dbPath, deps)) return;
|
||||
const daemonPid = deps.findLiveStatsDaemonPid(dbPath);
|
||||
if (daemonPid !== null) {
|
||||
deps.fail(
|
||||
`The SubMiner stats server is running (pid ${daemonPid}). Stop it with "subminer stats -s" (or close SubMiner) before syncing, or pass --force.`,
|
||||
);
|
||||
}
|
||||
if (context.mpvSocketPath && (await deps.canConnectUnixSocket(context.mpvSocketPath))) {
|
||||
deps.fail(
|
||||
`An mpv/SubMiner session appears to be running (socket ${context.mpvSocketPath}). Close it before syncing, or pass --force.`,
|
||||
);
|
||||
}
|
||||
runAppCommand: (appPath: string, appArgs: string[]) => void;
|
||||
fail: (message: string) => never;
|
||||
}
|
||||
|
||||
const defaultSyncCommandDeps: SyncCommandDeps = {
|
||||
createDbSnapshot,
|
||||
mergeSnapshotIntoDb,
|
||||
formatMergeSummary,
|
||||
findLiveStatsDaemonPid,
|
||||
assertSafeSshHost,
|
||||
resolveRemoteSubminerCommand,
|
||||
runScp,
|
||||
runSsh,
|
||||
runAppCommand: runAppCommandInteractive,
|
||||
fail,
|
||||
log,
|
||||
canConnectUnixSocket,
|
||||
realpathSync: fs.realpathSync,
|
||||
mkdtempSync: fs.mkdtempSync,
|
||||
rmSync: fs.rmSync,
|
||||
consoleLog: console.log,
|
||||
writeStdout: process.stdout.write.bind(process.stdout),
|
||||
ensureTrackerQuiescent: async (context, dbPath) => ensureTrackerQuiescent(context, dbPath),
|
||||
};
|
||||
|
||||
function resolveSyncCommandDeps(inputDeps: Partial<SyncCommandDeps> = {}): SyncCommandDeps {
|
||||
return { ...defaultSyncCommandDeps, ...inputDeps };
|
||||
}
|
||||
|
||||
export function runSnapshotMode(
|
||||
context: LauncherCommandContext,
|
||||
dbPath: string,
|
||||
inputDeps: Partial<SyncCommandDeps> = {},
|
||||
): void {
|
||||
const deps = resolveSyncCommandDeps(inputDeps);
|
||||
const outPath = resolvePathMaybe(context.args.syncSnapshotPath);
|
||||
deps.createDbSnapshot(dbPath, outPath);
|
||||
deps.consoleLog(outPath);
|
||||
}
|
||||
|
||||
export async function runMergeMode(
|
||||
context: LauncherCommandContext,
|
||||
dbPath: string,
|
||||
inputDeps: Partial<SyncCommandDeps> = {},
|
||||
): Promise<void> {
|
||||
const deps = resolveSyncCommandDeps(inputDeps);
|
||||
await deps.ensureTrackerQuiescent(context, dbPath);
|
||||
const snapshotPath = resolvePathMaybe(context.args.syncMergePath);
|
||||
const summary = deps.mergeSnapshotIntoDb(dbPath, snapshotPath);
|
||||
deps.consoleLog(deps.formatMergeSummary(summary));
|
||||
}
|
||||
|
||||
function cleanupRemote(host: string, remoteTmpDir: string, deps: SyncCommandDeps): void {
|
||||
if (!remoteTmpDir.startsWith('/tmp/')) return;
|
||||
deps.runSsh(host, `rm -rf ${shellQuote(remoteTmpDir)}`);
|
||||
}
|
||||
|
||||
function formatRemoteRunError(message: string, run: RemoteRunResult): string {
|
||||
const stderr = run.stderr.trim();
|
||||
return stderr ? `${message}\n${stderr}` : message;
|
||||
}
|
||||
|
||||
export async function runHostSync(
|
||||
context: LauncherCommandContext,
|
||||
dbPath: string,
|
||||
inputDeps: Partial<SyncCommandDeps> = {},
|
||||
): Promise<void> {
|
||||
const deps = resolveSyncCommandDeps(inputDeps);
|
||||
const { args } = context;
|
||||
const host = args.syncHost;
|
||||
const direction = args.syncDirection ?? 'both';
|
||||
const shouldPull = direction !== 'push';
|
||||
const shouldPush = direction !== 'pull';
|
||||
deps.assertSafeSshHost(host);
|
||||
|
||||
await deps.ensureTrackerQuiescent(context, dbPath);
|
||||
|
||||
const remoteCmd = deps.resolveRemoteSubminerCommand(host, args.syncRemoteCmd || null);
|
||||
deps.log('debug', args.logLevel, `Remote subminer command: ${remoteCmd}`);
|
||||
|
||||
const localTmpDir = deps.mkdtempSync(path.join(os.tmpdir(), 'subminer-sync-'));
|
||||
let remoteTmpDir = '';
|
||||
try {
|
||||
// Signal failures by throwing (not fail(), which exits synchronously and
|
||||
// would skip the finally cleanup, leaking temp dirs holding snapshot data).
|
||||
// main().catch() reports the message the same way fail() would.
|
||||
const mktemp = deps.runSsh(host, 'mktemp -d /tmp/subminer-sync.XXXXXX');
|
||||
remoteTmpDir = mktemp.stdout.trim();
|
||||
if (mktemp.status !== 0 || !remoteTmpDir.startsWith('/tmp/')) {
|
||||
throw new Error(`Could not create a temporary directory on ${host}.`);
|
||||
}
|
||||
|
||||
const forceFlag = args.syncForce ? ' --force' : '';
|
||||
|
||||
const localSnapshot = path.join(localTmpDir, 'local.sqlite');
|
||||
if (shouldPush) {
|
||||
deps.consoleLog(`Snapshotting local database (${dbPath})...`);
|
||||
deps.createDbSnapshot(dbPath, localSnapshot);
|
||||
}
|
||||
|
||||
const remoteSnapshot = `${remoteTmpDir}/snapshot.sqlite`;
|
||||
if (shouldPull) {
|
||||
deps.consoleLog(`Snapshotting ${host}...`);
|
||||
const snapshotRun = deps.runSsh(
|
||||
host,
|
||||
`${remoteCmd} sync --snapshot ${shellQuote(remoteSnapshot)}${forceFlag}`,
|
||||
);
|
||||
if (snapshotRun.status !== 0) {
|
||||
throw new Error(formatRemoteRunError(`Remote snapshot failed on ${host}.`, snapshotRun));
|
||||
}
|
||||
}
|
||||
|
||||
const pulledSnapshot = path.join(localTmpDir, 'remote.sqlite');
|
||||
if (shouldPull) deps.runScp(`${host}:${remoteSnapshot}`, pulledSnapshot);
|
||||
const incomingSnapshot = `${remoteTmpDir}/incoming.sqlite`;
|
||||
if (shouldPush) deps.runScp(localSnapshot, `${host}:${incomingSnapshot}`);
|
||||
|
||||
if (shouldPull) {
|
||||
deps.consoleLog(`\nMerging ${host} -> local:`);
|
||||
await deps.ensureTrackerQuiescent(context, dbPath);
|
||||
const summary = deps.mergeSnapshotIntoDb(dbPath, pulledSnapshot);
|
||||
deps.consoleLog(deps.formatMergeSummary(summary));
|
||||
}
|
||||
|
||||
if (shouldPush) {
|
||||
deps.consoleLog(`\nMerging local -> ${host}:`);
|
||||
await deps.ensureTrackerQuiescent(context, dbPath);
|
||||
const mergeRun = deps.runSsh(
|
||||
host,
|
||||
`${remoteCmd} sync --merge ${shellQuote(incomingSnapshot)}${forceFlag}`,
|
||||
);
|
||||
deps.writeStdout(mergeRun.stdout);
|
||||
if (mergeRun.status !== 0) {
|
||||
const retryCommand =
|
||||
direction === 'push' ? `subminer sync ${host} --push` : `subminer sync ${host}`;
|
||||
const localUpdate = shouldPull ? ' The local database was updated;' : '';
|
||||
throw new Error(
|
||||
formatRemoteRunError(
|
||||
`Remote merge failed on ${host}.${localUpdate} re-run "${retryCommand}" once the remote issue is fixed.`,
|
||||
mergeRun,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
deps.consoleLog('\nSync complete.');
|
||||
} finally {
|
||||
deps.rmSync(localTmpDir, { recursive: true, force: true });
|
||||
if (remoteTmpDir) {
|
||||
try {
|
||||
cleanupRemote(host, remoteTmpDir, deps);
|
||||
} catch {
|
||||
// best effort
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* `subminer sync` is a thin proxy: the sync engine only executes inside the
|
||||
* SubMiner app (--sync-cli mode, libsql). The launcher contributes its
|
||||
* parser/help and app discovery; the app's parseSyncCliTokens owns validation,
|
||||
* so its errors reach the terminal through the child's inherited stdio. The
|
||||
* child owns the terminal and its exit code becomes the launcher's.
|
||||
*/
|
||||
export async function runSyncCommand(
|
||||
context: LauncherCommandContext,
|
||||
inputDeps: Partial<SyncCommandDeps> = {},
|
||||
): Promise<boolean> {
|
||||
const deps = resolveSyncCommandDeps(inputDeps);
|
||||
const { args } = context;
|
||||
if (!args.sync) return false;
|
||||
|
||||
const dbPath = resolveDbPath(context);
|
||||
if (args.syncSnapshotPath) {
|
||||
runSnapshotMode(context, dbPath, deps);
|
||||
} else if (args.syncMergePath) {
|
||||
await runMergeMode(context, dbPath, deps);
|
||||
} else if (args.syncHost) {
|
||||
await runHostSync(context, dbPath, deps);
|
||||
} else {
|
||||
deps.fail('sync requires a host, --snapshot <file>, or --merge <file>.');
|
||||
const deps = { ...defaultSyncCommandDeps, ...inputDeps };
|
||||
if (!context.args.sync) return false;
|
||||
if (!context.appPath) {
|
||||
deps.fail(
|
||||
context.processAdapter.platform() === 'darwin'
|
||||
? 'SubMiner app binary not found (sync runs inside the app). Install SubMiner.app to /Applications or ~/Applications, or set SUBMINER_APPIMAGE_PATH.'
|
||||
: 'SubMiner app binary not found (sync runs inside the app). Install the SubMiner app, or set SUBMINER_APPIMAGE_PATH.',
|
||||
);
|
||||
return true; // fail() never returns; this only satisfies control-flow analysis
|
||||
}
|
||||
deps.runAppCommand(context.appPath, [
|
||||
SYNC_CLI_FLAG,
|
||||
'sync',
|
||||
...context.args.syncCliTokens,
|
||||
'--log-level',
|
||||
context.args.logLevel,
|
||||
]);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -136,14 +136,10 @@ test('applyInvocationsToArgs maps config and jellyfin invocation state', () => {
|
||||
statsCleanupLifetime: false,
|
||||
statsLogLevel: null,
|
||||
syncTriggered: false,
|
||||
syncHost: null,
|
||||
syncSnapshotPath: null,
|
||||
syncMergePath: null,
|
||||
syncDirection: 'both',
|
||||
syncRemoteCmd: null,
|
||||
syncDbPath: null,
|
||||
syncForce: false,
|
||||
syncCliTokens: [],
|
||||
syncLogLevel: null,
|
||||
syncUiTriggered: false,
|
||||
syncUiLogLevel: null,
|
||||
doctorTriggered: false,
|
||||
doctorLogLevel: null,
|
||||
doctorRefreshKnownWords: false,
|
||||
@@ -191,14 +187,10 @@ test('applyInvocationsToArgs maps settings invocation to settings window', () =>
|
||||
statsCleanupLifetime: false,
|
||||
statsLogLevel: null,
|
||||
syncTriggered: false,
|
||||
syncHost: null,
|
||||
syncSnapshotPath: null,
|
||||
syncMergePath: null,
|
||||
syncDirection: 'both',
|
||||
syncRemoteCmd: null,
|
||||
syncDbPath: null,
|
||||
syncForce: false,
|
||||
syncCliTokens: [],
|
||||
syncLogLevel: null,
|
||||
syncUiTriggered: false,
|
||||
syncUiLogLevel: null,
|
||||
doctorTriggered: false,
|
||||
doctorLogLevel: null,
|
||||
doctorRefreshKnownWords: false,
|
||||
@@ -239,14 +231,10 @@ test('applyInvocationsToArgs fails when config invocation has no action', () =>
|
||||
statsCleanupLifetime: false,
|
||||
statsLogLevel: null,
|
||||
syncTriggered: false,
|
||||
syncHost: null,
|
||||
syncSnapshotPath: null,
|
||||
syncMergePath: null,
|
||||
syncDirection: 'both',
|
||||
syncRemoteCmd: null,
|
||||
syncDbPath: null,
|
||||
syncForce: false,
|
||||
syncCliTokens: [],
|
||||
syncLogLevel: null,
|
||||
syncUiTriggered: false,
|
||||
syncUiLogLevel: null,
|
||||
doctorTriggered: false,
|
||||
doctorLogLevel: null,
|
||||
doctorRefreshKnownWords: false,
|
||||
@@ -285,14 +273,10 @@ test('applyInvocationsToArgs maps texthooker browser-open request', () => {
|
||||
statsCleanupLifetime: false,
|
||||
statsLogLevel: null,
|
||||
syncTriggered: false,
|
||||
syncHost: null,
|
||||
syncSnapshotPath: null,
|
||||
syncMergePath: null,
|
||||
syncDirection: 'both',
|
||||
syncRemoteCmd: null,
|
||||
syncDbPath: null,
|
||||
syncForce: false,
|
||||
syncCliTokens: [],
|
||||
syncLogLevel: null,
|
||||
syncUiTriggered: false,
|
||||
syncUiLogLevel: null,
|
||||
doctorTriggered: false,
|
||||
doctorLogLevel: null,
|
||||
doctorRefreshKnownWords: false,
|
||||
|
||||
@@ -200,13 +200,8 @@ export function createDefaultArgs(
|
||||
useRofi: false,
|
||||
history: false,
|
||||
sync: false,
|
||||
syncHost: '',
|
||||
syncSnapshotPath: '',
|
||||
syncMergePath: '',
|
||||
syncDirection: 'both',
|
||||
syncRemoteCmd: '',
|
||||
syncDbPath: '',
|
||||
syncForce: false,
|
||||
syncCliTokens: [],
|
||||
syncUi: false,
|
||||
logLevel: loggingConfig.level ?? 'warn',
|
||||
logRotation: loggingConfig.rotation ?? 7,
|
||||
passwordStore: '',
|
||||
@@ -274,15 +269,13 @@ export function applyInvocationsToArgs(parsed: Args, invocations: CliInvocations
|
||||
}
|
||||
if (invocations.syncTriggered) {
|
||||
parsed.sync = true;
|
||||
parsed.syncHost = invocations.syncHost ?? '';
|
||||
parsed.syncSnapshotPath = invocations.syncSnapshotPath ?? '';
|
||||
parsed.syncMergePath = invocations.syncMergePath ?? '';
|
||||
parsed.syncDirection = invocations.syncDirection;
|
||||
parsed.syncRemoteCmd = invocations.syncRemoteCmd ?? '';
|
||||
parsed.syncDbPath = invocations.syncDbPath ?? '';
|
||||
parsed.syncForce = invocations.syncForce;
|
||||
parsed.syncCliTokens = invocations.syncCliTokens;
|
||||
if (invocations.syncLogLevel) parsed.logLevel = parseLogLevel(invocations.syncLogLevel);
|
||||
}
|
||||
if (invocations.syncUiTriggered) {
|
||||
parsed.syncUi = true;
|
||||
if (invocations.syncUiLogLevel) parsed.logLevel = parseLogLevel(invocations.syncUiLogLevel);
|
||||
}
|
||||
if (invocations.doctorTriggered) parsed.doctor = true;
|
||||
if (invocations.doctorRefreshKnownWords) parsed.doctorRefreshKnownWords = true;
|
||||
if (invocations.logsTriggered && !invocations.logsExport) {
|
||||
|
||||
@@ -43,21 +43,88 @@ test('parseCliPrograms captures texthooker browser-open flag', () => {
|
||||
assert.equal(result.invocations.texthookerOpenBrowser, true);
|
||||
});
|
||||
|
||||
test('parseCliPrograms captures one-way sync directions', () => {
|
||||
test('parseCliPrograms lowers sync options into app-owned CLI tokens', () => {
|
||||
const push = parseCliPrograms(['sync', 'media-box', '--push'], 'subminer');
|
||||
assert.equal(push.invocations.syncTriggered, true);
|
||||
assert.deepEqual(push.invocations.syncCliTokens, ['media-box', '--push']);
|
||||
|
||||
const pull = parseCliPrograms(['sync', 'media-box', '--pull'], 'subminer');
|
||||
assert.deepEqual(pull.invocations.syncCliTokens, ['media-box', '--pull']);
|
||||
|
||||
assert.equal(push.invocations.syncDirection, 'push');
|
||||
assert.equal(pull.invocations.syncDirection, 'pull');
|
||||
const check = parseCliPrograms(['sync', 'media-box', '--check', '--json'], 'subminer');
|
||||
assert.deepEqual(check.invocations.syncCliTokens, ['media-box', '--check', '--json']);
|
||||
|
||||
const full = parseCliPrograms(
|
||||
[
|
||||
'sync',
|
||||
'media-box',
|
||||
'--remote-cmd',
|
||||
'/opt/SubMiner.AppImage',
|
||||
'--db',
|
||||
'/tmp/db.sqlite',
|
||||
'--force',
|
||||
'--log-level',
|
||||
'debug',
|
||||
],
|
||||
'subminer',
|
||||
);
|
||||
assert.deepEqual(full.invocations.syncCliTokens, [
|
||||
'media-box',
|
||||
'--remote-cmd',
|
||||
'/opt/SubMiner.AppImage',
|
||||
'--db',
|
||||
'/tmp/db.sqlite',
|
||||
'--force',
|
||||
]);
|
||||
assert.equal(full.invocations.syncLogLevel, 'debug');
|
||||
|
||||
const snapshot = parseCliPrograms(['sync', '--snapshot', '/tmp/out.sqlite'], 'subminer');
|
||||
assert.deepEqual(snapshot.invocations.syncCliTokens, ['--snapshot', '/tmp/out.sqlite']);
|
||||
|
||||
const merge = parseCliPrograms(['sync', '--merge', '/tmp/in.sqlite'], 'subminer');
|
||||
assert.deepEqual(merge.invocations.syncCliTokens, ['--merge', '/tmp/in.sqlite']);
|
||||
|
||||
const makeTemp = parseCliPrograms(['sync', '--make-temp'], 'subminer');
|
||||
assert.deepEqual(makeTemp.invocations.syncCliTokens, ['--make-temp']);
|
||||
|
||||
const removeTemp = parseCliPrograms(
|
||||
['sync', '--remove-temp', '/tmp/subminer-sync-x'],
|
||||
'subminer',
|
||||
);
|
||||
assert.deepEqual(removeTemp.invocations.syncCliTokens, ['--remove-temp', '/tmp/subminer-sync-x']);
|
||||
});
|
||||
|
||||
test('parseCliPrograms rejects conflicting or hostless one-way sync directions', () => {
|
||||
test('parseCliPrograms leaves sync validation to the app parser', () => {
|
||||
// Invalid combinations are forwarded; the app's parseSyncCliTokens rejects them.
|
||||
const invalid = parseCliPrograms(['sync', 'media-box', '--push', '--pull'], 'subminer');
|
||||
assert.equal(invalid.invocations.syncTriggered, true);
|
||||
assert.deepEqual(invalid.invocations.syncCliTokens, ['media-box', '--push', '--pull']);
|
||||
|
||||
const empty = parseCliPrograms(['sync'], 'subminer');
|
||||
assert.equal(empty.invocations.syncTriggered, true);
|
||||
assert.deepEqual(empty.invocations.syncCliTokens, []);
|
||||
});
|
||||
|
||||
test('parseCliPrograms captures sync --ui', () => {
|
||||
const result = parseCliPrograms(['sync', '--ui'], 'subminer');
|
||||
assert.equal(result.invocations.syncUiTriggered, true);
|
||||
assert.equal(result.invocations.syncTriggered, false);
|
||||
|
||||
assert.throws(
|
||||
() => parseCliPrograms(['sync', 'media-box', '--push', '--pull'], 'subminer'),
|
||||
/--push and --pull cannot be combined/,
|
||||
);
|
||||
assert.throws(
|
||||
() => parseCliPrograms(['sync', '--snapshot', '/tmp/stats.sqlite', '--push'], 'subminer'),
|
||||
/--push and --pull require a host/,
|
||||
() => parseCliPrograms(['sync', 'media-box', '--ui'], 'subminer'),
|
||||
/--ui cannot be combined/,
|
||||
);
|
||||
});
|
||||
|
||||
test('parseCliPrograms rejects sync --ui with --remote-cmd', () => {
|
||||
assert.throws(
|
||||
() => parseCliPrograms(['sync', '--ui', '--remote-cmd', '/opt/SubMiner.AppImage'], 'subminer'),
|
||||
{ message: 'Sync --ui cannot be combined with other sync options.' },
|
||||
);
|
||||
});
|
||||
|
||||
test('parseCliPrograms rejects sync --ui with --db', () => {
|
||||
assert.throws(() => parseCliPrograms(['sync', '--ui', '--db', '/tmp/db.sqlite'], 'subminer'), {
|
||||
message: 'Sync --ui cannot be combined with other sync options.',
|
||||
});
|
||||
});
|
||||
|
||||
@@ -39,14 +39,10 @@ export interface CliInvocations {
|
||||
statsCleanupLifetime: boolean;
|
||||
statsLogLevel: string | null;
|
||||
syncTriggered: boolean;
|
||||
syncHost: string | null;
|
||||
syncSnapshotPath: string | null;
|
||||
syncMergePath: string | null;
|
||||
syncDirection: 'both' | 'push' | 'pull';
|
||||
syncRemoteCmd: string | null;
|
||||
syncDbPath: string | null;
|
||||
syncForce: boolean;
|
||||
syncCliTokens: string[];
|
||||
syncLogLevel: string | null;
|
||||
syncUiTriggered: boolean;
|
||||
syncUiLogLevel: string | null;
|
||||
doctorTriggered: boolean;
|
||||
doctorLogLevel: string | null;
|
||||
doctorRefreshKnownWords: boolean;
|
||||
@@ -75,7 +71,11 @@ function applyRootOptions(program: Command): void {
|
||||
.option('-R, --rofi', 'Use rofi picker')
|
||||
.option('-H, --history', 'Browse local watch history')
|
||||
.option('-S, --start-overlay', 'Auto-start overlay')
|
||||
.option('-T, --no-texthooker', 'Disable texthooker-ui server');
|
||||
.option('-T, --no-texthooker', 'Disable texthooker-ui server')
|
||||
// The SubMiner app answers sync commands when invoked with --sync-cli.
|
||||
// Remote-command resolution may address the launcher the same way, so
|
||||
// accept the flag as a no-op to keep both invocation shapes equivalent.
|
||||
.option('--sync-cli', 'Compatibility no-op (sync commands work with or without it)');
|
||||
}
|
||||
|
||||
function buildSubcommandHelpText(program: Command): string {
|
||||
@@ -171,14 +171,10 @@ export function parseCliPrograms(
|
||||
let statsCleanupLifetime = false;
|
||||
let statsLogLevel: string | null = null;
|
||||
let syncTriggered = false;
|
||||
let syncHost: string | null = null;
|
||||
let syncSnapshotPath: string | null = null;
|
||||
let syncMergePath: string | null = null;
|
||||
let syncDirection: 'both' | 'push' | 'pull' = 'both';
|
||||
let syncRemoteCmd: string | null = null;
|
||||
let syncDbPath: string | null = null;
|
||||
let syncForce = false;
|
||||
let syncCliTokens: string[] = [];
|
||||
let syncLogLevel: string | null = null;
|
||||
let syncUiTriggered = false;
|
||||
let syncUiLogLevel: string | null = null;
|
||||
let doctorLogLevel: string | null = null;
|
||||
let doctorRefreshKnownWords = false;
|
||||
let logsTriggered = false;
|
||||
@@ -319,6 +315,11 @@ export function parseCliPrograms(
|
||||
.option('--db <file>', 'Override the local stats database path')
|
||||
.option('--remote-cmd <cmd>', 'subminer command to run on the remote host')
|
||||
.option('-f, --force', 'Skip the running-app safety check')
|
||||
.option('--check', 'Test the SSH connection and remote subminer availability')
|
||||
.option('--json', 'Emit machine-readable NDJSON progress output')
|
||||
.option('--make-temp', 'Create a sync temp directory and print its path (used over SSH)')
|
||||
.option('--remove-temp <dir>', 'Remove a sync temp directory created by --make-temp')
|
||||
.option('--ui', 'Open the SubMiner sync window')
|
||||
.option('--log-level <level>', 'Log level')
|
||||
.action((rawHost: string | undefined, options: Record<string, unknown>) => {
|
||||
const host = typeof rawHost === 'string' ? rawHost.trim() : '';
|
||||
@@ -326,28 +327,49 @@ export function parseCliPrograms(
|
||||
const merge = typeof options.merge === 'string' ? options.merge.trim() : '';
|
||||
const push = options.push === true;
|
||||
const pull = options.pull === true;
|
||||
if (push && pull) {
|
||||
throw new Error('Sync --push and --pull cannot be combined.');
|
||||
}
|
||||
if ((push || pull) && !host) {
|
||||
throw new Error('Sync --push and --pull require a host.');
|
||||
}
|
||||
const modes = [Boolean(host), Boolean(snapshot), Boolean(merge)].filter(Boolean).length;
|
||||
if (modes === 0) {
|
||||
throw new Error('Sync requires a host, --snapshot <file>, or --merge <file>.');
|
||||
}
|
||||
if (modes > 1) {
|
||||
throw new Error('Sync host, --snapshot, and --merge cannot be combined.');
|
||||
const check = options.check === true;
|
||||
const makeTemp = options.makeTemp === true;
|
||||
const removeTemp = typeof options.removeTemp === 'string' ? options.removeTemp.trim() : '';
|
||||
if (options.ui === true) {
|
||||
if (
|
||||
host ||
|
||||
snapshot ||
|
||||
merge ||
|
||||
push ||
|
||||
pull ||
|
||||
check ||
|
||||
makeTemp ||
|
||||
removeTemp ||
|
||||
options.remoteCmd !== undefined ||
|
||||
options.db !== undefined ||
|
||||
options.json === true ||
|
||||
options.force === true
|
||||
) {
|
||||
throw new Error('Sync --ui cannot be combined with other sync options.');
|
||||
}
|
||||
syncUiTriggered = true;
|
||||
syncUiLogLevel = typeof options.logLevel === 'string' ? options.logLevel : null;
|
||||
return;
|
||||
}
|
||||
// No validation here: the app's parseSyncCliTokens owns the sync rules
|
||||
// and its error text reaches the terminal through the child's stdio.
|
||||
const remoteCmd = typeof options.remoteCmd === 'string' ? options.remoteCmd.trim() : '';
|
||||
const dbPath = typeof options.db === 'string' ? options.db.trim() : '';
|
||||
const tokens: string[] = [];
|
||||
if (host) tokens.push(host);
|
||||
if (snapshot) tokens.push('--snapshot', snapshot);
|
||||
if (merge) tokens.push('--merge', merge);
|
||||
if (makeTemp) tokens.push('--make-temp');
|
||||
if (removeTemp) tokens.push('--remove-temp', removeTemp);
|
||||
if (push) tokens.push('--push');
|
||||
if (pull) tokens.push('--pull');
|
||||
if (check) tokens.push('--check');
|
||||
if (remoteCmd) tokens.push('--remote-cmd', remoteCmd);
|
||||
if (dbPath) tokens.push('--db', dbPath);
|
||||
if (options.force === true) tokens.push('--force');
|
||||
if (options.json === true) tokens.push('--json');
|
||||
syncTriggered = true;
|
||||
syncHost = host || null;
|
||||
syncSnapshotPath = snapshot || null;
|
||||
syncMergePath = merge || null;
|
||||
syncDirection = push ? 'push' : pull ? 'pull' : 'both';
|
||||
syncRemoteCmd =
|
||||
typeof options.remoteCmd === 'string' ? options.remoteCmd.trim() || null : null;
|
||||
syncDbPath = typeof options.db === 'string' ? options.db.trim() || null : null;
|
||||
syncForce = options.force === true;
|
||||
syncCliTokens = tokens;
|
||||
syncLogLevel = typeof options.logLevel === 'string' ? options.logLevel : null;
|
||||
});
|
||||
|
||||
@@ -463,14 +485,10 @@ export function parseCliPrograms(
|
||||
statsCleanupLifetime,
|
||||
statsLogLevel,
|
||||
syncTriggered,
|
||||
syncHost,
|
||||
syncSnapshotPath,
|
||||
syncMergePath,
|
||||
syncDirection,
|
||||
syncRemoteCmd,
|
||||
syncDbPath,
|
||||
syncForce,
|
||||
syncCliTokens,
|
||||
syncLogLevel,
|
||||
syncUiTriggered,
|
||||
syncUiLogLevel,
|
||||
doctorTriggered,
|
||||
doctorLogLevel,
|
||||
doctorRefreshKnownWords,
|
||||
|
||||
+6
-69
@@ -1,32 +1,12 @@
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { Database } from 'bun:sqlite';
|
||||
import { resolveConfigDir } from '../src/config/path-resolution.js';
|
||||
import { readLauncherMainConfigObject } from './config/shared-config-reader.js';
|
||||
import type { HistoryVideoRow } from './history-types.js';
|
||||
import { resolvePathMaybe } from './util.js';
|
||||
import { resolveImmersionDbPath } from '../src/core/services/stats-sync/db-path.js';
|
||||
import {
|
||||
isReadonlyWalRetryError,
|
||||
withReadonlyWalRetry,
|
||||
} from '../src/core/services/stats-sync/wal-retry.js';
|
||||
|
||||
export function resolveImmersionDbPath(): string {
|
||||
const root = readLauncherMainConfigObject();
|
||||
const tracking =
|
||||
root?.immersionTracking &&
|
||||
typeof root.immersionTracking === 'object' &&
|
||||
!Array.isArray(root.immersionTracking)
|
||||
? (root.immersionTracking as Record<string, unknown>)
|
||||
: null;
|
||||
const configured = typeof tracking?.dbPath === 'string' ? tracking.dbPath.trim() : '';
|
||||
if (configured) return resolvePathMaybe(configured);
|
||||
|
||||
const configDir = resolveConfigDir({
|
||||
platform: process.platform,
|
||||
appDataDir: process.env.APPDATA,
|
||||
xdgConfigHome: process.env.XDG_CONFIG_HOME,
|
||||
homeDir: os.homedir(),
|
||||
existsSync: fs.existsSync,
|
||||
});
|
||||
return path.join(configDir, 'immersion.sqlite');
|
||||
}
|
||||
export { isReadonlyWalRetryError, resolveImmersionDbPath, withReadonlyWalRetry };
|
||||
|
||||
interface RawHistoryRow {
|
||||
video_id: number;
|
||||
@@ -43,49 +23,6 @@ export function queryLocalWatchHistory(dbPath: string): HistoryVideoRow[] {
|
||||
return withReadonlyWalRetry(dbPath, (options) => readHistoryRows(dbPath, options));
|
||||
}
|
||||
|
||||
export function withReadonlyWalRetry<T>(
|
||||
dbPath: string,
|
||||
query: (options: { readonly?: boolean; readwrite?: boolean; create?: boolean }) => T,
|
||||
): T {
|
||||
try {
|
||||
return query({ readonly: true });
|
||||
} catch (error) {
|
||||
if (!isReadonlyWalRetryError(error, dbPath)) throw error;
|
||||
return query({ readwrite: true, create: false });
|
||||
}
|
||||
}
|
||||
|
||||
export function isReadonlyWalRetryError(error: unknown, dbPath: string): boolean {
|
||||
if (!isWalModeSqliteDatabase(dbPath)) return false;
|
||||
const code =
|
||||
typeof error === 'object' && error !== null && 'code' in error
|
||||
? String((error as { code?: unknown }).code ?? '')
|
||||
: '';
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const text = `${code} ${message}`.toLowerCase();
|
||||
return (
|
||||
text.includes('readonly') ||
|
||||
text.includes('read-only') ||
|
||||
text.includes('attempt to write a readonly database') ||
|
||||
text.includes('sqlite_cantopen') ||
|
||||
text.includes('unable to open database file')
|
||||
);
|
||||
}
|
||||
|
||||
function isWalModeSqliteDatabase(dbPath: string): boolean {
|
||||
const header = Buffer.alloc(20);
|
||||
let fd: number | null = null;
|
||||
try {
|
||||
fd = fs.openSync(dbPath, 'r');
|
||||
if (fs.readSync(fd, header, 0, header.length, 0) < header.length) return false;
|
||||
} catch {
|
||||
return false;
|
||||
} finally {
|
||||
if (fd !== null) fs.closeSync(fd);
|
||||
}
|
||||
return header.subarray(0, 16).toString('ascii') === 'SQLite format 3\0' && header[18] === 2;
|
||||
}
|
||||
|
||||
function tableExists(db: Database, tableName: string): boolean {
|
||||
return Boolean(
|
||||
db.query(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`).get(tableName),
|
||||
|
||||
@@ -31,13 +31,8 @@ function createArgs(): Args {
|
||||
useRofi: false,
|
||||
history: false,
|
||||
sync: false,
|
||||
syncHost: '',
|
||||
syncSnapshotPath: '',
|
||||
syncMergePath: '',
|
||||
syncDirection: 'both',
|
||||
syncRemoteCmd: '',
|
||||
syncDbPath: '',
|
||||
syncForce: false,
|
||||
syncCliTokens: [],
|
||||
syncUi: false,
|
||||
logLevel: 'info',
|
||||
logRotation: 7,
|
||||
passwordStore: '',
|
||||
|
||||
+33
-7
@@ -119,6 +119,37 @@ test('runAppCommandCaptureOutput transports Linux AppImage args through environm
|
||||
}
|
||||
});
|
||||
|
||||
test('runAppCommandCaptureOutput runs Linux AppImage sync in Node-only mode', () => {
|
||||
const { dir } = createTempSocketPath();
|
||||
const appPath = path.join(dir, 'SubMiner.AppImage');
|
||||
fs.writeFileSync(
|
||||
appPath,
|
||||
[
|
||||
'#!/bin/sh',
|
||||
'printf "args:%s\\n" "$*"',
|
||||
'printf "electron-node:%s\\n" "$ELECTRON_RUN_AS_NODE"',
|
||||
'printf "argc:%s\\n" "$SUBMINER_APP_ARGC"',
|
||||
'printf "arg0:%s\\n" "$SUBMINER_APP_ARG_0"',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
fs.chmodSync(appPath, 0o755);
|
||||
|
||||
try {
|
||||
const result = withPlatform('linux', () =>
|
||||
runAppCommandCaptureOutput(appPath, ['--sync-cli', 'sync', '--snapshot', '/tmp/out']),
|
||||
);
|
||||
|
||||
assert.equal(result.status, 0);
|
||||
assert.match(result.stdout, /^args:-e /m);
|
||||
assert.match(result.stdout, /^electron-node:1$/m);
|
||||
assert.match(result.stdout, /^argc:4$/m);
|
||||
assert.match(result.stdout, /^arg0:--sync-cli$/m);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('parseMpvArgString preserves empty quoted tokens', () => {
|
||||
assert.deepEqual(parseMpvArgString('--title "" --force-media-title \'\' --pause'), [
|
||||
'--title',
|
||||
@@ -572,13 +603,8 @@ function makeArgs(overrides: Partial<Args> = {}): Args {
|
||||
useRofi: false,
|
||||
history: false,
|
||||
sync: false,
|
||||
syncHost: '',
|
||||
syncSnapshotPath: '',
|
||||
syncMergePath: '',
|
||||
syncDirection: 'both',
|
||||
syncRemoteCmd: '',
|
||||
syncDbPath: '',
|
||||
syncForce: false,
|
||||
syncCliTokens: [],
|
||||
syncUi: false,
|
||||
logLevel: 'error',
|
||||
logRotation: 7,
|
||||
passwordStore: '',
|
||||
|
||||
+43
-2
@@ -3,6 +3,7 @@ import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import net from 'node:net';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import type { StdioOptions } from 'node:child_process';
|
||||
import { buildMpvLaunchModeArgs } from '../src/shared/mpv-launch-mode.js';
|
||||
import { buildMpvLoggingArgs } from '../src/shared/mpv-logging-args.js';
|
||||
import {
|
||||
@@ -1276,6 +1277,15 @@ function shouldTransportAppArgsForAppImage(appPath: string): boolean {
|
||||
return process.platform === 'linux' && /\.AppImage$/i.test(appPath);
|
||||
}
|
||||
|
||||
const APPIMAGE_SYNC_NODE_RUNNER = [
|
||||
'const root=process.env.APPDIR+"/resources/app.asar";',
|
||||
'const {runSyncCliFromProcess}=require(root+"/dist/main/sync-cli.js");',
|
||||
'const count=Number(process.env.SUBMINER_APP_ARGC);',
|
||||
'const argv=[process.execPath,...Array.from({length:count},(_,i)=>process.env["SUBMINER_APP_ARG_"+i]??"")];',
|
||||
'runSyncCliFromProcess(argv,require(root+"/package.json").version)',
|
||||
'.then(code=>process.exit(code),error=>{console.error(error);process.exit(1)});',
|
||||
].join('');
|
||||
|
||||
function buildAppEnv(
|
||||
baseEnv: NodeJS.ProcessEnv = process.env,
|
||||
extraEnv: NodeJS.ProcessEnv = {},
|
||||
@@ -1431,6 +1441,16 @@ function maybeCaptureAppArgs(appArgs: string[]): boolean {
|
||||
|
||||
function resolveAppSpawnTarget(appPath: string, appArgs: string[]): SpawnTarget {
|
||||
if (shouldTransportAppArgsForAppImage(appPath)) {
|
||||
if (appArgs[0] === '--sync-cli') {
|
||||
return {
|
||||
command: appPath,
|
||||
args: ['-e', APPIMAGE_SYNC_NODE_RUNNER],
|
||||
env: {
|
||||
...buildTransportedAppArgsEnv(appArgs),
|
||||
ELECTRON_RUN_AS_NODE: '1',
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
command: appPath,
|
||||
args: [],
|
||||
@@ -1444,16 +1464,37 @@ function resolveAppSpawnTarget(appPath: string, appArgs: string[]): SpawnTarget
|
||||
}
|
||||
|
||||
export function runAppCommandWithInherit(appPath: string, appArgs: string[]): void {
|
||||
runAppCommand(appPath, appArgs, ['ignore', 'pipe', 'pipe'], true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Like runAppCommandWithInherit, but with the terminal fully attached: the
|
||||
* child owns stdin (ssh password/host-key prompts must reach the user) and
|
||||
* writes stdout/stderr directly (NDJSON --json output must stay unwrapped).
|
||||
* Used for `subminer sync`, which proxies to the app's --sync-cli mode.
|
||||
*/
|
||||
export function runAppCommandInteractive(appPath: string, appArgs: string[]): void {
|
||||
runAppCommand(appPath, appArgs, ['inherit', 'inherit', 'inherit'], false);
|
||||
}
|
||||
|
||||
function runAppCommand(
|
||||
appPath: string,
|
||||
appArgs: string[],
|
||||
stdio: StdioOptions,
|
||||
attachLogging: boolean,
|
||||
): void {
|
||||
if (maybeCaptureAppArgs(appArgs)) {
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const target = resolveAppSpawnTarget(appPath, appArgs);
|
||||
const proc = spawn(target.command, target.args, {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
stdio,
|
||||
env: buildAppEnv(process.env, target.env),
|
||||
});
|
||||
attachAppProcessLogging(proc, { mirrorStdout: true, mirrorStderr: true });
|
||||
if (attachLogging) {
|
||||
attachAppProcessLogging(proc, { mirrorStdout: true, mirrorStderr: true });
|
||||
}
|
||||
proc.once('error', (error) => {
|
||||
fail(`Failed to run app command: ${error.message}`);
|
||||
});
|
||||
|
||||
@@ -4,7 +4,11 @@ import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { Database } from 'bun:sqlite';
|
||||
import { createDbSnapshot, mergeSnapshotIntoDb } from './sync-db.js';
|
||||
// The engine executes on libsql in production; these merge tests run through
|
||||
// that same driver. bun:sqlite is used only to build fixtures and inspect
|
||||
// results.
|
||||
import { createDbSnapshot } from '../../src/core/services/stats-sync/shared.js';
|
||||
import { mergeSnapshotIntoDb } from '../../src/core/services/stats-sync/merge.js';
|
||||
import {
|
||||
createImmersionDbFixture,
|
||||
insertFixtureSession,
|
||||
@@ -596,3 +600,61 @@ test('createDbSnapshot produces a mergeable copy', () => {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('adopted word frequency excludes active-session counts that merge later', () => {
|
||||
const { dir, localPath, remotePath } = makeDbPair();
|
||||
try {
|
||||
// Remote has an ended session and a stale ACTIVE one (e.g. app crashed).
|
||||
// The remote tracker increments frequency live, so the word's frequency (5)
|
||||
// already includes the active session's 4 occurrences even though that
|
||||
// session's lines are skipped by the merge.
|
||||
insertFixtureSession(remotePath, {
|
||||
uuid: 'remote-ended',
|
||||
videoKey: 'showb-e1',
|
||||
animeTitleKey: 'showb',
|
||||
startedAtMs: BASE_MS,
|
||||
applyLifetime: true,
|
||||
words: [{ headword: '食べる', word: '食べた', reading: 'たべた', count: 1 }],
|
||||
});
|
||||
insertFixtureSession(remotePath, {
|
||||
uuid: 'remote-active',
|
||||
videoKey: 'showb-e2',
|
||||
animeTitleKey: 'showb',
|
||||
startedAtMs: BASE_MS + DAY_MS,
|
||||
endedAtMs: null,
|
||||
words: [{ headword: '食べる', word: '食べた', reading: 'たべた', count: 4 }],
|
||||
});
|
||||
|
||||
const first = mergeSnapshotIntoDb(localPath, remotePath);
|
||||
assert.equal(first.sessionsMerged, 1);
|
||||
assert.equal(first.activeSessionsSkipped, 1);
|
||||
assert.equal(first.wordsAdded, 1);
|
||||
// Only the ended session's count is adopted; the active session's slice
|
||||
// is re-added when that session finalizes and syncs.
|
||||
assert.equal(
|
||||
queryOne<{ frequency: number }>(localPath, `SELECT frequency FROM imm_words WHERE word = '食べた'`)
|
||||
?.frequency,
|
||||
1,
|
||||
);
|
||||
|
||||
// The remote app restarts and finalizes the stale session.
|
||||
withWritableDb(remotePath, (db) => {
|
||||
db.prepare(
|
||||
`UPDATE imm_sessions SET ended_at_ms = ?, status = 2
|
||||
WHERE session_uuid = 'remote-active'`,
|
||||
).run(String(BASE_MS + DAY_MS + 1_500_000));
|
||||
});
|
||||
|
||||
const second = mergeSnapshotIntoDb(localPath, remotePath);
|
||||
assert.equal(second.sessionsMerged, 1);
|
||||
assert.equal(second.sessionsAlreadyPresent, 1);
|
||||
// 1 (ended session) + 4 (finalized session), not 5 + 4 = 9.
|
||||
assert.equal(
|
||||
queryOne<{ frequency: number }>(localPath, `SELECT frequency FROM imm_words WHERE word = '食べた'`)
|
||||
?.frequency,
|
||||
5,
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -1,423 +0,0 @@
|
||||
import { Database } from 'bun:sqlite';
|
||||
import { insertRow, tableExists, type SyncMergeSummary } from './sync-shared.js';
|
||||
|
||||
const ANIME_COPY_COLUMNS = [
|
||||
'normalized_title_key',
|
||||
'canonical_title',
|
||||
'anilist_id',
|
||||
'title_romaji',
|
||||
'title_english',
|
||||
'title_native',
|
||||
'episodes_total',
|
||||
'description',
|
||||
'metadata_json',
|
||||
'CREATED_DATE',
|
||||
'LAST_UPDATE_DATE',
|
||||
] as const;
|
||||
|
||||
const VIDEO_COPY_COLUMNS = [
|
||||
'video_key',
|
||||
'canonical_title',
|
||||
'source_type',
|
||||
'source_path',
|
||||
'source_url',
|
||||
'parsed_basename',
|
||||
'parsed_title',
|
||||
'parsed_season',
|
||||
'parsed_episode',
|
||||
'parser_source',
|
||||
'parser_confidence',
|
||||
'parse_metadata_json',
|
||||
'watched',
|
||||
'duration_ms',
|
||||
'file_size_bytes',
|
||||
'codec_id',
|
||||
'container_id',
|
||||
'width_px',
|
||||
'height_px',
|
||||
'fps_x100',
|
||||
'bitrate_kbps',
|
||||
'audio_codec_id',
|
||||
'hash_sha256',
|
||||
'screenshot_path',
|
||||
'metadata_json',
|
||||
'CREATED_DATE',
|
||||
'LAST_UPDATE_DATE',
|
||||
] as const;
|
||||
|
||||
const MEDIA_ART_COPY_COLUMNS = [
|
||||
'anilist_id',
|
||||
'cover_url',
|
||||
'cover_blob',
|
||||
'cover_blob_hash',
|
||||
'title_romaji',
|
||||
'title_english',
|
||||
'episodes_total',
|
||||
'fetched_at_ms',
|
||||
'CREATED_DATE',
|
||||
'LAST_UPDATE_DATE',
|
||||
] as const;
|
||||
|
||||
const YOUTUBE_COPY_COLUMNS = [
|
||||
'youtube_video_id',
|
||||
'video_url',
|
||||
'video_title',
|
||||
'video_thumbnail_url',
|
||||
'channel_id',
|
||||
'channel_name',
|
||||
'channel_url',
|
||||
'channel_thumbnail_url',
|
||||
'uploader_id',
|
||||
'uploader_url',
|
||||
'description',
|
||||
'metadata_json',
|
||||
'fetched_at_ms',
|
||||
'CREATED_DATE',
|
||||
'LAST_UPDATE_DATE',
|
||||
] as const;
|
||||
|
||||
const WORD_COPY_COLUMNS = [
|
||||
'headword',
|
||||
'word',
|
||||
'reading',
|
||||
'part_of_speech',
|
||||
'pos1',
|
||||
'pos2',
|
||||
'pos3',
|
||||
'first_seen',
|
||||
'last_seen',
|
||||
'frequency',
|
||||
'frequency_rank',
|
||||
] as const;
|
||||
|
||||
type SqlRow = Record<string, unknown>;
|
||||
|
||||
function selectAll(db: Database, sql: string, params: unknown[] = []): SqlRow[] {
|
||||
return db.query<SqlRow>(sql).all(...params);
|
||||
}
|
||||
|
||||
export function mergeAnime(
|
||||
local: Database,
|
||||
remote: Database,
|
||||
summary: SyncMergeSummary,
|
||||
): Map<number, number> {
|
||||
const map = new Map<number, number>();
|
||||
const byAnilist = local.prepare<SqlRow>('SELECT anime_id FROM imm_anime WHERE anilist_id = ?');
|
||||
const byTitleKey = local.prepare<SqlRow>(
|
||||
'SELECT anime_id FROM imm_anime WHERE normalized_title_key = ?',
|
||||
);
|
||||
const fillMissing = local.prepare(
|
||||
`UPDATE imm_anime
|
||||
SET
|
||||
title_romaji = COALESCE(title_romaji, ?),
|
||||
title_english = COALESCE(title_english, ?),
|
||||
title_native = COALESCE(title_native, ?),
|
||||
episodes_total = COALESCE(episodes_total, ?),
|
||||
description = COALESCE(description, ?)
|
||||
WHERE anime_id = ?`,
|
||||
);
|
||||
|
||||
for (const row of selectAll(
|
||||
remote,
|
||||
`SELECT anime_id, ${ANIME_COPY_COLUMNS.join(', ')} FROM imm_anime`,
|
||||
)) {
|
||||
const remoteId = Number(row.anime_id);
|
||||
const existing =
|
||||
(row.anilist_id !== null ? byAnilist.get(row.anilist_id) : undefined) ??
|
||||
byTitleKey.get(row.normalized_title_key);
|
||||
if (existing) {
|
||||
const localId = Number(existing.anime_id);
|
||||
map.set(remoteId, localId);
|
||||
fillMissing.run(
|
||||
row.title_romaji,
|
||||
row.title_english,
|
||||
row.title_native,
|
||||
row.episodes_total,
|
||||
row.description,
|
||||
localId,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// No local row matched by anilist_id (checked first in `existing` above)
|
||||
// or title key, so the remote anilist_id — if any — is free to insert as-is.
|
||||
const values = ANIME_COPY_COLUMNS.map((column) => row[column]);
|
||||
map.set(remoteId, insertRow(local, 'imm_anime', ANIME_COPY_COLUMNS, values));
|
||||
summary.animeAdded += 1;
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
export interface VideoMergeResult {
|
||||
videoIdMap: Map<number, number>;
|
||||
addedVideoIds: Set<number>;
|
||||
}
|
||||
|
||||
export function mergeVideos(
|
||||
local: Database,
|
||||
remote: Database,
|
||||
animeIdMap: Map<number, number>,
|
||||
summary: SyncMergeSummary,
|
||||
): VideoMergeResult {
|
||||
const videoIdMap = new Map<number, number>();
|
||||
const addedVideoIds = new Set<number>();
|
||||
const byKey = local.prepare<SqlRow>(
|
||||
'SELECT video_id, watched FROM imm_videos WHERE video_key = ?',
|
||||
);
|
||||
const setWatched = local.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?');
|
||||
|
||||
for (const row of selectAll(
|
||||
remote,
|
||||
`SELECT video_id, anime_id, ${VIDEO_COPY_COLUMNS.join(', ')} FROM imm_videos`,
|
||||
)) {
|
||||
const remoteId = Number(row.video_id);
|
||||
const mappedAnimeId =
|
||||
row.anime_id === null ? null : (animeIdMap.get(Number(row.anime_id)) ?? null);
|
||||
const existing = byKey.get(row.video_key);
|
||||
if (existing) {
|
||||
const localId = Number(existing.video_id);
|
||||
videoIdMap.set(remoteId, localId);
|
||||
if (Number(row.watched) > 0 && Number(existing.watched) <= 0) {
|
||||
setWatched.run(localId);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const columns = ['anime_id', ...VIDEO_COPY_COLUMNS];
|
||||
const values = [mappedAnimeId, ...VIDEO_COPY_COLUMNS.map((column) => row[column])];
|
||||
const localId = insertRow(local, 'imm_videos', columns, values);
|
||||
videoIdMap.set(remoteId, localId);
|
||||
addedVideoIds.add(remoteId);
|
||||
summary.videosAdded += 1;
|
||||
}
|
||||
return { videoIdMap, addedVideoIds };
|
||||
}
|
||||
|
||||
export function mergeMediaMetadata(
|
||||
local: Database,
|
||||
remote: Database,
|
||||
videoIdMap: Map<number, number>,
|
||||
addedVideoIds: Set<number>,
|
||||
): void {
|
||||
if (videoIdMap.size === 0) return;
|
||||
const metadataVideoIds = new Set<number>([...addedVideoIds, ...videoIdMap.keys()]);
|
||||
|
||||
const hasBlobStore =
|
||||
tableExists(local, 'imm_cover_art_blobs') && tableExists(remote, 'imm_cover_art_blobs');
|
||||
const copyBlob = hasBlobStore
|
||||
? local.prepare(
|
||||
`INSERT INTO imm_cover_art_blobs (blob_hash, cover_blob, CREATED_DATE, LAST_UPDATE_DATE)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(blob_hash) DO NOTHING`,
|
||||
)
|
||||
: null;
|
||||
const readBlob = hasBlobStore
|
||||
? remote.prepare<SqlRow>('SELECT * FROM imm_cover_art_blobs WHERE blob_hash = ?')
|
||||
: null;
|
||||
|
||||
if (tableExists(remote, 'imm_media_art') && tableExists(local, 'imm_media_art')) {
|
||||
const localArtExists = local.prepare<SqlRow>(
|
||||
'SELECT 1 FROM imm_media_art WHERE video_id = ? LIMIT 1',
|
||||
);
|
||||
for (const remoteVideoId of metadataVideoIds) {
|
||||
const localVideoId = videoIdMap.get(remoteVideoId)!;
|
||||
if (localArtExists.get(localVideoId)) continue;
|
||||
const row = remote
|
||||
.query<SqlRow>(
|
||||
`SELECT ${MEDIA_ART_COPY_COLUMNS.join(', ')} FROM imm_media_art WHERE video_id = ?`,
|
||||
)
|
||||
.get(remoteVideoId);
|
||||
if (!row) continue;
|
||||
if (row.cover_blob_hash && copyBlob && readBlob) {
|
||||
const blob = readBlob.get(row.cover_blob_hash);
|
||||
if (blob) {
|
||||
copyBlob.run(blob.blob_hash, blob.cover_blob, blob.CREATED_DATE, blob.LAST_UPDATE_DATE);
|
||||
}
|
||||
}
|
||||
insertRow(
|
||||
local,
|
||||
'imm_media_art',
|
||||
['video_id', ...MEDIA_ART_COPY_COLUMNS],
|
||||
[localVideoId, ...MEDIA_ART_COPY_COLUMNS.map((column) => row[column])],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (tableExists(remote, 'imm_youtube_videos') && tableExists(local, 'imm_youtube_videos')) {
|
||||
const localYoutubeExists = local.prepare<SqlRow>(
|
||||
'SELECT 1 FROM imm_youtube_videos WHERE video_id = ? LIMIT 1',
|
||||
);
|
||||
for (const remoteVideoId of metadataVideoIds) {
|
||||
const localVideoId = videoIdMap.get(remoteVideoId)!;
|
||||
if (localYoutubeExists.get(localVideoId)) continue;
|
||||
const row = remote
|
||||
.query<SqlRow>(
|
||||
`SELECT ${YOUTUBE_COPY_COLUMNS.join(', ')} FROM imm_youtube_videos WHERE video_id = ?`,
|
||||
)
|
||||
.get(remoteVideoId);
|
||||
if (!row) continue;
|
||||
insertRow(
|
||||
local,
|
||||
'imm_youtube_videos',
|
||||
['video_id', ...YOUTUBE_COPY_COLUMNS],
|
||||
[localVideoId, ...YOUTUBE_COPY_COLUMNS.map((column) => row[column])],
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function mergeExcludedWords(
|
||||
local: Database,
|
||||
remote: Database,
|
||||
summary: SyncMergeSummary,
|
||||
): void {
|
||||
if (
|
||||
!tableExists(remote, 'imm_stats_excluded_words') ||
|
||||
!tableExists(local, 'imm_stats_excluded_words')
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const insert = local.prepare(
|
||||
`INSERT INTO imm_stats_excluded_words (headword, word, reading, CREATED_DATE, LAST_UPDATE_DATE)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(headword, word, reading) DO NOTHING`,
|
||||
);
|
||||
for (const row of selectAll(
|
||||
remote,
|
||||
'SELECT headword, word, reading, CREATED_DATE, LAST_UPDATE_DATE FROM imm_stats_excluded_words',
|
||||
)) {
|
||||
const result = insert.run(
|
||||
row.headword,
|
||||
row.word,
|
||||
row.reading,
|
||||
row.CREATED_DATE,
|
||||
row.LAST_UPDATE_DATE,
|
||||
);
|
||||
summary.excludedWordsAdded += result.changes;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazily maps remote imm_words / imm_kanji ids onto local rows by natural key
|
||||
* ((headword, word, reading) / kanji). New rows are copied with the remote's
|
||||
* accumulated frequency; rows that already exist locally get their frequency
|
||||
* incremented later with only the occurrence counts this merge adds (the
|
||||
* remote total would double-count lines merged in earlier syncs).
|
||||
*/
|
||||
export class LexiconResolver {
|
||||
private readonly wordMap = new Map<number, { localId: number; isNew: boolean }>();
|
||||
private readonly kanjiMap = new Map<number, { localId: number; isNew: boolean }>();
|
||||
readonly wordFrequencyDeltas = new Map<number, number>();
|
||||
readonly kanjiFrequencyDeltas = new Map<number, number>();
|
||||
|
||||
constructor(
|
||||
private readonly local: Database,
|
||||
private readonly remote: Database,
|
||||
private readonly summary: SyncMergeSummary,
|
||||
) {}
|
||||
|
||||
resolveWord(remoteWordId: number): number {
|
||||
const cached = this.wordMap.get(remoteWordId);
|
||||
if (cached) return cached.localId;
|
||||
|
||||
const row = this.remote
|
||||
.query<SqlRow>(`SELECT ${WORD_COPY_COLUMNS.join(', ')} FROM imm_words WHERE id = ?`)
|
||||
.get(remoteWordId);
|
||||
if (!row) throw new Error(`Snapshot references missing imm_words row ${remoteWordId}`);
|
||||
|
||||
const existing = this.local
|
||||
.query<SqlRow>('SELECT id FROM imm_words WHERE headword IS ? AND word IS ? AND reading IS ?')
|
||||
.get(row.headword, row.word, row.reading);
|
||||
let entry: { localId: number; isNew: boolean };
|
||||
if (existing) {
|
||||
entry = { localId: Number(existing.id), isNew: false };
|
||||
this.local
|
||||
.prepare(
|
||||
`UPDATE imm_words
|
||||
SET first_seen = MIN(COALESCE(first_seen, ?), COALESCE(?, first_seen)),
|
||||
last_seen = MAX(COALESCE(last_seen, ?), COALESCE(?, last_seen))
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(row.first_seen, row.first_seen, row.last_seen, row.last_seen, entry.localId);
|
||||
} else {
|
||||
const localId = insertRow(
|
||||
this.local,
|
||||
'imm_words',
|
||||
WORD_COPY_COLUMNS,
|
||||
WORD_COPY_COLUMNS.map((column) => row[column]),
|
||||
);
|
||||
entry = { localId, isNew: true };
|
||||
this.summary.wordsAdded += 1;
|
||||
}
|
||||
this.wordMap.set(remoteWordId, entry);
|
||||
return entry.localId;
|
||||
}
|
||||
|
||||
resolveKanji(remoteKanjiId: number): number {
|
||||
const cached = this.kanjiMap.get(remoteKanjiId);
|
||||
if (cached) return cached.localId;
|
||||
|
||||
const row = this.remote
|
||||
.query<SqlRow>('SELECT kanji, first_seen, last_seen, frequency FROM imm_kanji WHERE id = ?')
|
||||
.get(remoteKanjiId);
|
||||
if (!row) throw new Error(`Snapshot references missing imm_kanji row ${remoteKanjiId}`);
|
||||
|
||||
const existing = this.local
|
||||
.query<SqlRow>('SELECT id FROM imm_kanji WHERE kanji IS ?')
|
||||
.get(row.kanji);
|
||||
let entry: { localId: number; isNew: boolean };
|
||||
if (existing) {
|
||||
entry = { localId: Number(existing.id), isNew: false };
|
||||
this.local
|
||||
.prepare(
|
||||
`UPDATE imm_kanji
|
||||
SET first_seen = MIN(COALESCE(first_seen, ?), COALESCE(?, first_seen)),
|
||||
last_seen = MAX(COALESCE(last_seen, ?), COALESCE(?, last_seen))
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(row.first_seen, row.first_seen, row.last_seen, row.last_seen, entry.localId);
|
||||
} else {
|
||||
const localId = insertRow(
|
||||
this.local,
|
||||
'imm_kanji',
|
||||
['kanji', 'first_seen', 'last_seen', 'frequency'],
|
||||
[row.kanji, row.first_seen, row.last_seen, row.frequency],
|
||||
);
|
||||
entry = { localId, isNew: true };
|
||||
this.summary.kanjiAdded += 1;
|
||||
}
|
||||
this.kanjiMap.set(remoteKanjiId, entry);
|
||||
return entry.localId;
|
||||
}
|
||||
|
||||
addWordOccurrences(remoteWordId: number, count: number): void {
|
||||
const entry = this.wordMap.get(remoteWordId);
|
||||
if (!entry || entry.isNew) return;
|
||||
this.wordFrequencyDeltas.set(
|
||||
entry.localId,
|
||||
(this.wordFrequencyDeltas.get(entry.localId) ?? 0) + count,
|
||||
);
|
||||
}
|
||||
|
||||
addKanjiOccurrences(remoteKanjiId: number, count: number): void {
|
||||
const entry = this.kanjiMap.get(remoteKanjiId);
|
||||
if (!entry || entry.isNew) return;
|
||||
this.kanjiFrequencyDeltas.set(
|
||||
entry.localId,
|
||||
(this.kanjiFrequencyDeltas.get(entry.localId) ?? 0) + count,
|
||||
);
|
||||
}
|
||||
|
||||
applyFrequencyDeltas(): void {
|
||||
const updateWord = this.local.prepare(
|
||||
'UPDATE imm_words SET frequency = COALESCE(frequency, 0) + ? WHERE id = ?',
|
||||
);
|
||||
for (const [localId, delta] of this.wordFrequencyDeltas) {
|
||||
updateWord.run(delta, localId);
|
||||
}
|
||||
const updateKanji = this.local.prepare(
|
||||
'UPDATE imm_kanji SET frequency = COALESCE(frequency, 0) + ? WHERE id = ?',
|
||||
);
|
||||
for (const [localId, delta] of this.kanjiFrequencyDeltas) {
|
||||
updateKanji.run(delta, localId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,265 +0,0 @@
|
||||
import { Database } from 'bun:sqlite';
|
||||
import { nowDbTimestamp, tableExists, type SyncMergeSummary } from './sync-shared.js';
|
||||
|
||||
type SqlRow = Record<string, unknown>;
|
||||
|
||||
const LOCAL_DAY_EXPR = `CAST(julianday(CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER)`;
|
||||
const LOCAL_MONTH_EXPR = `CAST(strftime('%Y%m', CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER)`;
|
||||
|
||||
// Ported from upsertDailyRollupsForGroups / upsertMonthlyRollupsForGroups in
|
||||
// src/core/services/immersion-tracker/maintenance.ts — must stay in sync.
|
||||
const DAILY_ROLLUP_UPSERT = `
|
||||
WITH matching_sessions AS (
|
||||
SELECT * FROM imm_sessions
|
||||
WHERE ${LOCAL_DAY_EXPR} = ? AND video_id = ?
|
||||
),
|
||||
session_metrics AS (
|
||||
SELECT
|
||||
t.session_id,
|
||||
MAX(t.active_watched_ms) AS max_active_ms,
|
||||
MAX(t.lines_seen) AS max_lines,
|
||||
MAX(t.tokens_seen) AS max_tokens,
|
||||
MAX(t.cards_mined) AS max_cards,
|
||||
MAX(t.lookup_count) AS max_lookups,
|
||||
MAX(t.lookup_hits) AS max_hits
|
||||
FROM imm_session_telemetry t
|
||||
JOIN matching_sessions s ON s.session_id = t.session_id
|
||||
GROUP BY t.session_id
|
||||
)
|
||||
INSERT INTO imm_daily_rollups (
|
||||
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards, cards_per_hour, tokens_per_min, lookup_hit_rate,
|
||||
CREATED_DATE, LAST_UPDATE_DATE
|
||||
)
|
||||
SELECT
|
||||
${LOCAL_DAY_EXPR.replace('started_at_ms', 's.started_at_ms')} AS rollup_day,
|
||||
s.video_id AS video_id,
|
||||
COUNT(DISTINCT s.session_id) AS total_sessions,
|
||||
COALESCE(SUM(COALESCE(sm.max_active_ms, s.active_watched_ms)), 0) / 60000.0 AS total_active_min,
|
||||
COALESCE(SUM(COALESCE(sm.max_lines, s.lines_seen)), 0) AS total_lines_seen,
|
||||
COALESCE(SUM(COALESCE(sm.max_tokens, s.tokens_seen)), 0) AS total_tokens_seen,
|
||||
COALESCE(SUM(COALESCE(sm.max_cards, s.cards_mined)), 0) AS total_cards,
|
||||
CASE
|
||||
WHEN COALESCE(SUM(COALESCE(sm.max_active_ms, s.active_watched_ms)), 0) > 0
|
||||
THEN (COALESCE(SUM(COALESCE(sm.max_cards, s.cards_mined)), 0) * 60.0)
|
||||
/ (COALESCE(SUM(COALESCE(sm.max_active_ms, s.active_watched_ms)), 0) / 60000.0)
|
||||
ELSE NULL
|
||||
END AS cards_per_hour,
|
||||
CASE
|
||||
WHEN COALESCE(SUM(COALESCE(sm.max_active_ms, s.active_watched_ms)), 0) > 0
|
||||
THEN COALESCE(SUM(COALESCE(sm.max_tokens, s.tokens_seen)), 0)
|
||||
/ (COALESCE(SUM(COALESCE(sm.max_active_ms, s.active_watched_ms)), 0) / 60000.0)
|
||||
ELSE NULL
|
||||
END AS tokens_per_min,
|
||||
CASE
|
||||
WHEN COALESCE(SUM(COALESCE(sm.max_lookups, s.lookup_count)), 0) > 0
|
||||
THEN CAST(COALESCE(SUM(COALESCE(sm.max_hits, s.lookup_hits)), 0) AS REAL)
|
||||
/ CAST(COALESCE(SUM(COALESCE(sm.max_lookups, s.lookup_count)), 0) AS REAL)
|
||||
ELSE NULL
|
||||
END AS lookup_hit_rate,
|
||||
? AS CREATED_DATE,
|
||||
? AS LAST_UPDATE_DATE
|
||||
FROM matching_sessions s
|
||||
LEFT JOIN session_metrics sm ON s.session_id = sm.session_id
|
||||
GROUP BY rollup_day, s.video_id
|
||||
ON CONFLICT (rollup_day, video_id) DO UPDATE SET
|
||||
total_sessions = excluded.total_sessions,
|
||||
total_active_min = excluded.total_active_min,
|
||||
total_lines_seen = excluded.total_lines_seen,
|
||||
total_tokens_seen = excluded.total_tokens_seen,
|
||||
total_cards = excluded.total_cards,
|
||||
cards_per_hour = excluded.cards_per_hour,
|
||||
tokens_per_min = excluded.tokens_per_min,
|
||||
lookup_hit_rate = excluded.lookup_hit_rate,
|
||||
CREATED_DATE = COALESCE(imm_daily_rollups.CREATED_DATE, excluded.CREATED_DATE),
|
||||
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
|
||||
`;
|
||||
|
||||
const MONTHLY_ROLLUP_UPSERT = `
|
||||
WITH matching_sessions AS (
|
||||
SELECT * FROM imm_sessions
|
||||
WHERE ${LOCAL_MONTH_EXPR} = ? AND video_id = ?
|
||||
),
|
||||
session_metrics AS (
|
||||
SELECT
|
||||
t.session_id,
|
||||
MAX(t.active_watched_ms) AS max_active_ms,
|
||||
MAX(t.lines_seen) AS max_lines,
|
||||
MAX(t.tokens_seen) AS max_tokens,
|
||||
MAX(t.cards_mined) AS max_cards
|
||||
FROM imm_session_telemetry t
|
||||
JOIN matching_sessions s ON s.session_id = t.session_id
|
||||
GROUP BY t.session_id
|
||||
)
|
||||
INSERT INTO imm_monthly_rollups (
|
||||
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
|
||||
)
|
||||
SELECT
|
||||
${LOCAL_MONTH_EXPR.replace('started_at_ms', 's.started_at_ms')} AS rollup_month,
|
||||
s.video_id AS video_id,
|
||||
COUNT(DISTINCT s.session_id) AS total_sessions,
|
||||
COALESCE(SUM(COALESCE(sm.max_active_ms, s.active_watched_ms)), 0) / 60000.0 AS total_active_min,
|
||||
COALESCE(SUM(COALESCE(sm.max_lines, s.lines_seen)), 0) AS total_lines_seen,
|
||||
COALESCE(SUM(COALESCE(sm.max_tokens, s.tokens_seen)), 0) AS total_tokens_seen,
|
||||
COALESCE(SUM(COALESCE(sm.max_cards, s.cards_mined)), 0) AS total_cards,
|
||||
? AS CREATED_DATE,
|
||||
? AS LAST_UPDATE_DATE
|
||||
FROM matching_sessions s
|
||||
LEFT JOIN session_metrics sm ON s.session_id = sm.session_id
|
||||
GROUP BY rollup_month, s.video_id
|
||||
ON CONFLICT (rollup_month, video_id) DO UPDATE SET
|
||||
total_sessions = excluded.total_sessions,
|
||||
total_active_min = excluded.total_active_min,
|
||||
total_lines_seen = excluded.total_lines_seen,
|
||||
total_tokens_seen = excluded.total_tokens_seen,
|
||||
total_cards = excluded.total_cards,
|
||||
CREATED_DATE = COALESCE(imm_monthly_rollups.CREATED_DATE, excluded.CREATED_DATE),
|
||||
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
|
||||
`;
|
||||
|
||||
/**
|
||||
* Recompute daily/monthly rollup groups touched by the newly merged sessions
|
||||
* from the (now merged) local session + telemetry data. The maintenance
|
||||
* watermark is left alone: telemetry newer than it gets recomputed again by
|
||||
* the app later, which is idempotent.
|
||||
*/
|
||||
export function refreshRollupsForNewSessions(
|
||||
local: Database,
|
||||
newSessionIds: number[],
|
||||
summary: SyncMergeSummary,
|
||||
): void {
|
||||
if (newSessionIds.length === 0) return;
|
||||
|
||||
const groups = new Map<string, { day: number; month: number; videoId: number }>();
|
||||
for (let offset = 0; offset < newSessionIds.length; offset += 500) {
|
||||
const chunk = newSessionIds.slice(offset, offset + 500);
|
||||
const rows = local
|
||||
.query<SqlRow>(
|
||||
`SELECT DISTINCT ${LOCAL_DAY_EXPR} AS rollup_day, ${LOCAL_MONTH_EXPR} AS rollup_month, video_id
|
||||
FROM imm_sessions WHERE session_id IN (${chunk.map(() => '?').join(',')})`,
|
||||
)
|
||||
.all(...chunk);
|
||||
for (const row of rows) {
|
||||
const day = Number(row.rollup_day);
|
||||
const month = Number(row.rollup_month);
|
||||
const videoId = Number(row.video_id);
|
||||
groups.set(`${day}-${videoId}`, { day, month, videoId });
|
||||
}
|
||||
}
|
||||
|
||||
const stampMs = nowDbTimestamp();
|
||||
const deleteDaily = local.prepare(
|
||||
'DELETE FROM imm_daily_rollups WHERE rollup_day = ? AND video_id = ?',
|
||||
);
|
||||
const deleteMonthly = local.prepare(
|
||||
'DELETE FROM imm_monthly_rollups WHERE rollup_month = ? AND video_id = ?',
|
||||
);
|
||||
const upsertDaily = local.prepare(DAILY_ROLLUP_UPSERT);
|
||||
const upsertMonthly = local.prepare(MONTHLY_ROLLUP_UPSERT);
|
||||
|
||||
const monthlyGroups = new Set<string>();
|
||||
for (const { day, month, videoId } of groups.values()) {
|
||||
deleteDaily.run(day, videoId);
|
||||
upsertDaily.run(day, videoId, stampMs, stampMs);
|
||||
summary.rollupGroupsRecomputed += 1;
|
||||
const monthKey = `${month}-${videoId}`;
|
||||
if (!monthlyGroups.has(monthKey)) {
|
||||
monthlyGroups.add(monthKey);
|
||||
deleteMonthly.run(month, videoId);
|
||||
upsertMonthly.run(month, videoId, stampMs, stampMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sessions are pruned after a retention window, but rollups are kept much
|
||||
* longer — the remote's older rollup history can't be reconstructed from
|
||||
* merged sessions. Copy remote rollup rows for groups where the local DB has
|
||||
* neither a rollup row nor any sessions (i.e. history only the remote knows).
|
||||
* Groups both machines have data for are never summed, to avoid
|
||||
* double-counting sessions that earlier syncs already shared.
|
||||
*/
|
||||
export function copyRemoteOnlyRollups(
|
||||
local: Database,
|
||||
remote: Database,
|
||||
videoIdMap: Map<number, number>,
|
||||
summary: SyncMergeSummary,
|
||||
): void {
|
||||
if (!tableExists(remote, 'imm_daily_rollups') || !tableExists(local, 'imm_daily_rollups')) return;
|
||||
|
||||
const localDailyExists = local.prepare(
|
||||
'SELECT 1 FROM imm_daily_rollups WHERE rollup_day = ? AND video_id = ? LIMIT 1',
|
||||
);
|
||||
const localDaySessions = local.prepare(
|
||||
`SELECT 1 FROM imm_sessions WHERE video_id = ? AND ${LOCAL_DAY_EXPR} = ? LIMIT 1`,
|
||||
);
|
||||
const localMonthSessions = local.prepare(
|
||||
`SELECT 1 FROM imm_sessions WHERE video_id = ? AND ${LOCAL_MONTH_EXPR} = ? LIMIT 1`,
|
||||
);
|
||||
const localMonthSessionsForDay = local.prepare(
|
||||
`SELECT 1 FROM imm_sessions
|
||||
WHERE video_id = ?
|
||||
AND ${LOCAL_MONTH_EXPR} = CAST(strftime('%Y%m', CAST(? AS INTEGER) * 86400, 'unixepoch', 'localtime') AS INTEGER)
|
||||
LIMIT 1`,
|
||||
);
|
||||
const insertDaily = local.prepare(
|
||||
`INSERT INTO imm_daily_rollups (
|
||||
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards, cards_per_hour, tokens_per_min, lookup_hit_rate,
|
||||
CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
for (const row of remote.query<SqlRow>('SELECT * FROM imm_daily_rollups').all()) {
|
||||
if (row.video_id === null) continue;
|
||||
const localVideoId = videoIdMap.get(Number(row.video_id));
|
||||
if (localVideoId === undefined) continue;
|
||||
if (localDailyExists.get(row.rollup_day, localVideoId)) continue;
|
||||
if (localDaySessions.get(localVideoId, row.rollup_day)) continue;
|
||||
if (localMonthSessionsForDay.get(localVideoId, row.rollup_day)) continue;
|
||||
insertDaily.run(
|
||||
row.rollup_day,
|
||||
localVideoId,
|
||||
row.total_sessions,
|
||||
row.total_active_min,
|
||||
row.total_lines_seen,
|
||||
row.total_tokens_seen,
|
||||
row.total_cards,
|
||||
row.cards_per_hour,
|
||||
row.tokens_per_min,
|
||||
row.lookup_hit_rate,
|
||||
row.CREATED_DATE,
|
||||
row.LAST_UPDATE_DATE,
|
||||
);
|
||||
summary.dailyRollupsCopied += 1;
|
||||
}
|
||||
|
||||
const localMonthlyExists = local.prepare(
|
||||
'SELECT 1 FROM imm_monthly_rollups WHERE rollup_month = ? AND video_id = ? LIMIT 1',
|
||||
);
|
||||
const insertMonthly = local.prepare(
|
||||
`INSERT INTO imm_monthly_rollups (
|
||||
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
for (const row of remote.query<SqlRow>('SELECT * FROM imm_monthly_rollups').all()) {
|
||||
if (row.video_id === null) continue;
|
||||
const localVideoId = videoIdMap.get(Number(row.video_id));
|
||||
if (localVideoId === undefined) continue;
|
||||
if (localMonthlyExists.get(row.rollup_month, localVideoId)) continue;
|
||||
if (localMonthSessions.get(localVideoId, row.rollup_month)) continue;
|
||||
insertMonthly.run(
|
||||
row.rollup_month,
|
||||
localVideoId,
|
||||
row.total_sessions,
|
||||
row.total_active_min,
|
||||
row.total_lines_seen,
|
||||
row.total_tokens_seen,
|
||||
row.total_cards,
|
||||
row.CREATED_DATE,
|
||||
row.LAST_UPDATE_DATE,
|
||||
);
|
||||
summary.monthlyRollupsCopied += 1;
|
||||
}
|
||||
}
|
||||
@@ -1,465 +0,0 @@
|
||||
import { Database } from 'bun:sqlite';
|
||||
import type { LexiconResolver } from './merge-catalog.js';
|
||||
import { insertRow, nowDbTimestamp, type SyncMergeSummary } from './sync-shared.js';
|
||||
|
||||
const SESSION_COPY_COLUMNS = [
|
||||
'session_uuid',
|
||||
'started_at_ms',
|
||||
'ended_at_ms',
|
||||
'status',
|
||||
'locale_id',
|
||||
'target_lang_id',
|
||||
'difficulty_tier',
|
||||
'subtitle_mode',
|
||||
'ended_media_ms',
|
||||
'total_watched_ms',
|
||||
'active_watched_ms',
|
||||
'lines_seen',
|
||||
'tokens_seen',
|
||||
'cards_mined',
|
||||
'lookup_count',
|
||||
'lookup_hits',
|
||||
'yomitan_lookup_count',
|
||||
'pause_count',
|
||||
'pause_ms',
|
||||
'seek_forward_count',
|
||||
'seek_backward_count',
|
||||
'media_buffer_events',
|
||||
'CREATED_DATE',
|
||||
'LAST_UPDATE_DATE',
|
||||
] as const;
|
||||
|
||||
const TELEMETRY_COPY_COLUMNS = [
|
||||
'sample_ms',
|
||||
'total_watched_ms',
|
||||
'active_watched_ms',
|
||||
'lines_seen',
|
||||
'tokens_seen',
|
||||
'cards_mined',
|
||||
'lookup_count',
|
||||
'lookup_hits',
|
||||
'yomitan_lookup_count',
|
||||
'pause_count',
|
||||
'pause_ms',
|
||||
'seek_forward_count',
|
||||
'seek_backward_count',
|
||||
'media_buffer_events',
|
||||
'CREATED_DATE',
|
||||
'LAST_UPDATE_DATE',
|
||||
] as const;
|
||||
|
||||
const EVENT_COPY_COLUMNS = [
|
||||
'ts_ms',
|
||||
'event_type',
|
||||
'line_index',
|
||||
'segment_start_ms',
|
||||
'segment_end_ms',
|
||||
'tokens_delta',
|
||||
'cards_delta',
|
||||
'payload_json',
|
||||
'CREATED_DATE',
|
||||
'LAST_UPDATE_DATE',
|
||||
] as const;
|
||||
|
||||
const LINE_COPY_COLUMNS = [
|
||||
'line_index',
|
||||
'segment_start_ms',
|
||||
'segment_end_ms',
|
||||
'text',
|
||||
'secondary_text',
|
||||
'CREATED_DATE',
|
||||
'LAST_UPDATE_DATE',
|
||||
] as const;
|
||||
|
||||
type SqlRow = Record<string, unknown>;
|
||||
|
||||
export interface SessionMergeResult {
|
||||
newSessionIds: number[];
|
||||
}
|
||||
|
||||
export function mergeSessions(
|
||||
local: Database,
|
||||
remote: Database,
|
||||
videoIdMap: Map<number, number>,
|
||||
animeIdMap: Map<number, number>,
|
||||
lexicon: LexiconResolver,
|
||||
summary: SyncMergeSummary,
|
||||
): SessionMergeResult {
|
||||
const newSessionIds: number[] = [];
|
||||
const uuidExists = local.prepare<SqlRow>(
|
||||
'SELECT session_id FROM imm_sessions WHERE session_uuid = ?',
|
||||
);
|
||||
|
||||
const remoteSessions = remote
|
||||
.query<SqlRow>(
|
||||
`SELECT session_id, video_id, ${SESSION_COPY_COLUMNS.join(', ')}
|
||||
FROM imm_sessions
|
||||
ORDER BY CAST(started_at_ms AS REAL) ASC, session_id ASC`,
|
||||
)
|
||||
.all();
|
||||
|
||||
for (const session of remoteSessions) {
|
||||
if (session.ended_at_ms === null) {
|
||||
// Stale ACTIVE sessions are finalized by the app on its next startup;
|
||||
// they will sync once they carry final numbers.
|
||||
summary.activeSessionsSkipped += 1;
|
||||
continue;
|
||||
}
|
||||
if (uuidExists.get(session.session_uuid)) {
|
||||
summary.sessionsAlreadyPresent += 1;
|
||||
continue;
|
||||
}
|
||||
const localVideoId = videoIdMap.get(Number(session.video_id));
|
||||
if (localVideoId === undefined) {
|
||||
throw new Error(`Snapshot session ${String(session.session_uuid)} references missing video row`);
|
||||
}
|
||||
|
||||
const localSessionId = insertRow(
|
||||
local,
|
||||
'imm_sessions',
|
||||
['video_id', ...SESSION_COPY_COLUMNS],
|
||||
[localVideoId, ...SESSION_COPY_COLUMNS.map((column) => session[column])],
|
||||
);
|
||||
newSessionIds.push(localSessionId);
|
||||
summary.sessionsMerged += 1;
|
||||
|
||||
const remoteSessionId = Number(session.session_id);
|
||||
copyTelemetry(local, remote, remoteSessionId, localSessionId, summary);
|
||||
const eventIdMap = copyEvents(local, remote, remoteSessionId, localSessionId, summary);
|
||||
copySubtitleLines(
|
||||
local,
|
||||
remote,
|
||||
remoteSessionId,
|
||||
localSessionId,
|
||||
localVideoId,
|
||||
animeIdMap,
|
||||
eventIdMap,
|
||||
lexicon,
|
||||
summary,
|
||||
);
|
||||
applyMergedSessionLifetime(local, localSessionId, localVideoId, session);
|
||||
}
|
||||
|
||||
return { newSessionIds };
|
||||
}
|
||||
|
||||
function copyTelemetry(
|
||||
local: Database,
|
||||
remote: Database,
|
||||
remoteSessionId: number,
|
||||
localSessionId: number,
|
||||
summary: SyncMergeSummary,
|
||||
): void {
|
||||
const rows = remote
|
||||
.query<SqlRow>(
|
||||
`SELECT ${TELEMETRY_COPY_COLUMNS.join(', ')} FROM imm_session_telemetry
|
||||
WHERE session_id = ? ORDER BY telemetry_id ASC`,
|
||||
)
|
||||
.all(remoteSessionId);
|
||||
for (const row of rows) {
|
||||
insertRow(
|
||||
local,
|
||||
'imm_session_telemetry',
|
||||
['session_id', ...TELEMETRY_COPY_COLUMNS],
|
||||
[localSessionId, ...TELEMETRY_COPY_COLUMNS.map((column) => row[column])],
|
||||
);
|
||||
summary.telemetryRowsAdded += 1;
|
||||
}
|
||||
}
|
||||
|
||||
function copyEvents(
|
||||
local: Database,
|
||||
remote: Database,
|
||||
remoteSessionId: number,
|
||||
localSessionId: number,
|
||||
summary: SyncMergeSummary,
|
||||
): Map<number, number> {
|
||||
const eventIdMap = new Map<number, number>();
|
||||
const rows = remote
|
||||
.query<SqlRow>(
|
||||
`SELECT event_id, ${EVENT_COPY_COLUMNS.join(', ')} FROM imm_session_events
|
||||
WHERE session_id = ? ORDER BY event_id ASC`,
|
||||
)
|
||||
.all(remoteSessionId);
|
||||
for (const row of rows) {
|
||||
const localEventId = insertRow(
|
||||
local,
|
||||
'imm_session_events',
|
||||
['session_id', ...EVENT_COPY_COLUMNS],
|
||||
[localSessionId, ...EVENT_COPY_COLUMNS.map((column) => row[column])],
|
||||
);
|
||||
eventIdMap.set(Number(row.event_id), localEventId);
|
||||
summary.eventsAdded += 1;
|
||||
}
|
||||
return eventIdMap;
|
||||
}
|
||||
|
||||
function copySubtitleLines(
|
||||
local: Database,
|
||||
remote: Database,
|
||||
remoteSessionId: number,
|
||||
localSessionId: number,
|
||||
localVideoId: number,
|
||||
animeIdMap: Map<number, number>,
|
||||
eventIdMap: Map<number, number>,
|
||||
lexicon: LexiconResolver,
|
||||
summary: SyncMergeSummary,
|
||||
): void {
|
||||
const rows = remote
|
||||
.query<SqlRow>(
|
||||
`SELECT line_id, event_id, anime_id, ${LINE_COPY_COLUMNS.join(', ')} FROM imm_subtitle_lines
|
||||
WHERE session_id = ? ORDER BY line_id ASC`,
|
||||
)
|
||||
.all(remoteSessionId);
|
||||
const wordOccurrences = remote.prepare<SqlRow>(
|
||||
'SELECT word_id, occurrence_count FROM imm_word_line_occurrences WHERE line_id = ?',
|
||||
);
|
||||
const kanjiOccurrences = remote.prepare<SqlRow>(
|
||||
'SELECT kanji_id, occurrence_count FROM imm_kanji_line_occurrences WHERE line_id = ?',
|
||||
);
|
||||
const insertWordOccurrence = local.prepare(
|
||||
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count) VALUES (?, ?, ?)
|
||||
ON CONFLICT(line_id, word_id) DO UPDATE SET occurrence_count = occurrence_count + excluded.occurrence_count`,
|
||||
);
|
||||
const insertKanjiOccurrence = local.prepare(
|
||||
`INSERT INTO imm_kanji_line_occurrences (line_id, kanji_id, occurrence_count) VALUES (?, ?, ?)
|
||||
ON CONFLICT(line_id, kanji_id) DO UPDATE SET occurrence_count = occurrence_count + excluded.occurrence_count`,
|
||||
);
|
||||
|
||||
for (const row of rows) {
|
||||
const localEventId = row.event_id === null ? null : (eventIdMap.get(Number(row.event_id)) ?? null);
|
||||
const localAnimeId = row.anime_id === null ? null : (animeIdMap.get(Number(row.anime_id)) ?? null);
|
||||
const localLineId = insertRow(
|
||||
local,
|
||||
'imm_subtitle_lines',
|
||||
['session_id', 'event_id', 'video_id', 'anime_id', ...LINE_COPY_COLUMNS],
|
||||
[
|
||||
localSessionId,
|
||||
localEventId,
|
||||
localVideoId,
|
||||
localAnimeId,
|
||||
...LINE_COPY_COLUMNS.map((column) => row[column]),
|
||||
],
|
||||
);
|
||||
summary.subtitleLinesAdded += 1;
|
||||
|
||||
for (const occurrence of wordOccurrences.all(row.line_id)) {
|
||||
const localWordId = lexicon.resolveWord(Number(occurrence.word_id));
|
||||
const count = Number(occurrence.occurrence_count);
|
||||
insertWordOccurrence.run(localLineId, localWordId, count);
|
||||
lexicon.addWordOccurrences(Number(occurrence.word_id), count);
|
||||
}
|
||||
for (const occurrence of kanjiOccurrences.all(row.line_id)) {
|
||||
const localKanjiId = lexicon.resolveKanji(Number(occurrence.kanji_id));
|
||||
const count = Number(occurrence.occurrence_count);
|
||||
insertKanjiOccurrence.run(localLineId, localKanjiId, count);
|
||||
lexicon.addKanjiOccurrences(Number(occurrence.kanji_id), count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Port of applySessionLifetimeSummary (src/core/services/immersion-tracker/
|
||||
* lifetime.ts) for sessions arriving out of chronological order. The
|
||||
* "first session of the day / for this video" checks are order-independent
|
||||
* here (any other session counts, not just earlier ones): the local machine
|
||||
* already credited active_days/episodes_started when its own session was
|
||||
* applied, even if the merged session started earlier that day.
|
||||
*/
|
||||
function applyMergedSessionLifetime(
|
||||
local: Database,
|
||||
sessionId: number,
|
||||
videoId: number,
|
||||
session: SqlRow,
|
||||
): void {
|
||||
const updatedAtMs = nowDbTimestamp();
|
||||
const applied = local
|
||||
.prepare(
|
||||
`INSERT INTO imm_lifetime_applied_sessions (session_id, applied_at_ms, CREATED_DATE, LAST_UPDATE_DATE)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(session_id) DO NOTHING`,
|
||||
)
|
||||
.run(sessionId, session.ended_at_ms, updatedAtMs, updatedAtMs);
|
||||
if (applied.changes <= 0) return;
|
||||
|
||||
const telemetry = local
|
||||
.query<SqlRow>(
|
||||
`SELECT active_watched_ms, cards_mined, lines_seen, tokens_seen
|
||||
FROM imm_session_telemetry
|
||||
WHERE session_id = ?
|
||||
ORDER BY sample_ms DESC, telemetry_id DESC
|
||||
LIMIT 1`,
|
||||
)
|
||||
.get(sessionId);
|
||||
|
||||
const metric = (telemetryValue: unknown, sessionValue: unknown): number => {
|
||||
const fromTelemetry = telemetry ? Number(telemetryValue) : Number.NaN;
|
||||
const value = Number.isFinite(fromTelemetry) ? fromTelemetry : Number(sessionValue);
|
||||
return Math.max(0, Math.floor(Number.isFinite(value) ? value : 0));
|
||||
};
|
||||
const activeMs = metric(telemetry?.active_watched_ms, session.active_watched_ms);
|
||||
const cardsMined = metric(telemetry?.cards_mined, session.cards_mined);
|
||||
const linesSeen = metric(telemetry?.lines_seen, session.lines_seen);
|
||||
const tokensSeen = metric(telemetry?.tokens_seen, session.tokens_seen);
|
||||
|
||||
const video = local
|
||||
.query<SqlRow>('SELECT anime_id, watched FROM imm_videos WHERE video_id = ?')
|
||||
.get(videoId);
|
||||
const watched = Number(video?.watched ?? 0);
|
||||
const animeId = video?.anime_id === null || video?.anime_id === undefined ? null : Number(video.anime_id);
|
||||
|
||||
const mediaLifetime = local
|
||||
.query<SqlRow>('SELECT completed FROM imm_lifetime_media WHERE video_id = ?')
|
||||
.get(videoId);
|
||||
const hasOtherSessionForVideo = Boolean(
|
||||
local
|
||||
.query('SELECT 1 FROM imm_sessions WHERE video_id = ? AND session_id != ? LIMIT 1')
|
||||
.get(videoId, sessionId),
|
||||
);
|
||||
const isFirstSessionForVideoRun = !mediaLifetime && !hasOtherSessionForVideo;
|
||||
const isFirstCompletedSessionForVideoRun = watched > 0 && Number(mediaLifetime?.completed ?? 0) <= 0;
|
||||
|
||||
const hasOtherSessionOnDay = Boolean(
|
||||
local
|
||||
.query(
|
||||
`SELECT 1 FROM imm_sessions
|
||||
WHERE session_id != ?
|
||||
AND CAST(julianday(CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER)
|
||||
= CAST(julianday(CAST(? AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER)
|
||||
LIMIT 1`,
|
||||
)
|
||||
.get(sessionId, session.started_at_ms),
|
||||
);
|
||||
|
||||
let animeCompletedDelta = 0;
|
||||
if (animeId !== null && watched > 0 && isFirstCompletedSessionForVideoRun) {
|
||||
const animeLifetime = local
|
||||
.query<SqlRow>('SELECT episodes_completed FROM imm_lifetime_anime WHERE anime_id = ?')
|
||||
.get(animeId);
|
||||
const anime = local
|
||||
.query<SqlRow>('SELECT episodes_total FROM imm_anime WHERE anime_id = ?')
|
||||
.get(animeId);
|
||||
const episodesCompletedBefore = Number(animeLifetime?.episodes_completed ?? 0);
|
||||
const episodesTotal = anime?.episodes_total === null || anime?.episodes_total === undefined
|
||||
? null
|
||||
: Number(anime.episodes_total);
|
||||
if (
|
||||
episodesTotal !== null &&
|
||||
episodesTotal > 0 &&
|
||||
episodesCompletedBefore < episodesTotal &&
|
||||
episodesCompletedBefore + 1 >= episodesTotal
|
||||
) {
|
||||
animeCompletedDelta = 1;
|
||||
}
|
||||
}
|
||||
|
||||
local
|
||||
.prepare(
|
||||
`UPDATE imm_lifetime_global
|
||||
SET total_sessions = total_sessions + 1,
|
||||
total_active_ms = total_active_ms + ?,
|
||||
total_cards = total_cards + ?,
|
||||
active_days = active_days + ?,
|
||||
episodes_started = episodes_started + ?,
|
||||
episodes_completed = episodes_completed + ?,
|
||||
anime_completed = anime_completed + ?,
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE global_id = 1`,
|
||||
)
|
||||
.run(
|
||||
activeMs,
|
||||
cardsMined,
|
||||
hasOtherSessionOnDay ? 0 : 1,
|
||||
isFirstSessionForVideoRun ? 1 : 0,
|
||||
isFirstCompletedSessionForVideoRun ? 1 : 0,
|
||||
animeCompletedDelta,
|
||||
updatedAtMs,
|
||||
);
|
||||
|
||||
local
|
||||
.prepare(
|
||||
`INSERT INTO imm_lifetime_media(
|
||||
video_id, total_sessions, total_active_ms, total_cards, total_lines_seen,
|
||||
total_tokens_seen, completed, first_watched_ms, last_watched_ms, CREATED_DATE, LAST_UPDATE_DATE
|
||||
)
|
||||
VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(video_id) DO UPDATE SET
|
||||
total_sessions = total_sessions + 1,
|
||||
total_active_ms = total_active_ms + excluded.total_active_ms,
|
||||
total_cards = total_cards + excluded.total_cards,
|
||||
total_lines_seen = total_lines_seen + excluded.total_lines_seen,
|
||||
total_tokens_seen = total_tokens_seen + excluded.total_tokens_seen,
|
||||
completed = MAX(completed, excluded.completed),
|
||||
first_watched_ms = CASE
|
||||
WHEN excluded.first_watched_ms IS NULL THEN first_watched_ms
|
||||
WHEN first_watched_ms IS NULL THEN excluded.first_watched_ms
|
||||
WHEN excluded.first_watched_ms < first_watched_ms THEN excluded.first_watched_ms
|
||||
ELSE first_watched_ms
|
||||
END,
|
||||
last_watched_ms = CASE
|
||||
WHEN excluded.last_watched_ms IS NULL THEN last_watched_ms
|
||||
WHEN last_watched_ms IS NULL THEN excluded.last_watched_ms
|
||||
WHEN excluded.last_watched_ms > last_watched_ms THEN excluded.last_watched_ms
|
||||
ELSE last_watched_ms
|
||||
END,
|
||||
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE`,
|
||||
)
|
||||
.run(
|
||||
videoId,
|
||||
activeMs,
|
||||
cardsMined,
|
||||
linesSeen,
|
||||
tokensSeen,
|
||||
watched > 0 ? 1 : 0,
|
||||
session.started_at_ms,
|
||||
session.ended_at_ms,
|
||||
updatedAtMs,
|
||||
updatedAtMs,
|
||||
);
|
||||
|
||||
if (animeId !== null) {
|
||||
local
|
||||
.prepare(
|
||||
`INSERT INTO imm_lifetime_anime(
|
||||
anime_id, total_sessions, total_active_ms, total_cards, total_lines_seen,
|
||||
total_tokens_seen, episodes_started, episodes_completed, first_watched_ms,
|
||||
last_watched_ms, CREATED_DATE, LAST_UPDATE_DATE
|
||||
)
|
||||
VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(anime_id) DO UPDATE SET
|
||||
total_sessions = total_sessions + 1,
|
||||
total_active_ms = total_active_ms + excluded.total_active_ms,
|
||||
total_cards = total_cards + excluded.total_cards,
|
||||
total_lines_seen = total_lines_seen + excluded.total_lines_seen,
|
||||
total_tokens_seen = total_tokens_seen + excluded.total_tokens_seen,
|
||||
episodes_started = episodes_started + excluded.episodes_started,
|
||||
episodes_completed = episodes_completed + excluded.episodes_completed,
|
||||
first_watched_ms = CASE
|
||||
WHEN excluded.first_watched_ms IS NULL THEN first_watched_ms
|
||||
WHEN first_watched_ms IS NULL THEN excluded.first_watched_ms
|
||||
WHEN excluded.first_watched_ms < first_watched_ms THEN excluded.first_watched_ms
|
||||
ELSE first_watched_ms
|
||||
END,
|
||||
last_watched_ms = CASE
|
||||
WHEN excluded.last_watched_ms IS NULL THEN last_watched_ms
|
||||
WHEN last_watched_ms IS NULL THEN excluded.last_watched_ms
|
||||
WHEN excluded.last_watched_ms > last_watched_ms THEN excluded.last_watched_ms
|
||||
ELSE last_watched_ms
|
||||
END,
|
||||
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE`,
|
||||
)
|
||||
.run(
|
||||
animeId,
|
||||
activeMs,
|
||||
cardsMined,
|
||||
linesSeen,
|
||||
tokensSeen,
|
||||
isFirstSessionForVideoRun ? 1 : 0,
|
||||
isFirstCompletedSessionForVideoRun ? 1 : 0,
|
||||
session.started_at_ms,
|
||||
session.ended_at_ms,
|
||||
updatedAtMs,
|
||||
updatedAtMs,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { assertSafeSshHost, resolveRemoteSubminerCommand, runScp, shellQuote } from './ssh.js';
|
||||
|
||||
test('assertSafeSshHost rejects option-like hosts', () => {
|
||||
assert.throws(() => assertSafeSshHost('-oProxyCommand=touch pwned'), /looks like an option/);
|
||||
assert.throws(() => assertSafeSshHost('-lroot'), /looks like an option/);
|
||||
});
|
||||
|
||||
test('assertSafeSshHost accepts normal destinations', () => {
|
||||
assert.doesNotThrow(() => assertSafeSshHost('macbook'));
|
||||
assert.doesNotThrow(() => assertSafeSshHost('user@192.168.1.20'));
|
||||
assert.doesNotThrow(() => assertSafeSshHost('ssh-alias'));
|
||||
});
|
||||
|
||||
test('shellQuote escapes single quotes and wraps in quotes', () => {
|
||||
assert.equal(shellQuote('subminer'), `'subminer'`);
|
||||
assert.equal(shellQuote(`a'; rm -rf ~; '`), `'a'\\''; rm -rf ~; '\\'''`);
|
||||
});
|
||||
|
||||
test('runScp rejects option-like local endpoints before spawning scp', () => {
|
||||
assert.throws(() => runScp('-oProxyCommand=sh', '/tmp/out.sqlite'), /looks like an option/);
|
||||
assert.throws(() => runScp('/tmp/in.sqlite', '-bad-destination'), /looks like an option/);
|
||||
});
|
||||
|
||||
test('runScp rejects option-like remote host components', () => {
|
||||
assert.throws(
|
||||
() => runScp('-oProxyCommand=sh:/tmp/in.sqlite', '/tmp/out.sqlite'),
|
||||
/SSH host that looks like an option/,
|
||||
);
|
||||
});
|
||||
|
||||
test('resolveRemoteSubminerCommand verifies the launcher under the remote runtime PATH', () => {
|
||||
const calls: Array<{ host: string; remoteCommand: string }> = [];
|
||||
const command = resolveRemoteSubminerCommand('macbook', null, (host, remoteCommand) => {
|
||||
calls.push({ host, remoteCommand });
|
||||
return { status: 0, stdout: '', stderr: '' };
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
command,
|
||||
'PATH="$HOME/.local/bin:$HOME/.bun/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$PATH" subminer',
|
||||
);
|
||||
assert.deepEqual(calls, [
|
||||
{
|
||||
host: 'macbook',
|
||||
remoteCommand:
|
||||
'PATH="$HOME/.local/bin:$HOME/.bun/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$PATH" subminer --help >/dev/null 2>&1',
|
||||
},
|
||||
]);
|
||||
});
|
||||
@@ -1,106 +0,0 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
export interface RemoteRunResult {
|
||||
status: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* ssh/scp have no `--` terminator for the destination, so a host that starts
|
||||
* with `-` (e.g. `-oProxyCommand=...`) is parsed as an option. Reject those
|
||||
* before spawning.
|
||||
*/
|
||||
export function assertSafeSshHost(host: string): void {
|
||||
if (host.startsWith('-')) {
|
||||
throw new Error(`Refusing to use SSH host that looks like an option: ${host}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a command on the SSH host. stdin stays attached so interactive prompts
|
||||
* can still read from the terminal; stdout/stderr are captured for callers
|
||||
* that need actionable remote failure messages.
|
||||
*/
|
||||
export function runSsh(host: string, remoteCommand: string): RemoteRunResult {
|
||||
assertSafeSshHost(host);
|
||||
const result = spawnSync('ssh', [host, remoteCommand], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['inherit', 'pipe', 'pipe'],
|
||||
});
|
||||
if (result.error) {
|
||||
throw new Error(`Failed to run ssh: ${(result.error as Error).message}`);
|
||||
}
|
||||
return { status: result.status ?? 1, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
|
||||
}
|
||||
|
||||
function assertSafeScpEndpoint(endpoint: string): void {
|
||||
const colon = endpoint.indexOf(':');
|
||||
const slash = endpoint.indexOf('/');
|
||||
if (colon <= 0 || (slash !== -1 && slash < colon)) {
|
||||
if (endpoint.startsWith('-')) {
|
||||
throw new Error(`Refusing to use scp endpoint that looks like an option: ${endpoint}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const host = endpoint.slice(0, colon);
|
||||
const remotePath = endpoint.slice(colon + 1);
|
||||
assertSafeSshHost(host);
|
||||
if (remotePath.startsWith('-')) {
|
||||
throw new Error(`Refusing to use scp remote path that looks like an option: ${remotePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function runScp(from: string, to: string): void {
|
||||
assertSafeScpEndpoint(from);
|
||||
assertSafeScpEndpoint(to);
|
||||
const result = spawnSync('scp', ['-q', from, to], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['inherit', 'inherit', 'inherit'],
|
||||
});
|
||||
if (result.error) {
|
||||
throw new Error(`Failed to run scp: ${(result.error as Error).message}`);
|
||||
}
|
||||
if ((result.status ?? 1) !== 0) {
|
||||
throw new Error(`scp failed copying ${from} -> ${to}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function shellQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", `'\\''`)}'`;
|
||||
}
|
||||
|
||||
const REMOTE_RUNTIME_PATH =
|
||||
'PATH="$HOME/.local/bin:$HOME/.bun/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$PATH"';
|
||||
|
||||
/**
|
||||
* Non-interactive SSH shells often miss user-installed launchers and Bun.
|
||||
* Probe the launcher under the same deterministic PATH used by sync itself.
|
||||
*/
|
||||
export function resolveRemoteSubminerCommand(
|
||||
host: string,
|
||||
preferred: string | null,
|
||||
runRemote: typeof runSsh = runSsh,
|
||||
): string {
|
||||
// Trusted defaults stay unquoted so the remote shell expands `~`; a
|
||||
// user-supplied override is shell-quoted to prevent command injection.
|
||||
const candidates: Array<{ value: string; invocation: string }> = preferred
|
||||
? [{ value: preferred, invocation: shellQuote(preferred) }]
|
||||
: [
|
||||
{ value: 'subminer', invocation: 'subminer' },
|
||||
{ value: '~/.local/bin/subminer', invocation: '~/.local/bin/subminer' },
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
const command = `${REMOTE_RUNTIME_PATH} ${candidate.invocation}`;
|
||||
const probe = runRemote(host, `${command} --help >/dev/null 2>&1`);
|
||||
if (probe.status === 0) {
|
||||
return command;
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
preferred
|
||||
? `Remote command not found on ${host}: ${preferred}`
|
||||
: `subminer not found on ${host} (tried PATH and ~/.local/bin/subminer). Pass --remote-cmd <path>.`,
|
||||
);
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
import fs from 'node:fs';
|
||||
import { Database } from 'bun:sqlite';
|
||||
import {
|
||||
LexiconResolver,
|
||||
mergeAnime,
|
||||
mergeExcludedWords,
|
||||
mergeMediaMetadata,
|
||||
mergeVideos,
|
||||
} from './merge-catalog.js';
|
||||
import { mergeSessions } from './merge-sessions.js';
|
||||
import { copyRemoteOnlyRollups, refreshRollupsForNewSessions } from './merge-rollups.js';
|
||||
import {
|
||||
assertMergeableSchema,
|
||||
createEmptyMergeSummary,
|
||||
type SyncMergeSummary,
|
||||
} from './sync-shared.js';
|
||||
|
||||
export type { SyncMergeSummary } from './sync-shared.js';
|
||||
export { createDbSnapshot, findLiveStatsDaemonPid } from './sync-shared.js';
|
||||
|
||||
/**
|
||||
* Merge a snapshot of another machine's immersion database into the local
|
||||
* one. Insert-only union keyed on natural keys (session_uuid, video_key,
|
||||
* normalized_title_key, word/kanji identity); lifetime and rollup aggregates
|
||||
* are updated incrementally so history older than the session retention
|
||||
* window is preserved on both sides. Idempotent: re-merging the same
|
||||
* snapshot is a no-op.
|
||||
*/
|
||||
export function mergeSnapshotIntoDb(localDbPath: string, snapshotPath: string): SyncMergeSummary {
|
||||
if (!fs.existsSync(localDbPath)) {
|
||||
throw new Error(`Local stats database not found: ${localDbPath}`);
|
||||
}
|
||||
if (!fs.existsSync(snapshotPath)) {
|
||||
throw new Error(`Snapshot database not found: ${snapshotPath}`);
|
||||
}
|
||||
|
||||
const remote = new Database(snapshotPath, { readonly: true });
|
||||
let local: Database;
|
||||
try {
|
||||
local = new Database(localDbPath, { readwrite: true, create: false });
|
||||
} catch (error) {
|
||||
remote.close();
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
assertMergeableSchema(remote, 'Snapshot');
|
||||
assertMergeableSchema(local, 'Local');
|
||||
|
||||
const summary = createEmptyMergeSummary();
|
||||
local.run('PRAGMA foreign_keys = ON');
|
||||
local.run('PRAGMA busy_timeout = 5000');
|
||||
local.run('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const animeIdMap = mergeAnime(local, remote, summary);
|
||||
const { videoIdMap, addedVideoIds } = mergeVideos(local, remote, animeIdMap, summary);
|
||||
mergeMediaMetadata(local, remote, videoIdMap, addedVideoIds);
|
||||
mergeExcludedWords(local, remote, summary);
|
||||
|
||||
const lexicon = new LexiconResolver(local, remote, summary);
|
||||
const { newSessionIds } = mergeSessions(
|
||||
local,
|
||||
remote,
|
||||
videoIdMap,
|
||||
animeIdMap,
|
||||
lexicon,
|
||||
summary,
|
||||
);
|
||||
lexicon.applyFrequencyDeltas();
|
||||
refreshRollupsForNewSessions(local, newSessionIds, summary);
|
||||
copyRemoteOnlyRollups(local, remote, videoIdMap, summary);
|
||||
|
||||
local.run('COMMIT');
|
||||
return summary;
|
||||
} catch (error) {
|
||||
local.run('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
local.close();
|
||||
remote.close();
|
||||
}
|
||||
}
|
||||
|
||||
export function formatMergeSummary(summary: SyncMergeSummary): string {
|
||||
const lines = [
|
||||
`Sessions merged: ${summary.sessionsMerged} (${summary.sessionsAlreadyPresent} already present, ${summary.activeSessionsSkipped} unfinished skipped)`,
|
||||
];
|
||||
const detail: string[] = [];
|
||||
if (summary.animeAdded) detail.push(`${summary.animeAdded} series`);
|
||||
if (summary.videosAdded) detail.push(`${summary.videosAdded} videos`);
|
||||
if (summary.wordsAdded) detail.push(`${summary.wordsAdded} words`);
|
||||
if (summary.kanjiAdded) detail.push(`${summary.kanjiAdded} kanji`);
|
||||
if (summary.subtitleLinesAdded) detail.push(`${summary.subtitleLinesAdded} subtitle lines`);
|
||||
if (summary.excludedWordsAdded) detail.push(`${summary.excludedWordsAdded} excluded words`);
|
||||
if (detail.length > 0) lines.push(`Added: ${detail.join(', ')}`);
|
||||
if (summary.dailyRollupsCopied || summary.monthlyRollupsCopied) {
|
||||
lines.push(
|
||||
`Historical rollups copied: ${summary.dailyRollupsCopied} daily, ${summary.monthlyRollupsCopied} monthly`,
|
||||
);
|
||||
}
|
||||
if (summary.rollupGroupsRecomputed) {
|
||||
lines.push(`Rollup groups recomputed: ${summary.rollupGroupsRecomputed}`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { Database } from 'bun:sqlite';
|
||||
import { SCHEMA_VERSION } from '../../src/core/services/immersion-tracker/types.js';
|
||||
import { withReadonlyWalRetry } from '../history-db.js';
|
||||
import { resolveConfigDir } from '../../src/config/path-resolution.js';
|
||||
|
||||
export { SCHEMA_VERSION };
|
||||
|
||||
export interface SyncMergeSummary {
|
||||
sessionsMerged: number;
|
||||
sessionsAlreadyPresent: number;
|
||||
activeSessionsSkipped: number;
|
||||
animeAdded: number;
|
||||
videosAdded: number;
|
||||
wordsAdded: number;
|
||||
kanjiAdded: number;
|
||||
subtitleLinesAdded: number;
|
||||
telemetryRowsAdded: number;
|
||||
eventsAdded: number;
|
||||
excludedWordsAdded: number;
|
||||
dailyRollupsCopied: number;
|
||||
monthlyRollupsCopied: number;
|
||||
rollupGroupsRecomputed: number;
|
||||
}
|
||||
|
||||
export function createEmptyMergeSummary(): SyncMergeSummary {
|
||||
return {
|
||||
sessionsMerged: 0,
|
||||
sessionsAlreadyPresent: 0,
|
||||
activeSessionsSkipped: 0,
|
||||
animeAdded: 0,
|
||||
videosAdded: 0,
|
||||
wordsAdded: 0,
|
||||
kanjiAdded: 0,
|
||||
subtitleLinesAdded: 0,
|
||||
telemetryRowsAdded: 0,
|
||||
eventsAdded: 0,
|
||||
excludedWordsAdded: 0,
|
||||
dailyRollupsCopied: 0,
|
||||
monthlyRollupsCopied: 0,
|
||||
rollupGroupsRecomputed: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function nowDbTimestamp(): string {
|
||||
return String(Date.now());
|
||||
}
|
||||
|
||||
export function tableExists(db: Database, tableName: string): boolean {
|
||||
return Boolean(
|
||||
db.query(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`).get(tableName),
|
||||
);
|
||||
}
|
||||
|
||||
export function readSchemaVersion(db: Database): number | null {
|
||||
if (!tableExists(db, 'imm_schema_version')) return null;
|
||||
const row = db
|
||||
.query<{ schema_version: number }>(
|
||||
'SELECT MAX(schema_version) AS schema_version FROM imm_schema_version',
|
||||
)
|
||||
.get();
|
||||
return typeof row?.schema_version === 'number' ? row.schema_version : null;
|
||||
}
|
||||
|
||||
export function assertMergeableSchema(db: Database, label: string): void {
|
||||
const version = readSchemaVersion(db);
|
||||
if (version === null) {
|
||||
throw new Error(
|
||||
`${label} database has no schema version. Run SubMiner once on that machine so the stats database is initialized.`,
|
||||
);
|
||||
}
|
||||
if (version !== SCHEMA_VERSION) {
|
||||
throw new Error(
|
||||
`${label} database is at schema version ${version} but this launcher expects ${SCHEMA_VERSION}. Update SubMiner on both machines to the same version and run each app once before syncing.`,
|
||||
);
|
||||
}
|
||||
for (const table of ['imm_sessions', 'imm_videos', 'imm_lifetime_global']) {
|
||||
if (!tableExists(db, table)) {
|
||||
throw new Error(`${label} database is missing table ${table}; cannot sync.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function insertRow(
|
||||
db: Database,
|
||||
table: string,
|
||||
columns: readonly string[],
|
||||
values: unknown[],
|
||||
): number {
|
||||
const sql = `INSERT INTO ${table} (${columns.join(', ')}) VALUES (${columns.map(() => '?').join(', ')})`;
|
||||
// db.query() caches the prepared statement per SQL string; this runs once
|
||||
// per copied row, so re-preparing via db.prepare() would dominate merge time.
|
||||
const result = db.query(sql).run(...values);
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
|
||||
export function createDbSnapshot(dbPath: string, outPath: string): void {
|
||||
if (!fs.existsSync(dbPath)) {
|
||||
throw new Error(`Stats database not found: ${dbPath}`);
|
||||
}
|
||||
fs.rmSync(outPath, { force: true });
|
||||
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
||||
withReadonlyWalRetry(dbPath, (options) => {
|
||||
const db = new Database(dbPath, options);
|
||||
try {
|
||||
assertMergeableSchema(db, 'Local');
|
||||
db.prepare('VACUUM INTO ?').run(outPath);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
interface DaemonStateFile {
|
||||
pid?: unknown;
|
||||
}
|
||||
|
||||
function isProcessAlive(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
// EPERM means the process exists but we can't signal it → still alive.
|
||||
// Only ESRCH (no such process) means it's actually gone.
|
||||
return (error as NodeJS.ErrnoException)?.code === 'EPERM';
|
||||
}
|
||||
}
|
||||
|
||||
function statsDaemonStateCandidates(dbPath: string): string[] {
|
||||
const homeDir = os.homedir();
|
||||
const candidates = new Set<string>([path.join(path.dirname(dbPath), 'stats-daemon.json')]);
|
||||
const configDir = resolveConfigDir({
|
||||
platform: process.platform,
|
||||
appDataDir: process.env.APPDATA,
|
||||
xdgConfigHome: process.env.XDG_CONFIG_HOME,
|
||||
homeDir,
|
||||
existsSync: fs.existsSync,
|
||||
});
|
||||
candidates.add(path.join(configDir, 'stats-daemon.json'));
|
||||
if (process.platform === 'darwin') {
|
||||
candidates.add(path.join(homeDir, 'Library', 'Application Support', 'SubMiner', 'stats-daemon.json'));
|
||||
}
|
||||
return [...candidates];
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort guard against merging while a SubMiner process holds the
|
||||
* tracker's write queue in memory. Detects the background stats daemon via
|
||||
* its pid state file; the interactive app is caught by the mpv-socket check
|
||||
* in the sync command.
|
||||
*/
|
||||
export function findLiveStatsDaemonPid(dbPath: string): number | null {
|
||||
for (const statePath of statsDaemonStateCandidates(dbPath)) {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(statePath, 'utf8');
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as DaemonStateFile;
|
||||
const pid = typeof parsed.pid === 'number' && Number.isInteger(parsed.pid) ? parsed.pid : 0;
|
||||
if (pid > 0 && isProcessAlive(pid)) {
|
||||
return pid;
|
||||
}
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
+3
-7
@@ -114,13 +114,9 @@ export interface Args {
|
||||
useRofi: boolean;
|
||||
history: boolean;
|
||||
sync: boolean;
|
||||
syncHost: string;
|
||||
syncSnapshotPath: string;
|
||||
syncMergePath: string;
|
||||
syncDirection: 'both' | 'push' | 'pull';
|
||||
syncRemoteCmd: string;
|
||||
syncDbPath: string;
|
||||
syncForce: boolean;
|
||||
/** App-owned sync argv tokens forwarded verbatim to `--sync-cli sync`. */
|
||||
syncCliTokens: string[];
|
||||
syncUi: boolean;
|
||||
logLevel: LogLevel;
|
||||
logRotation: LogRotation;
|
||||
passwordStore: string;
|
||||
|
||||
Reference in New Issue
Block a user