mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-09-10 17:16:20 -07:00
perf(stats): use compressed incremental snapshot transfers
- Prefer checksum-based fuzzy rsync with compressed scp fallback - Preserve bidirectional snapshot bases during transfers
This commit is contained in:
@@ -0,0 +1,71 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { assertSafeSshHost, runScp, runSsh, shellQuote, type RemoteShellFlavor } from './ssh';
|
||||
|
||||
const RSYNC_OPTIONS = ['--compress', '--checksum', '--fuzzy'];
|
||||
|
||||
function runRsync(args: string[]) {
|
||||
return spawnSync('rsync', args, {
|
||||
encoding: 'utf8',
|
||||
stdio: ['inherit', 'pipe', 'pipe'],
|
||||
// Quote remote paths ourselves for both modern rsync and macOS openrsync.
|
||||
env: { ...process.env, RSYNC_OLD_ARGS: '1' },
|
||||
});
|
||||
}
|
||||
|
||||
interface TransferDeps {
|
||||
platform: NodeJS.Platform;
|
||||
runRsync: typeof runRsync;
|
||||
runSsh: typeof runSsh;
|
||||
runScp: typeof runScp;
|
||||
}
|
||||
|
||||
export interface SnapshotTransfer {
|
||||
kind: 'rsync' | 'scp';
|
||||
copy: (request: {
|
||||
direction: 'download' | 'upload';
|
||||
localPath: string;
|
||||
remotePath: string;
|
||||
}) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reuse the receiving machine's snapshot as rsync's fuzzy basis. Transfers
|
||||
* always write a separate file; the basis remains available for the other
|
||||
* direction. Missing rsync and Windows endpoints use compressed scp.
|
||||
*/
|
||||
export function createSnapshotTransfer(
|
||||
host: string,
|
||||
flavor: RemoteShellFlavor,
|
||||
deps: TransferDeps = { platform: process.platform, runRsync, runSsh, runScp },
|
||||
): SnapshotTransfer {
|
||||
assertSafeSshHost(host);
|
||||
const canUseRsync =
|
||||
deps.platform !== 'win32' &&
|
||||
flavor === 'posix' &&
|
||||
deps.runRsync([...RSYNC_OPTIONS, '--version']).status === 0 &&
|
||||
deps.runSsh(host, `rsync ${RSYNC_OPTIONS.join(' ')} --version`, {
|
||||
batchMode: true,
|
||||
connectTimeoutSeconds: 10,
|
||||
timeoutMs: 15_000,
|
||||
}).status === 0;
|
||||
|
||||
return {
|
||||
kind: canUseRsync ? 'rsync' : 'scp',
|
||||
copy: ({ direction, localPath, remotePath }) => {
|
||||
const remote = `${host}:${canUseRsync ? shellQuote(remotePath) : remotePath}`;
|
||||
const endpoints = direction === 'download' ? [remote, localPath] : [localPath, remote];
|
||||
if (!canUseRsync) {
|
||||
deps.runScp(endpoints[0]!, endpoints[1]!);
|
||||
return;
|
||||
}
|
||||
// --checksum prevents a same-size, same-mtime snapshot being skipped.
|
||||
// Avoid --inplace: neither a failed transfer nor a matching basis may
|
||||
// modify the snapshot we still need to send in the opposite direction.
|
||||
const result = deps.runRsync([...RSYNC_OPTIONS, '--quiet', '--', ...endpoints]);
|
||||
if (result.error) throw new Error(`Failed to run rsync: ${result.error.message}`);
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`rsync ${direction} failed for ${host}: ${result.stderr.trim()}`);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -74,7 +74,7 @@ function assertSafeScpEndpoint(endpoint: string): void {
|
||||
export function runScp(from: string, to: string): void {
|
||||
assertSafeScpEndpoint(from);
|
||||
assertSafeScpEndpoint(to);
|
||||
const result = spawnSync('scp', ['-q', from, to], {
|
||||
const result = spawnSync('scp', ['-C', '-q', from, to], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['inherit', 'inherit', 'inherit'],
|
||||
});
|
||||
|
||||
@@ -3,6 +3,7 @@ import path from 'node:path';
|
||||
import { formatMergeSummary } from './merge';
|
||||
import { quoteForRemoteShell } from './ssh';
|
||||
import type { RemoteRunResult, RemoteShellFlavor, RunSshOptions } from './ssh';
|
||||
import type { createSnapshotTransfer } from './snapshot-transfer';
|
||||
import {
|
||||
parseSyncProgressLine,
|
||||
type SyncMergeSummary,
|
||||
@@ -31,7 +32,7 @@ export interface SyncFlowContext {
|
||||
}
|
||||
|
||||
/**
|
||||
* Process/IO seams the sync flow needs stubbed in tests: SSH/scp, the DB
|
||||
* Process/IO seams the sync flow needs stubbed in tests: SSH/transfers, the DB
|
||||
* snapshot/merge engine, filesystem, and progress/bookkeeping output. The
|
||||
* app's --sync-cli mode (src/main/sync-cli.ts) provides the only production
|
||||
* binding; pure helpers are imported directly.
|
||||
@@ -51,7 +52,7 @@ export interface SyncFlowDeps {
|
||||
flavor: RemoteShellFlavor,
|
||||
runRemote?: (host: string, remoteCommand: string) => RemoteRunResult,
|
||||
) => string;
|
||||
runScp: (from: string, to: string) => void;
|
||||
createSnapshotTransfer: typeof createSnapshotTransfer;
|
||||
runSsh: (host: string, remoteCommand: string, options?: RunSshOptions) => RemoteRunResult;
|
||||
canConnectUnixSocket: (socketPath: string) => Promise<boolean>;
|
||||
realpathSync: (candidate: string) => string;
|
||||
@@ -309,6 +310,7 @@ export async function runHostSync(
|
||||
|
||||
const flavor = deps.detectRemoteShellFlavor(host, deps.runSsh);
|
||||
const remoteCmd = deps.resolveRemoteSubminerCommand(host, args.syncRemoteCmd || null, flavor);
|
||||
const transfer = deps.createSnapshotTransfer(host, flavor);
|
||||
const quote = (value: string) => quoteForRemoteShell(flavor, value);
|
||||
if (args.logLevel === 'debug') {
|
||||
console.error(`Remote subminer command (${flavor}): ${remoteCmd}`);
|
||||
@@ -332,7 +334,7 @@ export async function runHostSync(
|
||||
const forceFlag = args.syncForce ? ' --force' : '';
|
||||
|
||||
const localSnapshot = path.join(localTmpDir, 'local.sqlite');
|
||||
if (shouldPush) {
|
||||
if (shouldPush || transfer.kind === 'rsync') {
|
||||
deps.consoleLog(`Snapshotting local database (${dbPath})...`);
|
||||
deps.emitEvent({
|
||||
type: 'stage',
|
||||
@@ -343,7 +345,7 @@ export async function runHostSync(
|
||||
}
|
||||
|
||||
const remoteSnapshot = `${remoteTmpDir}/snapshot.sqlite`;
|
||||
if (shouldPull) {
|
||||
if (shouldPull || transfer.kind === 'rsync') {
|
||||
deps.consoleLog(`Snapshotting ${host}...`);
|
||||
deps.emitEvent({ type: 'stage', stage: 'snapshot-remote', message: `Snapshotting ${host}` });
|
||||
const snapshotRun = deps.runSsh(
|
||||
@@ -362,12 +364,12 @@ export async function runHostSync(
|
||||
stage: 'download',
|
||||
message: `Copying snapshot from ${host}`,
|
||||
});
|
||||
deps.runScp(`${host}:${remoteSnapshot}`, pulledSnapshot);
|
||||
transfer.copy({ direction: 'download', remotePath: remoteSnapshot, localPath: pulledSnapshot });
|
||||
}
|
||||
const incomingSnapshot = `${remoteTmpDir}/incoming.sqlite`;
|
||||
if (shouldPush) {
|
||||
deps.emitEvent({ type: 'stage', stage: 'upload', message: `Copying snapshot to ${host}` });
|
||||
deps.runScp(localSnapshot, `${host}:${incomingSnapshot}`);
|
||||
transfer.copy({ direction: 'upload', localPath: localSnapshot, remotePath: incomingSnapshot });
|
||||
}
|
||||
|
||||
if (shouldPull) {
|
||||
|
||||
@@ -14,9 +14,9 @@ import {
|
||||
assertSafeSshHost,
|
||||
detectRemoteShellFlavor,
|
||||
resolveRemoteSubminerCommand,
|
||||
runScp,
|
||||
runSsh,
|
||||
} from '../core/services/stats-sync/ssh';
|
||||
import { createSnapshotTransfer } from '../core/services/stats-sync/snapshot-transfer';
|
||||
import {
|
||||
ensureTrackerQuiescentFlow,
|
||||
runSyncFlow,
|
||||
@@ -63,7 +63,7 @@ function buildSyncCliDeps(): SyncFlowDeps {
|
||||
assertSafeSshHost,
|
||||
detectRemoteShellFlavor,
|
||||
resolveRemoteSubminerCommand,
|
||||
runScp,
|
||||
createSnapshotTransfer,
|
||||
runSsh,
|
||||
canConnectUnixSocket: canConnectSocket,
|
||||
realpathSync: (candidate) => fs.realpathSync(candidate),
|
||||
|
||||
Reference in New Issue
Block a user