mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-09-10 17:16:20 -07:00
fix(sync): force SSH and bound rsync transfers
This commit is contained in:
@@ -3,3 +3,4 @@ area: sync
|
||||
|
||||
- Sync uses compressed, incremental rsync transfers on compatible macOS and Linux machines, caching the last received snapshot per peer to reduce traffic on subsequent syncs.
|
||||
- Machines without compatible rsync, including Windows endpoints, automatically use compressed scp transfers.
|
||||
- Rsync explicitly uses SSH and aborts transfers that exceed 30 minutes before merging.
|
||||
|
||||
@@ -107,6 +107,8 @@ On macOS and Linux, sync automatically uses compressed `rsync` transfers when co
|
||||
|
||||
For a one-way transfer, `--push` snapshots the local database and merges it into the host without changing the local database. `--pull` snapshots the host and merges it into the local database without changing the host. These modes add missing data; they do not delete destination-only data or make the destination an exact mirror.
|
||||
|
||||
Each rsync transfer explicitly uses SSH and has a 30-minute time limit. A timed-out transfer stops the sync before merging the incomplete snapshot.
|
||||
|
||||
Transfers write separate temporary files and verify the reconstructed content before merging. Cached comparison snapshots are preserved throughout the transfer. After a successful rsync sync, each receiver keeps one snapshot per peer/database identity in `sync-transfer-cache/` under its SubMiner config directory. This uses roughly one database-sized file per identity; deleting that cache is safe and only makes the next sync transfer more data. Missing or unwritable caches do not prevent syncing. Older peers without the cache helper still support compressed transfers, but cannot retain the upload comparison copy.
|
||||
|
||||
Command-line sync defaults to a cold-start safety check: close SubMiner (and stop the background stats daemon with `subminer stats -s`) on both machines before running it, or pass `--force`. Syncs started from the Sync window use live mode automatically, including scheduled auto-syncs while SubMiner or playback is active. SQLite WAL provides a consistent snapshot, the transactional merge serializes with live writes, and each machine's unfinished session is excluded from the transfer; that session syncs normally after it finishes. The mpv safety check requires a live socket connection, so a stale socket file left after mpv exits does not block command-line sync. Both machines must be on the same SubMiner version; otherwise, the sync aborts on a stats schema mismatch.
|
||||
|
||||
@@ -5,7 +5,7 @@ import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createSnapshotTransfer } from './snapshot-transfer';
|
||||
import { createSnapshotTransfer, runRsync } from './snapshot-transfer';
|
||||
import { createTransferCache, transferCacheKey } from './transfer-cache';
|
||||
|
||||
type TransferDeps = NonNullable<Parameters<typeof createSnapshotTransfer>[2]>;
|
||||
@@ -77,6 +77,58 @@ test('failed rsync transfers report errors without silently retrying through scp
|
||||
|
||||
const hasRsync = process.platform !== 'win32' && spawnSync('rsync', ['--version']).status === 0;
|
||||
|
||||
test(
|
||||
'rsync forces SSH, preserves its environment, and fails on a process timeout',
|
||||
{ skip: process.platform === 'win32' },
|
||||
() => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-rsync-process-test-'));
|
||||
const previousPath = process.env.PATH;
|
||||
const previousRsh = process.env.RSYNC_RSH;
|
||||
try {
|
||||
process.env.PATH = `${dir}${path.delimiter}${previousPath ?? ''}`;
|
||||
process.env.RSYNC_RSH = 'unexpected-transport';
|
||||
const executable = path.join(dir, 'rsync');
|
||||
fs.writeFileSync(
|
||||
executable,
|
||||
'#!/bin/sh\nprintf "%s\\n" "$@" "$RSYNC_RSH" "$RSYNC_OLD_ARGS"\n',
|
||||
{ mode: 0o700 },
|
||||
);
|
||||
const result = runRsync(['--version']);
|
||||
assert.equal(result.status, 0);
|
||||
assert.deepEqual(result.stdout.trim().split('\n'), [
|
||||
'--rsh=ssh',
|
||||
'--version',
|
||||
'unexpected-transport',
|
||||
'1',
|
||||
]);
|
||||
|
||||
fs.writeFileSync(executable, '#!/bin/sh\nexec /bin/sleep 5\n');
|
||||
const transfer = createSnapshotTransfer(
|
||||
'macbook',
|
||||
'posix',
|
||||
makeDeps({
|
||||
runRsync: (args) => (args.includes('--version') ? commandResult() : runRsync(args, 50)),
|
||||
}),
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
transfer.copy({
|
||||
direction: 'download',
|
||||
localPath: '/local.sqlite',
|
||||
remotePath: '/remote.sqlite',
|
||||
}),
|
||||
/rsync download timed out for macbook/,
|
||||
);
|
||||
} finally {
|
||||
if (previousPath === undefined) delete process.env.PATH;
|
||||
else process.env.PATH = previousPath;
|
||||
if (previousRsh === undefined) delete process.env.RSYNC_RSH;
|
||||
else process.env.RSYNC_RSH = previousRsh;
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
for (const direction of ['download', 'upload'] as const) {
|
||||
test(
|
||||
`rsync ${direction} reuses snapshot blocks and preserves the basis`,
|
||||
|
||||
@@ -4,10 +4,12 @@ import { assertSafeSshHost, runScp, runSsh, shellQuote, type RemoteShellFlavor }
|
||||
|
||||
const RSYNC_OPTIONS = ['--compress', '--checksum'];
|
||||
|
||||
function runRsync(args: string[]) {
|
||||
return spawnSync('rsync', args, {
|
||||
export function runRsync(args: string[], timeoutMs = 30 * 60_000) {
|
||||
return spawnSync('rsync', ['--rsh=ssh', ...args], {
|
||||
encoding: 'utf8',
|
||||
stdio: ['inherit', 'pipe', 'pipe'],
|
||||
timeout: timeoutMs,
|
||||
killSignal: 'SIGKILL',
|
||||
// Quote remote paths ourselves for both modern rsync and macOS openrsync.
|
||||
env: { ...process.env, RSYNC_OLD_ARGS: '1' },
|
||||
});
|
||||
@@ -71,6 +73,9 @@ export function createSnapshotTransfer(
|
||||
// Without --inplace, rsync replaces the staged basis only after the
|
||||
// reconstructed file passes its transfer checksum.
|
||||
const result = deps.runRsync([...RSYNC_OPTIONS, '--quiet', '--', from, to]);
|
||||
if (result.error && 'code' in result.error && result.error.code === 'ETIMEDOUT') {
|
||||
throw new Error(`rsync ${direction} timed out for ${host}`);
|
||||
}
|
||||
if (result.error) throw new Error(`Failed to run rsync: ${result.error.message}`);
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`rsync ${direction} failed for ${host}: ${result.stderr.trim()}`);
|
||||
|
||||
Reference in New Issue
Block a user