sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-07-27 16:49:51 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat(launcher): add subminer sync for cross-machine stats/history merge over SSH (#152)

Browse Source
This commit is contained in:
Kyle Yasuda
2026-07-11 00:40:15 -07:00
committed by GitHub
parent beeab564b4
commit 321461c50f
27 changed files with 3707 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
launcher/sync/merge-catalog.ts
+423
View File
@@ -0,0 +1,423 @@
import { Database } from 'bun:sqlite';
import { insertRow, tableExists, type SyncMergeSummary } from './sync-shared.js';
const ANIME_COPY_COLUMNS = [
'normalized_title_key',
'canonical_title',
'anilist_id',
'title_romaji',
'title_english',
'title_native',
'episodes_total',
'description',
'metadata_json',
'CREATED_DATE',
'LAST_UPDATE_DATE',
] as const;
const VIDEO_COPY_COLUMNS = [
'video_key',
'canonical_title',
'source_type',
'source_path',
'source_url',
'parsed_basename',
'parsed_title',
'parsed_season',
'parsed_episode',
'parser_source',
'parser_confidence',
'parse_metadata_json',
'watched',
'duration_ms',
'file_size_bytes',
'codec_id',
'container_id',
'width_px',
'height_px',
'fps_x100',
'bitrate_kbps',
'audio_codec_id',
'hash_sha256',
'screenshot_path',
'metadata_json',
'CREATED_DATE',
'LAST_UPDATE_DATE',
] as const;
const MEDIA_ART_COPY_COLUMNS = [
'anilist_id',
'cover_url',
'cover_blob',
'cover_blob_hash',
'title_romaji',
'title_english',
'episodes_total',
'fetched_at_ms',
'CREATED_DATE',
'LAST_UPDATE_DATE',
] as const;
const YOUTUBE_COPY_COLUMNS = [
'youtube_video_id',
'video_url',
'video_title',
'video_thumbnail_url',
'channel_id',
'channel_name',
'channel_url',
'channel_thumbnail_url',
'uploader_id',
'uploader_url',
'description',
'metadata_json',
'fetched_at_ms',
'CREATED_DATE',
'LAST_UPDATE_DATE',
] as const;
const WORD_COPY_COLUMNS = [
'headword',
'word',
'reading',
'part_of_speech',
'pos1',
'pos2',
'pos3',
'first_seen',
'last_seen',
'frequency',
'frequency_rank',
] as const;
type SqlRow = Record<string, unknown>;
function selectAll(db: Database, sql: string, params: unknown[] = []): SqlRow[] {
return db.query<SqlRow>(sql).all(...params);
}
export function mergeAnime(
local: Database,
remote: Database,
summary: SyncMergeSummary,
): Map<number, number> {
const map = new Map<number, number>();
const byAnilist = local.prepare<SqlRow>('SELECT anime_id FROM imm_anime WHERE anilist_id = ?');
const byTitleKey = local.prepare<SqlRow>(
'SELECT anime_id FROM imm_anime WHERE normalized_title_key = ?',
);
const fillMissing = local.prepare(
`UPDATE imm_anime
SET
title_romaji = COALESCE(title_romaji, ?),
title_english = COALESCE(title_english, ?),
title_native = COALESCE(title_native, ?),
episodes_total = COALESCE(episodes_total, ?),
description = COALESCE(description, ?)
WHERE anime_id = ?`,
);
for (const row of selectAll(
remote,
`SELECT anime_id, ${ANIME_COPY_COLUMNS.join(', ')} FROM imm_anime`,
)) {
const remoteId = Number(row.anime_id);
const existing =
(row.anilist_id !== null ? byAnilist.get(row.anilist_id) : undefined) ??
byTitleKey.get(row.normalized_title_key);
if (existing) {
const localId = Number(existing.anime_id);
map.set(remoteId, localId);
fillMissing.run(
row.title_romaji,
row.title_english,
row.title_native,
row.episodes_total,
row.description,
localId,
);
continue;
}
// No local row matched by anilist_id (checked first in `existing` above)
// or title key, so the remote anilist_id — if any — is free to insert as-is.
const values = ANIME_COPY_COLUMNS.map((column) => row[column]);
map.set(remoteId, insertRow(local, 'imm_anime', ANIME_COPY_COLUMNS, values));
summary.animeAdded += 1;
}
return map;
}
export interface VideoMergeResult {
videoIdMap: Map<number, number>;
addedVideoIds: Set<number>;
}
export function mergeVideos(
local: Database,
remote: Database,
animeIdMap: Map<number, number>,
summary: SyncMergeSummary,
): VideoMergeResult {
const videoIdMap = new Map<number, number>();
const addedVideoIds = new Set<number>();
const byKey = local.prepare<SqlRow>(
'SELECT video_id, watched FROM imm_videos WHERE video_key = ?',
);
const setWatched = local.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?');
for (const row of selectAll(
remote,
`SELECT video_id, anime_id, ${VIDEO_COPY_COLUMNS.join(', ')} FROM imm_videos`,
)) {
const remoteId = Number(row.video_id);
const mappedAnimeId =
row.anime_id === null ? null : (animeIdMap.get(Number(row.anime_id)) ?? null);
const existing = byKey.get(row.video_key);
if (existing) {
const localId = Number(existing.video_id);
videoIdMap.set(remoteId, localId);
if (Number(row.watched) > 0 && Number(existing.watched) <= 0) {
setWatched.run(localId);
}
continue;
}
const columns = ['anime_id', ...VIDEO_COPY_COLUMNS];
const values = [mappedAnimeId, ...VIDEO_COPY_COLUMNS.map((column) => row[column])];
const localId = insertRow(local, 'imm_videos', columns, values);
videoIdMap.set(remoteId, localId);
addedVideoIds.add(remoteId);
summary.videosAdded += 1;
}
return { videoIdMap, addedVideoIds };
}
export function mergeMediaMetadata(
local: Database,
remote: Database,
videoIdMap: Map<number, number>,
addedVideoIds: Set<number>,
): void {
if (videoIdMap.size === 0) return;
const metadataVideoIds = new Set<number>([...addedVideoIds, ...videoIdMap.keys()]);
const hasBlobStore =
tableExists(local, 'imm_cover_art_blobs') && tableExists(remote, 'imm_cover_art_blobs');
const copyBlob = hasBlobStore
? local.prepare(
`INSERT INTO imm_cover_art_blobs (blob_hash, cover_blob, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (?, ?, ?, ?)
ON CONFLICT(blob_hash) DO NOTHING`,
)
: null;
const readBlob = hasBlobStore
? remote.prepare<SqlRow>('SELECT * FROM imm_cover_art_blobs WHERE blob_hash = ?')
: null;
if (tableExists(remote, 'imm_media_art') && tableExists(local, 'imm_media_art')) {
const localArtExists = local.prepare<SqlRow>(
'SELECT 1 FROM imm_media_art WHERE video_id = ? LIMIT 1',
);
for (const remoteVideoId of metadataVideoIds) {
const localVideoId = videoIdMap.get(remoteVideoId)!;
if (localArtExists.get(localVideoId)) continue;
const row = remote
.query<SqlRow>(
`SELECT ${MEDIA_ART_COPY_COLUMNS.join(', ')} FROM imm_media_art WHERE video_id = ?`,
)
.get(remoteVideoId);
if (!row) continue;
if (row.cover_blob_hash && copyBlob && readBlob) {
const blob = readBlob.get(row.cover_blob_hash);
if (blob) {
copyBlob.run(blob.blob_hash, blob.cover_blob, blob.CREATED_DATE, blob.LAST_UPDATE_DATE);
}
}
insertRow(
local,
'imm_media_art',
['video_id', ...MEDIA_ART_COPY_COLUMNS],
[localVideoId, ...MEDIA_ART_COPY_COLUMNS.map((column) => row[column])],
);
}
}
if (tableExists(remote, 'imm_youtube_videos') && tableExists(local, 'imm_youtube_videos')) {
const localYoutubeExists = local.prepare<SqlRow>(
'SELECT 1 FROM imm_youtube_videos WHERE video_id = ? LIMIT 1',
);
for (const remoteVideoId of metadataVideoIds) {
const localVideoId = videoIdMap.get(remoteVideoId)!;
if (localYoutubeExists.get(localVideoId)) continue;
const row = remote
.query<SqlRow>(
`SELECT ${YOUTUBE_COPY_COLUMNS.join(', ')} FROM imm_youtube_videos WHERE video_id = ?`,
)
.get(remoteVideoId);
if (!row) continue;
insertRow(
local,
'imm_youtube_videos',
['video_id', ...YOUTUBE_COPY_COLUMNS],
[localVideoId, ...YOUTUBE_COPY_COLUMNS.map((column) => row[column])],
);
}
}
}
export function mergeExcludedWords(
local: Database,
remote: Database,
summary: SyncMergeSummary,
): void {
if (
!tableExists(remote, 'imm_stats_excluded_words') ||
!tableExists(local, 'imm_stats_excluded_words')
) {
return;
}
const insert = local.prepare(
`INSERT INTO imm_stats_excluded_words (headword, word, reading, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(headword, word, reading) DO NOTHING`,
);
for (const row of selectAll(
remote,
'SELECT headword, word, reading, CREATED_DATE, LAST_UPDATE_DATE FROM imm_stats_excluded_words',
)) {
const result = insert.run(
row.headword,
row.word,
row.reading,
row.CREATED_DATE,
row.LAST_UPDATE_DATE,
);
summary.excludedWordsAdded += result.changes;
}
}
/**
* Lazily maps remote imm_words / imm_kanji ids onto local rows by natural key
* ((headword, word, reading) / kanji). New rows are copied with the remote's
* accumulated frequency; rows that already exist locally get their frequency
* incremented later with only the occurrence counts this merge adds (the
* remote total would double-count lines merged in earlier syncs).
*/
export class LexiconResolver {
private readonly wordMap = new Map<number, { localId: number; isNew: boolean }>();
private readonly kanjiMap = new Map<number, { localId: number; isNew: boolean }>();
readonly wordFrequencyDeltas = new Map<number, number>();
readonly kanjiFrequencyDeltas = new Map<number, number>();
constructor(
private readonly local: Database,
private readonly remote: Database,
private readonly summary: SyncMergeSummary,
) {}
resolveWord(remoteWordId: number): number {
const cached = this.wordMap.get(remoteWordId);
if (cached) return cached.localId;
const row = this.remote
.query<SqlRow>(`SELECT ${WORD_COPY_COLUMNS.join(', ')} FROM imm_words WHERE id = ?`)
.get(remoteWordId);
if (!row) throw new Error(`Snapshot references missing imm_words row ${remoteWordId}`);
const existing = this.local
.query<SqlRow>('SELECT id FROM imm_words WHERE headword IS ? AND word IS ? AND reading IS ?')
.get(row.headword, row.word, row.reading);
let entry: { localId: number; isNew: boolean };
if (existing) {
entry = { localId: Number(existing.id), isNew: false };
this.local
.prepare(
`UPDATE imm_words
SET first_seen = MIN(COALESCE(first_seen, ?), COALESCE(?, first_seen)),
last_seen = MAX(COALESCE(last_seen, ?), COALESCE(?, last_seen))
WHERE id = ?`,
)
.run(row.first_seen, row.first_seen, row.last_seen, row.last_seen, entry.localId);
} else {
const localId = insertRow(
this.local,
'imm_words',
WORD_COPY_COLUMNS,
WORD_COPY_COLUMNS.map((column) => row[column]),
);
entry = { localId, isNew: true };
this.summary.wordsAdded += 1;
}
this.wordMap.set(remoteWordId, entry);
return entry.localId;
}
resolveKanji(remoteKanjiId: number): number {
const cached = this.kanjiMap.get(remoteKanjiId);
if (cached) return cached.localId;
const row = this.remote
.query<SqlRow>('SELECT kanji, first_seen, last_seen, frequency FROM imm_kanji WHERE id = ?')
.get(remoteKanjiId);
if (!row) throw new Error(`Snapshot references missing imm_kanji row ${remoteKanjiId}`);
const existing = this.local
.query<SqlRow>('SELECT id FROM imm_kanji WHERE kanji IS ?')
.get(row.kanji);
let entry: { localId: number; isNew: boolean };
if (existing) {
entry = { localId: Number(existing.id), isNew: false };
this.local
.prepare(
`UPDATE imm_kanji
SET first_seen = MIN(COALESCE(first_seen, ?), COALESCE(?, first_seen)),
last_seen = MAX(COALESCE(last_seen, ?), COALESCE(?, last_seen))
WHERE id = ?`,
)
.run(row.first_seen, row.first_seen, row.last_seen, row.last_seen, entry.localId);
} else {
const localId = insertRow(
this.local,
'imm_kanji',
['kanji', 'first_seen', 'last_seen', 'frequency'],
[row.kanji, row.first_seen, row.last_seen, row.frequency],
);
entry = { localId, isNew: true };
this.summary.kanjiAdded += 1;
}
this.kanjiMap.set(remoteKanjiId, entry);
return entry.localId;
}
addWordOccurrences(remoteWordId: number, count: number): void {
const entry = this.wordMap.get(remoteWordId);
if (!entry || entry.isNew) return;
this.wordFrequencyDeltas.set(
entry.localId,
(this.wordFrequencyDeltas.get(entry.localId) ?? 0) + count,
);
}
addKanjiOccurrences(remoteKanjiId: number, count: number): void {
const entry = this.kanjiMap.get(remoteKanjiId);
if (!entry || entry.isNew) return;
this.kanjiFrequencyDeltas.set(
entry.localId,
(this.kanjiFrequencyDeltas.get(entry.localId) ?? 0) + count,
);
}
applyFrequencyDeltas(): void {
const updateWord = this.local.prepare(
'UPDATE imm_words SET frequency = COALESCE(frequency, 0) + ? WHERE id = ?',
);
for (const [localId, delta] of this.wordFrequencyDeltas) {
updateWord.run(delta, localId);
}
const updateKanji = this.local.prepare(
'UPDATE imm_kanji SET frequency = COALESCE(frequency, 0) + ? WHERE id = ?',
);
for (const [localId, delta] of this.kanjiFrequencyDeltas) {
updateKanji.run(delta, localId);
}
}
}
launcher/sync/merge-rollups.ts
+265
View File
@@ -0,0 +1,265 @@
import { Database } from 'bun:sqlite';
import { nowDbTimestamp, tableExists, type SyncMergeSummary } from './sync-shared.js';
type SqlRow = Record<string, unknown>;
const LOCAL_DAY_EXPR = `CAST(julianday(CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER)`;
const LOCAL_MONTH_EXPR = `CAST(strftime('%Y%m', CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER)`;
// Ported from upsertDailyRollupsForGroups / upsertMonthlyRollupsForGroups in
// src/core/services/immersion-tracker/maintenance.ts — must stay in sync.
const DAILY_ROLLUP_UPSERT = `
WITH matching_sessions AS (
SELECT * FROM imm_sessions
WHERE ${LOCAL_DAY_EXPR} = ? AND video_id = ?
),
session_metrics AS (
SELECT
t.session_id,
MAX(t.active_watched_ms) AS max_active_ms,
MAX(t.lines_seen) AS max_lines,
MAX(t.tokens_seen) AS max_tokens,
MAX(t.cards_mined) AS max_cards,
MAX(t.lookup_count) AS max_lookups,
MAX(t.lookup_hits) AS max_hits
FROM imm_session_telemetry t
JOIN matching_sessions s ON s.session_id = t.session_id
GROUP BY t.session_id
)
INSERT INTO imm_daily_rollups (
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards, cards_per_hour, tokens_per_min, lookup_hit_rate,
CREATED_DATE, LAST_UPDATE_DATE
)
SELECT
${LOCAL_DAY_EXPR.replace('started_at_ms', 's.started_at_ms')} AS rollup_day,
s.video_id AS video_id,
COUNT(DISTINCT s.session_id) AS total_sessions,
COALESCE(SUM(COALESCE(sm.max_active_ms, s.active_watched_ms)), 0) / 60000.0 AS total_active_min,
COALESCE(SUM(COALESCE(sm.max_lines, s.lines_seen)), 0) AS total_lines_seen,
COALESCE(SUM(COALESCE(sm.max_tokens, s.tokens_seen)), 0) AS total_tokens_seen,
COALESCE(SUM(COALESCE(sm.max_cards, s.cards_mined)), 0) AS total_cards,
CASE
WHEN COALESCE(SUM(COALESCE(sm.max_active_ms, s.active_watched_ms)), 0) > 0
THEN (COALESCE(SUM(COALESCE(sm.max_cards, s.cards_mined)), 0) * 60.0)
/ (COALESCE(SUM(COALESCE(sm.max_active_ms, s.active_watched_ms)), 0) / 60000.0)
ELSE NULL
END AS cards_per_hour,
CASE
WHEN COALESCE(SUM(COALESCE(sm.max_active_ms, s.active_watched_ms)), 0) > 0
THEN COALESCE(SUM(COALESCE(sm.max_tokens, s.tokens_seen)), 0)
/ (COALESCE(SUM(COALESCE(sm.max_active_ms, s.active_watched_ms)), 0) / 60000.0)
ELSE NULL
END AS tokens_per_min,
CASE
WHEN COALESCE(SUM(COALESCE(sm.max_lookups, s.lookup_count)), 0) > 0
THEN CAST(COALESCE(SUM(COALESCE(sm.max_hits, s.lookup_hits)), 0) AS REAL)
/ CAST(COALESCE(SUM(COALESCE(sm.max_lookups, s.lookup_count)), 0) AS REAL)
ELSE NULL
END AS lookup_hit_rate,
? AS CREATED_DATE,
? AS LAST_UPDATE_DATE
FROM matching_sessions s
LEFT JOIN session_metrics sm ON s.session_id = sm.session_id
GROUP BY rollup_day, s.video_id
ON CONFLICT (rollup_day, video_id) DO UPDATE SET
total_sessions = excluded.total_sessions,
total_active_min = excluded.total_active_min,
total_lines_seen = excluded.total_lines_seen,
total_tokens_seen = excluded.total_tokens_seen,
total_cards = excluded.total_cards,
cards_per_hour = excluded.cards_per_hour,
tokens_per_min = excluded.tokens_per_min,
lookup_hit_rate = excluded.lookup_hit_rate,
CREATED_DATE = COALESCE(imm_daily_rollups.CREATED_DATE, excluded.CREATED_DATE),
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
`;
const MONTHLY_ROLLUP_UPSERT = `
WITH matching_sessions AS (
SELECT * FROM imm_sessions
WHERE ${LOCAL_MONTH_EXPR} = ? AND video_id = ?
),
session_metrics AS (
SELECT
t.session_id,
MAX(t.active_watched_ms) AS max_active_ms,
MAX(t.lines_seen) AS max_lines,
MAX(t.tokens_seen) AS max_tokens,
MAX(t.cards_mined) AS max_cards
FROM imm_session_telemetry t
JOIN matching_sessions s ON s.session_id = t.session_id
GROUP BY t.session_id
)
INSERT INTO imm_monthly_rollups (
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
)
SELECT
${LOCAL_MONTH_EXPR.replace('started_at_ms', 's.started_at_ms')} AS rollup_month,
s.video_id AS video_id,
COUNT(DISTINCT s.session_id) AS total_sessions,
COALESCE(SUM(COALESCE(sm.max_active_ms, s.active_watched_ms)), 0) / 60000.0 AS total_active_min,
COALESCE(SUM(COALESCE(sm.max_lines, s.lines_seen)), 0) AS total_lines_seen,
COALESCE(SUM(COALESCE(sm.max_tokens, s.tokens_seen)), 0) AS total_tokens_seen,
COALESCE(SUM(COALESCE(sm.max_cards, s.cards_mined)), 0) AS total_cards,
? AS CREATED_DATE,
? AS LAST_UPDATE_DATE
FROM matching_sessions s
LEFT JOIN session_metrics sm ON s.session_id = sm.session_id
GROUP BY rollup_month, s.video_id
ON CONFLICT (rollup_month, video_id) DO UPDATE SET
total_sessions = excluded.total_sessions,
total_active_min = excluded.total_active_min,
total_lines_seen = excluded.total_lines_seen,
total_tokens_seen = excluded.total_tokens_seen,
total_cards = excluded.total_cards,
CREATED_DATE = COALESCE(imm_monthly_rollups.CREATED_DATE, excluded.CREATED_DATE),
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
`;
/**
* Recompute daily/monthly rollup groups touched by the newly merged sessions
* from the (now merged) local session + telemetry data. The maintenance
* watermark is left alone: telemetry newer than it gets recomputed again by
* the app later, which is idempotent.
*/
export function refreshRollupsForNewSessions(
local: Database,
newSessionIds: number[],
summary: SyncMergeSummary,
): void {
if (newSessionIds.length === 0) return;
const groups = new Map<string, { day: number; month: number; videoId: number }>();
for (let offset = 0; offset < newSessionIds.length; offset += 500) {
const chunk = newSessionIds.slice(offset, offset + 500);
const rows = local
.query<SqlRow>(
`SELECT DISTINCT ${LOCAL_DAY_EXPR} AS rollup_day, ${LOCAL_MONTH_EXPR} AS rollup_month, video_id
FROM imm_sessions WHERE session_id IN (${chunk.map(() => '?').join(',')})`,
)
.all(...chunk);
for (const row of rows) {
const day = Number(row.rollup_day);
const month = Number(row.rollup_month);
const videoId = Number(row.video_id);
groups.set(`${day}-${videoId}`, { day, month, videoId });
}
}
const stampMs = nowDbTimestamp();
const deleteDaily = local.prepare(
'DELETE FROM imm_daily_rollups WHERE rollup_day = ? AND video_id = ?',
);
const deleteMonthly = local.prepare(
'DELETE FROM imm_monthly_rollups WHERE rollup_month = ? AND video_id = ?',
);
const upsertDaily = local.prepare(DAILY_ROLLUP_UPSERT);
const upsertMonthly = local.prepare(MONTHLY_ROLLUP_UPSERT);
const monthlyGroups = new Set<string>();
for (const { day, month, videoId } of groups.values()) {
deleteDaily.run(day, videoId);
upsertDaily.run(day, videoId, stampMs, stampMs);
summary.rollupGroupsRecomputed += 1;
const monthKey = `${month}-${videoId}`;
if (!monthlyGroups.has(monthKey)) {
monthlyGroups.add(monthKey);
deleteMonthly.run(month, videoId);
upsertMonthly.run(month, videoId, stampMs, stampMs);
}
}
}
/**
* Sessions are pruned after a retention window, but rollups are kept much
* longer — the remote's older rollup history can't be reconstructed from
* merged sessions. Copy remote rollup rows for groups where the local DB has
* neither a rollup row nor any sessions (i.e. history only the remote knows).
* Groups both machines have data for are never summed, to avoid
* double-counting sessions that earlier syncs already shared.
*/
export function copyRemoteOnlyRollups(
local: Database,
remote: Database,
videoIdMap: Map<number, number>,
summary: SyncMergeSummary,
): void {
if (!tableExists(remote, 'imm_daily_rollups') || !tableExists(local, 'imm_daily_rollups')) return;
const localDailyExists = local.prepare(
'SELECT 1 FROM imm_daily_rollups WHERE rollup_day = ? AND video_id = ? LIMIT 1',
);
const localDaySessions = local.prepare(
`SELECT 1 FROM imm_sessions WHERE video_id = ? AND ${LOCAL_DAY_EXPR} = ? LIMIT 1`,
);
const localMonthSessions = local.prepare(
`SELECT 1 FROM imm_sessions WHERE video_id = ? AND ${LOCAL_MONTH_EXPR} = ? LIMIT 1`,
);
const localMonthSessionsForDay = local.prepare(
`SELECT 1 FROM imm_sessions
WHERE video_id = ?
AND ${LOCAL_MONTH_EXPR} = CAST(strftime('%Y%m', CAST(? AS INTEGER) * 86400, 'unixepoch', 'localtime') AS INTEGER)
LIMIT 1`,
);
const insertDaily = local.prepare(
`INSERT INTO imm_daily_rollups (
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards, cards_per_hour, tokens_per_min, lookup_hit_rate,
CREATED_DATE, LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
);
for (const row of remote.query<SqlRow>('SELECT * FROM imm_daily_rollups').all()) {
if (row.video_id === null) continue;
const localVideoId = videoIdMap.get(Number(row.video_id));
if (localVideoId === undefined) continue;
if (localDailyExists.get(row.rollup_day, localVideoId)) continue;
if (localDaySessions.get(localVideoId, row.rollup_day)) continue;
if (localMonthSessionsForDay.get(localVideoId, row.rollup_day)) continue;
insertDaily.run(
row.rollup_day,
localVideoId,
row.total_sessions,
row.total_active_min,
row.total_lines_seen,
row.total_tokens_seen,
row.total_cards,
row.cards_per_hour,
row.tokens_per_min,
row.lookup_hit_rate,
row.CREATED_DATE,
row.LAST_UPDATE_DATE,
);
summary.dailyRollupsCopied += 1;
}
const localMonthlyExists = local.prepare(
'SELECT 1 FROM imm_monthly_rollups WHERE rollup_month = ? AND video_id = ? LIMIT 1',
);
const insertMonthly = local.prepare(
`INSERT INTO imm_monthly_rollups (
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
);
for (const row of remote.query<SqlRow>('SELECT * FROM imm_monthly_rollups').all()) {
if (row.video_id === null) continue;
const localVideoId = videoIdMap.get(Number(row.video_id));
if (localVideoId === undefined) continue;
if (localMonthlyExists.get(row.rollup_month, localVideoId)) continue;
if (localMonthSessions.get(localVideoId, row.rollup_month)) continue;
insertMonthly.run(
row.rollup_month,
localVideoId,
row.total_sessions,
row.total_active_min,
row.total_lines_seen,
row.total_tokens_seen,
row.total_cards,
row.CREATED_DATE,
row.LAST_UPDATE_DATE,
);
summary.monthlyRollupsCopied += 1;
}
}
launcher/sync/merge-sessions.ts
+465
View File
@@ -0,0 +1,465 @@
import { Database } from 'bun:sqlite';
import type { LexiconResolver } from './merge-catalog.js';
import { insertRow, nowDbTimestamp, type SyncMergeSummary } from './sync-shared.js';
const SESSION_COPY_COLUMNS = [
'session_uuid',
'started_at_ms',
'ended_at_ms',
'status',
'locale_id',
'target_lang_id',
'difficulty_tier',
'subtitle_mode',
'ended_media_ms',
'total_watched_ms',
'active_watched_ms',
'lines_seen',
'tokens_seen',
'cards_mined',
'lookup_count',
'lookup_hits',
'yomitan_lookup_count',
'pause_count',
'pause_ms',
'seek_forward_count',
'seek_backward_count',
'media_buffer_events',
'CREATED_DATE',
'LAST_UPDATE_DATE',
] as const;
const TELEMETRY_COPY_COLUMNS = [
'sample_ms',
'total_watched_ms',
'active_watched_ms',
'lines_seen',
'tokens_seen',
'cards_mined',
'lookup_count',
'lookup_hits',
'yomitan_lookup_count',
'pause_count',
'pause_ms',
'seek_forward_count',
'seek_backward_count',
'media_buffer_events',
'CREATED_DATE',
'LAST_UPDATE_DATE',
] as const;
const EVENT_COPY_COLUMNS = [
'ts_ms',
'event_type',
'line_index',
'segment_start_ms',
'segment_end_ms',
'tokens_delta',
'cards_delta',
'payload_json',
'CREATED_DATE',
'LAST_UPDATE_DATE',
] as const;
const LINE_COPY_COLUMNS = [
'line_index',
'segment_start_ms',
'segment_end_ms',
'text',
'secondary_text',
'CREATED_DATE',
'LAST_UPDATE_DATE',
] as const;
type SqlRow = Record<string, unknown>;
export interface SessionMergeResult {
newSessionIds: number[];
}
export function mergeSessions(
local: Database,
remote: Database,
videoIdMap: Map<number, number>,
animeIdMap: Map<number, number>,
lexicon: LexiconResolver,
summary: SyncMergeSummary,
): SessionMergeResult {
const newSessionIds: number[] = [];
const uuidExists = local.prepare<SqlRow>(
'SELECT session_id FROM imm_sessions WHERE session_uuid = ?',
);
const remoteSessions = remote
.query<SqlRow>(
`SELECT session_id, video_id, ${SESSION_COPY_COLUMNS.join(', ')}
FROM imm_sessions
ORDER BY CAST(started_at_ms AS REAL) ASC, session_id ASC`,
)
.all();
for (const session of remoteSessions) {
if (session.ended_at_ms === null) {
// Stale ACTIVE sessions are finalized by the app on its next startup;
// they will sync once they carry final numbers.
summary.activeSessionsSkipped += 1;
continue;
}
if (uuidExists.get(session.session_uuid)) {
summary.sessionsAlreadyPresent += 1;
continue;
}
const localVideoId = videoIdMap.get(Number(session.video_id));
if (localVideoId === undefined) {
throw new Error(`Snapshot session ${String(session.session_uuid)} references missing video row`);
}
const localSessionId = insertRow(
local,
'imm_sessions',
['video_id', ...SESSION_COPY_COLUMNS],
[localVideoId, ...SESSION_COPY_COLUMNS.map((column) => session[column])],
);
newSessionIds.push(localSessionId);
summary.sessionsMerged += 1;
const remoteSessionId = Number(session.session_id);
copyTelemetry(local, remote, remoteSessionId, localSessionId, summary);
const eventIdMap = copyEvents(local, remote, remoteSessionId, localSessionId, summary);
copySubtitleLines(
local,
remote,
remoteSessionId,
localSessionId,
localVideoId,
animeIdMap,
eventIdMap,
lexicon,
summary,
);
applyMergedSessionLifetime(local, localSessionId, localVideoId, session);
}
return { newSessionIds };
}
function copyTelemetry(
local: Database,
remote: Database,
remoteSessionId: number,
localSessionId: number,
summary: SyncMergeSummary,
): void {
const rows = remote
.query<SqlRow>(
`SELECT ${TELEMETRY_COPY_COLUMNS.join(', ')} FROM imm_session_telemetry
WHERE session_id = ? ORDER BY telemetry_id ASC`,
)
.all(remoteSessionId);
for (const row of rows) {
insertRow(
local,
'imm_session_telemetry',
['session_id', ...TELEMETRY_COPY_COLUMNS],
[localSessionId, ...TELEMETRY_COPY_COLUMNS.map((column) => row[column])],
);
summary.telemetryRowsAdded += 1;
}
}
function copyEvents(
local: Database,
remote: Database,
remoteSessionId: number,
localSessionId: number,
summary: SyncMergeSummary,
): Map<number, number> {
const eventIdMap = new Map<number, number>();
const rows = remote
.query<SqlRow>(
`SELECT event_id, ${EVENT_COPY_COLUMNS.join(', ')} FROM imm_session_events
WHERE session_id = ? ORDER BY event_id ASC`,
)
.all(remoteSessionId);
for (const row of rows) {
const localEventId = insertRow(
local,
'imm_session_events',
['session_id', ...EVENT_COPY_COLUMNS],
[localSessionId, ...EVENT_COPY_COLUMNS.map((column) => row[column])],
);
eventIdMap.set(Number(row.event_id), localEventId);
summary.eventsAdded += 1;
}
return eventIdMap;
}
function copySubtitleLines(
local: Database,
remote: Database,
remoteSessionId: number,
localSessionId: number,
localVideoId: number,
animeIdMap: Map<number, number>,
eventIdMap: Map<number, number>,
lexicon: LexiconResolver,
summary: SyncMergeSummary,
): void {
const rows = remote
.query<SqlRow>(
`SELECT line_id, event_id, anime_id, ${LINE_COPY_COLUMNS.join(', ')} FROM imm_subtitle_lines
WHERE session_id = ? ORDER BY line_id ASC`,
)
.all(remoteSessionId);
const wordOccurrences = remote.prepare<SqlRow>(
'SELECT word_id, occurrence_count FROM imm_word_line_occurrences WHERE line_id = ?',
);
const kanjiOccurrences = remote.prepare<SqlRow>(
'SELECT kanji_id, occurrence_count FROM imm_kanji_line_occurrences WHERE line_id = ?',
);
const insertWordOccurrence = local.prepare(
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count) VALUES (?, ?, ?)
ON CONFLICT(line_id, word_id) DO UPDATE SET occurrence_count = occurrence_count + excluded.occurrence_count`,
);
const insertKanjiOccurrence = local.prepare(
`INSERT INTO imm_kanji_line_occurrences (line_id, kanji_id, occurrence_count) VALUES (?, ?, ?)
ON CONFLICT(line_id, kanji_id) DO UPDATE SET occurrence_count = occurrence_count + excluded.occurrence_count`,
);
for (const row of rows) {
const localEventId = row.event_id === null ? null : (eventIdMap.get(Number(row.event_id)) ?? null);
const localAnimeId = row.anime_id === null ? null : (animeIdMap.get(Number(row.anime_id)) ?? null);
const localLineId = insertRow(
local,
'imm_subtitle_lines',
['session_id', 'event_id', 'video_id', 'anime_id', ...LINE_COPY_COLUMNS],
[
localSessionId,
localEventId,
localVideoId,
localAnimeId,
...LINE_COPY_COLUMNS.map((column) => row[column]),
],
);
summary.subtitleLinesAdded += 1;
for (const occurrence of wordOccurrences.all(row.line_id)) {
const localWordId = lexicon.resolveWord(Number(occurrence.word_id));
const count = Number(occurrence.occurrence_count);
insertWordOccurrence.run(localLineId, localWordId, count);
lexicon.addWordOccurrences(Number(occurrence.word_id), count);
}
for (const occurrence of kanjiOccurrences.all(row.line_id)) {
const localKanjiId = lexicon.resolveKanji(Number(occurrence.kanji_id));
const count = Number(occurrence.occurrence_count);
insertKanjiOccurrence.run(localLineId, localKanjiId, count);
lexicon.addKanjiOccurrences(Number(occurrence.kanji_id), count);
}
}
}
/**
* Port of applySessionLifetimeSummary (src/core/services/immersion-tracker/
* lifetime.ts) for sessions arriving out of chronological order. The
* "first session of the day / for this video" checks are order-independent
* here (any other session counts, not just earlier ones): the local machine
* already credited active_days/episodes_started when its own session was
* applied, even if the merged session started earlier that day.
*/
function applyMergedSessionLifetime(
local: Database,
sessionId: number,
videoId: number,
session: SqlRow,
): void {
const updatedAtMs = nowDbTimestamp();
const applied = local
.prepare(
`INSERT INTO imm_lifetime_applied_sessions (session_id, applied_at_ms, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (?, ?, ?, ?)
ON CONFLICT(session_id) DO NOTHING`,
)
.run(sessionId, session.ended_at_ms, updatedAtMs, updatedAtMs);
if (applied.changes <= 0) return;
const telemetry = local
.query<SqlRow>(
`SELECT active_watched_ms, cards_mined, lines_seen, tokens_seen
FROM imm_session_telemetry
WHERE session_id = ?
ORDER BY sample_ms DESC, telemetry_id DESC
LIMIT 1`,
)
.get(sessionId);
const metric = (telemetryValue: unknown, sessionValue: unknown): number => {
const fromTelemetry = telemetry ? Number(telemetryValue) : Number.NaN;
const value = Number.isFinite(fromTelemetry) ? fromTelemetry : Number(sessionValue);
return Math.max(0, Math.floor(Number.isFinite(value) ? value : 0));
};
const activeMs = metric(telemetry?.active_watched_ms, session.active_watched_ms);
const cardsMined = metric(telemetry?.cards_mined, session.cards_mined);
const linesSeen = metric(telemetry?.lines_seen, session.lines_seen);
const tokensSeen = metric(telemetry?.tokens_seen, session.tokens_seen);
const video = local
.query<SqlRow>('SELECT anime_id, watched FROM imm_videos WHERE video_id = ?')
.get(videoId);
const watched = Number(video?.watched ?? 0);
const animeId = video?.anime_id === null || video?.anime_id === undefined ? null : Number(video.anime_id);
const mediaLifetime = local
.query<SqlRow>('SELECT completed FROM imm_lifetime_media WHERE video_id = ?')
.get(videoId);
const hasOtherSessionForVideo = Boolean(
local
.query('SELECT 1 FROM imm_sessions WHERE video_id = ? AND session_id != ? LIMIT 1')
.get(videoId, sessionId),
);
const isFirstSessionForVideoRun = !mediaLifetime && !hasOtherSessionForVideo;
const isFirstCompletedSessionForVideoRun = watched > 0 && Number(mediaLifetime?.completed ?? 0) <= 0;
const hasOtherSessionOnDay = Boolean(
local
.query(
`SELECT 1 FROM imm_sessions
WHERE session_id != ?
AND CAST(julianday(CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER)
= CAST(julianday(CAST(? AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER)
LIMIT 1`,
)
.get(sessionId, session.started_at_ms),
);
let animeCompletedDelta = 0;
if (animeId !== null && watched > 0 && isFirstCompletedSessionForVideoRun) {
const animeLifetime = local
.query<SqlRow>('SELECT episodes_completed FROM imm_lifetime_anime WHERE anime_id = ?')
.get(animeId);
const anime = local
.query<SqlRow>('SELECT episodes_total FROM imm_anime WHERE anime_id = ?')
.get(animeId);
const episodesCompletedBefore = Number(animeLifetime?.episodes_completed ?? 0);
const episodesTotal = anime?.episodes_total === null || anime?.episodes_total === undefined
? null
: Number(anime.episodes_total);
if (
episodesTotal !== null &&
episodesTotal > 0 &&
episodesCompletedBefore < episodesTotal &&
episodesCompletedBefore + 1 >= episodesTotal
) {
animeCompletedDelta = 1;
}
}
local
.prepare(
`UPDATE imm_lifetime_global
SET total_sessions = total_sessions + 1,
total_active_ms = total_active_ms + ?,
total_cards = total_cards + ?,
active_days = active_days + ?,
episodes_started = episodes_started + ?,
episodes_completed = episodes_completed + ?,
anime_completed = anime_completed + ?,
LAST_UPDATE_DATE = ?
WHERE global_id = 1`,
)
.run(
activeMs,
cardsMined,
hasOtherSessionOnDay ? 0 : 1,
isFirstSessionForVideoRun ? 1 : 0,
isFirstCompletedSessionForVideoRun ? 1 : 0,
animeCompletedDelta,
updatedAtMs,
);
local
.prepare(
`INSERT INTO imm_lifetime_media(
video_id, total_sessions, total_active_ms, total_cards, total_lines_seen,
total_tokens_seen, completed, first_watched_ms, last_watched_ms, CREATED_DATE, LAST_UPDATE_DATE
)
VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(video_id) DO UPDATE SET
total_sessions = total_sessions + 1,
total_active_ms = total_active_ms + excluded.total_active_ms,
total_cards = total_cards + excluded.total_cards,
total_lines_seen = total_lines_seen + excluded.total_lines_seen,
total_tokens_seen = total_tokens_seen + excluded.total_tokens_seen,
completed = MAX(completed, excluded.completed),
first_watched_ms = CASE
WHEN excluded.first_watched_ms IS NULL THEN first_watched_ms
WHEN first_watched_ms IS NULL THEN excluded.first_watched_ms
WHEN excluded.first_watched_ms < first_watched_ms THEN excluded.first_watched_ms
ELSE first_watched_ms
END,
last_watched_ms = CASE
WHEN excluded.last_watched_ms IS NULL THEN last_watched_ms
WHEN last_watched_ms IS NULL THEN excluded.last_watched_ms
WHEN excluded.last_watched_ms > last_watched_ms THEN excluded.last_watched_ms
ELSE last_watched_ms
END,
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE`,
)
.run(
videoId,
activeMs,
cardsMined,
linesSeen,
tokensSeen,
watched > 0 ? 1 : 0,
session.started_at_ms,
session.ended_at_ms,
updatedAtMs,
updatedAtMs,
);
if (animeId !== null) {
local
.prepare(
`INSERT INTO imm_lifetime_anime(
anime_id, total_sessions, total_active_ms, total_cards, total_lines_seen,
total_tokens_seen, episodes_started, episodes_completed, first_watched_ms,
last_watched_ms, CREATED_DATE, LAST_UPDATE_DATE
)
VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(anime_id) DO UPDATE SET
total_sessions = total_sessions + 1,
total_active_ms = total_active_ms + excluded.total_active_ms,
total_cards = total_cards + excluded.total_cards,
total_lines_seen = total_lines_seen + excluded.total_lines_seen,
total_tokens_seen = total_tokens_seen + excluded.total_tokens_seen,
episodes_started = episodes_started + excluded.episodes_started,
episodes_completed = episodes_completed + excluded.episodes_completed,
first_watched_ms = CASE
WHEN excluded.first_watched_ms IS NULL THEN first_watched_ms
WHEN first_watched_ms IS NULL THEN excluded.first_watched_ms
WHEN excluded.first_watched_ms < first_watched_ms THEN excluded.first_watched_ms
ELSE first_watched_ms
END,
last_watched_ms = CASE
WHEN excluded.last_watched_ms IS NULL THEN last_watched_ms
WHEN last_watched_ms IS NULL THEN excluded.last_watched_ms
WHEN excluded.last_watched_ms > last_watched_ms THEN excluded.last_watched_ms
ELSE last_watched_ms
END,
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE`,
)
.run(
animeId,
activeMs,
cardsMined,
linesSeen,
tokensSeen,
isFirstSessionForVideoRun ? 1 : 0,
isFirstCompletedSessionForVideoRun ? 1 : 0,
session.started_at_ms,
session.ended_at_ms,
updatedAtMs,
updatedAtMs,
);
}
}
launcher/sync/ssh.test.ts
+51
View File
@@ -0,0 +1,51 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { assertSafeSshHost, resolveRemoteSubminerCommand, runScp, shellQuote } from './ssh.js';
test('assertSafeSshHost rejects option-like hosts', () => {
assert.throws(() => assertSafeSshHost('-oProxyCommand=touch pwned'), /looks like an option/);
assert.throws(() => assertSafeSshHost('-lroot'), /looks like an option/);
});
test('assertSafeSshHost accepts normal destinations', () => {
assert.doesNotThrow(() => assertSafeSshHost('macbook'));
assert.doesNotThrow(() => assertSafeSshHost('user@192.168.1.20'));
assert.doesNotThrow(() => assertSafeSshHost('ssh-alias'));
});
test('shellQuote escapes single quotes and wraps in quotes', () => {
assert.equal(shellQuote('subminer'), `'subminer'`);
assert.equal(shellQuote(`a'; rm -rf ~; '`), `'a'\\''; rm -rf ~; '\\'''`);
});
test('runScp rejects option-like local endpoints before spawning scp', () => {
assert.throws(() => runScp('-oProxyCommand=sh', '/tmp/out.sqlite'), /looks like an option/);
assert.throws(() => runScp('/tmp/in.sqlite', '-bad-destination'), /looks like an option/);
});
test('runScp rejects option-like remote host components', () => {
assert.throws(
() => runScp('-oProxyCommand=sh:/tmp/in.sqlite', '/tmp/out.sqlite'),
/SSH host that looks like an option/,
);
});
test('resolveRemoteSubminerCommand verifies the launcher under the remote runtime PATH', () => {
const calls: Array<{ host: string; remoteCommand: string }> = [];
const command = resolveRemoteSubminerCommand('macbook', null, (host, remoteCommand) => {
calls.push({ host, remoteCommand });
return { status: 0, stdout: '', stderr: '' };
});
assert.equal(
command,
'PATH="$HOME/.local/bin:$HOME/.bun/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$PATH" subminer',
);
assert.deepEqual(calls, [
{
host: 'macbook',
remoteCommand:
'PATH="$HOME/.local/bin:$HOME/.bun/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$PATH" subminer --help >/dev/null 2>&1',
},
]);
});
launcher/sync/ssh.ts
+106
View File
@@ -0,0 +1,106 @@
import { spawnSync } from 'node:child_process';
export interface RemoteRunResult {
status: number;
stdout: string;
stderr: string;
}
/**
* ssh/scp have no `--` terminator for the destination, so a host that starts
* with `-` (e.g. `-oProxyCommand=...`) is parsed as an option. Reject those
* before spawning.
*/
export function assertSafeSshHost(host: string): void {
if (host.startsWith('-')) {
throw new Error(`Refusing to use SSH host that looks like an option: ${host}`);
}
}
/**
* Run a command on the SSH host. stdin stays attached so interactive prompts
* can still read from the terminal; stdout/stderr are captured for callers
* that need actionable remote failure messages.
*/
export function runSsh(host: string, remoteCommand: string): RemoteRunResult {
assertSafeSshHost(host);
const result = spawnSync('ssh', [host, remoteCommand], {
encoding: 'utf8',
stdio: ['inherit', 'pipe', 'pipe'],
});
if (result.error) {
throw new Error(`Failed to run ssh: ${(result.error as Error).message}`);
}
return { status: result.status ?? 1, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
}
function assertSafeScpEndpoint(endpoint: string): void {
const colon = endpoint.indexOf(':');
const slash = endpoint.indexOf('/');
if (colon <= 0 || (slash !== -1 && slash < colon)) {
if (endpoint.startsWith('-')) {
throw new Error(`Refusing to use scp endpoint that looks like an option: ${endpoint}`);
}
return;
}
const host = endpoint.slice(0, colon);
const remotePath = endpoint.slice(colon + 1);
assertSafeSshHost(host);
if (remotePath.startsWith('-')) {
throw new Error(`Refusing to use scp remote path that looks like an option: ${remotePath}`);
}
}
export function runScp(from: string, to: string): void {
assertSafeScpEndpoint(from);
assertSafeScpEndpoint(to);
const result = spawnSync('scp', ['-q', from, to], {
encoding: 'utf8',
stdio: ['inherit', 'inherit', 'inherit'],
});
if (result.error) {
throw new Error(`Failed to run scp: ${(result.error as Error).message}`);
}
if ((result.status ?? 1) !== 0) {
throw new Error(`scp failed copying ${from} -> ${to}`);
}
}
export function shellQuote(value: string): string {
return `'${value.replaceAll("'", `'\\''`)}'`;
}
const REMOTE_RUNTIME_PATH =
'PATH="$HOME/.local/bin:$HOME/.bun/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$PATH"';
/**
* Non-interactive SSH shells often miss user-installed launchers and Bun.
* Probe the launcher under the same deterministic PATH used by sync itself.
*/
export function resolveRemoteSubminerCommand(
host: string,
preferred: string | null,
runRemote: typeof runSsh = runSsh,
): string {
// Trusted defaults stay unquoted so the remote shell expands `~`; a
// user-supplied override is shell-quoted to prevent command injection.
const candidates: Array<{ value: string; invocation: string }> = preferred
? [{ value: preferred, invocation: shellQuote(preferred) }]
: [
{ value: 'subminer', invocation: 'subminer' },
{ value: '~/.local/bin/subminer', invocation: '~/.local/bin/subminer' },
];
for (const candidate of candidates) {
const command = `${REMOTE_RUNTIME_PATH} ${candidate.invocation}`;
const probe = runRemote(host, `${command} --help >/dev/null 2>&1`);
if (probe.status === 0) {
return command;
}
}
throw new Error(
preferred
? `Remote command not found on ${host}: ${preferred}`
: `subminer not found on ${host} (tried PATH and ~/.local/bin/subminer). Pass --remote-cmd <path>.`,
);
}
launcher/sync/sync-db.test.ts
+598
View File
@@ -0,0 +1,598 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { Database } from 'bun:sqlite';
import { createDbSnapshot, mergeSnapshotIntoDb } from './sync-db.js';
import {
createImmersionDbFixture,
insertFixtureSession,
} from '../test-support/immersion-db-fixture.js';
const DAY_MS = 86_400_000;
const BASE_MS = Date.UTC(2026, 5, 1, 12, 0, 0);
function makeTmpDir(): string {
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-sync-test-'));
}
function makeDbPair(): { dir: string; localPath: string; remotePath: string } {
const dir = makeTmpDir();
const localPath = path.join(dir, 'local.sqlite');
const remotePath = path.join(dir, 'remote.sqlite');
createImmersionDbFixture(localPath);
createImmersionDbFixture(remotePath);
return { dir, localPath, remotePath };
}
function queryOne<T extends Record<string, unknown>>(
dbPath: string,
sql: string,
params: unknown[] = [],
): T | undefined {
const db = new Database(dbPath, { readonly: true });
try {
return db.query<T>(sql).get(...params) as T | undefined;
} finally {
db.close();
}
}
function count(dbPath: string, sql: string, params: unknown[] = []): number {
return Number(queryOne<{ n: number }>(dbPath, sql, params)?.n ?? 0);
}
function withWritableDb<T>(dbPath: string, fn: (db: Database) => T): T {
const db = new Database(dbPath, { readwrite: true });
try {
return fn(db);
} finally {
db.close();
}
}
test('merges remote-only sessions with catalog, lifetime, and rollups', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
insertFixtureSession(localPath, {
uuid: 'local-1',
videoKey: 'showa-e1',
animeTitleKey: 'showa',
startedAtMs: BASE_MS,
applyLifetime: true,
words: [{ headword: '見る', word: '見た', reading: 'みた', count: 2 }],
});
insertFixtureSession(remotePath, {
uuid: 'remote-1',
videoKey: 'showb-e1',
animeTitleKey: 'showb',
startedAtMs: BASE_MS + DAY_MS,
activeWatchedMs: 900_000,
cardsMined: 3,
applyLifetime: true,
words: [
{ headword: '見る', word: '見た', reading: 'みた', count: 5 },
{ headword: '食べる', word: '食べた', reading: 'たべた', count: 1 },
],
});
const summary = mergeSnapshotIntoDb(localPath, remotePath);
assert.equal(summary.sessionsMerged, 1);
assert.equal(summary.animeAdded, 1);
assert.equal(summary.videosAdded, 1);
assert.equal(summary.wordsAdded, 1);
assert.equal(summary.subtitleLinesAdded, 2);
assert.equal(summary.telemetryRowsAdded, 1);
assert.equal(count(localPath, 'SELECT COUNT(*) AS n FROM imm_sessions'), 2);
const global = queryOne<{
total_sessions: number;
total_active_ms: number;
total_cards: number;
active_days: number;
episodes_started: number;
}>(
localPath,
'SELECT total_sessions, total_active_ms, total_cards, active_days, episodes_started FROM imm_lifetime_global WHERE global_id = 1',
);
assert.equal(global?.total_sessions, 2);
assert.equal(global?.total_active_ms, 1_200_000 + 900_000);
assert.equal(global?.total_cards, 2 + 3);
assert.equal(global?.active_days, 2);
assert.equal(global?.episodes_started, 2);
// Existing word: local 2 + merged 5; new word carries remote frequency.
const sharedWord = queryOne<{ frequency: number }>(
localPath,
`SELECT frequency FROM imm_words WHERE word = '見た'`,
);
assert.equal(sharedWord?.frequency, 7);
const newWord = queryOne<{ frequency: number }>(
localPath,
`SELECT frequency FROM imm_words WHERE word = '食べた'`,
);
assert.equal(newWord?.frequency, 1);
// The merged session's rollup group was recomputed.
assert.equal(summary.rollupGroupsRecomputed, 1);
const mergedVideoId = Number(
queryOne<{ video_id: number }>(
localPath,
`SELECT video_id FROM imm_videos WHERE video_key = 'showb-e1'`,
)?.video_id,
);
assert.equal(
count(localPath, 'SELECT COUNT(*) AS n FROM imm_daily_rollups WHERE video_id = ?', [
mergedVideoId,
]),
1,
);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('is idempotent: re-merging the same snapshot changes nothing', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
insertFixtureSession(remotePath, {
uuid: 'remote-1',
videoKey: 'showa-e1',
animeTitleKey: 'showa',
startedAtMs: BASE_MS,
applyLifetime: true,
words: [{ headword: '見る', word: '見た', reading: 'みた', count: 4 }],
});
mergeSnapshotIntoDb(localPath, remotePath);
const globalAfterFirst = queryOne<Record<string, unknown>>(
localPath,
'SELECT * FROM imm_lifetime_global WHERE global_id = 1',
);
const summary = mergeSnapshotIntoDb(localPath, remotePath);
assert.equal(summary.sessionsMerged, 0);
assert.equal(summary.sessionsAlreadyPresent, 1);
assert.equal(summary.wordsAdded, 0);
assert.equal(count(localPath, 'SELECT COUNT(*) AS n FROM imm_sessions'), 1);
assert.equal(count(localPath, 'SELECT COUNT(*) AS n FROM imm_subtitle_lines'), 1);
const globalAfterSecond = queryOne<Record<string, unknown>>(
localPath,
'SELECT * FROM imm_lifetime_global WHERE global_id = 1',
);
assert.deepEqual(
{ ...globalAfterSecond, LAST_UPDATE_DATE: null },
{ ...globalAfterFirst, LAST_UPDATE_DATE: null },
);
assert.equal(
Number(
queryOne<{ frequency: number }>(
localPath,
`SELECT frequency FROM imm_words WHERE word = '見た'`,
)?.frequency,
),
4,
);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('preserves anilist_id when inserting a new anime from the snapshot', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
insertFixtureSession(remotePath, {
uuid: 'remote-1',
videoKey: 'showb-e1',
animeTitleKey: 'showb',
startedAtMs: BASE_MS,
applyLifetime: true,
});
withWritableDb(remotePath, (remoteDb) => {
remoteDb
.prepare(`UPDATE imm_anime SET anilist_id = 12345 WHERE normalized_title_key = 'showb'`)
.run();
});
const summary = mergeSnapshotIntoDb(localPath, remotePath);
assert.equal(summary.animeAdded, 1);
const anime = queryOne<{ anilist_id: number }>(
localPath,
`SELECT anilist_id FROM imm_anime WHERE normalized_title_key = 'showb'`,
);
assert.equal(anime?.anilist_id, 12345);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('matches an existing local anime by anilist_id even when the title key differs', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
insertFixtureSession(localPath, {
uuid: 'local-1',
videoKey: 'showa-e1',
animeTitleKey: 'show-romaji',
startedAtMs: BASE_MS,
applyLifetime: true,
});
insertFixtureSession(remotePath, {
uuid: 'remote-1',
videoKey: 'showa-e2',
animeTitleKey: 'show-native',
startedAtMs: BASE_MS + DAY_MS,
applyLifetime: true,
});
withWritableDb(localPath, (localDb) => {
localDb
.prepare(`UPDATE imm_anime SET anilist_id = 999 WHERE normalized_title_key = 'show-romaji'`)
.run();
});
withWritableDb(remotePath, (remoteDb) => {
remoteDb
.prepare(`UPDATE imm_anime SET anilist_id = 999 WHERE normalized_title_key = 'show-native'`)
.run();
});
const summary = mergeSnapshotIntoDb(localPath, remotePath);
// Same anilist_id → one anime, not two.
assert.equal(summary.animeAdded, 0);
assert.equal(count(localPath, 'SELECT COUNT(*) AS n FROM imm_anime'), 1);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('matches shared videos by video_key and merges the watched flag', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
insertFixtureSession(localPath, {
uuid: 'local-1',
videoKey: 'showa-e1',
animeTitleKey: 'showa',
startedAtMs: BASE_MS,
applyLifetime: true,
});
insertFixtureSession(remotePath, {
uuid: 'remote-1',
videoKey: 'showa-e1',
animeTitleKey: 'showa',
startedAtMs: BASE_MS + DAY_MS,
watched: true,
applyLifetime: true,
});
const summary = mergeSnapshotIntoDb(localPath, remotePath);
assert.equal(summary.videosAdded, 0);
assert.equal(summary.animeAdded, 0);
assert.equal(count(localPath, 'SELECT COUNT(*) AS n FROM imm_videos'), 1);
const video = queryOne<{ watched: number }>(
localPath,
`SELECT watched FROM imm_videos WHERE video_key = 'showa-e1'`,
);
assert.equal(video?.watched, 1);
// Both sessions now credit the same video; episode was started once and
// completed once (by the remote session that watched it to the end).
const global = queryOne<{
episodes_started: number;
episodes_completed: number;
total_sessions: number;
}>(
localPath,
'SELECT episodes_started, episodes_completed, total_sessions FROM imm_lifetime_global WHERE global_id = 1',
);
assert.equal(global?.total_sessions, 2);
assert.equal(global?.episodes_started, 1);
assert.equal(global?.episodes_completed, 1);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('backfills media metadata for matched videos when local metadata is absent', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
insertFixtureSession(localPath, {
uuid: 'local-1',
videoKey: 'showa-e1',
animeTitleKey: 'showa',
startedAtMs: BASE_MS,
applyLifetime: true,
});
insertFixtureSession(remotePath, {
uuid: 'remote-1',
videoKey: 'showa-e1',
animeTitleKey: 'showa',
startedAtMs: BASE_MS + DAY_MS,
applyLifetime: true,
});
const remoteDb = new Database(remotePath, { readwrite: true });
try {
const remoteVideoId = Number(
remoteDb
.query<{
video_id: number;
}>(`SELECT video_id FROM imm_videos WHERE video_key = 'showa-e1'`)
.get()?.video_id,
);
remoteDb
.prepare(
`INSERT INTO imm_media_art (video_id, anilist_id, cover_url, title_romaji, fetched_at_ms)
VALUES (?, 123, 'https://example.test/cover.jpg', 'Show A', ?)`,
)
.run(remoteVideoId, String(BASE_MS));
remoteDb
.prepare(
`INSERT INTO imm_youtube_videos (video_id, youtube_video_id, video_url, video_title, fetched_at_ms)
VALUES (?, 'yt-1', 'https://youtube.test/watch?v=yt-1', 'Remote Video', ?)`,
)
.run(remoteVideoId, String(BASE_MS));
} finally {
remoteDb.close();
}
const summary = mergeSnapshotIntoDb(localPath, remotePath);
assert.equal(summary.videosAdded, 0);
const localVideoId = Number(
queryOne<{ video_id: number }>(
localPath,
`SELECT video_id FROM imm_videos WHERE video_key = 'showa-e1'`,
)?.video_id,
);
const art = queryOne<{ cover_url: string; title_romaji: string }>(
localPath,
'SELECT cover_url, title_romaji FROM imm_media_art WHERE video_id = ?',
[localVideoId],
);
assert.equal(art?.cover_url, 'https://example.test/cover.jpg');
assert.equal(art?.title_romaji, 'Show A');
const youtube = queryOne<{ youtube_video_id: string; video_title: string }>(
localPath,
'SELECT youtube_video_id, video_title FROM imm_youtube_videos WHERE video_id = ?',
[localVideoId],
);
assert.equal(youtube?.youtube_video_id, 'yt-1');
assert.equal(youtube?.video_title, 'Remote Video');
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('does not double-count active_days when a merged session starts earlier on an already-credited day', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
insertFixtureSession(localPath, {
uuid: 'local-1',
videoKey: 'showa-e1',
startedAtMs: BASE_MS + 3_600_000,
applyLifetime: true,
});
insertFixtureSession(remotePath, {
uuid: 'remote-1',
videoKey: 'showb-e1',
startedAtMs: BASE_MS,
applyLifetime: true,
});
mergeSnapshotIntoDb(localPath, remotePath);
const global = queryOne<{ active_days: number; total_sessions: number }>(
localPath,
'SELECT active_days, total_sessions FROM imm_lifetime_global WHERE global_id = 1',
);
assert.equal(global?.total_sessions, 2);
assert.equal(global?.active_days, 1);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('skips unfinished sessions', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
insertFixtureSession(remotePath, {
uuid: 'remote-active',
videoKey: 'showa-e1',
startedAtMs: BASE_MS,
endedAtMs: null,
});
const summary = mergeSnapshotIntoDb(localPath, remotePath);
assert.equal(summary.sessionsMerged, 0);
assert.equal(summary.activeSessionsSkipped, 1);
assert.equal(count(localPath, 'SELECT COUNT(*) AS n FROM imm_sessions'), 0);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('copies remote-only historical rollups but never sums shared groups', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
// Remote has a video plus an old rollup row whose sessions were pruned.
insertFixtureSession(remotePath, {
uuid: 'remote-1',
videoKey: 'old-show-e1',
startedAtMs: BASE_MS,
applyLifetime: true,
});
withWritableDb(remotePath, (remoteDb) => {
const remoteVideoId = Number(
remoteDb
.query<{
video_id: number;
}>(`SELECT video_id FROM imm_videos WHERE video_key = 'old-show-e1'`)
.get()?.video_id,
);
remoteDb
.prepare(
`INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards)
VALUES (10000, ?, 4, 120.5, 400, 3200, 9)`,
)
.run(remoteVideoId);
});
// Local already has its own rollup row for a shared group.
insertFixtureSession(localPath, {
uuid: 'local-1',
videoKey: 'old-show-e1',
startedAtMs: BASE_MS,
applyLifetime: true,
});
const localVideoId = withWritableDb(localPath, (localDb) => {
const videoId = Number(
localDb
.query<{
video_id: number;
}>(`SELECT video_id FROM imm_videos WHERE video_key = 'old-show-e1'`)
.get()?.video_id,
);
localDb
.prepare(
`INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards)
VALUES (10000, ?, 2, 60.0, 200, 1600, 4)`,
)
.run(videoId);
return videoId;
});
const summary = mergeSnapshotIntoDb(localPath, remotePath);
// Shared group (day 10000) untouched; the remote-only session's own group
// was recomputed, not copied.
assert.equal(summary.dailyRollupsCopied, 0);
const shared = queryOne<{ total_sessions: number }>(
localPath,
'SELECT total_sessions FROM imm_daily_rollups WHERE rollup_day = 10000 AND video_id = ?',
[localVideoId],
);
assert.equal(shared?.total_sessions, 2);
// Now a rollup for a video with no local sessions at all gets copied.
withWritableDb(remotePath, (remoteDb2) => {
const orphanVideoId = Number(
remoteDb2
.prepare(
`INSERT INTO imm_videos (video_key, canonical_title, source_type, watched, duration_ms)
VALUES ('pruned-show-e1', 'pruned-show-e1', 1, 1, 1440000)`,
)
.run().lastInsertRowid,
);
remoteDb2
.prepare(
`INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards)
VALUES (9000, ?, 3, 90.0, 300, 2400, 6)`,
)
.run(orphanVideoId);
});
const summary2 = mergeSnapshotIntoDb(localPath, remotePath);
assert.equal(summary2.dailyRollupsCopied, 1);
const copiedVideoId = Number(
queryOne<{ video_id: number }>(
localPath,
`SELECT video_id FROM imm_videos WHERE video_key = 'pruned-show-e1'`,
)?.video_id,
);
const copied = queryOne<{ total_sessions: number; total_active_min: number }>(
localPath,
'SELECT total_sessions, total_active_min FROM imm_daily_rollups WHERE rollup_day = 9000 AND video_id = ?',
[copiedVideoId],
);
assert.equal(copied?.total_sessions, 3);
assert.equal(copied?.total_active_min, 90.0);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('does not copy remote-only daily rollups into months with local sessions', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
insertFixtureSession(localPath, {
uuid: 'local-1',
videoKey: 'mixed-month-e1',
animeTitleKey: 'mixed-month',
startedAtMs: BASE_MS,
applyLifetime: true,
});
const remoteDb = new Database(remotePath, { readwrite: true });
try {
const remoteVideoId = Number(
remoteDb
.prepare(
`INSERT INTO imm_videos (video_key, canonical_title, source_type, watched, duration_ms)
VALUES ('mixed-month-e1', 'mixed-month-e1', 1, 1, 1440000)`,
)
.run().lastInsertRowid,
);
remoteDb
.prepare(
`INSERT INTO imm_daily_rollups (rollup_day, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards)
VALUES (20615, ?, 3, 90.0, 300, 2400, 6)`,
)
.run(remoteVideoId);
remoteDb
.prepare(
`INSERT INTO imm_monthly_rollups (rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, total_tokens_seen, total_cards)
VALUES (202606, ?, 3, 90.0, 300, 2400, 6)`,
)
.run(remoteVideoId);
} finally {
remoteDb.close();
}
const summary = mergeSnapshotIntoDb(localPath, remotePath);
assert.equal(summary.dailyRollupsCopied, 0);
assert.equal(summary.monthlyRollupsCopied, 0);
const localVideoId = Number(
queryOne<{ video_id: number }>(
localPath,
`SELECT video_id FROM imm_videos WHERE video_key = 'mixed-month-e1'`,
)?.video_id,
);
assert.equal(
count(
localPath,
'SELECT COUNT(*) AS n FROM imm_daily_rollups WHERE rollup_day = 20615 AND video_id = ?',
[localVideoId],
),
0,
);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('rejects snapshots at a different schema version', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
withWritableDb(remotePath, (remoteDb) => {
remoteDb.prepare('UPDATE imm_schema_version SET schema_version = 17').run();
});
assert.throws(() => mergeSnapshotIntoDb(localPath, remotePath), /schema version 17/);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('createDbSnapshot produces a mergeable copy', () => {
const { dir, localPath, remotePath } = makeDbPair();
try {
insertFixtureSession(remotePath, {
uuid: 'remote-1',
videoKey: 'showa-e1',
startedAtMs: BASE_MS,
applyLifetime: true,
});
const snapshotPath = path.join(dir, 'snapshot.sqlite');
createDbSnapshot(remotePath, snapshotPath);
assert.ok(fs.existsSync(snapshotPath));
const summary = mergeSnapshotIntoDb(localPath, snapshotPath);
assert.equal(summary.sessionsMerged, 1);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
launcher/sync/sync-db.ts
+105
View File
@@ -0,0 +1,105 @@
import fs from 'node:fs';
import { Database } from 'bun:sqlite';
import {
LexiconResolver,
mergeAnime,
mergeExcludedWords,
mergeMediaMetadata,
mergeVideos,
} from './merge-catalog.js';
import { mergeSessions } from './merge-sessions.js';
import { copyRemoteOnlyRollups, refreshRollupsForNewSessions } from './merge-rollups.js';
import {
assertMergeableSchema,
createEmptyMergeSummary,
type SyncMergeSummary,
} from './sync-shared.js';
export type { SyncMergeSummary } from './sync-shared.js';
export { createDbSnapshot, findLiveStatsDaemonPid } from './sync-shared.js';
/**
* Merge a snapshot of another machine's immersion database into the local
* one. Insert-only union keyed on natural keys (session_uuid, video_key,
* normalized_title_key, word/kanji identity); lifetime and rollup aggregates
* are updated incrementally so history older than the session retention
* window is preserved on both sides. Idempotent: re-merging the same
* snapshot is a no-op.
*/
export function mergeSnapshotIntoDb(localDbPath: string, snapshotPath: string): SyncMergeSummary {
if (!fs.existsSync(localDbPath)) {
throw new Error(`Local stats database not found: ${localDbPath}`);
}
if (!fs.existsSync(snapshotPath)) {
throw new Error(`Snapshot database not found: ${snapshotPath}`);
}
const remote = new Database(snapshotPath, { readonly: true });
let local: Database;
try {
local = new Database(localDbPath, { readwrite: true, create: false });
} catch (error) {
remote.close();
throw error;
}
try {
assertMergeableSchema(remote, 'Snapshot');
assertMergeableSchema(local, 'Local');
const summary = createEmptyMergeSummary();
local.run('PRAGMA foreign_keys = ON');
local.run('PRAGMA busy_timeout = 5000');
local.run('BEGIN IMMEDIATE');
try {
const animeIdMap = mergeAnime(local, remote, summary);
const { videoIdMap, addedVideoIds } = mergeVideos(local, remote, animeIdMap, summary);
mergeMediaMetadata(local, remote, videoIdMap, addedVideoIds);
mergeExcludedWords(local, remote, summary);
const lexicon = new LexiconResolver(local, remote, summary);
const { newSessionIds } = mergeSessions(
local,
remote,
videoIdMap,
animeIdMap,
lexicon,
summary,
);
lexicon.applyFrequencyDeltas();
refreshRollupsForNewSessions(local, newSessionIds, summary);
copyRemoteOnlyRollups(local, remote, videoIdMap, summary);
local.run('COMMIT');
return summary;
} catch (error) {
local.run('ROLLBACK');
throw error;
}
} finally {
local.close();
remote.close();
}
}
export function formatMergeSummary(summary: SyncMergeSummary): string {
const lines = [
`Sessions merged: ${summary.sessionsMerged} (${summary.sessionsAlreadyPresent} already present, ${summary.activeSessionsSkipped} unfinished skipped)`,
];
const detail: string[] = [];
if (summary.animeAdded) detail.push(`${summary.animeAdded} series`);
if (summary.videosAdded) detail.push(`${summary.videosAdded} videos`);
if (summary.wordsAdded) detail.push(`${summary.wordsAdded} words`);
if (summary.kanjiAdded) detail.push(`${summary.kanjiAdded} kanji`);
if (summary.subtitleLinesAdded) detail.push(`${summary.subtitleLinesAdded} subtitle lines`);
if (summary.excludedWordsAdded) detail.push(`${summary.excludedWordsAdded} excluded words`);
if (detail.length > 0) lines.push(`Added: ${detail.join(', ')}`);
if (summary.dailyRollupsCopied || summary.monthlyRollupsCopied) {
lines.push(
`Historical rollups copied: ${summary.dailyRollupsCopied} daily, ${summary.monthlyRollupsCopied} monthly`,
);
}
if (summary.rollupGroupsRecomputed) {
lines.push(`Rollup groups recomputed: ${summary.rollupGroupsRecomputed}`);
}
return lines.join('\n');
}
launcher/sync/sync-shared.ts
+173
View File
@@ -0,0 +1,173 @@
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { Database } from 'bun:sqlite';
import { SCHEMA_VERSION } from '../../src/core/services/immersion-tracker/types.js';
import { withReadonlyWalRetry } from '../history-db.js';
import { resolveConfigDir } from '../../src/config/path-resolution.js';
export { SCHEMA_VERSION };
export interface SyncMergeSummary {
sessionsMerged: number;
sessionsAlreadyPresent: number;
activeSessionsSkipped: number;
animeAdded: number;
videosAdded: number;
wordsAdded: number;
kanjiAdded: number;
subtitleLinesAdded: number;
telemetryRowsAdded: number;
eventsAdded: number;
excludedWordsAdded: number;
dailyRollupsCopied: number;
monthlyRollupsCopied: number;
rollupGroupsRecomputed: number;
}
export function createEmptyMergeSummary(): SyncMergeSummary {
return {
sessionsMerged: 0,
sessionsAlreadyPresent: 0,
activeSessionsSkipped: 0,
animeAdded: 0,
videosAdded: 0,
wordsAdded: 0,
kanjiAdded: 0,
subtitleLinesAdded: 0,
telemetryRowsAdded: 0,
eventsAdded: 0,
excludedWordsAdded: 0,
dailyRollupsCopied: 0,
monthlyRollupsCopied: 0,
rollupGroupsRecomputed: 0,
};
}
export function nowDbTimestamp(): string {
return String(Date.now());
}
export function tableExists(db: Database, tableName: string): boolean {
return Boolean(
db.query(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`).get(tableName),
);
}
export function readSchemaVersion(db: Database): number | null {
if (!tableExists(db, 'imm_schema_version')) return null;
const row = db
.query<{ schema_version: number }>(
'SELECT MAX(schema_version) AS schema_version FROM imm_schema_version',
)
.get();
return typeof row?.schema_version === 'number' ? row.schema_version : null;
}
export function assertMergeableSchema(db: Database, label: string): void {
const version = readSchemaVersion(db);
if (version === null) {
throw new Error(
`${label} database has no schema version. Run SubMiner once on that machine so the stats database is initialized.`,
);
}
if (version !== SCHEMA_VERSION) {
throw new Error(
`${label} database is at schema version ${version} but this launcher expects ${SCHEMA_VERSION}. Update SubMiner on both machines to the same version and run each app once before syncing.`,
);
}
for (const table of ['imm_sessions', 'imm_videos', 'imm_lifetime_global']) {
if (!tableExists(db, table)) {
throw new Error(`${label} database is missing table ${table}; cannot sync.`);
}
}
}
export function insertRow(
db: Database,
table: string,
columns: readonly string[],
values: unknown[],
): number {
const sql = `INSERT INTO ${table} (${columns.join(', ')}) VALUES (${columns.map(() => '?').join(', ')})`;
// db.query() caches the prepared statement per SQL string; this runs once
// per copied row, so re-preparing via db.prepare() would dominate merge time.
const result = db.query(sql).run(...values);
return Number(result.lastInsertRowid);
}
export function createDbSnapshot(dbPath: string, outPath: string): void {
if (!fs.existsSync(dbPath)) {
throw new Error(`Stats database not found: ${dbPath}`);
}
fs.rmSync(outPath, { force: true });
fs.mkdirSync(path.dirname(outPath), { recursive: true });
withReadonlyWalRetry(dbPath, (options) => {
const db = new Database(dbPath, options);
try {
assertMergeableSchema(db, 'Local');
db.prepare('VACUUM INTO ?').run(outPath);
} finally {
db.close();
}
});
}
interface DaemonStateFile {
pid?: unknown;
}
function isProcessAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch (error) {
// EPERM means the process exists but we can't signal it → still alive.
// Only ESRCH (no such process) means it's actually gone.
return (error as NodeJS.ErrnoException)?.code === 'EPERM';
}
}
function statsDaemonStateCandidates(dbPath: string): string[] {
const homeDir = os.homedir();
const candidates = new Set<string>([path.join(path.dirname(dbPath), 'stats-daemon.json')]);
const configDir = resolveConfigDir({
platform: process.platform,
appDataDir: process.env.APPDATA,
xdgConfigHome: process.env.XDG_CONFIG_HOME,
homeDir,
existsSync: fs.existsSync,
});
candidates.add(path.join(configDir, 'stats-daemon.json'));
if (process.platform === 'darwin') {
candidates.add(path.join(homeDir, 'Library', 'Application Support', 'SubMiner', 'stats-daemon.json'));
}
return [...candidates];
}
/**
* Best-effort guard against merging while a SubMiner process holds the
* tracker's write queue in memory. Detects the background stats daemon via
* its pid state file; the interactive app is caught by the mpv-socket check
* in the sync command.
*/
export function findLiveStatsDaemonPid(dbPath: string): number | null {
for (const statePath of statsDaemonStateCandidates(dbPath)) {
let raw: string;
try {
raw = fs.readFileSync(statePath, 'utf8');
} catch {
continue;
}
try {
const parsed = JSON.parse(raw) as DaemonStateFile;
const pid = typeof parsed.pid === 'number' && Number.isInteger(parsed.pid) ? parsed.pid : 0;
if (pid > 0 && isProcessAlive(pid)) {
return pid;
}
} catch {
continue;
}
}
return null;
}