From f8c10edce0bddfa31bb6d103deebd10af1d5d095 Mon Sep 17 00:00:00 2001 From: sudacode Date: Sun, 12 Jul 2026 02:18:41 -0700 Subject: [PATCH] fix(sync): adopted lexicon frequency no longer double-counts active-session lines The remote tracker increments imm_words/imm_kanji.frequency live, so a snapshot holding a stale ACTIVE session (skipped by the merge) has that session's partial occurrences baked into frequency. A word new to the local DB adopted that full total, then received the same occurrences again when the session finalized and merged on a later sync. Subtract active-session occurrence counts when adopting a new row; they are re-added exactly once when the session completes. Existing rows were already safe (isNew guard in addWordOccurrences/addKanjiOccurrences). Verified red/green: the new two-sync regression test fails without the subtraction (frequency 9 instead of 5). --- ...c-active-session-frequency-double-count.md | 4 ++ launcher/sync/engine-merge.test.ts | 58 +++++++++++++++++++ src/core/services/stats-sync/merge-catalog.ts | 53 +++++++++++++++-- 3 files changed, 110 insertions(+), 5 deletions(-) create mode 100644 changes/sync-active-session-frequency-double-count.md diff --git a/changes/sync-active-session-frequency-double-count.md b/changes/sync-active-session-frequency-double-count.md new file mode 100644 index 00000000..99e227e0 --- /dev/null +++ b/changes/sync-active-session-frequency-double-count.md @@ -0,0 +1,4 @@ +type: fixed +area: sync + +- Fixed word/kanji frequencies double-counting across syncs when the remote snapshot contained a stale active session (e.g. after a crash): a word new to the local machine adopted the remote's full lifetime frequency, which already included the active session's partial occurrences, and those occurrences were added again when the session finalized and synced. Newly adopted words/kanji now exclude active-session counts, which arrive once the session completes. diff --git a/launcher/sync/engine-merge.test.ts b/launcher/sync/engine-merge.test.ts index d76b0123..25402040 100644 --- a/launcher/sync/engine-merge.test.ts +++ b/launcher/sync/engine-merge.test.ts @@ -606,3 +606,61 @@ test('createDbSnapshot produces a mergeable copy', () => { fs.rmSync(dir, { recursive: true, force: true }); } }); + +test('adopted word frequency excludes active-session counts that merge later', () => { + const { dir, localPath, remotePath } = makeDbPair(); + try { + // Remote has an ended session and a stale ACTIVE one (e.g. app crashed). + // The remote tracker increments frequency live, so the word's frequency (5) + // already includes the active session's 4 occurrences even though that + // session's lines are skipped by the merge. + insertFixtureSession(remotePath, { + uuid: 'remote-ended', + videoKey: 'showb-e1', + animeTitleKey: 'showb', + startedAtMs: BASE_MS, + applyLifetime: true, + words: [{ headword: '食べる', word: '食べた', reading: 'たべた', count: 1 }], + }); + insertFixtureSession(remotePath, { + uuid: 'remote-active', + videoKey: 'showb-e2', + animeTitleKey: 'showb', + startedAtMs: BASE_MS + DAY_MS, + endedAtMs: null, + words: [{ headword: '食べる', word: '食べた', reading: 'たべた', count: 4 }], + }); + + const first = mergeSnapshotIntoDb(localPath, remotePath); + assert.equal(first.sessionsMerged, 1); + assert.equal(first.activeSessionsSkipped, 1); + assert.equal(first.wordsAdded, 1); + // Only the ended session's count is adopted; the active session's slice + // is re-added when that session finalizes and syncs. + assert.equal( + queryOne<{ frequency: number }>(localPath, `SELECT frequency FROM imm_words WHERE word = '食べた'`) + ?.frequency, + 1, + ); + + // The remote app restarts and finalizes the stale session. + withWritableDb(remotePath, (db) => { + db.prepare( + `UPDATE imm_sessions SET ended_at_ms = ?, status = 2 + WHERE session_uuid = 'remote-active'`, + ).run(String(BASE_MS + DAY_MS + 1_500_000)); + }); + + const second = mergeSnapshotIntoDb(localPath, remotePath); + assert.equal(second.sessionsMerged, 1); + assert.equal(second.sessionsAlreadyPresent, 1); + // 1 (ended session) + 4 (finalized session), not 5 + 4 = 9. + assert.equal( + queryOne<{ frequency: number }>(localPath, `SELECT frequency FROM imm_words WHERE word = '食べた'`) + ?.frequency, + 5, + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/src/core/services/stats-sync/merge-catalog.ts b/src/core/services/stats-sync/merge-catalog.ts index 265abbbd..2b4fc56f 100644 --- a/src/core/services/stats-sync/merge-catalog.ts +++ b/src/core/services/stats-sync/merge-catalog.ts @@ -285,9 +285,11 @@ export function mergeExcludedWords( /** * Lazily maps remote imm_words / imm_kanji ids onto local rows by natural key * ((headword, word, reading) / kanji). New rows are copied with the remote's - * accumulated frequency; rows that already exist locally get their frequency - * incremented later with only the occurrence counts this merge adds (the - * remote total would double-count lines merged in earlier syncs). + * accumulated frequency minus any counts owed to skipped ACTIVE sessions + * (those merge later and re-add their counts); rows that already exist locally + * get their frequency incremented later with only the occurrence counts this + * merge adds (the remote total would double-count lines merged in earlier + * syncs). */ export class LexiconResolver { private readonly wordMap = new Map(); @@ -301,6 +303,35 @@ export class LexiconResolver { private readonly summary: SyncMergeSummary, ) {} + /** + * Occurrence counts the remote's live tracker already baked into `frequency` + * but that belong to skipped ACTIVE sessions. Those lines are not copied this + * merge; they are re-added via addWordOccurrences/addKanjiOccurrences when + * the session finalizes and syncs, so a newly adopted row must not carry them + * or that slice would be counted twice. + */ + private pendingActiveSessionOccurrences( + occurrenceTable: 'imm_word_line_occurrences' | 'imm_kanji_line_occurrences', + idColumn: 'word_id' | 'kanji_id', + remoteId: number, + ): number { + const row = selectOne( + this.remote, + `SELECT COALESCE(SUM(o.occurrence_count), 0) AS pending + FROM ${occurrenceTable} o + JOIN imm_subtitle_lines l ON l.line_id = o.line_id + JOIN imm_sessions s ON s.session_id = l.session_id + WHERE o.${idColumn} = ? AND s.ended_at_ms IS NULL`, + [remoteId], + ); + return Number(row?.pending ?? 0); + } + + private adoptedFrequency(frequency: unknown, pending: number): unknown { + if (pending <= 0 || typeof frequency !== 'number') return frequency; + return Math.max(0, frequency - pending); + } + resolveWord(remoteWordId: number): number { const cached = this.wordMap.get(remoteWordId); if (cached) return cached.localId; @@ -329,11 +360,18 @@ export class LexiconResolver { ) .run(row.first_seen, row.first_seen, row.last_seen, row.last_seen, entry.localId); } else { + const pending = this.pendingActiveSessionOccurrences( + 'imm_word_line_occurrences', + 'word_id', + remoteWordId, + ); const localId = insertRow( this.local, 'imm_words', WORD_COPY_COLUMNS, - WORD_COPY_COLUMNS.map((column) => row[column]), + WORD_COPY_COLUMNS.map((column) => + column === 'frequency' ? this.adoptedFrequency(row[column], pending) : row[column], + ), ); entry = { localId, isNew: true }; this.summary.wordsAdded += 1; @@ -368,11 +406,16 @@ export class LexiconResolver { ) .run(row.first_seen, row.first_seen, row.last_seen, row.last_seen, entry.localId); } else { + const pending = this.pendingActiveSessionOccurrences( + 'imm_kanji_line_occurrences', + 'kanji_id', + remoteKanjiId, + ); const localId = insertRow( this.local, 'imm_kanji', ['kanji', 'first_seen', 'last_seen', 'frequency'], - [row.kanji, row.first_seen, row.last_seen, row.frequency], + [row.kanji, row.first_seen, row.last_seen, this.adoptedFrequency(row.frequency, pending)], ); entry = { localId, isNew: true }; this.summary.kanjiAdded += 1;