mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-27 04:49:49 -07:00
fix(sync): adopted lexicon frequency no longer double-counts active-session lines
The remote tracker increments imm_words/imm_kanji.frequency live, so a snapshot holding a stale ACTIVE session (skipped by the merge) has that session's partial occurrences baked into frequency. A word new to the local DB adopted that full total, then received the same occurrences again when the session finalized and merged on a later sync. Subtract active-session occurrence counts when adopting a new row; they are re-added exactly once when the session completes. Existing rows were already safe (isNew guard in addWordOccurrences/addKanjiOccurrences). Verified red/green: the new two-sync regression test fails without the subtraction (frequency 9 instead of 5).
This commit is contained in:
@@ -0,0 +1,4 @@
|
|||||||
|
type: fixed
|
||||||
|
area: sync
|
||||||
|
|
||||||
|
- Fixed word/kanji frequencies double-counting across syncs when the remote snapshot contained a stale active session (e.g. after a crash): a word new to the local machine adopted the remote's full lifetime frequency, which already included the active session's partial occurrences, and those occurrences were added again when the session finalized and synced. Newly adopted words/kanji now exclude active-session counts, which arrive once the session completes.
|
||||||
@@ -606,3 +606,61 @@ test('createDbSnapshot produces a mergeable copy', () => {
|
|||||||
fs.rmSync(dir, { recursive: true, force: true });
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('adopted word frequency excludes active-session counts that merge later', () => {
|
||||||
|
const { dir, localPath, remotePath } = makeDbPair();
|
||||||
|
try {
|
||||||
|
// Remote has an ended session and a stale ACTIVE one (e.g. app crashed).
|
||||||
|
// The remote tracker increments frequency live, so the word's frequency (5)
|
||||||
|
// already includes the active session's 4 occurrences even though that
|
||||||
|
// session's lines are skipped by the merge.
|
||||||
|
insertFixtureSession(remotePath, {
|
||||||
|
uuid: 'remote-ended',
|
||||||
|
videoKey: 'showb-e1',
|
||||||
|
animeTitleKey: 'showb',
|
||||||
|
startedAtMs: BASE_MS,
|
||||||
|
applyLifetime: true,
|
||||||
|
words: [{ headword: '食べる', word: '食べた', reading: 'たべた', count: 1 }],
|
||||||
|
});
|
||||||
|
insertFixtureSession(remotePath, {
|
||||||
|
uuid: 'remote-active',
|
||||||
|
videoKey: 'showb-e2',
|
||||||
|
animeTitleKey: 'showb',
|
||||||
|
startedAtMs: BASE_MS + DAY_MS,
|
||||||
|
endedAtMs: null,
|
||||||
|
words: [{ headword: '食べる', word: '食べた', reading: 'たべた', count: 4 }],
|
||||||
|
});
|
||||||
|
|
||||||
|
const first = mergeSnapshotIntoDb(localPath, remotePath);
|
||||||
|
assert.equal(first.sessionsMerged, 1);
|
||||||
|
assert.equal(first.activeSessionsSkipped, 1);
|
||||||
|
assert.equal(first.wordsAdded, 1);
|
||||||
|
// Only the ended session's count is adopted; the active session's slice
|
||||||
|
// is re-added when that session finalizes and syncs.
|
||||||
|
assert.equal(
|
||||||
|
queryOne<{ frequency: number }>(localPath, `SELECT frequency FROM imm_words WHERE word = '食べた'`)
|
||||||
|
?.frequency,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
|
||||||
|
// The remote app restarts and finalizes the stale session.
|
||||||
|
withWritableDb(remotePath, (db) => {
|
||||||
|
db.prepare(
|
||||||
|
`UPDATE imm_sessions SET ended_at_ms = ?, status = 2
|
||||||
|
WHERE session_uuid = 'remote-active'`,
|
||||||
|
).run(String(BASE_MS + DAY_MS + 1_500_000));
|
||||||
|
});
|
||||||
|
|
||||||
|
const second = mergeSnapshotIntoDb(localPath, remotePath);
|
||||||
|
assert.equal(second.sessionsMerged, 1);
|
||||||
|
assert.equal(second.sessionsAlreadyPresent, 1);
|
||||||
|
// 1 (ended session) + 4 (finalized session), not 5 + 4 = 9.
|
||||||
|
assert.equal(
|
||||||
|
queryOne<{ frequency: number }>(localPath, `SELECT frequency FROM imm_words WHERE word = '食べた'`)
|
||||||
|
?.frequency,
|
||||||
|
5,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|||||||
@@ -285,9 +285,11 @@ export function mergeExcludedWords(
|
|||||||
/**
|
/**
|
||||||
* Lazily maps remote imm_words / imm_kanji ids onto local rows by natural key
|
* Lazily maps remote imm_words / imm_kanji ids onto local rows by natural key
|
||||||
* ((headword, word, reading) / kanji). New rows are copied with the remote's
|
* ((headword, word, reading) / kanji). New rows are copied with the remote's
|
||||||
* accumulated frequency; rows that already exist locally get their frequency
|
* accumulated frequency minus any counts owed to skipped ACTIVE sessions
|
||||||
* incremented later with only the occurrence counts this merge adds (the
|
* (those merge later and re-add their counts); rows that already exist locally
|
||||||
* remote total would double-count lines merged in earlier syncs).
|
* get their frequency incremented later with only the occurrence counts this
|
||||||
|
* merge adds (the remote total would double-count lines merged in earlier
|
||||||
|
* syncs).
|
||||||
*/
|
*/
|
||||||
export class LexiconResolver {
|
export class LexiconResolver {
|
||||||
private readonly wordMap = new Map<number, { localId: number; isNew: boolean }>();
|
private readonly wordMap = new Map<number, { localId: number; isNew: boolean }>();
|
||||||
@@ -301,6 +303,35 @@ export class LexiconResolver {
|
|||||||
private readonly summary: SyncMergeSummary,
|
private readonly summary: SyncMergeSummary,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Occurrence counts the remote's live tracker already baked into `frequency`
|
||||||
|
* but that belong to skipped ACTIVE sessions. Those lines are not copied this
|
||||||
|
* merge; they are re-added via addWordOccurrences/addKanjiOccurrences when
|
||||||
|
* the session finalizes and syncs, so a newly adopted row must not carry them
|
||||||
|
* or that slice would be counted twice.
|
||||||
|
*/
|
||||||
|
private pendingActiveSessionOccurrences(
|
||||||
|
occurrenceTable: 'imm_word_line_occurrences' | 'imm_kanji_line_occurrences',
|
||||||
|
idColumn: 'word_id' | 'kanji_id',
|
||||||
|
remoteId: number,
|
||||||
|
): number {
|
||||||
|
const row = selectOne(
|
||||||
|
this.remote,
|
||||||
|
`SELECT COALESCE(SUM(o.occurrence_count), 0) AS pending
|
||||||
|
FROM ${occurrenceTable} o
|
||||||
|
JOIN imm_subtitle_lines l ON l.line_id = o.line_id
|
||||||
|
JOIN imm_sessions s ON s.session_id = l.session_id
|
||||||
|
WHERE o.${idColumn} = ? AND s.ended_at_ms IS NULL`,
|
||||||
|
[remoteId],
|
||||||
|
);
|
||||||
|
return Number(row?.pending ?? 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
private adoptedFrequency(frequency: unknown, pending: number): unknown {
|
||||||
|
if (pending <= 0 || typeof frequency !== 'number') return frequency;
|
||||||
|
return Math.max(0, frequency - pending);
|
||||||
|
}
|
||||||
|
|
||||||
resolveWord(remoteWordId: number): number {
|
resolveWord(remoteWordId: number): number {
|
||||||
const cached = this.wordMap.get(remoteWordId);
|
const cached = this.wordMap.get(remoteWordId);
|
||||||
if (cached) return cached.localId;
|
if (cached) return cached.localId;
|
||||||
@@ -329,11 +360,18 @@ export class LexiconResolver {
|
|||||||
)
|
)
|
||||||
.run(row.first_seen, row.first_seen, row.last_seen, row.last_seen, entry.localId);
|
.run(row.first_seen, row.first_seen, row.last_seen, row.last_seen, entry.localId);
|
||||||
} else {
|
} else {
|
||||||
|
const pending = this.pendingActiveSessionOccurrences(
|
||||||
|
'imm_word_line_occurrences',
|
||||||
|
'word_id',
|
||||||
|
remoteWordId,
|
||||||
|
);
|
||||||
const localId = insertRow(
|
const localId = insertRow(
|
||||||
this.local,
|
this.local,
|
||||||
'imm_words',
|
'imm_words',
|
||||||
WORD_COPY_COLUMNS,
|
WORD_COPY_COLUMNS,
|
||||||
WORD_COPY_COLUMNS.map((column) => row[column]),
|
WORD_COPY_COLUMNS.map((column) =>
|
||||||
|
column === 'frequency' ? this.adoptedFrequency(row[column], pending) : row[column],
|
||||||
|
),
|
||||||
);
|
);
|
||||||
entry = { localId, isNew: true };
|
entry = { localId, isNew: true };
|
||||||
this.summary.wordsAdded += 1;
|
this.summary.wordsAdded += 1;
|
||||||
@@ -368,11 +406,16 @@ export class LexiconResolver {
|
|||||||
)
|
)
|
||||||
.run(row.first_seen, row.first_seen, row.last_seen, row.last_seen, entry.localId);
|
.run(row.first_seen, row.first_seen, row.last_seen, row.last_seen, entry.localId);
|
||||||
} else {
|
} else {
|
||||||
|
const pending = this.pendingActiveSessionOccurrences(
|
||||||
|
'imm_kanji_line_occurrences',
|
||||||
|
'kanji_id',
|
||||||
|
remoteKanjiId,
|
||||||
|
);
|
||||||
const localId = insertRow(
|
const localId = insertRow(
|
||||||
this.local,
|
this.local,
|
||||||
'imm_kanji',
|
'imm_kanji',
|
||||||
['kanji', 'first_seen', 'last_seen', 'frequency'],
|
['kanji', 'first_seen', 'last_seen', 'frequency'],
|
||||||
[row.kanji, row.first_seen, row.last_seen, row.frequency],
|
[row.kanji, row.first_seen, row.last_seen, this.adoptedFrequency(row.frequency, pending)],
|
||||||
);
|
);
|
||||||
entry = { localId, isNew: true };
|
entry = { localId, isNew: true };
|
||||||
this.summary.kanjiAdded += 1;
|
this.summary.kanjiAdded += 1;
|
||||||
|
|||||||
Reference in New Issue
Block a user