mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-11-14 16:46:17 -08:00
fix(stats): drop first_seen repair from lexical rollup rebuild
A stale first_seen is indistinguishable from real history whose occurrences retention pruned, and frequency does not reliably separate them (it diverges from surviving occurrences on databases with neither retention nor pre-occurrence history), so the guarded repair either did nothing or risked overwriting correct dates. The delete path already keeps first_seen exact for new deletes; the rebuild now repairs only last_seen.
This commit is contained in:
2 files changed
+28
-42
No files matched your search
@@ -182,7 +182,7 @@ test('re-seeing a filtered legacy word keeps charts consistent with the summary'
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('lexical rollup rebuild repairs seen bounds left pointing at deleted history', () => {
|
test('lexical rollup rebuild repairs last_seen left pointing at deleted history', () => {
|
||||||
const db = createDb();
|
const db = createDb();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -197,21 +197,20 @@ test('lexical rollup rebuild repairs seen bounds left pointing at deleted histor
|
|||||||
.run(sessionId, videoId, BASE_MS, BASE_MS).lastInsertRowid,
|
.run(sessionId, videoId, BASE_MS, BASE_MS).lastInsertRowid,
|
||||||
);
|
);
|
||||||
const seen = (BASE_MS + 251) / 1000;
|
const seen = (BASE_MS + 251) / 1000;
|
||||||
const staleFirst = seen - 86_400.5;
|
// first_seen this early may be real history pruned by retention; never touched.
|
||||||
|
const earlyFirst = seen - 86_400.5;
|
||||||
const staleLast = seen + 86_400.631;
|
const staleLast = seen + 86_400.631;
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`INSERT INTO imm_words(id, headword, word, reading, first_seen, last_seen, frequency)
|
`INSERT INTO imm_words(id, headword, word, reading, first_seen, last_seen, frequency)
|
||||||
VALUES (1, '猫', '猫', 'ねこ', ?, ?, 1), (2, '犬', '犬', 'いぬ', ?, ?, 1),
|
VALUES (1, '猫', '猫', 'ねこ', ?, ?, 1), (2, '犬', '犬', 'いぬ', ?, ?, 1)`,
|
||||||
(3, '鳥', '鳥', 'とり', ?, ?, 5)`,
|
).run(earlyFirst, staleLast, seen - 0.1, seen - 0.1);
|
||||||
// Word 3's earlier history was pruned by retention: frequency still counts it.
|
|
||||||
).run(staleFirst, staleLast, seen - 0.1, seen - 0.1, staleFirst, seen);
|
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency) VALUES (1, '猫', ?, ?, 1)`,
|
`INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency) VALUES (1, '猫', ?, ?, 1)`,
|
||||||
).run(staleFirst, staleLast);
|
).run(earlyFirst, staleLast);
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms)
|
`INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms)
|
||||||
VALUES (?, 1, 1, ?), (?, 2, 1, ?), (?, 3, 1, ?)`,
|
VALUES (?, 1, 1, ?), (?, 2, 1, ?)`,
|
||||||
).run(lineId, BASE_MS + 251, lineId, BASE_MS + 251, lineId, BASE_MS + 251);
|
).run(lineId, BASE_MS + 251, lineId, BASE_MS + 251);
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`INSERT INTO imm_kanji_line_occurrences(line_id, kanji_id, occurrence_count, seen_ms)
|
`INSERT INTO imm_kanji_line_occurrences(line_id, kanji_id, occurrence_count, seen_ms)
|
||||||
VALUES (?, 1, 1, ?)`,
|
VALUES (?, 1, 1, ?)`,
|
||||||
@@ -231,13 +230,12 @@ test('lexical rollup rebuild repairs seen bounds left pointing at deleted histor
|
|||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
bounds('imm_words'),
|
bounds('imm_words'),
|
||||||
[
|
[
|
||||||
[seen, seen],
|
[earlyFirst, seen],
|
||||||
[seen - 0.1, seen - 0.1],
|
[seen - 0.1, seen - 0.1],
|
||||||
[staleFirst, seen],
|
|
||||||
],
|
],
|
||||||
'flush-lag and retention-pruned rows keep their first_seen',
|
'healthy rows are untouched',
|
||||||
);
|
);
|
||||||
assert.deepEqual(bounds('imm_kanji'), [[seen, seen]]);
|
assert.deepEqual(bounds('imm_kanji'), [[earlyFirst, seen]]);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,49 +19,37 @@ const LOCAL_EPOCH_DAY_SQL = `
|
|||||||
`;
|
`;
|
||||||
|
|
||||||
// v3 re-derives visibility for rows a re-seen word forced visible and repairs
|
// v3 re-derives visibility for rows a re-seen word forced visible and repairs
|
||||||
// first_seen/last_seen values left behind by session deletes.
|
// last_seen values left behind by session deletes.
|
||||||
const LEXICAL_DAILY_ROLLUP_VERSION = '3';
|
const LEXICAL_DAILY_ROLLUP_VERSION = '3';
|
||||||
const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
|
const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
|
||||||
const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
|
const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
|
||||||
// Healthy first_seen trails its oldest seen_ms only by queue-to-flush lag.
|
|
||||||
const FIRST_SEEN_FLUSH_LAG_SLACK_SEC = 60;
|
|
||||||
|
|
||||||
export function localEpochDaySql(value: string): string {
|
export function localEpochDaySql(value: string): string {
|
||||||
return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
|
return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Session deletes once compared rounded timestamps against stored bounds and
|
// Session deletes once rounded removed timestamps to whole seconds, missing
|
||||||
// could leave first_seen/last_seen pointing at deleted history. Bounds are
|
// fractional last_seen extremes that pointed at deleted history. last_seen is
|
||||||
// stamped at queue time, before seen_ms is stamped at flush, so healthy rows
|
// stamped at queue time, before seen_ms is stamped at flush, and retention
|
||||||
// never exceed their newest occurrence and only trail their oldest by flush lag.
|
// prunes the oldest sessions first, so healthy rows never exceed their newest
|
||||||
// Rows with undated occurrences are skipped.
|
// surviving occurrence. Rows with undated occurrences are skipped.
|
||||||
//
|
//
|
||||||
// first_seen can also legitimately predate every surviving occurrence: retention
|
// first_seen is deliberately not repaired: a stale bound is indistinguishable
|
||||||
// pruning cascades away old lines, and early history predates the occurrence
|
// from real history whose occurrences retention pruned, and frequency does not
|
||||||
// tables, while both keep the lifetime frequency. Deletes subtract frequency
|
// reliably tell them apart.
|
||||||
// exactly, so only rows whose frequency matches their surviving occurrences have
|
function repairStaleLastSeen(db: DatabaseSync): void {
|
||||||
// their whole history on hand and a provably stale first_seen.
|
|
||||||
function repairStaleSeenBounds(db: DatabaseSync): void {
|
|
||||||
for (const [table, occurrenceTable, col] of [
|
for (const [table, occurrenceTable, col] of [
|
||||||
['imm_words', 'imm_word_line_occurrences', 'word_id'],
|
['imm_words', 'imm_word_line_occurrences', 'word_id'],
|
||||||
['imm_kanji', 'imm_kanji_line_occurrences', 'kanji_id'],
|
['imm_kanji', 'imm_kanji_line_occurrences', 'kanji_id'],
|
||||||
] as const) {
|
] as const) {
|
||||||
const seenSec = (fn: 'MIN' | 'MAX'): string =>
|
const newestSeen = `(SELECT MAX(o.seen_ms) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id) / 1000.0`;
|
||||||
`(SELECT ${fn}(o.seen_ms) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id) / 1000.0`;
|
|
||||||
const hasNoUndated = `NOT EXISTS (
|
|
||||||
SELECT 1 FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id AND o.seen_ms IS NULL
|
|
||||||
)`;
|
|
||||||
db.exec(`
|
db.exec(`
|
||||||
UPDATE ${table}
|
UPDATE ${table}
|
||||||
SET first_seen = ${seenSec('MIN')}
|
SET last_seen = ${newestSeen}
|
||||||
WHERE first_seen < ${seenSec('MIN')} - ${FIRST_SEEN_FLUSH_LAG_SLACK_SEC}
|
WHERE last_seen > ${newestSeen}
|
||||||
AND ${hasNoUndated}
|
AND NOT EXISTS (
|
||||||
AND frequency = (
|
SELECT 1 FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id AND o.seen_ms IS NULL
|
||||||
SELECT SUM(o.occurrence_count) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id
|
)
|
||||||
);
|
|
||||||
UPDATE ${table}
|
|
||||||
SET last_seen = ${seenSec('MAX')}
|
|
||||||
WHERE last_seen > ${seenSec('MAX')} AND ${hasNoUndated};
|
|
||||||
`);
|
`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -226,7 +214,7 @@ export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
|
|||||||
lastId = vocabularyRows[vocabularyRows.length - 1]!.id;
|
lastId = vocabularyRows[vocabularyRows.length - 1]!.id;
|
||||||
if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break;
|
if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break;
|
||||||
}
|
}
|
||||||
repairStaleSeenBounds(db);
|
repairStaleLastSeen(db);
|
||||||
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||||
db.exec(`
|
db.exec(`
|
||||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||||
|
|||||||
Reference in new issue
Block a user