mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-11-10 04:46:07 -08:00
fix(stats): keep first_seen for words with retention-pruned history
- Only repair stale first_seen when frequency matches surviving occurrences - Rows whose older lines were pruned (or predate occurrence tables) keep their original first_seen - Extend rollup rebuild test with a retention-pruned word
This commit is contained in:
2 files changed
+19
-6
No files matched your search
@@ -201,15 +201,17 @@ test('lexical rollup rebuild repairs seen bounds left pointing at deleted histor
|
|||||||
const staleLast = seen + 86_400.631;
|
const staleLast = seen + 86_400.631;
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`INSERT INTO imm_words(id, headword, word, reading, first_seen, last_seen, frequency)
|
`INSERT INTO imm_words(id, headword, word, reading, first_seen, last_seen, frequency)
|
||||||
VALUES (1, '猫', '猫', 'ねこ', ?, ?, 1), (2, '犬', '犬', 'いぬ', ?, ?, 1)`,
|
VALUES (1, '猫', '猫', 'ねこ', ?, ?, 1), (2, '犬', '犬', 'いぬ', ?, ?, 1),
|
||||||
).run(staleFirst, staleLast, seen - 0.1, seen - 0.1);
|
(3, '鳥', '鳥', 'とり', ?, ?, 5)`,
|
||||||
|
// Word 3's earlier history was pruned by retention: frequency still counts it.
|
||||||
|
).run(staleFirst, staleLast, seen - 0.1, seen - 0.1, staleFirst, seen);
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency) VALUES (1, '猫', ?, ?, 1)`,
|
`INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency) VALUES (1, '猫', ?, ?, 1)`,
|
||||||
).run(staleFirst, staleLast);
|
).run(staleFirst, staleLast);
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms)
|
`INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms)
|
||||||
VALUES (?, 1, 1, ?), (?, 2, 1, ?)`,
|
VALUES (?, 1, 1, ?), (?, 2, 1, ?), (?, 3, 1, ?)`,
|
||||||
).run(lineId, BASE_MS + 251, lineId, BASE_MS + 251);
|
).run(lineId, BASE_MS + 251, lineId, BASE_MS + 251, lineId, BASE_MS + 251);
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`INSERT INTO imm_kanji_line_occurrences(line_id, kanji_id, occurrence_count, seen_ms)
|
`INSERT INTO imm_kanji_line_occurrences(line_id, kanji_id, occurrence_count, seen_ms)
|
||||||
VALUES (?, 1, 1, ?)`,
|
VALUES (?, 1, 1, ?)`,
|
||||||
@@ -231,8 +233,9 @@ test('lexical rollup rebuild repairs seen bounds left pointing at deleted histor
|
|||||||
[
|
[
|
||||||
[seen, seen],
|
[seen, seen],
|
||||||
[seen - 0.1, seen - 0.1],
|
[seen - 0.1, seen - 0.1],
|
||||||
|
[staleFirst, seen],
|
||||||
],
|
],
|
||||||
'healthy rows within flush lag are untouched',
|
'flush-lag and retention-pruned rows keep their first_seen',
|
||||||
);
|
);
|
||||||
assert.deepEqual(bounds('imm_kanji'), [[seen, seen]]);
|
assert.deepEqual(bounds('imm_kanji'), [[seen, seen]]);
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -35,6 +35,12 @@ export function localEpochDaySql(value: string): string {
|
|||||||
// stamped at queue time, before seen_ms is stamped at flush, so healthy rows
|
// stamped at queue time, before seen_ms is stamped at flush, so healthy rows
|
||||||
// never exceed their newest occurrence and only trail their oldest by flush lag.
|
// never exceed their newest occurrence and only trail their oldest by flush lag.
|
||||||
// Rows with undated occurrences are skipped.
|
// Rows with undated occurrences are skipped.
|
||||||
|
//
|
||||||
|
// first_seen can also legitimately predate every surviving occurrence: retention
|
||||||
|
// pruning cascades away old lines, and early history predates the occurrence
|
||||||
|
// tables, while both keep the lifetime frequency. Deletes subtract frequency
|
||||||
|
// exactly, so only rows whose frequency matches their surviving occurrences have
|
||||||
|
// their whole history on hand and a provably stale first_seen.
|
||||||
function repairStaleSeenBounds(db: DatabaseSync): void {
|
function repairStaleSeenBounds(db: DatabaseSync): void {
|
||||||
for (const [table, occurrenceTable, col] of [
|
for (const [table, occurrenceTable, col] of [
|
||||||
['imm_words', 'imm_word_line_occurrences', 'word_id'],
|
['imm_words', 'imm_word_line_occurrences', 'word_id'],
|
||||||
@@ -48,7 +54,11 @@ function repairStaleSeenBounds(db: DatabaseSync): void {
|
|||||||
db.exec(`
|
db.exec(`
|
||||||
UPDATE ${table}
|
UPDATE ${table}
|
||||||
SET first_seen = ${seenSec('MIN')}
|
SET first_seen = ${seenSec('MIN')}
|
||||||
WHERE first_seen < ${seenSec('MIN')} - ${FIRST_SEEN_FLUSH_LAG_SLACK_SEC} AND ${hasNoUndated};
|
WHERE first_seen < ${seenSec('MIN')} - ${FIRST_SEEN_FLUSH_LAG_SLACK_SEC}
|
||||||
|
AND ${hasNoUndated}
|
||||||
|
AND frequency = (
|
||||||
|
SELECT SUM(o.occurrence_count) FROM ${occurrenceTable} o WHERE o.${col} = ${table}.id
|
||||||
|
);
|
||||||
UPDATE ${table}
|
UPDATE ${table}
|
||||||
SET last_seen = ${seenSec('MAX')}
|
SET last_seen = ${seenSec('MAX')}
|
||||||
WHERE last_seen > ${seenSec('MAX')} AND ${hasNoUndated};
|
WHERE last_seen > ${seenSec('MAX')} AND ${hasNoUndated};
|
||||||
|
|||||||
Reference in new issue
Block a user