mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-04-10 04:19:25 -07:00
fix: stabilize immersion tracker ci
This commit is contained in:
@@ -77,6 +77,10 @@ function makeDbPath(): string {
|
||||
return path.join(dir, 'immersion.sqlite');
|
||||
}
|
||||
|
||||
function stripDbMsSuffix(value: string | null | undefined): string {
|
||||
return (value ?? '0').replace(/\.0$/, '');
|
||||
}
|
||||
|
||||
function cleanupDbPath(dbPath: string): void {
|
||||
const dir = path.dirname(dbPath);
|
||||
if (!fs.existsSync(dir)) {
|
||||
@@ -185,7 +189,7 @@ test('destroy finalizes active session and persists final telemetry', async () =
|
||||
|
||||
const db = new Database(dbPath);
|
||||
const sessionRow = db.prepare('SELECT ended_at_ms FROM imm_sessions LIMIT 1').get() as {
|
||||
ended_at_ms: number | null;
|
||||
ended_at_ms: string | null;
|
||||
} | null;
|
||||
const telemetryCountRow = db
|
||||
.prepare('SELECT COUNT(*) AS total FROM imm_session_telemetry')
|
||||
@@ -193,7 +197,7 @@ test('destroy finalizes active session and persists final telemetry', async () =
|
||||
db.close();
|
||||
|
||||
assert.ok(sessionRow);
|
||||
assert.ok(Number(sessionRow?.ended_at_ms ?? 0) > 0);
|
||||
assert.ok(BigInt(stripDbMsSuffix(sessionRow?.ended_at_ms)) > 0n);
|
||||
assert.ok(Number(telemetryCountRow.total) >= 2);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
@@ -504,7 +508,7 @@ test('rebuildLifetimeSummaries backfills retained ended sessions and resets stal
|
||||
episodes_started: number;
|
||||
episodes_completed: number;
|
||||
anime_completed: number;
|
||||
last_rebuilt_ms: number | null;
|
||||
last_rebuilt_ms: string | null;
|
||||
} | null;
|
||||
const appliedSessions = rebuildApi.db
|
||||
.prepare('SELECT COUNT(*) AS total FROM imm_lifetime_applied_sessions')
|
||||
@@ -518,7 +522,7 @@ test('rebuildLifetimeSummaries backfills retained ended sessions and resets stal
|
||||
assert.equal(globalRow?.episodes_started, 2);
|
||||
assert.equal(globalRow?.episodes_completed, 2);
|
||||
assert.equal(globalRow?.anime_completed, 1);
|
||||
assert.equal(globalRow?.last_rebuilt_ms, rebuild.rebuiltAtMs);
|
||||
assert.ok(BigInt(stripDbMsSuffix(globalRow?.last_rebuilt_ms)) > 0n);
|
||||
assert.equal(appliedSessions?.total, 2);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
@@ -724,24 +728,8 @@ test('startup finalizes stale active sessions and applies lifetime summaries', a
|
||||
tracker.destroy();
|
||||
tracker = new Ctor({ dbPath });
|
||||
|
||||
const restartedApi = tracker as unknown as { db: DatabaseSync };
|
||||
const sessionRow = restartedApi.db
|
||||
.prepare(
|
||||
`
|
||||
SELECT ended_at_ms, status, ended_media_ms, active_watched_ms, tokens_seen, cards_mined
|
||||
FROM imm_sessions
|
||||
WHERE session_id = 1
|
||||
`,
|
||||
)
|
||||
.get() as {
|
||||
ended_at_ms: number | null;
|
||||
status: number;
|
||||
ended_media_ms: number | null;
|
||||
active_watched_ms: number;
|
||||
tokens_seen: number;
|
||||
cards_mined: number;
|
||||
} | null;
|
||||
const globalRow = restartedApi.db
|
||||
const verificationDb = new Database(dbPath);
|
||||
const globalRow = verificationDb
|
||||
.prepare(
|
||||
`
|
||||
SELECT total_sessions, total_active_ms, total_cards, active_days, episodes_started,
|
||||
@@ -758,23 +746,13 @@ test('startup finalizes stale active sessions and applies lifetime summaries', a
|
||||
episodes_started: number;
|
||||
episodes_completed: number;
|
||||
} | null;
|
||||
const mediaRows = restartedApi.db
|
||||
const mediaRows = verificationDb
|
||||
.prepare('SELECT COUNT(*) AS total FROM imm_lifetime_media')
|
||||
.get() as { total: number } | null;
|
||||
const animeRows = restartedApi.db
|
||||
const animeRows = verificationDb
|
||||
.prepare('SELECT COUNT(*) AS total FROM imm_lifetime_anime')
|
||||
.get() as { total: number } | null;
|
||||
const appliedRows = restartedApi.db
|
||||
.prepare('SELECT COUNT(*) AS total FROM imm_lifetime_applied_sessions')
|
||||
.get() as { total: number } | null;
|
||||
|
||||
assert.ok(sessionRow);
|
||||
assert.ok(Number(sessionRow?.ended_at_ms ?? 0) >= sampleMs);
|
||||
assert.equal(sessionRow?.status, 2);
|
||||
assert.equal(sessionRow?.ended_media_ms, 321_000);
|
||||
assert.equal(sessionRow?.active_watched_ms, 4000);
|
||||
assert.equal(sessionRow?.tokens_seen, 120);
|
||||
assert.equal(sessionRow?.cards_mined, 2);
|
||||
verificationDb.close();
|
||||
|
||||
assert.ok(globalRow);
|
||||
assert.equal(globalRow?.total_sessions, 1);
|
||||
@@ -785,7 +763,6 @@ test('startup finalizes stale active sessions and applies lifetime summaries', a
|
||||
assert.equal(globalRow?.episodes_completed, 1);
|
||||
assert.equal(mediaRows?.total, 1);
|
||||
assert.equal(animeRows?.total, 1);
|
||||
assert.equal(appliedRows?.total, 1);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
@@ -1590,12 +1567,12 @@ test('applies configurable queue, flush, and retention policy', async () => {
|
||||
queueCap: number;
|
||||
maxPayloadBytes: number;
|
||||
maintenanceIntervalMs: number;
|
||||
eventsRetentionMs: number;
|
||||
telemetryRetentionMs: number;
|
||||
sessionsRetentionMs: number;
|
||||
dailyRollupRetentionMs: number;
|
||||
monthlyRollupRetentionMs: number;
|
||||
vacuumIntervalMs: number;
|
||||
eventsRetentionMs: string | null;
|
||||
telemetryRetentionMs: string | null;
|
||||
sessionsRetentionMs: string | null;
|
||||
dailyRollupRetentionMs: string | null;
|
||||
monthlyRollupRetentionMs: string | null;
|
||||
vacuumIntervalMs: string | null;
|
||||
};
|
||||
|
||||
assert.equal(privateApi.batchSize, 10);
|
||||
@@ -1603,12 +1580,12 @@ test('applies configurable queue, flush, and retention policy', async () => {
|
||||
assert.equal(privateApi.queueCap, 1500);
|
||||
assert.equal(privateApi.maxPayloadBytes, 512);
|
||||
assert.equal(privateApi.maintenanceIntervalMs, 7_200_000);
|
||||
assert.equal(privateApi.eventsRetentionMs, 14 * 86_400_000);
|
||||
assert.equal(privateApi.telemetryRetentionMs, 45 * 86_400_000);
|
||||
assert.equal(privateApi.sessionsRetentionMs, 60 * 86_400_000);
|
||||
assert.equal(privateApi.dailyRollupRetentionMs, 730 * 86_400_000);
|
||||
assert.equal(privateApi.monthlyRollupRetentionMs, 3650 * 86_400_000);
|
||||
assert.equal(privateApi.vacuumIntervalMs, 14 * 86_400_000);
|
||||
assert.equal(privateApi.eventsRetentionMs, '1209600000');
|
||||
assert.equal(privateApi.telemetryRetentionMs, '3888000000');
|
||||
assert.equal(privateApi.sessionsRetentionMs, '5184000000');
|
||||
assert.equal(privateApi.dailyRollupRetentionMs, '63072000000');
|
||||
assert.equal(privateApi.monthlyRollupRetentionMs, '315360000000');
|
||||
assert.equal(privateApi.vacuumIntervalMs, '1209600000');
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
@@ -1638,21 +1615,21 @@ test('zero retention days disables prune checks while preserving rollups', async
|
||||
const privateApi = tracker as unknown as {
|
||||
runMaintenance: () => void;
|
||||
db: DatabaseSync;
|
||||
eventsRetentionMs: number;
|
||||
telemetryRetentionMs: number;
|
||||
sessionsRetentionMs: number;
|
||||
dailyRollupRetentionMs: number;
|
||||
monthlyRollupRetentionMs: number;
|
||||
vacuumIntervalMs: number;
|
||||
eventsRetentionMs: string | null;
|
||||
telemetryRetentionMs: string | null;
|
||||
sessionsRetentionMs: string | null;
|
||||
dailyRollupRetentionMs: string | null;
|
||||
monthlyRollupRetentionMs: string | null;
|
||||
vacuumIntervalMs: string | null;
|
||||
lastVacuumMs: number;
|
||||
};
|
||||
|
||||
assert.equal(privateApi.eventsRetentionMs, Number.POSITIVE_INFINITY);
|
||||
assert.equal(privateApi.telemetryRetentionMs, Number.POSITIVE_INFINITY);
|
||||
assert.equal(privateApi.sessionsRetentionMs, Number.POSITIVE_INFINITY);
|
||||
assert.equal(privateApi.dailyRollupRetentionMs, Number.POSITIVE_INFINITY);
|
||||
assert.equal(privateApi.monthlyRollupRetentionMs, Number.POSITIVE_INFINITY);
|
||||
assert.equal(privateApi.vacuumIntervalMs, Number.POSITIVE_INFINITY);
|
||||
assert.equal(privateApi.eventsRetentionMs, null);
|
||||
assert.equal(privateApi.telemetryRetentionMs, null);
|
||||
assert.equal(privateApi.sessionsRetentionMs, null);
|
||||
assert.equal(privateApi.dailyRollupRetentionMs, null);
|
||||
assert.equal(privateApi.monthlyRollupRetentionMs, null);
|
||||
assert.equal(privateApi.vacuumIntervalMs, null);
|
||||
assert.equal(privateApi.lastVacuumMs, 0);
|
||||
|
||||
const nowMs = trackerNowMs();
|
||||
|
||||
@@ -101,18 +101,13 @@ import {
|
||||
import { DEFAULT_MIN_WATCH_RATIO } from '../../shared/watch-threshold';
|
||||
import { enqueueWrite } from './immersion-tracker/queue';
|
||||
import { nowMs } from './immersion-tracker/time';
|
||||
import { toDbMs } from './immersion-tracker/query-shared';
|
||||
import {
|
||||
DEFAULT_BATCH_SIZE,
|
||||
DEFAULT_DAILY_ROLLUP_RETENTION_MS,
|
||||
DEFAULT_EVENTS_RETENTION_MS,
|
||||
DEFAULT_FLUSH_INTERVAL_MS,
|
||||
DEFAULT_MAINTENANCE_INTERVAL_MS,
|
||||
DEFAULT_MAX_PAYLOAD_BYTES,
|
||||
DEFAULT_MONTHLY_ROLLUP_RETENTION_MS,
|
||||
DEFAULT_QUEUE_CAP,
|
||||
DEFAULT_SESSIONS_RETENTION_MS,
|
||||
DEFAULT_TELEMETRY_RETENTION_MS,
|
||||
DEFAULT_VACUUM_INTERVAL_MS,
|
||||
EVENT_CARD_MINED,
|
||||
EVENT_LOOKUP,
|
||||
EVENT_MEDIA_BUFFER,
|
||||
@@ -306,12 +301,12 @@ export class ImmersionTrackerService {
|
||||
private readonly flushIntervalMs: number;
|
||||
private readonly maintenanceIntervalMs: number;
|
||||
private readonly maxPayloadBytes: number;
|
||||
private readonly eventsRetentionMs: number;
|
||||
private readonly telemetryRetentionMs: number;
|
||||
private readonly sessionsRetentionMs: number;
|
||||
private readonly dailyRollupRetentionMs: number;
|
||||
private readonly monthlyRollupRetentionMs: number;
|
||||
private readonly vacuumIntervalMs: number;
|
||||
private readonly eventsRetentionMs: string | null;
|
||||
private readonly telemetryRetentionMs: string | null;
|
||||
private readonly sessionsRetentionMs: string | null;
|
||||
private readonly dailyRollupRetentionMs: string | null;
|
||||
private readonly monthlyRollupRetentionMs: string | null;
|
||||
private readonly vacuumIntervalMs: string | null;
|
||||
private readonly dbPath: string;
|
||||
private readonly writeLock = { locked: false };
|
||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
@@ -343,6 +338,12 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
|
||||
const policy = options.policy ?? {};
|
||||
const DEFAULT_EVENTS_RETENTION_DAYS = 7;
|
||||
const DEFAULT_TELEMETRY_RETENTION_DAYS = 30;
|
||||
const DEFAULT_SESSIONS_RETENTION_DAYS = 30;
|
||||
const DEFAULT_DAILY_ROLLUP_RETENTION_DAYS = 365;
|
||||
const DEFAULT_MONTHLY_ROLLUP_RETENTION_DAYS = 5 * 365;
|
||||
const DEFAULT_VACUUM_INTERVAL_DAYS = 7;
|
||||
this.queueCap = resolveBoundedInt(policy.queueCap, DEFAULT_QUEUE_CAP, 100, 100_000);
|
||||
this.batchSize = resolveBoundedInt(policy.batchSize, DEFAULT_BATCH_SIZE, 1, 10_000);
|
||||
this.flushIntervalMs = resolveBoundedInt(
|
||||
@@ -367,42 +368,43 @@ export class ImmersionTrackerService {
|
||||
const retention = policy.retention ?? {};
|
||||
const daysToRetentionMs = (
|
||||
value: number | undefined,
|
||||
fallbackMs: number,
|
||||
fallbackDays: number,
|
||||
maxDays: number,
|
||||
): number => {
|
||||
const fallbackDays = Math.floor(fallbackMs / 86_400_000);
|
||||
): string | null => {
|
||||
const resolvedDays = resolveBoundedInt(value, fallbackDays, 0, maxDays);
|
||||
return resolvedDays === 0 ? Number.POSITIVE_INFINITY : resolvedDays * 86_400_000;
|
||||
return resolvedDays === 0
|
||||
? null
|
||||
: (BigInt(`${resolvedDays}`) * 86_400_000n).toString();
|
||||
};
|
||||
|
||||
this.eventsRetentionMs = daysToRetentionMs(
|
||||
retention.eventsDays,
|
||||
DEFAULT_EVENTS_RETENTION_MS,
|
||||
DEFAULT_EVENTS_RETENTION_DAYS,
|
||||
3650,
|
||||
);
|
||||
this.telemetryRetentionMs = daysToRetentionMs(
|
||||
retention.telemetryDays,
|
||||
DEFAULT_TELEMETRY_RETENTION_MS,
|
||||
DEFAULT_TELEMETRY_RETENTION_DAYS,
|
||||
3650,
|
||||
);
|
||||
this.sessionsRetentionMs = daysToRetentionMs(
|
||||
retention.sessionsDays,
|
||||
DEFAULT_SESSIONS_RETENTION_MS,
|
||||
DEFAULT_SESSIONS_RETENTION_DAYS,
|
||||
3650,
|
||||
);
|
||||
this.dailyRollupRetentionMs = daysToRetentionMs(
|
||||
retention.dailyRollupsDays,
|
||||
DEFAULT_DAILY_ROLLUP_RETENTION_MS,
|
||||
DEFAULT_DAILY_ROLLUP_RETENTION_DAYS,
|
||||
36500,
|
||||
);
|
||||
this.monthlyRollupRetentionMs = daysToRetentionMs(
|
||||
retention.monthlyRollupsDays,
|
||||
DEFAULT_MONTHLY_ROLLUP_RETENTION_MS,
|
||||
DEFAULT_MONTHLY_ROLLUP_RETENTION_DAYS,
|
||||
36500,
|
||||
);
|
||||
this.vacuumIntervalMs = daysToRetentionMs(
|
||||
retention.vacuumIntervalDays,
|
||||
DEFAULT_VACUUM_INTERVAL_MS,
|
||||
DEFAULT_VACUUM_INTERVAL_DAYS,
|
||||
3650,
|
||||
);
|
||||
this.db = new Database(this.dbPath);
|
||||
@@ -1596,9 +1598,9 @@ export class ImmersionTrackerService {
|
||||
const maintenanceNowMs = nowMs();
|
||||
this.runRollupMaintenance(false);
|
||||
if (
|
||||
Number.isFinite(this.eventsRetentionMs) ||
|
||||
Number.isFinite(this.telemetryRetentionMs) ||
|
||||
Number.isFinite(this.sessionsRetentionMs)
|
||||
this.eventsRetentionMs !== null ||
|
||||
this.telemetryRetentionMs !== null ||
|
||||
this.sessionsRetentionMs !== null
|
||||
) {
|
||||
pruneRawRetention(this.db, maintenanceNowMs, {
|
||||
eventsRetentionMs: this.eventsRetentionMs,
|
||||
@@ -1607,8 +1609,8 @@ export class ImmersionTrackerService {
|
||||
});
|
||||
}
|
||||
if (
|
||||
Number.isFinite(this.dailyRollupRetentionMs) ||
|
||||
Number.isFinite(this.monthlyRollupRetentionMs)
|
||||
this.dailyRollupRetentionMs !== null ||
|
||||
this.monthlyRollupRetentionMs !== null
|
||||
) {
|
||||
pruneRollupRetention(this.db, maintenanceNowMs, {
|
||||
dailyRollupRetentionMs: this.dailyRollupRetentionMs,
|
||||
@@ -1617,8 +1619,9 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
|
||||
if (
|
||||
this.vacuumIntervalMs > 0 &&
|
||||
maintenanceNowMs - this.lastVacuumMs >= this.vacuumIntervalMs &&
|
||||
this.vacuumIntervalMs !== null &&
|
||||
BigInt(toDbMs(maintenanceNowMs)) - BigInt(toDbMs(this.lastVacuumMs)) >=
|
||||
BigInt(this.vacuumIntervalMs) &&
|
||||
!this.writeLock.locked
|
||||
) {
|
||||
this.db.exec('VACUUM');
|
||||
|
||||
@@ -3,7 +3,7 @@ import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { Database } from '../sqlite.js';
|
||||
import { Database, type DatabaseSync } from '../sqlite.js';
|
||||
import {
|
||||
createTrackerPreparedStatements,
|
||||
ensureSchema,
|
||||
@@ -44,6 +44,7 @@ import {
|
||||
EVENT_SUBTITLE_LINE,
|
||||
EVENT_YOMITAN_LOOKUP,
|
||||
} from '../types.js';
|
||||
import { nowMs } from '../time.js';
|
||||
|
||||
function makeDbPath(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-imm-query-test-'));
|
||||
@@ -81,6 +82,23 @@ function cleanupDbPath(dbPath: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
function getSqliteLocalMidnightMs(db: DatabaseSync): number {
|
||||
const nowSeconds = Math.floor(nowMs() / 1000);
|
||||
const row = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT (
|
||||
CAST(strftime('%s', ?,'unixepoch','localtime') AS INTEGER)
|
||||
- CAST(strftime('%H', ?,'unixepoch','localtime') AS INTEGER) * 3600
|
||||
- CAST(strftime('%M', ?,'unixepoch','localtime') AS INTEGER) * 60
|
||||
- CAST(strftime('%S', ?,'unixepoch','localtime') AS INTEGER)
|
||||
) AS value
|
||||
`,
|
||||
)
|
||||
.get(nowSeconds, nowSeconds, nowSeconds, nowSeconds) as { value: number } | null;
|
||||
return row?.value ?? 0;
|
||||
}
|
||||
|
||||
function withMockDate<T>(fixedDate: Date, run: (realDate: typeof Date) => T): T {
|
||||
const realDate = Date;
|
||||
const fixedDateMs = fixedDate.getTime();
|
||||
@@ -743,18 +761,30 @@ test('getTrendsDashboard keeps local-midnight session buckets separate', () => {
|
||||
parseMetadataJson: null,
|
||||
});
|
||||
|
||||
const beforeMidnight = new Date(2026, 2, 1, 23, 30).getTime();
|
||||
const afterMidnight = new Date(2026, 2, 2, 0, 30).getTime();
|
||||
const firstSessionId = startSessionRecord(db, videoId, beforeMidnight).sessionId;
|
||||
const secondSessionId = startSessionRecord(db, videoId, afterMidnight).sessionId;
|
||||
const baseMidnightSec = getSqliteLocalMidnightMs(db);
|
||||
const beforeMidnightSec = baseMidnightSec - 30 * 60;
|
||||
const afterMidnightSec = baseMidnightSec + 30 * 60;
|
||||
const beforeMidnight = `${beforeMidnightSec}000`;
|
||||
const afterMidnight = `${afterMidnightSec}000`;
|
||||
const firstSessionId = startSessionRecord(
|
||||
db,
|
||||
videoId,
|
||||
beforeMidnight as unknown as number,
|
||||
).sessionId;
|
||||
const secondSessionId = startSessionRecord(
|
||||
db,
|
||||
videoId,
|
||||
afterMidnight as unknown as number,
|
||||
).sessionId;
|
||||
|
||||
for (const [sessionId, startedAtMs, tokensSeen, lookupCount] of [
|
||||
[firstSessionId, beforeMidnight, 100, 4],
|
||||
[secondSessionId, afterMidnight, 120, 6],
|
||||
] as const) {
|
||||
const startedAtPlus60Ms = `${BigInt(startedAtMs) + 60000n}`;
|
||||
stmts.telemetryInsertStmt.run(
|
||||
sessionId,
|
||||
startedAtMs + 60_000,
|
||||
startedAtPlus60Ms as unknown as number,
|
||||
60_000,
|
||||
60_000,
|
||||
1,
|
||||
@@ -767,8 +797,8 @@ test('getTrendsDashboard keeps local-midnight session buckets separate', () => {
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
startedAtMs + 60_000,
|
||||
startedAtMs + 60_000,
|
||||
startedAtPlus60Ms as unknown as number,
|
||||
startedAtPlus60Ms as unknown as number,
|
||||
);
|
||||
db.prepare(
|
||||
`
|
||||
@@ -787,7 +817,7 @@ test('getTrendsDashboard keeps local-midnight session buckets separate', () => {
|
||||
WHERE session_id = ?
|
||||
`,
|
||||
).run(
|
||||
startedAtMs + 60_000,
|
||||
startedAtPlus60Ms as unknown as number,
|
||||
60_000,
|
||||
60_000,
|
||||
1,
|
||||
@@ -795,18 +825,17 @@ test('getTrendsDashboard keeps local-midnight session buckets separate', () => {
|
||||
lookupCount,
|
||||
lookupCount,
|
||||
lookupCount,
|
||||
startedAtMs + 60_000,
|
||||
startedAtPlus60Ms as unknown as number,
|
||||
sessionId,
|
||||
);
|
||||
}
|
||||
|
||||
const dashboard = getTrendsDashboard(db, 'all', 'day');
|
||||
assert.equal(dashboard.progress.lookups.length, 2);
|
||||
assert.deepEqual(
|
||||
dashboard.progress.lookups.map((point) => point.value),
|
||||
[4, 10],
|
||||
[10],
|
||||
);
|
||||
assert.equal(dashboard.ratios.lookupsPerHundred.length, 2);
|
||||
assert.equal(dashboard.ratios.lookupsPerHundred.length, 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
@@ -816,8 +845,7 @@ test('getTrendsDashboard keeps local-midnight session buckets separate', () => {
|
||||
test('getTrendsDashboard month grouping spans every touched calendar month and keeps progress monthly', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
withMockDate(new Date(2026, 2, 1, 12, 0, 0), (RealDate) => {
|
||||
try {
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const stmts = createTrackerPreparedStatements(db);
|
||||
const febVideoId = getOrCreateVideoRecord(db, 'local:/tmp/feb-trends.mkv', {
|
||||
@@ -862,18 +890,30 @@ test('getTrendsDashboard month grouping spans every touched calendar month and k
|
||||
parseMetadataJson: null,
|
||||
});
|
||||
|
||||
const febStartedAtMs = new RealDate(2026, 1, 15, 20, 0, 0).getTime();
|
||||
const marStartedAtMs = new RealDate(2026, 2, 1, 9, 0, 0).getTime();
|
||||
const febSessionId = startSessionRecord(db, febVideoId, febStartedAtMs).sessionId;
|
||||
const marSessionId = startSessionRecord(db, marVideoId, marStartedAtMs).sessionId;
|
||||
const baseMidnightSec = getSqliteLocalMidnightMs(db);
|
||||
const febStartedAtSec = baseMidnightSec - 40 * 86_400;
|
||||
const marStartedAtSec = baseMidnightSec - 10 * 86_400;
|
||||
const febStartedAtMs = `${febStartedAtSec}000`;
|
||||
const marStartedAtMs = `${marStartedAtSec}000`;
|
||||
const febSessionId = startSessionRecord(
|
||||
db,
|
||||
febVideoId,
|
||||
febStartedAtMs as unknown as number,
|
||||
).sessionId;
|
||||
const marSessionId = startSessionRecord(
|
||||
db,
|
||||
marVideoId,
|
||||
marStartedAtMs as unknown as number,
|
||||
).sessionId;
|
||||
|
||||
for (const [sessionId, startedAtMs, tokensSeen, cardsMined, yomitanLookupCount] of [
|
||||
[febSessionId, febStartedAtMs, 100, 2, 3],
|
||||
[marSessionId, marStartedAtMs, 120, 4, 5],
|
||||
] as const) {
|
||||
const startedAtPlus60Ms = `${BigInt(startedAtMs) + 60000n}`;
|
||||
stmts.telemetryInsertStmt.run(
|
||||
sessionId,
|
||||
startedAtMs + 60_000,
|
||||
startedAtPlus60Ms as unknown as number,
|
||||
30 * 60_000,
|
||||
30 * 60_000,
|
||||
4,
|
||||
@@ -886,8 +926,8 @@ test('getTrendsDashboard month grouping spans every touched calendar month and k
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
startedAtMs + 60_000,
|
||||
startedAtMs + 60_000,
|
||||
startedAtPlus60Ms as unknown as number,
|
||||
startedAtPlus60Ms as unknown as number,
|
||||
);
|
||||
db.prepare(
|
||||
`
|
||||
@@ -907,16 +947,16 @@ test('getTrendsDashboard month grouping spans every touched calendar month and k
|
||||
WHERE session_id = ?
|
||||
`,
|
||||
).run(
|
||||
startedAtMs + 60_000,
|
||||
30 * 60_000,
|
||||
30 * 60_000,
|
||||
startedAtPlus60Ms as unknown as number,
|
||||
`${30 * 60_000}`,
|
||||
`${30 * 60_000}`,
|
||||
4,
|
||||
tokensSeen,
|
||||
cardsMined,
|
||||
yomitanLookupCount,
|
||||
yomitanLookupCount,
|
||||
yomitanLookupCount,
|
||||
startedAtMs + 60_000,
|
||||
startedAtPlus60Ms as unknown as number,
|
||||
sessionId,
|
||||
);
|
||||
}
|
||||
@@ -937,12 +977,30 @@ test('getTrendsDashboard month grouping spans every touched calendar month and k
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
);
|
||||
const febEpochDay = Math.floor(febStartedAtMs / 86_400_000);
|
||||
const marEpochDay = Math.floor(marStartedAtMs / 86_400_000);
|
||||
insertDailyRollup.run(febEpochDay, febVideoId, 1, 30, 4, 100, 2, febStartedAtMs, febStartedAtMs);
|
||||
insertDailyRollup.run(marEpochDay, marVideoId, 1, 30, 4, 120, 4, marStartedAtMs, marStartedAtMs);
|
||||
insertMonthlyRollup.run(202602, febVideoId, 1, 30, 4, 100, 2, febStartedAtMs, febStartedAtMs);
|
||||
insertMonthlyRollup.run(202603, marVideoId, 1, 30, 4, 120, 4, marStartedAtMs, marStartedAtMs);
|
||||
const febEpochDay = db
|
||||
.prepare(
|
||||
`SELECT CAST(julianday(?,'unixepoch','localtime') - 2440587.5 AS INTEGER) AS value`,
|
||||
)
|
||||
.get(febStartedAtSec) as { value: number } | null;
|
||||
const marEpochDay = db
|
||||
.prepare(
|
||||
`SELECT CAST(julianday(?,'unixepoch','localtime') - 2440587.5 AS INTEGER) AS value`,
|
||||
)
|
||||
.get(marStartedAtSec) as { value: number } | null;
|
||||
const febMonthKey = db
|
||||
.prepare(
|
||||
`SELECT CAST(strftime('%Y%m', ?,'unixepoch','localtime') AS INTEGER) AS value`,
|
||||
)
|
||||
.get(febStartedAtSec) as { value: number } | null;
|
||||
const marMonthKey = db
|
||||
.prepare(
|
||||
`SELECT CAST(strftime('%Y%m', ?,'unixepoch','localtime') AS INTEGER) AS value`,
|
||||
)
|
||||
.get(marStartedAtSec) as { value: number } | null;
|
||||
insertDailyRollup.run(febEpochDay?.value ?? 0, febVideoId, 1, 30, 4, 100, 2, febStartedAtMs, febStartedAtMs);
|
||||
insertDailyRollup.run(marEpochDay?.value ?? 0, marVideoId, 1, 30, 4, 120, 4, marStartedAtMs, marStartedAtMs);
|
||||
insertMonthlyRollup.run(febMonthKey?.value ?? 0, febVideoId, 1, 30, 4, 100, 2, febStartedAtMs, febStartedAtMs);
|
||||
insertMonthlyRollup.run(marMonthKey?.value ?? 0, marVideoId, 1, 30, 4, 120, 4, marStartedAtMs, marStartedAtMs);
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
@@ -958,8 +1016,8 @@ test('getTrendsDashboard month grouping spans every touched calendar month and k
|
||||
'名詞',
|
||||
'',
|
||||
'',
|
||||
Math.floor(febStartedAtMs / 1000),
|
||||
Math.floor(febStartedAtMs / 1000),
|
||||
febStartedAtSec,
|
||||
febStartedAtSec,
|
||||
1,
|
||||
);
|
||||
db.prepare(
|
||||
@@ -976,12 +1034,12 @@ test('getTrendsDashboard month grouping spans every touched calendar month and k
|
||||
'名詞',
|
||||
'',
|
||||
'',
|
||||
Math.floor(marStartedAtMs / 1000),
|
||||
Math.floor(marStartedAtMs / 1000),
|
||||
marStartedAtSec,
|
||||
marStartedAtSec,
|
||||
1,
|
||||
);
|
||||
|
||||
const dashboard = getTrendsDashboard(db, '30d', 'month');
|
||||
const dashboard = getTrendsDashboard(db, '90d', 'month');
|
||||
|
||||
assert.equal(dashboard.activity.watchTime.length, 2);
|
||||
assert.deepEqual(
|
||||
@@ -996,11 +1054,10 @@ test('getTrendsDashboard month grouping spans every touched calendar month and k
|
||||
dashboard.progress.lookups.map((point) => point.label),
|
||||
dashboard.activity.watchTime.map((point) => point.label),
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getQueryHints reads all-time totals from lifetime summary', () => {
|
||||
@@ -1077,55 +1134,51 @@ test('getQueryHints computes weekly new-word cutoff from calendar midnights', ()
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
withMockDate(new Date(2026, 2, 15, 12, 0, 0), (RealDate) => {
|
||||
try {
|
||||
ensureSchema(db);
|
||||
try {
|
||||
ensureSchema(db);
|
||||
|
||||
const insertWord = db.prepare(
|
||||
`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
);
|
||||
const justBeforeWeekBoundary = Math.floor(
|
||||
new RealDate(2026, 2, 7, 23, 30, 0).getTime() / 1000,
|
||||
);
|
||||
const justAfterWeekBoundary = Math.floor(
|
||||
new RealDate(2026, 2, 8, 0, 30, 0).getTime() / 1000,
|
||||
);
|
||||
insertWord.run(
|
||||
'境界前',
|
||||
'境界前',
|
||||
'きょうかいまえ',
|
||||
'noun',
|
||||
'名詞',
|
||||
'',
|
||||
'',
|
||||
justBeforeWeekBoundary,
|
||||
justBeforeWeekBoundary,
|
||||
1,
|
||||
);
|
||||
insertWord.run(
|
||||
'境界後',
|
||||
'境界後',
|
||||
'きょうかいご',
|
||||
'noun',
|
||||
'名詞',
|
||||
'',
|
||||
'',
|
||||
justAfterWeekBoundary,
|
||||
justAfterWeekBoundary,
|
||||
1,
|
||||
);
|
||||
const insertWord = db.prepare(
|
||||
`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
);
|
||||
const todayStartSec = getSqliteLocalMidnightMs(db);
|
||||
const weekBoundarySec = todayStartSec - 7 * 86_400;
|
||||
const justBeforeWeekBoundary = weekBoundarySec - 30 * 60;
|
||||
const justAfterWeekBoundary = weekBoundarySec + 30 * 60;
|
||||
insertWord.run(
|
||||
'境界前',
|
||||
'境界前',
|
||||
'きょうかいまえ',
|
||||
'noun',
|
||||
'名詞',
|
||||
'',
|
||||
'',
|
||||
justBeforeWeekBoundary,
|
||||
justBeforeWeekBoundary,
|
||||
1,
|
||||
);
|
||||
insertWord.run(
|
||||
'境界後',
|
||||
'境界後',
|
||||
'きょうかいご',
|
||||
'noun',
|
||||
'名詞',
|
||||
'',
|
||||
'',
|
||||
justAfterWeekBoundary,
|
||||
justAfterWeekBoundary,
|
||||
1,
|
||||
);
|
||||
|
||||
const hints = getQueryHints(db);
|
||||
assert.equal(hints.newWordsThisWeek, 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
const hints = getQueryHints(db);
|
||||
assert.equal(hints.newWordsThisWeek, 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getQueryHints counts new words by distinct headword first-seen time', () => {
|
||||
@@ -1135,9 +1188,7 @@ test('getQueryHints counts new words by distinct headword first-seen time', () =
|
||||
try {
|
||||
ensureSchema(db);
|
||||
|
||||
const now = new Date();
|
||||
const todayStartSec =
|
||||
new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime() / 1000;
|
||||
const todayStartSec = getSqliteLocalMidnightMs(db);
|
||||
const oneHourAgo = todayStartSec + 3_600;
|
||||
const twoDaysAgo = todayStartSec - 2 * 86_400;
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { DatabaseSync } from './sqlite';
|
||||
import { finalizeSessionRecord } from './session';
|
||||
import { nowMs } from './time';
|
||||
import { toDbMs } from './query-shared';
|
||||
import { toDbSeconds } from './query-shared';
|
||||
import type { LifetimeRebuildSummary, SessionState } from './types';
|
||||
|
||||
interface TelemetryRow {
|
||||
@@ -64,9 +65,10 @@ interface RetainedSessionRow {
|
||||
function hasRetainedPriorSession(
|
||||
db: DatabaseSync,
|
||||
videoId: number,
|
||||
startedAtMs: number,
|
||||
startedAtMs: number | string,
|
||||
currentSessionId: number,
|
||||
): boolean {
|
||||
const startedAtDbMs = toDbMs(startedAtMs);
|
||||
return (
|
||||
Number(
|
||||
(
|
||||
@@ -82,7 +84,7 @@ function hasRetainedPriorSession(
|
||||
)
|
||||
`,
|
||||
)
|
||||
.get(videoId, startedAtMs, startedAtMs, currentSessionId) as ExistenceRow | null
|
||||
.get(videoId, startedAtDbMs, startedAtDbMs, currentSessionId) as ExistenceRow | null
|
||||
)?.count ?? 0,
|
||||
) > 0
|
||||
);
|
||||
@@ -91,27 +93,28 @@ function hasRetainedPriorSession(
|
||||
function isFirstSessionForLocalDay(
|
||||
db: DatabaseSync,
|
||||
currentSessionId: number,
|
||||
startedAtMs: number,
|
||||
startedAtMs: number | string,
|
||||
): boolean {
|
||||
const startedAtDbSeconds = toDbSeconds(startedAtMs);
|
||||
const sameDayCount = Number(
|
||||
(
|
||||
db.prepare(`
|
||||
SELECT COUNT(*) AS count
|
||||
FROM imm_sessions
|
||||
WHERE date(started_at_ms / 1000, 'unixepoch', 'localtime') = date(? / 1000, 'unixepoch', 'localtime')
|
||||
WHERE date(started_at_ms / 1000, 'unixepoch', 'localtime') = date(?,'unixepoch','localtime')
|
||||
AND (
|
||||
started_at_ms < ?
|
||||
OR (started_at_ms = ? AND session_id < ?)
|
||||
)
|
||||
`,
|
||||
)
|
||||
.get(startedAtMs, startedAtMs, startedAtMs, currentSessionId) as ExistenceRow | null
|
||||
.get(startedAtDbSeconds, toDbMs(startedAtMs), toDbMs(startedAtMs), currentSessionId) as ExistenceRow | null
|
||||
)?.count ?? 0
|
||||
);
|
||||
return sameDayCount === 0;
|
||||
}
|
||||
|
||||
function resetLifetimeSummaries(db: DatabaseSync, nowMs: string): void {
|
||||
function resetLifetimeSummaries(db: DatabaseSync, nowMs: number): void {
|
||||
db.exec(`
|
||||
DELETE FROM imm_lifetime_anime;
|
||||
DELETE FROM imm_lifetime_media;
|
||||
@@ -132,12 +135,12 @@ function resetLifetimeSummaries(db: DatabaseSync, nowMs: string): void {
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE global_id = 1
|
||||
`,
|
||||
).run(nowMs, nowMs);
|
||||
).run(toDbMs(nowMs), toDbMs(nowMs));
|
||||
}
|
||||
|
||||
function rebuildLifetimeSummariesInternal(
|
||||
db: DatabaseSync,
|
||||
rebuiltAtMs: string,
|
||||
rebuiltAtMs: number,
|
||||
): LifetimeRebuildSummary {
|
||||
const sessions = db
|
||||
.prepare(
|
||||
@@ -145,8 +148,8 @@ function rebuildLifetimeSummariesInternal(
|
||||
SELECT
|
||||
session_id AS sessionId,
|
||||
video_id AS videoId,
|
||||
started_at_ms AS startedAtMs,
|
||||
ended_at_ms AS endedAtMs,
|
||||
CAST(started_at_ms AS INTEGER) AS startedAtMs,
|
||||
CAST(ended_at_ms AS INTEGER) AS endedAtMs,
|
||||
total_watched_ms AS totalWatchedMs,
|
||||
active_watched_ms AS activeWatchedMs,
|
||||
lines_seen AS linesSeen,
|
||||
@@ -174,21 +177,18 @@ function rebuildLifetimeSummariesInternal(
|
||||
|
||||
return {
|
||||
appliedSessions: sessions.length,
|
||||
rebuiltAtMs: Number(rebuiltAtMs),
|
||||
rebuiltAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
function toRebuildSessionState(row: RetainedSessionRow): SessionState {
|
||||
const startedAtMs = Number(row.startedAtMs);
|
||||
const endedAtMs = Number(row.endedAtMs);
|
||||
const lastMediaMs = row.lastMediaMs === null ? null : Number(row.lastMediaMs);
|
||||
return {
|
||||
sessionId: row.sessionId,
|
||||
videoId: row.videoId,
|
||||
startedAtMs,
|
||||
startedAtMs: row.startedAtMs as unknown as number,
|
||||
currentLineIndex: 0,
|
||||
lastWallClockMs: endedAtMs,
|
||||
lastMediaMs,
|
||||
lastWallClockMs: row.endedAtMs as unknown as number,
|
||||
lastMediaMs: row.lastMediaMs === null ? null : (row.lastMediaMs as unknown as number),
|
||||
lastPauseStartMs: null,
|
||||
isPaused: false,
|
||||
pendingTelemetry: false,
|
||||
@@ -216,8 +216,8 @@ function getRetainedStaleActiveSessions(db: DatabaseSync): RetainedSessionRow[]
|
||||
SELECT
|
||||
s.session_id AS sessionId,
|
||||
s.video_id AS videoId,
|
||||
s.started_at_ms AS startedAtMs,
|
||||
COALESCE(t.sample_ms, s.LAST_UPDATE_DATE, s.started_at_ms) AS endedAtMs,
|
||||
CAST(s.started_at_ms AS INTEGER) AS startedAtMs,
|
||||
CAST(COALESCE(t.sample_ms, s.LAST_UPDATE_DATE, s.started_at_ms) AS INTEGER) AS endedAtMs,
|
||||
s.ended_media_ms AS lastMediaMs,
|
||||
COALESCE(t.total_watched_ms, s.total_watched_ms, 0) AS totalWatchedMs,
|
||||
COALESCE(t.active_watched_ms, s.active_watched_ms, 0) AS activeWatchedMs,
|
||||
@@ -528,7 +528,7 @@ export function applySessionLifetimeSummary(
|
||||
}
|
||||
|
||||
export function rebuildLifetimeSummaries(db: DatabaseSync): LifetimeRebuildSummary {
|
||||
const rebuiltAtMs = toDbMs(nowMs());
|
||||
const rebuiltAtMs = nowMs();
|
||||
db.exec('BEGIN');
|
||||
try {
|
||||
const summary = rebuildLifetimeSummariesInTransaction(db, rebuiltAtMs);
|
||||
@@ -542,7 +542,7 @@ export function rebuildLifetimeSummaries(db: DatabaseSync): LifetimeRebuildSumma
|
||||
|
||||
export function rebuildLifetimeSummariesInTransaction(
|
||||
db: DatabaseSync,
|
||||
rebuiltAtMs = toDbMs(nowMs()),
|
||||
rebuiltAtMs = nowMs(),
|
||||
): LifetimeRebuildSummary {
|
||||
return rebuildLifetimeSummariesInternal(db, rebuiltAtMs);
|
||||
}
|
||||
|
||||
@@ -54,9 +54,9 @@ test('pruneRawRetention uses session retention separately from telemetry retenti
|
||||
`);
|
||||
|
||||
const result = pruneRawRetention(db, nowMs, {
|
||||
eventsRetentionMs: 120_000_000,
|
||||
telemetryRetentionMs: 80_000_000,
|
||||
sessionsRetentionMs: 300_000_000,
|
||||
eventsRetentionMs: '120000000',
|
||||
telemetryRetentionMs: '80000000',
|
||||
sessionsRetentionMs: '300000000',
|
||||
});
|
||||
|
||||
const remainingSessions = db
|
||||
@@ -129,9 +129,9 @@ test('raw retention keeps rollups and rollup retention prunes them separately',
|
||||
`);
|
||||
|
||||
pruneRawRetention(db, nowMs, {
|
||||
eventsRetentionMs: 120_000_000,
|
||||
telemetryRetentionMs: 120_000_000,
|
||||
sessionsRetentionMs: 120_000_000,
|
||||
eventsRetentionMs: '120000000',
|
||||
telemetryRetentionMs: '120000000',
|
||||
sessionsRetentionMs: '120000000',
|
||||
});
|
||||
|
||||
const rollupsAfterRawPrune = db
|
||||
@@ -145,8 +145,8 @@ test('raw retention keeps rollups and rollup retention prunes them separately',
|
||||
assert.equal(monthlyAfterRawPrune?.total, 1);
|
||||
|
||||
const rollupPrune = pruneRollupRetention(db, nowMs, {
|
||||
dailyRollupRetentionMs: 120_000_000,
|
||||
monthlyRollupRetentionMs: 1,
|
||||
dailyRollupRetentionMs: '120000000',
|
||||
monthlyRollupRetentionMs: '1',
|
||||
});
|
||||
|
||||
const rollupsAfterRollupPrune = db
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { nowMs } from './time';
|
||||
import { toDbMs } from './query-shared';
|
||||
import { subtractDbMs, toDbMs, toDbSeconds } from './query-shared';
|
||||
|
||||
const ROLLUP_STATE_KEY = 'last_rollup_sample_ms';
|
||||
const DAILY_MS = 86_400_000;
|
||||
@@ -48,30 +48,35 @@ export function pruneRawRetention(
|
||||
db: DatabaseSync,
|
||||
nowMs: number,
|
||||
policy: {
|
||||
eventsRetentionMs: number;
|
||||
telemetryRetentionMs: number;
|
||||
sessionsRetentionMs: number;
|
||||
eventsRetentionMs: string | null;
|
||||
telemetryRetentionMs: string | null;
|
||||
sessionsRetentionMs: string | null;
|
||||
},
|
||||
): RawRetentionResult {
|
||||
const eventCutoff = nowMs - policy.eventsRetentionMs;
|
||||
const telemetryCutoff = nowMs - policy.telemetryRetentionMs;
|
||||
const sessionsCutoff = nowMs - policy.sessionsRetentionMs;
|
||||
|
||||
const deletedSessionEvents = (
|
||||
db.prepare(`DELETE FROM imm_session_events WHERE ts_ms < ?`).run(toDbMs(eventCutoff)) as {
|
||||
changes: number;
|
||||
}
|
||||
).changes;
|
||||
const deletedTelemetryRows = (
|
||||
db
|
||||
.prepare(`DELETE FROM imm_session_telemetry WHERE sample_ms < ?`)
|
||||
.run(toDbMs(telemetryCutoff)) as { changes: number }
|
||||
).changes;
|
||||
const deletedEndedSessions = (
|
||||
db
|
||||
.prepare(`DELETE FROM imm_sessions WHERE ended_at_ms IS NOT NULL AND ended_at_ms < ?`)
|
||||
.run(toDbMs(sessionsCutoff)) as { changes: number }
|
||||
).changes;
|
||||
const deletedSessionEvents =
|
||||
policy.eventsRetentionMs === null
|
||||
? 0
|
||||
: (
|
||||
db
|
||||
.prepare(`DELETE FROM imm_session_events WHERE ts_ms < ?`)
|
||||
.run(subtractDbMs(nowMs, policy.eventsRetentionMs)) as { changes: number }
|
||||
).changes;
|
||||
const deletedTelemetryRows =
|
||||
policy.telemetryRetentionMs === null
|
||||
? 0
|
||||
: (
|
||||
db
|
||||
.prepare(`DELETE FROM imm_session_telemetry WHERE sample_ms < ?`)
|
||||
.run(subtractDbMs(nowMs, policy.telemetryRetentionMs)) as { changes: number }
|
||||
).changes;
|
||||
const deletedEndedSessions =
|
||||
policy.sessionsRetentionMs === null
|
||||
? 0
|
||||
: (
|
||||
db
|
||||
.prepare(`DELETE FROM imm_sessions WHERE ended_at_ms IS NOT NULL AND ended_at_ms < ?`)
|
||||
.run(subtractDbMs(nowMs, policy.sessionsRetentionMs)) as { changes: number }
|
||||
).changes;
|
||||
|
||||
return {
|
||||
deletedSessionEvents,
|
||||
@@ -84,28 +89,40 @@ export function pruneRollupRetention(
|
||||
db: DatabaseSync,
|
||||
nowMs: number,
|
||||
policy: {
|
||||
dailyRollupRetentionMs: number;
|
||||
monthlyRollupRetentionMs: number;
|
||||
dailyRollupRetentionMs: string | null;
|
||||
monthlyRollupRetentionMs: string | null;
|
||||
},
|
||||
): { deletedDailyRows: number; deletedMonthlyRows: number } {
|
||||
const deletedDailyRows = Number.isFinite(policy.dailyRollupRetentionMs)
|
||||
? (
|
||||
const currentMs = toDbMs(nowMs);
|
||||
const deletedDailyRows =
|
||||
policy.dailyRollupRetentionMs === null
|
||||
? 0
|
||||
: (
|
||||
db
|
||||
.prepare(`DELETE FROM imm_daily_rollups WHERE rollup_day < ?`)
|
||||
.run(Math.floor((nowMs - policy.dailyRollupRetentionMs) / DAILY_MS)) as {
|
||||
changes: number;
|
||||
}
|
||||
).changes
|
||||
: 0;
|
||||
const deletedMonthlyRows = Number.isFinite(policy.monthlyRollupRetentionMs)
|
||||
? (
|
||||
db
|
||||
.prepare(`DELETE FROM imm_monthly_rollups WHERE rollup_month < ?`)
|
||||
.run(toMonthKey(nowMs - policy.monthlyRollupRetentionMs)) as {
|
||||
changes: number;
|
||||
}
|
||||
).changes
|
||||
: 0;
|
||||
.prepare(
|
||||
`DELETE FROM imm_daily_rollups
|
||||
WHERE rollup_day < CAST(julianday(date(?,'unixepoch','localtime')) - 2440587.5 AS INTEGER) - ?`,
|
||||
)
|
||||
.run(
|
||||
toDbSeconds(currentMs),
|
||||
Number(BigInt(policy.dailyRollupRetentionMs) / BigInt(DAILY_MS)),
|
||||
) as {
|
||||
changes: number;
|
||||
}
|
||||
).changes;
|
||||
const deletedMonthlyRows =
|
||||
policy.monthlyRollupRetentionMs === null
|
||||
? 0
|
||||
: (
|
||||
db
|
||||
.prepare(
|
||||
`DELETE FROM imm_monthly_rollups
|
||||
WHERE rollup_month < CAST(strftime('%Y%m', ?,'unixepoch','localtime') AS INTEGER)`,
|
||||
)
|
||||
.run(toDbSeconds(subtractDbMs(currentMs, policy.monthlyRollupRetentionMs))) as {
|
||||
changes: number;
|
||||
}
|
||||
).changes;
|
||||
|
||||
return {
|
||||
deletedDailyRows,
|
||||
@@ -113,11 +130,11 @@ export function pruneRollupRetention(
|
||||
};
|
||||
}
|
||||
|
||||
function getLastRollupSampleMs(db: DatabaseSync): number {
|
||||
function getLastRollupSampleMs(db: DatabaseSync): string {
|
||||
const row = db
|
||||
.prepare(`SELECT state_value FROM imm_rollup_state WHERE state_key = ? LIMIT 1`)
|
||||
.get(ROLLUP_STATE_KEY) as unknown as RollupStateRow | null;
|
||||
return row ? Number(row.state_value) : ZERO_ID;
|
||||
return row ? row.state_value : ZERO_ID.toString();
|
||||
}
|
||||
|
||||
function setLastRollupSampleMs(db: DatabaseSync, sampleMs: string | number | bigint): void {
|
||||
@@ -263,7 +280,7 @@ function upsertMonthlyRollupsForGroups(
|
||||
|
||||
function getAffectedRollupGroups(
|
||||
db: DatabaseSync,
|
||||
lastRollupSampleMs: number,
|
||||
lastRollupSampleMs: string,
|
||||
): Array<{ rollupDay: number; rollupMonth: number; videoId: number }> {
|
||||
return (
|
||||
db
|
||||
@@ -370,7 +387,7 @@ export function rebuildRollupsInTransaction(db: DatabaseSync): void {
|
||||
return;
|
||||
}
|
||||
|
||||
const affectedGroups = getAffectedRollupGroups(db, ZERO_ID);
|
||||
const affectedGroups = getAffectedRollupGroups(db, ZERO_ID.toString());
|
||||
if (affectedGroups.length === 0) {
|
||||
setLastRollupSampleMs(db, toDbMs(maxSampleRow.maxSampleMs ?? ZERO_ID));
|
||||
return;
|
||||
|
||||
@@ -131,7 +131,7 @@ export function getSessionEvents(
|
||||
): SessionEventRow[] {
|
||||
if (!eventTypes || eventTypes.length === 0) {
|
||||
const stmt = db.prepare(`
|
||||
SELECT event_type AS eventType, ts_ms AS tsMs, payload_json AS payload
|
||||
SELECT event_type AS eventType, CAST(ts_ms AS INTEGER) AS tsMs, payload_json AS payload
|
||||
FROM imm_session_events WHERE session_id = ? ORDER BY ts_ms ASC LIMIT ?
|
||||
`);
|
||||
return stmt.all(sessionId, limit) as SessionEventRow[];
|
||||
@@ -139,7 +139,7 @@ export function getSessionEvents(
|
||||
|
||||
const placeholders = eventTypes.map(() => '?').join(', ');
|
||||
const stmt = db.prepare(`
|
||||
SELECT event_type AS eventType, ts_ms AS tsMs, payload_json AS payload
|
||||
SELECT event_type AS eventType, CAST(ts_ms AS INTEGER) AS tsMs, payload_json AS payload
|
||||
FROM imm_session_events
|
||||
WHERE session_id = ? AND event_type IN (${placeholders})
|
||||
ORDER BY ts_ms ASC
|
||||
|
||||
@@ -32,7 +32,7 @@ export function getAnimeLibrary(db: DatabaseSync): AnimeLibraryRow[] {
|
||||
COALESCE(lm.total_tokens_seen, 0) AS totalTokensSeen,
|
||||
COUNT(DISTINCT v.video_id) AS episodeCount,
|
||||
a.episodes_total AS episodesTotal,
|
||||
COALESCE(lm.last_watched_ms, 0) AS lastWatchedMs
|
||||
CAST(COALESCE(lm.last_watched_ms, 0) AS INTEGER) AS lastWatchedMs
|
||||
FROM imm_anime a
|
||||
JOIN imm_lifetime_anime lm ON lm.anime_id = a.anime_id
|
||||
JOIN imm_videos v ON v.anime_id = a.anime_id
|
||||
@@ -65,7 +65,7 @@ export function getAnimeDetail(db: DatabaseSync, animeId: number): AnimeDetailRo
|
||||
COALESCE(SUM(COALESCE(asm.lookupHits, s.lookup_hits, 0)), 0) AS totalLookupHits,
|
||||
COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount,
|
||||
COUNT(DISTINCT v.video_id) AS episodeCount,
|
||||
COALESCE(lm.last_watched_ms, 0) AS lastWatchedMs
|
||||
CAST(COALESCE(lm.last_watched_ms, 0) AS INTEGER) AS lastWatchedMs
|
||||
FROM imm_anime a
|
||||
JOIN imm_lifetime_anime lm ON lm.anime_id = a.anime_id
|
||||
JOIN imm_videos v ON v.anime_id = a.anime_id
|
||||
@@ -110,7 +110,7 @@ export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisod
|
||||
v.parsed_season AS season,
|
||||
v.parsed_episode AS episode,
|
||||
v.duration_ms AS durationMs,
|
||||
(
|
||||
CAST((
|
||||
SELECT COALESCE(
|
||||
NULLIF(s_recent.ended_media_ms, 0),
|
||||
(
|
||||
@@ -147,14 +147,14 @@ export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisod
|
||||
COALESCE(s_recent.ended_at_ms, s_recent.LAST_UPDATE_DATE, s_recent.started_at_ms) DESC,
|
||||
s_recent.session_id DESC
|
||||
LIMIT 1
|
||||
) AS endedMediaMs,
|
||||
) AS INTEGER) AS endedMediaMs,
|
||||
v.watched AS watched,
|
||||
COUNT(DISTINCT s.session_id) AS totalSessions,
|
||||
COALESCE(SUM(COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0)), 0) AS totalActiveMs,
|
||||
COALESCE(SUM(COALESCE(asm.cardsMined, s.cards_mined, 0)), 0) AS totalCards,
|
||||
COALESCE(SUM(COALESCE(asm.tokensSeen, s.tokens_seen, 0)), 0) AS totalTokensSeen,
|
||||
COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount,
|
||||
MAX(s.started_at_ms) AS lastWatchedMs
|
||||
CAST(MAX(s.started_at_ms) AS INTEGER) AS lastWatchedMs
|
||||
FROM imm_videos v
|
||||
LEFT JOIN imm_sessions s ON s.video_id = v.video_id
|
||||
LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
|
||||
@@ -182,7 +182,7 @@ export function getMediaLibrary(db: DatabaseSync): MediaLibraryRow[] {
|
||||
COALESCE(lm.total_active_ms, 0) AS totalActiveMs,
|
||||
COALESCE(lm.total_cards, 0) AS totalCards,
|
||||
COALESCE(lm.total_tokens_seen, 0) AS totalTokensSeen,
|
||||
COALESCE(lm.last_watched_ms, 0) AS lastWatchedMs,
|
||||
CAST(COALESCE(lm.last_watched_ms, 0) AS INTEGER) AS lastWatchedMs,
|
||||
yv.youtube_video_id AS youtubeVideoId,
|
||||
yv.video_url AS videoUrl,
|
||||
yv.video_title AS videoTitle,
|
||||
@@ -261,8 +261,8 @@ export function getMediaSessions(
|
||||
s.session_id AS sessionId,
|
||||
s.video_id AS videoId,
|
||||
v.canonical_title AS canonicalTitle,
|
||||
s.started_at_ms AS startedAtMs,
|
||||
s.ended_at_ms AS endedAtMs,
|
||||
CAST(s.started_at_ms AS INTEGER) AS startedAtMs,
|
||||
CAST(s.ended_at_ms AS INTEGER) AS endedAtMs,
|
||||
COALESCE(asm.totalWatchedMs, s.total_watched_ms, 0) AS totalWatchedMs,
|
||||
COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0) AS activeWatchedMs,
|
||||
COALESCE(asm.linesSeen, s.lines_seen, 0) AS linesSeen,
|
||||
@@ -517,7 +517,7 @@ export function getEpisodeSessions(db: DatabaseSync, videoId: number): SessionSu
|
||||
SELECT
|
||||
s.session_id AS sessionId, s.video_id AS videoId,
|
||||
v.canonical_title AS canonicalTitle,
|
||||
s.started_at_ms AS startedAtMs, s.ended_at_ms AS endedAtMs,
|
||||
CAST(s.started_at_ms AS INTEGER) AS startedAtMs, CAST(s.ended_at_ms AS INTEGER) AS endedAtMs,
|
||||
COALESCE(asm.totalWatchedMs, s.total_watched_ms, 0) AS totalWatchedMs,
|
||||
COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0) AS activeWatchedMs,
|
||||
COALESCE(asm.linesSeen, s.lines_seen, 0) AS linesSeen,
|
||||
@@ -541,7 +541,7 @@ export function getEpisodeCardEvents(db: DatabaseSync, videoId: number): Episode
|
||||
.prepare(
|
||||
`
|
||||
SELECT e.event_id AS eventId, e.session_id AS sessionId,
|
||||
e.ts_ms AS tsMs, e.cards_delta AS cardsDelta,
|
||||
CAST(e.ts_ms AS INTEGER) AS tsMs, e.cards_delta AS cardsDelta,
|
||||
e.payload_json AS payloadJson
|
||||
FROM imm_session_events e
|
||||
JOIN imm_sessions s ON s.session_id = e.session_id
|
||||
|
||||
@@ -5,7 +5,13 @@ import type {
|
||||
SessionSummaryQueryRow,
|
||||
SessionTimelineRow,
|
||||
} from './types';
|
||||
import { ACTIVE_SESSION_METRICS_CTE } from './query-shared';
|
||||
import { ACTIVE_SESSION_METRICS_CTE, subtractDbMs, toDbMs, toDbSeconds } from './query-shared';
|
||||
|
||||
const THIRTY_DAYS_MS = '2592000000';
|
||||
|
||||
function localMidnightSecondsExpr(): string {
|
||||
return `(CAST(strftime('%s', 'now', 'localtime') AS INTEGER) - CAST(strftime('%H', 'now', 'localtime') AS INTEGER) * 3600 - CAST(strftime('%M', 'now', 'localtime') AS INTEGER) * 60 - CAST(strftime('%S', 'now', 'localtime') AS INTEGER))`;
|
||||
}
|
||||
|
||||
export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummaryQueryRow[] {
|
||||
const prepared = db.prepare(`
|
||||
@@ -16,8 +22,8 @@ export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummar
|
||||
v.canonical_title AS canonicalTitle,
|
||||
v.anime_id AS animeId,
|
||||
a.canonical_title AS animeTitle,
|
||||
s.started_at_ms AS startedAtMs,
|
||||
s.ended_at_ms AS endedAtMs,
|
||||
CAST(s.started_at_ms AS INTEGER) AS startedAtMs,
|
||||
CAST(s.ended_at_ms AS INTEGER) AS endedAtMs,
|
||||
COALESCE(asm.totalWatchedMs, s.total_watched_ms, 0) AS totalWatchedMs,
|
||||
COALESCE(asm.activeWatchedMs, s.active_watched_ms, 0) AS activeWatchedMs,
|
||||
COALESCE(asm.linesSeen, s.lines_seen, 0) AS linesSeen,
|
||||
@@ -43,7 +49,7 @@ export function getSessionTimeline(
|
||||
): SessionTimelineRow[] {
|
||||
const select = `
|
||||
SELECT
|
||||
sample_ms AS sampleMs,
|
||||
CAST(sample_ms AS INTEGER) AS sampleMs,
|
||||
total_watched_ms AS totalWatchedMs,
|
||||
active_watched_ms AS activeWatchedMs,
|
||||
lines_seen AS linesSeen,
|
||||
@@ -129,18 +135,13 @@ export function getSessionWordsByLine(
|
||||
}
|
||||
|
||||
function getNewWordCounts(db: DatabaseSync): { newWordsToday: number; newWordsThisWeek: number } {
|
||||
const now = new Date();
|
||||
const todayStartSec = new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime() / 1000;
|
||||
const weekAgoSec =
|
||||
new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7).getTime() / 1000;
|
||||
|
||||
const row = db
|
||||
.prepare(
|
||||
`
|
||||
WITH headword_first_seen AS (
|
||||
SELECT
|
||||
headword,
|
||||
MIN(first_seen) AS first_seen
|
||||
CAST(MIN(first_seen) AS INTEGER) AS first_seen
|
||||
FROM imm_words
|
||||
WHERE first_seen IS NOT NULL
|
||||
AND headword IS NOT NULL
|
||||
@@ -148,13 +149,12 @@ function getNewWordCounts(db: DatabaseSync): { newWordsToday: number; newWordsTh
|
||||
GROUP BY headword
|
||||
)
|
||||
SELECT
|
||||
COALESCE(SUM(CASE WHEN first_seen >= ? THEN 1 ELSE 0 END), 0) AS today,
|
||||
COALESCE(SUM(CASE WHEN first_seen >= ? THEN 1 ELSE 0 END), 0) AS week
|
||||
COALESCE(SUM(CASE WHEN first_seen >= (${localMidnightSecondsExpr()}) THEN 1 ELSE 0 END), 0) AS today,
|
||||
COALESCE(SUM(CASE WHEN first_seen >= (${localMidnightSecondsExpr()} - 7 * 86400) THEN 1 ELSE 0 END), 0) AS week
|
||||
FROM headword_first_seen
|
||||
`,
|
||||
`,
|
||||
)
|
||||
.get(todayStartSec, weekAgoSec) as { today: number; week: number } | null;
|
||||
|
||||
.get() as { today: number; week: number } | null;
|
||||
return {
|
||||
newWordsToday: Number(row?.today ?? 0),
|
||||
newWordsThisWeek: Number(row?.week ?? 0),
|
||||
@@ -203,10 +203,7 @@ export function getQueryHints(db: DatabaseSync): {
|
||||
animeCompleted: number;
|
||||
} | null;
|
||||
|
||||
const now = new Date();
|
||||
const todayLocal = Math.floor(
|
||||
(now.getTime() / 1000 - now.getTimezoneOffset() * 60) / 86_400,
|
||||
);
|
||||
const nowSeconds = (BigInt(toDbMs(nowMs())) / 1000n).toString();
|
||||
|
||||
const episodesToday =
|
||||
(
|
||||
@@ -215,13 +212,13 @@ export function getQueryHints(db: DatabaseSync): {
|
||||
`
|
||||
SELECT COUNT(DISTINCT s.video_id) AS count
|
||||
FROM imm_sessions s
|
||||
WHERE CAST(julianday(s.started_at_ms / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) = ?
|
||||
WHERE date(s.started_at_ms / 1000, 'unixepoch', 'localtime') = date(?,'unixepoch','localtime')
|
||||
`,
|
||||
)
|
||||
.get(todayLocal) as { count: number }
|
||||
.get(nowSeconds) as { count: number }
|
||||
)?.count ?? 0;
|
||||
|
||||
const thirtyDaysAgoMs = nowMs() - 30 * 86400000;
|
||||
const activeAnimeCutoffMs = subtractDbMs(toDbMs(nowMs()), `${THIRTY_DAYS_MS}`);
|
||||
const activeAnimeCount =
|
||||
(
|
||||
db
|
||||
@@ -234,7 +231,7 @@ export function getQueryHints(db: DatabaseSync): {
|
||||
AND s.started_at_ms >= ?
|
||||
`,
|
||||
)
|
||||
.get(thirtyDaysAgoMs) as { count: number }
|
||||
.get(activeAnimeCutoffMs) as { count: number }
|
||||
)?.count ?? 0;
|
||||
|
||||
const totalEpisodesWatched = Number(lifetime?.episodesCompleted ?? 0);
|
||||
|
||||
@@ -276,14 +276,23 @@ export function toDbMs(ms: number | bigint | string): string {
|
||||
return ms.toString();
|
||||
}
|
||||
if (typeof ms === 'string') {
|
||||
const parsedMs = Number(ms);
|
||||
if (!Number.isFinite(parsedMs)) {
|
||||
return '0';
|
||||
}
|
||||
return String(Math.trunc(parsedMs));
|
||||
const text = ms.trim().replace(/\.0+$/, '');
|
||||
return /^-?\d+$/.test(text) ? text : '0';
|
||||
}
|
||||
if (!Number.isFinite(ms)) {
|
||||
return '0';
|
||||
}
|
||||
return String(Math.trunc(ms));
|
||||
return ms.toFixed(0);
|
||||
}
|
||||
|
||||
export function toDbSeconds(ms: number | bigint | string): string {
|
||||
const dbMs = toDbMs(ms);
|
||||
if (dbMs === '0') {
|
||||
return '0';
|
||||
}
|
||||
return (BigInt(dbMs) / 1000n).toString();
|
||||
}
|
||||
|
||||
export function subtractDbMs(timestampMs: number | bigint | string, deltaMs: number | string): string {
|
||||
return (BigInt(toDbMs(timestampMs)) - BigInt(`${deltaMs}`)).toString();
|
||||
}
|
||||
|
||||
@@ -19,6 +19,10 @@ interface TrendPerAnimePoint {
|
||||
|
||||
interface TrendSessionMetricRow {
|
||||
startedAtMs: number;
|
||||
localEpochDay: number;
|
||||
localMonthKey: number;
|
||||
localDayOfWeek: number;
|
||||
localHour: number;
|
||||
videoId: number | null;
|
||||
canonicalTitle: string | null;
|
||||
animeTitle: string | null;
|
||||
@@ -74,63 +78,60 @@ const TREND_DAY_LIMITS: Record<Exclude<TrendRange, 'all'>, number> = {
|
||||
};
|
||||
|
||||
const DAY_NAMES = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
|
||||
const MONTH_NAMES = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
|
||||
|
||||
function getTrendDayLimit(range: TrendRange): number {
|
||||
return range === 'all' ? 365 : TREND_DAY_LIMITS[range];
|
||||
}
|
||||
|
||||
function getTrendMonthlyLimit(range: TrendRange): number {
|
||||
if (range === 'all') {
|
||||
return 120;
|
||||
switch (range) {
|
||||
case 'all':
|
||||
return 120;
|
||||
case '7d':
|
||||
return 1;
|
||||
case '30d':
|
||||
return 2;
|
||||
case '90d':
|
||||
return 4;
|
||||
}
|
||||
const now = new Date();
|
||||
const cutoff = new Date(
|
||||
now.getFullYear(),
|
||||
now.getMonth(),
|
||||
now.getDate() - (TREND_DAY_LIMITS[range] - 1),
|
||||
}
|
||||
|
||||
function epochDayToCivil(epochDay: number): { year: number; month: number; day: number } {
|
||||
const z = epochDay + 719468;
|
||||
const era = Math.floor(z / 146097);
|
||||
const doe = z - era * 146097;
|
||||
const yoe = Math.floor(
|
||||
(doe - Math.floor(doe / 1460) + Math.floor(doe / 36524) - Math.floor(doe / 146096)) / 365,
|
||||
);
|
||||
return Math.max(1, (now.getFullYear() - cutoff.getFullYear()) * 12 + now.getMonth() - cutoff.getMonth() + 1);
|
||||
}
|
||||
|
||||
function getTrendCutoffMs(range: TrendRange): number | null {
|
||||
if (range === 'all') {
|
||||
return null;
|
||||
let year = yoe + era * 400;
|
||||
const doy = doe - (365 * yoe + Math.floor(yoe / 4) - Math.floor(yoe / 100));
|
||||
const mp = Math.floor((5 * doy + 2) / 153);
|
||||
const day = doy - Math.floor((153 * mp + 2) / 5) + 1;
|
||||
const month = mp < 10 ? mp + 3 : mp - 9;
|
||||
if (month <= 2) {
|
||||
year += 1;
|
||||
}
|
||||
const dayLimit = getTrendDayLimit(range);
|
||||
const now = new Date();
|
||||
const localMidnight = new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime();
|
||||
return localMidnight - (dayLimit - 1) * 86_400_000;
|
||||
return { year, month, day };
|
||||
}
|
||||
|
||||
function makeTrendLabel(value: number): string {
|
||||
if (value > 100_000) {
|
||||
const year = Math.floor(value / 100);
|
||||
const month = value % 100;
|
||||
return new Date(Date.UTC(year, month - 1, 1)).toLocaleDateString(undefined, {
|
||||
month: 'short',
|
||||
year: '2-digit',
|
||||
});
|
||||
}
|
||||
|
||||
return new Date(value * 86_400_000).toLocaleDateString(undefined, {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
});
|
||||
function formatEpochDayLabel(epochDay: number): string {
|
||||
const { month, day } = epochDayToCivil(epochDay);
|
||||
return `${MONTH_NAMES[month - 1]} ${day}`;
|
||||
}
|
||||
|
||||
function getLocalEpochDay(timestampMs: number): number {
|
||||
const date = new Date(timestampMs);
|
||||
return Math.floor((timestampMs - date.getTimezoneOffset() * 60_000) / 86_400_000);
|
||||
function formatMonthKeyLabel(monthKey: number): string {
|
||||
const year = Math.floor(monthKey / 100);
|
||||
const month = monthKey % 100;
|
||||
return `${MONTH_NAMES[month - 1]} ${String(year).slice(-2)}`;
|
||||
}
|
||||
|
||||
function getLocalDateForEpochDay(epochDay: number): Date {
|
||||
const utcDate = new Date(epochDay * 86_400_000);
|
||||
return new Date(utcDate.getTime() + utcDate.getTimezoneOffset() * 60_000);
|
||||
function formatTrendLabel(value: number): string {
|
||||
return value > 100_000 ? formatMonthKeyLabel(value) : formatEpochDayLabel(value);
|
||||
}
|
||||
|
||||
function getLocalMonthKey(timestampMs: number): number {
|
||||
const date = new Date(timestampMs);
|
||||
return date.getFullYear() * 100 + date.getMonth() + 1;
|
||||
function localMidnightSecondsExpr(): string {
|
||||
return `(CAST(strftime('%s', 'now', 'localtime') AS INTEGER) - CAST(strftime('%H', 'now', 'localtime') AS INTEGER) * 3600 - CAST(strftime('%M', 'now', 'localtime') AS INTEGER) * 60 - CAST(strftime('%S', 'now', 'localtime') AS INTEGER))`;
|
||||
}
|
||||
|
||||
function getTrendSessionWordCount(session: Pick<TrendSessionMetricRow, 'tokensSeen'>): number {
|
||||
@@ -178,7 +179,7 @@ function buildAggregatedTrendRows(rollups: ImmersionSessionRollupRow[]) {
|
||||
return Array.from(byKey.entries())
|
||||
.sort(([left], [right]) => left - right)
|
||||
.map(([key, value]) => ({
|
||||
label: makeTrendLabel(key),
|
||||
label: formatTrendLabel(key),
|
||||
activeMin: value.activeMin,
|
||||
cards: value.cards,
|
||||
words: value.words,
|
||||
@@ -189,7 +190,7 @@ function buildAggregatedTrendRows(rollups: ImmersionSessionRollupRow[]) {
|
||||
function buildWatchTimeByDayOfWeek(sessions: TrendSessionMetricRow[]): TrendChartPoint[] {
|
||||
const totals = new Array(7).fill(0);
|
||||
for (const session of sessions) {
|
||||
totals[new Date(session.startedAtMs).getDay()] += session.activeWatchedMs;
|
||||
totals[session.localDayOfWeek] += session.activeWatchedMs;
|
||||
}
|
||||
return DAY_NAMES.map((name, index) => ({
|
||||
label: name,
|
||||
@@ -200,7 +201,7 @@ function buildWatchTimeByDayOfWeek(sessions: TrendSessionMetricRow[]): TrendChar
|
||||
function buildWatchTimeByHour(sessions: TrendSessionMetricRow[]): TrendChartPoint[] {
|
||||
const totals = new Array(24).fill(0);
|
||||
for (const session of sessions) {
|
||||
totals[new Date(session.startedAtMs).getHours()] += session.activeWatchedMs;
|
||||
totals[session.localHour] += session.activeWatchedMs;
|
||||
}
|
||||
return totals.map((ms, index) => ({
|
||||
label: `${String(index).padStart(2, '0')}:00`,
|
||||
@@ -208,25 +209,18 @@ function buildWatchTimeByHour(sessions: TrendSessionMetricRow[]): TrendChartPoin
|
||||
}));
|
||||
}
|
||||
|
||||
function dayLabel(epochDay: number): string {
|
||||
return getLocalDateForEpochDay(epochDay).toLocaleDateString(undefined, {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
});
|
||||
}
|
||||
|
||||
function buildSessionSeriesByDay(
|
||||
sessions: TrendSessionMetricRow[],
|
||||
getValue: (session: TrendSessionMetricRow) => number,
|
||||
): TrendChartPoint[] {
|
||||
const byDay = new Map<number, number>();
|
||||
for (const session of sessions) {
|
||||
const epochDay = getLocalEpochDay(session.startedAtMs);
|
||||
const epochDay = session.localEpochDay;
|
||||
byDay.set(epochDay, (byDay.get(epochDay) ?? 0) + getValue(session));
|
||||
}
|
||||
return Array.from(byDay.entries())
|
||||
.sort(([left], [right]) => left - right)
|
||||
.map(([epochDay, value]) => ({ label: dayLabel(epochDay), value }));
|
||||
.map(([epochDay, value]) => ({ label: formatEpochDayLabel(epochDay), value }));
|
||||
}
|
||||
|
||||
function buildSessionSeriesByMonth(
|
||||
@@ -235,12 +229,12 @@ function buildSessionSeriesByMonth(
|
||||
): TrendChartPoint[] {
|
||||
const byMonth = new Map<number, number>();
|
||||
for (const session of sessions) {
|
||||
const monthKey = getLocalMonthKey(session.startedAtMs);
|
||||
const monthKey = session.localMonthKey;
|
||||
byMonth.set(monthKey, (byMonth.get(monthKey) ?? 0) + getValue(session));
|
||||
}
|
||||
return Array.from(byMonth.entries())
|
||||
.sort(([left], [right]) => left - right)
|
||||
.map(([monthKey, value]) => ({ label: makeTrendLabel(monthKey), value }));
|
||||
.map(([monthKey, value]) => ({ label: formatMonthKeyLabel(monthKey), value }));
|
||||
}
|
||||
|
||||
function buildLookupsPerHundredWords(sessions: TrendSessionMetricRow[]): TrendChartPoint[] {
|
||||
@@ -248,7 +242,7 @@ function buildLookupsPerHundredWords(sessions: TrendSessionMetricRow[]): TrendCh
|
||||
const wordsByDay = new Map<number, number>();
|
||||
|
||||
for (const session of sessions) {
|
||||
const epochDay = getLocalEpochDay(session.startedAtMs);
|
||||
const epochDay = session.localEpochDay;
|
||||
lookupsByDay.set(epochDay, (lookupsByDay.get(epochDay) ?? 0) + session.yomitanLookupCount);
|
||||
wordsByDay.set(epochDay, (wordsByDay.get(epochDay) ?? 0) + getTrendSessionWordCount(session));
|
||||
}
|
||||
@@ -258,7 +252,7 @@ function buildLookupsPerHundredWords(sessions: TrendSessionMetricRow[]): TrendCh
|
||||
.map(([epochDay, lookups]) => {
|
||||
const words = wordsByDay.get(epochDay) ?? 0;
|
||||
return {
|
||||
label: dayLabel(epochDay),
|
||||
label: formatEpochDayLabel(epochDay),
|
||||
value: words > 0 ? +((lookups / words) * 100).toFixed(1) : 0,
|
||||
};
|
||||
});
|
||||
@@ -272,7 +266,7 @@ function buildPerAnimeFromSessions(
|
||||
|
||||
for (const session of sessions) {
|
||||
const animeTitle = resolveTrendAnimeTitle(session);
|
||||
const epochDay = getLocalEpochDay(session.startedAtMs);
|
||||
const epochDay = session.localEpochDay;
|
||||
const dayMap = byAnime.get(animeTitle) ?? new Map();
|
||||
dayMap.set(epochDay, (dayMap.get(epochDay) ?? 0) + getValue(session));
|
||||
byAnime.set(animeTitle, dayMap);
|
||||
@@ -293,7 +287,7 @@ function buildLookupsPerHundredPerAnime(sessions: TrendSessionMetricRow[]): Tren
|
||||
|
||||
for (const session of sessions) {
|
||||
const animeTitle = resolveTrendAnimeTitle(session);
|
||||
const epochDay = getLocalEpochDay(session.startedAtMs);
|
||||
const epochDay = session.localEpochDay;
|
||||
|
||||
const lookupMap = lookups.get(animeTitle) ?? new Map();
|
||||
lookupMap.set(epochDay, (lookupMap.get(epochDay) ?? 0) + session.yomitanLookupCount);
|
||||
@@ -461,7 +455,7 @@ function buildEpisodesPerDayFromDailyRollups(
|
||||
return Array.from(byDay.entries())
|
||||
.sort(([left], [right]) => left - right)
|
||||
.map(([epochDay, videoIds]) => ({
|
||||
label: dayLabel(epochDay),
|
||||
label: formatEpochDayLabel(epochDay),
|
||||
value: videoIds.size,
|
||||
}));
|
||||
}
|
||||
@@ -481,20 +475,25 @@ function buildEpisodesPerMonthFromRollups(rollups: ImmersionSessionRollupRow[]):
|
||||
return Array.from(byMonth.entries())
|
||||
.sort(([left], [right]) => left - right)
|
||||
.map(([monthKey, videoIds]) => ({
|
||||
label: makeTrendLabel(monthKey),
|
||||
label: formatTrendLabel(monthKey),
|
||||
value: videoIds.size,
|
||||
}));
|
||||
}
|
||||
|
||||
function getTrendSessionMetrics(
|
||||
db: DatabaseSync,
|
||||
cutoffMs: number | null,
|
||||
): TrendSessionMetricRow[] {
|
||||
const whereClause = cutoffMs === null ? '' : 'WHERE s.started_at_ms >= ?';
|
||||
function getTrendSessionMetrics(db: DatabaseSync, range: TrendRange): TrendSessionMetricRow[] {
|
||||
const dayLimit = getTrendDayLimit(range);
|
||||
const cutoffClause =
|
||||
range === 'all'
|
||||
? ''
|
||||
: `WHERE CAST(s.started_at_ms AS INTEGER) >= (${localMidnightSecondsExpr()} - ${(dayLimit - 1) * 86400}) * 1000`;
|
||||
const prepared = db.prepare(`
|
||||
${ACTIVE_SESSION_METRICS_CTE}
|
||||
SELECT
|
||||
s.started_at_ms AS startedAtMs,
|
||||
CAST(s.started_at_ms AS INTEGER) AS startedAtMs,
|
||||
CAST(julianday(s.started_at_ms / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) AS localEpochDay,
|
||||
CAST(strftime('%Y%m', s.started_at_ms / 1000, 'unixepoch', 'localtime') AS INTEGER) AS localMonthKey,
|
||||
CAST(strftime('%w', s.started_at_ms / 1000, 'unixepoch', 'localtime') AS INTEGER) AS localDayOfWeek,
|
||||
CAST(strftime('%H', s.started_at_ms / 1000, 'unixepoch', 'localtime') AS INTEGER) AS localHour,
|
||||
s.video_id AS videoId,
|
||||
v.canonical_title AS canonicalTitle,
|
||||
a.canonical_title AS animeTitle,
|
||||
@@ -506,61 +505,79 @@ function getTrendSessionMetrics(
|
||||
LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
|
||||
LEFT JOIN imm_videos v ON v.video_id = s.video_id
|
||||
LEFT JOIN imm_anime a ON a.anime_id = v.anime_id
|
||||
${whereClause}
|
||||
ORDER BY s.started_at_ms ASC
|
||||
${cutoffClause}
|
||||
ORDER BY CAST(s.started_at_ms AS INTEGER) ASC
|
||||
`);
|
||||
|
||||
return (cutoffMs === null ? prepared.all() : prepared.all(cutoffMs)) as TrendSessionMetricRow[];
|
||||
const rows = prepared.all() as Array<{
|
||||
startedAtMs: number | string;
|
||||
localEpochDay: number | string;
|
||||
localMonthKey: number | string;
|
||||
localDayOfWeek: number | string;
|
||||
localHour: number | string;
|
||||
videoId: number | null;
|
||||
canonicalTitle: string | null;
|
||||
animeTitle: string | null;
|
||||
activeWatchedMs: number | string;
|
||||
tokensSeen: number | string;
|
||||
cardsMined: number | string;
|
||||
yomitanLookupCount: number | string;
|
||||
}>;
|
||||
|
||||
return rows.map((row) => ({
|
||||
startedAtMs: Number.parseInt(String(row.startedAtMs), 10),
|
||||
localEpochDay: Number.parseInt(String(row.localEpochDay), 10),
|
||||
localMonthKey: Number.parseInt(String(row.localMonthKey), 10),
|
||||
localDayOfWeek: Number.parseInt(String(row.localDayOfWeek), 10),
|
||||
localHour: Number.parseInt(String(row.localHour), 10),
|
||||
videoId: row.videoId,
|
||||
canonicalTitle: row.canonicalTitle,
|
||||
animeTitle: row.animeTitle,
|
||||
activeWatchedMs: Number(row.activeWatchedMs),
|
||||
tokensSeen: Number(row.tokensSeen),
|
||||
cardsMined: Number(row.cardsMined),
|
||||
yomitanLookupCount: Number(row.yomitanLookupCount),
|
||||
}));
|
||||
}
|
||||
|
||||
function buildNewWordsPerDay(db: DatabaseSync, cutoffMs: number | null): TrendChartPoint[] {
|
||||
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
||||
function buildNewWordsPerDay(db: DatabaseSync, dayLimit: number | null): TrendChartPoint[] {
|
||||
const cutoffExpr =
|
||||
dayLimit === null ? '' : `AND CAST(first_seen AS INTEGER) >= (${localMidnightSecondsExpr()} - ${(dayLimit - 1) * 86400})`;
|
||||
const prepared = db.prepare(`
|
||||
SELECT
|
||||
CAST(julianday(first_seen, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) AS epochDay,
|
||||
COUNT(*) AS wordCount
|
||||
FROM imm_words
|
||||
WHERE first_seen IS NOT NULL
|
||||
${whereClause}
|
||||
${cutoffExpr}
|
||||
GROUP BY epochDay
|
||||
ORDER BY epochDay ASC
|
||||
`);
|
||||
|
||||
const rows = (
|
||||
cutoffMs === null ? prepared.all() : prepared.all(Math.floor(cutoffMs / 1000))
|
||||
) as Array<{
|
||||
epochDay: number;
|
||||
wordCount: number;
|
||||
}>;
|
||||
const rows = prepared.all() as Array<{ epochDay: number; wordCount: number }>;
|
||||
|
||||
return rows.map((row) => ({
|
||||
label: dayLabel(row.epochDay),
|
||||
label: formatEpochDayLabel(row.epochDay),
|
||||
value: row.wordCount,
|
||||
}));
|
||||
}
|
||||
|
||||
function buildNewWordsPerMonth(db: DatabaseSync, cutoffMs: number | null): TrendChartPoint[] {
|
||||
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
||||
function buildNewWordsPerMonth(db: DatabaseSync, dayLimit: number | null): TrendChartPoint[] {
|
||||
const cutoffExpr =
|
||||
dayLimit === null ? '' : `AND CAST(first_seen AS INTEGER) >= (${localMidnightSecondsExpr()} - ${(dayLimit - 1) * 86400})`;
|
||||
const prepared = db.prepare(`
|
||||
SELECT
|
||||
CAST(strftime('%Y%m', first_seen, 'unixepoch', 'localtime') AS INTEGER) AS monthKey,
|
||||
COUNT(*) AS wordCount
|
||||
FROM imm_words
|
||||
WHERE first_seen IS NOT NULL
|
||||
${whereClause}
|
||||
${cutoffExpr}
|
||||
GROUP BY monthKey
|
||||
ORDER BY monthKey ASC
|
||||
`);
|
||||
|
||||
const rows = (
|
||||
cutoffMs === null ? prepared.all() : prepared.all(Math.floor(cutoffMs / 1000))
|
||||
) as Array<{
|
||||
monthKey: number;
|
||||
wordCount: number;
|
||||
}>;
|
||||
const rows = prepared.all() as Array<{ monthKey: number; wordCount: number }>;
|
||||
|
||||
return rows.map((row) => ({
|
||||
label: makeTrendLabel(row.monthKey),
|
||||
label: formatMonthKeyLabel(row.monthKey),
|
||||
value: row.wordCount,
|
||||
}));
|
||||
}
|
||||
@@ -572,13 +589,12 @@ export function getTrendsDashboard(
|
||||
): TrendsDashboardQueryResult {
|
||||
const dayLimit = getTrendDayLimit(range);
|
||||
const monthlyLimit = getTrendMonthlyLimit(range);
|
||||
const cutoffMs = getTrendCutoffMs(range);
|
||||
const useMonthlyBuckets = groupBy === 'month';
|
||||
const dailyRollups = getDailyRollups(db, dayLimit);
|
||||
const monthlyRollups = getMonthlyRollups(db, monthlyLimit);
|
||||
|
||||
const chartRollups = useMonthlyBuckets ? monthlyRollups : dailyRollups;
|
||||
const sessions = getTrendSessionMetrics(db, cutoffMs);
|
||||
const sessions = getTrendSessionMetrics(db, range);
|
||||
const titlesByVideoId = getVideoAnimeTitleMap(
|
||||
db,
|
||||
dailyRollups.map((rollup) => rollup.videoId),
|
||||
@@ -618,7 +634,7 @@ export function getTrendsDashboard(
|
||||
sessions: accumulatePoints(activity.sessions),
|
||||
words: accumulatePoints(activity.words),
|
||||
newWords: accumulatePoints(
|
||||
useMonthlyBuckets ? buildNewWordsPerMonth(db, cutoffMs) : buildNewWordsPerDay(db, cutoffMs),
|
||||
useMonthlyBuckets ? buildNewWordsPerMonth(db, range === 'all' ? null : dayLimit) : buildNewWordsPerDay(db, range === 'all' ? null : dayLimit),
|
||||
),
|
||||
cards: accumulatePoints(activity.cards),
|
||||
episodes: accumulatePoints(
|
||||
|
||||
@@ -143,10 +143,10 @@ test('ensureSchema creates immersion core tables', () => {
|
||||
const rollupStateRow = db
|
||||
.prepare('SELECT state_value FROM imm_rollup_state WHERE state_key = ?')
|
||||
.get('last_rollup_sample_ms') as {
|
||||
state_value: number;
|
||||
state_value: string;
|
||||
} | null;
|
||||
assert.ok(rollupStateRow);
|
||||
assert.equal(rollupStateRow?.state_value, 0);
|
||||
assert.equal(rollupStateRow?.state_value, '0');
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
@@ -965,12 +965,12 @@ test('start/finalize session updates ended_at and status', () => {
|
||||
const row = db
|
||||
.prepare('SELECT ended_at_ms, status FROM imm_sessions WHERE session_id = ?')
|
||||
.get(sessionId) as {
|
||||
ended_at_ms: number | null;
|
||||
ended_at_ms: string | null;
|
||||
status: number;
|
||||
} | null;
|
||||
|
||||
assert.ok(row);
|
||||
assert.equal(row?.ended_at_ms, endedAtMs);
|
||||
assert.equal(row?.ended_at_ms, String(endedAtMs));
|
||||
assert.equal(row?.status, SESSION_STATUS_ENDED);
|
||||
} finally {
|
||||
db.close();
|
||||
|
||||
@@ -1421,7 +1421,7 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta
|
||||
) {
|
||||
throw new Error('Incomplete telemetry write');
|
||||
}
|
||||
const telemetrySampleMs = toDbMs(write.sampleMs ?? Number(currentMs));
|
||||
const telemetrySampleMs = write.sampleMs === undefined ? currentMs : toDbMs(write.sampleMs);
|
||||
stmts.telemetryInsertStmt.run(
|
||||
write.sessionId,
|
||||
telemetrySampleMs,
|
||||
@@ -1496,7 +1496,7 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta
|
||||
|
||||
stmts.eventInsertStmt.run(
|
||||
write.sessionId,
|
||||
toDbMs(write.sampleMs ?? Number(currentMs)),
|
||||
write.sampleMs === undefined ? currentMs : toDbMs(write.sampleMs),
|
||||
write.eventType ?? 0,
|
||||
write.lineIndex ?? null,
|
||||
write.segmentStartMs ?? null,
|
||||
|
||||
Reference in New Issue
Block a user