mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-04-12 04:19:25 -07:00
fix(immersion-tracker): preserve timestamps under Bun libsql
This commit is contained in:
4
changes/immersion-tracker-timestamp-compat.md
Normal file
4
changes/immersion-tracker-timestamp-compat.md
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
type: fixed
|
||||||
|
area: stats
|
||||||
|
|
||||||
|
- Fixed immersion-tracker timestamp handling under Bun/libsql so library rows, session timelines, and lifetime summaries keep real wall-clock millisecond values instead of truncating to invalid negative timestamps.
|
||||||
@@ -1938,6 +1938,50 @@ test('getSessionEvents returns events ordered by ts_ms ascending', () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('getSessionEvents round-trips wall-clock timestamps written through event inserts', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const stmts = createTrackerPreparedStatements(db);
|
||||||
|
|
||||||
|
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/events-wall-clock.mkv', {
|
||||||
|
canonicalTitle: 'Events Wall Clock',
|
||||||
|
sourcePath: '/tmp/events-wall-clock.mkv',
|
||||||
|
sourceUrl: null,
|
||||||
|
sourceType: SOURCE_TYPE_LOCAL,
|
||||||
|
});
|
||||||
|
|
||||||
|
const startedAtMs = Date.now() - 10_000;
|
||||||
|
const eventTsMs = startedAtMs + 5_000;
|
||||||
|
const { sessionId } = startSessionRecord(db, videoId, startedAtMs);
|
||||||
|
|
||||||
|
stmts.eventInsertStmt.run(
|
||||||
|
sessionId,
|
||||||
|
toDbTimestamp(eventTsMs),
|
||||||
|
EVENT_SUBTITLE_LINE,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
500,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
'{"line":"wall-clock"}',
|
||||||
|
toDbTimestamp(eventTsMs),
|
||||||
|
toDbTimestamp(eventTsMs),
|
||||||
|
);
|
||||||
|
|
||||||
|
const events = getSessionEvents(db, sessionId, 10);
|
||||||
|
|
||||||
|
assert.equal(events.length, 1);
|
||||||
|
assert.equal(events[0]?.tsMs, eventTsMs);
|
||||||
|
assert.equal(events[0]?.payload, '{"line":"wall-clock"}');
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('getSessionEvents returns empty array for session with no events', () => {
|
test('getSessionEvents returns empty array for session with no events', () => {
|
||||||
const dbPath = makeDbPath();
|
const dbPath = makeDbPath();
|
||||||
const db = new Database(dbPath);
|
const db = new Database(dbPath);
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ export function pruneRawRetention(
|
|||||||
const deletedSessionEvents = Number.isFinite(policy.eventsRetentionMs)
|
const deletedSessionEvents = Number.isFinite(policy.eventsRetentionMs)
|
||||||
? (
|
? (
|
||||||
db
|
db
|
||||||
.prepare(`DELETE FROM imm_session_events WHERE ts_ms < ?`)
|
.prepare(`DELETE FROM imm_session_events WHERE CAST(ts_ms AS REAL) < CAST(? AS REAL)`)
|
||||||
.run(resolveCutoff(policy.eventsRetentionMs, policy.eventsRetentionDays)) as {
|
.run(resolveCutoff(policy.eventsRetentionMs, policy.eventsRetentionDays)) as {
|
||||||
changes: number;
|
changes: number;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -133,7 +133,7 @@ export function getSessionEvents(
|
|||||||
if (!eventTypes || eventTypes.length === 0) {
|
if (!eventTypes || eventTypes.length === 0) {
|
||||||
const stmt = db.prepare(`
|
const stmt = db.prepare(`
|
||||||
SELECT event_type AS eventType, ts_ms AS tsMs, payload_json AS payload
|
SELECT event_type AS eventType, ts_ms AS tsMs, payload_json AS payload
|
||||||
FROM imm_session_events WHERE session_id = ? ORDER BY ts_ms ASC LIMIT ?
|
FROM imm_session_events WHERE session_id = ? ORDER BY CAST(ts_ms AS REAL) ASC LIMIT ?
|
||||||
`);
|
`);
|
||||||
const rows = stmt.all(sessionId, limit) as Array<SessionEventRow & { tsMs: number | string }>;
|
const rows = stmt.all(sessionId, limit) as Array<SessionEventRow & { tsMs: number | string }>;
|
||||||
return rows.map((row) => ({
|
return rows.map((row) => ({
|
||||||
@@ -147,7 +147,7 @@ export function getSessionEvents(
|
|||||||
SELECT event_type AS eventType, ts_ms AS tsMs, payload_json AS payload
|
SELECT event_type AS eventType, ts_ms AS tsMs, payload_json AS payload
|
||||||
FROM imm_session_events
|
FROM imm_session_events
|
||||||
WHERE session_id = ? AND event_type IN (${placeholders})
|
WHERE session_id = ? AND event_type IN (${placeholders})
|
||||||
ORDER BY ts_ms ASC
|
ORDER BY CAST(ts_ms AS REAL) ASC
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
`);
|
`);
|
||||||
const rows = stmt.all(sessionId, ...eventTypes, limit) as Array<
|
const rows = stmt.all(sessionId, ...eventTypes, limit) as Array<
|
||||||
|
|||||||
@@ -602,7 +602,7 @@ export function getEpisodeCardEvents(db: DatabaseSync, videoId: number): Episode
|
|||||||
FROM imm_session_events e
|
FROM imm_session_events e
|
||||||
JOIN imm_sessions s ON s.session_id = e.session_id
|
JOIN imm_sessions s ON s.session_id = e.session_id
|
||||||
WHERE s.video_id = ? AND e.event_type = 4
|
WHERE s.video_id = ? AND e.event_type = 4
|
||||||
ORDER BY e.ts_ms DESC
|
ORDER BY CAST(e.ts_ms AS REAL) DESC
|
||||||
`,
|
`,
|
||||||
)
|
)
|
||||||
.all(videoId) as Array<{
|
.all(videoId) as Array<{
|
||||||
|
|||||||
@@ -345,7 +345,11 @@ export function fromDbTimestamp(ms: number | bigint | string | null | undefined)
|
|||||||
if (typeof ms === 'bigint') {
|
if (typeof ms === 'bigint') {
|
||||||
return Number(ms);
|
return Number(ms);
|
||||||
}
|
}
|
||||||
return Number(ms);
|
const normalized = normalizeTimestampString(ms);
|
||||||
|
if (/^-?\d+$/.test(normalized)) {
|
||||||
|
return Number(BigInt(normalized));
|
||||||
|
}
|
||||||
|
return Math.trunc(Number.parseFloat(normalized));
|
||||||
}
|
}
|
||||||
|
|
||||||
function getNumericCalendarValue(
|
function getNumericCalendarValue(
|
||||||
|
|||||||
@@ -263,6 +263,370 @@ test('ensureSchema adds youtube metadata table to existing schema version 15 dat
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('ensureSchema migrates session event timestamps to text and repairs libsql-truncated wall-clock values', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
db.exec(`
|
||||||
|
CREATE TABLE imm_schema_version (
|
||||||
|
schema_version INTEGER PRIMARY KEY,
|
||||||
|
applied_at_ms INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
INSERT INTO imm_schema_version(schema_version, applied_at_ms) VALUES (16, 1000);
|
||||||
|
|
||||||
|
CREATE TABLE imm_rollup_state(
|
||||||
|
state_key TEXT PRIMARY KEY,
|
||||||
|
state_value INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('last_rollup_sample_ms', 0);
|
||||||
|
|
||||||
|
CREATE TABLE imm_anime(
|
||||||
|
anime_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
normalized_title_key TEXT NOT NULL UNIQUE,
|
||||||
|
canonical_title TEXT NOT NULL,
|
||||||
|
anilist_id INTEGER UNIQUE,
|
||||||
|
title_romaji TEXT,
|
||||||
|
title_english TEXT,
|
||||||
|
title_native TEXT,
|
||||||
|
episodes_total INTEGER,
|
||||||
|
description TEXT,
|
||||||
|
metadata_json TEXT,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_videos(
|
||||||
|
video_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
video_key TEXT NOT NULL UNIQUE,
|
||||||
|
anime_id INTEGER,
|
||||||
|
canonical_title TEXT NOT NULL,
|
||||||
|
source_type INTEGER NOT NULL,
|
||||||
|
source_path TEXT,
|
||||||
|
source_url TEXT,
|
||||||
|
parsed_basename TEXT,
|
||||||
|
parsed_title TEXT,
|
||||||
|
parsed_season INTEGER,
|
||||||
|
parsed_episode INTEGER,
|
||||||
|
parser_source TEXT,
|
||||||
|
parser_confidence REAL,
|
||||||
|
parse_metadata_json TEXT,
|
||||||
|
watched INTEGER NOT NULL DEFAULT 0,
|
||||||
|
duration_ms INTEGER NOT NULL CHECK(duration_ms>=0),
|
||||||
|
file_size_bytes INTEGER CHECK(file_size_bytes>=0),
|
||||||
|
codec_id INTEGER, container_id INTEGER,
|
||||||
|
width_px INTEGER, height_px INTEGER, fps_x100 INTEGER,
|
||||||
|
bitrate_kbps INTEGER, audio_codec_id INTEGER,
|
||||||
|
hash_sha256 TEXT, screenshot_path TEXT,
|
||||||
|
metadata_json TEXT,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT,
|
||||||
|
FOREIGN KEY(anime_id) REFERENCES imm_anime(anime_id) ON DELETE SET NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_sessions(
|
||||||
|
session_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
session_uuid TEXT NOT NULL UNIQUE,
|
||||||
|
video_id INTEGER NOT NULL,
|
||||||
|
started_at_ms TEXT NOT NULL,
|
||||||
|
ended_at_ms TEXT,
|
||||||
|
status INTEGER NOT NULL,
|
||||||
|
locale_id INTEGER,
|
||||||
|
target_lang_id INTEGER,
|
||||||
|
difficulty_tier INTEGER,
|
||||||
|
subtitle_mode INTEGER,
|
||||||
|
ended_media_ms INTEGER,
|
||||||
|
total_watched_ms INTEGER NOT NULL DEFAULT 0,
|
||||||
|
active_watched_ms INTEGER NOT NULL DEFAULT 0,
|
||||||
|
lines_seen INTEGER NOT NULL DEFAULT 0,
|
||||||
|
tokens_seen INTEGER NOT NULL DEFAULT 0,
|
||||||
|
cards_mined INTEGER NOT NULL DEFAULT 0,
|
||||||
|
lookup_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
lookup_hits INTEGER NOT NULL DEFAULT 0,
|
||||||
|
yomitan_lookup_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
pause_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
pause_ms INTEGER NOT NULL DEFAULT 0,
|
||||||
|
seek_forward_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
seek_backward_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
media_buffer_events INTEGER NOT NULL DEFAULT 0,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT,
|
||||||
|
FOREIGN KEY(video_id) REFERENCES imm_videos(video_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_session_telemetry(
|
||||||
|
telemetry_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
session_id INTEGER NOT NULL,
|
||||||
|
sample_ms TEXT NOT NULL,
|
||||||
|
total_watched_ms INTEGER NOT NULL DEFAULT 0,
|
||||||
|
active_watched_ms INTEGER NOT NULL DEFAULT 0,
|
||||||
|
lines_seen INTEGER NOT NULL DEFAULT 0,
|
||||||
|
tokens_seen INTEGER NOT NULL DEFAULT 0,
|
||||||
|
cards_mined INTEGER NOT NULL DEFAULT 0,
|
||||||
|
lookup_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
lookup_hits INTEGER NOT NULL DEFAULT 0,
|
||||||
|
yomitan_lookup_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
pause_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
pause_ms INTEGER NOT NULL DEFAULT 0,
|
||||||
|
seek_forward_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
seek_backward_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
media_buffer_events INTEGER NOT NULL DEFAULT 0,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT,
|
||||||
|
FOREIGN KEY(session_id) REFERENCES imm_sessions(session_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_session_events(
|
||||||
|
event_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
session_id INTEGER NOT NULL,
|
||||||
|
ts_ms INTEGER NOT NULL,
|
||||||
|
event_type INTEGER NOT NULL,
|
||||||
|
line_index INTEGER,
|
||||||
|
segment_start_ms INTEGER,
|
||||||
|
segment_end_ms INTEGER,
|
||||||
|
tokens_delta INTEGER NOT NULL DEFAULT 0,
|
||||||
|
cards_delta INTEGER NOT NULL DEFAULT 0,
|
||||||
|
payload_json TEXT,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT,
|
||||||
|
FOREIGN KEY(session_id) REFERENCES imm_sessions(session_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_daily_rollups(
|
||||||
|
rollup_day INTEGER NOT NULL,
|
||||||
|
video_id INTEGER,
|
||||||
|
total_sessions INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_active_min REAL NOT NULL DEFAULT 0,
|
||||||
|
total_lines_seen INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_tokens_seen INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_cards INTEGER NOT NULL DEFAULT 0,
|
||||||
|
cards_per_hour REAL,
|
||||||
|
tokens_per_min REAL,
|
||||||
|
lookup_hit_rate REAL,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT,
|
||||||
|
PRIMARY KEY (rollup_day, video_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_monthly_rollups(
|
||||||
|
rollup_month INTEGER NOT NULL,
|
||||||
|
video_id INTEGER,
|
||||||
|
total_sessions INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_active_min REAL NOT NULL DEFAULT 0,
|
||||||
|
total_lines_seen INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_tokens_seen INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_cards INTEGER NOT NULL DEFAULT 0,
|
||||||
|
cards_per_hour REAL,
|
||||||
|
tokens_per_min REAL,
|
||||||
|
lookup_hit_rate REAL,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT,
|
||||||
|
PRIMARY KEY (rollup_month, video_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_words(
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
headword TEXT NOT NULL,
|
||||||
|
word TEXT NOT NULL,
|
||||||
|
reading TEXT NOT NULL,
|
||||||
|
part_of_speech TEXT,
|
||||||
|
pos1 TEXT,
|
||||||
|
pos2 TEXT,
|
||||||
|
pos3 TEXT,
|
||||||
|
first_seen INTEGER NOT NULL,
|
||||||
|
last_seen INTEGER NOT NULL,
|
||||||
|
frequency INTEGER NOT NULL DEFAULT 0,
|
||||||
|
frequency_rank INTEGER,
|
||||||
|
UNIQUE(headword, word, reading)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_kanji(
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
kanji TEXT NOT NULL UNIQUE,
|
||||||
|
first_seen INTEGER NOT NULL,
|
||||||
|
last_seen INTEGER NOT NULL,
|
||||||
|
frequency INTEGER NOT NULL DEFAULT 0
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_subtitle_lines(
|
||||||
|
line_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
session_id INTEGER NOT NULL,
|
||||||
|
event_id INTEGER,
|
||||||
|
video_id INTEGER NOT NULL,
|
||||||
|
anime_id INTEGER,
|
||||||
|
line_index INTEGER NOT NULL,
|
||||||
|
segment_start_ms INTEGER,
|
||||||
|
segment_end_ms INTEGER,
|
||||||
|
text TEXT NOT NULL,
|
||||||
|
secondary_text TEXT,
|
||||||
|
CREATED_DATE INTEGER,
|
||||||
|
LAST_UPDATE_DATE INTEGER,
|
||||||
|
FOREIGN KEY(session_id) REFERENCES imm_sessions(session_id) ON DELETE CASCADE,
|
||||||
|
FOREIGN KEY(event_id) REFERENCES imm_session_events(event_id) ON DELETE SET NULL,
|
||||||
|
FOREIGN KEY(video_id) REFERENCES imm_videos(video_id) ON DELETE CASCADE,
|
||||||
|
FOREIGN KEY(anime_id) REFERENCES imm_anime(anime_id) ON DELETE SET NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_word_line_occurrences(
|
||||||
|
line_id INTEGER NOT NULL,
|
||||||
|
word_id INTEGER NOT NULL,
|
||||||
|
occurrence_count INTEGER NOT NULL,
|
||||||
|
PRIMARY KEY(line_id, word_id),
|
||||||
|
FOREIGN KEY(line_id) REFERENCES imm_subtitle_lines(line_id) ON DELETE CASCADE,
|
||||||
|
FOREIGN KEY(word_id) REFERENCES imm_words(id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_kanji_line_occurrences(
|
||||||
|
line_id INTEGER NOT NULL,
|
||||||
|
kanji_id INTEGER NOT NULL,
|
||||||
|
occurrence_count INTEGER NOT NULL,
|
||||||
|
PRIMARY KEY(line_id, kanji_id),
|
||||||
|
FOREIGN KEY(line_id) REFERENCES imm_subtitle_lines(line_id) ON DELETE CASCADE,
|
||||||
|
FOREIGN KEY(kanji_id) REFERENCES imm_kanji(id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_lifetime_global(
|
||||||
|
global_id INTEGER PRIMARY KEY CHECK(global_id = 1),
|
||||||
|
total_sessions INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_active_ms INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_cards INTEGER NOT NULL DEFAULT 0,
|
||||||
|
active_days INTEGER NOT NULL DEFAULT 0,
|
||||||
|
episodes_started INTEGER NOT NULL DEFAULT 0,
|
||||||
|
episodes_completed INTEGER NOT NULL DEFAULT 0,
|
||||||
|
anime_completed INTEGER NOT NULL DEFAULT 0,
|
||||||
|
last_rebuilt_ms TEXT,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_lifetime_anime(
|
||||||
|
anime_id INTEGER PRIMARY KEY,
|
||||||
|
total_sessions INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_active_ms INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_cards INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_lines_seen INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_tokens_seen INTEGER NOT NULL DEFAULT 0,
|
||||||
|
episodes_started INTEGER NOT NULL DEFAULT 0,
|
||||||
|
episodes_completed INTEGER NOT NULL DEFAULT 0,
|
||||||
|
first_watched_ms TEXT,
|
||||||
|
last_watched_ms TEXT,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT,
|
||||||
|
FOREIGN KEY(anime_id) REFERENCES imm_anime(anime_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_lifetime_media(
|
||||||
|
video_id INTEGER PRIMARY KEY,
|
||||||
|
total_sessions INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_active_ms INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_cards INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_lines_seen INTEGER NOT NULL DEFAULT 0,
|
||||||
|
total_tokens_seen INTEGER NOT NULL DEFAULT 0,
|
||||||
|
completed INTEGER NOT NULL DEFAULT 0,
|
||||||
|
first_watched_ms TEXT,
|
||||||
|
last_watched_ms TEXT,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT,
|
||||||
|
FOREIGN KEY(video_id) REFERENCES imm_videos(video_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_lifetime_applied_sessions(
|
||||||
|
session_id INTEGER PRIMARY KEY,
|
||||||
|
applied_at_ms TEXT NOT NULL,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT,
|
||||||
|
FOREIGN KEY(session_id) REFERENCES imm_sessions(session_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_media_art(
|
||||||
|
video_id INTEGER PRIMARY KEY,
|
||||||
|
anilist_id INTEGER,
|
||||||
|
cover_url TEXT,
|
||||||
|
cover_blob BLOB,
|
||||||
|
cover_blob_hash TEXT,
|
||||||
|
fetched_at_ms TEXT,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT,
|
||||||
|
FOREIGN KEY(video_id) REFERENCES imm_videos(video_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_cover_art_blobs(
|
||||||
|
blob_hash TEXT PRIMARY KEY,
|
||||||
|
cover_blob BLOB NOT NULL,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE imm_youtube_videos(
|
||||||
|
video_id INTEGER PRIMARY KEY,
|
||||||
|
youtube_video_id TEXT,
|
||||||
|
video_url TEXT,
|
||||||
|
video_title TEXT,
|
||||||
|
video_thumbnail_url TEXT,
|
||||||
|
channel_id TEXT,
|
||||||
|
channel_name TEXT,
|
||||||
|
channel_url TEXT,
|
||||||
|
channel_thumbnail_url TEXT,
|
||||||
|
uploader_id TEXT,
|
||||||
|
uploader_url TEXT,
|
||||||
|
description TEXT,
|
||||||
|
metadata_json TEXT,
|
||||||
|
fetched_at_ms TEXT NOT NULL,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT,
|
||||||
|
FOREIGN KEY(video_id) REFERENCES imm_videos(video_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO imm_videos (
|
||||||
|
video_id, video_key, canonical_title, source_type, source_path, source_url, watched, duration_ms,
|
||||||
|
CREATED_DATE, LAST_UPDATE_DATE
|
||||||
|
) VALUES (
|
||||||
|
1, 'local:/tmp/repaired-event.mkv', 'Repaired Event', 1, '/tmp/repaired-event.mkv', NULL, 0, 0, '1000', '1000'
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO imm_sessions (
|
||||||
|
session_id, session_uuid, video_id, started_at_ms, status, CREATED_DATE, LAST_UPDATE_DATE
|
||||||
|
) VALUES (
|
||||||
|
1, 'session-1', 1, '1775940000000', 1, '1775940000000', '1775940000000'
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO imm_session_events (
|
||||||
|
event_id, session_id, ts_ms, event_type, line_index, segment_start_ms, segment_end_ms,
|
||||||
|
tokens_delta, cards_delta, payload_json, CREATED_DATE, LAST_UPDATE_DATE
|
||||||
|
) VALUES (
|
||||||
|
1, 1, -2147483648, 4, NULL, NULL, NULL, 0, 1, '{\"noteIds\":[1]}', '1775943304128', '1775943304128'
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
|
||||||
|
ensureSchema(db);
|
||||||
|
|
||||||
|
const column = db.prepare(`PRAGMA table_info(imm_session_events)`).all() as Array<{
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
}>;
|
||||||
|
assert.equal(column.find((entry) => entry.name === 'ts_ms')?.type, 'TEXT');
|
||||||
|
|
||||||
|
const row = db.prepare(
|
||||||
|
`
|
||||||
|
SELECT ts_ms AS tsMs, typeof(ts_ms) AS tsType, CREATED_DATE AS createdDate
|
||||||
|
FROM imm_session_events
|
||||||
|
WHERE event_id = 1
|
||||||
|
`,
|
||||||
|
).get() as {
|
||||||
|
tsMs: string;
|
||||||
|
tsType: string;
|
||||||
|
createdDate: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
assert.equal(row.tsType, 'text');
|
||||||
|
assert.equal(row.tsMs, '1775943304128');
|
||||||
|
assert.equal(row.createdDate, '1775943304128');
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('ensureSchema creates large-history performance indexes', () => {
|
test('ensureSchema creates large-history performance indexes', () => {
|
||||||
const dbPath = makeDbPath();
|
const dbPath = makeDbPath();
|
||||||
const db = new Database(dbPath);
|
const db = new Database(dbPath);
|
||||||
|
|||||||
@@ -170,6 +170,14 @@ function hasColumn(db: DatabaseSync, tableName: string, columnName: string): boo
|
|||||||
.some((row: unknown) => (row as { name: string }).name === columnName);
|
.some((row: unknown) => (row as { name: string }).name === columnName);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getColumnType(db: DatabaseSync, tableName: string, columnName: string): string | null {
|
||||||
|
const row = (db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
}>).find((entry) => entry.name === columnName);
|
||||||
|
return row?.type ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
function addColumnIfMissing(
|
function addColumnIfMissing(
|
||||||
db: DatabaseSync,
|
db: DatabaseSync,
|
||||||
tableName: string,
|
tableName: string,
|
||||||
@@ -187,6 +195,92 @@ function dropColumnIfExists(db: DatabaseSync, tableName: string, columnName: str
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function migrateSessionEventTimestampsToText(db: DatabaseSync): void {
|
||||||
|
if (getColumnType(db, 'imm_session_events', 'ts_ms') === 'TEXT') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const lineIndexExpr = hasColumn(db, 'imm_session_events', 'line_index') ? 'line_index' : 'NULL';
|
||||||
|
const segmentStartExpr = hasColumn(db, 'imm_session_events', 'segment_start_ms')
|
||||||
|
? 'segment_start_ms'
|
||||||
|
: 'NULL';
|
||||||
|
const segmentEndExpr = hasColumn(db, 'imm_session_events', 'segment_end_ms')
|
||||||
|
? 'segment_end_ms'
|
||||||
|
: 'NULL';
|
||||||
|
const tokensDeltaExpr = hasColumn(db, 'imm_session_events', 'tokens_delta')
|
||||||
|
? 'tokens_delta'
|
||||||
|
: '0';
|
||||||
|
const cardsDeltaExpr = hasColumn(db, 'imm_session_events', 'cards_delta') ? 'cards_delta' : '0';
|
||||||
|
const payloadExpr = hasColumn(db, 'imm_session_events', 'payload_json') ? 'payload_json' : 'NULL';
|
||||||
|
const createdDateExpr = hasColumn(db, 'imm_session_events', 'CREATED_DATE')
|
||||||
|
? 'CAST(CREATED_DATE AS TEXT)'
|
||||||
|
: 'NULL';
|
||||||
|
const lastUpdateExpr = hasColumn(db, 'imm_session_events', 'LAST_UPDATE_DATE')
|
||||||
|
? 'CAST(LAST_UPDATE_DATE AS TEXT)'
|
||||||
|
: 'NULL';
|
||||||
|
const repairedTimestampExpr =
|
||||||
|
hasColumn(db, 'imm_session_events', 'CREATED_DATE') ||
|
||||||
|
hasColumn(db, 'imm_session_events', 'LAST_UPDATE_DATE')
|
||||||
|
? `CASE
|
||||||
|
WHEN ts_ms < 0 AND COALESCE(CREATED_DATE, LAST_UPDATE_DATE) IS NOT NULL
|
||||||
|
THEN CAST(COALESCE(CREATED_DATE, LAST_UPDATE_DATE) AS TEXT)
|
||||||
|
ELSE CAST(ts_ms AS TEXT)
|
||||||
|
END`
|
||||||
|
: 'CAST(ts_ms AS TEXT)';
|
||||||
|
|
||||||
|
db.exec('PRAGMA foreign_keys = OFF');
|
||||||
|
db.exec(`
|
||||||
|
CREATE TABLE imm_session_events_new(
|
||||||
|
event_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
session_id INTEGER NOT NULL,
|
||||||
|
ts_ms TEXT NOT NULL,
|
||||||
|
event_type INTEGER NOT NULL,
|
||||||
|
line_index INTEGER,
|
||||||
|
segment_start_ms INTEGER,
|
||||||
|
segment_end_ms INTEGER,
|
||||||
|
tokens_delta INTEGER NOT NULL DEFAULT 0,
|
||||||
|
cards_delta INTEGER NOT NULL DEFAULT 0,
|
||||||
|
payload_json TEXT,
|
||||||
|
CREATED_DATE TEXT,
|
||||||
|
LAST_UPDATE_DATE TEXT,
|
||||||
|
FOREIGN KEY(session_id) REFERENCES imm_sessions(session_id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
db.exec(`
|
||||||
|
INSERT INTO imm_session_events_new(
|
||||||
|
event_id,
|
||||||
|
session_id,
|
||||||
|
ts_ms,
|
||||||
|
event_type,
|
||||||
|
line_index,
|
||||||
|
segment_start_ms,
|
||||||
|
segment_end_ms,
|
||||||
|
tokens_delta,
|
||||||
|
cards_delta,
|
||||||
|
payload_json,
|
||||||
|
CREATED_DATE,
|
||||||
|
LAST_UPDATE_DATE
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
event_id,
|
||||||
|
session_id,
|
||||||
|
${repairedTimestampExpr},
|
||||||
|
event_type,
|
||||||
|
${lineIndexExpr},
|
||||||
|
${segmentStartExpr},
|
||||||
|
${segmentEndExpr},
|
||||||
|
${tokensDeltaExpr},
|
||||||
|
${cardsDeltaExpr},
|
||||||
|
${payloadExpr},
|
||||||
|
${createdDateExpr},
|
||||||
|
${lastUpdateExpr}
|
||||||
|
FROM imm_session_events
|
||||||
|
`);
|
||||||
|
db.exec('DROP TABLE imm_session_events');
|
||||||
|
db.exec('ALTER TABLE imm_session_events_new RENAME TO imm_session_events');
|
||||||
|
db.exec('PRAGMA foreign_keys = ON');
|
||||||
|
}
|
||||||
|
|
||||||
export function applyPragmas(db: DatabaseSync): void {
|
export function applyPragmas(db: DatabaseSync): void {
|
||||||
db.exec('PRAGMA journal_mode = WAL');
|
db.exec('PRAGMA journal_mode = WAL');
|
||||||
db.exec('PRAGMA synchronous = NORMAL');
|
db.exec('PRAGMA synchronous = NORMAL');
|
||||||
@@ -685,7 +779,7 @@ export function ensureSchema(db: DatabaseSync): void {
|
|||||||
CREATE TABLE IF NOT EXISTS imm_session_events(
|
CREATE TABLE IF NOT EXISTS imm_session_events(
|
||||||
event_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
event_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
session_id INTEGER NOT NULL,
|
session_id INTEGER NOT NULL,
|
||||||
ts_ms INTEGER NOT NULL,
|
ts_ms TEXT NOT NULL,
|
||||||
event_type INTEGER NOT NULL,
|
event_type INTEGER NOT NULL,
|
||||||
line_index INTEGER,
|
line_index INTEGER,
|
||||||
segment_start_ms INTEGER,
|
segment_start_ms INTEGER,
|
||||||
@@ -1122,6 +1216,8 @@ export function ensureSchema(db: DatabaseSync): void {
|
|||||||
addColumnIfMissing(db, 'imm_sessions', 'ended_media_ms', 'INTEGER');
|
addColumnIfMissing(db, 'imm_sessions', 'ended_media_ms', 'INTEGER');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
migrateSessionEventTimestampsToText(db);
|
||||||
|
|
||||||
ensureLifetimeSummaryTables(db);
|
ensureLifetimeSummaryTables(db);
|
||||||
|
|
||||||
db.exec(`
|
db.exec(`
|
||||||
@@ -1420,7 +1516,8 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta
|
|||||||
) {
|
) {
|
||||||
throw new Error('Incomplete telemetry write');
|
throw new Error('Incomplete telemetry write');
|
||||||
}
|
}
|
||||||
const telemetrySampleMs = toDbTimestamp(write.sampleMs ?? Number(currentMs));
|
const telemetrySampleMs =
|
||||||
|
write.sampleMs === undefined ? currentMs : toDbTimestamp(write.sampleMs);
|
||||||
stmts.telemetryInsertStmt.run(
|
stmts.telemetryInsertStmt.run(
|
||||||
write.sessionId,
|
write.sessionId,
|
||||||
telemetrySampleMs,
|
telemetrySampleMs,
|
||||||
@@ -1495,7 +1592,7 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta
|
|||||||
|
|
||||||
stmts.eventInsertStmt.run(
|
stmts.eventInsertStmt.run(
|
||||||
write.sessionId,
|
write.sessionId,
|
||||||
toDbTimestamp(write.sampleMs ?? Number(currentMs)),
|
write.sampleMs === undefined ? currentMs : toDbTimestamp(write.sampleMs),
|
||||||
write.eventType ?? 0,
|
write.eventType ?? 0,
|
||||||
write.lineIndex ?? null,
|
write.lineIndex ?? null,
|
||||||
write.segmentStartMs ?? null,
|
write.segmentStartMs ?? null,
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
export const SCHEMA_VERSION = 16;
|
export const SCHEMA_VERSION = 17;
|
||||||
export const DEFAULT_QUEUE_CAP = 1_000;
|
export const DEFAULT_QUEUE_CAP = 1_000;
|
||||||
export const DEFAULT_BATCH_SIZE = 25;
|
export const DEFAULT_BATCH_SIZE = 25;
|
||||||
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
|
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
|
||||||
|
|||||||
Reference in New Issue
Block a user