sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-03-25 00:11:26 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): add changelog fragment for immersion changes

Browse Source
This commit is contained in:
Kyle Yasuda
2026-03-22 19:07:07 -07:00
parent 8928bfdf7e
commit 8da3a26855
17 changed files with 1109 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
src/core/services/immersion-tracker/__tests__/query.test.ts
View File

@@ -39,6 +39,7 @@ import {
} from '../query.js';
import {
SOURCE_TYPE_LOCAL,
SOURCE_TYPE_REMOTE,
EVENT_CARD_MINED,
EVENT_SUBTITLE_LINE,
EVENT_YOMITAN_LOOKUP,
@@ -1956,6 +1957,100 @@ test('media library and detail queries read lifetime totals', () => {
}
});
test('media library and detail queries include joined youtube metadata when present', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const mediaOne = getOrCreateVideoRecord(db, 'yt:https://www.youtube.com/watch?v=abc123', {
canonicalTitle: 'Local Fallback Title',
sourcePath: null,
sourceUrl: 'https://www.youtube.com/watch?v=abc123',
sourceType: SOURCE_TYPE_REMOTE,
});
db.prepare(
`
INSERT INTO imm_lifetime_media (
video_id,
total_sessions,
total_active_ms,
total_cards,
total_lines_seen,
total_tokens_seen,
completed,
first_watched_ms,
last_watched_ms,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(mediaOne, 2, 6_000, 1, 5, 80, 0, 1_000, 9_000, 9_000, 9_000);
db.prepare(
`
INSERT INTO imm_youtube_videos (
video_id,
youtube_video_id,
video_url,
video_title,
video_thumbnail_url,
channel_id,
channel_name,
channel_url,
channel_thumbnail_url,
uploader_id,
uploader_url,
description,
metadata_json,
fetched_at_ms,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(
mediaOne,
'abc123',
'https://www.youtube.com/watch?v=abc123',
'Tracked Video Title',
'https://i.ytimg.com/vi/abc123/hqdefault.jpg',
'UCcreator123',
'Creator Name',
'https://www.youtube.com/channel/UCcreator123',
'https://yt3.googleusercontent.com/channel-avatar=s88',
'@creator',
'https://www.youtube.com/@creator',
'Video description',
'{"source":"test"}',
10_000,
10_000,
10_000,
);
const library = getMediaLibrary(db);
const detail = getMediaDetail(db, mediaOne);
assert.equal(library.length, 1);
assert.equal(library[0]?.youtubeVideoId, 'abc123');
assert.equal(library[0]?.videoTitle, 'Tracked Video Title');
assert.equal(library[0]?.channelId, 'UCcreator123');
assert.equal(library[0]?.channelName, 'Creator Name');
assert.equal(library[0]?.channelUrl, 'https://www.youtube.com/channel/UCcreator123');
assert.equal(detail?.youtubeVideoId, 'abc123');
assert.equal(detail?.videoUrl, 'https://www.youtube.com/watch?v=abc123');
assert.equal(detail?.videoThumbnailUrl, 'https://i.ytimg.com/vi/abc123/hqdefault.jpg');
assert.equal(detail?.channelThumbnailUrl, 'https://yt3.googleusercontent.com/channel-avatar=s88');
assert.equal(detail?.uploaderId, '@creator');
assert.equal(detail?.uploaderUrl, 'https://www.youtube.com/@creator');
assert.equal(detail?.description, 'Video description');
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('cover art queries reuse a shared blob across duplicate anime art rows', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);

26
src/core/services/immersion-tracker/query.ts
View File

@@ -1817,6 +1817,17 @@ export function getMediaLibrary(db: DatabaseSync): MediaLibraryRow[] {
COALESCE(lm.total_cards, 0) AS totalCards,
COALESCE(lm.total_tokens_seen, 0) AS totalTokensSeen,
COALESCE(lm.last_watched_ms, 0) AS lastWatchedMs,
yv.youtube_video_id AS youtubeVideoId,
yv.video_url AS videoUrl,
yv.video_title AS videoTitle,
yv.video_thumbnail_url AS videoThumbnailUrl,
yv.channel_id AS channelId,
yv.channel_name AS channelName,
yv.channel_url AS channelUrl,
yv.channel_thumbnail_url AS channelThumbnailUrl,
yv.uploader_id AS uploaderId,
yv.uploader_url AS uploaderUrl,
yv.description AS description,
CASE
WHEN ma.cover_blob_hash IS NOT NULL OR ma.cover_blob IS NOT NULL THEN 1
ELSE 0
@@ -1824,6 +1835,7 @@ export function getMediaLibrary(db: DatabaseSync): MediaLibraryRow[] {
FROM imm_videos v
JOIN imm_lifetime_media lm ON lm.video_id = v.video_id
LEFT JOIN imm_media_art ma ON ma.video_id = v.video_id
LEFT JOIN imm_youtube_videos yv ON yv.video_id = v.video_id
ORDER BY lm.last_watched_ms DESC
`,
)
@@ -1846,9 +1858,21 @@ export function getMediaDetail(db: DatabaseSync, videoId: number): MediaDetailRo
COALESCE(lm.total_lines_seen, 0) AS totalLinesSeen,
COALESCE(SUM(COALESCE(asm.lookupCount, s.lookup_count, 0)), 0) AS totalLookupCount,
COALESCE(SUM(COALESCE(asm.lookupHits, s.lookup_hits, 0)), 0) AS totalLookupHits,
COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount
COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount,
yv.youtube_video_id AS youtubeVideoId,
yv.video_url AS videoUrl,
yv.video_title AS videoTitle,
yv.video_thumbnail_url AS videoThumbnailUrl,
yv.channel_id AS channelId,
yv.channel_name AS channelName,
yv.channel_url AS channelUrl,
yv.channel_thumbnail_url AS channelThumbnailUrl,
yv.uploader_id AS uploaderId,
yv.uploader_url AS uploaderUrl,
yv.description AS description
FROM imm_videos v
JOIN imm_lifetime_media lm ON lm.video_id = v.video_id
LEFT JOIN imm_youtube_videos yv ON yv.video_id = v.video_id
LEFT JOIN imm_sessions s ON s.video_id = v.video_id
LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
WHERE v.video_id = ?

109
src/core/services/immersion-tracker/storage-session.test.ts
View File

@@ -106,6 +106,7 @@ test('ensureSchema creates immersion core tables', () => {
assert.ok(tableNames.has('imm_kanji_line_occurrences'));
assert.ok(tableNames.has('imm_rollup_state'));
assert.ok(tableNames.has('imm_cover_art_blobs'));
assert.ok(tableNames.has('imm_youtube_videos'));
const videoColumns = new Set(
(
@@ -146,6 +147,114 @@ test('ensureSchema creates immersion core tables', () => {
}
});
test('ensureSchema adds youtube metadata table to existing schema version 15 databases', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
db.exec(`
CREATE TABLE imm_schema_version (
schema_version INTEGER PRIMARY KEY,
applied_at_ms INTEGER NOT NULL
);
INSERT INTO imm_schema_version(schema_version, applied_at_ms) VALUES (15, 1000);
CREATE TABLE imm_rollup_state(
state_key TEXT PRIMARY KEY,
state_value INTEGER NOT NULL
);
INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('last_rollup_sample_ms', 123);
CREATE TABLE imm_anime(
anime_id INTEGER PRIMARY KEY AUTOINCREMENT,
normalized_title_key TEXT NOT NULL UNIQUE,
canonical_title TEXT NOT NULL,
anilist_id INTEGER UNIQUE,
title_romaji TEXT,
title_english TEXT,
title_native TEXT,
episodes_total INTEGER,
description TEXT,
metadata_json TEXT,
CREATED_DATE INTEGER,
LAST_UPDATE_DATE INTEGER
);
CREATE TABLE imm_videos(
video_id INTEGER PRIMARY KEY AUTOINCREMENT,
video_key TEXT NOT NULL UNIQUE,
anime_id INTEGER,
canonical_title TEXT NOT NULL,
source_type INTEGER NOT NULL,
source_path TEXT,
source_url TEXT,
parsed_basename TEXT,
parsed_title TEXT,
parsed_season INTEGER,
parsed_episode INTEGER,
parser_source TEXT,
parser_confidence REAL,
parse_metadata_json TEXT,
watched INTEGER NOT NULL DEFAULT 0,
duration_ms INTEGER NOT NULL CHECK(duration_ms>=0),
file_size_bytes INTEGER CHECK(file_size_bytes>=0),
codec_id INTEGER, container_id INTEGER,
width_px INTEGER, height_px INTEGER, fps_x100 INTEGER,
bitrate_kbps INTEGER, audio_codec_id INTEGER,
hash_sha256 TEXT, screenshot_path TEXT,
metadata_json TEXT,
CREATED_DATE INTEGER,
LAST_UPDATE_DATE INTEGER,
FOREIGN KEY(anime_id) REFERENCES imm_anime(anime_id) ON DELETE SET NULL
);
`);
ensureSchema(db);
const tables = new Set(
(
db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'imm_%'`).all() as Array<{
name: string;
}>
).map((row) => row.name),
);
assert.ok(tables.has('imm_youtube_videos'));
const columns = new Set(
(
db.prepare('PRAGMA table_info(imm_youtube_videos)').all() as Array<{
name: string;
}>
).map((row) => row.name),
);
assert.deepEqual(
columns,
new Set([
'video_id',
'youtube_video_id',
'video_url',
'video_title',
'video_thumbnail_url',
'channel_id',
'channel_name',
'channel_url',
'channel_thumbnail_url',
'uploader_id',
'uploader_url',
'description',
'metadata_json',
'fetched_at_ms',
'CREATED_DATE',
'LAST_UPDATE_DATE',
]),
);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('ensureSchema creates large-history performance indexes', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);

93
src/core/services/immersion-tracker/storage.ts
View File

@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto';
import { parseMediaInfo } from '../../../jimaku/utils';
import type { DatabaseSync } from './sqlite';
import { SCHEMA_VERSION } from './types';
import type { QueuedWrite, VideoMetadata } from './types';
import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types';
export interface TrackerPreparedStatements {
telemetryInsertStmt: ReturnType<DatabaseSync['prepare']>;
@@ -743,6 +743,27 @@ export function ensureSchema(db: DatabaseSync): void {
FOREIGN KEY(video_id) REFERENCES imm_videos(video_id) ON DELETE CASCADE
);
`);
db.exec(`
CREATE TABLE IF NOT EXISTS imm_youtube_videos(
video_id INTEGER PRIMARY KEY,
youtube_video_id TEXT NOT NULL,
video_url TEXT NOT NULL,
video_title TEXT,
video_thumbnail_url TEXT,
channel_id TEXT,
channel_name TEXT,
channel_url TEXT,
channel_thumbnail_url TEXT,
uploader_id TEXT,
uploader_url TEXT,
description TEXT,
metadata_json TEXT,
fetched_at_ms INTEGER NOT NULL,
CREATED_DATE INTEGER,
LAST_UPDATE_DATE INTEGER,
FOREIGN KEY(video_id) REFERENCES imm_videos(video_id) ON DELETE CASCADE
);
`);
db.exec(`
CREATE TABLE IF NOT EXISTS imm_cover_art_blobs(
blob_hash TEXT PRIMARY KEY,
@@ -1134,6 +1155,14 @@ export function ensureSchema(db: DatabaseSync): void {
CREATE INDEX IF NOT EXISTS idx_media_art_cover_url
ON imm_media_art(cover_url)
`);
db.exec(`
CREATE INDEX IF NOT EXISTS idx_youtube_videos_channel_id
ON imm_youtube_videos(channel_id)
`);
db.exec(`
CREATE INDEX IF NOT EXISTS idx_youtube_videos_youtube_video_id
ON imm_youtube_videos(youtube_video_id)
`);
if (currentVersion?.schema_version && currentVersion.schema_version < SCHEMA_VERSION) {
db.exec('DELETE FROM imm_daily_rollups');
@@ -1506,3 +1535,65 @@ export function updateVideoTitleRecord(
`,
).run(canonicalTitle, Date.now(), videoId);
}
export function upsertYoutubeVideoMetadata(
db: DatabaseSync,
videoId: number,
metadata: YoutubeVideoMetadata,
): void {
const nowMs = Date.now();
db.prepare(
`
INSERT INTO imm_youtube_videos (
video_id,
youtube_video_id,
video_url,
video_title,
video_thumbnail_url,
channel_id,
channel_name,
channel_url,
channel_thumbnail_url,
uploader_id,
uploader_url,
description,
metadata_json,
fetched_at_ms,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(video_id) DO UPDATE SET
youtube_video_id = excluded.youtube_video_id,
video_url = excluded.video_url,
video_title = excluded.video_title,
video_thumbnail_url = excluded.video_thumbnail_url,
channel_id = excluded.channel_id,
channel_name = excluded.channel_name,
channel_url = excluded.channel_url,
channel_thumbnail_url = excluded.channel_thumbnail_url,
uploader_id = excluded.uploader_id,
uploader_url = excluded.uploader_url,
description = excluded.description,
metadata_json = excluded.metadata_json,
fetched_at_ms = excluded.fetched_at_ms,
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
`,
).run(
videoId,
metadata.youtubeVideoId,
metadata.videoUrl,
metadata.videoTitle ?? null,
metadata.videoThumbnailUrl ?? null,
metadata.channelId ?? null,
metadata.channelName ?? null,
metadata.channelUrl ?? null,
metadata.channelThumbnailUrl ?? null,
metadata.uploaderId ?? null,
metadata.uploaderUrl ?? null,
metadata.description ?? null,
metadata.metadataJson ?? null,
nowMs,
nowMs,
nowMs,
);
}

39
src/core/services/immersion-tracker/types.ts
View File

@@ -1,4 +1,4 @@
export const SCHEMA_VERSION = 15;
export const SCHEMA_VERSION = 16;
export const DEFAULT_QUEUE_CAP = 1_000;
export const DEFAULT_BATCH_SIZE = 25;
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
@@ -420,6 +420,17 @@ export interface MediaLibraryRow {
totalTokensSeen: number;
lastWatchedMs: number;
hasCoverArt: number;
youtubeVideoId: string | null;
videoUrl: string | null;
videoTitle: string | null;
videoThumbnailUrl: string | null;
channelId: string | null;
channelName: string | null;
channelUrl: string | null;
channelThumbnailUrl: string | null;
uploaderId: string | null;
uploaderUrl: string | null;
description: string | null;
}
export interface MediaDetailRow {
@@ -434,6 +445,32 @@ export interface MediaDetailRow {
totalLookupCount: number;
totalLookupHits: number;
totalYomitanLookupCount: number;
youtubeVideoId: string | null;
videoUrl: string | null;
videoTitle: string | null;
videoThumbnailUrl: string | null;
channelId: string | null;
channelName: string | null;
channelUrl: string | null;
channelThumbnailUrl: string | null;
uploaderId: string | null;
uploaderUrl: string | null;
description: string | null;
}
export interface YoutubeVideoMetadata {
youtubeVideoId: string;
videoUrl: string;
videoTitle: string | null;
videoThumbnailUrl: string | null;
channelId: string | null;
channelName: string | null;
channelUrl: string | null;
channelThumbnailUrl: string | null;
uploaderId: string | null;
uploaderUrl: string | null;
description: string | null;
metadataJson: string | null;
}
export interface AnimeLibraryRow {