mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-03-25 00:11:26 -07:00
fix(ci): add changelog fragment for immersion changes
This commit is contained in:
@@ -2297,6 +2297,99 @@ test('reassignAnimeAnilist preserves existing description when description is om
|
||||
}
|
||||
});
|
||||
|
||||
test('handleMediaChange stores youtube metadata for new youtube sessions', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
const originalFetch = globalThis.fetch;
|
||||
const originalPath = process.env.PATH;
|
||||
|
||||
try {
|
||||
const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-yt-dlp-bin-'));
|
||||
const scriptPath = path.join(fakeBinDir, 'yt-dlp');
|
||||
fs.writeFileSync(
|
||||
scriptPath,
|
||||
`#!/bin/sh
|
||||
printf '%s\n' '{"id":"abc123","title":"Video Name","webpage_url":"https://www.youtube.com/watch?v=abc123","thumbnail":"https://i.ytimg.com/vi/abc123/hqdefault.jpg","channel_id":"UCcreator123","channel":"Creator Name","channel_url":"https://www.youtube.com/channel/UCcreator123","uploader_id":"@creator","uploader_url":"https://www.youtube.com/@creator","description":"Video description","channel_follower_count":12345,"thumbnails":[{"url":"https://i.ytimg.com/vi/abc123/hqdefault.jpg"},{"url":"https://yt3.googleusercontent.com/channel-avatar=s88"}]}'\n`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
process.env.PATH = `${fakeBinDir}${path.delimiter}${originalPath ?? ''}`;
|
||||
|
||||
globalThis.fetch = async (input) => {
|
||||
const url = String(input);
|
||||
if (url.includes('/oembed')) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
thumbnail_url: 'https://i.ytimg.com/vi/abc123/hqdefault.jpg',
|
||||
}),
|
||||
{ status: 200, headers: { 'Content-Type': 'application/json' } },
|
||||
);
|
||||
}
|
||||
return new Response(new Uint8Array([1, 2, 3]), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'image/jpeg' },
|
||||
});
|
||||
};
|
||||
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor({ dbPath });
|
||||
tracker.handleMediaChange('https://www.youtube.com/watch?v=abc123', 'Player Title');
|
||||
|
||||
await waitForPendingAnimeMetadata(tracker);
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
|
||||
const privateApi = tracker as unknown as { db: DatabaseSync };
|
||||
const row = privateApi.db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
youtube_video_id AS youtubeVideoId,
|
||||
video_url AS videoUrl,
|
||||
video_title AS videoTitle,
|
||||
video_thumbnail_url AS videoThumbnailUrl,
|
||||
channel_id AS channelId,
|
||||
channel_name AS channelName,
|
||||
channel_url AS channelUrl,
|
||||
channel_thumbnail_url AS channelThumbnailUrl,
|
||||
uploader_id AS uploaderId,
|
||||
uploader_url AS uploaderUrl,
|
||||
description AS description
|
||||
FROM imm_youtube_videos
|
||||
`,
|
||||
)
|
||||
.get() as {
|
||||
youtubeVideoId: string;
|
||||
videoUrl: string;
|
||||
videoTitle: string;
|
||||
videoThumbnailUrl: string;
|
||||
channelId: string;
|
||||
channelName: string;
|
||||
channelUrl: string;
|
||||
channelThumbnailUrl: string;
|
||||
uploaderId: string;
|
||||
uploaderUrl: string;
|
||||
description: string;
|
||||
} | null;
|
||||
|
||||
assert.ok(row);
|
||||
assert.equal(row.youtubeVideoId, 'abc123');
|
||||
assert.equal(row.videoUrl, 'https://www.youtube.com/watch?v=abc123');
|
||||
assert.equal(row.videoTitle, 'Video Name');
|
||||
assert.equal(row.videoThumbnailUrl, 'https://i.ytimg.com/vi/abc123/hqdefault.jpg');
|
||||
assert.equal(row.channelId, 'UCcreator123');
|
||||
assert.equal(row.channelName, 'Creator Name');
|
||||
assert.equal(row.channelUrl, 'https://www.youtube.com/channel/UCcreator123');
|
||||
assert.equal(row.channelThumbnailUrl, 'https://yt3.googleusercontent.com/channel-avatar=s88');
|
||||
assert.equal(row.uploaderId, '@creator');
|
||||
assert.equal(row.uploaderUrl, 'https://www.youtube.com/@creator');
|
||||
assert.equal(row.description, 'Video description');
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
globalThis.fetch = originalFetch;
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('reassignAnimeAnilist clears description when description is explicitly null', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import path from 'node:path';
|
||||
import * as fs from 'node:fs';
|
||||
import { createLogger } from '../../logger';
|
||||
import { MediaGenerator } from '../../media-generator';
|
||||
import type { CoverArtFetcher } from './anilist/cover-art-fetcher';
|
||||
import { getLocalVideoMetadata, guessAnimeVideoMetadata } from './immersion-tracker/metadata';
|
||||
import {
|
||||
@@ -22,6 +23,7 @@ import {
|
||||
type TrackerPreparedStatements,
|
||||
updateVideoMetadataRecord,
|
||||
updateVideoTitleRecord,
|
||||
upsertYoutubeVideoMetadata,
|
||||
} from './immersion-tracker/storage';
|
||||
import {
|
||||
applySessionLifetimeSummary,
|
||||
@@ -153,6 +155,104 @@ import {
|
||||
import type { MergedToken } from '../../types';
|
||||
import { shouldExcludeTokenFromVocabularyPersistence } from './tokenizer/annotation-stage';
|
||||
import { deriveStoredPartOfSpeech } from './tokenizer/part-of-speech';
|
||||
import { probeYoutubeVideoMetadata } from './youtube/metadata-probe';
|
||||
|
||||
const YOUTUBE_COVER_RETRY_MS = 5 * 60 * 1000;
|
||||
const YOUTUBE_SCREENSHOT_MAX_SECONDS = 120;
|
||||
const YOUTUBE_OEMBED_ENDPOINT = 'https://www.youtube.com/oembed';
|
||||
const YOUTUBE_ID_PATTERN = /^[A-Za-z0-9_-]{6,}$/;
|
||||
|
||||
function isValidYouTubeVideoId(value: string | null): boolean {
|
||||
return Boolean(value && YOUTUBE_ID_PATTERN.test(value));
|
||||
}
|
||||
|
||||
function extractYouTubeVideoId(mediaUrl: string): string | null {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(mediaUrl);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const host = parsed.hostname.toLowerCase();
|
||||
if (
|
||||
host !== 'youtu.be' &&
|
||||
!host.endsWith('.youtu.be') &&
|
||||
!host.endsWith('youtube.com') &&
|
||||
!host.endsWith('youtube-nocookie.com')
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (host === 'youtu.be' || host.endsWith('.youtu.be')) {
|
||||
const pathId = parsed.pathname.split('/').filter(Boolean)[0];
|
||||
return isValidYouTubeVideoId(pathId ?? null) ? (pathId as string) : null;
|
||||
}
|
||||
|
||||
const queryId = parsed.searchParams.get('v') ?? parsed.searchParams.get('vi') ?? null;
|
||||
if (isValidYouTubeVideoId(queryId)) {
|
||||
return queryId;
|
||||
}
|
||||
|
||||
const pathParts = parsed.pathname.split('/').filter(Boolean);
|
||||
for (let i = 0; i < pathParts.length; i += 1) {
|
||||
const current = pathParts[i];
|
||||
const next = pathParts[i + 1];
|
||||
if (!current || !next) continue;
|
||||
if (
|
||||
current.toLowerCase() === 'shorts' ||
|
||||
current.toLowerCase() === 'embed' ||
|
||||
current.toLowerCase() === 'live' ||
|
||||
current.toLowerCase() === 'v'
|
||||
) {
|
||||
const candidate = decodeURIComponent(next);
|
||||
if (isValidYouTubeVideoId(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function buildYouTubeThumbnailUrls(videoId: string): string[] {
|
||||
return [
|
||||
`https://i.ytimg.com/vi/${videoId}/maxresdefault.jpg`,
|
||||
`https://i.ytimg.com/vi/${videoId}/hqdefault.jpg`,
|
||||
`https://i.ytimg.com/vi/${videoId}/sddefault.jpg`,
|
||||
`https://i.ytimg.com/vi/${videoId}/mqdefault.jpg`,
|
||||
`https://i.ytimg.com/vi/${videoId}/0.jpg`,
|
||||
`https://i.ytimg.com/vi/${videoId}/default.jpg`,
|
||||
];
|
||||
}
|
||||
|
||||
async function fetchYouTubeOEmbedThumbnail(mediaUrl: string): Promise<string | null> {
|
||||
try {
|
||||
const response = await fetch(`${YOUTUBE_OEMBED_ENDPOINT}?url=${encodeURIComponent(mediaUrl)}&format=json`);
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
const payload = (await response.json()) as { thumbnail_url?: unknown };
|
||||
const candidate = typeof payload.thumbnail_url === 'string' ? payload.thumbnail_url.trim() : '';
|
||||
return candidate || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadImage(url: string): Promise<Buffer | null> {
|
||||
try {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) return null;
|
||||
const contentType = response.headers.get('content-type');
|
||||
if (contentType && !contentType.toLowerCase().startsWith('image/')) {
|
||||
return null;
|
||||
}
|
||||
return Buffer.from(await response.arrayBuffer());
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export type {
|
||||
AnimeAnilistEntryRow,
|
||||
@@ -212,9 +312,11 @@ export class ImmersionTrackerService {
|
||||
private sessionState: SessionState | null = null;
|
||||
private currentVideoKey = '';
|
||||
private currentMediaPathOrUrl = '';
|
||||
private readonly mediaGenerator = new MediaGenerator();
|
||||
private readonly preparedStatements: TrackerPreparedStatements;
|
||||
private coverArtFetcher: CoverArtFetcher | null = null;
|
||||
private readonly pendingCoverFetches = new Map<number, Promise<boolean>>();
|
||||
private readonly pendingYoutubeMetadataFetches = new Map<number, Promise<void>>();
|
||||
private readonly recordedSubtitleKeys = new Set<string>();
|
||||
private readonly pendingAnimeMetadataUpdates = new Map<number, Promise<void>>();
|
||||
private readonly resolveLegacyVocabularyPos:
|
||||
@@ -647,6 +749,17 @@ export class ImmersionTrackerService {
|
||||
if (existing?.coverBlob) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const row = this.db
|
||||
.prepare('SELECT source_url AS sourceUrl FROM imm_videos WHERE video_id = ?')
|
||||
.get(videoId) as { sourceUrl: string | null } | null;
|
||||
const sourceUrl = row?.sourceUrl?.trim() ?? '';
|
||||
const youtubeVideoId = sourceUrl ? extractYouTubeVideoId(sourceUrl) : null;
|
||||
if (youtubeVideoId) {
|
||||
const youtubePromise = this.ensureYouTubeCoverArt(videoId, sourceUrl, youtubeVideoId);
|
||||
return await youtubePromise;
|
||||
}
|
||||
|
||||
if (!this.coverArtFetcher) {
|
||||
return false;
|
||||
}
|
||||
@@ -677,6 +790,140 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
}
|
||||
|
||||
private ensureYouTubeCoverArt(videoId: number, sourceUrl: string, youtubeVideoId: string): Promise<boolean> {
|
||||
const existing = this.pendingCoverFetches.get(videoId);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const promise = this.captureYouTubeCoverArt(videoId, sourceUrl, youtubeVideoId);
|
||||
this.pendingCoverFetches.set(videoId, promise);
|
||||
promise.finally(() => {
|
||||
this.pendingCoverFetches.delete(videoId);
|
||||
});
|
||||
return promise;
|
||||
}
|
||||
|
||||
private async captureYouTubeCoverArt(
|
||||
videoId: number,
|
||||
sourceUrl: string,
|
||||
youtubeVideoId: string,
|
||||
): Promise<boolean> {
|
||||
if (this.isDestroyed) return false;
|
||||
const existing = await this.getCoverArt(videoId);
|
||||
if (existing?.coverBlob) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
existing?.coverUrl === null &&
|
||||
existing?.anilistId === null &&
|
||||
existing?.coverBlob === null &&
|
||||
Date.now() - existing.fetchedAtMs < YOUTUBE_COVER_RETRY_MS
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let coverBlob: Buffer | null = null;
|
||||
let coverUrl: string | null = null;
|
||||
|
||||
const embedThumbnailUrl = await fetchYouTubeOEmbedThumbnail(sourceUrl);
|
||||
if (embedThumbnailUrl) {
|
||||
const embedBlob = await downloadImage(embedThumbnailUrl);
|
||||
if (embedBlob) {
|
||||
coverBlob = embedBlob;
|
||||
coverUrl = embedThumbnailUrl;
|
||||
}
|
||||
}
|
||||
|
||||
if (!coverBlob) {
|
||||
for (const candidate of buildYouTubeThumbnailUrls(youtubeVideoId)) {
|
||||
const candidateBlob = await downloadImage(candidate);
|
||||
if (!candidateBlob) {
|
||||
continue;
|
||||
}
|
||||
coverBlob = candidateBlob;
|
||||
coverUrl = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!coverBlob) {
|
||||
const durationMs = getVideoDurationMs(this.db, videoId);
|
||||
const maxSeconds = durationMs > 0 ? Math.min(durationMs / 1000, YOUTUBE_SCREENSHOT_MAX_SECONDS) : null;
|
||||
const seekSecond = Math.random() * (maxSeconds ?? YOUTUBE_SCREENSHOT_MAX_SECONDS);
|
||||
try {
|
||||
coverBlob = await this.mediaGenerator.generateScreenshot(
|
||||
sourceUrl,
|
||||
seekSecond,
|
||||
{
|
||||
format: 'jpg',
|
||||
quality: 90,
|
||||
maxWidth: 640,
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
'cover-art: failed to generate YouTube screenshot for videoId=%d: %s',
|
||||
videoId,
|
||||
(error as Error).message,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (coverBlob) {
|
||||
upsertCoverArt(this.db, videoId, {
|
||||
anilistId: existing?.anilistId ?? null,
|
||||
coverUrl,
|
||||
coverBlob,
|
||||
titleRomaji: existing?.titleRomaji ?? null,
|
||||
titleEnglish: existing?.titleEnglish ?? null,
|
||||
episodesTotal: existing?.episodesTotal ?? null,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
const shouldCacheNoMatch =
|
||||
!existing || (existing.coverUrl === null && existing.anilistId === null);
|
||||
if (shouldCacheNoMatch) {
|
||||
upsertCoverArt(this.db, videoId, {
|
||||
anilistId: null,
|
||||
coverUrl: null,
|
||||
coverBlob: null,
|
||||
titleRomaji: existing?.titleRomaji ?? null,
|
||||
titleEnglish: existing?.titleEnglish ?? null,
|
||||
episodesTotal: existing?.episodesTotal ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private captureYoutubeMetadataAsync(videoId: number, sourceUrl: string): void {
|
||||
if (this.pendingYoutubeMetadataFetches.has(videoId)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const pending = (async () => {
|
||||
try {
|
||||
const metadata = await probeYoutubeVideoMetadata(sourceUrl);
|
||||
if (!metadata) {
|
||||
return;
|
||||
}
|
||||
upsertYoutubeVideoMetadata(this.db, videoId, metadata);
|
||||
} catch (error) {
|
||||
this.logger.debug(
|
||||
'youtube metadata capture skipped for videoId=%d: %s',
|
||||
videoId,
|
||||
(error as Error).message,
|
||||
);
|
||||
}
|
||||
})();
|
||||
|
||||
this.pendingYoutubeMetadataFetches.set(videoId, pending);
|
||||
pending.finally(() => {
|
||||
this.pendingYoutubeMetadataFetches.delete(videoId);
|
||||
});
|
||||
}
|
||||
|
||||
handleMediaChange(mediaPath: string | null, mediaTitle: string | null): void {
|
||||
const normalizedPath = normalizeMediaPath(mediaPath);
|
||||
const normalizedTitle = normalizeText(mediaTitle);
|
||||
@@ -721,6 +968,13 @@ export class ImmersionTrackerService {
|
||||
`Starting immersion session for path=${normalizedPath} videoId=${sessionInfo.videoId}`,
|
||||
);
|
||||
this.startSession(sessionInfo.videoId, sessionInfo.startedAtMs);
|
||||
if (sourceType === SOURCE_TYPE_REMOTE) {
|
||||
const youtubeVideoId = extractYouTubeVideoId(normalizedPath);
|
||||
if (youtubeVideoId) {
|
||||
void this.ensureYouTubeCoverArt(sessionInfo.videoId, normalizedPath, youtubeVideoId);
|
||||
this.captureYoutubeMetadataAsync(sessionInfo.videoId, normalizedPath);
|
||||
}
|
||||
}
|
||||
this.captureAnimeMetadataAsync(sessionInfo.videoId, normalizedPath, normalizedTitle || null);
|
||||
this.captureVideoMetadataAsync(sessionInfo.videoId, sourceType, normalizedPath);
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@ import {
|
||||
} from '../query.js';
|
||||
import {
|
||||
SOURCE_TYPE_LOCAL,
|
||||
SOURCE_TYPE_REMOTE,
|
||||
EVENT_CARD_MINED,
|
||||
EVENT_SUBTITLE_LINE,
|
||||
EVENT_YOMITAN_LOOKUP,
|
||||
@@ -1956,6 +1957,100 @@ test('media library and detail queries read lifetime totals', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('media library and detail queries include joined youtube metadata when present', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
|
||||
const mediaOne = getOrCreateVideoRecord(db, 'yt:https://www.youtube.com/watch?v=abc123', {
|
||||
canonicalTitle: 'Local Fallback Title',
|
||||
sourcePath: null,
|
||||
sourceUrl: 'https://www.youtube.com/watch?v=abc123',
|
||||
sourceType: SOURCE_TYPE_REMOTE,
|
||||
});
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_lifetime_media (
|
||||
video_id,
|
||||
total_sessions,
|
||||
total_active_ms,
|
||||
total_cards,
|
||||
total_lines_seen,
|
||||
total_tokens_seen,
|
||||
completed,
|
||||
first_watched_ms,
|
||||
last_watched_ms,
|
||||
CREATED_DATE,
|
||||
LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
).run(mediaOne, 2, 6_000, 1, 5, 80, 0, 1_000, 9_000, 9_000, 9_000);
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_youtube_videos (
|
||||
video_id,
|
||||
youtube_video_id,
|
||||
video_url,
|
||||
video_title,
|
||||
video_thumbnail_url,
|
||||
channel_id,
|
||||
channel_name,
|
||||
channel_url,
|
||||
channel_thumbnail_url,
|
||||
uploader_id,
|
||||
uploader_url,
|
||||
description,
|
||||
metadata_json,
|
||||
fetched_at_ms,
|
||||
CREATED_DATE,
|
||||
LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
).run(
|
||||
mediaOne,
|
||||
'abc123',
|
||||
'https://www.youtube.com/watch?v=abc123',
|
||||
'Tracked Video Title',
|
||||
'https://i.ytimg.com/vi/abc123/hqdefault.jpg',
|
||||
'UCcreator123',
|
||||
'Creator Name',
|
||||
'https://www.youtube.com/channel/UCcreator123',
|
||||
'https://yt3.googleusercontent.com/channel-avatar=s88',
|
||||
'@creator',
|
||||
'https://www.youtube.com/@creator',
|
||||
'Video description',
|
||||
'{"source":"test"}',
|
||||
10_000,
|
||||
10_000,
|
||||
10_000,
|
||||
);
|
||||
|
||||
const library = getMediaLibrary(db);
|
||||
const detail = getMediaDetail(db, mediaOne);
|
||||
|
||||
assert.equal(library.length, 1);
|
||||
assert.equal(library[0]?.youtubeVideoId, 'abc123');
|
||||
assert.equal(library[0]?.videoTitle, 'Tracked Video Title');
|
||||
assert.equal(library[0]?.channelId, 'UCcreator123');
|
||||
assert.equal(library[0]?.channelName, 'Creator Name');
|
||||
assert.equal(library[0]?.channelUrl, 'https://www.youtube.com/channel/UCcreator123');
|
||||
assert.equal(detail?.youtubeVideoId, 'abc123');
|
||||
assert.equal(detail?.videoUrl, 'https://www.youtube.com/watch?v=abc123');
|
||||
assert.equal(detail?.videoThumbnailUrl, 'https://i.ytimg.com/vi/abc123/hqdefault.jpg');
|
||||
assert.equal(detail?.channelThumbnailUrl, 'https://yt3.googleusercontent.com/channel-avatar=s88');
|
||||
assert.equal(detail?.uploaderId, '@creator');
|
||||
assert.equal(detail?.uploaderUrl, 'https://www.youtube.com/@creator');
|
||||
assert.equal(detail?.description, 'Video description');
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('cover art queries reuse a shared blob across duplicate anime art rows', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
@@ -1817,6 +1817,17 @@ export function getMediaLibrary(db: DatabaseSync): MediaLibraryRow[] {
|
||||
COALESCE(lm.total_cards, 0) AS totalCards,
|
||||
COALESCE(lm.total_tokens_seen, 0) AS totalTokensSeen,
|
||||
COALESCE(lm.last_watched_ms, 0) AS lastWatchedMs,
|
||||
yv.youtube_video_id AS youtubeVideoId,
|
||||
yv.video_url AS videoUrl,
|
||||
yv.video_title AS videoTitle,
|
||||
yv.video_thumbnail_url AS videoThumbnailUrl,
|
||||
yv.channel_id AS channelId,
|
||||
yv.channel_name AS channelName,
|
||||
yv.channel_url AS channelUrl,
|
||||
yv.channel_thumbnail_url AS channelThumbnailUrl,
|
||||
yv.uploader_id AS uploaderId,
|
||||
yv.uploader_url AS uploaderUrl,
|
||||
yv.description AS description,
|
||||
CASE
|
||||
WHEN ma.cover_blob_hash IS NOT NULL OR ma.cover_blob IS NOT NULL THEN 1
|
||||
ELSE 0
|
||||
@@ -1824,6 +1835,7 @@ export function getMediaLibrary(db: DatabaseSync): MediaLibraryRow[] {
|
||||
FROM imm_videos v
|
||||
JOIN imm_lifetime_media lm ON lm.video_id = v.video_id
|
||||
LEFT JOIN imm_media_art ma ON ma.video_id = v.video_id
|
||||
LEFT JOIN imm_youtube_videos yv ON yv.video_id = v.video_id
|
||||
ORDER BY lm.last_watched_ms DESC
|
||||
`,
|
||||
)
|
||||
@@ -1846,9 +1858,21 @@ export function getMediaDetail(db: DatabaseSync, videoId: number): MediaDetailRo
|
||||
COALESCE(lm.total_lines_seen, 0) AS totalLinesSeen,
|
||||
COALESCE(SUM(COALESCE(asm.lookupCount, s.lookup_count, 0)), 0) AS totalLookupCount,
|
||||
COALESCE(SUM(COALESCE(asm.lookupHits, s.lookup_hits, 0)), 0) AS totalLookupHits,
|
||||
COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount
|
||||
COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount,
|
||||
yv.youtube_video_id AS youtubeVideoId,
|
||||
yv.video_url AS videoUrl,
|
||||
yv.video_title AS videoTitle,
|
||||
yv.video_thumbnail_url AS videoThumbnailUrl,
|
||||
yv.channel_id AS channelId,
|
||||
yv.channel_name AS channelName,
|
||||
yv.channel_url AS channelUrl,
|
||||
yv.channel_thumbnail_url AS channelThumbnailUrl,
|
||||
yv.uploader_id AS uploaderId,
|
||||
yv.uploader_url AS uploaderUrl,
|
||||
yv.description AS description
|
||||
FROM imm_videos v
|
||||
JOIN imm_lifetime_media lm ON lm.video_id = v.video_id
|
||||
LEFT JOIN imm_youtube_videos yv ON yv.video_id = v.video_id
|
||||
LEFT JOIN imm_sessions s ON s.video_id = v.video_id
|
||||
LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
|
||||
WHERE v.video_id = ?
|
||||
|
||||
@@ -106,6 +106,7 @@ test('ensureSchema creates immersion core tables', () => {
|
||||
assert.ok(tableNames.has('imm_kanji_line_occurrences'));
|
||||
assert.ok(tableNames.has('imm_rollup_state'));
|
||||
assert.ok(tableNames.has('imm_cover_art_blobs'));
|
||||
assert.ok(tableNames.has('imm_youtube_videos'));
|
||||
|
||||
const videoColumns = new Set(
|
||||
(
|
||||
@@ -146,6 +147,114 @@ test('ensureSchema creates immersion core tables', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('ensureSchema adds youtube metadata table to existing schema version 15 databases', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
db.exec(`
|
||||
CREATE TABLE imm_schema_version (
|
||||
schema_version INTEGER PRIMARY KEY,
|
||||
applied_at_ms INTEGER NOT NULL
|
||||
);
|
||||
INSERT INTO imm_schema_version(schema_version, applied_at_ms) VALUES (15, 1000);
|
||||
|
||||
CREATE TABLE imm_rollup_state(
|
||||
state_key TEXT PRIMARY KEY,
|
||||
state_value INTEGER NOT NULL
|
||||
);
|
||||
INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('last_rollup_sample_ms', 123);
|
||||
|
||||
CREATE TABLE imm_anime(
|
||||
anime_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
normalized_title_key TEXT NOT NULL UNIQUE,
|
||||
canonical_title TEXT NOT NULL,
|
||||
anilist_id INTEGER UNIQUE,
|
||||
title_romaji TEXT,
|
||||
title_english TEXT,
|
||||
title_native TEXT,
|
||||
episodes_total INTEGER,
|
||||
description TEXT,
|
||||
metadata_json TEXT,
|
||||
CREATED_DATE INTEGER,
|
||||
LAST_UPDATE_DATE INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE imm_videos(
|
||||
video_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
video_key TEXT NOT NULL UNIQUE,
|
||||
anime_id INTEGER,
|
||||
canonical_title TEXT NOT NULL,
|
||||
source_type INTEGER NOT NULL,
|
||||
source_path TEXT,
|
||||
source_url TEXT,
|
||||
parsed_basename TEXT,
|
||||
parsed_title TEXT,
|
||||
parsed_season INTEGER,
|
||||
parsed_episode INTEGER,
|
||||
parser_source TEXT,
|
||||
parser_confidence REAL,
|
||||
parse_metadata_json TEXT,
|
||||
watched INTEGER NOT NULL DEFAULT 0,
|
||||
duration_ms INTEGER NOT NULL CHECK(duration_ms>=0),
|
||||
file_size_bytes INTEGER CHECK(file_size_bytes>=0),
|
||||
codec_id INTEGER, container_id INTEGER,
|
||||
width_px INTEGER, height_px INTEGER, fps_x100 INTEGER,
|
||||
bitrate_kbps INTEGER, audio_codec_id INTEGER,
|
||||
hash_sha256 TEXT, screenshot_path TEXT,
|
||||
metadata_json TEXT,
|
||||
CREATED_DATE INTEGER,
|
||||
LAST_UPDATE_DATE INTEGER,
|
||||
FOREIGN KEY(anime_id) REFERENCES imm_anime(anime_id) ON DELETE SET NULL
|
||||
);
|
||||
`);
|
||||
|
||||
ensureSchema(db);
|
||||
|
||||
const tables = new Set(
|
||||
(
|
||||
db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'imm_%'`).all() as Array<{
|
||||
name: string;
|
||||
}>
|
||||
).map((row) => row.name),
|
||||
);
|
||||
assert.ok(tables.has('imm_youtube_videos'));
|
||||
|
||||
const columns = new Set(
|
||||
(
|
||||
db.prepare('PRAGMA table_info(imm_youtube_videos)').all() as Array<{
|
||||
name: string;
|
||||
}>
|
||||
).map((row) => row.name),
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
columns,
|
||||
new Set([
|
||||
'video_id',
|
||||
'youtube_video_id',
|
||||
'video_url',
|
||||
'video_title',
|
||||
'video_thumbnail_url',
|
||||
'channel_id',
|
||||
'channel_name',
|
||||
'channel_url',
|
||||
'channel_thumbnail_url',
|
||||
'uploader_id',
|
||||
'uploader_url',
|
||||
'description',
|
||||
'metadata_json',
|
||||
'fetched_at_ms',
|
||||
'CREATED_DATE',
|
||||
'LAST_UPDATE_DATE',
|
||||
]),
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('ensureSchema creates large-history performance indexes', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto';
|
||||
import { parseMediaInfo } from '../../../jimaku/utils';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { SCHEMA_VERSION } from './types';
|
||||
import type { QueuedWrite, VideoMetadata } from './types';
|
||||
import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types';
|
||||
|
||||
export interface TrackerPreparedStatements {
|
||||
telemetryInsertStmt: ReturnType<DatabaseSync['prepare']>;
|
||||
@@ -743,6 +743,27 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
FOREIGN KEY(video_id) REFERENCES imm_videos(video_id) ON DELETE CASCADE
|
||||
);
|
||||
`);
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS imm_youtube_videos(
|
||||
video_id INTEGER PRIMARY KEY,
|
||||
youtube_video_id TEXT NOT NULL,
|
||||
video_url TEXT NOT NULL,
|
||||
video_title TEXT,
|
||||
video_thumbnail_url TEXT,
|
||||
channel_id TEXT,
|
||||
channel_name TEXT,
|
||||
channel_url TEXT,
|
||||
channel_thumbnail_url TEXT,
|
||||
uploader_id TEXT,
|
||||
uploader_url TEXT,
|
||||
description TEXT,
|
||||
metadata_json TEXT,
|
||||
fetched_at_ms INTEGER NOT NULL,
|
||||
CREATED_DATE INTEGER,
|
||||
LAST_UPDATE_DATE INTEGER,
|
||||
FOREIGN KEY(video_id) REFERENCES imm_videos(video_id) ON DELETE CASCADE
|
||||
);
|
||||
`);
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS imm_cover_art_blobs(
|
||||
blob_hash TEXT PRIMARY KEY,
|
||||
@@ -1134,6 +1155,14 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
CREATE INDEX IF NOT EXISTS idx_media_art_cover_url
|
||||
ON imm_media_art(cover_url)
|
||||
`);
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_youtube_videos_channel_id
|
||||
ON imm_youtube_videos(channel_id)
|
||||
`);
|
||||
db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_youtube_videos_youtube_video_id
|
||||
ON imm_youtube_videos(youtube_video_id)
|
||||
`);
|
||||
|
||||
if (currentVersion?.schema_version && currentVersion.schema_version < SCHEMA_VERSION) {
|
||||
db.exec('DELETE FROM imm_daily_rollups');
|
||||
@@ -1506,3 +1535,65 @@ export function updateVideoTitleRecord(
|
||||
`,
|
||||
).run(canonicalTitle, Date.now(), videoId);
|
||||
}
|
||||
|
||||
export function upsertYoutubeVideoMetadata(
|
||||
db: DatabaseSync,
|
||||
videoId: number,
|
||||
metadata: YoutubeVideoMetadata,
|
||||
): void {
|
||||
const nowMs = Date.now();
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_youtube_videos (
|
||||
video_id,
|
||||
youtube_video_id,
|
||||
video_url,
|
||||
video_title,
|
||||
video_thumbnail_url,
|
||||
channel_id,
|
||||
channel_name,
|
||||
channel_url,
|
||||
channel_thumbnail_url,
|
||||
uploader_id,
|
||||
uploader_url,
|
||||
description,
|
||||
metadata_json,
|
||||
fetched_at_ms,
|
||||
CREATED_DATE,
|
||||
LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(video_id) DO UPDATE SET
|
||||
youtube_video_id = excluded.youtube_video_id,
|
||||
video_url = excluded.video_url,
|
||||
video_title = excluded.video_title,
|
||||
video_thumbnail_url = excluded.video_thumbnail_url,
|
||||
channel_id = excluded.channel_id,
|
||||
channel_name = excluded.channel_name,
|
||||
channel_url = excluded.channel_url,
|
||||
channel_thumbnail_url = excluded.channel_thumbnail_url,
|
||||
uploader_id = excluded.uploader_id,
|
||||
uploader_url = excluded.uploader_url,
|
||||
description = excluded.description,
|
||||
metadata_json = excluded.metadata_json,
|
||||
fetched_at_ms = excluded.fetched_at_ms,
|
||||
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
|
||||
`,
|
||||
).run(
|
||||
videoId,
|
||||
metadata.youtubeVideoId,
|
||||
metadata.videoUrl,
|
||||
metadata.videoTitle ?? null,
|
||||
metadata.videoThumbnailUrl ?? null,
|
||||
metadata.channelId ?? null,
|
||||
metadata.channelName ?? null,
|
||||
metadata.channelUrl ?? null,
|
||||
metadata.channelThumbnailUrl ?? null,
|
||||
metadata.uploaderId ?? null,
|
||||
metadata.uploaderUrl ?? null,
|
||||
metadata.description ?? null,
|
||||
metadata.metadataJson ?? null,
|
||||
nowMs,
|
||||
nowMs,
|
||||
nowMs,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export const SCHEMA_VERSION = 15;
|
||||
export const SCHEMA_VERSION = 16;
|
||||
export const DEFAULT_QUEUE_CAP = 1_000;
|
||||
export const DEFAULT_BATCH_SIZE = 25;
|
||||
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
|
||||
@@ -420,6 +420,17 @@ export interface MediaLibraryRow {
|
||||
totalTokensSeen: number;
|
||||
lastWatchedMs: number;
|
||||
hasCoverArt: number;
|
||||
youtubeVideoId: string | null;
|
||||
videoUrl: string | null;
|
||||
videoTitle: string | null;
|
||||
videoThumbnailUrl: string | null;
|
||||
channelId: string | null;
|
||||
channelName: string | null;
|
||||
channelUrl: string | null;
|
||||
channelThumbnailUrl: string | null;
|
||||
uploaderId: string | null;
|
||||
uploaderUrl: string | null;
|
||||
description: string | null;
|
||||
}
|
||||
|
||||
export interface MediaDetailRow {
|
||||
@@ -434,6 +445,32 @@ export interface MediaDetailRow {
|
||||
totalLookupCount: number;
|
||||
totalLookupHits: number;
|
||||
totalYomitanLookupCount: number;
|
||||
youtubeVideoId: string | null;
|
||||
videoUrl: string | null;
|
||||
videoTitle: string | null;
|
||||
videoThumbnailUrl: string | null;
|
||||
channelId: string | null;
|
||||
channelName: string | null;
|
||||
channelUrl: string | null;
|
||||
channelThumbnailUrl: string | null;
|
||||
uploaderId: string | null;
|
||||
uploaderUrl: string | null;
|
||||
description: string | null;
|
||||
}
|
||||
|
||||
export interface YoutubeVideoMetadata {
|
||||
youtubeVideoId: string;
|
||||
videoUrl: string;
|
||||
videoTitle: string | null;
|
||||
videoThumbnailUrl: string | null;
|
||||
channelId: string | null;
|
||||
channelName: string | null;
|
||||
channelUrl: string | null;
|
||||
channelThumbnailUrl: string | null;
|
||||
uploaderId: string | null;
|
||||
uploaderUrl: string | null;
|
||||
description: string | null;
|
||||
metadataJson: string | null;
|
||||
}
|
||||
|
||||
export interface AnimeLibraryRow {
|
||||
|
||||
103
src/core/services/youtube/metadata-probe.ts
Normal file
103
src/core/services/youtube/metadata-probe.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import type { YoutubeVideoMetadata } from '../immersion-tracker/types';
|
||||
|
||||
type YtDlpThumbnail = {
|
||||
url?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
};
|
||||
|
||||
type YtDlpYoutubeMetadata = {
|
||||
id?: string;
|
||||
title?: string;
|
||||
webpage_url?: string;
|
||||
thumbnail?: string;
|
||||
thumbnails?: YtDlpThumbnail[];
|
||||
channel_id?: string;
|
||||
channel?: string;
|
||||
channel_url?: string;
|
||||
uploader_id?: string;
|
||||
uploader_url?: string;
|
||||
description?: string;
|
||||
};
|
||||
|
||||
function runCapture(command: string, args: string[]): Promise<{ stdout: string; stderr: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const proc = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
proc.stdout.setEncoding('utf8');
|
||||
proc.stderr.setEncoding('utf8');
|
||||
proc.stdout.on('data', (chunk) => {
|
||||
stdout += String(chunk);
|
||||
});
|
||||
proc.stderr.on('data', (chunk) => {
|
||||
stderr += String(chunk);
|
||||
});
|
||||
proc.once('error', reject);
|
||||
proc.once('close', (code) => {
|
||||
if (code === 0) {
|
||||
resolve({ stdout, stderr });
|
||||
return;
|
||||
}
|
||||
reject(new Error(stderr.trim() || `yt-dlp exited with status ${code ?? 'unknown'}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function pickChannelThumbnail(thumbnails: YtDlpThumbnail[] | undefined): string | null {
|
||||
if (!Array.isArray(thumbnails)) return null;
|
||||
for (const thumbnail of thumbnails) {
|
||||
const candidate = thumbnail.url?.trim();
|
||||
if (!candidate) continue;
|
||||
if (candidate.includes('/vi/')) continue;
|
||||
if (
|
||||
typeof thumbnail.width === 'number' &&
|
||||
typeof thumbnail.height === 'number' &&
|
||||
thumbnail.width > 0 &&
|
||||
thumbnail.height > 0
|
||||
) {
|
||||
const ratio = thumbnail.width / thumbnail.height;
|
||||
if (ratio >= 0.8 && ratio <= 1.25) {
|
||||
return candidate;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (candidate.includes('yt3.googleusercontent.com')) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function probeYoutubeVideoMetadata(
|
||||
targetUrl: string,
|
||||
): Promise<YoutubeVideoMetadata | null> {
|
||||
const { stdout } = await runCapture('yt-dlp', [
|
||||
'--dump-single-json',
|
||||
'--no-warnings',
|
||||
'--skip-download',
|
||||
targetUrl,
|
||||
]);
|
||||
const info = JSON.parse(stdout) as YtDlpYoutubeMetadata;
|
||||
const youtubeVideoId = info.id?.trim();
|
||||
const videoUrl = info.webpage_url?.trim() || targetUrl.trim();
|
||||
if (!youtubeVideoId || !videoUrl) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
youtubeVideoId,
|
||||
videoUrl,
|
||||
videoTitle: info.title?.trim() || null,
|
||||
videoThumbnailUrl: info.thumbnail?.trim() || null,
|
||||
channelId: info.channel_id?.trim() || null,
|
||||
channelName: info.channel?.trim() || null,
|
||||
channelUrl: info.channel_url?.trim() || null,
|
||||
channelThumbnailUrl: pickChannelThumbnail(info.thumbnails),
|
||||
uploaderId: info.uploader_id?.trim() || null,
|
||||
uploaderUrl: info.uploader_url?.trim() || null,
|
||||
description: info.description?.trim() || null,
|
||||
metadataJson: JSON.stringify(info),
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user