fix(immersion): special-case youtube media paths in runtime and tracking

This commit is contained in:
2026-03-23 00:36:19 -07:00
parent 3e7615b3bd
commit 2e43d95396
20 changed files with 1481 additions and 56 deletions

View File

@@ -31,11 +31,12 @@ function checkDependencies(args: Args): void {
if (!commandExists('mpv')) missing.push('mpv'); if (!commandExists('mpv')) missing.push('mpv');
if (args.targetKind === 'url' && isYoutubeTarget(args.target) && !commandExists('yt-dlp')) { const isYoutubeUrl = args.targetKind === 'url' && isYoutubeTarget(args.target);
if (args.targetKind === 'url' && !isYoutubeUrl && !commandExists('yt-dlp')) {
missing.push('yt-dlp'); missing.push('yt-dlp');
} }
if (args.targetKind === 'url' && isYoutubeTarget(args.target) && !commandExists('ffmpeg')) { if (args.targetKind === 'url' && !isYoutubeUrl && !commandExists('ffmpeg')) {
missing.push('ffmpeg'); missing.push('ffmpeg');
} }

View File

@@ -1284,6 +1284,40 @@ test('flushTelemetry checkpoints latest playback position on the active session
} }
}); });
test('recordSubtitleLine advances session checkpoint progress when playback position is unavailable', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath });
tracker.handleMediaChange('https://stream.example.com/subtitle-progress.m3u8', 'Subtitle Progress');
tracker.recordSubtitleLine('line one', 170, 185, [], null);
const privateApi = tracker as unknown as {
db: DatabaseSync;
sessionState: { sessionId: number } | null;
flushTelemetry: (force?: boolean) => void;
flushNow: () => void;
};
const sessionId = privateApi.sessionState?.sessionId;
assert.ok(sessionId);
privateApi.flushTelemetry(true);
privateApi.flushNow();
const row = privateApi.db
.prepare('SELECT ended_media_ms FROM imm_sessions WHERE session_id = ?')
.get(sessionId) as { ended_media_ms: number | null } | null;
assert.equal(row?.ended_media_ms, 185_000);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('deleteSession ignores the currently active session and keeps new writes flushable', async () => { test('deleteSession ignores the currently active session and keeps new writes flushable', async () => {
const dbPath = makeDbPath(); const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null; let tracker: ImmersionTrackerService | null = null;
@@ -2412,6 +2446,23 @@ printf '%s\n' '${ytDlpOutput}'
`, `,
) )
.get() as { canonicalTitle: string } | null; .get() as { canonicalTitle: string } | null;
const animeRow = privateApi.db
.prepare(
`
SELECT
a.canonical_title AS canonicalTitle,
v.parsed_title AS parsedTitle,
v.parser_source AS parserSource
FROM imm_videos v
JOIN imm_anime a ON a.anime_id = v.anime_id
WHERE v.video_id = 1
`,
)
.get() as {
canonicalTitle: string;
parsedTitle: string | null;
parserSource: string | null;
} | null;
assert.ok(row); assert.ok(row);
assert.ok(videoRow); assert.ok(videoRow);
@@ -2427,6 +2478,9 @@ printf '%s\n' '${ytDlpOutput}'
assert.equal(row.uploaderUrl, 'https://www.youtube.com/@creator'); assert.equal(row.uploaderUrl, 'https://www.youtube.com/@creator');
assert.equal(row.description, 'Video description'); assert.equal(row.description, 'Video description');
assert.equal(videoRow.canonicalTitle, 'Video Name'); assert.equal(videoRow.canonicalTitle, 'Video Name');
assert.equal(animeRow?.canonicalTitle, 'Creator Name');
assert.equal(animeRow?.parsedTitle, 'Creator Name');
assert.equal(animeRow?.parserSource, 'youtube');
} finally { } finally {
process.env.PATH = originalPath; process.env.PATH = originalPath;
globalThis.fetch = originalFetch; globalThis.fetch = originalFetch;
@@ -2438,6 +2492,419 @@ printf '%s\n' '${ytDlpOutput}'
} }
}); });
test('getMediaLibrary lazily backfills missing youtube metadata for existing rows', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
const originalPath = process.env.PATH;
let fakeBinDir: string | null = null;
try {
fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-yt-dlp-bin-'));
const ytDlpOutput =
'{"id":"backfill123","title":"Backfilled Video Title","webpage_url":"https://www.youtube.com/watch?v=backfill123","thumbnail":"https://i.ytimg.com/vi/backfill123/hqdefault.jpg","channel_id":"UCbackfill123","channel":"Backfill Creator","channel_url":"https://www.youtube.com/channel/UCbackfill123","uploader_id":"@backfill","uploader_url":"https://www.youtube.com/@backfill","description":"Backfilled description","thumbnails":[{"url":"https://i.ytimg.com/vi/backfill123/hqdefault.jpg"},{"url":"https://yt3.googleusercontent.com/backfill-avatar=s88"}]}';
if (process.platform === 'win32') {
const outputPath = path.join(fakeBinDir, 'output.json');
fs.writeFileSync(outputPath, ytDlpOutput, 'utf8');
fs.writeFileSync(
path.join(fakeBinDir, 'yt-dlp.cmd'),
'@echo off\r\ntype "%~dp0output.json"\r\n',
'utf8',
);
} else {
const scriptPath = path.join(fakeBinDir, 'yt-dlp');
fs.writeFileSync(
scriptPath,
`#!/bin/sh
printf '%s\n' '${ytDlpOutput}'
`,
{ mode: 0o755 },
);
}
process.env.PATH = `${fakeBinDir}${path.delimiter}${originalPath ?? ''}`;
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath });
const privateApi = tracker as unknown as { db: DatabaseSync };
const nowMs = Date.now();
privateApi.db
.prepare(
`
INSERT INTO imm_videos (
video_key,
canonical_title,
source_type,
source_path,
source_url,
duration_ms,
file_size_bytes,
codec_id,
container_id,
width_px,
height_px,
fps_x100,
bitrate_kbps,
audio_codec_id,
hash_sha256,
screenshot_path,
metadata_json,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
)
.run(
'remote:https://www.youtube.com/watch?v=backfill123',
'watch?v=backfill123',
2,
null,
'https://www.youtube.com/watch?v=backfill123',
0,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
nowMs,
nowMs,
);
privateApi.db
.prepare(
`
INSERT INTO imm_lifetime_media (
video_id,
total_sessions,
total_active_ms,
total_cards,
total_lines_seen,
total_tokens_seen,
completed,
first_watched_ms,
last_watched_ms,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
)
.run(1, 1, 5_000, 0, 0, 50, 0, nowMs, nowMs, nowMs, nowMs);
const before = await tracker.getMediaLibrary();
assert.equal(before[0]?.channelName ?? null, null);
await waitForCondition(() => {
const row = privateApi.db
.prepare(
`
SELECT
video_title AS videoTitle,
channel_name AS channelName,
channel_thumbnail_url AS channelThumbnailUrl
FROM imm_youtube_videos
WHERE video_id = 1
`,
)
.get() as {
videoTitle: string | null;
channelName: string | null;
channelThumbnailUrl: string | null;
} | null;
return (
row?.videoTitle === 'Backfilled Video Title' &&
row.channelName === 'Backfill Creator' &&
row.channelThumbnailUrl === 'https://yt3.googleusercontent.com/backfill-avatar=s88'
);
}, 5_000);
const after = await tracker.getMediaLibrary();
assert.equal(after[0]?.videoTitle, 'Backfilled Video Title');
assert.equal(after[0]?.channelName, 'Backfill Creator');
assert.equal(
after[0]?.channelThumbnailUrl,
'https://yt3.googleusercontent.com/backfill-avatar=s88',
);
} finally {
process.env.PATH = originalPath;
tracker?.destroy();
cleanupDbPath(dbPath);
if (fakeBinDir) {
fs.rmSync(fakeBinDir, { recursive: true, force: true });
}
}
});
test('getAnimeLibrary lazily relinks youtube rows to channel groupings', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath });
const privateApi = tracker as unknown as { db: DatabaseSync };
const nowMs = Date.now();
privateApi.db.exec(`
INSERT INTO imm_anime (
anime_id,
normalized_title_key,
canonical_title,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES
(1, 'watch v first', 'watch?v first', ${nowMs}, ${nowMs}),
(2, 'watch v second', 'watch?v second', ${nowMs}, ${nowMs});
INSERT INTO imm_videos (
video_id,
anime_id,
video_key,
canonical_title,
parsed_title,
parser_source,
source_type,
source_path,
source_url,
duration_ms,
file_size_bytes,
codec_id,
container_id,
width_px,
height_px,
fps_x100,
bitrate_kbps,
audio_codec_id,
hash_sha256,
screenshot_path,
metadata_json,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES
(
1,
1,
'remote:https://www.youtube.com/watch?v=first',
'watch?v first',
'watch?v first',
'fallback',
2,
NULL,
'https://www.youtube.com/watch?v=first',
0,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
${nowMs},
${nowMs}
),
(
2,
2,
'remote:https://www.youtube.com/watch?v=second',
'watch?v second',
'watch?v second',
'fallback',
2,
NULL,
'https://www.youtube.com/watch?v=second',
0,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
NULL,
${nowMs},
${nowMs}
);
INSERT INTO imm_youtube_videos (
video_id,
youtube_video_id,
video_url,
video_title,
video_thumbnail_url,
channel_id,
channel_name,
channel_url,
channel_thumbnail_url,
uploader_id,
uploader_url,
description,
metadata_json,
fetched_at_ms,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES
(
1,
'first',
'https://www.youtube.com/watch?v=first',
'First Video',
'https://i.ytimg.com/vi/first/hqdefault.jpg',
'UCchannel1',
'Shared Channel',
'https://www.youtube.com/channel/UCchannel1',
'https://yt3.googleusercontent.com/shared=s88',
'@shared',
'https://www.youtube.com/@shared',
NULL,
'{}',
${nowMs},
${nowMs},
${nowMs}
),
(
2,
'second',
'https://www.youtube.com/watch?v=second',
'Second Video',
'https://i.ytimg.com/vi/second/hqdefault.jpg',
'UCchannel1',
'Shared Channel',
'https://www.youtube.com/channel/UCchannel1',
'https://yt3.googleusercontent.com/shared=s88',
'@shared',
'https://www.youtube.com/@shared',
NULL,
'{}',
${nowMs},
${nowMs},
${nowMs}
);
INSERT INTO imm_sessions (
session_id,
session_uuid,
video_id,
started_at_ms,
ended_at_ms,
status,
total_watched_ms,
active_watched_ms,
lines_seen,
tokens_seen,
cards_mined,
lookup_count,
lookup_hits,
yomitan_lookup_count,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES
(
1,
'session-youtube-1',
1,
${nowMs - 70000},
${nowMs - 10000},
2,
65000,
60000,
0,
100,
0,
0,
0,
0,
${nowMs},
${nowMs}
),
(
2,
'session-youtube-2',
2,
${nowMs - 50000},
${nowMs - 5000},
2,
35000,
30000,
0,
50,
0,
0,
0,
0,
${nowMs},
${nowMs}
);
INSERT INTO imm_lifetime_anime (
anime_id,
total_sessions,
total_active_ms,
total_cards,
total_lines_seen,
total_tokens_seen,
episodes_started,
episodes_completed,
first_watched_ms,
last_watched_ms,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES
(1, 1, 60000, 0, 0, 100, 1, 0, ${nowMs}, ${nowMs}, ${nowMs}, ${nowMs}),
(2, 1, 30000, 0, 0, 50, 1, 0, ${nowMs}, ${nowMs}, ${nowMs}, ${nowMs});
INSERT INTO imm_lifetime_media (
video_id,
total_sessions,
total_active_ms,
total_cards,
total_lines_seen,
total_tokens_seen,
completed,
first_watched_ms,
last_watched_ms,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES
(1, 1, 60000, 0, 0, 100, 0, ${nowMs}, ${nowMs}, ${nowMs}, ${nowMs}),
(2, 1, 30000, 0, 0, 50, 0, ${nowMs}, ${nowMs}, ${nowMs}, ${nowMs});
`);
const rows = await tracker.getAnimeLibrary();
const sharedRows = rows.filter((row) => row.canonicalTitle === 'Shared Channel');
assert.equal(sharedRows.length, 1);
assert.equal(sharedRows[0]?.episodeCount, 2);
const relinked = privateApi.db
.prepare(
`
SELECT a.canonical_title AS canonicalTitle, COUNT(*) AS total
FROM imm_videos v
JOIN imm_anime a ON a.anime_id = v.anime_id
GROUP BY a.anime_id, a.canonical_title
ORDER BY total DESC, a.anime_id ASC
`,
)
.all() as Array<{ canonicalTitle: string; total: number }>;
assert.equal(relinked[0]?.canonicalTitle, 'Shared Channel');
assert.equal(relinked[0]?.total, 2);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('reassignAnimeAnilist clears description when description is explicitly null', async () => { test('reassignAnimeAnilist clears description when description is explicitly null', async () => {
const dbPath = makeDbPath(); const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null; let tracker: ImmersionTrackerService | null = null;

View File

@@ -20,6 +20,7 @@ import {
getOrCreateAnimeRecord, getOrCreateAnimeRecord,
getOrCreateVideoRecord, getOrCreateVideoRecord,
linkVideoToAnimeRecord, linkVideoToAnimeRecord,
linkYoutubeVideoToAnimeRecord,
type TrackerPreparedStatements, type TrackerPreparedStatements,
updateVideoMetadataRecord, updateVideoMetadataRecord,
updateVideoTitleRecord, updateVideoTitleRecord,
@@ -161,6 +162,7 @@ const YOUTUBE_COVER_RETRY_MS = 5 * 60 * 1000;
const YOUTUBE_SCREENSHOT_MAX_SECONDS = 120; const YOUTUBE_SCREENSHOT_MAX_SECONDS = 120;
const YOUTUBE_OEMBED_ENDPOINT = 'https://www.youtube.com/oembed'; const YOUTUBE_OEMBED_ENDPOINT = 'https://www.youtube.com/oembed';
const YOUTUBE_ID_PATTERN = /^[A-Za-z0-9_-]{6,}$/; const YOUTUBE_ID_PATTERN = /^[A-Za-z0-9_-]{6,}$/;
const YOUTUBE_METADATA_REFRESH_MS = 24 * 60 * 60 * 1000;
function isValidYouTubeVideoId(value: string | null): boolean { function isValidYouTubeVideoId(value: string | null): boolean {
return Boolean(value && YOUTUBE_ID_PATTERN.test(value)); return Boolean(value && YOUTUBE_ID_PATTERN.test(value));
@@ -535,11 +537,15 @@ export class ImmersionTrackerService {
} }
async getMediaLibrary(): Promise<MediaLibraryRow[]> { async getMediaLibrary(): Promise<MediaLibraryRow[]> {
return getMediaLibrary(this.db); const rows = getMediaLibrary(this.db);
this.backfillYoutubeMetadataForLibrary();
return rows;
} }
async getMediaDetail(videoId: number): Promise<MediaDetailRow | null> { async getMediaDetail(videoId: number): Promise<MediaDetailRow | null> {
return getMediaDetail(this.db, videoId); const detail = getMediaDetail(this.db, videoId);
this.backfillYoutubeMetadataForVideo(videoId);
return detail;
} }
async getMediaSessions(videoId: number, limit = 100): Promise<SessionSummaryQueryRow[]> { async getMediaSessions(videoId: number, limit = 100): Promise<SessionSummaryQueryRow[]> {
@@ -555,10 +561,12 @@ export class ImmersionTrackerService {
} }
async getAnimeLibrary(): Promise<AnimeLibraryRow[]> { async getAnimeLibrary(): Promise<AnimeLibraryRow[]> {
this.relinkYoutubeAnimeLibrary();
return getAnimeLibrary(this.db); return getAnimeLibrary(this.db);
} }
async getAnimeDetail(animeId: number): Promise<AnimeDetailRow | null> { async getAnimeDetail(animeId: number): Promise<AnimeDetailRow | null> {
this.relinkYoutubeAnimeLibrary();
return getAnimeDetail(this.db, animeId); return getAnimeDetail(this.db, animeId);
} }
@@ -909,6 +917,7 @@ export class ImmersionTrackerService {
return; return;
} }
upsertYoutubeVideoMetadata(this.db, videoId, metadata); upsertYoutubeVideoMetadata(this.db, videoId, metadata);
linkYoutubeVideoToAnimeRecord(this.db, videoId, metadata);
if (metadata.videoTitle?.trim()) { if (metadata.videoTitle?.trim()) {
updateVideoTitleRecord(this.db, videoId, metadata.videoTitle.trim()); updateVideoTitleRecord(this.db, videoId, metadata.videoTitle.trim());
} }
@@ -927,6 +936,174 @@ export class ImmersionTrackerService {
}); });
} }
private backfillYoutubeMetadataForLibrary(): void {
const candidate = this.db
.prepare(
`
SELECT
v.video_id AS videoId,
v.source_url AS sourceUrl
FROM imm_videos v
JOIN imm_lifetime_media lm ON lm.video_id = v.video_id
LEFT JOIN imm_youtube_videos yv ON yv.video_id = v.video_id
WHERE
v.source_type = ?
AND v.source_url IS NOT NULL
AND (
LOWER(v.source_url) LIKE 'https://www.youtube.com/%'
OR LOWER(v.source_url) LIKE 'https://youtube.com/%'
OR LOWER(v.source_url) LIKE 'https://m.youtube.com/%'
OR LOWER(v.source_url) LIKE 'https://youtu.be/%'
)
AND (
yv.video_id IS NULL
OR yv.video_title IS NULL
OR yv.channel_name IS NULL
OR yv.channel_thumbnail_url IS NULL
)
AND (
yv.fetched_at_ms IS NULL
OR yv.fetched_at_ms <= ?
)
ORDER BY lm.last_watched_ms DESC, v.video_id DESC
LIMIT 1
`,
)
.get(
SOURCE_TYPE_REMOTE,
Date.now() - YOUTUBE_METADATA_REFRESH_MS,
) as { videoId: number; sourceUrl: string | null } | null;
if (!candidate?.sourceUrl) {
return;
}
this.captureYoutubeMetadataAsync(candidate.videoId, candidate.sourceUrl);
}
private backfillYoutubeMetadataForVideo(videoId: number): void {
const candidate = this.db
.prepare(
`
SELECT
v.source_url AS sourceUrl
FROM imm_videos v
LEFT JOIN imm_youtube_videos yv ON yv.video_id = v.video_id
WHERE
v.video_id = ?
AND v.source_type = ?
AND v.source_url IS NOT NULL
AND (
LOWER(v.source_url) LIKE 'https://www.youtube.com/%'
OR LOWER(v.source_url) LIKE 'https://youtube.com/%'
OR LOWER(v.source_url) LIKE 'https://m.youtube.com/%'
OR LOWER(v.source_url) LIKE 'https://youtu.be/%'
)
AND (
yv.video_id IS NULL
OR yv.video_title IS NULL
OR yv.channel_name IS NULL
OR yv.channel_thumbnail_url IS NULL
)
AND (
yv.fetched_at_ms IS NULL
OR yv.fetched_at_ms <= ?
)
`,
)
.get(
videoId,
SOURCE_TYPE_REMOTE,
Date.now() - YOUTUBE_METADATA_REFRESH_MS,
) as { sourceUrl: string | null } | null;
if (!candidate?.sourceUrl) {
return;
}
this.captureYoutubeMetadataAsync(videoId, candidate.sourceUrl);
}
private relinkYoutubeAnimeLibrary(): void {
const candidates = this.db
.prepare(
`
SELECT
v.video_id AS videoId,
yv.youtube_video_id AS youtubeVideoId,
yv.video_url AS videoUrl,
yv.video_title AS videoTitle,
yv.video_thumbnail_url AS videoThumbnailUrl,
yv.channel_id AS channelId,
yv.channel_name AS channelName,
yv.channel_url AS channelUrl,
yv.channel_thumbnail_url AS channelThumbnailUrl,
yv.uploader_id AS uploaderId,
yv.uploader_url AS uploaderUrl,
yv.description AS description,
yv.metadata_json AS metadataJson
FROM imm_videos v
JOIN imm_youtube_videos yv ON yv.video_id = v.video_id
LEFT JOIN imm_anime a ON a.anime_id = v.anime_id
LEFT JOIN imm_lifetime_media lm ON lm.video_id = v.video_id
WHERE
v.source_type = ?
AND v.source_url IS NOT NULL
AND (
LOWER(v.source_url) LIKE 'https://www.youtube.com/%'
OR LOWER(v.source_url) LIKE 'https://youtube.com/%'
OR LOWER(v.source_url) LIKE 'https://m.youtube.com/%'
OR LOWER(v.source_url) LIKE 'https://youtu.be/%'
)
AND yv.channel_name IS NOT NULL
AND (
v.anime_id IS NULL
OR a.metadata_json IS NULL
OR a.metadata_json NOT LIKE '%"source":"youtube-channel"%'
OR a.canonical_title IS NULL
OR TRIM(a.canonical_title) != TRIM(yv.channel_name)
)
ORDER BY lm.last_watched_ms DESC, v.video_id DESC
`,
)
.all(SOURCE_TYPE_REMOTE) as Array<{
videoId: number;
youtubeVideoId: string | null;
videoUrl: string | null;
videoTitle: string | null;
videoThumbnailUrl: string | null;
channelId: string | null;
channelName: string | null;
channelUrl: string | null;
channelThumbnailUrl: string | null;
uploaderId: string | null;
uploaderUrl: string | null;
description: string | null;
metadataJson: string | null;
}>;
if (candidates.length === 0) {
return;
}
for (const candidate of candidates) {
if (!candidate.youtubeVideoId || !candidate.videoUrl) {
continue;
}
linkYoutubeVideoToAnimeRecord(this.db, candidate.videoId, {
youtubeVideoId: candidate.youtubeVideoId,
videoUrl: candidate.videoUrl,
videoTitle: candidate.videoTitle,
videoThumbnailUrl: candidate.videoThumbnailUrl,
channelId: candidate.channelId,
channelName: candidate.channelName,
channelUrl: candidate.channelUrl,
channelThumbnailUrl: candidate.channelThumbnailUrl,
uploaderId: candidate.uploaderId,
uploaderUrl: candidate.uploaderUrl,
description: candidate.description,
metadataJson: candidate.metadataJson,
});
}
rebuildLifetimeSummaryTables(this.db);
}
handleMediaChange(mediaPath: string | null, mediaTitle: string | null): void { handleMediaChange(mediaPath: string | null, mediaTitle: string | null): void {
const normalizedPath = normalizeMediaPath(mediaPath); const normalizedPath = normalizeMediaPath(mediaPath);
const normalizedTitle = normalizeText(mediaTitle); const normalizedTitle = normalizeText(mediaTitle);
@@ -971,14 +1148,14 @@ export class ImmersionTrackerService {
`Starting immersion session for path=${normalizedPath} videoId=${sessionInfo.videoId}`, `Starting immersion session for path=${normalizedPath} videoId=${sessionInfo.videoId}`,
); );
this.startSession(sessionInfo.videoId, sessionInfo.startedAtMs); this.startSession(sessionInfo.videoId, sessionInfo.startedAtMs);
if (sourceType === SOURCE_TYPE_REMOTE) { const youtubeVideoId =
const youtubeVideoId = extractYouTubeVideoId(normalizedPath); sourceType === SOURCE_TYPE_REMOTE ? extractYouTubeVideoId(normalizedPath) : null;
if (youtubeVideoId) { if (youtubeVideoId) {
void this.ensureYouTubeCoverArt(sessionInfo.videoId, normalizedPath, youtubeVideoId); void this.ensureYouTubeCoverArt(sessionInfo.videoId, normalizedPath, youtubeVideoId);
this.captureYoutubeMetadataAsync(sessionInfo.videoId, normalizedPath); this.captureYoutubeMetadataAsync(sessionInfo.videoId, normalizedPath);
} } else {
this.captureAnimeMetadataAsync(sessionInfo.videoId, normalizedPath, normalizedTitle || null);
} }
this.captureAnimeMetadataAsync(sessionInfo.videoId, normalizedPath, normalizedTitle || null);
this.captureVideoMetadataAsync(sessionInfo.videoId, sourceType, normalizedPath); this.captureVideoMetadataAsync(sessionInfo.videoId, sourceType, normalizedPath);
} }
@@ -1006,6 +1183,7 @@ export class ImmersionTrackerService {
} }
const startMs = secToMs(startSec); const startMs = secToMs(startSec);
const endMs = secToMs(endSec);
const subtitleKey = `${startMs}:${cleaned}`; const subtitleKey = `${startMs}:${cleaned}`;
if (this.recordedSubtitleKeys.has(subtitleKey)) { if (this.recordedSubtitleKeys.has(subtitleKey)) {
return; return;
@@ -1019,6 +1197,9 @@ export class ImmersionTrackerService {
this.sessionState.currentLineIndex += 1; this.sessionState.currentLineIndex += 1;
this.sessionState.linesSeen += 1; this.sessionState.linesSeen += 1;
this.sessionState.tokensSeen += tokenCount; this.sessionState.tokensSeen += tokenCount;
if (this.sessionState.lastMediaMs === null || endMs > this.sessionState.lastMediaMs) {
this.sessionState.lastMediaMs = endMs;
}
this.sessionState.pendingTelemetry = true; this.sessionState.pendingTelemetry = true;
const wordOccurrences = new Map<string, CountedWordOccurrence>(); const wordOccurrences = new Map<string, CountedWordOccurrence>();
@@ -1068,8 +1249,8 @@ export class ImmersionTrackerService {
sessionId: this.sessionState.sessionId, sessionId: this.sessionState.sessionId,
videoId: this.sessionState.videoId, videoId: this.sessionState.videoId,
lineIndex: this.sessionState.currentLineIndex, lineIndex: this.sessionState.currentLineIndex,
segmentStartMs: secToMs(startSec), segmentStartMs: startMs,
segmentEndMs: secToMs(endSec), segmentEndMs: endMs,
text: cleaned, text: cleaned,
secondaryText: secondaryText ?? null, secondaryText: secondaryText ?? null,
wordOccurrences: Array.from(wordOccurrences.values()), wordOccurrences: Array.from(wordOccurrences.values()),

View File

@@ -280,6 +280,78 @@ test('getAnimeEpisodes falls back to the latest subtitle segment end when sessio
} }
}); });
test('getAnimeEpisodes ignores zero-valued session checkpoints and falls back to subtitle progress', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const stmts = createTrackerPreparedStatements(db);
const videoId = getOrCreateVideoRecord(db, 'remote:https://www.youtube.com/watch?v=zero123', {
canonicalTitle: 'Zero Checkpoint Stream',
sourcePath: null,
sourceUrl: 'https://www.youtube.com/watch?v=zero123',
sourceType: SOURCE_TYPE_REMOTE,
});
const animeId = getOrCreateAnimeRecord(db, {
parsedTitle: 'Zero Checkpoint Anime',
canonicalTitle: 'Zero Checkpoint Anime',
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
linkVideoToAnimeRecord(db, videoId, {
animeId,
parsedBasename: 'watch?v=zero123',
parsedTitle: 'Zero Checkpoint Anime',
parsedSeason: 1,
parsedEpisode: 1,
parserSource: 'fallback',
parserConfidence: 1,
parseMetadataJson: '{"episode":1}',
});
db.prepare('UPDATE imm_videos SET duration_ms = ? WHERE video_id = ?').run(600_000, videoId);
const startedAtMs = 1_200_000;
const sessionId = startSessionRecord(db, videoId, startedAtMs).sessionId;
db.prepare(
`
UPDATE imm_sessions
SET
ended_at_ms = ?,
status = 2,
ended_media_ms = 0,
active_watched_ms = ?,
LAST_UPDATE_DATE = ?
WHERE session_id = ?
`,
).run(startedAtMs + 30_000, 180_000, startedAtMs + 30_000, sessionId);
stmts.eventInsertStmt.run(
sessionId,
startedAtMs + 29_000,
EVENT_SUBTITLE_LINE,
1,
170_000,
185_000,
4,
0,
'{"line":"stream progress"}',
startedAtMs + 29_000,
startedAtMs + 29_000,
);
const [episode] = getAnimeEpisodes(db, animeId);
assert.ok(episode);
assert.equal(episode?.endedMediaMs, 185_000);
assert.equal(episode?.durationMs, 600_000);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getSessionTimeline returns the full session when no limit is provided', () => { test('getSessionTimeline returns the full session when no limit is provided', () => {
const dbPath = makeDbPath(); const dbPath = makeDbPath();
const db = new Database(dbPath); const db = new Database(dbPath);
@@ -2774,3 +2846,200 @@ test('deleteSession rebuilds word and kanji aggregates from retained subtitle li
cleanupDbPath(dbPath); cleanupDbPath(dbPath);
} }
}); });
test('deleteSession removes zero-session media from library and trends', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const animeId = getOrCreateAnimeRecord(db, {
parsedTitle: 'Delete Me Anime',
canonicalTitle: 'Delete Me Anime',
anilistId: 404_404,
titleRomaji: 'Delete Me Anime',
titleEnglish: 'Delete Me Anime',
titleNative: 'Delete Me Anime',
metadataJson: null,
});
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/delete-last-session.mkv', {
canonicalTitle: 'Delete Last Session',
sourcePath: '/tmp/delete-last-session.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
linkVideoToAnimeRecord(db, videoId, {
animeId,
parsedBasename: 'Delete Last Session',
parsedTitle: 'Delete Me Anime',
parsedSeason: 1,
parsedEpisode: 1,
parserSource: 'fallback',
parserConfidence: 1,
parseMetadataJson: '{"episode":1}',
});
const startedAtMs = 9_000_000;
const endedAtMs = startedAtMs + 120_000;
const rollupDay = Math.floor(startedAtMs / 86_400_000);
const rollupMonth = 197001;
const { sessionId } = startSessionRecord(db, videoId, startedAtMs);
db.prepare(
`
UPDATE imm_sessions
SET
ended_at_ms = ?,
ended_media_ms = ?,
total_watched_ms = ?,
active_watched_ms = ?,
lines_seen = ?,
tokens_seen = ?,
cards_mined = ?,
LAST_UPDATE_DATE = ?
WHERE session_id = ?
`,
).run(endedAtMs, 120000, 120000, 120000, 12, 120, 3, endedAtMs, sessionId);
db.prepare(
`
INSERT INTO imm_lifetime_applied_sessions (
session_id,
applied_at_ms,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?)
`,
).run(sessionId, endedAtMs, endedAtMs, endedAtMs);
db.prepare(
`
INSERT INTO imm_lifetime_media (
video_id,
total_sessions,
total_active_ms,
total_cards,
total_lines_seen,
total_tokens_seen,
completed,
first_watched_ms,
last_watched_ms,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(videoId, 1, 120_000, 3, 12, 120, 0, startedAtMs, endedAtMs, endedAtMs, endedAtMs);
db.prepare(
`
INSERT INTO imm_lifetime_anime (
anime_id,
total_sessions,
total_active_ms,
total_cards,
total_lines_seen,
total_tokens_seen,
episodes_started,
episodes_completed,
first_watched_ms,
last_watched_ms,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(animeId, 1, 120000, 3, 12, 120, 1, 0, startedAtMs, endedAtMs, endedAtMs, endedAtMs);
db.prepare(
`
UPDATE imm_lifetime_global
SET
total_sessions = 1,
total_active_ms = 120000,
total_cards = 3,
active_days = 1,
episodes_started = 1,
episodes_completed = 0,
anime_completed = 0,
last_rebuilt_ms = ?,
LAST_UPDATE_DATE = ?
WHERE global_id = 1
`,
).run(endedAtMs, endedAtMs);
db.prepare(
`
INSERT INTO imm_daily_rollups (
rollup_day,
video_id,
total_sessions,
total_active_min,
total_lines_seen,
total_tokens_seen,
total_cards,
cards_per_hour,
tokens_per_min,
lookup_hit_rate,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(rollupDay, videoId, 1, 2, 12, 120, 3, 90, 60, null, endedAtMs, endedAtMs);
db.prepare(
`
INSERT INTO imm_monthly_rollups (
rollup_month,
video_id,
total_sessions,
total_active_min,
total_lines_seen,
total_tokens_seen,
total_cards,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(rollupMonth, videoId, 1, 2, 12, 120, 3, endedAtMs, endedAtMs);
deleteSession(db, sessionId);
assert.deepEqual(getMediaLibrary(db), []);
assert.equal(getMediaDetail(db, videoId) ?? null, null);
assert.deepEqual(getAnimeLibrary(db), []);
assert.equal(getAnimeDetail(db, animeId) ?? null, null);
const trends = getTrendsDashboard(db, 'all', 'day');
assert.deepEqual(trends.activity.watchTime, []);
assert.deepEqual(trends.activity.sessions, []);
const dailyRollups = getDailyRollups(db, 30);
const monthlyRollups = getMonthlyRollups(db, 30);
assert.deepEqual(dailyRollups, []);
assert.deepEqual(monthlyRollups, []);
const lifetimeMediaCount = Number(
(
db.prepare('SELECT COUNT(*) AS total FROM imm_lifetime_media WHERE video_id = ?').get(
videoId,
) as { total: number }
).total,
);
const lifetimeAnimeCount = Number(
(
db.prepare('SELECT COUNT(*) AS total FROM imm_lifetime_anime WHERE anime_id = ?').get(
animeId,
) as { total: number }
).total,
);
const appliedSessionCount = Number(
(
db
.prepare('SELECT COUNT(*) AS total FROM imm_lifetime_applied_sessions WHERE session_id = ?')
.get(sessionId) as { total: number }
).total,
);
assert.equal(lifetimeMediaCount, 0);
assert.equal(lifetimeAnimeCount, 0);
assert.equal(appliedSessionCount, 0);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});

View File

@@ -134,6 +134,49 @@ function resetLifetimeSummaries(db: DatabaseSync, nowMs: number): void {
).run(nowMs, nowMs); ).run(nowMs, nowMs);
} }
function rebuildLifetimeSummariesInternal(
db: DatabaseSync,
rebuiltAtMs: number,
): LifetimeRebuildSummary {
const sessions = db
.prepare(
`
SELECT
session_id AS sessionId,
video_id AS videoId,
started_at_ms AS startedAtMs,
ended_at_ms AS endedAtMs,
total_watched_ms AS totalWatchedMs,
active_watched_ms AS activeWatchedMs,
lines_seen AS linesSeen,
tokens_seen AS tokensSeen,
cards_mined AS cardsMined,
lookup_count AS lookupCount,
lookup_hits AS lookupHits,
yomitan_lookup_count AS yomitanLookupCount,
pause_count AS pauseCount,
pause_ms AS pauseMs,
seek_forward_count AS seekForwardCount,
seek_backward_count AS seekBackwardCount,
media_buffer_events AS mediaBufferEvents
FROM imm_sessions
WHERE ended_at_ms IS NOT NULL
ORDER BY started_at_ms ASC, session_id ASC
`,
)
.all() as RetainedSessionRow[];
resetLifetimeSummaries(db, rebuiltAtMs);
for (const session of sessions) {
applySessionLifetimeSummary(db, toRebuildSessionState(session), session.endedAtMs);
}
return {
appliedSessions: sessions.length,
rebuiltAtMs,
};
}
function toRebuildSessionState(row: RetainedSessionRow): SessionState { function toRebuildSessionState(row: RetainedSessionRow): SessionState {
return { return {
sessionId: row.sessionId, sessionId: row.sessionId,
@@ -482,50 +525,22 @@ export function applySessionLifetimeSummary(
export function rebuildLifetimeSummaries(db: DatabaseSync): LifetimeRebuildSummary { export function rebuildLifetimeSummaries(db: DatabaseSync): LifetimeRebuildSummary {
const rebuiltAtMs = Date.now(); const rebuiltAtMs = Date.now();
const sessions = db
.prepare(
`
SELECT
session_id AS sessionId,
video_id AS videoId,
started_at_ms AS startedAtMs,
ended_at_ms AS endedAtMs,
total_watched_ms AS totalWatchedMs,
active_watched_ms AS activeWatchedMs,
lines_seen AS linesSeen,
tokens_seen AS tokensSeen,
cards_mined AS cardsMined,
lookup_count AS lookupCount,
lookup_hits AS lookupHits,
yomitan_lookup_count AS yomitanLookupCount,
pause_count AS pauseCount,
pause_ms AS pauseMs,
seek_forward_count AS seekForwardCount,
seek_backward_count AS seekBackwardCount,
media_buffer_events AS mediaBufferEvents
FROM imm_sessions
WHERE ended_at_ms IS NOT NULL
ORDER BY started_at_ms ASC, session_id ASC
`,
)
.all() as RetainedSessionRow[];
db.exec('BEGIN'); db.exec('BEGIN');
try { try {
resetLifetimeSummaries(db, rebuiltAtMs); const summary = rebuildLifetimeSummariesInTransaction(db, rebuiltAtMs);
for (const session of sessions) {
applySessionLifetimeSummary(db, toRebuildSessionState(session), session.endedAtMs);
}
db.exec('COMMIT'); db.exec('COMMIT');
return summary;
} catch (error) { } catch (error) {
db.exec('ROLLBACK'); db.exec('ROLLBACK');
throw error; throw error;
} }
}
return { export function rebuildLifetimeSummariesInTransaction(
appliedSessions: sessions.length, db: DatabaseSync,
rebuiltAtMs, rebuiltAtMs = Date.now(),
}; ): LifetimeRebuildSummary {
return rebuildLifetimeSummariesInternal(db, rebuiltAtMs);
} }
export function reconcileStaleActiveSessions(db: DatabaseSync): number { export function reconcileStaleActiveSessions(db: DatabaseSync): number {

View File

@@ -113,6 +113,14 @@ function setLastRollupSampleMs(db: DatabaseSync, sampleMs: number): void {
).run(ROLLUP_STATE_KEY, sampleMs); ).run(ROLLUP_STATE_KEY, sampleMs);
} }
function resetRollups(db: DatabaseSync): void {
db.exec(`
DELETE FROM imm_daily_rollups;
DELETE FROM imm_monthly_rollups;
`);
setLastRollupSampleMs(db, ZERO_ID);
}
function upsertDailyRollupsForGroups( function upsertDailyRollupsForGroups(
db: DatabaseSync, db: DatabaseSync,
groups: Array<{ rollupDay: number; videoId: number }>, groups: Array<{ rollupDay: number; videoId: number }>,
@@ -281,8 +289,20 @@ function dedupeGroups<T extends { rollupDay?: number; rollupMonth?: number; vide
} }
export function runRollupMaintenance(db: DatabaseSync, forceRebuild = false): void { export function runRollupMaintenance(db: DatabaseSync, forceRebuild = false): void {
if (forceRebuild) {
db.exec('BEGIN IMMEDIATE');
try {
rebuildRollupsInTransaction(db);
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
return;
}
const rollupNowMs = Date.now(); const rollupNowMs = Date.now();
const lastRollupSampleMs = forceRebuild ? ZERO_ID : getLastRollupSampleMs(db); const lastRollupSampleMs = getLastRollupSampleMs(db);
const maxSampleRow = db const maxSampleRow = db
.prepare('SELECT MAX(sample_ms) AS maxSampleMs FROM imm_session_telemetry') .prepare('SELECT MAX(sample_ms) AS maxSampleMs FROM imm_session_telemetry')
@@ -324,6 +344,41 @@ export function runRollupMaintenance(db: DatabaseSync, forceRebuild = false): vo
} }
} }
export function rebuildRollupsInTransaction(db: DatabaseSync): void {
const rollupNowMs = Date.now();
const maxSampleRow = db
.prepare('SELECT MAX(sample_ms) AS maxSampleMs FROM imm_session_telemetry')
.get() as unknown as RollupTelemetryResult | null;
resetRollups(db);
if (!maxSampleRow?.maxSampleMs) {
return;
}
const affectedGroups = getAffectedRollupGroups(db, ZERO_ID);
if (affectedGroups.length === 0) {
setLastRollupSampleMs(db, Number(maxSampleRow.maxSampleMs));
return;
}
const dailyGroups = dedupeGroups(
affectedGroups.map((group) => ({
rollupDay: group.rollupDay,
videoId: group.videoId,
})),
);
const monthlyGroups = dedupeGroups(
affectedGroups.map((group) => ({
rollupMonth: group.rollupMonth,
videoId: group.videoId,
})),
);
upsertDailyRollupsForGroups(db, dailyGroups, rollupNowMs);
upsertMonthlyRollupsForGroups(db, monthlyGroups, rollupNowMs);
setLastRollupSampleMs(db, Number(maxSampleRow.maxSampleMs));
}
export function runOptimizeMaintenance(db: DatabaseSync): void { export function runOptimizeMaintenance(db: DatabaseSync): void {
db.exec('PRAGMA optimize'); db.exec('PRAGMA optimize');
} }

View File

@@ -31,6 +31,8 @@ import type {
VocabularyStatsRow, VocabularyStatsRow,
} from './types'; } from './types';
import { buildCoverBlobReference, normalizeCoverBlobBytes } from './storage'; import { buildCoverBlobReference, normalizeCoverBlobBytes } from './storage';
import { rebuildLifetimeSummariesInTransaction } from './lifetime';
import { rebuildRollupsInTransaction } from './maintenance';
import { PartOfSpeech, type MergedToken } from '../../../types'; import { PartOfSpeech, type MergedToken } from '../../../types';
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage'; import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
import { deriveStoredPartOfSpeech } from '../tokenizer/part-of-speech'; import { deriveStoredPartOfSpeech } from '../tokenizer/part-of-speech';
@@ -1746,7 +1748,7 @@ export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisod
v.duration_ms AS durationMs, v.duration_ms AS durationMs,
( (
SELECT COALESCE( SELECT COALESCE(
s_recent.ended_media_ms, NULLIF(s_recent.ended_media_ms, 0),
( (
SELECT MAX(line.segment_end_ms) SELECT MAX(line.segment_end_ms)
FROM imm_subtitle_lines line FROM imm_subtitle_lines line
@@ -2467,6 +2469,8 @@ export function deleteSession(db: DatabaseSync, sessionId: number): void {
try { try {
deleteSessionsByIds(db, sessionIds); deleteSessionsByIds(db, sessionIds);
refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds); refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds);
rebuildLifetimeSummariesInTransaction(db);
rebuildRollupsInTransaction(db);
db.exec('COMMIT'); db.exec('COMMIT');
} catch (error) { } catch (error) {
db.exec('ROLLBACK'); db.exec('ROLLBACK');
@@ -2483,6 +2487,8 @@ export function deleteSessions(db: DatabaseSync, sessionIds: number[]): void {
try { try {
deleteSessionsByIds(db, sessionIds); deleteSessionsByIds(db, sessionIds);
refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds); refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds);
rebuildLifetimeSummariesInTransaction(db);
rebuildRollupsInTransaction(db);
db.exec('COMMIT'); db.exec('COMMIT');
} catch (error) { } catch (error) {
db.exec('ROLLBACK'); db.exec('ROLLBACK');
@@ -2519,6 +2525,8 @@ export function deleteVideo(db: DatabaseSync, videoId: number): void {
cleanupUnusedCoverArtBlobHash(db, artRow?.coverBlobHash ?? null); cleanupUnusedCoverArtBlobHash(db, artRow?.coverBlobHash ?? null);
db.prepare('DELETE FROM imm_videos WHERE video_id = ?').run(videoId); db.prepare('DELETE FROM imm_videos WHERE video_id = ?').run(videoId);
refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds); refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds);
rebuildLifetimeSummariesInTransaction(db);
rebuildRollupsInTransaction(db);
db.exec('COMMIT'); db.exec('COMMIT');
} catch (error) { } catch (error) {
db.exec('ROLLBACK'); db.exec('ROLLBACK');

View File

@@ -15,8 +15,14 @@ import {
getOrCreateAnimeRecord, getOrCreateAnimeRecord,
getOrCreateVideoRecord, getOrCreateVideoRecord,
linkVideoToAnimeRecord, linkVideoToAnimeRecord,
linkYoutubeVideoToAnimeRecord,
} from './storage'; } from './storage';
import { EVENT_SUBTITLE_LINE, SESSION_STATUS_ENDED, SOURCE_TYPE_LOCAL } from './types'; import {
EVENT_SUBTITLE_LINE,
SESSION_STATUS_ENDED,
SOURCE_TYPE_LOCAL,
SOURCE_TYPE_REMOTE,
} from './types';
function makeDbPath(): string { function makeDbPath(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-imm-storage-session-')); const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-imm-storage-session-'));
@@ -817,6 +823,123 @@ test('anime rows are reused by normalized parsed title and upgraded with AniList
} }
}); });
test('youtube videos can be regrouped under a shared channel anime identity', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const firstVideoId = getOrCreateVideoRecord(
db,
'remote:https://www.youtube.com/watch?v=video-1',
{
canonicalTitle: 'watch?v video-1',
sourcePath: null,
sourceUrl: 'https://www.youtube.com/watch?v=video-1',
sourceType: SOURCE_TYPE_REMOTE,
},
);
const secondVideoId = getOrCreateVideoRecord(
db,
'remote:https://www.youtube.com/watch?v=video-2',
{
canonicalTitle: 'watch?v video-2',
sourcePath: null,
sourceUrl: 'https://www.youtube.com/watch?v=video-2',
sourceType: SOURCE_TYPE_REMOTE,
},
);
const firstAnimeId = getOrCreateAnimeRecord(db, {
parsedTitle: 'watch?v video-1',
canonicalTitle: 'watch?v video-1',
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
linkVideoToAnimeRecord(db, firstVideoId, {
animeId: firstAnimeId,
parsedBasename: null,
parsedTitle: 'watch?v video-1',
parsedSeason: null,
parsedEpisode: null,
parserSource: 'fallback',
parserConfidence: 0.2,
parseMetadataJson: '{"source":"fallback"}',
});
const secondAnimeId = getOrCreateAnimeRecord(db, {
parsedTitle: 'watch?v video-2',
canonicalTitle: 'watch?v video-2',
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
linkVideoToAnimeRecord(db, secondVideoId, {
animeId: secondAnimeId,
parsedBasename: null,
parsedTitle: 'watch?v video-2',
parsedSeason: null,
parsedEpisode: null,
parserSource: 'fallback',
parserConfidence: 0.2,
parseMetadataJson: '{"source":"fallback"}',
});
linkYoutubeVideoToAnimeRecord(db, firstVideoId, {
youtubeVideoId: 'video-1',
videoUrl: 'https://www.youtube.com/watch?v=video-1',
videoTitle: 'Video One',
videoThumbnailUrl: 'https://i.ytimg.com/vi/video-1/hqdefault.jpg',
channelId: 'UC123',
channelName: 'Channel Name',
channelUrl: 'https://www.youtube.com/channel/UC123',
channelThumbnailUrl: 'https://yt3.googleusercontent.com/channel-123=s176-c-k-c0x00ffffff-no-rj',
uploaderId: '@channelname',
uploaderUrl: 'https://www.youtube.com/@channelname',
description: null,
metadataJson: '{"id":"video-1"}',
});
linkYoutubeVideoToAnimeRecord(db, secondVideoId, {
youtubeVideoId: 'video-2',
videoUrl: 'https://www.youtube.com/watch?v=video-2',
videoTitle: 'Video Two',
videoThumbnailUrl: 'https://i.ytimg.com/vi/video-2/hqdefault.jpg',
channelId: 'UC123',
channelName: 'Channel Name',
channelUrl: 'https://www.youtube.com/channel/UC123',
channelThumbnailUrl: 'https://yt3.googleusercontent.com/channel-123=s176-c-k-c0x00ffffff-no-rj',
uploaderId: '@channelname',
uploaderUrl: 'https://www.youtube.com/@channelname',
description: null,
metadataJson: '{"id":"video-2"}',
});
const animeRows = db.prepare('SELECT anime_id, canonical_title FROM imm_anime').all() as Array<{
anime_id: number;
canonical_title: string;
}>;
const videoRows = db
.prepare('SELECT video_id, anime_id, parsed_title FROM imm_videos ORDER BY video_id ASC')
.all() as Array<{ video_id: number; anime_id: number | null; parsed_title: string | null }>;
const channelAnimeRows = animeRows.filter((row) => row.canonical_title === 'Channel Name');
assert.equal(channelAnimeRows.length, 1);
assert.equal(videoRows[0]?.anime_id, channelAnimeRows[0]?.anime_id);
assert.equal(videoRows[1]?.anime_id, channelAnimeRows[0]?.anime_id);
assert.equal(videoRows[0]?.parsed_title, 'Channel Name');
assert.equal(videoRows[1]?.parsed_title, 'Channel Name');
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('start/finalize session updates ended_at and status', () => { test('start/finalize session updates ended_at and status', () => {
const dbPath = makeDbPath(); const dbPath = makeDbPath();
const db = new Database(dbPath); const db = new Database(dbPath);

View File

@@ -39,6 +39,41 @@ export interface VideoAnimeLinkInput {
parseMetadataJson: string | null; parseMetadataJson: string | null;
} }
function buildYoutubeChannelAnimeIdentity(metadata: YoutubeVideoMetadata): {
parsedTitle: string;
canonicalTitle: string;
metadataJson: string;
} | null {
const channelId = metadata.channelId?.trim() || null;
const channelUrl = metadata.channelUrl?.trim() || null;
const channelName = metadata.channelName?.trim() || null;
const uploaderId = metadata.uploaderId?.trim() || null;
const videoTitle = metadata.videoTitle?.trim() || null;
const parsedTitle = channelId
? `youtube-channel:${channelId}`
: channelUrl
? `youtube-channel-url:${channelUrl}`
: channelName
? `youtube-channel-name:${channelName}`
: null;
if (!parsedTitle) {
return null;
}
return {
parsedTitle,
canonicalTitle: channelName || uploaderId || videoTitle || parsedTitle,
metadataJson: JSON.stringify({
source: 'youtube-channel',
channelId,
channelUrl,
channelName,
uploaderId,
}),
};
}
const COVER_BLOB_REFERENCE_PREFIX = '__subminer_cover_blob_ref__:'; const COVER_BLOB_REFERENCE_PREFIX = '__subminer_cover_blob_ref__:';
const WAL_JOURNAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024; const WAL_JOURNAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024;
@@ -439,6 +474,38 @@ export function linkVideoToAnimeRecord(
); );
} }
export function linkYoutubeVideoToAnimeRecord(
db: DatabaseSync,
videoId: number,
metadata: YoutubeVideoMetadata,
): number | null {
const identity = buildYoutubeChannelAnimeIdentity(metadata);
if (!identity) {
return null;
}
const animeId = getOrCreateAnimeRecord(db, {
parsedTitle: identity.parsedTitle,
canonicalTitle: identity.canonicalTitle,
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: identity.metadataJson,
});
linkVideoToAnimeRecord(db, videoId, {
animeId,
parsedBasename: null,
parsedTitle: identity.canonicalTitle,
parsedSeason: null,
parsedEpisode: null,
parserSource: 'youtube',
parserConfidence: 1,
parseMetadataJson: identity.metadataJson,
});
return animeId;
}
function migrateLegacyAnimeMetadata(db: DatabaseSync): void { function migrateLegacyAnimeMetadata(db: DatabaseSync): void {
addColumnIfMissing(db, 'imm_videos', 'anime_id', 'INTEGER REFERENCES imm_anime(anime_id)'); addColumnIfMissing(db, 'imm_videos', 'anime_id', 'INTEGER REFERENCES imm_anime(anime_id)');
addColumnIfMissing(db, 'imm_videos', 'parsed_basename', 'TEXT'); addColumnIfMissing(db, 'imm_videos', 'parsed_basename', 'TEXT');

View File

@@ -412,6 +412,7 @@ import { handleCharacterDictionaryAutoSyncComplete } from './main/runtime/charac
import { notifyCharacterDictionaryAutoSyncStatus } from './main/runtime/character-dictionary-auto-sync-notifications'; import { notifyCharacterDictionaryAutoSyncStatus } from './main/runtime/character-dictionary-auto-sync-notifications';
import { createCurrentMediaTokenizationGate } from './main/runtime/current-media-tokenization-gate'; import { createCurrentMediaTokenizationGate } from './main/runtime/current-media-tokenization-gate';
import { createStartupOsdSequencer } from './main/runtime/startup-osd-sequencer'; import { createStartupOsdSequencer } from './main/runtime/startup-osd-sequencer';
import { isYoutubePlaybackActive } from './main/runtime/youtube-playback';
import { createYomitanProfilePolicy } from './main/runtime/yomitan-profile-policy'; import { createYomitanProfilePolicy } from './main/runtime/yomitan-profile-policy';
import { formatSkippedYomitanWriteAction } from './main/runtime/yomitan-read-only-log'; import { formatSkippedYomitanWriteAction } from './main/runtime/yomitan-read-only-log';
import { import {
@@ -1231,6 +1232,13 @@ const startupOsdSequencer = createStartupOsdSequencer({
showOsd: (message) => showMpvOsd(message), showOsd: (message) => showMpvOsd(message),
}); });
function isYoutubePlaybackActiveNow(): boolean {
return isYoutubePlaybackActive(
appState.currentMediaPath,
appState.mpvClient?.currentVideoPath ?? null,
);
}
function maybeSignalPluginAutoplayReady( function maybeSignalPluginAutoplayReady(
payload: SubtitleData, payload: SubtitleData,
options?: { forceWhilePaused?: boolean }, options?: { forceWhilePaused?: boolean },
@@ -1741,7 +1749,10 @@ const characterDictionaryAutoSyncRuntime = createCharacterDictionaryAutoSyncRunt
getConfig: () => { getConfig: () => {
const config = getResolvedConfig().anilist.characterDictionary; const config = getResolvedConfig().anilist.characterDictionary;
return { return {
enabled: config.enabled && yomitanProfilePolicy.isCharacterDictionaryEnabled(), enabled:
config.enabled &&
yomitanProfilePolicy.isCharacterDictionaryEnabled() &&
!isYoutubePlaybackActiveNow(),
maxLoaded: config.maxLoaded, maxLoaded: config.maxLoaded,
profileScope: config.profileScope, profileScope: config.profileScope,
}; };
@@ -3518,7 +3529,7 @@ const {
); );
}, },
scheduleCharacterDictionarySync: () => { scheduleCharacterDictionarySync: () => {
if (!yomitanProfilePolicy.isCharacterDictionaryEnabled()) { if (!yomitanProfilePolicy.isCharacterDictionaryEnabled() || isYoutubePlaybackActiveNow()) {
return; return;
} }
characterDictionaryAutoSyncRuntime.scheduleSync(); characterDictionaryAutoSyncRuntime.scheduleSync();
@@ -3613,7 +3624,8 @@ const {
), ),
getCharacterDictionaryEnabled: () => getCharacterDictionaryEnabled: () =>
getResolvedConfig().anilist.characterDictionary.enabled && getResolvedConfig().anilist.characterDictionary.enabled &&
yomitanProfilePolicy.isCharacterDictionaryEnabled(), yomitanProfilePolicy.isCharacterDictionaryEnabled() &&
!isYoutubePlaybackActiveNow(),
getNameMatchEnabled: () => getResolvedConfig().subtitleStyle.nameMatchEnabled, getNameMatchEnabled: () => getResolvedConfig().subtitleStyle.nameMatchEnabled,
getFrequencyDictionaryEnabled: () => getFrequencyDictionaryEnabled: () =>
getRuntimeBooleanOption( getRuntimeBooleanOption(

View File

@@ -68,3 +68,32 @@ test('ensureAnilistMediaGuess memoizes in-flight guess promise', async () => {
}); });
assert.equal(state.mediaGuessPromise, null); assert.equal(state.mediaGuessPromise, null);
}); });
test('ensureAnilistMediaGuess skips youtube playback urls', async () => {
let state: AnilistMediaGuessRuntimeState = {
mediaKey: 'https://www.youtube.com/watch?v=abc123',
mediaDurationSec: null,
mediaGuess: null,
mediaGuessPromise: null,
lastDurationProbeAtMs: 0,
};
let calls = 0;
const ensureGuess = createEnsureAnilistMediaGuessHandler({
getState: () => state,
setState: (next) => {
state = next;
},
resolveMediaPathForJimaku: (value) => value,
getCurrentMediaPath: () => 'https://www.youtube.com/watch?v=abc123',
getCurrentMediaTitle: () => 'Video',
guessAnilistMediaInfo: async () => {
calls += 1;
return { title: 'Show', season: null, episode: 1, source: 'guessit' };
},
});
const guess = await ensureGuess('https://www.youtube.com/watch?v=abc123');
assert.equal(guess, null);
assert.equal(calls, 0);
assert.equal(state.mediaGuess, null);
});

View File

@@ -1,4 +1,5 @@
import type { AnilistMediaGuess } from '../../core/services/anilist/anilist-updater'; import type { AnilistMediaGuess } from '../../core/services/anilist/anilist-updater';
import { isYoutubeMediaPath } from './youtube-playback';
export type AnilistMediaGuessRuntimeState = { export type AnilistMediaGuessRuntimeState = {
mediaKey: string | null; mediaKey: string | null;
@@ -26,6 +27,9 @@ export function createMaybeProbeAnilistDurationHandler(deps: {
if (state.mediaKey !== mediaKey) { if (state.mediaKey !== mediaKey) {
return null; return null;
} }
if (isYoutubeMediaPath(mediaKey)) {
return null;
}
if (typeof state.mediaDurationSec === 'number' && state.mediaDurationSec > 0) { if (typeof state.mediaDurationSec === 'number' && state.mediaDurationSec > 0) {
return state.mediaDurationSec; return state.mediaDurationSec;
} }
@@ -73,6 +77,9 @@ export function createEnsureAnilistMediaGuessHandler(deps: {
if (state.mediaKey !== mediaKey) { if (state.mediaKey !== mediaKey) {
return null; return null;
} }
if (isYoutubeMediaPath(mediaKey)) {
return null;
}
if (state.mediaGuess) { if (state.mediaGuess) {
return state.mediaGuess; return state.mediaGuess;
} }

View File

@@ -20,6 +20,18 @@ test('get current anilist media key trims and normalizes empty path', () => {
assert.equal(getEmptyKey(), null); assert.equal(getEmptyKey(), null);
}); });
test('get current anilist media key skips youtube playback urls', () => {
const getYoutubeKey = createGetCurrentAnilistMediaKeyHandler({
getCurrentMediaPath: () => ' https://www.youtube.com/watch?v=abc123 ',
});
const getShortYoutubeKey = createGetCurrentAnilistMediaKeyHandler({
getCurrentMediaPath: () => 'https://youtu.be/abc123',
});
assert.equal(getYoutubeKey(), null);
assert.equal(getShortYoutubeKey(), null);
});
test('reset anilist media tracking clears duration/guess/probe state', () => { test('reset anilist media tracking clears duration/guess/probe state', () => {
let mediaKey: string | null = 'old'; let mediaKey: string | null = 'old';
let mediaDurationSec: number | null = 123; let mediaDurationSec: number | null = 123;

View File

@@ -1,11 +1,15 @@
import type { AnilistMediaGuessRuntimeState } from './anilist-media-guess'; import type { AnilistMediaGuessRuntimeState } from './anilist-media-guess';
import { isYoutubeMediaPath } from './youtube-playback';
export function createGetCurrentAnilistMediaKeyHandler(deps: { export function createGetCurrentAnilistMediaKeyHandler(deps: {
getCurrentMediaPath: () => string | null; getCurrentMediaPath: () => string | null;
}) { }) {
return (): string | null => { return (): string | null => {
const mediaPath = deps.getCurrentMediaPath()?.trim(); const mediaPath = deps.getCurrentMediaPath()?.trim();
return mediaPath && mediaPath.length > 0 ? mediaPath : null; if (!mediaPath || mediaPath.length === 0 || isYoutubeMediaPath(mediaPath)) {
return null;
}
return mediaPath;
}; };
} }

View File

@@ -76,3 +76,52 @@ test('createMaybeRunAnilistPostWatchUpdateHandler queues when token missing', as
assert.ok(calls.includes('inflight:true')); assert.ok(calls.includes('inflight:true'));
assert.ok(calls.includes('inflight:false')); assert.ok(calls.includes('inflight:false'));
}); });
test('createMaybeRunAnilistPostWatchUpdateHandler skips youtube playback entirely', async () => {
const calls: string[] = [];
const handler = createMaybeRunAnilistPostWatchUpdateHandler({
getInFlight: () => false,
setInFlight: (value) => calls.push(`inflight:${value}`),
getResolvedConfig: () => ({}),
isAnilistTrackingEnabled: () => true,
getCurrentMediaKey: () => 'https://www.youtube.com/watch?v=abc123',
hasMpvClient: () => true,
getTrackedMediaKey: () => 'https://www.youtube.com/watch?v=abc123',
resetTrackedMedia: () => calls.push('reset'),
getWatchedSeconds: () => 1000,
maybeProbeAnilistDuration: async () => {
calls.push('probe');
return 1000;
},
ensureAnilistMediaGuess: async () => {
calls.push('guess');
return { title: 'Show', season: null, episode: 1 };
},
hasAttemptedUpdateKey: () => false,
processNextAnilistRetryUpdate: async () => {
calls.push('process-retry');
return { ok: true, message: 'noop' };
},
refreshAnilistClientSecretState: async () => {
calls.push('refresh-token');
return 'token';
},
enqueueRetry: () => calls.push('enqueue'),
markRetryFailure: () => calls.push('mark-failure'),
markRetrySuccess: () => calls.push('mark-success'),
refreshRetryQueueState: () => calls.push('refresh'),
updateAnilistPostWatchProgress: async () => {
calls.push('update');
return { status: 'updated', message: 'ok' };
},
rememberAttemptedUpdateKey: () => calls.push('remember'),
showMpvOsd: (message) => calls.push(`osd:${message}`),
logInfo: (message) => calls.push(`info:${message}`),
logWarn: (message) => calls.push(`warn:${message}`),
minWatchSeconds: 600,
minWatchRatio: 0.85,
});
await handler();
assert.deepEqual(calls, []);
});

View File

@@ -1,3 +1,5 @@
import { isYoutubeMediaPath } from './youtube-playback';
type AnilistGuess = { type AnilistGuess = {
title: string; title: string;
episode: number | null; episode: number | null;
@@ -130,6 +132,9 @@ export function createMaybeRunAnilistPostWatchUpdateHandler(deps: {
if (!mediaKey || !deps.hasMpvClient()) { if (!mediaKey || !deps.hasMpvClient()) {
return; return;
} }
if (isYoutubeMediaPath(mediaKey)) {
return;
}
if (deps.getTrackedMediaKey() !== mediaKey) { if (deps.getTrackedMediaKey() !== mediaKey) {
deps.resetTrackedMedia(mediaKey); deps.resetTrackedMedia(mediaKey);
} }

View File

@@ -56,6 +56,57 @@ test('createImmersionTrackerStartupHandler skips when disabled', () => {
assert.equal(tracker, 'unchanged'); assert.equal(tracker, 'unchanged');
}); });
test('createImmersionTrackerStartupHandler skips when env disables session tracking', () => {
const calls: string[] = [];
const originalEnv = process.env.SUBMINER_DISABLE_IMMERSION_TRACKING;
process.env.SUBMINER_DISABLE_IMMERSION_TRACKING = '1';
try {
let tracker: unknown = 'unchanged';
const handler = createImmersionTrackerStartupHandler({
getResolvedConfig: () => {
calls.push('getResolvedConfig');
return makeConfig();
},
getConfiguredDbPath: () => {
calls.push('getConfiguredDbPath');
return '/tmp/subminer.db';
},
createTrackerService: () => {
calls.push('createTrackerService');
return {};
},
setTracker: (nextTracker) => {
tracker = nextTracker;
},
getMpvClient: () => null,
seedTrackerFromCurrentMedia: () => calls.push('seedTracker'),
logInfo: (message) => calls.push(`info:${message}`),
logDebug: (message) => calls.push(`debug:${message}`),
logWarn: (message) => calls.push(`warn:${message}`),
});
handler();
assert.equal(calls.includes('getResolvedConfig'), false);
assert.equal(calls.includes('getConfiguredDbPath'), false);
assert.equal(calls.includes('createTrackerService'), false);
assert.equal(calls.includes('seedTracker'), false);
assert.equal(tracker, 'unchanged');
assert.ok(
calls.includes(
'info:Immersion tracking disabled for this session by SUBMINER_DISABLE_IMMERSION_TRACKING=1.',
),
);
} finally {
if (originalEnv === undefined) {
delete process.env.SUBMINER_DISABLE_IMMERSION_TRACKING;
} else {
process.env.SUBMINER_DISABLE_IMMERSION_TRACKING = originalEnv;
}
}
});
test('createImmersionTrackerStartupHandler creates tracker and auto-connects mpv', () => { test('createImmersionTrackerStartupHandler creates tracker and auto-connects mpv', () => {
const calls: string[] = []; const calls: string[] = [];
const trackerInstance = { kind: 'tracker' }; const trackerInstance = { kind: 'tracker' };

View File

@@ -23,6 +23,8 @@ type ImmersionTrackingConfig = {
type ImmersionTrackerPolicy = Omit<ImmersionTrackingPolicy, 'enabled'>; type ImmersionTrackerPolicy = Omit<ImmersionTrackingPolicy, 'enabled'>;
const DISABLE_IMMERSION_TRACKING_SESSION_ENV = 'SUBMINER_DISABLE_IMMERSION_TRACKING';
type ImmersionTrackerServiceParams = { type ImmersionTrackerServiceParams = {
dbPath: string; dbPath: string;
policy: ImmersionTrackerPolicy; policy: ImmersionTrackerPolicy;
@@ -49,7 +51,16 @@ export type ImmersionTrackerStartupDeps = {
export function createImmersionTrackerStartupHandler( export function createImmersionTrackerStartupHandler(
deps: ImmersionTrackerStartupDeps, deps: ImmersionTrackerStartupDeps,
): () => void { ): () => void {
const isSessionTrackingDisabled = process.env[DISABLE_IMMERSION_TRACKING_SESSION_ENV] === '1';
return () => { return () => {
if (isSessionTrackingDisabled) {
deps.logInfo(
`Immersion tracking disabled for this session by ${DISABLE_IMMERSION_TRACKING_SESSION_ENV}=1.`,
);
return;
}
const config = deps.getResolvedConfig(); const config = deps.getResolvedConfig();
if (config.immersionTracking?.enabled === false) { if (config.immersionTracking?.enabled === false) {
deps.logInfo('Immersion tracking disabled in config'); deps.logInfo('Immersion tracking disabled in config');

View File

@@ -0,0 +1,23 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { isYoutubeMediaPath, isYoutubePlaybackActive } from './youtube-playback';
test('isYoutubeMediaPath detects youtube watch and short urls', () => {
assert.equal(isYoutubeMediaPath('https://www.youtube.com/watch?v=abc123'), true);
assert.equal(isYoutubeMediaPath('https://m.youtube.com/watch?v=abc123'), true);
assert.equal(isYoutubeMediaPath('https://youtu.be/abc123'), true);
assert.equal(isYoutubeMediaPath('https://www.youtube-nocookie.com/embed/abc123'), true);
});
test('isYoutubeMediaPath ignores local files and non-youtube urls', () => {
assert.equal(isYoutubeMediaPath('/tmp/video.mkv'), false);
assert.equal(isYoutubeMediaPath('https://example.com/watch?v=abc123'), false);
assert.equal(isYoutubeMediaPath(' '), false);
assert.equal(isYoutubeMediaPath(null), false);
});
test('isYoutubePlaybackActive checks both current media and mpv video paths', () => {
assert.equal(isYoutubePlaybackActive('/tmp/video.mkv', 'https://youtu.be/abc123'), true);
assert.equal(isYoutubePlaybackActive('https://www.youtube.com/watch?v=abc123', null), true);
assert.equal(isYoutubePlaybackActive('/tmp/video.mkv', '/tmp/video.mkv'), false);
});

View File

@@ -0,0 +1,36 @@
function trimToNull(value: string | null | undefined): string | null {
if (typeof value !== 'string') {
return null;
}
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : null;
}
export function isYoutubeMediaPath(mediaPath: string | null | undefined): boolean {
const normalized = trimToNull(mediaPath);
if (!normalized) {
return false;
}
let parsed: URL;
try {
parsed = new URL(normalized);
} catch {
return false;
}
const host = parsed.hostname.toLowerCase();
return (
host === 'youtu.be' ||
host.endsWith('.youtu.be') ||
host.endsWith('youtube.com') ||
host.endsWith('youtube-nocookie.com')
);
}
export function isYoutubePlaybackActive(
currentMediaPath: string | null | undefined,
currentVideoPath: string | null | undefined,
): boolean {
return isYoutubeMediaPath(currentMediaPath) || isYoutubeMediaPath(currentVideoPath);
}