feat(stats): separate YouTube channels in the Library

- Store YouTube channels as a distinct media kind and migrate legacy entries safely
- Add All Titles, Anime, and YouTube filters with channel-specific stats views
This commit is contained in:
2026-09-18 23:53:26 -07:00
parent 026d495fac
commit d71aab37dc
30 changed files with 475 additions and 47 deletions
@@ -192,6 +192,16 @@ export function createCoverArtFetcher(
return {
async fetchIfMissing(db, videoId, canonicalTitle): Promise<boolean> {
const channel = db
.prepare(
`
SELECT 1 FROM imm_videos v
JOIN imm_anime a ON a.anime_id = v.anime_id
WHERE v.video_id = ? AND a.media_kind != 'anime'
`,
)
.get(videoId);
if (channel) return false;
const existing = getCoverArt(db, videoId);
if (existing?.coverBlob) {
return true;
@@ -27,7 +27,7 @@ function getAnimeTitles(db: DatabaseSync, animeId: number): AnimeTitleRow | null
.prepare(
`SELECT canonical_title, title_romaji, title_english, title_native
FROM imm_anime
WHERE anime_id = ?`,
WHERE anime_id = ? AND media_kind = 'anime'`,
)
.get(animeId) as AnimeTitleRow | null;
}
@@ -77,6 +77,7 @@ export function shouldRecommendAnilistConflict(
conflictAnimeId: number,
options: AnimeConflictRecommendationOptions,
): boolean {
if (!getAnimeTitles(db, targetAnimeId) || !getAnimeTitles(db, conflictAnimeId)) return false;
if (options.survivor === 'target' || options.matchConfidence === 'manual') return false;
if (
!animeSeasonsAreMergeCompatible(
@@ -99,6 +100,8 @@ export function recordAnimeMergeRecommendation(
secondCandidateAnimeId: number,
anilistId: number,
): void {
if (!getAnimeTitles(db, firstCandidateAnimeId) || !getAnimeTitles(db, secondCandidateAnimeId))
return;
const firstAnimeId = Math.min(firstCandidateAnimeId, secondCandidateAnimeId);
const secondAnimeId = Math.max(firstCandidateAnimeId, secondCandidateAnimeId);
const timestamp = toDbTimestamp(nowMs());
@@ -141,6 +144,8 @@ export function getAnimeMergeRecommendations(db: DatabaseSync): AnimeMergeRecomm
second_anime_id AS secondAnimeId
FROM imm_anime_merge_recommendations
WHERE status = 'pending'
AND first_anime_id IN (SELECT anime_id FROM imm_anime WHERE media_kind = 'anime')
AND second_anime_id IN (SELECT anime_id FROM imm_anime WHERE media_kind = 'anime')
ORDER BY recommendation_id ASC`,
)
.all() as Array<{
@@ -134,7 +134,7 @@ function getAnimeRow(db: DatabaseSync, animeId: number): AnimeRow | null {
episodes_total,
description
FROM imm_anime
WHERE anime_id = ?
WHERE anime_id = ? AND media_kind = 'anime'
`,
)
.get(animeId) as AnimeRow | null;
@@ -335,7 +335,8 @@ export function repairLegacySeasonlessAnimeRows(db: DatabaseSync): AnimeSeasonRe
SELECT a.anime_id AS animeId
FROM imm_anime a
JOIN imm_videos v ON v.anime_id = a.anime_id
WHERE v.parsed_title IS NOT NULL
WHERE a.media_kind = 'anime'
AND v.parsed_title IS NOT NULL
AND TRIM(v.parsed_title) != ''
AND v.parsed_season IS NOT NULL
AND v.parsed_season > 0
@@ -372,6 +373,11 @@ export function resolveAnimeAnilistConflict(
anilistId: number,
options: AnimeAnilistConflictOptions = {},
): AnimeSeasonRepairSummary {
if (!getAnimeRow(db, targetAnimeId)) {
const summary = emptySummary();
summary.anilistAssignmentBlocked = true;
return summary;
}
const conflict = db
.prepare(
`
@@ -387,6 +393,11 @@ export function resolveAnimeAnilistConflict(
return emptySummary();
}
if (!getAnimeRow(db, conflict.animeId)) {
const summary = emptySummary();
summary.anilistAssignmentBlocked = true;
return summary;
}
return runInTransaction(db, () => {
const targetRow = getAnimeRow(db, targetAnimeId);
if (
@@ -33,6 +33,7 @@ export function getAnimeLibrary(db: DatabaseSync): AnimeLibraryRow[] {
SELECT
a.anime_id AS animeId,
a.canonical_title AS canonicalTitle,
a.media_kind AS mediaKind,
a.anilist_id AS anilistId,
COALESCE(lm.total_sessions, 0) AS totalSessions,
COALESCE(lm.total_active_ms, 0) AS totalActiveMs,
@@ -63,6 +64,7 @@ export function getAnimeDetail(db: DatabaseSync, animeId: number): AnimeDetailRo
SELECT
a.anime_id AS animeId,
a.canonical_title AS canonicalTitle,
a.media_kind AS mediaKind,
a.anilist_id AS anilistId,
a.title_romaji AS titleRomaji,
a.title_english AS titleEnglish,
+32 -1
View File
@@ -1,3 +1,4 @@
import type { MediaKind } from '../../../shared/media-kind';
import { createHash } from 'node:crypto';
import path from 'node:path';
import { parseMediaInfo } from '../../../jimaku/utils';
@@ -24,6 +25,7 @@ export interface TrackerPreparedStatements {
}
export interface AnimeRecordInput {
mediaKind?: MediaKind;
parsedTitle: string;
canonicalTitle: string;
seasonScope?: number | null;
@@ -600,6 +602,7 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
`
UPDATE imm_anime
SET
media_kind = COALESCE(?, media_kind),
canonical_title = COALESCE(NULLIF(?, ''), canonical_title),
anilist_id = COALESCE(?, anilist_id),
title_romaji = COALESCE(?, title_romaji),
@@ -610,6 +613,7 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
WHERE anime_id = ?
`,
).run(
byAnilistId || byNormalizedTitle ? (input.mediaKind ?? null) : null,
canonicalTitleUpdate,
input.anilistId,
input.titleRomaji,
@@ -627,6 +631,7 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
.prepare(
`
INSERT INTO imm_anime(
media_kind,
normalized_title_key,
canonical_title,
anilist_id,
@@ -636,10 +641,11 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
metadata_json,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
)
.run(
input.mediaKind ?? 'anime',
normalizedTitleKey,
canonicalTitle,
input.anilistId,
@@ -803,6 +809,7 @@ export function linkYoutubeVideoToAnimeRecord(
}
const animeId = getOrCreateAnimeRecord(db, {
mediaKind: 'youtube',
parsedTitle: identity.parsedTitle,
canonicalTitle: identity.canonicalTitle,
anilistId: null,
@@ -875,6 +882,22 @@ function migrateLegacyAnimeMetadata(db: DatabaseSync): void {
}
}
// Older builds can create channel rows with the default anime kind even after
// the schema upgrade. Repair classification on every startup without moving videos.
function classifyYoutubeChannels(db: DatabaseSync): void {
db.exec(`
UPDATE imm_anime
SET media_kind = 'youtube'
WHERE media_kind = 'anime'
AND (
normalized_title_key LIKE 'youtube channel %'
OR CASE WHEN json_valid(metadata_json)
THEN json_extract(metadata_json, '$.source') = 'youtube-channel'
ELSE 0 END
)
`);
}
export function ensureSchema(db: DatabaseSync): void {
db.exec(`
CREATE TABLE IF NOT EXISTS imm_schema_version (
@@ -897,6 +920,7 @@ export function ensureSchema(db: DatabaseSync): void {
.prepare('SELECT schema_version FROM imm_schema_version ORDER BY schema_version DESC LIMIT 1')
.get() as { schema_version: number } | null;
if (currentVersion?.schema_version === SCHEMA_VERSION) {
classifyYoutubeChannels(db);
ensureLexicalDailyRollupTables(db);
ensureLifetimeSummaryTables(db);
ensureStatsExcludedWordsTable(db);
@@ -921,6 +945,13 @@ export function ensureSchema(db: DatabaseSync): void {
LAST_UPDATE_DATE TEXT
);
`);
addColumnIfMissing(
db,
'imm_anime',
'media_kind',
"TEXT NOT NULL DEFAULT 'anime' CHECK(media_kind IN ('anime', 'youtube'))",
);
classifyYoutubeChannels(db);
db.exec(`
CREATE TABLE IF NOT EXISTS imm_videos(
video_id INTEGER PRIMARY KEY AUTOINCREMENT,
+5 -1
View File
@@ -1,4 +1,6 @@
export const SCHEMA_VERSION = 23;
import type { MediaKind } from '../../../shared/media-kind';
export const SCHEMA_VERSION = 24;
export const DEFAULT_QUEUE_CAP = 1_000;
export const DEFAULT_BATCH_SIZE = 25;
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
@@ -518,6 +520,7 @@ export interface YoutubeVideoMetadata {
}
export interface AnimeLibraryRow {
mediaKind: MediaKind;
animeId: number;
canonicalTitle: string;
anilistId: number | null;
@@ -531,6 +534,7 @@ export interface AnimeLibraryRow {
}
export interface AnimeDetailRow {
mediaKind: MediaKind;
animeId: number;
canonicalTitle: string;
anilistId: number | null;
@@ -0,0 +1,188 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { Database, type DatabaseSync } from './sqlite';
import {
ensureSchema,
getOrCreateAnimeRecord,
getOrCreateVideoRecord,
linkYoutubeVideoToAnimeRecord,
} from './storage';
import { getAnimeDetail, getAnimeLibrary } from './query-library';
import { updateAnimeAnilistInfo } from './query-maintenance';
import {
repairLegacySeasonlessAnimeRows,
resolveAnimeAnilistConflict,
} from './anime-season-repair';
import {
getAnimeMergeRecommendations,
recordAnimeMergeRecommendation,
} from './anime-merge-recommendations';
import { createCoverArtFetcher } from '../anilist/cover-art-fetcher';
import { SCHEMA_VERSION, SOURCE_TYPE_REMOTE, type YoutubeVideoMetadata } from './types';
const metadata: YoutubeVideoMetadata = {
youtubeVideoId: 'video1',
videoUrl: 'https://www.youtube.com/watch?v=video1',
videoTitle: 'Video title',
videoThumbnailUrl: null,
channelId: 'UC123',
channelName: 'Channel name',
channelUrl: 'https://www.youtube.com/channel/UC123',
channelThumbnailUrl: null,
uploaderId: null,
uploaderUrl: null,
description: null,
metadataJson: null,
};
function createVideo(db: DatabaseSync, key: string): number {
return getOrCreateVideoRecord(db, key, {
canonicalTitle: 'Video title',
sourcePath: null,
sourceUrl: metadata.videoUrl,
sourceType: SOURCE_TYPE_REMOTE,
});
}
function createAnime(
db: DatabaseSync,
parsedTitle: string,
metadataJson: string | null = null,
): number {
return getOrCreateAnimeRecord(db, {
parsedTitle,
canonicalTitle: parsedTitle,
metadataJson,
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
});
}
test('schema 23 channel migration preserves history and manual assignments and is idempotent', () => {
const db = new Database(':memory:');
try {
ensureSchema(db);
const ids = [
createAnime(db, 'youtube-channel:UC123'),
createAnime(db, 'youtube-channel-url:https://www.youtube.com/@creator'),
createAnime(db, 'youtube-channel-name:Creator'),
createAnime(db, 'Renamed channel', '{ "source": "youtube-channel" }'),
];
const animeId = createAnime(db, 'Anime title', 'legacy non-JSON metadata');
const videoId = createVideo(db, 'manual');
db.prepare(
'UPDATE imm_videos SET anime_id = ?, anime_assignment_locked = 1 WHERE video_id = ?',
).run(animeId, videoId);
db.prepare(
'INSERT INTO imm_lifetime_anime(anime_id, total_active_ms, total_cards) VALUES (?, 123456, 7)',
).run(animeId);
const history = getAnimeLibrary(db);
// Reproduce the previous schema, including its lack of a media kind column.
db.exec(
'ALTER TABLE imm_anime DROP COLUMN media_kind; DELETE FROM imm_schema_version; INSERT INTO imm_schema_version VALUES (23, 0)',
);
ensureSchema(db);
ensureSchema(db);
for (const id of ids) {
const row = db.prepare('SELECT media_kind FROM imm_anime WHERE anime_id = ?').get(id);
assert.ok(row && typeof row === 'object' && 'media_kind' in row);
assert.equal(row.media_kind, 'youtube');
}
assert.deepEqual(getAnimeLibrary(db), history);
assert.equal(linkYoutubeVideoToAnimeRecord(db, videoId, metadata), animeId);
const version = db
.prepare('SELECT MAX(schema_version) AS version FROM imm_schema_version')
.get();
assert.ok(version && typeof version === 'object' && 'version' in version);
assert.equal(version.version, SCHEMA_VERSION);
} finally {
db.close();
}
});
test('channel creation and repeated linking expose youtube in library and detail without losing totals', async () => {
const db = new Database(':memory:');
try {
ensureSchema(db);
const videoId = createVideo(db, 'first');
const channelId = linkYoutubeVideoToAnimeRecord(db, videoId, metadata);
assert.ok(channelId);
const secondVideoId = createVideo(db, 'second');
assert.equal(linkYoutubeVideoToAnimeRecord(db, secondVideoId, metadata), channelId);
db.prepare(
'INSERT INTO imm_lifetime_anime(anime_id, total_active_ms, total_cards) VALUES (?, 123456, 7)',
).run(channelId);
assert.equal(getAnimeLibrary(db)[0]?.mediaKind, 'youtube');
const detail = getAnimeDetail(db, channelId);
assert.equal(detail?.mediaKind, 'youtube');
assert.equal(detail?.episodeCount, 2);
assert.equal(detail?.totalActiveMs, 123456);
assert.equal(detail?.totalCards, 7);
// Even parsed season numbers and a matching anime name must not trigger repairs.
db.prepare('UPDATE imm_videos SET parsed_season = video_id').run();
assert.equal(repairLegacySeasonlessAnimeRows(db).repaired, 0);
const animeId = createAnime(db, 'Channel name');
for (const matchConfidence of ['exact', 'weak', 'manual'] as const) {
assert.equal(
resolveAnimeAnilistConflict(db, channelId, 123, { matchConfidence })
.anilistAssignmentBlocked,
true,
);
}
updateAnimeAnilistInfo(db, videoId, {
anilistId: 123,
titleRomaji: 'Wrong title',
titleEnglish: null,
titleNative: null,
episodesTotal: 12,
});
recordAnimeMergeRecommendation(db, channelId, animeId, 123);
assert.deepEqual(getAnimeMergeRecommendations(db), []);
assert.equal(getAnimeDetail(db, channelId)?.anilistId, null);
const fetcher = createCoverArtFetcher(
{
acquire: async () => {
assert.fail('YouTube must not query AniList');
},
recordResponse: () => {},
},
console,
);
assert.equal(await fetcher.fetchIfMissing(db, videoId, 'Channel name'), false);
} finally {
db.close();
}
});
test('startup reclassifies channels created by an older build after the schema upgrade', () => {
const db = new Database(':memory:');
try {
ensureSchema(db);
// An old build omits media_kind when creating a channel in the upgraded DB.
const channelId = createAnime(db, 'youtube-channel:UCnew');
const animeId = createAnime(db, 'Regular anime');
const videoId = createVideo(db, 'older-build');
db.prepare(
'UPDATE imm_videos SET anime_id = ?, anime_assignment_locked = 1 WHERE video_id = ?',
).run(channelId, videoId);
db.prepare(
'INSERT INTO imm_lifetime_anime(anime_id, total_active_ms, total_cards) VALUES (?, 120000, 5)',
).run(channelId);
assert.equal(getAnimeLibrary(db)[0]?.mediaKind, 'anime');
ensureSchema(db);
const channel = getAnimeLibrary(db)[0];
assert.equal(channel?.animeId, channelId);
assert.equal(channel?.mediaKind, 'youtube');
assert.equal(channel?.totalActiveMs, 120000);
assert.equal(channel?.totalCards, 5);
assert.equal(linkYoutubeVideoToAnimeRecord(db, videoId, metadata), channelId);
const anime = db.prepare('SELECT media_kind FROM imm_anime WHERE anime_id = ?').get(animeId);
assert.ok(anime && typeof anime === 'object' && 'media_kind' in anime);
assert.equal(anime.media_kind, 'anime');
} finally {
db.close();
}
});
@@ -2,6 +2,7 @@ import { selectAll, selectOne, type SqlRow, type SyncDb } from './libsql-driver'
import { insertRow, tableExists, type SyncMergeSummary } from './shared';
const ANIME_COPY_COLUMNS = [
'media_kind',
'normalized_title_key',
'canonical_title',
'anilist_id',
@@ -98,3 +98,37 @@ for (const legacyOccurrences of [false, true]) {
}
});
}
test('sync preserves the YouTube media kind when adding a channel', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-sync-youtube-'));
try {
const localPath = buildDb(dir, 'local.sqlite', {
word: '猫',
seenMs: BASE_MS,
legacyOccurrences: false,
});
const remotePath = buildDb(dir, 'remote.sqlite', {
word: '犬',
seenMs: BASE_MS,
legacyOccurrences: false,
});
const remote = new Database(remotePath);
remote.exec("UPDATE imm_anime SET media_kind = 'youtube'");
remote.close();
mergeSnapshotIntoDb(localPath, remotePath);
const local = new Database(localPath);
try {
const row = local
.prepare(
"SELECT media_kind FROM imm_anime WHERE normalized_title_key = 'key-remote.sqlite'",
)
.get();
assert.ok(row && typeof row === 'object' && 'media_kind' in row);
assert.equal(row.media_kind, 'youtube');
} finally {
local.close();
}
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});