feat(stats): separate YouTube channels in the Library (#253)

This commit is contained in:
2026-09-20 19:57:32 -07:00
committed by GitHub
parent 76e43d7e6d
commit 8097ed27e5
35 changed files with 888 additions and 81 deletions
@@ -192,6 +192,16 @@ export function createCoverArtFetcher(
return {
async fetchIfMissing(db, videoId, canonicalTitle): Promise<boolean> {
const channel = db
.prepare(
`
SELECT 1 FROM imm_videos v
JOIN imm_anime a ON a.anime_id = v.anime_id
WHERE v.video_id = ? AND a.media_kind != 'anime'
`,
)
.get(videoId);
if (channel) return false;
const existing = getCoverArt(db, videoId);
if (existing?.coverBlob) {
return true;
@@ -13,7 +13,11 @@ import {
getOrCreateAnimeRecord,
linkVideoToAnimeRecord,
} from '../storage.js';
import { mergeAnimeRecords, moveVideoToAnime } from '../anime-merge.js';
import {
MEDIA_KIND_MISMATCH_MESSAGE,
mergeAnimeRecords,
moveVideoToAnime,
} from '../anime-merge.js';
import {
dismissAnimeMergeRecommendation,
getAnimeMergeRecommendations,
@@ -940,3 +944,36 @@ test('automatic AniList update onto an entry that already links elsewhere does n
);
});
});
test('merge and move refuse to mix anime entries with YouTube channels', () => {
withDb((db) => {
insertAnime(db, { animeId: 1, key: 'some anime', title: 'Some Anime', anilistId: 555 });
insertAnime(db, { animeId: 2, key: 'youtube channel uc123', title: 'Channel' });
db.prepare("UPDATE imm_anime SET media_kind = 'youtube' WHERE anime_id = 2").run();
insertEpisode(db, { videoId: 10, animeId: 1 });
insertEpisode(db, { videoId: 20, animeId: 2 });
assert.throws(() => mergeAnimeRecords(db, 1, [2]), { message: MEDIA_KIND_MISMATCH_MESSAGE });
assert.throws(() => mergeAnimeRecords(db, 2, [1]), { message: MEDIA_KIND_MISMATCH_MESSAGE });
assert.throws(() => moveVideoToAnime(db, 20, 1), { message: MEDIA_KIND_MISMATCH_MESSAGE });
assert.throws(() => moveVideoToAnime(db, 10, 2), { message: MEDIA_KIND_MISMATCH_MESSAGE });
const rows = db
.prepare(
'SELECT anime_id AS animeId, media_kind AS mediaKind FROM imm_anime ORDER BY anime_id',
)
.all() as Array<{ animeId: number; mediaKind: string }>;
assert.deepEqual(rows, [
{ animeId: 1, mediaKind: 'anime' },
{ animeId: 2, mediaKind: 'youtube' },
]);
assert.equal(
(
db.prepare('SELECT anime_id AS animeId FROM imm_videos WHERE video_id = 20').get() as {
animeId: number;
}
).animeId,
2,
);
});
});
@@ -27,7 +27,7 @@ function getAnimeTitles(db: DatabaseSync, animeId: number): AnimeTitleRow | null
.prepare(
`SELECT canonical_title, title_romaji, title_english, title_native
FROM imm_anime
WHERE anime_id = ?`,
WHERE anime_id = ? AND media_kind = 'anime'`,
)
.get(animeId) as AnimeTitleRow | null;
}
@@ -77,6 +77,7 @@ export function shouldRecommendAnilistConflict(
conflictAnimeId: number,
options: AnimeConflictRecommendationOptions,
): boolean {
if (!getAnimeTitles(db, targetAnimeId) || !getAnimeTitles(db, conflictAnimeId)) return false;
if (options.survivor === 'target' || options.matchConfidence === 'manual') return false;
if (
!animeSeasonsAreMergeCompatible(
@@ -99,6 +100,8 @@ export function recordAnimeMergeRecommendation(
secondCandidateAnimeId: number,
anilistId: number,
): void {
if (!getAnimeTitles(db, firstCandidateAnimeId) || !getAnimeTitles(db, secondCandidateAnimeId))
return;
const firstAnimeId = Math.min(firstCandidateAnimeId, secondCandidateAnimeId);
const secondAnimeId = Math.max(firstCandidateAnimeId, secondCandidateAnimeId);
const timestamp = toDbTimestamp(nowMs());
@@ -141,6 +144,8 @@ export function getAnimeMergeRecommendations(db: DatabaseSync): AnimeMergeRecomm
second_anime_id AS secondAnimeId
FROM imm_anime_merge_recommendations
WHERE status = 'pending'
AND first_anime_id IN (SELECT anime_id FROM imm_anime WHERE media_kind = 'anime')
AND second_anime_id IN (SELECT anime_id FROM imm_anime WHERE media_kind = 'anime')
ORDER BY recommendation_id ASC`,
)
.all() as Array<{
@@ -1,3 +1,4 @@
import type { MediaKind } from '../../../shared/media-kind';
import type { DatabaseSync } from './sqlite';
import { recomputeLifetimeAnimeAggregatesInTransaction } from './lifetime';
import { toDbTimestamp } from './query-shared';
@@ -5,6 +6,8 @@ import { nowMs } from './time';
/** Thrown when a move names an episode or destination entry that is not there. */
export const UNKNOWN_MOVE_TARGET_MESSAGE = 'Unknown episode or target library entry';
/** Thrown when a merge or move would mix an anime entry with a YouTube channel. */
export const MEDIA_KIND_MISMATCH_MESSAGE = 'Anime and YouTube channel entries cannot be combined';
export interface AnimeMergeSummary {
/** Library entry that owns every moved episode once the merge finishes. */
@@ -60,8 +63,11 @@ function readAnimeMetadata(db: DatabaseSync, animeId: number): AnimeMetadataRow
.get(animeId) ?? null) as AnimeMetadataRow | null;
}
function animeExists(db: DatabaseSync, animeId: number): boolean {
return Boolean(db.prepare('SELECT 1 FROM imm_anime WHERE anime_id = ?').get(animeId));
function readMediaKind(db: DatabaseSync, animeId: number): MediaKind | null {
const row = db
.prepare('SELECT media_kind AS mediaKind FROM imm_anime WHERE anime_id = ?')
.get(animeId) as { mediaKind: MediaKind } | undefined;
return row?.mediaKind ?? null;
}
function hasAnimeReferences(db: DatabaseSync, animeId: number): boolean {
@@ -161,7 +167,8 @@ export function mergeAnimeRecordsInTransaction(
sourceAnimeIds: number[],
): AnimeMergeSummary {
const summary = emptyMergeSummary(targetAnimeId);
if (!animeExists(db, targetAnimeId)) {
const targetKind = readMediaKind(db, targetAnimeId);
if (targetKind === null) {
return summary;
}
@@ -195,8 +202,13 @@ export function mergeAnimeRecordsInTransaction(
const dropAnimeStmt = db.prepare('DELETE FROM imm_anime WHERE anime_id = ?');
for (const sourceAnimeId of new Set(sourceAnimeIds)) {
if (sourceAnimeId === targetAnimeId || !animeExists(db, sourceAnimeId)) {
continue;
if (sourceAnimeId === targetAnimeId) continue;
const sourceKind = readMediaKind(db, sourceAnimeId);
if (sourceKind === null) continue;
// A channel folded into an anime would only be recreated on the next
// watch, because title lookups never cross kinds; refuse instead.
if (sourceKind !== targetKind) {
throw new Error(MEDIA_KIND_MISMATCH_MESSAGE);
}
const sourceMetadata = readAnimeMetadata(db, sourceAnimeId);
@@ -257,11 +269,15 @@ export function moveVideoToAnime(
const videoRow = db
.prepare('SELECT anime_id AS animeId FROM imm_videos WHERE video_id = ?')
.get(videoId) as { animeId: number | null } | null;
if (!videoRow || !animeExists(db, targetAnimeId)) {
const targetKind = readMediaKind(db, targetAnimeId);
if (!videoRow || targetKind === null) {
throw new Error(UNKNOWN_MOVE_TARGET_MESSAGE);
}
const previousAnimeId = videoRow.animeId;
if (previousAnimeId !== null && readMediaKind(db, previousAnimeId) !== targetKind) {
throw new Error(MEDIA_KIND_MISMATCH_MESSAGE);
}
if (previousAnimeId === targetAnimeId) {
db.prepare(
'UPDATE imm_videos SET anime_assignment_locked = 1, LAST_UPDATE_DATE = ? WHERE video_id = ?',
@@ -134,7 +134,7 @@ function getAnimeRow(db: DatabaseSync, animeId: number): AnimeRow | null {
episodes_total,
description
FROM imm_anime
WHERE anime_id = ?
WHERE anime_id = ? AND media_kind = 'anime'
`,
)
.get(animeId) as AnimeRow | null;
@@ -335,7 +335,8 @@ export function repairLegacySeasonlessAnimeRows(db: DatabaseSync): AnimeSeasonRe
SELECT a.anime_id AS animeId
FROM imm_anime a
JOIN imm_videos v ON v.anime_id = a.anime_id
WHERE v.parsed_title IS NOT NULL
WHERE a.media_kind = 'anime'
AND v.parsed_title IS NOT NULL
AND TRIM(v.parsed_title) != ''
AND v.parsed_season IS NOT NULL
AND v.parsed_season > 0
@@ -372,6 +373,11 @@ export function resolveAnimeAnilistConflict(
anilistId: number,
options: AnimeAnilistConflictOptions = {},
): AnimeSeasonRepairSummary {
if (!getAnimeRow(db, targetAnimeId)) {
const summary = emptySummary();
summary.anilistAssignmentBlocked = true;
return summary;
}
const conflict = db
.prepare(
`
@@ -387,6 +393,11 @@ export function resolveAnimeAnilistConflict(
return emptySummary();
}
if (!getAnimeRow(db, conflict.animeId)) {
const summary = emptySummary();
summary.anilistAssignmentBlocked = true;
return summary;
}
return runInTransaction(db, () => {
const targetRow = getAnimeRow(db, targetAnimeId);
if (
@@ -33,6 +33,7 @@ export function getAnimeLibrary(db: DatabaseSync): AnimeLibraryRow[] {
SELECT
a.anime_id AS animeId,
a.canonical_title AS canonicalTitle,
a.media_kind AS mediaKind,
a.anilist_id AS anilistId,
COALESCE(lm.total_sessions, 0) AS totalSessions,
COALESCE(lm.total_active_ms, 0) AS totalActiveMs,
@@ -63,6 +64,7 @@ export function getAnimeDetail(db: DatabaseSync, animeId: number): AnimeDetailRo
SELECT
a.anime_id AS animeId,
a.canonical_title AS canonicalTitle,
a.media_kind AS mediaKind,
a.anilist_id AS anilistId,
a.title_romaji AS titleRomaji,
a.title_english AS titleEnglish,
+98 -11
View File
@@ -1,3 +1,4 @@
import type { MediaKind } from '../../../shared/media-kind';
import { createHash } from 'node:crypto';
import path from 'node:path';
import { parseMediaInfo } from '../../../jimaku/utils';
@@ -24,6 +25,7 @@ export interface TrackerPreparedStatements {
}
export interface AnimeRecordInput {
mediaKind?: MediaKind;
parsedTitle: string;
canonicalTitle: string;
seasonScope?: number | null;
@@ -569,6 +571,8 @@ function ensureSubtitleLineEventIndex(db: DatabaseSync): void {
}
export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput): number {
const mediaKind = input.mediaKind ?? 'anime';
const anilistId = mediaKind === 'anime' ? input.anilistId : null;
const seasonScope = normalizeSeasonScope(input.seasonScope);
const identityTitle = buildSeasonScopedAnimeTitle(input.parsedTitle, seasonScope);
const canonicalTitle =
@@ -580,17 +584,23 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
}
const byAnilistId =
input.anilistId !== null
? (db.prepare('SELECT anime_id FROM imm_anime WHERE anilist_id = ?').get(input.anilistId) as {
anilistId !== null
? (db
.prepare("SELECT anime_id FROM imm_anime WHERE anilist_id = ? AND media_kind = 'anime'")
.get(anilistId) as {
anime_id: number;
} | null)
: null;
const byNormalizedTitle = db
.prepare('SELECT anime_id FROM imm_anime WHERE normalized_title_key = ?')
.get(normalizedTitleKey) as { anime_id: number } | null;
.prepare('SELECT anime_id FROM imm_anime WHERE normalized_title_key = ? AND media_kind = ?')
.get(normalizedTitleKey, mediaKind) as { anime_id: number } | null;
const byTitleAlias = db
.prepare('SELECT anime_id FROM imm_anime_title_aliases WHERE normalized_title_key = ?')
.get(normalizedTitleKey) as { anime_id: number } | null;
.prepare(
`SELECT a.anime_id FROM imm_anime_title_aliases AS alias
JOIN imm_anime AS a ON a.anime_id = alias.anime_id
WHERE alias.normalized_title_key = ? AND a.media_kind = ?`,
)
.get(normalizedTitleKey, mediaKind) as { anime_id: number } | null;
const existing = byAnilistId ?? byNormalizedTitle ?? byTitleAlias;
if (existing?.anime_id) {
// An alias remembers an intentionally merged-away spelling. Reusing it
@@ -601,7 +611,7 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
UPDATE imm_anime
SET
canonical_title = COALESCE(NULLIF(?, ''), canonical_title),
anilist_id = COALESCE(?, anilist_id),
anilist_id = CASE WHEN ? = 'youtube' THEN NULL ELSE COALESCE(?, anilist_id) END,
title_romaji = COALESCE(?, title_romaji),
title_english = COALESCE(?, title_english),
title_native = COALESCE(?, title_native),
@@ -611,7 +621,8 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
`,
).run(
canonicalTitleUpdate,
input.anilistId,
mediaKind,
anilistId,
input.titleRomaji,
input.titleEnglish,
input.titleNative,
@@ -627,6 +638,7 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
.prepare(
`
INSERT INTO imm_anime(
media_kind,
normalized_title_key,
canonical_title,
anilist_id,
@@ -636,13 +648,14 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
metadata_json,
CREATED_DATE,
LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
)
.run(
input.mediaKind ?? 'anime',
normalizedTitleKey,
canonicalTitle,
input.anilistId,
anilistId,
input.titleRomaji,
input.titleEnglish,
input.titleNative,
@@ -803,6 +816,7 @@ export function linkYoutubeVideoToAnimeRecord(
}
const animeId = getOrCreateAnimeRecord(db, {
mediaKind: 'youtube',
parsedTitle: identity.parsedTitle,
canonicalTitle: identity.canonicalTitle,
anilistId: null,
@@ -875,6 +889,70 @@ function migrateLegacyAnimeMetadata(db: DatabaseSync): void {
}
}
// SQLite cannot drop a table-level UNIQUE constraint. Rebuild with IDs intact
// and foreign keys disabled so dependent history and manual assignments survive.
function migrateAnimeTitleUniqueness(db: DatabaseSync): void {
const schema = db.prepare("SELECT sql FROM sqlite_master WHERE name = 'imm_anime'").get() as {
sql: string;
};
if (/normalized_title_key TEXT NOT NULL UNIQUE/i.test(schema.sql)) {
const foreignKeys = db.prepare('PRAGMA foreign_keys').get() as { foreign_keys: number };
const sequence = db
.prepare("SELECT seq FROM sqlite_sequence WHERE name = 'imm_anime'")
.get() as { seq: number } | null;
db.exec('PRAGMA foreign_keys = OFF');
try {
db.exec('BEGIN IMMEDIATE');
db.exec(
schema.sql
.replace(
/CREATE TABLE (?:IF NOT EXISTS )?["`]?imm_anime["`]?/i,
'CREATE TABLE imm_anime_new',
)
.replace(
/normalized_title_key TEXT NOT NULL UNIQUE/i,
'normalized_title_key TEXT NOT NULL',
),
);
db.exec(`INSERT INTO imm_anime_new SELECT * FROM imm_anime;
DROP TABLE imm_anime;
ALTER TABLE imm_anime_new RENAME TO imm_anime;`);
if (sequence) {
db.prepare("UPDATE sqlite_sequence SET seq = MAX(seq, ?) WHERE name = 'imm_anime'").run(
sequence.seq,
);
}
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
} finally {
db.exec(`PRAGMA foreign_keys = ${foreignKeys.foreign_keys}`);
}
}
db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_anime_kind_title
ON imm_anime(media_kind, normalized_title_key)`);
}
// Older builds can create channel rows with the default anime kind even after
// the schema upgrade. Repair classification on every startup without moving videos.
function classifyYoutubeChannels(db: DatabaseSync): void {
db.exec(`
UPDATE imm_anime
SET media_kind = 'youtube', anilist_id = NULL
WHERE media_kind = 'anime'
AND NOT EXISTS (SELECT 1 FROM imm_anime AS channel
WHERE channel.media_kind = 'youtube'
AND channel.normalized_title_key = imm_anime.normalized_title_key)
AND (
normalized_title_key LIKE 'youtube channel %'
OR CASE WHEN json_valid(metadata_json)
THEN json_extract(metadata_json, '$.source') = 'youtube-channel'
ELSE 0 END
)
`);
}
export function ensureSchema(db: DatabaseSync): void {
db.exec(`
CREATE TABLE IF NOT EXISTS imm_schema_version (
@@ -897,6 +975,7 @@ export function ensureSchema(db: DatabaseSync): void {
.prepare('SELECT schema_version FROM imm_schema_version ORDER BY schema_version DESC LIMIT 1')
.get() as { schema_version: number } | null;
if (currentVersion?.schema_version === SCHEMA_VERSION) {
classifyYoutubeChannels(db);
ensureLexicalDailyRollupTables(db);
ensureLifetimeSummaryTables(db);
ensureStatsExcludedWordsTable(db);
@@ -908,7 +987,7 @@ export function ensureSchema(db: DatabaseSync): void {
db.exec(`
CREATE TABLE IF NOT EXISTS imm_anime(
anime_id INTEGER PRIMARY KEY AUTOINCREMENT,
normalized_title_key TEXT NOT NULL UNIQUE,
normalized_title_key TEXT NOT NULL,
canonical_title TEXT NOT NULL,
anilist_id INTEGER UNIQUE,
title_romaji TEXT,
@@ -921,6 +1000,12 @@ export function ensureSchema(db: DatabaseSync): void {
LAST_UPDATE_DATE TEXT
);
`);
addColumnIfMissing(
db,
'imm_anime',
'media_kind',
"TEXT NOT NULL DEFAULT 'anime' CHECK(media_kind IN ('anime', 'youtube'))",
);
db.exec(`
CREATE TABLE IF NOT EXISTS imm_videos(
video_id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -1464,6 +1549,8 @@ export function ensureSchema(db: DatabaseSync): void {
);
}
migrateAnimeTitleUniqueness(db);
classifyYoutubeChannels(db);
migrateSessionEventTimestampsToText(db);
ensureLexicalDailyRollupTables(db);
+5 -1
View File
@@ -1,4 +1,6 @@
export const SCHEMA_VERSION = 23;
import type { MediaKind } from '../../../shared/media-kind';
export const SCHEMA_VERSION = 25;
export const DEFAULT_QUEUE_CAP = 1_000;
export const DEFAULT_BATCH_SIZE = 25;
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
@@ -518,6 +520,7 @@ export interface YoutubeVideoMetadata {
}
export interface AnimeLibraryRow {
mediaKind: MediaKind;
animeId: number;
canonicalTitle: string;
anilistId: number | null;
@@ -531,6 +534,7 @@ export interface AnimeLibraryRow {
}
export interface AnimeDetailRow {
mediaKind: MediaKind;
animeId: number;
canonicalTitle: string;
anilistId: number | null;
@@ -0,0 +1,310 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { Database, type DatabaseSync } from './sqlite';
import {
ensureSchema,
getOrCreateAnimeRecord,
getOrCreateVideoRecord,
linkYoutubeVideoToAnimeRecord,
} from './storage';
import { getAnimeDetail, getAnimeLibrary } from './query-library';
import { updateAnimeAnilistInfo } from './query-maintenance';
import {
repairLegacySeasonlessAnimeRows,
resolveAnimeAnilistConflict,
} from './anime-season-repair';
import {
getAnimeMergeRecommendations,
recordAnimeMergeRecommendation,
} from './anime-merge-recommendations';
import { createCoverArtFetcher } from '../anilist/cover-art-fetcher';
import { SCHEMA_VERSION, SOURCE_TYPE_REMOTE, type YoutubeVideoMetadata } from './types';
const metadata: YoutubeVideoMetadata = {
youtubeVideoId: 'video1',
videoUrl: 'https://www.youtube.com/watch?v=video1',
videoTitle: 'Video title',
videoThumbnailUrl: null,
channelId: 'UC123',
channelName: 'Channel name',
channelUrl: 'https://www.youtube.com/channel/UC123',
channelThumbnailUrl: null,
uploaderId: null,
uploaderUrl: null,
description: null,
metadataJson: null,
};
function createVideo(db: DatabaseSync, key: string): number {
return getOrCreateVideoRecord(db, key, {
canonicalTitle: 'Video title',
sourcePath: null,
sourceUrl: metadata.videoUrl,
sourceType: SOURCE_TYPE_REMOTE,
});
}
function createAnime(
db: DatabaseSync,
parsedTitle: string,
metadataJson: string | null = null,
): number {
return getOrCreateAnimeRecord(db, {
parsedTitle,
canonicalTitle: parsedTitle,
metadataJson,
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
});
}
test('schema 23 channel migration preserves history and manual assignments and is idempotent', () => {
const db = new Database(':memory:');
try {
ensureSchema(db);
const ids = [
createAnime(db, 'youtube-channel:UC123'),
createAnime(db, 'youtube-channel-url:https://www.youtube.com/@creator'),
createAnime(db, 'youtube-channel-name:Creator'),
createAnime(db, 'Renamed channel', '{ "source": "youtube-channel" }'),
];
const animeId = createAnime(db, 'Anime title', 'legacy non-JSON metadata');
const videoId = createVideo(db, 'manual');
db.prepare(
'UPDATE imm_videos SET anime_id = ?, anime_assignment_locked = 1 WHERE video_id = ?',
).run(animeId, videoId);
db.prepare(
'INSERT INTO imm_lifetime_anime(anime_id, total_active_ms, total_cards) VALUES (?, 123456, 7)',
).run(animeId);
const history = getAnimeLibrary(db);
// Reproduce the previous schema, including its lack of a media kind column.
db.exec(
'DROP INDEX idx_anime_kind_title; ALTER TABLE imm_anime DROP COLUMN media_kind; DELETE FROM imm_schema_version; INSERT INTO imm_schema_version VALUES (23, 0)',
);
ensureSchema(db);
ensureSchema(db);
for (const id of ids) {
const row = db.prepare('SELECT media_kind FROM imm_anime WHERE anime_id = ?').get(id);
assert.ok(row && typeof row === 'object' && 'media_kind' in row);
assert.equal(row.media_kind, 'youtube');
}
assert.deepEqual(getAnimeLibrary(db), history);
assert.equal(linkYoutubeVideoToAnimeRecord(db, videoId, metadata), animeId);
const version = db
.prepare('SELECT MAX(schema_version) AS version FROM imm_schema_version')
.get();
assert.ok(version && typeof version === 'object' && 'version' in version);
assert.equal(version.version, SCHEMA_VERSION);
} finally {
db.close();
}
});
test('channel creation and repeated linking expose youtube in library and detail without losing totals', async () => {
const db = new Database(':memory:');
try {
ensureSchema(db);
const videoId = createVideo(db, 'first');
const channelId = linkYoutubeVideoToAnimeRecord(db, videoId, metadata);
assert.ok(channelId);
const secondVideoId = createVideo(db, 'second');
assert.equal(linkYoutubeVideoToAnimeRecord(db, secondVideoId, metadata), channelId);
db.prepare(
'INSERT INTO imm_lifetime_anime(anime_id, total_active_ms, total_cards) VALUES (?, 123456, 7)',
).run(channelId);
assert.equal(getAnimeLibrary(db)[0]?.mediaKind, 'youtube');
const detail = getAnimeDetail(db, channelId);
assert.equal(detail?.mediaKind, 'youtube');
assert.equal(detail?.episodeCount, 2);
assert.equal(detail?.totalActiveMs, 123456);
assert.equal(detail?.totalCards, 7);
// Even parsed season numbers and a matching anime name must not trigger repairs.
db.prepare('UPDATE imm_videos SET parsed_season = video_id').run();
assert.equal(repairLegacySeasonlessAnimeRows(db).repaired, 0);
const animeId = createAnime(db, 'Channel name');
for (const matchConfidence of ['exact', 'weak', 'manual'] as const) {
assert.equal(
resolveAnimeAnilistConflict(db, channelId, 123, { matchConfidence })
.anilistAssignmentBlocked,
true,
);
}
updateAnimeAnilistInfo(db, videoId, {
anilistId: 123,
titleRomaji: 'Wrong title',
titleEnglish: null,
titleNative: null,
episodesTotal: 12,
});
recordAnimeMergeRecommendation(db, channelId, animeId, 123);
assert.deepEqual(getAnimeMergeRecommendations(db), []);
assert.equal(getAnimeDetail(db, channelId)?.anilistId, null);
const fetcher = createCoverArtFetcher(
{
acquire: async () => {
assert.fail('YouTube must not query AniList');
},
recordResponse: () => {},
},
console,
);
assert.equal(await fetcher.fetchIfMissing(db, videoId, 'Channel name'), false);
} finally {
db.close();
}
});
test('startup reclassifies channels created by an older build after the schema upgrade', () => {
const db = new Database(':memory:');
try {
ensureSchema(db);
// An old build omits media_kind when creating a channel in the upgraded DB.
const channelId = createAnime(db, 'youtube-channel:UCnew');
db.prepare('UPDATE imm_anime SET anilist_id = 321 WHERE anime_id = ?').run(channelId);
const animeId = createAnime(db, 'Regular anime');
const videoId = createVideo(db, 'older-build');
db.prepare(
'UPDATE imm_videos SET anime_id = ?, anime_assignment_locked = 1 WHERE video_id = ?',
).run(channelId, videoId);
db.prepare(
'INSERT INTO imm_lifetime_anime(anime_id, total_active_ms, total_cards) VALUES (?, 120000, 5)',
).run(channelId);
assert.equal(getAnimeLibrary(db)[0]?.mediaKind, 'anime');
ensureSchema(db);
const channel = getAnimeLibrary(db)[0];
assert.equal(channel?.animeId, channelId);
assert.equal(channel?.mediaKind, 'youtube');
assert.equal(
(
db.prepare('SELECT anilist_id FROM imm_anime WHERE anime_id = ?').get(channelId) as {
anilist_id: number | null;
}
).anilist_id,
null,
);
assert.equal(channel?.totalActiveMs, 120000);
assert.equal(channel?.totalCards, 5);
assert.equal(linkYoutubeVideoToAnimeRecord(db, videoId, metadata), channelId);
const anime = db.prepare('SELECT media_kind FROM imm_anime WHERE anime_id = ?').get(animeId);
assert.ok(anime && typeof anime === 'object' && 'media_kind' in anime);
assert.equal(anime.media_kind, 'anime');
} finally {
db.close();
}
});
test('title identity and aliases never cross media kinds', () => {
const db = new Database(':memory:');
try {
ensureSchema(db);
const animeId = createAnime(db, 'Shared title');
const input = {
mediaKind: 'youtube' as const,
parsedTitle: 'Shared title',
canonicalTitle: 'Shared title',
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
};
const channelId = getOrCreateAnimeRecord(db, input);
assert.notEqual(channelId, animeId);
db.prepare('UPDATE imm_anime SET anilist_id = 123 WHERE anime_id = ?').run(channelId);
assert.equal(getOrCreateAnimeRecord(db, input), channelId);
assert.equal(
(
db.prepare('SELECT anilist_id FROM imm_anime WHERE anime_id = ?').get(channelId) as {
anilist_id: number | null;
}
).anilist_id,
null,
);
assert.equal(createAnime(db, 'Shared title'), animeId);
db.prepare(
'INSERT INTO imm_anime_title_aliases(normalized_title_key, anime_id) VALUES (?, ?)',
).run('alias title', animeId);
assert.notEqual(getOrCreateAnimeRecord(db, { ...input, parsedTitle: 'Alias title' }), animeId);
assert.throws(() =>
db
.prepare(
"INSERT INTO imm_anime(normalized_title_key, canonical_title, media_kind) VALUES ('shared title', 'duplicate', 'youtube')",
)
.run(),
);
} finally {
db.close();
}
});
test('schema 24 title constraint migration preserves referenced data', () => {
const db = new Database(':memory:');
try {
// Reproduce the original table-level uniqueness constraint.
db.exec(`CREATE TABLE imm_anime(
anime_id INTEGER PRIMARY KEY AUTOINCREMENT,
normalized_title_key TEXT NOT NULL UNIQUE,
canonical_title TEXT NOT NULL,
anilist_id INTEGER UNIQUE,
title_romaji TEXT, title_english TEXT, title_native TEXT,
episodes_total INTEGER, description TEXT, metadata_json TEXT,
CREATED_DATE TEXT, LAST_UPDATE_DATE TEXT,
media_kind TEXT NOT NULL DEFAULT 'anime' CHECK(media_kind IN ('anime', 'youtube'))
)`);
ensureSchema(db);
const animeId = createAnime(db, 'Shared title');
const videoId = createVideo(db, 'migration-video');
db.prepare(
'UPDATE imm_videos SET anime_id = ?, anime_assignment_locked = 1 WHERE video_id = ?',
).run(animeId, videoId);
db.prepare(
'INSERT INTO imm_lifetime_anime(anime_id, total_active_ms, total_cards) VALUES (?, 123, 4)',
).run(animeId);
// Restore the old constraint while leaving child rows populated.
db.exec(`PRAGMA foreign_keys = OFF;
CREATE TABLE old_anime AS SELECT * FROM imm_anime;
DROP TABLE imm_anime;
CREATE TABLE imm_anime(
anime_id INTEGER PRIMARY KEY AUTOINCREMENT, normalized_title_key TEXT NOT NULL UNIQUE,
canonical_title TEXT NOT NULL, anilist_id INTEGER UNIQUE,
title_romaji TEXT, title_english TEXT, title_native TEXT, episodes_total INTEGER,
description TEXT, metadata_json TEXT, CREATED_DATE TEXT, LAST_UPDATE_DATE TEXT,
media_kind TEXT NOT NULL DEFAULT 'anime' CHECK(media_kind IN ('anime', 'youtube')));
INSERT INTO imm_anime SELECT * FROM old_anime;
DROP TABLE old_anime;
DELETE FROM imm_schema_version;
INSERT INTO imm_schema_version VALUES (24, 0);
PRAGMA foreign_keys = ON;`);
ensureSchema(db);
ensureSchema(db);
assert.deepEqual(db.prepare('PRAGMA foreign_key_check').all(), []);
assert.equal(
(db.prepare('PRAGMA foreign_keys').get() as { foreign_keys: number }).foreign_keys,
1,
);
assert.equal(
(
db.prepare('SELECT anime_id FROM imm_videos WHERE video_id = ?').get(videoId) as {
anime_id: number;
}
).anime_id,
animeId,
);
assert.equal(
(
db
.prepare('SELECT total_active_ms FROM imm_lifetime_anime WHERE anime_id = ?')
.get(animeId) as { total_active_ms: number }
).total_active_ms,
123,
);
db.prepare(
"INSERT INTO imm_anime(normalized_title_key, canonical_title, media_kind) VALUES ('shared title', 'Channel', 'youtube')",
).run();
} finally {
db.close();
}
});
@@ -1,6 +1,9 @@
import type { Hono } from 'hono';
import { statsJson } from '../../../types/stats-http-contract.js';
import { UNKNOWN_MOVE_TARGET_MESSAGE } from '../immersion-tracker/anime-merge.js';
import {
MEDIA_KIND_MISMATCH_MESSAGE,
UNKNOWN_MOVE_TARGET_MESSAGE,
} from '../immersion-tracker/anime-merge.js';
import type { ImmersionTrackerService } from '../immersion-tracker-service.js';
import {
buildSentenceSearchOptions,
@@ -245,7 +248,15 @@ export function registerStatsLibraryRoutes(
const body = await c.req.json().catch(() => null);
const sourceAnimeIds = parsePositiveIdList(body?.sourceAnimeIds).filter((id) => id !== animeId);
if (sourceAnimeIds.length === 0) return c.body(null, 400);
const summary = await tracker.mergeAnime(animeId, sourceAnimeIds);
let summary: Awaited<ReturnType<typeof tracker.mergeAnime>>;
try {
summary = await tracker.mergeAnime(animeId, sourceAnimeIds);
} catch (error) {
if (error instanceof Error && error.message === MEDIA_KIND_MISMATCH_MESSAGE) {
return c.text(MEDIA_KIND_MISMATCH_MESSAGE, 409);
}
throw error;
}
// Nothing folded means the target or every source was already gone, so the
// caller should not be told the merge succeeded.
if (summary.mergedAnimeIds.length === 0) return c.body(null, 404);
@@ -281,6 +292,9 @@ export function registerStatsLibraryRoutes(
if (error instanceof Error && error.message === UNKNOWN_MOVE_TARGET_MESSAGE) {
return c.body(null, 404);
}
if (error instanceof Error && error.message === MEDIA_KIND_MISMATCH_MESSAGE) {
return c.text(MEDIA_KIND_MISMATCH_MESSAGE, 409);
}
throw error;
}
});
+37 -5
View File
@@ -2,6 +2,7 @@ import { selectAll, selectOne, type SqlRow, type SyncDb } from './libsql-driver'
import { insertRow, tableExists, type SyncMergeSummary } from './shared';
const ANIME_COPY_COLUMNS = [
'media_kind',
'normalized_title_key',
'canonical_title',
'anilist_id',
@@ -98,11 +99,28 @@ export function mergeAnime(
summary: SyncMergeSummary,
): Map<number, number> {
const map = new Map<number, number>();
const byAnilist = local.query('SELECT anime_id FROM imm_anime WHERE anilist_id = ?');
const byTitleKey = local.query('SELECT anime_id FROM imm_anime WHERE normalized_title_key = ?');
const byAnilist = local.query(
"SELECT anime_id FROM imm_anime WHERE anilist_id = ? AND media_kind = 'anime'",
);
const byTitleKey = local.query(
'SELECT anime_id FROM imm_anime WHERE normalized_title_key = ? AND media_kind = ?',
);
// A pre-classification channel can be repaired, but a genuine anime sharing
// its title must remain a separate entry.
const legacyChannel = local.query(`SELECT anime_id FROM imm_anime
WHERE normalized_title_key = ? AND media_kind = 'anime' AND (
normalized_title_key LIKE 'youtube channel %'
OR CASE WHEN json_valid(metadata_json)
THEN json_extract(metadata_json, '$.source') = 'youtube-channel' ELSE 0 END
)`);
const releaseChannelAnilistId = local.query(
"UPDATE imm_anime SET anilist_id = NULL WHERE media_kind = 'youtube' AND anilist_id = ?",
);
const fillMissing = local.query(
`UPDATE imm_anime
SET
media_kind = ?,
anilist_id = CASE WHEN ? = 'youtube' THEN NULL ELSE anilist_id END,
title_romaji = COALESCE(title_romaji, ?),
title_english = COALESCE(title_english, ?),
title_native = COALESCE(title_native, ?),
@@ -116,12 +134,24 @@ export function mergeAnime(
`SELECT anime_id, ${ANIME_COPY_COLUMNS.join(', ')} FROM imm_anime`,
)) {
const remoteId = Number(row.anime_id);
const existing = ((row.anilist_id !== null ? byAnilist.get(row.anilist_id) : undefined) ??
byTitleKey.get(row.normalized_title_key)) as SqlRow | undefined;
if (row.media_kind === 'anime' && row.anilist_id !== null) {
// AniList identifiers belong to anime, including when an older peer
// incorrectly attached one to a channel.
releaseChannelAnilistId.run(row.anilist_id);
}
const existing = ((row.media_kind === 'anime' && row.anilist_id !== null
? byAnilist.get(row.anilist_id)
: undefined) ??
byTitleKey.get(row.normalized_title_key, row.media_kind) ??
(row.media_kind === 'youtube' ? legacyChannel.get(row.normalized_title_key) : undefined)) as
| SqlRow
| undefined;
if (existing) {
const localId = Number(existing.anime_id);
map.set(remoteId, localId);
fillMissing.run(
row.media_kind,
row.media_kind,
row.title_romaji,
row.title_english,
row.title_native,
@@ -133,7 +163,9 @@ export function mergeAnime(
}
// No local row matched by anilist_id (checked first in `existing` above)
// or title key, so the remote anilist_id — if any — is free to insert as-is.
const values = ANIME_COPY_COLUMNS.map((column) => row[column]);
const values = ANIME_COPY_COLUMNS.map((column) =>
column === 'anilist_id' && row.media_kind !== 'anime' ? null : row[column],
);
map.set(remoteId, insertRow(local, 'imm_anime', ANIME_COPY_COLUMNS, values));
summary.animeAdded += 1;
}
@@ -98,3 +98,122 @@ for (const legacyOccurrences of [false, true]) {
}
});
}
test('sync preserves the YouTube media kind when adding a channel', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-sync-youtube-'));
try {
const localPath = buildDb(dir, 'local.sqlite', {
word: '猫',
seenMs: BASE_MS,
legacyOccurrences: false,
});
const remotePath = buildDb(dir, 'remote.sqlite', {
word: '犬',
seenMs: BASE_MS,
legacyOccurrences: false,
});
const remote = new Database(remotePath);
remote.exec("UPDATE imm_anime SET media_kind = 'youtube'");
remote.close();
mergeSnapshotIntoDb(localPath, remotePath);
const local = new Database(localPath);
try {
const row = local
.prepare(
"SELECT media_kind FROM imm_anime WHERE normalized_title_key = 'key-remote.sqlite'",
)
.get();
assert.ok(row && typeof row === 'object' && 'media_kind' in row);
assert.equal(row.media_kind, 'youtube');
} finally {
local.close();
}
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('sync separates same-title kinds and repairs only identifiable legacy channels', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-sync-kinds-'));
const localPath = buildDb(dir, 'local', {
word: 'local',
seenMs: BASE_MS,
legacyOccurrences: false,
});
const remotePath = buildDb(dir, 'remote', {
word: 'remote',
seenMs: BASE_MS,
legacyOccurrences: false,
});
try {
const local = new Database(localPath);
local.exec(`UPDATE imm_anime SET normalized_title_key = 'shared', anilist_id = 42;
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, metadata_json, title_english, LAST_UPDATE_DATE)
VALUES (2, 'legacy', 'Local channel', '{"source":"youtube-channel"}', 'Keep local', '9999999999999');
UPDATE imm_anime SET anilist_id = 99 WHERE anime_id = 2;
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, media_kind, anilist_id)
VALUES (3, 'other shared', 'Channel with bad ID', 'youtube', 77);`);
local.close();
const remote = new Database(remotePath);
remote.exec(`UPDATE imm_anime SET normalized_title_key = 'shared', media_kind = 'youtube', anilist_id = 42;
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, media_kind, title_english, description, LAST_UPDATE_DATE)
VALUES (2, 'legacy', 'Remote channel', 'youtube', 'Remote title', 'Fill missing', '1');
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, media_kind, anilist_id)
VALUES (3, 'other shared', 'Real anime', 'anime', 77);`);
remote.close();
mergeSnapshotIntoDb(localPath, remotePath);
mergeSnapshotIntoDb(localPath, remotePath);
const db = new Database(localPath);
try {
const rows = db
.prepare(
'SELECT media_kind FROM imm_anime WHERE normalized_title_key = ? ORDER BY media_kind',
)
.all('shared');
assert.deepEqual(rows, [{ media_kind: 'anime' }, { media_kind: 'youtube' }]);
assert.deepEqual(
db
.prepare(
"SELECT media_kind FROM imm_anime WHERE normalized_title_key = 'other shared' ORDER BY media_kind",
)
.all(),
[{ media_kind: 'anime' }, { media_kind: 'youtube' }],
);
assert.equal(
(
db.prepare('SELECT media_kind FROM imm_anime WHERE anilist_id = 77').get() as {
media_kind: string;
}
).media_kind,
'anime',
);
assert.deepEqual(
db
.prepare(
'SELECT media_kind, anilist_id, title_english, description FROM imm_anime WHERE anime_id = 2',
)
.all()[0],
{
media_kind: 'youtube',
anilist_id: null,
title_english: 'Keep local',
description: 'Fill missing',
},
);
assert.equal(
(
db
.prepare(
"SELECT a.media_kind FROM imm_videos v JOIN imm_anime a ON a.anime_id = v.anime_id WHERE v.video_key = 'video-remote'",
)
.get() as { media_kind: string }
).media_kind,
'youtube',
);
} finally {
db.close();
}
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
+3
View File
@@ -0,0 +1,3 @@
export const MEDIA_KINDS = ['anime', 'youtube'] as const;
export type MediaKind = (typeof MEDIA_KINDS)[number];
+3
View File
@@ -1,3 +1,4 @@
import type { MediaKind } from '../shared/media-kind';
export interface SessionSummary {
sessionId: number;
canonicalTitle: string | null;
@@ -240,6 +241,7 @@ export const EventType = {
export type EventType = (typeof EventType)[keyof typeof EventType];
export interface AnimeLibraryItem {
mediaKind: MediaKind;
animeId: number;
canonicalTitle: string;
anilistId: number | null;
@@ -261,6 +263,7 @@ export interface AnilistEntry {
export interface AnimeDetailData {
detail: {
mediaKind: MediaKind;
animeId: number;
canonicalTitle: string;
anilistId: number | null;