fix(stats): keep same-title anime and YouTube records separate

- Scope title identity, aliases, AniList matching, and sync by media kind
- Migrate legacy schemas while preserving IDs, history, and assignments
This commit is contained in:
2026-09-19 00:40:54 -07:00
parent e79db73cd9
commit c961a84dce
10 changed files with 323 additions and 22 deletions
+70 -12
View File
@@ -571,6 +571,8 @@ function ensureSubtitleLineEventIndex(db: DatabaseSync): void {
}
export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput): number {
const mediaKind = input.mediaKind ?? 'anime';
const anilistId = mediaKind === 'anime' ? input.anilistId : null;
const seasonScope = normalizeSeasonScope(input.seasonScope);
const identityTitle = buildSeasonScopedAnimeTitle(input.parsedTitle, seasonScope);
const canonicalTitle =
@@ -582,17 +584,23 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
}
const byAnilistId =
input.anilistId !== null
? (db.prepare('SELECT anime_id FROM imm_anime WHERE anilist_id = ?').get(input.anilistId) as {
anilistId !== null
? (db
.prepare("SELECT anime_id FROM imm_anime WHERE anilist_id = ? AND media_kind = 'anime'")
.get(anilistId) as {
anime_id: number;
} | null)
: null;
const byNormalizedTitle = db
.prepare('SELECT anime_id FROM imm_anime WHERE normalized_title_key = ?')
.get(normalizedTitleKey) as { anime_id: number } | null;
.prepare('SELECT anime_id FROM imm_anime WHERE normalized_title_key = ? AND media_kind = ?')
.get(normalizedTitleKey, mediaKind) as { anime_id: number } | null;
const byTitleAlias = db
.prepare('SELECT anime_id FROM imm_anime_title_aliases WHERE normalized_title_key = ?')
.get(normalizedTitleKey) as { anime_id: number } | null;
.prepare(
`SELECT a.anime_id FROM imm_anime_title_aliases AS alias
JOIN imm_anime AS a ON a.anime_id = alias.anime_id
WHERE alias.normalized_title_key = ? AND a.media_kind = ?`,
)
.get(normalizedTitleKey, mediaKind) as { anime_id: number } | null;
const existing = byAnilistId ?? byNormalizedTitle ?? byTitleAlias;
if (existing?.anime_id) {
// An alias remembers an intentionally merged-away spelling. Reusing it
@@ -604,7 +612,7 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
SET
media_kind = COALESCE(?, media_kind),
canonical_title = COALESCE(NULLIF(?, ''), canonical_title),
anilist_id = COALESCE(?, anilist_id),
anilist_id = CASE WHEN ? = 'youtube' THEN NULL ELSE COALESCE(?, anilist_id) END,
title_romaji = COALESCE(?, title_romaji),
title_english = COALESCE(?, title_english),
title_native = COALESCE(?, title_native),
@@ -615,7 +623,8 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
).run(
byAnilistId || byNormalizedTitle ? (input.mediaKind ?? null) : null,
canonicalTitleUpdate,
input.anilistId,
mediaKind,
anilistId,
input.titleRomaji,
input.titleEnglish,
input.titleNative,
@@ -648,7 +657,7 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
input.mediaKind ?? 'anime',
normalizedTitleKey,
canonicalTitle,
input.anilistId,
anilistId,
input.titleRomaji,
input.titleEnglish,
input.titleNative,
@@ -882,13 +891,61 @@ function migrateLegacyAnimeMetadata(db: DatabaseSync): void {
}
}
// SQLite cannot drop a table-level UNIQUE constraint. Rebuild with IDs intact
// and foreign keys disabled so dependent history and manual assignments survive.
function migrateAnimeTitleUniqueness(db: DatabaseSync): void {
const schema = db.prepare("SELECT sql FROM sqlite_master WHERE name = 'imm_anime'").get() as {
sql: string;
};
if (/normalized_title_key TEXT NOT NULL UNIQUE/i.test(schema.sql)) {
const foreignKeys = db.prepare('PRAGMA foreign_keys').get() as { foreign_keys: number };
const sequence = db
.prepare("SELECT seq FROM sqlite_sequence WHERE name = 'imm_anime'")
.get() as { seq: number } | null;
db.exec('PRAGMA foreign_keys = OFF');
try {
db.exec('BEGIN IMMEDIATE');
db.exec(
schema.sql
.replace(
/CREATE TABLE (?:IF NOT EXISTS )?["`]?imm_anime["`]?/i,
'CREATE TABLE imm_anime_new',
)
.replace(
/normalized_title_key TEXT NOT NULL UNIQUE/i,
'normalized_title_key TEXT NOT NULL',
),
);
db.exec(`INSERT INTO imm_anime_new SELECT * FROM imm_anime;
DROP TABLE imm_anime;
ALTER TABLE imm_anime_new RENAME TO imm_anime;`);
if (sequence) {
db.prepare("UPDATE sqlite_sequence SET seq = MAX(seq, ?) WHERE name = 'imm_anime'").run(
sequence.seq,
);
}
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
} finally {
db.exec(`PRAGMA foreign_keys = ${foreignKeys.foreign_keys}`);
}
}
db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_anime_kind_title
ON imm_anime(media_kind, normalized_title_key)`);
}
// Older builds can create channel rows with the default anime kind even after
// the schema upgrade. Repair classification on every startup without moving videos.
function classifyYoutubeChannels(db: DatabaseSync): void {
db.exec(`
UPDATE imm_anime
SET media_kind = 'youtube'
SET media_kind = 'youtube', anilist_id = NULL
WHERE media_kind = 'anime'
AND NOT EXISTS (SELECT 1 FROM imm_anime AS channel
WHERE channel.media_kind = 'youtube'
AND channel.normalized_title_key = imm_anime.normalized_title_key)
AND (
normalized_title_key LIKE 'youtube channel %'
OR CASE WHEN json_valid(metadata_json)
@@ -932,7 +989,7 @@ export function ensureSchema(db: DatabaseSync): void {
db.exec(`
CREATE TABLE IF NOT EXISTS imm_anime(
anime_id INTEGER PRIMARY KEY AUTOINCREMENT,
normalized_title_key TEXT NOT NULL UNIQUE,
normalized_title_key TEXT NOT NULL,
canonical_title TEXT NOT NULL,
anilist_id INTEGER UNIQUE,
title_romaji TEXT,
@@ -951,7 +1008,6 @@ export function ensureSchema(db: DatabaseSync): void {
'media_kind',
"TEXT NOT NULL DEFAULT 'anime' CHECK(media_kind IN ('anime', 'youtube'))",
);
classifyYoutubeChannels(db);
db.exec(`
CREATE TABLE IF NOT EXISTS imm_videos(
video_id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -1495,6 +1551,8 @@ export function ensureSchema(db: DatabaseSync): void {
);
}
migrateAnimeTitleUniqueness(db);
classifyYoutubeChannels(db);
migrateSessionEventTimestampsToText(db);
ensureLexicalDailyRollupTables(db);
+1 -1
View File
@@ -1,6 +1,6 @@
import type { MediaKind } from '../../../shared/media-kind';
export const SCHEMA_VERSION = 24;
export const SCHEMA_VERSION = 25;
export const DEFAULT_QUEUE_CAP = 1_000;
export const DEFAULT_BATCH_SIZE = 25;
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
@@ -81,7 +81,7 @@ test('schema 23 channel migration preserves history and manual assignments and i
const history = getAnimeLibrary(db);
// Reproduce the previous schema, including its lack of a media kind column.
db.exec(
'ALTER TABLE imm_anime DROP COLUMN media_kind; DELETE FROM imm_schema_version; INSERT INTO imm_schema_version VALUES (23, 0)',
'DROP INDEX idx_anime_kind_title; ALTER TABLE imm_anime DROP COLUMN media_kind; DELETE FROM imm_schema_version; INSERT INTO imm_schema_version VALUES (23, 0)',
);
ensureSchema(db);
ensureSchema(db);
@@ -163,6 +163,7 @@ test('startup reclassifies channels created by an older build after the schema u
ensureSchema(db);
// An old build omits media_kind when creating a channel in the upgraded DB.
const channelId = createAnime(db, 'youtube-channel:UCnew');
db.prepare('UPDATE imm_anime SET anilist_id = 321 WHERE anime_id = ?').run(channelId);
const animeId = createAnime(db, 'Regular anime');
const videoId = createVideo(db, 'older-build');
db.prepare(
@@ -176,6 +177,14 @@ test('startup reclassifies channels created by an older build after the schema u
const channel = getAnimeLibrary(db)[0];
assert.equal(channel?.animeId, channelId);
assert.equal(channel?.mediaKind, 'youtube');
assert.equal(
(
db.prepare('SELECT anilist_id FROM imm_anime WHERE anime_id = ?').get(channelId) as {
anilist_id: number | null;
}
).anilist_id,
null,
);
assert.equal(channel?.totalActiveMs, 120000);
assert.equal(channel?.totalCards, 5);
assert.equal(linkYoutubeVideoToAnimeRecord(db, videoId, metadata), channelId);
@@ -186,3 +195,116 @@ test('startup reclassifies channels created by an older build after the schema u
db.close();
}
});
test('title identity and aliases never cross media kinds', () => {
const db = new Database(':memory:');
try {
ensureSchema(db);
const animeId = createAnime(db, 'Shared title');
const input = {
mediaKind: 'youtube' as const,
parsedTitle: 'Shared title',
canonicalTitle: 'Shared title',
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
};
const channelId = getOrCreateAnimeRecord(db, input);
assert.notEqual(channelId, animeId);
db.prepare('UPDATE imm_anime SET anilist_id = 123 WHERE anime_id = ?').run(channelId);
assert.equal(getOrCreateAnimeRecord(db, input), channelId);
assert.equal(
(
db.prepare('SELECT anilist_id FROM imm_anime WHERE anime_id = ?').get(channelId) as {
anilist_id: number | null;
}
).anilist_id,
null,
);
assert.equal(createAnime(db, 'Shared title'), animeId);
db.prepare(
'INSERT INTO imm_anime_title_aliases(normalized_title_key, anime_id) VALUES (?, ?)',
).run('alias title', animeId);
assert.notEqual(getOrCreateAnimeRecord(db, { ...input, parsedTitle: 'Alias title' }), animeId);
assert.throws(() =>
db
.prepare(
"INSERT INTO imm_anime(normalized_title_key, canonical_title, media_kind) VALUES ('shared title', 'duplicate', 'youtube')",
)
.run(),
);
} finally {
db.close();
}
});
test('schema 24 title constraint migration preserves referenced data', () => {
const db = new Database(':memory:');
try {
// Reproduce the original table-level uniqueness constraint.
db.exec(`CREATE TABLE imm_anime(
anime_id INTEGER PRIMARY KEY AUTOINCREMENT,
normalized_title_key TEXT NOT NULL UNIQUE,
canonical_title TEXT NOT NULL,
anilist_id INTEGER UNIQUE,
title_romaji TEXT, title_english TEXT, title_native TEXT,
episodes_total INTEGER, description TEXT, metadata_json TEXT,
CREATED_DATE TEXT, LAST_UPDATE_DATE TEXT,
media_kind TEXT NOT NULL DEFAULT 'anime' CHECK(media_kind IN ('anime', 'youtube'))
)`);
ensureSchema(db);
const animeId = createAnime(db, 'Shared title');
const videoId = createVideo(db, 'migration-video');
db.prepare(
'UPDATE imm_videos SET anime_id = ?, anime_assignment_locked = 1 WHERE video_id = ?',
).run(animeId, videoId);
db.prepare(
'INSERT INTO imm_lifetime_anime(anime_id, total_active_ms, total_cards) VALUES (?, 123, 4)',
).run(animeId);
// Restore the old constraint while leaving child rows populated.
db.exec(`PRAGMA foreign_keys = OFF;
CREATE TABLE old_anime AS SELECT * FROM imm_anime;
DROP TABLE imm_anime;
CREATE TABLE imm_anime(
anime_id INTEGER PRIMARY KEY AUTOINCREMENT, normalized_title_key TEXT NOT NULL UNIQUE,
canonical_title TEXT NOT NULL, anilist_id INTEGER UNIQUE,
title_romaji TEXT, title_english TEXT, title_native TEXT, episodes_total INTEGER,
description TEXT, metadata_json TEXT, CREATED_DATE TEXT, LAST_UPDATE_DATE TEXT,
media_kind TEXT NOT NULL DEFAULT 'anime' CHECK(media_kind IN ('anime', 'youtube')));
INSERT INTO imm_anime SELECT * FROM old_anime;
DROP TABLE old_anime;
DELETE FROM imm_schema_version;
INSERT INTO imm_schema_version VALUES (24, 0);
PRAGMA foreign_keys = ON;`);
ensureSchema(db);
ensureSchema(db);
assert.deepEqual(db.prepare('PRAGMA foreign_key_check').all(), []);
assert.equal(
(db.prepare('PRAGMA foreign_keys').get() as { foreign_keys: number }).foreign_keys,
1,
);
assert.equal(
(
db.prepare('SELECT anime_id FROM imm_videos WHERE video_id = ?').get(videoId) as {
anime_id: number;
}
).anime_id,
animeId,
);
assert.equal(
(
db
.prepare('SELECT total_active_ms FROM imm_lifetime_anime WHERE anime_id = ?')
.get(animeId) as { total_active_ms: number }
).total_active_ms,
123,
);
db.prepare(
"INSERT INTO imm_anime(normalized_title_key, canonical_title, media_kind) VALUES ('shared title', 'Channel', 'youtube')",
).run();
} finally {
db.close();
}
});
+36 -5
View File
@@ -99,11 +99,28 @@ export function mergeAnime(
summary: SyncMergeSummary,
): Map<number, number> {
const map = new Map<number, number>();
const byAnilist = local.query('SELECT anime_id FROM imm_anime WHERE anilist_id = ?');
const byTitleKey = local.query('SELECT anime_id FROM imm_anime WHERE normalized_title_key = ?');
const byAnilist = local.query(
"SELECT anime_id FROM imm_anime WHERE anilist_id = ? AND media_kind = 'anime'",
);
const byTitleKey = local.query(
'SELECT anime_id FROM imm_anime WHERE normalized_title_key = ? AND media_kind = ?',
);
// A pre-classification channel can be repaired, but a genuine anime sharing
// its title must remain a separate entry.
const legacyChannel = local.query(`SELECT anime_id FROM imm_anime
WHERE normalized_title_key = ? AND media_kind = 'anime' AND (
normalized_title_key LIKE 'youtube channel %'
OR CASE WHEN json_valid(metadata_json)
THEN json_extract(metadata_json, '$.source') = 'youtube-channel' ELSE 0 END
)`);
const releaseChannelAnilistId = local.query(
"UPDATE imm_anime SET anilist_id = NULL WHERE media_kind = 'youtube' AND anilist_id = ?",
);
const fillMissing = local.query(
`UPDATE imm_anime
SET
media_kind = ?,
anilist_id = CASE WHEN ? = 'youtube' THEN NULL ELSE anilist_id END,
title_romaji = COALESCE(title_romaji, ?),
title_english = COALESCE(title_english, ?),
title_native = COALESCE(title_native, ?),
@@ -117,12 +134,24 @@ export function mergeAnime(
`SELECT anime_id, ${ANIME_COPY_COLUMNS.join(', ')} FROM imm_anime`,
)) {
const remoteId = Number(row.anime_id);
const existing = ((row.anilist_id !== null ? byAnilist.get(row.anilist_id) : undefined) ??
byTitleKey.get(row.normalized_title_key)) as SqlRow | undefined;
if (row.media_kind === 'anime' && row.anilist_id !== null) {
// AniList identifiers belong to anime, including when an older peer
// incorrectly attached one to a channel.
releaseChannelAnilistId.run(row.anilist_id);
}
const existing = ((row.media_kind === 'anime' && row.anilist_id !== null
? byAnilist.get(row.anilist_id)
: undefined) ??
byTitleKey.get(row.normalized_title_key, row.media_kind) ??
(row.media_kind === 'youtube' ? legacyChannel.get(row.normalized_title_key) : undefined)) as
| SqlRow
| undefined;
if (existing) {
const localId = Number(existing.anime_id);
map.set(remoteId, localId);
fillMissing.run(
row.media_kind,
row.media_kind,
row.title_romaji,
row.title_english,
row.title_native,
@@ -134,7 +163,9 @@ export function mergeAnime(
}
// No local row matched by anilist_id (checked first in `existing` above)
// or title key, so the remote anilist_id — if any — is free to insert as-is.
const values = ANIME_COPY_COLUMNS.map((column) => row[column]);
const values = ANIME_COPY_COLUMNS.map((column) =>
column === 'anilist_id' && row.media_kind !== 'anime' ? null : row[column],
);
map.set(remoteId, insertRow(local, 'imm_anime', ANIME_COPY_COLUMNS, values));
summary.animeAdded += 1;
}
@@ -132,3 +132,88 @@ test('sync preserves the YouTube media kind when adding a channel', () => {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('sync separates same-title kinds and repairs only identifiable legacy channels', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-sync-kinds-'));
const localPath = buildDb(dir, 'local', {
word: 'local',
seenMs: BASE_MS,
legacyOccurrences: false,
});
const remotePath = buildDb(dir, 'remote', {
word: 'remote',
seenMs: BASE_MS,
legacyOccurrences: false,
});
try {
const local = new Database(localPath);
local.exec(`UPDATE imm_anime SET normalized_title_key = 'shared', anilist_id = 42;
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, metadata_json, title_english, LAST_UPDATE_DATE)
VALUES (2, 'legacy', 'Local channel', '{"source":"youtube-channel"}', 'Keep local', '9999999999999');
UPDATE imm_anime SET anilist_id = 99 WHERE anime_id = 2;
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, media_kind, anilist_id)
VALUES (3, 'other shared', 'Channel with bad ID', 'youtube', 77);`);
local.close();
const remote = new Database(remotePath);
remote.exec(`UPDATE imm_anime SET normalized_title_key = 'shared', media_kind = 'youtube', anilist_id = 42;
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, media_kind, title_english, description, LAST_UPDATE_DATE)
VALUES (2, 'legacy', 'Remote channel', 'youtube', 'Remote title', 'Fill missing', '1');
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, media_kind, anilist_id)
VALUES (3, 'other shared', 'Real anime', 'anime', 77);`);
remote.close();
mergeSnapshotIntoDb(localPath, remotePath);
mergeSnapshotIntoDb(localPath, remotePath);
const db = new Database(localPath);
try {
const rows = db
.prepare(
'SELECT media_kind FROM imm_anime WHERE normalized_title_key = ? ORDER BY media_kind',
)
.all('shared');
assert.deepEqual(rows, [{ media_kind: 'anime' }, { media_kind: 'youtube' }]);
assert.deepEqual(
db
.prepare(
"SELECT media_kind FROM imm_anime WHERE normalized_title_key = 'other shared' ORDER BY media_kind",
)
.all(),
[{ media_kind: 'anime' }, { media_kind: 'youtube' }],
);
assert.equal(
(
db.prepare('SELECT media_kind FROM imm_anime WHERE anilist_id = 77').get() as {
media_kind: string;
}
).media_kind,
'anime',
);
assert.deepEqual(
db
.prepare(
'SELECT media_kind, anilist_id, title_english, description FROM imm_anime WHERE anime_id = 2',
)
.all()[0],
{
media_kind: 'youtube',
anilist_id: null,
title_english: 'Keep local',
description: 'Fill missing',
},
);
assert.equal(
(
db
.prepare(
"SELECT a.media_kind FROM imm_videos v JOIN imm_anime a ON a.anime_id = v.anime_id WHERE v.video_key = 'video-remote'",
)
.get() as { media_kind: string }
).media_kind,
'youtube',
);
} finally {
db.close();
}
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});