mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-09-19 05:16:27 -07:00
feat(stats): separate YouTube channels in the Library
- Store YouTube channels as a distinct media kind and migrate legacy entries safely - Add All Titles, Anime, and YouTube filters with channel-specific stats views
This commit is contained in:
@@ -192,6 +192,16 @@ export function createCoverArtFetcher(
|
||||
|
||||
return {
|
||||
async fetchIfMissing(db, videoId, canonicalTitle): Promise<boolean> {
|
||||
const channel = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT 1 FROM imm_videos v
|
||||
JOIN imm_anime a ON a.anime_id = v.anime_id
|
||||
WHERE v.video_id = ? AND a.media_kind != 'anime'
|
||||
`,
|
||||
)
|
||||
.get(videoId);
|
||||
if (channel) return false;
|
||||
const existing = getCoverArt(db, videoId);
|
||||
if (existing?.coverBlob) {
|
||||
return true;
|
||||
|
||||
@@ -27,7 +27,7 @@ function getAnimeTitles(db: DatabaseSync, animeId: number): AnimeTitleRow | null
|
||||
.prepare(
|
||||
`SELECT canonical_title, title_romaji, title_english, title_native
|
||||
FROM imm_anime
|
||||
WHERE anime_id = ?`,
|
||||
WHERE anime_id = ? AND media_kind = 'anime'`,
|
||||
)
|
||||
.get(animeId) as AnimeTitleRow | null;
|
||||
}
|
||||
@@ -77,6 +77,7 @@ export function shouldRecommendAnilistConflict(
|
||||
conflictAnimeId: number,
|
||||
options: AnimeConflictRecommendationOptions,
|
||||
): boolean {
|
||||
if (!getAnimeTitles(db, targetAnimeId) || !getAnimeTitles(db, conflictAnimeId)) return false;
|
||||
if (options.survivor === 'target' || options.matchConfidence === 'manual') return false;
|
||||
if (
|
||||
!animeSeasonsAreMergeCompatible(
|
||||
@@ -99,6 +100,8 @@ export function recordAnimeMergeRecommendation(
|
||||
secondCandidateAnimeId: number,
|
||||
anilistId: number,
|
||||
): void {
|
||||
if (!getAnimeTitles(db, firstCandidateAnimeId) || !getAnimeTitles(db, secondCandidateAnimeId))
|
||||
return;
|
||||
const firstAnimeId = Math.min(firstCandidateAnimeId, secondCandidateAnimeId);
|
||||
const secondAnimeId = Math.max(firstCandidateAnimeId, secondCandidateAnimeId);
|
||||
const timestamp = toDbTimestamp(nowMs());
|
||||
@@ -141,6 +144,8 @@ export function getAnimeMergeRecommendations(db: DatabaseSync): AnimeMergeRecomm
|
||||
second_anime_id AS secondAnimeId
|
||||
FROM imm_anime_merge_recommendations
|
||||
WHERE status = 'pending'
|
||||
AND first_anime_id IN (SELECT anime_id FROM imm_anime WHERE media_kind = 'anime')
|
||||
AND second_anime_id IN (SELECT anime_id FROM imm_anime WHERE media_kind = 'anime')
|
||||
ORDER BY recommendation_id ASC`,
|
||||
)
|
||||
.all() as Array<{
|
||||
|
||||
@@ -134,7 +134,7 @@ function getAnimeRow(db: DatabaseSync, animeId: number): AnimeRow | null {
|
||||
episodes_total,
|
||||
description
|
||||
FROM imm_anime
|
||||
WHERE anime_id = ?
|
||||
WHERE anime_id = ? AND media_kind = 'anime'
|
||||
`,
|
||||
)
|
||||
.get(animeId) as AnimeRow | null;
|
||||
@@ -335,7 +335,8 @@ export function repairLegacySeasonlessAnimeRows(db: DatabaseSync): AnimeSeasonRe
|
||||
SELECT a.anime_id AS animeId
|
||||
FROM imm_anime a
|
||||
JOIN imm_videos v ON v.anime_id = a.anime_id
|
||||
WHERE v.parsed_title IS NOT NULL
|
||||
WHERE a.media_kind = 'anime'
|
||||
AND v.parsed_title IS NOT NULL
|
||||
AND TRIM(v.parsed_title) != ''
|
||||
AND v.parsed_season IS NOT NULL
|
||||
AND v.parsed_season > 0
|
||||
@@ -372,6 +373,11 @@ export function resolveAnimeAnilistConflict(
|
||||
anilistId: number,
|
||||
options: AnimeAnilistConflictOptions = {},
|
||||
): AnimeSeasonRepairSummary {
|
||||
if (!getAnimeRow(db, targetAnimeId)) {
|
||||
const summary = emptySummary();
|
||||
summary.anilistAssignmentBlocked = true;
|
||||
return summary;
|
||||
}
|
||||
const conflict = db
|
||||
.prepare(
|
||||
`
|
||||
@@ -387,6 +393,11 @@ export function resolveAnimeAnilistConflict(
|
||||
return emptySummary();
|
||||
}
|
||||
|
||||
if (!getAnimeRow(db, conflict.animeId)) {
|
||||
const summary = emptySummary();
|
||||
summary.anilistAssignmentBlocked = true;
|
||||
return summary;
|
||||
}
|
||||
return runInTransaction(db, () => {
|
||||
const targetRow = getAnimeRow(db, targetAnimeId);
|
||||
if (
|
||||
|
||||
@@ -33,6 +33,7 @@ export function getAnimeLibrary(db: DatabaseSync): AnimeLibraryRow[] {
|
||||
SELECT
|
||||
a.anime_id AS animeId,
|
||||
a.canonical_title AS canonicalTitle,
|
||||
a.media_kind AS mediaKind,
|
||||
a.anilist_id AS anilistId,
|
||||
COALESCE(lm.total_sessions, 0) AS totalSessions,
|
||||
COALESCE(lm.total_active_ms, 0) AS totalActiveMs,
|
||||
@@ -63,6 +64,7 @@ export function getAnimeDetail(db: DatabaseSync, animeId: number): AnimeDetailRo
|
||||
SELECT
|
||||
a.anime_id AS animeId,
|
||||
a.canonical_title AS canonicalTitle,
|
||||
a.media_kind AS mediaKind,
|
||||
a.anilist_id AS anilistId,
|
||||
a.title_romaji AS titleRomaji,
|
||||
a.title_english AS titleEnglish,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { MediaKind } from '../../../shared/media-kind';
|
||||
import { createHash } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import { parseMediaInfo } from '../../../jimaku/utils';
|
||||
@@ -24,6 +25,7 @@ export interface TrackerPreparedStatements {
|
||||
}
|
||||
|
||||
export interface AnimeRecordInput {
|
||||
mediaKind?: MediaKind;
|
||||
parsedTitle: string;
|
||||
canonicalTitle: string;
|
||||
seasonScope?: number | null;
|
||||
@@ -600,6 +602,7 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
|
||||
`
|
||||
UPDATE imm_anime
|
||||
SET
|
||||
media_kind = COALESCE(?, media_kind),
|
||||
canonical_title = COALESCE(NULLIF(?, ''), canonical_title),
|
||||
anilist_id = COALESCE(?, anilist_id),
|
||||
title_romaji = COALESCE(?, title_romaji),
|
||||
@@ -610,6 +613,7 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
|
||||
WHERE anime_id = ?
|
||||
`,
|
||||
).run(
|
||||
byAnilistId || byNormalizedTitle ? (input.mediaKind ?? null) : null,
|
||||
canonicalTitleUpdate,
|
||||
input.anilistId,
|
||||
input.titleRomaji,
|
||||
@@ -627,6 +631,7 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
|
||||
.prepare(
|
||||
`
|
||||
INSERT INTO imm_anime(
|
||||
media_kind,
|
||||
normalized_title_key,
|
||||
canonical_title,
|
||||
anilist_id,
|
||||
@@ -636,10 +641,11 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
|
||||
metadata_json,
|
||||
CREATED_DATE,
|
||||
LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
)
|
||||
.run(
|
||||
input.mediaKind ?? 'anime',
|
||||
normalizedTitleKey,
|
||||
canonicalTitle,
|
||||
input.anilistId,
|
||||
@@ -803,6 +809,7 @@ export function linkYoutubeVideoToAnimeRecord(
|
||||
}
|
||||
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
mediaKind: 'youtube',
|
||||
parsedTitle: identity.parsedTitle,
|
||||
canonicalTitle: identity.canonicalTitle,
|
||||
anilistId: null,
|
||||
@@ -875,6 +882,22 @@ function migrateLegacyAnimeMetadata(db: DatabaseSync): void {
|
||||
}
|
||||
}
|
||||
|
||||
// Older builds can create channel rows with the default anime kind even after
|
||||
// the schema upgrade. Repair classification on every startup without moving videos.
|
||||
function classifyYoutubeChannels(db: DatabaseSync): void {
|
||||
db.exec(`
|
||||
UPDATE imm_anime
|
||||
SET media_kind = 'youtube'
|
||||
WHERE media_kind = 'anime'
|
||||
AND (
|
||||
normalized_title_key LIKE 'youtube channel %'
|
||||
OR CASE WHEN json_valid(metadata_json)
|
||||
THEN json_extract(metadata_json, '$.source') = 'youtube-channel'
|
||||
ELSE 0 END
|
||||
)
|
||||
`);
|
||||
}
|
||||
|
||||
export function ensureSchema(db: DatabaseSync): void {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS imm_schema_version (
|
||||
@@ -897,6 +920,7 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
.prepare('SELECT schema_version FROM imm_schema_version ORDER BY schema_version DESC LIMIT 1')
|
||||
.get() as { schema_version: number } | null;
|
||||
if (currentVersion?.schema_version === SCHEMA_VERSION) {
|
||||
classifyYoutubeChannels(db);
|
||||
ensureLexicalDailyRollupTables(db);
|
||||
ensureLifetimeSummaryTables(db);
|
||||
ensureStatsExcludedWordsTable(db);
|
||||
@@ -921,6 +945,13 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
LAST_UPDATE_DATE TEXT
|
||||
);
|
||||
`);
|
||||
addColumnIfMissing(
|
||||
db,
|
||||
'imm_anime',
|
||||
'media_kind',
|
||||
"TEXT NOT NULL DEFAULT 'anime' CHECK(media_kind IN ('anime', 'youtube'))",
|
||||
);
|
||||
classifyYoutubeChannels(db);
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS imm_videos(
|
||||
video_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
export const SCHEMA_VERSION = 23;
|
||||
import type { MediaKind } from '../../../shared/media-kind';
|
||||
|
||||
export const SCHEMA_VERSION = 24;
|
||||
export const DEFAULT_QUEUE_CAP = 1_000;
|
||||
export const DEFAULT_BATCH_SIZE = 25;
|
||||
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
|
||||
@@ -518,6 +520,7 @@ export interface YoutubeVideoMetadata {
|
||||
}
|
||||
|
||||
export interface AnimeLibraryRow {
|
||||
mediaKind: MediaKind;
|
||||
animeId: number;
|
||||
canonicalTitle: string;
|
||||
anilistId: number | null;
|
||||
@@ -531,6 +534,7 @@ export interface AnimeLibraryRow {
|
||||
}
|
||||
|
||||
export interface AnimeDetailRow {
|
||||
mediaKind: MediaKind;
|
||||
animeId: number;
|
||||
canonicalTitle: string;
|
||||
anilistId: number | null;
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { Database, type DatabaseSync } from './sqlite';
|
||||
import {
|
||||
ensureSchema,
|
||||
getOrCreateAnimeRecord,
|
||||
getOrCreateVideoRecord,
|
||||
linkYoutubeVideoToAnimeRecord,
|
||||
} from './storage';
|
||||
import { getAnimeDetail, getAnimeLibrary } from './query-library';
|
||||
import { updateAnimeAnilistInfo } from './query-maintenance';
|
||||
import {
|
||||
repairLegacySeasonlessAnimeRows,
|
||||
resolveAnimeAnilistConflict,
|
||||
} from './anime-season-repair';
|
||||
import {
|
||||
getAnimeMergeRecommendations,
|
||||
recordAnimeMergeRecommendation,
|
||||
} from './anime-merge-recommendations';
|
||||
import { createCoverArtFetcher } from '../anilist/cover-art-fetcher';
|
||||
import { SCHEMA_VERSION, SOURCE_TYPE_REMOTE, type YoutubeVideoMetadata } from './types';
|
||||
|
||||
const metadata: YoutubeVideoMetadata = {
|
||||
youtubeVideoId: 'video1',
|
||||
videoUrl: 'https://www.youtube.com/watch?v=video1',
|
||||
videoTitle: 'Video title',
|
||||
videoThumbnailUrl: null,
|
||||
channelId: 'UC123',
|
||||
channelName: 'Channel name',
|
||||
channelUrl: 'https://www.youtube.com/channel/UC123',
|
||||
channelThumbnailUrl: null,
|
||||
uploaderId: null,
|
||||
uploaderUrl: null,
|
||||
description: null,
|
||||
metadataJson: null,
|
||||
};
|
||||
|
||||
function createVideo(db: DatabaseSync, key: string): number {
|
||||
return getOrCreateVideoRecord(db, key, {
|
||||
canonicalTitle: 'Video title',
|
||||
sourcePath: null,
|
||||
sourceUrl: metadata.videoUrl,
|
||||
sourceType: SOURCE_TYPE_REMOTE,
|
||||
});
|
||||
}
|
||||
|
||||
function createAnime(
|
||||
db: DatabaseSync,
|
||||
parsedTitle: string,
|
||||
metadataJson: string | null = null,
|
||||
): number {
|
||||
return getOrCreateAnimeRecord(db, {
|
||||
parsedTitle,
|
||||
canonicalTitle: parsedTitle,
|
||||
metadataJson,
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
});
|
||||
}
|
||||
|
||||
test('schema 23 channel migration preserves history and manual assignments and is idempotent', () => {
|
||||
const db = new Database(':memory:');
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const ids = [
|
||||
createAnime(db, 'youtube-channel:UC123'),
|
||||
createAnime(db, 'youtube-channel-url:https://www.youtube.com/@creator'),
|
||||
createAnime(db, 'youtube-channel-name:Creator'),
|
||||
createAnime(db, 'Renamed channel', '{ "source": "youtube-channel" }'),
|
||||
];
|
||||
const animeId = createAnime(db, 'Anime title', 'legacy non-JSON metadata');
|
||||
const videoId = createVideo(db, 'manual');
|
||||
db.prepare(
|
||||
'UPDATE imm_videos SET anime_id = ?, anime_assignment_locked = 1 WHERE video_id = ?',
|
||||
).run(animeId, videoId);
|
||||
db.prepare(
|
||||
'INSERT INTO imm_lifetime_anime(anime_id, total_active_ms, total_cards) VALUES (?, 123456, 7)',
|
||||
).run(animeId);
|
||||
const history = getAnimeLibrary(db);
|
||||
// Reproduce the previous schema, including its lack of a media kind column.
|
||||
db.exec(
|
||||
'ALTER TABLE imm_anime DROP COLUMN media_kind; DELETE FROM imm_schema_version; INSERT INTO imm_schema_version VALUES (23, 0)',
|
||||
);
|
||||
ensureSchema(db);
|
||||
ensureSchema(db);
|
||||
for (const id of ids) {
|
||||
const row = db.prepare('SELECT media_kind FROM imm_anime WHERE anime_id = ?').get(id);
|
||||
assert.ok(row && typeof row === 'object' && 'media_kind' in row);
|
||||
assert.equal(row.media_kind, 'youtube');
|
||||
}
|
||||
assert.deepEqual(getAnimeLibrary(db), history);
|
||||
assert.equal(linkYoutubeVideoToAnimeRecord(db, videoId, metadata), animeId);
|
||||
const version = db
|
||||
.prepare('SELECT MAX(schema_version) AS version FROM imm_schema_version')
|
||||
.get();
|
||||
assert.ok(version && typeof version === 'object' && 'version' in version);
|
||||
assert.equal(version.version, SCHEMA_VERSION);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('channel creation and repeated linking expose youtube in library and detail without losing totals', async () => {
|
||||
const db = new Database(':memory:');
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const videoId = createVideo(db, 'first');
|
||||
const channelId = linkYoutubeVideoToAnimeRecord(db, videoId, metadata);
|
||||
assert.ok(channelId);
|
||||
const secondVideoId = createVideo(db, 'second');
|
||||
assert.equal(linkYoutubeVideoToAnimeRecord(db, secondVideoId, metadata), channelId);
|
||||
db.prepare(
|
||||
'INSERT INTO imm_lifetime_anime(anime_id, total_active_ms, total_cards) VALUES (?, 123456, 7)',
|
||||
).run(channelId);
|
||||
assert.equal(getAnimeLibrary(db)[0]?.mediaKind, 'youtube');
|
||||
const detail = getAnimeDetail(db, channelId);
|
||||
assert.equal(detail?.mediaKind, 'youtube');
|
||||
assert.equal(detail?.episodeCount, 2);
|
||||
assert.equal(detail?.totalActiveMs, 123456);
|
||||
assert.equal(detail?.totalCards, 7);
|
||||
|
||||
// Even parsed season numbers and a matching anime name must not trigger repairs.
|
||||
db.prepare('UPDATE imm_videos SET parsed_season = video_id').run();
|
||||
assert.equal(repairLegacySeasonlessAnimeRows(db).repaired, 0);
|
||||
const animeId = createAnime(db, 'Channel name');
|
||||
for (const matchConfidence of ['exact', 'weak', 'manual'] as const) {
|
||||
assert.equal(
|
||||
resolveAnimeAnilistConflict(db, channelId, 123, { matchConfidence })
|
||||
.anilistAssignmentBlocked,
|
||||
true,
|
||||
);
|
||||
}
|
||||
updateAnimeAnilistInfo(db, videoId, {
|
||||
anilistId: 123,
|
||||
titleRomaji: 'Wrong title',
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
episodesTotal: 12,
|
||||
});
|
||||
recordAnimeMergeRecommendation(db, channelId, animeId, 123);
|
||||
assert.deepEqual(getAnimeMergeRecommendations(db), []);
|
||||
assert.equal(getAnimeDetail(db, channelId)?.anilistId, null);
|
||||
const fetcher = createCoverArtFetcher(
|
||||
{
|
||||
acquire: async () => {
|
||||
assert.fail('YouTube must not query AniList');
|
||||
},
|
||||
recordResponse: () => {},
|
||||
},
|
||||
console,
|
||||
);
|
||||
assert.equal(await fetcher.fetchIfMissing(db, videoId, 'Channel name'), false);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('startup reclassifies channels created by an older build after the schema upgrade', () => {
|
||||
const db = new Database(':memory:');
|
||||
try {
|
||||
ensureSchema(db);
|
||||
// An old build omits media_kind when creating a channel in the upgraded DB.
|
||||
const channelId = createAnime(db, 'youtube-channel:UCnew');
|
||||
const animeId = createAnime(db, 'Regular anime');
|
||||
const videoId = createVideo(db, 'older-build');
|
||||
db.prepare(
|
||||
'UPDATE imm_videos SET anime_id = ?, anime_assignment_locked = 1 WHERE video_id = ?',
|
||||
).run(channelId, videoId);
|
||||
db.prepare(
|
||||
'INSERT INTO imm_lifetime_anime(anime_id, total_active_ms, total_cards) VALUES (?, 120000, 5)',
|
||||
).run(channelId);
|
||||
assert.equal(getAnimeLibrary(db)[0]?.mediaKind, 'anime');
|
||||
ensureSchema(db);
|
||||
const channel = getAnimeLibrary(db)[0];
|
||||
assert.equal(channel?.animeId, channelId);
|
||||
assert.equal(channel?.mediaKind, 'youtube');
|
||||
assert.equal(channel?.totalActiveMs, 120000);
|
||||
assert.equal(channel?.totalCards, 5);
|
||||
assert.equal(linkYoutubeVideoToAnimeRecord(db, videoId, metadata), channelId);
|
||||
const anime = db.prepare('SELECT media_kind FROM imm_anime WHERE anime_id = ?').get(animeId);
|
||||
assert.ok(anime && typeof anime === 'object' && 'media_kind' in anime);
|
||||
assert.equal(anime.media_kind, 'anime');
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { selectAll, selectOne, type SqlRow, type SyncDb } from './libsql-driver'
|
||||
import { insertRow, tableExists, type SyncMergeSummary } from './shared';
|
||||
|
||||
const ANIME_COPY_COLUMNS = [
|
||||
'media_kind',
|
||||
'normalized_title_key',
|
||||
'canonical_title',
|
||||
'anilist_id',
|
||||
|
||||
@@ -98,3 +98,37 @@ for (const legacyOccurrences of [false, true]) {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('sync preserves the YouTube media kind when adding a channel', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-sync-youtube-'));
|
||||
try {
|
||||
const localPath = buildDb(dir, 'local.sqlite', {
|
||||
word: '猫',
|
||||
seenMs: BASE_MS,
|
||||
legacyOccurrences: false,
|
||||
});
|
||||
const remotePath = buildDb(dir, 'remote.sqlite', {
|
||||
word: '犬',
|
||||
seenMs: BASE_MS,
|
||||
legacyOccurrences: false,
|
||||
});
|
||||
const remote = new Database(remotePath);
|
||||
remote.exec("UPDATE imm_anime SET media_kind = 'youtube'");
|
||||
remote.close();
|
||||
mergeSnapshotIntoDb(localPath, remotePath);
|
||||
const local = new Database(localPath);
|
||||
try {
|
||||
const row = local
|
||||
.prepare(
|
||||
"SELECT media_kind FROM imm_anime WHERE normalized_title_key = 'key-remote.sqlite'",
|
||||
)
|
||||
.get();
|
||||
assert.ok(row && typeof row === 'object' && 'media_kind' in row);
|
||||
assert.equal(row.media_kind, 'youtube');
|
||||
} finally {
|
||||
local.close();
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user