mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-27 16:49:51 -07:00
feat(stats): speed up session maintenance and improve stats UI (#111)
This commit is contained in:
GitHub
@@ -0,0 +1,18 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
|
||||
test('getRollupGroupsForSessions uses only localtime rollup keys', () => {
|
||||
const source = fs.readFileSync(
|
||||
path.join(process.cwd(), 'src/core/services/immersion-tracker/maintenance.ts'),
|
||||
'utf8',
|
||||
);
|
||||
const start = source.indexOf('export function getRollupGroupsForSessions');
|
||||
const end = source.indexOf('export function refreshRollupsForGroupsInTransaction');
|
||||
const functionSource = source.slice(start, end);
|
||||
|
||||
assert.match(functionSource, /'unixepoch', 'localtime'/);
|
||||
assert.doesNotMatch(functionSource, /UNION/);
|
||||
assert.doesNotMatch(functionSource, /86400000/);
|
||||
});
|
||||
@@ -356,6 +356,81 @@ test('split session and lexical helpers return distinct-headword, detail, appear
|
||||
}
|
||||
});
|
||||
|
||||
test('similar words use same reading and shared kanji without kana suffix noise', () => {
|
||||
const { db, dbPath, stmts } = createDb();
|
||||
|
||||
try {
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: 'Similar Words Anime',
|
||||
canonicalTitle: 'Similar Words Anime',
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: null,
|
||||
});
|
||||
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/similar-words.mkv', {
|
||||
canonicalTitle: 'Similar Words Episode',
|
||||
sourcePath: '/tmp/similar-words.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
const sessionId = startSessionRecord(db, videoId, 1_000_000).sessionId;
|
||||
|
||||
const araiId = insertWordOccurrence(db, stmts, {
|
||||
sessionId,
|
||||
videoId,
|
||||
animeId,
|
||||
lineIndex: 1,
|
||||
text: '荒い息',
|
||||
word: { headword: '荒い', word: '荒い', reading: 'あらい' },
|
||||
});
|
||||
insertWordOccurrence(db, stmts, {
|
||||
sessionId,
|
||||
videoId,
|
||||
animeId,
|
||||
lineIndex: 2,
|
||||
text: '洗い物',
|
||||
word: { headword: '洗い', word: '洗い', reading: 'あらい' },
|
||||
});
|
||||
insertWordOccurrence(db, stmts, {
|
||||
sessionId,
|
||||
videoId,
|
||||
animeId,
|
||||
lineIndex: 3,
|
||||
text: '荒波',
|
||||
word: { headword: '荒波', word: '荒波', reading: 'あらなみ' },
|
||||
});
|
||||
|
||||
for (let lineIndex = 4; lineIndex < 9; lineIndex++) {
|
||||
insertWordOccurrence(db, stmts, {
|
||||
sessionId,
|
||||
videoId,
|
||||
animeId,
|
||||
lineIndex,
|
||||
text: '良い',
|
||||
word: { headword: '良い', word: '良い', reading: 'よい' },
|
||||
});
|
||||
}
|
||||
insertWordOccurrence(db, stmts, {
|
||||
sessionId,
|
||||
videoId,
|
||||
animeId,
|
||||
lineIndex: 9,
|
||||
text: 'お構いなく',
|
||||
word: { headword: 'お構いなく', word: 'お構いなく', reading: 'おかまいなく' },
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
getSimilarWords(db, araiId, 10).map((row) => row.headword),
|
||||
['洗い', '荒波'],
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('split library helpers return anime/media session and analytics rows', () => {
|
||||
const { db, dbPath, stmts } = createDb();
|
||||
|
||||
@@ -605,6 +680,79 @@ test('split maintenance helpers update anime metadata and watched state', () =>
|
||||
}
|
||||
});
|
||||
|
||||
test('deleteSessions refreshes only rollups affected by deleted sessions', () => {
|
||||
const { db, dbPath } = createDb();
|
||||
|
||||
try {
|
||||
const keepVideoId = getOrCreateVideoRecord(db, 'local:/tmp/rollup-keep.mkv', {
|
||||
canonicalTitle: 'Rollup Keep',
|
||||
sourcePath: '/tmp/rollup-keep.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
const dropVideoId = getOrCreateVideoRecord(db, 'local:/tmp/rollup-drop.mkv', {
|
||||
canonicalTitle: 'Rollup Drop',
|
||||
sourcePath: '/tmp/rollup-drop.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
|
||||
const keepStartedAtMs = 1_700_000_000_000;
|
||||
const dropStartedAtMs = 1_700_086_400_000;
|
||||
const keepSessionId = startSessionRecord(db, keepVideoId, keepStartedAtMs).sessionId;
|
||||
const dropSessionId = startSessionRecord(db, dropVideoId, dropStartedAtMs).sessionId;
|
||||
finalizeSessionMetrics(db, keepSessionId, keepStartedAtMs, {
|
||||
activeWatchedMs: 30_000,
|
||||
cardsMined: 1,
|
||||
});
|
||||
finalizeSessionMetrics(db, dropSessionId, dropStartedAtMs, {
|
||||
activeWatchedMs: 60_000,
|
||||
cardsMined: 2,
|
||||
});
|
||||
|
||||
const keepDay = getLocalEpochDay(db, keepStartedAtMs);
|
||||
const dropDay = getLocalEpochDay(db, dropStartedAtMs);
|
||||
const keepMonth = 202311;
|
||||
const dropMonth = 202311;
|
||||
|
||||
const insertDaily = db.prepare(`
|
||||
INSERT INTO imm_daily_rollups (
|
||||
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
const insertMonthly = db.prepare(`
|
||||
INSERT INTO imm_monthly_rollups (
|
||||
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
insertDaily.run(keepDay, keepVideoId, 1, 0.5, 3, 6, 1, keepStartedAtMs, keepStartedAtMs);
|
||||
insertDaily.run(dropDay, dropVideoId, 1, 1, 3, 6, 2, dropStartedAtMs, dropStartedAtMs);
|
||||
insertMonthly.run(keepMonth, keepVideoId, 1, 0.5, 3, 6, 1, keepStartedAtMs, keepStartedAtMs);
|
||||
insertMonthly.run(dropMonth, dropVideoId, 1, 1, 3, 6, 2, dropStartedAtMs, dropStartedAtMs);
|
||||
|
||||
deleteSessions(db, [dropSessionId]);
|
||||
|
||||
const dailyRows = db
|
||||
.prepare('SELECT rollup_day, video_id, total_cards FROM imm_daily_rollups ORDER BY video_id')
|
||||
.all() as Array<{ rollup_day: number; video_id: number; total_cards: number }>;
|
||||
const monthlyRows = db
|
||||
.prepare(
|
||||
'SELECT rollup_month, video_id, total_cards FROM imm_monthly_rollups ORDER BY video_id',
|
||||
)
|
||||
.all() as Array<{ rollup_month: number; video_id: number; total_cards: number }>;
|
||||
|
||||
assert.deepEqual(dailyRows, [{ rollup_day: keepDay, video_id: keepVideoId, total_cards: 1 }]);
|
||||
assert.deepEqual(monthlyRows, [
|
||||
{ rollup_month: keepMonth, video_id: keepVideoId, total_cards: 1 },
|
||||
]);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('split maintenance helpers delete multiple sessions and whole videos with dependent rows', () => {
|
||||
const { db, dbPath, stmts } = createDb();
|
||||
|
||||
|
||||
@@ -35,9 +35,11 @@ import {
|
||||
getSessionTimeline,
|
||||
getSessionWordsByLine,
|
||||
getWordOccurrences,
|
||||
searchSubtitleSentences,
|
||||
upsertCoverArt,
|
||||
} from '../query.js';
|
||||
import {
|
||||
getLocalEpochDay,
|
||||
getShiftedLocalDaySec,
|
||||
getStartOfLocalDayTimestamp,
|
||||
toDbTimestamp,
|
||||
@@ -759,6 +761,10 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => {
|
||||
assert.equal(dashboard.progress.watchTime[1]?.value, 75);
|
||||
assert.equal(dashboard.progress.lookups[1]?.value, 18);
|
||||
assert.equal(dashboard.ratios.lookupsPerHundred[0]?.value, +((8 / 120) * 100).toFixed(1));
|
||||
assert.equal(dashboard.ratios.cardsPerHour[0]?.value, +(2 / (30 / 60)).toFixed(1));
|
||||
assert.equal(dashboard.ratios.cardsPerHour[1]?.value, +(3 / (45 / 60)).toFixed(1));
|
||||
assert.equal(dashboard.ratios.readingSpeed[0]?.value, +(120 / 30).toFixed(1));
|
||||
assert.equal(dashboard.ratios.readingSpeed[1]?.value, +(140 / 45).toFixed(1));
|
||||
assert.equal(dashboard.librarySummary[0]?.title, 'Trend Dashboard Anime');
|
||||
assert.equal(dashboard.animeCumulative.watchTime[1]?.value, 75);
|
||||
assert.equal(
|
||||
@@ -771,6 +777,84 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('getTrendsDashboard redacts legacy Jellyfin stream titles', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const rawStreamTitle =
|
||||
'stream?static true&api key secret-token&MediaSourceId ms-1&AudioStreamIndex 3&SubtitleStreamIndex 4';
|
||||
const videoId = getOrCreateVideoRecord(
|
||||
db,
|
||||
'remote:http://jellyfin.local/Videos/item-1/stream?static=true&api_key=secret-token&MediaSourceId=ms-1&AudioStreamIndex=3&SubtitleStreamIndex=4',
|
||||
{
|
||||
canonicalTitle: rawStreamTitle,
|
||||
sourcePath: null,
|
||||
sourceUrl:
|
||||
'http://jellyfin.local/Videos/item-1/stream?static=true&api_key=secret-token&MediaSourceId=ms-1&AudioStreamIndex=3&SubtitleStreamIndex=4',
|
||||
sourceType: SOURCE_TYPE_REMOTE,
|
||||
},
|
||||
);
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: rawStreamTitle,
|
||||
canonicalTitle: rawStreamTitle,
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: null,
|
||||
});
|
||||
linkVideoToAnimeRecord(db, videoId, {
|
||||
animeId,
|
||||
parsedBasename:
|
||||
'stream?static=true&api_key=secret-token&MediaSourceId=ms-1&AudioStreamIndex=3&SubtitleStreamIndex=4',
|
||||
parsedTitle: rawStreamTitle,
|
||||
parsedSeason: null,
|
||||
parsedEpisode: null,
|
||||
parserSource: 'guessit',
|
||||
parserConfidence: 1,
|
||||
parseMetadataJson: null,
|
||||
});
|
||||
const startedAtMs = 1_700_000_000_000;
|
||||
const session = startSessionRecord(db, videoId, startedAtMs);
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE imm_sessions
|
||||
SET
|
||||
ended_at_ms = ?,
|
||||
total_watched_ms = ?,
|
||||
active_watched_ms = ?,
|
||||
tokens_seen = ?
|
||||
WHERE session_id = ?
|
||||
`,
|
||||
).run(`${startedAtMs + 30 * 60_000}`, 30 * 60_000, 30 * 60_000, 120, session.sessionId);
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_daily_rollups (
|
||||
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
).run(Math.floor(startedAtMs / 86_400_000), videoId, 1, 30, 10, 120, 0);
|
||||
|
||||
const dashboard = getTrendsDashboard(db, 'all', 'day');
|
||||
const titles = [
|
||||
...dashboard.animeCumulative.watchTime.map((point) => point.animeTitle),
|
||||
...dashboard.librarySummary.map((row) => row.title),
|
||||
];
|
||||
|
||||
assert.deepEqual([...new Set(titles)], ['Jellyfin Video']);
|
||||
assert.equal(titles.some((title) => title.includes('api_key=')), false);
|
||||
assert.equal(titles.some((title) => title.includes('api key')), false);
|
||||
assert.equal(titles.some((title) => title.includes('secret-token')), false);
|
||||
assert.equal(titles.some((title) => title.includes('stream?')), false);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getTrendsDashboard keeps local-midnight session buckets separate', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
@@ -3686,6 +3770,187 @@ test('getWordOccurrences maps a normalized word back to anime, video, and subtit
|
||||
}
|
||||
});
|
||||
|
||||
test('searchSubtitleSentences searches known subtitle lines and returns media context', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: 'Dungeon Meshi',
|
||||
canonicalTitle: 'Dungeon Meshi',
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: '{"source":"test"}',
|
||||
});
|
||||
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/dungeon-meshi-01.mkv', {
|
||||
canonicalTitle: 'Episode 1',
|
||||
sourcePath: '/tmp/Dungeon Meshi 01.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
linkVideoToAnimeRecord(db, videoId, {
|
||||
animeId,
|
||||
parsedBasename: 'Dungeon Meshi 01.mkv',
|
||||
parsedTitle: 'Dungeon Meshi',
|
||||
parsedSeason: 1,
|
||||
parsedEpisode: 1,
|
||||
parserSource: 'fallback',
|
||||
parserConfidence: 1,
|
||||
parseMetadataJson: '{"episode":1}',
|
||||
});
|
||||
const { sessionId } = startSessionRecord(db, videoId, 3_000_000);
|
||||
|
||||
db.prepare(
|
||||
`INSERT INTO imm_subtitle_lines (
|
||||
session_id, event_id, video_id, anime_id, line_index, segment_start_ms, segment_end_ms,
|
||||
text, secondary_text, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
).run(
|
||||
sessionId,
|
||||
null,
|
||||
videoId,
|
||||
animeId,
|
||||
7,
|
||||
4_000,
|
||||
5_500,
|
||||
'魔物を食べるなんて信じられない',
|
||||
'I cannot believe we are eating monsters',
|
||||
3_000,
|
||||
3_000,
|
||||
);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_subtitle_lines (
|
||||
session_id, event_id, video_id, anime_id, line_index, segment_start_ms, segment_end_ms,
|
||||
text, secondary_text, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
).run(
|
||||
sessionId,
|
||||
null,
|
||||
videoId,
|
||||
animeId,
|
||||
8,
|
||||
6_000,
|
||||
7_000,
|
||||
'これは別の行です',
|
||||
'Another line',
|
||||
2_000,
|
||||
2_000,
|
||||
);
|
||||
|
||||
const rows = searchSubtitleSentences(db, '魔物 食べる', 10);
|
||||
|
||||
assert.deepEqual(rows, [
|
||||
{
|
||||
animeId,
|
||||
animeTitle: 'Dungeon Meshi',
|
||||
sourcePath: '/tmp/Dungeon Meshi 01.mkv',
|
||||
secondaryText: 'I cannot believe we are eating monsters',
|
||||
videoId,
|
||||
videoTitle: 'Episode 1',
|
||||
sessionId,
|
||||
lineIndex: 7,
|
||||
segmentStartMs: 4_000,
|
||||
segmentEndMs: 5_500,
|
||||
text: '魔物を食べるなんて信じられない',
|
||||
},
|
||||
]);
|
||||
|
||||
assert.deepEqual(searchSubtitleSentences(db, 'monsters', 10), []);
|
||||
assert.doesNotThrow(() => searchSubtitleSentences(db, '魔物', Number.POSITIVE_INFINITY));
|
||||
assert.equal(searchSubtitleSentences(db, '魔物', -1).length, 1);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('searchSubtitleSentences searches subtitle lines by resolved headword candidates', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: 'Little Witch Academia',
|
||||
canonicalTitle: 'Little Witch Academia',
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: '{"source":"test"}',
|
||||
});
|
||||
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/lwa-05.mkv', {
|
||||
canonicalTitle: 'Episode 5',
|
||||
sourcePath: '/tmp/Little Witch Academia S01E05.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
linkVideoToAnimeRecord(db, videoId, {
|
||||
animeId,
|
||||
parsedBasename: 'Little Witch Academia S01E05.mkv',
|
||||
parsedTitle: 'Little Witch Academia',
|
||||
parsedSeason: 1,
|
||||
parsedEpisode: 5,
|
||||
parserSource: 'fallback',
|
||||
parserConfidence: 1,
|
||||
parseMetadataJson: '{"episode":5}',
|
||||
});
|
||||
const { sessionId } = startSessionRecord(db, videoId, 4_000_000);
|
||||
const lineResult = db
|
||||
.prepare(
|
||||
`INSERT INTO imm_subtitle_lines (
|
||||
session_id, event_id, video_id, anime_id, line_index, segment_start_ms, segment_end_ms,
|
||||
text, secondary_text, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run(
|
||||
sessionId,
|
||||
null,
|
||||
videoId,
|
||||
animeId,
|
||||
20,
|
||||
247_000,
|
||||
250_000,
|
||||
'ああ、名無しが何だか知らねえが',
|
||||
null,
|
||||
4_000,
|
||||
4_000,
|
||||
);
|
||||
const wordResult = db
|
||||
.prepare(
|
||||
`INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run('知る', '知らねえ', 'しらねえ', 'verb', '動詞', '自立', '', 4_000, 4_000, 1);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count)
|
||||
VALUES (?, ?, ?)`,
|
||||
).run(Number(lineResult.lastInsertRowid), Number(wordResult.lastInsertRowid), 1);
|
||||
|
||||
assert.deepEqual(searchSubtitleSentences(db, '知らない', 10), []);
|
||||
|
||||
const rows = searchSubtitleSentences(db, '知らない', 10, {
|
||||
headwordTerms: [{ term: '知らない', headwords: ['知る'] }],
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
rows.map((row) => row.text),
|
||||
['ああ、名無しが何だか知らねえが'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
searchSubtitleSentences(db, '知らねえ', 10).map((row) => row.text),
|
||||
['ああ、名無しが何だか知らねえが'],
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getKanjiOccurrences maps a kanji back to anime, video, and subtitle line context', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
@@ -4100,8 +4365,14 @@ test('deleteSession removes zero-session media from library and trends', () => {
|
||||
|
||||
const startedAtMs = 9_000_000;
|
||||
const endedAtMs = startedAtMs + 120_000;
|
||||
const rollupDay = Math.floor(startedAtMs / 86_400_000);
|
||||
const rollupMonth = 197001;
|
||||
const rollupDay = getLocalEpochDay(db, startedAtMs);
|
||||
const rollupMonth = (
|
||||
db
|
||||
.prepare(
|
||||
"SELECT CAST(strftime('%Y%m', CAST(? AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) AS rollupMonth",
|
||||
)
|
||||
.get(startedAtMs) as { rollupMonth: number }
|
||||
).rollupMonth;
|
||||
const { sessionId } = startSessionRecord(db, videoId, startedAtMs);
|
||||
|
||||
db.prepare(
|
||||
|
||||
@@ -0,0 +1,413 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { normalizeText } from './reducer';
|
||||
import { normalizeAnimeIdentityKey } from './storage';
|
||||
import { nowMs } from './time';
|
||||
import { toDbTimestamp } from './query-shared';
|
||||
import type { JellyfinLinkRepairSummary } from './types';
|
||||
|
||||
type LegacyJellyfinVideoRow = {
|
||||
video_id: number;
|
||||
video_key: string;
|
||||
source_url: string | null;
|
||||
canonical_title: string;
|
||||
};
|
||||
|
||||
type JellyfinTargetVideoRow = {
|
||||
video_id: number;
|
||||
anime_id: number | null;
|
||||
canonical_title: string;
|
||||
parsed_basename: string | null;
|
||||
parsed_title: string | null;
|
||||
parsed_season: number | null;
|
||||
parsed_episode: number | null;
|
||||
parser_source: string | null;
|
||||
parser_confidence: number | null;
|
||||
parse_metadata_json: string | null;
|
||||
};
|
||||
|
||||
type LeakedAnimeTitleRow = {
|
||||
anime_id: number;
|
||||
canonical_title: string;
|
||||
normalized_title_key: string;
|
||||
title_romaji: string | null;
|
||||
title_english: string | null;
|
||||
title_native: string | null;
|
||||
linked_video_title: string | null;
|
||||
};
|
||||
|
||||
function looksLikeLeakedJellyfinTitle(value: string | null): boolean {
|
||||
if (!value) return false;
|
||||
const lowered = value.toLowerCase();
|
||||
const hasApiKey = /api[\s_-]*key(?:\s|=|$)/i.test(value);
|
||||
return (
|
||||
hasApiKey &&
|
||||
(lowered.includes('stream?') ||
|
||||
lowered.includes('/stream?') ||
|
||||
lowered.includes('/videos/') ||
|
||||
lowered.includes('mediasourceid'))
|
||||
);
|
||||
}
|
||||
|
||||
function chooseSafeAnimeTitle(row: LeakedAnimeTitleRow): string | null {
|
||||
const candidates = [
|
||||
row.title_english,
|
||||
row.title_romaji,
|
||||
row.title_native,
|
||||
row.linked_video_title?.replace(/^\[Jellyfin\/direct]\s*/i, ''),
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
const normalized = candidate?.trim();
|
||||
if (normalized && !looksLikeLeakedJellyfinTitle(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseLegacyJellyfinStreamUrl(value: string | null): URL | null {
|
||||
if (!value) return null;
|
||||
const trimmed = value.trim();
|
||||
const urlText = trimmed.startsWith('remote:') ? trimmed.slice('remote:'.length) : trimmed;
|
||||
try {
|
||||
const url = new URL(urlText);
|
||||
const pathSegments = url.pathname.split('/').filter(Boolean);
|
||||
const videosIndex = pathSegments.findIndex((segment) => segment.toLowerCase() === 'videos');
|
||||
if (
|
||||
videosIndex < 0 ||
|
||||
pathSegments[videosIndex + 1] === undefined ||
|
||||
pathSegments[videosIndex + 2]?.toLowerCase() !== 'stream'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
if (!url.searchParams.has('api_key')) {
|
||||
return null;
|
||||
}
|
||||
return url;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function buildJellyfinStatsUrlFromLegacyStream(url: URL): string | null {
|
||||
const pathSegments = url.pathname.split('/').filter(Boolean);
|
||||
const videosIndex = pathSegments.findIndex((segment) => segment.toLowerCase() === 'videos');
|
||||
const itemId = normalizeText(pathSegments[videosIndex + 1]);
|
||||
if (!itemId) return null;
|
||||
return `jellyfin://${url.host}/item/${encodeURIComponent(itemId)}`;
|
||||
}
|
||||
|
||||
function buildSanitizedJellyfinVideoKey(
|
||||
db: DatabaseSync,
|
||||
videoId: number,
|
||||
statsUrl: string,
|
||||
): string {
|
||||
const baseKey = `remote:${statsUrl}`;
|
||||
const existing = db
|
||||
.prepare('SELECT video_id FROM imm_videos WHERE video_key = ?')
|
||||
.get(baseKey) as { video_id: number } | null;
|
||||
if (!existing || existing.video_id === videoId) {
|
||||
return baseKey;
|
||||
}
|
||||
return `${baseKey}#legacy-${videoId}`;
|
||||
}
|
||||
|
||||
function repairLeakedJellyfinAnimeTitles(db: DatabaseSync, currentTimestamp: string): number {
|
||||
const candidates = (
|
||||
db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
a.anime_id,
|
||||
a.normalized_title_key,
|
||||
a.canonical_title,
|
||||
a.title_romaji,
|
||||
a.title_english,
|
||||
a.title_native,
|
||||
(
|
||||
SELECT v.canonical_title
|
||||
FROM imm_videos v
|
||||
WHERE v.anime_id = a.anime_id
|
||||
AND v.canonical_title NOT LIKE '%api_key=%'
|
||||
AND lower(v.canonical_title) NOT LIKE '%api key%'
|
||||
ORDER BY v.LAST_UPDATE_DATE DESC, v.video_id DESC
|
||||
LIMIT 1
|
||||
) AS linked_video_title
|
||||
FROM imm_anime a
|
||||
WHERE a.canonical_title LIKE '%api_key=%'
|
||||
OR lower(a.canonical_title) LIKE '%api key%'
|
||||
OR lower(a.normalized_title_key) LIKE '%api key%'
|
||||
`,
|
||||
)
|
||||
.all() as LeakedAnimeTitleRow[]
|
||||
).filter(
|
||||
(row) =>
|
||||
looksLikeLeakedJellyfinTitle(row.canonical_title) ||
|
||||
looksLikeLeakedJellyfinTitle(row.normalized_title_key),
|
||||
);
|
||||
|
||||
let repaired = 0;
|
||||
for (const candidate of candidates) {
|
||||
const replacementTitle = chooseSafeAnimeTitle(candidate);
|
||||
if (!replacementTitle) {
|
||||
continue;
|
||||
}
|
||||
const replacementKey = normalizeAnimeIdentityKey(replacementTitle);
|
||||
if (!replacementKey) {
|
||||
continue;
|
||||
}
|
||||
const existing = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT anime_id
|
||||
FROM imm_anime
|
||||
WHERE normalized_title_key = ?
|
||||
AND anime_id != ?
|
||||
`,
|
||||
)
|
||||
.get(replacementKey, candidate.anime_id) as { anime_id: number } | null;
|
||||
if (existing) {
|
||||
const videoUpdate = db
|
||||
.prepare(
|
||||
`
|
||||
UPDATE imm_videos
|
||||
SET anime_id = ?, LAST_UPDATE_DATE = ?
|
||||
WHERE anime_id = ?
|
||||
`,
|
||||
)
|
||||
.run(existing.anime_id, currentTimestamp, candidate.anime_id) as { changes: number };
|
||||
const subtitleUpdate = db
|
||||
.prepare(
|
||||
`
|
||||
UPDATE imm_subtitle_lines
|
||||
SET anime_id = ?, LAST_UPDATE_DATE = ?
|
||||
WHERE anime_id = ?
|
||||
`,
|
||||
)
|
||||
.run(existing.anime_id, currentTimestamp, candidate.anime_id) as { changes: number };
|
||||
const animeDelete = db
|
||||
.prepare(
|
||||
`
|
||||
DELETE FROM imm_anime
|
||||
WHERE anime_id = ?
|
||||
AND NOT EXISTS (SELECT 1 FROM imm_videos WHERE anime_id = ?)
|
||||
AND NOT EXISTS (SELECT 1 FROM imm_subtitle_lines WHERE anime_id = ?)
|
||||
`,
|
||||
)
|
||||
.run(candidate.anime_id, candidate.anime_id, candidate.anime_id) as { changes: number };
|
||||
if (videoUpdate.changes > 0 || subtitleUpdate.changes > 0) {
|
||||
repaired += 1;
|
||||
} else if (animeDelete.changes > 0) {
|
||||
repaired += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const updated = db
|
||||
.prepare(
|
||||
`
|
||||
UPDATE imm_anime
|
||||
SET
|
||||
normalized_title_key = ?,
|
||||
canonical_title = ?,
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE anime_id = ?
|
||||
`,
|
||||
)
|
||||
.run(replacementKey, replacementTitle, currentTimestamp, candidate.anime_id) as {
|
||||
changes: number;
|
||||
};
|
||||
if (updated.changes > 0) {
|
||||
repaired += 1;
|
||||
}
|
||||
}
|
||||
return repaired;
|
||||
}
|
||||
|
||||
function repairLeakedJellyfinVideoParseMetadata(
|
||||
db: DatabaseSync,
|
||||
currentTimestamp: string,
|
||||
): number {
|
||||
const updated = db
|
||||
.prepare(
|
||||
`
|
||||
UPDATE imm_videos
|
||||
SET
|
||||
parsed_basename = NULL,
|
||||
parsed_title = NULL,
|
||||
parse_metadata_json = NULL,
|
||||
parser_source = CASE
|
||||
WHEN parser_source = 'guessit' THEN 'jellyfin'
|
||||
ELSE parser_source
|
||||
END,
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE source_type = 2
|
||||
AND (
|
||||
parsed_basename LIKE '%api_key=%'
|
||||
OR lower(parsed_basename) LIKE '%api key%'
|
||||
OR parsed_title LIKE '%api_key=%'
|
||||
OR lower(parsed_title) LIKE '%api key%'
|
||||
OR parse_metadata_json LIKE '%api_key=%'
|
||||
OR lower(parse_metadata_json) LIKE '%api key%'
|
||||
)
|
||||
`,
|
||||
)
|
||||
.run(currentTimestamp) as { changes: number };
|
||||
return updated.changes;
|
||||
}
|
||||
|
||||
export function repairJellyfinStreamVideoLinks(db: DatabaseSync): JellyfinLinkRepairSummary {
|
||||
const candidates = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT video_id, video_key, source_url, canonical_title
|
||||
FROM imm_videos
|
||||
WHERE source_type = 2
|
||||
AND (
|
||||
video_key LIKE '%api_key=%'
|
||||
OR lower(video_key) LIKE '%api key%'
|
||||
OR source_url LIKE '%api_key=%'
|
||||
OR lower(source_url) LIKE '%api key%'
|
||||
OR canonical_title LIKE '%api_key=%'
|
||||
OR lower(canonical_title) LIKE '%api key%'
|
||||
)
|
||||
`,
|
||||
)
|
||||
.all() as LegacyJellyfinVideoRow[];
|
||||
|
||||
const summary: JellyfinLinkRepairSummary = {
|
||||
scanned: candidates.length,
|
||||
repaired: 0,
|
||||
};
|
||||
if (candidates.length === 0) {
|
||||
const currentTimestamp = toDbTimestamp(nowMs());
|
||||
const repaired =
|
||||
repairLeakedJellyfinAnimeTitles(db, currentTimestamp) +
|
||||
repairLeakedJellyfinVideoParseMetadata(db, currentTimestamp);
|
||||
summary.repaired += repaired;
|
||||
return summary;
|
||||
}
|
||||
|
||||
const currentTimestamp = toDbTimestamp(nowMs());
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
for (const candidate of candidates) {
|
||||
const legacyUrl =
|
||||
parseLegacyJellyfinStreamUrl(candidate.source_url) ??
|
||||
parseLegacyJellyfinStreamUrl(candidate.video_key);
|
||||
if (!legacyUrl) {
|
||||
continue;
|
||||
}
|
||||
const statsUrl = buildJellyfinStatsUrlFromLegacyStream(legacyUrl);
|
||||
if (!statsUrl) {
|
||||
continue;
|
||||
}
|
||||
const sanitizedVideoKey = buildSanitizedJellyfinVideoKey(db, candidate.video_id, statsUrl);
|
||||
const sanitizedCanonicalTitle = looksLikeLeakedJellyfinTitle(candidate.canonical_title)
|
||||
? 'Jellyfin Video'
|
||||
: candidate.canonical_title;
|
||||
const target = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
video_id,
|
||||
anime_id,
|
||||
canonical_title,
|
||||
parsed_basename,
|
||||
parsed_title,
|
||||
parsed_season,
|
||||
parsed_episode,
|
||||
parser_source,
|
||||
parser_confidence,
|
||||
parse_metadata_json
|
||||
FROM imm_videos
|
||||
WHERE video_id != ?
|
||||
AND (video_key = ? OR source_url = ?)
|
||||
ORDER BY parser_source = 'jellyfin' DESC, video_id DESC
|
||||
LIMIT 1
|
||||
`,
|
||||
)
|
||||
.get(candidate.video_id, `remote:${statsUrl}`, statsUrl) as JellyfinTargetVideoRow | null;
|
||||
if (!target) {
|
||||
const updated = db
|
||||
.prepare(
|
||||
`
|
||||
UPDATE imm_videos
|
||||
SET
|
||||
video_key = ?,
|
||||
source_url = ?,
|
||||
canonical_title = ?,
|
||||
parser_source = COALESCE(parser_source, 'jellyfin'),
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE video_id = ?
|
||||
AND (video_key != ? OR source_url != ? OR canonical_title != ?)
|
||||
`,
|
||||
)
|
||||
.run(
|
||||
sanitizedVideoKey,
|
||||
statsUrl,
|
||||
sanitizedCanonicalTitle,
|
||||
currentTimestamp,
|
||||
candidate.video_id,
|
||||
sanitizedVideoKey,
|
||||
statsUrl,
|
||||
sanitizedCanonicalTitle,
|
||||
) as { changes: number };
|
||||
if (updated.changes > 0) {
|
||||
summary.repaired += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE imm_videos
|
||||
SET
|
||||
video_key = ?,
|
||||
anime_id = ?,
|
||||
canonical_title = ?,
|
||||
source_url = ?,
|
||||
parsed_basename = ?,
|
||||
parsed_title = ?,
|
||||
parsed_season = ?,
|
||||
parsed_episode = ?,
|
||||
parser_source = ?,
|
||||
parser_confidence = ?,
|
||||
parse_metadata_json = ?,
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE video_id = ?
|
||||
`,
|
||||
).run(
|
||||
sanitizedVideoKey,
|
||||
target.anime_id,
|
||||
target.canonical_title,
|
||||
statsUrl,
|
||||
target.parsed_basename,
|
||||
target.parsed_title,
|
||||
target.parsed_season,
|
||||
target.parsed_episode,
|
||||
target.parser_source,
|
||||
target.parser_confidence,
|
||||
target.parse_metadata_json,
|
||||
currentTimestamp,
|
||||
candidate.video_id,
|
||||
);
|
||||
if (target.anime_id !== null) {
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE imm_subtitle_lines
|
||||
SET anime_id = ?, LAST_UPDATE_DATE = ?
|
||||
WHERE video_id = ?
|
||||
`,
|
||||
).run(target.anime_id, currentTimestamp, candidate.video_id);
|
||||
}
|
||||
summary.repaired += 1;
|
||||
}
|
||||
summary.repaired += repairLeakedJellyfinAnimeTitles(db, currentTimestamp);
|
||||
summary.repaired += repairLeakedJellyfinVideoParseMetadata(db, currentTimestamp);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
|
||||
return summary;
|
||||
}
|
||||
@@ -60,6 +60,34 @@ interface RetainedSessionRow {
|
||||
mediaBufferEvents: number;
|
||||
}
|
||||
|
||||
const RETAINED_SESSION_METRICS_CTE = `
|
||||
retained_sessions AS (
|
||||
SELECT
|
||||
s.session_id,
|
||||
s.video_id,
|
||||
v.anime_id,
|
||||
s.started_at_ms,
|
||||
s.ended_at_ms,
|
||||
MAX(COALESCE(t.active_watched_ms, s.active_watched_ms, 0), 0) AS active_ms,
|
||||
MAX(COALESCE(t.cards_mined, s.cards_mined, 0), 0) AS cards_mined,
|
||||
MAX(COALESCE(t.lines_seen, s.lines_seen, 0), 0) AS lines_seen,
|
||||
MAX(COALESCE(t.tokens_seen, s.tokens_seen, 0), 0) AS tokens_seen,
|
||||
CASE WHEN v.watched > 0 THEN 1 ELSE 0 END AS completed
|
||||
FROM imm_sessions s
|
||||
JOIN imm_videos v
|
||||
ON v.video_id = s.video_id
|
||||
LEFT JOIN imm_session_telemetry t
|
||||
ON t.telemetry_id = (
|
||||
SELECT telemetry_id
|
||||
FROM imm_session_telemetry
|
||||
WHERE session_id = s.session_id
|
||||
ORDER BY sample_ms DESC, telemetry_id DESC
|
||||
LIMIT 1
|
||||
)
|
||||
WHERE s.ended_at_ms IS NOT NULL
|
||||
)
|
||||
`;
|
||||
|
||||
function hasRetainedPriorSession(
|
||||
db: DatabaseSync,
|
||||
videoId: number,
|
||||
@@ -154,54 +182,150 @@ function rebuildLifetimeSummariesInternal(
|
||||
db: DatabaseSync,
|
||||
rebuiltAtMs: number,
|
||||
): LifetimeRebuildSummary {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
session_id AS sessionId,
|
||||
video_id AS videoId,
|
||||
started_at_ms AS startedAtMs,
|
||||
ended_at_ms AS endedAtMs,
|
||||
ended_media_ms AS lastMediaMs,
|
||||
total_watched_ms AS totalWatchedMs,
|
||||
active_watched_ms AS activeWatchedMs,
|
||||
lines_seen AS linesSeen,
|
||||
tokens_seen AS tokensSeen,
|
||||
cards_mined AS cardsMined,
|
||||
lookup_count AS lookupCount,
|
||||
lookup_hits AS lookupHits,
|
||||
yomitan_lookup_count AS yomitanLookupCount,
|
||||
pause_count AS pauseCount,
|
||||
pause_ms AS pauseMs,
|
||||
seek_forward_count AS seekForwardCount,
|
||||
seek_backward_count AS seekBackwardCount,
|
||||
media_buffer_events AS mediaBufferEvents
|
||||
FROM imm_sessions
|
||||
WHERE ended_at_ms IS NOT NULL
|
||||
ORDER BY started_at_ms ASC, session_id ASC
|
||||
`,
|
||||
)
|
||||
.all() as Array<
|
||||
Omit<RetainedSessionRow, 'startedAtMs' | 'endedAtMs' | 'lastMediaMs'> & {
|
||||
startedAtMs: number | string;
|
||||
endedAtMs: number | string;
|
||||
lastMediaMs: number | string | null;
|
||||
}
|
||||
>;
|
||||
const sessions = rows.map((row) => ({
|
||||
...row,
|
||||
startedAtMs: row.startedAtMs,
|
||||
endedAtMs: row.endedAtMs,
|
||||
lastMediaMs: row.lastMediaMs === null ? null : Number(row.lastMediaMs),
|
||||
})) as RetainedSessionRow[];
|
||||
const rebuiltAtDbMs = toDbTimestamp(rebuiltAtMs);
|
||||
const appliedSessions = Number(
|
||||
(
|
||||
db
|
||||
.prepare('SELECT COUNT(*) AS total FROM imm_sessions WHERE ended_at_ms IS NOT NULL')
|
||||
.get() as { total: number }
|
||||
).total,
|
||||
);
|
||||
|
||||
resetLifetimeSummaries(db, rebuiltAtMs);
|
||||
for (const session of sessions) {
|
||||
applySessionLifetimeSummary(db, toRebuildSessionState(session), session.endedAtMs);
|
||||
}
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_lifetime_applied_sessions (
|
||||
session_id,
|
||||
applied_at_ms,
|
||||
CREATED_DATE,
|
||||
LAST_UPDATE_DATE
|
||||
)
|
||||
SELECT
|
||||
session_id,
|
||||
ended_at_ms,
|
||||
?,
|
||||
?
|
||||
FROM imm_sessions
|
||||
WHERE ended_at_ms IS NOT NULL
|
||||
`,
|
||||
).run(rebuiltAtDbMs, rebuiltAtDbMs);
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
WITH ${RETAINED_SESSION_METRICS_CTE}
|
||||
INSERT INTO imm_lifetime_media (
|
||||
video_id,
|
||||
total_sessions,
|
||||
total_active_ms,
|
||||
total_cards,
|
||||
total_lines_seen,
|
||||
total_tokens_seen,
|
||||
completed,
|
||||
first_watched_ms,
|
||||
last_watched_ms,
|
||||
CREATED_DATE,
|
||||
LAST_UPDATE_DATE
|
||||
)
|
||||
SELECT
|
||||
video_id,
|
||||
COUNT(*) AS total_sessions,
|
||||
COALESCE(SUM(active_ms), 0) AS total_active_ms,
|
||||
COALESCE(SUM(cards_mined), 0) AS total_cards,
|
||||
COALESCE(SUM(lines_seen), 0) AS total_lines_seen,
|
||||
COALESCE(SUM(tokens_seen), 0) AS total_tokens_seen,
|
||||
MAX(completed) AS completed,
|
||||
MIN(started_at_ms) AS first_watched_ms,
|
||||
MAX(ended_at_ms) AS last_watched_ms,
|
||||
? AS CREATED_DATE,
|
||||
? AS LAST_UPDATE_DATE
|
||||
FROM retained_sessions
|
||||
GROUP BY video_id
|
||||
`,
|
||||
).run(rebuiltAtDbMs, rebuiltAtDbMs);
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
WITH ${RETAINED_SESSION_METRICS_CTE}
|
||||
INSERT INTO imm_lifetime_anime (
|
||||
anime_id,
|
||||
total_sessions,
|
||||
total_active_ms,
|
||||
total_cards,
|
||||
total_lines_seen,
|
||||
total_tokens_seen,
|
||||
episodes_started,
|
||||
episodes_completed,
|
||||
first_watched_ms,
|
||||
last_watched_ms,
|
||||
CREATED_DATE,
|
||||
LAST_UPDATE_DATE
|
||||
)
|
||||
SELECT
|
||||
anime_id,
|
||||
COUNT(*) AS total_sessions,
|
||||
COALESCE(SUM(active_ms), 0) AS total_active_ms,
|
||||
COALESCE(SUM(cards_mined), 0) AS total_cards,
|
||||
COALESCE(SUM(lines_seen), 0) AS total_lines_seen,
|
||||
COALESCE(SUM(tokens_seen), 0) AS total_tokens_seen,
|
||||
COUNT(DISTINCT video_id) AS episodes_started,
|
||||
COUNT(DISTINCT CASE WHEN completed > 0 THEN video_id END) AS episodes_completed,
|
||||
MIN(started_at_ms) AS first_watched_ms,
|
||||
MAX(ended_at_ms) AS last_watched_ms,
|
||||
? AS CREATED_DATE,
|
||||
? AS LAST_UPDATE_DATE
|
||||
FROM retained_sessions
|
||||
WHERE anime_id IS NOT NULL
|
||||
GROUP BY anime_id
|
||||
`,
|
||||
).run(rebuiltAtDbMs, rebuiltAtDbMs);
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
WITH ${RETAINED_SESSION_METRICS_CTE},
|
||||
anime_completion AS (
|
||||
SELECT
|
||||
rs.anime_id,
|
||||
MAX(a.episodes_total) AS episodes_total,
|
||||
COUNT(DISTINCT CASE WHEN rs.completed > 0 THEN rs.video_id END) AS completed_videos
|
||||
FROM retained_sessions rs
|
||||
JOIN imm_anime a
|
||||
ON a.anime_id = rs.anime_id
|
||||
WHERE rs.anime_id IS NOT NULL
|
||||
GROUP BY rs.anime_id
|
||||
)
|
||||
UPDATE imm_lifetime_global
|
||||
SET
|
||||
total_sessions = (SELECT COUNT(*) FROM retained_sessions),
|
||||
total_active_ms = (SELECT COALESCE(SUM(active_ms), 0) FROM retained_sessions),
|
||||
total_cards = (SELECT COALESCE(SUM(cards_mined), 0) FROM retained_sessions),
|
||||
active_days = (
|
||||
SELECT COUNT(DISTINCT CAST(
|
||||
julianday(CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5
|
||||
AS INTEGER
|
||||
))
|
||||
FROM retained_sessions
|
||||
),
|
||||
episodes_started = (SELECT COUNT(DISTINCT video_id) FROM retained_sessions),
|
||||
episodes_completed = (
|
||||
SELECT COUNT(DISTINCT CASE WHEN completed > 0 THEN video_id END)
|
||||
FROM retained_sessions
|
||||
),
|
||||
anime_completed = (
|
||||
SELECT COUNT(*)
|
||||
FROM anime_completion
|
||||
WHERE episodes_total IS NOT NULL
|
||||
AND episodes_total > 0
|
||||
AND completed_videos >= episodes_total
|
||||
),
|
||||
last_rebuilt_ms = ?,
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE global_id = 1
|
||||
`,
|
||||
).run(rebuiltAtDbMs, rebuiltAtDbMs);
|
||||
|
||||
return {
|
||||
appliedSessions: sessions.length,
|
||||
appliedSessions,
|
||||
rebuiltAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { nowMs } from './time';
|
||||
import { subtractDbTimestamp, toDbTimestamp } from './query-shared';
|
||||
import { makePlaceholders, subtractDbTimestamp, toDbTimestamp } from './query-shared';
|
||||
|
||||
const ROLLUP_STATE_KEY = 'last_rollup_sample_ms';
|
||||
const DAILY_MS = 86_400_000;
|
||||
@@ -20,6 +20,12 @@ interface RollupTelemetryResult {
|
||||
maxSampleMs: number | null;
|
||||
}
|
||||
|
||||
export interface RollupGroup {
|
||||
rollupDay: number;
|
||||
rollupMonth: number;
|
||||
videoId: number;
|
||||
}
|
||||
|
||||
interface RawRetentionResult {
|
||||
deletedSessionEvents: number;
|
||||
deletedTelemetryRows: number;
|
||||
@@ -164,6 +170,26 @@ function upsertDailyRollupsForGroups(
|
||||
}
|
||||
|
||||
const upsertStmt = db.prepare(`
|
||||
WITH matching_sessions AS (
|
||||
SELECT *
|
||||
FROM imm_sessions
|
||||
WHERE CAST(julianday(CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) = ?
|
||||
AND video_id = ?
|
||||
),
|
||||
session_metrics AS (
|
||||
SELECT
|
||||
t.session_id,
|
||||
MAX(t.active_watched_ms) AS max_active_ms,
|
||||
MAX(t.lines_seen) AS max_lines,
|
||||
MAX(t.tokens_seen) AS max_tokens,
|
||||
MAX(t.cards_mined) AS max_cards,
|
||||
MAX(t.lookup_count) AS max_lookups,
|
||||
MAX(t.lookup_hits) AS max_hits
|
||||
FROM imm_session_telemetry t
|
||||
JOIN matching_sessions s
|
||||
ON s.session_id = t.session_id
|
||||
GROUP BY t.session_id
|
||||
)
|
||||
INSERT INTO imm_daily_rollups (
|
||||
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards, cards_per_hour,
|
||||
@@ -197,20 +223,8 @@ function upsertDailyRollupsForGroups(
|
||||
END AS lookup_hit_rate,
|
||||
? AS CREATED_DATE,
|
||||
? AS LAST_UPDATE_DATE
|
||||
FROM imm_sessions s
|
||||
LEFT JOIN (
|
||||
SELECT
|
||||
t.session_id,
|
||||
MAX(t.active_watched_ms) AS max_active_ms,
|
||||
MAX(t.lines_seen) AS max_lines,
|
||||
MAX(t.tokens_seen) AS max_tokens,
|
||||
MAX(t.cards_mined) AS max_cards,
|
||||
MAX(t.lookup_count) AS max_lookups,
|
||||
MAX(t.lookup_hits) AS max_hits
|
||||
FROM imm_session_telemetry t
|
||||
GROUP BY t.session_id
|
||||
) sm ON s.session_id = sm.session_id
|
||||
WHERE CAST(julianday(s.started_at_ms / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) = ? AND s.video_id = ?
|
||||
FROM matching_sessions s
|
||||
LEFT JOIN session_metrics sm ON s.session_id = sm.session_id
|
||||
GROUP BY rollup_day, s.video_id
|
||||
ON CONFLICT (rollup_day, video_id) DO UPDATE SET
|
||||
total_sessions = excluded.total_sessions,
|
||||
@@ -226,7 +240,7 @@ function upsertDailyRollupsForGroups(
|
||||
`);
|
||||
|
||||
for (const { rollupDay, videoId } of groups) {
|
||||
upsertStmt.run(rollupNowMs, rollupNowMs, rollupDay, videoId);
|
||||
upsertStmt.run(rollupDay, videoId, rollupNowMs, rollupNowMs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,6 +254,24 @@ function upsertMonthlyRollupsForGroups(
|
||||
}
|
||||
|
||||
const upsertStmt = db.prepare(`
|
||||
WITH matching_sessions AS (
|
||||
SELECT *
|
||||
FROM imm_sessions
|
||||
WHERE CAST(strftime('%Y%m', CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) = ?
|
||||
AND video_id = ?
|
||||
),
|
||||
session_metrics AS (
|
||||
SELECT
|
||||
t.session_id,
|
||||
MAX(t.active_watched_ms) AS max_active_ms,
|
||||
MAX(t.lines_seen) AS max_lines,
|
||||
MAX(t.tokens_seen) AS max_tokens,
|
||||
MAX(t.cards_mined) AS max_cards
|
||||
FROM imm_session_telemetry t
|
||||
JOIN matching_sessions s
|
||||
ON s.session_id = t.session_id
|
||||
GROUP BY t.session_id
|
||||
)
|
||||
INSERT INTO imm_monthly_rollups (
|
||||
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
|
||||
@@ -254,18 +286,8 @@ function upsertMonthlyRollupsForGroups(
|
||||
COALESCE(SUM(COALESCE(sm.max_cards, s.cards_mined)), 0) AS total_cards,
|
||||
? AS CREATED_DATE,
|
||||
? AS LAST_UPDATE_DATE
|
||||
FROM imm_sessions s
|
||||
LEFT JOIN (
|
||||
SELECT
|
||||
t.session_id,
|
||||
MAX(t.active_watched_ms) AS max_active_ms,
|
||||
MAX(t.lines_seen) AS max_lines,
|
||||
MAX(t.tokens_seen) AS max_tokens,
|
||||
MAX(t.cards_mined) AS max_cards
|
||||
FROM imm_session_telemetry t
|
||||
GROUP BY t.session_id
|
||||
) sm ON s.session_id = sm.session_id
|
||||
WHERE CAST(strftime('%Y%m', s.started_at_ms / 1000, 'unixepoch', 'localtime') AS INTEGER) = ? AND s.video_id = ?
|
||||
FROM matching_sessions s
|
||||
LEFT JOIN session_metrics sm ON s.session_id = sm.session_id
|
||||
GROUP BY rollup_month, s.video_id
|
||||
ON CONFLICT (rollup_month, video_id) DO UPDATE SET
|
||||
total_sessions = excluded.total_sessions,
|
||||
@@ -278,10 +300,75 @@ function upsertMonthlyRollupsForGroups(
|
||||
`);
|
||||
|
||||
for (const { rollupMonth, videoId } of groups) {
|
||||
upsertStmt.run(rollupNowMs, rollupNowMs, rollupMonth, videoId);
|
||||
upsertStmt.run(rollupMonth, videoId, rollupNowMs, rollupNowMs);
|
||||
}
|
||||
}
|
||||
|
||||
export function getRollupGroupsForSessions(db: DatabaseSync, sessionIds: number[]): RollupGroup[] {
|
||||
if (sessionIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const placeholders = makePlaceholders(sessionIds);
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT DISTINCT
|
||||
CAST(julianday(CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) AS rollup_day,
|
||||
CAST(strftime('%Y%m', CAST(started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) AS rollup_month,
|
||||
video_id
|
||||
FROM imm_sessions
|
||||
WHERE session_id IN (${placeholders})
|
||||
`,
|
||||
)
|
||||
.all(...sessionIds) as RollupGroupRow[];
|
||||
|
||||
return rows.map((row) => ({
|
||||
rollupDay: row.rollup_day,
|
||||
rollupMonth: row.rollup_month,
|
||||
videoId: row.video_id,
|
||||
}));
|
||||
}
|
||||
|
||||
export function refreshRollupsForGroupsInTransaction(
|
||||
db: DatabaseSync,
|
||||
groups: RollupGroup[],
|
||||
): void {
|
||||
if (groups.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const rollupNowMs = toDbTimestamp(nowMs());
|
||||
const dailyGroups = dedupeGroups(
|
||||
groups.map((group) => ({
|
||||
rollupDay: group.rollupDay,
|
||||
videoId: group.videoId,
|
||||
})),
|
||||
);
|
||||
const monthlyGroups = dedupeGroups(
|
||||
groups.map((group) => ({
|
||||
rollupMonth: group.rollupMonth,
|
||||
videoId: group.videoId,
|
||||
})),
|
||||
);
|
||||
const deleteDailyStmt = db.prepare(
|
||||
'DELETE FROM imm_daily_rollups WHERE rollup_day = ? AND video_id = ?',
|
||||
);
|
||||
const deleteMonthlyStmt = db.prepare(
|
||||
'DELETE FROM imm_monthly_rollups WHERE rollup_month = ? AND video_id = ?',
|
||||
);
|
||||
|
||||
for (const { rollupDay, videoId } of dailyGroups) {
|
||||
deleteDailyStmt.run(rollupDay, videoId);
|
||||
}
|
||||
for (const { rollupMonth, videoId } of monthlyGroups) {
|
||||
deleteMonthlyStmt.run(rollupMonth, videoId);
|
||||
}
|
||||
|
||||
upsertDailyRollupsForGroups(db, dailyGroups, rollupNowMs);
|
||||
upsertMonthlyRollupsForGroups(db, monthlyGroups, rollupNowMs);
|
||||
}
|
||||
|
||||
function getAffectedRollupGroups(
|
||||
db: DatabaseSync,
|
||||
lastRollupSampleMs: number | string,
|
||||
|
||||
@@ -179,6 +179,32 @@ test('guessAnimeVideoMetadata uses guessit basename output first when available'
|
||||
});
|
||||
});
|
||||
|
||||
test('guessAnimeVideoMetadata keeps season directory scope when guessit omits season', async () => {
|
||||
const parsed = await guessAnimeVideoMetadata(
|
||||
'/tmp/KonoSuba/Season 2/KonoSuba - 05.mkv',
|
||||
'Episode 5',
|
||||
{
|
||||
runGuessit: async () =>
|
||||
JSON.stringify({
|
||||
title: 'KonoSuba',
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
assert.deepEqual(parsed, {
|
||||
parsedBasename: 'KonoSuba - 05.mkv',
|
||||
parsedTitle: 'KonoSuba',
|
||||
parsedSeason: 2,
|
||||
parsedEpisode: null,
|
||||
parserSource: 'guessit',
|
||||
parserConfidence: 1,
|
||||
parseMetadataJson: JSON.stringify({
|
||||
filename: 'KonoSuba - 05.mkv',
|
||||
source: 'guessit',
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
test('guessAnimeVideoMetadata falls back to parser when guessit throws', async () => {
|
||||
const parsed = await guessAnimeVideoMetadata(
|
||||
'/tmp/Little Witch Academia S02E05.mkv',
|
||||
|
||||
@@ -7,6 +7,8 @@ import type {
|
||||
KanjiOccurrenceRow,
|
||||
KanjiStatsRow,
|
||||
KanjiWordRow,
|
||||
SentenceSearchOptions,
|
||||
SentenceSearchResultRow,
|
||||
SessionEventRow,
|
||||
SimilarWordRow,
|
||||
StatsExcludedWordRow,
|
||||
@@ -20,6 +22,56 @@ import { nowMs } from './time';
|
||||
|
||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
|
||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
|
||||
const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
|
||||
const SENTENCE_SEARCH_MAX_LIMIT = 100;
|
||||
const KANJI_PATTERN = /\p{Script=Han}/gu;
|
||||
|
||||
function resolveSentenceSearchLimit(limit: number): number {
|
||||
if (!Number.isFinite(limit)) return SENTENCE_SEARCH_DEFAULT_LIMIT;
|
||||
const normalized = Math.floor(limit);
|
||||
if (normalized <= 0) return SENTENCE_SEARCH_DEFAULT_LIMIT;
|
||||
return Math.min(normalized, SENTENCE_SEARCH_MAX_LIMIT);
|
||||
}
|
||||
|
||||
export function splitSentenceSearchTerms(query: string): string[] {
|
||||
return query
|
||||
.trim()
|
||||
.split(/\s+/)
|
||||
.map((term) => term.trim())
|
||||
.filter(Boolean)
|
||||
.slice(0, 8);
|
||||
}
|
||||
|
||||
function escapeLikeTerm(term: string): string {
|
||||
return term.replace(/[\\%_]/g, (match) => `\\${match}`);
|
||||
}
|
||||
|
||||
function uniqueNonEmptyTerms(values: readonly string[] | undefined): string[] {
|
||||
const seen = new Set<string>();
|
||||
const terms: string[] = [];
|
||||
for (const value of values ?? []) {
|
||||
const term = value.trim();
|
||||
if (!term || seen.has(term)) continue;
|
||||
seen.add(term);
|
||||
terms.push(term);
|
||||
}
|
||||
return terms;
|
||||
}
|
||||
|
||||
function getHeadwordCandidatesForSentenceSearchTerm(
|
||||
term: string,
|
||||
options: SentenceSearchOptions | undefined,
|
||||
): string[] {
|
||||
const headwords =
|
||||
options?.headwordTerms
|
||||
?.filter((entry) => entry.term === term)
|
||||
.flatMap((entry) => entry.headwords) ?? [];
|
||||
return uniqueNonEmptyTerms(headwords);
|
||||
}
|
||||
|
||||
function uniqueKanji(text: string): string[] {
|
||||
return Array.from(new Set(text.match(KANJI_PATTERN) ?? []));
|
||||
}
|
||||
|
||||
function toVocabularyToken(row: VocabularyStatsRow): MergedToken {
|
||||
const partOfSpeech =
|
||||
@@ -211,6 +263,70 @@ export function getKanjiOccurrences(
|
||||
.all(kanji, limit, offset) as unknown as KanjiOccurrenceRow[];
|
||||
}
|
||||
|
||||
export function searchSubtitleSentences(
|
||||
db: DatabaseSync,
|
||||
query: string,
|
||||
limit = SENTENCE_SEARCH_DEFAULT_LIMIT,
|
||||
options?: SentenceSearchOptions,
|
||||
): SentenceSearchResultRow[] {
|
||||
const terms = splitSentenceSearchTerms(query);
|
||||
if (terms.length === 0) return [];
|
||||
const resolvedLimit = resolveSentenceSearchLimit(limit);
|
||||
|
||||
const clauses: string[] = [];
|
||||
const params: string[] = [];
|
||||
for (const term of terms) {
|
||||
const likeTerm = `%${escapeLikeTerm(term)}%`;
|
||||
const headwords = getHeadwordCandidatesForSentenceSearchTerm(term, options);
|
||||
const headwordClause =
|
||||
headwords.length > 0
|
||||
? `
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM imm_word_line_occurrences o
|
||||
JOIN imm_words w ON w.id = o.word_id
|
||||
WHERE o.line_id = l.line_id
|
||||
AND w.headword IN (${headwords.map(() => '?').join(', ')})
|
||||
)
|
||||
`
|
||||
: '';
|
||||
clauses.push(`
|
||||
(
|
||||
l.text LIKE ? ESCAPE '\\'
|
||||
OR v.canonical_title LIKE ? ESCAPE '\\'
|
||||
OR COALESCE(a.canonical_title, '') LIKE ? ESCAPE '\\'
|
||||
${headwordClause}
|
||||
)
|
||||
`);
|
||||
params.push(likeTerm, likeTerm, likeTerm, ...headwords);
|
||||
}
|
||||
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
l.anime_id AS animeId,
|
||||
a.canonical_title AS animeTitle,
|
||||
l.video_id AS videoId,
|
||||
v.canonical_title AS videoTitle,
|
||||
v.source_path AS sourcePath,
|
||||
l.secondary_text AS secondaryText,
|
||||
l.session_id AS sessionId,
|
||||
l.line_index AS lineIndex,
|
||||
l.segment_start_ms AS segmentStartMs,
|
||||
l.segment_end_ms AS segmentEndMs,
|
||||
l.text AS text
|
||||
FROM imm_subtitle_lines l
|
||||
JOIN imm_videos v ON v.video_id = l.video_id
|
||||
LEFT JOIN imm_anime a ON a.anime_id = l.anime_id
|
||||
WHERE ${clauses.join(' AND ')}
|
||||
ORDER BY l.CREATED_DATE DESC, l.line_id DESC
|
||||
LIMIT ?
|
||||
`,
|
||||
)
|
||||
.all(...params, resolvedLimit) as unknown as SentenceSearchResultRow[];
|
||||
}
|
||||
|
||||
export function getSessionEvents(
|
||||
db: DatabaseSync,
|
||||
sessionId: number,
|
||||
@@ -287,24 +403,38 @@ export function getSimilarWords(db: DatabaseSync, wordId: number, limit = 10): S
|
||||
reading: string;
|
||||
} | null;
|
||||
if (!word || word.headword.trim() === '') return [];
|
||||
|
||||
const clauses: string[] = [];
|
||||
const params: string[] = [];
|
||||
const reading = word.reading.trim();
|
||||
if (reading !== '') {
|
||||
clauses.push('reading = ?');
|
||||
params.push(word.reading);
|
||||
}
|
||||
|
||||
for (const kanji of uniqueKanji(word.headword)) {
|
||||
clauses.push("headword LIKE ? ESCAPE '\\'");
|
||||
params.push(`%${escapeLikeTerm(kanji)}%`);
|
||||
}
|
||||
|
||||
if (clauses.length === 0) return [];
|
||||
|
||||
const orderBy =
|
||||
reading !== '' ? 'CASE WHEN reading = ? THEN 0 ELSE 1 END, frequency DESC' : 'frequency DESC';
|
||||
const orderParams = reading !== '' ? [word.reading] : [];
|
||||
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id AS wordId, headword, word, reading, frequency
|
||||
FROM imm_words
|
||||
WHERE id != ?
|
||||
AND (reading = ? OR headword LIKE ? OR headword LIKE ?)
|
||||
ORDER BY frequency DESC
|
||||
AND (${clauses.join(' OR ')})
|
||||
ORDER BY ${orderBy}
|
||||
LIMIT ?
|
||||
`,
|
||||
)
|
||||
.all(
|
||||
wordId,
|
||||
word.reading,
|
||||
`%${word.headword.charAt(0)}%`,
|
||||
`%${word.headword.charAt(word.headword.length - 1)}%`,
|
||||
limit,
|
||||
) as SimilarWordRow[];
|
||||
.all(wordId, ...params, ...orderParams, limit) as SimilarWordRow[];
|
||||
}
|
||||
|
||||
export function getKanjiDetail(db: DatabaseSync, kanjiId: number): KanjiDetailRow | null {
|
||||
|
||||
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { buildCoverBlobReference, normalizeCoverBlobBytes } from './storage';
|
||||
import { rebuildLifetimeSummariesInTransaction } from './lifetime';
|
||||
import { rebuildRollupsInTransaction } from './maintenance';
|
||||
import { getRollupGroupsForSessions, refreshRollupsForGroupsInTransaction } from './maintenance';
|
||||
import { nowMs } from './time';
|
||||
import { PartOfSpeech, type MergedToken } from '../../../types';
|
||||
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
|
||||
@@ -474,13 +474,14 @@ export function deleteSession(db: DatabaseSync, sessionId: number): void {
|
||||
const sessionIds = [sessionId];
|
||||
const affectedWordIds = getAffectedWordIdsForSessions(db, sessionIds);
|
||||
const affectedKanjiIds = getAffectedKanjiIdsForSessions(db, sessionIds);
|
||||
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
|
||||
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
deleteSessionsByIds(db, sessionIds);
|
||||
refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds);
|
||||
rebuildLifetimeSummariesInTransaction(db);
|
||||
rebuildRollupsInTransaction(db);
|
||||
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
@@ -492,13 +493,14 @@ export function deleteSessions(db: DatabaseSync, sessionIds: number[]): void {
|
||||
if (sessionIds.length === 0) return;
|
||||
const affectedWordIds = getAffectedWordIdsForSessions(db, sessionIds);
|
||||
const affectedKanjiIds = getAffectedKanjiIdsForSessions(db, sessionIds);
|
||||
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
|
||||
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
deleteSessionsByIds(db, sessionIds);
|
||||
refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds);
|
||||
rebuildLifetimeSummariesInTransaction(db);
|
||||
rebuildRollupsInTransaction(db);
|
||||
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
@@ -536,7 +538,6 @@ export function deleteVideo(db: DatabaseSync, videoId: number): void {
|
||||
db.prepare('DELETE FROM imm_videos WHERE video_id = ?').run(videoId);
|
||||
refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds);
|
||||
rebuildLifetimeSummariesInTransaction(db);
|
||||
rebuildRollupsInTransaction(db);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
db.exec('ROLLBACK');
|
||||
|
||||
@@ -74,6 +74,8 @@ export interface TrendsDashboardQueryResult {
|
||||
};
|
||||
ratios: {
|
||||
lookupsPerHundred: TrendChartPoint[];
|
||||
cardsPerHour: TrendChartPoint[];
|
||||
readingSpeed: TrendChartPoint[];
|
||||
};
|
||||
animeCumulative: {
|
||||
watchTime: TrendPerAnimePoint[];
|
||||
@@ -176,11 +178,31 @@ function getTrendSessionWordCount(session: Pick<TrendSessionMetricRow, 'tokensSe
|
||||
return session.tokensSeen;
|
||||
}
|
||||
|
||||
function looksLikeJellyfinStreamTitle(title: string): boolean {
|
||||
const lowered = title.toLowerCase();
|
||||
const hasApiKey = /api[\s_-]*key(?:\s|=|$)/i.test(title);
|
||||
return (
|
||||
hasApiKey &&
|
||||
(lowered.includes('stream?') ||
|
||||
lowered.includes('/stream?') ||
|
||||
lowered.includes('/videos/') ||
|
||||
lowered.includes('mediasourceid'))
|
||||
);
|
||||
}
|
||||
|
||||
function sanitizeTrendTitle(title: string): string {
|
||||
const normalized = title.trim();
|
||||
if (!normalized) {
|
||||
return 'Unknown';
|
||||
}
|
||||
return looksLikeJellyfinStreamTitle(normalized) ? 'Jellyfin Video' : normalized;
|
||||
}
|
||||
|
||||
function resolveTrendAnimeTitle(value: {
|
||||
animeTitle: string | null;
|
||||
canonicalTitle: string | null;
|
||||
}): string {
|
||||
return value.animeTitle ?? value.canonicalTitle ?? 'Unknown';
|
||||
return sanitizeTrendTitle(value.animeTitle ?? value.canonicalTitle ?? 'Unknown');
|
||||
}
|
||||
|
||||
function accumulatePoints(points: TrendChartPoint[]): TrendChartPoint[] {
|
||||
@@ -225,6 +247,26 @@ function buildAggregatedTrendRows(rollups: ImmersionSessionRollupRow[]) {
|
||||
}));
|
||||
}
|
||||
|
||||
function buildEfficiencyRates(rows: ReturnType<typeof buildAggregatedTrendRows>): {
|
||||
cardsPerHour: TrendChartPoint[];
|
||||
readingSpeed: TrendChartPoint[];
|
||||
} {
|
||||
const cardsPerHour: TrendChartPoint[] = [];
|
||||
const readingSpeed: TrendChartPoint[] = [];
|
||||
for (const row of rows) {
|
||||
const hours = row.activeMin / 60;
|
||||
cardsPerHour.push({
|
||||
label: row.label,
|
||||
value: hours > 0 ? +(row.cards / hours).toFixed(1) : 0,
|
||||
});
|
||||
readingSpeed.push({
|
||||
label: row.label,
|
||||
value: row.activeMin > 0 ? +(row.words / row.activeMin).toFixed(1) : 0,
|
||||
});
|
||||
}
|
||||
return { cardsPerHour, readingSpeed };
|
||||
}
|
||||
|
||||
function buildWatchTimeByDayOfWeek(sessions: TrendSessionMetricRow[]): TrendChartPoint[] {
|
||||
const totals = new Array(7).fill(0);
|
||||
for (const session of sessions) {
|
||||
@@ -449,7 +491,7 @@ function getVideoAnimeTitleMap(
|
||||
)
|
||||
.all(...uniqueIds) as Array<{ videoId: number; animeTitle: string }>;
|
||||
|
||||
return new Map(rows.map((row) => [row.videoId, row.animeTitle]));
|
||||
return new Map(rows.map((row) => [row.videoId, sanitizeTrendTitle(row.animeTitle)]));
|
||||
}
|
||||
|
||||
function resolveVideoAnimeTitle(
|
||||
@@ -675,6 +717,7 @@ export function getTrendsDashboard(
|
||||
);
|
||||
|
||||
const aggregatedRows = buildAggregatedTrendRows(chartRollups);
|
||||
const efficiency = buildEfficiencyRates(aggregatedRows);
|
||||
const activity = {
|
||||
watchTime: aggregatedRows.map((row) => ({ label: row.label, value: row.activeMin })),
|
||||
cards: aggregatedRows.map((row) => ({ label: row.label, value: row.cards })),
|
||||
@@ -724,6 +767,8 @@ export function getTrendsDashboard(
|
||||
},
|
||||
ratios: {
|
||||
lookupsPerHundred: buildLookupsPerHundredWords(sessions, groupBy),
|
||||
cardsPerHour: efficiency.cardsPerHour,
|
||||
readingSpeed: efficiency.readingSpeed,
|
||||
},
|
||||
animeCumulative: {
|
||||
watchTime: buildCumulativePerAnime(animePerDay.watchTime),
|
||||
|
||||
@@ -813,7 +813,7 @@ test('ensureSchema migrates legacy videos and backfills anime metadata from file
|
||||
.all() as Array<{ canonical_title: string }>;
|
||||
assert.deepEqual(
|
||||
animeRows.map((row) => row.canonical_title),
|
||||
['Frieren', 'Little Witch Academia'],
|
||||
['Frieren', 'Little Witch Academia Season 2'],
|
||||
);
|
||||
|
||||
const littleWitchRows = db
|
||||
@@ -855,7 +855,7 @@ test('ensureSchema migrates legacy videos and backfills anime metadata from file
|
||||
})),
|
||||
[
|
||||
{
|
||||
animeTitle: 'Little Witch Academia',
|
||||
animeTitle: 'Little Witch Academia Season 2',
|
||||
parsedTitle: 'Little Witch Academia',
|
||||
parsedBasename: 'Little Witch Academia S02E05.mkv',
|
||||
parsedSeason: 2,
|
||||
@@ -863,7 +863,7 @@ test('ensureSchema migrates legacy videos and backfills anime metadata from file
|
||||
parserSource: 'fallback',
|
||||
},
|
||||
{
|
||||
animeTitle: 'Little Witch Academia',
|
||||
animeTitle: 'Little Witch Academia Season 2',
|
||||
parsedTitle: 'Little Witch Academia',
|
||||
parsedBasename: 'Little Witch Academia S02E06.mkv',
|
||||
parsedSeason: 2,
|
||||
|
||||
@@ -23,6 +23,7 @@ export interface TrackerPreparedStatements {
|
||||
export interface AnimeRecordInput {
|
||||
parsedTitle: string;
|
||||
canonicalTitle: string;
|
||||
seasonScope?: number | null;
|
||||
anilistId: number | null;
|
||||
titleRomaji: string | null;
|
||||
titleEnglish: string | null;
|
||||
@@ -300,6 +301,31 @@ export function normalizeAnimeIdentityKey(title: string): string {
|
||||
.replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
function normalizeSeasonScope(value: number | null | undefined): number | null {
|
||||
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) {
|
||||
return null;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function titleAlreadyHasSeasonScope(title: string, season: number): boolean {
|
||||
const normalized = title.normalize('NFKC').toLowerCase();
|
||||
const padded = String(season).padStart(2, '0');
|
||||
return (
|
||||
new RegExp(`\\bseason\\s*0?${season}\\b`, 'i').test(normalized) ||
|
||||
new RegExp(`\\bs0?${season}\\b`, 'i').test(normalized) ||
|
||||
new RegExp(`\\bs${padded}\\b`, 'i').test(normalized)
|
||||
);
|
||||
}
|
||||
|
||||
function buildSeasonScopedAnimeTitle(title: string, season: number | null): string {
|
||||
const trimmed = title.trim();
|
||||
if (!trimmed || season === null || titleAlreadyHasSeasonScope(trimmed, season)) {
|
||||
return trimmed;
|
||||
}
|
||||
return `${trimmed} Season ${season}`;
|
||||
}
|
||||
|
||||
function looksLikeEpisodeOnlyTitle(title: string): boolean {
|
||||
const normalized = title.normalize('NFKC').toLowerCase().replace(/\s+/g, ' ').trim();
|
||||
return /^(episode|ep)\s*\d{1,3}$/.test(normalized) || /^第\s*\d{1,3}\s*話$/.test(normalized);
|
||||
@@ -478,7 +504,12 @@ function ensureStatsExcludedWordsTable(db: DatabaseSync): void {
|
||||
}
|
||||
|
||||
export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput): number {
|
||||
const normalizedTitleKey = normalizeAnimeIdentityKey(input.parsedTitle);
|
||||
const seasonScope = normalizeSeasonScope(input.seasonScope);
|
||||
const identityTitle = buildSeasonScopedAnimeTitle(input.parsedTitle, seasonScope);
|
||||
const canonicalTitle =
|
||||
buildSeasonScopedAnimeTitle(input.canonicalTitle || input.parsedTitle, seasonScope) ||
|
||||
identityTitle;
|
||||
const normalizedTitleKey = normalizeAnimeIdentityKey(identityTitle);
|
||||
if (!normalizedTitleKey) {
|
||||
throw new Error('parsedTitle is required to create or update an anime record');
|
||||
}
|
||||
@@ -508,7 +539,7 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
|
||||
WHERE anime_id = ?
|
||||
`,
|
||||
).run(
|
||||
input.canonicalTitle,
|
||||
canonicalTitle,
|
||||
input.anilistId,
|
||||
input.titleRomaji,
|
||||
input.titleEnglish,
|
||||
@@ -539,7 +570,7 @@ export function getOrCreateAnimeRecord(db: DatabaseSync, input: AnimeRecordInput
|
||||
)
|
||||
.run(
|
||||
normalizedTitleKey,
|
||||
input.canonicalTitle,
|
||||
canonicalTitle,
|
||||
input.anilistId,
|
||||
input.titleRomaji,
|
||||
input.titleEnglish,
|
||||
@@ -648,6 +679,7 @@ function migrateLegacyAnimeMetadata(db: DatabaseSync): void {
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: parsed.title,
|
||||
canonicalTitle: parsed.title,
|
||||
seasonScope: parsed.season,
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
|
||||
@@ -52,6 +52,11 @@ export interface ImmersionTrackerPolicy {
|
||||
};
|
||||
}
|
||||
|
||||
export interface JellyfinLinkRepairSummary {
|
||||
scanned: number;
|
||||
repaired: number;
|
||||
}
|
||||
|
||||
export interface TelemetryAccumulator {
|
||||
totalWatchedMs: number;
|
||||
activeWatchedMs: number;
|
||||
@@ -367,6 +372,29 @@ export interface KanjiOccurrenceRow {
|
||||
occurrenceCount: number;
|
||||
}
|
||||
|
||||
export interface SentenceSearchResultRow {
|
||||
animeId: number | null;
|
||||
animeTitle: string | null;
|
||||
videoId: number;
|
||||
videoTitle: string;
|
||||
sourcePath: string | null;
|
||||
secondaryText: string | null;
|
||||
sessionId: number;
|
||||
lineIndex: number;
|
||||
segmentStartMs: number | null;
|
||||
segmentEndMs: number | null;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface SentenceSearchHeadwordTerm {
|
||||
term: string;
|
||||
headwords: string[];
|
||||
}
|
||||
|
||||
export interface SentenceSearchOptions {
|
||||
headwordTerms?: SentenceSearchHeadwordTerm[];
|
||||
}
|
||||
|
||||
export interface SessionEventRow {
|
||||
eventType: number;
|
||||
tsMs: number;
|
||||
|
||||
Reference in New Issue
Block a user