feat(stats): add library entry deletion and app-wide delete progress (#174)

This commit is contained in:
2026-07-29 02:00:16 -07:00
committed by GitHub
parent 0d7084c8aa
commit 6d2a72e13b
40 changed files with 2013 additions and 125 deletions
@@ -1100,12 +1100,56 @@ describe('stats server API routes', () => {
assert.deepEqual(assignments, [{ animeId: 1, body }]);
});
it('GET /api/stats/anime/:animeId/cover returns cover art', async () => {
it('GET /api/stats/anime/:animeId/cover returns revalidated cover art', async () => {
const app = createStatsApp(createMockTracker());
const res = await app.request('/api/stats/anime/1/cover');
assert.equal(res.status, 200);
assert.equal(res.headers.get('content-type'), 'image/jpeg');
assert.equal(res.headers.get('cache-control'), 'public, max-age=86400');
assert.equal(res.headers.get('cache-control'), 'no-cache');
assert.ok(res.headers.get('etag'));
});
it('GET /api/stats/anime/:animeId/cover answers 304 for a matching ETag', async () => {
const app = createStatsApp(createMockTracker());
const first = await app.request('/api/stats/anime/1/cover');
const etag = first.headers.get('etag');
assert.ok(etag);
const cached = await app.request('/api/stats/anime/1/cover', {
headers: { 'If-None-Match': `W/${etag}, "other"` },
});
assert.equal(cached.status, 304);
assert.equal(cached.headers.get('etag'), etag);
});
it('GET /api/stats/anime/:animeId/cover resends art when the ETag no longer matches', async () => {
// A relinked AniList entry swaps the bytes behind the same cover URL.
let coverBlob = Buffer.from([0xff, 0xd8, 0xff, 0xd9]);
const app = createStatsApp(
createMockTracker({
getAnimeCoverArt: async () => ({
videoId: 1,
anilistId: 21858,
coverUrl: 'https://example.com/cover.jpg',
coverBlob,
titleRomaji: 'Little Witch Academia',
titleEnglish: 'Little Witch Academia',
episodesTotal: 25,
fetchedAtMs: Date.now(),
}),
}),
);
const first = await app.request('/api/stats/anime/1/cover');
const staleEtag = first.headers.get('etag');
assert.ok(staleEtag);
coverBlob = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
const res = await app.request('/api/stats/anime/1/cover', {
headers: { 'If-None-Match': staleEtag },
});
assert.equal(res.status, 200);
assert.equal(res.headers.get('content-type'), 'image/png');
assert.notEqual(res.headers.get('etag'), staleEtag);
});
it('GET /api/stats/anime/:animeId/cover serves detected cover MIME type', async () => {
@@ -2873,6 +2917,39 @@ Aligned English subtitle
assert.equal(deleteCalls, 0);
});
it('DELETE /api/stats/anime/:animeId deletes the whole library entry', async () => {
let deletedAnimeId: number | null = null;
const app = createStatsApp(
createMockTracker({
deleteAnime: async (animeId: number) => {
deletedAnimeId = animeId;
},
} as Partial<ImmersionTrackerService>),
);
const res = await app.request('/api/stats/anime/7', { method: 'DELETE' });
assert.equal(res.status, 200);
assert.equal(deletedAnimeId, 7);
assert.deepEqual(await res.json(), { ok: true });
});
it('DELETE /api/stats/anime/:animeId rejects non-positive anime ids', async () => {
let deleteCalls = 0;
const app = createStatsApp(
createMockTracker({
deleteAnime: async () => {
deleteCalls += 1;
},
} as Partial<ImmersionTrackerService>),
);
const res = await app.request('/api/stats/anime/0', { method: 'DELETE' });
assert.equal(res.status, 400);
assert.equal(deleteCalls, 0);
});
it('POST /api/stats/anki/browse returns 400 for missing noteId', async () => {
const app = createStatsApp(createMockTracker());
const res = await app.request('/api/stats/anki/browse', { method: 'POST' });
@@ -1466,6 +1466,190 @@ test('deleteVideo ignores the currently active video and keeps new writes flusha
}
});
/**
* Attach the derived rows a real session would produce to every recorded line
* of `animeId`: word/kanji entries and their occurrences, monthly rollups, and
* cover art backed by a shared blob.
*/
function seedDerivedAnimeData(db: DatabaseSync, animeId: number): void {
const lines = db
.prepare(
'SELECT line_id AS lineId, CREATED_DATE AS seenMs FROM imm_subtitle_lines WHERE anime_id = ?',
)
.all(animeId) as Array<{ lineId: number; seenMs: number }>;
assert.ok(lines.length > 0, 'expected recorded subtitle lines to decorate');
db.prepare(
`INSERT INTO imm_words(id, headword, word, reading, part_of_speech, pos1, first_seen, last_seen, frequency)
VALUES (9001, '天気', '天気', 'てんき', 'noun', '名詞', 0, 0, 0)`,
).run();
db.prepare(
`INSERT INTO imm_kanji(id, kanji, first_seen, last_seen, frequency) VALUES (9101, '気', 0, 0, 0)`,
).run();
const insertWordOccurrence = db.prepare(
'INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms) VALUES (?, 9001, 1, ?)',
);
const insertKanjiOccurrence = db.prepare(
'INSERT INTO imm_kanji_line_occurrences(line_id, kanji_id, occurrence_count, seen_ms) VALUES (?, 9101, 1, ?)',
);
for (const line of lines) {
insertWordOccurrence.run(line.lineId, line.seenMs);
insertKanjiOccurrence.run(line.lineId, line.seenMs);
}
db.prepare(
`UPDATE imm_words SET frequency = ?, first_seen = ?, last_seen = ? WHERE id = 9001`,
).run(lines.length, 0, 0);
const videoIds = (
db
.prepare('SELECT video_id AS videoId FROM imm_videos WHERE anime_id = ?')
.all(animeId) as Array<{ videoId: number }>
).map((row) => row.videoId);
const insertMonthlyRollup = db.prepare(
`INSERT INTO imm_monthly_rollups(rollup_month, video_id, total_sessions, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (202401, ?, 1, '0', '0')`,
);
const insertArt = db.prepare(
`INSERT INTO imm_media_art(video_id, anilist_id, cover_url, cover_blob, cover_blob_hash, fetched_at_ms, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (?, 4242, 'https://example.test/cover.jpg', NULL, 'deadbeef', '0', '0', '0')`,
);
db.prepare(
`INSERT INTO imm_cover_art_blobs(blob_hash, cover_blob, CREATED_DATE, LAST_UPDATE_DATE)
VALUES ('deadbeef', X'FFD8FFD9', '0', '0')`,
).run();
for (const videoId of videoIds) {
insertMonthlyRollup.run(videoId);
insertArt.run(videoId);
}
}
test('deleteAnime removes every episode, session and library row for the title', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
try {
const Ctor = await loadTrackerCtor();
for (const episode of ['S02E05', 'S02E06']) {
tracker = new Ctor({ dbPath });
tracker.handleMediaChange(`/tmp/Little Witch Academia ${episode}.mkv`, `Episode ${episode}`);
await waitForPendingAnimeMetadata(tracker);
tracker.recordSubtitleLine('今日は晴れです', 0, 1.2);
tracker.recordCardsMined(1);
tracker.destroy();
tracker = null;
}
tracker = new Ctor({ dbPath });
const privateApi = tracker as unknown as { db: DatabaseSync };
const animeId = (
privateApi.db.prepare('SELECT anime_id FROM imm_anime LIMIT 1').get() as {
anime_id: number;
} | null
)?.anime_id;
assert.ok(animeId);
// The tokenizer does not run in this harness, so attach vocabulary, kanji,
// rollups and cover art to the recorded lines by hand. Without them the
// "everything is gone" assertions below would pass against empty tables.
seedDerivedAnimeData(privateApi.db, animeId);
const countOf = (sql: string): number =>
(privateApi.db.prepare(sql).get() as { total: number }).total;
for (const table of [
'imm_words',
'imm_kanji',
'imm_word_line_occurrences',
'imm_kanji_line_occurrences',
'imm_daily_rollups',
'imm_monthly_rollups',
'imm_media_art',
'imm_cover_art_blobs',
]) {
assert.ok(
countOf(`SELECT COUNT(*) AS total FROM ${table}`) > 0,
`precondition: ${table} should hold rows before the delete`,
);
}
const libraryBefore = await tracker.getAnimeLibrary();
assert.equal(libraryBefore.length, 1);
assert.equal(libraryBefore[0]?.episodeCount, 2);
await tracker.deleteAnime(animeId);
const libraryAfter = await tracker.getAnimeLibrary();
assert.equal(libraryAfter.length, 0);
for (const table of [
'imm_anime',
'imm_lifetime_anime',
'imm_videos',
'imm_sessions',
'imm_subtitle_lines',
'imm_daily_rollups',
'imm_monthly_rollups',
'imm_lifetime_media',
'imm_words',
'imm_kanji',
'imm_word_line_occurrences',
'imm_kanji_line_occurrences',
'imm_media_art',
'imm_cover_art_blobs',
]) {
assert.equal(
countOf(`SELECT COUNT(*) AS total FROM ${table}`),
0,
`${table} should be empty after deleting the only title`,
);
}
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('deleteAnime ignores the title of the currently active session', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath });
tracker.handleMediaChange('/tmp/Little Witch Academia S02E05.mkv', 'Episode 5');
await waitForPendingAnimeMetadata(tracker);
const privateApi = tracker as unknown as {
db: DatabaseSync;
sessionState: { sessionId: number; videoId: number } | null;
};
const videoId = privateApi.sessionState?.videoId;
assert.ok(videoId);
const animeId = (
privateApi.db.prepare('SELECT anime_id FROM imm_videos WHERE video_id = ?').get(videoId) as {
anime_id: number | null;
} | null
)?.anime_id;
assert.ok(animeId);
await tracker.deleteAnime(animeId);
const animeCountRow = privateApi.db
.prepare('SELECT COUNT(*) AS total FROM imm_anime WHERE anime_id = ?')
.get(animeId) as { total: number };
const videoCountRow = privateApi.db
.prepare('SELECT COUNT(*) AS total FROM imm_videos WHERE video_id = ?')
.get(videoId) as { total: number };
assert.equal(animeCountRow.total, 1);
assert.equal(videoCountRow.total, 1);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('handleMediaChange links parsed anime metadata on the active video row', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
@@ -83,6 +83,7 @@ import {
} from './immersion-tracker/query-library';
import {
cleanupVocabularyStats,
deleteAnime as deleteAnimeQuery,
deleteSession as deleteSessionQuery,
deleteSessions as deleteSessionsQuery,
deleteVideo as deleteVideoQuery,
@@ -733,6 +734,28 @@ export class ImmersionTrackerService {
deleteVideoQuery(this.db, videoId);
}
async deleteAnime(animeId: number): Promise<void> {
// The active video's anime link is assigned asynchronously after the title
// is parsed, so a guard reading imm_videos too early sees a null and lets
// the delete through — then the late update recreates the anime row.
const pendingVideoId = this.sessionState?.videoId;
if (pendingVideoId !== undefined) {
await this.pendingAnimeMetadataUpdates.get(pendingVideoId);
}
const activeVideoId = this.sessionState?.videoId;
if (activeVideoId !== undefined) {
const activeAnime = this.db
.prepare('SELECT anime_id FROM imm_videos WHERE video_id = ?')
.get(activeVideoId) as { anime_id: number | null } | null;
if (activeAnime?.anime_id === animeId) {
this.logger.warn(`Ignoring delete request for active immersion anime ${animeId}`);
return;
}
}
deleteAnimeQuery(this.db, animeId);
}
async reassignAnimeAnilist(
animeId: number,
info: {
@@ -0,0 +1,331 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { Database } from '../sqlite.js';
import type { DatabaseSync } from '../sqlite.js';
import { ensureSchema } from '../storage.js';
import { deleteSession, deleteSessions, deleteVideo } from '../query-maintenance.js';
const DAY_MS = 86_400_000;
const BASE_MS = 1_700_000_000_000;
function makeDbPath(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-removal-test-'));
return path.join(dir, 'immersion.sqlite');
}
function cleanupDbPath(dbPath: string): void {
const dir = path.dirname(dbPath);
if (!fs.existsSync(dir)) return;
fs.rmSync(dir, { recursive: true, force: true });
}
/**
* Seed two episodes of one anime, each with one ended session.
*
* `lines` places a word occurrence on a specific day so tests can control which
* session holds a word's first/last occurrence.
*/
function seed(
db: DatabaseSync,
lines: Array<{ session: 1 | 2; wordId: number; dayOffset: number; count?: number }>,
options: { legacyRows?: boolean } = {},
): void {
db.exec(`
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (1, 'show', 'Show', ${BASE_MS}, ${BASE_MS});
INSERT INTO imm_videos(video_id, video_key, anime_id, canonical_title, source_type, watched, duration_ms, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (1, 'v1', 1, 'Ep 1', 1, 1, 1440000, ${BASE_MS}, ${BASE_MS}),
(2, 'v2', 1, 'Ep 2', 1, 1, 1440000, ${BASE_MS}, ${BASE_MS});
INSERT INTO imm_sessions(session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (1, 's1', 1, '${BASE_MS}', '${BASE_MS + 1000}', 2, ${BASE_MS}, ${BASE_MS}),
(2, 's2', 2, '${BASE_MS + DAY_MS}', '${BASE_MS + DAY_MS + 1000}', 2, ${BASE_MS}, ${BASE_MS});
`);
const insertLine = db.prepare(
`INSERT INTO imm_subtitle_lines(line_id, session_id, video_id, anime_id, line_index, text, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (?, ?, ?, 1, ?, ?, ?, ?)`,
);
const insertWord = db.prepare(
`INSERT OR IGNORE INTO imm_words(id, headword, word, reading, part_of_speech, pos1, first_seen, last_seen, frequency)
VALUES (?, ?, ?, '', 'noun', '名詞', 0, 0, 0)`,
);
// `legacyRows` reproduces databases written before the seen_ms column existed,
// where the timestamp has to be read back off the subtitle line.
const insertOccurrence = options.legacyRows
? db.prepare(
`INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count) VALUES (?, ?, ?)`,
)
: db.prepare(
`INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms) VALUES (?, ?, ?, ?)`,
);
let lineId = 0;
for (const line of lines) {
lineId += 1;
const seenMs = BASE_MS + line.dayOffset * DAY_MS;
insertLine.run(lineId, line.session, line.session, lineId, `line ${lineId}`, seenMs, seenMs);
insertWord.run(line.wordId, `${line.wordId}`, `${line.wordId}`);
if (options.legacyRows) {
insertOccurrence.run(lineId, line.wordId, line.count ?? 1);
} else {
insertOccurrence.run(lineId, line.wordId, line.count ?? 1, seenMs);
}
}
// Match what the tracker maintains: aggregates derived from the occurrences.
db.exec(`
UPDATE imm_words SET
frequency = (
SELECT COALESCE(SUM(o.occurrence_count), 0)
FROM imm_word_line_occurrences o WHERE o.word_id = imm_words.id
),
first_seen = (
SELECT MIN(sl.CREATED_DATE) / 1000
FROM imm_word_line_occurrences o
JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
WHERE o.word_id = imm_words.id
),
last_seen = (
SELECT MAX(sl.LAST_UPDATE_DATE) / 1000
FROM imm_word_line_occurrences o
JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
WHERE o.word_id = imm_words.id
)
`);
}
function createDb(
lines: Parameters<typeof seed>[1],
options: Parameters<typeof seed>[2] = {},
): { db: DatabaseSync; dbPath: string } {
const dbPath = makeDbPath();
const db = new Database(dbPath);
ensureSchema(db);
seed(db, lines, options);
return { db, dbPath };
}
function readWord(
db: DatabaseSync,
wordId: number,
): { frequency: number; firstSeen: number; lastSeen: number } | null {
return (
(db
.prepare(
'SELECT frequency, first_seen AS firstSeen, last_seen AS lastSeen FROM imm_words WHERE id = ?',
)
.get(wordId) as { frequency: number; firstSeen: number; lastSeen: number } | null) ?? null
);
}
test('deleting a session subtracts only the occurrences it removed', () => {
const { db, dbPath } = createDb([
{ session: 1, wordId: 10, dayOffset: 0, count: 3 },
{ session: 2, wordId: 10, dayOffset: 1, count: 4 },
]);
try {
assert.equal(readWord(db, 10)?.frequency, 7);
deleteSession(db, 1);
assert.equal(readWord(db, 10)?.frequency, 4, 'only session 1 occurrences are subtracted');
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('deleting the session that held a word first drops the word entirely', () => {
const { db, dbPath } = createDb([
{ session: 1, wordId: 11, dayOffset: 0 },
{ session: 2, wordId: 12, dayOffset: 1 },
]);
try {
deleteSession(db, 1);
assert.equal(readWord(db, 11), null, 'word seen only in the deleted session is removed');
assert.ok(readWord(db, 12), 'word seen elsewhere survives');
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('deleting the earliest session moves first_seen forward to the surviving line', () => {
const { db, dbPath } = createDb([
{ session: 1, wordId: 13, dayOffset: 0 },
{ session: 2, wordId: 13, dayOffset: 5 },
]);
try {
assert.equal(readWord(db, 13)?.firstSeen, Math.floor(BASE_MS / 1000));
deleteSession(db, 1);
const word = readWord(db, 13);
assert.equal(word?.frequency, 1);
assert.equal(
word?.firstSeen,
Math.floor((BASE_MS + 5 * DAY_MS) / 1000),
'first_seen advances to the remaining occurrence',
);
assert.equal(word?.lastSeen, Math.floor((BASE_MS + 5 * DAY_MS) / 1000));
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('deleting the latest session moves last_seen back to the surviving line', () => {
const { db, dbPath } = createDb([
{ session: 1, wordId: 14, dayOffset: 0 },
{ session: 2, wordId: 14, dayOffset: 5 },
]);
try {
deleteSession(db, 2);
const word = readWord(db, 14);
assert.equal(word?.frequency, 1);
assert.equal(word?.lastSeen, Math.floor(BASE_MS / 1000), 'last_seen falls back to session 1');
assert.equal(word?.firstSeen, Math.floor(BASE_MS / 1000));
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('deleting an interior occurrence leaves the surrounding extremes untouched', () => {
// Session 2 carries the middle occurrence; sessions bracket it in time.
const { db, dbPath } = createDb([
{ session: 1, wordId: 15, dayOffset: 0 },
{ session: 2, wordId: 15, dayOffset: 3 },
{ session: 1, wordId: 15, dayOffset: 9 },
]);
try {
deleteSessions(db, [2]);
const word = readWord(db, 15);
assert.equal(word?.frequency, 2);
assert.equal(word?.firstSeen, Math.floor(BASE_MS / 1000));
assert.equal(word?.lastSeen, Math.floor((BASE_MS + 9 * DAY_MS) / 1000));
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a stored frequency that has drifted low is repaired instead of dropping a live word', () => {
const { db, dbPath } = createDb([
{ session: 1, wordId: 16, dayOffset: 0, count: 5 },
{ session: 2, wordId: 16, dayOffset: 4, count: 5 },
]);
try {
// Simulate drift: the stored total is lower than the occurrences justify, so
// naive subtraction would take the word to zero while rows still reference it.
db.prepare('UPDATE imm_words SET frequency = 5 WHERE id = ?').run(16);
deleteSession(db, 1);
const word = readWord(db, 16);
assert.ok(word, 'word with surviving occurrences is not deleted');
assert.equal(word?.frequency, 5, 'frequency is recomputed from the surviving occurrences');
assert.equal(word?.firstSeen, Math.floor((BASE_MS + 4 * DAY_MS) / 1000));
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('deleting a video subtracts every occurrence carried by its lines', () => {
const { db, dbPath } = createDb([
{ session: 1, wordId: 17, dayOffset: 0, count: 2 },
{ session: 1, wordId: 17, dayOffset: 1, count: 3 },
{ session: 2, wordId: 17, dayOffset: 2, count: 4 },
{ session: 2, wordId: 18, dayOffset: 2, count: 1 },
]);
try {
deleteVideo(db, 1);
assert.equal(readWord(db, 17)?.frequency, 4, 'both lines from video 1 are subtracted');
assert.equal(readWord(db, 18)?.frequency, 1, 'untouched video keeps its counts');
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('occurrence rows written before the seen_ms column still resolve their dates', () => {
const { db, dbPath } = createDb(
[
{ session: 1, wordId: 20, dayOffset: 0 },
{ session: 2, wordId: 20, dayOffset: 6 },
],
{ legacyRows: true },
);
try {
assert.equal(
(
db
.prepare('SELECT COUNT(*) AS total FROM imm_word_line_occurrences WHERE seen_ms IS NULL')
.get() as { total: number }
).total,
2,
'precondition: rows carry no denormalised timestamp',
);
deleteSession(db, 1);
const word = readWord(db, 20);
assert.equal(word?.frequency, 1);
assert.equal(
word?.firstSeen,
Math.floor((BASE_MS + 6 * DAY_MS) / 1000),
'falls back to the subtitle line timestamp',
);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('upgrading an older database backfills seen_ms from the subtitle lines', () => {
const { db, dbPath } = createDb(
[
{ session: 1, wordId: 21, dayOffset: 0 },
{ session: 2, wordId: 21, dayOffset: 2 },
],
{ legacyRows: true },
);
try {
// Re-run ensureSchema the way a pre-19 database would be opened.
db.exec('DELETE FROM imm_schema_version');
db.exec(`INSERT INTO imm_schema_version(schema_version, applied_at_ms) VALUES (18, '0')`);
ensureSchema(db);
const rows = db
.prepare('SELECT line_id AS lineId, seen_ms AS seenMs FROM imm_word_line_occurrences')
.all() as Array<{ lineId: number; seenMs: number | null }>;
assert.equal(rows.length, 2);
for (const row of rows) {
assert.ok(row.seenMs, `line ${row.lineId} should have a backfilled timestamp`);
}
assert.deepEqual(
rows.map((row) => row.seenMs).sort((a, b) => Number(a) - Number(b)),
[BASE_MS, BASE_MS + 2 * DAY_MS],
);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
@@ -4993,3 +4993,145 @@ test('getTrendsDashboard librarySummary is empty when no rollups exist', () => {
cleanupDbPath(dbPath);
}
});
test('getVocabularyStats counts the distinct anime each word appeared in', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const startedAtMs = 1_700_000_000_000;
db.exec(`
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (11, 'vocab-a', 'Vocab A', ${startedAtMs}, ${startedAtMs}),
(22, 'vocab-b', 'Vocab B', ${startedAtMs}, ${startedAtMs});
`);
const videoIds = [1, 2].map((n) =>
getOrCreateVideoRecord(db, `local:/tmp/vocab-anime-${n}.mkv`, {
canonicalTitle: `Vocab Anime ${n}`,
sourcePath: `/tmp/vocab-anime-${n}.mkv`,
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
}),
);
const sessionIds = videoIds.map(
(videoId) => startSessionRecord(db, videoId, startedAtMs).sessionId,
);
// 猫 spans both titles, twice in the first, so the count has to deduplicate
// by anime rather than just tallying occurrence rows. 犬 stays in one title.
insertFilteredWordOccurrence(db, {
sessionId: sessionIds[0]!,
videoId: videoIds[0]!,
animeId: 11,
lineIndex: 1,
occurrenceCount: 5,
startedAtMs,
word: '猫',
reading: 'ねこ',
});
insertFilteredWordOccurrence(db, {
sessionId: sessionIds[0]!,
videoId: videoIds[0]!,
animeId: 11,
lineIndex: 2,
occurrenceCount: 1,
startedAtMs,
word: '猫',
reading: 'ねこ',
});
insertFilteredWordOccurrence(db, {
sessionId: sessionIds[1]!,
videoId: videoIds[1]!,
animeId: 22,
lineIndex: 3,
occurrenceCount: 4,
startedAtMs,
word: '猫',
reading: 'ねこ',
});
insertFilteredWordOccurrence(db, {
sessionId: sessionIds[0]!,
videoId: videoIds[0]!,
animeId: 11,
lineIndex: 4,
occurrenceCount: 2,
startedAtMs,
word: '犬',
reading: 'いぬ',
});
const rows = getVocabularyStats(db, 10);
const cat = rows.find((row) => row.headword === '猫');
const dog = rows.find((row) => row.headword === '犬');
assert.equal(cat?.animeCount, 2, '猫 was seen in two anime across three lines');
assert.equal(dog?.animeCount, 1, '犬 was seen in one');
assert.equal(cat?.frequency, 10, 'frequency still aggregates across both titles');
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getVocabularyStats still applies part-of-speech exclusions', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const startedAtMs = 1_700_000_000_000;
db.exec(`
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (11, 'vocab-a', 'Vocab A', ${startedAtMs}, ${startedAtMs});
`);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/vocab-exclude.mkv', {
canonicalTitle: 'Vocab Exclude',
sourcePath: '/tmp/vocab-exclude.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const { sessionId } = startSessionRecord(db, videoId, startedAtMs);
insertFilteredWordOccurrence(db, {
sessionId,
videoId,
animeId: 11,
lineIndex: 1,
occurrenceCount: 9,
startedAtMs,
word: '走る',
reading: 'はしる',
partOfSpeech: 'verb',
});
insertFilteredWordOccurrence(db, {
sessionId,
videoId,
animeId: 11,
lineIndex: 2,
occurrenceCount: 3,
startedAtMs,
word: '猫',
reading: 'ねこ',
partOfSpeech: 'noun',
});
const all = getVocabularyStats(db, 10);
assert.deepEqual(
all.map((row) => row.headword).sort(),
['猫', '走る'],
'precondition: both parts of speech are present',
);
const nounsOnly = getVocabularyStats(db, 10, ['verb']);
assert.deepEqual(
nounsOnly.map((row) => row.headword),
['猫'],
'excluded part of speech is filtered out even though it ranks higher',
);
assert.equal(nounsOnly[0]?.animeCount, 1, 'surviving rows keep their anime count');
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
@@ -115,18 +115,29 @@ export function getVocabularyStats(
const whereClause = hasExclude
? `WHERE (part_of_speech IS NULL OR part_of_speech NOT IN (${placeholders}))`
: '';
// The page is selected before the join so `animeCount` is only computed for the
// rows being returned. Aggregating first made every request walk each word's
// entire occurrence history — seconds of blocked event loop on a large library,
// because only the ordering, not the aggregate, decides which rows survive.
const stmt = db.prepare(`
SELECT w.id AS wordId, w.headword, w.word, w.reading,
w.part_of_speech AS partOfSpeech, w.pos1, w.pos2, w.pos3,
w.frequency, w.frequency_rank AS frequencyRank,
w.first_seen AS firstSeen, w.last_seen AS lastSeen,
WITH page AS (
SELECT id, headword, word, reading, part_of_speech, pos1, pos2, pos3,
frequency, frequency_rank, first_seen, last_seen
FROM imm_words
${whereClause}
ORDER BY frequency DESC, id
LIMIT ? OFFSET ?
)
SELECT p.id AS wordId, p.headword, p.word, p.reading,
p.part_of_speech AS partOfSpeech, p.pos1, p.pos2, p.pos3,
p.frequency, p.frequency_rank AS frequencyRank,
p.first_seen AS firstSeen, p.last_seen AS lastSeen,
COUNT(DISTINCT sl.anime_id) AS animeCount
FROM imm_words w
LEFT JOIN imm_word_line_occurrences o ON o.word_id = w.id
FROM page p
LEFT JOIN imm_word_line_occurrences o ON o.word_id = p.id
LEFT JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id AND sl.anime_id IS NOT NULL
${whereClause ? whereClause.replace('part_of_speech', 'w.part_of_speech') : ''}
GROUP BY w.id
ORDER BY w.frequency DESC LIMIT ? OFFSET ?
GROUP BY p.id
ORDER BY p.frequency DESC, p.id
`);
const visibleRows: VocabularyStatsRow[] = [];
let offset = 0;
@@ -9,14 +9,12 @@ import { PartOfSpeech, type MergedToken } from '../../../types';
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
import { deriveStoredPartOfSpeech } from '../tokenizer/part-of-speech';
import {
applyLexicalRemovals,
cleanupUnusedCoverArtBlobHash,
deleteSessionsByIds,
findSharedCoverBlobHash,
getAffectedKanjiIdsForSessions,
getAffectedKanjiIdsForVideo,
getAffectedWordIdsForSessions,
getAffectedWordIdsForVideo,
refreshLexicalAggregates,
planLexicalRemovalsForSessions,
planLexicalRemovalsForVideos,
toDbMs,
toDbTimestamp,
} from './query-shared';
@@ -232,12 +230,13 @@ export async function cleanupVocabularyStats(
WHERE id = ?`,
);
const moveOccurrencesStmt = db.prepare(
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count)
SELECT line_id, ?, occurrence_count
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count, seen_ms)
SELECT line_id, ?, occurrence_count, seen_ms
FROM imm_word_line_occurrences
WHERE word_id = ?
ON CONFLICT(line_id, word_id) DO UPDATE SET
occurrence_count = imm_word_line_occurrences.occurrence_count + excluded.occurrence_count`,
occurrence_count = imm_word_line_occurrences.occurrence_count + excluded.occurrence_count,
seen_ms = COALESCE(imm_word_line_occurrences.seen_ms, excluded.seen_ms)`,
);
const deleteOccurrencesStmt = db.prepare(
'DELETE FROM imm_word_line_occurrences WHERE word_id = ?',
@@ -484,14 +483,16 @@ export function isVideoWatched(db: DatabaseSync, videoId: number): boolean {
export function deleteSession(db: DatabaseSync, sessionId: number): void {
const sessionIds = [sessionId];
const affectedWordIds = getAffectedWordIdsForSessions(db, sessionIds);
const affectedKanjiIds = getAffectedKanjiIdsForSessions(db, sessionIds);
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
db.exec('BEGIN IMMEDIATE');
try {
// Measured inside the write lock: the plan records what the delete removes,
// and applying a plan taken against a different snapshot would subtract the
// wrong totals from imm_words/imm_kanji.
const lexicalRemovals = planLexicalRemovalsForSessions(db, sessionIds);
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
deleteSessionsByIds(db, sessionIds);
refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds);
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
db.exec('COMMIT');
@@ -503,14 +504,13 @@ export function deleteSession(db: DatabaseSync, sessionId: number): void {
export function deleteSessions(db: DatabaseSync, sessionIds: number[]): void {
if (sessionIds.length === 0) return;
const affectedWordIds = getAffectedWordIdsForSessions(db, sessionIds);
const affectedKanjiIds = getAffectedKanjiIdsForSessions(db, sessionIds);
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
db.exec('BEGIN IMMEDIATE');
try {
const lexicalRemovals = planLexicalRemovalsForSessions(db, sessionIds);
const affectedRollupGroups = getRollupGroupsForSessions(db, sessionIds);
deleteSessionsByIds(db, sessionIds);
refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds);
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
refreshRollupsForGroupsInTransaction(db, affectedRollupGroups);
db.exec('COMMIT');
@@ -520,24 +520,83 @@ export function deleteSessions(db: DatabaseSync, sessionIds: number[]): void {
}
}
/**
* Delete an entire library entry: every episode of the anime, all of their
* sessions and derived stats, and the anime row itself.
*
* Mirrors {@link deleteVideo} per episode, but batches the lexical refresh and
* lifetime rebuild into a single transaction so a multi-episode title doesn't
* pay for one full rebuild per episode.
*/
export function deleteAnime(db: DatabaseSync, animeId: number): void {
db.exec('BEGIN IMMEDIATE');
try {
const videoIds = (
db.prepare('SELECT video_id FROM imm_videos WHERE anime_id = ?').all(animeId) as Array<{
video_id: number;
}>
).map((row) => row.video_id);
const lexicalRemovals = planLexicalRemovalsForVideos(db, videoIds);
const coverBlobHashes: string[] = [];
const sessionIds: number[] = [];
for (const videoId of videoIds) {
const artRow = db
.prepare('SELECT cover_blob_hash AS coverBlobHash FROM imm_media_art WHERE video_id = ?')
.get(videoId) as { coverBlobHash: string | null } | undefined;
if (artRow?.coverBlobHash) {
coverBlobHashes.push(artRow.coverBlobHash);
}
const sessions = db
.prepare('SELECT session_id FROM imm_sessions WHERE video_id = ?')
.all(videoId) as Array<{ session_id: number }>;
sessionIds.push(...sessions.map((session) => session.session_id));
}
deleteSessionsByIds(db, sessionIds);
const deleteLinesStmt = db.prepare('DELETE FROM imm_subtitle_lines WHERE video_id = ?');
const deleteDailyStmt = db.prepare('DELETE FROM imm_daily_rollups WHERE video_id = ?');
const deleteMonthlyStmt = db.prepare('DELETE FROM imm_monthly_rollups WHERE video_id = ?');
const deleteArtStmt = db.prepare('DELETE FROM imm_media_art WHERE video_id = ?');
const deleteVideoStmt = db.prepare('DELETE FROM imm_videos WHERE video_id = ?');
for (const videoId of videoIds) {
deleteLinesStmt.run(videoId);
deleteDailyStmt.run(videoId);
deleteMonthlyStmt.run(videoId);
deleteArtStmt.run(videoId);
deleteVideoStmt.run(videoId);
}
for (const coverBlobHash of new Set(coverBlobHashes)) {
cleanupUnusedCoverArtBlobHash(db, coverBlobHash);
}
db.prepare('DELETE FROM imm_lifetime_anime WHERE anime_id = ?').run(animeId);
db.prepare('DELETE FROM imm_anime WHERE anime_id = ?').run(animeId);
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
db.exec('COMMIT');
} catch (error) {
db.exec('ROLLBACK');
throw error;
}
}
export function deleteVideo(db: DatabaseSync, videoId: number): void {
const artRow = db
.prepare(
`
db.exec('BEGIN IMMEDIATE');
try {
const artRow = db
.prepare(
`
SELECT cover_blob_hash AS coverBlobHash
FROM imm_media_art
WHERE video_id = ?
`,
)
.get(videoId) as { coverBlobHash: string | null } | undefined;
const affectedWordIds = getAffectedWordIdsForVideo(db, videoId);
const affectedKanjiIds = getAffectedKanjiIdsForVideo(db, videoId);
const sessions = db
.prepare('SELECT session_id FROM imm_sessions WHERE video_id = ?')
.all(videoId) as Array<{ session_id: number }>;
)
.get(videoId) as { coverBlobHash: string | null } | undefined;
const lexicalRemovals = planLexicalRemovalsForVideos(db, [videoId]);
const sessions = db
.prepare('SELECT session_id FROM imm_sessions WHERE video_id = ?')
.all(videoId) as Array<{ session_id: number }>;
db.exec('BEGIN IMMEDIATE');
try {
deleteSessionsByIds(
db,
sessions.map((session) => session.session_id),
@@ -548,7 +607,7 @@ export function deleteVideo(db: DatabaseSync, videoId: number): void {
db.prepare('DELETE FROM imm_media_art WHERE video_id = ?').run(videoId);
cleanupUnusedCoverArtBlobHash(db, artRow?.coverBlobHash ?? null);
db.prepare('DELETE FROM imm_videos WHERE video_id = ?').run(videoId);
refreshLexicalAggregates(db, affectedWordIds, affectedKanjiIds);
applyLexicalRemovals(db, lexicalRemovals);
rebuildLifetimeSummariesInTransaction(db);
db.exec('COMMIT');
} catch (error) {
@@ -203,6 +203,167 @@ export function getAffectedKanjiIdsForVideo(db: DatabaseSync, videoId: number):
return getAffectedIdsForVideo(db, 'kanji', videoId);
}
/** Per-entity totals that a pending delete is about to remove. */
interface LexicalRemoval {
id: number;
removedFrequency: number;
removedFirstSeenMs: number | null;
removedLastSeenMs: number | null;
}
/** What a pending delete removes from `imm_words` and `imm_kanji`. */
export interface LexicalRemovalPlan {
words: LexicalRemoval[];
kanji: LexicalRemoval[];
}
export const EMPTY_LEXICAL_REMOVAL_PLAN: LexicalRemovalPlan = { words: [], kanji: [] };
function collectLexicalRemovals(
db: DatabaseSync,
entity: LexicalEntity,
lineScopeSql: string,
params: number[],
): LexicalRemoval[] {
const table = entity === 'word' ? 'imm_word_line_occurrences' : 'imm_kanji_line_occurrences';
const col = `${entity}_id`;
return db
.prepare(
`SELECT
o.${col} AS id,
COALESCE(SUM(o.occurrence_count), 0) AS removedFrequency,
MIN(COALESCE(o.seen_ms, sl.CREATED_DATE, sl.LAST_UPDATE_DATE)) AS removedFirstSeenMs,
MAX(COALESCE(o.seen_ms, sl.LAST_UPDATE_DATE, sl.CREATED_DATE)) AS removedLastSeenMs
FROM imm_subtitle_lines sl
JOIN ${table} o ON o.line_id = sl.line_id
WHERE ${lineScopeSql}
GROUP BY o.${col}`,
)
.all(...params) as LexicalRemoval[];
}
function planLexicalRemovals(
db: DatabaseSync,
lineScopeSql: string,
params: number[],
): LexicalRemovalPlan {
return {
words: collectLexicalRemovals(db, 'word', lineScopeSql, params),
kanji: collectLexicalRemovals(db, 'kanji', lineScopeSql, params),
};
}
/**
* Measure what deleting these sessions removes from the vocabulary tables.
*
* Must run before the rows are deleted. Reads only the subtitle lines in scope,
* unlike {@link refreshLexicalAggregates}, which re-reads every occurrence of
* every affected word across the whole library.
*/
export function planLexicalRemovalsForSessions(
db: DatabaseSync,
sessionIds: number[],
): LexicalRemovalPlan {
if (sessionIds.length === 0) return EMPTY_LEXICAL_REMOVAL_PLAN;
return planLexicalRemovals(db, `sl.session_id IN (${makePlaceholders(sessionIds)})`, sessionIds);
}
/** Measure what deleting these videos removes from the vocabulary tables. */
export function planLexicalRemovalsForVideos(
db: DatabaseSync,
videoIds: number[],
): LexicalRemovalPlan {
if (videoIds.length === 0) return EMPTY_LEXICAL_REMOVAL_PLAN;
return planLexicalRemovals(db, `sl.video_id IN (${makePlaceholders(videoIds)})`, videoIds);
}
function toStoredSeenSeconds(ms: number | null): number | null {
if (ms === null || !Number.isFinite(ms)) return null;
return Math.floor(ms / 1000);
}
/**
* Apply a removal plan to the vocabulary aggregates.
*
* Frequencies are adjusted by subtraction, which is exact and touches only the
* affected rows. `first_seen`/`last_seen` only need a rescan when the removed
* lines held the current extreme, and rows whose frequency reaches zero are
* verified against the surviving occurrences before deletion — so stored counts
* that have drifted still converge on the truth instead of dropping a live row.
*/
export function applyLexicalRemovals(db: DatabaseSync, plan: LexicalRemovalPlan): void {
applyRemovalsForEntity(db, 'word', plan.words);
applyRemovalsForEntity(db, 'kanji', plan.kanji);
}
function applyRemovalsForEntity(
db: DatabaseSync,
entity: LexicalEntity,
removals: LexicalRemoval[],
): void {
if (removals.length === 0) return;
const entityTable = entity === 'word' ? 'imm_words' : 'imm_kanji';
const occurrenceTable =
entity === 'word' ? 'imm_word_line_occurrences' : 'imm_kanji_line_occurrences';
const col = `${entity}_id`;
const selectStmt = db.prepare(
`SELECT frequency, first_seen AS firstSeen, last_seen AS lastSeen
FROM ${entityTable}
WHERE id = ?`,
);
const updateFrequencyStmt = db.prepare(`UPDATE ${entityTable} SET frequency = ? WHERE id = ?`);
const hasOccurrencesStmt = db.prepare(
`SELECT 1 AS found FROM ${occurrenceTable} WHERE ${col} = ? LIMIT 1`,
);
const deleteStmt = db.prepare(`DELETE FROM ${entityTable} WHERE id = ?`);
const needsExactRefresh: number[] = [];
for (const removal of removals) {
const current = selectStmt.get(removal.id) as {
frequency: number | null;
firstSeen: number | null;
lastSeen: number | null;
} | null;
if (!current) continue;
const nextFrequency = (current.frequency ?? 0) - removal.removedFrequency;
if (nextFrequency <= 0) {
// The rows in scope are already gone by now, so anything still pointing at
// this entity means the stored frequency was stale rather than exhausted.
if (hasOccurrencesStmt.get(removal.id)) {
needsExactRefresh.push(removal.id);
} else {
deleteStmt.run(removal.id);
}
continue;
}
const removedFirstSeen = toStoredSeenSeconds(removal.removedFirstSeenMs);
const removedLastSeen = toStoredSeenSeconds(removal.removedLastSeenMs);
const firstSeenMayHaveMoved =
current.firstSeen === null ||
(removedFirstSeen !== null && removedFirstSeen <= current.firstSeen);
const lastSeenMayHaveMoved =
current.lastSeen === null ||
(removedLastSeen !== null && removedLastSeen >= current.lastSeen);
if (firstSeenMayHaveMoved || lastSeenMayHaveMoved) {
needsExactRefresh.push(removal.id);
continue;
}
updateFrequencyStmt.run(nextFrequency, removal.id);
}
if (entity === 'word') {
refreshWordAggregates(db, needsExactRefresh);
} else {
refreshKanjiAggregates(db, needsExactRefresh);
}
}
function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void {
if (wordIds.length === 0) {
return;
@@ -214,11 +375,18 @@ function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void {
SELECT
w.id AS wordId,
COALESCE(SUM(o.occurrence_count), 0) AS frequency,
MIN(COALESCE(sl.CREATED_DATE, sl.LAST_UPDATE_DATE)) AS firstSeen,
MAX(COALESCE(sl.LAST_UPDATE_DATE, sl.CREATED_DATE)) AS lastSeen
MIN(COALESCE(o.seen_ms, (
SELECT COALESCE(sl.CREATED_DATE, sl.LAST_UPDATE_DATE)
FROM imm_subtitle_lines sl
WHERE sl.line_id = o.line_id
))) AS firstSeen,
MAX(COALESCE(o.seen_ms, (
SELECT COALESCE(sl.CREATED_DATE, sl.LAST_UPDATE_DATE)
FROM imm_subtitle_lines sl
WHERE sl.line_id = o.line_id
))) AS lastSeen
FROM imm_words w
LEFT JOIN imm_word_line_occurrences o ON o.word_id = w.id
LEFT JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
WHERE w.id IN (${makePlaceholders(wordIds)})
GROUP BY w.id
`,
@@ -263,11 +431,18 @@ function refreshKanjiAggregates(db: DatabaseSync, kanjiIds: number[]): void {
SELECT
k.id AS kanjiId,
COALESCE(SUM(o.occurrence_count), 0) AS frequency,
MIN(COALESCE(sl.CREATED_DATE, sl.LAST_UPDATE_DATE)) AS firstSeen,
MAX(COALESCE(sl.LAST_UPDATE_DATE, sl.CREATED_DATE)) AS lastSeen
MIN(COALESCE(o.seen_ms, (
SELECT COALESCE(sl.CREATED_DATE, sl.LAST_UPDATE_DATE)
FROM imm_subtitle_lines sl
WHERE sl.line_id = o.line_id
))) AS firstSeen,
MAX(COALESCE(o.seen_ms, (
SELECT COALESCE(sl.CREATED_DATE, sl.LAST_UPDATE_DATE)
FROM imm_subtitle_lines sl
WHERE sl.line_id = o.line_id
))) AS lastSeen
FROM imm_kanji k
LEFT JOIN imm_kanji_line_occurrences o ON o.kanji_id = k.id
LEFT JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
WHERE k.id IN (${makePlaceholders(kanjiIds)})
GROUP BY k.id
`,
+56 -12
View File
@@ -192,6 +192,33 @@ function addColumnIfMissing(
}
}
/**
* Copy each subtitle line's timestamp onto its word/kanji occurrence rows.
*
* Vocabulary aggregates used to be recomputed by joining every occurrence back
* to `imm_subtitle_lines` just to read two timestamps, which meant one random
* read into the widest table per occurrence the reason deleting a session cost
* seconds on a large library. With the timestamp stored alongside the count, the
* covering index answers those aggregates on its own.
*/
function backfillLexicalOccurrenceSeenMs(db: DatabaseSync): void {
for (const table of ['imm_word_line_occurrences', 'imm_kanji_line_occurrences']) {
addColumnIfMissing(db, table, 'seen_ms', 'INTEGER');
db.exec(`
UPDATE ${table}
SET seen_ms = (
SELECT COALESCE(sl.CREATED_DATE, sl.LAST_UPDATE_DATE)
FROM imm_subtitle_lines sl
WHERE sl.line_id = ${table}.line_id
)
WHERE seen_ms IS NULL
`);
}
// Superseded by the covering (entity, seen_ms, occurrence_count, line_id) indexes.
db.exec('DROP INDEX IF EXISTS idx_word_line_occurrences_word');
db.exec('DROP INDEX IF EXISTS idx_kanji_line_occurrences_kanji');
}
function dropColumnIfExists(db: DatabaseSync, tableName: string, columnName: string): void {
if (hasColumn(db, tableName, columnName)) {
db.exec(`ALTER TABLE ${tableName} DROP COLUMN ${columnName}`);
@@ -923,6 +950,7 @@ export function ensureSchema(db: DatabaseSync): void {
line_id INTEGER NOT NULL,
word_id INTEGER NOT NULL,
occurrence_count INTEGER NOT NULL,
seen_ms INTEGER,
PRIMARY KEY(line_id, word_id),
FOREIGN KEY(line_id) REFERENCES imm_subtitle_lines(line_id) ON DELETE CASCADE,
FOREIGN KEY(word_id) REFERENCES imm_words(id) ON DELETE CASCADE
@@ -933,6 +961,7 @@ export function ensureSchema(db: DatabaseSync): void {
line_id INTEGER NOT NULL,
kanji_id INTEGER NOT NULL,
occurrence_count INTEGER NOT NULL,
seen_ms INTEGER,
PRIMARY KEY(line_id, kanji_id),
FOREIGN KEY(line_id) REFERENCES imm_subtitle_lines(line_id) ON DELETE CASCADE,
FOREIGN KEY(kanji_id) REFERENCES imm_kanji(id) ON DELETE CASCADE
@@ -1093,6 +1122,7 @@ export function ensureSchema(db: DatabaseSync): void {
line_id INTEGER NOT NULL,
word_id INTEGER NOT NULL,
occurrence_count INTEGER NOT NULL,
seen_ms INTEGER,
PRIMARY KEY(line_id, word_id),
FOREIGN KEY(line_id) REFERENCES imm_subtitle_lines(line_id) ON DELETE CASCADE,
FOREIGN KEY(word_id) REFERENCES imm_words(id) ON DELETE CASCADE
@@ -1103,6 +1133,7 @@ export function ensureSchema(db: DatabaseSync): void {
line_id INTEGER NOT NULL,
kanji_id INTEGER NOT NULL,
occurrence_count INTEGER NOT NULL,
seen_ms INTEGER,
PRIMARY KEY(line_id, kanji_id),
FOREIGN KEY(line_id) REFERENCES imm_subtitle_lines(line_id) ON DELETE CASCADE,
FOREIGN KEY(kanji_id) REFERENCES imm_kanji(id) ON DELETE CASCADE
@@ -1349,13 +1380,19 @@ export function ensureSchema(db: DatabaseSync): void {
CREATE INDEX IF NOT EXISTS idx_subtitle_lines_anime_line
ON imm_subtitle_lines(anime_id, line_index)
`);
if (currentVersion?.schema_version && currentVersion.schema_version < 19) {
backfillLexicalOccurrenceSeenMs(db);
}
// Covering indexes: vocabulary aggregates (frequency, first/last seen) are
// answered from the index alone, without reading the wide subtitle-line rows.
db.exec(`
CREATE INDEX IF NOT EXISTS idx_word_line_occurrences_word
ON imm_word_line_occurrences(word_id, line_id)
CREATE INDEX IF NOT EXISTS idx_word_line_occurrences_word_seen
ON imm_word_line_occurrences(word_id, seen_ms, occurrence_count, line_id)
`);
db.exec(`
CREATE INDEX IF NOT EXISTS idx_kanji_line_occurrences_kanji
ON imm_kanji_line_occurrences(kanji_id, line_id)
CREATE INDEX IF NOT EXISTS idx_kanji_line_occurrences_kanji_seen
ON imm_kanji_line_occurrences(kanji_id, seen_ms, occurrence_count, line_id)
`);
db.exec(`
CREATE INDEX IF NOT EXISTS idx_media_art_cover_blob_hash
@@ -1475,21 +1512,23 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
`),
wordLineOccurrenceUpsertStmt: db.prepare(`
INSERT INTO imm_word_line_occurrences (
line_id, word_id, occurrence_count
line_id, word_id, occurrence_count, seen_ms
) VALUES (
?, ?, ?
?, ?, ?, ?
)
ON CONFLICT(line_id, word_id) DO UPDATE SET
occurrence_count = imm_word_line_occurrences.occurrence_count + excluded.occurrence_count
occurrence_count = imm_word_line_occurrences.occurrence_count + excluded.occurrence_count,
seen_ms = COALESCE(imm_word_line_occurrences.seen_ms, excluded.seen_ms)
`),
kanjiLineOccurrenceUpsertStmt: db.prepare(`
INSERT INTO imm_kanji_line_occurrences (
line_id, kanji_id, occurrence_count
line_id, kanji_id, occurrence_count, seen_ms
) VALUES (
?, ?, ?
?, ?, ?, ?
)
ON CONFLICT(line_id, kanji_id) DO UPDATE SET
occurrence_count = imm_kanji_line_occurrences.occurrence_count + excluded.occurrence_count
occurrence_count = imm_kanji_line_occurrences.occurrence_count + excluded.occurrence_count,
seen_ms = COALESCE(imm_kanji_line_occurrences.seen_ms, excluded.seen_ms)
`),
videoAnimeIdSelectStmt: db.prepare(`
SELECT anime_id FROM imm_videos
@@ -1630,11 +1669,16 @@ export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedSta
const lineId = Number(lineResult.lastInsertRowid);
for (const occurrence of write.wordOccurrences) {
const wordId = incrementWordAggregate(stmts, occurrence, write.firstSeen, write.lastSeen);
stmts.wordLineOccurrenceUpsertStmt.run(lineId, wordId, occurrence.occurrenceCount);
stmts.wordLineOccurrenceUpsertStmt.run(lineId, wordId, occurrence.occurrenceCount, currentMs);
}
for (const occurrence of write.kanjiOccurrences) {
const kanjiId = incrementKanjiAggregate(stmts, occurrence, write.firstSeen, write.lastSeen);
stmts.kanjiLineOccurrenceUpsertStmt.run(lineId, kanjiId, occurrence.occurrenceCount);
stmts.kanjiLineOccurrenceUpsertStmt.run(
lineId,
kanjiId,
occurrence.occurrenceCount,
currentMs,
);
}
return;
}
+1 -1
View File
@@ -1,4 +1,4 @@
export const SCHEMA_VERSION = 18;
export const SCHEMA_VERSION = 19;
export const DEFAULT_QUEUE_CAP = 1_000;
export const DEFAULT_BATCH_SIZE = 25;
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
+28 -14
View File
@@ -1,3 +1,4 @@
import { createHash } from 'node:crypto';
import type { Hono } from 'hono';
import type { ImmersionTrackerService } from './immersion-tracker-service.js';
import { statsJson, type StatsCoverImagesRequest } from '../../types/stats-http-contract.js';
@@ -74,6 +75,31 @@ function detectImageContentType(bytes: Uint8Array): string {
return 'application/octet-stream';
}
function ifNoneMatchHits(header: string | undefined, etag: string): boolean {
if (!header) return false;
if (header.trim() === '*') return true;
return header
.split(',')
.map((candidate) => candidate.trim().replace(/^W\//, ''))
.some((candidate) => candidate === etag);
}
// Cover URLs are stable per anime/video, so relinking an AniList entry swaps the
// bytes behind an unchanged URL. Long max-age would keep serving the old art from
// the browser cache; revalidate instead and let the ETag skip the transfer.
function coverResponse(bytes: Uint8Array<ArrayBuffer>, ifNoneMatch: string | undefined): Response {
const etag = `"${createHash('sha256').update(bytes).digest('hex')}"`;
const headers: Record<string, string> = {
'Content-Type': detectImageContentType(bytes),
'Cache-Control': 'no-cache',
ETag: etag,
};
if (ifNoneMatchHits(ifNoneMatch, etag)) {
return new Response(null, { status: 304, headers });
}
return new Response(bytes, { headers });
}
function createLimitedTaskRunner(maxConcurrentTasks: number): (task: () => Promise<void>) => void {
const queue: Array<() => Promise<void>> = [];
let activeTasks = 0;
@@ -137,13 +163,7 @@ export function registerStatsCoverRoutes(app: Hono, tracker: ImmersionTrackerSer
art = await tracker.getAnimeCoverArt(animeId);
}
if (!art?.coverBlob) return c.body(null, 404);
const bytes = new Uint8Array(art.coverBlob);
return new Response(bytes, {
headers: {
'Content-Type': detectImageContentType(bytes),
'Cache-Control': 'public, max-age=86400',
},
});
return coverResponse(new Uint8Array(art.coverBlob), c.req.header('if-none-match'));
});
app.get('/api/stats/media/:videoId/cover', async (c) => {
@@ -155,12 +175,6 @@ export function registerStatsCoverRoutes(app: Hono, tracker: ImmersionTrackerSer
art = await tracker.getCoverArt(videoId);
}
if (!art?.coverBlob) return c.body(null, 404);
const bytes = new Uint8Array(art.coverBlob);
return new Response(bytes, {
headers: {
'Content-Type': detectImageContentType(bytes),
'Cache-Control': 'public, max-age=604800',
},
});
return coverResponse(new Uint8Array(art.coverBlob), c.req.header('if-none-match'));
});
}
@@ -190,4 +190,11 @@ export function registerStatsLibraryRoutes(
await tracker.deleteVideo(videoId);
return c.json(statsJson('deleteVideo', { ok: true }));
});
app.delete('/api/stats/anime/:animeId', async (c) => {
const animeId = parseIntQuery(c.req.param('animeId'), 0);
if (animeId <= 0) return c.body(null, 400);
await tracker.deleteAnime(animeId);
return c.json(statsJson('deleteAnime', { ok: true }));
});
}
@@ -0,0 +1,100 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { Database } from '../immersion-tracker/sqlite';
import { ensureSchema } from '../immersion-tracker/storage';
import { mergeSnapshotIntoDb } from './merge';
const BASE_MS = 1_700_000_000_000;
const DAY_MS = 86_400_000;
/**
* Build a database holding one anime, one episode, one ended session and a
* single word occurrence.
*
* `legacyOccurrences` reproduces a peer whose schema predates the denormalised
* `seen_ms` column, so the merge has to recover the timestamp from the line.
*/
function buildDb(
dir: string,
name: string,
options: { word: string; seenMs: number; legacyOccurrences: boolean },
): string {
const dbPath = path.join(dir, name);
const db = new Database(dbPath);
ensureSchema(db);
db.exec(`
INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (1, 'key-${name}', 'Show ${name}', ${BASE_MS}, ${BASE_MS});
INSERT INTO imm_videos(video_id, video_key, anime_id, canonical_title, source_type, watched, duration_ms, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (1, 'video-${name}', 1, 'Episode 1', 1, 1, 1440000, ${BASE_MS}, ${BASE_MS});
INSERT INTO imm_sessions(session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (1, 'uuid-${name}', 1, '${options.seenMs}', '${options.seenMs + 1000}', 2, ${BASE_MS}, ${BASE_MS});
INSERT INTO imm_subtitle_lines(line_id, session_id, video_id, anime_id, line_index, text, CREATED_DATE, LAST_UPDATE_DATE)
VALUES (1, 1, 1, 1, 0, 'line', ${options.seenMs}, ${options.seenMs});
INSERT INTO imm_words(id, headword, word, reading, part_of_speech, pos1, first_seen, last_seen, frequency)
VALUES (1, '${options.word}', '${options.word}', '', 'noun', '名詞',
${Math.floor(options.seenMs / 1000)}, ${Math.floor(options.seenMs / 1000)}, 1);
`);
db.exec(
options.legacyOccurrences
? 'INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count) VALUES (1, 1, 1)'
: `INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms) VALUES (1, 1, 1, ${options.seenMs})`,
);
db.close();
return dbPath;
}
function mergedOccurrences(dbPath: string): Array<{ word: string; seenMs: number | null }> {
const db = new Database(dbPath);
try {
return db
.prepare(
`SELECT w.word AS word, o.seen_ms AS seenMs
FROM imm_word_line_occurrences o
JOIN imm_words w ON w.id = o.word_id
ORDER BY w.word`,
)
.all() as Array<{ word: string; seenMs: number | null }>;
} finally {
db.close();
}
}
for (const legacyOccurrences of [false, true]) {
const label = legacyOccurrences ? 'a peer predating the seen_ms column' : 'a current peer';
test(`sync merge carries occurrence timestamps in from ${label}`, () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-merge-occurrences-test-'));
try {
const local = buildDb(dir, 'local.sqlite', {
word: '猫',
seenMs: BASE_MS,
legacyOccurrences: false,
});
const remoteSeenMs = BASE_MS + 5 * DAY_MS;
const remote = buildDb(dir, 'remote.sqlite', {
word: '犬',
seenMs: remoteSeenMs,
legacyOccurrences,
});
mergeSnapshotIntoDb(local, remote);
const rows = mergedOccurrences(local);
assert.equal(rows.length, 2, 'local and remote occurrences both survive the merge');
for (const row of rows) {
assert.ok(row.seenMs, `${row.word} must carry a timestamp so aggregates stay index-only`);
}
assert.equal(
Number(rows.find((row) => row.word === '犬')?.seenMs),
remoteSeenMs,
"the remote line's own timestamp is preserved rather than stamped with merge time",
);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
}
+13 -6
View File
@@ -215,12 +215,16 @@ function copySubtitleLines(
'SELECT kanji_id, occurrence_count FROM imm_kanji_line_occurrences WHERE line_id = ?',
);
const insertWordOccurrence = local.query(
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count) VALUES (?, ?, ?)
ON CONFLICT(line_id, word_id) DO UPDATE SET occurrence_count = occurrence_count + excluded.occurrence_count`,
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count, seen_ms) VALUES (?, ?, ?, ?)
ON CONFLICT(line_id, word_id) DO UPDATE SET
occurrence_count = occurrence_count + excluded.occurrence_count,
seen_ms = COALESCE(seen_ms, excluded.seen_ms)`,
);
const insertKanjiOccurrence = local.query(
`INSERT INTO imm_kanji_line_occurrences (line_id, kanji_id, occurrence_count) VALUES (?, ?, ?)
ON CONFLICT(line_id, kanji_id) DO UPDATE SET occurrence_count = occurrence_count + excluded.occurrence_count`,
`INSERT INTO imm_kanji_line_occurrences (line_id, kanji_id, occurrence_count, seen_ms) VALUES (?, ?, ?, ?)
ON CONFLICT(line_id, kanji_id) DO UPDATE SET
occurrence_count = occurrence_count + excluded.occurrence_count,
seen_ms = COALESCE(seen_ms, excluded.seen_ms)`,
);
for (const row of rows) {
@@ -241,17 +245,20 @@ function copySubtitleLines(
],
);
summary.subtitleLinesAdded += 1;
// Taken from the line rather than the remote occurrence row so this also
// works when the remote database predates the seen_ms column.
const seenMs = row.CREATED_DATE ?? row.LAST_UPDATE_DATE ?? null;
for (const occurrence of wordOccurrences.all(row.line_id) as SqlRow[]) {
const localWordId = lexicon.resolveWord(Number(occurrence.word_id));
const count = Number(occurrence.occurrence_count);
insertWordOccurrence.run(localLineId, localWordId, count);
insertWordOccurrence.run(localLineId, localWordId, count, seenMs);
lexicon.addWordOccurrences(Number(occurrence.word_id), count);
}
for (const occurrence of kanjiOccurrences.all(row.line_id) as SqlRow[]) {
const localKanjiId = lexicon.resolveKanji(Number(occurrence.kanji_id));
const count = Number(occurrence.occurrence_count);
insertKanjiOccurrence.run(localLineId, localKanjiId, count);
insertKanjiOccurrence.run(localLineId, localKanjiId, count, seenMs);
lexicon.addKanjiOccurrences(Number(occurrence.kanji_id), count);
}
}
+2
View File
@@ -141,6 +141,7 @@ export interface StatsJsonResponseMap {
deleteSessions: StatsOkResponse;
deleteSession: StatsOkResponse;
deleteVideo: StatsOkResponse;
deleteAnime: StatsOkResponse;
anilistSearch: StatsAnilistSearchResult[];
knownWords: string[];
knownWordsSummary: StatsKnownWordsSummary;
@@ -219,6 +220,7 @@ export interface StatsHttpClient {
deleteSession: (sessionId: number) => Promise<void>;
deleteSessions: (sessionIds: number[]) => Promise<void>;
deleteVideo: (videoId: number) => Promise<void>;
deleteAnime: (animeId: number) => Promise<void>;
getKnownWords: () => Promise<string[]>;
getKnownWordsSummary: () => Promise<StatsKnownWordsSummary>;
getAnimeKnownWordsSummary: (animeId: number) => Promise<StatsKnownWordsSummary>;