fix: address CodeRabbit review feedback

This commit is contained in:
2026-03-27 03:05:32 -07:00
parent ed32f985c6
commit 4d95de51a0
14 changed files with 673 additions and 159 deletions

View File

@@ -72,6 +72,14 @@ add_blocker() {
BLOCKED=1
}
validate_artifact_dir() {
local candidate=$1
if [[ ! "$candidate" =~ ^[A-Za-z0-9._/@:+-]+$ ]]; then
echo "Invalid characters in --artifact-dir path" >&2
exit 2
fi
}
append_step_record() {
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
"$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" >>"$STEPS_TSV"
@@ -411,6 +419,7 @@ if [[ -z "${ARTIFACT_DIR:-}" ]]; then
SESSION_ID=$(generate_session_id)
ARTIFACT_DIR="$REPO_ROOT/.tmp/skill-verification/$SESSION_ID"
else
validate_artifact_dir "$ARTIFACT_DIR"
SESSION_ID=$(basename "$ARTIFACT_DIR")
fi

View File

@@ -1982,6 +1982,7 @@ test('flushSingle reuses cached prepared statements', async () => {
cardsMined?: number;
lookupCount?: number;
lookupHits?: number;
yomitanLookupCount?: number;
pauseCount?: number;
pauseMs?: number;
seekForwardCount?: number;
@@ -2051,6 +2052,7 @@ test('flushSingle reuses cached prepared statements', async () => {
cardsMined: 0,
lookupCount: 0,
lookupHits: 0,
yomitanLookupCount: 0,
pauseCount: 0,
pauseMs: 0,
seekForwardCount: 0,

View File

@@ -208,6 +208,104 @@ test('getAnimeEpisodes prefers the latest session media position when the latest
}
});
test('getAnimeEpisodes includes unwatched episodes for the anime', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const watchedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/watched-episode.mkv', {
canonicalTitle: 'Watched Episode',
sourcePath: '/tmp/watched-episode.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const unwatchedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/unwatched-episode.mkv', {
canonicalTitle: 'Unwatched Episode',
sourcePath: '/tmp/unwatched-episode.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const animeId = getOrCreateAnimeRecord(db, {
parsedTitle: 'Episode Coverage Anime',
canonicalTitle: 'Episode Coverage Anime',
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
linkVideoToAnimeRecord(db, watchedVideoId, {
animeId,
parsedBasename: 'watched-episode.mkv',
parsedTitle: 'Episode Coverage Anime',
parsedSeason: 1,
parsedEpisode: 1,
parserSource: 'fallback',
parserConfidence: 1,
parseMetadataJson: '{"episode":1}',
});
linkVideoToAnimeRecord(db, unwatchedVideoId, {
animeId,
parsedBasename: 'unwatched-episode.mkv',
parsedTitle: 'Episode Coverage Anime',
parsedSeason: 1,
parsedEpisode: 2,
parserSource: 'fallback',
parserConfidence: 1,
parseMetadataJson: '{"episode":2}',
});
const watchedSessionId = startSessionRecord(db, watchedVideoId, 1_000_000).sessionId;
db.prepare(
`
UPDATE imm_sessions
SET
ended_at_ms = ?,
status = 2,
ended_media_ms = ?,
active_watched_ms = ?,
cards_mined = ?,
tokens_seen = ?,
yomitan_lookup_count = ?,
LAST_UPDATE_DATE = ?
WHERE session_id = ?
`,
).run(1_005_000, 7_000, 3_000, 2, 20, 4, 1_005_000, watchedSessionId);
const episodes = getAnimeEpisodes(db, animeId);
assert.equal(episodes.length, 2);
assert.deepEqual(
episodes.map((episode) => ({
videoId: episode.videoId,
totalSessions: episode.totalSessions,
totalActiveMs: episode.totalActiveMs,
totalCards: episode.totalCards,
totalTokensSeen: episode.totalTokensSeen,
})),
[
{
videoId: watchedVideoId,
totalSessions: 1,
totalActiveMs: 3_000,
totalCards: 2,
totalTokensSeen: 20,
},
{
videoId: unwatchedVideoId,
totalSessions: 0,
totalActiveMs: 0,
totalCards: 0,
totalTokensSeen: 0,
},
],
);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getAnimeEpisodes falls back to the latest subtitle segment end when session progress checkpoints are missing', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -586,6 +684,109 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => {
}
});
test('getTrendsDashboard keeps local-midnight session buckets separate', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const stmts = createTrackerPreparedStatements(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/local-midnight-trends.mkv', {
canonicalTitle: 'Local Midnight Trends',
sourcePath: '/tmp/local-midnight-trends.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const animeId = getOrCreateAnimeRecord(db, {
parsedTitle: 'Local Midnight Trends',
canonicalTitle: 'Local Midnight Trends',
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
linkVideoToAnimeRecord(db, videoId, {
animeId,
parsedBasename: 'local-midnight-trends.mkv',
parsedTitle: 'Local Midnight Trends',
parsedSeason: 1,
parsedEpisode: 1,
parserSource: 'test',
parserConfidence: 1,
parseMetadataJson: null,
});
const beforeMidnight = new Date(2026, 2, 1, 23, 30).getTime();
const afterMidnight = new Date(2026, 2, 2, 0, 30).getTime();
const firstSessionId = startSessionRecord(db, videoId, beforeMidnight).sessionId;
const secondSessionId = startSessionRecord(db, videoId, afterMidnight).sessionId;
for (const [sessionId, startedAtMs, tokensSeen, lookupCount] of [
[firstSessionId, beforeMidnight, 100, 4],
[secondSessionId, afterMidnight, 120, 6],
] as const) {
stmts.telemetryInsertStmt.run(
sessionId,
startedAtMs + 60_000,
60_000,
60_000,
1,
tokensSeen,
0,
lookupCount,
lookupCount,
lookupCount,
0,
0,
0,
0,
startedAtMs + 60_000,
startedAtMs + 60_000,
);
db.prepare(
`
UPDATE imm_sessions
SET
ended_at_ms = ?,
status = 2,
total_watched_ms = ?,
active_watched_ms = ?,
lines_seen = ?,
tokens_seen = ?,
lookup_count = ?,
lookup_hits = ?,
yomitan_lookup_count = ?,
LAST_UPDATE_DATE = ?
WHERE session_id = ?
`,
).run(
startedAtMs + 60_000,
60_000,
60_000,
1,
tokensSeen,
lookupCount,
lookupCount,
lookupCount,
startedAtMs + 60_000,
sessionId,
);
}
const dashboard = getTrendsDashboard(db, 'all', 'day');
assert.equal(dashboard.progress.lookups.length, 2);
assert.deepEqual(
dashboard.progress.lookups.map((point) => point.value),
[4, 10],
);
assert.equal(dashboard.ratios.lookupsPerHundred.length, 2);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getQueryHints reads all-time totals from lifetime summary', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -1024,6 +1225,36 @@ test('getMonthlyRollups returns all rows for the most recent rollup months', ()
}
});
test('getMonthlyRollups derives rate metrics from stored monthly totals', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const insertRollup = db.prepare(
`
INSERT INTO imm_monthly_rollups (
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
);
const nowMs = Date.now();
insertRollup.run(202602, 1, 2, 30, 20, 90, 15, nowMs, nowMs);
insertRollup.run(202602, 2, 1, 0, 10, 25, 5, nowMs, nowMs);
const rows = getMonthlyRollups(db, 1);
assert.equal(rows.length, 2);
assert.equal(rows[1]?.cardsPerHour, 30);
assert.equal(rows[1]?.tokensPerMin, 3);
assert.equal(rows[1]?.lookupHitRate ?? null, null);
assert.equal(rows[0]?.cardsPerHour ?? null, null);
assert.equal(rows[0]?.tokensPerMin ?? null, null);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getAnimeDailyRollups returns all rows for the most recent rollup days', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -2234,9 +2465,8 @@ test('cover art queries reuse a shared blob across duplicate anime art rows', ()
const animeArt = getAnimeCoverArt(db, animeId);
const library = getMediaLibrary(db);
assert.equal(artOne?.coverBlob?.length, 4);
assert.equal(artTwo?.coverBlob?.length, 4);
assert.deepEqual(artOne?.coverBlob, artTwo?.coverBlob);
assert.deepEqual(artOne?.coverBlob, Buffer.from([1, 2, 3, 4]));
assert.deepEqual(artTwo?.coverBlob, Buffer.from([9, 9, 9, 9]));
assert.equal(animeArt?.coverBlob?.length, 4);
assert.deepEqual(
library.map((row) => ({
@@ -2254,6 +2484,52 @@ test('cover art queries reuse a shared blob across duplicate anime art rows', ()
}
});
test('upsertCoverArt prefers freshly fetched bytes over a reused shared hash', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const originalVideoId = getOrCreateVideoRecord(db, 'local:/tmp/shared-cover-original.mkv', {
canonicalTitle: 'Shared Cover Original',
sourcePath: '/tmp/shared-cover-original.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const refreshedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/shared-cover-refresh.mkv', {
canonicalTitle: 'Shared Cover Refresh',
sourcePath: '/tmp/shared-cover-refresh.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
upsertCoverArt(db, originalVideoId, {
anilistId: 999,
coverUrl: 'https://images.test/shared-refresh.jpg',
coverBlob: Buffer.from([1, 2, 3, 4]),
titleRomaji: 'Shared Cover Refresh',
titleEnglish: 'Shared Cover Refresh',
episodesTotal: 12,
});
upsertCoverArt(db, refreshedVideoId, {
anilistId: 999,
coverUrl: 'https://images.test/shared-refresh.jpg',
coverBlob: Buffer.from([9, 8, 7, 6]),
titleRomaji: 'Shared Cover Refresh',
titleEnglish: 'Shared Cover Refresh',
episodesTotal: 12,
});
const originalArt = getCoverArt(db, originalVideoId);
const refreshedArt = getCoverArt(db, refreshedVideoId);
assert.deepEqual(originalArt?.coverBlob, Buffer.from([1, 2, 3, 4]));
assert.deepEqual(refreshedArt?.coverBlob, Buffer.from([9, 8, 7, 6]));
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('anime/media detail and episode queries use ended-session metrics when telemetry rows are absent', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -2836,13 +3112,13 @@ test('deleteSession rebuilds word and kanji aggregates from retained subtitle li
assert.ok(sharedWordRow);
assert.equal(sharedWordRow.frequency, 1);
assert.equal(sharedWordRow.first_seen, keptTs);
assert.equal(sharedWordRow.last_seen, keptTs);
assert.equal(sharedWordRow.first_seen, Math.floor(keptTs / 1000));
assert.equal(sharedWordRow.last_seen, Math.floor(keptTs / 1000));
assert.equal(deletedOnlyWordRow ?? null, null);
assert.ok(sharedKanjiRow);
assert.equal(sharedKanjiRow.frequency, 1);
assert.equal(sharedKanjiRow.first_seen, keptTs);
assert.equal(sharedKanjiRow.last_seen, keptTs);
assert.equal(sharedKanjiRow.first_seen, Math.floor(keptTs / 1000));
assert.equal(sharedKanjiRow.last_seen, Math.floor(keptTs / 1000));
assert.equal(deletedOnlyKanjiRow ?? null, null);
} finally {
db.close();

View File

@@ -156,7 +156,7 @@ export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisod
COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount,
MAX(s.started_at_ms) AS lastWatchedMs
FROM imm_videos v
JOIN imm_sessions s ON s.video_id = v.video_id
LEFT JOIN imm_sessions s ON s.video_id = v.video_id
LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
WHERE v.anime_id = ?
GROUP BY v.video_id

View File

@@ -352,15 +352,16 @@ export function upsertCoverArt(
const sharedCoverBlobHash = findSharedCoverBlobHash(db, videoId, art.anilistId, art.coverUrl);
const fetchedAtMs = toDbMs(nowMs());
const coverBlob = normalizeCoverBlobBytes(art.coverBlob);
let coverBlobHash = sharedCoverBlobHash ?? null;
if (!coverBlobHash && coverBlob && coverBlob.length > 0) {
coverBlobHash = createHash('sha256').update(coverBlob).digest('hex');
}
const computedCoverBlobHash =
coverBlob && coverBlob.length > 0
? createHash('sha256').update(coverBlob).digest('hex')
: null;
let coverBlobHash = computedCoverBlobHash ?? sharedCoverBlobHash ?? null;
if (!coverBlobHash && (!coverBlob || coverBlob.length === 0)) {
coverBlobHash = existing?.coverBlobHash ?? null;
}
if (coverBlobHash && coverBlob && coverBlob.length > 0 && !sharedCoverBlobHash) {
if (computedCoverBlobHash && coverBlob && coverBlob.length > 0) {
db.prepare(
`
INSERT INTO imm_cover_art_blobs (blob_hash, cover_blob, CREATED_DATE, LAST_UPDATE_DATE)
@@ -368,7 +369,7 @@ export function upsertCoverArt(
ON CONFLICT(blob_hash) DO UPDATE SET
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
`,
).run(coverBlobHash, coverBlob, fetchedAtMs, fetchedAtMs);
).run(computedCoverBlobHash, coverBlob, fetchedAtMs, fetchedAtMs);
}
db.prepare(

View File

@@ -204,7 +204,7 @@ export function getQueryHints(db: DatabaseSync): {
const now = new Date();
const todayLocal = Math.floor(
new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime() / 86_400_000,
(now.getTime() / 1000 - now.getTimezoneOffset() * 60) / 86_400,
);
const episodesToday =
@@ -333,9 +333,15 @@ export function getMonthlyRollups(db: DatabaseSync, limit = 24): ImmersionSessio
total_lines_seen AS totalLinesSeen,
total_tokens_seen AS totalTokensSeen,
total_cards AS totalCards,
0 AS cardsPerHour,
0 AS tokensPerMin,
0 AS lookupHitRate
CASE
WHEN total_active_min > 0 THEN (total_cards * 60.0) / total_active_min
ELSE NULL
END AS cardsPerHour,
CASE
WHEN total_active_min > 0 THEN total_tokens_seen * 1.0 / total_active_min
ELSE NULL
END AS tokensPerMin,
NULL AS lookupHitRate
FROM imm_monthly_rollups
WHERE rollup_month IN (SELECT rollup_month FROM recent_months)
ORDER BY rollup_month DESC, video_id DESC

View File

@@ -197,7 +197,12 @@ function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void {
deleteStmt.run(row.wordId);
continue;
}
updateStmt.run(row.frequency, row.firstSeen, row.lastSeen, row.wordId);
updateStmt.run(
row.frequency,
Math.floor(row.firstSeen / 1000),
Math.floor(row.lastSeen / 1000),
row.wordId,
);
}
}
@@ -241,7 +246,12 @@ function refreshKanjiAggregates(db: DatabaseSync, kanjiIds: number[]): void {
deleteStmt.run(row.kanjiId);
continue;
}
updateStmt.run(row.frequency, row.firstSeen, row.lastSeen, row.kanjiId);
updateStmt.run(
row.frequency,
Math.floor(row.firstSeen / 1000),
Math.floor(row.lastSeen / 1000),
row.kanjiId,
);
}
}

View File

@@ -112,6 +112,16 @@ function makeTrendLabel(value: number): string {
});
}
function getLocalEpochDay(timestampMs: number): number {
const date = new Date(timestampMs);
return Math.floor((timestampMs - date.getTimezoneOffset() * 60_000) / 86_400_000);
}
function getLocalDateForEpochDay(epochDay: number): Date {
const utcDate = new Date(epochDay * 86_400_000);
return new Date(utcDate.getTime() + utcDate.getTimezoneOffset() * 60_000);
}
function getTrendSessionWordCount(session: Pick<TrendSessionMetricRow, 'tokensSeen'>): number {
return session.tokensSeen;
}
@@ -188,7 +198,7 @@ function buildWatchTimeByHour(sessions: TrendSessionMetricRow[]): TrendChartPoin
}
function dayLabel(epochDay: number): string {
return new Date(epochDay * 86_400_000).toLocaleDateString(undefined, {
return getLocalDateForEpochDay(epochDay).toLocaleDateString(undefined, {
month: 'short',
day: 'numeric',
});
@@ -200,7 +210,7 @@ function buildSessionSeriesByDay(
): TrendChartPoint[] {
const byDay = new Map<number, number>();
for (const session of sessions) {
const epochDay = Math.floor(session.startedAtMs / 86_400_000);
const epochDay = getLocalEpochDay(session.startedAtMs);
byDay.set(epochDay, (byDay.get(epochDay) ?? 0) + getValue(session));
}
return Array.from(byDay.entries())
@@ -213,7 +223,7 @@ function buildLookupsPerHundredWords(sessions: TrendSessionMetricRow[]): TrendCh
const wordsByDay = new Map<number, number>();
for (const session of sessions) {
const epochDay = Math.floor(session.startedAtMs / 86_400_000);
const epochDay = getLocalEpochDay(session.startedAtMs);
lookupsByDay.set(epochDay, (lookupsByDay.get(epochDay) ?? 0) + session.yomitanLookupCount);
wordsByDay.set(epochDay, (wordsByDay.get(epochDay) ?? 0) + getTrendSessionWordCount(session));
}
@@ -237,7 +247,7 @@ function buildPerAnimeFromSessions(
for (const session of sessions) {
const animeTitle = resolveTrendAnimeTitle(session);
const epochDay = Math.floor(session.startedAtMs / 86_400_000);
const epochDay = getLocalEpochDay(session.startedAtMs);
const dayMap = byAnime.get(animeTitle) ?? new Map();
dayMap.set(epochDay, (dayMap.get(epochDay) ?? 0) + getValue(session));
byAnime.set(animeTitle, dayMap);
@@ -258,7 +268,7 @@ function buildLookupsPerHundredPerAnime(sessions: TrendSessionMetricRow[]): Tren
for (const session of sessions) {
const animeTitle = resolveTrendAnimeTitle(session);
const epochDay = Math.floor(session.startedAtMs / 86_400_000);
const epochDay = getLocalEpochDay(session.startedAtMs);
const lookupMap = lookups.get(animeTitle) ?? new Map();
lookupMap.set(epochDay, (lookupMap.get(epochDay) ?? 0) + session.yomitanLookupCount);
@@ -462,7 +472,7 @@ function buildNewWordsPerDay(db: DatabaseSync, cutoffMs: number | null): TrendCh
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
const prepared = db.prepare(`
SELECT
CAST(first_seen / 86400 AS INTEGER) AS epochDay,
CAST(julianday(first_seen, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) AS epochDay,
COUNT(*) AS wordCount
FROM imm_words
WHERE first_seen IS NOT NULL

View File

@@ -1078,6 +1078,56 @@ test('executeQueuedWrite inserts event and telemetry rows', () => {
}
});
test('executeQueuedWrite rejects partial telemetry writes instead of zero-filling', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const stmts = createTrackerPreparedStatements(db);
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/partial-telemetry.mkv', {
canonicalTitle: 'Partial Telemetry',
sourcePath: '/tmp/partial-telemetry.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const { sessionId } = startSessionRecord(db, videoId, 5_000);
assert.throws(
() =>
executeQueuedWrite(
{
kind: 'telemetry',
sessionId,
sampleMs: 6_000,
totalWatchedMs: 1_000,
activeWatchedMs: 900,
linesSeen: 3,
cardsMined: 1,
lookupCount: 2,
lookupHits: 1,
yomitanLookupCount: 0,
pauseCount: 1,
pauseMs: 50,
seekForwardCount: 0,
seekBackwardCount: 0,
mediaBufferEvents: 0,
},
stmts,
),
/Incomplete telemetry write/,
);
const telemetryCount = db
.prepare('SELECT COUNT(*) AS total FROM imm_session_telemetry WHERE session_id = ?')
.get(sessionId) as { total: number };
assert.equal(telemetryCount.total, 0);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('executeQueuedWrite inserts and upserts word and kanji rows', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);

View File

@@ -1406,27 +1406,46 @@ function incrementKanjiAggregate(
export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedStatements): void {
const currentMs = toDbMs(nowMs());
if (write.kind === 'telemetry') {
if (
write.totalWatchedMs === undefined ||
write.activeWatchedMs === undefined ||
write.linesSeen === undefined ||
write.tokensSeen === undefined ||
write.cardsMined === undefined ||
write.lookupCount === undefined ||
write.lookupHits === undefined ||
write.yomitanLookupCount === undefined ||
write.pauseCount === undefined ||
write.pauseMs === undefined ||
write.seekForwardCount === undefined ||
write.seekBackwardCount === undefined ||
write.mediaBufferEvents === undefined
) {
throw new Error('Incomplete telemetry write');
}
const telemetrySampleMs = toDbMs(write.sampleMs ?? Number(currentMs));
stmts.telemetryInsertStmt.run(
write.sessionId,
telemetrySampleMs,
write.totalWatchedMs ?? 0,
write.activeWatchedMs ?? 0,
write.linesSeen ?? 0,
write.tokensSeen ?? 0,
write.cardsMined ?? 0,
write.lookupCount ?? 0,
write.lookupHits ?? 0,
write.yomitanLookupCount ?? 0,
write.pauseCount ?? 0,
write.pauseMs ?? 0,
write.seekForwardCount ?? 0,
write.seekBackwardCount ?? 0,
write.mediaBufferEvents ?? 0,
write.totalWatchedMs,
write.activeWatchedMs,
write.linesSeen,
write.tokensSeen,
write.cardsMined,
write.lookupCount,
write.lookupHits,
write.yomitanLookupCount,
write.pauseCount,
write.pauseMs,
write.seekForwardCount,
write.seekBackwardCount,
write.mediaBufferEvents,
currentMs,
currentMs,
);
if (write.lastMediaMs !== undefined) {
stmts.sessionCheckpointStmt.run(write.lastMediaMs ?? null, currentMs, write.sessionId);
}
return;
}
if (write.kind === 'word') {

View File

@@ -0,0 +1,7 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { nowMs } from './time.js';
test('nowMs returns wall-clock epoch milliseconds', () => {
assert.ok(nowMs() > 1_600_000_000_000);
});

View File

@@ -1,10 +1,8 @@
const SQLITE_SAFE_EPOCH_BASE_MS = 2_000_000_000;
export function nowMs(): number {
const perf = globalThis.performance;
if (perf) {
return SQLITE_SAFE_EPOCH_BASE_MS + Math.floor(perf.now());
if (perf && Number.isFinite(perf.timeOrigin)) {
return Math.floor(perf.timeOrigin + perf.now());
}
return SQLITE_SAFE_EPOCH_BASE_MS;
return Date.now();
}

View File

@@ -0,0 +1,98 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { buildDictionaryZip } from './zip';
import type { CharacterDictionaryTermEntry } from './types';
function makeTempDir(): string {
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-character-zip-'));
}
function cleanupDir(dirPath: string): void {
fs.rmSync(dirPath, { recursive: true, force: true });
}
function readStoredZipEntries(zipPath: string): Map<string, Buffer> {
const archive = fs.readFileSync(zipPath);
const entries = new Map<string, Buffer>();
let cursor = 0;
while (cursor + 4 <= archive.length) {
const signature = archive.readUInt32LE(cursor);
if (signature === 0x02014b50 || signature === 0x06054b50) {
break;
}
assert.equal(signature, 0x04034b50, `unexpected local file header at offset ${cursor}`);
const compressedSize = archive.readUInt32LE(cursor + 18);
const fileNameLength = archive.readUInt16LE(cursor + 26);
const extraLength = archive.readUInt16LE(cursor + 28);
const fileNameStart = cursor + 30;
const dataStart = fileNameStart + fileNameLength + extraLength;
const fileName = archive.subarray(fileNameStart, fileNameStart + fileNameLength).toString(
'utf8',
);
const data = archive.subarray(dataStart, dataStart + compressedSize);
entries.set(fileName, Buffer.from(data));
cursor = dataStart + compressedSize;
}
return entries;
}
test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', () => {
const tempDir = makeTempDir();
const outputPath = path.join(tempDir, 'dictionary.zip');
const termEntries: CharacterDictionaryTermEntry[] = [
['アルファ', 'あるふぁ', '', '', 0, ['Alpha entry'], 0, 'name'],
];
const originalBufferConcat = Buffer.concat;
try {
Buffer.concat = ((...args: Parameters<typeof Buffer.concat>) => {
throw new Error(`buildDictionaryZip should not Buffer.concat the full archive (${args[0].length} chunks)`);
}) as typeof Buffer.concat;
const result = buildDictionaryZip(
outputPath,
'Dictionary Title',
'Dictionary Description',
'2026-03-27',
termEntries,
[{ path: 'images/alpha.bin', dataBase64: Buffer.from([1, 2, 3]).toString('base64') }],
);
assert.equal(result.zipPath, outputPath);
assert.equal(result.entryCount, 1);
const entries = readStoredZipEntries(outputPath);
assert.deepEqual([...entries.keys()].sort(), [
'images/alpha.bin',
'index.json',
'tag_bank_1.json',
'term_bank_1.json',
]);
const indexJson = JSON.parse(entries.get('index.json')!.toString('utf8')) as {
title: string;
description: string;
revision: string;
format: number;
};
assert.equal(indexJson.title, 'Dictionary Title');
assert.equal(indexJson.description, 'Dictionary Description');
assert.equal(indexJson.revision, '2026-03-27');
assert.equal(indexJson.format, 3);
const termBank = JSON.parse(entries.get('term_bank_1.json')!.toString('utf8')) as
CharacterDictionaryTermEntry[];
assert.equal(termBank.length, 1);
assert.equal(termBank[0]?.[0], 'アルファ');
assert.deepEqual(entries.get('images/alpha.bin'), Buffer.from([1, 2, 3]));
} finally {
Buffer.concat = originalBufferConcat;
cleanupDir(tempDir);
}
});

View File

@@ -5,8 +5,8 @@ import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } f
type ZipEntry = {
name: string;
data: Buffer;
crc32: number;
size: number;
localHeaderOffset: number;
};
@@ -67,15 +67,7 @@ function crc32(data: Buffer): number {
return (crc ^ 0xffffffff) >>> 0;
}
function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
const chunks: Buffer[] = [];
const entries: ZipEntry[] = [];
let offset = 0;
for (const file of files) {
const fileName = Buffer.from(file.name, 'utf8');
const fileData = file.data;
const fileCrc32 = crc32(fileData);
function createLocalFileHeader(fileName: Buffer, fileCrc32: number, fileSize: number): Buffer {
const local = Buffer.alloc(30 + fileName.length);
let cursor = 0;
writeUint32LE(local, 0x04034b50, cursor);
@@ -92,29 +84,19 @@ function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
cursor += 2;
writeUint32LE(local, fileCrc32, cursor);
cursor += 4;
writeUint32LE(local, fileData.length, cursor);
writeUint32LE(local, fileSize, cursor);
cursor += 4;
writeUint32LE(local, fileData.length, cursor);
writeUint32LE(local, fileSize, cursor);
cursor += 4;
local.writeUInt16LE(fileName.length, cursor);
cursor += 2;
local.writeUInt16LE(0, cursor);
cursor += 2;
fileName.copy(local, cursor);
chunks.push(local, fileData);
entries.push({
name: file.name,
data: fileData,
crc32: fileCrc32,
localHeaderOffset: offset,
});
offset += local.length + fileData.length;
return local;
}
const centralStart = offset;
const centralChunks: Buffer[] = [];
for (const entry of entries) {
function createCentralDirectoryHeader(entry: ZipEntry): Buffer {
const fileName = Buffer.from(entry.name, 'utf8');
const central = Buffer.alloc(46 + fileName.length);
let cursor = 0;
@@ -134,9 +116,9 @@ function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
cursor += 2;
writeUint32LE(central, entry.crc32, cursor);
cursor += 4;
writeUint32LE(central, entry.data.length, cursor);
writeUint32LE(central, entry.size, cursor);
cursor += 4;
writeUint32LE(central, entry.data.length, cursor);
writeUint32LE(central, entry.size, cursor);
cursor += 4;
central.writeUInt16LE(fileName.length, cursor);
cursor += 2;
@@ -153,11 +135,10 @@ function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
writeUint32LE(central, entry.localHeaderOffset, cursor);
cursor += 4;
fileName.copy(central, cursor);
centralChunks.push(central);
offset += central.length;
return central;
}
const centralSize = offset - centralStart;
function createEndOfCentralDirectory(entriesLength: number, centralSize: number, centralStart: number): Buffer {
const end = Buffer.alloc(22);
let cursor = 0;
writeUint32LE(end, 0x06054b50, cursor);
@@ -166,17 +147,63 @@ function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
cursor += 2;
end.writeUInt16LE(0, cursor);
cursor += 2;
end.writeUInt16LE(entries.length, cursor);
end.writeUInt16LE(entriesLength, cursor);
cursor += 2;
end.writeUInt16LE(entries.length, cursor);
end.writeUInt16LE(entriesLength, cursor);
cursor += 2;
writeUint32LE(end, centralSize, cursor);
cursor += 4;
writeUint32LE(end, centralStart, cursor);
cursor += 4;
end.writeUInt16LE(0, cursor);
return end;
}
return Buffer.concat([...chunks, ...centralChunks, end]);
function writeBuffer(fd: number, buffer: Buffer): void {
let written = 0;
while (written < buffer.length) {
written += fs.writeSync(fd, buffer, written, buffer.length - written);
}
}
function writeStoredZip(outputPath: string, files: Iterable<{ name: string; data: Buffer }>): void {
const entries: ZipEntry[] = [];
let offset = 0;
const fd = fs.openSync(outputPath, 'w');
try {
for (const file of files) {
const fileName = Buffer.from(file.name, 'utf8');
const fileSize = file.data.length;
const fileCrc32 = crc32(file.data);
const localHeader = createLocalFileHeader(fileName, fileCrc32, fileSize);
writeBuffer(fd, localHeader);
writeBuffer(fd, file.data);
entries.push({
name: file.name,
crc32: fileCrc32,
size: fileSize,
localHeaderOffset: offset,
});
offset += localHeader.length + fileSize;
}
const centralStart = offset;
for (const entry of entries) {
const centralHeader = createCentralDirectoryHeader(entry);
writeBuffer(fd, centralHeader);
offset += centralHeader.length;
}
const centralSize = offset - centralStart;
writeBuffer(fd, createEndOfCentralDirectory(entries.length, centralSize, centralStart));
} catch (error) {
fs.closeSync(fd);
fs.rmSync(outputPath, { force: true });
throw error;
}
fs.closeSync(fd);
}
export function buildDictionaryZip(
@@ -187,36 +214,37 @@ export function buildDictionaryZip(
termEntries: CharacterDictionaryTermEntry[],
images: CharacterDictionarySnapshotImage[],
): { zipPath: string; entryCount: number } {
const zipFiles: Array<{ name: string; data: Buffer }> = [
{
ensureDir(path.dirname(outputPath));
function* zipFiles(): Iterable<{ name: string; data: Buffer }> {
yield {
name: 'index.json',
data: Buffer.from(
JSON.stringify(createIndex(dictionaryTitle, description, revision), null, 2),
'utf8',
),
},
{
};
yield {
name: 'tag_bank_1.json',
data: Buffer.from(JSON.stringify(createTagBank()), 'utf8'),
},
];
};
for (const image of images) {
zipFiles.push({
yield {
name: image.path,
data: Buffer.from(image.dataBase64, 'base64'),
});
};
}
const entriesPerBank = 10_000;
for (let i = 0; i < termEntries.length; i += entriesPerBank) {
zipFiles.push({
yield {
name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`,
data: Buffer.from(JSON.stringify(termEntries.slice(i, i + entriesPerBank)), 'utf8'),
});
};
}
}
ensureDir(path.dirname(outputPath));
fs.writeFileSync(outputPath, createStoredZip(zipFiles));
writeStoredZip(outputPath, zipFiles());
return { zipPath: outputPath, entryCount: termEntries.length };
}