mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-03-27 06:12:05 -07:00
fix: address CodeRabbit review feedback
This commit is contained in:
@@ -72,6 +72,14 @@ add_blocker() {
|
||||
BLOCKED=1
|
||||
}
|
||||
|
||||
validate_artifact_dir() {
|
||||
local candidate=$1
|
||||
if [[ ! "$candidate" =~ ^[A-Za-z0-9._/@:+-]+$ ]]; then
|
||||
echo "Invalid characters in --artifact-dir path" >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
append_step_record() {
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" >>"$STEPS_TSV"
|
||||
@@ -411,6 +419,7 @@ if [[ -z "${ARTIFACT_DIR:-}" ]]; then
|
||||
SESSION_ID=$(generate_session_id)
|
||||
ARTIFACT_DIR="$REPO_ROOT/.tmp/skill-verification/$SESSION_ID"
|
||||
else
|
||||
validate_artifact_dir "$ARTIFACT_DIR"
|
||||
SESSION_ID=$(basename "$ARTIFACT_DIR")
|
||||
fi
|
||||
|
||||
|
||||
@@ -1982,6 +1982,7 @@ test('flushSingle reuses cached prepared statements', async () => {
|
||||
cardsMined?: number;
|
||||
lookupCount?: number;
|
||||
lookupHits?: number;
|
||||
yomitanLookupCount?: number;
|
||||
pauseCount?: number;
|
||||
pauseMs?: number;
|
||||
seekForwardCount?: number;
|
||||
@@ -2051,6 +2052,7 @@ test('flushSingle reuses cached prepared statements', async () => {
|
||||
cardsMined: 0,
|
||||
lookupCount: 0,
|
||||
lookupHits: 0,
|
||||
yomitanLookupCount: 0,
|
||||
pauseCount: 0,
|
||||
pauseMs: 0,
|
||||
seekForwardCount: 0,
|
||||
|
||||
@@ -208,6 +208,104 @@ test('getAnimeEpisodes prefers the latest session media position when the latest
|
||||
}
|
||||
});
|
||||
|
||||
test('getAnimeEpisodes includes unwatched episodes for the anime', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const watchedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/watched-episode.mkv', {
|
||||
canonicalTitle: 'Watched Episode',
|
||||
sourcePath: '/tmp/watched-episode.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
const unwatchedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/unwatched-episode.mkv', {
|
||||
canonicalTitle: 'Unwatched Episode',
|
||||
sourcePath: '/tmp/unwatched-episode.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: 'Episode Coverage Anime',
|
||||
canonicalTitle: 'Episode Coverage Anime',
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: null,
|
||||
});
|
||||
linkVideoToAnimeRecord(db, watchedVideoId, {
|
||||
animeId,
|
||||
parsedBasename: 'watched-episode.mkv',
|
||||
parsedTitle: 'Episode Coverage Anime',
|
||||
parsedSeason: 1,
|
||||
parsedEpisode: 1,
|
||||
parserSource: 'fallback',
|
||||
parserConfidence: 1,
|
||||
parseMetadataJson: '{"episode":1}',
|
||||
});
|
||||
linkVideoToAnimeRecord(db, unwatchedVideoId, {
|
||||
animeId,
|
||||
parsedBasename: 'unwatched-episode.mkv',
|
||||
parsedTitle: 'Episode Coverage Anime',
|
||||
parsedSeason: 1,
|
||||
parsedEpisode: 2,
|
||||
parserSource: 'fallback',
|
||||
parserConfidence: 1,
|
||||
parseMetadataJson: '{"episode":2}',
|
||||
});
|
||||
|
||||
const watchedSessionId = startSessionRecord(db, watchedVideoId, 1_000_000).sessionId;
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE imm_sessions
|
||||
SET
|
||||
ended_at_ms = ?,
|
||||
status = 2,
|
||||
ended_media_ms = ?,
|
||||
active_watched_ms = ?,
|
||||
cards_mined = ?,
|
||||
tokens_seen = ?,
|
||||
yomitan_lookup_count = ?,
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE session_id = ?
|
||||
`,
|
||||
).run(1_005_000, 7_000, 3_000, 2, 20, 4, 1_005_000, watchedSessionId);
|
||||
|
||||
const episodes = getAnimeEpisodes(db, animeId);
|
||||
assert.equal(episodes.length, 2);
|
||||
assert.deepEqual(
|
||||
episodes.map((episode) => ({
|
||||
videoId: episode.videoId,
|
||||
totalSessions: episode.totalSessions,
|
||||
totalActiveMs: episode.totalActiveMs,
|
||||
totalCards: episode.totalCards,
|
||||
totalTokensSeen: episode.totalTokensSeen,
|
||||
})),
|
||||
[
|
||||
{
|
||||
videoId: watchedVideoId,
|
||||
totalSessions: 1,
|
||||
totalActiveMs: 3_000,
|
||||
totalCards: 2,
|
||||
totalTokensSeen: 20,
|
||||
},
|
||||
{
|
||||
videoId: unwatchedVideoId,
|
||||
totalSessions: 0,
|
||||
totalActiveMs: 0,
|
||||
totalCards: 0,
|
||||
totalTokensSeen: 0,
|
||||
},
|
||||
],
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getAnimeEpisodes falls back to the latest subtitle segment end when session progress checkpoints are missing', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
@@ -586,6 +684,109 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('getTrendsDashboard keeps local-midnight session buckets separate', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const stmts = createTrackerPreparedStatements(db);
|
||||
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/local-midnight-trends.mkv', {
|
||||
canonicalTitle: 'Local Midnight Trends',
|
||||
sourcePath: '/tmp/local-midnight-trends.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: 'Local Midnight Trends',
|
||||
canonicalTitle: 'Local Midnight Trends',
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: null,
|
||||
});
|
||||
linkVideoToAnimeRecord(db, videoId, {
|
||||
animeId,
|
||||
parsedBasename: 'local-midnight-trends.mkv',
|
||||
parsedTitle: 'Local Midnight Trends',
|
||||
parsedSeason: 1,
|
||||
parsedEpisode: 1,
|
||||
parserSource: 'test',
|
||||
parserConfidence: 1,
|
||||
parseMetadataJson: null,
|
||||
});
|
||||
|
||||
const beforeMidnight = new Date(2026, 2, 1, 23, 30).getTime();
|
||||
const afterMidnight = new Date(2026, 2, 2, 0, 30).getTime();
|
||||
const firstSessionId = startSessionRecord(db, videoId, beforeMidnight).sessionId;
|
||||
const secondSessionId = startSessionRecord(db, videoId, afterMidnight).sessionId;
|
||||
|
||||
for (const [sessionId, startedAtMs, tokensSeen, lookupCount] of [
|
||||
[firstSessionId, beforeMidnight, 100, 4],
|
||||
[secondSessionId, afterMidnight, 120, 6],
|
||||
] as const) {
|
||||
stmts.telemetryInsertStmt.run(
|
||||
sessionId,
|
||||
startedAtMs + 60_000,
|
||||
60_000,
|
||||
60_000,
|
||||
1,
|
||||
tokensSeen,
|
||||
0,
|
||||
lookupCount,
|
||||
lookupCount,
|
||||
lookupCount,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
startedAtMs + 60_000,
|
||||
startedAtMs + 60_000,
|
||||
);
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE imm_sessions
|
||||
SET
|
||||
ended_at_ms = ?,
|
||||
status = 2,
|
||||
total_watched_ms = ?,
|
||||
active_watched_ms = ?,
|
||||
lines_seen = ?,
|
||||
tokens_seen = ?,
|
||||
lookup_count = ?,
|
||||
lookup_hits = ?,
|
||||
yomitan_lookup_count = ?,
|
||||
LAST_UPDATE_DATE = ?
|
||||
WHERE session_id = ?
|
||||
`,
|
||||
).run(
|
||||
startedAtMs + 60_000,
|
||||
60_000,
|
||||
60_000,
|
||||
1,
|
||||
tokensSeen,
|
||||
lookupCount,
|
||||
lookupCount,
|
||||
lookupCount,
|
||||
startedAtMs + 60_000,
|
||||
sessionId,
|
||||
);
|
||||
}
|
||||
|
||||
const dashboard = getTrendsDashboard(db, 'all', 'day');
|
||||
assert.equal(dashboard.progress.lookups.length, 2);
|
||||
assert.deepEqual(
|
||||
dashboard.progress.lookups.map((point) => point.value),
|
||||
[4, 10],
|
||||
);
|
||||
assert.equal(dashboard.ratios.lookupsPerHundred.length, 2);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getQueryHints reads all-time totals from lifetime summary', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
@@ -1024,6 +1225,36 @@ test('getMonthlyRollups returns all rows for the most recent rollup months', ()
|
||||
}
|
||||
});
|
||||
|
||||
test('getMonthlyRollups derives rate metrics from stored monthly totals', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertRollup = db.prepare(
|
||||
`
|
||||
INSERT INTO imm_monthly_rollups (
|
||||
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
);
|
||||
const nowMs = Date.now();
|
||||
insertRollup.run(202602, 1, 2, 30, 20, 90, 15, nowMs, nowMs);
|
||||
insertRollup.run(202602, 2, 1, 0, 10, 25, 5, nowMs, nowMs);
|
||||
|
||||
const rows = getMonthlyRollups(db, 1);
|
||||
assert.equal(rows.length, 2);
|
||||
assert.equal(rows[1]?.cardsPerHour, 30);
|
||||
assert.equal(rows[1]?.tokensPerMin, 3);
|
||||
assert.equal(rows[1]?.lookupHitRate ?? null, null);
|
||||
assert.equal(rows[0]?.cardsPerHour ?? null, null);
|
||||
assert.equal(rows[0]?.tokensPerMin ?? null, null);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getAnimeDailyRollups returns all rows for the most recent rollup days', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
@@ -2234,9 +2465,8 @@ test('cover art queries reuse a shared blob across duplicate anime art rows', ()
|
||||
const animeArt = getAnimeCoverArt(db, animeId);
|
||||
const library = getMediaLibrary(db);
|
||||
|
||||
assert.equal(artOne?.coverBlob?.length, 4);
|
||||
assert.equal(artTwo?.coverBlob?.length, 4);
|
||||
assert.deepEqual(artOne?.coverBlob, artTwo?.coverBlob);
|
||||
assert.deepEqual(artOne?.coverBlob, Buffer.from([1, 2, 3, 4]));
|
||||
assert.deepEqual(artTwo?.coverBlob, Buffer.from([9, 9, 9, 9]));
|
||||
assert.equal(animeArt?.coverBlob?.length, 4);
|
||||
assert.deepEqual(
|
||||
library.map((row) => ({
|
||||
@@ -2254,6 +2484,52 @@ test('cover art queries reuse a shared blob across duplicate anime art rows', ()
|
||||
}
|
||||
});
|
||||
|
||||
test('upsertCoverArt prefers freshly fetched bytes over a reused shared hash', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const originalVideoId = getOrCreateVideoRecord(db, 'local:/tmp/shared-cover-original.mkv', {
|
||||
canonicalTitle: 'Shared Cover Original',
|
||||
sourcePath: '/tmp/shared-cover-original.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
const refreshedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/shared-cover-refresh.mkv', {
|
||||
canonicalTitle: 'Shared Cover Refresh',
|
||||
sourcePath: '/tmp/shared-cover-refresh.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
|
||||
upsertCoverArt(db, originalVideoId, {
|
||||
anilistId: 999,
|
||||
coverUrl: 'https://images.test/shared-refresh.jpg',
|
||||
coverBlob: Buffer.from([1, 2, 3, 4]),
|
||||
titleRomaji: 'Shared Cover Refresh',
|
||||
titleEnglish: 'Shared Cover Refresh',
|
||||
episodesTotal: 12,
|
||||
});
|
||||
upsertCoverArt(db, refreshedVideoId, {
|
||||
anilistId: 999,
|
||||
coverUrl: 'https://images.test/shared-refresh.jpg',
|
||||
coverBlob: Buffer.from([9, 8, 7, 6]),
|
||||
titleRomaji: 'Shared Cover Refresh',
|
||||
titleEnglish: 'Shared Cover Refresh',
|
||||
episodesTotal: 12,
|
||||
});
|
||||
|
||||
const originalArt = getCoverArt(db, originalVideoId);
|
||||
const refreshedArt = getCoverArt(db, refreshedVideoId);
|
||||
assert.deepEqual(originalArt?.coverBlob, Buffer.from([1, 2, 3, 4]));
|
||||
assert.deepEqual(refreshedArt?.coverBlob, Buffer.from([9, 8, 7, 6]));
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('anime/media detail and episode queries use ended-session metrics when telemetry rows are absent', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
@@ -2836,13 +3112,13 @@ test('deleteSession rebuilds word and kanji aggregates from retained subtitle li
|
||||
|
||||
assert.ok(sharedWordRow);
|
||||
assert.equal(sharedWordRow.frequency, 1);
|
||||
assert.equal(sharedWordRow.first_seen, keptTs);
|
||||
assert.equal(sharedWordRow.last_seen, keptTs);
|
||||
assert.equal(sharedWordRow.first_seen, Math.floor(keptTs / 1000));
|
||||
assert.equal(sharedWordRow.last_seen, Math.floor(keptTs / 1000));
|
||||
assert.equal(deletedOnlyWordRow ?? null, null);
|
||||
assert.ok(sharedKanjiRow);
|
||||
assert.equal(sharedKanjiRow.frequency, 1);
|
||||
assert.equal(sharedKanjiRow.first_seen, keptTs);
|
||||
assert.equal(sharedKanjiRow.last_seen, keptTs);
|
||||
assert.equal(sharedKanjiRow.first_seen, Math.floor(keptTs / 1000));
|
||||
assert.equal(sharedKanjiRow.last_seen, Math.floor(keptTs / 1000));
|
||||
assert.equal(deletedOnlyKanjiRow ?? null, null);
|
||||
} finally {
|
||||
db.close();
|
||||
|
||||
@@ -156,7 +156,7 @@ export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisod
|
||||
COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount,
|
||||
MAX(s.started_at_ms) AS lastWatchedMs
|
||||
FROM imm_videos v
|
||||
JOIN imm_sessions s ON s.video_id = v.video_id
|
||||
LEFT JOIN imm_sessions s ON s.video_id = v.video_id
|
||||
LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
|
||||
WHERE v.anime_id = ?
|
||||
GROUP BY v.video_id
|
||||
|
||||
@@ -352,15 +352,16 @@ export function upsertCoverArt(
|
||||
const sharedCoverBlobHash = findSharedCoverBlobHash(db, videoId, art.anilistId, art.coverUrl);
|
||||
const fetchedAtMs = toDbMs(nowMs());
|
||||
const coverBlob = normalizeCoverBlobBytes(art.coverBlob);
|
||||
let coverBlobHash = sharedCoverBlobHash ?? null;
|
||||
if (!coverBlobHash && coverBlob && coverBlob.length > 0) {
|
||||
coverBlobHash = createHash('sha256').update(coverBlob).digest('hex');
|
||||
}
|
||||
const computedCoverBlobHash =
|
||||
coverBlob && coverBlob.length > 0
|
||||
? createHash('sha256').update(coverBlob).digest('hex')
|
||||
: null;
|
||||
let coverBlobHash = computedCoverBlobHash ?? sharedCoverBlobHash ?? null;
|
||||
if (!coverBlobHash && (!coverBlob || coverBlob.length === 0)) {
|
||||
coverBlobHash = existing?.coverBlobHash ?? null;
|
||||
}
|
||||
|
||||
if (coverBlobHash && coverBlob && coverBlob.length > 0 && !sharedCoverBlobHash) {
|
||||
if (computedCoverBlobHash && coverBlob && coverBlob.length > 0) {
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_cover_art_blobs (blob_hash, cover_blob, CREATED_DATE, LAST_UPDATE_DATE)
|
||||
@@ -368,7 +369,7 @@ export function upsertCoverArt(
|
||||
ON CONFLICT(blob_hash) DO UPDATE SET
|
||||
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
|
||||
`,
|
||||
).run(coverBlobHash, coverBlob, fetchedAtMs, fetchedAtMs);
|
||||
).run(computedCoverBlobHash, coverBlob, fetchedAtMs, fetchedAtMs);
|
||||
}
|
||||
|
||||
db.prepare(
|
||||
|
||||
@@ -204,7 +204,7 @@ export function getQueryHints(db: DatabaseSync): {
|
||||
|
||||
const now = new Date();
|
||||
const todayLocal = Math.floor(
|
||||
new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime() / 86_400_000,
|
||||
(now.getTime() / 1000 - now.getTimezoneOffset() * 60) / 86_400,
|
||||
);
|
||||
|
||||
const episodesToday =
|
||||
@@ -333,9 +333,15 @@ export function getMonthlyRollups(db: DatabaseSync, limit = 24): ImmersionSessio
|
||||
total_lines_seen AS totalLinesSeen,
|
||||
total_tokens_seen AS totalTokensSeen,
|
||||
total_cards AS totalCards,
|
||||
0 AS cardsPerHour,
|
||||
0 AS tokensPerMin,
|
||||
0 AS lookupHitRate
|
||||
CASE
|
||||
WHEN total_active_min > 0 THEN (total_cards * 60.0) / total_active_min
|
||||
ELSE NULL
|
||||
END AS cardsPerHour,
|
||||
CASE
|
||||
WHEN total_active_min > 0 THEN total_tokens_seen * 1.0 / total_active_min
|
||||
ELSE NULL
|
||||
END AS tokensPerMin,
|
||||
NULL AS lookupHitRate
|
||||
FROM imm_monthly_rollups
|
||||
WHERE rollup_month IN (SELECT rollup_month FROM recent_months)
|
||||
ORDER BY rollup_month DESC, video_id DESC
|
||||
|
||||
@@ -197,7 +197,12 @@ function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void {
|
||||
deleteStmt.run(row.wordId);
|
||||
continue;
|
||||
}
|
||||
updateStmt.run(row.frequency, row.firstSeen, row.lastSeen, row.wordId);
|
||||
updateStmt.run(
|
||||
row.frequency,
|
||||
Math.floor(row.firstSeen / 1000),
|
||||
Math.floor(row.lastSeen / 1000),
|
||||
row.wordId,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,7 +246,12 @@ function refreshKanjiAggregates(db: DatabaseSync, kanjiIds: number[]): void {
|
||||
deleteStmt.run(row.kanjiId);
|
||||
continue;
|
||||
}
|
||||
updateStmt.run(row.frequency, row.firstSeen, row.lastSeen, row.kanjiId);
|
||||
updateStmt.run(
|
||||
row.frequency,
|
||||
Math.floor(row.firstSeen / 1000),
|
||||
Math.floor(row.lastSeen / 1000),
|
||||
row.kanjiId,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -112,6 +112,16 @@ function makeTrendLabel(value: number): string {
|
||||
});
|
||||
}
|
||||
|
||||
function getLocalEpochDay(timestampMs: number): number {
|
||||
const date = new Date(timestampMs);
|
||||
return Math.floor((timestampMs - date.getTimezoneOffset() * 60_000) / 86_400_000);
|
||||
}
|
||||
|
||||
function getLocalDateForEpochDay(epochDay: number): Date {
|
||||
const utcDate = new Date(epochDay * 86_400_000);
|
||||
return new Date(utcDate.getTime() + utcDate.getTimezoneOffset() * 60_000);
|
||||
}
|
||||
|
||||
function getTrendSessionWordCount(session: Pick<TrendSessionMetricRow, 'tokensSeen'>): number {
|
||||
return session.tokensSeen;
|
||||
}
|
||||
@@ -188,7 +198,7 @@ function buildWatchTimeByHour(sessions: TrendSessionMetricRow[]): TrendChartPoin
|
||||
}
|
||||
|
||||
function dayLabel(epochDay: number): string {
|
||||
return new Date(epochDay * 86_400_000).toLocaleDateString(undefined, {
|
||||
return getLocalDateForEpochDay(epochDay).toLocaleDateString(undefined, {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
});
|
||||
@@ -200,7 +210,7 @@ function buildSessionSeriesByDay(
|
||||
): TrendChartPoint[] {
|
||||
const byDay = new Map<number, number>();
|
||||
for (const session of sessions) {
|
||||
const epochDay = Math.floor(session.startedAtMs / 86_400_000);
|
||||
const epochDay = getLocalEpochDay(session.startedAtMs);
|
||||
byDay.set(epochDay, (byDay.get(epochDay) ?? 0) + getValue(session));
|
||||
}
|
||||
return Array.from(byDay.entries())
|
||||
@@ -213,7 +223,7 @@ function buildLookupsPerHundredWords(sessions: TrendSessionMetricRow[]): TrendCh
|
||||
const wordsByDay = new Map<number, number>();
|
||||
|
||||
for (const session of sessions) {
|
||||
const epochDay = Math.floor(session.startedAtMs / 86_400_000);
|
||||
const epochDay = getLocalEpochDay(session.startedAtMs);
|
||||
lookupsByDay.set(epochDay, (lookupsByDay.get(epochDay) ?? 0) + session.yomitanLookupCount);
|
||||
wordsByDay.set(epochDay, (wordsByDay.get(epochDay) ?? 0) + getTrendSessionWordCount(session));
|
||||
}
|
||||
@@ -237,7 +247,7 @@ function buildPerAnimeFromSessions(
|
||||
|
||||
for (const session of sessions) {
|
||||
const animeTitle = resolveTrendAnimeTitle(session);
|
||||
const epochDay = Math.floor(session.startedAtMs / 86_400_000);
|
||||
const epochDay = getLocalEpochDay(session.startedAtMs);
|
||||
const dayMap = byAnime.get(animeTitle) ?? new Map();
|
||||
dayMap.set(epochDay, (dayMap.get(epochDay) ?? 0) + getValue(session));
|
||||
byAnime.set(animeTitle, dayMap);
|
||||
@@ -258,7 +268,7 @@ function buildLookupsPerHundredPerAnime(sessions: TrendSessionMetricRow[]): Tren
|
||||
|
||||
for (const session of sessions) {
|
||||
const animeTitle = resolveTrendAnimeTitle(session);
|
||||
const epochDay = Math.floor(session.startedAtMs / 86_400_000);
|
||||
const epochDay = getLocalEpochDay(session.startedAtMs);
|
||||
|
||||
const lookupMap = lookups.get(animeTitle) ?? new Map();
|
||||
lookupMap.set(epochDay, (lookupMap.get(epochDay) ?? 0) + session.yomitanLookupCount);
|
||||
@@ -462,7 +472,7 @@ function buildNewWordsPerDay(db: DatabaseSync, cutoffMs: number | null): TrendCh
|
||||
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
||||
const prepared = db.prepare(`
|
||||
SELECT
|
||||
CAST(first_seen / 86400 AS INTEGER) AS epochDay,
|
||||
CAST(julianday(first_seen, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) AS epochDay,
|
||||
COUNT(*) AS wordCount
|
||||
FROM imm_words
|
||||
WHERE first_seen IS NOT NULL
|
||||
|
||||
@@ -1078,6 +1078,56 @@ test('executeQueuedWrite inserts event and telemetry rows', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('executeQueuedWrite rejects partial telemetry writes instead of zero-filling', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const stmts = createTrackerPreparedStatements(db);
|
||||
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/partial-telemetry.mkv', {
|
||||
canonicalTitle: 'Partial Telemetry',
|
||||
sourcePath: '/tmp/partial-telemetry.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
const { sessionId } = startSessionRecord(db, videoId, 5_000);
|
||||
|
||||
assert.throws(
|
||||
() =>
|
||||
executeQueuedWrite(
|
||||
{
|
||||
kind: 'telemetry',
|
||||
sessionId,
|
||||
sampleMs: 6_000,
|
||||
totalWatchedMs: 1_000,
|
||||
activeWatchedMs: 900,
|
||||
linesSeen: 3,
|
||||
cardsMined: 1,
|
||||
lookupCount: 2,
|
||||
lookupHits: 1,
|
||||
yomitanLookupCount: 0,
|
||||
pauseCount: 1,
|
||||
pauseMs: 50,
|
||||
seekForwardCount: 0,
|
||||
seekBackwardCount: 0,
|
||||
mediaBufferEvents: 0,
|
||||
},
|
||||
stmts,
|
||||
),
|
||||
/Incomplete telemetry write/,
|
||||
);
|
||||
|
||||
const telemetryCount = db
|
||||
.prepare('SELECT COUNT(*) AS total FROM imm_session_telemetry WHERE session_id = ?')
|
||||
.get(sessionId) as { total: number };
|
||||
assert.equal(telemetryCount.total, 0);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('executeQueuedWrite inserts and upserts word and kanji rows', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
@@ -1406,27 +1406,46 @@ function incrementKanjiAggregate(
|
||||
export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedStatements): void {
|
||||
const currentMs = toDbMs(nowMs());
|
||||
if (write.kind === 'telemetry') {
|
||||
if (
|
||||
write.totalWatchedMs === undefined ||
|
||||
write.activeWatchedMs === undefined ||
|
||||
write.linesSeen === undefined ||
|
||||
write.tokensSeen === undefined ||
|
||||
write.cardsMined === undefined ||
|
||||
write.lookupCount === undefined ||
|
||||
write.lookupHits === undefined ||
|
||||
write.yomitanLookupCount === undefined ||
|
||||
write.pauseCount === undefined ||
|
||||
write.pauseMs === undefined ||
|
||||
write.seekForwardCount === undefined ||
|
||||
write.seekBackwardCount === undefined ||
|
||||
write.mediaBufferEvents === undefined
|
||||
) {
|
||||
throw new Error('Incomplete telemetry write');
|
||||
}
|
||||
const telemetrySampleMs = toDbMs(write.sampleMs ?? Number(currentMs));
|
||||
stmts.telemetryInsertStmt.run(
|
||||
write.sessionId,
|
||||
telemetrySampleMs,
|
||||
write.totalWatchedMs ?? 0,
|
||||
write.activeWatchedMs ?? 0,
|
||||
write.linesSeen ?? 0,
|
||||
write.tokensSeen ?? 0,
|
||||
write.cardsMined ?? 0,
|
||||
write.lookupCount ?? 0,
|
||||
write.lookupHits ?? 0,
|
||||
write.yomitanLookupCount ?? 0,
|
||||
write.pauseCount ?? 0,
|
||||
write.pauseMs ?? 0,
|
||||
write.seekForwardCount ?? 0,
|
||||
write.seekBackwardCount ?? 0,
|
||||
write.mediaBufferEvents ?? 0,
|
||||
write.totalWatchedMs,
|
||||
write.activeWatchedMs,
|
||||
write.linesSeen,
|
||||
write.tokensSeen,
|
||||
write.cardsMined,
|
||||
write.lookupCount,
|
||||
write.lookupHits,
|
||||
write.yomitanLookupCount,
|
||||
write.pauseCount,
|
||||
write.pauseMs,
|
||||
write.seekForwardCount,
|
||||
write.seekBackwardCount,
|
||||
write.mediaBufferEvents,
|
||||
currentMs,
|
||||
currentMs,
|
||||
);
|
||||
stmts.sessionCheckpointStmt.run(write.lastMediaMs ?? null, currentMs, write.sessionId);
|
||||
if (write.lastMediaMs !== undefined) {
|
||||
stmts.sessionCheckpointStmt.run(write.lastMediaMs ?? null, currentMs, write.sessionId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (write.kind === 'word') {
|
||||
|
||||
7
src/core/services/immersion-tracker/time.test.ts
Normal file
7
src/core/services/immersion-tracker/time.test.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { nowMs } from './time.js';
|
||||
|
||||
test('nowMs returns wall-clock epoch milliseconds', () => {
|
||||
assert.ok(nowMs() > 1_600_000_000_000);
|
||||
});
|
||||
@@ -1,10 +1,8 @@
|
||||
const SQLITE_SAFE_EPOCH_BASE_MS = 2_000_000_000;
|
||||
|
||||
export function nowMs(): number {
|
||||
const perf = globalThis.performance;
|
||||
if (perf) {
|
||||
return SQLITE_SAFE_EPOCH_BASE_MS + Math.floor(perf.now());
|
||||
if (perf && Number.isFinite(perf.timeOrigin)) {
|
||||
return Math.floor(perf.timeOrigin + perf.now());
|
||||
}
|
||||
|
||||
return SQLITE_SAFE_EPOCH_BASE_MS;
|
||||
return Date.now();
|
||||
}
|
||||
|
||||
98
src/main/character-dictionary-runtime/zip.test.ts
Normal file
98
src/main/character-dictionary-runtime/zip.test.ts
Normal file
@@ -0,0 +1,98 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { buildDictionaryZip } from './zip';
|
||||
import type { CharacterDictionaryTermEntry } from './types';
|
||||
|
||||
function makeTempDir(): string {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-character-zip-'));
|
||||
}
|
||||
|
||||
function cleanupDir(dirPath: string): void {
|
||||
fs.rmSync(dirPath, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function readStoredZipEntries(zipPath: string): Map<string, Buffer> {
|
||||
const archive = fs.readFileSync(zipPath);
|
||||
const entries = new Map<string, Buffer>();
|
||||
let cursor = 0;
|
||||
|
||||
while (cursor + 4 <= archive.length) {
|
||||
const signature = archive.readUInt32LE(cursor);
|
||||
if (signature === 0x02014b50 || signature === 0x06054b50) {
|
||||
break;
|
||||
}
|
||||
assert.equal(signature, 0x04034b50, `unexpected local file header at offset ${cursor}`);
|
||||
|
||||
const compressedSize = archive.readUInt32LE(cursor + 18);
|
||||
const fileNameLength = archive.readUInt16LE(cursor + 26);
|
||||
const extraLength = archive.readUInt16LE(cursor + 28);
|
||||
const fileNameStart = cursor + 30;
|
||||
const dataStart = fileNameStart + fileNameLength + extraLength;
|
||||
const fileName = archive.subarray(fileNameStart, fileNameStart + fileNameLength).toString(
|
||||
'utf8',
|
||||
);
|
||||
const data = archive.subarray(dataStart, dataStart + compressedSize);
|
||||
entries.set(fileName, Buffer.from(data));
|
||||
cursor = dataStart + compressedSize;
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', () => {
|
||||
const tempDir = makeTempDir();
|
||||
const outputPath = path.join(tempDir, 'dictionary.zip');
|
||||
const termEntries: CharacterDictionaryTermEntry[] = [
|
||||
['アルファ', 'あるふぁ', '', '', 0, ['Alpha entry'], 0, 'name'],
|
||||
];
|
||||
const originalBufferConcat = Buffer.concat;
|
||||
|
||||
try {
|
||||
Buffer.concat = ((...args: Parameters<typeof Buffer.concat>) => {
|
||||
throw new Error(`buildDictionaryZip should not Buffer.concat the full archive (${args[0].length} chunks)`);
|
||||
}) as typeof Buffer.concat;
|
||||
|
||||
const result = buildDictionaryZip(
|
||||
outputPath,
|
||||
'Dictionary Title',
|
||||
'Dictionary Description',
|
||||
'2026-03-27',
|
||||
termEntries,
|
||||
[{ path: 'images/alpha.bin', dataBase64: Buffer.from([1, 2, 3]).toString('base64') }],
|
||||
);
|
||||
|
||||
assert.equal(result.zipPath, outputPath);
|
||||
assert.equal(result.entryCount, 1);
|
||||
|
||||
const entries = readStoredZipEntries(outputPath);
|
||||
assert.deepEqual([...entries.keys()].sort(), [
|
||||
'images/alpha.bin',
|
||||
'index.json',
|
||||
'tag_bank_1.json',
|
||||
'term_bank_1.json',
|
||||
]);
|
||||
|
||||
const indexJson = JSON.parse(entries.get('index.json')!.toString('utf8')) as {
|
||||
title: string;
|
||||
description: string;
|
||||
revision: string;
|
||||
format: number;
|
||||
};
|
||||
assert.equal(indexJson.title, 'Dictionary Title');
|
||||
assert.equal(indexJson.description, 'Dictionary Description');
|
||||
assert.equal(indexJson.revision, '2026-03-27');
|
||||
assert.equal(indexJson.format, 3);
|
||||
|
||||
const termBank = JSON.parse(entries.get('term_bank_1.json')!.toString('utf8')) as
|
||||
CharacterDictionaryTermEntry[];
|
||||
assert.equal(termBank.length, 1);
|
||||
assert.equal(termBank[0]?.[0], 'アルファ');
|
||||
assert.deepEqual(entries.get('images/alpha.bin'), Buffer.from([1, 2, 3]));
|
||||
} finally {
|
||||
Buffer.concat = originalBufferConcat;
|
||||
cleanupDir(tempDir);
|
||||
}
|
||||
});
|
||||
@@ -5,8 +5,8 @@ import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } f
|
||||
|
||||
type ZipEntry = {
|
||||
name: string;
|
||||
data: Buffer;
|
||||
crc32: number;
|
||||
size: number;
|
||||
localHeaderOffset: number;
|
||||
};
|
||||
|
||||
@@ -67,97 +67,78 @@ function crc32(data: Buffer): number {
|
||||
return (crc ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
|
||||
const chunks: Buffer[] = [];
|
||||
const entries: ZipEntry[] = [];
|
||||
let offset = 0;
|
||||
function createLocalFileHeader(fileName: Buffer, fileCrc32: number, fileSize: number): Buffer {
|
||||
const local = Buffer.alloc(30 + fileName.length);
|
||||
let cursor = 0;
|
||||
writeUint32LE(local, 0x04034b50, cursor);
|
||||
cursor += 4;
|
||||
local.writeUInt16LE(20, cursor);
|
||||
cursor += 2;
|
||||
local.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
local.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
local.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
local.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
writeUint32LE(local, fileCrc32, cursor);
|
||||
cursor += 4;
|
||||
writeUint32LE(local, fileSize, cursor);
|
||||
cursor += 4;
|
||||
writeUint32LE(local, fileSize, cursor);
|
||||
cursor += 4;
|
||||
local.writeUInt16LE(fileName.length, cursor);
|
||||
cursor += 2;
|
||||
local.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
fileName.copy(local, cursor);
|
||||
return local;
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
const fileName = Buffer.from(file.name, 'utf8');
|
||||
const fileData = file.data;
|
||||
const fileCrc32 = crc32(fileData);
|
||||
const local = Buffer.alloc(30 + fileName.length);
|
||||
let cursor = 0;
|
||||
writeUint32LE(local, 0x04034b50, cursor);
|
||||
cursor += 4;
|
||||
local.writeUInt16LE(20, cursor);
|
||||
cursor += 2;
|
||||
local.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
local.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
local.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
local.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
writeUint32LE(local, fileCrc32, cursor);
|
||||
cursor += 4;
|
||||
writeUint32LE(local, fileData.length, cursor);
|
||||
cursor += 4;
|
||||
writeUint32LE(local, fileData.length, cursor);
|
||||
cursor += 4;
|
||||
local.writeUInt16LE(fileName.length, cursor);
|
||||
cursor += 2;
|
||||
local.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
fileName.copy(local, cursor);
|
||||
function createCentralDirectoryHeader(entry: ZipEntry): Buffer {
|
||||
const fileName = Buffer.from(entry.name, 'utf8');
|
||||
const central = Buffer.alloc(46 + fileName.length);
|
||||
let cursor = 0;
|
||||
writeUint32LE(central, 0x02014b50, cursor);
|
||||
cursor += 4;
|
||||
central.writeUInt16LE(20, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(20, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
writeUint32LE(central, entry.crc32, cursor);
|
||||
cursor += 4;
|
||||
writeUint32LE(central, entry.size, cursor);
|
||||
cursor += 4;
|
||||
writeUint32LE(central, entry.size, cursor);
|
||||
cursor += 4;
|
||||
central.writeUInt16LE(fileName.length, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
writeUint32LE(central, 0, cursor);
|
||||
cursor += 4;
|
||||
writeUint32LE(central, entry.localHeaderOffset, cursor);
|
||||
cursor += 4;
|
||||
fileName.copy(central, cursor);
|
||||
return central;
|
||||
}
|
||||
|
||||
chunks.push(local, fileData);
|
||||
entries.push({
|
||||
name: file.name,
|
||||
data: fileData,
|
||||
crc32: fileCrc32,
|
||||
localHeaderOffset: offset,
|
||||
});
|
||||
offset += local.length + fileData.length;
|
||||
}
|
||||
|
||||
const centralStart = offset;
|
||||
const centralChunks: Buffer[] = [];
|
||||
for (const entry of entries) {
|
||||
const fileName = Buffer.from(entry.name, 'utf8');
|
||||
const central = Buffer.alloc(46 + fileName.length);
|
||||
let cursor = 0;
|
||||
writeUint32LE(central, 0x02014b50, cursor);
|
||||
cursor += 4;
|
||||
central.writeUInt16LE(20, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(20, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
writeUint32LE(central, entry.crc32, cursor);
|
||||
cursor += 4;
|
||||
writeUint32LE(central, entry.data.length, cursor);
|
||||
cursor += 4;
|
||||
writeUint32LE(central, entry.data.length, cursor);
|
||||
cursor += 4;
|
||||
central.writeUInt16LE(fileName.length, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
central.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
writeUint32LE(central, 0, cursor);
|
||||
cursor += 4;
|
||||
writeUint32LE(central, entry.localHeaderOffset, cursor);
|
||||
cursor += 4;
|
||||
fileName.copy(central, cursor);
|
||||
centralChunks.push(central);
|
||||
offset += central.length;
|
||||
}
|
||||
|
||||
const centralSize = offset - centralStart;
|
||||
function createEndOfCentralDirectory(entriesLength: number, centralSize: number, centralStart: number): Buffer {
|
||||
const end = Buffer.alloc(22);
|
||||
let cursor = 0;
|
||||
writeUint32LE(end, 0x06054b50, cursor);
|
||||
@@ -166,17 +147,63 @@ function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
|
||||
cursor += 2;
|
||||
end.writeUInt16LE(0, cursor);
|
||||
cursor += 2;
|
||||
end.writeUInt16LE(entries.length, cursor);
|
||||
end.writeUInt16LE(entriesLength, cursor);
|
||||
cursor += 2;
|
||||
end.writeUInt16LE(entries.length, cursor);
|
||||
end.writeUInt16LE(entriesLength, cursor);
|
||||
cursor += 2;
|
||||
writeUint32LE(end, centralSize, cursor);
|
||||
cursor += 4;
|
||||
writeUint32LE(end, centralStart, cursor);
|
||||
cursor += 4;
|
||||
end.writeUInt16LE(0, cursor);
|
||||
return end;
|
||||
}
|
||||
|
||||
return Buffer.concat([...chunks, ...centralChunks, end]);
|
||||
function writeBuffer(fd: number, buffer: Buffer): void {
|
||||
let written = 0;
|
||||
while (written < buffer.length) {
|
||||
written += fs.writeSync(fd, buffer, written, buffer.length - written);
|
||||
}
|
||||
}
|
||||
|
||||
function writeStoredZip(outputPath: string, files: Iterable<{ name: string; data: Buffer }>): void {
|
||||
const entries: ZipEntry[] = [];
|
||||
let offset = 0;
|
||||
const fd = fs.openSync(outputPath, 'w');
|
||||
|
||||
try {
|
||||
for (const file of files) {
|
||||
const fileName = Buffer.from(file.name, 'utf8');
|
||||
const fileSize = file.data.length;
|
||||
const fileCrc32 = crc32(file.data);
|
||||
const localHeader = createLocalFileHeader(fileName, fileCrc32, fileSize);
|
||||
writeBuffer(fd, localHeader);
|
||||
writeBuffer(fd, file.data);
|
||||
entries.push({
|
||||
name: file.name,
|
||||
crc32: fileCrc32,
|
||||
size: fileSize,
|
||||
localHeaderOffset: offset,
|
||||
});
|
||||
offset += localHeader.length + fileSize;
|
||||
}
|
||||
|
||||
const centralStart = offset;
|
||||
for (const entry of entries) {
|
||||
const centralHeader = createCentralDirectoryHeader(entry);
|
||||
writeBuffer(fd, centralHeader);
|
||||
offset += centralHeader.length;
|
||||
}
|
||||
|
||||
const centralSize = offset - centralStart;
|
||||
writeBuffer(fd, createEndOfCentralDirectory(entries.length, centralSize, centralStart));
|
||||
} catch (error) {
|
||||
fs.closeSync(fd);
|
||||
fs.rmSync(outputPath, { force: true });
|
||||
throw error;
|
||||
}
|
||||
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
|
||||
export function buildDictionaryZip(
|
||||
@@ -187,36 +214,37 @@ export function buildDictionaryZip(
|
||||
termEntries: CharacterDictionaryTermEntry[],
|
||||
images: CharacterDictionarySnapshotImage[],
|
||||
): { zipPath: string; entryCount: number } {
|
||||
const zipFiles: Array<{ name: string; data: Buffer }> = [
|
||||
{
|
||||
ensureDir(path.dirname(outputPath));
|
||||
|
||||
function* zipFiles(): Iterable<{ name: string; data: Buffer }> {
|
||||
yield {
|
||||
name: 'index.json',
|
||||
data: Buffer.from(
|
||||
JSON.stringify(createIndex(dictionaryTitle, description, revision), null, 2),
|
||||
'utf8',
|
||||
),
|
||||
},
|
||||
{
|
||||
};
|
||||
yield {
|
||||
name: 'tag_bank_1.json',
|
||||
data: Buffer.from(JSON.stringify(createTagBank()), 'utf8'),
|
||||
},
|
||||
];
|
||||
};
|
||||
|
||||
for (const image of images) {
|
||||
zipFiles.push({
|
||||
name: image.path,
|
||||
data: Buffer.from(image.dataBase64, 'base64'),
|
||||
});
|
||||
for (const image of images) {
|
||||
yield {
|
||||
name: image.path,
|
||||
data: Buffer.from(image.dataBase64, 'base64'),
|
||||
};
|
||||
}
|
||||
|
||||
const entriesPerBank = 10_000;
|
||||
for (let i = 0; i < termEntries.length; i += entriesPerBank) {
|
||||
yield {
|
||||
name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`,
|
||||
data: Buffer.from(JSON.stringify(termEntries.slice(i, i + entriesPerBank)), 'utf8'),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const entriesPerBank = 10_000;
|
||||
for (let i = 0; i < termEntries.length; i += entriesPerBank) {
|
||||
zipFiles.push({
|
||||
name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`,
|
||||
data: Buffer.from(JSON.stringify(termEntries.slice(i, i + entriesPerBank)), 'utf8'),
|
||||
});
|
||||
}
|
||||
|
||||
ensureDir(path.dirname(outputPath));
|
||||
fs.writeFileSync(outputPath, createStoredZip(zipFiles));
|
||||
writeStoredZip(outputPath, zipFiles());
|
||||
return { zipPath: outputPath, entryCount: termEntries.length };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user