mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-03-27 06:12:05 -07:00
fix: address CodeRabbit review feedback
This commit is contained in:
@@ -72,6 +72,14 @@ add_blocker() {
|
|||||||
BLOCKED=1
|
BLOCKED=1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
validate_artifact_dir() {
|
||||||
|
local candidate=$1
|
||||||
|
if [[ ! "$candidate" =~ ^[A-Za-z0-9._/@:+-]+$ ]]; then
|
||||||
|
echo "Invalid characters in --artifact-dir path" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
append_step_record() {
|
append_step_record() {
|
||||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||||
"$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" >>"$STEPS_TSV"
|
"$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" >>"$STEPS_TSV"
|
||||||
@@ -411,6 +419,7 @@ if [[ -z "${ARTIFACT_DIR:-}" ]]; then
|
|||||||
SESSION_ID=$(generate_session_id)
|
SESSION_ID=$(generate_session_id)
|
||||||
ARTIFACT_DIR="$REPO_ROOT/.tmp/skill-verification/$SESSION_ID"
|
ARTIFACT_DIR="$REPO_ROOT/.tmp/skill-verification/$SESSION_ID"
|
||||||
else
|
else
|
||||||
|
validate_artifact_dir "$ARTIFACT_DIR"
|
||||||
SESSION_ID=$(basename "$ARTIFACT_DIR")
|
SESSION_ID=$(basename "$ARTIFACT_DIR")
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
@@ -1982,6 +1982,7 @@ test('flushSingle reuses cached prepared statements', async () => {
|
|||||||
cardsMined?: number;
|
cardsMined?: number;
|
||||||
lookupCount?: number;
|
lookupCount?: number;
|
||||||
lookupHits?: number;
|
lookupHits?: number;
|
||||||
|
yomitanLookupCount?: number;
|
||||||
pauseCount?: number;
|
pauseCount?: number;
|
||||||
pauseMs?: number;
|
pauseMs?: number;
|
||||||
seekForwardCount?: number;
|
seekForwardCount?: number;
|
||||||
@@ -2051,6 +2052,7 @@ test('flushSingle reuses cached prepared statements', async () => {
|
|||||||
cardsMined: 0,
|
cardsMined: 0,
|
||||||
lookupCount: 0,
|
lookupCount: 0,
|
||||||
lookupHits: 0,
|
lookupHits: 0,
|
||||||
|
yomitanLookupCount: 0,
|
||||||
pauseCount: 0,
|
pauseCount: 0,
|
||||||
pauseMs: 0,
|
pauseMs: 0,
|
||||||
seekForwardCount: 0,
|
seekForwardCount: 0,
|
||||||
|
|||||||
@@ -208,6 +208,104 @@ test('getAnimeEpisodes prefers the latest session media position when the latest
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('getAnimeEpisodes includes unwatched episodes for the anime', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const watchedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/watched-episode.mkv', {
|
||||||
|
canonicalTitle: 'Watched Episode',
|
||||||
|
sourcePath: '/tmp/watched-episode.mkv',
|
||||||
|
sourceUrl: null,
|
||||||
|
sourceType: SOURCE_TYPE_LOCAL,
|
||||||
|
});
|
||||||
|
const unwatchedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/unwatched-episode.mkv', {
|
||||||
|
canonicalTitle: 'Unwatched Episode',
|
||||||
|
sourcePath: '/tmp/unwatched-episode.mkv',
|
||||||
|
sourceUrl: null,
|
||||||
|
sourceType: SOURCE_TYPE_LOCAL,
|
||||||
|
});
|
||||||
|
const animeId = getOrCreateAnimeRecord(db, {
|
||||||
|
parsedTitle: 'Episode Coverage Anime',
|
||||||
|
canonicalTitle: 'Episode Coverage Anime',
|
||||||
|
anilistId: null,
|
||||||
|
titleRomaji: null,
|
||||||
|
titleEnglish: null,
|
||||||
|
titleNative: null,
|
||||||
|
metadataJson: null,
|
||||||
|
});
|
||||||
|
linkVideoToAnimeRecord(db, watchedVideoId, {
|
||||||
|
animeId,
|
||||||
|
parsedBasename: 'watched-episode.mkv',
|
||||||
|
parsedTitle: 'Episode Coverage Anime',
|
||||||
|
parsedSeason: 1,
|
||||||
|
parsedEpisode: 1,
|
||||||
|
parserSource: 'fallback',
|
||||||
|
parserConfidence: 1,
|
||||||
|
parseMetadataJson: '{"episode":1}',
|
||||||
|
});
|
||||||
|
linkVideoToAnimeRecord(db, unwatchedVideoId, {
|
||||||
|
animeId,
|
||||||
|
parsedBasename: 'unwatched-episode.mkv',
|
||||||
|
parsedTitle: 'Episode Coverage Anime',
|
||||||
|
parsedSeason: 1,
|
||||||
|
parsedEpisode: 2,
|
||||||
|
parserSource: 'fallback',
|
||||||
|
parserConfidence: 1,
|
||||||
|
parseMetadataJson: '{"episode":2}',
|
||||||
|
});
|
||||||
|
|
||||||
|
const watchedSessionId = startSessionRecord(db, watchedVideoId, 1_000_000).sessionId;
|
||||||
|
db.prepare(
|
||||||
|
`
|
||||||
|
UPDATE imm_sessions
|
||||||
|
SET
|
||||||
|
ended_at_ms = ?,
|
||||||
|
status = 2,
|
||||||
|
ended_media_ms = ?,
|
||||||
|
active_watched_ms = ?,
|
||||||
|
cards_mined = ?,
|
||||||
|
tokens_seen = ?,
|
||||||
|
yomitan_lookup_count = ?,
|
||||||
|
LAST_UPDATE_DATE = ?
|
||||||
|
WHERE session_id = ?
|
||||||
|
`,
|
||||||
|
).run(1_005_000, 7_000, 3_000, 2, 20, 4, 1_005_000, watchedSessionId);
|
||||||
|
|
||||||
|
const episodes = getAnimeEpisodes(db, animeId);
|
||||||
|
assert.equal(episodes.length, 2);
|
||||||
|
assert.deepEqual(
|
||||||
|
episodes.map((episode) => ({
|
||||||
|
videoId: episode.videoId,
|
||||||
|
totalSessions: episode.totalSessions,
|
||||||
|
totalActiveMs: episode.totalActiveMs,
|
||||||
|
totalCards: episode.totalCards,
|
||||||
|
totalTokensSeen: episode.totalTokensSeen,
|
||||||
|
})),
|
||||||
|
[
|
||||||
|
{
|
||||||
|
videoId: watchedVideoId,
|
||||||
|
totalSessions: 1,
|
||||||
|
totalActiveMs: 3_000,
|
||||||
|
totalCards: 2,
|
||||||
|
totalTokensSeen: 20,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
videoId: unwatchedVideoId,
|
||||||
|
totalSessions: 0,
|
||||||
|
totalActiveMs: 0,
|
||||||
|
totalCards: 0,
|
||||||
|
totalTokensSeen: 0,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('getAnimeEpisodes falls back to the latest subtitle segment end when session progress checkpoints are missing', () => {
|
test('getAnimeEpisodes falls back to the latest subtitle segment end when session progress checkpoints are missing', () => {
|
||||||
const dbPath = makeDbPath();
|
const dbPath = makeDbPath();
|
||||||
const db = new Database(dbPath);
|
const db = new Database(dbPath);
|
||||||
@@ -586,6 +684,109 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('getTrendsDashboard keeps local-midnight session buckets separate', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const stmts = createTrackerPreparedStatements(db);
|
||||||
|
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/local-midnight-trends.mkv', {
|
||||||
|
canonicalTitle: 'Local Midnight Trends',
|
||||||
|
sourcePath: '/tmp/local-midnight-trends.mkv',
|
||||||
|
sourceUrl: null,
|
||||||
|
sourceType: SOURCE_TYPE_LOCAL,
|
||||||
|
});
|
||||||
|
const animeId = getOrCreateAnimeRecord(db, {
|
||||||
|
parsedTitle: 'Local Midnight Trends',
|
||||||
|
canonicalTitle: 'Local Midnight Trends',
|
||||||
|
anilistId: null,
|
||||||
|
titleRomaji: null,
|
||||||
|
titleEnglish: null,
|
||||||
|
titleNative: null,
|
||||||
|
metadataJson: null,
|
||||||
|
});
|
||||||
|
linkVideoToAnimeRecord(db, videoId, {
|
||||||
|
animeId,
|
||||||
|
parsedBasename: 'local-midnight-trends.mkv',
|
||||||
|
parsedTitle: 'Local Midnight Trends',
|
||||||
|
parsedSeason: 1,
|
||||||
|
parsedEpisode: 1,
|
||||||
|
parserSource: 'test',
|
||||||
|
parserConfidence: 1,
|
||||||
|
parseMetadataJson: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
const beforeMidnight = new Date(2026, 2, 1, 23, 30).getTime();
|
||||||
|
const afterMidnight = new Date(2026, 2, 2, 0, 30).getTime();
|
||||||
|
const firstSessionId = startSessionRecord(db, videoId, beforeMidnight).sessionId;
|
||||||
|
const secondSessionId = startSessionRecord(db, videoId, afterMidnight).sessionId;
|
||||||
|
|
||||||
|
for (const [sessionId, startedAtMs, tokensSeen, lookupCount] of [
|
||||||
|
[firstSessionId, beforeMidnight, 100, 4],
|
||||||
|
[secondSessionId, afterMidnight, 120, 6],
|
||||||
|
] as const) {
|
||||||
|
stmts.telemetryInsertStmt.run(
|
||||||
|
sessionId,
|
||||||
|
startedAtMs + 60_000,
|
||||||
|
60_000,
|
||||||
|
60_000,
|
||||||
|
1,
|
||||||
|
tokensSeen,
|
||||||
|
0,
|
||||||
|
lookupCount,
|
||||||
|
lookupCount,
|
||||||
|
lookupCount,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
startedAtMs + 60_000,
|
||||||
|
startedAtMs + 60_000,
|
||||||
|
);
|
||||||
|
db.prepare(
|
||||||
|
`
|
||||||
|
UPDATE imm_sessions
|
||||||
|
SET
|
||||||
|
ended_at_ms = ?,
|
||||||
|
status = 2,
|
||||||
|
total_watched_ms = ?,
|
||||||
|
active_watched_ms = ?,
|
||||||
|
lines_seen = ?,
|
||||||
|
tokens_seen = ?,
|
||||||
|
lookup_count = ?,
|
||||||
|
lookup_hits = ?,
|
||||||
|
yomitan_lookup_count = ?,
|
||||||
|
LAST_UPDATE_DATE = ?
|
||||||
|
WHERE session_id = ?
|
||||||
|
`,
|
||||||
|
).run(
|
||||||
|
startedAtMs + 60_000,
|
||||||
|
60_000,
|
||||||
|
60_000,
|
||||||
|
1,
|
||||||
|
tokensSeen,
|
||||||
|
lookupCount,
|
||||||
|
lookupCount,
|
||||||
|
lookupCount,
|
||||||
|
startedAtMs + 60_000,
|
||||||
|
sessionId,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const dashboard = getTrendsDashboard(db, 'all', 'day');
|
||||||
|
assert.equal(dashboard.progress.lookups.length, 2);
|
||||||
|
assert.deepEqual(
|
||||||
|
dashboard.progress.lookups.map((point) => point.value),
|
||||||
|
[4, 10],
|
||||||
|
);
|
||||||
|
assert.equal(dashboard.ratios.lookupsPerHundred.length, 2);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('getQueryHints reads all-time totals from lifetime summary', () => {
|
test('getQueryHints reads all-time totals from lifetime summary', () => {
|
||||||
const dbPath = makeDbPath();
|
const dbPath = makeDbPath();
|
||||||
const db = new Database(dbPath);
|
const db = new Database(dbPath);
|
||||||
@@ -1024,6 +1225,36 @@ test('getMonthlyRollups returns all rows for the most recent rollup months', ()
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('getMonthlyRollups derives rate metrics from stored monthly totals', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const insertRollup = db.prepare(
|
||||||
|
`
|
||||||
|
INSERT INTO imm_monthly_rollups (
|
||||||
|
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||||
|
total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
`,
|
||||||
|
);
|
||||||
|
const nowMs = Date.now();
|
||||||
|
insertRollup.run(202602, 1, 2, 30, 20, 90, 15, nowMs, nowMs);
|
||||||
|
insertRollup.run(202602, 2, 1, 0, 10, 25, 5, nowMs, nowMs);
|
||||||
|
|
||||||
|
const rows = getMonthlyRollups(db, 1);
|
||||||
|
assert.equal(rows.length, 2);
|
||||||
|
assert.equal(rows[1]?.cardsPerHour, 30);
|
||||||
|
assert.equal(rows[1]?.tokensPerMin, 3);
|
||||||
|
assert.equal(rows[1]?.lookupHitRate ?? null, null);
|
||||||
|
assert.equal(rows[0]?.cardsPerHour ?? null, null);
|
||||||
|
assert.equal(rows[0]?.tokensPerMin ?? null, null);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('getAnimeDailyRollups returns all rows for the most recent rollup days', () => {
|
test('getAnimeDailyRollups returns all rows for the most recent rollup days', () => {
|
||||||
const dbPath = makeDbPath();
|
const dbPath = makeDbPath();
|
||||||
const db = new Database(dbPath);
|
const db = new Database(dbPath);
|
||||||
@@ -2234,9 +2465,8 @@ test('cover art queries reuse a shared blob across duplicate anime art rows', ()
|
|||||||
const animeArt = getAnimeCoverArt(db, animeId);
|
const animeArt = getAnimeCoverArt(db, animeId);
|
||||||
const library = getMediaLibrary(db);
|
const library = getMediaLibrary(db);
|
||||||
|
|
||||||
assert.equal(artOne?.coverBlob?.length, 4);
|
assert.deepEqual(artOne?.coverBlob, Buffer.from([1, 2, 3, 4]));
|
||||||
assert.equal(artTwo?.coverBlob?.length, 4);
|
assert.deepEqual(artTwo?.coverBlob, Buffer.from([9, 9, 9, 9]));
|
||||||
assert.deepEqual(artOne?.coverBlob, artTwo?.coverBlob);
|
|
||||||
assert.equal(animeArt?.coverBlob?.length, 4);
|
assert.equal(animeArt?.coverBlob?.length, 4);
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
library.map((row) => ({
|
library.map((row) => ({
|
||||||
@@ -2254,6 +2484,52 @@ test('cover art queries reuse a shared blob across duplicate anime art rows', ()
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('upsertCoverArt prefers freshly fetched bytes over a reused shared hash', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const originalVideoId = getOrCreateVideoRecord(db, 'local:/tmp/shared-cover-original.mkv', {
|
||||||
|
canonicalTitle: 'Shared Cover Original',
|
||||||
|
sourcePath: '/tmp/shared-cover-original.mkv',
|
||||||
|
sourceUrl: null,
|
||||||
|
sourceType: SOURCE_TYPE_LOCAL,
|
||||||
|
});
|
||||||
|
const refreshedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/shared-cover-refresh.mkv', {
|
||||||
|
canonicalTitle: 'Shared Cover Refresh',
|
||||||
|
sourcePath: '/tmp/shared-cover-refresh.mkv',
|
||||||
|
sourceUrl: null,
|
||||||
|
sourceType: SOURCE_TYPE_LOCAL,
|
||||||
|
});
|
||||||
|
|
||||||
|
upsertCoverArt(db, originalVideoId, {
|
||||||
|
anilistId: 999,
|
||||||
|
coverUrl: 'https://images.test/shared-refresh.jpg',
|
||||||
|
coverBlob: Buffer.from([1, 2, 3, 4]),
|
||||||
|
titleRomaji: 'Shared Cover Refresh',
|
||||||
|
titleEnglish: 'Shared Cover Refresh',
|
||||||
|
episodesTotal: 12,
|
||||||
|
});
|
||||||
|
upsertCoverArt(db, refreshedVideoId, {
|
||||||
|
anilistId: 999,
|
||||||
|
coverUrl: 'https://images.test/shared-refresh.jpg',
|
||||||
|
coverBlob: Buffer.from([9, 8, 7, 6]),
|
||||||
|
titleRomaji: 'Shared Cover Refresh',
|
||||||
|
titleEnglish: 'Shared Cover Refresh',
|
||||||
|
episodesTotal: 12,
|
||||||
|
});
|
||||||
|
|
||||||
|
const originalArt = getCoverArt(db, originalVideoId);
|
||||||
|
const refreshedArt = getCoverArt(db, refreshedVideoId);
|
||||||
|
assert.deepEqual(originalArt?.coverBlob, Buffer.from([1, 2, 3, 4]));
|
||||||
|
assert.deepEqual(refreshedArt?.coverBlob, Buffer.from([9, 8, 7, 6]));
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('anime/media detail and episode queries use ended-session metrics when telemetry rows are absent', () => {
|
test('anime/media detail and episode queries use ended-session metrics when telemetry rows are absent', () => {
|
||||||
const dbPath = makeDbPath();
|
const dbPath = makeDbPath();
|
||||||
const db = new Database(dbPath);
|
const db = new Database(dbPath);
|
||||||
@@ -2836,13 +3112,13 @@ test('deleteSession rebuilds word and kanji aggregates from retained subtitle li
|
|||||||
|
|
||||||
assert.ok(sharedWordRow);
|
assert.ok(sharedWordRow);
|
||||||
assert.equal(sharedWordRow.frequency, 1);
|
assert.equal(sharedWordRow.frequency, 1);
|
||||||
assert.equal(sharedWordRow.first_seen, keptTs);
|
assert.equal(sharedWordRow.first_seen, Math.floor(keptTs / 1000));
|
||||||
assert.equal(sharedWordRow.last_seen, keptTs);
|
assert.equal(sharedWordRow.last_seen, Math.floor(keptTs / 1000));
|
||||||
assert.equal(deletedOnlyWordRow ?? null, null);
|
assert.equal(deletedOnlyWordRow ?? null, null);
|
||||||
assert.ok(sharedKanjiRow);
|
assert.ok(sharedKanjiRow);
|
||||||
assert.equal(sharedKanjiRow.frequency, 1);
|
assert.equal(sharedKanjiRow.frequency, 1);
|
||||||
assert.equal(sharedKanjiRow.first_seen, keptTs);
|
assert.equal(sharedKanjiRow.first_seen, Math.floor(keptTs / 1000));
|
||||||
assert.equal(sharedKanjiRow.last_seen, keptTs);
|
assert.equal(sharedKanjiRow.last_seen, Math.floor(keptTs / 1000));
|
||||||
assert.equal(deletedOnlyKanjiRow ?? null, null);
|
assert.equal(deletedOnlyKanjiRow ?? null, null);
|
||||||
} finally {
|
} finally {
|
||||||
db.close();
|
db.close();
|
||||||
|
|||||||
@@ -156,7 +156,7 @@ export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisod
|
|||||||
COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount,
|
COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount,
|
||||||
MAX(s.started_at_ms) AS lastWatchedMs
|
MAX(s.started_at_ms) AS lastWatchedMs
|
||||||
FROM imm_videos v
|
FROM imm_videos v
|
||||||
JOIN imm_sessions s ON s.video_id = v.video_id
|
LEFT JOIN imm_sessions s ON s.video_id = v.video_id
|
||||||
LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
|
LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
|
||||||
WHERE v.anime_id = ?
|
WHERE v.anime_id = ?
|
||||||
GROUP BY v.video_id
|
GROUP BY v.video_id
|
||||||
|
|||||||
@@ -352,15 +352,16 @@ export function upsertCoverArt(
|
|||||||
const sharedCoverBlobHash = findSharedCoverBlobHash(db, videoId, art.anilistId, art.coverUrl);
|
const sharedCoverBlobHash = findSharedCoverBlobHash(db, videoId, art.anilistId, art.coverUrl);
|
||||||
const fetchedAtMs = toDbMs(nowMs());
|
const fetchedAtMs = toDbMs(nowMs());
|
||||||
const coverBlob = normalizeCoverBlobBytes(art.coverBlob);
|
const coverBlob = normalizeCoverBlobBytes(art.coverBlob);
|
||||||
let coverBlobHash = sharedCoverBlobHash ?? null;
|
const computedCoverBlobHash =
|
||||||
if (!coverBlobHash && coverBlob && coverBlob.length > 0) {
|
coverBlob && coverBlob.length > 0
|
||||||
coverBlobHash = createHash('sha256').update(coverBlob).digest('hex');
|
? createHash('sha256').update(coverBlob).digest('hex')
|
||||||
}
|
: null;
|
||||||
|
let coverBlobHash = computedCoverBlobHash ?? sharedCoverBlobHash ?? null;
|
||||||
if (!coverBlobHash && (!coverBlob || coverBlob.length === 0)) {
|
if (!coverBlobHash && (!coverBlob || coverBlob.length === 0)) {
|
||||||
coverBlobHash = existing?.coverBlobHash ?? null;
|
coverBlobHash = existing?.coverBlobHash ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (coverBlobHash && coverBlob && coverBlob.length > 0 && !sharedCoverBlobHash) {
|
if (computedCoverBlobHash && coverBlob && coverBlob.length > 0) {
|
||||||
db.prepare(
|
db.prepare(
|
||||||
`
|
`
|
||||||
INSERT INTO imm_cover_art_blobs (blob_hash, cover_blob, CREATED_DATE, LAST_UPDATE_DATE)
|
INSERT INTO imm_cover_art_blobs (blob_hash, cover_blob, CREATED_DATE, LAST_UPDATE_DATE)
|
||||||
@@ -368,7 +369,7 @@ export function upsertCoverArt(
|
|||||||
ON CONFLICT(blob_hash) DO UPDATE SET
|
ON CONFLICT(blob_hash) DO UPDATE SET
|
||||||
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
|
LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE
|
||||||
`,
|
`,
|
||||||
).run(coverBlobHash, coverBlob, fetchedAtMs, fetchedAtMs);
|
).run(computedCoverBlobHash, coverBlob, fetchedAtMs, fetchedAtMs);
|
||||||
}
|
}
|
||||||
|
|
||||||
db.prepare(
|
db.prepare(
|
||||||
|
|||||||
@@ -204,7 +204,7 @@ export function getQueryHints(db: DatabaseSync): {
|
|||||||
|
|
||||||
const now = new Date();
|
const now = new Date();
|
||||||
const todayLocal = Math.floor(
|
const todayLocal = Math.floor(
|
||||||
new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime() / 86_400_000,
|
(now.getTime() / 1000 - now.getTimezoneOffset() * 60) / 86_400,
|
||||||
);
|
);
|
||||||
|
|
||||||
const episodesToday =
|
const episodesToday =
|
||||||
@@ -333,9 +333,15 @@ export function getMonthlyRollups(db: DatabaseSync, limit = 24): ImmersionSessio
|
|||||||
total_lines_seen AS totalLinesSeen,
|
total_lines_seen AS totalLinesSeen,
|
||||||
total_tokens_seen AS totalTokensSeen,
|
total_tokens_seen AS totalTokensSeen,
|
||||||
total_cards AS totalCards,
|
total_cards AS totalCards,
|
||||||
0 AS cardsPerHour,
|
CASE
|
||||||
0 AS tokensPerMin,
|
WHEN total_active_min > 0 THEN (total_cards * 60.0) / total_active_min
|
||||||
0 AS lookupHitRate
|
ELSE NULL
|
||||||
|
END AS cardsPerHour,
|
||||||
|
CASE
|
||||||
|
WHEN total_active_min > 0 THEN total_tokens_seen * 1.0 / total_active_min
|
||||||
|
ELSE NULL
|
||||||
|
END AS tokensPerMin,
|
||||||
|
NULL AS lookupHitRate
|
||||||
FROM imm_monthly_rollups
|
FROM imm_monthly_rollups
|
||||||
WHERE rollup_month IN (SELECT rollup_month FROM recent_months)
|
WHERE rollup_month IN (SELECT rollup_month FROM recent_months)
|
||||||
ORDER BY rollup_month DESC, video_id DESC
|
ORDER BY rollup_month DESC, video_id DESC
|
||||||
|
|||||||
@@ -197,7 +197,12 @@ function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void {
|
|||||||
deleteStmt.run(row.wordId);
|
deleteStmt.run(row.wordId);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
updateStmt.run(row.frequency, row.firstSeen, row.lastSeen, row.wordId);
|
updateStmt.run(
|
||||||
|
row.frequency,
|
||||||
|
Math.floor(row.firstSeen / 1000),
|
||||||
|
Math.floor(row.lastSeen / 1000),
|
||||||
|
row.wordId,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -241,7 +246,12 @@ function refreshKanjiAggregates(db: DatabaseSync, kanjiIds: number[]): void {
|
|||||||
deleteStmt.run(row.kanjiId);
|
deleteStmt.run(row.kanjiId);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
updateStmt.run(row.frequency, row.firstSeen, row.lastSeen, row.kanjiId);
|
updateStmt.run(
|
||||||
|
row.frequency,
|
||||||
|
Math.floor(row.firstSeen / 1000),
|
||||||
|
Math.floor(row.lastSeen / 1000),
|
||||||
|
row.kanjiId,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -112,6 +112,16 @@ function makeTrendLabel(value: number): string {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getLocalEpochDay(timestampMs: number): number {
|
||||||
|
const date = new Date(timestampMs);
|
||||||
|
return Math.floor((timestampMs - date.getTimezoneOffset() * 60_000) / 86_400_000);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getLocalDateForEpochDay(epochDay: number): Date {
|
||||||
|
const utcDate = new Date(epochDay * 86_400_000);
|
||||||
|
return new Date(utcDate.getTime() + utcDate.getTimezoneOffset() * 60_000);
|
||||||
|
}
|
||||||
|
|
||||||
function getTrendSessionWordCount(session: Pick<TrendSessionMetricRow, 'tokensSeen'>): number {
|
function getTrendSessionWordCount(session: Pick<TrendSessionMetricRow, 'tokensSeen'>): number {
|
||||||
return session.tokensSeen;
|
return session.tokensSeen;
|
||||||
}
|
}
|
||||||
@@ -188,7 +198,7 @@ function buildWatchTimeByHour(sessions: TrendSessionMetricRow[]): TrendChartPoin
|
|||||||
}
|
}
|
||||||
|
|
||||||
function dayLabel(epochDay: number): string {
|
function dayLabel(epochDay: number): string {
|
||||||
return new Date(epochDay * 86_400_000).toLocaleDateString(undefined, {
|
return getLocalDateForEpochDay(epochDay).toLocaleDateString(undefined, {
|
||||||
month: 'short',
|
month: 'short',
|
||||||
day: 'numeric',
|
day: 'numeric',
|
||||||
});
|
});
|
||||||
@@ -200,7 +210,7 @@ function buildSessionSeriesByDay(
|
|||||||
): TrendChartPoint[] {
|
): TrendChartPoint[] {
|
||||||
const byDay = new Map<number, number>();
|
const byDay = new Map<number, number>();
|
||||||
for (const session of sessions) {
|
for (const session of sessions) {
|
||||||
const epochDay = Math.floor(session.startedAtMs / 86_400_000);
|
const epochDay = getLocalEpochDay(session.startedAtMs);
|
||||||
byDay.set(epochDay, (byDay.get(epochDay) ?? 0) + getValue(session));
|
byDay.set(epochDay, (byDay.get(epochDay) ?? 0) + getValue(session));
|
||||||
}
|
}
|
||||||
return Array.from(byDay.entries())
|
return Array.from(byDay.entries())
|
||||||
@@ -213,7 +223,7 @@ function buildLookupsPerHundredWords(sessions: TrendSessionMetricRow[]): TrendCh
|
|||||||
const wordsByDay = new Map<number, number>();
|
const wordsByDay = new Map<number, number>();
|
||||||
|
|
||||||
for (const session of sessions) {
|
for (const session of sessions) {
|
||||||
const epochDay = Math.floor(session.startedAtMs / 86_400_000);
|
const epochDay = getLocalEpochDay(session.startedAtMs);
|
||||||
lookupsByDay.set(epochDay, (lookupsByDay.get(epochDay) ?? 0) + session.yomitanLookupCount);
|
lookupsByDay.set(epochDay, (lookupsByDay.get(epochDay) ?? 0) + session.yomitanLookupCount);
|
||||||
wordsByDay.set(epochDay, (wordsByDay.get(epochDay) ?? 0) + getTrendSessionWordCount(session));
|
wordsByDay.set(epochDay, (wordsByDay.get(epochDay) ?? 0) + getTrendSessionWordCount(session));
|
||||||
}
|
}
|
||||||
@@ -237,7 +247,7 @@ function buildPerAnimeFromSessions(
|
|||||||
|
|
||||||
for (const session of sessions) {
|
for (const session of sessions) {
|
||||||
const animeTitle = resolveTrendAnimeTitle(session);
|
const animeTitle = resolveTrendAnimeTitle(session);
|
||||||
const epochDay = Math.floor(session.startedAtMs / 86_400_000);
|
const epochDay = getLocalEpochDay(session.startedAtMs);
|
||||||
const dayMap = byAnime.get(animeTitle) ?? new Map();
|
const dayMap = byAnime.get(animeTitle) ?? new Map();
|
||||||
dayMap.set(epochDay, (dayMap.get(epochDay) ?? 0) + getValue(session));
|
dayMap.set(epochDay, (dayMap.get(epochDay) ?? 0) + getValue(session));
|
||||||
byAnime.set(animeTitle, dayMap);
|
byAnime.set(animeTitle, dayMap);
|
||||||
@@ -258,7 +268,7 @@ function buildLookupsPerHundredPerAnime(sessions: TrendSessionMetricRow[]): Tren
|
|||||||
|
|
||||||
for (const session of sessions) {
|
for (const session of sessions) {
|
||||||
const animeTitle = resolveTrendAnimeTitle(session);
|
const animeTitle = resolveTrendAnimeTitle(session);
|
||||||
const epochDay = Math.floor(session.startedAtMs / 86_400_000);
|
const epochDay = getLocalEpochDay(session.startedAtMs);
|
||||||
|
|
||||||
const lookupMap = lookups.get(animeTitle) ?? new Map();
|
const lookupMap = lookups.get(animeTitle) ?? new Map();
|
||||||
lookupMap.set(epochDay, (lookupMap.get(epochDay) ?? 0) + session.yomitanLookupCount);
|
lookupMap.set(epochDay, (lookupMap.get(epochDay) ?? 0) + session.yomitanLookupCount);
|
||||||
@@ -462,7 +472,7 @@ function buildNewWordsPerDay(db: DatabaseSync, cutoffMs: number | null): TrendCh
|
|||||||
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
||||||
const prepared = db.prepare(`
|
const prepared = db.prepare(`
|
||||||
SELECT
|
SELECT
|
||||||
CAST(first_seen / 86400 AS INTEGER) AS epochDay,
|
CAST(julianday(first_seen, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) AS epochDay,
|
||||||
COUNT(*) AS wordCount
|
COUNT(*) AS wordCount
|
||||||
FROM imm_words
|
FROM imm_words
|
||||||
WHERE first_seen IS NOT NULL
|
WHERE first_seen IS NOT NULL
|
||||||
|
|||||||
@@ -1078,6 +1078,56 @@ test('executeQueuedWrite inserts event and telemetry rows', () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('executeQueuedWrite rejects partial telemetry writes instead of zero-filling', () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
const db = new Database(dbPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensureSchema(db);
|
||||||
|
const stmts = createTrackerPreparedStatements(db);
|
||||||
|
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/partial-telemetry.mkv', {
|
||||||
|
canonicalTitle: 'Partial Telemetry',
|
||||||
|
sourcePath: '/tmp/partial-telemetry.mkv',
|
||||||
|
sourceUrl: null,
|
||||||
|
sourceType: SOURCE_TYPE_LOCAL,
|
||||||
|
});
|
||||||
|
const { sessionId } = startSessionRecord(db, videoId, 5_000);
|
||||||
|
|
||||||
|
assert.throws(
|
||||||
|
() =>
|
||||||
|
executeQueuedWrite(
|
||||||
|
{
|
||||||
|
kind: 'telemetry',
|
||||||
|
sessionId,
|
||||||
|
sampleMs: 6_000,
|
||||||
|
totalWatchedMs: 1_000,
|
||||||
|
activeWatchedMs: 900,
|
||||||
|
linesSeen: 3,
|
||||||
|
cardsMined: 1,
|
||||||
|
lookupCount: 2,
|
||||||
|
lookupHits: 1,
|
||||||
|
yomitanLookupCount: 0,
|
||||||
|
pauseCount: 1,
|
||||||
|
pauseMs: 50,
|
||||||
|
seekForwardCount: 0,
|
||||||
|
seekBackwardCount: 0,
|
||||||
|
mediaBufferEvents: 0,
|
||||||
|
},
|
||||||
|
stmts,
|
||||||
|
),
|
||||||
|
/Incomplete telemetry write/,
|
||||||
|
);
|
||||||
|
|
||||||
|
const telemetryCount = db
|
||||||
|
.prepare('SELECT COUNT(*) AS total FROM imm_session_telemetry WHERE session_id = ?')
|
||||||
|
.get(sessionId) as { total: number };
|
||||||
|
assert.equal(telemetryCount.total, 0);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('executeQueuedWrite inserts and upserts word and kanji rows', () => {
|
test('executeQueuedWrite inserts and upserts word and kanji rows', () => {
|
||||||
const dbPath = makeDbPath();
|
const dbPath = makeDbPath();
|
||||||
const db = new Database(dbPath);
|
const db = new Database(dbPath);
|
||||||
|
|||||||
@@ -1406,27 +1406,46 @@ function incrementKanjiAggregate(
|
|||||||
export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedStatements): void {
|
export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedStatements): void {
|
||||||
const currentMs = toDbMs(nowMs());
|
const currentMs = toDbMs(nowMs());
|
||||||
if (write.kind === 'telemetry') {
|
if (write.kind === 'telemetry') {
|
||||||
|
if (
|
||||||
|
write.totalWatchedMs === undefined ||
|
||||||
|
write.activeWatchedMs === undefined ||
|
||||||
|
write.linesSeen === undefined ||
|
||||||
|
write.tokensSeen === undefined ||
|
||||||
|
write.cardsMined === undefined ||
|
||||||
|
write.lookupCount === undefined ||
|
||||||
|
write.lookupHits === undefined ||
|
||||||
|
write.yomitanLookupCount === undefined ||
|
||||||
|
write.pauseCount === undefined ||
|
||||||
|
write.pauseMs === undefined ||
|
||||||
|
write.seekForwardCount === undefined ||
|
||||||
|
write.seekBackwardCount === undefined ||
|
||||||
|
write.mediaBufferEvents === undefined
|
||||||
|
) {
|
||||||
|
throw new Error('Incomplete telemetry write');
|
||||||
|
}
|
||||||
const telemetrySampleMs = toDbMs(write.sampleMs ?? Number(currentMs));
|
const telemetrySampleMs = toDbMs(write.sampleMs ?? Number(currentMs));
|
||||||
stmts.telemetryInsertStmt.run(
|
stmts.telemetryInsertStmt.run(
|
||||||
write.sessionId,
|
write.sessionId,
|
||||||
telemetrySampleMs,
|
telemetrySampleMs,
|
||||||
write.totalWatchedMs ?? 0,
|
write.totalWatchedMs,
|
||||||
write.activeWatchedMs ?? 0,
|
write.activeWatchedMs,
|
||||||
write.linesSeen ?? 0,
|
write.linesSeen,
|
||||||
write.tokensSeen ?? 0,
|
write.tokensSeen,
|
||||||
write.cardsMined ?? 0,
|
write.cardsMined,
|
||||||
write.lookupCount ?? 0,
|
write.lookupCount,
|
||||||
write.lookupHits ?? 0,
|
write.lookupHits,
|
||||||
write.yomitanLookupCount ?? 0,
|
write.yomitanLookupCount,
|
||||||
write.pauseCount ?? 0,
|
write.pauseCount,
|
||||||
write.pauseMs ?? 0,
|
write.pauseMs,
|
||||||
write.seekForwardCount ?? 0,
|
write.seekForwardCount,
|
||||||
write.seekBackwardCount ?? 0,
|
write.seekBackwardCount,
|
||||||
write.mediaBufferEvents ?? 0,
|
write.mediaBufferEvents,
|
||||||
currentMs,
|
currentMs,
|
||||||
currentMs,
|
currentMs,
|
||||||
);
|
);
|
||||||
|
if (write.lastMediaMs !== undefined) {
|
||||||
stmts.sessionCheckpointStmt.run(write.lastMediaMs ?? null, currentMs, write.sessionId);
|
stmts.sessionCheckpointStmt.run(write.lastMediaMs ?? null, currentMs, write.sessionId);
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (write.kind === 'word') {
|
if (write.kind === 'word') {
|
||||||
|
|||||||
7
src/core/services/immersion-tracker/time.test.ts
Normal file
7
src/core/services/immersion-tracker/time.test.ts
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import test from 'node:test';
|
||||||
|
import { nowMs } from './time.js';
|
||||||
|
|
||||||
|
test('nowMs returns wall-clock epoch milliseconds', () => {
|
||||||
|
assert.ok(nowMs() > 1_600_000_000_000);
|
||||||
|
});
|
||||||
@@ -1,10 +1,8 @@
|
|||||||
const SQLITE_SAFE_EPOCH_BASE_MS = 2_000_000_000;
|
|
||||||
|
|
||||||
export function nowMs(): number {
|
export function nowMs(): number {
|
||||||
const perf = globalThis.performance;
|
const perf = globalThis.performance;
|
||||||
if (perf) {
|
if (perf && Number.isFinite(perf.timeOrigin)) {
|
||||||
return SQLITE_SAFE_EPOCH_BASE_MS + Math.floor(perf.now());
|
return Math.floor(perf.timeOrigin + perf.now());
|
||||||
}
|
}
|
||||||
|
|
||||||
return SQLITE_SAFE_EPOCH_BASE_MS;
|
return Date.now();
|
||||||
}
|
}
|
||||||
|
|||||||
98
src/main/character-dictionary-runtime/zip.test.ts
Normal file
98
src/main/character-dictionary-runtime/zip.test.ts
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import test from 'node:test';
|
||||||
|
import { buildDictionaryZip } from './zip';
|
||||||
|
import type { CharacterDictionaryTermEntry } from './types';
|
||||||
|
|
||||||
|
function makeTempDir(): string {
|
||||||
|
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-character-zip-'));
|
||||||
|
}
|
||||||
|
|
||||||
|
function cleanupDir(dirPath: string): void {
|
||||||
|
fs.rmSync(dirPath, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
function readStoredZipEntries(zipPath: string): Map<string, Buffer> {
|
||||||
|
const archive = fs.readFileSync(zipPath);
|
||||||
|
const entries = new Map<string, Buffer>();
|
||||||
|
let cursor = 0;
|
||||||
|
|
||||||
|
while (cursor + 4 <= archive.length) {
|
||||||
|
const signature = archive.readUInt32LE(cursor);
|
||||||
|
if (signature === 0x02014b50 || signature === 0x06054b50) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
assert.equal(signature, 0x04034b50, `unexpected local file header at offset ${cursor}`);
|
||||||
|
|
||||||
|
const compressedSize = archive.readUInt32LE(cursor + 18);
|
||||||
|
const fileNameLength = archive.readUInt16LE(cursor + 26);
|
||||||
|
const extraLength = archive.readUInt16LE(cursor + 28);
|
||||||
|
const fileNameStart = cursor + 30;
|
||||||
|
const dataStart = fileNameStart + fileNameLength + extraLength;
|
||||||
|
const fileName = archive.subarray(fileNameStart, fileNameStart + fileNameLength).toString(
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
const data = archive.subarray(dataStart, dataStart + compressedSize);
|
||||||
|
entries.set(fileName, Buffer.from(data));
|
||||||
|
cursor = dataStart + compressedSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
|
|
||||||
|
test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', () => {
|
||||||
|
const tempDir = makeTempDir();
|
||||||
|
const outputPath = path.join(tempDir, 'dictionary.zip');
|
||||||
|
const termEntries: CharacterDictionaryTermEntry[] = [
|
||||||
|
['アルファ', 'あるふぁ', '', '', 0, ['Alpha entry'], 0, 'name'],
|
||||||
|
];
|
||||||
|
const originalBufferConcat = Buffer.concat;
|
||||||
|
|
||||||
|
try {
|
||||||
|
Buffer.concat = ((...args: Parameters<typeof Buffer.concat>) => {
|
||||||
|
throw new Error(`buildDictionaryZip should not Buffer.concat the full archive (${args[0].length} chunks)`);
|
||||||
|
}) as typeof Buffer.concat;
|
||||||
|
|
||||||
|
const result = buildDictionaryZip(
|
||||||
|
outputPath,
|
||||||
|
'Dictionary Title',
|
||||||
|
'Dictionary Description',
|
||||||
|
'2026-03-27',
|
||||||
|
termEntries,
|
||||||
|
[{ path: 'images/alpha.bin', dataBase64: Buffer.from([1, 2, 3]).toString('base64') }],
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(result.zipPath, outputPath);
|
||||||
|
assert.equal(result.entryCount, 1);
|
||||||
|
|
||||||
|
const entries = readStoredZipEntries(outputPath);
|
||||||
|
assert.deepEqual([...entries.keys()].sort(), [
|
||||||
|
'images/alpha.bin',
|
||||||
|
'index.json',
|
||||||
|
'tag_bank_1.json',
|
||||||
|
'term_bank_1.json',
|
||||||
|
]);
|
||||||
|
|
||||||
|
const indexJson = JSON.parse(entries.get('index.json')!.toString('utf8')) as {
|
||||||
|
title: string;
|
||||||
|
description: string;
|
||||||
|
revision: string;
|
||||||
|
format: number;
|
||||||
|
};
|
||||||
|
assert.equal(indexJson.title, 'Dictionary Title');
|
||||||
|
assert.equal(indexJson.description, 'Dictionary Description');
|
||||||
|
assert.equal(indexJson.revision, '2026-03-27');
|
||||||
|
assert.equal(indexJson.format, 3);
|
||||||
|
|
||||||
|
const termBank = JSON.parse(entries.get('term_bank_1.json')!.toString('utf8')) as
|
||||||
|
CharacterDictionaryTermEntry[];
|
||||||
|
assert.equal(termBank.length, 1);
|
||||||
|
assert.equal(termBank[0]?.[0], 'アルファ');
|
||||||
|
assert.deepEqual(entries.get('images/alpha.bin'), Buffer.from([1, 2, 3]));
|
||||||
|
} finally {
|
||||||
|
Buffer.concat = originalBufferConcat;
|
||||||
|
cleanupDir(tempDir);
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -5,8 +5,8 @@ import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } f
|
|||||||
|
|
||||||
type ZipEntry = {
|
type ZipEntry = {
|
||||||
name: string;
|
name: string;
|
||||||
data: Buffer;
|
|
||||||
crc32: number;
|
crc32: number;
|
||||||
|
size: number;
|
||||||
localHeaderOffset: number;
|
localHeaderOffset: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -67,15 +67,7 @@ function crc32(data: Buffer): number {
|
|||||||
return (crc ^ 0xffffffff) >>> 0;
|
return (crc ^ 0xffffffff) >>> 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
|
function createLocalFileHeader(fileName: Buffer, fileCrc32: number, fileSize: number): Buffer {
|
||||||
const chunks: Buffer[] = [];
|
|
||||||
const entries: ZipEntry[] = [];
|
|
||||||
let offset = 0;
|
|
||||||
|
|
||||||
for (const file of files) {
|
|
||||||
const fileName = Buffer.from(file.name, 'utf8');
|
|
||||||
const fileData = file.data;
|
|
||||||
const fileCrc32 = crc32(fileData);
|
|
||||||
const local = Buffer.alloc(30 + fileName.length);
|
const local = Buffer.alloc(30 + fileName.length);
|
||||||
let cursor = 0;
|
let cursor = 0;
|
||||||
writeUint32LE(local, 0x04034b50, cursor);
|
writeUint32LE(local, 0x04034b50, cursor);
|
||||||
@@ -92,29 +84,19 @@ function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
|
|||||||
cursor += 2;
|
cursor += 2;
|
||||||
writeUint32LE(local, fileCrc32, cursor);
|
writeUint32LE(local, fileCrc32, cursor);
|
||||||
cursor += 4;
|
cursor += 4;
|
||||||
writeUint32LE(local, fileData.length, cursor);
|
writeUint32LE(local, fileSize, cursor);
|
||||||
cursor += 4;
|
cursor += 4;
|
||||||
writeUint32LE(local, fileData.length, cursor);
|
writeUint32LE(local, fileSize, cursor);
|
||||||
cursor += 4;
|
cursor += 4;
|
||||||
local.writeUInt16LE(fileName.length, cursor);
|
local.writeUInt16LE(fileName.length, cursor);
|
||||||
cursor += 2;
|
cursor += 2;
|
||||||
local.writeUInt16LE(0, cursor);
|
local.writeUInt16LE(0, cursor);
|
||||||
cursor += 2;
|
cursor += 2;
|
||||||
fileName.copy(local, cursor);
|
fileName.copy(local, cursor);
|
||||||
|
return local;
|
||||||
chunks.push(local, fileData);
|
|
||||||
entries.push({
|
|
||||||
name: file.name,
|
|
||||||
data: fileData,
|
|
||||||
crc32: fileCrc32,
|
|
||||||
localHeaderOffset: offset,
|
|
||||||
});
|
|
||||||
offset += local.length + fileData.length;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const centralStart = offset;
|
function createCentralDirectoryHeader(entry: ZipEntry): Buffer {
|
||||||
const centralChunks: Buffer[] = [];
|
|
||||||
for (const entry of entries) {
|
|
||||||
const fileName = Buffer.from(entry.name, 'utf8');
|
const fileName = Buffer.from(entry.name, 'utf8');
|
||||||
const central = Buffer.alloc(46 + fileName.length);
|
const central = Buffer.alloc(46 + fileName.length);
|
||||||
let cursor = 0;
|
let cursor = 0;
|
||||||
@@ -134,9 +116,9 @@ function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
|
|||||||
cursor += 2;
|
cursor += 2;
|
||||||
writeUint32LE(central, entry.crc32, cursor);
|
writeUint32LE(central, entry.crc32, cursor);
|
||||||
cursor += 4;
|
cursor += 4;
|
||||||
writeUint32LE(central, entry.data.length, cursor);
|
writeUint32LE(central, entry.size, cursor);
|
||||||
cursor += 4;
|
cursor += 4;
|
||||||
writeUint32LE(central, entry.data.length, cursor);
|
writeUint32LE(central, entry.size, cursor);
|
||||||
cursor += 4;
|
cursor += 4;
|
||||||
central.writeUInt16LE(fileName.length, cursor);
|
central.writeUInt16LE(fileName.length, cursor);
|
||||||
cursor += 2;
|
cursor += 2;
|
||||||
@@ -153,11 +135,10 @@ function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
|
|||||||
writeUint32LE(central, entry.localHeaderOffset, cursor);
|
writeUint32LE(central, entry.localHeaderOffset, cursor);
|
||||||
cursor += 4;
|
cursor += 4;
|
||||||
fileName.copy(central, cursor);
|
fileName.copy(central, cursor);
|
||||||
centralChunks.push(central);
|
return central;
|
||||||
offset += central.length;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const centralSize = offset - centralStart;
|
function createEndOfCentralDirectory(entriesLength: number, centralSize: number, centralStart: number): Buffer {
|
||||||
const end = Buffer.alloc(22);
|
const end = Buffer.alloc(22);
|
||||||
let cursor = 0;
|
let cursor = 0;
|
||||||
writeUint32LE(end, 0x06054b50, cursor);
|
writeUint32LE(end, 0x06054b50, cursor);
|
||||||
@@ -166,17 +147,63 @@ function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer {
|
|||||||
cursor += 2;
|
cursor += 2;
|
||||||
end.writeUInt16LE(0, cursor);
|
end.writeUInt16LE(0, cursor);
|
||||||
cursor += 2;
|
cursor += 2;
|
||||||
end.writeUInt16LE(entries.length, cursor);
|
end.writeUInt16LE(entriesLength, cursor);
|
||||||
cursor += 2;
|
cursor += 2;
|
||||||
end.writeUInt16LE(entries.length, cursor);
|
end.writeUInt16LE(entriesLength, cursor);
|
||||||
cursor += 2;
|
cursor += 2;
|
||||||
writeUint32LE(end, centralSize, cursor);
|
writeUint32LE(end, centralSize, cursor);
|
||||||
cursor += 4;
|
cursor += 4;
|
||||||
writeUint32LE(end, centralStart, cursor);
|
writeUint32LE(end, centralStart, cursor);
|
||||||
cursor += 4;
|
cursor += 4;
|
||||||
end.writeUInt16LE(0, cursor);
|
end.writeUInt16LE(0, cursor);
|
||||||
|
return end;
|
||||||
|
}
|
||||||
|
|
||||||
return Buffer.concat([...chunks, ...centralChunks, end]);
|
function writeBuffer(fd: number, buffer: Buffer): void {
|
||||||
|
let written = 0;
|
||||||
|
while (written < buffer.length) {
|
||||||
|
written += fs.writeSync(fd, buffer, written, buffer.length - written);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeStoredZip(outputPath: string, files: Iterable<{ name: string; data: Buffer }>): void {
|
||||||
|
const entries: ZipEntry[] = [];
|
||||||
|
let offset = 0;
|
||||||
|
const fd = fs.openSync(outputPath, 'w');
|
||||||
|
|
||||||
|
try {
|
||||||
|
for (const file of files) {
|
||||||
|
const fileName = Buffer.from(file.name, 'utf8');
|
||||||
|
const fileSize = file.data.length;
|
||||||
|
const fileCrc32 = crc32(file.data);
|
||||||
|
const localHeader = createLocalFileHeader(fileName, fileCrc32, fileSize);
|
||||||
|
writeBuffer(fd, localHeader);
|
||||||
|
writeBuffer(fd, file.data);
|
||||||
|
entries.push({
|
||||||
|
name: file.name,
|
||||||
|
crc32: fileCrc32,
|
||||||
|
size: fileSize,
|
||||||
|
localHeaderOffset: offset,
|
||||||
|
});
|
||||||
|
offset += localHeader.length + fileSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
const centralStart = offset;
|
||||||
|
for (const entry of entries) {
|
||||||
|
const centralHeader = createCentralDirectoryHeader(entry);
|
||||||
|
writeBuffer(fd, centralHeader);
|
||||||
|
offset += centralHeader.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
const centralSize = offset - centralStart;
|
||||||
|
writeBuffer(fd, createEndOfCentralDirectory(entries.length, centralSize, centralStart));
|
||||||
|
} catch (error) {
|
||||||
|
fs.closeSync(fd);
|
||||||
|
fs.rmSync(outputPath, { force: true });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.closeSync(fd);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildDictionaryZip(
|
export function buildDictionaryZip(
|
||||||
@@ -187,36 +214,37 @@ export function buildDictionaryZip(
|
|||||||
termEntries: CharacterDictionaryTermEntry[],
|
termEntries: CharacterDictionaryTermEntry[],
|
||||||
images: CharacterDictionarySnapshotImage[],
|
images: CharacterDictionarySnapshotImage[],
|
||||||
): { zipPath: string; entryCount: number } {
|
): { zipPath: string; entryCount: number } {
|
||||||
const zipFiles: Array<{ name: string; data: Buffer }> = [
|
ensureDir(path.dirname(outputPath));
|
||||||
{
|
|
||||||
|
function* zipFiles(): Iterable<{ name: string; data: Buffer }> {
|
||||||
|
yield {
|
||||||
name: 'index.json',
|
name: 'index.json',
|
||||||
data: Buffer.from(
|
data: Buffer.from(
|
||||||
JSON.stringify(createIndex(dictionaryTitle, description, revision), null, 2),
|
JSON.stringify(createIndex(dictionaryTitle, description, revision), null, 2),
|
||||||
'utf8',
|
'utf8',
|
||||||
),
|
),
|
||||||
},
|
};
|
||||||
{
|
yield {
|
||||||
name: 'tag_bank_1.json',
|
name: 'tag_bank_1.json',
|
||||||
data: Buffer.from(JSON.stringify(createTagBank()), 'utf8'),
|
data: Buffer.from(JSON.stringify(createTagBank()), 'utf8'),
|
||||||
},
|
};
|
||||||
];
|
|
||||||
|
|
||||||
for (const image of images) {
|
for (const image of images) {
|
||||||
zipFiles.push({
|
yield {
|
||||||
name: image.path,
|
name: image.path,
|
||||||
data: Buffer.from(image.dataBase64, 'base64'),
|
data: Buffer.from(image.dataBase64, 'base64'),
|
||||||
});
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const entriesPerBank = 10_000;
|
const entriesPerBank = 10_000;
|
||||||
for (let i = 0; i < termEntries.length; i += entriesPerBank) {
|
for (let i = 0; i < termEntries.length; i += entriesPerBank) {
|
||||||
zipFiles.push({
|
yield {
|
||||||
name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`,
|
name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`,
|
||||||
data: Buffer.from(JSON.stringify(termEntries.slice(i, i + entriesPerBank)), 'utf8'),
|
data: Buffer.from(JSON.stringify(termEntries.slice(i, i + entriesPerBank)), 'utf8'),
|
||||||
});
|
};
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ensureDir(path.dirname(outputPath));
|
writeStoredZip(outputPath, zipFiles());
|
||||||
fs.writeFileSync(outputPath, createStoredZip(zipFiles));
|
|
||||||
return { zipPath: outputPath, entryCount: termEntries.length };
|
return { zipPath: outputPath, entryCount: termEntries.length };
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user