diff --git a/plugins/subminer-workflow/skills/subminer-change-verification/scripts/verify_subminer_change.sh b/plugins/subminer-workflow/skills/subminer-change-verification/scripts/verify_subminer_change.sh index 3284ece..692b521 100755 --- a/plugins/subminer-workflow/skills/subminer-change-verification/scripts/verify_subminer_change.sh +++ b/plugins/subminer-workflow/skills/subminer-change-verification/scripts/verify_subminer_change.sh @@ -72,6 +72,14 @@ add_blocker() { BLOCKED=1 } +validate_artifact_dir() { + local candidate=$1 + if [[ ! "$candidate" =~ ^[A-Za-z0-9._/@:+-]+$ ]]; then + echo "Invalid characters in --artifact-dir path" >&2 + exit 2 + fi +} + append_step_record() { printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" >>"$STEPS_TSV" @@ -411,6 +419,7 @@ if [[ -z "${ARTIFACT_DIR:-}" ]]; then SESSION_ID=$(generate_session_id) ARTIFACT_DIR="$REPO_ROOT/.tmp/skill-verification/$SESSION_ID" else + validate_artifact_dir "$ARTIFACT_DIR" SESSION_ID=$(basename "$ARTIFACT_DIR") fi diff --git a/src/core/services/immersion-tracker-service.test.ts b/src/core/services/immersion-tracker-service.test.ts index 4c94ffb..e286f12 100644 --- a/src/core/services/immersion-tracker-service.test.ts +++ b/src/core/services/immersion-tracker-service.test.ts @@ -1982,6 +1982,7 @@ test('flushSingle reuses cached prepared statements', async () => { cardsMined?: number; lookupCount?: number; lookupHits?: number; + yomitanLookupCount?: number; pauseCount?: number; pauseMs?: number; seekForwardCount?: number; @@ -2051,6 +2052,7 @@ test('flushSingle reuses cached prepared statements', async () => { cardsMined: 0, lookupCount: 0, lookupHits: 0, + yomitanLookupCount: 0, pauseCount: 0, pauseMs: 0, seekForwardCount: 0, diff --git a/src/core/services/immersion-tracker/__tests__/query.test.ts b/src/core/services/immersion-tracker/__tests__/query.test.ts index 21fbb47..ccc90bd 100644 --- a/src/core/services/immersion-tracker/__tests__/query.test.ts +++ b/src/core/services/immersion-tracker/__tests__/query.test.ts @@ -208,6 +208,104 @@ test('getAnimeEpisodes prefers the latest session media position when the latest } }); +test('getAnimeEpisodes includes unwatched episodes for the anime', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const watchedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/watched-episode.mkv', { + canonicalTitle: 'Watched Episode', + sourcePath: '/tmp/watched-episode.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + const unwatchedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/unwatched-episode.mkv', { + canonicalTitle: 'Unwatched Episode', + sourcePath: '/tmp/unwatched-episode.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + const animeId = getOrCreateAnimeRecord(db, { + parsedTitle: 'Episode Coverage Anime', + canonicalTitle: 'Episode Coverage Anime', + anilistId: null, + titleRomaji: null, + titleEnglish: null, + titleNative: null, + metadataJson: null, + }); + linkVideoToAnimeRecord(db, watchedVideoId, { + animeId, + parsedBasename: 'watched-episode.mkv', + parsedTitle: 'Episode Coverage Anime', + parsedSeason: 1, + parsedEpisode: 1, + parserSource: 'fallback', + parserConfidence: 1, + parseMetadataJson: '{"episode":1}', + }); + linkVideoToAnimeRecord(db, unwatchedVideoId, { + animeId, + parsedBasename: 'unwatched-episode.mkv', + parsedTitle: 'Episode Coverage Anime', + parsedSeason: 1, + parsedEpisode: 2, + parserSource: 'fallback', + parserConfidence: 1, + parseMetadataJson: '{"episode":2}', + }); + + const watchedSessionId = startSessionRecord(db, watchedVideoId, 1_000_000).sessionId; + db.prepare( + ` + UPDATE imm_sessions + SET + ended_at_ms = ?, + status = 2, + ended_media_ms = ?, + active_watched_ms = ?, + cards_mined = ?, + tokens_seen = ?, + yomitan_lookup_count = ?, + LAST_UPDATE_DATE = ? + WHERE session_id = ? + `, + ).run(1_005_000, 7_000, 3_000, 2, 20, 4, 1_005_000, watchedSessionId); + + const episodes = getAnimeEpisodes(db, animeId); + assert.equal(episodes.length, 2); + assert.deepEqual( + episodes.map((episode) => ({ + videoId: episode.videoId, + totalSessions: episode.totalSessions, + totalActiveMs: episode.totalActiveMs, + totalCards: episode.totalCards, + totalTokensSeen: episode.totalTokensSeen, + })), + [ + { + videoId: watchedVideoId, + totalSessions: 1, + totalActiveMs: 3_000, + totalCards: 2, + totalTokensSeen: 20, + }, + { + videoId: unwatchedVideoId, + totalSessions: 0, + totalActiveMs: 0, + totalCards: 0, + totalTokensSeen: 0, + }, + ], + ); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('getAnimeEpisodes falls back to the latest subtitle segment end when session progress checkpoints are missing', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -586,6 +684,109 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => { } }); +test('getTrendsDashboard keeps local-midnight session buckets separate', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const stmts = createTrackerPreparedStatements(db); + const videoId = getOrCreateVideoRecord(db, 'local:/tmp/local-midnight-trends.mkv', { + canonicalTitle: 'Local Midnight Trends', + sourcePath: '/tmp/local-midnight-trends.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + const animeId = getOrCreateAnimeRecord(db, { + parsedTitle: 'Local Midnight Trends', + canonicalTitle: 'Local Midnight Trends', + anilistId: null, + titleRomaji: null, + titleEnglish: null, + titleNative: null, + metadataJson: null, + }); + linkVideoToAnimeRecord(db, videoId, { + animeId, + parsedBasename: 'local-midnight-trends.mkv', + parsedTitle: 'Local Midnight Trends', + parsedSeason: 1, + parsedEpisode: 1, + parserSource: 'test', + parserConfidence: 1, + parseMetadataJson: null, + }); + + const beforeMidnight = new Date(2026, 2, 1, 23, 30).getTime(); + const afterMidnight = new Date(2026, 2, 2, 0, 30).getTime(); + const firstSessionId = startSessionRecord(db, videoId, beforeMidnight).sessionId; + const secondSessionId = startSessionRecord(db, videoId, afterMidnight).sessionId; + + for (const [sessionId, startedAtMs, tokensSeen, lookupCount] of [ + [firstSessionId, beforeMidnight, 100, 4], + [secondSessionId, afterMidnight, 120, 6], + ] as const) { + stmts.telemetryInsertStmt.run( + sessionId, + startedAtMs + 60_000, + 60_000, + 60_000, + 1, + tokensSeen, + 0, + lookupCount, + lookupCount, + lookupCount, + 0, + 0, + 0, + 0, + startedAtMs + 60_000, + startedAtMs + 60_000, + ); + db.prepare( + ` + UPDATE imm_sessions + SET + ended_at_ms = ?, + status = 2, + total_watched_ms = ?, + active_watched_ms = ?, + lines_seen = ?, + tokens_seen = ?, + lookup_count = ?, + lookup_hits = ?, + yomitan_lookup_count = ?, + LAST_UPDATE_DATE = ? + WHERE session_id = ? + `, + ).run( + startedAtMs + 60_000, + 60_000, + 60_000, + 1, + tokensSeen, + lookupCount, + lookupCount, + lookupCount, + startedAtMs + 60_000, + sessionId, + ); + } + + const dashboard = getTrendsDashboard(db, 'all', 'day'); + assert.equal(dashboard.progress.lookups.length, 2); + assert.deepEqual( + dashboard.progress.lookups.map((point) => point.value), + [4, 10], + ); + assert.equal(dashboard.ratios.lookupsPerHundred.length, 2); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('getQueryHints reads all-time totals from lifetime summary', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -1024,6 +1225,36 @@ test('getMonthlyRollups returns all rows for the most recent rollup months', () } }); +test('getMonthlyRollups derives rate metrics from stored monthly totals', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + try { + ensureSchema(db); + const insertRollup = db.prepare( + ` + INSERT INTO imm_monthly_rollups ( + rollup_month, video_id, total_sessions, total_active_min, total_lines_seen, + total_tokens_seen, total_cards, CREATED_DATE, LAST_UPDATE_DATE + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + ); + const nowMs = Date.now(); + insertRollup.run(202602, 1, 2, 30, 20, 90, 15, nowMs, nowMs); + insertRollup.run(202602, 2, 1, 0, 10, 25, 5, nowMs, nowMs); + + const rows = getMonthlyRollups(db, 1); + assert.equal(rows.length, 2); + assert.equal(rows[1]?.cardsPerHour, 30); + assert.equal(rows[1]?.tokensPerMin, 3); + assert.equal(rows[1]?.lookupHitRate ?? null, null); + assert.equal(rows[0]?.cardsPerHour ?? null, null); + assert.equal(rows[0]?.tokensPerMin ?? null, null); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('getAnimeDailyRollups returns all rows for the most recent rollup days', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -2234,9 +2465,8 @@ test('cover art queries reuse a shared blob across duplicate anime art rows', () const animeArt = getAnimeCoverArt(db, animeId); const library = getMediaLibrary(db); - assert.equal(artOne?.coverBlob?.length, 4); - assert.equal(artTwo?.coverBlob?.length, 4); - assert.deepEqual(artOne?.coverBlob, artTwo?.coverBlob); + assert.deepEqual(artOne?.coverBlob, Buffer.from([1, 2, 3, 4])); + assert.deepEqual(artTwo?.coverBlob, Buffer.from([9, 9, 9, 9])); assert.equal(animeArt?.coverBlob?.length, 4); assert.deepEqual( library.map((row) => ({ @@ -2254,6 +2484,52 @@ test('cover art queries reuse a shared blob across duplicate anime art rows', () } }); +test('upsertCoverArt prefers freshly fetched bytes over a reused shared hash', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const originalVideoId = getOrCreateVideoRecord(db, 'local:/tmp/shared-cover-original.mkv', { + canonicalTitle: 'Shared Cover Original', + sourcePath: '/tmp/shared-cover-original.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + const refreshedVideoId = getOrCreateVideoRecord(db, 'local:/tmp/shared-cover-refresh.mkv', { + canonicalTitle: 'Shared Cover Refresh', + sourcePath: '/tmp/shared-cover-refresh.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + + upsertCoverArt(db, originalVideoId, { + anilistId: 999, + coverUrl: 'https://images.test/shared-refresh.jpg', + coverBlob: Buffer.from([1, 2, 3, 4]), + titleRomaji: 'Shared Cover Refresh', + titleEnglish: 'Shared Cover Refresh', + episodesTotal: 12, + }); + upsertCoverArt(db, refreshedVideoId, { + anilistId: 999, + coverUrl: 'https://images.test/shared-refresh.jpg', + coverBlob: Buffer.from([9, 8, 7, 6]), + titleRomaji: 'Shared Cover Refresh', + titleEnglish: 'Shared Cover Refresh', + episodesTotal: 12, + }); + + const originalArt = getCoverArt(db, originalVideoId); + const refreshedArt = getCoverArt(db, refreshedVideoId); + assert.deepEqual(originalArt?.coverBlob, Buffer.from([1, 2, 3, 4])); + assert.deepEqual(refreshedArt?.coverBlob, Buffer.from([9, 8, 7, 6])); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('anime/media detail and episode queries use ended-session metrics when telemetry rows are absent', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -2836,13 +3112,13 @@ test('deleteSession rebuilds word and kanji aggregates from retained subtitle li assert.ok(sharedWordRow); assert.equal(sharedWordRow.frequency, 1); - assert.equal(sharedWordRow.first_seen, keptTs); - assert.equal(sharedWordRow.last_seen, keptTs); + assert.equal(sharedWordRow.first_seen, Math.floor(keptTs / 1000)); + assert.equal(sharedWordRow.last_seen, Math.floor(keptTs / 1000)); assert.equal(deletedOnlyWordRow ?? null, null); assert.ok(sharedKanjiRow); assert.equal(sharedKanjiRow.frequency, 1); - assert.equal(sharedKanjiRow.first_seen, keptTs); - assert.equal(sharedKanjiRow.last_seen, keptTs); + assert.equal(sharedKanjiRow.first_seen, Math.floor(keptTs / 1000)); + assert.equal(sharedKanjiRow.last_seen, Math.floor(keptTs / 1000)); assert.equal(deletedOnlyKanjiRow ?? null, null); } finally { db.close(); diff --git a/src/core/services/immersion-tracker/query-library.ts b/src/core/services/immersion-tracker/query-library.ts index fadaa30..a13daaf 100644 --- a/src/core/services/immersion-tracker/query-library.ts +++ b/src/core/services/immersion-tracker/query-library.ts @@ -156,7 +156,7 @@ export function getAnimeEpisodes(db: DatabaseSync, animeId: number): AnimeEpisod COALESCE(SUM(COALESCE(asm.yomitanLookupCount, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount, MAX(s.started_at_ms) AS lastWatchedMs FROM imm_videos v - JOIN imm_sessions s ON s.video_id = v.video_id + LEFT JOIN imm_sessions s ON s.video_id = v.video_id LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id WHERE v.anime_id = ? GROUP BY v.video_id diff --git a/src/core/services/immersion-tracker/query-maintenance.ts b/src/core/services/immersion-tracker/query-maintenance.ts index d23a730..0b2b229 100644 --- a/src/core/services/immersion-tracker/query-maintenance.ts +++ b/src/core/services/immersion-tracker/query-maintenance.ts @@ -352,15 +352,16 @@ export function upsertCoverArt( const sharedCoverBlobHash = findSharedCoverBlobHash(db, videoId, art.anilistId, art.coverUrl); const fetchedAtMs = toDbMs(nowMs()); const coverBlob = normalizeCoverBlobBytes(art.coverBlob); - let coverBlobHash = sharedCoverBlobHash ?? null; - if (!coverBlobHash && coverBlob && coverBlob.length > 0) { - coverBlobHash = createHash('sha256').update(coverBlob).digest('hex'); - } + const computedCoverBlobHash = + coverBlob && coverBlob.length > 0 + ? createHash('sha256').update(coverBlob).digest('hex') + : null; + let coverBlobHash = computedCoverBlobHash ?? sharedCoverBlobHash ?? null; if (!coverBlobHash && (!coverBlob || coverBlob.length === 0)) { coverBlobHash = existing?.coverBlobHash ?? null; } - if (coverBlobHash && coverBlob && coverBlob.length > 0 && !sharedCoverBlobHash) { + if (computedCoverBlobHash && coverBlob && coverBlob.length > 0) { db.prepare( ` INSERT INTO imm_cover_art_blobs (blob_hash, cover_blob, CREATED_DATE, LAST_UPDATE_DATE) @@ -368,7 +369,7 @@ export function upsertCoverArt( ON CONFLICT(blob_hash) DO UPDATE SET LAST_UPDATE_DATE = excluded.LAST_UPDATE_DATE `, - ).run(coverBlobHash, coverBlob, fetchedAtMs, fetchedAtMs); + ).run(computedCoverBlobHash, coverBlob, fetchedAtMs, fetchedAtMs); } db.prepare( diff --git a/src/core/services/immersion-tracker/query-sessions.ts b/src/core/services/immersion-tracker/query-sessions.ts index 8b96b5c..6199570 100644 --- a/src/core/services/immersion-tracker/query-sessions.ts +++ b/src/core/services/immersion-tracker/query-sessions.ts @@ -204,7 +204,7 @@ export function getQueryHints(db: DatabaseSync): { const now = new Date(); const todayLocal = Math.floor( - new Date(now.getFullYear(), now.getMonth(), now.getDate()).getTime() / 86_400_000, + (now.getTime() / 1000 - now.getTimezoneOffset() * 60) / 86_400, ); const episodesToday = @@ -333,9 +333,15 @@ export function getMonthlyRollups(db: DatabaseSync, limit = 24): ImmersionSessio total_lines_seen AS totalLinesSeen, total_tokens_seen AS totalTokensSeen, total_cards AS totalCards, - 0 AS cardsPerHour, - 0 AS tokensPerMin, - 0 AS lookupHitRate + CASE + WHEN total_active_min > 0 THEN (total_cards * 60.0) / total_active_min + ELSE NULL + END AS cardsPerHour, + CASE + WHEN total_active_min > 0 THEN total_tokens_seen * 1.0 / total_active_min + ELSE NULL + END AS tokensPerMin, + NULL AS lookupHitRate FROM imm_monthly_rollups WHERE rollup_month IN (SELECT rollup_month FROM recent_months) ORDER BY rollup_month DESC, video_id DESC diff --git a/src/core/services/immersion-tracker/query-shared.ts b/src/core/services/immersion-tracker/query-shared.ts index 448578f..6e5dc4a 100644 --- a/src/core/services/immersion-tracker/query-shared.ts +++ b/src/core/services/immersion-tracker/query-shared.ts @@ -197,7 +197,12 @@ function refreshWordAggregates(db: DatabaseSync, wordIds: number[]): void { deleteStmt.run(row.wordId); continue; } - updateStmt.run(row.frequency, row.firstSeen, row.lastSeen, row.wordId); + updateStmt.run( + row.frequency, + Math.floor(row.firstSeen / 1000), + Math.floor(row.lastSeen / 1000), + row.wordId, + ); } } @@ -241,7 +246,12 @@ function refreshKanjiAggregates(db: DatabaseSync, kanjiIds: number[]): void { deleteStmt.run(row.kanjiId); continue; } - updateStmt.run(row.frequency, row.firstSeen, row.lastSeen, row.kanjiId); + updateStmt.run( + row.frequency, + Math.floor(row.firstSeen / 1000), + Math.floor(row.lastSeen / 1000), + row.kanjiId, + ); } } diff --git a/src/core/services/immersion-tracker/query-trends.ts b/src/core/services/immersion-tracker/query-trends.ts index 2aac8b1..7f8946f 100644 --- a/src/core/services/immersion-tracker/query-trends.ts +++ b/src/core/services/immersion-tracker/query-trends.ts @@ -112,6 +112,16 @@ function makeTrendLabel(value: number): string { }); } +function getLocalEpochDay(timestampMs: number): number { + const date = new Date(timestampMs); + return Math.floor((timestampMs - date.getTimezoneOffset() * 60_000) / 86_400_000); +} + +function getLocalDateForEpochDay(epochDay: number): Date { + const utcDate = new Date(epochDay * 86_400_000); + return new Date(utcDate.getTime() + utcDate.getTimezoneOffset() * 60_000); +} + function getTrendSessionWordCount(session: Pick): number { return session.tokensSeen; } @@ -188,7 +198,7 @@ function buildWatchTimeByHour(sessions: TrendSessionMetricRow[]): TrendChartPoin } function dayLabel(epochDay: number): string { - return new Date(epochDay * 86_400_000).toLocaleDateString(undefined, { + return getLocalDateForEpochDay(epochDay).toLocaleDateString(undefined, { month: 'short', day: 'numeric', }); @@ -200,7 +210,7 @@ function buildSessionSeriesByDay( ): TrendChartPoint[] { const byDay = new Map(); for (const session of sessions) { - const epochDay = Math.floor(session.startedAtMs / 86_400_000); + const epochDay = getLocalEpochDay(session.startedAtMs); byDay.set(epochDay, (byDay.get(epochDay) ?? 0) + getValue(session)); } return Array.from(byDay.entries()) @@ -213,7 +223,7 @@ function buildLookupsPerHundredWords(sessions: TrendSessionMetricRow[]): TrendCh const wordsByDay = new Map(); for (const session of sessions) { - const epochDay = Math.floor(session.startedAtMs / 86_400_000); + const epochDay = getLocalEpochDay(session.startedAtMs); lookupsByDay.set(epochDay, (lookupsByDay.get(epochDay) ?? 0) + session.yomitanLookupCount); wordsByDay.set(epochDay, (wordsByDay.get(epochDay) ?? 0) + getTrendSessionWordCount(session)); } @@ -237,7 +247,7 @@ function buildPerAnimeFromSessions( for (const session of sessions) { const animeTitle = resolveTrendAnimeTitle(session); - const epochDay = Math.floor(session.startedAtMs / 86_400_000); + const epochDay = getLocalEpochDay(session.startedAtMs); const dayMap = byAnime.get(animeTitle) ?? new Map(); dayMap.set(epochDay, (dayMap.get(epochDay) ?? 0) + getValue(session)); byAnime.set(animeTitle, dayMap); @@ -258,7 +268,7 @@ function buildLookupsPerHundredPerAnime(sessions: TrendSessionMetricRow[]): Tren for (const session of sessions) { const animeTitle = resolveTrendAnimeTitle(session); - const epochDay = Math.floor(session.startedAtMs / 86_400_000); + const epochDay = getLocalEpochDay(session.startedAtMs); const lookupMap = lookups.get(animeTitle) ?? new Map(); lookupMap.set(epochDay, (lookupMap.get(epochDay) ?? 0) + session.yomitanLookupCount); @@ -462,7 +472,7 @@ function buildNewWordsPerDay(db: DatabaseSync, cutoffMs: number | null): TrendCh const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?'; const prepared = db.prepare(` SELECT - CAST(first_seen / 86400 AS INTEGER) AS epochDay, + CAST(julianday(first_seen, 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) AS epochDay, COUNT(*) AS wordCount FROM imm_words WHERE first_seen IS NOT NULL diff --git a/src/core/services/immersion-tracker/storage-session.test.ts b/src/core/services/immersion-tracker/storage-session.test.ts index 184557a..d00e09b 100644 --- a/src/core/services/immersion-tracker/storage-session.test.ts +++ b/src/core/services/immersion-tracker/storage-session.test.ts @@ -1078,6 +1078,56 @@ test('executeQueuedWrite inserts event and telemetry rows', () => { } }); +test('executeQueuedWrite rejects partial telemetry writes instead of zero-filling', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const stmts = createTrackerPreparedStatements(db); + const videoId = getOrCreateVideoRecord(db, 'local:/tmp/partial-telemetry.mkv', { + canonicalTitle: 'Partial Telemetry', + sourcePath: '/tmp/partial-telemetry.mkv', + sourceUrl: null, + sourceType: SOURCE_TYPE_LOCAL, + }); + const { sessionId } = startSessionRecord(db, videoId, 5_000); + + assert.throws( + () => + executeQueuedWrite( + { + kind: 'telemetry', + sessionId, + sampleMs: 6_000, + totalWatchedMs: 1_000, + activeWatchedMs: 900, + linesSeen: 3, + cardsMined: 1, + lookupCount: 2, + lookupHits: 1, + yomitanLookupCount: 0, + pauseCount: 1, + pauseMs: 50, + seekForwardCount: 0, + seekBackwardCount: 0, + mediaBufferEvents: 0, + }, + stmts, + ), + /Incomplete telemetry write/, + ); + + const telemetryCount = db + .prepare('SELECT COUNT(*) AS total FROM imm_session_telemetry WHERE session_id = ?') + .get(sessionId) as { total: number }; + assert.equal(telemetryCount.total, 0); + } finally { + db.close(); + cleanupDbPath(dbPath); + } +}); + test('executeQueuedWrite inserts and upserts word and kanji rows', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); diff --git a/src/core/services/immersion-tracker/storage.ts b/src/core/services/immersion-tracker/storage.ts index d8f2f69..3fa4174 100644 --- a/src/core/services/immersion-tracker/storage.ts +++ b/src/core/services/immersion-tracker/storage.ts @@ -1406,27 +1406,46 @@ function incrementKanjiAggregate( export function executeQueuedWrite(write: QueuedWrite, stmts: TrackerPreparedStatements): void { const currentMs = toDbMs(nowMs()); if (write.kind === 'telemetry') { + if ( + write.totalWatchedMs === undefined || + write.activeWatchedMs === undefined || + write.linesSeen === undefined || + write.tokensSeen === undefined || + write.cardsMined === undefined || + write.lookupCount === undefined || + write.lookupHits === undefined || + write.yomitanLookupCount === undefined || + write.pauseCount === undefined || + write.pauseMs === undefined || + write.seekForwardCount === undefined || + write.seekBackwardCount === undefined || + write.mediaBufferEvents === undefined + ) { + throw new Error('Incomplete telemetry write'); + } const telemetrySampleMs = toDbMs(write.sampleMs ?? Number(currentMs)); stmts.telemetryInsertStmt.run( write.sessionId, telemetrySampleMs, - write.totalWatchedMs ?? 0, - write.activeWatchedMs ?? 0, - write.linesSeen ?? 0, - write.tokensSeen ?? 0, - write.cardsMined ?? 0, - write.lookupCount ?? 0, - write.lookupHits ?? 0, - write.yomitanLookupCount ?? 0, - write.pauseCount ?? 0, - write.pauseMs ?? 0, - write.seekForwardCount ?? 0, - write.seekBackwardCount ?? 0, - write.mediaBufferEvents ?? 0, + write.totalWatchedMs, + write.activeWatchedMs, + write.linesSeen, + write.tokensSeen, + write.cardsMined, + write.lookupCount, + write.lookupHits, + write.yomitanLookupCount, + write.pauseCount, + write.pauseMs, + write.seekForwardCount, + write.seekBackwardCount, + write.mediaBufferEvents, currentMs, currentMs, ); - stmts.sessionCheckpointStmt.run(write.lastMediaMs ?? null, currentMs, write.sessionId); + if (write.lastMediaMs !== undefined) { + stmts.sessionCheckpointStmt.run(write.lastMediaMs ?? null, currentMs, write.sessionId); + } return; } if (write.kind === 'word') { diff --git a/src/core/services/immersion-tracker/time.test.ts b/src/core/services/immersion-tracker/time.test.ts new file mode 100644 index 0000000..08c5f54 --- /dev/null +++ b/src/core/services/immersion-tracker/time.test.ts @@ -0,0 +1,7 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { nowMs } from './time.js'; + +test('nowMs returns wall-clock epoch milliseconds', () => { + assert.ok(nowMs() > 1_600_000_000_000); +}); diff --git a/src/core/services/immersion-tracker/time.ts b/src/core/services/immersion-tracker/time.ts index 9870e1b..8ea2081 100644 --- a/src/core/services/immersion-tracker/time.ts +++ b/src/core/services/immersion-tracker/time.ts @@ -1,10 +1,8 @@ -const SQLITE_SAFE_EPOCH_BASE_MS = 2_000_000_000; - export function nowMs(): number { const perf = globalThis.performance; - if (perf) { - return SQLITE_SAFE_EPOCH_BASE_MS + Math.floor(perf.now()); + if (perf && Number.isFinite(perf.timeOrigin)) { + return Math.floor(perf.timeOrigin + perf.now()); } - return SQLITE_SAFE_EPOCH_BASE_MS; + return Date.now(); } diff --git a/src/main/character-dictionary-runtime/zip.test.ts b/src/main/character-dictionary-runtime/zip.test.ts new file mode 100644 index 0000000..53419cb --- /dev/null +++ b/src/main/character-dictionary-runtime/zip.test.ts @@ -0,0 +1,98 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { buildDictionaryZip } from './zip'; +import type { CharacterDictionaryTermEntry } from './types'; + +function makeTempDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-character-zip-')); +} + +function cleanupDir(dirPath: string): void { + fs.rmSync(dirPath, { recursive: true, force: true }); +} + +function readStoredZipEntries(zipPath: string): Map { + const archive = fs.readFileSync(zipPath); + const entries = new Map(); + let cursor = 0; + + while (cursor + 4 <= archive.length) { + const signature = archive.readUInt32LE(cursor); + if (signature === 0x02014b50 || signature === 0x06054b50) { + break; + } + assert.equal(signature, 0x04034b50, `unexpected local file header at offset ${cursor}`); + + const compressedSize = archive.readUInt32LE(cursor + 18); + const fileNameLength = archive.readUInt16LE(cursor + 26); + const extraLength = archive.readUInt16LE(cursor + 28); + const fileNameStart = cursor + 30; + const dataStart = fileNameStart + fileNameLength + extraLength; + const fileName = archive.subarray(fileNameStart, fileNameStart + fileNameLength).toString( + 'utf8', + ); + const data = archive.subarray(dataStart, dataStart + compressedSize); + entries.set(fileName, Buffer.from(data)); + cursor = dataStart + compressedSize; + } + + return entries; +} + +test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', () => { + const tempDir = makeTempDir(); + const outputPath = path.join(tempDir, 'dictionary.zip'); + const termEntries: CharacterDictionaryTermEntry[] = [ + ['アルファ', 'あるふぁ', '', '', 0, ['Alpha entry'], 0, 'name'], + ]; + const originalBufferConcat = Buffer.concat; + + try { + Buffer.concat = ((...args: Parameters) => { + throw new Error(`buildDictionaryZip should not Buffer.concat the full archive (${args[0].length} chunks)`); + }) as typeof Buffer.concat; + + const result = buildDictionaryZip( + outputPath, + 'Dictionary Title', + 'Dictionary Description', + '2026-03-27', + termEntries, + [{ path: 'images/alpha.bin', dataBase64: Buffer.from([1, 2, 3]).toString('base64') }], + ); + + assert.equal(result.zipPath, outputPath); + assert.equal(result.entryCount, 1); + + const entries = readStoredZipEntries(outputPath); + assert.deepEqual([...entries.keys()].sort(), [ + 'images/alpha.bin', + 'index.json', + 'tag_bank_1.json', + 'term_bank_1.json', + ]); + + const indexJson = JSON.parse(entries.get('index.json')!.toString('utf8')) as { + title: string; + description: string; + revision: string; + format: number; + }; + assert.equal(indexJson.title, 'Dictionary Title'); + assert.equal(indexJson.description, 'Dictionary Description'); + assert.equal(indexJson.revision, '2026-03-27'); + assert.equal(indexJson.format, 3); + + const termBank = JSON.parse(entries.get('term_bank_1.json')!.toString('utf8')) as + CharacterDictionaryTermEntry[]; + assert.equal(termBank.length, 1); + assert.equal(termBank[0]?.[0], 'アルファ'); + assert.deepEqual(entries.get('images/alpha.bin'), Buffer.from([1, 2, 3])); + } finally { + Buffer.concat = originalBufferConcat; + cleanupDir(tempDir); + } +}); diff --git a/src/main/character-dictionary-runtime/zip.ts b/src/main/character-dictionary-runtime/zip.ts index 5b31ce8..85bf34b 100644 --- a/src/main/character-dictionary-runtime/zip.ts +++ b/src/main/character-dictionary-runtime/zip.ts @@ -5,8 +5,8 @@ import type { CharacterDictionarySnapshotImage, CharacterDictionaryTermEntry } f type ZipEntry = { name: string; - data: Buffer; crc32: number; + size: number; localHeaderOffset: number; }; @@ -67,97 +67,78 @@ function crc32(data: Buffer): number { return (crc ^ 0xffffffff) >>> 0; } -function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer { - const chunks: Buffer[] = []; - const entries: ZipEntry[] = []; - let offset = 0; +function createLocalFileHeader(fileName: Buffer, fileCrc32: number, fileSize: number): Buffer { + const local = Buffer.alloc(30 + fileName.length); + let cursor = 0; + writeUint32LE(local, 0x04034b50, cursor); + cursor += 4; + local.writeUInt16LE(20, cursor); + cursor += 2; + local.writeUInt16LE(0, cursor); + cursor += 2; + local.writeUInt16LE(0, cursor); + cursor += 2; + local.writeUInt16LE(0, cursor); + cursor += 2; + local.writeUInt16LE(0, cursor); + cursor += 2; + writeUint32LE(local, fileCrc32, cursor); + cursor += 4; + writeUint32LE(local, fileSize, cursor); + cursor += 4; + writeUint32LE(local, fileSize, cursor); + cursor += 4; + local.writeUInt16LE(fileName.length, cursor); + cursor += 2; + local.writeUInt16LE(0, cursor); + cursor += 2; + fileName.copy(local, cursor); + return local; +} - for (const file of files) { - const fileName = Buffer.from(file.name, 'utf8'); - const fileData = file.data; - const fileCrc32 = crc32(fileData); - const local = Buffer.alloc(30 + fileName.length); - let cursor = 0; - writeUint32LE(local, 0x04034b50, cursor); - cursor += 4; - local.writeUInt16LE(20, cursor); - cursor += 2; - local.writeUInt16LE(0, cursor); - cursor += 2; - local.writeUInt16LE(0, cursor); - cursor += 2; - local.writeUInt16LE(0, cursor); - cursor += 2; - local.writeUInt16LE(0, cursor); - cursor += 2; - writeUint32LE(local, fileCrc32, cursor); - cursor += 4; - writeUint32LE(local, fileData.length, cursor); - cursor += 4; - writeUint32LE(local, fileData.length, cursor); - cursor += 4; - local.writeUInt16LE(fileName.length, cursor); - cursor += 2; - local.writeUInt16LE(0, cursor); - cursor += 2; - fileName.copy(local, cursor); +function createCentralDirectoryHeader(entry: ZipEntry): Buffer { + const fileName = Buffer.from(entry.name, 'utf8'); + const central = Buffer.alloc(46 + fileName.length); + let cursor = 0; + writeUint32LE(central, 0x02014b50, cursor); + cursor += 4; + central.writeUInt16LE(20, cursor); + cursor += 2; + central.writeUInt16LE(20, cursor); + cursor += 2; + central.writeUInt16LE(0, cursor); + cursor += 2; + central.writeUInt16LE(0, cursor); + cursor += 2; + central.writeUInt16LE(0, cursor); + cursor += 2; + central.writeUInt16LE(0, cursor); + cursor += 2; + writeUint32LE(central, entry.crc32, cursor); + cursor += 4; + writeUint32LE(central, entry.size, cursor); + cursor += 4; + writeUint32LE(central, entry.size, cursor); + cursor += 4; + central.writeUInt16LE(fileName.length, cursor); + cursor += 2; + central.writeUInt16LE(0, cursor); + cursor += 2; + central.writeUInt16LE(0, cursor); + cursor += 2; + central.writeUInt16LE(0, cursor); + cursor += 2; + central.writeUInt16LE(0, cursor); + cursor += 2; + writeUint32LE(central, 0, cursor); + cursor += 4; + writeUint32LE(central, entry.localHeaderOffset, cursor); + cursor += 4; + fileName.copy(central, cursor); + return central; +} - chunks.push(local, fileData); - entries.push({ - name: file.name, - data: fileData, - crc32: fileCrc32, - localHeaderOffset: offset, - }); - offset += local.length + fileData.length; - } - - const centralStart = offset; - const centralChunks: Buffer[] = []; - for (const entry of entries) { - const fileName = Buffer.from(entry.name, 'utf8'); - const central = Buffer.alloc(46 + fileName.length); - let cursor = 0; - writeUint32LE(central, 0x02014b50, cursor); - cursor += 4; - central.writeUInt16LE(20, cursor); - cursor += 2; - central.writeUInt16LE(20, cursor); - cursor += 2; - central.writeUInt16LE(0, cursor); - cursor += 2; - central.writeUInt16LE(0, cursor); - cursor += 2; - central.writeUInt16LE(0, cursor); - cursor += 2; - central.writeUInt16LE(0, cursor); - cursor += 2; - writeUint32LE(central, entry.crc32, cursor); - cursor += 4; - writeUint32LE(central, entry.data.length, cursor); - cursor += 4; - writeUint32LE(central, entry.data.length, cursor); - cursor += 4; - central.writeUInt16LE(fileName.length, cursor); - cursor += 2; - central.writeUInt16LE(0, cursor); - cursor += 2; - central.writeUInt16LE(0, cursor); - cursor += 2; - central.writeUInt16LE(0, cursor); - cursor += 2; - central.writeUInt16LE(0, cursor); - cursor += 2; - writeUint32LE(central, 0, cursor); - cursor += 4; - writeUint32LE(central, entry.localHeaderOffset, cursor); - cursor += 4; - fileName.copy(central, cursor); - centralChunks.push(central); - offset += central.length; - } - - const centralSize = offset - centralStart; +function createEndOfCentralDirectory(entriesLength: number, centralSize: number, centralStart: number): Buffer { const end = Buffer.alloc(22); let cursor = 0; writeUint32LE(end, 0x06054b50, cursor); @@ -166,17 +147,63 @@ function createStoredZip(files: Array<{ name: string; data: Buffer }>): Buffer { cursor += 2; end.writeUInt16LE(0, cursor); cursor += 2; - end.writeUInt16LE(entries.length, cursor); + end.writeUInt16LE(entriesLength, cursor); cursor += 2; - end.writeUInt16LE(entries.length, cursor); + end.writeUInt16LE(entriesLength, cursor); cursor += 2; writeUint32LE(end, centralSize, cursor); cursor += 4; writeUint32LE(end, centralStart, cursor); cursor += 4; end.writeUInt16LE(0, cursor); + return end; +} - return Buffer.concat([...chunks, ...centralChunks, end]); +function writeBuffer(fd: number, buffer: Buffer): void { + let written = 0; + while (written < buffer.length) { + written += fs.writeSync(fd, buffer, written, buffer.length - written); + } +} + +function writeStoredZip(outputPath: string, files: Iterable<{ name: string; data: Buffer }>): void { + const entries: ZipEntry[] = []; + let offset = 0; + const fd = fs.openSync(outputPath, 'w'); + + try { + for (const file of files) { + const fileName = Buffer.from(file.name, 'utf8'); + const fileSize = file.data.length; + const fileCrc32 = crc32(file.data); + const localHeader = createLocalFileHeader(fileName, fileCrc32, fileSize); + writeBuffer(fd, localHeader); + writeBuffer(fd, file.data); + entries.push({ + name: file.name, + crc32: fileCrc32, + size: fileSize, + localHeaderOffset: offset, + }); + offset += localHeader.length + fileSize; + } + + const centralStart = offset; + for (const entry of entries) { + const centralHeader = createCentralDirectoryHeader(entry); + writeBuffer(fd, centralHeader); + offset += centralHeader.length; + } + + const centralSize = offset - centralStart; + writeBuffer(fd, createEndOfCentralDirectory(entries.length, centralSize, centralStart)); + } catch (error) { + fs.closeSync(fd); + fs.rmSync(outputPath, { force: true }); + throw error; + } + + fs.closeSync(fd); } export function buildDictionaryZip( @@ -187,36 +214,37 @@ export function buildDictionaryZip( termEntries: CharacterDictionaryTermEntry[], images: CharacterDictionarySnapshotImage[], ): { zipPath: string; entryCount: number } { - const zipFiles: Array<{ name: string; data: Buffer }> = [ - { + ensureDir(path.dirname(outputPath)); + + function* zipFiles(): Iterable<{ name: string; data: Buffer }> { + yield { name: 'index.json', data: Buffer.from( JSON.stringify(createIndex(dictionaryTitle, description, revision), null, 2), 'utf8', ), - }, - { + }; + yield { name: 'tag_bank_1.json', data: Buffer.from(JSON.stringify(createTagBank()), 'utf8'), - }, - ]; + }; - for (const image of images) { - zipFiles.push({ - name: image.path, - data: Buffer.from(image.dataBase64, 'base64'), - }); + for (const image of images) { + yield { + name: image.path, + data: Buffer.from(image.dataBase64, 'base64'), + }; + } + + const entriesPerBank = 10_000; + for (let i = 0; i < termEntries.length; i += entriesPerBank) { + yield { + name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`, + data: Buffer.from(JSON.stringify(termEntries.slice(i, i + entriesPerBank)), 'utf8'), + }; + } } - const entriesPerBank = 10_000; - for (let i = 0; i < termEntries.length; i += entriesPerBank) { - zipFiles.push({ - name: `term_bank_${Math.floor(i / entriesPerBank) + 1}.json`, - data: Buffer.from(JSON.stringify(termEntries.slice(i, i + entriesPerBank)), 'utf8'), - }); - } - - ensureDir(path.dirname(outputPath)); - fs.writeFileSync(outputPath, createStoredZip(zipFiles)); + writeStoredZip(outputPath, zipFiles()); return { zipPath: outputPath, entryCount: termEntries.length }; }