From 06745ff63a17b2436e2e58960b768f1ada1322bf Mon Sep 17 00:00:00 2001 From: sudacode Date: Sat, 14 Mar 2026 22:15:37 -0700 Subject: [PATCH] feat: add AniList rate limiter and remaining backlog tasks --- ...nt-immersion-stats-dashboard-and-config.md | 45 +++++++++++ ...evel-immersion-metadata-and-link-videos.md | 79 +++++++++++++++++++ ...etric-from-Vocabulary-tab-summary-cards.md | 40 ++++++++++ src/core/services/anilist/rate-limiter.ts | 72 +++++++++++++++++ 4 files changed, 236 insertions(+) create mode 100644 backlog/tasks/task-168 - Document-immersion-stats-dashboard-and-config.md create mode 100644 backlog/tasks/task-169 - Add-anime-level-immersion-metadata-and-link-videos.md create mode 100644 backlog/tasks/task-173 - Remove-Avg-Frequency-metric-from-Vocabulary-tab-summary-cards.md create mode 100644 src/core/services/anilist/rate-limiter.ts diff --git a/backlog/tasks/task-168 - Document-immersion-stats-dashboard-and-config.md b/backlog/tasks/task-168 - Document-immersion-stats-dashboard-and-config.md new file mode 100644 index 0000000..236fd9b --- /dev/null +++ b/backlog/tasks/task-168 - Document-immersion-stats-dashboard-and-config.md @@ -0,0 +1,45 @@ +--- +id: TASK-168 +title: Document immersion stats dashboard and config +status: Done +assignee: + - codex +created_date: '2026-03-12 22:53' +updated_date: '2026-03-12 22:53' +labels: + - docs + - immersion +dependencies: [] +priority: medium +--- + +## Description + + + +Refresh user-facing docs for the new immersion stats dashboard so README, docs-site pages, changelog notes, and generated config examples describe how to access the dashboard and which `stats.*` settings control it. + + + +## Acceptance Criteria + + + +- [x] #1 README mentions the new stats surface in product-facing feature/docs copy. +- [x] #2 Docs explain how to access the stats dashboard in-app and via localhost, and document the `stats` config block. +- [x] #3 Changelog/release-note input includes the new stats dashboard. +- [x] #4 Generated config examples include the new `stats` section. + + + +## Final Summary + + + +Updated README and the docs-site immersion/config/mining/shortcut/homepage copy to describe the new stats dashboard, including the overlay toggle (`stats.toggleKey`, default `Backquote`) and the localhost browser UI (`http://127.0.0.1:5175` by default). + +Added a changelog fragment for the stats dashboard release notes and extended the config template sections so regenerated `config.example.jsonc` artifacts now include the `stats` block. + +Verified with `bun run test:config`, `bun run generate:config-example`, `bun run docs:test`, `bun run docs:build`, and `bun run changelog:lint`. + + diff --git a/backlog/tasks/task-169 - Add-anime-level-immersion-metadata-and-link-videos.md b/backlog/tasks/task-169 - Add-anime-level-immersion-metadata-and-link-videos.md new file mode 100644 index 0000000..80e4506 --- /dev/null +++ b/backlog/tasks/task-169 - Add-anime-level-immersion-metadata-and-link-videos.md @@ -0,0 +1,79 @@ +--- +id: TASK-169 +title: Add anime-level immersion metadata and link videos +status: Done +assignee: + - codex +created_date: '2026-03-13 19:34' +updated_date: '2026-03-13 21:46' +labels: + - immersion + - stats + - database + - anilist +dependencies: [] +references: + - /home/sudacode/projects/japanese/SubMiner/docs/plans/2026-03-13-immersion-anime-metadata-design.md + - /home/sudacode/projects/japanese/SubMiner/docs/plans/2026-03-13-immersion-anime-metadata.md +--- + +## Description + + +Add first-class anime metadata to the immersion tracker so stats can group sessions and videos by anime, season, and episode instead of relying only on per-video canonical titles. The new model should deduplicate anime-level metadata across rewatches and multiple files, use guessit-first filename parsing with built-in parser fallback, and create provisional anime rows even when AniList lookup fails. + + +## Acceptance Criteria + + +- [x] #1 The immersion schema includes a new anime-level table plus additive video linkage/parsed metadata fields needed for anime, season, and episode stats. +- [x] #2 Media ingest creates or reuses anime rows, stores parsed season/episode metadata on videos, and upgrades provisional anime rows when AniList data becomes available. +- [x] #3 Query surfaces expose anime-level aggregation suitable for library/detail/episode stats without breaking current video/session queries. +- [x] #4 Focused regression coverage exists for schema/storage/query/service behavior, including provisional anime rows and guessit-first parser fallback behavior. +- [x] #5 Verification covers the SQLite immersion lane and any broader lanes required by the touched runtime/query files. + + +## Implementation Plan + + +1. Add red tests for the new schema shape in the SQLite immersion lane before changing storage code. +2. Implement `imm_anime` plus additive `imm_videos` metadata fields and focused storage helpers for provisional anime creation and AniList upgrade. +3. Add a guessit-first parser helper with built-in fallback and wire media ingest to persist anime/video metadata during `handleMediaChange(...)`. +4. Add anime-level query surfaces for library/detail/episode aggregation and expose them only where needed. +5. Run focused SQLite verification first, then broader verification lanes only if touched runtime/API files require them. + + +## Implementation Notes + + +2026-03-13: Design approved in-thread. Initial scope excluded migration/backfill work, but implementation was corrected in-thread to add a legacy DB migration/backfill path based on filename parsing. +2026-03-13: Detailed implementation plan written at `docs/plans/2026-03-13-immersion-anime-metadata.md`. +2026-03-13: Task 6 export/API work was intentionally skipped because no current stats API/UI consumer needs the anime query surface yet, and widening the contract would have touched unrelated dirty stats files. +2026-03-13: Verification commands run: + - `bun test src/core/services/immersion-tracker/storage-session.test.ts` + - `bun test src/core/services/immersion-tracker/metadata.test.ts` + - `bun test src/core/services/immersion-tracker-service.test.ts` + - `bun test src/core/services/immersion-tracker/__tests__/query.test.ts` + - `bun run test:immersion:sqlite:src` + - `bash .agents/skills/subminer-change-verification/scripts/classify_subminer_diff.sh src/core/services/immersion-tracker/storage.ts src/core/services/immersion-tracker/storage-session.test.ts src/core/services/immersion-tracker/metadata.ts src/core/services/immersion-tracker/metadata.test.ts src/core/services/immersion-tracker/query.ts src/core/services/immersion-tracker/types.ts src/core/services/immersion-tracker/__tests__/query.test.ts src/core/services/immersion-tracker-service.ts src/core/services/immersion-tracker-service.test.ts` + - `bash .agents/skills/subminer-change-verification/scripts/verify_subminer_change.sh --lane core src/core/services/immersion-tracker/storage.ts src/core/services/immersion-tracker/storage-session.test.ts src/core/services/immersion-tracker/metadata.ts src/core/services/immersion-tracker/metadata.test.ts src/core/services/immersion-tracker/query.ts src/core/services/immersion-tracker/types.ts src/core/services/immersion-tracker/__tests__/query.test.ts src/core/services/immersion-tracker-service.ts src/core/services/immersion-tracker-service.test.ts` +2026-03-13: Verification results: + - `bun run test:immersion:sqlite:src`: passed + - verifier lane selection: `core` + - verifier result: passed (`bun run typecheck`, `bun run test:fast`) + - verifier artifacts: `.tmp/skill-verification/subminer-verify-20260313-214533-Ciw3L0/` + + +## Final Summary + + + +Added `imm_anime`, additive `imm_videos` anime/parser metadata fields, and a legacy migration/backfill path that links existing videos to provisional anime rows from parsed filenames. + +Added focused storage helpers for normalized anime identity reuse, later AniList upgrades, and per-video season/episode/parser metadata linking. Media ingest now parses and links anime metadata during `handleMediaChange(...)`. + +Added anime-level query surfaces for library/detail/episode aggregation and regression coverage for schema, migration, storage, parser fallback, service ingest wiring, and anime stats queries. + +Verified with the focused SQLite lane plus verifier-selected `core` coverage (`typecheck`, `test:fast`). No stats API/UI export was added yet because there is no current consumer for the new anime query surface. + + diff --git a/backlog/tasks/task-173 - Remove-Avg-Frequency-metric-from-Vocabulary-tab-summary-cards.md b/backlog/tasks/task-173 - Remove-Avg-Frequency-metric-from-Vocabulary-tab-summary-cards.md new file mode 100644 index 0000000..bc802a2 --- /dev/null +++ b/backlog/tasks/task-173 - Remove-Avg-Frequency-metric-from-Vocabulary-tab-summary-cards.md @@ -0,0 +1,40 @@ +--- +id: TASK-173 +title: Remove Avg Frequency metric from Vocabulary tab summary cards +status: Done +assignee: [] +created_date: '2026-03-15 00:13' +updated_date: '2026-03-15 00:15' +labels: + - stats + - ui +dependencies: [] +priority: low +--- + +## Description + + +User requested removing the Avg Frequency card/metric because it is not useful. Remove the UI card and stop computing/storing the summary field in dashboard summary shaping code. + + +## Acceptance Criteria + +- [x] #1 Vocabulary tab no longer renders an "Avg Frequency" stat card. +- [x] #2 Vocabulary summary model no longer exposes or computes averageFrequency. +- [x] #3 Typecheck/tests covering dashboard summary and vocabulary tab pass. + + +## Final Summary + + +Removed the Vocabulary tab "Avg Frequency" card and deleted the corresponding `averageFrequency` field from `VocabularySummary` and `buildVocabularySummary`. + +Verification run: +- `bun test stats/src/lib/dashboard-data.test.ts` +- `bun run typecheck` +- `bun run test:fast` +- `bun run build` +- `bun run test:env` +- `bun run test:smoke:dist` + diff --git a/src/core/services/anilist/rate-limiter.ts b/src/core/services/anilist/rate-limiter.ts new file mode 100644 index 0000000..5753494 --- /dev/null +++ b/src/core/services/anilist/rate-limiter.ts @@ -0,0 +1,72 @@ +const DEFAULT_MAX_PER_MINUTE = 20; +const WINDOW_MS = 60_000; +const SAFETY_REMAINING_THRESHOLD = 5; + +export interface AnilistRateLimiter { + acquire(): Promise; + recordResponse(headers: Headers): void; +} + +export function createAnilistRateLimiter( + maxPerMinute = DEFAULT_MAX_PER_MINUTE, +): AnilistRateLimiter { + const timestamps: number[] = []; + let pauseUntilMs = 0; + + function pruneOld(now: number): void { + const cutoff = now - WINDOW_MS; + while (timestamps.length > 0 && timestamps[0]! < cutoff) { + timestamps.shift(); + } + } + + return { + async acquire(): Promise { + const now = Date.now(); + + if (now < pauseUntilMs) { + const waitMs = pauseUntilMs - now; + await new Promise((resolve) => setTimeout(resolve, waitMs)); + } + + pruneOld(Date.now()); + + if (timestamps.length >= maxPerMinute) { + const oldest = timestamps[0]!; + const waitMs = oldest + WINDOW_MS - Date.now() + 100; + if (waitMs > 0) { + await new Promise((resolve) => setTimeout(resolve, waitMs)); + } + pruneOld(Date.now()); + } + + timestamps.push(Date.now()); + }, + + recordResponse(headers: Headers): void { + const remaining = headers.get('x-ratelimit-remaining'); + if (remaining !== null) { + const n = parseInt(remaining, 10); + if (Number.isFinite(n) && n < SAFETY_REMAINING_THRESHOLD) { + const reset = headers.get('x-ratelimit-reset'); + if (reset) { + const resetMs = parseInt(reset, 10) * 1000; + if (Number.isFinite(resetMs)) { + pauseUntilMs = Math.max(pauseUntilMs, resetMs); + } + } else { + pauseUntilMs = Math.max(pauseUntilMs, Date.now() + WINDOW_MS); + } + } + } + + const retryAfter = headers.get('retry-after'); + if (retryAfter) { + const seconds = parseInt(retryAfter, 10); + if (Number.isFinite(seconds) && seconds > 0) { + pauseUntilMs = Math.max(pauseUntilMs, Date.now() + seconds * 1000); + } + } + }, + }; +}