From 3c3bf3bb1835c2498e82165fc5d60fa772d93057 Mon Sep 17 00:00:00 2001 From: sudacode Date: Sat, 6 Jun 2026 14:48:12 -0700 Subject: [PATCH] fix(stats): persist filter prefs, fix cover MIME types, and dedup alass - Remember Hide Known/Hide Kana filter state in localStorage across sessions - Detect PNG and WebP cover art MIME types instead of hardcoding image/jpeg - Use configured AnkiConnect URL for the browse action - Deduplicate concurrent in-flight alass retime calls via promise caching - Prefer request-provided secondary subtitle text over retimed sidecar fallback - Fix cover image record key types from string to number --- changes/stats-updates.md | 6 +- .../services/__tests__/stats-server.test.ts | 92 ++++++++++++++++++- .../services/secondary-subtitle-sidecar.ts | 36 +++++--- src/core/services/stats-server.ts | 50 ++++++++-- .../vocabulary/FrequencyRankTable.test.tsx | 77 ++++++++++++++++ .../vocabulary/FrequencyRankTable.tsx | 47 +++++++++- stats/src/types/stats.ts | 4 +- 7 files changed, 275 insertions(+), 37 deletions(-) diff --git a/changes/stats-updates.md b/changes/stats-updates.md index 2de390fe..b422f083 100644 --- a/changes/stats-updates.md +++ b/changes/stats-updates.md @@ -2,7 +2,7 @@ type: changed area: stats - Added the Stats Search tab for realtime subtitle sentence search with media context, headword matching, and mining actions for source-backed sentence cards or exact-match word/audio cards. -- Improved Stats mining from Search and vocabulary examples: empty `ankiConnect.deck` can use Yomitan's mining deck, sentence cards are created before slow media generation finishes, secondary subtitles from stored lines, sidecar files, or temporary alass-retimed English sidecars populate sentence Selection Text, invalid stored timings are blocked before FFmpeg runs, future out-of-order subtitle timing pairs are skipped until valid timings arrive, and partial media failures are shown. +- Improved Stats mining from Search and vocabulary examples: empty `ankiConnect.deck` can use Yomitan's mining deck, sentence cards are created before slow media generation finishes, stored/requested secondary subtitles are preserved before falling back to sidecar files or temporary alass-retimed English sidecars for sentence Selection Text, invalid stored timings are blocked before FFmpeg runs, future out-of-order subtitle timing pairs are skipped until valid timings arrive, and partial media failures are shown. - Fixed Stats mining field/audio behavior so sentence clips update `SentenceAudio`, word audio uses the configured Yomitan sources, English subtitle text is not written onto word cards, and secondary subtitle auto-selection prefers regular English tracks over Signs/Songs tracks. -- Improved vocabulary review with a Hide Kana filter, duplicate-collapsed exclusions across token variants, and Related Seen Words matching based on shared readings or kanji. -- Improved Stats browsing reliability by remembering library card size, retrying stored cover art without extra AniList lookups, showing progress during session deletes, and making session deletes refresh faster. +- Improved vocabulary review with remembered Hide Known/Hide Kana filters, duplicate-collapsed exclusions across token variants, and Related Seen Words matching based on shared readings or kanji. +- Improved Stats browsing reliability by remembering library card size, retrying stored cover art without extra AniList lookups, preserving PNG/WebP cover MIME types, honoring custom AnkiConnect URLs for Browse, showing progress during session deletes, and making session deletes refresh faster. diff --git a/src/core/services/__tests__/stats-server.test.ts b/src/core/services/__tests__/stats-server.test.ts index fc88ff8f..f354bca8 100644 --- a/src/core/services/__tests__/stats-server.test.ts +++ b/src/core/services/__tests__/stats-server.test.ts @@ -966,7 +966,7 @@ describe('stats server API routes', () => { videoId, anilistId: null, coverUrl: null, - coverBlob: Buffer.from([0x89, 0x50]), + coverBlob: Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), titleRomaji: null, titleEnglish: null, episodesTotal: null, @@ -997,8 +997,8 @@ describe('stats server API routes', () => { }, media: { 7: { - contentType: 'image/jpeg', - dataUrl: 'data:image/jpeg;base64,iVA=', + contentType: 'image/png', + dataUrl: 'data:image/png;base64,iVBORw0KGgo=', }, 99999: null, }, @@ -1365,7 +1365,7 @@ describe('stats server API routes', () => { }); }); - it('POST /api/stats/mine-card prefers retimed sidecar secondary text for sentence cards', async () => { + it('POST /api/stats/mine-card prefers request secondary text over retimed fallback', async () => { await withTempDir(async (dir) => { const sourcePath = path.join(dir, 'episode.mkv'); fs.writeFileSync(sourcePath, 'fake media'); @@ -1414,7 +1414,7 @@ describe('stats server API routes', () => { const addNoteRequest = requests.find((request) => request.action === 'addNote'); assert.equal( addNoteRequest?.params?.note?.fields?.SelectionText, - 'Aligned English subtitle', + 'Stale stored English subtitle', ); }); }); @@ -1484,6 +1484,74 @@ Aligned English subtitle }); }); + it('shares in-flight retimed secondary subtitle work for concurrent requests', async () => { + await withTempDir(async (dir) => { + const sourcePath = path.join(dir, 'episode.mkv'); + const japanesePath = path.join(dir, 'episode.ja.srt'); + const englishPath = path.join(dir, 'episode.en.srt'); + const alassPath = path.join(dir, 'alass-cli'); + fs.writeFileSync(sourcePath, 'fake media'); + fs.writeFileSync(alassPath, 'fake alass'); + fs.writeFileSync( + japanesePath, + `1 +00:00:01,000 --> 00:00:02,000 +猫を見た +`, + ); + fs.writeFileSync( + englishPath, + `1 +00:00:09,000 --> 00:00:10,000 +Stale English subtitle +`, + ); + + let alassRuns = 0; + let releaseAlass!: () => void; + const alassGate = new Promise((resolve) => { + releaseAlass = resolve; + }); + const input = { + sourcePath, + startMs: 1_000, + endMs: 2_000, + alassPath, + runAlass: async ( + _alassPath: string, + _referencePath: string, + _inputPath: string, + outputPath: string, + ) => { + alassRuns += 1; + await alassGate; + fs.writeFileSync( + outputPath, + `1 +00:00:01,000 --> 00:00:02,000 +Aligned English subtitle +`, + ); + return { ok: true, code: 0, stdout: '', stderr: '' }; + }, + }; + + try { + const first = resolveRetimedSecondarySubtitleTextFromSidecar(input); + const second = resolveRetimedSecondarySubtitleTextFromSidecar(input); + releaseAlass(); + + assert.deepEqual(await Promise.all([first, second]), [ + 'Aligned English subtitle', + 'Aligned English subtitle', + ]); + assert.equal(alassRuns, 1); + } finally { + clearRetimedSecondarySubtitleCache(); + } + }); + }); + it('POST /api/stats/mine-card adds direct sentence cards before slow media finishes', async () => { await withTempDir(async (dir) => { const sourcePath = path.join(dir, 'episode.mkv'); @@ -2387,6 +2455,20 @@ Aligned English subtitle assert.equal(res.status, 400); }); + it('POST /api/stats/anki/browse uses configured AnkiConnect URL', async () => { + await withFakeAnkiConnect(async (requests, url) => { + const app = createStatsApp(createMockTracker(), { + ankiConnectConfig: { url }, + }); + + const res = await app.request('/api/stats/anki/browse?noteId=12345', { method: 'POST' }); + + assert.equal(res.status, 200); + assert.equal(requests[0]?.action, 'guiBrowse'); + assert.deepEqual(requests[0]?.params, { query: 'nid:12345' }); + }); + }); + it('GET /api/stats/anilist/search uses the configured AniList rate limiter', async () => { const originalFetch = globalThis.fetch; let acquireCalls = 0; diff --git a/src/core/services/secondary-subtitle-sidecar.ts b/src/core/services/secondary-subtitle-sidecar.ts index ac290d81..988db6f7 100644 --- a/src/core/services/secondary-subtitle-sidecar.ts +++ b/src/core/services/secondary-subtitle-sidecar.ts @@ -29,6 +29,7 @@ type SidecarCandidate = { type RetimedSubtitleCacheEntry = { path: string; cleanupDir: string; + promise?: Promise; }; export type RetimedSubtitleCommandRunner = ( @@ -353,6 +354,9 @@ async function retimeSecondarySubtitle(input: { if (!key) return ''; const cached = retimedSubtitleCache.get(key); + if (cached?.promise) { + return cached.promise; + } if (cached && existsSync(cached.path)) { return cached.path; } @@ -371,19 +375,25 @@ async function retimeSecondarySubtitle(input: { `${parsedSecondary.name}.retimed${parsedSecondary.ext || '.srt'}`, ); - const result = await input.runAlass( - input.alassPath, - input.primaryPath, - input.secondaryPath, - outputPath, - ); - if (!result.ok || !existsSync(outputPath)) { - rmSync(cleanupDir, { recursive: true, force: true }); - return ''; - } - - retimedSubtitleCache.set(key, { path: outputPath, cleanupDir }); - return outputPath; + const entry: RetimedSubtitleCacheEntry = { path: outputPath, cleanupDir }; + entry.promise = input + .runAlass(input.alassPath, input.primaryPath, input.secondaryPath, outputPath) + .then((result) => { + if (!result.ok || !existsSync(outputPath)) { + rmSync(cleanupDir, { recursive: true, force: true }); + retimedSubtitleCache.delete(key); + return ''; + } + entry.promise = undefined; + return outputPath; + }) + .catch(() => { + rmSync(cleanupDir, { recursive: true, force: true }); + retimedSubtitleCache.delete(key); + return ''; + }); + retimedSubtitleCache.set(key, entry); + return entry.promise; } export function resolveSecondarySubtitleTextFromSidecar(input: { diff --git a/src/core/services/stats-server.ts b/src/core/services/stats-server.ts index a9a4866d..0e64f9fd 100644 --- a/src/core/services/stats-server.ts +++ b/src/core/services/stats-server.ts @@ -52,7 +52,7 @@ type StatsExcludedWordPayload = { }; type StatsCoverImagePayload = { - contentType: 'image/jpeg'; + contentType: string; dataUrl: string; } | null; @@ -126,12 +126,43 @@ function coverImagePayload( art: { coverBlob?: Uint8Array | null } | null | undefined, ): StatsCoverImagePayload { if (!art?.coverBlob) return null; + const bytes = new Uint8Array(art.coverBlob); + const contentType = detectImageContentType(bytes); return { - contentType: 'image/jpeg', - dataUrl: `data:image/jpeg;base64,${Buffer.from(art.coverBlob).toString('base64')}`, + contentType, + dataUrl: `data:${contentType};base64,${Buffer.from(bytes).toString('base64')}`, }; } +function detectImageContentType(bytes: Uint8Array): string { + if ( + bytes.length >= 8 && + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47 + ) { + return 'image/png'; + } + if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return 'image/jpeg'; + } + if ( + bytes.length >= 12 && + bytes[0] === 0x52 && + bytes[1] === 0x49 && + bytes[2] === 0x46 && + bytes[3] === 0x46 && + bytes[8] === 0x57 && + bytes[9] === 0x45 && + bytes[10] === 0x42 && + bytes[11] === 0x50 + ) { + return 'image/webp'; + } + return 'application/octet-stream'; +} + function resolveStatsNoteFieldName( noteInfo: StatsServerNoteInfo, ...preferredNames: (string | undefined)[] @@ -960,17 +991,17 @@ export function createStatsApp( const body = (await c.req.json().catch(() => null)) as StatsCoverBatchBody | null; const animeIds = parsePositiveIdList(body?.animeIds); const videoIds = parsePositiveIdList(body?.videoIds); - const anime: Record = {}; - const media: Record = {}; + const anime: Record = {}; + const media: Record = {}; await Promise.all( animeIds.map(async (animeId) => { - anime[String(animeId)] = coverImagePayload(await tracker.getAnimeCoverArt(animeId)); + anime[animeId] = coverImagePayload(await tracker.getAnimeCoverArt(animeId)); }), ); await Promise.all( videoIds.map(async (videoId) => { - media[String(videoId)] = coverImagePayload(await tracker.getCoverArt(videoId)); + media[videoId] = coverImagePayload(await tracker.getCoverArt(videoId)); }), ); @@ -1024,8 +1055,9 @@ export function createStatsApp( app.post('/api/stats/anki/browse', async (c) => { const noteId = parseIntQuery(c.req.query('noteId'), 0); if (noteId <= 0) return c.body(null, 400); + const ankiConfig = getAnkiConnectConfig(); try { - const response = await fetch('http://127.0.0.1:8765', { + const response = await fetch(ankiConfig?.url ?? 'http://127.0.0.1:8765', { method: 'POST', headers: { 'Content-Type': 'application/json' }, signal: AbortSignal.timeout(ANKI_CONNECT_FETCH_TIMEOUT_MS), @@ -1136,8 +1168,8 @@ export function createStatsApp( } } const secondaryText = - retimedSecondaryText || bodySecondaryText || + retimedSecondaryText || resolveSecondarySubtitleTextFromSidecar({ sourcePath, startMs, diff --git a/stats/src/components/vocabulary/FrequencyRankTable.test.tsx b/stats/src/components/vocabulary/FrequencyRankTable.test.tsx index ea4dd274..96c0cfff 100644 --- a/stats/src/components/vocabulary/FrequencyRankTable.test.tsx +++ b/stats/src/components/vocabulary/FrequencyRankTable.test.tsx @@ -24,6 +24,46 @@ function makeEntry(over: Partial): VocabularyEntry { } as VocabularyEntry; } +function withLocalStorage(initial: Record, run: () => T): T { + const previous = Object.getOwnPropertyDescriptor(globalThis, 'localStorage'); + const values = new Map(Object.entries(initial)); + const storage = { + get length() { + return values.size; + }, + clear() { + values.clear(); + }, + getItem(key: string) { + return values.get(key) ?? null; + }, + key(index: number) { + return Array.from(values.keys())[index] ?? null; + }, + removeItem(key: string) { + values.delete(key); + }, + setItem(key: string, value: string) { + values.set(key, value); + }, + } as Storage; + + Object.defineProperty(globalThis, 'localStorage', { + configurable: true, + value: storage, + }); + + try { + return run(); + } finally { + if (previous) { + Object.defineProperty(globalThis, 'localStorage', previous); + } else { + delete (globalThis as { localStorage?: unknown }).localStorage; + } + } +} + test('renders headword and reading inline in a single column (no separate Reading header)', () => { const entry = makeEntry({}); const markup = renderToStaticMarkup( @@ -84,3 +124,40 @@ test('renders a Hide Kana filter button', () => { ); assert.match(markup, /Hide Kana/); }); + +test('uses saved Hide Kana preference on first render', () => { + const markup = withLocalStorage({ 'subminer.stats.frequencyRank.hideKanaOnly': 'true' }, () => + renderToStaticMarkup( + , + ), + ); + + assert.doesNotMatch(markup, />さらに前に { + const markup = withLocalStorage({ 'subminer.stats.frequencyRank.hideKnown': 'false' }, () => + renderToStaticMarkup( + , + ), + ); + + assert.match(markup, />Most Common Words Seen日本語, @@ -70,8 +99,12 @@ export function buildFrequencyRankRows( export function FrequencyRankTable({ words, knownWords, onSelectWord }: FrequencyRankTableProps) { const [page, setPage] = useState(0); - const [hideKnown, setHideKnown] = useState(true); - const [hideKanaOnly, setHideKanaOnly] = useState(false); + const [hideKnown, setHideKnown] = useState(() => + readBooleanPreference(HIDE_KNOWN_STORAGE_KEY, true), + ); + const [hideKanaOnly, setHideKanaOnly] = useState(() => + readBooleanPreference(HIDE_KANA_ONLY_STORAGE_KEY, false), + ); const [collapsed, setCollapsed] = useState(false); const hasKnownData = knownWords.size > 0; @@ -116,7 +149,9 @@ export function FrequencyRankTable({ words, knownWords, onSelectWord }: Frequenc type="button" aria-pressed={hideKnown} onClick={() => { - setHideKnown(!hideKnown); + const next = !hideKnown; + setHideKnown(next); + writeBooleanPreference(HIDE_KNOWN_STORAGE_KEY, next); setPage(0); }} className={`px-2.5 py-1 rounded-lg text-xs transition-colors border ${ @@ -132,7 +167,9 @@ export function FrequencyRankTable({ words, knownWords, onSelectWord }: Frequenc type="button" aria-pressed={hideKanaOnly} onClick={() => { - setHideKanaOnly(!hideKanaOnly); + const next = !hideKanaOnly; + setHideKanaOnly(next); + writeBooleanPreference(HIDE_KANA_ONLY_STORAGE_KEY, next); setPage(0); }} className={`px-2.5 py-1 rounded-lg text-xs transition-colors border ${ @@ -150,7 +187,7 @@ export function FrequencyRankTable({ words, knownWords, onSelectWord }: Frequenc
{hideKnown && hasKnownData && !hideKanaOnly ? 'All ranked words are already in Anki!' - : hideKnown || hideKanaOnly + : (hideKnown && hasKnownData) || hideKanaOnly ? 'No ranked words match the active filters.' : 'No words with frequency data.'}
diff --git a/stats/src/types/stats.ts b/stats/src/types/stats.ts index c606d09b..c8ba14d2 100644 --- a/stats/src/types/stats.ts +++ b/stats/src/types/stats.ts @@ -88,8 +88,8 @@ export interface StatsCoverImage { } export interface StatsCoverImagesData { - anime: Record; - media: Record; + anime: Record; + media: Record; } export interface KanjiEntry {