fix(stats): persist filter prefs, fix cover MIME types, and dedup alass

- Remember Hide Known/Hide Kana filter state in localStorage across sessions
- Detect PNG and WebP cover art MIME types instead of hardcoding image/jpeg
- Use configured AnkiConnect URL for the browse action
- Deduplicate concurrent in-flight alass retime calls via promise caching
- Prefer request-provided secondary subtitle text over retimed sidecar fallback
- Fix cover image record key types from string to number
This commit is contained in:
2026-06-06 14:48:12 -07:00
parent 99401e5a70
commit 3c3bf3bb18
7 changed files with 275 additions and 37 deletions
+3 -3
View File
@@ -2,7 +2,7 @@ type: changed
area: stats area: stats
- Added the Stats Search tab for realtime subtitle sentence search with media context, headword matching, and mining actions for source-backed sentence cards or exact-match word/audio cards. - Added the Stats Search tab for realtime subtitle sentence search with media context, headword matching, and mining actions for source-backed sentence cards or exact-match word/audio cards.
- Improved Stats mining from Search and vocabulary examples: empty `ankiConnect.deck` can use Yomitan's mining deck, sentence cards are created before slow media generation finishes, secondary subtitles from stored lines, sidecar files, or temporary alass-retimed English sidecars populate sentence Selection Text, invalid stored timings are blocked before FFmpeg runs, future out-of-order subtitle timing pairs are skipped until valid timings arrive, and partial media failures are shown. - Improved Stats mining from Search and vocabulary examples: empty `ankiConnect.deck` can use Yomitan's mining deck, sentence cards are created before slow media generation finishes, stored/requested secondary subtitles are preserved before falling back to sidecar files or temporary alass-retimed English sidecars for sentence Selection Text, invalid stored timings are blocked before FFmpeg runs, future out-of-order subtitle timing pairs are skipped until valid timings arrive, and partial media failures are shown.
- Fixed Stats mining field/audio behavior so sentence clips update `SentenceAudio`, word audio uses the configured Yomitan sources, English subtitle text is not written onto word cards, and secondary subtitle auto-selection prefers regular English tracks over Signs/Songs tracks. - Fixed Stats mining field/audio behavior so sentence clips update `SentenceAudio`, word audio uses the configured Yomitan sources, English subtitle text is not written onto word cards, and secondary subtitle auto-selection prefers regular English tracks over Signs/Songs tracks.
- Improved vocabulary review with a Hide Kana filter, duplicate-collapsed exclusions across token variants, and Related Seen Words matching based on shared readings or kanji. - Improved vocabulary review with remembered Hide Known/Hide Kana filters, duplicate-collapsed exclusions across token variants, and Related Seen Words matching based on shared readings or kanji.
- Improved Stats browsing reliability by remembering library card size, retrying stored cover art without extra AniList lookups, showing progress during session deletes, and making session deletes refresh faster. - Improved Stats browsing reliability by remembering library card size, retrying stored cover art without extra AniList lookups, preserving PNG/WebP cover MIME types, honoring custom AnkiConnect URLs for Browse, showing progress during session deletes, and making session deletes refresh faster.
@@ -966,7 +966,7 @@ describe('stats server API routes', () => {
videoId, videoId,
anilistId: null, anilistId: null,
coverUrl: null, coverUrl: null,
coverBlob: Buffer.from([0x89, 0x50]), coverBlob: Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
titleRomaji: null, titleRomaji: null,
titleEnglish: null, titleEnglish: null,
episodesTotal: null, episodesTotal: null,
@@ -997,8 +997,8 @@ describe('stats server API routes', () => {
}, },
media: { media: {
7: { 7: {
contentType: 'image/jpeg', contentType: 'image/png',
dataUrl: 'data:image/jpeg;base64,iVA=', dataUrl: 'data:image/png;base64,iVBORw0KGgo=',
}, },
99999: null, 99999: null,
}, },
@@ -1365,7 +1365,7 @@ describe('stats server API routes', () => {
}); });
}); });
it('POST /api/stats/mine-card prefers retimed sidecar secondary text for sentence cards', async () => { it('POST /api/stats/mine-card prefers request secondary text over retimed fallback', async () => {
await withTempDir(async (dir) => { await withTempDir(async (dir) => {
const sourcePath = path.join(dir, 'episode.mkv'); const sourcePath = path.join(dir, 'episode.mkv');
fs.writeFileSync(sourcePath, 'fake media'); fs.writeFileSync(sourcePath, 'fake media');
@@ -1414,7 +1414,7 @@ describe('stats server API routes', () => {
const addNoteRequest = requests.find((request) => request.action === 'addNote'); const addNoteRequest = requests.find((request) => request.action === 'addNote');
assert.equal( assert.equal(
addNoteRequest?.params?.note?.fields?.SelectionText, addNoteRequest?.params?.note?.fields?.SelectionText,
'Aligned English subtitle', 'Stale stored English subtitle',
); );
}); });
}); });
@@ -1484,6 +1484,74 @@ Aligned English subtitle
}); });
}); });
it('shares in-flight retimed secondary subtitle work for concurrent requests', async () => {
await withTempDir(async (dir) => {
const sourcePath = path.join(dir, 'episode.mkv');
const japanesePath = path.join(dir, 'episode.ja.srt');
const englishPath = path.join(dir, 'episode.en.srt');
const alassPath = path.join(dir, 'alass-cli');
fs.writeFileSync(sourcePath, 'fake media');
fs.writeFileSync(alassPath, 'fake alass');
fs.writeFileSync(
japanesePath,
`1
00:00:01,000 --> 00:00:02,000
猫を見た
`,
);
fs.writeFileSync(
englishPath,
`1
00:00:09,000 --> 00:00:10,000
Stale English subtitle
`,
);
let alassRuns = 0;
let releaseAlass!: () => void;
const alassGate = new Promise<void>((resolve) => {
releaseAlass = resolve;
});
const input = {
sourcePath,
startMs: 1_000,
endMs: 2_000,
alassPath,
runAlass: async (
_alassPath: string,
_referencePath: string,
_inputPath: string,
outputPath: string,
) => {
alassRuns += 1;
await alassGate;
fs.writeFileSync(
outputPath,
`1
00:00:01,000 --> 00:00:02,000
Aligned English subtitle
`,
);
return { ok: true, code: 0, stdout: '', stderr: '' };
},
};
try {
const first = resolveRetimedSecondarySubtitleTextFromSidecar(input);
const second = resolveRetimedSecondarySubtitleTextFromSidecar(input);
releaseAlass();
assert.deepEqual(await Promise.all([first, second]), [
'Aligned English subtitle',
'Aligned English subtitle',
]);
assert.equal(alassRuns, 1);
} finally {
clearRetimedSecondarySubtitleCache();
}
});
});
it('POST /api/stats/mine-card adds direct sentence cards before slow media finishes', async () => { it('POST /api/stats/mine-card adds direct sentence cards before slow media finishes', async () => {
await withTempDir(async (dir) => { await withTempDir(async (dir) => {
const sourcePath = path.join(dir, 'episode.mkv'); const sourcePath = path.join(dir, 'episode.mkv');
@@ -2387,6 +2455,20 @@ Aligned English subtitle
assert.equal(res.status, 400); assert.equal(res.status, 400);
}); });
it('POST /api/stats/anki/browse uses configured AnkiConnect URL', async () => {
await withFakeAnkiConnect(async (requests, url) => {
const app = createStatsApp(createMockTracker(), {
ankiConnectConfig: { url },
});
const res = await app.request('/api/stats/anki/browse?noteId=12345', { method: 'POST' });
assert.equal(res.status, 200);
assert.equal(requests[0]?.action, 'guiBrowse');
assert.deepEqual(requests[0]?.params, { query: 'nid:12345' });
});
});
it('GET /api/stats/anilist/search uses the configured AniList rate limiter', async () => { it('GET /api/stats/anilist/search uses the configured AniList rate limiter', async () => {
const originalFetch = globalThis.fetch; const originalFetch = globalThis.fetch;
let acquireCalls = 0; let acquireCalls = 0;
+23 -13
View File
@@ -29,6 +29,7 @@ type SidecarCandidate = {
type RetimedSubtitleCacheEntry = { type RetimedSubtitleCacheEntry = {
path: string; path: string;
cleanupDir: string; cleanupDir: string;
promise?: Promise<string>;
}; };
export type RetimedSubtitleCommandRunner = ( export type RetimedSubtitleCommandRunner = (
@@ -353,6 +354,9 @@ async function retimeSecondarySubtitle(input: {
if (!key) return ''; if (!key) return '';
const cached = retimedSubtitleCache.get(key); const cached = retimedSubtitleCache.get(key);
if (cached?.promise) {
return cached.promise;
}
if (cached && existsSync(cached.path)) { if (cached && existsSync(cached.path)) {
return cached.path; return cached.path;
} }
@@ -371,19 +375,25 @@ async function retimeSecondarySubtitle(input: {
`${parsedSecondary.name}.retimed${parsedSecondary.ext || '.srt'}`, `${parsedSecondary.name}.retimed${parsedSecondary.ext || '.srt'}`,
); );
const result = await input.runAlass( const entry: RetimedSubtitleCacheEntry = { path: outputPath, cleanupDir };
input.alassPath, entry.promise = input
input.primaryPath, .runAlass(input.alassPath, input.primaryPath, input.secondaryPath, outputPath)
input.secondaryPath, .then((result) => {
outputPath, if (!result.ok || !existsSync(outputPath)) {
); rmSync(cleanupDir, { recursive: true, force: true });
if (!result.ok || !existsSync(outputPath)) { retimedSubtitleCache.delete(key);
rmSync(cleanupDir, { recursive: true, force: true }); return '';
return ''; }
} entry.promise = undefined;
return outputPath;
retimedSubtitleCache.set(key, { path: outputPath, cleanupDir }); })
return outputPath; .catch(() => {
rmSync(cleanupDir, { recursive: true, force: true });
retimedSubtitleCache.delete(key);
return '';
});
retimedSubtitleCache.set(key, entry);
return entry.promise;
} }
export function resolveSecondarySubtitleTextFromSidecar(input: { export function resolveSecondarySubtitleTextFromSidecar(input: {
+41 -9
View File
@@ -52,7 +52,7 @@ type StatsExcludedWordPayload = {
}; };
type StatsCoverImagePayload = { type StatsCoverImagePayload = {
contentType: 'image/jpeg'; contentType: string;
dataUrl: string; dataUrl: string;
} | null; } | null;
@@ -126,12 +126,43 @@ function coverImagePayload(
art: { coverBlob?: Uint8Array | null } | null | undefined, art: { coverBlob?: Uint8Array | null } | null | undefined,
): StatsCoverImagePayload { ): StatsCoverImagePayload {
if (!art?.coverBlob) return null; if (!art?.coverBlob) return null;
const bytes = new Uint8Array(art.coverBlob);
const contentType = detectImageContentType(bytes);
return { return {
contentType: 'image/jpeg', contentType,
dataUrl: `data:image/jpeg;base64,${Buffer.from(art.coverBlob).toString('base64')}`, dataUrl: `data:${contentType};base64,${Buffer.from(bytes).toString('base64')}`,
}; };
} }
function detectImageContentType(bytes: Uint8Array): string {
if (
bytes.length >= 8 &&
bytes[0] === 0x89 &&
bytes[1] === 0x50 &&
bytes[2] === 0x4e &&
bytes[3] === 0x47
) {
return 'image/png';
}
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
return 'image/jpeg';
}
if (
bytes.length >= 12 &&
bytes[0] === 0x52 &&
bytes[1] === 0x49 &&
bytes[2] === 0x46 &&
bytes[3] === 0x46 &&
bytes[8] === 0x57 &&
bytes[9] === 0x45 &&
bytes[10] === 0x42 &&
bytes[11] === 0x50
) {
return 'image/webp';
}
return 'application/octet-stream';
}
function resolveStatsNoteFieldName( function resolveStatsNoteFieldName(
noteInfo: StatsServerNoteInfo, noteInfo: StatsServerNoteInfo,
...preferredNames: (string | undefined)[] ...preferredNames: (string | undefined)[]
@@ -960,17 +991,17 @@ export function createStatsApp(
const body = (await c.req.json().catch(() => null)) as StatsCoverBatchBody | null; const body = (await c.req.json().catch(() => null)) as StatsCoverBatchBody | null;
const animeIds = parsePositiveIdList(body?.animeIds); const animeIds = parsePositiveIdList(body?.animeIds);
const videoIds = parsePositiveIdList(body?.videoIds); const videoIds = parsePositiveIdList(body?.videoIds);
const anime: Record<string, StatsCoverImagePayload> = {}; const anime: Record<number, StatsCoverImagePayload> = {};
const media: Record<string, StatsCoverImagePayload> = {}; const media: Record<number, StatsCoverImagePayload> = {};
await Promise.all( await Promise.all(
animeIds.map(async (animeId) => { animeIds.map(async (animeId) => {
anime[String(animeId)] = coverImagePayload(await tracker.getAnimeCoverArt(animeId)); anime[animeId] = coverImagePayload(await tracker.getAnimeCoverArt(animeId));
}), }),
); );
await Promise.all( await Promise.all(
videoIds.map(async (videoId) => { videoIds.map(async (videoId) => {
media[String(videoId)] = coverImagePayload(await tracker.getCoverArt(videoId)); media[videoId] = coverImagePayload(await tracker.getCoverArt(videoId));
}), }),
); );
@@ -1024,8 +1055,9 @@ export function createStatsApp(
app.post('/api/stats/anki/browse', async (c) => { app.post('/api/stats/anki/browse', async (c) => {
const noteId = parseIntQuery(c.req.query('noteId'), 0); const noteId = parseIntQuery(c.req.query('noteId'), 0);
if (noteId <= 0) return c.body(null, 400); if (noteId <= 0) return c.body(null, 400);
const ankiConfig = getAnkiConnectConfig();
try { try {
const response = await fetch('http://127.0.0.1:8765', { const response = await fetch(ankiConfig?.url ?? 'http://127.0.0.1:8765', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
signal: AbortSignal.timeout(ANKI_CONNECT_FETCH_TIMEOUT_MS), signal: AbortSignal.timeout(ANKI_CONNECT_FETCH_TIMEOUT_MS),
@@ -1136,8 +1168,8 @@ export function createStatsApp(
} }
} }
const secondaryText = const secondaryText =
retimedSecondaryText ||
bodySecondaryText || bodySecondaryText ||
retimedSecondaryText ||
resolveSecondarySubtitleTextFromSidecar({ resolveSecondarySubtitleTextFromSidecar({
sourcePath, sourcePath,
startMs, startMs,
@@ -24,6 +24,46 @@ function makeEntry(over: Partial<VocabularyEntry>): VocabularyEntry {
} as VocabularyEntry; } as VocabularyEntry;
} }
function withLocalStorage<T>(initial: Record<string, string>, run: () => T): T {
const previous = Object.getOwnPropertyDescriptor(globalThis, 'localStorage');
const values = new Map(Object.entries(initial));
const storage = {
get length() {
return values.size;
},
clear() {
values.clear();
},
getItem(key: string) {
return values.get(key) ?? null;
},
key(index: number) {
return Array.from(values.keys())[index] ?? null;
},
removeItem(key: string) {
values.delete(key);
},
setItem(key: string, value: string) {
values.set(key, value);
},
} as Storage;
Object.defineProperty(globalThis, 'localStorage', {
configurable: true,
value: storage,
});
try {
return run();
} finally {
if (previous) {
Object.defineProperty(globalThis, 'localStorage', previous);
} else {
delete (globalThis as { localStorage?: unknown }).localStorage;
}
}
}
test('renders headword and reading inline in a single column (no separate Reading header)', () => { test('renders headword and reading inline in a single column (no separate Reading header)', () => {
const entry = makeEntry({}); const entry = makeEntry({});
const markup = renderToStaticMarkup( const markup = renderToStaticMarkup(
@@ -84,3 +124,40 @@ test('renders a Hide Kana filter button', () => {
); );
assert.match(markup, /Hide Kana/); assert.match(markup, /Hide Kana/);
}); });
test('uses saved Hide Kana preference on first render', () => {
const markup = withLocalStorage({ 'subminer.stats.frequencyRank.hideKanaOnly': 'true' }, () =>
renderToStaticMarkup(
<FrequencyRankTable
words={[
makeEntry({ wordId: 1, headword: 'さらに', word: 'さらに', frequencyRank: 10 }),
makeEntry({
wordId: 2,
headword: '前に',
word: '前に',
reading: 'まえに',
frequencyRank: 20,
}),
]}
knownWords={new Set()}
/>,
),
);
assert.doesNotMatch(markup, />さらに</);
assert.match(markup, />前に</);
});
test('uses saved Hide Known preference on first render', () => {
const markup = withLocalStorage({ 'subminer.stats.frequencyRank.hideKnown': 'false' }, () =>
renderToStaticMarkup(
<FrequencyRankTable
words={[makeEntry({ headword: '日本語', word: '日本語', frequencyRank: 10 })]}
knownWords={new Set(['日本語'])}
/>,
),
);
assert.match(markup, />Most Common Words Seen</);
assert.match(markup, />日本語</);
});
@@ -10,6 +10,8 @@ interface FrequencyRankTableProps {
} }
const PAGE_SIZE = 25; const PAGE_SIZE = 25;
const HIDE_KNOWN_STORAGE_KEY = 'subminer.stats.frequencyRank.hideKnown';
const HIDE_KANA_ONLY_STORAGE_KEY = 'subminer.stats.frequencyRank.hideKanaOnly';
interface FrequencyRankOptions { interface FrequencyRankOptions {
hideKnown: boolean; hideKnown: boolean;
@@ -31,6 +33,33 @@ function isKanaOnlyWord(w: VocabularyEntry): boolean {
return isKanaOnlyTokenText(w.headword || w.word); return isKanaOnlyTokenText(w.headword || w.word);
} }
function getPreferenceStorage(): Storage | null {
try {
return globalThis.localStorage ?? null;
} catch {
return null;
}
}
function readBooleanPreference(key: string, fallback: boolean): boolean {
try {
const value = getPreferenceStorage()?.getItem(key);
if (value === 'true') return true;
if (value === 'false') return false;
return fallback;
} catch {
return fallback;
}
}
function writeBooleanPreference(key: string, value: boolean): void {
try {
getPreferenceStorage()?.setItem(key, String(value));
} catch {
// Storage can be blocked in private/restricted contexts; keep the in-memory choice.
}
}
export function buildFrequencyRankRows( export function buildFrequencyRankRows(
words: VocabularyEntry[], words: VocabularyEntry[],
knownWords: Set<string>, knownWords: Set<string>,
@@ -70,8 +99,12 @@ export function buildFrequencyRankRows(
export function FrequencyRankTable({ words, knownWords, onSelectWord }: FrequencyRankTableProps) { export function FrequencyRankTable({ words, knownWords, onSelectWord }: FrequencyRankTableProps) {
const [page, setPage] = useState(0); const [page, setPage] = useState(0);
const [hideKnown, setHideKnown] = useState(true); const [hideKnown, setHideKnown] = useState(() =>
const [hideKanaOnly, setHideKanaOnly] = useState(false); readBooleanPreference(HIDE_KNOWN_STORAGE_KEY, true),
);
const [hideKanaOnly, setHideKanaOnly] = useState(() =>
readBooleanPreference(HIDE_KANA_ONLY_STORAGE_KEY, false),
);
const [collapsed, setCollapsed] = useState(false); const [collapsed, setCollapsed] = useState(false);
const hasKnownData = knownWords.size > 0; const hasKnownData = knownWords.size > 0;
@@ -116,7 +149,9 @@ export function FrequencyRankTable({ words, knownWords, onSelectWord }: Frequenc
type="button" type="button"
aria-pressed={hideKnown} aria-pressed={hideKnown}
onClick={() => { onClick={() => {
setHideKnown(!hideKnown); const next = !hideKnown;
setHideKnown(next);
writeBooleanPreference(HIDE_KNOWN_STORAGE_KEY, next);
setPage(0); setPage(0);
}} }}
className={`px-2.5 py-1 rounded-lg text-xs transition-colors border ${ className={`px-2.5 py-1 rounded-lg text-xs transition-colors border ${
@@ -132,7 +167,9 @@ export function FrequencyRankTable({ words, knownWords, onSelectWord }: Frequenc
type="button" type="button"
aria-pressed={hideKanaOnly} aria-pressed={hideKanaOnly}
onClick={() => { onClick={() => {
setHideKanaOnly(!hideKanaOnly); const next = !hideKanaOnly;
setHideKanaOnly(next);
writeBooleanPreference(HIDE_KANA_ONLY_STORAGE_KEY, next);
setPage(0); setPage(0);
}} }}
className={`px-2.5 py-1 rounded-lg text-xs transition-colors border ${ className={`px-2.5 py-1 rounded-lg text-xs transition-colors border ${
@@ -150,7 +187,7 @@ export function FrequencyRankTable({ words, knownWords, onSelectWord }: Frequenc
<div className="text-xs text-ctp-overlay2 mt-3"> <div className="text-xs text-ctp-overlay2 mt-3">
{hideKnown && hasKnownData && !hideKanaOnly {hideKnown && hasKnownData && !hideKanaOnly
? 'All ranked words are already in Anki!' ? 'All ranked words are already in Anki!'
: hideKnown || hideKanaOnly : (hideKnown && hasKnownData) || hideKanaOnly
? 'No ranked words match the active filters.' ? 'No ranked words match the active filters.'
: 'No words with frequency data.'} : 'No words with frequency data.'}
</div> </div>
+2 -2
View File
@@ -88,8 +88,8 @@ export interface StatsCoverImage {
} }
export interface StatsCoverImagesData { export interface StatsCoverImagesData {
anime: Record<string, StatsCoverImage | null>; anime: Record<number, StatsCoverImage | null>;
media: Record<string, StatsCoverImage | null>; media: Record<number, StatsCoverImage | null>;
} }
export interface KanjiEntry { export interface KanjiEntry {