mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-29 19:21:33 -07:00
fix(stats): persist filter prefs, fix cover MIME types, and dedup alass
- Remember Hide Known/Hide Kana filter state in localStorage across sessions - Detect PNG and WebP cover art MIME types instead of hardcoding image/jpeg - Use configured AnkiConnect URL for the browse action - Deduplicate concurrent in-flight alass retime calls via promise caching - Prefer request-provided secondary subtitle text over retimed sidecar fallback - Fix cover image record key types from string to number
This commit is contained in:
@@ -2,7 +2,7 @@ type: changed
|
||||
area: stats
|
||||
|
||||
- Added the Stats Search tab for realtime subtitle sentence search with media context, headword matching, and mining actions for source-backed sentence cards or exact-match word/audio cards.
|
||||
- Improved Stats mining from Search and vocabulary examples: empty `ankiConnect.deck` can use Yomitan's mining deck, sentence cards are created before slow media generation finishes, secondary subtitles from stored lines, sidecar files, or temporary alass-retimed English sidecars populate sentence Selection Text, invalid stored timings are blocked before FFmpeg runs, future out-of-order subtitle timing pairs are skipped until valid timings arrive, and partial media failures are shown.
|
||||
- Improved Stats mining from Search and vocabulary examples: empty `ankiConnect.deck` can use Yomitan's mining deck, sentence cards are created before slow media generation finishes, stored/requested secondary subtitles are preserved before falling back to sidecar files or temporary alass-retimed English sidecars for sentence Selection Text, invalid stored timings are blocked before FFmpeg runs, future out-of-order subtitle timing pairs are skipped until valid timings arrive, and partial media failures are shown.
|
||||
- Fixed Stats mining field/audio behavior so sentence clips update `SentenceAudio`, word audio uses the configured Yomitan sources, English subtitle text is not written onto word cards, and secondary subtitle auto-selection prefers regular English tracks over Signs/Songs tracks.
|
||||
- Improved vocabulary review with a Hide Kana filter, duplicate-collapsed exclusions across token variants, and Related Seen Words matching based on shared readings or kanji.
|
||||
- Improved Stats browsing reliability by remembering library card size, retrying stored cover art without extra AniList lookups, showing progress during session deletes, and making session deletes refresh faster.
|
||||
- Improved vocabulary review with remembered Hide Known/Hide Kana filters, duplicate-collapsed exclusions across token variants, and Related Seen Words matching based on shared readings or kanji.
|
||||
- Improved Stats browsing reliability by remembering library card size, retrying stored cover art without extra AniList lookups, preserving PNG/WebP cover MIME types, honoring custom AnkiConnect URLs for Browse, showing progress during session deletes, and making session deletes refresh faster.
|
||||
|
||||
@@ -966,7 +966,7 @@ describe('stats server API routes', () => {
|
||||
videoId,
|
||||
anilistId: null,
|
||||
coverUrl: null,
|
||||
coverBlob: Buffer.from([0x89, 0x50]),
|
||||
coverBlob: Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
episodesTotal: null,
|
||||
@@ -997,8 +997,8 @@ describe('stats server API routes', () => {
|
||||
},
|
||||
media: {
|
||||
7: {
|
||||
contentType: 'image/jpeg',
|
||||
dataUrl: 'data:image/jpeg;base64,iVA=',
|
||||
contentType: 'image/png',
|
||||
dataUrl: 'data:image/png;base64,iVBORw0KGgo=',
|
||||
},
|
||||
99999: null,
|
||||
},
|
||||
@@ -1365,7 +1365,7 @@ describe('stats server API routes', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('POST /api/stats/mine-card prefers retimed sidecar secondary text for sentence cards', async () => {
|
||||
it('POST /api/stats/mine-card prefers request secondary text over retimed fallback', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
fs.writeFileSync(sourcePath, 'fake media');
|
||||
@@ -1414,7 +1414,7 @@ describe('stats server API routes', () => {
|
||||
const addNoteRequest = requests.find((request) => request.action === 'addNote');
|
||||
assert.equal(
|
||||
addNoteRequest?.params?.note?.fields?.SelectionText,
|
||||
'Aligned English subtitle',
|
||||
'Stale stored English subtitle',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1484,6 +1484,74 @@ Aligned English subtitle
|
||||
});
|
||||
});
|
||||
|
||||
it('shares in-flight retimed secondary subtitle work for concurrent requests', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
const japanesePath = path.join(dir, 'episode.ja.srt');
|
||||
const englishPath = path.join(dir, 'episode.en.srt');
|
||||
const alassPath = path.join(dir, 'alass-cli');
|
||||
fs.writeFileSync(sourcePath, 'fake media');
|
||||
fs.writeFileSync(alassPath, 'fake alass');
|
||||
fs.writeFileSync(
|
||||
japanesePath,
|
||||
`1
|
||||
00:00:01,000 --> 00:00:02,000
|
||||
猫を見た
|
||||
`,
|
||||
);
|
||||
fs.writeFileSync(
|
||||
englishPath,
|
||||
`1
|
||||
00:00:09,000 --> 00:00:10,000
|
||||
Stale English subtitle
|
||||
`,
|
||||
);
|
||||
|
||||
let alassRuns = 0;
|
||||
let releaseAlass!: () => void;
|
||||
const alassGate = new Promise<void>((resolve) => {
|
||||
releaseAlass = resolve;
|
||||
});
|
||||
const input = {
|
||||
sourcePath,
|
||||
startMs: 1_000,
|
||||
endMs: 2_000,
|
||||
alassPath,
|
||||
runAlass: async (
|
||||
_alassPath: string,
|
||||
_referencePath: string,
|
||||
_inputPath: string,
|
||||
outputPath: string,
|
||||
) => {
|
||||
alassRuns += 1;
|
||||
await alassGate;
|
||||
fs.writeFileSync(
|
||||
outputPath,
|
||||
`1
|
||||
00:00:01,000 --> 00:00:02,000
|
||||
Aligned English subtitle
|
||||
`,
|
||||
);
|
||||
return { ok: true, code: 0, stdout: '', stderr: '' };
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
const first = resolveRetimedSecondarySubtitleTextFromSidecar(input);
|
||||
const second = resolveRetimedSecondarySubtitleTextFromSidecar(input);
|
||||
releaseAlass();
|
||||
|
||||
assert.deepEqual(await Promise.all([first, second]), [
|
||||
'Aligned English subtitle',
|
||||
'Aligned English subtitle',
|
||||
]);
|
||||
assert.equal(alassRuns, 1);
|
||||
} finally {
|
||||
clearRetimedSecondarySubtitleCache();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it('POST /api/stats/mine-card adds direct sentence cards before slow media finishes', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
@@ -2387,6 +2455,20 @@ Aligned English subtitle
|
||||
assert.equal(res.status, 400);
|
||||
});
|
||||
|
||||
it('POST /api/stats/anki/browse uses configured AnkiConnect URL', async () => {
|
||||
await withFakeAnkiConnect(async (requests, url) => {
|
||||
const app = createStatsApp(createMockTracker(), {
|
||||
ankiConnectConfig: { url },
|
||||
});
|
||||
|
||||
const res = await app.request('/api/stats/anki/browse?noteId=12345', { method: 'POST' });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(requests[0]?.action, 'guiBrowse');
|
||||
assert.deepEqual(requests[0]?.params, { query: 'nid:12345' });
|
||||
});
|
||||
});
|
||||
|
||||
it('GET /api/stats/anilist/search uses the configured AniList rate limiter', async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let acquireCalls = 0;
|
||||
|
||||
@@ -29,6 +29,7 @@ type SidecarCandidate = {
|
||||
type RetimedSubtitleCacheEntry = {
|
||||
path: string;
|
||||
cleanupDir: string;
|
||||
promise?: Promise<string>;
|
||||
};
|
||||
|
||||
export type RetimedSubtitleCommandRunner = (
|
||||
@@ -353,6 +354,9 @@ async function retimeSecondarySubtitle(input: {
|
||||
if (!key) return '';
|
||||
|
||||
const cached = retimedSubtitleCache.get(key);
|
||||
if (cached?.promise) {
|
||||
return cached.promise;
|
||||
}
|
||||
if (cached && existsSync(cached.path)) {
|
||||
return cached.path;
|
||||
}
|
||||
@@ -371,19 +375,25 @@ async function retimeSecondarySubtitle(input: {
|
||||
`${parsedSecondary.name}.retimed${parsedSecondary.ext || '.srt'}`,
|
||||
);
|
||||
|
||||
const result = await input.runAlass(
|
||||
input.alassPath,
|
||||
input.primaryPath,
|
||||
input.secondaryPath,
|
||||
outputPath,
|
||||
);
|
||||
if (!result.ok || !existsSync(outputPath)) {
|
||||
rmSync(cleanupDir, { recursive: true, force: true });
|
||||
return '';
|
||||
}
|
||||
|
||||
retimedSubtitleCache.set(key, { path: outputPath, cleanupDir });
|
||||
return outputPath;
|
||||
const entry: RetimedSubtitleCacheEntry = { path: outputPath, cleanupDir };
|
||||
entry.promise = input
|
||||
.runAlass(input.alassPath, input.primaryPath, input.secondaryPath, outputPath)
|
||||
.then((result) => {
|
||||
if (!result.ok || !existsSync(outputPath)) {
|
||||
rmSync(cleanupDir, { recursive: true, force: true });
|
||||
retimedSubtitleCache.delete(key);
|
||||
return '';
|
||||
}
|
||||
entry.promise = undefined;
|
||||
return outputPath;
|
||||
})
|
||||
.catch(() => {
|
||||
rmSync(cleanupDir, { recursive: true, force: true });
|
||||
retimedSubtitleCache.delete(key);
|
||||
return '';
|
||||
});
|
||||
retimedSubtitleCache.set(key, entry);
|
||||
return entry.promise;
|
||||
}
|
||||
|
||||
export function resolveSecondarySubtitleTextFromSidecar(input: {
|
||||
|
||||
@@ -52,7 +52,7 @@ type StatsExcludedWordPayload = {
|
||||
};
|
||||
|
||||
type StatsCoverImagePayload = {
|
||||
contentType: 'image/jpeg';
|
||||
contentType: string;
|
||||
dataUrl: string;
|
||||
} | null;
|
||||
|
||||
@@ -126,12 +126,43 @@ function coverImagePayload(
|
||||
art: { coverBlob?: Uint8Array | null } | null | undefined,
|
||||
): StatsCoverImagePayload {
|
||||
if (!art?.coverBlob) return null;
|
||||
const bytes = new Uint8Array(art.coverBlob);
|
||||
const contentType = detectImageContentType(bytes);
|
||||
return {
|
||||
contentType: 'image/jpeg',
|
||||
dataUrl: `data:image/jpeg;base64,${Buffer.from(art.coverBlob).toString('base64')}`,
|
||||
contentType,
|
||||
dataUrl: `data:${contentType};base64,${Buffer.from(bytes).toString('base64')}`,
|
||||
};
|
||||
}
|
||||
|
||||
function detectImageContentType(bytes: Uint8Array): string {
|
||||
if (
|
||||
bytes.length >= 8 &&
|
||||
bytes[0] === 0x89 &&
|
||||
bytes[1] === 0x50 &&
|
||||
bytes[2] === 0x4e &&
|
||||
bytes[3] === 0x47
|
||||
) {
|
||||
return 'image/png';
|
||||
}
|
||||
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
|
||||
return 'image/jpeg';
|
||||
}
|
||||
if (
|
||||
bytes.length >= 12 &&
|
||||
bytes[0] === 0x52 &&
|
||||
bytes[1] === 0x49 &&
|
||||
bytes[2] === 0x46 &&
|
||||
bytes[3] === 0x46 &&
|
||||
bytes[8] === 0x57 &&
|
||||
bytes[9] === 0x45 &&
|
||||
bytes[10] === 0x42 &&
|
||||
bytes[11] === 0x50
|
||||
) {
|
||||
return 'image/webp';
|
||||
}
|
||||
return 'application/octet-stream';
|
||||
}
|
||||
|
||||
function resolveStatsNoteFieldName(
|
||||
noteInfo: StatsServerNoteInfo,
|
||||
...preferredNames: (string | undefined)[]
|
||||
@@ -960,17 +991,17 @@ export function createStatsApp(
|
||||
const body = (await c.req.json().catch(() => null)) as StatsCoverBatchBody | null;
|
||||
const animeIds = parsePositiveIdList(body?.animeIds);
|
||||
const videoIds = parsePositiveIdList(body?.videoIds);
|
||||
const anime: Record<string, StatsCoverImagePayload> = {};
|
||||
const media: Record<string, StatsCoverImagePayload> = {};
|
||||
const anime: Record<number, StatsCoverImagePayload> = {};
|
||||
const media: Record<number, StatsCoverImagePayload> = {};
|
||||
|
||||
await Promise.all(
|
||||
animeIds.map(async (animeId) => {
|
||||
anime[String(animeId)] = coverImagePayload(await tracker.getAnimeCoverArt(animeId));
|
||||
anime[animeId] = coverImagePayload(await tracker.getAnimeCoverArt(animeId));
|
||||
}),
|
||||
);
|
||||
await Promise.all(
|
||||
videoIds.map(async (videoId) => {
|
||||
media[String(videoId)] = coverImagePayload(await tracker.getCoverArt(videoId));
|
||||
media[videoId] = coverImagePayload(await tracker.getCoverArt(videoId));
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -1024,8 +1055,9 @@ export function createStatsApp(
|
||||
app.post('/api/stats/anki/browse', async (c) => {
|
||||
const noteId = parseIntQuery(c.req.query('noteId'), 0);
|
||||
if (noteId <= 0) return c.body(null, 400);
|
||||
const ankiConfig = getAnkiConnectConfig();
|
||||
try {
|
||||
const response = await fetch('http://127.0.0.1:8765', {
|
||||
const response = await fetch(ankiConfig?.url ?? 'http://127.0.0.1:8765', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
signal: AbortSignal.timeout(ANKI_CONNECT_FETCH_TIMEOUT_MS),
|
||||
@@ -1136,8 +1168,8 @@ export function createStatsApp(
|
||||
}
|
||||
}
|
||||
const secondaryText =
|
||||
retimedSecondaryText ||
|
||||
bodySecondaryText ||
|
||||
retimedSecondaryText ||
|
||||
resolveSecondarySubtitleTextFromSidecar({
|
||||
sourcePath,
|
||||
startMs,
|
||||
|
||||
@@ -24,6 +24,46 @@ function makeEntry(over: Partial<VocabularyEntry>): VocabularyEntry {
|
||||
} as VocabularyEntry;
|
||||
}
|
||||
|
||||
function withLocalStorage<T>(initial: Record<string, string>, run: () => T): T {
|
||||
const previous = Object.getOwnPropertyDescriptor(globalThis, 'localStorage');
|
||||
const values = new Map(Object.entries(initial));
|
||||
const storage = {
|
||||
get length() {
|
||||
return values.size;
|
||||
},
|
||||
clear() {
|
||||
values.clear();
|
||||
},
|
||||
getItem(key: string) {
|
||||
return values.get(key) ?? null;
|
||||
},
|
||||
key(index: number) {
|
||||
return Array.from(values.keys())[index] ?? null;
|
||||
},
|
||||
removeItem(key: string) {
|
||||
values.delete(key);
|
||||
},
|
||||
setItem(key: string, value: string) {
|
||||
values.set(key, value);
|
||||
},
|
||||
} as Storage;
|
||||
|
||||
Object.defineProperty(globalThis, 'localStorage', {
|
||||
configurable: true,
|
||||
value: storage,
|
||||
});
|
||||
|
||||
try {
|
||||
return run();
|
||||
} finally {
|
||||
if (previous) {
|
||||
Object.defineProperty(globalThis, 'localStorage', previous);
|
||||
} else {
|
||||
delete (globalThis as { localStorage?: unknown }).localStorage;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
test('renders headword and reading inline in a single column (no separate Reading header)', () => {
|
||||
const entry = makeEntry({});
|
||||
const markup = renderToStaticMarkup(
|
||||
@@ -84,3 +124,40 @@ test('renders a Hide Kana filter button', () => {
|
||||
);
|
||||
assert.match(markup, /Hide Kana/);
|
||||
});
|
||||
|
||||
test('uses saved Hide Kana preference on first render', () => {
|
||||
const markup = withLocalStorage({ 'subminer.stats.frequencyRank.hideKanaOnly': 'true' }, () =>
|
||||
renderToStaticMarkup(
|
||||
<FrequencyRankTable
|
||||
words={[
|
||||
makeEntry({ wordId: 1, headword: 'さらに', word: 'さらに', frequencyRank: 10 }),
|
||||
makeEntry({
|
||||
wordId: 2,
|
||||
headword: '前に',
|
||||
word: '前に',
|
||||
reading: 'まえに',
|
||||
frequencyRank: 20,
|
||||
}),
|
||||
]}
|
||||
knownWords={new Set()}
|
||||
/>,
|
||||
),
|
||||
);
|
||||
|
||||
assert.doesNotMatch(markup, />さらに</);
|
||||
assert.match(markup, />前に</);
|
||||
});
|
||||
|
||||
test('uses saved Hide Known preference on first render', () => {
|
||||
const markup = withLocalStorage({ 'subminer.stats.frequencyRank.hideKnown': 'false' }, () =>
|
||||
renderToStaticMarkup(
|
||||
<FrequencyRankTable
|
||||
words={[makeEntry({ headword: '日本語', word: '日本語', frequencyRank: 10 })]}
|
||||
knownWords={new Set(['日本語'])}
|
||||
/>,
|
||||
),
|
||||
);
|
||||
|
||||
assert.match(markup, />Most Common Words Seen</);
|
||||
assert.match(markup, />日本語</);
|
||||
});
|
||||
|
||||
@@ -10,6 +10,8 @@ interface FrequencyRankTableProps {
|
||||
}
|
||||
|
||||
const PAGE_SIZE = 25;
|
||||
const HIDE_KNOWN_STORAGE_KEY = 'subminer.stats.frequencyRank.hideKnown';
|
||||
const HIDE_KANA_ONLY_STORAGE_KEY = 'subminer.stats.frequencyRank.hideKanaOnly';
|
||||
|
||||
interface FrequencyRankOptions {
|
||||
hideKnown: boolean;
|
||||
@@ -31,6 +33,33 @@ function isKanaOnlyWord(w: VocabularyEntry): boolean {
|
||||
return isKanaOnlyTokenText(w.headword || w.word);
|
||||
}
|
||||
|
||||
function getPreferenceStorage(): Storage | null {
|
||||
try {
|
||||
return globalThis.localStorage ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function readBooleanPreference(key: string, fallback: boolean): boolean {
|
||||
try {
|
||||
const value = getPreferenceStorage()?.getItem(key);
|
||||
if (value === 'true') return true;
|
||||
if (value === 'false') return false;
|
||||
return fallback;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
function writeBooleanPreference(key: string, value: boolean): void {
|
||||
try {
|
||||
getPreferenceStorage()?.setItem(key, String(value));
|
||||
} catch {
|
||||
// Storage can be blocked in private/restricted contexts; keep the in-memory choice.
|
||||
}
|
||||
}
|
||||
|
||||
export function buildFrequencyRankRows(
|
||||
words: VocabularyEntry[],
|
||||
knownWords: Set<string>,
|
||||
@@ -70,8 +99,12 @@ export function buildFrequencyRankRows(
|
||||
|
||||
export function FrequencyRankTable({ words, knownWords, onSelectWord }: FrequencyRankTableProps) {
|
||||
const [page, setPage] = useState(0);
|
||||
const [hideKnown, setHideKnown] = useState(true);
|
||||
const [hideKanaOnly, setHideKanaOnly] = useState(false);
|
||||
const [hideKnown, setHideKnown] = useState(() =>
|
||||
readBooleanPreference(HIDE_KNOWN_STORAGE_KEY, true),
|
||||
);
|
||||
const [hideKanaOnly, setHideKanaOnly] = useState(() =>
|
||||
readBooleanPreference(HIDE_KANA_ONLY_STORAGE_KEY, false),
|
||||
);
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
|
||||
const hasKnownData = knownWords.size > 0;
|
||||
@@ -116,7 +149,9 @@ export function FrequencyRankTable({ words, knownWords, onSelectWord }: Frequenc
|
||||
type="button"
|
||||
aria-pressed={hideKnown}
|
||||
onClick={() => {
|
||||
setHideKnown(!hideKnown);
|
||||
const next = !hideKnown;
|
||||
setHideKnown(next);
|
||||
writeBooleanPreference(HIDE_KNOWN_STORAGE_KEY, next);
|
||||
setPage(0);
|
||||
}}
|
||||
className={`px-2.5 py-1 rounded-lg text-xs transition-colors border ${
|
||||
@@ -132,7 +167,9 @@ export function FrequencyRankTable({ words, knownWords, onSelectWord }: Frequenc
|
||||
type="button"
|
||||
aria-pressed={hideKanaOnly}
|
||||
onClick={() => {
|
||||
setHideKanaOnly(!hideKanaOnly);
|
||||
const next = !hideKanaOnly;
|
||||
setHideKanaOnly(next);
|
||||
writeBooleanPreference(HIDE_KANA_ONLY_STORAGE_KEY, next);
|
||||
setPage(0);
|
||||
}}
|
||||
className={`px-2.5 py-1 rounded-lg text-xs transition-colors border ${
|
||||
@@ -150,7 +187,7 @@ export function FrequencyRankTable({ words, knownWords, onSelectWord }: Frequenc
|
||||
<div className="text-xs text-ctp-overlay2 mt-3">
|
||||
{hideKnown && hasKnownData && !hideKanaOnly
|
||||
? 'All ranked words are already in Anki!'
|
||||
: hideKnown || hideKanaOnly
|
||||
: (hideKnown && hasKnownData) || hideKanaOnly
|
||||
? 'No ranked words match the active filters.'
|
||||
: 'No words with frequency data.'}
|
||||
</div>
|
||||
|
||||
@@ -88,8 +88,8 @@ export interface StatsCoverImage {
|
||||
}
|
||||
|
||||
export interface StatsCoverImagesData {
|
||||
anime: Record<string, StatsCoverImage | null>;
|
||||
media: Record<string, StatsCoverImage | null>;
|
||||
anime: Record<number, StatsCoverImage | null>;
|
||||
media: Record<number, StatsCoverImage | null>;
|
||||
}
|
||||
|
||||
export interface KanjiEntry {
|
||||
|
||||
Reference in New Issue
Block a user