mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-05-28 12:55:17 -07:00
fix(character-dictionary): cache AniList media resolution to skip repeat
- Add anilist-resolution-cache.json to persist seriesKey→mediaId mappings - Skip AniList search when a cached resolution or matching snapshot exists - Expose readCachedSnapshots and readCachedMediaResolution from cache module
This commit is contained in:
@@ -0,0 +1,4 @@
|
|||||||
|
type: fixed
|
||||||
|
area: character-dictionary
|
||||||
|
|
||||||
|
- Reused cached character-dictionary media matches so loading a title with an existing snapshot no longer sends another AniList search request.
|
||||||
@@ -35,7 +35,7 @@ Character dictionary sync is disabled by default. To turn it on:
|
|||||||
```
|
```
|
||||||
|
|
||||||
::: tip
|
::: tip
|
||||||
The first sync for a media title takes a few seconds while character data and portraits are fetched from AniList. Subsequent launches reuse the cached snapshot.
|
The first sync for a media title takes a few seconds while character data and portraits are fetched from AniList. Subsequent launches reuse the cached media match and snapshot without a fresh AniList lookup.
|
||||||
:::
|
:::
|
||||||
|
|
||||||
::: warning
|
::: warning
|
||||||
@@ -139,7 +139,7 @@ When `characterDictionary.enabled` is `true`, SubMiner runs an auto-sync routine
|
|||||||
5. **importing** — Push the ZIP into Yomitan. Waits for Yomitan mutation readiness (7-second timeout per operation).
|
5. **importing** — Push the ZIP into Yomitan. Waits for Yomitan mutation readiness (7-second timeout per operation).
|
||||||
6. **ready** — Dictionary is live. Character names will match on the next subtitle line.
|
6. **ready** — Dictionary is live. Character names will match on the next subtitle line.
|
||||||
|
|
||||||
**State tracking** is persisted in `character-dictionaries/auto-sync-state.json`:
|
**State tracking** is persisted in `character-dictionaries/auto-sync-state.json`. AniList media matches are cached separately in `character-dictionaries/anilist-resolution-cache.json` so snapshot hits do not need another AniList search.
|
||||||
|
|
||||||
```jsonc
|
```jsonc
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1459,7 +1459,7 @@ test('generateForCurrentMedia preserves duplicate surface forms across different
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('getOrCreateCurrentSnapshot persists and reuses normalized snapshot data', async () => {
|
test('getOrCreateCurrentSnapshot reuses cached media resolution without AniList requests', async () => {
|
||||||
const userDataPath = makeTempDir();
|
const userDataPath = makeTempDir();
|
||||||
const originalFetch = globalThis.fetch;
|
const originalFetch = globalThis.fetch;
|
||||||
let searchQueryCount = 0;
|
let searchQueryCount = 0;
|
||||||
@@ -1567,11 +1567,18 @@ test('getOrCreateCurrentSnapshot persists and reuses normalized snapshot data',
|
|||||||
});
|
});
|
||||||
|
|
||||||
const first = await runtime.getOrCreateCurrentSnapshot();
|
const first = await runtime.getOrCreateCurrentSnapshot();
|
||||||
|
assert.equal(searchQueryCount, 1);
|
||||||
|
assert.equal(characterQueryCount, 1);
|
||||||
|
|
||||||
|
fs.rmSync(path.join(userDataPath, 'character-dictionaries', 'anilist-resolution-cache.json'), {
|
||||||
|
force: true,
|
||||||
|
});
|
||||||
|
|
||||||
const second = await runtime.getOrCreateCurrentSnapshot();
|
const second = await runtime.getOrCreateCurrentSnapshot();
|
||||||
|
|
||||||
assert.equal(first.fromCache, false);
|
assert.equal(first.fromCache, false);
|
||||||
assert.equal(second.fromCache, true);
|
assert.equal(second.fromCache, true);
|
||||||
assert.equal(searchQueryCount, 2);
|
assert.equal(searchQueryCount, 1);
|
||||||
assert.equal(characterQueryCount, 1);
|
assert.equal(characterQueryCount, 1);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
fs.existsSync(path.join(userDataPath, 'character-dictionaries', 'cache.json')),
|
fs.existsSync(path.join(userDataPath, 'character-dictionaries', 'cache.json')),
|
||||||
|
|||||||
@@ -15,7 +15,10 @@ import {
|
|||||||
getMergedZipPath,
|
getMergedZipPath,
|
||||||
getSnapshotPath,
|
getSnapshotPath,
|
||||||
normalizeMergedMediaIds,
|
normalizeMergedMediaIds,
|
||||||
|
readCachedMediaResolution,
|
||||||
|
readCachedSnapshots,
|
||||||
readSnapshot,
|
readSnapshot,
|
||||||
|
writeCachedMediaResolution,
|
||||||
writeSnapshot,
|
writeSnapshot,
|
||||||
} from './character-dictionary-runtime/cache';
|
} from './character-dictionary-runtime/cache';
|
||||||
import {
|
import {
|
||||||
@@ -41,6 +44,7 @@ import type {
|
|||||||
CharacterDictionaryManualSelectionResult,
|
CharacterDictionaryManualSelectionResult,
|
||||||
CharacterDictionaryManualSelectionSnapshot,
|
CharacterDictionaryManualSelectionSnapshot,
|
||||||
CharacterDictionaryRuntimeDeps,
|
CharacterDictionaryRuntimeDeps,
|
||||||
|
CharacterDictionarySnapshot,
|
||||||
CharacterDictionarySnapshotImage,
|
CharacterDictionarySnapshotImage,
|
||||||
CharacterDictionarySnapshotProgress,
|
CharacterDictionarySnapshotProgress,
|
||||||
CharacterDictionarySnapshotProgressCallbacks,
|
CharacterDictionarySnapshotProgressCallbacks,
|
||||||
@@ -204,6 +208,26 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const findCachedSnapshotForSeriesKey = (
|
||||||
|
seriesKey: string,
|
||||||
|
): CharacterDictionarySnapshot | null => {
|
||||||
|
return (
|
||||||
|
readCachedSnapshots(outputDir).find((snapshot) => {
|
||||||
|
const snapshotSeriesKey = buildCharacterDictionarySeriesKey({
|
||||||
|
mediaPath: null,
|
||||||
|
mediaTitle: snapshot.mediaTitle,
|
||||||
|
guess: {
|
||||||
|
title: snapshot.mediaTitle,
|
||||||
|
season: null,
|
||||||
|
episode: null,
|
||||||
|
source: 'fallback',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return snapshotSeriesKey === seriesKey;
|
||||||
|
}) ?? null
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
const resolveCurrentMedia = async (
|
const resolveCurrentMedia = async (
|
||||||
targetPath?: string,
|
targetPath?: string,
|
||||||
beforeRequest?: () => Promise<void>,
|
beforeRequest?: () => Promise<void>,
|
||||||
@@ -228,7 +252,43 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
|
|||||||
staleMediaIds: override.staleMediaIds,
|
staleMediaIds: override.staleMediaIds,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const cachedResolution = readCachedMediaResolution(outputDir, seriesKey);
|
||||||
|
if (cachedResolution) {
|
||||||
|
const cachedSnapshot = readSnapshot(getSnapshotPath(outputDir, cachedResolution.mediaId));
|
||||||
|
if (cachedSnapshot) {
|
||||||
|
deps.logInfo?.(
|
||||||
|
`[dictionary] cached AniList match: ${cachedSnapshot.mediaTitle} -> AniList ${cachedSnapshot.mediaId}`,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
id: cachedSnapshot.mediaId,
|
||||||
|
title: cachedSnapshot.mediaTitle,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const cachedSnapshot = findCachedSnapshotForSeriesKey(seriesKey);
|
||||||
|
if (cachedSnapshot) {
|
||||||
|
writeCachedMediaResolution(outputDir, {
|
||||||
|
seriesKey,
|
||||||
|
mediaId: cachedSnapshot.mediaId,
|
||||||
|
mediaTitle: cachedSnapshot.mediaTitle,
|
||||||
|
});
|
||||||
|
deps.logInfo?.(
|
||||||
|
`[dictionary] cached snapshot match: ${cachedSnapshot.mediaTitle} -> AniList ${cachedSnapshot.mediaId}`,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
id: cachedSnapshot.mediaId,
|
||||||
|
title: cachedSnapshot.mediaTitle,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const resolved = await resolveAniListMediaIdFromGuess(guessed, beforeRequest);
|
const resolved = await resolveAniListMediaIdFromGuess(guessed, beforeRequest);
|
||||||
|
writeCachedMediaResolution(outputDir, {
|
||||||
|
seriesKey,
|
||||||
|
mediaId: resolved.id,
|
||||||
|
mediaTitle: resolved.title,
|
||||||
|
});
|
||||||
deps.logInfo?.(`[dictionary] AniList match: ${resolved.title} -> AniList ${resolved.id}`);
|
deps.logInfo?.(`[dictionary] AniList match: ${resolved.title} -> AniList ${resolved.id}`);
|
||||||
return resolved;
|
return resolved;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -21,6 +21,102 @@ export function getMergedZipPath(outputDir: string): string {
|
|||||||
return path.join(outputDir, 'merged.zip');
|
return path.join(outputDir, 'merged.zip');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type MediaResolutionCacheEntry = {
|
||||||
|
seriesKey: string;
|
||||||
|
mediaId: number;
|
||||||
|
mediaTitle: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type MediaResolutionCacheFile = {
|
||||||
|
entries?: MediaResolutionCacheEntry[];
|
||||||
|
};
|
||||||
|
|
||||||
|
function getMediaResolutionCachePath(outputDir: string): string {
|
||||||
|
return path.join(outputDir, 'anilist-resolution-cache.json');
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeMediaResolutionEntry(value: unknown): MediaResolutionCacheEntry | null {
|
||||||
|
if (!value || typeof value !== 'object') return null;
|
||||||
|
const raw = value as Partial<MediaResolutionCacheEntry>;
|
||||||
|
const seriesKey = typeof raw.seriesKey === 'string' ? raw.seriesKey.trim() : '';
|
||||||
|
const mediaTitle = typeof raw.mediaTitle === 'string' ? raw.mediaTitle.trim() : '';
|
||||||
|
if (typeof raw.mediaId !== 'number' || !Number.isFinite(raw.mediaId)) return null;
|
||||||
|
const mediaId = Math.floor(raw.mediaId);
|
||||||
|
if (!seriesKey || mediaId <= 0 || !mediaTitle) return null;
|
||||||
|
return {
|
||||||
|
seriesKey,
|
||||||
|
mediaId,
|
||||||
|
mediaTitle,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function readMediaResolutionEntries(outputDir: string): MediaResolutionCacheEntry[] {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(
|
||||||
|
fs.readFileSync(getMediaResolutionCachePath(outputDir), 'utf8'),
|
||||||
|
) as MediaResolutionCacheFile;
|
||||||
|
if (!Array.isArray(parsed.entries)) return [];
|
||||||
|
const byKey = new Map<string, MediaResolutionCacheEntry>();
|
||||||
|
for (const value of parsed.entries) {
|
||||||
|
const normalized = normalizeMediaResolutionEntry(value);
|
||||||
|
if (normalized) byKey.set(normalized.seriesKey, normalized);
|
||||||
|
}
|
||||||
|
return [...byKey.values()];
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeMediaResolutionEntries(
|
||||||
|
outputDir: string,
|
||||||
|
entries: MediaResolutionCacheEntry[],
|
||||||
|
): void {
|
||||||
|
ensureDir(outputDir);
|
||||||
|
fs.writeFileSync(
|
||||||
|
getMediaResolutionCachePath(outputDir),
|
||||||
|
JSON.stringify({ entries }, null, 2),
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function readCachedMediaResolution(
|
||||||
|
outputDir: string,
|
||||||
|
seriesKey: string,
|
||||||
|
): MediaResolutionCacheEntry | null {
|
||||||
|
const normalizedKey = seriesKey.trim();
|
||||||
|
if (!normalizedKey) return null;
|
||||||
|
return (
|
||||||
|
readMediaResolutionEntries(outputDir).find((entry) => entry.seriesKey === normalizedKey) ?? null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function writeCachedMediaResolution(
|
||||||
|
outputDir: string,
|
||||||
|
entry: MediaResolutionCacheEntry,
|
||||||
|
): void {
|
||||||
|
const normalized = normalizeMediaResolutionEntry(entry);
|
||||||
|
if (!normalized) return;
|
||||||
|
const remaining = readMediaResolutionEntries(outputDir).filter(
|
||||||
|
(existing) => existing.seriesKey !== normalized.seriesKey,
|
||||||
|
);
|
||||||
|
writeMediaResolutionEntries(outputDir, [...remaining, normalized]);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function readCachedSnapshots(outputDir: string): CharacterDictionarySnapshot[] {
|
||||||
|
let entries: fs.Dirent[] = [];
|
||||||
|
try {
|
||||||
|
entries = fs.readdirSync(getSnapshotsDir(outputDir), { withFileTypes: true });
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return entries
|
||||||
|
.filter((entry) => entry.isFile() && /^anilist-\d+\.json$/.test(entry.name))
|
||||||
|
.sort((left, right) => left.name.localeCompare(right.name))
|
||||||
|
.map((entry) => readSnapshot(path.join(getSnapshotsDir(outputDir), entry.name)))
|
||||||
|
.filter((snapshot): snapshot is CharacterDictionarySnapshot => snapshot !== null);
|
||||||
|
}
|
||||||
|
|
||||||
export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot | null {
|
export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot | null {
|
||||||
try {
|
try {
|
||||||
const raw = fs.readFileSync(snapshotPath, 'utf8');
|
const raw = fs.readFileSync(snapshotPath, 'utf8');
|
||||||
|
|||||||
Reference in New Issue
Block a user