mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-29 01:01:34 -07:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d253710c2e | |||
|
c3df510e4f
|
|||
| 187f68e5b6 | |||
|
0e254cbbef
|
@@ -1,4 +1,4 @@
|
|||||||
type: fixed
|
type: added
|
||||||
area: mining
|
area: mining
|
||||||
|
|
||||||
- Normalized generated card audio by default during media extraction, with `ankiConnect.media.normalizeAudio` available to keep raw source loudness when needed.
|
- Normalized generated card audio by default during media extraction, with `ankiConnect.media.normalizeAudio` available to keep raw source loudness when needed.
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
type: fixed
|
||||||
|
area: stats
|
||||||
|
|
||||||
|
- Cover art is now fetched eagerly when a new series starts playing, instead of waiting for the first visit to its series detail page, so the stats timeline shows the best-guess AniList image right away. The stats covers endpoint also backfills missing series art in the background, so existing series without an image pick one up on the next stats page load.
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
type: changed
|
||||||
|
area: launcher
|
||||||
|
|
||||||
|
- Moved fzf previews below launcher menus so long titles and metadata have more horizontal room.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
type: fixed
|
||||||
|
area: overlay
|
||||||
|
|
||||||
|
- Fixed single-kana grammar tokens counting as known words by borrowing the reading of an unrelated Anki note (よ in 全然いいよ matched a card read よ such as 夜, standalone え matched 絵), which painted them with the known-word highlight. Reading-only known-word matching now requires at least two kana; single-kana cards still match by their word field, and tokens genuinely present in the known-words cache (e.g. です) keep their highlight.
|
||||||
|
- Standalone suffix tokens (MeCab pos2 接尾, e.g. さん, れる) are now excluded from JLPT/frequency/N+1 annotations by default, matching how particles and interjections are treated. Cache-backed known-word highlighting still applies; override via the pos2 exclusion config if you want them annotated.
|
||||||
+2
-2
@@ -56,7 +56,7 @@ export function showFzfFlatMenu(
|
|||||||
`--prompt=${prompt}`,
|
`--prompt=${prompt}`,
|
||||||
'--delimiter=\t',
|
'--delimiter=\t',
|
||||||
'--with-nth=2',
|
'--with-nth=2',
|
||||||
'--preview-window=right:50%:wrap',
|
'--preview-window=down:50%:wrap',
|
||||||
'--preview',
|
'--preview',
|
||||||
previewCommand,
|
previewCommand,
|
||||||
];
|
];
|
||||||
@@ -468,7 +468,7 @@ thumb=$(get_thumb)
|
|||||||
'--prompt=Select Video: ',
|
'--prompt=Select Video: ',
|
||||||
'--delimiter=\t',
|
'--delimiter=\t',
|
||||||
'--with-nth=1',
|
'--with-nth=1',
|
||||||
'--preview-window=right:50%:wrap',
|
'--preview-window=down:50%:wrap',
|
||||||
'--preview',
|
'--preview',
|
||||||
previewCmd,
|
previewCmd,
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -7,6 +7,9 @@
|
|||||||
- **Watch History Browser**
|
- **Watch History Browser**
|
||||||
- New `subminer -H` / `--history` command to browse your local watch history, replay the last episode, jump to the next one, or pick an episode via fzf/rofi.
|
- New `subminer -H` / `--history` command to browse your local watch history, replay the last episode, jump to the next one, or pick an episode via fzf/rofi.
|
||||||
- The rofi picker now shows AniList cover art for each show, making it easier to spot the right title at a glance.
|
- The rofi picker now shows AniList cover art for each show, making it easier to spot the right title at a glance.
|
||||||
|
- **Card Audio Normalization**
|
||||||
|
- Audio extracted for Anki cards is now volume-normalized by default for more consistent playback loudness.
|
||||||
|
- If you prefer the original source volume, disable it via the new `ankiConnect.media.normalizeAudio` setting.
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
- **New App Icon**
|
- **New App Icon**
|
||||||
@@ -27,9 +30,6 @@
|
|||||||
- Fixed a duplicate "Field grouping cancelled" notification appearing when grouping was cancelled via the trigger shortcut.
|
- Fixed a duplicate "Field grouping cancelled" notification appearing when grouping was cancelled via the trigger shortcut.
|
||||||
- **Secondary Subtitles**
|
- **Secondary Subtitles**
|
||||||
- Karaoke-style secondary subtitles (common in opening/ending songs) no longer spam dozens of lines down the screen; repeated lines are now collapsed and the subtitle area is capped to a strip at the top.
|
- Karaoke-style secondary subtitles (common in opening/ending songs) no longer spam dozens of lines down the screen; repeated lines are now collapsed and the subtitle area is capped to a strip at the top.
|
||||||
- **Card Audio Normalization**
|
|
||||||
- Audio extracted for Anki cards is now volume-normalized by default for more consistent playback loudness.
|
|
||||||
- If you prefer the original source volume, disable it via the new `ankiConnect.media.normalizeAudio` setting.
|
|
||||||
- **YouTube Extraction**
|
- **YouTube Extraction**
|
||||||
- Fixed direct YouTube stream extraction occasionally corrupting the stream URL and causing failed audio/video capture.
|
- Fixed direct YouTube stream extraction occasionally corrupting the stream URL and causing failed audio/video capture.
|
||||||
- **Background Stats Server**
|
- **Background Stats Server**
|
||||||
|
|||||||
@@ -734,6 +734,49 @@ test('KnownWordCacheManager disambiguates known words by note reading', async ()
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('KnownWordCacheManager does not match single-kana text by reading alone', async () => {
|
||||||
|
const config: AnkiConnectConfig = {
|
||||||
|
fields: {
|
||||||
|
word: 'Word',
|
||||||
|
},
|
||||||
|
knownWords: {
|
||||||
|
highlightEnabled: true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
|
||||||
|
|
||||||
|
try {
|
||||||
|
clientState.findNotesResult = [1, 2];
|
||||||
|
clientState.notesInfoResult = [
|
||||||
|
{
|
||||||
|
noteId: 1,
|
||||||
|
fields: {
|
||||||
|
Word: { value: '夜' },
|
||||||
|
'Word Reading': { value: 'よ' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
noteId: 2,
|
||||||
|
fields: {
|
||||||
|
Word: { value: 'え' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
await manager.refresh(true);
|
||||||
|
|
||||||
|
// よ must not count as known just because 夜 is read よ.
|
||||||
|
assert.equal(manager.isKnownWord('よ'), false);
|
||||||
|
assert.equal(manager.isKnownWord('ヨ'), false);
|
||||||
|
assert.equal(manager.isKnownWord('夜'), true);
|
||||||
|
assert.equal(manager.isKnownWord('夜', 'よ'), true);
|
||||||
|
// A literal single-kana word entry still matches via the word map.
|
||||||
|
assert.equal(manager.isKnownWord('え'), true);
|
||||||
|
} finally {
|
||||||
|
cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('KnownWordCacheManager probes reading fields even with per-deck word fields configured', async () => {
|
test('KnownWordCacheManager probes reading fields even with per-deck word fields configured', async () => {
|
||||||
const config: AnkiConnectConfig = {
|
const config: AnkiConnectConfig = {
|
||||||
fields: {
|
fields: {
|
||||||
|
|||||||
@@ -163,7 +163,14 @@ export class KnownWordCacheManager {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.readingCounts.has(convertKatakanaToHiragana(normalized));
|
// Reading-only fallback, except for single-kana text: particles and
|
||||||
|
// interjections (よ, ね, え…) would otherwise borrow the reading of an
|
||||||
|
// unrelated note (夜「よ」, 絵「え」) and count as known.
|
||||||
|
const hiragana = convertKatakanaToHiragana(normalized);
|
||||||
|
if ([...hiragana].length === 1) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return this.readingCounts.has(hiragana);
|
||||||
}
|
}
|
||||||
|
|
||||||
refresh(force = false): Promise<void> {
|
refresh(force = false): Promise<void> {
|
||||||
|
|||||||
@@ -301,6 +301,7 @@ function createMockTracker(
|
|||||||
{ epochDay: Math.floor(Date.now() / 86_400_000) - 1, totalActiveMin: 30 },
|
{ epochDay: Math.floor(Date.now() / 86_400_000) - 1, totalActiveMin: 30 },
|
||||||
{ epochDay: Math.floor(Date.now() / 86_400_000), totalActiveMin: 45 },
|
{ epochDay: Math.floor(Date.now() / 86_400_000), totalActiveMin: 45 },
|
||||||
],
|
],
|
||||||
|
ensureAnimeCoverArt: async () => false,
|
||||||
getAnimeCoverArt: async (animeId: number) =>
|
getAnimeCoverArt: async (animeId: number) =>
|
||||||
animeId === 1
|
animeId === 1
|
||||||
? {
|
? {
|
||||||
@@ -994,8 +995,9 @@ describe('stats server API routes', () => {
|
|||||||
assert.equal(res.status, 404);
|
assert.equal(res.status, 404);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('POST /api/stats/covers batches stored cover art without fetching missing art', async () => {
|
it('POST /api/stats/covers batches stored cover art and backfills missing anime art in the background', async () => {
|
||||||
let ensureCoverArtCalls = 0;
|
let ensureCoverArtCalls = 0;
|
||||||
|
const ensureAnimeCoverArtCalls: number[] = [];
|
||||||
const app = createStatsApp(
|
const app = createStatsApp(
|
||||||
createMockTracker({
|
createMockTracker({
|
||||||
getCoverArt: async (videoId: number) =>
|
getCoverArt: async (videoId: number) =>
|
||||||
@@ -1015,6 +1017,10 @@ describe('stats server API routes', () => {
|
|||||||
ensureCoverArtCalls += 1;
|
ensureCoverArtCalls += 1;
|
||||||
return true;
|
return true;
|
||||||
},
|
},
|
||||||
|
ensureAnimeCoverArt: async (animeId: number) => {
|
||||||
|
ensureAnimeCoverArtCalls.push(animeId);
|
||||||
|
return false;
|
||||||
|
},
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -1042,6 +1048,68 @@ describe('stats server API routes', () => {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
assert.equal(ensureCoverArtCalls, 0);
|
assert.equal(ensureCoverArtCalls, 0);
|
||||||
|
assert.deepEqual(ensureAnimeCoverArtCalls, [99999]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('POST /api/stats/covers limits concurrent missing anime cover backfills', async () => {
|
||||||
|
let activeBackfills = 0;
|
||||||
|
let maxActiveBackfills = 0;
|
||||||
|
const pendingBackfills: Array<() => void> = [];
|
||||||
|
const app = createStatsApp(
|
||||||
|
createMockTracker({
|
||||||
|
getAnimeCoverArt: async () => null,
|
||||||
|
ensureAnimeCoverArt: async () => {
|
||||||
|
activeBackfills += 1;
|
||||||
|
maxActiveBackfills = Math.max(maxActiveBackfills, activeBackfills);
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
pendingBackfills.push(resolve);
|
||||||
|
});
|
||||||
|
activeBackfills -= 1;
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const res = await app.request('/api/stats/covers', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ animeIds: [101, 102, 103, 104, 105] }),
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(res.status, 200);
|
||||||
|
assert.equal(maxActiveBackfills, 3);
|
||||||
|
for (const resolveBackfill of pendingBackfills) {
|
||||||
|
resolveBackfill();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('GET /api/stats/anime/:animeId/cover fetches missing art before serving', async () => {
|
||||||
|
let fetched = false;
|
||||||
|
const app = createStatsApp(
|
||||||
|
createMockTracker({
|
||||||
|
getAnimeCoverArt: async () =>
|
||||||
|
fetched
|
||||||
|
? {
|
||||||
|
videoId: 1,
|
||||||
|
anilistId: 21858,
|
||||||
|
coverUrl: 'https://example.com/cover.jpg',
|
||||||
|
coverBlob: Buffer.from([0xff, 0xd8, 0xff, 0xd9]),
|
||||||
|
titleRomaji: 'Little Witch Academia',
|
||||||
|
titleEnglish: 'Little Witch Academia',
|
||||||
|
episodesTotal: 25,
|
||||||
|
fetchedAtMs: Date.now(),
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
ensureAnimeCoverArt: async () => {
|
||||||
|
fetched = true;
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const res = await app.request('/api/stats/anime/1/cover');
|
||||||
|
assert.equal(res.status, 200);
|
||||||
|
assert.equal(res.headers.get('content-type'), 'image/jpeg');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('GET /api/stats/anime/:animeId/words returns top words for an anime', async () => {
|
it('GET /api/stats/anime/:animeId/words returns top words for an anime', async () => {
|
||||||
|
|||||||
@@ -4041,3 +4041,91 @@ test('markActiveVideoWatched returns false when no active session', async () =>
|
|||||||
cleanupDbPath(dbPath);
|
cleanupDbPath(dbPath);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('handleMediaChange prefetches cover art at session start', async () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
let tracker: ImmersionTrackerService | null = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const Ctor = await loadTrackerCtor();
|
||||||
|
tracker = new Ctor({ dbPath });
|
||||||
|
|
||||||
|
const fetchedVideoIds: number[] = [];
|
||||||
|
tracker.setCoverArtFetcher({
|
||||||
|
fetchIfMissing: async (_db, videoId) => {
|
||||||
|
fetchedVideoIds.push(videoId);
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
tracker.handleMediaChange('/tmp/Little Witch Academia S02E05.mkv', 'Episode 5');
|
||||||
|
await waitForPendingAnimeMetadata(tracker);
|
||||||
|
await waitForCondition(() => fetchedVideoIds.length > 0);
|
||||||
|
|
||||||
|
const privateApi = tracker as unknown as {
|
||||||
|
sessionState: { videoId: number } | null;
|
||||||
|
};
|
||||||
|
assert.deepEqual(fetchedVideoIds, [privateApi.sessionState?.videoId]);
|
||||||
|
} finally {
|
||||||
|
tracker?.destroy();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('ensureAnimeCoverArt fetches art via the latest video of the anime', async () => {
|
||||||
|
const dbPath = makeDbPath();
|
||||||
|
let tracker: ImmersionTrackerService | null = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const Ctor = await loadTrackerCtor();
|
||||||
|
tracker = new Ctor({ dbPath });
|
||||||
|
const privateApi = tracker as unknown as { db: DatabaseSync };
|
||||||
|
|
||||||
|
privateApi.db.exec(`
|
||||||
|
INSERT INTO imm_anime (
|
||||||
|
anime_id,
|
||||||
|
normalized_title_key,
|
||||||
|
canonical_title,
|
||||||
|
CREATED_DATE,
|
||||||
|
LAST_UPDATE_DATE
|
||||||
|
) VALUES (
|
||||||
|
1,
|
||||||
|
'little witch academia',
|
||||||
|
'Little Witch Academia',
|
||||||
|
1000,
|
||||||
|
1000
|
||||||
|
);
|
||||||
|
INSERT INTO imm_videos (
|
||||||
|
video_id,
|
||||||
|
video_key,
|
||||||
|
canonical_title,
|
||||||
|
source_type,
|
||||||
|
duration_ms,
|
||||||
|
anime_id,
|
||||||
|
CREATED_DATE,
|
||||||
|
LAST_UPDATE_DATE
|
||||||
|
) VALUES
|
||||||
|
(1, 'local:/tmp/lwa-1.mkv', 'Little Witch Academia S01E01', 1, 0, 1, 1000, 1000),
|
||||||
|
(2, 'local:/tmp/lwa-2.mkv', 'Little Witch Academia S01E02', 1, 0, 1, 1000, 1000);
|
||||||
|
`);
|
||||||
|
|
||||||
|
const fetchedVideoIds: number[] = [];
|
||||||
|
tracker.setCoverArtFetcher({
|
||||||
|
fetchIfMissing: async (_db, videoId) => {
|
||||||
|
fetchedVideoIds.push(videoId);
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await tracker.ensureAnimeCoverArt(1);
|
||||||
|
assert.equal(result, false);
|
||||||
|
assert.deepEqual(fetchedVideoIds, [2]);
|
||||||
|
|
||||||
|
const missing = await tracker.ensureAnimeCoverArt(999);
|
||||||
|
assert.equal(missing, false);
|
||||||
|
assert.deepEqual(fetchedVideoIds, [2]);
|
||||||
|
} finally {
|
||||||
|
tracker?.destroy();
|
||||||
|
cleanupDbPath(dbPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|||||||
@@ -854,6 +854,22 @@ export class ImmersionTrackerService {
|
|||||||
this.coverArtFetcher = fetcher;
|
this.coverArtFetcher = fetcher;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async ensureAnimeCoverArt(animeId: number): Promise<boolean> {
|
||||||
|
const existing = await this.getAnimeCoverArt(animeId);
|
||||||
|
if (existing?.coverBlob) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const row = this.db
|
||||||
|
.prepare(
|
||||||
|
'SELECT video_id AS videoId FROM imm_videos WHERE anime_id = ? ORDER BY video_id DESC LIMIT 1',
|
||||||
|
)
|
||||||
|
.get(animeId) as { videoId: number } | undefined;
|
||||||
|
if (!row?.videoId) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return this.ensureCoverArt(row.videoId);
|
||||||
|
}
|
||||||
|
|
||||||
async ensureCoverArt(videoId: number): Promise<boolean> {
|
async ensureCoverArt(videoId: number): Promise<boolean> {
|
||||||
const existing = await this.getCoverArt(videoId);
|
const existing = await this.getCoverArt(videoId);
|
||||||
if (existing?.coverBlob) {
|
if (existing?.coverBlob) {
|
||||||
@@ -879,8 +895,10 @@ export class ImmersionTrackerService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const fetchPromise = (async () => {
|
const fetchPromise = (async () => {
|
||||||
const detail = getMediaDetail(this.db, videoId);
|
const titleRow = this.db
|
||||||
const canonicalTitle = detail?.canonicalTitle?.trim();
|
.prepare('SELECT canonical_title AS canonicalTitle FROM imm_videos WHERE video_id = ?')
|
||||||
|
.get(videoId) as { canonicalTitle: string | null } | undefined;
|
||||||
|
const canonicalTitle = titleRow?.canonicalTitle?.trim();
|
||||||
if (!canonicalTitle) {
|
if (!canonicalTitle) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -1342,6 +1360,9 @@ export class ImmersionTrackerService {
|
|||||||
} else if (!this.hasJellyfinMetadata(sessionInfo.videoId)) {
|
} else if (!this.hasJellyfinMetadata(sessionInfo.videoId)) {
|
||||||
this.captureAnimeMetadataAsync(sessionInfo.videoId, normalizedPath, normalizedTitle || null);
|
this.captureAnimeMetadataAsync(sessionInfo.videoId, normalizedPath, normalizedTitle || null);
|
||||||
}
|
}
|
||||||
|
if (!youtubeVideoId) {
|
||||||
|
this.prefetchCoverArtAsync(sessionInfo.videoId);
|
||||||
|
}
|
||||||
this.captureVideoMetadataAsync(sessionInfo.videoId, sourceType, normalizedPath);
|
this.captureVideoMetadataAsync(sessionInfo.videoId, sourceType, normalizedPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1924,6 +1945,24 @@ export class ImmersionTrackerService {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fetch cover art eagerly at session start (after anime metadata parsing
|
||||||
|
// settles) so new series show art on the stats timeline without requiring a
|
||||||
|
// visit to the series detail page first.
|
||||||
|
private prefetchCoverArtAsync(videoId: number): void {
|
||||||
|
const pendingMetadata = this.pendingAnimeMetadataUpdates.get(videoId);
|
||||||
|
void (async () => {
|
||||||
|
try {
|
||||||
|
await pendingMetadata;
|
||||||
|
if (this.isDestroyed) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await this.ensureCoverArt(videoId);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.warn('Unable to prefetch cover art', (error as Error).message);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
|
||||||
private updateVideoTitleForActiveSession(canonicalTitle: string): void {
|
private updateVideoTitleForActiveSession(canonicalTitle: string): void {
|
||||||
if (!this.sessionState) return;
|
if (!this.sessionState) return;
|
||||||
updateVideoTitleRecord(this.db, this.sessionState.videoId, canonicalTitle);
|
updateVideoTitleRecord(this.db, this.sessionState.videoId, canonicalTitle);
|
||||||
|
|||||||
@@ -0,0 +1,171 @@
|
|||||||
|
import type { Hono } from 'hono';
|
||||||
|
import type { ImmersionTrackerService } from './immersion-tracker-service.js';
|
||||||
|
|
||||||
|
type StatsCoverImagePayload = {
|
||||||
|
contentType: string;
|
||||||
|
dataUrl: string;
|
||||||
|
} | null;
|
||||||
|
|
||||||
|
type StatsCoverBatchBody = {
|
||||||
|
animeIds?: unknown;
|
||||||
|
videoIds?: unknown;
|
||||||
|
};
|
||||||
|
|
||||||
|
const MAX_BACKGROUND_ANIME_COVER_FETCHES = 3;
|
||||||
|
|
||||||
|
function parseIntQuery(raw: string | undefined, fallback: number, maxLimit?: number): number {
|
||||||
|
if (raw === undefined) return fallback;
|
||||||
|
const n = Number(raw);
|
||||||
|
if (!Number.isFinite(n) || n < 0) {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
const parsed = Math.floor(n);
|
||||||
|
return maxLimit === undefined ? parsed : Math.min(parsed, maxLimit);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parsePositiveIdList(raw: unknown, maxItems = 100): number[] {
|
||||||
|
if (!Array.isArray(raw)) return [];
|
||||||
|
|
||||||
|
const ids = new Set<number>();
|
||||||
|
for (const rawId of raw) {
|
||||||
|
const id = typeof rawId === 'number' ? rawId : typeof rawId === 'string' ? Number(rawId) : NaN;
|
||||||
|
if (Number.isFinite(id) && id > 0) {
|
||||||
|
ids.add(Math.floor(id));
|
||||||
|
if (ids.size >= maxItems) break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return Array.from(ids).sort((a, b) => a - b);
|
||||||
|
}
|
||||||
|
|
||||||
|
function coverImagePayload(
|
||||||
|
art: { coverBlob?: Uint8Array | null } | null | undefined,
|
||||||
|
): StatsCoverImagePayload {
|
||||||
|
if (!art?.coverBlob) return null;
|
||||||
|
const bytes = new Uint8Array(art.coverBlob);
|
||||||
|
const contentType = detectImageContentType(bytes);
|
||||||
|
return {
|
||||||
|
contentType,
|
||||||
|
dataUrl: `data:${contentType};base64,${Buffer.from(bytes).toString('base64')}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function detectImageContentType(bytes: Uint8Array): string {
|
||||||
|
if (
|
||||||
|
bytes.length >= 8 &&
|
||||||
|
bytes[0] === 0x89 &&
|
||||||
|
bytes[1] === 0x50 &&
|
||||||
|
bytes[2] === 0x4e &&
|
||||||
|
bytes[3] === 0x47
|
||||||
|
) {
|
||||||
|
return 'image/png';
|
||||||
|
}
|
||||||
|
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
|
||||||
|
return 'image/jpeg';
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
bytes.length >= 12 &&
|
||||||
|
bytes[0] === 0x52 &&
|
||||||
|
bytes[1] === 0x49 &&
|
||||||
|
bytes[2] === 0x46 &&
|
||||||
|
bytes[3] === 0x46 &&
|
||||||
|
bytes[8] === 0x57 &&
|
||||||
|
bytes[9] === 0x45 &&
|
||||||
|
bytes[10] === 0x42 &&
|
||||||
|
bytes[11] === 0x50
|
||||||
|
) {
|
||||||
|
return 'image/webp';
|
||||||
|
}
|
||||||
|
return 'application/octet-stream';
|
||||||
|
}
|
||||||
|
|
||||||
|
function createLimitedTaskRunner(maxConcurrentTasks: number): (task: () => Promise<void>) => void {
|
||||||
|
const queue: Array<() => Promise<void>> = [];
|
||||||
|
let activeTasks = 0;
|
||||||
|
|
||||||
|
const drain = (): void => {
|
||||||
|
while (activeTasks < maxConcurrentTasks && queue.length > 0) {
|
||||||
|
const task = queue.shift();
|
||||||
|
if (!task) return;
|
||||||
|
activeTasks += 1;
|
||||||
|
void task()
|
||||||
|
.catch(() => {})
|
||||||
|
.finally(() => {
|
||||||
|
activeTasks -= 1;
|
||||||
|
drain();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (task: () => Promise<void>): void => {
|
||||||
|
queue.push(task);
|
||||||
|
drain();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function registerStatsCoverRoutes(app: Hono, tracker: ImmersionTrackerService): void {
|
||||||
|
const enqueueAnimeCoverBackfill = createLimitedTaskRunner(MAX_BACKGROUND_ANIME_COVER_FETCHES);
|
||||||
|
|
||||||
|
app.post('/api/stats/covers', async (c) => {
|
||||||
|
const body = (await c.req.json().catch(() => null)) as StatsCoverBatchBody | null;
|
||||||
|
const animeIds = parsePositiveIdList(body?.animeIds);
|
||||||
|
const videoIds = parsePositiveIdList(body?.videoIds);
|
||||||
|
const anime: Record<number, StatsCoverImagePayload> = {};
|
||||||
|
const media: Record<number, StatsCoverImagePayload> = {};
|
||||||
|
|
||||||
|
await Promise.all(
|
||||||
|
animeIds.map(async (animeId) => {
|
||||||
|
const art = await tracker.getAnimeCoverArt(animeId);
|
||||||
|
if (!art?.coverBlob) {
|
||||||
|
enqueueAnimeCoverBackfill(async () => {
|
||||||
|
await tracker.ensureAnimeCoverArt(animeId);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
anime[animeId] = coverImagePayload(art);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
await Promise.all(
|
||||||
|
videoIds.map(async (videoId) => {
|
||||||
|
media[videoId] = coverImagePayload(await tracker.getCoverArt(videoId));
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
return c.json({ anime, media });
|
||||||
|
});
|
||||||
|
|
||||||
|
app.get('/api/stats/anime/:animeId/cover', async (c) => {
|
||||||
|
const animeId = parseIntQuery(c.req.param('animeId'), 0);
|
||||||
|
if (animeId <= 0) return c.body(null, 404);
|
||||||
|
let art = await tracker.getAnimeCoverArt(animeId);
|
||||||
|
if (!art?.coverBlob) {
|
||||||
|
await tracker.ensureAnimeCoverArt(animeId);
|
||||||
|
art = await tracker.getAnimeCoverArt(animeId);
|
||||||
|
}
|
||||||
|
if (!art?.coverBlob) return c.body(null, 404);
|
||||||
|
const bytes = new Uint8Array(art.coverBlob);
|
||||||
|
return new Response(bytes, {
|
||||||
|
headers: {
|
||||||
|
'Content-Type': detectImageContentType(bytes),
|
||||||
|
'Cache-Control': 'public, max-age=86400',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
app.get('/api/stats/media/:videoId/cover', async (c) => {
|
||||||
|
const videoId = parseIntQuery(c.req.param('videoId'), 0);
|
||||||
|
if (videoId <= 0) return c.body(null, 404);
|
||||||
|
let art = await tracker.getCoverArt(videoId);
|
||||||
|
if (!art?.coverBlob) {
|
||||||
|
await tracker.ensureCoverArt(videoId);
|
||||||
|
art = await tracker.getCoverArt(videoId);
|
||||||
|
}
|
||||||
|
if (!art?.coverBlob) return c.body(null, 404);
|
||||||
|
const bytes = new Uint8Array(art.coverBlob);
|
||||||
|
return new Response(bytes, {
|
||||||
|
headers: {
|
||||||
|
'Content-Type': detectImageContentType(bytes),
|
||||||
|
'Cache-Control': 'public, max-age=604800',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -17,6 +17,7 @@ import {
|
|||||||
} from '../../anki-field-config.js';
|
} from '../../anki-field-config.js';
|
||||||
import { resolveAnimatedImageLeadInSeconds } from '../../anki-integration/animated-image-sync.js';
|
import { resolveAnimatedImageLeadInSeconds } from '../../anki-integration/animated-image-sync.js';
|
||||||
import type { AnilistRateLimiter } from './anilist/rate-limiter.js';
|
import type { AnilistRateLimiter } from './anilist/rate-limiter.js';
|
||||||
|
import { registerStatsCoverRoutes } from './stats-cover-routes.js';
|
||||||
import {
|
import {
|
||||||
resolveRetimedSecondarySubtitleTextFromSidecar,
|
resolveRetimedSecondarySubtitleTextFromSidecar,
|
||||||
resolveSecondarySubtitleTextFromSidecar,
|
resolveSecondarySubtitleTextFromSidecar,
|
||||||
@@ -51,16 +52,6 @@ type StatsExcludedWordPayload = {
|
|||||||
reading: string;
|
reading: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
type StatsCoverImagePayload = {
|
|
||||||
contentType: string;
|
|
||||||
dataUrl: string;
|
|
||||||
} | null;
|
|
||||||
|
|
||||||
type StatsCoverBatchBody = {
|
|
||||||
animeIds?: unknown;
|
|
||||||
videoIds?: unknown;
|
|
||||||
};
|
|
||||||
|
|
||||||
function parseIntQuery(raw: string | undefined, fallback: number, maxLimit?: number): number {
|
function parseIntQuery(raw: string | undefined, fallback: number, maxLimit?: number): number {
|
||||||
if (raw === undefined) return fallback;
|
if (raw === undefined) return fallback;
|
||||||
const n = Number(raw);
|
const n = Number(raw);
|
||||||
@@ -113,62 +104,6 @@ function parseExcludedWordsBody(body: unknown): StatsExcludedWordPayload[] | nul
|
|||||||
return words;
|
return words;
|
||||||
}
|
}
|
||||||
|
|
||||||
function parsePositiveIdList(raw: unknown, maxItems = 100): number[] {
|
|
||||||
if (!Array.isArray(raw)) return [];
|
|
||||||
|
|
||||||
const ids = new Set<number>();
|
|
||||||
for (const rawId of raw) {
|
|
||||||
const id = typeof rawId === 'number' ? rawId : typeof rawId === 'string' ? Number(rawId) : NaN;
|
|
||||||
if (Number.isFinite(id) && id > 0) {
|
|
||||||
ids.add(Math.floor(id));
|
|
||||||
if (ids.size >= maxItems) break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Array.from(ids).sort((a, b) => a - b);
|
|
||||||
}
|
|
||||||
|
|
||||||
function coverImagePayload(
|
|
||||||
art: { coverBlob?: Uint8Array | null } | null | undefined,
|
|
||||||
): StatsCoverImagePayload {
|
|
||||||
if (!art?.coverBlob) return null;
|
|
||||||
const bytes = new Uint8Array(art.coverBlob);
|
|
||||||
const contentType = detectImageContentType(bytes);
|
|
||||||
return {
|
|
||||||
contentType,
|
|
||||||
dataUrl: `data:${contentType};base64,${Buffer.from(bytes).toString('base64')}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function detectImageContentType(bytes: Uint8Array): string {
|
|
||||||
if (
|
|
||||||
bytes.length >= 8 &&
|
|
||||||
bytes[0] === 0x89 &&
|
|
||||||
bytes[1] === 0x50 &&
|
|
||||||
bytes[2] === 0x4e &&
|
|
||||||
bytes[3] === 0x47
|
|
||||||
) {
|
|
||||||
return 'image/png';
|
|
||||||
}
|
|
||||||
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
|
|
||||||
return 'image/jpeg';
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
bytes.length >= 12 &&
|
|
||||||
bytes[0] === 0x52 &&
|
|
||||||
bytes[1] === 0x49 &&
|
|
||||||
bytes[2] === 0x46 &&
|
|
||||||
bytes[3] === 0x46 &&
|
|
||||||
bytes[8] === 0x57 &&
|
|
||||||
bytes[9] === 0x45 &&
|
|
||||||
bytes[10] === 0x42 &&
|
|
||||||
bytes[11] === 0x50
|
|
||||||
) {
|
|
||||||
return 'image/webp';
|
|
||||||
}
|
|
||||||
return 'application/octet-stream';
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveStatsNoteFieldName(
|
function resolveStatsNoteFieldName(
|
||||||
noteInfo: StatsServerNoteInfo,
|
noteInfo: StatsServerNoteInfo,
|
||||||
...preferredNames: (string | undefined)[]
|
...preferredNames: (string | undefined)[]
|
||||||
@@ -1017,58 +952,7 @@ export function createStatsApp(
|
|||||||
return c.json({ ok: true });
|
return c.json({ ok: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
app.post('/api/stats/covers', async (c) => {
|
registerStatsCoverRoutes(app, tracker);
|
||||||
const body = (await c.req.json().catch(() => null)) as StatsCoverBatchBody | null;
|
|
||||||
const animeIds = parsePositiveIdList(body?.animeIds);
|
|
||||||
const videoIds = parsePositiveIdList(body?.videoIds);
|
|
||||||
const anime: Record<number, StatsCoverImagePayload> = {};
|
|
||||||
const media: Record<number, StatsCoverImagePayload> = {};
|
|
||||||
|
|
||||||
await Promise.all(
|
|
||||||
animeIds.map(async (animeId) => {
|
|
||||||
anime[animeId] = coverImagePayload(await tracker.getAnimeCoverArt(animeId));
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
await Promise.all(
|
|
||||||
videoIds.map(async (videoId) => {
|
|
||||||
media[videoId] = coverImagePayload(await tracker.getCoverArt(videoId));
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
return c.json({ anime, media });
|
|
||||||
});
|
|
||||||
|
|
||||||
app.get('/api/stats/anime/:animeId/cover', async (c) => {
|
|
||||||
const animeId = parseIntQuery(c.req.param('animeId'), 0);
|
|
||||||
if (animeId <= 0) return c.body(null, 404);
|
|
||||||
const art = await tracker.getAnimeCoverArt(animeId);
|
|
||||||
if (!art?.coverBlob) return c.body(null, 404);
|
|
||||||
const bytes = new Uint8Array(art.coverBlob);
|
|
||||||
return new Response(bytes, {
|
|
||||||
headers: {
|
|
||||||
'Content-Type': detectImageContentType(bytes),
|
|
||||||
'Cache-Control': 'public, max-age=86400',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
app.get('/api/stats/media/:videoId/cover', async (c) => {
|
|
||||||
const videoId = parseIntQuery(c.req.param('videoId'), 0);
|
|
||||||
if (videoId <= 0) return c.body(null, 404);
|
|
||||||
let art = await tracker.getCoverArt(videoId);
|
|
||||||
if (!art?.coverBlob) {
|
|
||||||
await tracker.ensureCoverArt(videoId);
|
|
||||||
art = await tracker.getCoverArt(videoId);
|
|
||||||
}
|
|
||||||
if (!art?.coverBlob) return c.body(null, 404);
|
|
||||||
const bytes = new Uint8Array(art.coverBlob);
|
|
||||||
return new Response(bytes, {
|
|
||||||
headers: {
|
|
||||||
'Content-Type': detectImageContentType(bytes),
|
|
||||||
'Cache-Control': 'public, max-age=604800',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
app.get('/api/stats/episode/:videoId/detail', async (c) => {
|
app.get('/api/stats/episode/:videoId/detail', async (c) => {
|
||||||
const videoId = parseIntQuery(c.req.param('videoId'), 0);
|
const videoId = parseIntQuery(c.req.param('videoId'), 0);
|
||||||
|
|||||||
@@ -1722,6 +1722,36 @@ test('annotateTokens keeps known status while clearing other annotations for sta
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('annotateTokens excludes standalone noun-suffix tokens from annotations while keeping cache-backed known status', () => {
|
||||||
|
const tokens = [
|
||||||
|
makeToken({
|
||||||
|
surface: 'さん',
|
||||||
|
headword: 'さん',
|
||||||
|
reading: 'サン',
|
||||||
|
partOfSpeech: PartOfSpeech.noun,
|
||||||
|
pos1: '名詞',
|
||||||
|
pos2: '接尾',
|
||||||
|
startPos: 0,
|
||||||
|
endPos: 2,
|
||||||
|
frequencyRank: 33,
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = annotateTokens(
|
||||||
|
tokens,
|
||||||
|
makeDeps({
|
||||||
|
isKnownWord: (text) => text === 'さん',
|
||||||
|
getJlptLevel: (text) => (text === 'さん' ? 'N5' : null),
|
||||||
|
}),
|
||||||
|
{ minSentenceWordsForNPlusOne: 1 },
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(result[0]?.isKnown, true);
|
||||||
|
assert.equal(result[0]?.isNPlusOneTarget, false);
|
||||||
|
assert.equal(result[0]?.frequencyRank, undefined);
|
||||||
|
assert.equal(result[0]?.jlptLevel, undefined);
|
||||||
|
});
|
||||||
|
|
||||||
test('annotateTokens keeps known status while clearing other annotations for auxiliary-only te-kureru helper spans', () => {
|
test('annotateTokens keeps known status while clearing other annotations for auxiliary-only te-kureru helper spans', () => {
|
||||||
const tokens = [
|
const tokens = [
|
||||||
makeToken({
|
makeToken({
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { normalizePos1ExclusionList } from './token-pos1-exclusions';
|
|||||||
|
|
||||||
export const DEFAULT_ANNOTATION_POS2_EXCLUSION_DEFAULTS = Object.freeze([
|
export const DEFAULT_ANNOTATION_POS2_EXCLUSION_DEFAULTS = Object.freeze([
|
||||||
'非自立',
|
'非自立',
|
||||||
|
'接尾',
|
||||||
]) as readonly string[];
|
]) as readonly string[];
|
||||||
|
|
||||||
export const DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG: ResolvedTokenPos2ExclusionConfig = {
|
export const DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG: ResolvedTokenPos2ExclusionConfig = {
|
||||||
|
|||||||
Reference in New Issue
Block a user