mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-29 19:21:33 -07:00
feat(stats): add headword sentence search and rename related words
- Search by headword enabled by default; finds inflected variants (e.g. 知らない → 知らねえ) - Add "Search by headword" toggle to switch back to exact text/title matching - Rename "Similar Words" → "Related Seen Words" with tighter matching (same reading/shared kanji) - ankiConnect.deck falls back to Yomitan mining deck when empty
This commit is contained in:
@@ -338,7 +338,10 @@ function withTempDir<T>(fn: (dir: string) => Promise<T> | T): Promise<T> | T {
|
||||
type CapturedAnkiRequest = {
|
||||
action?: string;
|
||||
params?: {
|
||||
cards?: number[];
|
||||
deck?: string;
|
||||
notes?: number[];
|
||||
query?: string;
|
||||
note?: {
|
||||
id?: number;
|
||||
deckName?: string;
|
||||
@@ -384,6 +387,10 @@ async function withFakeAnkiConnect<T>(
|
||||
})),
|
||||
error: null,
|
||||
};
|
||||
} else if (payload.action === 'findCards') {
|
||||
body = { result: [9001], error: null };
|
||||
} else if (payload.action === 'changeDeck') {
|
||||
body = { result: null, error: null };
|
||||
}
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
@@ -435,6 +442,43 @@ describe('stats server API routes', () => {
|
||||
assert.ok(Array.isArray(body));
|
||||
});
|
||||
|
||||
it('GET /api/stats/sentences/search resolves headword candidates by default', async () => {
|
||||
const seen: Array<{
|
||||
query: string;
|
||||
limit: number;
|
||||
options: unknown;
|
||||
}> = [];
|
||||
const resolvedTerms: string[] = [];
|
||||
const app = createStatsApp(
|
||||
createMockTracker({
|
||||
searchSubtitleSentences: async (query: string, limit: number, options: unknown) => {
|
||||
seen.push({ query, limit, options });
|
||||
return [];
|
||||
},
|
||||
}),
|
||||
{
|
||||
resolveSentenceSearchHeadwords: async (term: string) => {
|
||||
resolvedTerms.push(term);
|
||||
return term === '知らない' ? ['知る'] : [];
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const res = await app.request(
|
||||
'/api/stats/sentences/search?q=%E7%9F%A5%E3%82%89%E3%81%AA%E3%81%84&limit=12',
|
||||
);
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(resolvedTerms, ['知らない']);
|
||||
assert.deepEqual(seen, [
|
||||
{
|
||||
query: '知らない',
|
||||
limit: 12,
|
||||
options: { headwordTerms: [{ term: '知らない', headwords: ['知る'] }] },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('GET /api/stats/sessions enriches known-word metrics using filtered persisted totals', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const cachePath = path.join(dir, 'known-words.json');
|
||||
@@ -1162,12 +1206,62 @@ describe('stats server API routes', () => {
|
||||
const addNoteRequest = requests.find((request) => request.action === 'addNote');
|
||||
assert.equal(addNoteRequest?.params?.note?.deckName, 'Default');
|
||||
assert.equal(addNoteRequest?.params?.note?.modelName, 'Lapis Morph');
|
||||
assert.equal(addNoteRequest?.params?.note?.fields?.Sentence, '<b>猫</b>を見た');
|
||||
assert.equal(addNoteRequest?.params?.note?.fields?.Sentence, '猫を見た');
|
||||
assert.equal(addNoteRequest?.params?.note?.fields?.IsSentenceCard, 'x');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('POST /api/stats/mine-card uses Yomitan deck for direct sentence cards when config deck is empty', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
fs.writeFileSync(sourcePath, 'fake media');
|
||||
|
||||
await withFakeAnkiConnect(async (requests, url) => {
|
||||
const app = createStatsApp(createMockTracker(), {
|
||||
getYomitanAnkiDeckName: async () => 'Minecraft',
|
||||
ankiConnectConfig: {
|
||||
url,
|
||||
deck: '',
|
||||
tags: ['SubMiner'],
|
||||
fields: {
|
||||
word: 'Expression',
|
||||
sentence: 'Sentence',
|
||||
translation: 'SelectionText',
|
||||
},
|
||||
media: {
|
||||
generateAudio: false,
|
||||
generateImage: false,
|
||||
},
|
||||
isLapis: {
|
||||
enabled: true,
|
||||
sentenceCardModel: 'Lapis Morph',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const res = await app.request('/api/stats/mine-card?mode=sentence', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
sourcePath,
|
||||
startMs: 1_000,
|
||||
endMs: 2_000,
|
||||
sentence: '猫を見た',
|
||||
word: '猫',
|
||||
secondaryText: 'I saw a cat',
|
||||
videoTitle: 'Episode 1',
|
||||
}),
|
||||
});
|
||||
|
||||
const body = await res.json();
|
||||
assert.equal(res.status, 200, JSON.stringify(body));
|
||||
const addNoteRequest = requests.find((request) => request.action === 'addNote');
|
||||
assert.equal(addNoteRequest?.params?.note?.deckName, 'Minecraft');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('POST /api/stats/mine-card resolves Anki config at request time', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
@@ -1294,7 +1388,7 @@ describe('stats server API routes', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('POST /api/stats/mine-card leaves word card selection text to Yomitan glossary fields', async () => {
|
||||
it('POST /api/stats/mine-card writes secondary subtitles to word card selection text', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
fs.writeFileSync(sourcePath, 'fake media');
|
||||
@@ -1340,7 +1434,286 @@ describe('stats server API routes', () => {
|
||||
const updateRequest = requests.find((request) => request.action === 'updateNoteFields');
|
||||
assert.equal(updateRequest?.params?.note?.id, 777);
|
||||
assert.equal(updateRequest?.params?.note?.fields?.Sentence, '<b>猫</b>を見た');
|
||||
assert.equal(updateRequest?.params?.note?.fields?.SelectionText, undefined);
|
||||
assert.equal(updateRequest?.params?.note?.fields?.SelectionText, 'I saw a cat');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('POST /api/stats/mine-card moves Yomitan-created word notes to the configured deck', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
fs.writeFileSync(sourcePath, 'fake media');
|
||||
|
||||
await withFakeAnkiConnect(async (requests, url) => {
|
||||
const app = createStatsApp(createMockTracker(), {
|
||||
addYomitanNote: async () => 777,
|
||||
ankiConnectConfig: {
|
||||
url,
|
||||
deck: 'Mining',
|
||||
fields: {
|
||||
sentence: 'Sentence',
|
||||
translation: 'SelectionText',
|
||||
},
|
||||
media: {
|
||||
generateAudio: false,
|
||||
generateImage: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const res = await app.request('/api/stats/mine-card?mode=word', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
sourcePath,
|
||||
startMs: 1_000,
|
||||
endMs: 2_000,
|
||||
sentence: '猫を見た',
|
||||
word: '猫',
|
||||
videoTitle: 'Episode 1',
|
||||
}),
|
||||
});
|
||||
|
||||
const body = await res.json();
|
||||
assert.equal(res.status, 200, JSON.stringify(body));
|
||||
|
||||
const findCardsRequest = requests.find((request) => request.action === 'findCards');
|
||||
assert.equal(findCardsRequest?.params?.query, 'nid:777');
|
||||
const changeDeckRequest = requests.find((request) => request.action === 'changeDeck');
|
||||
assert.deepEqual(changeDeckRequest?.params?.cards, [9001]);
|
||||
assert.equal(changeDeckRequest?.params?.deck, 'Mining');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('POST /api/stats/mine-card moves Yomitan-created word notes to Yomitan deck when config deck is empty', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
fs.writeFileSync(sourcePath, 'fake media');
|
||||
|
||||
await withFakeAnkiConnect(async (requests, url) => {
|
||||
const app = createStatsApp(createMockTracker(), {
|
||||
getYomitanAnkiDeckName: async () => 'Minecraft',
|
||||
addYomitanNote: async () => 777,
|
||||
ankiConnectConfig: {
|
||||
url,
|
||||
deck: '',
|
||||
fields: {
|
||||
sentence: 'Sentence',
|
||||
translation: 'SelectionText',
|
||||
},
|
||||
media: {
|
||||
generateAudio: false,
|
||||
generateImage: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const res = await app.request('/api/stats/mine-card?mode=word', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
sourcePath,
|
||||
startMs: 1_000,
|
||||
endMs: 2_000,
|
||||
sentence: '猫を見た',
|
||||
word: '猫',
|
||||
videoTitle: 'Episode 1',
|
||||
}),
|
||||
});
|
||||
|
||||
const body = await res.json();
|
||||
assert.equal(res.status, 200, JSON.stringify(body));
|
||||
|
||||
const findCardsRequest = requests.find((request) => request.action === 'findCards');
|
||||
assert.equal(findCardsRequest?.params?.query, 'nid:777');
|
||||
const changeDeckRequest = requests.find((request) => request.action === 'changeDeck');
|
||||
assert.deepEqual(changeDeckRequest?.params?.cards, [9001]);
|
||||
assert.equal(changeDeckRequest?.params?.deck, 'Minecraft');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('POST /api/stats/mine-card uses the full sentence as sentence-card expression', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
fs.writeFileSync(sourcePath, 'fake media');
|
||||
|
||||
await withFakeAnkiConnect(async (requests, url) => {
|
||||
const app = createStatsApp(createMockTracker(), {
|
||||
ankiConnectConfig: {
|
||||
url,
|
||||
deck: 'Minecraft',
|
||||
tags: ['SubMiner'],
|
||||
fields: {
|
||||
word: 'Expression',
|
||||
sentence: 'Sentence',
|
||||
translation: 'SelectionText',
|
||||
},
|
||||
media: {
|
||||
generateAudio: false,
|
||||
generateImage: false,
|
||||
},
|
||||
isLapis: {
|
||||
enabled: true,
|
||||
sentenceCardModel: 'Lapis Morph',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const res = await app.request('/api/stats/mine-card?mode=sentence', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
sourcePath,
|
||||
startMs: 1_000,
|
||||
endMs: 2_000,
|
||||
sentence: '猫を見た',
|
||||
word: '猫',
|
||||
secondaryText: 'I saw a cat',
|
||||
videoTitle: 'Episode 1',
|
||||
}),
|
||||
});
|
||||
|
||||
const body = await res.json();
|
||||
assert.equal(res.status, 200, JSON.stringify(body));
|
||||
|
||||
const addNoteRequest = requests.find((request) => request.action === 'addNote');
|
||||
assert.equal(addNoteRequest?.params?.note?.deckName, 'Minecraft');
|
||||
assert.equal(addNoteRequest?.params?.note?.fields?.Expression, '猫を見た');
|
||||
assert.equal(addNoteRequest?.params?.note?.fields?.Sentence, '猫を見た');
|
||||
assert.equal(addNoteRequest?.params?.note?.fields?.SelectionText, 'I saw a cat');
|
||||
assert.equal(addNoteRequest?.params?.note?.fields?.IsSentenceCard, 'x');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('POST /api/stats/mine-card fills selection text from a matching secondary sidecar subtitle', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
fs.writeFileSync(sourcePath, 'fake media');
|
||||
fs.writeFileSync(
|
||||
path.join(dir, 'episode.en.srt'),
|
||||
[
|
||||
'1',
|
||||
'00:00:00,800 --> 00:00:02,500',
|
||||
'I saw a cat.',
|
||||
'',
|
||||
'2',
|
||||
'00:00:03,000 --> 00:00:04,000',
|
||||
'Not this line.',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
await withFakeAnkiConnect(async (requests, url) => {
|
||||
const app = createStatsApp(createMockTracker(), {
|
||||
ankiConnectConfig: {
|
||||
url,
|
||||
deck: 'Minecraft',
|
||||
tags: ['SubMiner'],
|
||||
fields: {
|
||||
word: 'Expression',
|
||||
sentence: 'Sentence',
|
||||
translation: 'SelectionText',
|
||||
},
|
||||
media: {
|
||||
generateAudio: false,
|
||||
generateImage: false,
|
||||
},
|
||||
isLapis: {
|
||||
enabled: true,
|
||||
sentenceCardModel: 'Lapis Morph',
|
||||
},
|
||||
},
|
||||
secondarySubtitleLanguages: ['en'],
|
||||
} as Parameters<typeof createStatsApp>[1]);
|
||||
|
||||
const res = await app.request('/api/stats/mine-card?mode=sentence', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
sourcePath,
|
||||
startMs: 1_000,
|
||||
endMs: 2_000,
|
||||
sentence: '猫を見た',
|
||||
word: '猫',
|
||||
videoTitle: 'Episode 1',
|
||||
}),
|
||||
});
|
||||
|
||||
const body = await res.json();
|
||||
assert.equal(res.status, 200, JSON.stringify(body));
|
||||
|
||||
const addNoteRequest = requests.find((request) => request.action === 'addNote');
|
||||
assert.equal(addNoteRequest?.params?.note?.deckName, 'Minecraft');
|
||||
assert.equal(addNoteRequest?.params?.note?.fields?.SelectionText, 'I saw a cat.');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('POST /api/stats/mine-card does not append the next sidecar cue near a timing boundary', async () => {
|
||||
await withTempDir(async (dir) => {
|
||||
const sourcePath = path.join(dir, 'episode.mkv');
|
||||
fs.writeFileSync(sourcePath, 'fake media');
|
||||
fs.writeFileSync(
|
||||
path.join(dir, 'episode.en.srt'),
|
||||
[
|
||||
'1',
|
||||
'00:00:00,800 --> 00:00:01,500',
|
||||
"I don't give a damn what family she's from.",
|
||||
'',
|
||||
'2',
|
||||
'00:00:01,700 --> 00:00:03,000',
|
||||
'That snobby attitude just pisses me off!',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
await withFakeAnkiConnect(async (requests, url) => {
|
||||
const app = createStatsApp(createMockTracker(), {
|
||||
ankiConnectConfig: {
|
||||
url,
|
||||
deck: 'Minecraft',
|
||||
tags: ['SubMiner'],
|
||||
fields: {
|
||||
word: 'Expression',
|
||||
sentence: 'Sentence',
|
||||
translation: 'SelectionText',
|
||||
},
|
||||
media: {
|
||||
generateAudio: false,
|
||||
generateImage: false,
|
||||
},
|
||||
isLapis: {
|
||||
enabled: true,
|
||||
sentenceCardModel: 'Lapis Morph',
|
||||
},
|
||||
},
|
||||
secondarySubtitleLanguages: ['en'],
|
||||
} as Parameters<typeof createStatsApp>[1]);
|
||||
|
||||
const res = await app.request('/api/stats/mine-card?mode=sentence', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
sourcePath,
|
||||
startMs: 1_000,
|
||||
endMs: 2_000,
|
||||
sentence: '名門か何だか知らねえが',
|
||||
word: '名門',
|
||||
videoTitle: 'Episode 1',
|
||||
}),
|
||||
});
|
||||
|
||||
const body = await res.json();
|
||||
assert.equal(res.status, 200, JSON.stringify(body));
|
||||
|
||||
const addNoteRequest = requests.find((request) => request.action === 'addNote');
|
||||
assert.equal(
|
||||
addNoteRequest?.params?.note?.fields?.SelectionText,
|
||||
"I don't give a damn what family she's from.",
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1466,6 +1839,8 @@ describe('stats server API routes', () => {
|
||||
'generateAudio',
|
||||
'generateScreenshot',
|
||||
'addNote',
|
||||
'findCards',
|
||||
'changeDeck',
|
||||
'uploadAudio',
|
||||
'uploadImage',
|
||||
'updateNoteFields',
|
||||
|
||||
@@ -149,6 +149,7 @@ import {
|
||||
type MediaLibraryRow,
|
||||
type NewAnimePerDayRow,
|
||||
type QueuedWrite,
|
||||
type SentenceSearchOptions,
|
||||
type SentenceSearchResultRow,
|
||||
type SessionEventRow,
|
||||
type SessionState,
|
||||
@@ -570,8 +571,12 @@ export class ImmersionTrackerService {
|
||||
return getKanjiOccurrences(this.db, kanji, limit, offset);
|
||||
}
|
||||
|
||||
async searchSubtitleSentences(query: string, limit = 50): Promise<SentenceSearchResultRow[]> {
|
||||
return searchSubtitleSentences(this.db, query, limit);
|
||||
async searchSubtitleSentences(
|
||||
query: string,
|
||||
limit = 50,
|
||||
options?: SentenceSearchOptions,
|
||||
): Promise<SentenceSearchResultRow[]> {
|
||||
return searchSubtitleSentences(this.db, query, limit, options);
|
||||
}
|
||||
|
||||
async getSessionEvents(
|
||||
|
||||
@@ -356,6 +356,81 @@ test('split session and lexical helpers return distinct-headword, detail, appear
|
||||
}
|
||||
});
|
||||
|
||||
test('similar words use same reading and shared kanji without kana suffix noise', () => {
|
||||
const { db, dbPath, stmts } = createDb();
|
||||
|
||||
try {
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: 'Similar Words Anime',
|
||||
canonicalTitle: 'Similar Words Anime',
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: null,
|
||||
});
|
||||
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/similar-words.mkv', {
|
||||
canonicalTitle: 'Similar Words Episode',
|
||||
sourcePath: '/tmp/similar-words.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
const sessionId = startSessionRecord(db, videoId, 1_000_000).sessionId;
|
||||
|
||||
const araiId = insertWordOccurrence(db, stmts, {
|
||||
sessionId,
|
||||
videoId,
|
||||
animeId,
|
||||
lineIndex: 1,
|
||||
text: '荒い息',
|
||||
word: { headword: '荒い', word: '荒い', reading: 'あらい' },
|
||||
});
|
||||
insertWordOccurrence(db, stmts, {
|
||||
sessionId,
|
||||
videoId,
|
||||
animeId,
|
||||
lineIndex: 2,
|
||||
text: '洗い物',
|
||||
word: { headword: '洗い', word: '洗い', reading: 'あらい' },
|
||||
});
|
||||
insertWordOccurrence(db, stmts, {
|
||||
sessionId,
|
||||
videoId,
|
||||
animeId,
|
||||
lineIndex: 3,
|
||||
text: '荒波',
|
||||
word: { headword: '荒波', word: '荒波', reading: 'あらなみ' },
|
||||
});
|
||||
|
||||
for (let lineIndex = 4; lineIndex < 9; lineIndex++) {
|
||||
insertWordOccurrence(db, stmts, {
|
||||
sessionId,
|
||||
videoId,
|
||||
animeId,
|
||||
lineIndex,
|
||||
text: '良い',
|
||||
word: { headword: '良い', word: '良い', reading: 'よい' },
|
||||
});
|
||||
}
|
||||
insertWordOccurrence(db, stmts, {
|
||||
sessionId,
|
||||
videoId,
|
||||
animeId,
|
||||
lineIndex: 9,
|
||||
text: 'お構いなく',
|
||||
word: { headword: 'お構いなく', word: 'お構いなく', reading: 'おかまいなく' },
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
getSimilarWords(db, araiId, 10).map((row) => row.headword),
|
||||
['洗い', '荒波'],
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('split library helpers return anime/media session and analytics rows', () => {
|
||||
const { db, dbPath, stmts } = createDb();
|
||||
|
||||
|
||||
@@ -3785,6 +3785,90 @@ test('searchSubtitleSentences searches known subtitle lines and returns media co
|
||||
}
|
||||
});
|
||||
|
||||
test('searchSubtitleSentences searches subtitle lines by resolved headword candidates', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: 'Little Witch Academia',
|
||||
canonicalTitle: 'Little Witch Academia',
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: '{"source":"test"}',
|
||||
});
|
||||
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/lwa-05.mkv', {
|
||||
canonicalTitle: 'Episode 5',
|
||||
sourcePath: '/tmp/Little Witch Academia S01E05.mkv',
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
linkVideoToAnimeRecord(db, videoId, {
|
||||
animeId,
|
||||
parsedBasename: 'Little Witch Academia S01E05.mkv',
|
||||
parsedTitle: 'Little Witch Academia',
|
||||
parsedSeason: 1,
|
||||
parsedEpisode: 5,
|
||||
parserSource: 'fallback',
|
||||
parserConfidence: 1,
|
||||
parseMetadataJson: '{"episode":5}',
|
||||
});
|
||||
const { sessionId } = startSessionRecord(db, videoId, 4_000_000);
|
||||
const lineResult = db
|
||||
.prepare(
|
||||
`INSERT INTO imm_subtitle_lines (
|
||||
session_id, event_id, video_id, anime_id, line_index, segment_start_ms, segment_end_ms,
|
||||
text, secondary_text, CREATED_DATE, LAST_UPDATE_DATE
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run(
|
||||
sessionId,
|
||||
null,
|
||||
videoId,
|
||||
animeId,
|
||||
20,
|
||||
247_000,
|
||||
250_000,
|
||||
'ああ、名無しが何だか知らねえが',
|
||||
null,
|
||||
4_000,
|
||||
4_000,
|
||||
);
|
||||
const wordResult = db
|
||||
.prepare(
|
||||
`INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run('知る', '知らねえ', 'しらねえ', 'verb', '動詞', '自立', '', 4_000, 4_000, 1);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count)
|
||||
VALUES (?, ?, ?)`,
|
||||
).run(Number(lineResult.lastInsertRowid), Number(wordResult.lastInsertRowid), 1);
|
||||
|
||||
assert.deepEqual(searchSubtitleSentences(db, '知らない', 10), []);
|
||||
|
||||
const rows = searchSubtitleSentences(db, '知らない', 10, {
|
||||
headwordTerms: [{ term: '知らない', headwords: ['知る'] }],
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
rows.map((row) => row.text),
|
||||
['ああ、名無しが何だか知らねえが'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
searchSubtitleSentences(db, '知らねえ', 10).map((row) => row.text),
|
||||
['ああ、名無しが何だか知らねえが'],
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getKanjiOccurrences maps a kanji back to anime, video, and subtitle line context', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
@@ -7,6 +7,7 @@ import type {
|
||||
KanjiOccurrenceRow,
|
||||
KanjiStatsRow,
|
||||
KanjiWordRow,
|
||||
SentenceSearchOptions,
|
||||
SentenceSearchResultRow,
|
||||
SessionEventRow,
|
||||
SimilarWordRow,
|
||||
@@ -23,6 +24,7 @@ const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
|
||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
|
||||
const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
|
||||
const SENTENCE_SEARCH_MAX_LIMIT = 100;
|
||||
const KANJI_PATTERN = /\p{Script=Han}/gu;
|
||||
|
||||
function resolveSentenceSearchLimit(limit: number): number {
|
||||
if (!Number.isFinite(limit)) return SENTENCE_SEARCH_DEFAULT_LIMIT;
|
||||
@@ -31,7 +33,7 @@ function resolveSentenceSearchLimit(limit: number): number {
|
||||
return Math.min(normalized, SENTENCE_SEARCH_MAX_LIMIT);
|
||||
}
|
||||
|
||||
function splitSearchTerms(query: string): string[] {
|
||||
export function splitSentenceSearchTerms(query: string): string[] {
|
||||
return query
|
||||
.trim()
|
||||
.split(/\s+/)
|
||||
@@ -44,6 +46,33 @@ function escapeLikeTerm(term: string): string {
|
||||
return term.replace(/[\\%_]/g, (match) => `\\${match}`);
|
||||
}
|
||||
|
||||
function uniqueNonEmptyTerms(values: readonly string[] | undefined): string[] {
|
||||
const seen = new Set<string>();
|
||||
const terms: string[] = [];
|
||||
for (const value of values ?? []) {
|
||||
const term = value.trim();
|
||||
if (!term || seen.has(term)) continue;
|
||||
seen.add(term);
|
||||
terms.push(term);
|
||||
}
|
||||
return terms;
|
||||
}
|
||||
|
||||
function getHeadwordCandidatesForSentenceSearchTerm(
|
||||
term: string,
|
||||
options: SentenceSearchOptions | undefined,
|
||||
): string[] {
|
||||
const headwords =
|
||||
options?.headwordTerms
|
||||
?.filter((entry) => entry.term === term)
|
||||
.flatMap((entry) => entry.headwords) ?? [];
|
||||
return uniqueNonEmptyTerms(headwords);
|
||||
}
|
||||
|
||||
function uniqueKanji(text: string): string[] {
|
||||
return Array.from(new Set(text.match(KANJI_PATTERN) ?? []));
|
||||
}
|
||||
|
||||
function toVocabularyToken(row: VocabularyStatsRow): MergedToken {
|
||||
const partOfSpeech =
|
||||
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
|
||||
@@ -238,8 +267,9 @@ export function searchSubtitleSentences(
|
||||
db: DatabaseSync,
|
||||
query: string,
|
||||
limit = SENTENCE_SEARCH_DEFAULT_LIMIT,
|
||||
options?: SentenceSearchOptions,
|
||||
): SentenceSearchResultRow[] {
|
||||
const terms = splitSearchTerms(query);
|
||||
const terms = splitSentenceSearchTerms(query);
|
||||
if (terms.length === 0) return [];
|
||||
const resolvedLimit = resolveSentenceSearchLimit(limit);
|
||||
|
||||
@@ -247,14 +277,28 @@ export function searchSubtitleSentences(
|
||||
const params: string[] = [];
|
||||
for (const term of terms) {
|
||||
const likeTerm = `%${escapeLikeTerm(term)}%`;
|
||||
const headwords = getHeadwordCandidatesForSentenceSearchTerm(term, options);
|
||||
const headwordClause =
|
||||
headwords.length > 0
|
||||
? `
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM imm_word_line_occurrences o
|
||||
JOIN imm_words w ON w.id = o.word_id
|
||||
WHERE o.line_id = l.line_id
|
||||
AND w.headword IN (${headwords.map(() => '?').join(', ')})
|
||||
)
|
||||
`
|
||||
: '';
|
||||
clauses.push(`
|
||||
(
|
||||
l.text LIKE ? ESCAPE '\\'
|
||||
OR v.canonical_title LIKE ? ESCAPE '\\'
|
||||
OR COALESCE(a.canonical_title, '') LIKE ? ESCAPE '\\'
|
||||
${headwordClause}
|
||||
)
|
||||
`);
|
||||
params.push(likeTerm, likeTerm, likeTerm);
|
||||
params.push(likeTerm, likeTerm, likeTerm, ...headwords);
|
||||
}
|
||||
|
||||
return db
|
||||
@@ -359,24 +403,38 @@ export function getSimilarWords(db: DatabaseSync, wordId: number, limit = 10): S
|
||||
reading: string;
|
||||
} | null;
|
||||
if (!word || word.headword.trim() === '') return [];
|
||||
|
||||
const clauses: string[] = [];
|
||||
const params: string[] = [];
|
||||
const reading = word.reading.trim();
|
||||
if (reading !== '') {
|
||||
clauses.push('reading = ?');
|
||||
params.push(word.reading);
|
||||
}
|
||||
|
||||
for (const kanji of uniqueKanji(word.headword)) {
|
||||
clauses.push("headword LIKE ? ESCAPE '\\'");
|
||||
params.push(`%${escapeLikeTerm(kanji)}%`);
|
||||
}
|
||||
|
||||
if (clauses.length === 0) return [];
|
||||
|
||||
const orderBy =
|
||||
reading !== '' ? 'CASE WHEN reading = ? THEN 0 ELSE 1 END, frequency DESC' : 'frequency DESC';
|
||||
const orderParams = reading !== '' ? [word.reading] : [];
|
||||
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id AS wordId, headword, word, reading, frequency
|
||||
FROM imm_words
|
||||
WHERE id != ?
|
||||
AND (reading = ? OR headword LIKE ? OR headword LIKE ?)
|
||||
ORDER BY frequency DESC
|
||||
AND (${clauses.join(' OR ')})
|
||||
ORDER BY ${orderBy}
|
||||
LIMIT ?
|
||||
`,
|
||||
)
|
||||
.all(
|
||||
wordId,
|
||||
word.reading,
|
||||
`%${word.headword.charAt(0)}%`,
|
||||
`%${word.headword.charAt(word.headword.length - 1)}%`,
|
||||
limit,
|
||||
) as SimilarWordRow[];
|
||||
.all(wordId, ...params, ...orderParams, limit) as SimilarWordRow[];
|
||||
}
|
||||
|
||||
export function getKanjiDetail(db: DatabaseSync, kanjiId: number): KanjiDetailRow | null {
|
||||
|
||||
@@ -381,6 +381,15 @@ export interface SentenceSearchResultRow {
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface SentenceSearchHeadwordTerm {
|
||||
term: string;
|
||||
headwords: string[];
|
||||
}
|
||||
|
||||
export interface SentenceSearchOptions {
|
||||
headwordTerms?: SentenceSearchHeadwordTerm[];
|
||||
}
|
||||
|
||||
export interface SessionEventRow {
|
||||
eventType: number;
|
||||
tsMs: number;
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { parseSubtitleCues, type SubtitleCue } from './subtitle-cue-parser.js';
|
||||
import { isEnglishYoutubeLang, normalizeYoutubeLangCode } from './youtube/labels.js';
|
||||
|
||||
const DEFAULT_SECONDARY_SUBTITLE_LANGUAGES = ['en', 'eng', 'english', 'en-us', 'enus'];
|
||||
const SUPPORTED_SUBTITLE_EXTENSIONS = new Set(['.srt', '.vtt', '.ass', '.ssa']);
|
||||
const TIMING_TOLERANCE_SECONDS = 0.25;
|
||||
const SAME_TIMING_EPSILON_SECONDS = 0.001;
|
||||
|
||||
type SidecarCandidate = {
|
||||
path: string;
|
||||
languageRank: number;
|
||||
extensionRank: number;
|
||||
name: string;
|
||||
};
|
||||
|
||||
function unique(values: string[]): string[] {
|
||||
return values.filter((value, index) => value.length > 0 && values.indexOf(value) === index);
|
||||
}
|
||||
|
||||
function expandPreferredLanguages(languages: readonly string[] | undefined): string[] {
|
||||
const normalized = unique(
|
||||
(languages ?? []).map((language) => normalizeYoutubeLangCode(language)).filter(Boolean),
|
||||
);
|
||||
const base = normalized.length > 0 ? normalized : DEFAULT_SECONDARY_SUBTITLE_LANGUAGES;
|
||||
const expanded: string[] = [];
|
||||
for (const language of base) {
|
||||
expanded.push(language);
|
||||
if (isEnglishYoutubeLang(language)) {
|
||||
expanded.push(...DEFAULT_SECONDARY_SUBTITLE_LANGUAGES);
|
||||
}
|
||||
}
|
||||
return unique(expanded);
|
||||
}
|
||||
|
||||
function splitLanguageSuffix(value: string): string[] {
|
||||
const normalizedWhole = normalizeYoutubeLangCode(value);
|
||||
const tokens = value
|
||||
.split(/[^A-Za-z0-9-]+/g)
|
||||
.map((token) => normalizeYoutubeLangCode(token))
|
||||
.filter(Boolean);
|
||||
return unique([normalizedWhole, ...tokens]);
|
||||
}
|
||||
|
||||
function languageTokenMatches(token: string, preferredLanguage: string): boolean {
|
||||
if (token === preferredLanguage) {
|
||||
return true;
|
||||
}
|
||||
if (token.startsWith(`${preferredLanguage}-`) || preferredLanguage.startsWith(`${token}-`)) {
|
||||
return true;
|
||||
}
|
||||
return isEnglishYoutubeLang(token) && isEnglishYoutubeLang(preferredLanguage);
|
||||
}
|
||||
|
||||
function resolveLanguageRank(suffix: string, preferredLanguages: string[]): number {
|
||||
const tokens = splitLanguageSuffix(suffix);
|
||||
for (let index = 0; index < preferredLanguages.length; index += 1) {
|
||||
const preferredLanguage = preferredLanguages[index]!;
|
||||
if (tokens.some((token) => languageTokenMatches(token, preferredLanguage))) {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
return Number.POSITIVE_INFINITY;
|
||||
}
|
||||
|
||||
function extensionRank(ext: string): number {
|
||||
if (ext === '.srt') return 0;
|
||||
if (ext === '.vtt') return 1;
|
||||
if (ext === '.ass') return 2;
|
||||
if (ext === '.ssa') return 3;
|
||||
return 4;
|
||||
}
|
||||
|
||||
function findSidecarSubtitleCandidates(
|
||||
sourcePath: string,
|
||||
preferredLanguages: string[],
|
||||
): SidecarCandidate[] {
|
||||
const source = path.parse(sourcePath);
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = readdirSync(source.dir);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
const prefix = `${source.name}.`;
|
||||
return entries
|
||||
.map((entry) => {
|
||||
const parsed = path.parse(entry);
|
||||
const ext = parsed.ext.toLowerCase();
|
||||
if (!SUPPORTED_SUBTITLE_EXTENSIONS.has(ext) || !parsed.name.startsWith(prefix)) {
|
||||
return null;
|
||||
}
|
||||
const suffix = parsed.name.slice(prefix.length);
|
||||
const languageRank = resolveLanguageRank(suffix, preferredLanguages);
|
||||
if (!Number.isFinite(languageRank)) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
path: path.join(source.dir, entry),
|
||||
languageRank,
|
||||
extensionRank: extensionRank(ext),
|
||||
name: entry,
|
||||
};
|
||||
})
|
||||
.filter((candidate): candidate is SidecarCandidate => candidate !== null)
|
||||
.sort((left, right) => {
|
||||
if (left.languageRank !== right.languageRank) return left.languageRank - right.languageRank;
|
||||
if (left.extensionRank !== right.extensionRank)
|
||||
return left.extensionRank - right.extensionRank;
|
||||
return left.name.localeCompare(right.name);
|
||||
});
|
||||
}
|
||||
|
||||
function combineCueText(cues: SubtitleCue[]): string {
|
||||
return unique(cues.map((cue) => cue.text.trim()).filter(Boolean))
|
||||
.join('\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function overlapSeconds(cue: SubtitleCue, startSeconds: number, endSeconds: number): number {
|
||||
return (
|
||||
Math.min(cue.endTime, endSeconds + TIMING_TOLERANCE_SECONDS) -
|
||||
Math.max(cue.startTime, startSeconds - TIMING_TOLERANCE_SECONDS)
|
||||
);
|
||||
}
|
||||
|
||||
function isSameCueTiming(left: SubtitleCue, right: SubtitleCue): boolean {
|
||||
return (
|
||||
Math.abs(left.startTime - right.startTime) <= SAME_TIMING_EPSILON_SECONDS &&
|
||||
Math.abs(left.endTime - right.endTime) <= SAME_TIMING_EPSILON_SECONDS
|
||||
);
|
||||
}
|
||||
|
||||
function compareCueTimingMatch(
|
||||
startSeconds: number,
|
||||
endSeconds: number,
|
||||
left: { cue: SubtitleCue; overlap: number },
|
||||
right: { cue: SubtitleCue; overlap: number },
|
||||
): number {
|
||||
if (left.overlap !== right.overlap) {
|
||||
return right.overlap - left.overlap;
|
||||
}
|
||||
|
||||
const leftStartDistance = Math.abs(left.cue.startTime - startSeconds);
|
||||
const rightStartDistance = Math.abs(right.cue.startTime - startSeconds);
|
||||
if (leftStartDistance !== rightStartDistance) {
|
||||
return leftStartDistance - rightStartDistance;
|
||||
}
|
||||
|
||||
const leftEndDistance = Math.abs(left.cue.endTime - endSeconds);
|
||||
const rightEndDistance = Math.abs(right.cue.endTime - endSeconds);
|
||||
if (leftEndDistance !== rightEndDistance) {
|
||||
return leftEndDistance - rightEndDistance;
|
||||
}
|
||||
|
||||
return left.cue.startTime - right.cue.startTime;
|
||||
}
|
||||
|
||||
function findCueTextAtTiming(cues: SubtitleCue[], startMs: number, endMs: number): string {
|
||||
const startSeconds = startMs / 1000;
|
||||
const endSeconds = endMs / 1000;
|
||||
const midpointSeconds = (startSeconds + endSeconds) / 2;
|
||||
|
||||
const midpointMatches = cues
|
||||
.filter(
|
||||
(cue) =>
|
||||
cue.startTime - TIMING_TOLERANCE_SECONDS <= midpointSeconds &&
|
||||
cue.endTime + TIMING_TOLERANCE_SECONDS >= midpointSeconds,
|
||||
)
|
||||
.map((cue) => ({ cue, overlap: overlapSeconds(cue, startSeconds, endSeconds) }))
|
||||
.sort((left, right) => compareCueTimingMatch(startSeconds, endSeconds, left, right));
|
||||
const [bestMidpointMatch] = midpointMatches;
|
||||
const midpointText = bestMidpointMatch
|
||||
? combineCueText(
|
||||
midpointMatches
|
||||
.filter((match) => isSameCueTiming(match.cue, bestMidpointMatch.cue))
|
||||
.map((match) => match.cue),
|
||||
)
|
||||
: '';
|
||||
if (midpointText) {
|
||||
return midpointText;
|
||||
}
|
||||
|
||||
const [bestOverlap] = cues
|
||||
.map((cue) => ({ cue, overlap: overlapSeconds(cue, startSeconds, endSeconds) }))
|
||||
.filter((entry) => entry.overlap > 0)
|
||||
.sort((left, right) => compareCueTimingMatch(startSeconds, endSeconds, left, right));
|
||||
return bestOverlap ? bestOverlap.cue.text.trim() : '';
|
||||
}
|
||||
|
||||
export function resolveSecondarySubtitleTextFromSidecar(input: {
|
||||
sourcePath: string;
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
languages?: readonly string[];
|
||||
}): string {
|
||||
if (!input.sourcePath || !existsSync(input.sourcePath)) {
|
||||
return '';
|
||||
}
|
||||
try {
|
||||
if (!statSync(input.sourcePath).isFile()) {
|
||||
return '';
|
||||
}
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
|
||||
const preferredLanguages = expandPreferredLanguages(input.languages);
|
||||
const candidates = findSidecarSubtitleCandidates(input.sourcePath, preferredLanguages);
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
const content = readFileSync(candidate.path, 'utf8');
|
||||
const cues = parseSubtitleCues(content, candidate.path);
|
||||
const text = findCueTextAtTiming(cues, input.startMs, input.endMs);
|
||||
if (text) {
|
||||
return text;
|
||||
}
|
||||
} catch {
|
||||
// Try the next matching sidecar.
|
||||
}
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Hono } from 'hono';
|
||||
import type { ImmersionTrackerService } from './immersion-tracker-service.js';
|
||||
import { splitSentenceSearchTerms } from './immersion-tracker/query-lexical.js';
|
||||
import http, { type IncomingMessage, type ServerResponse } from 'node:http';
|
||||
import { basename, extname, resolve, sep } from 'node:path';
|
||||
import { readFileSync, existsSync, statSync } from 'node:fs';
|
||||
@@ -16,6 +17,7 @@ import {
|
||||
} from '../../anki-field-config.js';
|
||||
import { resolveAnimatedImageLeadInSeconds } from '../../anki-integration/animated-image-sync.js';
|
||||
import type { AnilistRateLimiter } from './anilist/rate-limiter.js';
|
||||
import { resolveSecondarySubtitleTextFromSidecar } from './secondary-subtitle-sidecar.js';
|
||||
|
||||
type StatsServerNoteInfo = {
|
||||
noteId: number;
|
||||
@@ -356,9 +358,13 @@ export interface StatsServerConfig {
|
||||
mpvSocketPath?: string;
|
||||
ankiConnectConfig?: AnkiConnectConfig;
|
||||
getAnkiConnectConfig?: () => AnkiConnectConfig | undefined;
|
||||
getYomitanAnkiDeckName?: () => Promise<string | null | undefined> | string | null | undefined;
|
||||
secondarySubtitleLanguages?: string[];
|
||||
getSecondarySubtitleLanguages?: () => string[] | undefined;
|
||||
anilistRateLimiter?: AnilistRateLimiter;
|
||||
addYomitanNote?: (word: string) => Promise<number | null>;
|
||||
resolveAnkiNoteId?: (noteId: number) => number;
|
||||
resolveSentenceSearchHeadwords?: (term: string) => Promise<string[]> | string[];
|
||||
}
|
||||
|
||||
const STATS_STATIC_CONTENT_TYPES: Record<string, string> = {
|
||||
@@ -385,6 +391,47 @@ function defaultNowMs(): number {
|
||||
return Date.now();
|
||||
}
|
||||
|
||||
function parseBooleanQuery(raw: string | undefined, fallback: boolean): boolean {
|
||||
if (raw === undefined) return fallback;
|
||||
const normalized = raw.trim().toLowerCase();
|
||||
if (!normalized) return fallback;
|
||||
return !['0', 'false', 'no', 'off'].includes(normalized);
|
||||
}
|
||||
|
||||
function uniqueNonEmptyStrings(values: readonly string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const result: string[] = [];
|
||||
for (const value of values) {
|
||||
const normalized = value.trim();
|
||||
if (!normalized || seen.has(normalized)) continue;
|
||||
seen.add(normalized);
|
||||
result.push(normalized);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function buildSentenceSearchOptions(
|
||||
query: string,
|
||||
searchByHeadword: boolean,
|
||||
resolveSentenceSearchHeadwords: ((term: string) => Promise<string[]> | string[]) | undefined,
|
||||
): Promise<{ headwordTerms: Array<{ term: string; headwords: string[] }> } | undefined> {
|
||||
if (!searchByHeadword) return undefined;
|
||||
|
||||
const terms = splitSentenceSearchTerms(query);
|
||||
const headwordTerms: Array<{ term: string; headwords: string[] }> = [];
|
||||
for (const term of terms) {
|
||||
const resolved = resolveSentenceSearchHeadwords
|
||||
? await resolveSentenceSearchHeadwords(term)
|
||||
: [term];
|
||||
const headwords = uniqueNonEmptyStrings(resolved);
|
||||
if (headwords.length > 0) {
|
||||
headwordTerms.push({ term, headwords });
|
||||
}
|
||||
}
|
||||
|
||||
return headwordTerms.length > 0 ? { headwordTerms } : undefined;
|
||||
}
|
||||
|
||||
function buildAnkiNotePreview(
|
||||
fields: Record<string, { value: string }>,
|
||||
ankiConfig?: Pick<AnkiConnectConfig, 'fields'>,
|
||||
@@ -446,9 +493,13 @@ export function createStatsApp(
|
||||
mpvSocketPath?: string;
|
||||
ankiConnectConfig?: AnkiConnectConfig;
|
||||
getAnkiConnectConfig?: () => AnkiConnectConfig | undefined;
|
||||
getYomitanAnkiDeckName?: () => Promise<string | null | undefined> | string | null | undefined;
|
||||
secondarySubtitleLanguages?: string[];
|
||||
getSecondarySubtitleLanguages?: () => string[] | undefined;
|
||||
anilistRateLimiter?: AnilistRateLimiter;
|
||||
addYomitanNote?: (word: string) => Promise<number | null>;
|
||||
resolveAnkiNoteId?: (noteId: number) => number;
|
||||
resolveSentenceSearchHeadwords?: (term: string) => Promise<string[]> | string[];
|
||||
createMediaGenerator?: () => StatsServerMediaGenerator;
|
||||
onMiningTiming?: (event: StatsMiningTimingEvent) => void;
|
||||
nowMs?: () => number;
|
||||
@@ -458,6 +509,23 @@ export function createStatsApp(
|
||||
const nowMs = options?.nowMs ?? defaultNowMs;
|
||||
const getAnkiConnectConfig = (): AnkiConnectConfig | undefined =>
|
||||
options?.getAnkiConnectConfig?.() ?? options?.ankiConnectConfig;
|
||||
const getSecondarySubtitleLanguages = (): string[] =>
|
||||
options?.getSecondarySubtitleLanguages?.() ?? options?.secondarySubtitleLanguages ?? [];
|
||||
const getEffectiveMiningDeckName = async (ankiConfig: AnkiConnectConfig): Promise<string> => {
|
||||
const configuredDeckName = ankiConfig.deck?.trim() ?? '';
|
||||
if (configuredDeckName) return configuredDeckName;
|
||||
|
||||
try {
|
||||
const yomitanDeckName = await options?.getYomitanAnkiDeckName?.();
|
||||
return typeof yomitanDeckName === 'string' ? yomitanDeckName.trim() : '';
|
||||
} catch (error) {
|
||||
statsMiningLogger.warn(
|
||||
'Failed to resolve Yomitan Anki deck for stats mining:',
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
return '';
|
||||
}
|
||||
};
|
||||
|
||||
const recordMiningTiming = (event: StatsMiningTimingEvent): void => {
|
||||
options?.onMiningTiming?.(event);
|
||||
@@ -659,7 +727,13 @@ export function createStatsApp(
|
||||
const query = (c.req.query('q') ?? '').trim();
|
||||
if (!query) return c.json([]);
|
||||
const limit = parseIntQuery(c.req.query('limit'), 50, 100);
|
||||
const rows = await tracker.searchSubtitleSentences(query, limit);
|
||||
const searchByHeadword = parseBooleanQuery(c.req.query('headword'), true);
|
||||
const searchOptions = await buildSentenceSearchOptions(
|
||||
query,
|
||||
searchByHeadword,
|
||||
options?.resolveSentenceSearchHeadwords,
|
||||
);
|
||||
const rows = await tracker.searchSubtitleSentences(query, limit, searchOptions);
|
||||
return c.json(rows);
|
||||
});
|
||||
|
||||
@@ -997,7 +1071,8 @@ export function createStatsApp(
|
||||
const endMs = typeof body?.endMs === 'number' ? body.endMs : NaN;
|
||||
const sentence = typeof body?.sentence === 'string' ? body.sentence.trim() : '';
|
||||
const word = typeof body?.word === 'string' ? body.word.trim() : '';
|
||||
const secondaryText = typeof body?.secondaryText === 'string' ? body.secondaryText.trim() : '';
|
||||
const bodySecondaryText =
|
||||
typeof body?.secondaryText === 'string' ? body.secondaryText.trim() : '';
|
||||
const videoTitle = typeof body?.videoTitle === 'string' ? body.videoTitle.trim() : '';
|
||||
const rawMode = c.req.query('mode');
|
||||
const mode = rawMode === 'audio' ? 'audio' : rawMode === 'word' ? 'word' : 'sentence';
|
||||
@@ -1014,6 +1089,14 @@ export function createStatsApp(
|
||||
if (!ankiConfig) {
|
||||
return c.json({ error: 'AnkiConnect is not configured' }, 500);
|
||||
}
|
||||
const secondaryText =
|
||||
bodySecondaryText ||
|
||||
resolveSecondarySubtitleTextFromSidecar({
|
||||
sourcePath,
|
||||
startMs,
|
||||
endMs,
|
||||
languages: getSecondarySubtitleLanguages(),
|
||||
});
|
||||
|
||||
const client = new AnkiConnectClient(ankiConfig.url ?? 'http://127.0.0.1:8765');
|
||||
const mediaGen = options?.createMediaGenerator?.() ?? new MediaGenerator();
|
||||
@@ -1080,6 +1163,25 @@ export function createStatsApp(
|
||||
|
||||
const errors: string[] = [];
|
||||
let noteId: number;
|
||||
let effectiveDeckNamePromise: Promise<string> | null = null;
|
||||
const getEffectiveDeckNameForRequest = (): Promise<string> => {
|
||||
effectiveDeckNamePromise ??= getEffectiveMiningDeckName(ankiConfig);
|
||||
return effectiveDeckNamePromise;
|
||||
};
|
||||
const moveNoteToConfiguredDeck = async (id: number): Promise<void> => {
|
||||
const deckName = await getEffectiveDeckNameForRequest();
|
||||
if (!deckName) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const cardIds = await timeMiningPhase(mode, 'findCards', () =>
|
||||
client.findCards(`nid:${id}`),
|
||||
);
|
||||
await timeMiningPhase(mode, 'changeDeck', () => client.changeDeck(cardIds, deckName));
|
||||
} catch (err) {
|
||||
errors.push(`deck: ${(err as Error).message}`);
|
||||
}
|
||||
};
|
||||
|
||||
if (mode === 'word') {
|
||||
if (!options?.addYomitanNote) {
|
||||
@@ -1107,6 +1209,7 @@ export function createStatsApp(
|
||||
}
|
||||
|
||||
noteId = yomitanResult.value;
|
||||
await moveNoteToConfiguredDeck(noteId);
|
||||
const audioBuffer = audioResult.status === 'fulfilled' ? audioResult.value : null;
|
||||
if (audioResult.status === 'rejected')
|
||||
errors.push(`audio: ${(audioResult.reason as Error).message}`);
|
||||
@@ -1145,10 +1248,14 @@ export function createStatsApp(
|
||||
const mediaFields: Record<string, string> = {};
|
||||
const timestamp = Date.now();
|
||||
const sentenceFieldName = ankiConfig.fields?.sentence ?? 'Sentence';
|
||||
const translationFieldName = ankiConfig.fields?.translation ?? 'SelectionText';
|
||||
const audioFieldName = getStatsWordMiningAudioFieldName(ankiConfig, noteInfo);
|
||||
const imageFieldName = ankiConfig.fields?.image ?? 'Picture';
|
||||
|
||||
mediaFields[sentenceFieldName] = highlightedSentence;
|
||||
if (secondaryText) {
|
||||
mediaFields[translationFieldName] = secondaryText;
|
||||
}
|
||||
|
||||
if (audioBuffer) {
|
||||
const audioFilename = `subminer_audio_${timestamp}.mp3`;
|
||||
@@ -1214,7 +1321,7 @@ export function createStatsApp(
|
||||
const miscInfoFieldName = ankiConfig.fields?.miscInfo ?? '';
|
||||
|
||||
const fields: Record<string, string> = {
|
||||
[sentenceFieldName]: highlightedSentence,
|
||||
[sentenceFieldName]: mode === 'sentence' ? sentence : highlightedSentence,
|
||||
};
|
||||
|
||||
if (mode === 'sentence' && secondaryText) {
|
||||
@@ -1222,7 +1329,9 @@ export function createStatsApp(
|
||||
}
|
||||
|
||||
if (ankiConfig.isLapis?.enabled || ankiConfig.isKiku?.enabled) {
|
||||
if (word) {
|
||||
if (mode === 'sentence') {
|
||||
fields[wordFieldName] = sentence;
|
||||
} else if (word) {
|
||||
fields[wordFieldName] = word;
|
||||
}
|
||||
if (mode === 'sentence') {
|
||||
@@ -1233,13 +1342,13 @@ export function createStatsApp(
|
||||
}
|
||||
|
||||
const model = ankiConfig.isLapis?.sentenceCardModel || 'Basic';
|
||||
const deck = ankiConfig.deck?.trim() || 'Default';
|
||||
const tags = ankiConfig.tags ?? ['SubMiner'];
|
||||
|
||||
const addNotePromise = timeMiningPhase(
|
||||
mode,
|
||||
'addNote',
|
||||
() => client.addNote(deck, model, fields, tags),
|
||||
async () =>
|
||||
client.addNote((await getEffectiveDeckNameForRequest()) || 'Default', model, fields, tags),
|
||||
(id) => ({
|
||||
noteId: id,
|
||||
}),
|
||||
@@ -1265,6 +1374,7 @@ export function createStatsApp(
|
||||
);
|
||||
}
|
||||
noteId = addNoteResult.value;
|
||||
await moveNoteToConfiguredDeck(noteId);
|
||||
|
||||
const mediaFields: Record<string, string> = {};
|
||||
const timestamp = Date.now();
|
||||
@@ -1370,9 +1480,13 @@ export function startStatsServer(config: StatsServerConfig): { close: () => void
|
||||
mpvSocketPath: config.mpvSocketPath,
|
||||
ankiConnectConfig: config.ankiConnectConfig,
|
||||
getAnkiConnectConfig: config.getAnkiConnectConfig,
|
||||
getYomitanAnkiDeckName: config.getYomitanAnkiDeckName,
|
||||
secondarySubtitleLanguages: config.secondarySubtitleLanguages,
|
||||
getSecondarySubtitleLanguages: config.getSecondarySubtitleLanguages,
|
||||
anilistRateLimiter: config.anilistRateLimiter,
|
||||
addYomitanNote: config.addYomitanNote,
|
||||
resolveAnkiNoteId: config.resolveAnkiNoteId,
|
||||
resolveSentenceSearchHeadwords: config.resolveSentenceSearchHeadwords,
|
||||
});
|
||||
|
||||
const bunRuntime = globalThis as typeof globalThis & {
|
||||
|
||||
@@ -151,6 +151,56 @@ test('syncYomitanDefaultAnkiServer injects force override when enabled', async (
|
||||
assert.match(scriptValue, /forceOverride = true/);
|
||||
});
|
||||
|
||||
test('syncYomitanDefaultAnkiServer updates the active profile Anki deck', async () => {
|
||||
const optionsFull = {
|
||||
profileCurrent: 0,
|
||||
profiles: [
|
||||
{
|
||||
options: {
|
||||
anki: {
|
||||
server: 'http://127.0.0.1:8766',
|
||||
cardFormats: [
|
||||
{ type: 'term', deck: 'Default', model: 'Mining Note', fields: {} },
|
||||
{ type: 'kanji', deck: 'Kanji', model: 'Kanji Note', fields: {} },
|
||||
],
|
||||
terms: { deck: 'Default', model: 'Legacy Note', fields: {} },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
let savedOptions: typeof optionsFull | null = null;
|
||||
const deps = createDeps((script) =>
|
||||
runInjectedYomitanScript(script, (action, params) => {
|
||||
if (action === 'optionsGetFull') {
|
||||
return JSON.parse(JSON.stringify(optionsFull));
|
||||
}
|
||||
if (action === 'setAllSettings') {
|
||||
savedOptions = (params as { value: typeof optionsFull }).value;
|
||||
return true;
|
||||
}
|
||||
throw new Error(`Unexpected action: ${action}`);
|
||||
}),
|
||||
);
|
||||
|
||||
const synced = await syncYomitanDefaultAnkiServer(
|
||||
'http://127.0.0.1:8766',
|
||||
deps,
|
||||
{
|
||||
error: () => undefined,
|
||||
info: () => undefined,
|
||||
},
|
||||
{ deck: 'Minecraft', forceOverride: true },
|
||||
);
|
||||
|
||||
assert.equal(synced, true);
|
||||
assert.ok(savedOptions);
|
||||
const saved = savedOptions as typeof optionsFull;
|
||||
assert.equal(saved.profiles[0]?.options.anki.cardFormats[0]?.deck, 'Minecraft');
|
||||
assert.equal(saved.profiles[0]?.options.anki.cardFormats[1]?.deck, 'Kanji');
|
||||
assert.equal(saved.profiles[0]?.options.anki.terms.deck, 'Minecraft');
|
||||
});
|
||||
|
||||
test('syncYomitanDefaultAnkiServer logs and returns false on script failure', async () => {
|
||||
const deps = createDeps(async () => {
|
||||
throw new Error('execute failed');
|
||||
|
||||
@@ -1783,6 +1783,7 @@ export async function syncYomitanDefaultAnkiServer(
|
||||
logger: LoggerLike,
|
||||
options?: {
|
||||
forceOverride?: boolean;
|
||||
deck?: string;
|
||||
},
|
||||
): Promise<boolean> {
|
||||
const normalizedTargetServer = serverUrl.trim();
|
||||
@@ -1790,6 +1791,7 @@ export async function syncYomitanDefaultAnkiServer(
|
||||
return false;
|
||||
}
|
||||
const forceOverride = options?.forceOverride === true;
|
||||
const normalizedTargetDeck = options?.deck?.trim() ?? '';
|
||||
|
||||
const isReady = await ensureYomitanParserWindow(deps, logger);
|
||||
const parserWindow = deps.getYomitanParserWindow();
|
||||
@@ -1819,6 +1821,7 @@ export async function syncYomitanDefaultAnkiServer(
|
||||
});
|
||||
|
||||
const targetServer = ${JSON.stringify(normalizedTargetServer)};
|
||||
const targetDeck = ${JSON.stringify(normalizedTargetDeck)};
|
||||
const forceOverride = ${forceOverride ? 'true' : 'false'};
|
||||
const optionsFull = await invoke("optionsGetFull", undefined);
|
||||
const profiles = Array.isArray(optionsFull.profiles) ? optionsFull.profiles : [];
|
||||
@@ -1843,18 +1846,54 @@ export async function syncYomitanDefaultAnkiServer(
|
||||
|
||||
const currentServerRaw = targetProfile.options.anki.server;
|
||||
const currentServer = typeof currentServerRaw === "string" ? currentServerRaw.trim() : "";
|
||||
if (currentServer === targetServer) {
|
||||
return { updated: false, matched: true, reason: "already-target", currentServer, targetServer };
|
||||
}
|
||||
const canReplaceCurrent =
|
||||
forceOverride || currentServer.length === 0 || currentServer === "http://127.0.0.1:8765";
|
||||
if (!canReplaceCurrent) {
|
||||
return { updated: false, matched: false, reason: "blocked-existing-server", currentServer, targetServer };
|
||||
let changed = false;
|
||||
if (currentServer !== targetServer) {
|
||||
const canReplaceCurrent =
|
||||
forceOverride || currentServer.length === 0 || currentServer === "http://127.0.0.1:8765";
|
||||
if (!canReplaceCurrent) {
|
||||
return { updated: false, matched: false, reason: "blocked-existing-server", currentServer, targetServer };
|
||||
}
|
||||
|
||||
targetProfile.options.anki.server = targetServer;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if (targetDeck) {
|
||||
const cardFormats = Array.isArray(targetProfile.options.anki.cardFormats)
|
||||
? targetProfile.options.anki.cardFormats
|
||||
: [];
|
||||
for (const cardFormat of cardFormats) {
|
||||
if (
|
||||
!cardFormat ||
|
||||
typeof cardFormat !== "object" ||
|
||||
cardFormat.type !== "term" ||
|
||||
cardFormat.enabled === false
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const currentDeck = typeof cardFormat.deck === "string" ? cardFormat.deck.trim() : "";
|
||||
if (currentDeck !== targetDeck) {
|
||||
cardFormat.deck = targetDeck;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
const terms = targetProfile.options.anki.terms;
|
||||
if (terms && typeof terms === "object") {
|
||||
const currentTermDeck = typeof terms.deck === "string" ? terms.deck.trim() : "";
|
||||
if (currentTermDeck !== targetDeck) {
|
||||
terms.deck = targetDeck;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!changed) {
|
||||
return { updated: false, matched: true, reason: "already-target", currentServer, targetServer, targetDeck };
|
||||
}
|
||||
|
||||
targetProfile.options.anki.server = targetServer;
|
||||
await invoke("setAllSettings", { value: optionsFull, source: "subminer" });
|
||||
return { updated: true, matched: true, currentServer, targetServer };
|
||||
return { updated: true, matched: true, currentServer, targetServer, targetDeck };
|
||||
})();
|
||||
`;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user