feat(stats): add headword sentence search and rename related words

- Search by headword enabled by default; finds inflected variants (e.g. 知らない → 知らねえ)
- Add "Search by headword" toggle to switch back to exact text/title matching
- Rename "Similar Words" → "Related Seen Words" with tighter matching (same reading/shared kanji)
- ankiConnect.deck falls back to Yomitan mining deck when empty
This commit is contained in:
2026-06-06 01:50:01 -07:00
parent e18ccfe288
commit 83897c81d1
31 changed files with 1522 additions and 183 deletions
@@ -338,7 +338,10 @@ function withTempDir<T>(fn: (dir: string) => Promise<T> | T): Promise<T> | T {
type CapturedAnkiRequest = {
action?: string;
params?: {
cards?: number[];
deck?: string;
notes?: number[];
query?: string;
note?: {
id?: number;
deckName?: string;
@@ -384,6 +387,10 @@ async function withFakeAnkiConnect<T>(
})),
error: null,
};
} else if (payload.action === 'findCards') {
body = { result: [9001], error: null };
} else if (payload.action === 'changeDeck') {
body = { result: null, error: null };
}
res.writeHead(200, { 'Content-Type': 'application/json' });
@@ -435,6 +442,43 @@ describe('stats server API routes', () => {
assert.ok(Array.isArray(body));
});
it('GET /api/stats/sentences/search resolves headword candidates by default', async () => {
const seen: Array<{
query: string;
limit: number;
options: unknown;
}> = [];
const resolvedTerms: string[] = [];
const app = createStatsApp(
createMockTracker({
searchSubtitleSentences: async (query: string, limit: number, options: unknown) => {
seen.push({ query, limit, options });
return [];
},
}),
{
resolveSentenceSearchHeadwords: async (term: string) => {
resolvedTerms.push(term);
return term === '知らない' ? ['知る'] : [];
},
},
);
const res = await app.request(
'/api/stats/sentences/search?q=%E7%9F%A5%E3%82%89%E3%81%AA%E3%81%84&limit=12',
);
assert.equal(res.status, 200);
assert.deepEqual(resolvedTerms, ['知らない']);
assert.deepEqual(seen, [
{
query: '知らない',
limit: 12,
options: { headwordTerms: [{ term: '知らない', headwords: ['知る'] }] },
},
]);
});
it('GET /api/stats/sessions enriches known-word metrics using filtered persisted totals', async () => {
await withTempDir(async (dir) => {
const cachePath = path.join(dir, 'known-words.json');
@@ -1162,12 +1206,62 @@ describe('stats server API routes', () => {
const addNoteRequest = requests.find((request) => request.action === 'addNote');
assert.equal(addNoteRequest?.params?.note?.deckName, 'Default');
assert.equal(addNoteRequest?.params?.note?.modelName, 'Lapis Morph');
assert.equal(addNoteRequest?.params?.note?.fields?.Sentence, '<b>猫</b>を見た');
assert.equal(addNoteRequest?.params?.note?.fields?.Sentence, 'を見た');
assert.equal(addNoteRequest?.params?.note?.fields?.IsSentenceCard, 'x');
});
});
});
it('POST /api/stats/mine-card uses Yomitan deck for direct sentence cards when config deck is empty', async () => {
await withTempDir(async (dir) => {
const sourcePath = path.join(dir, 'episode.mkv');
fs.writeFileSync(sourcePath, 'fake media');
await withFakeAnkiConnect(async (requests, url) => {
const app = createStatsApp(createMockTracker(), {
getYomitanAnkiDeckName: async () => 'Minecraft',
ankiConnectConfig: {
url,
deck: '',
tags: ['SubMiner'],
fields: {
word: 'Expression',
sentence: 'Sentence',
translation: 'SelectionText',
},
media: {
generateAudio: false,
generateImage: false,
},
isLapis: {
enabled: true,
sentenceCardModel: 'Lapis Morph',
},
},
});
const res = await app.request('/api/stats/mine-card?mode=sentence', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sourcePath,
startMs: 1_000,
endMs: 2_000,
sentence: '猫を見た',
word: '猫',
secondaryText: 'I saw a cat',
videoTitle: 'Episode 1',
}),
});
const body = await res.json();
assert.equal(res.status, 200, JSON.stringify(body));
const addNoteRequest = requests.find((request) => request.action === 'addNote');
assert.equal(addNoteRequest?.params?.note?.deckName, 'Minecraft');
});
});
});
it('POST /api/stats/mine-card resolves Anki config at request time', async () => {
await withTempDir(async (dir) => {
const sourcePath = path.join(dir, 'episode.mkv');
@@ -1294,7 +1388,7 @@ describe('stats server API routes', () => {
});
});
it('POST /api/stats/mine-card leaves word card selection text to Yomitan glossary fields', async () => {
it('POST /api/stats/mine-card writes secondary subtitles to word card selection text', async () => {
await withTempDir(async (dir) => {
const sourcePath = path.join(dir, 'episode.mkv');
fs.writeFileSync(sourcePath, 'fake media');
@@ -1340,7 +1434,286 @@ describe('stats server API routes', () => {
const updateRequest = requests.find((request) => request.action === 'updateNoteFields');
assert.equal(updateRequest?.params?.note?.id, 777);
assert.equal(updateRequest?.params?.note?.fields?.Sentence, '<b>猫</b>を見た');
assert.equal(updateRequest?.params?.note?.fields?.SelectionText, undefined);
assert.equal(updateRequest?.params?.note?.fields?.SelectionText, 'I saw a cat');
});
});
});
it('POST /api/stats/mine-card moves Yomitan-created word notes to the configured deck', async () => {
await withTempDir(async (dir) => {
const sourcePath = path.join(dir, 'episode.mkv');
fs.writeFileSync(sourcePath, 'fake media');
await withFakeAnkiConnect(async (requests, url) => {
const app = createStatsApp(createMockTracker(), {
addYomitanNote: async () => 777,
ankiConnectConfig: {
url,
deck: 'Mining',
fields: {
sentence: 'Sentence',
translation: 'SelectionText',
},
media: {
generateAudio: false,
generateImage: false,
},
},
});
const res = await app.request('/api/stats/mine-card?mode=word', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sourcePath,
startMs: 1_000,
endMs: 2_000,
sentence: '猫を見た',
word: '猫',
videoTitle: 'Episode 1',
}),
});
const body = await res.json();
assert.equal(res.status, 200, JSON.stringify(body));
const findCardsRequest = requests.find((request) => request.action === 'findCards');
assert.equal(findCardsRequest?.params?.query, 'nid:777');
const changeDeckRequest = requests.find((request) => request.action === 'changeDeck');
assert.deepEqual(changeDeckRequest?.params?.cards, [9001]);
assert.equal(changeDeckRequest?.params?.deck, 'Mining');
});
});
});
it('POST /api/stats/mine-card moves Yomitan-created word notes to Yomitan deck when config deck is empty', async () => {
await withTempDir(async (dir) => {
const sourcePath = path.join(dir, 'episode.mkv');
fs.writeFileSync(sourcePath, 'fake media');
await withFakeAnkiConnect(async (requests, url) => {
const app = createStatsApp(createMockTracker(), {
getYomitanAnkiDeckName: async () => 'Minecraft',
addYomitanNote: async () => 777,
ankiConnectConfig: {
url,
deck: '',
fields: {
sentence: 'Sentence',
translation: 'SelectionText',
},
media: {
generateAudio: false,
generateImage: false,
},
},
});
const res = await app.request('/api/stats/mine-card?mode=word', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sourcePath,
startMs: 1_000,
endMs: 2_000,
sentence: '猫を見た',
word: '猫',
videoTitle: 'Episode 1',
}),
});
const body = await res.json();
assert.equal(res.status, 200, JSON.stringify(body));
const findCardsRequest = requests.find((request) => request.action === 'findCards');
assert.equal(findCardsRequest?.params?.query, 'nid:777');
const changeDeckRequest = requests.find((request) => request.action === 'changeDeck');
assert.deepEqual(changeDeckRequest?.params?.cards, [9001]);
assert.equal(changeDeckRequest?.params?.deck, 'Minecraft');
});
});
});
it('POST /api/stats/mine-card uses the full sentence as sentence-card expression', async () => {
await withTempDir(async (dir) => {
const sourcePath = path.join(dir, 'episode.mkv');
fs.writeFileSync(sourcePath, 'fake media');
await withFakeAnkiConnect(async (requests, url) => {
const app = createStatsApp(createMockTracker(), {
ankiConnectConfig: {
url,
deck: 'Minecraft',
tags: ['SubMiner'],
fields: {
word: 'Expression',
sentence: 'Sentence',
translation: 'SelectionText',
},
media: {
generateAudio: false,
generateImage: false,
},
isLapis: {
enabled: true,
sentenceCardModel: 'Lapis Morph',
},
},
});
const res = await app.request('/api/stats/mine-card?mode=sentence', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sourcePath,
startMs: 1_000,
endMs: 2_000,
sentence: '猫を見た',
word: '猫',
secondaryText: 'I saw a cat',
videoTitle: 'Episode 1',
}),
});
const body = await res.json();
assert.equal(res.status, 200, JSON.stringify(body));
const addNoteRequest = requests.find((request) => request.action === 'addNote');
assert.equal(addNoteRequest?.params?.note?.deckName, 'Minecraft');
assert.equal(addNoteRequest?.params?.note?.fields?.Expression, '猫を見た');
assert.equal(addNoteRequest?.params?.note?.fields?.Sentence, '猫を見た');
assert.equal(addNoteRequest?.params?.note?.fields?.SelectionText, 'I saw a cat');
assert.equal(addNoteRequest?.params?.note?.fields?.IsSentenceCard, 'x');
});
});
});
it('POST /api/stats/mine-card fills selection text from a matching secondary sidecar subtitle', async () => {
await withTempDir(async (dir) => {
const sourcePath = path.join(dir, 'episode.mkv');
fs.writeFileSync(sourcePath, 'fake media');
fs.writeFileSync(
path.join(dir, 'episode.en.srt'),
[
'1',
'00:00:00,800 --> 00:00:02,500',
'I saw a cat.',
'',
'2',
'00:00:03,000 --> 00:00:04,000',
'Not this line.',
'',
].join('\n'),
);
await withFakeAnkiConnect(async (requests, url) => {
const app = createStatsApp(createMockTracker(), {
ankiConnectConfig: {
url,
deck: 'Minecraft',
tags: ['SubMiner'],
fields: {
word: 'Expression',
sentence: 'Sentence',
translation: 'SelectionText',
},
media: {
generateAudio: false,
generateImage: false,
},
isLapis: {
enabled: true,
sentenceCardModel: 'Lapis Morph',
},
},
secondarySubtitleLanguages: ['en'],
} as Parameters<typeof createStatsApp>[1]);
const res = await app.request('/api/stats/mine-card?mode=sentence', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sourcePath,
startMs: 1_000,
endMs: 2_000,
sentence: '猫を見た',
word: '猫',
videoTitle: 'Episode 1',
}),
});
const body = await res.json();
assert.equal(res.status, 200, JSON.stringify(body));
const addNoteRequest = requests.find((request) => request.action === 'addNote');
assert.equal(addNoteRequest?.params?.note?.deckName, 'Minecraft');
assert.equal(addNoteRequest?.params?.note?.fields?.SelectionText, 'I saw a cat.');
});
});
});
it('POST /api/stats/mine-card does not append the next sidecar cue near a timing boundary', async () => {
await withTempDir(async (dir) => {
const sourcePath = path.join(dir, 'episode.mkv');
fs.writeFileSync(sourcePath, 'fake media');
fs.writeFileSync(
path.join(dir, 'episode.en.srt'),
[
'1',
'00:00:00,800 --> 00:00:01,500',
"I don't give a damn what family she's from.",
'',
'2',
'00:00:01,700 --> 00:00:03,000',
'That snobby attitude just pisses me off!',
'',
].join('\n'),
);
await withFakeAnkiConnect(async (requests, url) => {
const app = createStatsApp(createMockTracker(), {
ankiConnectConfig: {
url,
deck: 'Minecraft',
tags: ['SubMiner'],
fields: {
word: 'Expression',
sentence: 'Sentence',
translation: 'SelectionText',
},
media: {
generateAudio: false,
generateImage: false,
},
isLapis: {
enabled: true,
sentenceCardModel: 'Lapis Morph',
},
},
secondarySubtitleLanguages: ['en'],
} as Parameters<typeof createStatsApp>[1]);
const res = await app.request('/api/stats/mine-card?mode=sentence', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sourcePath,
startMs: 1_000,
endMs: 2_000,
sentence: '名門か何だか知らねえが',
word: '名門',
videoTitle: 'Episode 1',
}),
});
const body = await res.json();
assert.equal(res.status, 200, JSON.stringify(body));
const addNoteRequest = requests.find((request) => request.action === 'addNote');
assert.equal(
addNoteRequest?.params?.note?.fields?.SelectionText,
"I don't give a damn what family she's from.",
);
});
});
});
@@ -1466,6 +1839,8 @@ describe('stats server API routes', () => {
'generateAudio',
'generateScreenshot',
'addNote',
'findCards',
'changeDeck',
'uploadAudio',
'uploadImage',
'updateNoteFields',
@@ -149,6 +149,7 @@ import {
type MediaLibraryRow,
type NewAnimePerDayRow,
type QueuedWrite,
type SentenceSearchOptions,
type SentenceSearchResultRow,
type SessionEventRow,
type SessionState,
@@ -570,8 +571,12 @@ export class ImmersionTrackerService {
return getKanjiOccurrences(this.db, kanji, limit, offset);
}
async searchSubtitleSentences(query: string, limit = 50): Promise<SentenceSearchResultRow[]> {
return searchSubtitleSentences(this.db, query, limit);
async searchSubtitleSentences(
query: string,
limit = 50,
options?: SentenceSearchOptions,
): Promise<SentenceSearchResultRow[]> {
return searchSubtitleSentences(this.db, query, limit, options);
}
async getSessionEvents(
@@ -356,6 +356,81 @@ test('split session and lexical helpers return distinct-headword, detail, appear
}
});
test('similar words use same reading and shared kanji without kana suffix noise', () => {
const { db, dbPath, stmts } = createDb();
try {
const animeId = getOrCreateAnimeRecord(db, {
parsedTitle: 'Similar Words Anime',
canonicalTitle: 'Similar Words Anime',
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/similar-words.mkv', {
canonicalTitle: 'Similar Words Episode',
sourcePath: '/tmp/similar-words.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const sessionId = startSessionRecord(db, videoId, 1_000_000).sessionId;
const araiId = insertWordOccurrence(db, stmts, {
sessionId,
videoId,
animeId,
lineIndex: 1,
text: '荒い息',
word: { headword: '荒い', word: '荒い', reading: 'あらい' },
});
insertWordOccurrence(db, stmts, {
sessionId,
videoId,
animeId,
lineIndex: 2,
text: '洗い物',
word: { headword: '洗い', word: '洗い', reading: 'あらい' },
});
insertWordOccurrence(db, stmts, {
sessionId,
videoId,
animeId,
lineIndex: 3,
text: '荒波',
word: { headword: '荒波', word: '荒波', reading: 'あらなみ' },
});
for (let lineIndex = 4; lineIndex < 9; lineIndex++) {
insertWordOccurrence(db, stmts, {
sessionId,
videoId,
animeId,
lineIndex,
text: '良い',
word: { headword: '良い', word: '良い', reading: 'よい' },
});
}
insertWordOccurrence(db, stmts, {
sessionId,
videoId,
animeId,
lineIndex: 9,
text: 'お構いなく',
word: { headword: 'お構いなく', word: 'お構いなく', reading: 'おかまいなく' },
});
assert.deepEqual(
getSimilarWords(db, araiId, 10).map((row) => row.headword),
['洗い', '荒波'],
);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('split library helpers return anime/media session and analytics rows', () => {
const { db, dbPath, stmts } = createDb();
@@ -3785,6 +3785,90 @@ test('searchSubtitleSentences searches known subtitle lines and returns media co
}
});
test('searchSubtitleSentences searches subtitle lines by resolved headword candidates', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const animeId = getOrCreateAnimeRecord(db, {
parsedTitle: 'Little Witch Academia',
canonicalTitle: 'Little Witch Academia',
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: '{"source":"test"}',
});
const videoId = getOrCreateVideoRecord(db, 'local:/tmp/lwa-05.mkv', {
canonicalTitle: 'Episode 5',
sourcePath: '/tmp/Little Witch Academia S01E05.mkv',
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
linkVideoToAnimeRecord(db, videoId, {
animeId,
parsedBasename: 'Little Witch Academia S01E05.mkv',
parsedTitle: 'Little Witch Academia',
parsedSeason: 1,
parsedEpisode: 5,
parserSource: 'fallback',
parserConfidence: 1,
parseMetadataJson: '{"episode":5}',
});
const { sessionId } = startSessionRecord(db, videoId, 4_000_000);
const lineResult = db
.prepare(
`INSERT INTO imm_subtitle_lines (
session_id, event_id, video_id, anime_id, line_index, segment_start_ms, segment_end_ms,
text, secondary_text, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
)
.run(
sessionId,
null,
videoId,
animeId,
20,
247_000,
250_000,
'ああ、名無しが何だか知らねえが',
null,
4_000,
4_000,
);
const wordResult = db
.prepare(
`INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
)
.run('知る', '知らねえ', 'しらねえ', 'verb', '動詞', '自立', '', 4_000, 4_000, 1);
db.prepare(
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count)
VALUES (?, ?, ?)`,
).run(Number(lineResult.lastInsertRowid), Number(wordResult.lastInsertRowid), 1);
assert.deepEqual(searchSubtitleSentences(db, '知らない', 10), []);
const rows = searchSubtitleSentences(db, '知らない', 10, {
headwordTerms: [{ term: '知らない', headwords: ['知る'] }],
});
assert.deepEqual(
rows.map((row) => row.text),
['ああ、名無しが何だか知らねえが'],
);
assert.deepEqual(
searchSubtitleSentences(db, '知らねえ', 10).map((row) => row.text),
['ああ、名無しが何だか知らねえが'],
);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getKanjiOccurrences maps a kanji back to anime, video, and subtitle line context', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -7,6 +7,7 @@ import type {
KanjiOccurrenceRow,
KanjiStatsRow,
KanjiWordRow,
SentenceSearchOptions,
SentenceSearchResultRow,
SessionEventRow,
SimilarWordRow,
@@ -23,6 +24,7 @@ const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
const SENTENCE_SEARCH_MAX_LIMIT = 100;
const KANJI_PATTERN = /\p{Script=Han}/gu;
function resolveSentenceSearchLimit(limit: number): number {
if (!Number.isFinite(limit)) return SENTENCE_SEARCH_DEFAULT_LIMIT;
@@ -31,7 +33,7 @@ function resolveSentenceSearchLimit(limit: number): number {
return Math.min(normalized, SENTENCE_SEARCH_MAX_LIMIT);
}
function splitSearchTerms(query: string): string[] {
export function splitSentenceSearchTerms(query: string): string[] {
return query
.trim()
.split(/\s+/)
@@ -44,6 +46,33 @@ function escapeLikeTerm(term: string): string {
return term.replace(/[\\%_]/g, (match) => `\\${match}`);
}
function uniqueNonEmptyTerms(values: readonly string[] | undefined): string[] {
const seen = new Set<string>();
const terms: string[] = [];
for (const value of values ?? []) {
const term = value.trim();
if (!term || seen.has(term)) continue;
seen.add(term);
terms.push(term);
}
return terms;
}
function getHeadwordCandidatesForSentenceSearchTerm(
term: string,
options: SentenceSearchOptions | undefined,
): string[] {
const headwords =
options?.headwordTerms
?.filter((entry) => entry.term === term)
.flatMap((entry) => entry.headwords) ?? [];
return uniqueNonEmptyTerms(headwords);
}
function uniqueKanji(text: string): string[] {
return Array.from(new Set(text.match(KANJI_PATTERN) ?? []));
}
function toVocabularyToken(row: VocabularyStatsRow): MergedToken {
const partOfSpeech =
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
@@ -238,8 +267,9 @@ export function searchSubtitleSentences(
db: DatabaseSync,
query: string,
limit = SENTENCE_SEARCH_DEFAULT_LIMIT,
options?: SentenceSearchOptions,
): SentenceSearchResultRow[] {
const terms = splitSearchTerms(query);
const terms = splitSentenceSearchTerms(query);
if (terms.length === 0) return [];
const resolvedLimit = resolveSentenceSearchLimit(limit);
@@ -247,14 +277,28 @@ export function searchSubtitleSentences(
const params: string[] = [];
for (const term of terms) {
const likeTerm = `%${escapeLikeTerm(term)}%`;
const headwords = getHeadwordCandidatesForSentenceSearchTerm(term, options);
const headwordClause =
headwords.length > 0
? `
OR EXISTS (
SELECT 1
FROM imm_word_line_occurrences o
JOIN imm_words w ON w.id = o.word_id
WHERE o.line_id = l.line_id
AND w.headword IN (${headwords.map(() => '?').join(', ')})
)
`
: '';
clauses.push(`
(
l.text LIKE ? ESCAPE '\\'
OR v.canonical_title LIKE ? ESCAPE '\\'
OR COALESCE(a.canonical_title, '') LIKE ? ESCAPE '\\'
${headwordClause}
)
`);
params.push(likeTerm, likeTerm, likeTerm);
params.push(likeTerm, likeTerm, likeTerm, ...headwords);
}
return db
@@ -359,24 +403,38 @@ export function getSimilarWords(db: DatabaseSync, wordId: number, limit = 10): S
reading: string;
} | null;
if (!word || word.headword.trim() === '') return [];
const clauses: string[] = [];
const params: string[] = [];
const reading = word.reading.trim();
if (reading !== '') {
clauses.push('reading = ?');
params.push(word.reading);
}
for (const kanji of uniqueKanji(word.headword)) {
clauses.push("headword LIKE ? ESCAPE '\\'");
params.push(`%${escapeLikeTerm(kanji)}%`);
}
if (clauses.length === 0) return [];
const orderBy =
reading !== '' ? 'CASE WHEN reading = ? THEN 0 ELSE 1 END, frequency DESC' : 'frequency DESC';
const orderParams = reading !== '' ? [word.reading] : [];
return db
.prepare(
`
SELECT id AS wordId, headword, word, reading, frequency
FROM imm_words
WHERE id != ?
AND (reading = ? OR headword LIKE ? OR headword LIKE ?)
ORDER BY frequency DESC
AND (${clauses.join(' OR ')})
ORDER BY ${orderBy}
LIMIT ?
`,
)
.all(
wordId,
word.reading,
`%${word.headword.charAt(0)}%`,
`%${word.headword.charAt(word.headword.length - 1)}%`,
limit,
) as SimilarWordRow[];
.all(wordId, ...params, ...orderParams, limit) as SimilarWordRow[];
}
export function getKanjiDetail(db: DatabaseSync, kanjiId: number): KanjiDetailRow | null {
@@ -381,6 +381,15 @@ export interface SentenceSearchResultRow {
text: string;
}
export interface SentenceSearchHeadwordTerm {
term: string;
headwords: string[];
}
export interface SentenceSearchOptions {
headwordTerms?: SentenceSearchHeadwordTerm[];
}
export interface SessionEventRow {
eventType: number;
tsMs: number;
@@ -0,0 +1,226 @@
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
import path from 'node:path';
import { parseSubtitleCues, type SubtitleCue } from './subtitle-cue-parser.js';
import { isEnglishYoutubeLang, normalizeYoutubeLangCode } from './youtube/labels.js';
const DEFAULT_SECONDARY_SUBTITLE_LANGUAGES = ['en', 'eng', 'english', 'en-us', 'enus'];
const SUPPORTED_SUBTITLE_EXTENSIONS = new Set(['.srt', '.vtt', '.ass', '.ssa']);
const TIMING_TOLERANCE_SECONDS = 0.25;
const SAME_TIMING_EPSILON_SECONDS = 0.001;
type SidecarCandidate = {
path: string;
languageRank: number;
extensionRank: number;
name: string;
};
function unique(values: string[]): string[] {
return values.filter((value, index) => value.length > 0 && values.indexOf(value) === index);
}
function expandPreferredLanguages(languages: readonly string[] | undefined): string[] {
const normalized = unique(
(languages ?? []).map((language) => normalizeYoutubeLangCode(language)).filter(Boolean),
);
const base = normalized.length > 0 ? normalized : DEFAULT_SECONDARY_SUBTITLE_LANGUAGES;
const expanded: string[] = [];
for (const language of base) {
expanded.push(language);
if (isEnglishYoutubeLang(language)) {
expanded.push(...DEFAULT_SECONDARY_SUBTITLE_LANGUAGES);
}
}
return unique(expanded);
}
function splitLanguageSuffix(value: string): string[] {
const normalizedWhole = normalizeYoutubeLangCode(value);
const tokens = value
.split(/[^A-Za-z0-9-]+/g)
.map((token) => normalizeYoutubeLangCode(token))
.filter(Boolean);
return unique([normalizedWhole, ...tokens]);
}
function languageTokenMatches(token: string, preferredLanguage: string): boolean {
if (token === preferredLanguage) {
return true;
}
if (token.startsWith(`${preferredLanguage}-`) || preferredLanguage.startsWith(`${token}-`)) {
return true;
}
return isEnglishYoutubeLang(token) && isEnglishYoutubeLang(preferredLanguage);
}
function resolveLanguageRank(suffix: string, preferredLanguages: string[]): number {
const tokens = splitLanguageSuffix(suffix);
for (let index = 0; index < preferredLanguages.length; index += 1) {
const preferredLanguage = preferredLanguages[index]!;
if (tokens.some((token) => languageTokenMatches(token, preferredLanguage))) {
return index;
}
}
return Number.POSITIVE_INFINITY;
}
function extensionRank(ext: string): number {
if (ext === '.srt') return 0;
if (ext === '.vtt') return 1;
if (ext === '.ass') return 2;
if (ext === '.ssa') return 3;
return 4;
}
function findSidecarSubtitleCandidates(
sourcePath: string,
preferredLanguages: string[],
): SidecarCandidate[] {
const source = path.parse(sourcePath);
let entries: string[];
try {
entries = readdirSync(source.dir);
} catch {
return [];
}
const prefix = `${source.name}.`;
return entries
.map((entry) => {
const parsed = path.parse(entry);
const ext = parsed.ext.toLowerCase();
if (!SUPPORTED_SUBTITLE_EXTENSIONS.has(ext) || !parsed.name.startsWith(prefix)) {
return null;
}
const suffix = parsed.name.slice(prefix.length);
const languageRank = resolveLanguageRank(suffix, preferredLanguages);
if (!Number.isFinite(languageRank)) {
return null;
}
return {
path: path.join(source.dir, entry),
languageRank,
extensionRank: extensionRank(ext),
name: entry,
};
})
.filter((candidate): candidate is SidecarCandidate => candidate !== null)
.sort((left, right) => {
if (left.languageRank !== right.languageRank) return left.languageRank - right.languageRank;
if (left.extensionRank !== right.extensionRank)
return left.extensionRank - right.extensionRank;
return left.name.localeCompare(right.name);
});
}
function combineCueText(cues: SubtitleCue[]): string {
return unique(cues.map((cue) => cue.text.trim()).filter(Boolean))
.join('\n')
.trim();
}
function overlapSeconds(cue: SubtitleCue, startSeconds: number, endSeconds: number): number {
return (
Math.min(cue.endTime, endSeconds + TIMING_TOLERANCE_SECONDS) -
Math.max(cue.startTime, startSeconds - TIMING_TOLERANCE_SECONDS)
);
}
function isSameCueTiming(left: SubtitleCue, right: SubtitleCue): boolean {
return (
Math.abs(left.startTime - right.startTime) <= SAME_TIMING_EPSILON_SECONDS &&
Math.abs(left.endTime - right.endTime) <= SAME_TIMING_EPSILON_SECONDS
);
}
function compareCueTimingMatch(
startSeconds: number,
endSeconds: number,
left: { cue: SubtitleCue; overlap: number },
right: { cue: SubtitleCue; overlap: number },
): number {
if (left.overlap !== right.overlap) {
return right.overlap - left.overlap;
}
const leftStartDistance = Math.abs(left.cue.startTime - startSeconds);
const rightStartDistance = Math.abs(right.cue.startTime - startSeconds);
if (leftStartDistance !== rightStartDistance) {
return leftStartDistance - rightStartDistance;
}
const leftEndDistance = Math.abs(left.cue.endTime - endSeconds);
const rightEndDistance = Math.abs(right.cue.endTime - endSeconds);
if (leftEndDistance !== rightEndDistance) {
return leftEndDistance - rightEndDistance;
}
return left.cue.startTime - right.cue.startTime;
}
function findCueTextAtTiming(cues: SubtitleCue[], startMs: number, endMs: number): string {
const startSeconds = startMs / 1000;
const endSeconds = endMs / 1000;
const midpointSeconds = (startSeconds + endSeconds) / 2;
const midpointMatches = cues
.filter(
(cue) =>
cue.startTime - TIMING_TOLERANCE_SECONDS <= midpointSeconds &&
cue.endTime + TIMING_TOLERANCE_SECONDS >= midpointSeconds,
)
.map((cue) => ({ cue, overlap: overlapSeconds(cue, startSeconds, endSeconds) }))
.sort((left, right) => compareCueTimingMatch(startSeconds, endSeconds, left, right));
const [bestMidpointMatch] = midpointMatches;
const midpointText = bestMidpointMatch
? combineCueText(
midpointMatches
.filter((match) => isSameCueTiming(match.cue, bestMidpointMatch.cue))
.map((match) => match.cue),
)
: '';
if (midpointText) {
return midpointText;
}
const [bestOverlap] = cues
.map((cue) => ({ cue, overlap: overlapSeconds(cue, startSeconds, endSeconds) }))
.filter((entry) => entry.overlap > 0)
.sort((left, right) => compareCueTimingMatch(startSeconds, endSeconds, left, right));
return bestOverlap ? bestOverlap.cue.text.trim() : '';
}
export function resolveSecondarySubtitleTextFromSidecar(input: {
sourcePath: string;
startMs: number;
endMs: number;
languages?: readonly string[];
}): string {
if (!input.sourcePath || !existsSync(input.sourcePath)) {
return '';
}
try {
if (!statSync(input.sourcePath).isFile()) {
return '';
}
} catch {
return '';
}
const preferredLanguages = expandPreferredLanguages(input.languages);
const candidates = findSidecarSubtitleCandidates(input.sourcePath, preferredLanguages);
for (const candidate of candidates) {
try {
const content = readFileSync(candidate.path, 'utf8');
const cues = parseSubtitleCues(content, candidate.path);
const text = findCueTextAtTiming(cues, input.startMs, input.endMs);
if (text) {
return text;
}
} catch {
// Try the next matching sidecar.
}
}
return '';
}
+120 -6
View File
@@ -1,5 +1,6 @@
import { Hono } from 'hono';
import type { ImmersionTrackerService } from './immersion-tracker-service.js';
import { splitSentenceSearchTerms } from './immersion-tracker/query-lexical.js';
import http, { type IncomingMessage, type ServerResponse } from 'node:http';
import { basename, extname, resolve, sep } from 'node:path';
import { readFileSync, existsSync, statSync } from 'node:fs';
@@ -16,6 +17,7 @@ import {
} from '../../anki-field-config.js';
import { resolveAnimatedImageLeadInSeconds } from '../../anki-integration/animated-image-sync.js';
import type { AnilistRateLimiter } from './anilist/rate-limiter.js';
import { resolveSecondarySubtitleTextFromSidecar } from './secondary-subtitle-sidecar.js';
type StatsServerNoteInfo = {
noteId: number;
@@ -356,9 +358,13 @@ export interface StatsServerConfig {
mpvSocketPath?: string;
ankiConnectConfig?: AnkiConnectConfig;
getAnkiConnectConfig?: () => AnkiConnectConfig | undefined;
getYomitanAnkiDeckName?: () => Promise<string | null | undefined> | string | null | undefined;
secondarySubtitleLanguages?: string[];
getSecondarySubtitleLanguages?: () => string[] | undefined;
anilistRateLimiter?: AnilistRateLimiter;
addYomitanNote?: (word: string) => Promise<number | null>;
resolveAnkiNoteId?: (noteId: number) => number;
resolveSentenceSearchHeadwords?: (term: string) => Promise<string[]> | string[];
}
const STATS_STATIC_CONTENT_TYPES: Record<string, string> = {
@@ -385,6 +391,47 @@ function defaultNowMs(): number {
return Date.now();
}
function parseBooleanQuery(raw: string | undefined, fallback: boolean): boolean {
if (raw === undefined) return fallback;
const normalized = raw.trim().toLowerCase();
if (!normalized) return fallback;
return !['0', 'false', 'no', 'off'].includes(normalized);
}
function uniqueNonEmptyStrings(values: readonly string[]): string[] {
const seen = new Set<string>();
const result: string[] = [];
for (const value of values) {
const normalized = value.trim();
if (!normalized || seen.has(normalized)) continue;
seen.add(normalized);
result.push(normalized);
}
return result;
}
async function buildSentenceSearchOptions(
query: string,
searchByHeadword: boolean,
resolveSentenceSearchHeadwords: ((term: string) => Promise<string[]> | string[]) | undefined,
): Promise<{ headwordTerms: Array<{ term: string; headwords: string[] }> } | undefined> {
if (!searchByHeadword) return undefined;
const terms = splitSentenceSearchTerms(query);
const headwordTerms: Array<{ term: string; headwords: string[] }> = [];
for (const term of terms) {
const resolved = resolveSentenceSearchHeadwords
? await resolveSentenceSearchHeadwords(term)
: [term];
const headwords = uniqueNonEmptyStrings(resolved);
if (headwords.length > 0) {
headwordTerms.push({ term, headwords });
}
}
return headwordTerms.length > 0 ? { headwordTerms } : undefined;
}
function buildAnkiNotePreview(
fields: Record<string, { value: string }>,
ankiConfig?: Pick<AnkiConnectConfig, 'fields'>,
@@ -446,9 +493,13 @@ export function createStatsApp(
mpvSocketPath?: string;
ankiConnectConfig?: AnkiConnectConfig;
getAnkiConnectConfig?: () => AnkiConnectConfig | undefined;
getYomitanAnkiDeckName?: () => Promise<string | null | undefined> | string | null | undefined;
secondarySubtitleLanguages?: string[];
getSecondarySubtitleLanguages?: () => string[] | undefined;
anilistRateLimiter?: AnilistRateLimiter;
addYomitanNote?: (word: string) => Promise<number | null>;
resolveAnkiNoteId?: (noteId: number) => number;
resolveSentenceSearchHeadwords?: (term: string) => Promise<string[]> | string[];
createMediaGenerator?: () => StatsServerMediaGenerator;
onMiningTiming?: (event: StatsMiningTimingEvent) => void;
nowMs?: () => number;
@@ -458,6 +509,23 @@ export function createStatsApp(
const nowMs = options?.nowMs ?? defaultNowMs;
const getAnkiConnectConfig = (): AnkiConnectConfig | undefined =>
options?.getAnkiConnectConfig?.() ?? options?.ankiConnectConfig;
const getSecondarySubtitleLanguages = (): string[] =>
options?.getSecondarySubtitleLanguages?.() ?? options?.secondarySubtitleLanguages ?? [];
const getEffectiveMiningDeckName = async (ankiConfig: AnkiConnectConfig): Promise<string> => {
const configuredDeckName = ankiConfig.deck?.trim() ?? '';
if (configuredDeckName) return configuredDeckName;
try {
const yomitanDeckName = await options?.getYomitanAnkiDeckName?.();
return typeof yomitanDeckName === 'string' ? yomitanDeckName.trim() : '';
} catch (error) {
statsMiningLogger.warn(
'Failed to resolve Yomitan Anki deck for stats mining:',
error instanceof Error ? error.message : String(error),
);
return '';
}
};
const recordMiningTiming = (event: StatsMiningTimingEvent): void => {
options?.onMiningTiming?.(event);
@@ -659,7 +727,13 @@ export function createStatsApp(
const query = (c.req.query('q') ?? '').trim();
if (!query) return c.json([]);
const limit = parseIntQuery(c.req.query('limit'), 50, 100);
const rows = await tracker.searchSubtitleSentences(query, limit);
const searchByHeadword = parseBooleanQuery(c.req.query('headword'), true);
const searchOptions = await buildSentenceSearchOptions(
query,
searchByHeadword,
options?.resolveSentenceSearchHeadwords,
);
const rows = await tracker.searchSubtitleSentences(query, limit, searchOptions);
return c.json(rows);
});
@@ -997,7 +1071,8 @@ export function createStatsApp(
const endMs = typeof body?.endMs === 'number' ? body.endMs : NaN;
const sentence = typeof body?.sentence === 'string' ? body.sentence.trim() : '';
const word = typeof body?.word === 'string' ? body.word.trim() : '';
const secondaryText = typeof body?.secondaryText === 'string' ? body.secondaryText.trim() : '';
const bodySecondaryText =
typeof body?.secondaryText === 'string' ? body.secondaryText.trim() : '';
const videoTitle = typeof body?.videoTitle === 'string' ? body.videoTitle.trim() : '';
const rawMode = c.req.query('mode');
const mode = rawMode === 'audio' ? 'audio' : rawMode === 'word' ? 'word' : 'sentence';
@@ -1014,6 +1089,14 @@ export function createStatsApp(
if (!ankiConfig) {
return c.json({ error: 'AnkiConnect is not configured' }, 500);
}
const secondaryText =
bodySecondaryText ||
resolveSecondarySubtitleTextFromSidecar({
sourcePath,
startMs,
endMs,
languages: getSecondarySubtitleLanguages(),
});
const client = new AnkiConnectClient(ankiConfig.url ?? 'http://127.0.0.1:8765');
const mediaGen = options?.createMediaGenerator?.() ?? new MediaGenerator();
@@ -1080,6 +1163,25 @@ export function createStatsApp(
const errors: string[] = [];
let noteId: number;
let effectiveDeckNamePromise: Promise<string> | null = null;
const getEffectiveDeckNameForRequest = (): Promise<string> => {
effectiveDeckNamePromise ??= getEffectiveMiningDeckName(ankiConfig);
return effectiveDeckNamePromise;
};
const moveNoteToConfiguredDeck = async (id: number): Promise<void> => {
const deckName = await getEffectiveDeckNameForRequest();
if (!deckName) {
return;
}
try {
const cardIds = await timeMiningPhase(mode, 'findCards', () =>
client.findCards(`nid:${id}`),
);
await timeMiningPhase(mode, 'changeDeck', () => client.changeDeck(cardIds, deckName));
} catch (err) {
errors.push(`deck: ${(err as Error).message}`);
}
};
if (mode === 'word') {
if (!options?.addYomitanNote) {
@@ -1107,6 +1209,7 @@ export function createStatsApp(
}
noteId = yomitanResult.value;
await moveNoteToConfiguredDeck(noteId);
const audioBuffer = audioResult.status === 'fulfilled' ? audioResult.value : null;
if (audioResult.status === 'rejected')
errors.push(`audio: ${(audioResult.reason as Error).message}`);
@@ -1145,10 +1248,14 @@ export function createStatsApp(
const mediaFields: Record<string, string> = {};
const timestamp = Date.now();
const sentenceFieldName = ankiConfig.fields?.sentence ?? 'Sentence';
const translationFieldName = ankiConfig.fields?.translation ?? 'SelectionText';
const audioFieldName = getStatsWordMiningAudioFieldName(ankiConfig, noteInfo);
const imageFieldName = ankiConfig.fields?.image ?? 'Picture';
mediaFields[sentenceFieldName] = highlightedSentence;
if (secondaryText) {
mediaFields[translationFieldName] = secondaryText;
}
if (audioBuffer) {
const audioFilename = `subminer_audio_${timestamp}.mp3`;
@@ -1214,7 +1321,7 @@ export function createStatsApp(
const miscInfoFieldName = ankiConfig.fields?.miscInfo ?? '';
const fields: Record<string, string> = {
[sentenceFieldName]: highlightedSentence,
[sentenceFieldName]: mode === 'sentence' ? sentence : highlightedSentence,
};
if (mode === 'sentence' && secondaryText) {
@@ -1222,7 +1329,9 @@ export function createStatsApp(
}
if (ankiConfig.isLapis?.enabled || ankiConfig.isKiku?.enabled) {
if (word) {
if (mode === 'sentence') {
fields[wordFieldName] = sentence;
} else if (word) {
fields[wordFieldName] = word;
}
if (mode === 'sentence') {
@@ -1233,13 +1342,13 @@ export function createStatsApp(
}
const model = ankiConfig.isLapis?.sentenceCardModel || 'Basic';
const deck = ankiConfig.deck?.trim() || 'Default';
const tags = ankiConfig.tags ?? ['SubMiner'];
const addNotePromise = timeMiningPhase(
mode,
'addNote',
() => client.addNote(deck, model, fields, tags),
async () =>
client.addNote((await getEffectiveDeckNameForRequest()) || 'Default', model, fields, tags),
(id) => ({
noteId: id,
}),
@@ -1265,6 +1374,7 @@ export function createStatsApp(
);
}
noteId = addNoteResult.value;
await moveNoteToConfiguredDeck(noteId);
const mediaFields: Record<string, string> = {};
const timestamp = Date.now();
@@ -1370,9 +1480,13 @@ export function startStatsServer(config: StatsServerConfig): { close: () => void
mpvSocketPath: config.mpvSocketPath,
ankiConnectConfig: config.ankiConnectConfig,
getAnkiConnectConfig: config.getAnkiConnectConfig,
getYomitanAnkiDeckName: config.getYomitanAnkiDeckName,
secondarySubtitleLanguages: config.secondarySubtitleLanguages,
getSecondarySubtitleLanguages: config.getSecondarySubtitleLanguages,
anilistRateLimiter: config.anilistRateLimiter,
addYomitanNote: config.addYomitanNote,
resolveAnkiNoteId: config.resolveAnkiNoteId,
resolveSentenceSearchHeadwords: config.resolveSentenceSearchHeadwords,
});
const bunRuntime = globalThis as typeof globalThis & {
@@ -151,6 +151,56 @@ test('syncYomitanDefaultAnkiServer injects force override when enabled', async (
assert.match(scriptValue, /forceOverride = true/);
});
test('syncYomitanDefaultAnkiServer updates the active profile Anki deck', async () => {
const optionsFull = {
profileCurrent: 0,
profiles: [
{
options: {
anki: {
server: 'http://127.0.0.1:8766',
cardFormats: [
{ type: 'term', deck: 'Default', model: 'Mining Note', fields: {} },
{ type: 'kanji', deck: 'Kanji', model: 'Kanji Note', fields: {} },
],
terms: { deck: 'Default', model: 'Legacy Note', fields: {} },
},
},
},
],
};
let savedOptions: typeof optionsFull | null = null;
const deps = createDeps((script) =>
runInjectedYomitanScript(script, (action, params) => {
if (action === 'optionsGetFull') {
return JSON.parse(JSON.stringify(optionsFull));
}
if (action === 'setAllSettings') {
savedOptions = (params as { value: typeof optionsFull }).value;
return true;
}
throw new Error(`Unexpected action: ${action}`);
}),
);
const synced = await syncYomitanDefaultAnkiServer(
'http://127.0.0.1:8766',
deps,
{
error: () => undefined,
info: () => undefined,
},
{ deck: 'Minecraft', forceOverride: true },
);
assert.equal(synced, true);
assert.ok(savedOptions);
const saved = savedOptions as typeof optionsFull;
assert.equal(saved.profiles[0]?.options.anki.cardFormats[0]?.deck, 'Minecraft');
assert.equal(saved.profiles[0]?.options.anki.cardFormats[1]?.deck, 'Kanji');
assert.equal(saved.profiles[0]?.options.anki.terms.deck, 'Minecraft');
});
test('syncYomitanDefaultAnkiServer logs and returns false on script failure', async () => {
const deps = createDeps(async () => {
throw new Error('execute failed');
@@ -1783,6 +1783,7 @@ export async function syncYomitanDefaultAnkiServer(
logger: LoggerLike,
options?: {
forceOverride?: boolean;
deck?: string;
},
): Promise<boolean> {
const normalizedTargetServer = serverUrl.trim();
@@ -1790,6 +1791,7 @@ export async function syncYomitanDefaultAnkiServer(
return false;
}
const forceOverride = options?.forceOverride === true;
const normalizedTargetDeck = options?.deck?.trim() ?? '';
const isReady = await ensureYomitanParserWindow(deps, logger);
const parserWindow = deps.getYomitanParserWindow();
@@ -1819,6 +1821,7 @@ export async function syncYomitanDefaultAnkiServer(
});
const targetServer = ${JSON.stringify(normalizedTargetServer)};
const targetDeck = ${JSON.stringify(normalizedTargetDeck)};
const forceOverride = ${forceOverride ? 'true' : 'false'};
const optionsFull = await invoke("optionsGetFull", undefined);
const profiles = Array.isArray(optionsFull.profiles) ? optionsFull.profiles : [];
@@ -1843,18 +1846,54 @@ export async function syncYomitanDefaultAnkiServer(
const currentServerRaw = targetProfile.options.anki.server;
const currentServer = typeof currentServerRaw === "string" ? currentServerRaw.trim() : "";
if (currentServer === targetServer) {
return { updated: false, matched: true, reason: "already-target", currentServer, targetServer };
}
const canReplaceCurrent =
forceOverride || currentServer.length === 0 || currentServer === "http://127.0.0.1:8765";
if (!canReplaceCurrent) {
return { updated: false, matched: false, reason: "blocked-existing-server", currentServer, targetServer };
let changed = false;
if (currentServer !== targetServer) {
const canReplaceCurrent =
forceOverride || currentServer.length === 0 || currentServer === "http://127.0.0.1:8765";
if (!canReplaceCurrent) {
return { updated: false, matched: false, reason: "blocked-existing-server", currentServer, targetServer };
}
targetProfile.options.anki.server = targetServer;
changed = true;
}
if (targetDeck) {
const cardFormats = Array.isArray(targetProfile.options.anki.cardFormats)
? targetProfile.options.anki.cardFormats
: [];
for (const cardFormat of cardFormats) {
if (
!cardFormat ||
typeof cardFormat !== "object" ||
cardFormat.type !== "term" ||
cardFormat.enabled === false
) {
continue;
}
const currentDeck = typeof cardFormat.deck === "string" ? cardFormat.deck.trim() : "";
if (currentDeck !== targetDeck) {
cardFormat.deck = targetDeck;
changed = true;
}
}
const terms = targetProfile.options.anki.terms;
if (terms && typeof terms === "object") {
const currentTermDeck = typeof terms.deck === "string" ? terms.deck.trim() : "";
if (currentTermDeck !== targetDeck) {
terms.deck = targetDeck;
changed = true;
}
}
}
if (!changed) {
return { updated: false, matched: true, reason: "already-target", currentServer, targetServer, targetDeck };
}
targetProfile.options.anki.server = targetServer;
await invoke("setAllSettings", { value: optionsFull, source: "subminer" });
return { updated: true, matched: true, currentServer, targetServer };
return { updated: true, matched: true, currentServer, targetServer, targetDeck };
})();
`;