sudacode/SubMiner
1
0 Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-11-10 04:46:07 -08:00
Code Issues Packages Projects Releases Wiki Activity

feat(stats): overhaul the Trends tab (#325)

Browse Source
This commit is contained in:
sudacode authored and GitHub committed 2026-11-04 21:55:18 -08:00
1 parent f89f78b861
commit 269c0708cb
73 files changed
+6085 -3354
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/core/services/immersion-tracker/__tests__/query.test.ts
+9 -15
View File
@@ -704,7 +704,7 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => {
total_tokens_seen, total_cards
) VALUES (?, ?, ?, ?, ?, ?, ?)
`,
).run(Math.floor(dayOneStart / 86_400_000), videoId, 1, 30, 10, 120, 2);
).run(getLocalEpochDay(db, dayOneStart), videoId, 1, 30, 10, 120, 2);
db.prepare(
`
@@ -713,7 +713,7 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => {
total_tokens_seen, total_cards
) VALUES (?, ?, ?, ?, ?, ?, ?)
`,
).run(Math.floor(dayTwoStart / 86_400_000), videoId, 1, 45, 10, 140, 3);
).run(getLocalEpochDay(db, dayTwoStart), videoId, 1, 45, 10, 140, 3);
db.prepare(
`
@@ -739,18 +739,17 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => {
assert.equal(dashboard.activity.watchTime[0]?.value, 30);
assert.equal(dashboard.progress.watchTime[1]?.value, 75);
assert.equal(dashboard.progress.lookups[1]?.value, 18);
assert.equal(dashboard.ratios.lookupsPerHundred[0]?.value, +((8 / 120) * 100).toFixed(1));
assert.equal(dashboard.ratios.cardsPerHour[0]?.value, +(2 / (30 / 60)).toFixed(1));
assert.equal(dashboard.ratios.cardsPerHour[1]?.value, +(3 / (45 / 60)).toFixed(1));
assert.equal(dashboard.ratios.cardsPerHour[1]?.value, +(5 / (75 / 60)).toFixed(1));
assert.equal(dashboard.ratios.readingSpeed[0]?.value, +(120 / 30).toFixed(1));
assert.equal(dashboard.ratios.readingSpeed[1]?.value, +(140 / 45).toFixed(1));
assert.equal(dashboard.ratios.readingSpeed[1]?.value, +(260 / 75).toFixed(1));
assert.equal(dashboard.librarySummary[0]?.title, 'Trend Dashboard Anime');
assert.deepEqual(
dashboard.animeDaily.watchTime.map((point) => point.value).sort((a, b) => a - b),
dashboard.titleTimeline.map((point) => point.value).sort((a, b) => a - b),
[30, 45],
);
assert.equal(
dashboard.patterns.watchTimeByDayOfWeek.reduce((sum, point) => sum + point.value, 0),
dashboard.watchTimeByDayHour.flat().reduce((sum, minutes) => sum + minutes, 0),
75,
);
} finally {
@@ -822,7 +821,7 @@ test('getTrendsDashboard redacts legacy Jellyfin stream titles', () => {
const dashboard = getTrendsDashboard(db, 'all', 'day');
const titles = [
...dashboard.animeDaily.watchTime.map((point) => point.animeTitle),
...dashboard.titleTimeline.map((point) => point.animeTitle),
...dashboard.librarySummary.map((row) => row.title),
];
@@ -4751,7 +4750,7 @@ test('deleteSession removes zero-session media from library and trends', () => {
const trends = getTrendsDashboard(db, 'all', 'day');
assert.deepEqual(trends.activity.watchTime, []);
assert.deepEqual(trends.activity.sessions, []);
assert.deepEqual(trends.activity.cards, []);
const dailyRollups = getDailyRollups(db, 30);
const monthlyRollups = getMonthlyRollups(db, 30);
@@ -5073,7 +5072,7 @@ test('getTrendsDashboard rollup word metrics keep persisted totals over partial
total_tokens_seen, total_cards
) VALUES (?, ?, ?, ?, ?, ?, ?)
`,
).run(Math.floor(row.start / 86_400_000), videoId, 1, 10, 1, row.rawWords, 0);
).run(getLocalEpochDay(db, row.start), videoId, 1, 10, 1, row.rawWords, 0);
}
const dashboard = getTrendsDashboard(db, 'all', 'day');
@@ -5085,13 +5084,8 @@ test('getTrendsDashboard rollup word metrics keep persisted totals over partial
dashboard.progress.words.map((point) => point.value),
[10, 30],
);
assert.equal(dashboard.ratios.lookupsPerHundred[0]?.value, 200);
assert.equal(dashboard.librarySummary[0]?.words, 30);
assert.equal(dashboard.librarySummary[0]?.lookupsPerHundred, 33.3);
assert.equal(
dashboard.animeDaily.words.reduce((sum, point) => sum + point.value, 0),
30,
);
} finally {
db.close();
cleanupDbPath(dbPath);
src/core/services/immersion-tracker/lexical-rollups.test.ts
+2 -62
View File
@@ -8,7 +8,6 @@ import {
getLexicalDailyRollups,
rebuildLexicalDailyRollups,
} from './lexical-rollups';
import { getTrendsDashboard } from './query-trends';
import {
getVocabularyChartData,
getVocabularySummary,
@@ -262,7 +261,7 @@ test('lexical rollup rebuild tolerates nullable legacy vocabulary text', () => {
assert.equal(areLexicalDailyRollupsReady(db), true);
assert.equal(getVocabularySummary(db, null).uniqueWords, 1);
assert.equal(getVocabularySummary(db, new Set(['猫'])).knownWordCount, 1);
assert.equal(getVocabularyChartData(db).topWords[0]?.headword, '猫');
assert.equal(getVocabularyChartData(db).newWordsTimeline[0]?.wordCount, 1);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
@@ -593,7 +592,7 @@ test('imm_words persists vocabulary visibility for rollup maintenance', () => {
}
});
test('vocabulary charts use complete top-word and lexical rollup data', () => {
test('vocabulary charts count every word in new-word history', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -611,8 +610,6 @@ test('vocabulary charts use complete top-word and lexical rollup data', () => {
const charts = getVocabularyChartData(db);
assert.equal(charts.topWords[0]?.headword, '語500');
assert.equal(charts.topWords[0]?.frequency, 10_000);
assert.equal(charts.newWordsTimeline[0]?.wordCount, 501);
} finally {
db.close();
@@ -620,41 +617,6 @@ test('vocabulary charts use complete top-word and lexical rollup data', () => {
}
});
test('vocabulary charts find full top-word sets beyond excluded and name rows', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const insertWord = db.prepare(
`INSERT INTO imm_words(headword, word, reading, pos2, first_seen, last_seen, frequency)
VALUES (?, ?, '', ?, 1700000000, 1700000000, ?)`,
);
const exclusions = [];
for (let index = 0; index < 100; index += 1) {
const headword = `語${index}`;
insertWord.run(
headword,
headword,
index < 80 && index >= 60 ? '固有名詞' : '一般',
100 - index,
);
if (index < 60) exclusions.push({ headword, word: headword, reading: '' });
}
replaceStatsExcludedWords(db, exclusions);
const charts = getVocabularyChartData(db);
assert.equal(charts.topWords.length, 12);
assert.equal(charts.topWords[0]?.headword, '語60');
assert.equal(charts.topWordsWithoutNames.length, 12);
assert.equal(charts.topWordsWithoutNames[0]?.headword, '語80');
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('vocabulary charts handle exclusion lists above one SQLite variable batch', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -674,7 +636,6 @@ test('vocabulary charts handle exclusion lists above one SQLite variable batch',
const charts = getVocabularyChartData(db);
assert.deepEqual(charts.topWords, []);
assert.deepEqual(charts.newWordsTimeline, []);
} finally {
db.close();
@@ -697,24 +658,3 @@ test('lexical rollup rebuild preserves the original error when rollback also fai
assert.throws(() => rebuildLexicalDailyRollups(db), originalError);
});
test('trends read historical new-word buckets from lexical rollups', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES ('海', '海', 'うみ', 1700000000, 1700000000, 1)`,
).run();
db.prepare(`UPDATE imm_lexical_daily_rollups SET word_count = 9`).run();
const dashboard = getTrendsDashboard(db, 'all', 'day', false);
assert.equal(dashboard.progress.newWords[0]?.value, 9);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
src/core/services/immersion-tracker/query-lexical.ts
+321 -135
View File
@@ -1,3 +1,4 @@
import type { TrendFrequencyBand } from '../../../types/stats-wire';
import type { DatabaseSync } from './sqlite';
import type {
KanjiAnimeAppearanceRow,
@@ -19,17 +20,44 @@ import type {
import { forEachIdChunk, fromDbTimestamp, makePlaceholders, toDbTimestamp } from './query-shared';
import { nowMs } from './time';
import { narrowFullwidthKana, widenHalfwidthKana } from '../../utils/halfwidth-kana';
import {
areLexicalDailyRollupsReady,
getLexicalDailyRollups,
localEpochDaySql,
} from './lexical-rollups';
import { areLexicalDailyRollupsReady, localEpochDaySql } from './lexical-rollups';
import { isVocabularyStatsRowVisible } from './vocabulary-visibility';
// Merged tokens join MeCab tags, so a name can be stored as 固有名詞|一般 or 固有名詞|接尾.
const PROPER_NOUN_TAG = '固有名詞';
const NOT_A_NAME_SQL = `COALESCE(pos2, '') NOT LIKE '%${PROPER_NOUN_TAG}%'`;
/** The dictionary word an `imm_words` row belongs to. Legacy rows can lack a headword or form. */
export const DICTIONARY_WORD_KEY_SQL = `COALESCE(NULLIF(TRIM(headword), ''), NULLIF(TRIM(word), ''), TRIM(reading))`;
/** True for lines from a stream: only local videos have a path to cut clips from. */
export const NO_LOCAL_FILE_SQL = `TRIM(COALESCE(v.source_path, '')) = ''`;
// Sorts lines that can be mined ahead of those that can't, because newest-first alone
// lets a run of recent streams bury every line with a usable clip.
// Mirrors getStatsMineCardUnavailableReason in the stats client.
const MINEABLE_LINES_FIRST_SQL = `
CASE
WHEN NOT (${NO_LOCAL_FILE_SQL})
AND l.segment_start_ms IS NOT NULL
AND l.segment_end_ms > l.segment_start_ms
THEN 0
ELSE 1
END`;
export function isProperNounPos2(pos2: string | null | undefined): boolean {
return pos2?.includes(PROPER_NOUN_TAG) ?? false;
}
export interface NewWordFirstSeenRow {
firstSeenDay: number;
firstSeenDayWithoutNames: number | null;
band: TrendFrequencyBand;
wordCount: number;
}
const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
const VOCABULARY_CHART_LIMIT = 12;
const VOCABULARY_CHART_PAGE_SIZE = 100;
const EXCLUSION_ALIAS_BATCH_SIZE = 300;
const VOCABULARY_SUMMARY_SCAN_BATCH_SIZE = 5_000;
const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
@@ -38,8 +66,6 @@ const KANJI_PATTERN = /\p{Script=Han}/gu;
export interface VocabularyChartData {
ready: boolean;
topWords: Array<{ wordId: number; headword: string; frequency: number }>;
topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>;
newWordsTimeline: Array<{ epochDay: number; wordCount: number }>;
newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>;
}
@@ -161,111 +187,137 @@ function getWordAnimeCounts(db: DatabaseSync, wordIds: number[]): Map<number, nu
}
/**
* Chart data is intentionally independent of the paginated vocabulary tables.
* Top words use the frequency index; new-word history reads permanent daily
* lexical rollups rather than loading every vocabulary row into the dashboard.
* New-word history is a SQL aggregate over first-seen dates rather than loading
* every vocabulary row into the dashboard.
*/
export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData {
const ready = areLexicalDailyRollupsReady(db);
const excludedAliases = new Set(
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
);
const isExcluded = (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>): boolean =>
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias));
const topWords = getTopVocabularyChartWords(db, isExcluded);
const rollups = ready ? getLexicalDailyRollups(db) : [];
const timeline = new Map(rollups.map((row) => [row.epochDay, { ...row }]));
if (excludedAliases.size > 0 && ready) {
const aliases = [...excludedAliases];
const excludedRows = new Map<
number,
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
wordId: number;
epochDay: number;
}
>();
for (let offset = 0; offset < aliases.length; offset += EXCLUSION_ALIAS_BATCH_SIZE) {
const batch = aliases.slice(offset, offset + EXCLUSION_ALIAS_BATCH_SIZE);
const placeholders = batch.map(() => '?').join(', ');
const rows = db
.prepare(
`
SELECT id AS wordId, headword, word, reading, pos2,
${localEpochDaySql('first_seen')} AS epochDay
FROM imm_words
WHERE vocabulary_visible = 1
AND (headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}))
`,
)
.all(...batch, ...batch, ...batch) as Array<
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
wordId: number;
epochDay: number;
}
>;
for (const row of rows) excludedRows.set(row.wordId, row);
}
for (const word of excludedRows.values()) {
if (!isExcluded(word)) continue;
const rollup = timeline.get(word.epochDay);
if (!rollup) continue;
rollup.wordCount -= 1;
if (word.pos2 !== '固有名詞') rollup.wordCountWithoutNames -= 1;
const firstSeenRows = ready ? getNewWordFirstSeenRows(db) : [];
const timeline = new Map<number, number>();
const timelineWithoutNames = new Map<number, number>();
for (const row of firstSeenRows) {
timeline.set(row.firstSeenDay, (timeline.get(row.firstSeenDay) ?? 0) + row.wordCount);
if (row.firstSeenDayWithoutNames !== null) {
timelineWithoutNames.set(
row.firstSeenDayWithoutNames,
(timelineWithoutNames.get(row.firstSeenDayWithoutNames) ?? 0) + row.wordCount,
);
}
}
return {
ready,
topWords: topWords.all.map((word) => ({
wordId: word.wordId,
headword: vocabularyDisplayHeadword(word),
frequency: word.frequency,
})),
topWordsWithoutNames: topWords.withoutNames.map((word) => ({
wordId: word.wordId,
headword: vocabularyDisplayHeadword(word),
frequency: word.frequency,
})),
newWordsTimeline: [...timeline.values()]
.filter((row) => row.wordCount > 0)
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCount })),
newWordsTimelineWithoutNames: [...timeline.values()]
.filter((row) => row.wordCountWithoutNames > 0)
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCountWithoutNames })),
newWordsTimeline: toNewWordTimeline(timeline),
newWordsTimelineWithoutNames: toNewWordTimeline(timelineWithoutNames),
};
}
function getTopVocabularyChartWords(
db: DatabaseSync,
isExcluded: (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>) => boolean,
): { all: VocabularyStatsRow[]; withoutNames: VocabularyStatsRow[] } {
const stmt = db.prepare(`
SELECT id AS wordId, headword, word, reading,
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
frequency, frequency_rank AS frequencyRank,
first_seen AS firstSeen, last_seen AS lastSeen,
0 AS animeCount
FROM imm_words
ORDER BY frequency DESC, id
LIMIT ? OFFSET ?
`);
const all: VocabularyStatsRow[] = [];
const withoutNames: VocabularyStatsRow[] = [];
let offset = 0;
function toNewWordTimeline(byDay: Map<number, number>): VocabularyChartData['newWordsTimeline'] {
return [...byDay.entries()]
.sort(([left], [right]) => left - right)
.map(([epochDay, wordCount]) => ({ epochDay, wordCount }));
}
while (all.length < VOCABULARY_CHART_LIMIT || withoutNames.length < VOCABULARY_CHART_LIMIT) {
const page = stmt.all(VOCABULARY_CHART_PAGE_SIZE, offset) as VocabularyStatsRow[];
if (page.length === 0) break;
for (const word of page) {
if (!isVocabularyStatsRowVisible(word) || isExcluded(word)) continue;
if (all.length < VOCABULARY_CHART_LIMIT) all.push(word);
if (word.pos2 !== '固有名詞' && withoutNames.length < VOCABULARY_CHART_LIMIT) {
withoutNames.push(word);
/** Word rows matching the user's stats exclusions, using the same aliases as the summary. */
export function getExcludedWordIds(db: DatabaseSync): number[] {
const excludedAliases = new Set(
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
);
const aliases = [...excludedAliases];
const ids = new Set<number>();
for (let offset = 0; offset < aliases.length; offset += EXCLUSION_ALIAS_BATCH_SIZE) {
const batch = aliases.slice(offset, offset + EXCLUSION_ALIAS_BATCH_SIZE);
const placeholders = batch.map(() => '?').join(', ');
const rows = db
.prepare(
`
SELECT id AS wordId, headword, word, reading
FROM imm_words
WHERE vocabulary_visible = 1
AND (headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}))
`,
)
.all(...batch, ...batch, ...batch) as Array<
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'> & { wordId: number }
>;
for (const row of rows) {
if (excludedVocabularyAliases(row).some((alias) => excludedAliases.has(alias))) {
ids.add(row.wordId);
}
}
offset += page.length;
}
return [...ids];
}
return { all, withoutNames };
// The aggregate below scans every word (about 200ms on 68k rows), and the Trends tab
// runs it on each control change. Results stay valid until something writes:
// total_changes covers this connection, data_version covers other connections.
const newWordFirstSeenCache = new WeakMap<
DatabaseSync,
{ version: string; rows: NewWordFirstSeenRow[] }
>();
export function databaseWriteVersion(db: DatabaseSync): string {
const { changes } = db.prepare('SELECT total_changes() AS changes').get() as { changes: number };
const { data_version: dataVersion } = db.prepare('PRAGMA data_version').get() as {
data_version: number;
};
return `${changes}:${dataVersion}`;
}
/**
* When each dictionary word was first seen, folded across its surface forms:
* `imm_words` stores 言う and 言った as separate rows, so counting rows would count
* conjugations as new words. Rows group identical (day, day without names, band)
* results so the caller gets a small aggregate rather than one row per word.
* `firstSeenDayWithoutNames` and `band` ignore proper-noun forms, and the day is
* null when every form of the word is a proper noun.
*/
export function getNewWordFirstSeenRows(db: DatabaseSync): NewWordFirstSeenRow[] {
const version = databaseWriteVersion(db);
const cached = newWordFirstSeenCache.get(db);
if (cached?.version === version) return cached.rows;
const excludedIds = getExcludedWordIds(db);
const exclusion =
excludedIds.length > 0 ? `AND id NOT IN (${makePlaceholders(excludedIds)})` : '';
const seconds = `CASE
WHEN ABS(CAST(first_seen AS REAL)) >= 10000000000 THEN CAST(first_seen AS REAL) / 1000
ELSE CAST(first_seen AS REAL)
END`;
const rows = db
.prepare(
`
WITH dictionary_words AS (
SELECT
MIN(${seconds}) AS firstSeen,
MIN(CASE WHEN ${NOT_A_NAME_SQL} THEN ${seconds} END) AS firstSeenWithoutNames,
MIN(CASE WHEN ${NOT_A_NAME_SQL} THEN frequency_rank END) AS rank
FROM imm_words
WHERE vocabulary_visible = 1 AND first_seen IS NOT NULL ${exclusion}
GROUP BY ${DICTIONARY_WORD_KEY_SQL}
)
SELECT
${localEpochDaySql('firstSeen')} AS firstSeenDay,
CASE
WHEN firstSeenWithoutNames IS NULL THEN NULL
ELSE ${localEpochDaySql('firstSeenWithoutNames')}
END AS firstSeenDayWithoutNames,
CASE
WHEN rank IS NULL THEN 'unranked'
WHEN rank <= 1000 THEN 'top1k'
WHEN rank <= 5000 THEN 'top5k'
WHEN rank <= 10000 THEN 'top10k'
WHEN rank <= 20000 THEN 'top20k'
ELSE 'beyond20k'
END AS band,
COUNT(*) AS wordCount
FROM dictionary_words
GROUP BY 1, 2, 3
`,
)
.all(...excludedIds) as unknown as NewWordFirstSeenRow[];
newWordFirstSeenCache.set(db, { version, rows });
return rows;
}
function excludedVocabularyAliases(
@@ -324,6 +376,12 @@ export function getVocabularySummary(
knownWordCount: knownWords ? 0 : null,
knownWordCountWithoutNames: knownWords ? 0 : null,
};
// Keyed by dictionary form, so a word's conjugations count once and it is new
// this week only if no form of it was seen earlier.
const dictionaryWords = new Map<
string,
{ firstSeenSec: number; firstSeenSecWithoutNames: number | null; known: boolean }
>();
let lastId = Number.MIN_SAFE_INTEGER;
for (;;) {
@@ -337,23 +395,40 @@ export function getVocabularySummary(
) {
continue;
}
const isName = word.pos2 === '固有名詞';
const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec;
const isKnown = knownWords?.has(vocabularyDisplayHeadword(word)) ?? false;
summary.uniqueWords += 1;
if (!isName) summary.uniqueWordsWithoutNames += 1;
if (isNewThisWeek) {
summary.newThisWeek += 1;
if (!isName) summary.newThisWeekWithoutNames += 1;
}
if (isKnown) {
summary.knownWordCount! += 1;
if (!isName) summary.knownWordCountWithoutNames! += 1;
const firstSeenSec = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0);
const key = vocabularyDisplayHeadword(word);
const entry = dictionaryWords.get(key) ?? {
firstSeenSec,
firstSeenSecWithoutNames: null,
known: false,
};
entry.known ||= (knownWords?.has(key) ?? false) || (knownWords?.has(word.word) ?? false);
entry.firstSeenSec = Math.min(entry.firstSeenSec, firstSeenSec);
if (!isProperNounPos2(word.pos2)) {
entry.firstSeenSecWithoutNames = Math.min(
entry.firstSeenSecWithoutNames ?? firstSeenSec,
firstSeenSec,
);
}
dictionaryWords.set(key, entry);
}
if (words.length < scanBatchSize) break;
}
for (const { firstSeenSec, firstSeenSecWithoutNames, known } of dictionaryWords.values()) {
const hasNonNameForm = firstSeenSecWithoutNames !== null;
summary.uniqueWords += 1;
if (hasNonNameForm) summary.uniqueWordsWithoutNames += 1;
if (firstSeenSec >= weekAgoSec) summary.newThisWeek += 1;
if (hasNonNameForm && firstSeenSecWithoutNames >= weekAgoSec) {
summary.newThisWeekWithoutNames += 1;
}
if (known) {
summary.knownWordCount! += 1;
if (hasNonNameForm) summary.knownWordCountWithoutNames! += 1;
}
}
return summary;
}
@@ -418,7 +493,12 @@ export function getWordOccurrences(
reading: string,
limit = 100,
offset = 0,
allForms = false,
): WordOccurrenceRow[] {
const wordFilter = allForms
? `w.id IN (SELECT id FROM imm_words WHERE ${DICTIONARY_WORD_KEY_SQL} = ?)`
: 'w.headword = ? AND w.word = ? AND w.reading = ?';
const wordParams = allForms ? [headword] : [headword, word, reading];
const rows = db
.prepare(
`
@@ -440,13 +520,13 @@ export function getWordOccurrences(
JOIN imm_subtitle_lines l ON l.line_id = o.line_id
JOIN imm_videos v ON v.video_id = l.video_id
LEFT JOIN imm_anime a ON a.anime_id = l.anime_id
WHERE w.headword = ? AND w.word = ? AND w.reading = ?
ORDER BY l.CREATED_DATE DESC, l.line_id DESC
WHERE ${wordFilter}
ORDER BY ${MINEABLE_LINES_FIRST_SQL}, l.CREATED_DATE DESC, l.line_id DESC
LIMIT ?
OFFSET ?
`,
)
.all(headword, word, reading, limit, offset) as unknown as WordOccurrenceRow[];
.all(...wordParams, limit, offset) as unknown as WordOccurrenceRow[];
return rows.map(widenLineText);
}
@@ -553,7 +633,7 @@ export function searchSubtitleSentences(
JOIN imm_videos v ON v.video_id = l.video_id
LEFT JOIN imm_anime a ON a.anime_id = l.anime_id
WHERE ${clauses.join(' AND ')}
ORDER BY l.CREATED_DATE DESC, l.line_id DESC
ORDER BY ${MINEABLE_LINES_FIRST_SQL}, l.CREATED_DATE DESC, l.line_id DESC
LIMIT ?
`,
)
@@ -598,17 +678,99 @@ export function getSessionEvents(
}));
}
const WORD_DETAIL_FORM_LIMIT = 30;
/**
* One entry per written form. A form can be stored more than once when its reading
* was recorded differently, as 見て is under both みて and み. Returns most seen first.
*/
export function mergeFormsByWord<T extends { word: string; seen: number }>(forms: T[]): T[] {
const byWord = new Map<string, T>();
for (const form of forms) {
const existing = byWord.get(form.word);
if (existing) {
existing.seen += form.seen;
} else {
byWord.set(form.word, { ...form });
}
}
return [...byWord.values()].sort((left, right) => right.seen - left.seen);
}
/** A word's form plus totals across every form of its dictionary word. */
export function getWordDetail(db: DatabaseSync, wordId: number): WordDetailRow | null {
return db
const word = db
.prepare(
`
SELECT id AS wordId, headword, word, reading,
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
frequency, first_seen AS firstSeen, last_seen AS lastSeen
SELECT id AS wordId, COALESCE(headword, '') AS headword, COALESCE(word, '') AS word,
COALESCE(reading, '') AS reading,
part_of_speech AS partOfSpeech, pos1, pos2, pos3
FROM imm_words WHERE id = ?
`,
)
.get(wordId) as WordDetailRow | null;
.get(wordId) as Pick<
WordDetailRow,
'wordId' | 'headword' | 'word' | 'reading' | 'partOfSpeech' | 'pos1' | 'pos2' | 'pos3'
> | null;
if (!word) return null;
const hasHeadword = word.headword.trim() !== '';
const forms = db
.prepare(
`
SELECT id AS wordId, COALESCE(word, '') AS word, COALESCE(reading, '') AS reading,
frequency AS seen, frequency_rank AS rank,
first_seen AS firstSeen, last_seen AS lastSeen
FROM imm_words
WHERE ${hasHeadword ? '(headword = ? AND vocabulary_visible = 1) OR id = ?' : 'id = ?'}
ORDER BY frequency DESC, id
`,
)
.all(...(hasHeadword ? [word.headword, wordId] : [wordId])) as Array<{
wordId: number;
word: string;
reading: string;
seen: number | null;
rank: number | null;
firstSeen: number;
lastSeen: number;
}>;
const lines = db
.prepare(
`
SELECT COUNT(DISTINCT l.line_id) AS lineCount,
COUNT(DISTINCT CASE WHEN ${NO_LOCAL_FILE_SQL} THEN l.line_id END) AS streamLineCount
FROM imm_word_line_occurrences o
JOIN imm_words w ON w.id = o.word_id
JOIN imm_subtitle_lines l ON l.line_id = o.line_id
JOIN imm_videos v ON v.video_id = l.video_id
WHERE ${hasHeadword ? 'w.headword = ?' : 'w.id = ?'}
`,
)
.get(hasHeadword ? word.headword : wordId) as { lineCount: number; streamLineCount: number };
const merged = mergeFormsByWord(forms.map((form) => ({ ...form, seen: form.seen ?? 0 })));
const toSeconds = (value: number): number => timestampSeconds(fromDbTimestamp(value) ?? 0);
const ranks = forms.map((form) => form.rank).filter((rank): rank is number => (rank ?? 0) > 0);
return {
...word,
frequency: forms.reduce((sum, form) => sum + (form.seen ?? 0), 0),
frequencyRank: ranks.length > 0 ? Math.min(...ranks) : null,
dictionaryReading:
forms.find((form) => form.word === word.headword && form.reading.trim() !== '')?.reading ??
'',
formCount: merged.length,
forms: merged.slice(0, WORD_DETAIL_FORM_LIMIT).map((form) => ({
wordId: form.wordId,
word: form.word,
reading: form.reading,
seen: form.seen,
})),
lineCount: lines.lineCount,
streamLineCount: lines.streamLineCount,
firstSeen: Math.min(...forms.map((form) => toSeconds(form.firstSeen))),
lastSeen: Math.max(...forms.map((form) => toSeconds(form.lastSeen))),
};
}
export function getWordAnimeAppearances(
@@ -621,21 +783,26 @@ export function getWordAnimeAppearances(
SELECT a.anime_id AS animeId, a.canonical_title AS animeTitle,
SUM(o.occurrence_count) AS occurrenceCount
FROM imm_word_line_occurrences o
JOIN imm_words w ON w.id = o.word_id
JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
JOIN imm_anime a ON a.anime_id = sl.anime_id
WHERE o.word_id = ? AND sl.anime_id IS NOT NULL
WHERE (w.id = ? OR w.headword = (SELECT NULLIF(TRIM(headword), '') FROM imm_words WHERE id = ?))
AND sl.anime_id IS NOT NULL
GROUP BY a.anime_id
ORDER BY occurrenceCount DESC
`,
)
.all(wordId) as WordAnimeAppearanceRow[];
.all(wordId, wordId) as WordAnimeAppearanceRow[];
}
/** Other dictionary words sharing a reading or kanji; a word's own forms don't count. */
export function getSimilarWords(db: DatabaseSync, wordId: number, limit = 10): SimilarWordRow[] {
const word = db.prepare('SELECT headword, reading FROM imm_words WHERE id = ?').get(wordId) as {
headword: string;
reading: string;
} | null;
const word = db
.prepare(
`SELECT COALESCE(headword, '') AS headword, COALESCE(reading, '') AS reading
FROM imm_words WHERE id = ?`,
)
.get(wordId) as { headword: string; reading: string } | null;
if (!word || word.headword.trim() === '') return [];
const clauses: string[] = [];
@@ -653,22 +820,41 @@ export function getSimilarWords(db: DatabaseSync, wordId: number, limit = 10): S
if (clauses.length === 0) return [];
const orderBy =
reading !== '' ? 'CASE WHEN reading = ? THEN 0 ELSE 1 END, frequency DESC' : 'frequency DESC';
const orderParams = reading !== '' ? [word.reading] : [];
const readingMatch =
reading !== ''
? 'MAX(CASE WHEN reading = ? THEN 1 ELSE 0 END) OVER (PARTITION BY headword)'
: '0';
const readingMatchParams = reading !== '' ? [word.reading] : [];
// One form matching is enough to relate a word, but its total and the form it shows
// come from all its visible forms, so a rare form with the same reading can't stand in.
// The blank check lists its characters because SQL TRIM alone strips only ASCII spaces.
return db
.prepare(
`
SELECT id AS wordId, headword, word, reading, frequency
FROM imm_words
WHERE id != ?
AND (${clauses.join(' OR ')})
ORDER BY ${orderBy}
WITH related AS (
SELECT DISTINCT headword
FROM imm_words
WHERE headword != ? AND vocabulary_visible = 1
AND TRIM(headword, ' ' || char(9, 10, 11, 12, 13, 12288)) != ''
AND (${clauses.join(' OR ')})
),
forms AS (
SELECT id, headword, word, reading,
SUM(frequency) OVER (PARTITION BY headword) AS totalSeen,
${readingMatch} AS readingMatch,
ROW_NUMBER() OVER (PARTITION BY headword ORDER BY frequency DESC, id) AS formOrder
FROM imm_words
WHERE vocabulary_visible = 1 AND headword IN (SELECT headword FROM related)
)
SELECT id AS wordId, headword, word, reading, totalSeen AS frequency
FROM forms
WHERE formOrder = 1
ORDER BY readingMatch DESC, totalSeen DESC, headword
LIMIT ?
`,
)
.all(wordId, ...params, ...orderParams, limit) as SimilarWordRow[];
.all(word.headword, ...params, ...readingMatchParams, limit) as SimilarWordRow[];
}
export function getKanjiDetail(db: DatabaseSync, kanjiId: number): KanjiDetailRow | null {
src/core/services/immersion-tracker/query-sessions.ts
+67 -2
View File
@@ -1,3 +1,4 @@
import type { StatsDayRange } from '../../../types/stats-http-contract';
import type { DatabaseSync } from './sqlite';
import type {
ImmersionSessionRollupRow,
@@ -15,7 +16,12 @@ import {
visibleWordSql,
} from './query-shared';
export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummaryQueryRow[] {
export function getSessionSummaries(
db: DatabaseSync,
limit = 50,
dayRange?: StatsDayRange,
): SessionSummaryQueryRow[] {
const dayFilter = dayRange ? `WHERE ${sessionRollupDaySql('s')} BETWEEN ? AND ?` : '';
const prepared = db.prepare(`
${ACTIVE_SESSION_METRICS_CTE}
SELECT
@@ -38,10 +44,12 @@ export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummar
LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
LEFT JOIN imm_videos v ON v.video_id = s.video_id
LEFT JOIN imm_anime a ON a.anime_id = v.anime_id
${dayFilter}
ORDER BY s.started_at_ms DESC
LIMIT ?
`);
const rows = prepared.all(limit) as Array<
const params = dayRange ? [dayRange.startDay, dayRange.endDay, limit] : [limit];
const rows = prepared.all(...params) as Array<
SessionSummaryQueryRow & {
startedAtMs: number | string;
endedAtMs: number | string | null;
@@ -393,6 +401,63 @@ export function selectDailyRollupRows(db: DatabaseSync, limit: number): RawRollu
.all(limit) as unknown as RawRollupRow[];
}
/** Raw daily rollup rows for an inclusive day span; a null start means no lower bound. */
export function selectDailyRollupRowsBetween(
db: DatabaseSync,
startDay: number | null,
endDay: number,
): RawRollupRow[] {
return db
.prepare(
`
SELECT
r.rollup_day AS rollupDayOrMonth,
r.video_id AS videoId,
r.total_sessions AS totalSessions,
r.total_active_min AS totalActiveMin,
r.total_lines_seen AS totalLinesSeen,
r.total_tokens_seen AS totalTokensSeen,
r.total_cards AS totalCards,
r.cards_per_hour AS cardsPerHour,
r.lookup_hit_rate AS lookupHitRate
FROM imm_daily_rollups r
WHERE r.rollup_day BETWEEN ? AND ?
ORDER BY r.rollup_day DESC, r.video_id DESC
`,
)
.all(startDay ?? Number.MIN_SAFE_INTEGER, endDay) as unknown as RawRollupRow[];
}
/** Raw monthly rollup rows for an inclusive `YYYYMM` span; a null start means no lower bound. */
export function selectMonthlyRollupRowsBetween(
db: DatabaseSync,
startMonth: number | null,
endMonth: number,
): RawRollupRow[] {
return db
.prepare(
`
SELECT
r.rollup_month AS rollupDayOrMonth,
r.video_id AS videoId,
r.total_sessions AS totalSessions,
r.total_active_min AS totalActiveMin,
r.total_lines_seen AS totalLinesSeen,
r.total_tokens_seen AS totalTokensSeen,
r.total_cards AS totalCards,
CASE
WHEN r.total_active_min > 0 THEN (r.total_cards * 60.0) / r.total_active_min
ELSE NULL
END AS cardsPerHour,
NULL AS lookupHitRate
FROM imm_monthly_rollups r
WHERE r.rollup_month BETWEEN ? AND ?
ORDER BY r.rollup_month DESC, r.video_id DESC
`,
)
.all(startMonth ?? Number.MIN_SAFE_INTEGER, endMonth) as unknown as RawRollupRow[];
}
/** Monthly counterpart of `selectDailyRollupRows`. */
export function selectMonthlyRollupRows(db: DatabaseSync, limit: number): RawRollupRow[] {
return db
src/core/services/immersion-tracker/query-trends.test.ts
+541
View File
@@ -0,0 +1,541 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { Database, type DatabaseSync } from './sqlite';
import { getOrCreateAnimeRecord, getOrCreateVideoRecord, linkVideoToAnimeRecord } from './storage';
import { ensureSchema } from './storage';
import { SOURCE_TYPE_LOCAL } from './types';
import { getTrendsDashboard } from './query-trends';
import { getSessionSummaries } from './query-sessions';
import { getVocabularyChartData, getVocabularySummary } from './query-lexical';
import { markLexicalDailyRollupsReady } from './lexical-rollups';
/** Noon, local time, on a calendar day, so the local epoch day holds in every time zone. */
function localNoonMs(epochDay: number): number {
const date = new Date(epochDay * 86_400_000);
return new Date(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), 12).getTime();
}
// Mocked "now" is noon on 2026-03-01, a Sunday.
const TODAY = Date.UTC(2026, 2, 1) / 86_400_000;
const NOW_MS = String(localNoonMs(TODAY));
function withDashboardDb(run: (db: DatabaseSync) => void): void {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-trends-'));
const db = new Database(path.join(dir, 'immersion.sqlite'));
const previousNow = globalThis.__subminerTestNowMs;
globalThis.__subminerTestNowMs = NOW_MS;
try {
ensureSchema(db);
run(db);
} finally {
globalThis.__subminerTestNowMs = previousNow;
db.close();
fs.rmSync(dir, { recursive: true, force: true });
}
}
function addVideo(db: DatabaseSync, title: string, episode: number, watched = false): number {
const key = `${title}-${episode}`;
const videoId = getOrCreateVideoRecord(db, `local:/tmp/${key}.mkv`, {
canonicalTitle: key,
sourcePath: `/tmp/${key}.mkv`,
sourceUrl: null,
sourceType: SOURCE_TYPE_LOCAL,
});
const animeId = getOrCreateAnimeRecord(db, {
parsedTitle: title,
canonicalTitle: title,
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
linkVideoToAnimeRecord(db, videoId, {
animeId,
parsedBasename: `${key}.mkv`,
parsedTitle: title,
parsedSeason: 1,
parsedEpisode: episode,
parserSource: 'test',
parserConfidence: 1,
parseMetadataJson: null,
});
if (watched) db.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?').run(videoId);
return videoId;
}
function addRollup(
db: DatabaseSync,
day: number,
videoId: number,
values: { minutes?: number; words?: number; cards?: number } = {},
): void {
db.prepare(
`INSERT INTO imm_daily_rollups (
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards
) VALUES (?, ?, 1, ?, 1, ?, ?)`,
).run(day, videoId, values.minutes ?? 30, values.words ?? 100, values.cards ?? 0);
}
function addSession(
db: DatabaseSync,
day: number,
videoId: number,
values: { minutes?: number; lookups?: number } = {},
): void {
const startedAtMs = localNoonMs(day).toString();
const activeMs = (values.minutes ?? 30) * 60_000;
db.prepare(
`INSERT INTO imm_sessions (
session_uuid, video_id, started_at_ms, ended_at_ms, status, active_watched_ms,
total_watched_ms, yomitan_lookup_count
) VALUES (?, ?, ?, ?, 2, ?, ?, ?)`,
).run(
`session-${day}-${videoId}-${Math.random()}`,
videoId,
startedAtMs,
(BigInt(startedAtMs) + BigInt(activeMs)).toString(),
activeMs,
activeMs,
values.lookups ?? 0,
);
}
function addWord(
db: DatabaseSync,
values: {
headword: string;
word?: string;
day: number;
pos2?: string | null;
rank?: number | null;
},
): void {
const firstSeen = localNoonMs(values.day) / 1000;
db.prepare(
`INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen,
frequency, frequency_rank
) VALUES (?, ?, ?, 'noun', '名詞', ?, '', ?, ?, 1, ?)`,
).run(
values.headword,
values.word ?? values.headword,
values.word ?? values.headword,
values.pos2 ?? null,
firstSeen,
firstSeen,
values.rank ?? null,
);
}
test('the title lists stay inside the calendar range when activity is sparse', () => {
withDashboardDb((db) => {
const recent = addVideo(db, 'Recent Show', 1);
const old = addVideo(db, 'Old Show', 1);
addRollup(db, TODAY, recent, { minutes: 20 });
// Outside a 7-day calendar window, though well within the last 7 active days.
addRollup(db, TODAY - 40, old, { minutes: 90 });
addSession(db, TODAY - 40, old);
const dashboard = getTrendsDashboard(db, '7d', 'day');
assert.deepEqual(
dashboard.librarySummary.map((row) => row.title),
['Recent Show'],
);
assert.deepEqual(
dashboard.titleTimeline.map((point) => point.animeTitle),
['Recent Show'],
);
assert.equal(dashboard.summary.current.watchTimeMin, 20);
assert.equal(
dashboard.librarySummary.reduce((sum, row) => sum + row.watchTimeMin, 0),
dashboard.activity.watchTime.reduce((sum, point) => sum + point.value, 0),
);
});
});
test('every grouping reports the same totals for a range', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
for (const day of [TODAY, TODAY - 3, TODAY - 20, TODAY - 29, TODAY - 45]) {
addRollup(db, day, videoId, { minutes: 10, cards: 1 });
}
// January is only partly in range, and its monthly total includes days that are not.
const insertMonthly = db.prepare(
`INSERT INTO imm_monthly_rollups (
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards
) VALUES (?, ?, 1, ?, 1, 100, 1)`,
);
insertMonthly.run(202601, videoId, 20);
insertMonthly.run(202602, videoId, 20);
insertMonthly.run(202603, videoId, 10);
const sum = (groupBy: 'day' | 'week' | 'month') =>
getTrendsDashboard(db, '30d', groupBy).activity.watchTime.reduce(
(total, point) => total + point.value,
0,
);
assert.equal(sum('day'), 40);
assert.equal(sum('week'), 40);
assert.equal(sum('month'), 40);
});
});
test('all-time month buckets use monthly rollups that outlive daily ones', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
const insertMonthly = db.prepare(
`INSERT INTO imm_monthly_rollups (
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards
) VALUES (?, ?, 3, ?, 1, 100, 50)`,
);
insertMonthly.run(202501, videoId, 500);
insertMonthly.run(202503, videoId, 200);
assert.deepEqual(getTrendsDashboard(db, 'all', 'month').activity.watchTime, [
{ label: 'Jan 2025', value: 500 },
{ label: 'Feb 2025', value: 0 },
{ label: 'Mar 2025', value: 200 },
]);
assert.deepEqual(getTrendsDashboard(db, 'all', 'month', false).activity.watchTime, [
{ label: 'Jan 2025', value: 500 },
{ label: 'Mar 2025', value: 200 },
]);
});
});
test('a finished video counts once however many days it was watched', () => {
withDashboardDb((db) => {
const finished = addVideo(db, 'Show', 1, true);
const unfinished = addVideo(db, 'Show', 2, false);
addRollup(db, TODAY - 2, finished);
addRollup(db, TODAY - 1, finished);
addRollup(db, TODAY, unfinished);
for (const groupBy of ['day', 'week', 'month'] as const) {
const dashboard = getTrendsDashboard(db, '30d', groupBy);
assert.equal(dashboard.progress.episodes.at(-1)?.value, 1, groupBy);
assert.equal(dashboard.summary.current.episodes, 1, groupBy);
}
const daily = getTrendsDashboard(db, '7d', 'day').activity.episodes;
// Attributed to the last day it was watched.
assert.equal(daily.at(-2)?.value, 1);
assert.equal(daily.at(-3)?.value, 0);
assert.equal(getTrendsDashboard(db, '30d', 'day').librarySummary[0]?.episodesFinished, 1);
assert.equal(getTrendsDashboard(db, '30d', 'day').librarySummary[0]?.videos, 2);
});
});
test('rates are gaps without enough data, not zeros', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
// A two-minute day with one card: 30 cards per hour if rated on its own.
addRollup(db, TODAY - 20, videoId, { minutes: 2, words: 50, cards: 1 });
const dashboard = getTrendsDashboard(db, '30d', 'day');
assert.equal(dashboard.ratios.cardsPerHour.at(-1)?.value, null);
assert.equal(
dashboard.ratios.cardsPerHour.every((point) => point.value === null),
true,
);
assert.equal(
dashboard.ratios.readingSpeed.every((point) => point.value === null),
true,
);
});
});
test('daily rates are volume-weighted over a trailing week, reaching before the range', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
// Eight days before today: outside the 7d range, inside the first day's lookback.
addRollup(db, TODAY - 8, videoId, { minutes: 60, words: 3_000, cards: 6 });
addRollup(db, TODAY - 6, videoId, { minutes: 60, words: 1_000, cards: 2 });
const speed = getTrendsDashboard(db, '7d', 'day').ratios.readingSpeed;
// (3000 + 1000) words / (60 + 60) min, not an average of 50 and 16.7 ...
assert.equal(speed[0]?.value, +(4_000 / 120).toFixed(1));
// ... and once the older day leaves the trailing week only the second remains.
assert.equal(speed[5]?.value, +(1_000 / 60).toFixed(1));
});
});
test('week grouping starts on Monday and clips the first bucket to the range', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
addRollup(db, TODAY, videoId);
const { buckets } = getTrendsDashboard(db, '30d', 'week');
assert.equal(buckets[0]?.startDay, TODAY - 29);
assert.equal(buckets.at(-1)?.endDay, TODAY);
// Today is a Sunday, so the final bucket is a full Monday-to-Sunday week.
assert.equal(buckets.at(-1)?.startDay, TODAY - 6);
for (let index = 1; index < buckets.length; index += 1) {
assert.equal(buckets[index]!.startDay, buckets[index - 1]!.endDay + 1);
}
});
});
test('showing empty periods on all-time fills the calendar between active days', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
addRollup(db, TODAY - 10, videoId);
addRollup(db, TODAY, videoId);
assert.equal(getTrendsDashboard(db, 'all', 'day', true).activity.watchTime.length, 11);
assert.equal(getTrendsDashboard(db, 'all', 'day', false).activity.watchTime.length, 2);
assert.deepEqual(getTrendsDashboard(db, 'all', 'day', true).window, {
startDay: TODAY - 10,
endDay: TODAY,
});
});
});
test('month labels name the year, and day labels do when the range crosses one', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
addRollup(db, TODAY, videoId);
assert.equal(getTrendsDashboard(db, '30d', 'month').buckets.at(-1)?.label, 'Mar 2026');
assert.equal(getTrendsDashboard(db, '7d', 'day').buckets.at(-1)?.label, 'Mar 1');
assert.equal(getTrendsDashboard(db, '90d', 'day').buckets.at(-1)?.label, 'Mar 1, 2026');
});
});
test('a custom range clips to its days and compares against the same-length span before it', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
addRollup(db, TODAY - 5, videoId, { minutes: 10 });
addRollup(db, TODAY - 15, videoId, { minutes: 20 });
addRollup(db, TODAY - 25, videoId, { minutes: 40 });
const dashboard = getTrendsDashboard(db, { startDay: TODAY - 19, endDay: TODAY - 10 });
assert.equal(dashboard.activity.watchTime.length, 10);
assert.equal(dashboard.summary.current.watchTimeMin, 20);
assert.equal(dashboard.summary.previous?.watchTimeMin, 40);
assert.deepEqual(dashboard.window, { startDay: TODAY - 19, endDay: TODAY - 10 });
});
});
test('summary compares with the previous period and all-time has no comparison', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
addRollup(db, TODAY, videoId, { minutes: 10, cards: 1 });
addRollup(db, TODAY - 10, videoId, { minutes: 25, cards: 4 });
const { current, previous } = getTrendsDashboard(db, '7d', 'day').summary;
assert.equal(current.watchTimeMin, 10);
assert.equal(current.activeDays, 1);
assert.equal(previous?.watchTimeMin, 25);
assert.equal(previous?.cards, 4);
assert.equal(getTrendsDashboard(db, 'all', 'day').summary.previous, null);
});
});
test('a title filter narrows the page but keeps the title lists complete', () => {
withDashboardDb((db) => {
const first = addVideo(db, 'First', 1);
const second = addVideo(db, 'Second', 1);
addRollup(db, TODAY, first, { minutes: 10 });
addRollup(db, TODAY, second, { minutes: 50 });
addSession(db, TODAY, first, { minutes: 10, lookups: 3 });
addSession(db, TODAY, second, { minutes: 50, lookups: 9 });
addWord(db, { headword: '海', day: TODAY });
const dashboard = getTrendsDashboard(db, '7d', 'day', true, 'First');
assert.equal(dashboard.summary.current.watchTimeMin, 10);
assert.equal(dashboard.progress.lookups.at(-1)?.value, 3);
assert.equal(
dashboard.watchTimeByDayHour.flat().reduce((sum, value) => sum + value, 0),
10,
);
assert.equal(dashboard.librarySummary.length, 2);
assert.equal(dashboard.titleTimeline.length, 2);
// New words are not tracked per title.
assert.equal(dashboard.summary.current.newWords, null);
assert.equal(dashboard.learning, null);
assert.deepEqual(dashboard.progress.newWords, []);
});
});
test('running totals follow the activity charts when empty days are hidden', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
addRollup(db, TODAY - 3, videoId, { minutes: 30 });
addRollup(db, TODAY, videoId, { minutes: 10 });
const dashboard = getTrendsDashboard(db, '7d', 'day', false);
assert.deepEqual(
dashboard.progress.watchTime.map((point) => point.value),
[30, 40],
);
assert.deepEqual(
dashboard.progress.watchTime.map((point) => point.label),
dashboard.activity.watchTime.map((point) => point.label),
);
});
});
test('the watch time running total rounds its sum, so it ends on the summary total', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
addRollup(db, TODAY - 2, videoId, { minutes: 10.4 });
addRollup(db, TODAY - 1, videoId, { minutes: 10.4 });
addRollup(db, TODAY, videoId, { minutes: 10.4 });
const dashboard = getTrendsDashboard(db, '7d', 'day', false);
assert.deepEqual(
dashboard.activity.watchTime.map((point) => point.value),
[10, 10, 10],
);
assert.deepEqual(
dashboard.progress.watchTime.map((point) => point.value),
[10, 21, 31],
);
assert.equal(dashboard.summary.current.watchTimeMin, 31);
});
});
test('the summary adds watch time in date order, so a total near .5 rounds like the chart', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
// Newest first these add up to 4.499999999999999, and oldest first to 4.5.
addRollup(db, TODAY - 2, videoId, { minutes: 0.1 });
addRollup(db, TODAY - 1, videoId, { minutes: 0.1 });
addRollup(db, TODAY, videoId, { minutes: 4.3 });
const dashboard = getTrendsDashboard(db, '7d', 'day');
assert.equal(
dashboard.summary.current.watchTimeMin,
dashboard.progress.watchTime.at(-1)?.value,
);
});
});
test('new words count a dictionary word once, on its first form, without names', () => {
withDashboardDb((db) => {
addWord(db, { headword: '言う', word: '言う', day: TODAY - 2, rank: 50 });
addWord(db, { headword: '言う', word: '言った', day: TODAY, rank: 50 });
addWord(db, { headword: '東京', day: TODAY - 1, pos2: '固有名詞', rank: 800 });
const dashboard = getTrendsDashboard(db, '7d', 'day');
assert.equal(dashboard.summary.current.newWords, 1);
assert.equal(dashboard.progress.newWords.at(-1)?.value, 1);
assert.equal(dashboard.progress.newWords.at(-3)?.value, 1);
assert.equal(dashboard.progress.newWords.at(-4)?.value, 0);
});
});
test('new words per thousand and frequency bands read dictionary words', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
addRollup(db, TODAY, videoId, { words: 2_000 });
addWord(db, { headword: '一', day: TODAY, rank: 500 });
addWord(db, { headword: '二', day: TODAY, rank: 3_000 });
addWord(db, { headword: '三', day: TODAY, rank: 30_000 });
addWord(db, { headword: '四', day: TODAY, rank: null });
const { learning } = getTrendsDashboard(db, '7d', 'day');
assert.equal(learning?.newWordsPerThousand.at(-1)?.value, 2);
assert.deepEqual(learning?.frequencyBands.at(-1)?.counts, {
top1k: 1,
top5k: 1,
top10k: 0,
top20k: 0,
beyond20k: 1,
unranked: 1,
});
});
});
test('vocabulary charts and summary count dictionary words, not surface forms', () => {
withDashboardDb((db) => {
markLexicalDailyRollupsReady(db);
addWord(db, { headword: '言う', word: '言う', day: TODAY - 20 });
addWord(db, { headword: '言う', word: '言った', day: TODAY });
addWord(db, { headword: '東京', day: TODAY, pos2: '固有名詞' });
const charts = getVocabularyChartData(db);
const summary = getVocabularySummary(db, null, Number(NOW_MS));
assert.deepEqual(charts.newWordsTimelineWithoutNames, [{ epochDay: TODAY - 20, wordCount: 1 }]);
assert.deepEqual(charts.newWordsTimeline, [
{ epochDay: TODAY - 20, wordCount: 1 },
{ epochDay: TODAY, wordCount: 1 },
]);
assert.equal(summary.uniqueWords, 2);
assert.equal(summary.uniqueWordsWithoutNames, 1);
// 言う was first seen long ago, so its newer conjugation is not a new word.
assert.equal(summary.newThisWeek, 1);
assert.equal(summary.newThisWeekWithoutNames, 0);
});
});
test('names tagged with merged parts of speech count as names', () => {
withDashboardDb((db) => {
markLexicalDailyRollupsReady(db);
addWord(db, { headword: '美琴', day: TODAY, pos2: '固有名詞|一般' });
addWord(db, { headword: '谷君', day: TODAY, pos2: '固有名詞|接尾' });
addWord(db, { headword: '海', day: TODAY, pos2: '一般' });
assert.equal(getTrendsDashboard(db, '7d', 'day').summary.current.newWords, 1);
const summary = getVocabularySummary(db, null, Number(NOW_MS));
assert.equal(summary.uniqueWords, 3);
assert.equal(summary.uniqueWordsWithoutNames, 1);
assert.deepEqual(getVocabularyChartData(db).newWordsTimelineWithoutNames, [
{ epochDay: TODAY, wordCount: 1 },
]);
});
});
test('new word counts pick up writes from this and other connections', () => {
withDashboardDb((db) => {
addWord(db, { headword: '海', day: TODAY });
assert.equal(getTrendsDashboard(db, '7d', 'day').summary.current.newWords, 1);
addWord(db, { headword: '山', day: TODAY });
assert.equal(getTrendsDashboard(db, '7d', 'day').summary.current.newWords, 2);
const dbPath = (db.prepare('PRAGMA database_list').get() as { file: string }).file;
const other = new Database(dbPath);
try {
addWord(other, { headword: '川', day: TODAY });
} finally {
other.close();
}
assert.equal(getTrendsDashboard(db, '7d', 'day').summary.current.newWords, 3);
});
});
test('getSessionSummaries can narrow to a span of local days', () => {
withDashboardDb((db) => {
const videoId = addVideo(db, 'Show', 1);
addSession(db, TODAY - 5, videoId);
addSession(db, TODAY - 4, videoId);
addSession(db, TODAY, videoId);
assert.equal(getSessionSummaries(db, 50).length, 3);
assert.equal(getSessionSummaries(db, 50, { startDay: TODAY - 5, endDay: TODAY - 4 }).length, 2);
});
});
src/core/services/immersion-tracker/query-trends.ts
+593 -641
View File
File diff suppressed because it is too large. Load diff
src/core/services/immersion-tracker/query-vocabulary-words.test.ts
+562
View File
@@ -0,0 +1,562 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import type { StatsDictionaryWordQuery } from '../../../types/stats-http-contract';
import {
getSimilarWords,
getWordAnimeAppearances,
getVocabularySummary,
getWordDetail,
getWordOccurrences,
replaceStatsExcludedWords,
searchSubtitleSentences,
} from './query-lexical';
import { markLexicalDailyRollupsReady } from './lexical-rollups';
import { getDictionaryWordPage, getVocabularyCoverage } from './query-vocabulary-words';
import { startSessionRecord } from './session';
import { Database, type DatabaseSync } from './sqlite';
import { ensureSchema, getOrCreateAnimeRecord, getOrCreateVideoRecord } from './storage';
import { SOURCE_TYPE_LOCAL, SOURCE_TYPE_REMOTE } from './types';
function withDb(run: (db: DatabaseSync) => void): void {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-vocabulary-'));
const db = new Database(path.join(dir, 'immersion.sqlite'));
try {
ensureSchema(db);
run(db);
} finally {
db.close();
fs.rmSync(dir, { recursive: true, force: true });
}
}
interface WordFixture {
headword: string;
word?: string;
reading?: string;
seen?: number;
rank?: number | null;
pos2?: string;
partOfSpeech?: string;
firstSeen?: number;
}
function addWord(db: DatabaseSync, word: WordFixture): number {
const firstSeen = word.firstSeen ?? 1_700_000_000;
const result = db
.prepare(
`INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen,
frequency, frequency_rank
) VALUES (?, ?, ?, ?, '', ?, '', ?, ?, ?, ?)`,
)
.run(
word.headword,
word.word ?? word.headword,
word.reading ?? '',
word.partOfSpeech ?? 'noun',
word.pos2 ?? '',
firstSeen,
firstSeen,
word.seen ?? 1,
word.rank ?? null,
);
return Number(result.lastInsertRowid);
}
function getIdOf(db: DatabaseSync, headword: string): number {
const row = db.prepare('SELECT id FROM imm_words WHERE headword = ?').get(headword) as {
id: number;
};
return row.id;
}
function addOccurrence(
db: DatabaseSync,
wordId: number,
animeTitle: string,
count = 1,
source: 'file' | 'stream' = 'file',
timing: { startMs: number | null; endMs: number | null } = { startMs: 1_000, endMs: 2_000 },
): void {
const animeId = getOrCreateAnimeRecord(db, {
parsedTitle: animeTitle,
canonicalTitle: animeTitle,
anilistId: null,
titleRomaji: null,
titleEnglish: null,
titleNative: null,
metadataJson: null,
});
const videoId = getOrCreateVideoRecord(
db,
source === 'file' ? `local:/tmp/${animeTitle}.mkv` : `remote:jellyfin://${animeTitle}`,
{
canonicalTitle: animeTitle,
sourcePath: source === 'file' ? `/tmp/${animeTitle}.mkv` : null,
sourceUrl: source === 'file' ? null : `jellyfin://${animeTitle}`,
sourceType: source === 'file' ? SOURCE_TYPE_LOCAL : SOURCE_TYPE_REMOTE,
},
);
const { sessionId } = startSessionRecord(db, videoId, 1_700_000_000_000);
const lineId = Number(
db
.prepare(
`INSERT INTO imm_subtitle_lines (
session_id, video_id, anime_id, line_index, segment_start_ms, segment_end_ms, text
) VALUES (?, ?, ?, 0, ?, ?, ?)`,
)
.run(sessionId, videoId, animeId, timing.startMs, timing.endMs, `line ${animeTitle}`)
.lastInsertRowid,
);
db.prepare(
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count) VALUES (?, ?, ?)`,
).run(lineId, wordId, count);
}
function query(overrides: Partial<StatsDictionaryWordQuery> = {}): StatsDictionaryWordQuery {
return {
sort: 'seen',
descending: true,
hideKnown: false,
hideNames: false,
hideKana: false,
limit: 25,
offset: 0,
...overrides,
};
}
const headwords = (db: DatabaseSync, overrides: Partial<StatsDictionaryWordQuery>) =>
getDictionaryWordPage(db, null, query(overrides)).rows.map((row) => row.headword);
test('conjugated forms fold into one word with its total and dictionary reading', () => {
withDb((db) => {
addWord(db, { headword: '見る', word: '見て', reading: 'みて', seen: 263, rank: 36 });
const dictionaryId = addWord(db, {
headword: '見る',
word: '見る',
reading: 'みる',
seen: 172,
rank: 36,
});
addWord(db, { headword: '見る', word: '見てる', reading: 'みてる', seen: 78, rank: 36 });
const { rows, total } = getDictionaryWordPage(db, null, query());
assert.equal(total, 1);
assert.equal(rows[0]?.seen, 513);
assert.equal(rows[0]?.reading, 'みる');
assert.equal(rows[0]?.wordId, dictionaryId);
assert.equal(rows[0]?.formCount, 3);
assert.deepEqual(
rows[0]?.forms.map((form) => form.word),
['見て', '見る', '見てる'],
);
});
});
test('a form stored under two readings counts as one form', () => {
withDb((db) => {
const first = addWord(db, { headword: '見る', word: '見て', reading: 'みて', seen: 263 });
addWord(db, { headword: '見る', word: '見て', reading: 'み', seen: 122 });
addWord(db, { headword: '見る', word: '見た', reading: 'みた', seen: 200 });
const row = getDictionaryWordPage(db, null, query()).rows[0];
assert.equal(row?.formCount, 2);
assert.deepEqual(
row?.forms.map((form) => [form.word, form.seen]),
[
['見て', 385],
['見た', 200],
],
);
assert.equal(getWordDetail(db, first)?.formCount, 2);
});
});
test('a word seen only conjugated has no reading rather than a conjugated one', () => {
withDb((db) => {
addWord(db, { headword: '待つ', word: '待って', reading: 'まって', seen: 5 });
assert.equal(getDictionaryWordPage(db, null, query()).rows[0]?.reading, '');
});
});
test('the list reaches rarely seen words, not just the most seen ones', () => {
withDb((db) => {
for (let index = 0; index < 600; index += 1) {
addWord(db, { headword: `語${index}`, seen: 1_000 - index });
}
const { rows, total } = getDictionaryWordPage(db, null, query({ limit: 10, offset: 590 }));
assert.equal(total, 600);
assert.equal(rows[9]?.headword, '語599');
});
});
test('search matches any form or reading, treating katakana and hiragana alike', () => {
withDb((db) => {
addWord(db, { headword: '猫', word: '猫', reading: 'ねこ' });
addWord(db, { headword: '行く', word: '行った', reading: 'いった' });
addWord(db, { headword: '犬', word: '犬', reading: 'いぬ' });
assert.deepEqual(headwords(db, { search: 'ネコ' }), ['猫']);
assert.deepEqual(headwords(db, { search: '行った' }), ['行く']);
assert.deepEqual(headwords(db, { search: 'いった' }), ['行く']);
});
});
for (const c of [
{ name: 'known words', filter: { hideKnown: true }, expected: ['海', '田中', 'なる'] },
{ name: 'names', filter: { hideNames: true }, expected: ['山', '海', 'なる'] },
{ name: 'kana', filter: { hideKana: true }, expected: ['山', '海', '田中'] },
{ name: 'a rank limit', filter: { maxRank: 500 }, expected: ['山', 'なる'] },
{ name: 'a part of speech', filter: { partOfSpeech: 'verb' as const }, expected: ['なる'] },
]) {
test(`the list filters by ${c.name}`, () => {
withDb((db) => {
addWord(db, { headword: '山', seen: 40, rank: 100 });
addWord(db, { headword: '海', seen: 30, rank: 9_000 });
addWord(db, { headword: '田中', seen: 20, rank: 800, pos2: '固有名詞|一般' });
addWord(db, { headword: 'なる', seen: 10, rank: 16, partOfSpeech: 'verb' });
const rows = getDictionaryWordPage(db, new Set(['山']), query(c.filter)).rows;
assert.deepEqual(
rows.map((row) => row.headword),
c.expected,
);
});
});
}
test('a word with a non-name form is not a name and ranks as that form', () => {
withDb((db) => {
addWord(db, { headword: '白', word: 'しろ', rank: 49_000, pos2: '固有名詞' });
addWord(db, { headword: '白', word: '白', rank: 1_500, pos2: '一般' });
const row = getDictionaryWordPage(db, null, query({ hideNames: true })).rows[0];
assert.equal(row?.isName, false);
assert.equal(row?.rank, 1_500);
});
});
test('excluded words stay out of the list', () => {
withDb((db) => {
addWord(db, { headword: '音', seen: 50 });
addWord(db, { headword: '海', seen: 10 });
assert.deepEqual(headwords(db, {}), ['音', '海']);
replaceStatsExcludedWords(db, [{ headword: '音', word: '音', reading: '' }]);
assert.deepEqual(headwords(db, {}), ['海']);
});
});
test('unranked words sort last in both rank directions', () => {
withDb((db) => {
addWord(db, { headword: '無', rank: null });
addWord(db, { headword: '高', rank: 10 });
addWord(db, { headword: '低', rank: 900 });
assert.deepEqual(headwords(db, { sort: 'rank', descending: false }), ['高', '低', '無']);
assert.deepEqual(headwords(db, { sort: 'rank', descending: true }), ['低', '高', '無']);
});
});
test('titles count distinct titles across every form and can filter and sort', () => {
withDb((db) => {
const seen = addWord(db, { headword: '見る', word: '見て', seen: 5 });
const seenAgain = addWord(db, { headword: '見る', word: '見た', seen: 5 });
const single = addWord(db, { headword: '海', seen: 50 });
addOccurrence(db, seen, 'A');
addOccurrence(db, seenAgain, 'B');
addOccurrence(db, single, 'C');
const page = getDictionaryWordPage(db, null, query());
assert.deepEqual(
page.rows.map((row) => [row.headword, row.titles]),
[
['海', 1],
['見る', 2],
],
);
const across = getDictionaryWordPage(db, null, query({ sort: 'titles', minTitles: 2 }));
assert.deepEqual(
across.rows.map((row) => [row.headword, row.titles]),
[['見る', 2]],
);
});
});
test('a word is stream only when every line, across its forms, came from a stream', () => {
withDb((db) => {
const streamed = addWord(db, { headword: '流', word: '流れ', seen: 30 });
addOccurrence(db, streamed, 'Stream A', 1, 'stream');
const mixed = addWord(db, { headword: '混', word: '混ぜ', seen: 20 });
const mixedFile = addWord(db, { headword: '混', word: '混ぜた', seen: 10 });
addOccurrence(db, mixed, 'Stream B', 1, 'stream');
addOccurrence(db, mixedFile, 'File A', 1, 'file');
addWord(db, { headword: '無', seen: 1 });
const streamOnly = Object.fromEntries(
getDictionaryWordPage(db, null, query()).rows.map((row) => [row.headword, row.streamOnly]),
);
// A word with no stored lines isn't blocked by streams.
assert.deepEqual(streamOnly, { 流: true, 混: false, 無: false });
});
});
test('word detail counts lines from streams across every form', () => {
withDb((db) => {
const streamed = addWord(db, { headword: '混', word: '混ぜ' });
const file = addWord(db, { headword: '混', word: '混ぜた' });
addOccurrence(db, streamed, 'Stream A', 3, 'stream');
addOccurrence(db, streamed, 'Stream B', 1, 'stream');
addOccurrence(db, file, 'File A', 2, 'file');
const detail = getWordDetail(db, file);
assert.equal(detail?.lineCount, 3);
assert.equal(detail?.streamLineCount, 2);
});
});
test('lines that can be mined come first in examples and search, newest first within each', () => {
withDb((db) => {
const wordId = addWord(db, { headword: '言う', word: '言った', reading: 'いった' });
addOccurrence(db, wordId, 'File old', 1, 'file');
addOccurrence(db, wordId, 'File new', 1, 'file');
addOccurrence(db, wordId, 'File untimed', 1, 'file', { startMs: null, endMs: null });
addOccurrence(db, wordId, 'Stream newest', 1, 'stream');
const expected = ['File new', 'File old', 'Stream newest', 'File untimed'];
assert.deepEqual(
getWordOccurrences(db, '言う', '言った', 'いった', 10, 0, true).map((row) => row.videoTitle),
expected,
);
assert.deepEqual(
searchSubtitleSentences(db, 'line').map((row) => row.videoTitle),
expected,
);
});
});
test('legacy rows with NULL text columns list, search, and open without failing', () => {
withDb((db) => {
const insert = db.prepare(
`INSERT INTO imm_words (headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, ?, 1700000000, 1700000000, 3)`,
);
const noHeadword = Number(insert.run(null, '猫', null).lastInsertRowid);
const noForm = Number(insert.run('犬', null, null).lastInsertRowid);
const { rows } = getDictionaryWordPage(db, null, query());
assert.deepEqual(rows.map((row) => [row.headword, row.forms[0]?.word, row.reading]).sort(), [
['犬', '犬', ''],
['猫', '猫', ''],
]);
// NULL must not become searchable text.
assert.deepEqual(headwords(db, { search: 'null' }), []);
for (const id of [noHeadword, noForm]) {
assert.equal(getWordDetail(db, id)?.frequency, 3);
assert.deepEqual(getWordAnimeAppearances(db, id), []);
assert.deepEqual(getSimilarWords(db, id), []);
}
});
});
test('maxSeen and total describe every page, for scaling bars and paging', () => {
withDb((db) => {
addWord(db, { headword: '一', seen: 100 });
addWord(db, { headword: '二', seen: 5 });
const page = getDictionaryWordPage(db, null, query({ limit: 1, offset: 1 }));
assert.equal(page.total, 2);
assert.equal(page.maxSeen, 100);
assert.deepEqual(
page.rows.map((row) => row.headword),
['二'],
);
});
});
test('the summary counts a word as known when any of its forms is known', () => {
withDb((db) => {
markLexicalDailyRollupsReady(db);
addWord(db, { headword: '見る', word: '見て', reading: 'みて' });
addWord(db, { headword: '海' });
assert.equal(getVocabularySummary(db, new Set(['見て'])).knownWordCount, 1);
assert.equal(
getDictionaryWordPage(db, new Set(['見て']), query()).rows.filter((row) => row.known).length,
1,
);
});
});
test('coverage counts seen and known words per frequency band, without names', () => {
withDb((db) => {
addWord(db, { headword: '一', rank: 500 });
addWord(db, { headword: '二', rank: 900 });
addWord(db, { headword: '三', rank: 4_000 });
addWord(db, { headword: '四', rank: 30_000 });
addWord(db, { headword: '田中', rank: 700, pos2: '固有名詞' });
addWord(db, { headword: '無', rank: null });
assert.deepEqual(getVocabularyCoverage(db, new Set(['一', '三', '田中'])), [
{ band: 'top1k', seen: 2, known: 1 },
{ band: 'top5k', seen: 1, known: 1 },
{ band: 'top10k', seen: 0, known: 0 },
{ band: 'top20k', seen: 0, known: 0 },
]);
});
});
test('word detail reports totals and a reading for the dictionary word, not one form', () => {
withDb((db) => {
const form = addWord(db, {
headword: '見る',
word: '見て',
reading: 'みて',
seen: 263,
rank: 36,
firstSeen: 1_700_000_500,
});
addWord(db, {
headword: '見る',
word: '見る',
reading: 'みる',
seen: 172,
rank: 36,
firstSeen: 1_700_000_100,
});
const detail = getWordDetail(db, form);
assert.equal(detail?.word, '見て');
assert.equal(detail?.frequency, 435);
assert.equal(detail?.dictionaryReading, 'みる');
assert.equal(detail?.frequencyRank, 36);
assert.equal(detail?.formCount, 2);
assert.equal(detail?.firstSeen, 1_700_000_100);
assert.equal(detail?.lastSeen, 1_700_000_500);
});
});
test('word appearances and examples can cover every form', () => {
withDb((db) => {
const first = addWord(db, { headword: '見る', word: '見て', reading: 'みて' });
const second = addWord(db, { headword: '見る', word: '見た', reading: 'みた' });
addOccurrence(db, first, 'A', 3);
addOccurrence(db, second, 'B', 2);
assert.deepEqual(
getWordAnimeAppearances(db, first).map((row) => [row.animeTitle, row.occurrenceCount]),
[
['A', 3],
['B', 2],
],
);
});
});
test("related words leave out a word's own forms and show one entry per word", () => {
withDb((db) => {
const form = addWord(db, { headword: '見る', word: '見て', reading: 'みて', seen: 10 });
addWord(db, { headword: '見る', word: '見た', reading: 'みた', seen: 9 });
addWord(db, { headword: '見える', word: '見えた', reading: 'みえた', seen: 4 });
addWord(db, { headword: '見える', word: '見える', reading: 'みえる', seen: 3 });
const similar = getSimilarWords(db, form);
assert.deepEqual(
similar.map((row) => [row.headword, row.frequency]),
[['見える', 7]],
);
assert.equal(similar[0]?.word, '見えた');
});
});
test('a reading match ranks a related word without choosing the form it shows', () => {
withDb((db) => {
const form = addWord(db, { headword: '見る', word: '見て', reading: 'みて', seen: 10 });
// Matches on the kanji with every form, and on the reading with its minor form.
const kanjiAndReading = addWord(db, {
headword: '見える',
word: '見えた',
reading: 'みえた',
seen: 9,
});
addWord(db, { headword: '見える', word: '見えて', reading: 'みて', seen: 2 });
// Matches only on the reading of its minor form, so its other forms must still count.
const readingOnly = addWord(db, {
headword: '満ちる',
word: '満ちた',
reading: 'みちた',
seen: 7,
});
addWord(db, { headword: '満ちる', word: '満ちて', reading: 'みて', seen: 1 });
// More seen, but no reading match, so it still ranks after both.
addWord(db, { headword: '見物', word: '見物', reading: 'けんぶつ', seen: 100 });
// A hidden form supplies neither the shown form nor the total.
const hidden = addWord(db, { headword: '満ちる', word: '満ち', reading: 'みち', seen: 50 });
db.prepare('UPDATE imm_words SET vocabulary_visible = 0 WHERE id = ?').run(hidden);
assert.deepEqual(
getSimilarWords(db, form).map((row) => [row.wordId, row.word, row.reading, row.frequency]),
[
[kanjiAndReading, '見えた', 'みえた', 11],
[readingOnly, '満ちた', 'みちた', 8],
[getIdOf(db, '見物'), '見物', 'けんぶつ', 100],
],
);
});
});
test('related words show the same form when a word has tied top forms', () => {
withDb((db) => {
const form = addWord(db, { headword: '見る', word: '見て', reading: 'みて', seen: 10 });
const first = addWord(db, { headword: '見える', word: '見えた', reading: 'みえた', seen: 5 });
addWord(db, { headword: '見える', word: '見えて', reading: 'みえて', seen: 5 });
assert.equal(getSimilarWords(db, form)[0]?.wordId, first);
});
});
test('related words leave out rows with a blank headword', () => {
withDb((db) => {
const form = addWord(db, { headword: '見る', word: '見て', reading: 'みて', seen: 10 });
for (const [index, headword] of ['', ' ', '\t', '\v', '\f', '　'].entries()) {
addWord(db, { headword, word: `空${index}`, reading: 'みて', seen: 50 });
}
addWord(db, { headword: '満ちる', word: '満ちて', reading: 'みて', seen: 1 });
assert.deepEqual(
getSimilarWords(db, form).map((row) => row.headword),
['満ちる'],
);
});
});
test('all-forms examples include legacy rows stored without a headword', () => {
withDb((db) => {
const insert = db.prepare(
`INSERT INTO imm_words (headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, ?, 1700000000, 1700000000, 1)`,
);
const titlesFor = (headword: string): string[] =>
getWordOccurrences(db, headword, '', '', 10, 0, true)
.map((row) => row.videoTitle)
.sort();
addOccurrence(db, Number(insert.run('猫', '猫', 'ねこ').lastInsertRowid), 'Headword');
addOccurrence(db, Number(insert.run(null, '猫', 'ねこ').lastInsertRowid), 'Null headword');
addOccurrence(db, Number(insert.run(' ', '猫', 'ねこ').lastInsertRowid), 'Blank headword');
addOccurrence(db, Number(insert.run(null, null, 'いぬ').lastInsertRowid), 'Reading only');
addOccurrence(db, Number(insert.run('犬', '猫', 'いぬ').lastInsertRowid), 'Other headword');
assert.deepEqual(titlesFor('猫'), ['Blank headword', 'Headword', 'Null headword']);
assert.deepEqual(titlesFor('いぬ'), ['Reading only']);
assert.deepEqual(titlesFor('犬'), ['Other headword']);
});
});
src/core/services/immersion-tracker/query-vocabulary-words.ts
+375
View File
@@ -0,0 +1,375 @@
import type {
StatsDictionaryWordPos,
StatsDictionaryWordQuery,
} from '../../../types/stats-http-contract';
import type {
DictionaryWordForm,
DictionaryWordPage,
DictionaryWordRow,
VocabularyCoverageBand,
VocabularyCoverageBandId,
} from '../../../types/stats-wire';
import type { DatabaseSync } from './sqlite';
import {
databaseWriteVersion,
DICTIONARY_WORD_KEY_SQL,
getExcludedWordIds,
isProperNounPos2,
mergeFormsByWord,
NO_LOCAL_FILE_SQL,
} from './query-lexical';
import { forEachIdChunk, makePlaceholders } from './query-shared';
import { nowMs } from './time';
const MAX_FORMS_PER_ROW = 8;
// Counting titles for every word walks every occurrence (about 900ms on 500k rows),
// and playback writes constantly, so that count tolerates being this stale.
const ALL_TITLE_COUNTS_MAX_AGE_MS = 5 * 60_000;
const COVERAGE_BAND_LIMITS: ReadonlyArray<{ band: VocabularyCoverageBandId; maxRank: number }> = [
{ band: 'top1k', maxRank: 1_000 },
{ band: 'top5k', maxRank: 5_000 },
{ band: 'top10k', maxRank: 10_000 },
{ band: 'top20k', maxRank: 20_000 },
];
const PART_OF_SPEECH_GROUPS: Record<StatsDictionaryWordPos, ReadonlySet<string>> = {
noun: new Set(['noun']),
verb: new Set(['verb']),
adjective: new Set(['i_adjective', 'na_adjective']),
};
const KANA_ONLY_TEXT = /^[\p{Script=Hiragana}\p{Script=Katakana}ーゝゞヽヾ]+$/u;
interface DictionaryWordFormRow {
id: number;
key: string;
word: string;
reading: string;
seen: number | null;
rank: number | null;
pos2: string | null;
partOfSpeech: string | null;
firstSeen: number | null;
lastSeen: number | null;
}
interface DictionaryWord {
key: string;
seen: number;
rank: number | null;
isName: boolean;
isKana: boolean;
firstSeen: number;
lastSeen: number;
searchText: string;
/** Most seen first. */
forms: DictionaryWordFormRow[];
}
const wordCache = new WeakMap<DatabaseSync, { version: string; words: DictionaryWord[] }>();
const allTitleCountsCache = new WeakMap<
DatabaseSync,
{ version: string; computedAtMs: number; counts: Map<string, number> }
>();
function toSeconds(value: number | null): number {
const numeric = Number(value ?? 0);
return Math.abs(numeric) >= 10_000_000_000 ? numeric / 1000 : numeric;
}
/** Lowercase hiragana, so a search matches regardless of kana type or case. */
function normalizeSearchText(text: string): string {
return text
.toLowerCase()
.replace(/[ァ-ヶ]/g, (char) => String.fromCharCode(char.charCodeAt(0) - 0x60));
}
function loadDictionaryWords(db: DatabaseSync): DictionaryWord[] {
const version = databaseWriteVersion(db);
const cached = wordCache.get(db);
if (cached?.version === version) return cached.words;
const excludedIds = getExcludedWordIds(db);
const exclusion =
excludedIds.length > 0 ? `AND id NOT IN (${makePlaceholders(excludedIds)})` : '';
const rows = db
.prepare(
`
SELECT id, ${DICTIONARY_WORD_KEY_SQL} AS key,
COALESCE(NULLIF(TRIM(word), ''), ${DICTIONARY_WORD_KEY_SQL}) AS word,
COALESCE(reading, '') AS reading, frequency AS seen,
frequency_rank AS rank, pos2, part_of_speech AS partOfSpeech,
first_seen AS firstSeen, last_seen AS lastSeen
FROM imm_words
WHERE vocabulary_visible = 1 ${exclusion}
ORDER BY frequency DESC, id
`,
)
.all(...excludedIds) as unknown as DictionaryWordFormRow[];
const byKey = new Map<string, DictionaryWord>();
for (const row of rows) {
if (!row.key) continue;
const firstSeen = toSeconds(row.firstSeen);
const lastSeen = toSeconds(row.lastSeen);
const isName = isProperNounPos2(row.pos2);
const entry = byKey.get(row.key);
if (entry) {
entry.seen += row.seen ?? 0;
entry.forms.push(row);
entry.firstSeen = Math.min(entry.firstSeen, firstSeen);
entry.lastSeen = Math.max(entry.lastSeen, lastSeen);
entry.isName &&= isName;
entry.searchText += `\n${row.word}\n${row.reading}`;
} else {
byKey.set(row.key, {
key: row.key,
seen: row.seen ?? 0,
rank: null,
isName,
isKana: KANA_ONLY_TEXT.test(row.key),
firstSeen,
lastSeen,
searchText: `${row.key}\n${row.word}\n${row.reading}`,
forms: [row],
});
}
}
const words = [...byKey.values()];
for (const word of words) {
word.searchText = normalizeSearchText(word.searchText);
word.rank = bestRank(word.forms);
}
wordCache.set(db, { version, words });
return words;
}
/** Rank from the word's non-name forms when it has any, so 白 ranks as the color. */
function bestRank(forms: DictionaryWordFormRow[]): number | null {
const nonNames = forms.filter((form) => !isProperNounPos2(form.pos2));
let best: number | null = null;
for (const form of nonNames.length > 0 ? nonNames : forms) {
if (form.rank !== null && form.rank > 0 && (best === null || form.rank < best)) {
best = form.rank;
}
}
return best;
}
function isKnown(word: DictionaryWord, knownWords: ReadonlySet<string> | null): boolean {
if (!knownWords) return false;
return knownWords.has(word.key) || word.forms.some((form) => knownWords.has(form.word));
}
function dictionaryForm(word: DictionaryWord): DictionaryWordFormRow | undefined {
return word.forms.find((form) => form.word === word.key);
}
function getAllTitleCounts(db: DatabaseSync): Map<string, number> {
const version = databaseWriteVersion(db);
const cached = allTitleCountsCache.get(db);
if (
cached &&
(cached.version === version || nowMs() - cached.computedAtMs < ALL_TITLE_COUNTS_MAX_AGE_MS)
) {
return cached.counts;
}
const rows = db
.prepare(
`
SELECT ${DICTIONARY_WORD_KEY_SQL} AS key, COUNT(DISTINCT sl.anime_id) AS titles
FROM imm_words
JOIN imm_word_line_occurrences o ON o.word_id = imm_words.id
JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
WHERE sl.anime_id IS NOT NULL AND vocabulary_visible = 1
GROUP BY key
`,
)
.all() as Array<{ key: string; titles: number }>;
const counts = new Map(rows.map((row) => [row.key, row.titles]));
allTitleCountsCache.set(db, { version, computedAtMs: nowMs(), counts });
return counts;
}
function getTitleCountsForWords(db: DatabaseSync, words: DictionaryWord[]): Map<string, number> {
const keyByWordId = new Map<number, string>();
for (const word of words) {
for (const form of word.forms) keyByWordId.set(form.id, word.key);
}
const animeByKey = new Map<string, Set<number>>();
forEachIdChunk([...keyByWordId.keys()], (chunk) => {
const rows = db
.prepare(
`
SELECT DISTINCT o.word_id AS wordId, sl.anime_id AS animeId
FROM imm_word_line_occurrences o
JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
WHERE o.word_id IN (${makePlaceholders(chunk)}) AND sl.anime_id IS NOT NULL
`,
)
.all(...chunk) as Array<{ wordId: number; animeId: number }>;
for (const row of rows) {
const key = keyByWordId.get(row.wordId);
if (key === undefined) continue;
const anime = animeByKey.get(key) ?? new Set<number>();
anime.add(row.animeId);
animeByKey.set(key, anime);
}
});
return new Map([...animeByKey].map(([key, anime]) => [key, anime.size]));
}
/** Words whose every stored line came from a stream. Words with no lines don't count. */
function getStreamOnlyWords(db: DatabaseSync, words: DictionaryWord[]): Set<string> {
const keyByWordId = new Map<number, string>();
for (const word of words) {
for (const form of word.forms) keyByWordId.set(form.id, word.key);
}
const withLines = new Set<string>();
const withFile = new Set<string>();
forEachIdChunk([...keyByWordId.keys()], (chunk) => {
const rows = db
.prepare(
`
SELECT o.word_id AS wordId, MIN(${NO_LOCAL_FILE_SQL}) AS allStreamed
FROM imm_word_line_occurrences o
JOIN imm_subtitle_lines l ON l.line_id = o.line_id
JOIN imm_videos v ON v.video_id = l.video_id
WHERE o.word_id IN (${makePlaceholders(chunk)})
GROUP BY o.word_id
`,
)
.all(...chunk) as Array<{ wordId: number; allStreamed: number }>;
for (const row of rows) {
const key = keyByWordId.get(row.wordId);
if (key === undefined) continue;
withLines.add(key);
if (row.allStreamed === 0) withFile.add(key);
}
});
return new Set([...withLines].filter((key) => !withFile.has(key)));
}
function compareWords(
left: DictionaryWord,
right: DictionaryWord,
query: StatsDictionaryWordQuery,
titleCounts: Map<string, number> | null,
): number {
const sign = query.descending ? -1 : 1;
const byText = left.key < right.key ? -1 : left.key > right.key ? 1 : 0;
switch (query.sort) {
case 'headword':
return byText * sign;
case 'rank':
// Unranked words go last in either direction.
if (left.rank === null || right.rank === null) {
return (left.rank === null ? 1 : 0) - (right.rank === null ? 1 : 0) || byText;
}
return (left.rank - right.rank) * sign || byText;
case 'titles':
return (
((titleCounts?.get(left.key) ?? 0) - (titleCounts?.get(right.key) ?? 0)) * sign ||
right.seen - left.seen ||
byText
);
case 'firstSeen':
return (left.firstSeen - right.firstSeen) * sign || byText;
case 'lastSeen':
return (left.lastSeen - right.lastSeen) * sign || byText;
case 'seen':
return (left.seen - right.seen) * sign || byText;
}
}
function toRow(
word: DictionaryWord,
knownWords: ReadonlySet<string> | null,
titleCounts: Map<string, number>,
streamOnlyWords: ReadonlySet<string>,
): DictionaryWordRow {
const dictionary = dictionaryForm(word);
const topForm = word.forms[0]!;
const reading = word.forms.find(
(form) => form.word === word.key && form.reading.trim() !== '',
)?.reading;
const forms = mergeFormsByWord<DictionaryWordForm>(
word.forms.map((form) => ({
wordId: form.id,
word: form.word,
reading: form.reading,
seen: form.seen ?? 0,
})),
);
return {
wordId: (dictionary ?? topForm).id,
headword: word.key,
reading: reading ?? '',
partOfSpeech: topForm.partOfSpeech,
rank: word.rank,
seen: word.seen,
titles: titleCounts.get(word.key) ?? 0,
firstSeen: word.firstSeen,
lastSeen: word.lastSeen,
known: isKnown(word, knownWords),
isName: word.isName,
streamOnly: streamOnlyWords.has(word.key),
formCount: forms.length,
forms: forms.slice(0, MAX_FORMS_PER_ROW),
};
}
/** One page of dictionary words, folded across their forms and filtered on the server. */
export function getDictionaryWordPage(
db: DatabaseSync,
knownWords: ReadonlySet<string> | null,
query: StatsDictionaryWordQuery,
): DictionaryWordPage {
const needle = query.search ? normalizeSearchText(query.search) : '';
const posGroup = query.partOfSpeech ? PART_OF_SPEECH_GROUPS[query.partOfSpeech] : null;
const needsAllTitleCounts = query.sort === 'titles' || query.minTitles !== undefined;
const allTitleCounts = needsAllTitleCounts ? getAllTitleCounts(db) : null;
const matches = loadDictionaryWords(db).filter((word) => {
if (query.hideNames && word.isName) return false;
if (query.hideKana && word.isKana) return false;
if (query.maxRank !== undefined && (word.rank === null || word.rank > query.maxRank)) {
return false;
}
if (posGroup && !posGroup.has(word.forms[0]?.partOfSpeech ?? '')) return false;
if (needle && !word.searchText.includes(needle)) return false;
if (query.minTitles !== undefined && (allTitleCounts?.get(word.key) ?? 0) < query.minTitles) {
return false;
}
return !(query.hideKnown && isKnown(word, knownWords));
});
matches.sort((left, right) => compareWords(left, right, query, allTitleCounts));
const pageWords = matches.slice(query.offset, query.offset + query.limit);
const titleCounts = allTitleCounts ?? getTitleCountsForWords(db, pageWords);
const streamOnlyWords = getStreamOnlyWords(db, pageWords);
return {
rows: pageWords.map((word) => toRow(word, knownWords, titleCounts, streamOnlyWords)),
total: matches.length,
maxSeen: matches.reduce((max, word) => Math.max(max, word.seen), 0),
};
}
/** How many seen words in each frequency band are known. Names don't count. */
export function getVocabularyCoverage(
db: DatabaseSync,
knownWords: ReadonlySet<string>,
): VocabularyCoverageBand[] {
const bands = COVERAGE_BAND_LIMITS.map(({ band }) => ({ band, seen: 0, known: 0 }));
for (const word of loadDictionaryWords(db)) {
if (word.isName || word.rank === null) continue;
const index = COVERAGE_BAND_LIMITS.findIndex(({ maxRank }) => word.rank! <= maxRank);
const band = bands[index];
if (!band) continue;
band.seen += 1;
if (isKnown(word, knownWords)) band.known += 1;
}
return bands;
}
src/core/services/immersion-tracker/types.ts
+7
View File
@@ -624,7 +624,14 @@ export interface WordDetailRow {
pos1: string | null;
pos2: string | null;
pos3: string | null;
/** Times seen across every form of this dictionary word. */
frequency: number;
frequencyRank: number | null;
dictionaryReading: string;
formCount: number;
forms: Array<{ wordId: number; word: string; reading: string; seen: number }>;
lineCount: number;
streamLineCount: number;
firstSeen: number;
lastSeen: number;
}