mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-11-10 04:46:07 -08:00
feat(stats): overhaul the Trends tab (#325)
This commit is contained in:
sudacode authored and 
GitHub committed
1 parent
f89f78b861
commit
269c0708cb
73 files changed
+6085
-3354
No files matched your search
@@ -155,15 +155,20 @@ const WATCH_TIME_PER_ANIME = [
|
||||
];
|
||||
|
||||
const TRENDS_DASHBOARD = {
|
||||
window: { startDay: 20_000, endDay: 20_000 },
|
||||
buckets: [{ label: 'Mar 1', startDay: 20_000, endDay: 20_000 }],
|
||||
summary: {
|
||||
current: { watchTimeMin: 25, words: 300, newWords: 12, cards: 5, episodes: 2, activeDays: 1 },
|
||||
previous: null,
|
||||
},
|
||||
activity: {
|
||||
watchTime: [{ label: 'Mar 1', value: 25 }],
|
||||
cards: [{ label: 'Mar 1', value: 5 }],
|
||||
words: [{ label: 'Mar 1', value: 300 }],
|
||||
sessions: [{ label: 'Mar 1', value: 3 }],
|
||||
episodes: [{ label: 'Mar 1', value: 2 }],
|
||||
},
|
||||
progress: {
|
||||
watchTime: [{ label: 'Mar 1', value: 25 }],
|
||||
sessions: [{ label: 'Mar 1', value: 3 }],
|
||||
words: [{ label: 'Mar 1', value: 300 }],
|
||||
newWords: [{ label: 'Mar 1', value: 12 }],
|
||||
cards: [{ label: 'Mar 1', value: 5 }],
|
||||
@@ -175,11 +180,19 @@ const TRENDS_DASHBOARD = {
|
||||
cardsPerHour: [{ label: 'Mar 1', value: 12 }],
|
||||
readingSpeed: [{ label: 'Mar 1', value: 180 }],
|
||||
},
|
||||
learning: {
|
||||
newWordsPerThousand: [{ label: 'Mar 1', value: null }],
|
||||
frequencyBands: [],
|
||||
},
|
||||
watchTimeByDayHour: [],
|
||||
librarySummary: [
|
||||
{
|
||||
title: 'Little Witch Academia',
|
||||
animeId: 1,
|
||||
videoId: 1,
|
||||
watchTimeMin: 25,
|
||||
videos: 1,
|
||||
episodesFinished: 1,
|
||||
sessions: 1,
|
||||
cards: 5,
|
||||
words: 300,
|
||||
@@ -189,16 +202,7 @@ const TRENDS_DASHBOARD = {
|
||||
lastWatched: 20_000,
|
||||
},
|
||||
],
|
||||
animeDaily: {
|
||||
watchTime: [{ epochDay: 20_000, animeTitle: 'Little Witch Academia', value: 25 }],
|
||||
episodes: [{ epochDay: 20_000, animeTitle: 'Little Witch Academia', value: 1 }],
|
||||
cards: [{ epochDay: 20_000, animeTitle: 'Little Witch Academia', value: 5 }],
|
||||
words: [{ epochDay: 20_000, animeTitle: 'Little Witch Academia', value: 300 }],
|
||||
},
|
||||
patterns: {
|
||||
watchTimeByDayOfWeek: [{ label: 'Sun', value: 25 }],
|
||||
watchTimeByHour: [{ label: '12:00', value: 25 }],
|
||||
},
|
||||
titleTimeline: [{ epochDay: 20_000, animeTitle: 'Little Witch Academia', value: 25 }],
|
||||
};
|
||||
|
||||
const ANIME_EPISODES = [
|
||||
@@ -227,6 +231,12 @@ const WORD_DETAIL = {
|
||||
pos2: '一般',
|
||||
pos3: null,
|
||||
frequency: 42,
|
||||
frequencyRank: 100,
|
||||
dictionaryReading: 'ねこ',
|
||||
formCount: 1,
|
||||
forms: [{ wordId: 1, word: '猫', reading: 'ねこ', seen: 42 }],
|
||||
lineCount: 40,
|
||||
streamLineCount: 0,
|
||||
firstSeen: Date.now() - 100_000,
|
||||
lastSeen: Date.now(),
|
||||
};
|
||||
@@ -313,11 +323,11 @@ function createMockTracker(
|
||||
}),
|
||||
getVocabularyChartData: async () => ({
|
||||
ready: true,
|
||||
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
}),
|
||||
getDictionaryWords: async () => ({ rows: [], total: 0, maxSeen: 0 }),
|
||||
getVocabularyCoverage: async () => [],
|
||||
getStatsExcludedWords: async () => [],
|
||||
replaceStatsExcludedWords: async () => {},
|
||||
getKanjiStats: async () => KANJI_STATS,
|
||||
@@ -674,9 +684,16 @@ const CLAMPED_QUERIES: Array<{
|
||||
];
|
||||
|
||||
const TRENDS_DASHBOARD_QUERIES = [
|
||||
{ query: 'range=90d&groupBy=month', expectedArgs: ['90d', 'month', true] },
|
||||
{ query: 'range=weird&groupBy=year', expectedArgs: ['30d', 'day', true] },
|
||||
{ query: 'range=30d&groupBy=day&fillEmpty=false', expectedArgs: ['30d', 'day', false] },
|
||||
{ query: 'range=90d&groupBy=month', expectedArgs: ['90d', 'month', true, null] },
|
||||
{ query: 'range=90d&groupBy=week', expectedArgs: ['90d', 'week', true, null] },
|
||||
{ query: 'range=weird&groupBy=year', expectedArgs: ['30d', 'day', true, null] },
|
||||
{ query: 'range=30d&groupBy=day&fillEmpty=false', expectedArgs: ['30d', 'day', false, null] },
|
||||
{
|
||||
query: 'range=custom&fromDay=20000&toDay=20010',
|
||||
expectedArgs: [{ startDay: 20000, endDay: 20010 }, 'day', true, null],
|
||||
},
|
||||
{ query: 'range=custom&fromDay=20010&toDay=20000', expectedArgs: ['30d', 'day', true, null] },
|
||||
{ query: 'range=30d&title=Bocchi%20%26%20Co', expectedArgs: ['30d', 'day', true, 'Bocchi & Co'] },
|
||||
];
|
||||
|
||||
const MALFORMED_IDS = [
|
||||
@@ -991,7 +1008,7 @@ describe('stats server API routes', () => {
|
||||
});
|
||||
|
||||
for (const { query, expectedArgs } of TRENDS_DASHBOARD_QUERIES) {
|
||||
it(`GET /api/stats/trends/dashboard?${query} requests ${expectedArgs.join('/')}`, async () => {
|
||||
it(`GET /api/stats/trends/dashboard?${query} requests ${JSON.stringify(expectedArgs)}`, async () => {
|
||||
let seenArgs: unknown[] = [];
|
||||
const app = createStatsApp(
|
||||
createMockTracker({
|
||||
@@ -1008,6 +1025,69 @@ describe('stats server API routes', () => {
|
||||
});
|
||||
}
|
||||
|
||||
it('GET /api/stats/vocabulary/words passes a parsed query and ignores unknown values', async () => {
|
||||
const seenQueries: unknown[] = [];
|
||||
const app = createStatsApp(
|
||||
createMockTracker({
|
||||
getDictionaryWords: async (_knownWords: unknown, query: unknown) => {
|
||||
seenQueries.push(query);
|
||||
return { rows: [], total: 0, maxSeen: 0 };
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await app.request(
|
||||
'/api/stats/vocabulary/words?search=%20%E7%8C%AB%20&sort=rank&order=asc&hideKnown=true&hideNames=true&pos=verb&maxRank=10000&minTitles=2&limit=25&offset=50',
|
||||
);
|
||||
await app.request('/api/stats/vocabulary/words?sort=bogus&pos=bogus&maxRank=-1&limit=99999');
|
||||
|
||||
assert.deepEqual(seenQueries, [
|
||||
{
|
||||
search: '猫',
|
||||
sort: 'rank',
|
||||
descending: false,
|
||||
hideKnown: true,
|
||||
hideNames: true,
|
||||
hideKana: false,
|
||||
partOfSpeech: 'verb',
|
||||
maxRank: 10000,
|
||||
minTitles: 2,
|
||||
limit: 25,
|
||||
offset: 50,
|
||||
},
|
||||
{
|
||||
search: undefined,
|
||||
sort: 'seen',
|
||||
descending: true,
|
||||
hideKnown: false,
|
||||
hideNames: false,
|
||||
hideKana: false,
|
||||
partOfSpeech: undefined,
|
||||
maxRank: undefined,
|
||||
minTitles: undefined,
|
||||
limit: 200,
|
||||
offset: 0,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('GET /api/stats/sessions narrows to a valid day range and ignores a reversed one', async () => {
|
||||
const seenRanges: unknown[] = [];
|
||||
const app = createStatsApp(
|
||||
createMockTracker({
|
||||
getSessionSummaries: async (_limit?: number, dayRange?: unknown) => {
|
||||
seenRanges.push(dayRange);
|
||||
return SESSION_SUMMARIES;
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await app.request('/api/stats/sessions?limit=500&fromDay=20000&toDay=20006');
|
||||
await app.request('/api/stats/sessions?fromDay=20006&toDay=20000');
|
||||
|
||||
assert.deepEqual(seenRanges, [{ startDay: 20000, endDay: 20006 }, undefined]);
|
||||
});
|
||||
|
||||
it('GET /api/stats/vocabulary/occurrences returns recent occurrence rows for a word', async () => {
|
||||
let seenArgs: unknown[] = [];
|
||||
const app = createStatsApp(
|
||||
@@ -1026,7 +1106,7 @@ describe('stats server API routes', () => {
|
||||
const body = await res.json();
|
||||
assert.ok(Array.isArray(body));
|
||||
assert.equal(body[0].animeTitle, 'Little Witch Academia');
|
||||
assert.deepEqual(seenArgs, ['猫', '猫', 'ねこ', 500, 25]);
|
||||
assert.deepEqual(seenArgs, ['猫', '猫', 'ねこ', 500, 25, false]);
|
||||
});
|
||||
|
||||
it('GET /api/stats/kanji/occurrences returns recent occurrence rows for a kanji', async () => {
|
||||
@@ -1056,6 +1136,33 @@ describe('stats server API routes', () => {
|
||||
assert.equal(res.status, 400);
|
||||
});
|
||||
|
||||
it('GET /api/stats/vocabulary/occurrences finds all forms of legacy words missing text', async () => {
|
||||
const seenHeadwords: unknown[] = [];
|
||||
const app = createStatsApp(
|
||||
createMockTracker({
|
||||
getWordOccurrences: async (...args: unknown[]) => {
|
||||
seenHeadwords.push(args[0]);
|
||||
return OCCURRENCES;
|
||||
},
|
||||
}),
|
||||
);
|
||||
const request = (params: string) =>
|
||||
app.request(`/api/stats/vocabulary/occurrences?allForms=true&${params}`);
|
||||
|
||||
assert.equal((await request('headword=&word=%E7%8C%AB&reading=')).status, 200);
|
||||
assert.equal((await request('headword=%E7%8A%AC&word=&reading=')).status, 200);
|
||||
assert.equal((await request('headword=&word=&reading=%E3%81%AD%E3%81%93')).status, 200);
|
||||
assert.deepEqual(seenHeadwords, ['猫', '犬', 'ねこ']);
|
||||
|
||||
assert.equal((await request('headword=%20&word=&reading=')).status, 400);
|
||||
});
|
||||
|
||||
it('GET /api/stats/vocabulary/occurrences needs a headword and form for an exact lookup', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
const res = await app.request('/api/stats/vocabulary/occurrences?headword=&word=%E7%8C%AB');
|
||||
assert.equal(res.status, 400);
|
||||
});
|
||||
|
||||
it('PUT /api/stats/excluded-words replaces tracker exclusion rows', async () => {
|
||||
let seenWords: unknown = null;
|
||||
const app = createStatsApp(
|
||||
|
||||
@@ -51,6 +51,16 @@ import {
|
||||
getSessionWordsByLine,
|
||||
} from './immersion-tracker/query-sessions';
|
||||
import { getTrendsDashboard } from './immersion-tracker/query-trends';
|
||||
import {
|
||||
getDictionaryWordPage,
|
||||
getVocabularyCoverage,
|
||||
} from './immersion-tracker/query-vocabulary-words';
|
||||
import type {
|
||||
StatsDayRange,
|
||||
StatsDictionaryWordQuery,
|
||||
StatsTrendGroupBy,
|
||||
StatsTrendRange,
|
||||
} from '../../types/stats-http-contract';
|
||||
import {
|
||||
getKanjiAnimeAppearances,
|
||||
getKanjiDetail,
|
||||
@@ -618,8 +628,11 @@ export class ImmersionTrackerService {
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
async getSessionSummaries(limit = 50): Promise<SessionSummaryQueryRow[]> {
|
||||
return getSessionSummaries(this.db, limit);
|
||||
async getSessionSummaries(
|
||||
limit = 50,
|
||||
dayRange?: StatsDayRange,
|
||||
): Promise<SessionSummaryQueryRow[]> {
|
||||
return getSessionSummaries(this.db, limit, dayRange);
|
||||
}
|
||||
|
||||
async getSessionTimeline(sessionId: number, limit?: number): Promise<SessionTimelineRow[]> {
|
||||
@@ -673,11 +686,12 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
|
||||
async getTrendsDashboard(
|
||||
range: '7d' | '30d' | '90d' | '365d' | 'all' = '30d',
|
||||
groupBy: 'day' | 'month' = 'day',
|
||||
range: StatsTrendRange = '30d',
|
||||
groupBy: StatsTrendGroupBy = 'day',
|
||||
fillEmptyBuckets = true,
|
||||
title: string | null = null,
|
||||
) {
|
||||
return getTrendsDashboard(this.db, range, groupBy, fillEmptyBuckets);
|
||||
return getTrendsDashboard(this.db, range, groupBy, fillEmptyBuckets, title);
|
||||
}
|
||||
|
||||
async getVocabularyStats(limit = 100, excludePos?: string[]): Promise<VocabularyStatsRow[]> {
|
||||
@@ -703,6 +717,17 @@ export class ImmersionTrackerService {
|
||||
return getVocabularyChartData(this.db);
|
||||
}
|
||||
|
||||
async getDictionaryWords(
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
query: StatsDictionaryWordQuery,
|
||||
) {
|
||||
return getDictionaryWordPage(this.db, knownWords, query);
|
||||
}
|
||||
|
||||
async getVocabularyCoverage(knownWords: ReadonlySet<string>) {
|
||||
return getVocabularyCoverage(this.db, knownWords);
|
||||
}
|
||||
|
||||
async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {
|
||||
return getStatsExcludedWords(this.db);
|
||||
}
|
||||
@@ -754,8 +779,9 @@ export class ImmersionTrackerService {
|
||||
reading: string,
|
||||
limit = 100,
|
||||
offset = 0,
|
||||
allForms = false,
|
||||
): Promise<WordOccurrenceRow[]> {
|
||||
return getWordOccurrences(this.db, headword, word, reading, limit, offset);
|
||||
return getWordOccurrences(this.db, headword, word, reading, limit, offset, allForms);
|
||||
}
|
||||
|
||||
async getKanjiOccurrences(kanji: string, limit = 100, offset = 0): Promise<KanjiOccurrenceRow[]> {
|
||||
|
||||
@@ -704,7 +704,7 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => {
|
||||
total_tokens_seen, total_cards
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
).run(Math.floor(dayOneStart / 86_400_000), videoId, 1, 30, 10, 120, 2);
|
||||
).run(getLocalEpochDay(db, dayOneStart), videoId, 1, 30, 10, 120, 2);
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
@@ -713,7 +713,7 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => {
|
||||
total_tokens_seen, total_cards
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
).run(Math.floor(dayTwoStart / 86_400_000), videoId, 1, 45, 10, 140, 3);
|
||||
).run(getLocalEpochDay(db, dayTwoStart), videoId, 1, 45, 10, 140, 3);
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
@@ -739,18 +739,17 @@ test('getTrendsDashboard returns chart-ready aggregated series', () => {
|
||||
assert.equal(dashboard.activity.watchTime[0]?.value, 30);
|
||||
assert.equal(dashboard.progress.watchTime[1]?.value, 75);
|
||||
assert.equal(dashboard.progress.lookups[1]?.value, 18);
|
||||
assert.equal(dashboard.ratios.lookupsPerHundred[0]?.value, +((8 / 120) * 100).toFixed(1));
|
||||
assert.equal(dashboard.ratios.cardsPerHour[0]?.value, +(2 / (30 / 60)).toFixed(1));
|
||||
assert.equal(dashboard.ratios.cardsPerHour[1]?.value, +(3 / (45 / 60)).toFixed(1));
|
||||
assert.equal(dashboard.ratios.cardsPerHour[1]?.value, +(5 / (75 / 60)).toFixed(1));
|
||||
assert.equal(dashboard.ratios.readingSpeed[0]?.value, +(120 / 30).toFixed(1));
|
||||
assert.equal(dashboard.ratios.readingSpeed[1]?.value, +(140 / 45).toFixed(1));
|
||||
assert.equal(dashboard.ratios.readingSpeed[1]?.value, +(260 / 75).toFixed(1));
|
||||
assert.equal(dashboard.librarySummary[0]?.title, 'Trend Dashboard Anime');
|
||||
assert.deepEqual(
|
||||
dashboard.animeDaily.watchTime.map((point) => point.value).sort((a, b) => a - b),
|
||||
dashboard.titleTimeline.map((point) => point.value).sort((a, b) => a - b),
|
||||
[30, 45],
|
||||
);
|
||||
assert.equal(
|
||||
dashboard.patterns.watchTimeByDayOfWeek.reduce((sum, point) => sum + point.value, 0),
|
||||
dashboard.watchTimeByDayHour.flat().reduce((sum, minutes) => sum + minutes, 0),
|
||||
75,
|
||||
);
|
||||
} finally {
|
||||
@@ -822,7 +821,7 @@ test('getTrendsDashboard redacts legacy Jellyfin stream titles', () => {
|
||||
|
||||
const dashboard = getTrendsDashboard(db, 'all', 'day');
|
||||
const titles = [
|
||||
...dashboard.animeDaily.watchTime.map((point) => point.animeTitle),
|
||||
...dashboard.titleTimeline.map((point) => point.animeTitle),
|
||||
...dashboard.librarySummary.map((row) => row.title),
|
||||
];
|
||||
|
||||
@@ -4751,7 +4750,7 @@ test('deleteSession removes zero-session media from library and trends', () => {
|
||||
|
||||
const trends = getTrendsDashboard(db, 'all', 'day');
|
||||
assert.deepEqual(trends.activity.watchTime, []);
|
||||
assert.deepEqual(trends.activity.sessions, []);
|
||||
assert.deepEqual(trends.activity.cards, []);
|
||||
|
||||
const dailyRollups = getDailyRollups(db, 30);
|
||||
const monthlyRollups = getMonthlyRollups(db, 30);
|
||||
@@ -5073,7 +5072,7 @@ test('getTrendsDashboard rollup word metrics keep persisted totals over partial
|
||||
total_tokens_seen, total_cards
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
).run(Math.floor(row.start / 86_400_000), videoId, 1, 10, 1, row.rawWords, 0);
|
||||
).run(getLocalEpochDay(db, row.start), videoId, 1, 10, 1, row.rawWords, 0);
|
||||
}
|
||||
|
||||
const dashboard = getTrendsDashboard(db, 'all', 'day');
|
||||
@@ -5085,13 +5084,8 @@ test('getTrendsDashboard rollup word metrics keep persisted totals over partial
|
||||
dashboard.progress.words.map((point) => point.value),
|
||||
[10, 30],
|
||||
);
|
||||
assert.equal(dashboard.ratios.lookupsPerHundred[0]?.value, 200);
|
||||
assert.equal(dashboard.librarySummary[0]?.words, 30);
|
||||
assert.equal(dashboard.librarySummary[0]?.lookupsPerHundred, 33.3);
|
||||
assert.equal(
|
||||
dashboard.animeDaily.words.reduce((sum, point) => sum + point.value, 0),
|
||||
30,
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
getLexicalDailyRollups,
|
||||
rebuildLexicalDailyRollups,
|
||||
} from './lexical-rollups';
|
||||
import { getTrendsDashboard } from './query-trends';
|
||||
import {
|
||||
getVocabularyChartData,
|
||||
getVocabularySummary,
|
||||
@@ -262,7 +261,7 @@ test('lexical rollup rebuild tolerates nullable legacy vocabulary text', () => {
|
||||
assert.equal(areLexicalDailyRollupsReady(db), true);
|
||||
assert.equal(getVocabularySummary(db, null).uniqueWords, 1);
|
||||
assert.equal(getVocabularySummary(db, new Set(['猫'])).knownWordCount, 1);
|
||||
assert.equal(getVocabularyChartData(db).topWords[0]?.headword, '猫');
|
||||
assert.equal(getVocabularyChartData(db).newWordsTimeline[0]?.wordCount, 1);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
@@ -593,7 +592,7 @@ test('imm_words persists vocabulary visibility for rollup maintenance', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary charts use complete top-word and lexical rollup data', () => {
|
||||
test('vocabulary charts count every word in new-word history', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
@@ -611,8 +610,6 @@ test('vocabulary charts use complete top-word and lexical rollup data', () => {
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.equal(charts.topWords[0]?.headword, '語500');
|
||||
assert.equal(charts.topWords[0]?.frequency, 10_000);
|
||||
assert.equal(charts.newWordsTimeline[0]?.wordCount, 501);
|
||||
} finally {
|
||||
db.close();
|
||||
@@ -620,41 +617,6 @@ test('vocabulary charts use complete top-word and lexical rollup data', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary charts find full top-word sets beyond excluded and name rows', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, pos2, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, '', ?, 1700000000, 1700000000, ?)`,
|
||||
);
|
||||
const exclusions = [];
|
||||
for (let index = 0; index < 100; index += 1) {
|
||||
const headword = `語${index}`;
|
||||
insertWord.run(
|
||||
headword,
|
||||
headword,
|
||||
index < 80 && index >= 60 ? '固有名詞' : '一般',
|
||||
100 - index,
|
||||
);
|
||||
if (index < 60) exclusions.push({ headword, word: headword, reading: '' });
|
||||
}
|
||||
replaceStatsExcludedWords(db, exclusions);
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.equal(charts.topWords.length, 12);
|
||||
assert.equal(charts.topWords[0]?.headword, '語60');
|
||||
assert.equal(charts.topWordsWithoutNames.length, 12);
|
||||
assert.equal(charts.topWordsWithoutNames[0]?.headword, '語80');
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary charts handle exclusion lists above one SQLite variable batch', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
@@ -674,7 +636,6 @@ test('vocabulary charts handle exclusion lists above one SQLite variable batch',
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.deepEqual(charts.topWords, []);
|
||||
assert.deepEqual(charts.newWordsTimeline, []);
|
||||
} finally {
|
||||
db.close();
|
||||
@@ -697,24 +658,3 @@ test('lexical rollup rebuild preserves the original error when rollback also fai
|
||||
|
||||
assert.throws(() => rebuildLexicalDailyRollups(db), originalError);
|
||||
});
|
||||
|
||||
test('trends read historical new-word buckets from lexical rollups', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES ('海', '海', 'うみ', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
db.prepare(`UPDATE imm_lexical_daily_rollups SET word_count = 9`).run();
|
||||
|
||||
const dashboard = getTrendsDashboard(db, 'all', 'day', false);
|
||||
|
||||
assert.equal(dashboard.progress.newWords[0]?.value, 9);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { TrendFrequencyBand } from '../../../types/stats-wire';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import type {
|
||||
KanjiAnimeAppearanceRow,
|
||||
@@ -19,17 +20,44 @@ import type {
|
||||
import { forEachIdChunk, fromDbTimestamp, makePlaceholders, toDbTimestamp } from './query-shared';
|
||||
import { nowMs } from './time';
|
||||
import { narrowFullwidthKana, widenHalfwidthKana } from '../../utils/halfwidth-kana';
|
||||
import {
|
||||
areLexicalDailyRollupsReady,
|
||||
getLexicalDailyRollups,
|
||||
localEpochDaySql,
|
||||
} from './lexical-rollups';
|
||||
import { areLexicalDailyRollupsReady, localEpochDaySql } from './lexical-rollups';
|
||||
import { isVocabularyStatsRowVisible } from './vocabulary-visibility';
|
||||
|
||||
// Merged tokens join MeCab tags, so a name can be stored as 固有名詞|一般 or 固有名詞|接尾.
|
||||
const PROPER_NOUN_TAG = '固有名詞';
|
||||
const NOT_A_NAME_SQL = `COALESCE(pos2, '') NOT LIKE '%${PROPER_NOUN_TAG}%'`;
|
||||
|
||||
/** The dictionary word an `imm_words` row belongs to. Legacy rows can lack a headword or form. */
|
||||
export const DICTIONARY_WORD_KEY_SQL = `COALESCE(NULLIF(TRIM(headword), ''), NULLIF(TRIM(word), ''), TRIM(reading))`;
|
||||
|
||||
/** True for lines from a stream: only local videos have a path to cut clips from. */
|
||||
export const NO_LOCAL_FILE_SQL = `TRIM(COALESCE(v.source_path, '')) = ''`;
|
||||
|
||||
// Sorts lines that can be mined ahead of those that can't, because newest-first alone
|
||||
// lets a run of recent streams bury every line with a usable clip.
|
||||
// Mirrors getStatsMineCardUnavailableReason in the stats client.
|
||||
const MINEABLE_LINES_FIRST_SQL = `
|
||||
CASE
|
||||
WHEN NOT (${NO_LOCAL_FILE_SQL})
|
||||
AND l.segment_start_ms IS NOT NULL
|
||||
AND l.segment_end_ms > l.segment_start_ms
|
||||
THEN 0
|
||||
ELSE 1
|
||||
END`;
|
||||
|
||||
export function isProperNounPos2(pos2: string | null | undefined): boolean {
|
||||
return pos2?.includes(PROPER_NOUN_TAG) ?? false;
|
||||
}
|
||||
|
||||
export interface NewWordFirstSeenRow {
|
||||
firstSeenDay: number;
|
||||
firstSeenDayWithoutNames: number | null;
|
||||
band: TrendFrequencyBand;
|
||||
wordCount: number;
|
||||
}
|
||||
|
||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
|
||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
|
||||
const VOCABULARY_CHART_LIMIT = 12;
|
||||
const VOCABULARY_CHART_PAGE_SIZE = 100;
|
||||
const EXCLUSION_ALIAS_BATCH_SIZE = 300;
|
||||
const VOCABULARY_SUMMARY_SCAN_BATCH_SIZE = 5_000;
|
||||
const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
|
||||
@@ -38,8 +66,6 @@ const KANJI_PATTERN = /\p{Script=Han}/gu;
|
||||
|
||||
export interface VocabularyChartData {
|
||||
ready: boolean;
|
||||
topWords: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||
topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||
newWordsTimeline: Array<{ epochDay: number; wordCount: number }>;
|
||||
newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>;
|
||||
}
|
||||
@@ -161,111 +187,137 @@ function getWordAnimeCounts(db: DatabaseSync, wordIds: number[]): Map<number, nu
|
||||
}
|
||||
|
||||
/**
|
||||
* Chart data is intentionally independent of the paginated vocabulary tables.
|
||||
* Top words use the frequency index; new-word history reads permanent daily
|
||||
* lexical rollups rather than loading every vocabulary row into the dashboard.
|
||||
* New-word history is a SQL aggregate over first-seen dates rather than loading
|
||||
* every vocabulary row into the dashboard.
|
||||
*/
|
||||
export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData {
|
||||
const ready = areLexicalDailyRollupsReady(db);
|
||||
const excludedAliases = new Set(
|
||||
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
|
||||
);
|
||||
const isExcluded = (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>): boolean =>
|
||||
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias));
|
||||
const topWords = getTopVocabularyChartWords(db, isExcluded);
|
||||
const rollups = ready ? getLexicalDailyRollups(db) : [];
|
||||
const timeline = new Map(rollups.map((row) => [row.epochDay, { ...row }]));
|
||||
if (excludedAliases.size > 0 && ready) {
|
||||
const aliases = [...excludedAliases];
|
||||
const excludedRows = new Map<
|
||||
number,
|
||||
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
|
||||
wordId: number;
|
||||
epochDay: number;
|
||||
}
|
||||
>();
|
||||
for (let offset = 0; offset < aliases.length; offset += EXCLUSION_ALIAS_BATCH_SIZE) {
|
||||
const batch = aliases.slice(offset, offset + EXCLUSION_ALIAS_BATCH_SIZE);
|
||||
const placeholders = batch.map(() => '?').join(', ');
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id AS wordId, headword, word, reading, pos2,
|
||||
${localEpochDaySql('first_seen')} AS epochDay
|
||||
FROM imm_words
|
||||
WHERE vocabulary_visible = 1
|
||||
AND (headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}))
|
||||
`,
|
||||
)
|
||||
.all(...batch, ...batch, ...batch) as Array<
|
||||
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
|
||||
wordId: number;
|
||||
epochDay: number;
|
||||
}
|
||||
>;
|
||||
for (const row of rows) excludedRows.set(row.wordId, row);
|
||||
}
|
||||
for (const word of excludedRows.values()) {
|
||||
if (!isExcluded(word)) continue;
|
||||
const rollup = timeline.get(word.epochDay);
|
||||
if (!rollup) continue;
|
||||
rollup.wordCount -= 1;
|
||||
if (word.pos2 !== '固有名詞') rollup.wordCountWithoutNames -= 1;
|
||||
const firstSeenRows = ready ? getNewWordFirstSeenRows(db) : [];
|
||||
const timeline = new Map<number, number>();
|
||||
const timelineWithoutNames = new Map<number, number>();
|
||||
for (const row of firstSeenRows) {
|
||||
timeline.set(row.firstSeenDay, (timeline.get(row.firstSeenDay) ?? 0) + row.wordCount);
|
||||
if (row.firstSeenDayWithoutNames !== null) {
|
||||
timelineWithoutNames.set(
|
||||
row.firstSeenDayWithoutNames,
|
||||
(timelineWithoutNames.get(row.firstSeenDayWithoutNames) ?? 0) + row.wordCount,
|
||||
);
|
||||
}
|
||||
}
|
||||
return {
|
||||
ready,
|
||||
topWords: topWords.all.map((word) => ({
|
||||
wordId: word.wordId,
|
||||
headword: vocabularyDisplayHeadword(word),
|
||||
frequency: word.frequency,
|
||||
})),
|
||||
topWordsWithoutNames: topWords.withoutNames.map((word) => ({
|
||||
wordId: word.wordId,
|
||||
headword: vocabularyDisplayHeadword(word),
|
||||
frequency: word.frequency,
|
||||
})),
|
||||
newWordsTimeline: [...timeline.values()]
|
||||
.filter((row) => row.wordCount > 0)
|
||||
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCount })),
|
||||
newWordsTimelineWithoutNames: [...timeline.values()]
|
||||
.filter((row) => row.wordCountWithoutNames > 0)
|
||||
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCountWithoutNames })),
|
||||
newWordsTimeline: toNewWordTimeline(timeline),
|
||||
newWordsTimelineWithoutNames: toNewWordTimeline(timelineWithoutNames),
|
||||
};
|
||||
}
|
||||
|
||||
function getTopVocabularyChartWords(
|
||||
db: DatabaseSync,
|
||||
isExcluded: (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>) => boolean,
|
||||
): { all: VocabularyStatsRow[]; withoutNames: VocabularyStatsRow[] } {
|
||||
const stmt = db.prepare(`
|
||||
SELECT id AS wordId, headword, word, reading,
|
||||
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
|
||||
frequency, frequency_rank AS frequencyRank,
|
||||
first_seen AS firstSeen, last_seen AS lastSeen,
|
||||
0 AS animeCount
|
||||
FROM imm_words
|
||||
ORDER BY frequency DESC, id
|
||||
LIMIT ? OFFSET ?
|
||||
`);
|
||||
const all: VocabularyStatsRow[] = [];
|
||||
const withoutNames: VocabularyStatsRow[] = [];
|
||||
let offset = 0;
|
||||
function toNewWordTimeline(byDay: Map<number, number>): VocabularyChartData['newWordsTimeline'] {
|
||||
return [...byDay.entries()]
|
||||
.sort(([left], [right]) => left - right)
|
||||
.map(([epochDay, wordCount]) => ({ epochDay, wordCount }));
|
||||
}
|
||||
|
||||
while (all.length < VOCABULARY_CHART_LIMIT || withoutNames.length < VOCABULARY_CHART_LIMIT) {
|
||||
const page = stmt.all(VOCABULARY_CHART_PAGE_SIZE, offset) as VocabularyStatsRow[];
|
||||
if (page.length === 0) break;
|
||||
for (const word of page) {
|
||||
if (!isVocabularyStatsRowVisible(word) || isExcluded(word)) continue;
|
||||
if (all.length < VOCABULARY_CHART_LIMIT) all.push(word);
|
||||
if (word.pos2 !== '固有名詞' && withoutNames.length < VOCABULARY_CHART_LIMIT) {
|
||||
withoutNames.push(word);
|
||||
/** Word rows matching the user's stats exclusions, using the same aliases as the summary. */
|
||||
export function getExcludedWordIds(db: DatabaseSync): number[] {
|
||||
const excludedAliases = new Set(
|
||||
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
|
||||
);
|
||||
const aliases = [...excludedAliases];
|
||||
const ids = new Set<number>();
|
||||
for (let offset = 0; offset < aliases.length; offset += EXCLUSION_ALIAS_BATCH_SIZE) {
|
||||
const batch = aliases.slice(offset, offset + EXCLUSION_ALIAS_BATCH_SIZE);
|
||||
const placeholders = batch.map(() => '?').join(', ');
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id AS wordId, headword, word, reading
|
||||
FROM imm_words
|
||||
WHERE vocabulary_visible = 1
|
||||
AND (headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}))
|
||||
`,
|
||||
)
|
||||
.all(...batch, ...batch, ...batch) as Array<
|
||||
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'> & { wordId: number }
|
||||
>;
|
||||
for (const row of rows) {
|
||||
if (excludedVocabularyAliases(row).some((alias) => excludedAliases.has(alias))) {
|
||||
ids.add(row.wordId);
|
||||
}
|
||||
}
|
||||
offset += page.length;
|
||||
}
|
||||
return [...ids];
|
||||
}
|
||||
|
||||
return { all, withoutNames };
|
||||
// The aggregate below scans every word (about 200ms on 68k rows), and the Trends tab
|
||||
// runs it on each control change. Results stay valid until something writes:
|
||||
// total_changes covers this connection, data_version covers other connections.
|
||||
const newWordFirstSeenCache = new WeakMap<
|
||||
DatabaseSync,
|
||||
{ version: string; rows: NewWordFirstSeenRow[] }
|
||||
>();
|
||||
|
||||
export function databaseWriteVersion(db: DatabaseSync): string {
|
||||
const { changes } = db.prepare('SELECT total_changes() AS changes').get() as { changes: number };
|
||||
const { data_version: dataVersion } = db.prepare('PRAGMA data_version').get() as {
|
||||
data_version: number;
|
||||
};
|
||||
return `${changes}:${dataVersion}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* When each dictionary word was first seen, folded across its surface forms:
|
||||
* `imm_words` stores 言う and 言った as separate rows, so counting rows would count
|
||||
* conjugations as new words. Rows group identical (day, day without names, band)
|
||||
* results so the caller gets a small aggregate rather than one row per word.
|
||||
* `firstSeenDayWithoutNames` and `band` ignore proper-noun forms, and the day is
|
||||
* null when every form of the word is a proper noun.
|
||||
*/
|
||||
export function getNewWordFirstSeenRows(db: DatabaseSync): NewWordFirstSeenRow[] {
|
||||
const version = databaseWriteVersion(db);
|
||||
const cached = newWordFirstSeenCache.get(db);
|
||||
if (cached?.version === version) return cached.rows;
|
||||
|
||||
const excludedIds = getExcludedWordIds(db);
|
||||
const exclusion =
|
||||
excludedIds.length > 0 ? `AND id NOT IN (${makePlaceholders(excludedIds)})` : '';
|
||||
const seconds = `CASE
|
||||
WHEN ABS(CAST(first_seen AS REAL)) >= 10000000000 THEN CAST(first_seen AS REAL) / 1000
|
||||
ELSE CAST(first_seen AS REAL)
|
||||
END`;
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
WITH dictionary_words AS (
|
||||
SELECT
|
||||
MIN(${seconds}) AS firstSeen,
|
||||
MIN(CASE WHEN ${NOT_A_NAME_SQL} THEN ${seconds} END) AS firstSeenWithoutNames,
|
||||
MIN(CASE WHEN ${NOT_A_NAME_SQL} THEN frequency_rank END) AS rank
|
||||
FROM imm_words
|
||||
WHERE vocabulary_visible = 1 AND first_seen IS NOT NULL ${exclusion}
|
||||
GROUP BY ${DICTIONARY_WORD_KEY_SQL}
|
||||
)
|
||||
SELECT
|
||||
${localEpochDaySql('firstSeen')} AS firstSeenDay,
|
||||
CASE
|
||||
WHEN firstSeenWithoutNames IS NULL THEN NULL
|
||||
ELSE ${localEpochDaySql('firstSeenWithoutNames')}
|
||||
END AS firstSeenDayWithoutNames,
|
||||
CASE
|
||||
WHEN rank IS NULL THEN 'unranked'
|
||||
WHEN rank <= 1000 THEN 'top1k'
|
||||
WHEN rank <= 5000 THEN 'top5k'
|
||||
WHEN rank <= 10000 THEN 'top10k'
|
||||
WHEN rank <= 20000 THEN 'top20k'
|
||||
ELSE 'beyond20k'
|
||||
END AS band,
|
||||
COUNT(*) AS wordCount
|
||||
FROM dictionary_words
|
||||
GROUP BY 1, 2, 3
|
||||
`,
|
||||
)
|
||||
.all(...excludedIds) as unknown as NewWordFirstSeenRow[];
|
||||
newWordFirstSeenCache.set(db, { version, rows });
|
||||
return rows;
|
||||
}
|
||||
|
||||
function excludedVocabularyAliases(
|
||||
@@ -324,6 +376,12 @@ export function getVocabularySummary(
|
||||
knownWordCount: knownWords ? 0 : null,
|
||||
knownWordCountWithoutNames: knownWords ? 0 : null,
|
||||
};
|
||||
// Keyed by dictionary form, so a word's conjugations count once and it is new
|
||||
// this week only if no form of it was seen earlier.
|
||||
const dictionaryWords = new Map<
|
||||
string,
|
||||
{ firstSeenSec: number; firstSeenSecWithoutNames: number | null; known: boolean }
|
||||
>();
|
||||
|
||||
let lastId = Number.MIN_SAFE_INTEGER;
|
||||
for (;;) {
|
||||
@@ -337,23 +395,40 @@ export function getVocabularySummary(
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const isName = word.pos2 === '固有名詞';
|
||||
const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec;
|
||||
const isKnown = knownWords?.has(vocabularyDisplayHeadword(word)) ?? false;
|
||||
summary.uniqueWords += 1;
|
||||
if (!isName) summary.uniqueWordsWithoutNames += 1;
|
||||
if (isNewThisWeek) {
|
||||
summary.newThisWeek += 1;
|
||||
if (!isName) summary.newThisWeekWithoutNames += 1;
|
||||
}
|
||||
if (isKnown) {
|
||||
summary.knownWordCount! += 1;
|
||||
if (!isName) summary.knownWordCountWithoutNames! += 1;
|
||||
const firstSeenSec = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0);
|
||||
const key = vocabularyDisplayHeadword(word);
|
||||
const entry = dictionaryWords.get(key) ?? {
|
||||
firstSeenSec,
|
||||
firstSeenSecWithoutNames: null,
|
||||
known: false,
|
||||
};
|
||||
entry.known ||= (knownWords?.has(key) ?? false) || (knownWords?.has(word.word) ?? false);
|
||||
entry.firstSeenSec = Math.min(entry.firstSeenSec, firstSeenSec);
|
||||
if (!isProperNounPos2(word.pos2)) {
|
||||
entry.firstSeenSecWithoutNames = Math.min(
|
||||
entry.firstSeenSecWithoutNames ?? firstSeenSec,
|
||||
firstSeenSec,
|
||||
);
|
||||
}
|
||||
dictionaryWords.set(key, entry);
|
||||
}
|
||||
if (words.length < scanBatchSize) break;
|
||||
}
|
||||
|
||||
for (const { firstSeenSec, firstSeenSecWithoutNames, known } of dictionaryWords.values()) {
|
||||
const hasNonNameForm = firstSeenSecWithoutNames !== null;
|
||||
summary.uniqueWords += 1;
|
||||
if (hasNonNameForm) summary.uniqueWordsWithoutNames += 1;
|
||||
if (firstSeenSec >= weekAgoSec) summary.newThisWeek += 1;
|
||||
if (hasNonNameForm && firstSeenSecWithoutNames >= weekAgoSec) {
|
||||
summary.newThisWeekWithoutNames += 1;
|
||||
}
|
||||
if (known) {
|
||||
summary.knownWordCount! += 1;
|
||||
if (hasNonNameForm) summary.knownWordCountWithoutNames! += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
@@ -418,7 +493,12 @@ export function getWordOccurrences(
|
||||
reading: string,
|
||||
limit = 100,
|
||||
offset = 0,
|
||||
allForms = false,
|
||||
): WordOccurrenceRow[] {
|
||||
const wordFilter = allForms
|
||||
? `w.id IN (SELECT id FROM imm_words WHERE ${DICTIONARY_WORD_KEY_SQL} = ?)`
|
||||
: 'w.headword = ? AND w.word = ? AND w.reading = ?';
|
||||
const wordParams = allForms ? [headword] : [headword, word, reading];
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
@@ -440,13 +520,13 @@ export function getWordOccurrences(
|
||||
JOIN imm_subtitle_lines l ON l.line_id = o.line_id
|
||||
JOIN imm_videos v ON v.video_id = l.video_id
|
||||
LEFT JOIN imm_anime a ON a.anime_id = l.anime_id
|
||||
WHERE w.headword = ? AND w.word = ? AND w.reading = ?
|
||||
ORDER BY l.CREATED_DATE DESC, l.line_id DESC
|
||||
WHERE ${wordFilter}
|
||||
ORDER BY ${MINEABLE_LINES_FIRST_SQL}, l.CREATED_DATE DESC, l.line_id DESC
|
||||
LIMIT ?
|
||||
OFFSET ?
|
||||
`,
|
||||
)
|
||||
.all(headword, word, reading, limit, offset) as unknown as WordOccurrenceRow[];
|
||||
.all(...wordParams, limit, offset) as unknown as WordOccurrenceRow[];
|
||||
return rows.map(widenLineText);
|
||||
}
|
||||
|
||||
@@ -553,7 +633,7 @@ export function searchSubtitleSentences(
|
||||
JOIN imm_videos v ON v.video_id = l.video_id
|
||||
LEFT JOIN imm_anime a ON a.anime_id = l.anime_id
|
||||
WHERE ${clauses.join(' AND ')}
|
||||
ORDER BY l.CREATED_DATE DESC, l.line_id DESC
|
||||
ORDER BY ${MINEABLE_LINES_FIRST_SQL}, l.CREATED_DATE DESC, l.line_id DESC
|
||||
LIMIT ?
|
||||
`,
|
||||
)
|
||||
@@ -598,17 +678,99 @@ export function getSessionEvents(
|
||||
}));
|
||||
}
|
||||
|
||||
const WORD_DETAIL_FORM_LIMIT = 30;
|
||||
|
||||
/**
|
||||
* One entry per written form. A form can be stored more than once when its reading
|
||||
* was recorded differently, as 見て is under both みて and み. Returns most seen first.
|
||||
*/
|
||||
export function mergeFormsByWord<T extends { word: string; seen: number }>(forms: T[]): T[] {
|
||||
const byWord = new Map<string, T>();
|
||||
for (const form of forms) {
|
||||
const existing = byWord.get(form.word);
|
||||
if (existing) {
|
||||
existing.seen += form.seen;
|
||||
} else {
|
||||
byWord.set(form.word, { ...form });
|
||||
}
|
||||
}
|
||||
return [...byWord.values()].sort((left, right) => right.seen - left.seen);
|
||||
}
|
||||
|
||||
/** A word's form plus totals across every form of its dictionary word. */
|
||||
export function getWordDetail(db: DatabaseSync, wordId: number): WordDetailRow | null {
|
||||
return db
|
||||
const word = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id AS wordId, headword, word, reading,
|
||||
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
|
||||
frequency, first_seen AS firstSeen, last_seen AS lastSeen
|
||||
SELECT id AS wordId, COALESCE(headword, '') AS headword, COALESCE(word, '') AS word,
|
||||
COALESCE(reading, '') AS reading,
|
||||
part_of_speech AS partOfSpeech, pos1, pos2, pos3
|
||||
FROM imm_words WHERE id = ?
|
||||
`,
|
||||
)
|
||||
.get(wordId) as WordDetailRow | null;
|
||||
.get(wordId) as Pick<
|
||||
WordDetailRow,
|
||||
'wordId' | 'headword' | 'word' | 'reading' | 'partOfSpeech' | 'pos1' | 'pos2' | 'pos3'
|
||||
> | null;
|
||||
if (!word) return null;
|
||||
|
||||
const hasHeadword = word.headword.trim() !== '';
|
||||
const forms = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id AS wordId, COALESCE(word, '') AS word, COALESCE(reading, '') AS reading,
|
||||
frequency AS seen, frequency_rank AS rank,
|
||||
first_seen AS firstSeen, last_seen AS lastSeen
|
||||
FROM imm_words
|
||||
WHERE ${hasHeadword ? '(headword = ? AND vocabulary_visible = 1) OR id = ?' : 'id = ?'}
|
||||
ORDER BY frequency DESC, id
|
||||
`,
|
||||
)
|
||||
.all(...(hasHeadword ? [word.headword, wordId] : [wordId])) as Array<{
|
||||
wordId: number;
|
||||
word: string;
|
||||
reading: string;
|
||||
seen: number | null;
|
||||
rank: number | null;
|
||||
firstSeen: number;
|
||||
lastSeen: number;
|
||||
}>;
|
||||
|
||||
const lines = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT COUNT(DISTINCT l.line_id) AS lineCount,
|
||||
COUNT(DISTINCT CASE WHEN ${NO_LOCAL_FILE_SQL} THEN l.line_id END) AS streamLineCount
|
||||
FROM imm_word_line_occurrences o
|
||||
JOIN imm_words w ON w.id = o.word_id
|
||||
JOIN imm_subtitle_lines l ON l.line_id = o.line_id
|
||||
JOIN imm_videos v ON v.video_id = l.video_id
|
||||
WHERE ${hasHeadword ? 'w.headword = ?' : 'w.id = ?'}
|
||||
`,
|
||||
)
|
||||
.get(hasHeadword ? word.headword : wordId) as { lineCount: number; streamLineCount: number };
|
||||
const merged = mergeFormsByWord(forms.map((form) => ({ ...form, seen: form.seen ?? 0 })));
|
||||
const toSeconds = (value: number): number => timestampSeconds(fromDbTimestamp(value) ?? 0);
|
||||
const ranks = forms.map((form) => form.rank).filter((rank): rank is number => (rank ?? 0) > 0);
|
||||
return {
|
||||
...word,
|
||||
frequency: forms.reduce((sum, form) => sum + (form.seen ?? 0), 0),
|
||||
frequencyRank: ranks.length > 0 ? Math.min(...ranks) : null,
|
||||
dictionaryReading:
|
||||
forms.find((form) => form.word === word.headword && form.reading.trim() !== '')?.reading ??
|
||||
'',
|
||||
formCount: merged.length,
|
||||
forms: merged.slice(0, WORD_DETAIL_FORM_LIMIT).map((form) => ({
|
||||
wordId: form.wordId,
|
||||
word: form.word,
|
||||
reading: form.reading,
|
||||
seen: form.seen,
|
||||
})),
|
||||
lineCount: lines.lineCount,
|
||||
streamLineCount: lines.streamLineCount,
|
||||
firstSeen: Math.min(...forms.map((form) => toSeconds(form.firstSeen))),
|
||||
lastSeen: Math.max(...forms.map((form) => toSeconds(form.lastSeen))),
|
||||
};
|
||||
}
|
||||
|
||||
export function getWordAnimeAppearances(
|
||||
@@ -621,21 +783,26 @@ export function getWordAnimeAppearances(
|
||||
SELECT a.anime_id AS animeId, a.canonical_title AS animeTitle,
|
||||
SUM(o.occurrence_count) AS occurrenceCount
|
||||
FROM imm_word_line_occurrences o
|
||||
JOIN imm_words w ON w.id = o.word_id
|
||||
JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
|
||||
JOIN imm_anime a ON a.anime_id = sl.anime_id
|
||||
WHERE o.word_id = ? AND sl.anime_id IS NOT NULL
|
||||
WHERE (w.id = ? OR w.headword = (SELECT NULLIF(TRIM(headword), '') FROM imm_words WHERE id = ?))
|
||||
AND sl.anime_id IS NOT NULL
|
||||
GROUP BY a.anime_id
|
||||
ORDER BY occurrenceCount DESC
|
||||
`,
|
||||
)
|
||||
.all(wordId) as WordAnimeAppearanceRow[];
|
||||
.all(wordId, wordId) as WordAnimeAppearanceRow[];
|
||||
}
|
||||
|
||||
/** Other dictionary words sharing a reading or kanji; a word's own forms don't count. */
|
||||
export function getSimilarWords(db: DatabaseSync, wordId: number, limit = 10): SimilarWordRow[] {
|
||||
const word = db.prepare('SELECT headword, reading FROM imm_words WHERE id = ?').get(wordId) as {
|
||||
headword: string;
|
||||
reading: string;
|
||||
} | null;
|
||||
const word = db
|
||||
.prepare(
|
||||
`SELECT COALESCE(headword, '') AS headword, COALESCE(reading, '') AS reading
|
||||
FROM imm_words WHERE id = ?`,
|
||||
)
|
||||
.get(wordId) as { headword: string; reading: string } | null;
|
||||
if (!word || word.headword.trim() === '') return [];
|
||||
|
||||
const clauses: string[] = [];
|
||||
@@ -653,22 +820,41 @@ export function getSimilarWords(db: DatabaseSync, wordId: number, limit = 10): S
|
||||
|
||||
if (clauses.length === 0) return [];
|
||||
|
||||
const orderBy =
|
||||
reading !== '' ? 'CASE WHEN reading = ? THEN 0 ELSE 1 END, frequency DESC' : 'frequency DESC';
|
||||
const orderParams = reading !== '' ? [word.reading] : [];
|
||||
const readingMatch =
|
||||
reading !== ''
|
||||
? 'MAX(CASE WHEN reading = ? THEN 1 ELSE 0 END) OVER (PARTITION BY headword)'
|
||||
: '0';
|
||||
const readingMatchParams = reading !== '' ? [word.reading] : [];
|
||||
|
||||
// One form matching is enough to relate a word, but its total and the form it shows
|
||||
// come from all its visible forms, so a rare form with the same reading can't stand in.
|
||||
// The blank check lists its characters because SQL TRIM alone strips only ASCII spaces.
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id AS wordId, headword, word, reading, frequency
|
||||
FROM imm_words
|
||||
WHERE id != ?
|
||||
AND (${clauses.join(' OR ')})
|
||||
ORDER BY ${orderBy}
|
||||
WITH related AS (
|
||||
SELECT DISTINCT headword
|
||||
FROM imm_words
|
||||
WHERE headword != ? AND vocabulary_visible = 1
|
||||
AND TRIM(headword, ' ' || char(9, 10, 11, 12, 13, 12288)) != ''
|
||||
AND (${clauses.join(' OR ')})
|
||||
),
|
||||
forms AS (
|
||||
SELECT id, headword, word, reading,
|
||||
SUM(frequency) OVER (PARTITION BY headword) AS totalSeen,
|
||||
${readingMatch} AS readingMatch,
|
||||
ROW_NUMBER() OVER (PARTITION BY headword ORDER BY frequency DESC, id) AS formOrder
|
||||
FROM imm_words
|
||||
WHERE vocabulary_visible = 1 AND headword IN (SELECT headword FROM related)
|
||||
)
|
||||
SELECT id AS wordId, headword, word, reading, totalSeen AS frequency
|
||||
FROM forms
|
||||
WHERE formOrder = 1
|
||||
ORDER BY readingMatch DESC, totalSeen DESC, headword
|
||||
LIMIT ?
|
||||
`,
|
||||
)
|
||||
.all(wordId, ...params, ...orderParams, limit) as SimilarWordRow[];
|
||||
.all(word.headword, ...params, ...readingMatchParams, limit) as SimilarWordRow[];
|
||||
}
|
||||
|
||||
export function getKanjiDetail(db: DatabaseSync, kanjiId: number): KanjiDetailRow | null {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { StatsDayRange } from '../../../types/stats-http-contract';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import type {
|
||||
ImmersionSessionRollupRow,
|
||||
@@ -15,7 +16,12 @@ import {
|
||||
visibleWordSql,
|
||||
} from './query-shared';
|
||||
|
||||
export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummaryQueryRow[] {
|
||||
export function getSessionSummaries(
|
||||
db: DatabaseSync,
|
||||
limit = 50,
|
||||
dayRange?: StatsDayRange,
|
||||
): SessionSummaryQueryRow[] {
|
||||
const dayFilter = dayRange ? `WHERE ${sessionRollupDaySql('s')} BETWEEN ? AND ?` : '';
|
||||
const prepared = db.prepare(`
|
||||
${ACTIVE_SESSION_METRICS_CTE}
|
||||
SELECT
|
||||
@@ -38,10 +44,12 @@ export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummar
|
||||
LEFT JOIN active_session_metrics asm ON asm.sessionId = s.session_id
|
||||
LEFT JOIN imm_videos v ON v.video_id = s.video_id
|
||||
LEFT JOIN imm_anime a ON a.anime_id = v.anime_id
|
||||
${dayFilter}
|
||||
ORDER BY s.started_at_ms DESC
|
||||
LIMIT ?
|
||||
`);
|
||||
const rows = prepared.all(limit) as Array<
|
||||
const params = dayRange ? [dayRange.startDay, dayRange.endDay, limit] : [limit];
|
||||
const rows = prepared.all(...params) as Array<
|
||||
SessionSummaryQueryRow & {
|
||||
startedAtMs: number | string;
|
||||
endedAtMs: number | string | null;
|
||||
@@ -393,6 +401,63 @@ export function selectDailyRollupRows(db: DatabaseSync, limit: number): RawRollu
|
||||
.all(limit) as unknown as RawRollupRow[];
|
||||
}
|
||||
|
||||
/** Raw daily rollup rows for an inclusive day span; a null start means no lower bound. */
|
||||
export function selectDailyRollupRowsBetween(
|
||||
db: DatabaseSync,
|
||||
startDay: number | null,
|
||||
endDay: number,
|
||||
): RawRollupRow[] {
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
r.rollup_day AS rollupDayOrMonth,
|
||||
r.video_id AS videoId,
|
||||
r.total_sessions AS totalSessions,
|
||||
r.total_active_min AS totalActiveMin,
|
||||
r.total_lines_seen AS totalLinesSeen,
|
||||
r.total_tokens_seen AS totalTokensSeen,
|
||||
r.total_cards AS totalCards,
|
||||
r.cards_per_hour AS cardsPerHour,
|
||||
r.lookup_hit_rate AS lookupHitRate
|
||||
FROM imm_daily_rollups r
|
||||
WHERE r.rollup_day BETWEEN ? AND ?
|
||||
ORDER BY r.rollup_day DESC, r.video_id DESC
|
||||
`,
|
||||
)
|
||||
.all(startDay ?? Number.MIN_SAFE_INTEGER, endDay) as unknown as RawRollupRow[];
|
||||
}
|
||||
|
||||
/** Raw monthly rollup rows for an inclusive `YYYYMM` span; a null start means no lower bound. */
|
||||
export function selectMonthlyRollupRowsBetween(
|
||||
db: DatabaseSync,
|
||||
startMonth: number | null,
|
||||
endMonth: number,
|
||||
): RawRollupRow[] {
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
r.rollup_month AS rollupDayOrMonth,
|
||||
r.video_id AS videoId,
|
||||
r.total_sessions AS totalSessions,
|
||||
r.total_active_min AS totalActiveMin,
|
||||
r.total_lines_seen AS totalLinesSeen,
|
||||
r.total_tokens_seen AS totalTokensSeen,
|
||||
r.total_cards AS totalCards,
|
||||
CASE
|
||||
WHEN r.total_active_min > 0 THEN (r.total_cards * 60.0) / r.total_active_min
|
||||
ELSE NULL
|
||||
END AS cardsPerHour,
|
||||
NULL AS lookupHitRate
|
||||
FROM imm_monthly_rollups r
|
||||
WHERE r.rollup_month BETWEEN ? AND ?
|
||||
ORDER BY r.rollup_month DESC, r.video_id DESC
|
||||
`,
|
||||
)
|
||||
.all(startMonth ?? Number.MIN_SAFE_INTEGER, endMonth) as unknown as RawRollupRow[];
|
||||
}
|
||||
|
||||
/** Monthly counterpart of `selectDailyRollupRows`. */
|
||||
export function selectMonthlyRollupRows(db: DatabaseSync, limit: number): RawRollupRow[] {
|
||||
return db
|
||||
|
||||
@@ -0,0 +1,541 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { Database, type DatabaseSync } from './sqlite';
|
||||
import { getOrCreateAnimeRecord, getOrCreateVideoRecord, linkVideoToAnimeRecord } from './storage';
|
||||
import { ensureSchema } from './storage';
|
||||
import { SOURCE_TYPE_LOCAL } from './types';
|
||||
import { getTrendsDashboard } from './query-trends';
|
||||
import { getSessionSummaries } from './query-sessions';
|
||||
import { getVocabularyChartData, getVocabularySummary } from './query-lexical';
|
||||
import { markLexicalDailyRollupsReady } from './lexical-rollups';
|
||||
|
||||
/** Noon, local time, on a calendar day, so the local epoch day holds in every time zone. */
|
||||
function localNoonMs(epochDay: number): number {
|
||||
const date = new Date(epochDay * 86_400_000);
|
||||
return new Date(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), 12).getTime();
|
||||
}
|
||||
|
||||
// Mocked "now" is noon on 2026-03-01, a Sunday.
|
||||
const TODAY = Date.UTC(2026, 2, 1) / 86_400_000;
|
||||
const NOW_MS = String(localNoonMs(TODAY));
|
||||
|
||||
function withDashboardDb(run: (db: DatabaseSync) => void): void {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-trends-'));
|
||||
const db = new Database(path.join(dir, 'immersion.sqlite'));
|
||||
const previousNow = globalThis.__subminerTestNowMs;
|
||||
globalThis.__subminerTestNowMs = NOW_MS;
|
||||
try {
|
||||
ensureSchema(db);
|
||||
run(db);
|
||||
} finally {
|
||||
globalThis.__subminerTestNowMs = previousNow;
|
||||
db.close();
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function addVideo(db: DatabaseSync, title: string, episode: number, watched = false): number {
|
||||
const key = `${title}-${episode}`;
|
||||
const videoId = getOrCreateVideoRecord(db, `local:/tmp/${key}.mkv`, {
|
||||
canonicalTitle: key,
|
||||
sourcePath: `/tmp/${key}.mkv`,
|
||||
sourceUrl: null,
|
||||
sourceType: SOURCE_TYPE_LOCAL,
|
||||
});
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: title,
|
||||
canonicalTitle: title,
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: null,
|
||||
});
|
||||
linkVideoToAnimeRecord(db, videoId, {
|
||||
animeId,
|
||||
parsedBasename: `${key}.mkv`,
|
||||
parsedTitle: title,
|
||||
parsedSeason: 1,
|
||||
parsedEpisode: episode,
|
||||
parserSource: 'test',
|
||||
parserConfidence: 1,
|
||||
parseMetadataJson: null,
|
||||
});
|
||||
if (watched) db.prepare('UPDATE imm_videos SET watched = 1 WHERE video_id = ?').run(videoId);
|
||||
return videoId;
|
||||
}
|
||||
|
||||
function addRollup(
|
||||
db: DatabaseSync,
|
||||
day: number,
|
||||
videoId: number,
|
||||
values: { minutes?: number; words?: number; cards?: number } = {},
|
||||
): void {
|
||||
db.prepare(
|
||||
`INSERT INTO imm_daily_rollups (
|
||||
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards
|
||||
) VALUES (?, ?, 1, ?, 1, ?, ?)`,
|
||||
).run(day, videoId, values.minutes ?? 30, values.words ?? 100, values.cards ?? 0);
|
||||
}
|
||||
|
||||
function addSession(
|
||||
db: DatabaseSync,
|
||||
day: number,
|
||||
videoId: number,
|
||||
values: { minutes?: number; lookups?: number } = {},
|
||||
): void {
|
||||
const startedAtMs = localNoonMs(day).toString();
|
||||
const activeMs = (values.minutes ?? 30) * 60_000;
|
||||
db.prepare(
|
||||
`INSERT INTO imm_sessions (
|
||||
session_uuid, video_id, started_at_ms, ended_at_ms, status, active_watched_ms,
|
||||
total_watched_ms, yomitan_lookup_count
|
||||
) VALUES (?, ?, ?, ?, 2, ?, ?, ?)`,
|
||||
).run(
|
||||
`session-${day}-${videoId}-${Math.random()}`,
|
||||
videoId,
|
||||
startedAtMs,
|
||||
(BigInt(startedAtMs) + BigInt(activeMs)).toString(),
|
||||
activeMs,
|
||||
activeMs,
|
||||
values.lookups ?? 0,
|
||||
);
|
||||
}
|
||||
|
||||
function addWord(
|
||||
db: DatabaseSync,
|
||||
values: {
|
||||
headword: string;
|
||||
word?: string;
|
||||
day: number;
|
||||
pos2?: string | null;
|
||||
rank?: number | null;
|
||||
},
|
||||
): void {
|
||||
const firstSeen = localNoonMs(values.day) / 1000;
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen,
|
||||
frequency, frequency_rank
|
||||
) VALUES (?, ?, ?, 'noun', '名詞', ?, '', ?, ?, 1, ?)`,
|
||||
).run(
|
||||
values.headword,
|
||||
values.word ?? values.headword,
|
||||
values.word ?? values.headword,
|
||||
values.pos2 ?? null,
|
||||
firstSeen,
|
||||
firstSeen,
|
||||
values.rank ?? null,
|
||||
);
|
||||
}
|
||||
|
||||
test('the title lists stay inside the calendar range when activity is sparse', () => {
|
||||
withDashboardDb((db) => {
|
||||
const recent = addVideo(db, 'Recent Show', 1);
|
||||
const old = addVideo(db, 'Old Show', 1);
|
||||
addRollup(db, TODAY, recent, { minutes: 20 });
|
||||
// Outside a 7-day calendar window, though well within the last 7 active days.
|
||||
addRollup(db, TODAY - 40, old, { minutes: 90 });
|
||||
addSession(db, TODAY - 40, old);
|
||||
|
||||
const dashboard = getTrendsDashboard(db, '7d', 'day');
|
||||
|
||||
assert.deepEqual(
|
||||
dashboard.librarySummary.map((row) => row.title),
|
||||
['Recent Show'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
dashboard.titleTimeline.map((point) => point.animeTitle),
|
||||
['Recent Show'],
|
||||
);
|
||||
assert.equal(dashboard.summary.current.watchTimeMin, 20);
|
||||
assert.equal(
|
||||
dashboard.librarySummary.reduce((sum, row) => sum + row.watchTimeMin, 0),
|
||||
dashboard.activity.watchTime.reduce((sum, point) => sum + point.value, 0),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('every grouping reports the same totals for a range', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
for (const day of [TODAY, TODAY - 3, TODAY - 20, TODAY - 29, TODAY - 45]) {
|
||||
addRollup(db, day, videoId, { minutes: 10, cards: 1 });
|
||||
}
|
||||
// January is only partly in range, and its monthly total includes days that are not.
|
||||
const insertMonthly = db.prepare(
|
||||
`INSERT INTO imm_monthly_rollups (
|
||||
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards
|
||||
) VALUES (?, ?, 1, ?, 1, 100, 1)`,
|
||||
);
|
||||
insertMonthly.run(202601, videoId, 20);
|
||||
insertMonthly.run(202602, videoId, 20);
|
||||
insertMonthly.run(202603, videoId, 10);
|
||||
|
||||
const sum = (groupBy: 'day' | 'week' | 'month') =>
|
||||
getTrendsDashboard(db, '30d', groupBy).activity.watchTime.reduce(
|
||||
(total, point) => total + point.value,
|
||||
0,
|
||||
);
|
||||
|
||||
assert.equal(sum('day'), 40);
|
||||
assert.equal(sum('week'), 40);
|
||||
assert.equal(sum('month'), 40);
|
||||
});
|
||||
});
|
||||
|
||||
test('all-time month buckets use monthly rollups that outlive daily ones', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
const insertMonthly = db.prepare(
|
||||
`INSERT INTO imm_monthly_rollups (
|
||||
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
|
||||
total_tokens_seen, total_cards
|
||||
) VALUES (?, ?, 3, ?, 1, 100, 50)`,
|
||||
);
|
||||
insertMonthly.run(202501, videoId, 500);
|
||||
insertMonthly.run(202503, videoId, 200);
|
||||
|
||||
assert.deepEqual(getTrendsDashboard(db, 'all', 'month').activity.watchTime, [
|
||||
{ label: 'Jan 2025', value: 500 },
|
||||
{ label: 'Feb 2025', value: 0 },
|
||||
{ label: 'Mar 2025', value: 200 },
|
||||
]);
|
||||
assert.deepEqual(getTrendsDashboard(db, 'all', 'month', false).activity.watchTime, [
|
||||
{ label: 'Jan 2025', value: 500 },
|
||||
{ label: 'Mar 2025', value: 200 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('a finished video counts once however many days it was watched', () => {
|
||||
withDashboardDb((db) => {
|
||||
const finished = addVideo(db, 'Show', 1, true);
|
||||
const unfinished = addVideo(db, 'Show', 2, false);
|
||||
addRollup(db, TODAY - 2, finished);
|
||||
addRollup(db, TODAY - 1, finished);
|
||||
addRollup(db, TODAY, unfinished);
|
||||
|
||||
for (const groupBy of ['day', 'week', 'month'] as const) {
|
||||
const dashboard = getTrendsDashboard(db, '30d', groupBy);
|
||||
assert.equal(dashboard.progress.episodes.at(-1)?.value, 1, groupBy);
|
||||
assert.equal(dashboard.summary.current.episodes, 1, groupBy);
|
||||
}
|
||||
|
||||
const daily = getTrendsDashboard(db, '7d', 'day').activity.episodes;
|
||||
// Attributed to the last day it was watched.
|
||||
assert.equal(daily.at(-2)?.value, 1);
|
||||
assert.equal(daily.at(-3)?.value, 0);
|
||||
assert.equal(getTrendsDashboard(db, '30d', 'day').librarySummary[0]?.episodesFinished, 1);
|
||||
assert.equal(getTrendsDashboard(db, '30d', 'day').librarySummary[0]?.videos, 2);
|
||||
});
|
||||
});
|
||||
|
||||
test('rates are gaps without enough data, not zeros', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
// A two-minute day with one card: 30 cards per hour if rated on its own.
|
||||
addRollup(db, TODAY - 20, videoId, { minutes: 2, words: 50, cards: 1 });
|
||||
|
||||
const dashboard = getTrendsDashboard(db, '30d', 'day');
|
||||
|
||||
assert.equal(dashboard.ratios.cardsPerHour.at(-1)?.value, null);
|
||||
assert.equal(
|
||||
dashboard.ratios.cardsPerHour.every((point) => point.value === null),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
dashboard.ratios.readingSpeed.every((point) => point.value === null),
|
||||
true,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('daily rates are volume-weighted over a trailing week, reaching before the range', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
// Eight days before today: outside the 7d range, inside the first day's lookback.
|
||||
addRollup(db, TODAY - 8, videoId, { minutes: 60, words: 3_000, cards: 6 });
|
||||
addRollup(db, TODAY - 6, videoId, { minutes: 60, words: 1_000, cards: 2 });
|
||||
|
||||
const speed = getTrendsDashboard(db, '7d', 'day').ratios.readingSpeed;
|
||||
|
||||
// (3000 + 1000) words / (60 + 60) min, not an average of 50 and 16.7 ...
|
||||
assert.equal(speed[0]?.value, +(4_000 / 120).toFixed(1));
|
||||
// ... and once the older day leaves the trailing week only the second remains.
|
||||
assert.equal(speed[5]?.value, +(1_000 / 60).toFixed(1));
|
||||
});
|
||||
});
|
||||
|
||||
test('week grouping starts on Monday and clips the first bucket to the range', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
addRollup(db, TODAY, videoId);
|
||||
|
||||
const { buckets } = getTrendsDashboard(db, '30d', 'week');
|
||||
|
||||
assert.equal(buckets[0]?.startDay, TODAY - 29);
|
||||
assert.equal(buckets.at(-1)?.endDay, TODAY);
|
||||
// Today is a Sunday, so the final bucket is a full Monday-to-Sunday week.
|
||||
assert.equal(buckets.at(-1)?.startDay, TODAY - 6);
|
||||
for (let index = 1; index < buckets.length; index += 1) {
|
||||
assert.equal(buckets[index]!.startDay, buckets[index - 1]!.endDay + 1);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test('showing empty periods on all-time fills the calendar between active days', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
addRollup(db, TODAY - 10, videoId);
|
||||
addRollup(db, TODAY, videoId);
|
||||
|
||||
assert.equal(getTrendsDashboard(db, 'all', 'day', true).activity.watchTime.length, 11);
|
||||
assert.equal(getTrendsDashboard(db, 'all', 'day', false).activity.watchTime.length, 2);
|
||||
assert.deepEqual(getTrendsDashboard(db, 'all', 'day', true).window, {
|
||||
startDay: TODAY - 10,
|
||||
endDay: TODAY,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test('month labels name the year, and day labels do when the range crosses one', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
addRollup(db, TODAY, videoId);
|
||||
|
||||
assert.equal(getTrendsDashboard(db, '30d', 'month').buckets.at(-1)?.label, 'Mar 2026');
|
||||
assert.equal(getTrendsDashboard(db, '7d', 'day').buckets.at(-1)?.label, 'Mar 1');
|
||||
assert.equal(getTrendsDashboard(db, '90d', 'day').buckets.at(-1)?.label, 'Mar 1, 2026');
|
||||
});
|
||||
});
|
||||
|
||||
test('a custom range clips to its days and compares against the same-length span before it', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
addRollup(db, TODAY - 5, videoId, { minutes: 10 });
|
||||
addRollup(db, TODAY - 15, videoId, { minutes: 20 });
|
||||
addRollup(db, TODAY - 25, videoId, { minutes: 40 });
|
||||
|
||||
const dashboard = getTrendsDashboard(db, { startDay: TODAY - 19, endDay: TODAY - 10 });
|
||||
|
||||
assert.equal(dashboard.activity.watchTime.length, 10);
|
||||
assert.equal(dashboard.summary.current.watchTimeMin, 20);
|
||||
assert.equal(dashboard.summary.previous?.watchTimeMin, 40);
|
||||
assert.deepEqual(dashboard.window, { startDay: TODAY - 19, endDay: TODAY - 10 });
|
||||
});
|
||||
});
|
||||
|
||||
test('summary compares with the previous period and all-time has no comparison', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
addRollup(db, TODAY, videoId, { minutes: 10, cards: 1 });
|
||||
addRollup(db, TODAY - 10, videoId, { minutes: 25, cards: 4 });
|
||||
|
||||
const { current, previous } = getTrendsDashboard(db, '7d', 'day').summary;
|
||||
|
||||
assert.equal(current.watchTimeMin, 10);
|
||||
assert.equal(current.activeDays, 1);
|
||||
assert.equal(previous?.watchTimeMin, 25);
|
||||
assert.equal(previous?.cards, 4);
|
||||
assert.equal(getTrendsDashboard(db, 'all', 'day').summary.previous, null);
|
||||
});
|
||||
});
|
||||
|
||||
test('a title filter narrows the page but keeps the title lists complete', () => {
|
||||
withDashboardDb((db) => {
|
||||
const first = addVideo(db, 'First', 1);
|
||||
const second = addVideo(db, 'Second', 1);
|
||||
addRollup(db, TODAY, first, { minutes: 10 });
|
||||
addRollup(db, TODAY, second, { minutes: 50 });
|
||||
addSession(db, TODAY, first, { minutes: 10, lookups: 3 });
|
||||
addSession(db, TODAY, second, { minutes: 50, lookups: 9 });
|
||||
addWord(db, { headword: '海', day: TODAY });
|
||||
|
||||
const dashboard = getTrendsDashboard(db, '7d', 'day', true, 'First');
|
||||
|
||||
assert.equal(dashboard.summary.current.watchTimeMin, 10);
|
||||
assert.equal(dashboard.progress.lookups.at(-1)?.value, 3);
|
||||
assert.equal(
|
||||
dashboard.watchTimeByDayHour.flat().reduce((sum, value) => sum + value, 0),
|
||||
10,
|
||||
);
|
||||
assert.equal(dashboard.librarySummary.length, 2);
|
||||
assert.equal(dashboard.titleTimeline.length, 2);
|
||||
// New words are not tracked per title.
|
||||
assert.equal(dashboard.summary.current.newWords, null);
|
||||
assert.equal(dashboard.learning, null);
|
||||
assert.deepEqual(dashboard.progress.newWords, []);
|
||||
});
|
||||
});
|
||||
|
||||
test('running totals follow the activity charts when empty days are hidden', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
addRollup(db, TODAY - 3, videoId, { minutes: 30 });
|
||||
addRollup(db, TODAY, videoId, { minutes: 10 });
|
||||
|
||||
const dashboard = getTrendsDashboard(db, '7d', 'day', false);
|
||||
|
||||
assert.deepEqual(
|
||||
dashboard.progress.watchTime.map((point) => point.value),
|
||||
[30, 40],
|
||||
);
|
||||
assert.deepEqual(
|
||||
dashboard.progress.watchTime.map((point) => point.label),
|
||||
dashboard.activity.watchTime.map((point) => point.label),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('the watch time running total rounds its sum, so it ends on the summary total', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
addRollup(db, TODAY - 2, videoId, { minutes: 10.4 });
|
||||
addRollup(db, TODAY - 1, videoId, { minutes: 10.4 });
|
||||
addRollup(db, TODAY, videoId, { minutes: 10.4 });
|
||||
|
||||
const dashboard = getTrendsDashboard(db, '7d', 'day', false);
|
||||
|
||||
assert.deepEqual(
|
||||
dashboard.activity.watchTime.map((point) => point.value),
|
||||
[10, 10, 10],
|
||||
);
|
||||
assert.deepEqual(
|
||||
dashboard.progress.watchTime.map((point) => point.value),
|
||||
[10, 21, 31],
|
||||
);
|
||||
assert.equal(dashboard.summary.current.watchTimeMin, 31);
|
||||
});
|
||||
});
|
||||
|
||||
test('the summary adds watch time in date order, so a total near .5 rounds like the chart', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
// Newest first these add up to 4.499999999999999, and oldest first to 4.5.
|
||||
addRollup(db, TODAY - 2, videoId, { minutes: 0.1 });
|
||||
addRollup(db, TODAY - 1, videoId, { minutes: 0.1 });
|
||||
addRollup(db, TODAY, videoId, { minutes: 4.3 });
|
||||
|
||||
const dashboard = getTrendsDashboard(db, '7d', 'day');
|
||||
|
||||
assert.equal(
|
||||
dashboard.summary.current.watchTimeMin,
|
||||
dashboard.progress.watchTime.at(-1)?.value,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('new words count a dictionary word once, on its first form, without names', () => {
|
||||
withDashboardDb((db) => {
|
||||
addWord(db, { headword: '言う', word: '言う', day: TODAY - 2, rank: 50 });
|
||||
addWord(db, { headword: '言う', word: '言った', day: TODAY, rank: 50 });
|
||||
addWord(db, { headword: '東京', day: TODAY - 1, pos2: '固有名詞', rank: 800 });
|
||||
|
||||
const dashboard = getTrendsDashboard(db, '7d', 'day');
|
||||
|
||||
assert.equal(dashboard.summary.current.newWords, 1);
|
||||
assert.equal(dashboard.progress.newWords.at(-1)?.value, 1);
|
||||
assert.equal(dashboard.progress.newWords.at(-3)?.value, 1);
|
||||
assert.equal(dashboard.progress.newWords.at(-4)?.value, 0);
|
||||
});
|
||||
});
|
||||
|
||||
test('new words per thousand and frequency bands read dictionary words', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
addRollup(db, TODAY, videoId, { words: 2_000 });
|
||||
addWord(db, { headword: '一', day: TODAY, rank: 500 });
|
||||
addWord(db, { headword: '二', day: TODAY, rank: 3_000 });
|
||||
addWord(db, { headword: '三', day: TODAY, rank: 30_000 });
|
||||
addWord(db, { headword: '四', day: TODAY, rank: null });
|
||||
|
||||
const { learning } = getTrendsDashboard(db, '7d', 'day');
|
||||
|
||||
assert.equal(learning?.newWordsPerThousand.at(-1)?.value, 2);
|
||||
assert.deepEqual(learning?.frequencyBands.at(-1)?.counts, {
|
||||
top1k: 1,
|
||||
top5k: 1,
|
||||
top10k: 0,
|
||||
top20k: 0,
|
||||
beyond20k: 1,
|
||||
unranked: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test('vocabulary charts and summary count dictionary words, not surface forms', () => {
|
||||
withDashboardDb((db) => {
|
||||
markLexicalDailyRollupsReady(db);
|
||||
addWord(db, { headword: '言う', word: '言う', day: TODAY - 20 });
|
||||
addWord(db, { headword: '言う', word: '言った', day: TODAY });
|
||||
addWord(db, { headword: '東京', day: TODAY, pos2: '固有名詞' });
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
const summary = getVocabularySummary(db, null, Number(NOW_MS));
|
||||
|
||||
assert.deepEqual(charts.newWordsTimelineWithoutNames, [{ epochDay: TODAY - 20, wordCount: 1 }]);
|
||||
assert.deepEqual(charts.newWordsTimeline, [
|
||||
{ epochDay: TODAY - 20, wordCount: 1 },
|
||||
{ epochDay: TODAY, wordCount: 1 },
|
||||
]);
|
||||
assert.equal(summary.uniqueWords, 2);
|
||||
assert.equal(summary.uniqueWordsWithoutNames, 1);
|
||||
// 言う was first seen long ago, so its newer conjugation is not a new word.
|
||||
assert.equal(summary.newThisWeek, 1);
|
||||
assert.equal(summary.newThisWeekWithoutNames, 0);
|
||||
});
|
||||
});
|
||||
|
||||
test('names tagged with merged parts of speech count as names', () => {
|
||||
withDashboardDb((db) => {
|
||||
markLexicalDailyRollupsReady(db);
|
||||
addWord(db, { headword: '美琴', day: TODAY, pos2: '固有名詞|一般' });
|
||||
addWord(db, { headword: '谷君', day: TODAY, pos2: '固有名詞|接尾' });
|
||||
addWord(db, { headword: '海', day: TODAY, pos2: '一般' });
|
||||
|
||||
assert.equal(getTrendsDashboard(db, '7d', 'day').summary.current.newWords, 1);
|
||||
const summary = getVocabularySummary(db, null, Number(NOW_MS));
|
||||
assert.equal(summary.uniqueWords, 3);
|
||||
assert.equal(summary.uniqueWordsWithoutNames, 1);
|
||||
assert.deepEqual(getVocabularyChartData(db).newWordsTimelineWithoutNames, [
|
||||
{ epochDay: TODAY, wordCount: 1 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('new word counts pick up writes from this and other connections', () => {
|
||||
withDashboardDb((db) => {
|
||||
addWord(db, { headword: '海', day: TODAY });
|
||||
assert.equal(getTrendsDashboard(db, '7d', 'day').summary.current.newWords, 1);
|
||||
|
||||
addWord(db, { headword: '山', day: TODAY });
|
||||
assert.equal(getTrendsDashboard(db, '7d', 'day').summary.current.newWords, 2);
|
||||
|
||||
const dbPath = (db.prepare('PRAGMA database_list').get() as { file: string }).file;
|
||||
const other = new Database(dbPath);
|
||||
try {
|
||||
addWord(other, { headword: '川', day: TODAY });
|
||||
} finally {
|
||||
other.close();
|
||||
}
|
||||
assert.equal(getTrendsDashboard(db, '7d', 'day').summary.current.newWords, 3);
|
||||
});
|
||||
});
|
||||
|
||||
test('getSessionSummaries can narrow to a span of local days', () => {
|
||||
withDashboardDb((db) => {
|
||||
const videoId = addVideo(db, 'Show', 1);
|
||||
addSession(db, TODAY - 5, videoId);
|
||||
addSession(db, TODAY - 4, videoId);
|
||||
addSession(db, TODAY, videoId);
|
||||
|
||||
assert.equal(getSessionSummaries(db, 50).length, 3);
|
||||
assert.equal(getSessionSummaries(db, 50, { startDay: TODAY - 5, endDay: TODAY - 4 }).length, 2);
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,562 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import type { StatsDictionaryWordQuery } from '../../../types/stats-http-contract';
|
||||
import {
|
||||
getSimilarWords,
|
||||
getWordAnimeAppearances,
|
||||
getVocabularySummary,
|
||||
getWordDetail,
|
||||
getWordOccurrences,
|
||||
replaceStatsExcludedWords,
|
||||
searchSubtitleSentences,
|
||||
} from './query-lexical';
|
||||
import { markLexicalDailyRollupsReady } from './lexical-rollups';
|
||||
import { getDictionaryWordPage, getVocabularyCoverage } from './query-vocabulary-words';
|
||||
import { startSessionRecord } from './session';
|
||||
import { Database, type DatabaseSync } from './sqlite';
|
||||
import { ensureSchema, getOrCreateAnimeRecord, getOrCreateVideoRecord } from './storage';
|
||||
import { SOURCE_TYPE_LOCAL, SOURCE_TYPE_REMOTE } from './types';
|
||||
|
||||
function withDb(run: (db: DatabaseSync) => void): void {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-vocabulary-'));
|
||||
const db = new Database(path.join(dir, 'immersion.sqlite'));
|
||||
try {
|
||||
ensureSchema(db);
|
||||
run(db);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
interface WordFixture {
|
||||
headword: string;
|
||||
word?: string;
|
||||
reading?: string;
|
||||
seen?: number;
|
||||
rank?: number | null;
|
||||
pos2?: string;
|
||||
partOfSpeech?: string;
|
||||
firstSeen?: number;
|
||||
}
|
||||
|
||||
function addWord(db: DatabaseSync, word: WordFixture): number {
|
||||
const firstSeen = word.firstSeen ?? 1_700_000_000;
|
||||
const result = db
|
||||
.prepare(
|
||||
`INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen,
|
||||
frequency, frequency_rank
|
||||
) VALUES (?, ?, ?, ?, '', ?, '', ?, ?, ?, ?)`,
|
||||
)
|
||||
.run(
|
||||
word.headword,
|
||||
word.word ?? word.headword,
|
||||
word.reading ?? '',
|
||||
word.partOfSpeech ?? 'noun',
|
||||
word.pos2 ?? '',
|
||||
firstSeen,
|
||||
firstSeen,
|
||||
word.seen ?? 1,
|
||||
word.rank ?? null,
|
||||
);
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
|
||||
function getIdOf(db: DatabaseSync, headword: string): number {
|
||||
const row = db.prepare('SELECT id FROM imm_words WHERE headword = ?').get(headword) as {
|
||||
id: number;
|
||||
};
|
||||
return row.id;
|
||||
}
|
||||
|
||||
function addOccurrence(
|
||||
db: DatabaseSync,
|
||||
wordId: number,
|
||||
animeTitle: string,
|
||||
count = 1,
|
||||
source: 'file' | 'stream' = 'file',
|
||||
timing: { startMs: number | null; endMs: number | null } = { startMs: 1_000, endMs: 2_000 },
|
||||
): void {
|
||||
const animeId = getOrCreateAnimeRecord(db, {
|
||||
parsedTitle: animeTitle,
|
||||
canonicalTitle: animeTitle,
|
||||
anilistId: null,
|
||||
titleRomaji: null,
|
||||
titleEnglish: null,
|
||||
titleNative: null,
|
||||
metadataJson: null,
|
||||
});
|
||||
const videoId = getOrCreateVideoRecord(
|
||||
db,
|
||||
source === 'file' ? `local:/tmp/${animeTitle}.mkv` : `remote:jellyfin://${animeTitle}`,
|
||||
{
|
||||
canonicalTitle: animeTitle,
|
||||
sourcePath: source === 'file' ? `/tmp/${animeTitle}.mkv` : null,
|
||||
sourceUrl: source === 'file' ? null : `jellyfin://${animeTitle}`,
|
||||
sourceType: source === 'file' ? SOURCE_TYPE_LOCAL : SOURCE_TYPE_REMOTE,
|
||||
},
|
||||
);
|
||||
const { sessionId } = startSessionRecord(db, videoId, 1_700_000_000_000);
|
||||
const lineId = Number(
|
||||
db
|
||||
.prepare(
|
||||
`INSERT INTO imm_subtitle_lines (
|
||||
session_id, video_id, anime_id, line_index, segment_start_ms, segment_end_ms, text
|
||||
) VALUES (?, ?, ?, 0, ?, ?, ?)`,
|
||||
)
|
||||
.run(sessionId, videoId, animeId, timing.startMs, timing.endMs, `line ${animeTitle}`)
|
||||
.lastInsertRowid,
|
||||
);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count) VALUES (?, ?, ?)`,
|
||||
).run(lineId, wordId, count);
|
||||
}
|
||||
|
||||
function query(overrides: Partial<StatsDictionaryWordQuery> = {}): StatsDictionaryWordQuery {
|
||||
return {
|
||||
sort: 'seen',
|
||||
descending: true,
|
||||
hideKnown: false,
|
||||
hideNames: false,
|
||||
hideKana: false,
|
||||
limit: 25,
|
||||
offset: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const headwords = (db: DatabaseSync, overrides: Partial<StatsDictionaryWordQuery>) =>
|
||||
getDictionaryWordPage(db, null, query(overrides)).rows.map((row) => row.headword);
|
||||
|
||||
test('conjugated forms fold into one word with its total and dictionary reading', () => {
|
||||
withDb((db) => {
|
||||
addWord(db, { headword: '見る', word: '見て', reading: 'みて', seen: 263, rank: 36 });
|
||||
const dictionaryId = addWord(db, {
|
||||
headword: '見る',
|
||||
word: '見る',
|
||||
reading: 'みる',
|
||||
seen: 172,
|
||||
rank: 36,
|
||||
});
|
||||
addWord(db, { headword: '見る', word: '見てる', reading: 'みてる', seen: 78, rank: 36 });
|
||||
|
||||
const { rows, total } = getDictionaryWordPage(db, null, query());
|
||||
|
||||
assert.equal(total, 1);
|
||||
assert.equal(rows[0]?.seen, 513);
|
||||
assert.equal(rows[0]?.reading, 'みる');
|
||||
assert.equal(rows[0]?.wordId, dictionaryId);
|
||||
assert.equal(rows[0]?.formCount, 3);
|
||||
assert.deepEqual(
|
||||
rows[0]?.forms.map((form) => form.word),
|
||||
['見て', '見る', '見てる'],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('a form stored under two readings counts as one form', () => {
|
||||
withDb((db) => {
|
||||
const first = addWord(db, { headword: '見る', word: '見て', reading: 'みて', seen: 263 });
|
||||
addWord(db, { headword: '見る', word: '見て', reading: 'み', seen: 122 });
|
||||
addWord(db, { headword: '見る', word: '見た', reading: 'みた', seen: 200 });
|
||||
|
||||
const row = getDictionaryWordPage(db, null, query()).rows[0];
|
||||
assert.equal(row?.formCount, 2);
|
||||
assert.deepEqual(
|
||||
row?.forms.map((form) => [form.word, form.seen]),
|
||||
[
|
||||
['見て', 385],
|
||||
['見た', 200],
|
||||
],
|
||||
);
|
||||
assert.equal(getWordDetail(db, first)?.formCount, 2);
|
||||
});
|
||||
});
|
||||
|
||||
test('a word seen only conjugated has no reading rather than a conjugated one', () => {
|
||||
withDb((db) => {
|
||||
addWord(db, { headword: '待つ', word: '待って', reading: 'まって', seen: 5 });
|
||||
assert.equal(getDictionaryWordPage(db, null, query()).rows[0]?.reading, '');
|
||||
});
|
||||
});
|
||||
|
||||
test('the list reaches rarely seen words, not just the most seen ones', () => {
|
||||
withDb((db) => {
|
||||
for (let index = 0; index < 600; index += 1) {
|
||||
addWord(db, { headword: `語${index}`, seen: 1_000 - index });
|
||||
}
|
||||
const { rows, total } = getDictionaryWordPage(db, null, query({ limit: 10, offset: 590 }));
|
||||
assert.equal(total, 600);
|
||||
assert.equal(rows[9]?.headword, '語599');
|
||||
});
|
||||
});
|
||||
|
||||
test('search matches any form or reading, treating katakana and hiragana alike', () => {
|
||||
withDb((db) => {
|
||||
addWord(db, { headword: '猫', word: '猫', reading: 'ねこ' });
|
||||
addWord(db, { headword: '行く', word: '行った', reading: 'いった' });
|
||||
addWord(db, { headword: '犬', word: '犬', reading: 'いぬ' });
|
||||
|
||||
assert.deepEqual(headwords(db, { search: 'ネコ' }), ['猫']);
|
||||
assert.deepEqual(headwords(db, { search: '行った' }), ['行く']);
|
||||
assert.deepEqual(headwords(db, { search: 'いった' }), ['行く']);
|
||||
});
|
||||
});
|
||||
|
||||
for (const c of [
|
||||
{ name: 'known words', filter: { hideKnown: true }, expected: ['海', '田中', 'なる'] },
|
||||
{ name: 'names', filter: { hideNames: true }, expected: ['山', '海', 'なる'] },
|
||||
{ name: 'kana', filter: { hideKana: true }, expected: ['山', '海', '田中'] },
|
||||
{ name: 'a rank limit', filter: { maxRank: 500 }, expected: ['山', 'なる'] },
|
||||
{ name: 'a part of speech', filter: { partOfSpeech: 'verb' as const }, expected: ['なる'] },
|
||||
]) {
|
||||
test(`the list filters by ${c.name}`, () => {
|
||||
withDb((db) => {
|
||||
addWord(db, { headword: '山', seen: 40, rank: 100 });
|
||||
addWord(db, { headword: '海', seen: 30, rank: 9_000 });
|
||||
addWord(db, { headword: '田中', seen: 20, rank: 800, pos2: '固有名詞|一般' });
|
||||
addWord(db, { headword: 'なる', seen: 10, rank: 16, partOfSpeech: 'verb' });
|
||||
|
||||
const rows = getDictionaryWordPage(db, new Set(['山']), query(c.filter)).rows;
|
||||
assert.deepEqual(
|
||||
rows.map((row) => row.headword),
|
||||
c.expected,
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test('a word with a non-name form is not a name and ranks as that form', () => {
|
||||
withDb((db) => {
|
||||
addWord(db, { headword: '白', word: 'しろ', rank: 49_000, pos2: '固有名詞' });
|
||||
addWord(db, { headword: '白', word: '白', rank: 1_500, pos2: '一般' });
|
||||
|
||||
const row = getDictionaryWordPage(db, null, query({ hideNames: true })).rows[0];
|
||||
assert.equal(row?.isName, false);
|
||||
assert.equal(row?.rank, 1_500);
|
||||
});
|
||||
});
|
||||
|
||||
test('excluded words stay out of the list', () => {
|
||||
withDb((db) => {
|
||||
addWord(db, { headword: '音', seen: 50 });
|
||||
addWord(db, { headword: '海', seen: 10 });
|
||||
assert.deepEqual(headwords(db, {}), ['音', '海']);
|
||||
|
||||
replaceStatsExcludedWords(db, [{ headword: '音', word: '音', reading: '' }]);
|
||||
assert.deepEqual(headwords(db, {}), ['海']);
|
||||
});
|
||||
});
|
||||
|
||||
test('unranked words sort last in both rank directions', () => {
|
||||
withDb((db) => {
|
||||
addWord(db, { headword: '無', rank: null });
|
||||
addWord(db, { headword: '高', rank: 10 });
|
||||
addWord(db, { headword: '低', rank: 900 });
|
||||
|
||||
assert.deepEqual(headwords(db, { sort: 'rank', descending: false }), ['高', '低', '無']);
|
||||
assert.deepEqual(headwords(db, { sort: 'rank', descending: true }), ['低', '高', '無']);
|
||||
});
|
||||
});
|
||||
|
||||
test('titles count distinct titles across every form and can filter and sort', () => {
|
||||
withDb((db) => {
|
||||
const seen = addWord(db, { headword: '見る', word: '見て', seen: 5 });
|
||||
const seenAgain = addWord(db, { headword: '見る', word: '見た', seen: 5 });
|
||||
const single = addWord(db, { headword: '海', seen: 50 });
|
||||
addOccurrence(db, seen, 'A');
|
||||
addOccurrence(db, seenAgain, 'B');
|
||||
addOccurrence(db, single, 'C');
|
||||
|
||||
const page = getDictionaryWordPage(db, null, query());
|
||||
assert.deepEqual(
|
||||
page.rows.map((row) => [row.headword, row.titles]),
|
||||
[
|
||||
['海', 1],
|
||||
['見る', 2],
|
||||
],
|
||||
);
|
||||
const across = getDictionaryWordPage(db, null, query({ sort: 'titles', minTitles: 2 }));
|
||||
assert.deepEqual(
|
||||
across.rows.map((row) => [row.headword, row.titles]),
|
||||
[['見る', 2]],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('a word is stream only when every line, across its forms, came from a stream', () => {
|
||||
withDb((db) => {
|
||||
const streamed = addWord(db, { headword: '流', word: '流れ', seen: 30 });
|
||||
addOccurrence(db, streamed, 'Stream A', 1, 'stream');
|
||||
const mixed = addWord(db, { headword: '混', word: '混ぜ', seen: 20 });
|
||||
const mixedFile = addWord(db, { headword: '混', word: '混ぜた', seen: 10 });
|
||||
addOccurrence(db, mixed, 'Stream B', 1, 'stream');
|
||||
addOccurrence(db, mixedFile, 'File A', 1, 'file');
|
||||
addWord(db, { headword: '無', seen: 1 });
|
||||
|
||||
const streamOnly = Object.fromEntries(
|
||||
getDictionaryWordPage(db, null, query()).rows.map((row) => [row.headword, row.streamOnly]),
|
||||
);
|
||||
// A word with no stored lines isn't blocked by streams.
|
||||
assert.deepEqual(streamOnly, { 流: true, 混: false, 無: false });
|
||||
});
|
||||
});
|
||||
|
||||
test('word detail counts lines from streams across every form', () => {
|
||||
withDb((db) => {
|
||||
const streamed = addWord(db, { headword: '混', word: '混ぜ' });
|
||||
const file = addWord(db, { headword: '混', word: '混ぜた' });
|
||||
addOccurrence(db, streamed, 'Stream A', 3, 'stream');
|
||||
addOccurrence(db, streamed, 'Stream B', 1, 'stream');
|
||||
addOccurrence(db, file, 'File A', 2, 'file');
|
||||
|
||||
const detail = getWordDetail(db, file);
|
||||
assert.equal(detail?.lineCount, 3);
|
||||
assert.equal(detail?.streamLineCount, 2);
|
||||
});
|
||||
});
|
||||
|
||||
test('lines that can be mined come first in examples and search, newest first within each', () => {
|
||||
withDb((db) => {
|
||||
const wordId = addWord(db, { headword: '言う', word: '言った', reading: 'いった' });
|
||||
addOccurrence(db, wordId, 'File old', 1, 'file');
|
||||
addOccurrence(db, wordId, 'File new', 1, 'file');
|
||||
addOccurrence(db, wordId, 'File untimed', 1, 'file', { startMs: null, endMs: null });
|
||||
addOccurrence(db, wordId, 'Stream newest', 1, 'stream');
|
||||
|
||||
const expected = ['File new', 'File old', 'Stream newest', 'File untimed'];
|
||||
assert.deepEqual(
|
||||
getWordOccurrences(db, '言う', '言った', 'いった', 10, 0, true).map((row) => row.videoTitle),
|
||||
expected,
|
||||
);
|
||||
assert.deepEqual(
|
||||
searchSubtitleSentences(db, 'line').map((row) => row.videoTitle),
|
||||
expected,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('legacy rows with NULL text columns list, search, and open without failing', () => {
|
||||
withDb((db) => {
|
||||
const insert = db.prepare(
|
||||
`INSERT INTO imm_words (headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1700000000, 1700000000, 3)`,
|
||||
);
|
||||
const noHeadword = Number(insert.run(null, '猫', null).lastInsertRowid);
|
||||
const noForm = Number(insert.run('犬', null, null).lastInsertRowid);
|
||||
|
||||
const { rows } = getDictionaryWordPage(db, null, query());
|
||||
assert.deepEqual(rows.map((row) => [row.headword, row.forms[0]?.word, row.reading]).sort(), [
|
||||
['犬', '犬', ''],
|
||||
['猫', '猫', ''],
|
||||
]);
|
||||
// NULL must not become searchable text.
|
||||
assert.deepEqual(headwords(db, { search: 'null' }), []);
|
||||
|
||||
for (const id of [noHeadword, noForm]) {
|
||||
assert.equal(getWordDetail(db, id)?.frequency, 3);
|
||||
assert.deepEqual(getWordAnimeAppearances(db, id), []);
|
||||
assert.deepEqual(getSimilarWords(db, id), []);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test('maxSeen and total describe every page, for scaling bars and paging', () => {
|
||||
withDb((db) => {
|
||||
addWord(db, { headword: '一', seen: 100 });
|
||||
addWord(db, { headword: '二', seen: 5 });
|
||||
|
||||
const page = getDictionaryWordPage(db, null, query({ limit: 1, offset: 1 }));
|
||||
assert.equal(page.total, 2);
|
||||
assert.equal(page.maxSeen, 100);
|
||||
assert.deepEqual(
|
||||
page.rows.map((row) => row.headword),
|
||||
['二'],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('the summary counts a word as known when any of its forms is known', () => {
|
||||
withDb((db) => {
|
||||
markLexicalDailyRollupsReady(db);
|
||||
addWord(db, { headword: '見る', word: '見て', reading: 'みて' });
|
||||
addWord(db, { headword: '海' });
|
||||
|
||||
assert.equal(getVocabularySummary(db, new Set(['見て'])).knownWordCount, 1);
|
||||
assert.equal(
|
||||
getDictionaryWordPage(db, new Set(['見て']), query()).rows.filter((row) => row.known).length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('coverage counts seen and known words per frequency band, without names', () => {
|
||||
withDb((db) => {
|
||||
addWord(db, { headword: '一', rank: 500 });
|
||||
addWord(db, { headword: '二', rank: 900 });
|
||||
addWord(db, { headword: '三', rank: 4_000 });
|
||||
addWord(db, { headword: '四', rank: 30_000 });
|
||||
addWord(db, { headword: '田中', rank: 700, pos2: '固有名詞' });
|
||||
addWord(db, { headword: '無', rank: null });
|
||||
|
||||
assert.deepEqual(getVocabularyCoverage(db, new Set(['一', '三', '田中'])), [
|
||||
{ band: 'top1k', seen: 2, known: 1 },
|
||||
{ band: 'top5k', seen: 1, known: 1 },
|
||||
{ band: 'top10k', seen: 0, known: 0 },
|
||||
{ band: 'top20k', seen: 0, known: 0 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test('word detail reports totals and a reading for the dictionary word, not one form', () => {
|
||||
withDb((db) => {
|
||||
const form = addWord(db, {
|
||||
headword: '見る',
|
||||
word: '見て',
|
||||
reading: 'みて',
|
||||
seen: 263,
|
||||
rank: 36,
|
||||
firstSeen: 1_700_000_500,
|
||||
});
|
||||
addWord(db, {
|
||||
headword: '見る',
|
||||
word: '見る',
|
||||
reading: 'みる',
|
||||
seen: 172,
|
||||
rank: 36,
|
||||
firstSeen: 1_700_000_100,
|
||||
});
|
||||
|
||||
const detail = getWordDetail(db, form);
|
||||
|
||||
assert.equal(detail?.word, '見て');
|
||||
assert.equal(detail?.frequency, 435);
|
||||
assert.equal(detail?.dictionaryReading, 'みる');
|
||||
assert.equal(detail?.frequencyRank, 36);
|
||||
assert.equal(detail?.formCount, 2);
|
||||
assert.equal(detail?.firstSeen, 1_700_000_100);
|
||||
assert.equal(detail?.lastSeen, 1_700_000_500);
|
||||
});
|
||||
});
|
||||
|
||||
test('word appearances and examples can cover every form', () => {
|
||||
withDb((db) => {
|
||||
const first = addWord(db, { headword: '見る', word: '見て', reading: 'みて' });
|
||||
const second = addWord(db, { headword: '見る', word: '見た', reading: 'みた' });
|
||||
addOccurrence(db, first, 'A', 3);
|
||||
addOccurrence(db, second, 'B', 2);
|
||||
|
||||
assert.deepEqual(
|
||||
getWordAnimeAppearances(db, first).map((row) => [row.animeTitle, row.occurrenceCount]),
|
||||
[
|
||||
['A', 3],
|
||||
['B', 2],
|
||||
],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("related words leave out a word's own forms and show one entry per word", () => {
|
||||
withDb((db) => {
|
||||
const form = addWord(db, { headword: '見る', word: '見て', reading: 'みて', seen: 10 });
|
||||
addWord(db, { headword: '見る', word: '見た', reading: 'みた', seen: 9 });
|
||||
addWord(db, { headword: '見える', word: '見えた', reading: 'みえた', seen: 4 });
|
||||
addWord(db, { headword: '見える', word: '見える', reading: 'みえる', seen: 3 });
|
||||
|
||||
const similar = getSimilarWords(db, form);
|
||||
|
||||
assert.deepEqual(
|
||||
similar.map((row) => [row.headword, row.frequency]),
|
||||
[['見える', 7]],
|
||||
);
|
||||
assert.equal(similar[0]?.word, '見えた');
|
||||
});
|
||||
});
|
||||
|
||||
test('a reading match ranks a related word without choosing the form it shows', () => {
|
||||
withDb((db) => {
|
||||
const form = addWord(db, { headword: '見る', word: '見て', reading: 'みて', seen: 10 });
|
||||
// Matches on the kanji with every form, and on the reading with its minor form.
|
||||
const kanjiAndReading = addWord(db, {
|
||||
headword: '見える',
|
||||
word: '見えた',
|
||||
reading: 'みえた',
|
||||
seen: 9,
|
||||
});
|
||||
addWord(db, { headword: '見える', word: '見えて', reading: 'みて', seen: 2 });
|
||||
// Matches only on the reading of its minor form, so its other forms must still count.
|
||||
const readingOnly = addWord(db, {
|
||||
headword: '満ちる',
|
||||
word: '満ちた',
|
||||
reading: 'みちた',
|
||||
seen: 7,
|
||||
});
|
||||
addWord(db, { headword: '満ちる', word: '満ちて', reading: 'みて', seen: 1 });
|
||||
// More seen, but no reading match, so it still ranks after both.
|
||||
addWord(db, { headword: '見物', word: '見物', reading: 'けんぶつ', seen: 100 });
|
||||
// A hidden form supplies neither the shown form nor the total.
|
||||
const hidden = addWord(db, { headword: '満ちる', word: '満ち', reading: 'みち', seen: 50 });
|
||||
db.prepare('UPDATE imm_words SET vocabulary_visible = 0 WHERE id = ?').run(hidden);
|
||||
|
||||
assert.deepEqual(
|
||||
getSimilarWords(db, form).map((row) => [row.wordId, row.word, row.reading, row.frequency]),
|
||||
[
|
||||
[kanjiAndReading, '見えた', 'みえた', 11],
|
||||
[readingOnly, '満ちた', 'みちた', 8],
|
||||
[getIdOf(db, '見物'), '見物', 'けんぶつ', 100],
|
||||
],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('related words show the same form when a word has tied top forms', () => {
|
||||
withDb((db) => {
|
||||
const form = addWord(db, { headword: '見る', word: '見て', reading: 'みて', seen: 10 });
|
||||
const first = addWord(db, { headword: '見える', word: '見えた', reading: 'みえた', seen: 5 });
|
||||
addWord(db, { headword: '見える', word: '見えて', reading: 'みえて', seen: 5 });
|
||||
|
||||
assert.equal(getSimilarWords(db, form)[0]?.wordId, first);
|
||||
});
|
||||
});
|
||||
|
||||
test('related words leave out rows with a blank headword', () => {
|
||||
withDb((db) => {
|
||||
const form = addWord(db, { headword: '見る', word: '見て', reading: 'みて', seen: 10 });
|
||||
for (const [index, headword] of ['', ' ', '\t', '\v', '\f', ' '].entries()) {
|
||||
addWord(db, { headword, word: `空${index}`, reading: 'みて', seen: 50 });
|
||||
}
|
||||
addWord(db, { headword: '満ちる', word: '満ちて', reading: 'みて', seen: 1 });
|
||||
|
||||
assert.deepEqual(
|
||||
getSimilarWords(db, form).map((row) => row.headword),
|
||||
['満ちる'],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('all-forms examples include legacy rows stored without a headword', () => {
|
||||
withDb((db) => {
|
||||
const insert = db.prepare(
|
||||
`INSERT INTO imm_words (headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1700000000, 1700000000, 1)`,
|
||||
);
|
||||
const titlesFor = (headword: string): string[] =>
|
||||
getWordOccurrences(db, headword, '', '', 10, 0, true)
|
||||
.map((row) => row.videoTitle)
|
||||
.sort();
|
||||
|
||||
addOccurrence(db, Number(insert.run('猫', '猫', 'ねこ').lastInsertRowid), 'Headword');
|
||||
addOccurrence(db, Number(insert.run(null, '猫', 'ねこ').lastInsertRowid), 'Null headword');
|
||||
addOccurrence(db, Number(insert.run(' ', '猫', 'ねこ').lastInsertRowid), 'Blank headword');
|
||||
addOccurrence(db, Number(insert.run(null, null, 'いぬ').lastInsertRowid), 'Reading only');
|
||||
addOccurrence(db, Number(insert.run('犬', '猫', 'いぬ').lastInsertRowid), 'Other headword');
|
||||
|
||||
assert.deepEqual(titlesFor('猫'), ['Blank headword', 'Headword', 'Null headword']);
|
||||
assert.deepEqual(titlesFor('いぬ'), ['Reading only']);
|
||||
assert.deepEqual(titlesFor('犬'), ['Other headword']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,375 @@
|
||||
import type {
|
||||
StatsDictionaryWordPos,
|
||||
StatsDictionaryWordQuery,
|
||||
} from '../../../types/stats-http-contract';
|
||||
import type {
|
||||
DictionaryWordForm,
|
||||
DictionaryWordPage,
|
||||
DictionaryWordRow,
|
||||
VocabularyCoverageBand,
|
||||
VocabularyCoverageBandId,
|
||||
} from '../../../types/stats-wire';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import {
|
||||
databaseWriteVersion,
|
||||
DICTIONARY_WORD_KEY_SQL,
|
||||
getExcludedWordIds,
|
||||
isProperNounPos2,
|
||||
mergeFormsByWord,
|
||||
NO_LOCAL_FILE_SQL,
|
||||
} from './query-lexical';
|
||||
import { forEachIdChunk, makePlaceholders } from './query-shared';
|
||||
import { nowMs } from './time';
|
||||
|
||||
const MAX_FORMS_PER_ROW = 8;
|
||||
// Counting titles for every word walks every occurrence (about 900ms on 500k rows),
|
||||
// and playback writes constantly, so that count tolerates being this stale.
|
||||
const ALL_TITLE_COUNTS_MAX_AGE_MS = 5 * 60_000;
|
||||
|
||||
const COVERAGE_BAND_LIMITS: ReadonlyArray<{ band: VocabularyCoverageBandId; maxRank: number }> = [
|
||||
{ band: 'top1k', maxRank: 1_000 },
|
||||
{ band: 'top5k', maxRank: 5_000 },
|
||||
{ band: 'top10k', maxRank: 10_000 },
|
||||
{ band: 'top20k', maxRank: 20_000 },
|
||||
];
|
||||
|
||||
const PART_OF_SPEECH_GROUPS: Record<StatsDictionaryWordPos, ReadonlySet<string>> = {
|
||||
noun: new Set(['noun']),
|
||||
verb: new Set(['verb']),
|
||||
adjective: new Set(['i_adjective', 'na_adjective']),
|
||||
};
|
||||
|
||||
const KANA_ONLY_TEXT = /^[\p{Script=Hiragana}\p{Script=Katakana}ーゝゞヽヾ]+$/u;
|
||||
|
||||
interface DictionaryWordFormRow {
|
||||
id: number;
|
||||
key: string;
|
||||
word: string;
|
||||
reading: string;
|
||||
seen: number | null;
|
||||
rank: number | null;
|
||||
pos2: string | null;
|
||||
partOfSpeech: string | null;
|
||||
firstSeen: number | null;
|
||||
lastSeen: number | null;
|
||||
}
|
||||
|
||||
interface DictionaryWord {
|
||||
key: string;
|
||||
seen: number;
|
||||
rank: number | null;
|
||||
isName: boolean;
|
||||
isKana: boolean;
|
||||
firstSeen: number;
|
||||
lastSeen: number;
|
||||
searchText: string;
|
||||
/** Most seen first. */
|
||||
forms: DictionaryWordFormRow[];
|
||||
}
|
||||
|
||||
const wordCache = new WeakMap<DatabaseSync, { version: string; words: DictionaryWord[] }>();
|
||||
const allTitleCountsCache = new WeakMap<
|
||||
DatabaseSync,
|
||||
{ version: string; computedAtMs: number; counts: Map<string, number> }
|
||||
>();
|
||||
|
||||
function toSeconds(value: number | null): number {
|
||||
const numeric = Number(value ?? 0);
|
||||
return Math.abs(numeric) >= 10_000_000_000 ? numeric / 1000 : numeric;
|
||||
}
|
||||
|
||||
/** Lowercase hiragana, so a search matches regardless of kana type or case. */
|
||||
function normalizeSearchText(text: string): string {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[ァ-ヶ]/g, (char) => String.fromCharCode(char.charCodeAt(0) - 0x60));
|
||||
}
|
||||
|
||||
function loadDictionaryWords(db: DatabaseSync): DictionaryWord[] {
|
||||
const version = databaseWriteVersion(db);
|
||||
const cached = wordCache.get(db);
|
||||
if (cached?.version === version) return cached.words;
|
||||
|
||||
const excludedIds = getExcludedWordIds(db);
|
||||
const exclusion =
|
||||
excludedIds.length > 0 ? `AND id NOT IN (${makePlaceholders(excludedIds)})` : '';
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id, ${DICTIONARY_WORD_KEY_SQL} AS key,
|
||||
COALESCE(NULLIF(TRIM(word), ''), ${DICTIONARY_WORD_KEY_SQL}) AS word,
|
||||
COALESCE(reading, '') AS reading, frequency AS seen,
|
||||
frequency_rank AS rank, pos2, part_of_speech AS partOfSpeech,
|
||||
first_seen AS firstSeen, last_seen AS lastSeen
|
||||
FROM imm_words
|
||||
WHERE vocabulary_visible = 1 ${exclusion}
|
||||
ORDER BY frequency DESC, id
|
||||
`,
|
||||
)
|
||||
.all(...excludedIds) as unknown as DictionaryWordFormRow[];
|
||||
|
||||
const byKey = new Map<string, DictionaryWord>();
|
||||
for (const row of rows) {
|
||||
if (!row.key) continue;
|
||||
const firstSeen = toSeconds(row.firstSeen);
|
||||
const lastSeen = toSeconds(row.lastSeen);
|
||||
const isName = isProperNounPos2(row.pos2);
|
||||
const entry = byKey.get(row.key);
|
||||
if (entry) {
|
||||
entry.seen += row.seen ?? 0;
|
||||
entry.forms.push(row);
|
||||
entry.firstSeen = Math.min(entry.firstSeen, firstSeen);
|
||||
entry.lastSeen = Math.max(entry.lastSeen, lastSeen);
|
||||
entry.isName &&= isName;
|
||||
entry.searchText += `\n${row.word}\n${row.reading}`;
|
||||
} else {
|
||||
byKey.set(row.key, {
|
||||
key: row.key,
|
||||
seen: row.seen ?? 0,
|
||||
rank: null,
|
||||
isName,
|
||||
isKana: KANA_ONLY_TEXT.test(row.key),
|
||||
firstSeen,
|
||||
lastSeen,
|
||||
searchText: `${row.key}\n${row.word}\n${row.reading}`,
|
||||
forms: [row],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const words = [...byKey.values()];
|
||||
for (const word of words) {
|
||||
word.searchText = normalizeSearchText(word.searchText);
|
||||
word.rank = bestRank(word.forms);
|
||||
}
|
||||
wordCache.set(db, { version, words });
|
||||
return words;
|
||||
}
|
||||
|
||||
/** Rank from the word's non-name forms when it has any, so 白 ranks as the color. */
|
||||
function bestRank(forms: DictionaryWordFormRow[]): number | null {
|
||||
const nonNames = forms.filter((form) => !isProperNounPos2(form.pos2));
|
||||
let best: number | null = null;
|
||||
for (const form of nonNames.length > 0 ? nonNames : forms) {
|
||||
if (form.rank !== null && form.rank > 0 && (best === null || form.rank < best)) {
|
||||
best = form.rank;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
function isKnown(word: DictionaryWord, knownWords: ReadonlySet<string> | null): boolean {
|
||||
if (!knownWords) return false;
|
||||
return knownWords.has(word.key) || word.forms.some((form) => knownWords.has(form.word));
|
||||
}
|
||||
|
||||
function dictionaryForm(word: DictionaryWord): DictionaryWordFormRow | undefined {
|
||||
return word.forms.find((form) => form.word === word.key);
|
||||
}
|
||||
|
||||
function getAllTitleCounts(db: DatabaseSync): Map<string, number> {
|
||||
const version = databaseWriteVersion(db);
|
||||
const cached = allTitleCountsCache.get(db);
|
||||
if (
|
||||
cached &&
|
||||
(cached.version === version || nowMs() - cached.computedAtMs < ALL_TITLE_COUNTS_MAX_AGE_MS)
|
||||
) {
|
||||
return cached.counts;
|
||||
}
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT ${DICTIONARY_WORD_KEY_SQL} AS key, COUNT(DISTINCT sl.anime_id) AS titles
|
||||
FROM imm_words
|
||||
JOIN imm_word_line_occurrences o ON o.word_id = imm_words.id
|
||||
JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
|
||||
WHERE sl.anime_id IS NOT NULL AND vocabulary_visible = 1
|
||||
GROUP BY key
|
||||
`,
|
||||
)
|
||||
.all() as Array<{ key: string; titles: number }>;
|
||||
const counts = new Map(rows.map((row) => [row.key, row.titles]));
|
||||
allTitleCountsCache.set(db, { version, computedAtMs: nowMs(), counts });
|
||||
return counts;
|
||||
}
|
||||
|
||||
function getTitleCountsForWords(db: DatabaseSync, words: DictionaryWord[]): Map<string, number> {
|
||||
const keyByWordId = new Map<number, string>();
|
||||
for (const word of words) {
|
||||
for (const form of word.forms) keyByWordId.set(form.id, word.key);
|
||||
}
|
||||
const animeByKey = new Map<string, Set<number>>();
|
||||
forEachIdChunk([...keyByWordId.keys()], (chunk) => {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT DISTINCT o.word_id AS wordId, sl.anime_id AS animeId
|
||||
FROM imm_word_line_occurrences o
|
||||
JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
|
||||
WHERE o.word_id IN (${makePlaceholders(chunk)}) AND sl.anime_id IS NOT NULL
|
||||
`,
|
||||
)
|
||||
.all(...chunk) as Array<{ wordId: number; animeId: number }>;
|
||||
for (const row of rows) {
|
||||
const key = keyByWordId.get(row.wordId);
|
||||
if (key === undefined) continue;
|
||||
const anime = animeByKey.get(key) ?? new Set<number>();
|
||||
anime.add(row.animeId);
|
||||
animeByKey.set(key, anime);
|
||||
}
|
||||
});
|
||||
return new Map([...animeByKey].map(([key, anime]) => [key, anime.size]));
|
||||
}
|
||||
|
||||
/** Words whose every stored line came from a stream. Words with no lines don't count. */
|
||||
function getStreamOnlyWords(db: DatabaseSync, words: DictionaryWord[]): Set<string> {
|
||||
const keyByWordId = new Map<number, string>();
|
||||
for (const word of words) {
|
||||
for (const form of word.forms) keyByWordId.set(form.id, word.key);
|
||||
}
|
||||
const withLines = new Set<string>();
|
||||
const withFile = new Set<string>();
|
||||
forEachIdChunk([...keyByWordId.keys()], (chunk) => {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT o.word_id AS wordId, MIN(${NO_LOCAL_FILE_SQL}) AS allStreamed
|
||||
FROM imm_word_line_occurrences o
|
||||
JOIN imm_subtitle_lines l ON l.line_id = o.line_id
|
||||
JOIN imm_videos v ON v.video_id = l.video_id
|
||||
WHERE o.word_id IN (${makePlaceholders(chunk)})
|
||||
GROUP BY o.word_id
|
||||
`,
|
||||
)
|
||||
.all(...chunk) as Array<{ wordId: number; allStreamed: number }>;
|
||||
for (const row of rows) {
|
||||
const key = keyByWordId.get(row.wordId);
|
||||
if (key === undefined) continue;
|
||||
withLines.add(key);
|
||||
if (row.allStreamed === 0) withFile.add(key);
|
||||
}
|
||||
});
|
||||
return new Set([...withLines].filter((key) => !withFile.has(key)));
|
||||
}
|
||||
|
||||
function compareWords(
|
||||
left: DictionaryWord,
|
||||
right: DictionaryWord,
|
||||
query: StatsDictionaryWordQuery,
|
||||
titleCounts: Map<string, number> | null,
|
||||
): number {
|
||||
const sign = query.descending ? -1 : 1;
|
||||
const byText = left.key < right.key ? -1 : left.key > right.key ? 1 : 0;
|
||||
switch (query.sort) {
|
||||
case 'headword':
|
||||
return byText * sign;
|
||||
case 'rank':
|
||||
// Unranked words go last in either direction.
|
||||
if (left.rank === null || right.rank === null) {
|
||||
return (left.rank === null ? 1 : 0) - (right.rank === null ? 1 : 0) || byText;
|
||||
}
|
||||
return (left.rank - right.rank) * sign || byText;
|
||||
case 'titles':
|
||||
return (
|
||||
((titleCounts?.get(left.key) ?? 0) - (titleCounts?.get(right.key) ?? 0)) * sign ||
|
||||
right.seen - left.seen ||
|
||||
byText
|
||||
);
|
||||
case 'firstSeen':
|
||||
return (left.firstSeen - right.firstSeen) * sign || byText;
|
||||
case 'lastSeen':
|
||||
return (left.lastSeen - right.lastSeen) * sign || byText;
|
||||
case 'seen':
|
||||
return (left.seen - right.seen) * sign || byText;
|
||||
}
|
||||
}
|
||||
|
||||
function toRow(
|
||||
word: DictionaryWord,
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
titleCounts: Map<string, number>,
|
||||
streamOnlyWords: ReadonlySet<string>,
|
||||
): DictionaryWordRow {
|
||||
const dictionary = dictionaryForm(word);
|
||||
const topForm = word.forms[0]!;
|
||||
const reading = word.forms.find(
|
||||
(form) => form.word === word.key && form.reading.trim() !== '',
|
||||
)?.reading;
|
||||
const forms = mergeFormsByWord<DictionaryWordForm>(
|
||||
word.forms.map((form) => ({
|
||||
wordId: form.id,
|
||||
word: form.word,
|
||||
reading: form.reading,
|
||||
seen: form.seen ?? 0,
|
||||
})),
|
||||
);
|
||||
return {
|
||||
wordId: (dictionary ?? topForm).id,
|
||||
headword: word.key,
|
||||
reading: reading ?? '',
|
||||
partOfSpeech: topForm.partOfSpeech,
|
||||
rank: word.rank,
|
||||
seen: word.seen,
|
||||
titles: titleCounts.get(word.key) ?? 0,
|
||||
firstSeen: word.firstSeen,
|
||||
lastSeen: word.lastSeen,
|
||||
known: isKnown(word, knownWords),
|
||||
isName: word.isName,
|
||||
streamOnly: streamOnlyWords.has(word.key),
|
||||
formCount: forms.length,
|
||||
forms: forms.slice(0, MAX_FORMS_PER_ROW),
|
||||
};
|
||||
}
|
||||
|
||||
/** One page of dictionary words, folded across their forms and filtered on the server. */
|
||||
export function getDictionaryWordPage(
|
||||
db: DatabaseSync,
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
query: StatsDictionaryWordQuery,
|
||||
): DictionaryWordPage {
|
||||
const needle = query.search ? normalizeSearchText(query.search) : '';
|
||||
const posGroup = query.partOfSpeech ? PART_OF_SPEECH_GROUPS[query.partOfSpeech] : null;
|
||||
const needsAllTitleCounts = query.sort === 'titles' || query.minTitles !== undefined;
|
||||
const allTitleCounts = needsAllTitleCounts ? getAllTitleCounts(db) : null;
|
||||
|
||||
const matches = loadDictionaryWords(db).filter((word) => {
|
||||
if (query.hideNames && word.isName) return false;
|
||||
if (query.hideKana && word.isKana) return false;
|
||||
if (query.maxRank !== undefined && (word.rank === null || word.rank > query.maxRank)) {
|
||||
return false;
|
||||
}
|
||||
if (posGroup && !posGroup.has(word.forms[0]?.partOfSpeech ?? '')) return false;
|
||||
if (needle && !word.searchText.includes(needle)) return false;
|
||||
if (query.minTitles !== undefined && (allTitleCounts?.get(word.key) ?? 0) < query.minTitles) {
|
||||
return false;
|
||||
}
|
||||
return !(query.hideKnown && isKnown(word, knownWords));
|
||||
});
|
||||
|
||||
matches.sort((left, right) => compareWords(left, right, query, allTitleCounts));
|
||||
const pageWords = matches.slice(query.offset, query.offset + query.limit);
|
||||
const titleCounts = allTitleCounts ?? getTitleCountsForWords(db, pageWords);
|
||||
const streamOnlyWords = getStreamOnlyWords(db, pageWords);
|
||||
return {
|
||||
rows: pageWords.map((word) => toRow(word, knownWords, titleCounts, streamOnlyWords)),
|
||||
total: matches.length,
|
||||
maxSeen: matches.reduce((max, word) => Math.max(max, word.seen), 0),
|
||||
};
|
||||
}
|
||||
|
||||
/** How many seen words in each frequency band are known. Names don't count. */
|
||||
export function getVocabularyCoverage(
|
||||
db: DatabaseSync,
|
||||
knownWords: ReadonlySet<string>,
|
||||
): VocabularyCoverageBand[] {
|
||||
const bands = COVERAGE_BAND_LIMITS.map(({ band }) => ({ band, seen: 0, known: 0 }));
|
||||
for (const word of loadDictionaryWords(db)) {
|
||||
if (word.isName || word.rank === null) continue;
|
||||
const index = COVERAGE_BAND_LIMITS.findIndex(({ maxRank }) => word.rank! <= maxRank);
|
||||
const band = bands[index];
|
||||
if (!band) continue;
|
||||
band.seen += 1;
|
||||
if (isKnown(word, knownWords)) band.known += 1;
|
||||
}
|
||||
return bands;
|
||||
}
|
||||
@@ -624,7 +624,14 @@ export interface WordDetailRow {
|
||||
pos1: string | null;
|
||||
pos2: string | null;
|
||||
pos3: string | null;
|
||||
/** Times seen across every form of this dictionary word. */
|
||||
frequency: number;
|
||||
frequencyRank: number | null;
|
||||
dictionaryReading: string;
|
||||
formCount: number;
|
||||
forms: Array<{ wordId: number; word: string; reading: string; seen: number }>;
|
||||
lineCount: number;
|
||||
streamLineCount: number;
|
||||
firstSeen: number;
|
||||
lastSeen: number;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
enrichSessionsWithKnownWordMetrics,
|
||||
loadKnownWordsSet,
|
||||
parseEventTypesQuery,
|
||||
parseDayRange,
|
||||
parseIntQuery,
|
||||
parsePositiveId,
|
||||
parseTrendFillEmpty,
|
||||
@@ -64,17 +65,26 @@ export function registerStatsAnalyticsRoutes(
|
||||
});
|
||||
|
||||
app.get('/api/stats/trends/dashboard', async (c) => {
|
||||
const range = parseTrendRange(c.req.query('range'));
|
||||
const range = parseTrendRange(
|
||||
c.req.query('range'),
|
||||
c.req.query('fromDay'),
|
||||
c.req.query('toDay'),
|
||||
);
|
||||
const groupBy = parseTrendGroupBy(c.req.query('groupBy'));
|
||||
const fillEmpty = parseTrendFillEmpty(c.req.query('fillEmpty'));
|
||||
const title = c.req.query('title')?.trim() || null;
|
||||
return c.json(
|
||||
statsJson('trendsDashboard', await tracker.getTrendsDashboard(range, groupBy, fillEmpty)),
|
||||
statsJson(
|
||||
'trendsDashboard',
|
||||
await tracker.getTrendsDashboard(range, groupBy, fillEmpty, title),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
app.get('/api/stats/sessions', async (c) => {
|
||||
const limit = parseIntQuery(c.req.query('limit'), 50, 500);
|
||||
const rawSessions = await tracker.getSessionSummaries(limit);
|
||||
const dayRange = parseDayRange(c.req.query('fromDay'), c.req.query('toDay')) ?? undefined;
|
||||
const rawSessions = await tracker.getSessionSummaries(limit, dayRange);
|
||||
const sessions = await enrichSessionsWithKnownWordMetrics(
|
||||
tracker,
|
||||
rawSessions,
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
parseBooleanQuery,
|
||||
parseDuplicateLineCleanupBody,
|
||||
parseExcludedWordsBody,
|
||||
parseDictionaryWordQuery,
|
||||
parseIntQuery,
|
||||
parsePositiveId,
|
||||
parsePositiveIdList,
|
||||
@@ -49,6 +50,20 @@ export function registerStatsLibraryRoutes(
|
||||
return c.json(statsJson('vocabularyCharts', await tracker.getVocabularyChartData()));
|
||||
});
|
||||
|
||||
app.get('/api/stats/vocabulary/words', async (c) => {
|
||||
const page = await tracker.getDictionaryWords(
|
||||
loadKnownWordsSet(options?.knownWordCachePath),
|
||||
parseDictionaryWordQuery((name) => c.req.query(name)),
|
||||
);
|
||||
return c.json(statsJson('dictionaryWords', page));
|
||||
});
|
||||
|
||||
app.get('/api/stats/vocabulary/coverage', async (c) => {
|
||||
const knownWords = loadKnownWordsSet(options?.knownWordCachePath);
|
||||
const coverage = knownWords ? await tracker.getVocabularyCoverage(knownWords) : [];
|
||||
return c.json(statsJson('vocabularyCoverage', coverage));
|
||||
});
|
||||
|
||||
app.get('/api/stats/excluded-words', async (c) => {
|
||||
return c.json(statsJson('excludedWords', await tracker.getStatsExcludedWords()));
|
||||
});
|
||||
@@ -78,12 +93,23 @@ export function registerStatsLibraryRoutes(
|
||||
const headword = (c.req.query('headword') ?? '').trim();
|
||||
const word = (c.req.query('word') ?? '').trim();
|
||||
const reading = (c.req.query('reading') ?? '').trim();
|
||||
if (!headword || !word) {
|
||||
const allForms = c.req.query('allForms') === 'true';
|
||||
// Legacy rows can lack a headword or form, so an all-forms lookup falls back the same
|
||||
// way the word list does. An exact lookup can only match a row that has both.
|
||||
const dictionaryWord = headword || word || reading;
|
||||
if (allForms ? !dictionaryWord : !headword || !word) {
|
||||
return c.json(statsJson('wordOccurrences', []), 400);
|
||||
}
|
||||
const limit = parseIntQuery(c.req.query('limit'), 50, 500);
|
||||
const offset = parseIntQuery(c.req.query('offset'), 0, 10_000);
|
||||
const occurrences = await tracker.getWordOccurrences(headword, word, reading, limit, offset);
|
||||
const occurrences = await tracker.getWordOccurrences(
|
||||
dictionaryWord,
|
||||
word,
|
||||
reading,
|
||||
limit,
|
||||
offset,
|
||||
allForms,
|
||||
);
|
||||
return c.json(statsJson('wordOccurrences', occurrences));
|
||||
});
|
||||
|
||||
@@ -102,7 +128,7 @@ export function registerStatsLibraryRoutes(
|
||||
});
|
||||
|
||||
app.get('/api/stats/kanji', async (c) => {
|
||||
const limit = parseIntQuery(c.req.query('limit'), 100, 500);
|
||||
const limit = parseIntQuery(c.req.query('limit'), 100, 10_000);
|
||||
const kanji = await tracker.getKanjiStats(limit);
|
||||
return c.json(statsJson('kanji', kanji));
|
||||
});
|
||||
|
||||
@@ -12,6 +12,13 @@ import {
|
||||
} from '../../../anki-integration/known-word-cache-format.js';
|
||||
import { createLogger } from '../../../logger.js';
|
||||
import type { AnkiConnectConfig } from '../../../types.js';
|
||||
import type {
|
||||
StatsDayRange,
|
||||
StatsDictionaryWordQuery,
|
||||
StatsDictionaryWordSort,
|
||||
StatsTrendGroupBy,
|
||||
StatsTrendRange,
|
||||
} from '../../../types/stats-http-contract.js';
|
||||
import type { StatsExcludedWord } from '../../../types/stats-wire.js';
|
||||
import type { ImmersionTrackerService } from '../immersion-tracker-service.js';
|
||||
import { splitSentenceSearchTerms } from '../immersion-tracker/query-lexical.js';
|
||||
@@ -58,14 +65,61 @@ export function parsePositiveId(raw: string | undefined): number | null {
|
||||
return isPositiveSafeInteger(value) && String(value) === raw ? value : null;
|
||||
}
|
||||
|
||||
export function parseTrendRange(raw: string | undefined): '7d' | '30d' | '90d' | '365d' | 'all' {
|
||||
/** Parses an inclusive `fromDay`/`toDay` pair; anything malformed or reversed means no range. */
|
||||
export function parseDayRange(
|
||||
rawFrom: string | undefined,
|
||||
rawTo: string | undefined,
|
||||
): StatsDayRange | null {
|
||||
const startDay = parsePositiveId(rawFrom);
|
||||
const endDay = parsePositiveId(rawTo);
|
||||
return startDay !== null && endDay !== null && startDay <= endDay ? { startDay, endDay } : null;
|
||||
}
|
||||
|
||||
export function parseTrendRange(
|
||||
raw: string | undefined,
|
||||
rawFrom?: string,
|
||||
rawTo?: string,
|
||||
): StatsTrendRange {
|
||||
if (raw === 'custom') return parseDayRange(rawFrom, rawTo) ?? '30d';
|
||||
return raw === '7d' || raw === '30d' || raw === '90d' || raw === '365d' || raw === 'all'
|
||||
? raw
|
||||
: '30d';
|
||||
}
|
||||
|
||||
export function parseTrendGroupBy(raw: string | undefined): 'day' | 'month' {
|
||||
return raw === 'month' ? 'month' : 'day';
|
||||
export function parseTrendGroupBy(raw: string | undefined): StatsTrendGroupBy {
|
||||
return raw === 'month' || raw === 'week' ? raw : 'day';
|
||||
}
|
||||
|
||||
const DICTIONARY_WORD_SORTS: readonly StatsDictionaryWordSort[] = [
|
||||
'seen',
|
||||
'rank',
|
||||
'titles',
|
||||
'firstSeen',
|
||||
'lastSeen',
|
||||
'headword',
|
||||
];
|
||||
const DICTIONARY_WORD_PAGE_LIMIT = 200;
|
||||
|
||||
/** Reads the word list query string. Anything unrecognized falls back to the default. */
|
||||
export function parseDictionaryWordQuery(
|
||||
read: (name: string) => string | undefined,
|
||||
): StatsDictionaryWordQuery {
|
||||
const sort = DICTIONARY_WORD_SORTS.find((value) => value === read('sort')) ?? 'seen';
|
||||
const pos = read('pos');
|
||||
const flag = (name: string): boolean => read(name) === 'true';
|
||||
return {
|
||||
search: read('search')?.trim() || undefined,
|
||||
sort,
|
||||
descending: read('order') !== 'asc',
|
||||
hideKnown: flag('hideKnown'),
|
||||
hideNames: flag('hideNames'),
|
||||
hideKana: flag('hideKana'),
|
||||
partOfSpeech: pos === 'noun' || pos === 'verb' || pos === 'adjective' ? pos : undefined,
|
||||
maxRank: parsePositiveId(read('maxRank')) ?? undefined,
|
||||
minTitles: parsePositiveId(read('minTitles')) ?? undefined,
|
||||
limit: parseIntQuery(read('limit'), 25, DICTIONARY_WORD_PAGE_LIMIT),
|
||||
offset: parseIntQuery(read('offset'), 0),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseTrendFillEmpty(raw: string | undefined): boolean {
|
||||
|
||||
@@ -3,6 +3,7 @@ import type {
|
||||
AnimeLibraryItem,
|
||||
AnimeWord,
|
||||
DailyRollup,
|
||||
DictionaryWordPage,
|
||||
EpisodeDetailData,
|
||||
EpisodesPerDay,
|
||||
KanjiDetailData,
|
||||
@@ -22,6 +23,7 @@ import type {
|
||||
StatsExcludedWord,
|
||||
StreakCalendarDay,
|
||||
TrendsDashboardData,
|
||||
VocabularyCoverageBand,
|
||||
VocabularyEntry,
|
||||
VocabularyOccurrenceEntry,
|
||||
WatchTimePerAnime,
|
||||
@@ -29,8 +31,14 @@ import type {
|
||||
} from './stats-wire';
|
||||
import type { TmdbMediaType } from '../shared/media-kind';
|
||||
|
||||
export type StatsTrendRange = '7d' | '30d' | '90d' | '365d' | 'all';
|
||||
export type StatsTrendGroupBy = 'day' | 'month';
|
||||
export type StatsTrendPresetRange = '7d' | '30d' | '90d' | '365d' | 'all';
|
||||
/** Inclusive local calendar days since the Unix epoch. */
|
||||
export interface StatsDayRange {
|
||||
startDay: number;
|
||||
endDay: number;
|
||||
}
|
||||
export type StatsTrendRange = StatsTrendPresetRange | StatsDayRange;
|
||||
export type StatsTrendGroupBy = 'day' | 'week' | 'month';
|
||||
export type StatsMineMode = 'word' | 'sentence' | 'audio';
|
||||
|
||||
/** Body of `POST /api/stats/maintenance/duplicate-lines`. */
|
||||
@@ -63,12 +71,37 @@ export interface StatsVocabularySummary {
|
||||
|
||||
export interface StatsVocabularyCharts {
|
||||
ready: boolean;
|
||||
topWords: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||
topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||
newWordsTimeline: Array<{ epochDay: number; wordCount: number }>;
|
||||
newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>;
|
||||
}
|
||||
|
||||
export type StatsDictionaryWordSort =
|
||||
| 'seen'
|
||||
| 'rank'
|
||||
| 'titles'
|
||||
| 'firstSeen'
|
||||
| 'lastSeen'
|
||||
| 'headword';
|
||||
export type StatsDictionaryWordPos = 'noun' | 'verb' | 'adjective';
|
||||
|
||||
/** Query for `GET /api/stats/vocabulary/words`. */
|
||||
export interface StatsDictionaryWordQuery {
|
||||
/** Matches any form or reading, treating katakana and hiragana alike. */
|
||||
search?: string;
|
||||
sort: StatsDictionaryWordSort;
|
||||
descending: boolean;
|
||||
hideKnown: boolean;
|
||||
hideNames: boolean;
|
||||
hideKana: boolean;
|
||||
partOfSpeech?: StatsDictionaryWordPos;
|
||||
/** Keep only ranked words at or better than this frequency rank. */
|
||||
maxRank?: number;
|
||||
/** Keep only words seen in at least this many titles. */
|
||||
minTitles?: number;
|
||||
limit: number;
|
||||
offset: number;
|
||||
}
|
||||
|
||||
export interface StatsAnilistSearchResult {
|
||||
id: number;
|
||||
episodes: number | null;
|
||||
@@ -204,6 +237,8 @@ export interface StatsJsonResponseMap {
|
||||
vocabulary: VocabularyEntry[];
|
||||
vocabularySummary: StatsVocabularySummary;
|
||||
vocabularyCharts: StatsVocabularyCharts;
|
||||
dictionaryWords: DictionaryWordPage;
|
||||
vocabularyCoverage: VocabularyCoverageBand[];
|
||||
excludedWords: StatsExcludedWord[];
|
||||
setExcludedWords: StatsOkResponse;
|
||||
duplicateLineCleanup: StatsDuplicateLineCleanupResult;
|
||||
@@ -259,13 +294,15 @@ export interface StatsHttpClient {
|
||||
getOverview: () => Promise<OverviewData>;
|
||||
getDailyRollups: (limit?: number) => Promise<DailyRollup[]>;
|
||||
getMonthlyRollups: (limit?: number) => Promise<MonthlyRollup[]>;
|
||||
getSessions: (limit?: number) => Promise<SessionSummary[]>;
|
||||
getSessions: (limit?: number, dayRange?: StatsDayRange) => Promise<SessionSummary[]>;
|
||||
getSessionTimeline: (id: number, limit?: number) => Promise<SessionTimelinePoint[]>;
|
||||
getSessionEvents: (id: number, limit?: number, eventTypes?: number[]) => Promise<SessionEvent[]>;
|
||||
getSessionKnownWordsTimeline: (id: number) => Promise<StatsSessionKnownWordsTimelinePoint[]>;
|
||||
getVocabulary: (limit?: number) => Promise<VocabularyEntry[]>;
|
||||
getVocabularySummary: () => Promise<StatsVocabularySummary>;
|
||||
getVocabularyCharts: () => Promise<StatsVocabularyCharts>;
|
||||
getDictionaryWords: (query: StatsDictionaryWordQuery) => Promise<DictionaryWordPage>;
|
||||
getVocabularyCoverage: () => Promise<VocabularyCoverageBand[]>;
|
||||
getExcludedWords: () => Promise<StatsExcludedWord[]>;
|
||||
setExcludedWords: (words: StatsExcludedWord[]) => Promise<void>;
|
||||
cleanupDuplicateLines: (
|
||||
@@ -277,6 +314,7 @@ export interface StatsHttpClient {
|
||||
reading: string,
|
||||
limit?: number,
|
||||
offset?: number,
|
||||
allForms?: boolean,
|
||||
) => Promise<VocabularyOccurrenceEntry[]>;
|
||||
searchSentences: (
|
||||
query: string,
|
||||
@@ -306,6 +344,7 @@ export interface StatsHttpClient {
|
||||
range: StatsTrendRange,
|
||||
groupBy: StatsTrendGroupBy,
|
||||
fillEmpty?: boolean,
|
||||
title?: string | null,
|
||||
) => Promise<TrendsDashboardData>;
|
||||
getWordDetail: (wordId: number) => Promise<WordDetailData>;
|
||||
getKanjiDetail: (kanjiId: number) => Promise<KanjiDetailData>;
|
||||
|
||||
+126
-16
@@ -78,6 +78,55 @@ export interface VocabularyEntry {
|
||||
lastSeen: number;
|
||||
}
|
||||
|
||||
/** One written form of a dictionary word, such as 見て for 見る. */
|
||||
export interface DictionaryWordForm {
|
||||
wordId: number;
|
||||
word: string;
|
||||
reading: string;
|
||||
seen: number;
|
||||
}
|
||||
|
||||
/** A dictionary word with every form it was seen in folded together. */
|
||||
export interface DictionaryWordRow {
|
||||
/** The form to open: the dictionary form when it was seen, otherwise the most seen one. */
|
||||
wordId: number;
|
||||
headword: string;
|
||||
/** Reading of the dictionary form, or empty when only conjugated forms were seen. */
|
||||
reading: string;
|
||||
partOfSpeech: string | null;
|
||||
rank: number | null;
|
||||
seen: number;
|
||||
/** Titles it appeared in. Computed for the returned rows. */
|
||||
titles: number;
|
||||
firstSeen: number;
|
||||
lastSeen: number;
|
||||
known: boolean;
|
||||
isName: boolean;
|
||||
/** Every line with this word came from a stream, so it can't be mined. */
|
||||
streamOnly: boolean;
|
||||
formCount: number;
|
||||
/** The most seen forms, capped so a word with hundreds of forms stays small. */
|
||||
forms: DictionaryWordForm[];
|
||||
}
|
||||
|
||||
export interface DictionaryWordPage {
|
||||
rows: DictionaryWordRow[];
|
||||
/** Words matching the query across all pages. */
|
||||
total: number;
|
||||
/** Highest `seen` across all pages, for scaling inline bars. */
|
||||
maxSeen: number;
|
||||
}
|
||||
|
||||
export const VOCABULARY_COVERAGE_BANDS = ['top1k', 'top5k', 'top10k', 'top20k'] as const;
|
||||
export type VocabularyCoverageBandId = (typeof VOCABULARY_COVERAGE_BANDS)[number];
|
||||
|
||||
/** Words seen in a frequency band, and how many of those are known. Names are left out. */
|
||||
export interface VocabularyCoverageBand {
|
||||
band: VocabularyCoverageBandId;
|
||||
seen: number;
|
||||
known: number;
|
||||
}
|
||||
|
||||
export interface StatsExcludedWord {
|
||||
headword: string;
|
||||
word: string;
|
||||
@@ -343,16 +392,59 @@ export interface TrendChartPoint {
|
||||
value: number;
|
||||
}
|
||||
|
||||
/** A rate over a bucket. `null` means too little data to rate, which charts draw as a gap. */
|
||||
export interface TrendRatePoint {
|
||||
label: string;
|
||||
value: number | null;
|
||||
}
|
||||
|
||||
/** One x-axis bucket, clipped to the requested date range. Epoch days are local calendar days. */
|
||||
export interface TrendBucket {
|
||||
label: string;
|
||||
startDay: number;
|
||||
endDay: number;
|
||||
}
|
||||
|
||||
export interface TrendPerAnimePoint {
|
||||
epochDay: number;
|
||||
animeTitle: string;
|
||||
value: number;
|
||||
}
|
||||
|
||||
export const TREND_FREQUENCY_BANDS = [
|
||||
'top1k',
|
||||
'top5k',
|
||||
'top10k',
|
||||
'top20k',
|
||||
'beyond20k',
|
||||
'unranked',
|
||||
] as const;
|
||||
export type TrendFrequencyBand = (typeof TREND_FREQUENCY_BANDS)[number];
|
||||
|
||||
export interface TrendFrequencyBandPoint {
|
||||
label: string;
|
||||
counts: Record<TrendFrequencyBand, number>;
|
||||
}
|
||||
|
||||
export interface TrendSummaryTotals {
|
||||
watchTimeMin: number;
|
||||
words: number;
|
||||
/** Null when the dashboard is filtered to one title: new words are not tracked per title. */
|
||||
newWords: number | null;
|
||||
cards: number;
|
||||
episodes: number;
|
||||
activeDays: number;
|
||||
}
|
||||
|
||||
export interface LibrarySummaryRow {
|
||||
title: string;
|
||||
/** Null for videos that are not linked to a library title. */
|
||||
animeId: number | null;
|
||||
/** Most recently watched video, the click-through target when `animeId` is null. */
|
||||
videoId: number | null;
|
||||
watchTimeMin: number;
|
||||
videos: number;
|
||||
episodesFinished: number;
|
||||
sessions: number;
|
||||
cards: number;
|
||||
words: number;
|
||||
@@ -363,15 +455,24 @@ export interface LibrarySummaryRow {
|
||||
}
|
||||
|
||||
export interface TrendsDashboardData {
|
||||
/** The days the dashboard covers. For `all` this is the span of recorded data. */
|
||||
window: { startDay: number; endDay: number };
|
||||
/** Axis shared by the `activity`, `progress`, and `ratios` series, index for index. */
|
||||
buckets: TrendBucket[];
|
||||
summary: {
|
||||
current: TrendSummaryTotals;
|
||||
/** The equally long period just before `window`; null for `all`. */
|
||||
previous: TrendSummaryTotals | null;
|
||||
};
|
||||
activity: {
|
||||
watchTime: TrendChartPoint[];
|
||||
cards: TrendChartPoint[];
|
||||
words: TrendChartPoint[];
|
||||
sessions: TrendChartPoint[];
|
||||
episodes: TrendChartPoint[];
|
||||
};
|
||||
/** Running totals counted from the start of `window`, each point the total at its period's end. */
|
||||
progress: {
|
||||
watchTime: TrendChartPoint[];
|
||||
sessions: TrendChartPoint[];
|
||||
words: TrendChartPoint[];
|
||||
newWords: TrendChartPoint[];
|
||||
cards: TrendChartPoint[];
|
||||
@@ -379,22 +480,21 @@ export interface TrendsDashboardData {
|
||||
lookups: TrendChartPoint[];
|
||||
};
|
||||
ratios: {
|
||||
lookupsPerHundred: TrendChartPoint[];
|
||||
cardsPerHour: TrendChartPoint[];
|
||||
readingSpeed: TrendChartPoint[];
|
||||
lookupsPerHundred: TrendRatePoint[];
|
||||
cardsPerHour: TrendRatePoint[];
|
||||
readingSpeed: TrendRatePoint[];
|
||||
};
|
||||
/** Vocabulary growth signals. Null when filtered to one title. */
|
||||
learning: {
|
||||
newWordsPerThousand: TrendRatePoint[];
|
||||
frequencyBands: TrendFrequencyBandPoint[];
|
||||
} | null;
|
||||
/** Active minutes as `[dayOfWeek][hour]`, with 0 = Sunday and local-time hours. */
|
||||
watchTimeByDayHour: number[][];
|
||||
/** Every title in `window`, unaffected by the title filter. */
|
||||
librarySummary: LibrarySummaryRow[];
|
||||
/** Sparse per-title values for each active day; the client accumulates them for charts. */
|
||||
animeDaily: {
|
||||
watchTime: TrendPerAnimePoint[];
|
||||
episodes: TrendPerAnimePoint[];
|
||||
cards: TrendPerAnimePoint[];
|
||||
words: TrendPerAnimePoint[];
|
||||
};
|
||||
patterns: {
|
||||
watchTimeByDayOfWeek: TrendChartPoint[];
|
||||
watchTimeByHour: TrendChartPoint[];
|
||||
};
|
||||
/** Sparse active minutes per title per day, unaffected by the title filter. */
|
||||
titleTimeline: TrendPerAnimePoint[];
|
||||
}
|
||||
|
||||
export interface WordDetailData {
|
||||
@@ -407,7 +507,17 @@ export interface WordDetailData {
|
||||
pos1: string | null;
|
||||
pos2: string | null;
|
||||
pos3: string | null;
|
||||
/** Times seen across every form of this dictionary word. */
|
||||
frequency: number;
|
||||
frequencyRank: number | null;
|
||||
/** Reading of the dictionary form, or empty when only conjugated forms were seen. */
|
||||
dictionaryReading: string;
|
||||
formCount: number;
|
||||
forms: DictionaryWordForm[];
|
||||
/** Stored subtitle lines with this word, across every form. */
|
||||
lineCount: number;
|
||||
/** Those lines that came from a stream, which can't be mined. */
|
||||
streamLineCount: number;
|
||||
firstSeen: number;
|
||||
lastSeen: number;
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user