sudacode/SubMiner
1
0
Fork 0
mirror of https://github.com/ksyasuda/SubMiner.git synced 2026-08-14 01:55:58 -07:00
Code Issues Packages Projects Releases Wiki Activity

Persist stats exclusions in DB and fix word metrics filtering (#60)

Browse Source
This commit is contained in:
Kyle Yasuda
2026-05-03 20:06:13 -07:00
committed by GitHub
parent db30c61327
commit 0915b23dc8
33 changed files with 1890 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/services/immersion-tracker/query-sessions.ts
+96 -22
View File
@@ -6,11 +6,14 @@ import type {
} from './types';
import {
ACTIVE_SESSION_METRICS_CTE,
SESSION_WORD_COUNTS_CTE,
SESSION_WORD_COUNTS_SELECT,
currentDbTimestamp,
fromDbTimestamp,
getLocalEpochDay,
getShiftedLocalDaySec,
toDbTimestamp,
sessionDisplayWordsExpr,
visibleWordSql,
} from './query-shared';
export function getSessionSummaries(db: DatabaseSync, limit = 50): SessionSummaryQueryRow[] {
@@ -94,7 +97,9 @@ export function getSessionTimeline(
/** Returns all distinct headwords in the vocabulary table (global). */
export function getAllDistinctHeadwords(db: DatabaseSync): string[] {
const rows = db.prepare('SELECT DISTINCT headword FROM imm_words').all() as Array<{
const rows = db
.prepare(`SELECT DISTINCT headword FROM imm_words w WHERE ${visibleWordSql('w')}`)
.all() as Array<{
headword: string;
}>;
return rows.map((r) => r.headword);
@@ -109,7 +114,7 @@ export function getAnimeDistinctHeadwords(db: DatabaseSync, animeId: number): st
FROM imm_word_line_occurrences o
JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
JOIN imm_words w ON w.id = o.word_id
WHERE sl.anime_id = ?
WHERE sl.anime_id = ? AND ${visibleWordSql('w')}
`,
)
.all(animeId) as Array<{ headword: string }>;
@@ -125,7 +130,7 @@ export function getMediaDistinctHeadwords(db: DatabaseSync, videoId: number): st
FROM imm_word_line_occurrences o
JOIN imm_subtitle_lines sl ON sl.line_id = o.line_id
JOIN imm_words w ON w.id = o.word_id
WHERE sl.video_id = ?
WHERE sl.video_id = ? AND ${visibleWordSql('w')}
`,
)
.all(videoId) as Array<{ headword: string }>;
@@ -148,7 +153,7 @@ export function getSessionWordsByLine(
FROM imm_subtitle_lines sl
JOIN imm_word_line_occurrences wlo ON wlo.line_id = sl.line_id
JOIN imm_words w ON w.id = wlo.word_id
WHERE sl.session_id = ?
WHERE sl.session_id = ? AND ${visibleWordSql('w')}
ORDER BY sl.line_index ASC
`);
return stmt.all(sessionId) as Array<{
@@ -290,11 +295,17 @@ export function getQueryHints(db: DatabaseSync): {
const totalCards = Number(lifetime?.totalCards ?? 0);
const activeDays = Number(lifetime?.activeDays ?? 0);
const lookupWordsExpr = sessionDisplayWordsExpr(
's',
'swc',
'COALESCE(t.tokens_seen, s.tokens_seen)',
);
const lookupTotals = db
.prepare(
`
${SESSION_WORD_COUNTS_CTE}
SELECT
COALESCE(SUM(COALESCE(t.tokens_seen, s.tokens_seen, 0)), 0) AS totalTokensSeen,
COALESCE(SUM(${lookupWordsExpr}), 0) AS totalTokensSeen,
COALESCE(SUM(COALESCE(t.lookup_count, s.lookup_count, 0)), 0) AS totalLookupCount,
COALESCE(SUM(COALESCE(t.lookup_hits, s.lookup_hits, 0)), 0) AS totalLookupHits,
COALESCE(SUM(COALESCE(t.yomitan_lookup_count, s.yomitan_lookup_count, 0)), 0) AS totalYomitanLookupCount
@@ -309,6 +320,7 @@ export function getQueryHints(db: DatabaseSync): {
FROM imm_session_telemetry
GROUP BY session_id
) t ON t.session_id = s.session_id
LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
WHERE s.ended_at_ms IS NOT NULL
`,
)
@@ -338,8 +350,25 @@ export function getQueryHints(db: DatabaseSync): {
}
export function getDailyRollups(db: DatabaseSync, limit = 60): ImmersionSessionRollupRow[] {
const wordsExpr = sessionDisplayWordsExpr('s', 'swc');
const prepared = db.prepare(`
WITH recent_days AS (
WITH session_word_counts AS (
${SESSION_WORD_COUNTS_SELECT}
),
daily_word_counts AS (
SELECT
CAST(
julianday(CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') - 2440587.5
AS INTEGER
) AS rollupDay,
s.video_id AS videoId,
SUM(${wordsExpr}) AS totalTokensSeen
FROM imm_sessions s
LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
WHERE s.ended_at_ms IS NOT NULL
GROUP BY rollupDay, s.video_id
),
recent_days AS (
SELECT DISTINCT rollup_day
FROM imm_daily_rollups
ORDER BY rollup_day DESC
@@ -351,12 +380,29 @@ export function getDailyRollups(db: DatabaseSync, limit = 60): ImmersionSessionR
r.total_sessions AS totalSessions,
r.total_active_min AS totalActiveMin,
r.total_lines_seen AS totalLinesSeen,
r.total_tokens_seen AS totalTokensSeen,
CASE
WHEN dwc.totalTokensSeen IS NOT NULL AND dwc.totalTokensSeen > r.total_tokens_seen THEN dwc.totalTokensSeen
ELSE r.total_tokens_seen
END AS totalTokensSeen,
r.total_cards AS totalCards,
r.cards_per_hour AS cardsPerHour,
r.tokens_per_min AS tokensPerMin,
CASE
WHEN r.total_active_min > 0 THEN (
CASE
WHEN dwc.totalTokensSeen IS NOT NULL AND dwc.totalTokensSeen > r.total_tokens_seen THEN dwc.totalTokensSeen
ELSE r.total_tokens_seen
END
) * 1.0 / r.total_active_min
ELSE NULL
END AS tokensPerMin,
r.lookup_hit_rate AS lookupHitRate
FROM imm_daily_rollups r
LEFT JOIN daily_word_counts dwc
ON dwc.rollupDay = r.rollup_day
AND (
(dwc.videoId IS NULL AND r.video_id IS NULL)
OR dwc.videoId = r.video_id
)
WHERE r.rollup_day IN (SELECT rollup_day FROM recent_days)
ORDER BY r.rollup_day DESC, r.video_id DESC
`);
@@ -365,33 +411,61 @@ export function getDailyRollups(db: DatabaseSync, limit = 60): ImmersionSessionR
}
export function getMonthlyRollups(db: DatabaseSync, limit = 24): ImmersionSessionRollupRow[] {
const wordsExpr = sessionDisplayWordsExpr('s', 'swc');
const prepared = db.prepare(`
WITH recent_months AS (
WITH session_word_counts AS (
${SESSION_WORD_COUNTS_SELECT}
),
monthly_word_counts AS (
SELECT
CAST(strftime('%Y%m', CAST(s.started_at_ms AS REAL) / 1000, 'unixepoch', 'localtime') AS INTEGER) AS rollupMonth,
s.video_id AS videoId,
SUM(${wordsExpr}) AS totalTokensSeen
FROM imm_sessions s
LEFT JOIN session_word_counts swc ON swc.sessionId = s.session_id
WHERE s.ended_at_ms IS NOT NULL
GROUP BY rollupMonth, s.video_id
),
recent_months AS (
SELECT DISTINCT rollup_month
FROM imm_monthly_rollups
ORDER BY rollup_month DESC
LIMIT ?
)
SELECT
rollup_month AS rollupDayOrMonth,
video_id AS videoId,
total_sessions AS totalSessions,
total_active_min AS totalActiveMin,
total_lines_seen AS totalLinesSeen,
total_tokens_seen AS totalTokensSeen,
total_cards AS totalCards,
r.rollup_month AS rollupDayOrMonth,
r.video_id AS videoId,
r.total_sessions AS totalSessions,
r.total_active_min AS totalActiveMin,
r.total_lines_seen AS totalLinesSeen,
CASE
WHEN total_active_min > 0 THEN (total_cards * 60.0) / total_active_min
WHEN mwc.totalTokensSeen IS NOT NULL AND mwc.totalTokensSeen > r.total_tokens_seen THEN mwc.totalTokensSeen
ELSE r.total_tokens_seen
END AS totalTokensSeen,
r.total_cards AS totalCards,
CASE
WHEN r.total_active_min > 0 THEN (r.total_cards * 60.0) / r.total_active_min
ELSE NULL
END AS cardsPerHour,
CASE
WHEN total_active_min > 0 THEN total_tokens_seen * 1.0 / total_active_min
WHEN r.total_active_min > 0 THEN (
CASE
WHEN mwc.totalTokensSeen IS NOT NULL AND mwc.totalTokensSeen > r.total_tokens_seen THEN mwc.totalTokensSeen
ELSE r.total_tokens_seen
END
) * 1.0 / r.total_active_min
ELSE NULL
END AS tokensPerMin,
NULL AS lookupHitRate
FROM imm_monthly_rollups
WHERE rollup_month IN (SELECT rollup_month FROM recent_months)
ORDER BY rollup_month DESC, video_id DESC
FROM imm_monthly_rollups r
LEFT JOIN monthly_word_counts mwc
ON mwc.rollupMonth = r.rollup_month
AND (
(mwc.videoId IS NULL AND r.video_id IS NULL)
OR mwc.videoId = r.video_id
)
WHERE r.rollup_month IN (SELECT rollup_month FROM recent_months)
ORDER BY r.rollup_month DESC, r.video_id DESC
`);
return prepared.all(limit) as unknown as ImmersionSessionRollupRow[];
}