import { existsSync, readFileSync, statSync } from 'node:fs'; import { extname, resolve, sep } from 'node:path'; import { getConfiguredSentenceFieldName, getConfiguredTranslationFieldName, getConfiguredWordFieldName, getPreferredNoteFieldValue, } from '../../../anki-field-config.js'; import { knownWordsFromState, parseKnownWordCacheState, } from '../../../anki-integration/known-word-cache-format.js'; import { createLogger } from '../../../logger.js'; import type { AnkiConnectConfig } from '../../../types.js'; import type { StatsDayRange, StatsDictionaryWordQuery, StatsDictionaryWordSort, StatsTrendGroupBy, StatsTrendRange, } from '../../../types/stats-http-contract.js'; import type { StatsExcludedWord } from '../../../types/stats-wire.js'; import type { ImmersionTrackerService } from '../immersion-tracker-service.js'; import { splitSentenceSearchTerms } from '../immersion-tracker/query-lexical.js'; const statsKnownWordsLogger = createLogger('stats:known-words'); const STATS_STATIC_CONTENT_TYPES: Record = { '.css': 'text/css; charset=utf-8', '.gif': 'image/gif', '.html': 'text/html; charset=utf-8', '.ico': 'image/x-icon', '.jpeg': 'image/jpeg', '.jpg': 'image/jpeg', '.js': 'text/javascript; charset=utf-8', '.json': 'application/json; charset=utf-8', '.mjs': 'text/javascript; charset=utf-8', '.png': 'image/png', '.svg': 'image/svg+xml', '.txt': 'text/plain; charset=utf-8', '.webp': 'image/webp', '.woff': 'font/woff', '.woff2': 'font/woff2', }; export function parseIntQuery( raw: string | undefined, fallback: number, maxLimit?: number, ): number { if (raw === undefined) return fallback; const n = Number(raw); if (!Number.isFinite(n) || n < 0) return fallback; const parsed = Math.floor(n); return maxLimit === undefined ? parsed : Math.min(parsed, maxLimit); } export function isPositiveSafeInteger(value: unknown): value is number { return typeof value === 'number' && Number.isSafeInteger(value) && value > 0; } export function parsePositiveId(raw: string | undefined): number | null { if (raw === undefined) return null; const value = Number(raw); return isPositiveSafeInteger(value) && String(value) === raw ? value : null; } /** Parses an inclusive `fromDay`/`toDay` pair; anything malformed or reversed means no range. */ export function parseDayRange( rawFrom: string | undefined, rawTo: string | undefined, ): StatsDayRange | null { const startDay = parsePositiveId(rawFrom); const endDay = parsePositiveId(rawTo); return startDay !== null && endDay !== null && startDay <= endDay ? { startDay, endDay } : null; } export function parseTrendRange( raw: string | undefined, rawFrom?: string, rawTo?: string, ): StatsTrendRange { if (raw === 'custom') return parseDayRange(rawFrom, rawTo) ?? '30d'; return raw === '7d' || raw === '30d' || raw === '90d' || raw === '365d' || raw === 'all' ? raw : '30d'; } export function parseTrendGroupBy(raw: string | undefined): StatsTrendGroupBy { return raw === 'month' || raw === 'week' ? raw : 'day'; } const DICTIONARY_WORD_SORTS: readonly StatsDictionaryWordSort[] = [ 'seen', 'rank', 'titles', 'firstSeen', 'lastSeen', 'headword', ]; const DICTIONARY_WORD_PAGE_LIMIT = 200; /** Reads the word list query string. Anything unrecognized falls back to the default. */ export function parseDictionaryWordQuery( read: (name: string) => string | undefined, ): StatsDictionaryWordQuery { const sort = DICTIONARY_WORD_SORTS.find((value) => value === read('sort')) ?? 'seen'; const pos = read('pos'); const flag = (name: string): boolean => read(name) === 'true'; return { search: read('search')?.trim() || undefined, sort, descending: read('order') !== 'asc', hideKnown: flag('hideKnown'), hideNames: flag('hideNames'), hideKana: flag('hideKana'), partOfSpeech: pos === 'noun' || pos === 'verb' || pos === 'adjective' ? pos : undefined, maxRank: parsePositiveId(read('maxRank')) ?? undefined, minTitles: parsePositiveId(read('minTitles')) ?? undefined, limit: parseIntQuery(read('limit'), 25, DICTIONARY_WORD_PAGE_LIMIT), offset: parseIntQuery(read('offset'), 0), }; } export function parseTrendFillEmpty(raw: string | undefined): boolean { return raw !== 'false'; } export function parseEventTypesQuery(raw: string | undefined): number[] | undefined { if (!raw) return undefined; const parsed = raw .split(',') .map((entry) => Number.parseInt(entry.trim(), 10)) .filter((entry) => Number.isInteger(entry) && entry > 0); return parsed.length > 0 ? parsed : undefined; } export function parseExcludedWordsBody(body: unknown): StatsExcludedWord[] | null { if (!body || typeof body !== 'object' || !Array.isArray((body as { words?: unknown }).words)) { return null; } const words: StatsExcludedWord[] = []; for (const row of (body as { words: unknown[] }).words) { if (!row || typeof row !== 'object') return null; const { headword, word, reading } = row as Record; if (typeof headword !== 'string' || typeof word !== 'string' || typeof reading !== 'string') { return null; } words.push({ headword, word, reading }); } return words; } /** * Read a duplicate-line cleanup request. An explicit object with no lookback scans all * history. Invalid bodies and invalid windows are rejected instead of broadening scope. */ export function parseDuplicateLineCleanupBody(body: unknown): { dryRun: boolean; lookbackDays: number | null; } | null { if (!body || typeof body !== 'object' || Array.isArray(body)) { return null; } const source = body as Record; if (source.dryRun !== undefined && typeof source.dryRun !== 'boolean') { return null; } const rawLookback = source.lookbackDays; if ( rawLookback !== undefined && rawLookback !== null && (typeof rawLookback !== 'number' || !Number.isFinite(rawLookback) || rawLookback < 1) ) { return null; } return { dryRun: source.dryRun === true, lookbackDays: typeof rawLookback === 'number' ? Math.floor(rawLookback) : null, }; } export function loadKnownWordsSet(cachePath: string | undefined): Set | null { if (!cachePath || !existsSync(cachePath)) return null; try { const state = parseKnownWordCacheState(JSON.parse(readFileSync(cachePath, 'utf-8')) as unknown); if (!state) { // A cache that exists but does not parse is a format mismatch, not an // empty collection; say so instead of reporting zero known words. statsKnownWordsLogger.warn(`Unrecognized known-word cache format at ${cachePath}`); return null; } return knownWordsFromState(state); } catch { // Treat an unreadable cache as unavailable. } return null; } export function countKnownWords( headwords: string[], knownWordsSet: Set, ): { totalUniqueWords: number; knownWordCount: number } { let knownWordCount = 0; for (const headword of headwords) { if (knownWordsSet.has(headword)) knownWordCount += 1; } return { totalUniqueWords: headwords.length, knownWordCount }; } function toKnownWordRate(knownWordsSeen: number, tokensSeen: number): number { if (!Number.isFinite(knownWordsSeen) || !Number.isFinite(tokensSeen) || tokensSeen <= 0) return 0; return Number(((knownWordsSeen / tokensSeen) * 100).toFixed(1)); } function summarizeFilteredWordOccurrences( wordsByLine: Array<{ lineIndex: number; headword: string; occurrenceCount: number }>, knownWordsSet: Set, ): { knownWordsSeen: number; totalWordsSeen: number } { let knownWordsSeen = 0; let totalWordsSeen = 0; for (const row of wordsByLine) { totalWordsSeen += row.occurrenceCount; if (knownWordsSet.has(row.headword)) knownWordsSeen += row.occurrenceCount; } return { knownWordsSeen, totalWordsSeen }; } export async function enrichSessionsWithKnownWordMetrics< Session extends { sessionId: number; tokensSeen: number }, >( tracker: ImmersionTrackerService, sessions: Session[], knownWordsCachePath?: string, ): Promise> { const knownWordsSet = loadKnownWordsSet(knownWordsCachePath); if (!knownWordsSet) { return sessions.map((session) => ({ ...session, knownWordsSeen: 0, knownWordRate: 0 })); } return Promise.all( sessions.map(async (session) => { let knownWordsSeen = 0; let totalWordsSeen = 0; try { const summary = summarizeFilteredWordOccurrences( await tracker.getSessionWordsByLine(session.sessionId), knownWordsSet, ); knownWordsSeen = summary.knownWordsSeen; totalWordsSeen = summary.totalWordsSeen; } catch { knownWordsSeen = 0; totalWordsSeen = 0; } return { ...session, knownWordsSeen, knownWordRate: toKnownWordRate(knownWordsSeen, totalWordsSeen), }; }), ); } /** Deduplicated positive safe integer ids from an untrusted JSON body field. */ export function parsePositiveIdList(raw: unknown, maxItems?: number): number[] | null { if (!Array.isArray(raw)) return null; const ids = new Set(); for (const value of raw) { if (!isPositiveSafeInteger(value)) return null; ids.add(value); } const parsed = [...ids]; return maxItems === undefined ? parsed : parsed.slice(0, maxItems); } export function parseBooleanQuery(raw: string | undefined, fallback: boolean): boolean { if (raw === undefined) return fallback; const normalized = raw.trim().toLowerCase(); if (!normalized) return fallback; return !['0', 'false', 'no', 'off'].includes(normalized); } function uniqueNonEmptyStrings(values: readonly string[]): string[] { const seen = new Set(); const result: string[] = []; for (const value of values) { const normalized = value.trim(); if (!normalized || seen.has(normalized)) continue; seen.add(normalized); result.push(normalized); } return result; } export async function buildSentenceSearchOptions( query: string, searchByHeadword: boolean, resolveSentenceSearchHeadwords: ((term: string) => Promise | string[]) | undefined, ): Promise<{ headwordTerms: Array<{ term: string; headwords: string[] }> } | undefined> { if (!searchByHeadword) return undefined; const headwordTerms: Array<{ term: string; headwords: string[] }> = []; for (const term of splitSentenceSearchTerms(query)) { const resolved = resolveSentenceSearchHeadwords ? await resolveSentenceSearchHeadwords(term) : [term]; const headwords = uniqueNonEmptyStrings(resolved); if (headwords.length > 0) headwordTerms.push({ term, headwords }); } return headwordTerms.length > 0 ? { headwordTerms } : undefined; } export function buildAnkiNotePreview( fields: Record, ankiConfig?: Pick, ): { word: string; sentence: string; translation: string } { return { word: getPreferredNoteFieldValue(fields, [getConfiguredWordFieldName(ankiConfig)]), sentence: getPreferredNoteFieldValue(fields, [getConfiguredSentenceFieldName(ankiConfig)]), translation: getPreferredNoteFieldValue(fields, [ getConfiguredTranslationFieldName(ankiConfig), ]), }; } function resolveStatsStaticPath(staticDir: string, requestPath: string): string | null { const normalizedPath = requestPath.replace(/^\/+/, '') || 'index.html'; const absoluteStaticDir = resolve(staticDir); let decodedPath: string; try { decodedPath = decodeURIComponent(normalizedPath); } catch { return null; } const absolutePath = resolve(absoluteStaticDir, decodedPath); if ( absolutePath !== absoluteStaticDir && !absolutePath.startsWith(`${absoluteStaticDir}${sep}`) ) { return null; } if (!existsSync(absolutePath) || !statSync(absolutePath).isFile()) return null; return absolutePath; } export function createStatsStaticResponse(staticDir: string, requestPath: string): Response | null { const absolutePath = resolveStatsStaticPath(staticDir, requestPath); if (!absolutePath) return null; const contentType = STATS_STATIC_CONTENT_TYPES[extname(absolutePath).toLowerCase()] ?? 'application/octet-stream'; return new Response(readFileSync(absolutePath), { headers: { 'Content-Type': contentType, 'Cache-Control': absolutePath.endsWith('index.html') ? 'no-cache' : 'public, max-age=31536000, immutable', }, }); }