fix(stats): report complete vocabulary totals and new-word history (#202)

This commit is contained in:
2026-08-18 00:42:46 -07:00
committed by GitHub
parent 273652f781
commit 7de73e16a1
38 changed files with 2995 additions and 85 deletions
@@ -284,6 +284,22 @@ function createMockTracker(
getSessionTimeline: async () => [],
getSessionEvents: async () => [],
getVocabularyStats: async () => VOCABULARY_STATS,
getVocabularySummary: async () => ({
uniqueWords: 501,
uniqueWordsWithoutNames: 500,
uniqueKanji: 201,
newThisWeek: 7,
newThisWeekWithoutNames: 6,
knownWordCount: 250,
knownWordCountWithoutNames: 249,
}),
getVocabularyChartData: async () => ({
ready: true,
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
}),
getStatsExcludedWords: async () => [],
replaceStatsExcludedWords: async () => {},
getKanjiStats: async () => KANJI_STATS,
@@ -711,6 +727,38 @@ describe('stats server API routes', () => {
assert.equal(body[0].headword, 'する');
});
it('GET /api/stats/vocabulary/summary returns database-wide card totals', async () => {
const app = createStatsApp(createMockTracker());
const res = await app.request('/api/stats/vocabulary/summary');
assert.equal(res.status, 200);
assert.deepEqual(await res.json(), {
uniqueWords: 501,
uniqueWordsWithoutNames: 500,
uniqueKanji: 201,
newThisWeek: 7,
newThisWeekWithoutNames: 6,
knownWordCount: 250,
knownWordCountWithoutNames: 249,
});
});
it('GET /api/stats/vocabulary/charts returns complete chart datasets', async () => {
const app = createStatsApp(createMockTracker());
const res = await app.request('/api/stats/vocabulary/charts');
assert.equal(res.status, 200);
assert.deepEqual(await res.json(), {
ready: true,
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
});
});
it('GET /api/stats/kanji returns kanji frequency data', async () => {
const app = createStatsApp(createMockTracker());
const res = await app.request('/api/stats/kanji');
@@ -559,6 +559,165 @@ test('fresh tracker DB creates lifetime summary tables', async () => {
}
});
test('fresh tracker DB skips lexical rollup backfill work', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let backfillRuns = 0;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath }, {
runLexicalRollupBackfillTask: async () => {
backfillRuns += 1;
},
} as never);
assert.equal(backfillRuns, 0);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('tracker starts the injected lexical rollup backfill when it is pending', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let backfillRuns = 0;
try {
const setupDb = new Database(dbPath);
const { ensureSchema } = await import('./immersion-tracker/storage');
ensureSchema(setupDb);
setupDb
.prepare(
`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
)
.run();
setupDb.close();
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath }, {
runLexicalRollupBackfillTask: async () => {
backfillRuns += 1;
},
} as never);
assert.equal(backfillRuns, 1);
await waitForCondition(
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
);
assert.equal(
(tracker as unknown as { preserveWriteQueueUntilDrained: boolean })
.preserveWriteQueueUntilDrained,
false,
);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('tracker queues playback writes until lexical rollup backfill settles', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let startBackfill = (): void => {};
let releaseBackfill = (): void => {};
let markBackfillStarted = (): void => {};
const backfillStartGate = new Promise<void>((resolve) => {
startBackfill = resolve;
});
const heldBackfill = new Promise<void>((resolve) => {
releaseBackfill = resolve;
});
const backfillStarted = new Promise<void>((resolve) => {
markBackfillStarted = resolve;
});
try {
const setupDb = new Database(dbPath);
const { ensureSchema } = await import('./immersion-tracker/storage');
ensureSchema(setupDb);
setupDb
.prepare(
`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
)
.run();
setupDb.close();
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath, policy: { queueCap: 100 } },
{
runLexicalRollupBackfillTask: async (workerDbPath) => {
await backfillStartGate;
const workerDb = new Database(workerDbPath);
try {
workerDb.exec('BEGIN IMMEDIATE');
markBackfillStarted();
await heldBackfill;
workerDb.exec('COMMIT');
} catch (error) {
try {
workerDb.exec('ROLLBACK');
} catch {
// Preserve the original worker failure.
}
throw error;
} finally {
workerDb.close();
}
},
},
);
tracker.handleMediaChange('https://example.com/backfill-test.mp4', 'Backfill Test');
startBackfill();
await backfillStarted;
for (let index = 0; index < 125; index += 1) tracker.recordCardsMined(1);
const privateApi = tracker as unknown as {
db: DatabaseSync;
queue: unknown[];
droppedWriteCount: number;
flushNow: () => void;
writeLock: { locked: boolean };
};
assert.equal(privateApi.writeLock.locked, true);
privateApi.flushNow();
assert.ok(privateApi.queue.length > 100, 'the protected queue may grow past its normal cap');
assert.equal(privateApi.droppedWriteCount, 0, 'backfill must not discard playback writes');
assert.equal(
(
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
total: number;
}
).total,
0,
);
releaseBackfill();
await waitForCondition(() => privateApi.queue.length === 0, 5_000);
assert.equal(
(
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
total: number;
}
).total,
125,
);
} finally {
releaseBackfill();
if (tracker) {
await waitForCondition(
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
5_000,
);
}
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('startup backfills lifetime summaries when retained sessions exist but summary tables are empty', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
@@ -4909,3 +5068,149 @@ test('ensureAnimeCoverArt fetches art via the latest video of the anime', async
cleanupDbPath(dbPath);
}
});
test('getVocabularySummary coalesces concurrent requests into one worker task', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let taskRuns = 0;
let releaseTask: (() => void) | null = null;
const seenKnownWords: Array<ReadonlySet<string> | null> = [];
const summary = {
uniqueWords: 1,
uniqueWordsWithoutNames: 1,
uniqueKanji: 0,
newThisWeek: 0,
newThisWeekWithoutNames: 0,
knownWordCount: null,
knownWordCountWithoutNames: null,
};
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath },
{
runVocabularySummaryTask: async (_dbPath, knownWords) => {
taskRuns += 1;
seenKnownWords.push(knownWords);
await new Promise<void>((resolve) => {
releaseTask = resolve;
});
return summary;
},
destroyVocabularySummaryRunner: () => {},
},
);
const knownWordsSnapshot = new Set(['猫']);
const first = tracker.getVocabularySummary(knownWordsSnapshot);
const second = tracker.getVocabularySummary(knownWordsSnapshot);
await waitForCondition(() => releaseTask !== null);
let release = releaseTask as (() => void) | null;
assert.ok(release);
release();
assert.deepEqual(await first, summary);
assert.equal(await second, await first);
assert.equal(taskRuns, 1);
assert.deepEqual(seenKnownWords, [knownWordsSnapshot]);
releaseTask = null;
const third = tracker.getVocabularySummary(null);
await waitForCondition(() => releaseTask !== null);
release = releaseTask as (() => void) | null;
assert.ok(release);
release();
assert.deepEqual(await third, summary);
assert.equal(taskRuns, 2);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('getVocabularySummary coalesces equivalent known-word snapshots by value', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let taskRuns = 0;
const releases: Array<() => void> = [];
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath },
{
runVocabularySummaryTask: async () => {
taskRuns += 1;
await new Promise<void>((resolve) => releases.push(resolve));
return {
uniqueWords: 2,
uniqueWordsWithoutNames: 2,
uniqueKanji: 2,
newThisWeek: 0,
newThisWeekWithoutNames: 0,
knownWordCount: 2,
knownWordCountWithoutNames: 2,
};
},
destroyVocabularySummaryRunner: () => {},
},
);
const first = tracker.getVocabularySummary(new Set(['猫', '犬']));
const second = tracker.getVocabularySummary(new Set(['犬', '猫']));
await waitForCondition(() => releases.length > 0);
const observedTaskRuns = taskRuns;
for (const release of releases) release();
await Promise.all([first, second]);
assert.equal(observedTaskRuns, 1);
const third = tracker.getVocabularySummary(new Set(['猫', '犬']));
await waitForCondition(() => releases.length === 2);
releases[1]!();
await third;
assert.equal(taskRuns, 2, 'a settled snapshot must be evicted from the in-flight map');
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('getVocabularySummary keeps different known-word snapshots independent', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
const releases: Array<() => void> = [];
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath },
{
runVocabularySummaryTask: async (_dbPath, knownWords) => {
await new Promise<void>((resolve) => releases.push(resolve));
return {
uniqueWords: 1,
uniqueWordsWithoutNames: 1,
uniqueKanji: 0,
newThisWeek: 0,
newThisWeekWithoutNames: 0,
knownWordCount: knownWords?.size ?? null,
knownWordCountWithoutNames: knownWords?.size ?? null,
};
},
destroyVocabularySummaryRunner: () => {},
},
);
const withoutKnownWords = tracker.getVocabularySummary(null);
const withKnownWords = tracker.getVocabularySummary(new Set(['猫']));
await waitForCondition(() => releases.length === 2);
for (const release of releases) release();
assert.equal((await withoutKnownWords).knownWordCount, null);
assert.equal((await withKnownWords).knownWordCount, 1);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
+106 -6
View File
@@ -58,6 +58,7 @@ import {
getSessionEvents,
getSimilarWords,
getStatsExcludedWords,
getVocabularyChartData,
getVocabularyStats,
replaceStatsExcludedWords,
searchSubtitleSentences,
@@ -96,6 +97,12 @@ import {
DeleteMaintenanceWorkerRuntime,
type RunDeleteMaintenanceTask,
} from './immersion-tracker/delete-maintenance-worker-runtime';
import {
VocabularySummaryWorkerRuntime,
type RunVocabularySummaryTask,
} from './immersion-tracker/vocabulary-summary-worker-runtime';
import { LexicalRollupWorkerRuntime } from './immersion-tracker/lexical-rollup-worker-runtime';
import { areLexicalDailyRollupsReady } from './immersion-tracker/lexical-rollups';
import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
import {
cleanupDuplicateSubtitleLines,
@@ -185,6 +192,7 @@ import {
type StatsExcludedWordRow,
type StreakCalendarRow,
type VocabularyCleanupSummary,
type VocabularyStatsSummary,
type WatchTimePerAnimeRow,
type WordAnimeAppearanceRow,
type WordDetailRow,
@@ -405,13 +413,24 @@ export class ImmersionTrackerService {
private readonly monthlyRollupRetentionMs: number;
private readonly vacuumIntervalMs: number;
private readonly dbPath: string;
private readonly writeLock = { locked: false };
private readonly writeLock = {
locked: false,
reasons: new Set<'flush' | 'delete-maintenance' | 'lexical-rollup-backfill'>(),
};
private readonly destroyDeleteMaintenanceRunner: () => void;
private readonly runVocabularySummaryTask: (
knownWords: ReadonlySet<string> | null,
) => Promise<VocabularyStatsSummary>;
private readonly vocabularySummariesInFlight = new Map<string, Promise<VocabularyStatsSummary>>();
private readonly destroyVocabularySummaryRunner: () => void;
private readonly runLexicalRollupBackfillTask: () => Promise<void>;
private readonly destroyLexicalRollupBackfillRunner: () => void;
private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
private flushTimer: ReturnType<typeof setTimeout> | null = null;
private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
private flushScheduled = false;
private droppedWriteCount = 0;
private preserveWriteQueueUntilDrained = false;
private lastVacuumMs = 0;
private isDestroyed = false;
private sessionState: SessionState | null = null;
@@ -434,6 +453,10 @@ export class ImmersionTrackerService {
dependencies: {
runDeleteMaintenanceTask?: RunDeleteMaintenanceTask;
destroyDeleteMaintenanceRunner?: () => void;
runVocabularySummaryTask?: RunVocabularySummaryTask;
destroyVocabularySummaryRunner?: () => void;
runLexicalRollupBackfillTask?: (dbPath: string) => Promise<void>;
destroyLexicalRollupBackfillRunner?: () => void;
} = {},
) {
this.dbPath = options.dbPath;
@@ -453,13 +476,34 @@ export class ImmersionTrackerService {
runTask: (task) => runDeleteMaintenanceTask(this.dbPath, task),
onBusy: () => {
this.requireWriteQueueDrained('delete maintenance');
this.writeLock.locked = true;
this.setWriteLock('delete-maintenance', true);
},
onIdle: () => {
this.writeLock.locked = false;
this.setWriteLock('delete-maintenance', false);
if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
},
});
if (dependencies.runVocabularySummaryTask) {
this.runVocabularySummaryTask = (knownWords) =>
dependencies.runVocabularySummaryTask!(this.dbPath, knownWords);
this.destroyVocabularySummaryRunner =
dependencies.destroyVocabularySummaryRunner ?? (() => {});
} else {
const vocabularySummaryRuntime = new VocabularySummaryWorkerRuntime();
this.runVocabularySummaryTask = (knownWords) =>
vocabularySummaryRuntime.run(this.dbPath, knownWords);
this.destroyVocabularySummaryRunner = () => vocabularySummaryRuntime.destroy();
}
if (dependencies.runLexicalRollupBackfillTask) {
this.runLexicalRollupBackfillTask = () =>
dependencies.runLexicalRollupBackfillTask!(this.dbPath);
this.destroyLexicalRollupBackfillRunner =
dependencies.destroyLexicalRollupBackfillRunner ?? (() => {});
} else {
const lexicalRollupRuntime = new LexicalRollupWorkerRuntime();
this.runLexicalRollupBackfillTask = () => lexicalRollupRuntime.run(this.dbPath);
this.destroyLexicalRollupBackfillRunner = () => lexicalRollupRuntime.destroy();
}
const parentDir = path.dirname(this.dbPath);
if (!fs.existsSync(parentDir)) {
fs.mkdirSync(parentDir, { recursive: true });
@@ -547,6 +591,7 @@ export class ImmersionTrackerService {
}
}
this.preparedStatements = createTrackerPreparedStatements(this.db);
if (!areLexicalDailyRollupsReady(this.db)) this.startLexicalRollupBackfill();
this.scheduleMaintenance();
this.scheduleFlush();
}
@@ -565,6 +610,8 @@ export class ImmersionTrackerService {
this.isDestroyed = true;
this.deleteMaintenanceScheduler.destroy();
this.destroyDeleteMaintenanceRunner();
this.destroyVocabularySummaryRunner();
this.destroyLexicalRollupBackfillRunner();
this.db.close();
}
@@ -634,6 +681,25 @@ export class ImmersionTrackerService {
return getVocabularyStats(this.db, limit, excludePos);
}
async getVocabularySummary(knownWords: ReadonlySet<string> | null) {
const key = knownWords ? JSON.stringify([...knownWords].sort()) : 'null';
const inFlight = this.vocabularySummariesInFlight.get(key);
if (inFlight) return inFlight;
const task = this.runVocabularySummaryTask(knownWords);
this.vocabularySummariesInFlight.set(key, task);
try {
return await task;
} finally {
if (this.vocabularySummariesInFlight.get(key) === task) {
this.vocabularySummariesInFlight.delete(key);
}
}
}
async getVocabularyChartData() {
return getVocabularyChartData(this.db);
}
async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {
return getStatsExcludedWords(this.db);
}
@@ -910,6 +976,33 @@ export class ImmersionTrackerService {
}
}
private setWriteLock(
reason: 'flush' | 'delete-maintenance' | 'lexical-rollup-backfill',
active: boolean,
): void {
if (active) this.writeLock.reasons.add(reason);
else this.writeLock.reasons.delete(reason);
this.writeLock.locked = this.writeLock.reasons.size > 0;
}
private startLexicalRollupBackfill(): void {
this.requireWriteQueueDrained('lexical rollup backfill');
this.preserveWriteQueueUntilDrained = true;
this.setWriteLock('lexical-rollup-backfill', true);
void this.runLexicalRollupBackfillTask()
.catch((error: unknown) => {
this.logger.warn(
'Lexical daily rollup backfill failed; it will retry on next startup',
error,
);
})
.finally(() => {
this.setWriteLock('lexical-rollup-backfill', false);
if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
else if (!this.isDestroyed) this.scheduleFlush(0);
});
}
async reassignAnimeAnilist(
animeId: number,
info: {
@@ -1906,7 +1999,12 @@ export class ImmersionTrackerService {
private recordWrite(write: QueuedWrite): void {
if (this.isDestroyed) return;
const { dropped } = enqueueWrite(this.queue, write, this.queueCap);
// A lexical migration owns the database write lock, so dropping the oldest
// entry cannot relieve pressure: nothing can flush until the worker exits.
// Preserve that finite startup burst and drain it as soon as the lock lifts.
const { dropped } = this.preserveWriteQueueUntilDrained
? (this.queue.push(write), { dropped: 0 })
: enqueueWrite(this.queue, write, this.queueCap);
if (dropped > 0) {
this.droppedWriteCount += dropped;
this.logger.warn(`Immersion tracker queue overflow; dropped ${dropped} oldest writes`);
@@ -1954,6 +2052,7 @@ export class ImmersionTrackerService {
private flushNow(): void {
if (this.writeLock.locked || this.isDestroyed) return;
if (this.queue.length === 0) {
this.preserveWriteQueueUntilDrained = false;
this.flushScheduled = false;
return;
}
@@ -1965,7 +2064,7 @@ export class ImmersionTrackerService {
}
const batch = this.queue.splice(0, Math.min(this.batchSize, this.queue.length));
this.writeLock.locked = true;
this.setWriteLock('flush', true);
try {
this.db.exec('BEGIN IMMEDIATE');
for (const write of batch) {
@@ -1977,8 +2076,9 @@ export class ImmersionTrackerService {
this.queue.unshift(...batch);
this.logger.warn('Immersion tracker flush failed, retrying later', error as Error);
} finally {
this.writeLock.locked = false;
this.setWriteLock('flush', false);
this.flushScheduled = false;
if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
if (this.queue.length > 0) {
this.scheduleFlush(this.flushIntervalMs);
}
@@ -31,6 +31,7 @@ import {
getKanjiOccurrences,
getSessionSummaries,
getVocabularyStats,
getVocabularySummary,
getKanjiStats,
getSessionEvents,
getSessionTimeline,
@@ -1875,6 +1876,115 @@ test('getVocabularyStats returns rows ordered by frequency descending', () => {
}
});
test('getVocabularySummary counts every tracked vocabulary row instead of a display page', () => {
const dbPath = makeDbPath();
const db = openTestDb(dbPath);
try {
ensureSchema(db);
const nowSec = Math.floor(Date.now() / 1000);
const insertWord = db.prepare(`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3,
first_seen, last_seen, frequency
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', ?, ?, 1)
`);
const insertKanji = db.prepare(`
INSERT INTO imm_kanji (kanji, first_seen, last_seen, frequency)
VALUES (?, ?, ?, 1)
`);
for (let index = 0; index < 501; index += 1) {
insertWord.run(`単語${index}`, `単語${index}`, nowSec - 8 * 86_400, nowSec - 8 * 86_400);
}
for (let index = 0; index < 201; index += 1) {
insertKanji.run(
String.fromCodePoint(0x4e00 + index),
nowSec - 8 * 86_400,
nowSec - 8 * 86_400,
);
}
insertWord.run('今週', '今週', nowSec - 86_400, nowSec - 86_400);
assert.deepEqual(getVocabularySummary(db, new Set(['単語0', '今週']), nowSec * 1000), {
uniqueWords: 502,
uniqueWordsWithoutNames: 502,
uniqueKanji: 201,
newThisWeek: 1,
newThisWeekWithoutNames: 1,
knownWordCount: 2,
knownWordCountWithoutNames: 2,
});
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getVocabularySummary applies vocabulary exclusions and Hide Names totals', () => {
const dbPath = makeDbPath();
const db = openTestDb(dbPath);
try {
ensureSchema(db);
const insertWord = db.prepare(`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3,
first_seen, last_seen, frequency
) VALUES (?, ?, '', 'noun', '名詞', ?, '', 1, 1, 1)
`);
insertWord.run('猫', '猫', '一般');
insertWord.run('太郎', '太郎', '固有名詞');
insertWord.run('東京', '東京都', '一般');
db.prepare(
`
INSERT INTO imm_stats_excluded_words (headword, word, reading)
VALUES ('東京', '東京', '')
`,
).run();
assert.deepEqual(getVocabularySummary(db, new Set(['猫', '太郎', '東京']), 9 * 86_400_000), {
uniqueWords: 2,
uniqueWordsWithoutNames: 1,
uniqueKanji: 0,
newThisWeek: 0,
newThisWeekWithoutNames: 0,
knownWordCount: 2,
knownWordCountWithoutNames: 1,
});
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getVocabularySummary counts identically across id-keyed scan batches', () => {
const dbPath = makeDbPath();
const db = openTestDb(dbPath);
try {
ensureSchema(db);
const insertWord = db.prepare(`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3,
first_seen, last_seen, frequency
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', 1, 1, 1)
`);
for (let index = 0; index < 5; index += 1) {
insertWord.run(`単語${index}`, `単語${index}`);
}
const fullScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000);
const batchedScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000, 2);
assert.equal(fullScan.uniqueWords, 5);
assert.deepEqual(batchedScan, fullScan);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () => {
const dbPath = makeDbPath();
const db = openTestDb(dbPath);
@@ -0,0 +1,136 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import {
LexicalRollupWorkerRuntime,
resolveLexicalRollupWorkerPath,
} from './lexical-rollup-worker-runtime';
import { areLexicalDailyRollupsReady } from './lexical-rollups';
import { Database } from './sqlite';
import { applyPragmas, ensureSchema } from './storage';
test('lexical rollup worker backfills without using the tracker connection', async () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-runtime-'));
const dbPath = path.join(directory, 'immersion.sqlite');
const runtime = new LexicalRollupWorkerRuntime();
const db = new Database(dbPath);
try {
applyPragmas(db);
ensureSchema(db);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES ('鳥', '鳥', 'とり', 1700000000, 1700000000, 1)`,
).run();
db.exec('DELETE FROM imm_lexical_daily_rollups');
db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
'lexical_daily_rollups_version',
);
db.close();
await runtime.run(dbPath);
const checkDb = new Database(dbPath);
try {
assert.equal(areLexicalDailyRollupsReady(checkDb), true);
} finally {
checkDb.close();
}
} finally {
runtime.destroy();
try {
db.close();
} catch {
// Closed before the worker starts.
}
fs.rmSync(directory, { recursive: true, force: true });
}
});
test('lexical rollup worker module resolves in the current layout', () => {
const workerPath = resolveLexicalRollupWorkerPath();
assert.ok(workerPath, 'expected the lexical rollup worker module to resolve');
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
});
test('lexical rollup worker leaves a backfill pending when no worker can start', async () => {
const runtime = new LexicalRollupWorkerRuntime({
resolveWorkerPath: () => null,
warn: () => {},
} as never);
try {
await assert.doesNotReject(runtime.run('/tmp/not-used.sqlite'));
} finally {
runtime.destroy();
}
});
test('lexical rollup worker absorbs termination failures after settling', async () => {
let sendMessage: ((message: { ok: boolean }) => void) | null = null;
const runtime = new LexicalRollupWorkerRuntime({
resolveWorkerPath: () => '/tmp/fake-worker.js',
createWorker: async () => ({
once(event: string, listener: (value: never) => void) {
if (event === 'message') sendMessage = listener as (message: { ok: boolean }) => void;
return this;
},
terminate: async () => {
throw new Error('termination failed');
},
}),
warn: () => {},
} as never);
const unhandled: unknown[] = [];
const captureUnhandled = (reason: unknown) => unhandled.push(reason);
process.on('unhandledRejection', captureUnhandled);
try {
const task = runtime.run('/tmp/not-used.sqlite');
await new Promise((resolve) => setImmediate(resolve));
const notify = sendMessage as ((message: { ok: boolean }) => void) | null;
assert.ok(notify);
notify({ ok: true });
await task;
await new Promise((resolve) => setImmediate(resolve));
assert.deepEqual(unhandled, []);
} finally {
process.off('unhandledRejection', captureUnhandled);
runtime.destroy();
}
});
test('lexical rollup worker times out when it never responds', async () => {
let terminated = false;
const runtime = new LexicalRollupWorkerRuntime({
resolveWorkerPath: () => '/tmp/fake-worker.js',
createWorker: async () => ({
once() {
return this;
},
terminate: async () => {
terminated = true;
return 0;
},
}),
timeoutMs: 1,
warn: () => {},
} as never);
try {
const outcome = await Promise.race([
runtime.run('/tmp/not-used.sqlite').then(
() => 'resolved',
(error: unknown) => String(error),
),
new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
]);
assert.match(outcome, /timed out/);
assert.equal(terminated, true);
} finally {
runtime.destroy();
}
});
@@ -0,0 +1,117 @@
import fs from 'node:fs';
import path from 'node:path';
import { createLogger } from '../../../logger';
interface WorkerResponse {
ok?: boolean;
error?: unknown;
}
interface WorkerHandle {
once(event: 'message', listener: (message: WorkerResponse) => void): this;
once(event: 'error', listener: (error: Error) => void): this;
once(event: 'exit', listener: (code: number) => void): this;
terminate(): Promise<number>;
}
interface LexicalRollupWorkerRuntimeOptions {
resolveWorkerPath?: () => string | null;
createWorker?: (workerPath: string, workerData: { dbPath: string }) => Promise<WorkerHandle>;
timeoutMs?: number;
warn?: (message: string, ...meta: unknown[]) => void;
}
const logger = createLogger('main:immersion-tracker:lexical-rollup-worker');
const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
export function resolveLexicalRollupWorkerPath(): string | null {
const fileName = __filename.endsWith('.ts')
? 'lexical-rollup-worker-thread.ts'
: 'lexical-rollup-worker-thread.js';
const workerPath = path.join(__dirname, fileName);
return fs.existsSync(workerPath) ? workerPath : null;
}
export class LexicalRollupWorkerRuntime {
private readonly activeWorkers = new Set<WorkerHandle>();
private destroyed = false;
constructor(private readonly options: LexicalRollupWorkerRuntimeOptions = {}) {}
async run(dbPath: string): Promise<void> {
if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
let worker: WorkerHandle;
try {
const workerPath = (this.options.resolveWorkerPath ?? resolveLexicalRollupWorkerPath)();
if (!workerPath) throw new Error('Emitted lexical rollup worker module was not found');
const createWorker =
this.options.createWorker ??
(async (resolvedPath, workerData) => {
const { Worker } = await import('node:worker_threads');
return new Worker(resolvedPath, { workerData });
});
worker = await createWorker(workerPath, { dbPath });
} catch (error) {
if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
(this.options.warn ?? logger.warn)(
'Lexical rollup worker unavailable; leaving backfill pending for a later startup',
error,
);
return;
}
if (this.destroyed) {
await worker.terminate().catch(() => undefined);
throw new Error('Lexical rollup worker is shut down');
}
return new Promise<void>((resolve, reject) => {
let settled = false;
let timeout: ReturnType<typeof setTimeout> | null = null;
this.activeWorkers.add(worker);
const settle = (error?: Error) => {
if (settled) return;
settled = true;
if (timeout) clearTimeout(timeout);
this.activeWorkers.delete(worker);
void worker.terminate().catch(() => undefined);
if (error) reject(error);
else resolve();
};
timeout = setTimeout(
() => settle(new Error('Lexical rollup worker timed out')),
this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
);
worker.once('message', (message) => {
if (message.ok) settle();
else
settle(
new Error(
`Lexical rollup backfill failed: ${String(message.error ?? 'unknown error')}`,
),
);
});
worker.once('error', (error) => settle(error));
worker.once('exit', (code) => {
if (!settled) {
settle(
new Error(
code === 0
? 'Lexical rollup worker exited without a response'
: `Lexical rollup worker exited with code ${code}`,
),
);
}
});
});
}
destroy(): void {
if (this.destroyed) return;
this.destroyed = true;
for (const worker of this.activeWorkers) {
void worker.terminate().catch(() => undefined);
}
this.activeWorkers.clear();
}
}
@@ -0,0 +1,11 @@
import { parentPort, workerData } from 'node:worker_threads';
import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
if (!parentPort) throw new Error('lexical rollup worker missing parent port');
try {
executeLexicalRollupBackfillTask((workerData as { dbPath: string }).dbPath);
parentPort.postMessage({ ok: true });
} catch (error) {
parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
}
@@ -0,0 +1,35 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
import { Database } from './sqlite';
import { ensureSchema } from './storage';
test('lexical rollup backfill materializes pre-existing vocabulary off the caller DB connection', () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-worker-'));
const dbPath = path.join(directory, 'immersion.sqlite');
const db = new Database(dbPath);
try {
ensureSchema(db);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, ?, ?, ?, 1)`,
).run('犬', '犬', 'いぬ', 1_700_000_000, 1_700_000_000);
db.exec('DELETE FROM imm_lexical_daily_rollups');
db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
'lexical_daily_rollups_version',
);
executeLexicalRollupBackfillTask(dbPath);
assert.equal(areLexicalDailyRollupsReady(db), true);
assert.equal(getLexicalDailyRollups(db)[0]?.wordCount, 1);
} finally {
db.close();
fs.rmSync(directory, { recursive: true, force: true });
}
});
@@ -0,0 +1,15 @@
import { areLexicalDailyRollupsReady, rebuildLexicalDailyRollups } from './lexical-rollups';
import { Database } from './sqlite';
import { applyPragmas } from './storage';
export function executeLexicalRollupBackfillTask(dbPath: string): void {
const db = new Database(dbPath);
try {
applyPragmas(db);
if (!areLexicalDailyRollupsReady(db)) {
rebuildLexicalDailyRollups(db);
}
} finally {
db.close();
}
}
@@ -0,0 +1,400 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import {
areLexicalDailyRollupsReady,
getLexicalDailyRollups,
rebuildLexicalDailyRollups,
} from './lexical-rollups';
import { getTrendsDashboard } from './query-trends';
import {
getVocabularyChartData,
getVocabularySummary,
replaceStatsExcludedWords,
} from './query-lexical';
import { Database } from './sqlite';
import type { DatabaseSync } from './sqlite';
import { ensureSchema } from './storage';
function makeDbPath(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollups-'));
return path.join(dir, 'immersion.sqlite');
}
test('lexical daily rollups follow first-seen corrections and deletions', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const firstDay = 19_500;
const correctedDay = firstDay + 2;
const firstSeen = firstDay * 86_400 + 43_200;
const correctedSeen = correctedDay * 86_400 + 43_200;
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, ?, ?, ?, 1)`,
).run('猫', '猫', 'ねこ', firstSeen, firstSeen);
db.prepare(
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
VALUES (?, ?, ?, 1)`,
).run('猫', firstSeen, firstSeen);
assert.deepEqual(getLexicalDailyRollups(db), [
{ epochDay: firstDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 1 },
]);
db.prepare(`UPDATE imm_words SET first_seen = ? WHERE headword = ?`).run(correctedSeen, '猫');
db.prepare(`DELETE FROM imm_kanji WHERE kanji = ?`).run('猫');
assert.deepEqual(getLexicalDailyRollups(db), [
{ epochDay: correctedDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
]);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('lexical daily rollups normalize second and millisecond timestamps', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const epochDay = 19_500;
const timestampSeconds = epochDay * 86_400 + 43_200;
const timestampMilliseconds = timestampSeconds * 1_000;
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, ?, ?, ?, 1)`,
).run('猫', '猫', 'ねこ', timestampSeconds, timestampSeconds);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, ?, ?, ?, 1)`,
).run('犬', '犬', 'いぬ', timestampMilliseconds, timestampMilliseconds);
db.prepare(
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
VALUES (?, ?, ?, 1)`,
).run('猫', timestampSeconds, timestampSeconds);
db.prepare(
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
VALUES (?, ?, ?, 1)`,
).run('犬', timestampMilliseconds, timestampMilliseconds);
assert.deepEqual(getLexicalDailyRollups(db), [
{ epochDay, wordCount: 2, wordCountWithoutNames: 2, kanjiCount: 2 },
]);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('lexical rollup rebuild excludes rows hidden by vocabulary persistence rules', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const epochDay = 19_500;
const firstSeen = epochDay * 86_400 + 43_200;
db.prepare(
`INSERT INTO imm_words(
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
db.prepare(
`INSERT INTO imm_words(
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
rebuildLexicalDailyRollups(db);
assert.deepEqual(getLexicalDailyRollups(db), [
{ epochDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
]);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('lexical rollup rebuild tolerates nullable legacy vocabulary text', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (NULL, NULL, NULL, 1700000000, 1700000000, 1)`,
).run();
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (NULL, '猫', 'ねこ', 1700000000, 1700000000, 1)`,
).run();
assert.doesNotThrow(() => rebuildLexicalDailyRollups(db));
assert.equal(areLexicalDailyRollupsReady(db), true);
assert.equal(getVocabularySummary(db, null).uniqueWords, 1);
assert.equal(getVocabularySummary(db, new Set(['猫'])).knownWordCount, 1);
assert.equal(getVocabularyChartData(db).topWords[0]?.headword, '猫');
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('lexical rollup rebuild scans vocabulary visibility in bounded id batches', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
const expectedBatchSize = 5_000;
try {
ensureSchema(db);
const insertWord = db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, '', 1700000000, 1700000000, 1)`,
);
db.exec('BEGIN');
for (let index = 0; index <= expectedBatchSize; index += 1) {
insertWord.run(`${index}`, `${index}`);
}
db.exec('COMMIT');
const scanPageSizes: number[] = [];
const instrumentedDb: DatabaseSync = {
prepare(source) {
const statement = db.prepare(source);
if (!source.includes('WHERE id > ?') || !source.includes('ORDER BY id')) {
return statement;
}
return {
run: (...params) => statement.run(...params),
get: (...params) => statement.get(...params),
all: (...params) => {
const rows = statement.all(...params);
scanPageSizes.push(rows.length);
return rows;
},
};
},
exec(source) {
db.exec(source);
return instrumentedDb;
},
close() {
return instrumentedDb;
},
};
rebuildLexicalDailyRollups(instrumentedDb);
assert.deepEqual(scanPageSizes, [expectedBatchSize, 1]);
assert.equal(getVocabularySummary(db, null).uniqueWords, expectedBatchSize + 1);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('chart exclusions do not subtract vocabulary rows already hidden from the rollup', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const epochDay = 19_500;
const firstSeen = epochDay * 86_400 + 43_200;
db.prepare(
`INSERT INTO imm_words(
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
db.prepare(
`INSERT INTO imm_words(
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
rebuildLexicalDailyRollups(db);
replaceStatsExcludedWords(db, [{ headword: 'は', word: 'は', reading: 'は' }]);
const charts = getVocabularyChartData(db);
assert.deepEqual(charts.newWordsTimeline, [{ epochDay, wordCount: 1 }]);
assert.deepEqual(charts.newWordsTimelineWithoutNames, [{ epochDay, wordCount: 1 }]);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('legacy lexical rollup readiness does not satisfy the current rollup version', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
db.prepare(
`INSERT INTO imm_rollup_state(state_key, state_value)
VALUES ('lexical_daily_rollups_ready', '1')
ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
).run();
db.prepare(
`DELETE FROM imm_rollup_state WHERE state_key = 'lexical_daily_rollups_version'`,
).run();
assert.equal(areLexicalDailyRollupsReady(db), false);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('imm_words persists vocabulary visibility for rollup maintenance', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const columns = db.prepare(`PRAGMA table_info(imm_words)`).all() as Array<{ name: string }>;
assert.equal(
columns.some((column) => column.name === 'vocabulary_visible'),
true,
);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('vocabulary charts use complete top-word and lexical rollup data', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const insertWord = db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, '', 1700000000, 1700000000, ?)`,
);
for (let index = 0; index < 501; index += 1) {
insertWord.run(`${index}`, `${index}`, index === 500 ? 10_000 : 1);
}
const charts = getVocabularyChartData(db);
assert.equal(charts.topWords[0]?.headword, '語500');
assert.equal(charts.topWords[0]?.frequency, 10_000);
assert.equal(charts.newWordsTimeline[0]?.wordCount, 501);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('vocabulary charts find full top-word sets beyond excluded and name rows', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const insertWord = db.prepare(
`INSERT INTO imm_words(headword, word, reading, pos2, first_seen, last_seen, frequency)
VALUES (?, ?, '', ?, 1700000000, 1700000000, ?)`,
);
const exclusions = [];
for (let index = 0; index < 100; index += 1) {
const headword = `${index}`;
insertWord.run(
headword,
headword,
index < 80 && index >= 60 ? '固有名詞' : '一般',
100 - index,
);
if (index < 60) exclusions.push({ headword, word: headword, reading: '' });
}
replaceStatsExcludedWords(db, exclusions);
const charts = getVocabularyChartData(db);
assert.equal(charts.topWords.length, 12);
assert.equal(charts.topWords[0]?.headword, '語60');
assert.equal(charts.topWordsWithoutNames.length, 12);
assert.equal(charts.topWordsWithoutNames[0]?.headword, '語80');
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('vocabulary charts handle exclusion lists above one SQLite variable batch', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES ('語0', '語0', '', 1700000000, 1700000000, 1)`,
).run();
const exclusions = Array.from({ length: 10_923 }, (_, index) => ({
headword: `${index}`,
word: `${index}`,
reading: '',
}));
replaceStatsExcludedWords(db, exclusions);
const charts = getVocabularyChartData(db);
assert.deepEqual(charts.topWords, []);
assert.deepEqual(charts.newWordsTimeline, []);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('lexical rollup rebuild preserves the original error when rollback also fails', () => {
const originalError = new Error('rebuild failed');
const db = {
exec(sql: string) {
if (sql === 'BEGIN IMMEDIATE') return;
if (sql === 'ROLLBACK') throw new Error('rollback failed');
throw originalError;
},
prepare() {
return { all: () => [], run: () => undefined };
},
} as unknown as DatabaseSync;
assert.throws(() => rebuildLexicalDailyRollups(db), originalError);
});
test('trends read historical new-word buckets from lexical rollups', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES ('海', '海', 'うみ', 1700000000, 1700000000, 1)`,
).run();
db.prepare(`UPDATE imm_lexical_daily_rollups SET word_count = 9`).run();
const dashboard = getTrendsDashboard(db, 'all', 'day', false);
assert.equal(dashboard.progress.newWords[0]?.value, 9);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
@@ -0,0 +1,226 @@
import type { DatabaseSync } from './sqlite';
import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility';
export interface LexicalDailyRollup {
epochDay: number;
wordCount: number;
wordCountWithoutNames: number;
kanjiCount: number;
}
const LOCAL_EPOCH_DAY_SQL = `
CAST(julianday(
CASE
WHEN ABS(CAST(%VALUE% AS REAL)) >= 10000000000 THEN CAST(%VALUE% AS REAL) / 1000
ELSE CAST(%VALUE% AS REAL)
END,
'unixepoch', 'localtime'
) - 2440587.5 AS INTEGER)
`;
const LEXICAL_DAILY_ROLLUP_VERSION = '2';
const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
export function localEpochDaySql(value: string): string {
return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
}
function createWordRollupTriggers(db: DatabaseSync): void {
const dayForNew = localEpochDaySql('NEW.first_seen');
const dayForOld = localEpochDaySql('OLD.first_seen');
db.exec(`
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_insert;
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_delete;
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_first_seen_update;
CREATE TRIGGER imm_words_lexical_rollup_insert
AFTER INSERT ON imm_words
WHEN NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
BEGIN
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
VALUES (${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0)
ON CONFLICT(epoch_day) DO UPDATE SET
word_count = word_count + 1,
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
END;
CREATE TRIGGER imm_words_lexical_rollup_delete
AFTER DELETE ON imm_words
WHEN OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
BEGIN
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
VALUES (${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0)
ON CONFLICT(epoch_day) DO UPDATE SET
word_count = word_count - 1,
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
DELETE FROM imm_lexical_daily_rollups
WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
END;
CREATE TRIGGER imm_words_lexical_rollup_first_seen_update
AFTER UPDATE OF first_seen, pos2, vocabulary_visible ON imm_words
WHEN OLD.first_seen IS NOT NEW.first_seen
OR OLD.pos2 IS NOT NEW.pos2
OR OLD.vocabulary_visible IS NOT NEW.vocabulary_visible
BEGIN
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
SELECT ${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0
WHERE OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
ON CONFLICT(epoch_day) DO UPDATE SET
word_count = word_count - 1,
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
SELECT ${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0
WHERE NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
ON CONFLICT(epoch_day) DO UPDATE SET
word_count = word_count + 1,
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
DELETE FROM imm_lexical_daily_rollups
WHERE word_count = 0 AND kanji_count = 0;
END;
`);
}
function createKanjiRollupTriggers(db: DatabaseSync): void {
const dayForNew = localEpochDaySql('NEW.first_seen');
const dayForOld = localEpochDaySql('OLD.first_seen');
db.exec(`
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_insert;
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_delete;
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_first_seen_update;
CREATE TRIGGER imm_kanji_lexical_rollup_insert
AFTER INSERT ON imm_kanji WHEN NEW.first_seen IS NOT NULL
BEGIN
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
VALUES (${dayForNew}, 0, 0, 1)
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
END;
CREATE TRIGGER imm_kanji_lexical_rollup_delete
AFTER DELETE ON imm_kanji WHEN OLD.first_seen IS NOT NULL
BEGIN
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
VALUES (${dayForOld}, 0, 0, -1)
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
DELETE FROM imm_lexical_daily_rollups
WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
END;
CREATE TRIGGER imm_kanji_lexical_rollup_first_seen_update
AFTER UPDATE OF first_seen ON imm_kanji WHEN OLD.first_seen IS NOT NEW.first_seen
BEGIN
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
SELECT ${dayForOld}, 0, 0, -1 WHERE OLD.first_seen IS NOT NULL
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
SELECT ${dayForNew}, 0, 0, 1 WHERE NEW.first_seen IS NOT NULL
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
DELETE FROM imm_lexical_daily_rollups WHERE word_count = 0 AND kanji_count = 0;
END;
`);
}
export function ensureLexicalDailyRollupTables(db: DatabaseSync): void {
db.exec(`
CREATE TABLE IF NOT EXISTS imm_lexical_daily_rollups(
epoch_day INTEGER PRIMARY KEY,
word_count INTEGER NOT NULL DEFAULT 0,
word_count_without_names INTEGER NOT NULL DEFAULT 0,
kanji_count INTEGER NOT NULL DEFAULT 0
);
INSERT INTO imm_rollup_state(state_key, state_value)
VALUES ('${LEXICAL_DAILY_ROLLUP_VERSION_KEY}', '0')
ON CONFLICT(state_key) DO NOTHING;
`);
createWordRollupTriggers(db);
createKanjiRollupTriggers(db);
}
export function areLexicalDailyRollupsReady(db: DatabaseSync): boolean {
const row = db
.prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
.get(LEXICAL_DAILY_ROLLUP_VERSION_KEY) as { value: string } | null;
return row?.value === LEXICAL_DAILY_ROLLUP_VERSION;
}
export function markLexicalDailyRollupsReady(db: DatabaseSync): void {
db.prepare(
`INSERT INTO imm_rollup_state(state_key, state_value)
VALUES (?, ?)
ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
).run(LEXICAL_DAILY_ROLLUP_VERSION_KEY, LEXICAL_DAILY_ROLLUP_VERSION);
}
/** Rebuild from the first-seen source of truth; run off the UI/main DB thread. */
export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
let transactionStarted = false;
try {
db.exec('BEGIN IMMEDIATE');
transactionStarted = true;
const scanVocabulary = db.prepare(
`SELECT id, word, headword, reading, part_of_speech AS partOfSpeech,
pos1, pos2, pos3, frequency_rank AS frequencyRank
FROM imm_words
WHERE id > ?
ORDER BY id
LIMIT ?`,
);
const updateVisibility = db.prepare(
`UPDATE imm_words SET vocabulary_visible = ? WHERE id = ? AND vocabulary_visible IS NOT ?`,
);
let lastId = Number.MIN_SAFE_INTEGER;
for (;;) {
const vocabularyRows = scanVocabulary.all(
lastId,
VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE,
) as Array<VocabularyVisibilityRow & { id: number }>;
if (vocabularyRows.length === 0) break;
for (const row of vocabularyRows) {
const visible = isVocabularyStatsRowVisible(row) ? 1 : 0;
updateVisibility.run(visible, row.id, visible);
}
lastId = vocabularyRows[vocabularyRows.length - 1]!.id;
if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break;
}
db.exec('DELETE FROM imm_lexical_daily_rollups');
db.exec(`
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
SELECT ${localEpochDaySql('first_seen')}, COUNT(*),
SUM(CASE WHEN pos2 = '固有名詞' THEN 0 ELSE 1 END), 0
FROM imm_words
WHERE first_seen IS NOT NULL AND vocabulary_visible = 1
GROUP BY ${localEpochDaySql('first_seen')};
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
SELECT ${localEpochDaySql('first_seen')}, 0, 0, COUNT(*)
FROM imm_kanji
WHERE first_seen IS NOT NULL
GROUP BY ${localEpochDaySql('first_seen')}
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + excluded.kanji_count;
`);
markLexicalDailyRollupsReady(db);
db.exec('COMMIT');
} catch (error) {
if (transactionStarted) {
try {
db.exec('ROLLBACK');
} catch {
// Preserve the rebuild failure; it is the actionable cause.
}
}
throw error;
}
}
export function getLexicalDailyRollups(db: DatabaseSync): LexicalDailyRollup[] {
return db
.prepare(
`
SELECT epoch_day AS epochDay, word_count AS wordCount,
word_count_without_names AS wordCountWithoutNames, kanji_count AS kanjiCount
FROM imm_lexical_daily_rollups
ORDER BY epoch_day ASC
`,
)
.all() as LexicalDailyRollup[];
}
@@ -1,6 +1,4 @@
import type { DatabaseSync } from './sqlite';
import { PartOfSpeech, type MergedToken } from '../../../types';
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
import type {
KanjiAnimeAppearanceRow,
KanjiDetailRow,
@@ -13,19 +11,38 @@ import type {
SimilarWordRow,
StatsExcludedWordRow,
VocabularyStatsRow,
VocabularyStatsSummary,
WordAnimeAppearanceRow,
WordDetailRow,
WordOccurrenceRow,
} from './types';
import { fromDbTimestamp, toDbTimestamp } from './query-shared';
import { nowMs } from './time';
import {
areLexicalDailyRollupsReady,
getLexicalDailyRollups,
localEpochDaySql,
} from './lexical-rollups';
import { isVocabularyStatsRowVisible } from './vocabulary-visibility';
const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
const VOCABULARY_CHART_LIMIT = 12;
const VOCABULARY_CHART_PAGE_SIZE = 100;
const EXCLUSION_ALIAS_BATCH_SIZE = 300;
const VOCABULARY_SUMMARY_SCAN_BATCH_SIZE = 5_000;
const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
const SENTENCE_SEARCH_MAX_LIMIT = 100;
const KANJI_PATTERN = /\p{Script=Han}/gu;
export interface VocabularyChartData {
ready: boolean;
topWords: Array<{ wordId: number; headword: string; frequency: number }>;
topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>;
newWordsTimeline: Array<{ epochDay: number; wordCount: number }>;
newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>;
}
function resolveSentenceSearchLimit(limit: number): number {
if (!Number.isFinite(limit)) return SENTENCE_SEARCH_DEFAULT_LIMIT;
const normalized = Math.floor(limit);
@@ -73,33 +90,6 @@ function uniqueKanji(text: string): string[] {
return Array.from(new Set(text.match(KANJI_PATTERN) ?? []));
}
function toVocabularyToken(row: VocabularyStatsRow): MergedToken {
const partOfSpeech =
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
? (row.partOfSpeech as PartOfSpeech)
: PartOfSpeech.other;
return {
surface: row.word,
reading: row.reading ?? '',
headword: row.headword,
startPos: 0,
endPos: row.word.length,
partOfSpeech,
pos1: row.pos1 ?? '',
pos2: row.pos2 ?? '',
pos3: row.pos3 ?? '',
frequencyRank: row.frequencyRank ?? undefined,
isMerged: false,
isKnown: false,
isNPlusOneTarget: false,
};
}
function isVocabularyStatsRowVisible(row: VocabularyStatsRow): boolean {
return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
}
export function getVocabularyStats(
db: DatabaseSync,
limit = 100,
@@ -153,6 +143,198 @@ export function getVocabularyStats(
return visibleRows.slice(0, limit);
}
/**
* Chart data is intentionally independent of the paginated vocabulary tables.
* Top words use the frequency index; new-word history reads permanent daily
* lexical rollups rather than loading every vocabulary row into the dashboard.
*/
export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData {
const ready = areLexicalDailyRollupsReady(db);
const excludedAliases = new Set(
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
);
const isExcluded = (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>): boolean =>
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias));
const topWords = getTopVocabularyChartWords(db, isExcluded);
const rollups = ready ? getLexicalDailyRollups(db) : [];
const timeline = new Map(rollups.map((row) => [row.epochDay, { ...row }]));
if (excludedAliases.size > 0 && ready) {
const aliases = [...excludedAliases];
const excludedRows = new Map<
number,
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
wordId: number;
epochDay: number;
}
>();
for (let offset = 0; offset < aliases.length; offset += EXCLUSION_ALIAS_BATCH_SIZE) {
const batch = aliases.slice(offset, offset + EXCLUSION_ALIAS_BATCH_SIZE);
const placeholders = batch.map(() => '?').join(', ');
const rows = db
.prepare(
`
SELECT id AS wordId, headword, word, reading, pos2,
${localEpochDaySql('first_seen')} AS epochDay
FROM imm_words
WHERE vocabulary_visible = 1
AND (headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}))
`,
)
.all(...batch, ...batch, ...batch) as Array<
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
wordId: number;
epochDay: number;
}
>;
for (const row of rows) excludedRows.set(row.wordId, row);
}
for (const word of excludedRows.values()) {
if (!isExcluded(word)) continue;
const rollup = timeline.get(word.epochDay);
if (!rollup) continue;
rollup.wordCount -= 1;
if (word.pos2 !== '固有名詞') rollup.wordCountWithoutNames -= 1;
}
}
return {
ready,
topWords: topWords.all.map((word) => ({
wordId: word.wordId,
headword: vocabularyDisplayHeadword(word),
frequency: word.frequency,
})),
topWordsWithoutNames: topWords.withoutNames.map((word) => ({
wordId: word.wordId,
headword: vocabularyDisplayHeadword(word),
frequency: word.frequency,
})),
newWordsTimeline: [...timeline.values()]
.filter((row) => row.wordCount > 0)
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCount })),
newWordsTimelineWithoutNames: [...timeline.values()]
.filter((row) => row.wordCountWithoutNames > 0)
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCountWithoutNames })),
};
}
function getTopVocabularyChartWords(
db: DatabaseSync,
isExcluded: (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>) => boolean,
): { all: VocabularyStatsRow[]; withoutNames: VocabularyStatsRow[] } {
const stmt = db.prepare(`
SELECT id AS wordId, headword, word, reading,
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
frequency, frequency_rank AS frequencyRank,
first_seen AS firstSeen, last_seen AS lastSeen,
0 AS animeCount
FROM imm_words
ORDER BY frequency DESC, id
LIMIT ? OFFSET ?
`);
const all: VocabularyStatsRow[] = [];
const withoutNames: VocabularyStatsRow[] = [];
let offset = 0;
while (all.length < VOCABULARY_CHART_LIMIT || withoutNames.length < VOCABULARY_CHART_LIMIT) {
const page = stmt.all(VOCABULARY_CHART_PAGE_SIZE, offset) as VocabularyStatsRow[];
if (page.length === 0) break;
for (const word of page) {
if (!isVocabularyStatsRowVisible(word) || isExcluded(word)) continue;
if (all.length < VOCABULARY_CHART_LIMIT) all.push(word);
if (word.pos2 !== '固有名詞' && withoutNames.length < VOCABULARY_CHART_LIMIT) {
withoutNames.push(word);
}
}
offset += page.length;
}
return { all, withoutNames };
}
function excludedVocabularyAliases(
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
): string[] {
const aliases = [word.headword?.trim() ?? '', word.word?.trim() ?? ''].filter(Boolean);
if (aliases.length === 0) aliases.push(word.reading?.trim() ?? '');
return [...new Set(aliases)];
}
function vocabularyDisplayHeadword(
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
): string {
return word.headword?.trim() || word.word?.trim() || word.reading?.trim() || '';
}
function timestampSeconds(timestamp: number): number {
return timestamp < 10_000_000_000 ? timestamp : Math.floor(timestamp / 1000);
}
export function getVocabularySummary(
db: DatabaseSync,
knownWords: ReadonlySet<string> | null,
nowMs: number = Date.now(),
scanBatchSize: number = VOCABULARY_SUMMARY_SCAN_BATCH_SIZE,
): VocabularyStatsSummary {
// Visibility and exclusion rules live in JS, so rows are scanned in id-keyed
// batches to keep memory bounded on large vocabularies.
const scanStmt = db.prepare(`
SELECT id AS wordId, headword, word, reading,
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
frequency, frequency_rank AS frequencyRank,
first_seen AS firstSeen, last_seen AS lastSeen,
0 AS animeCount
FROM imm_words
WHERE id > ?
ORDER BY id
LIMIT ?
`);
const excludedAliases = new Set(
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
);
const weekAgoSec = nowMs / 1000 - 7 * 86_400;
const summary: VocabularyStatsSummary = {
uniqueWords: 0,
uniqueWordsWithoutNames: 0,
uniqueKanji: (db.prepare('SELECT COUNT(*) AS count FROM imm_kanji').get() as { count: number })
.count,
newThisWeek: 0,
newThisWeekWithoutNames: 0,
knownWordCount: knownWords ? 0 : null,
knownWordCountWithoutNames: knownWords ? 0 : null,
};
let lastId = Number.MIN_SAFE_INTEGER;
for (;;) {
const words = scanStmt.all(lastId, scanBatchSize) as VocabularyStatsRow[];
if (words.length === 0) break;
lastId = words[words.length - 1]!.wordId;
for (const word of words) {
if (
!isVocabularyStatsRowVisible(word) ||
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias))
) {
continue;
}
const isName = word.pos2 === '固有名詞';
const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec;
const isKnown = knownWords?.has(vocabularyDisplayHeadword(word)) ?? false;
summary.uniqueWords += 1;
if (!isName) summary.uniqueWordsWithoutNames += 1;
if (isNewThisWeek) {
summary.newThisWeek += 1;
if (!isName) summary.newThisWeekWithoutNames += 1;
}
if (isKnown) {
summary.knownWordCount! += 1;
if (!isName) summary.knownWordCountWithoutNames! += 1;
}
}
if (words.length < scanBatchSize) break;
}
return summary;
}
export function getStatsExcludedWords(db: DatabaseSync): StatsExcludedWordRow[] {
return db
.prepare(
@@ -13,6 +13,7 @@ import {
toDbTimestamp,
} from './query-shared';
import { getDailyRollups, getMonthlyRollups } from './query-sessions';
import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
type TrendRange = '7d' | '30d' | '90d' | '365d' | 'all';
type TrendGroupBy = 'day' | 'month';
@@ -660,6 +661,16 @@ function buildNewWordsPerDay(
cutoffMs: string | null,
axis: number[] | null,
): TrendChartPoint[] {
if (areLexicalDailyRollupsReady(db)) {
// A trend range is defined in calendar buckets, so the rollup includes the
// complete local cutoff day rather than applying a time-of-day boundary.
const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
const rows = getLexicalDailyRollups(db).filter(
(row) => cutoffDay === null || row.epochDay >= cutoffDay,
);
return fillAxisPoints(axis, new Map(rows.map((row) => [row.epochDay, row.wordCount])));
}
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
const prepared = db.prepare(`
SELECT
@@ -691,6 +702,18 @@ function buildNewWordsPerMonth(
cutoffMs: string | null,
axis: number[] | null,
): TrendChartPoint[] {
if (areLexicalDailyRollupsReady(db)) {
const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
const byMonth = new Map<number, number>();
for (const row of getLexicalDailyRollups(db)) {
if (cutoffDay !== null && row.epochDay < cutoffDay) continue;
const { year, month } = dayPartsFromEpochDay(row.epochDay);
const monthKey = year * 100 + month;
byMonth.set(monthKey, (byMonth.get(monthKey) ?? 0) + row.wordCount);
}
return fillAxisPoints(axis, byMonth);
}
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
const prepared = db.prepare(`
SELECT
+23 -3
View File
@@ -4,6 +4,7 @@ import { parseMediaInfo } from '../../../jimaku/utils';
import { normalizeTitleIdentity } from '../../utils/title-normalization';
import type { DatabaseSync } from './sqlite';
import { nowMs } from './time';
import { ensureLexicalDailyRollupTables, markLexicalDailyRollupsReady } from './lexical-rollups';
import { SCHEMA_VERSION } from './types';
import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types';
import { toDbMs, toDbTimestamp } from './query-shared';
@@ -890,11 +891,11 @@ export function ensureSchema(db: DatabaseSync): void {
VALUES ('last_rollup_sample_ms', 0)
ON CONFLICT(state_key) DO NOTHING
`);
const currentVersion = db
.prepare('SELECT schema_version FROM imm_schema_version ORDER BY schema_version DESC LIMIT 1')
.get() as { schema_version: number } | null;
if (currentVersion?.schema_version === SCHEMA_VERSION) {
ensureLexicalDailyRollupTables(db);
ensureLifetimeSummaryTables(db);
ensureStatsExcludedWordsTable(db);
ensureAnimeMergeTables(db);
@@ -1068,6 +1069,7 @@ export function ensureSchema(db: DatabaseSync): void {
last_seen REAL,
frequency INTEGER,
frequency_rank INTEGER,
vocabulary_visible INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1)),
UNIQUE(headword, word, reading)
);
`);
@@ -1451,8 +1453,18 @@ export function ensureSchema(db: DatabaseSync): void {
addColumnIfMissing(db, 'imm_sessions', 'ended_media_ms', 'INTEGER');
}
if (currentVersion?.schema_version && currentVersion.schema_version < 23) {
addColumnIfMissing(
db,
'imm_words',
'vocabulary_visible',
'INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1))',
);
}
migrateSessionEventTimestampsToText(db);
ensureLexicalDailyRollupTables(db);
ensureLifetimeSummaryTables(db);
ensureStatsExcludedWordsTable(db);
@@ -1585,6 +1597,12 @@ export function ensureSchema(db: DatabaseSync): void {
VALUES (${SCHEMA_VERSION}, ${toDbTimestamp(nowMs())})
ON CONFLICT DO NOTHING
`);
// A new database has no history to materialize. Upgrades are populated by the
// background worker so startup never scans the existing vocabulary table.
if (!currentVersion) {
markLexicalDailyRollupsReady(db);
}
}
export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPreparedStatements {
@@ -1617,9 +1635,10 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
`),
wordUpsertStmt: db.prepare(`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency, frequency_rank
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen,
frequency, frequency_rank, vocabulary_visible
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?
?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, 1
)
ON CONFLICT(headword, word, reading) DO UPDATE SET
frequency = COALESCE(frequency, 0) + 1,
@@ -1632,6 +1651,7 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
pos1 = COALESCE(NULLIF(imm_words.pos1, ''), excluded.pos1),
pos2 = COALESCE(NULLIF(imm_words.pos2, ''), excluded.pos2),
pos3 = COALESCE(NULLIF(imm_words.pos3, ''), excluded.pos3),
vocabulary_visible = 1,
first_seen = MIN(COALESCE(first_seen, excluded.first_seen), excluded.first_seen),
last_seen = MAX(COALESCE(last_seen, excluded.last_seen), excluded.last_seen),
frequency_rank = CASE
+11 -1
View File
@@ -1,4 +1,4 @@
export const SCHEMA_VERSION = 21;
export const SCHEMA_VERSION = 23;
export const DEFAULT_QUEUE_CAP = 1_000;
export const DEFAULT_BATCH_SIZE = 25;
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
@@ -306,6 +306,16 @@ export interface VocabularyStatsRow {
lastSeen: number;
}
export interface VocabularyStatsSummary {
uniqueWords: number;
uniqueWordsWithoutNames: number;
uniqueKanji: number;
newThisWeek: number;
newThisWeekWithoutNames: number;
knownWordCount: number | null;
knownWordCountWithoutNames: number | null;
}
export interface StatsExcludedWordRow {
headword: string;
word: string;
@@ -0,0 +1,100 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import {
resolveVocabularySummaryWorkerPath,
VocabularySummaryWorkerRuntime,
} from './vocabulary-summary-worker-runtime';
import { Database } from './sqlite';
import { applyPragmas, ensureSchema } from './storage';
test('vocabulary summary worker reads the database from a separate connection', async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-vocabulary-summary-worker-'));
const dbPath = path.join(tempDir, 'immersion.sqlite');
const runtime = new VocabularySummaryWorkerRuntime();
const db = new Database(dbPath);
try {
applyPragmas(db);
ensureSchema(db);
db.prepare(
`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3,
first_seen, last_seen, frequency
) VALUES ('猫', '猫', 'ねこ', 'noun', '名詞', '一般', '', 1, 1, 1)
`,
).run();
db.close();
const summary = await runtime.run(dbPath, new Set(['猫']));
assert.equal(summary.uniqueWords, 1);
assert.equal(summary.knownWordCount, 1);
} finally {
runtime.destroy();
try {
db.close();
} catch {
// The worker needs the setup connection closed before it starts.
}
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
test('vocabulary summary worker module resolves in the current layout', () => {
const workerPath = resolveVocabularySummaryWorkerPath();
assert.ok(workerPath, 'expected the vocabulary summary worker module to resolve');
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
});
test('vocabulary summary worker never falls back to the caller thread', async () => {
const runtime = new VocabularySummaryWorkerRuntime({
resolveWorkerPath: () => null,
warn: () => {},
});
try {
await assert.rejects(
runtime.run('/tmp/subminer-summary-worker-not-used.sqlite', null),
/worker unavailable/i,
);
} finally {
runtime.destroy();
}
});
test('vocabulary summary worker times out when it never responds', async () => {
let terminated = false;
const runtime = new VocabularySummaryWorkerRuntime({
resolveWorkerPath: () => '/tmp/fake-worker.js',
createWorker: async () => ({
once() {
return this;
},
terminate: async () => {
terminated = true;
return 0;
},
}),
timeoutMs: 1,
warn: () => {},
} as never);
try {
const outcome = await Promise.race([
runtime.run('/tmp/not-used.sqlite', null).then(
() => 'resolved',
(error: unknown) => String(error),
),
new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
]);
assert.match(outcome, /timed out/);
assert.equal(terminated, true);
} finally {
runtime.destroy();
}
});
@@ -0,0 +1,133 @@
import fs from 'node:fs';
import path from 'node:path';
import { createLogger } from '../../../logger';
import type { VocabularyStatsSummary } from './types';
interface VocabularySummaryWorkerResponse {
summary?: VocabularyStatsSummary;
error?: unknown;
}
interface VocabularySummaryWorkerHandle {
once(event: 'message', listener: (message: VocabularySummaryWorkerResponse) => void): this;
once(event: 'error', listener: (error: Error) => void): this;
once(event: 'exit', listener: (code: number) => void): this;
terminate(): Promise<number>;
}
interface VocabularySummaryWorkerRuntimeOptions {
resolveWorkerPath?: () => string | null;
createWorker?: (
workerPath: string,
workerData: { dbPath: string; knownWords: string[] | null },
) => Promise<VocabularySummaryWorkerHandle>;
timeoutMs?: number;
warn?: (message: string, ...meta: unknown[]) => void;
}
export type RunVocabularySummaryTask = (
dbPath: string,
knownWords: ReadonlySet<string> | null,
) => Promise<VocabularyStatsSummary>;
export function resolveVocabularySummaryWorkerPath(): string | null {
const fileName = __filename.endsWith('.ts')
? 'vocabulary-summary-worker-thread.ts'
: 'vocabulary-summary-worker-thread.js';
const workerPath = path.join(__dirname, fileName);
return fs.existsSync(workerPath) ? workerPath : null;
}
const logger = createLogger('main:immersion-tracker:vocabulary-summary-worker');
const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
export class VocabularySummaryWorkerRuntime {
private readonly activeWorkers = new Set<VocabularySummaryWorkerHandle>();
private destroyed = false;
constructor(private readonly options: VocabularySummaryWorkerRuntimeOptions = {}) {}
async run(
dbPath: string,
knownWords: ReadonlySet<string> | null,
): Promise<VocabularyStatsSummary> {
if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
const workerData = { dbPath, knownWords: knownWords ? [...knownWords] : null };
let worker: VocabularySummaryWorkerHandle;
try {
const workerPath = (this.options.resolveWorkerPath ?? resolveVocabularySummaryWorkerPath)();
if (!workerPath) throw new Error('Emitted vocabulary summary worker module was not found');
const createWorker =
this.options.createWorker ??
(async (resolvedPath, data) => {
const { Worker } = await import('node:worker_threads');
return new Worker(resolvedPath, { workerData: data });
});
worker = await createWorker(workerPath, workerData);
} catch (error) {
if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
(this.options.warn ?? logger.warn)(
'Vocabulary summary worker unavailable; refusing to scan vocabulary on the current thread',
error,
);
throw new Error('Vocabulary summary worker unavailable');
}
if (this.destroyed) {
await worker.terminate().catch(() => undefined);
throw new Error('Vocabulary summary worker is shut down');
}
return new Promise<VocabularyStatsSummary>((resolve, reject) => {
let settled = false;
let timeout: ReturnType<typeof setTimeout> | null = null;
this.activeWorkers.add(worker);
const settle = (result: VocabularyStatsSummary | Error) => {
if (settled) return;
settled = true;
if (timeout) clearTimeout(timeout);
this.activeWorkers.delete(worker);
void worker.terminate().catch(() => undefined);
if (result instanceof Error) reject(result);
else resolve(result);
};
timeout = setTimeout(
() => settle(new Error('Vocabulary summary worker timed out')),
this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
);
worker.once('message', (message) => {
if (message.summary) {
settle(message.summary);
return;
}
settle(
new Error(
`Vocabulary summary failed: ${String(message.error ?? 'unknown worker error')}`,
),
);
});
worker.once('error', (error) => settle(error));
worker.once('exit', (code) => {
if (!settled) {
settle(
new Error(
code === 0
? 'Vocabulary summary worker exited without a response'
: `Vocabulary summary worker exited with code ${code}`,
),
);
}
});
});
}
destroy(): void {
if (this.destroyed) return;
this.destroyed = true;
for (const worker of this.activeWorkers) {
void worker.terminate().catch(() => undefined);
}
this.activeWorkers.clear();
}
}
@@ -0,0 +1,19 @@
import { parentPort, workerData } from 'node:worker_threads';
import { executeVocabularySummaryTask } from './vocabulary-summary-worker';
interface VocabularySummaryWorkerData {
dbPath: string;
knownWords: string[] | null;
}
if (!parentPort) throw new Error('vocabulary summary worker missing parent port');
const request = workerData as VocabularySummaryWorkerData;
try {
parentPort.postMessage({
summary: executeVocabularySummaryTask(request.dbPath, request.knownWords),
});
} catch (error) {
parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
}
@@ -0,0 +1,17 @@
import { getVocabularySummary } from './query-lexical';
import { Database } from './sqlite';
import { applyPragmas } from './storage';
import type { VocabularyStatsSummary } from './types';
export function executeVocabularySummaryTask(
dbPath: string,
knownWords: string[] | null,
): VocabularyStatsSummary {
const db = new Database(dbPath);
try {
applyPragmas(db);
return getVocabularySummary(db, knownWords ? new Set(knownWords) : null);
} finally {
db.close();
}
}
@@ -0,0 +1,43 @@
import { PartOfSpeech, type MergedToken } from '../../../types';
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
export interface VocabularyVisibilityRow {
word: string | null;
headword: string | null;
reading?: string | null;
partOfSpeech?: string | null;
pos1?: string | null;
pos2?: string | null;
pos3?: string | null;
frequencyRank?: number | null;
}
function toVocabularyToken(row: VocabularyVisibilityRow): MergedToken {
const word = row.word ?? '';
const headword = row.headword ?? word;
const partOfSpeech =
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
? (row.partOfSpeech as PartOfSpeech)
: PartOfSpeech.other;
return {
surface: word,
reading: row.reading ?? '',
headword,
startPos: 0,
endPos: word.length,
partOfSpeech,
pos1: row.pos1 ?? '',
pos2: row.pos2 ?? '',
pos3: row.pos3 ?? '',
frequencyRank: row.frequencyRank ?? undefined,
isMerged: false,
isKnown: false,
isNPlusOneTarget: false,
};
}
export function isVocabularyStatsRowVisible(row: VocabularyVisibilityRow): boolean {
if (!(row.word?.trim() || row.headword?.trim())) return false;
return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
}
@@ -10,6 +10,7 @@ import {
parseExcludedWordsBody,
parseIntQuery,
parsePositiveIdList,
loadKnownWordsSet,
} from './route-support.js';
export function registerStatsLibraryRoutes(
@@ -31,6 +32,17 @@ export function registerStatsLibraryRoutes(
return c.json(statsJson('vocabulary', vocab));
});
app.get('/api/stats/vocabulary/summary', async (c) => {
const summary = await tracker.getVocabularySummary(
loadKnownWordsSet(options?.knownWordCachePath),
);
return c.json(statsJson('vocabularySummary', summary));
});
app.get('/api/stats/vocabulary/charts', async (c) => {
return c.json(statsJson('vocabularyCharts', await tracker.getVocabularyChartData()));
});
app.get('/api/stats/excluded-words', async (c) => {
return c.json(statsJson('excludedWords', await tracker.getStatsExcludedWords()));
});
@@ -89,6 +89,7 @@ const WORD_COPY_COLUMNS = [
'last_seen',
'frequency',
'frequency_rank',
'vocabulary_visible',
] as const;
export function mergeAnime(