mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-21 12:15:25 -07:00
fix(stats): report complete vocabulary totals and new-word history (#202)
This commit is contained in:
@@ -284,6 +284,22 @@ function createMockTracker(
|
||||
getSessionTimeline: async () => [],
|
||||
getSessionEvents: async () => [],
|
||||
getVocabularyStats: async () => VOCABULARY_STATS,
|
||||
getVocabularySummary: async () => ({
|
||||
uniqueWords: 501,
|
||||
uniqueWordsWithoutNames: 500,
|
||||
uniqueKanji: 201,
|
||||
newThisWeek: 7,
|
||||
newThisWeekWithoutNames: 6,
|
||||
knownWordCount: 250,
|
||||
knownWordCountWithoutNames: 249,
|
||||
}),
|
||||
getVocabularyChartData: async () => ({
|
||||
ready: true,
|
||||
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
}),
|
||||
getStatsExcludedWords: async () => [],
|
||||
replaceStatsExcludedWords: async () => {},
|
||||
getKanjiStats: async () => KANJI_STATS,
|
||||
@@ -711,6 +727,38 @@ describe('stats server API routes', () => {
|
||||
assert.equal(body[0].headword, 'する');
|
||||
});
|
||||
|
||||
it('GET /api/stats/vocabulary/summary returns database-wide card totals', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
|
||||
const res = await app.request('/api/stats/vocabulary/summary');
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(await res.json(), {
|
||||
uniqueWords: 501,
|
||||
uniqueWordsWithoutNames: 500,
|
||||
uniqueKanji: 201,
|
||||
newThisWeek: 7,
|
||||
newThisWeekWithoutNames: 6,
|
||||
knownWordCount: 250,
|
||||
knownWordCountWithoutNames: 249,
|
||||
});
|
||||
});
|
||||
|
||||
it('GET /api/stats/vocabulary/charts returns complete chart datasets', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
|
||||
const res = await app.request('/api/stats/vocabulary/charts');
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(await res.json(), {
|
||||
ready: true,
|
||||
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
});
|
||||
});
|
||||
|
||||
it('GET /api/stats/kanji returns kanji frequency data', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
const res = await app.request('/api/stats/kanji');
|
||||
|
||||
@@ -559,6 +559,165 @@ test('fresh tracker DB creates lifetime summary tables', async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('fresh tracker DB skips lexical rollup backfill work', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let backfillRuns = 0;
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor({ dbPath }, {
|
||||
runLexicalRollupBackfillTask: async () => {
|
||||
backfillRuns += 1;
|
||||
},
|
||||
} as never);
|
||||
|
||||
assert.equal(backfillRuns, 0);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('tracker starts the injected lexical rollup backfill when it is pending', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let backfillRuns = 0;
|
||||
|
||||
try {
|
||||
const setupDb = new Database(dbPath);
|
||||
const { ensureSchema } = await import('./immersion-tracker/storage');
|
||||
ensureSchema(setupDb);
|
||||
setupDb
|
||||
.prepare(
|
||||
`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||
)
|
||||
.run();
|
||||
setupDb.close();
|
||||
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor({ dbPath }, {
|
||||
runLexicalRollupBackfillTask: async () => {
|
||||
backfillRuns += 1;
|
||||
},
|
||||
} as never);
|
||||
|
||||
assert.equal(backfillRuns, 1);
|
||||
await waitForCondition(
|
||||
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
|
||||
);
|
||||
assert.equal(
|
||||
(tracker as unknown as { preserveWriteQueueUntilDrained: boolean })
|
||||
.preserveWriteQueueUntilDrained,
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('tracker queues playback writes until lexical rollup backfill settles', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let startBackfill = (): void => {};
|
||||
let releaseBackfill = (): void => {};
|
||||
let markBackfillStarted = (): void => {};
|
||||
const backfillStartGate = new Promise<void>((resolve) => {
|
||||
startBackfill = resolve;
|
||||
});
|
||||
const heldBackfill = new Promise<void>((resolve) => {
|
||||
releaseBackfill = resolve;
|
||||
});
|
||||
const backfillStarted = new Promise<void>((resolve) => {
|
||||
markBackfillStarted = resolve;
|
||||
});
|
||||
|
||||
try {
|
||||
const setupDb = new Database(dbPath);
|
||||
const { ensureSchema } = await import('./immersion-tracker/storage');
|
||||
ensureSchema(setupDb);
|
||||
setupDb
|
||||
.prepare(
|
||||
`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||
)
|
||||
.run();
|
||||
setupDb.close();
|
||||
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath, policy: { queueCap: 100 } },
|
||||
{
|
||||
runLexicalRollupBackfillTask: async (workerDbPath) => {
|
||||
await backfillStartGate;
|
||||
const workerDb = new Database(workerDbPath);
|
||||
try {
|
||||
workerDb.exec('BEGIN IMMEDIATE');
|
||||
markBackfillStarted();
|
||||
await heldBackfill;
|
||||
workerDb.exec('COMMIT');
|
||||
} catch (error) {
|
||||
try {
|
||||
workerDb.exec('ROLLBACK');
|
||||
} catch {
|
||||
// Preserve the original worker failure.
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
workerDb.close();
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
tracker.handleMediaChange('https://example.com/backfill-test.mp4', 'Backfill Test');
|
||||
startBackfill();
|
||||
await backfillStarted;
|
||||
for (let index = 0; index < 125; index += 1) tracker.recordCardsMined(1);
|
||||
|
||||
const privateApi = tracker as unknown as {
|
||||
db: DatabaseSync;
|
||||
queue: unknown[];
|
||||
droppedWriteCount: number;
|
||||
flushNow: () => void;
|
||||
writeLock: { locked: boolean };
|
||||
};
|
||||
assert.equal(privateApi.writeLock.locked, true);
|
||||
privateApi.flushNow();
|
||||
|
||||
assert.ok(privateApi.queue.length > 100, 'the protected queue may grow past its normal cap');
|
||||
assert.equal(privateApi.droppedWriteCount, 0, 'backfill must not discard playback writes');
|
||||
assert.equal(
|
||||
(
|
||||
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
|
||||
total: number;
|
||||
}
|
||||
).total,
|
||||
0,
|
||||
);
|
||||
|
||||
releaseBackfill();
|
||||
await waitForCondition(() => privateApi.queue.length === 0, 5_000);
|
||||
assert.equal(
|
||||
(
|
||||
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
|
||||
total: number;
|
||||
}
|
||||
).total,
|
||||
125,
|
||||
);
|
||||
} finally {
|
||||
releaseBackfill();
|
||||
if (tracker) {
|
||||
await waitForCondition(
|
||||
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
|
||||
5_000,
|
||||
);
|
||||
}
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('startup backfills lifetime summaries when retained sessions exist but summary tables are empty', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
@@ -4909,3 +5068,149 @@ test('ensureAnimeCoverArt fetches art via the latest video of the anime', async
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary coalesces concurrent requests into one worker task', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let taskRuns = 0;
|
||||
let releaseTask: (() => void) | null = null;
|
||||
const seenKnownWords: Array<ReadonlySet<string> | null> = [];
|
||||
const summary = {
|
||||
uniqueWords: 1,
|
||||
uniqueWordsWithoutNames: 1,
|
||||
uniqueKanji: 0,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: null,
|
||||
knownWordCountWithoutNames: null,
|
||||
};
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runVocabularySummaryTask: async (_dbPath, knownWords) => {
|
||||
taskRuns += 1;
|
||||
seenKnownWords.push(knownWords);
|
||||
await new Promise<void>((resolve) => {
|
||||
releaseTask = resolve;
|
||||
});
|
||||
return summary;
|
||||
},
|
||||
destroyVocabularySummaryRunner: () => {},
|
||||
},
|
||||
);
|
||||
|
||||
const knownWordsSnapshot = new Set(['猫']);
|
||||
const first = tracker.getVocabularySummary(knownWordsSnapshot);
|
||||
const second = tracker.getVocabularySummary(knownWordsSnapshot);
|
||||
await waitForCondition(() => releaseTask !== null);
|
||||
let release = releaseTask as (() => void) | null;
|
||||
assert.ok(release);
|
||||
release();
|
||||
assert.deepEqual(await first, summary);
|
||||
assert.equal(await second, await first);
|
||||
assert.equal(taskRuns, 1);
|
||||
assert.deepEqual(seenKnownWords, [knownWordsSnapshot]);
|
||||
|
||||
releaseTask = null;
|
||||
const third = tracker.getVocabularySummary(null);
|
||||
await waitForCondition(() => releaseTask !== null);
|
||||
release = releaseTask as (() => void) | null;
|
||||
assert.ok(release);
|
||||
release();
|
||||
assert.deepEqual(await third, summary);
|
||||
assert.equal(taskRuns, 2);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary coalesces equivalent known-word snapshots by value', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let taskRuns = 0;
|
||||
const releases: Array<() => void> = [];
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runVocabularySummaryTask: async () => {
|
||||
taskRuns += 1;
|
||||
await new Promise<void>((resolve) => releases.push(resolve));
|
||||
return {
|
||||
uniqueWords: 2,
|
||||
uniqueWordsWithoutNames: 2,
|
||||
uniqueKanji: 2,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: 2,
|
||||
knownWordCountWithoutNames: 2,
|
||||
};
|
||||
},
|
||||
destroyVocabularySummaryRunner: () => {},
|
||||
},
|
||||
);
|
||||
|
||||
const first = tracker.getVocabularySummary(new Set(['猫', '犬']));
|
||||
const second = tracker.getVocabularySummary(new Set(['犬', '猫']));
|
||||
await waitForCondition(() => releases.length > 0);
|
||||
const observedTaskRuns = taskRuns;
|
||||
for (const release of releases) release();
|
||||
await Promise.all([first, second]);
|
||||
|
||||
assert.equal(observedTaskRuns, 1);
|
||||
|
||||
const third = tracker.getVocabularySummary(new Set(['猫', '犬']));
|
||||
await waitForCondition(() => releases.length === 2);
|
||||
releases[1]!();
|
||||
await third;
|
||||
assert.equal(taskRuns, 2, 'a settled snapshot must be evicted from the in-flight map');
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary keeps different known-word snapshots independent', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
const releases: Array<() => void> = [];
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runVocabularySummaryTask: async (_dbPath, knownWords) => {
|
||||
await new Promise<void>((resolve) => releases.push(resolve));
|
||||
return {
|
||||
uniqueWords: 1,
|
||||
uniqueWordsWithoutNames: 1,
|
||||
uniqueKanji: 0,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: knownWords?.size ?? null,
|
||||
knownWordCountWithoutNames: knownWords?.size ?? null,
|
||||
};
|
||||
},
|
||||
destroyVocabularySummaryRunner: () => {},
|
||||
},
|
||||
);
|
||||
|
||||
const withoutKnownWords = tracker.getVocabularySummary(null);
|
||||
const withKnownWords = tracker.getVocabularySummary(new Set(['猫']));
|
||||
await waitForCondition(() => releases.length === 2);
|
||||
for (const release of releases) release();
|
||||
|
||||
assert.equal((await withoutKnownWords).knownWordCount, null);
|
||||
assert.equal((await withKnownWords).knownWordCount, 1);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -58,6 +58,7 @@ import {
|
||||
getSessionEvents,
|
||||
getSimilarWords,
|
||||
getStatsExcludedWords,
|
||||
getVocabularyChartData,
|
||||
getVocabularyStats,
|
||||
replaceStatsExcludedWords,
|
||||
searchSubtitleSentences,
|
||||
@@ -96,6 +97,12 @@ import {
|
||||
DeleteMaintenanceWorkerRuntime,
|
||||
type RunDeleteMaintenanceTask,
|
||||
} from './immersion-tracker/delete-maintenance-worker-runtime';
|
||||
import {
|
||||
VocabularySummaryWorkerRuntime,
|
||||
type RunVocabularySummaryTask,
|
||||
} from './immersion-tracker/vocabulary-summary-worker-runtime';
|
||||
import { LexicalRollupWorkerRuntime } from './immersion-tracker/lexical-rollup-worker-runtime';
|
||||
import { areLexicalDailyRollupsReady } from './immersion-tracker/lexical-rollups';
|
||||
import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
|
||||
import {
|
||||
cleanupDuplicateSubtitleLines,
|
||||
@@ -185,6 +192,7 @@ import {
|
||||
type StatsExcludedWordRow,
|
||||
type StreakCalendarRow,
|
||||
type VocabularyCleanupSummary,
|
||||
type VocabularyStatsSummary,
|
||||
type WatchTimePerAnimeRow,
|
||||
type WordAnimeAppearanceRow,
|
||||
type WordDetailRow,
|
||||
@@ -405,13 +413,24 @@ export class ImmersionTrackerService {
|
||||
private readonly monthlyRollupRetentionMs: number;
|
||||
private readonly vacuumIntervalMs: number;
|
||||
private readonly dbPath: string;
|
||||
private readonly writeLock = { locked: false };
|
||||
private readonly writeLock = {
|
||||
locked: false,
|
||||
reasons: new Set<'flush' | 'delete-maintenance' | 'lexical-rollup-backfill'>(),
|
||||
};
|
||||
private readonly destroyDeleteMaintenanceRunner: () => void;
|
||||
private readonly runVocabularySummaryTask: (
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
) => Promise<VocabularyStatsSummary>;
|
||||
private readonly vocabularySummariesInFlight = new Map<string, Promise<VocabularyStatsSummary>>();
|
||||
private readonly destroyVocabularySummaryRunner: () => void;
|
||||
private readonly runLexicalRollupBackfillTask: () => Promise<void>;
|
||||
private readonly destroyLexicalRollupBackfillRunner: () => void;
|
||||
private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
|
||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private flushScheduled = false;
|
||||
private droppedWriteCount = 0;
|
||||
private preserveWriteQueueUntilDrained = false;
|
||||
private lastVacuumMs = 0;
|
||||
private isDestroyed = false;
|
||||
private sessionState: SessionState | null = null;
|
||||
@@ -434,6 +453,10 @@ export class ImmersionTrackerService {
|
||||
dependencies: {
|
||||
runDeleteMaintenanceTask?: RunDeleteMaintenanceTask;
|
||||
destroyDeleteMaintenanceRunner?: () => void;
|
||||
runVocabularySummaryTask?: RunVocabularySummaryTask;
|
||||
destroyVocabularySummaryRunner?: () => void;
|
||||
runLexicalRollupBackfillTask?: (dbPath: string) => Promise<void>;
|
||||
destroyLexicalRollupBackfillRunner?: () => void;
|
||||
} = {},
|
||||
) {
|
||||
this.dbPath = options.dbPath;
|
||||
@@ -453,13 +476,34 @@ export class ImmersionTrackerService {
|
||||
runTask: (task) => runDeleteMaintenanceTask(this.dbPath, task),
|
||||
onBusy: () => {
|
||||
this.requireWriteQueueDrained('delete maintenance');
|
||||
this.writeLock.locked = true;
|
||||
this.setWriteLock('delete-maintenance', true);
|
||||
},
|
||||
onIdle: () => {
|
||||
this.writeLock.locked = false;
|
||||
this.setWriteLock('delete-maintenance', false);
|
||||
if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
|
||||
},
|
||||
});
|
||||
if (dependencies.runVocabularySummaryTask) {
|
||||
this.runVocabularySummaryTask = (knownWords) =>
|
||||
dependencies.runVocabularySummaryTask!(this.dbPath, knownWords);
|
||||
this.destroyVocabularySummaryRunner =
|
||||
dependencies.destroyVocabularySummaryRunner ?? (() => {});
|
||||
} else {
|
||||
const vocabularySummaryRuntime = new VocabularySummaryWorkerRuntime();
|
||||
this.runVocabularySummaryTask = (knownWords) =>
|
||||
vocabularySummaryRuntime.run(this.dbPath, knownWords);
|
||||
this.destroyVocabularySummaryRunner = () => vocabularySummaryRuntime.destroy();
|
||||
}
|
||||
if (dependencies.runLexicalRollupBackfillTask) {
|
||||
this.runLexicalRollupBackfillTask = () =>
|
||||
dependencies.runLexicalRollupBackfillTask!(this.dbPath);
|
||||
this.destroyLexicalRollupBackfillRunner =
|
||||
dependencies.destroyLexicalRollupBackfillRunner ?? (() => {});
|
||||
} else {
|
||||
const lexicalRollupRuntime = new LexicalRollupWorkerRuntime();
|
||||
this.runLexicalRollupBackfillTask = () => lexicalRollupRuntime.run(this.dbPath);
|
||||
this.destroyLexicalRollupBackfillRunner = () => lexicalRollupRuntime.destroy();
|
||||
}
|
||||
const parentDir = path.dirname(this.dbPath);
|
||||
if (!fs.existsSync(parentDir)) {
|
||||
fs.mkdirSync(parentDir, { recursive: true });
|
||||
@@ -547,6 +591,7 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
}
|
||||
this.preparedStatements = createTrackerPreparedStatements(this.db);
|
||||
if (!areLexicalDailyRollupsReady(this.db)) this.startLexicalRollupBackfill();
|
||||
this.scheduleMaintenance();
|
||||
this.scheduleFlush();
|
||||
}
|
||||
@@ -565,6 +610,8 @@ export class ImmersionTrackerService {
|
||||
this.isDestroyed = true;
|
||||
this.deleteMaintenanceScheduler.destroy();
|
||||
this.destroyDeleteMaintenanceRunner();
|
||||
this.destroyVocabularySummaryRunner();
|
||||
this.destroyLexicalRollupBackfillRunner();
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
@@ -634,6 +681,25 @@ export class ImmersionTrackerService {
|
||||
return getVocabularyStats(this.db, limit, excludePos);
|
||||
}
|
||||
|
||||
async getVocabularySummary(knownWords: ReadonlySet<string> | null) {
|
||||
const key = knownWords ? JSON.stringify([...knownWords].sort()) : 'null';
|
||||
const inFlight = this.vocabularySummariesInFlight.get(key);
|
||||
if (inFlight) return inFlight;
|
||||
const task = this.runVocabularySummaryTask(knownWords);
|
||||
this.vocabularySummariesInFlight.set(key, task);
|
||||
try {
|
||||
return await task;
|
||||
} finally {
|
||||
if (this.vocabularySummariesInFlight.get(key) === task) {
|
||||
this.vocabularySummariesInFlight.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async getVocabularyChartData() {
|
||||
return getVocabularyChartData(this.db);
|
||||
}
|
||||
|
||||
async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {
|
||||
return getStatsExcludedWords(this.db);
|
||||
}
|
||||
@@ -910,6 +976,33 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
}
|
||||
|
||||
private setWriteLock(
|
||||
reason: 'flush' | 'delete-maintenance' | 'lexical-rollup-backfill',
|
||||
active: boolean,
|
||||
): void {
|
||||
if (active) this.writeLock.reasons.add(reason);
|
||||
else this.writeLock.reasons.delete(reason);
|
||||
this.writeLock.locked = this.writeLock.reasons.size > 0;
|
||||
}
|
||||
|
||||
private startLexicalRollupBackfill(): void {
|
||||
this.requireWriteQueueDrained('lexical rollup backfill');
|
||||
this.preserveWriteQueueUntilDrained = true;
|
||||
this.setWriteLock('lexical-rollup-backfill', true);
|
||||
void this.runLexicalRollupBackfillTask()
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
'Lexical daily rollup backfill failed; it will retry on next startup',
|
||||
error,
|
||||
);
|
||||
})
|
||||
.finally(() => {
|
||||
this.setWriteLock('lexical-rollup-backfill', false);
|
||||
if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
|
||||
else if (!this.isDestroyed) this.scheduleFlush(0);
|
||||
});
|
||||
}
|
||||
|
||||
async reassignAnimeAnilist(
|
||||
animeId: number,
|
||||
info: {
|
||||
@@ -1906,7 +1999,12 @@ export class ImmersionTrackerService {
|
||||
|
||||
private recordWrite(write: QueuedWrite): void {
|
||||
if (this.isDestroyed) return;
|
||||
const { dropped } = enqueueWrite(this.queue, write, this.queueCap);
|
||||
// A lexical migration owns the database write lock, so dropping the oldest
|
||||
// entry cannot relieve pressure: nothing can flush until the worker exits.
|
||||
// Preserve that finite startup burst and drain it as soon as the lock lifts.
|
||||
const { dropped } = this.preserveWriteQueueUntilDrained
|
||||
? (this.queue.push(write), { dropped: 0 })
|
||||
: enqueueWrite(this.queue, write, this.queueCap);
|
||||
if (dropped > 0) {
|
||||
this.droppedWriteCount += dropped;
|
||||
this.logger.warn(`Immersion tracker queue overflow; dropped ${dropped} oldest writes`);
|
||||
@@ -1954,6 +2052,7 @@ export class ImmersionTrackerService {
|
||||
private flushNow(): void {
|
||||
if (this.writeLock.locked || this.isDestroyed) return;
|
||||
if (this.queue.length === 0) {
|
||||
this.preserveWriteQueueUntilDrained = false;
|
||||
this.flushScheduled = false;
|
||||
return;
|
||||
}
|
||||
@@ -1965,7 +2064,7 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
|
||||
const batch = this.queue.splice(0, Math.min(this.batchSize, this.queue.length));
|
||||
this.writeLock.locked = true;
|
||||
this.setWriteLock('flush', true);
|
||||
try {
|
||||
this.db.exec('BEGIN IMMEDIATE');
|
||||
for (const write of batch) {
|
||||
@@ -1977,8 +2076,9 @@ export class ImmersionTrackerService {
|
||||
this.queue.unshift(...batch);
|
||||
this.logger.warn('Immersion tracker flush failed, retrying later', error as Error);
|
||||
} finally {
|
||||
this.writeLock.locked = false;
|
||||
this.setWriteLock('flush', false);
|
||||
this.flushScheduled = false;
|
||||
if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
|
||||
if (this.queue.length > 0) {
|
||||
this.scheduleFlush(this.flushIntervalMs);
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
getKanjiOccurrences,
|
||||
getSessionSummaries,
|
||||
getVocabularyStats,
|
||||
getVocabularySummary,
|
||||
getKanjiStats,
|
||||
getSessionEvents,
|
||||
getSessionTimeline,
|
||||
@@ -1875,6 +1876,115 @@ test('getVocabularyStats returns rows ordered by frequency descending', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary counts every tracked vocabulary row instead of a display page', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const nowSec = Math.floor(Date.now() / 1000);
|
||||
const insertWord = db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', ?, ?, 1)
|
||||
`);
|
||||
const insertKanji = db.prepare(`
|
||||
INSERT INTO imm_kanji (kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)
|
||||
`);
|
||||
|
||||
for (let index = 0; index < 501; index += 1) {
|
||||
insertWord.run(`単語${index}`, `単語${index}`, nowSec - 8 * 86_400, nowSec - 8 * 86_400);
|
||||
}
|
||||
for (let index = 0; index < 201; index += 1) {
|
||||
insertKanji.run(
|
||||
String.fromCodePoint(0x4e00 + index),
|
||||
nowSec - 8 * 86_400,
|
||||
nowSec - 8 * 86_400,
|
||||
);
|
||||
}
|
||||
insertWord.run('今週', '今週', nowSec - 86_400, nowSec - 86_400);
|
||||
|
||||
assert.deepEqual(getVocabularySummary(db, new Set(['単語0', '今週']), nowSec * 1000), {
|
||||
uniqueWords: 502,
|
||||
uniqueWordsWithoutNames: 502,
|
||||
uniqueKanji: 201,
|
||||
newThisWeek: 1,
|
||||
newThisWeekWithoutNames: 1,
|
||||
knownWordCount: 2,
|
||||
knownWordCountWithoutNames: 2,
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary applies vocabulary exclusions and Hide Names totals', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, '', 'noun', '名詞', ?, '', 1, 1, 1)
|
||||
`);
|
||||
insertWord.run('猫', '猫', '一般');
|
||||
insertWord.run('太郎', '太郎', '固有名詞');
|
||||
insertWord.run('東京', '東京都', '一般');
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_stats_excluded_words (headword, word, reading)
|
||||
VALUES ('東京', '東京', '')
|
||||
`,
|
||||
).run();
|
||||
|
||||
assert.deepEqual(getVocabularySummary(db, new Set(['猫', '太郎', '東京']), 9 * 86_400_000), {
|
||||
uniqueWords: 2,
|
||||
uniqueWordsWithoutNames: 1,
|
||||
uniqueKanji: 0,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: 2,
|
||||
knownWordCountWithoutNames: 1,
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary counts identically across id-keyed scan batches', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', 1, 1, 1)
|
||||
`);
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
insertWord.run(`単語${index}`, `単語${index}`);
|
||||
}
|
||||
|
||||
const fullScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000);
|
||||
const batchedScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000, 2);
|
||||
|
||||
assert.equal(fullScan.uniqueWords, 5);
|
||||
assert.deepEqual(batchedScan, fullScan);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
LexicalRollupWorkerRuntime,
|
||||
resolveLexicalRollupWorkerPath,
|
||||
} from './lexical-rollup-worker-runtime';
|
||||
import { areLexicalDailyRollupsReady } from './lexical-rollups';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas, ensureSchema } from './storage';
|
||||
|
||||
test('lexical rollup worker backfills without using the tracker connection', async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-runtime-'));
|
||||
const dbPath = path.join(directory, 'immersion.sqlite');
|
||||
const runtime = new LexicalRollupWorkerRuntime();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
applyPragmas(db);
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES ('鳥', '鳥', 'とり', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||
db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
|
||||
'lexical_daily_rollups_version',
|
||||
);
|
||||
db.close();
|
||||
|
||||
await runtime.run(dbPath);
|
||||
|
||||
const checkDb = new Database(dbPath);
|
||||
try {
|
||||
assert.equal(areLexicalDailyRollupsReady(checkDb), true);
|
||||
} finally {
|
||||
checkDb.close();
|
||||
}
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
try {
|
||||
db.close();
|
||||
} catch {
|
||||
// Closed before the worker starts.
|
||||
}
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup worker module resolves in the current layout', () => {
|
||||
const workerPath = resolveLexicalRollupWorkerPath();
|
||||
assert.ok(workerPath, 'expected the lexical rollup worker module to resolve');
|
||||
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
|
||||
});
|
||||
|
||||
test('lexical rollup worker leaves a backfill pending when no worker can start', async () => {
|
||||
const runtime = new LexicalRollupWorkerRuntime({
|
||||
resolveWorkerPath: () => null,
|
||||
warn: () => {},
|
||||
} as never);
|
||||
|
||||
try {
|
||||
await assert.doesNotReject(runtime.run('/tmp/not-used.sqlite'));
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup worker absorbs termination failures after settling', async () => {
|
||||
let sendMessage: ((message: { ok: boolean }) => void) | null = null;
|
||||
const runtime = new LexicalRollupWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/fake-worker.js',
|
||||
createWorker: async () => ({
|
||||
once(event: string, listener: (value: never) => void) {
|
||||
if (event === 'message') sendMessage = listener as (message: { ok: boolean }) => void;
|
||||
return this;
|
||||
},
|
||||
terminate: async () => {
|
||||
throw new Error('termination failed');
|
||||
},
|
||||
}),
|
||||
warn: () => {},
|
||||
} as never);
|
||||
|
||||
const unhandled: unknown[] = [];
|
||||
const captureUnhandled = (reason: unknown) => unhandled.push(reason);
|
||||
process.on('unhandledRejection', captureUnhandled);
|
||||
try {
|
||||
const task = runtime.run('/tmp/not-used.sqlite');
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
const notify = sendMessage as ((message: { ok: boolean }) => void) | null;
|
||||
assert.ok(notify);
|
||||
notify({ ok: true });
|
||||
await task;
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.deepEqual(unhandled, []);
|
||||
} finally {
|
||||
process.off('unhandledRejection', captureUnhandled);
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup worker times out when it never responds', async () => {
|
||||
let terminated = false;
|
||||
const runtime = new LexicalRollupWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/fake-worker.js',
|
||||
createWorker: async () => ({
|
||||
once() {
|
||||
return this;
|
||||
},
|
||||
terminate: async () => {
|
||||
terminated = true;
|
||||
return 0;
|
||||
},
|
||||
}),
|
||||
timeoutMs: 1,
|
||||
warn: () => {},
|
||||
} as never);
|
||||
|
||||
try {
|
||||
const outcome = await Promise.race([
|
||||
runtime.run('/tmp/not-used.sqlite').then(
|
||||
() => 'resolved',
|
||||
(error: unknown) => String(error),
|
||||
),
|
||||
new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
|
||||
]);
|
||||
|
||||
assert.match(outcome, /timed out/);
|
||||
assert.equal(terminated, true);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createLogger } from '../../../logger';
|
||||
|
||||
interface WorkerResponse {
|
||||
ok?: boolean;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
interface WorkerHandle {
|
||||
once(event: 'message', listener: (message: WorkerResponse) => void): this;
|
||||
once(event: 'error', listener: (error: Error) => void): this;
|
||||
once(event: 'exit', listener: (code: number) => void): this;
|
||||
terminate(): Promise<number>;
|
||||
}
|
||||
|
||||
interface LexicalRollupWorkerRuntimeOptions {
|
||||
resolveWorkerPath?: () => string | null;
|
||||
createWorker?: (workerPath: string, workerData: { dbPath: string }) => Promise<WorkerHandle>;
|
||||
timeoutMs?: number;
|
||||
warn?: (message: string, ...meta: unknown[]) => void;
|
||||
}
|
||||
|
||||
const logger = createLogger('main:immersion-tracker:lexical-rollup-worker');
|
||||
const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
|
||||
|
||||
export function resolveLexicalRollupWorkerPath(): string | null {
|
||||
const fileName = __filename.endsWith('.ts')
|
||||
? 'lexical-rollup-worker-thread.ts'
|
||||
: 'lexical-rollup-worker-thread.js';
|
||||
const workerPath = path.join(__dirname, fileName);
|
||||
return fs.existsSync(workerPath) ? workerPath : null;
|
||||
}
|
||||
|
||||
export class LexicalRollupWorkerRuntime {
|
||||
private readonly activeWorkers = new Set<WorkerHandle>();
|
||||
private destroyed = false;
|
||||
|
||||
constructor(private readonly options: LexicalRollupWorkerRuntimeOptions = {}) {}
|
||||
|
||||
async run(dbPath: string): Promise<void> {
|
||||
if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
|
||||
let worker: WorkerHandle;
|
||||
try {
|
||||
const workerPath = (this.options.resolveWorkerPath ?? resolveLexicalRollupWorkerPath)();
|
||||
if (!workerPath) throw new Error('Emitted lexical rollup worker module was not found');
|
||||
const createWorker =
|
||||
this.options.createWorker ??
|
||||
(async (resolvedPath, workerData) => {
|
||||
const { Worker } = await import('node:worker_threads');
|
||||
return new Worker(resolvedPath, { workerData });
|
||||
});
|
||||
worker = await createWorker(workerPath, { dbPath });
|
||||
} catch (error) {
|
||||
if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
|
||||
(this.options.warn ?? logger.warn)(
|
||||
'Lexical rollup worker unavailable; leaving backfill pending for a later startup',
|
||||
error,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.destroyed) {
|
||||
await worker.terminate().catch(() => undefined);
|
||||
throw new Error('Lexical rollup worker is shut down');
|
||||
}
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
let settled = false;
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
this.activeWorkers.add(worker);
|
||||
const settle = (error?: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timeout) clearTimeout(timeout);
|
||||
this.activeWorkers.delete(worker);
|
||||
void worker.terminate().catch(() => undefined);
|
||||
if (error) reject(error);
|
||||
else resolve();
|
||||
};
|
||||
timeout = setTimeout(
|
||||
() => settle(new Error('Lexical rollup worker timed out')),
|
||||
this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
|
||||
);
|
||||
worker.once('message', (message) => {
|
||||
if (message.ok) settle();
|
||||
else
|
||||
settle(
|
||||
new Error(
|
||||
`Lexical rollup backfill failed: ${String(message.error ?? 'unknown error')}`,
|
||||
),
|
||||
);
|
||||
});
|
||||
worker.once('error', (error) => settle(error));
|
||||
worker.once('exit', (code) => {
|
||||
if (!settled) {
|
||||
settle(
|
||||
new Error(
|
||||
code === 0
|
||||
? 'Lexical rollup worker exited without a response'
|
||||
: `Lexical rollup worker exited with code ${code}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
for (const worker of this.activeWorkers) {
|
||||
void worker.terminate().catch(() => undefined);
|
||||
}
|
||||
this.activeWorkers.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
|
||||
|
||||
if (!parentPort) throw new Error('lexical rollup worker missing parent port');
|
||||
|
||||
try {
|
||||
executeLexicalRollupBackfillTask((workerData as { dbPath: string }).dbPath);
|
||||
parentPort.postMessage({ ok: true });
|
||||
} catch (error) {
|
||||
parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
|
||||
import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
|
||||
import { Database } from './sqlite';
|
||||
import { ensureSchema } from './storage';
|
||||
|
||||
test('lexical rollup backfill materializes pre-existing vocabulary off the caller DB connection', () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-worker-'));
|
||||
const dbPath = path.join(directory, 'immersion.sqlite');
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||
).run('犬', '犬', 'いぬ', 1_700_000_000, 1_700_000_000);
|
||||
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||
db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
|
||||
'lexical_daily_rollups_version',
|
||||
);
|
||||
|
||||
executeLexicalRollupBackfillTask(dbPath);
|
||||
|
||||
assert.equal(areLexicalDailyRollupsReady(db), true);
|
||||
assert.equal(getLexicalDailyRollups(db)[0]?.wordCount, 1);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
import { areLexicalDailyRollupsReady, rebuildLexicalDailyRollups } from './lexical-rollups';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas } from './storage';
|
||||
|
||||
export function executeLexicalRollupBackfillTask(dbPath: string): void {
|
||||
const db = new Database(dbPath);
|
||||
try {
|
||||
applyPragmas(db);
|
||||
if (!areLexicalDailyRollupsReady(db)) {
|
||||
rebuildLexicalDailyRollups(db);
|
||||
}
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
areLexicalDailyRollupsReady,
|
||||
getLexicalDailyRollups,
|
||||
rebuildLexicalDailyRollups,
|
||||
} from './lexical-rollups';
|
||||
import { getTrendsDashboard } from './query-trends';
|
||||
import {
|
||||
getVocabularyChartData,
|
||||
getVocabularySummary,
|
||||
replaceStatsExcludedWords,
|
||||
} from './query-lexical';
|
||||
import { Database } from './sqlite';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { ensureSchema } from './storage';
|
||||
|
||||
function makeDbPath(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollups-'));
|
||||
return path.join(dir, 'immersion.sqlite');
|
||||
}
|
||||
|
||||
test('lexical daily rollups follow first-seen corrections and deletions', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const firstDay = 19_500;
|
||||
const correctedDay = firstDay + 2;
|
||||
const firstSeen = firstDay * 86_400 + 43_200;
|
||||
const correctedSeen = correctedDay * 86_400 + 43_200;
|
||||
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||
).run('猫', '猫', 'ねこ', firstSeen, firstSeen);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)`,
|
||||
).run('猫', firstSeen, firstSeen);
|
||||
|
||||
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||
{ epochDay: firstDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 1 },
|
||||
]);
|
||||
|
||||
db.prepare(`UPDATE imm_words SET first_seen = ? WHERE headword = ?`).run(correctedSeen, '猫');
|
||||
db.prepare(`DELETE FROM imm_kanji WHERE kanji = ?`).run('猫');
|
||||
|
||||
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||
{ epochDay: correctedDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
|
||||
]);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical daily rollups normalize second and millisecond timestamps', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const epochDay = 19_500;
|
||||
const timestampSeconds = epochDay * 86_400 + 43_200;
|
||||
const timestampMilliseconds = timestampSeconds * 1_000;
|
||||
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||
).run('猫', '猫', 'ねこ', timestampSeconds, timestampSeconds);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||
).run('犬', '犬', 'いぬ', timestampMilliseconds, timestampMilliseconds);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)`,
|
||||
).run('猫', timestampSeconds, timestampSeconds);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)`,
|
||||
).run('犬', timestampMilliseconds, timestampMilliseconds);
|
||||
|
||||
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||
{ epochDay, wordCount: 2, wordCountWithoutNames: 2, kanjiCount: 2 },
|
||||
]);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup rebuild excludes rows hidden by vocabulary persistence rules', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const epochDay = 19_500;
|
||||
const firstSeen = epochDay * 86_400 + 43_200;
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(
|
||||
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||
).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(
|
||||
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||
).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
|
||||
|
||||
rebuildLexicalDailyRollups(db);
|
||||
|
||||
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||
{ epochDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
|
||||
]);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup rebuild tolerates nullable legacy vocabulary text', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (NULL, NULL, NULL, 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (NULL, '猫', 'ねこ', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
|
||||
assert.doesNotThrow(() => rebuildLexicalDailyRollups(db));
|
||||
assert.equal(areLexicalDailyRollupsReady(db), true);
|
||||
assert.equal(getVocabularySummary(db, null).uniqueWords, 1);
|
||||
assert.equal(getVocabularySummary(db, new Set(['猫'])).knownWordCount, 1);
|
||||
assert.equal(getVocabularyChartData(db).topWords[0]?.headword, '猫');
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup rebuild scans vocabulary visibility in bounded id batches', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
const expectedBatchSize = 5_000;
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, '', 1700000000, 1700000000, 1)`,
|
||||
);
|
||||
db.exec('BEGIN');
|
||||
for (let index = 0; index <= expectedBatchSize; index += 1) {
|
||||
insertWord.run(`語${index}`, `語${index}`);
|
||||
}
|
||||
db.exec('COMMIT');
|
||||
|
||||
const scanPageSizes: number[] = [];
|
||||
const instrumentedDb: DatabaseSync = {
|
||||
prepare(source) {
|
||||
const statement = db.prepare(source);
|
||||
if (!source.includes('WHERE id > ?') || !source.includes('ORDER BY id')) {
|
||||
return statement;
|
||||
}
|
||||
return {
|
||||
run: (...params) => statement.run(...params),
|
||||
get: (...params) => statement.get(...params),
|
||||
all: (...params) => {
|
||||
const rows = statement.all(...params);
|
||||
scanPageSizes.push(rows.length);
|
||||
return rows;
|
||||
},
|
||||
};
|
||||
},
|
||||
exec(source) {
|
||||
db.exec(source);
|
||||
return instrumentedDb;
|
||||
},
|
||||
close() {
|
||||
return instrumentedDb;
|
||||
},
|
||||
};
|
||||
|
||||
rebuildLexicalDailyRollups(instrumentedDb);
|
||||
|
||||
assert.deepEqual(scanPageSizes, [expectedBatchSize, 1]);
|
||||
assert.equal(getVocabularySummary(db, null).uniqueWords, expectedBatchSize + 1);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('chart exclusions do not subtract vocabulary rows already hidden from the rollup', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const epochDay = 19_500;
|
||||
const firstSeen = epochDay * 86_400 + 43_200;
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(
|
||||
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||
).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(
|
||||
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||
).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
|
||||
rebuildLexicalDailyRollups(db);
|
||||
replaceStatsExcludedWords(db, [{ headword: 'は', word: 'は', reading: 'は' }]);
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.deepEqual(charts.newWordsTimeline, [{ epochDay, wordCount: 1 }]);
|
||||
assert.deepEqual(charts.newWordsTimelineWithoutNames, [{ epochDay, wordCount: 1 }]);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('legacy lexical rollup readiness does not satisfy the current rollup version', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_rollup_state(state_key, state_value)
|
||||
VALUES ('lexical_daily_rollups_ready', '1')
|
||||
ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`DELETE FROM imm_rollup_state WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||
).run();
|
||||
|
||||
assert.equal(areLexicalDailyRollupsReady(db), false);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('imm_words persists vocabulary visibility for rollup maintenance', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const columns = db.prepare(`PRAGMA table_info(imm_words)`).all() as Array<{ name: string }>;
|
||||
|
||||
assert.equal(
|
||||
columns.some((column) => column.name === 'vocabulary_visible'),
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary charts use complete top-word and lexical rollup data', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, '', 1700000000, 1700000000, ?)`,
|
||||
);
|
||||
for (let index = 0; index < 501; index += 1) {
|
||||
insertWord.run(`語${index}`, `語${index}`, index === 500 ? 10_000 : 1);
|
||||
}
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.equal(charts.topWords[0]?.headword, '語500');
|
||||
assert.equal(charts.topWords[0]?.frequency, 10_000);
|
||||
assert.equal(charts.newWordsTimeline[0]?.wordCount, 501);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary charts find full top-word sets beyond excluded and name rows', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, pos2, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, '', ?, 1700000000, 1700000000, ?)`,
|
||||
);
|
||||
const exclusions = [];
|
||||
for (let index = 0; index < 100; index += 1) {
|
||||
const headword = `語${index}`;
|
||||
insertWord.run(
|
||||
headword,
|
||||
headword,
|
||||
index < 80 && index >= 60 ? '固有名詞' : '一般',
|
||||
100 - index,
|
||||
);
|
||||
if (index < 60) exclusions.push({ headword, word: headword, reading: '' });
|
||||
}
|
||||
replaceStatsExcludedWords(db, exclusions);
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.equal(charts.topWords.length, 12);
|
||||
assert.equal(charts.topWords[0]?.headword, '語60');
|
||||
assert.equal(charts.topWordsWithoutNames.length, 12);
|
||||
assert.equal(charts.topWordsWithoutNames[0]?.headword, '語80');
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary charts handle exclusion lists above one SQLite variable batch', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES ('語0', '語0', '', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
const exclusions = Array.from({ length: 10_923 }, (_, index) => ({
|
||||
headword: `語${index}`,
|
||||
word: `語${index}`,
|
||||
reading: '',
|
||||
}));
|
||||
replaceStatsExcludedWords(db, exclusions);
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.deepEqual(charts.topWords, []);
|
||||
assert.deepEqual(charts.newWordsTimeline, []);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup rebuild preserves the original error when rollback also fails', () => {
|
||||
const originalError = new Error('rebuild failed');
|
||||
const db = {
|
||||
exec(sql: string) {
|
||||
if (sql === 'BEGIN IMMEDIATE') return;
|
||||
if (sql === 'ROLLBACK') throw new Error('rollback failed');
|
||||
throw originalError;
|
||||
},
|
||||
prepare() {
|
||||
return { all: () => [], run: () => undefined };
|
||||
},
|
||||
} as unknown as DatabaseSync;
|
||||
|
||||
assert.throws(() => rebuildLexicalDailyRollups(db), originalError);
|
||||
});
|
||||
|
||||
test('trends read historical new-word buckets from lexical rollups', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES ('海', '海', 'うみ', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
db.prepare(`UPDATE imm_lexical_daily_rollups SET word_count = 9`).run();
|
||||
|
||||
const dashboard = getTrendsDashboard(db, 'all', 'day', false);
|
||||
|
||||
assert.equal(dashboard.progress.newWords[0]?.value, 9);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,226 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility';
|
||||
|
||||
export interface LexicalDailyRollup {
|
||||
epochDay: number;
|
||||
wordCount: number;
|
||||
wordCountWithoutNames: number;
|
||||
kanjiCount: number;
|
||||
}
|
||||
|
||||
const LOCAL_EPOCH_DAY_SQL = `
|
||||
CAST(julianday(
|
||||
CASE
|
||||
WHEN ABS(CAST(%VALUE% AS REAL)) >= 10000000000 THEN CAST(%VALUE% AS REAL) / 1000
|
||||
ELSE CAST(%VALUE% AS REAL)
|
||||
END,
|
||||
'unixepoch', 'localtime'
|
||||
) - 2440587.5 AS INTEGER)
|
||||
`;
|
||||
|
||||
const LEXICAL_DAILY_ROLLUP_VERSION = '2';
|
||||
const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
|
||||
const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
|
||||
|
||||
export function localEpochDaySql(value: string): string {
|
||||
return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
|
||||
}
|
||||
|
||||
function createWordRollupTriggers(db: DatabaseSync): void {
|
||||
const dayForNew = localEpochDaySql('NEW.first_seen');
|
||||
const dayForOld = localEpochDaySql('OLD.first_seen');
|
||||
|
||||
db.exec(`
|
||||
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_insert;
|
||||
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_delete;
|
||||
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_first_seen_update;
|
||||
|
||||
CREATE TRIGGER imm_words_lexical_rollup_insert
|
||||
AFTER INSERT ON imm_words
|
||||
WHEN NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
VALUES (${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0)
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||
word_count = word_count + 1,
|
||||
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER imm_words_lexical_rollup_delete
|
||||
AFTER DELETE ON imm_words
|
||||
WHEN OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
VALUES (${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0)
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||
word_count = word_count - 1,
|
||||
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||
DELETE FROM imm_lexical_daily_rollups
|
||||
WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER imm_words_lexical_rollup_first_seen_update
|
||||
AFTER UPDATE OF first_seen, pos2, vocabulary_visible ON imm_words
|
||||
WHEN OLD.first_seen IS NOT NEW.first_seen
|
||||
OR OLD.pos2 IS NOT NEW.pos2
|
||||
OR OLD.vocabulary_visible IS NOT NEW.vocabulary_visible
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0
|
||||
WHERE OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||
word_count = word_count - 1,
|
||||
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0
|
||||
WHERE NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||
word_count = word_count + 1,
|
||||
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||
DELETE FROM imm_lexical_daily_rollups
|
||||
WHERE word_count = 0 AND kanji_count = 0;
|
||||
END;
|
||||
`);
|
||||
}
|
||||
|
||||
function createKanjiRollupTriggers(db: DatabaseSync): void {
|
||||
const dayForNew = localEpochDaySql('NEW.first_seen');
|
||||
const dayForOld = localEpochDaySql('OLD.first_seen');
|
||||
db.exec(`
|
||||
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_insert;
|
||||
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_delete;
|
||||
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_first_seen_update;
|
||||
|
||||
CREATE TRIGGER imm_kanji_lexical_rollup_insert
|
||||
AFTER INSERT ON imm_kanji WHEN NEW.first_seen IS NOT NULL
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
VALUES (${dayForNew}, 0, 0, 1)
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
|
||||
END;
|
||||
CREATE TRIGGER imm_kanji_lexical_rollup_delete
|
||||
AFTER DELETE ON imm_kanji WHEN OLD.first_seen IS NOT NULL
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
VALUES (${dayForOld}, 0, 0, -1)
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
|
||||
DELETE FROM imm_lexical_daily_rollups
|
||||
WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
|
||||
END;
|
||||
CREATE TRIGGER imm_kanji_lexical_rollup_first_seen_update
|
||||
AFTER UPDATE OF first_seen ON imm_kanji WHEN OLD.first_seen IS NOT NEW.first_seen
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${dayForOld}, 0, 0, -1 WHERE OLD.first_seen IS NOT NULL
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${dayForNew}, 0, 0, 1 WHERE NEW.first_seen IS NOT NULL
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
|
||||
DELETE FROM imm_lexical_daily_rollups WHERE word_count = 0 AND kanji_count = 0;
|
||||
END;
|
||||
`);
|
||||
}
|
||||
|
||||
export function ensureLexicalDailyRollupTables(db: DatabaseSync): void {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS imm_lexical_daily_rollups(
|
||||
epoch_day INTEGER PRIMARY KEY,
|
||||
word_count INTEGER NOT NULL DEFAULT 0,
|
||||
word_count_without_names INTEGER NOT NULL DEFAULT 0,
|
||||
kanji_count INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
INSERT INTO imm_rollup_state(state_key, state_value)
|
||||
VALUES ('${LEXICAL_DAILY_ROLLUP_VERSION_KEY}', '0')
|
||||
ON CONFLICT(state_key) DO NOTHING;
|
||||
`);
|
||||
createWordRollupTriggers(db);
|
||||
createKanjiRollupTriggers(db);
|
||||
}
|
||||
|
||||
export function areLexicalDailyRollupsReady(db: DatabaseSync): boolean {
|
||||
const row = db
|
||||
.prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
|
||||
.get(LEXICAL_DAILY_ROLLUP_VERSION_KEY) as { value: string } | null;
|
||||
return row?.value === LEXICAL_DAILY_ROLLUP_VERSION;
|
||||
}
|
||||
|
||||
export function markLexicalDailyRollupsReady(db: DatabaseSync): void {
|
||||
db.prepare(
|
||||
`INSERT INTO imm_rollup_state(state_key, state_value)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
|
||||
).run(LEXICAL_DAILY_ROLLUP_VERSION_KEY, LEXICAL_DAILY_ROLLUP_VERSION);
|
||||
}
|
||||
|
||||
/** Rebuild from the first-seen source of truth; run off the UI/main DB thread. */
|
||||
export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
|
||||
let transactionStarted = false;
|
||||
try {
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
transactionStarted = true;
|
||||
const scanVocabulary = db.prepare(
|
||||
`SELECT id, word, headword, reading, part_of_speech AS partOfSpeech,
|
||||
pos1, pos2, pos3, frequency_rank AS frequencyRank
|
||||
FROM imm_words
|
||||
WHERE id > ?
|
||||
ORDER BY id
|
||||
LIMIT ?`,
|
||||
);
|
||||
const updateVisibility = db.prepare(
|
||||
`UPDATE imm_words SET vocabulary_visible = ? WHERE id = ? AND vocabulary_visible IS NOT ?`,
|
||||
);
|
||||
let lastId = Number.MIN_SAFE_INTEGER;
|
||||
for (;;) {
|
||||
const vocabularyRows = scanVocabulary.all(
|
||||
lastId,
|
||||
VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE,
|
||||
) as Array<VocabularyVisibilityRow & { id: number }>;
|
||||
if (vocabularyRows.length === 0) break;
|
||||
for (const row of vocabularyRows) {
|
||||
const visible = isVocabularyStatsRowVisible(row) ? 1 : 0;
|
||||
updateVisibility.run(visible, row.id, visible);
|
||||
}
|
||||
lastId = vocabularyRows[vocabularyRows.length - 1]!.id;
|
||||
if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break;
|
||||
}
|
||||
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||
db.exec(`
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${localEpochDaySql('first_seen')}, COUNT(*),
|
||||
SUM(CASE WHEN pos2 = '固有名詞' THEN 0 ELSE 1 END), 0
|
||||
FROM imm_words
|
||||
WHERE first_seen IS NOT NULL AND vocabulary_visible = 1
|
||||
GROUP BY ${localEpochDaySql('first_seen')};
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${localEpochDaySql('first_seen')}, 0, 0, COUNT(*)
|
||||
FROM imm_kanji
|
||||
WHERE first_seen IS NOT NULL
|
||||
GROUP BY ${localEpochDaySql('first_seen')}
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + excluded.kanji_count;
|
||||
`);
|
||||
markLexicalDailyRollupsReady(db);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
if (transactionStarted) {
|
||||
try {
|
||||
db.exec('ROLLBACK');
|
||||
} catch {
|
||||
// Preserve the rebuild failure; it is the actionable cause.
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function getLexicalDailyRollups(db: DatabaseSync): LexicalDailyRollup[] {
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT epoch_day AS epochDay, word_count AS wordCount,
|
||||
word_count_without_names AS wordCountWithoutNames, kanji_count AS kanjiCount
|
||||
FROM imm_lexical_daily_rollups
|
||||
ORDER BY epoch_day ASC
|
||||
`,
|
||||
)
|
||||
.all() as LexicalDailyRollup[];
|
||||
}
|
||||
@@ -1,6 +1,4 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { PartOfSpeech, type MergedToken } from '../../../types';
|
||||
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
|
||||
import type {
|
||||
KanjiAnimeAppearanceRow,
|
||||
KanjiDetailRow,
|
||||
@@ -13,19 +11,38 @@ import type {
|
||||
SimilarWordRow,
|
||||
StatsExcludedWordRow,
|
||||
VocabularyStatsRow,
|
||||
VocabularyStatsSummary,
|
||||
WordAnimeAppearanceRow,
|
||||
WordDetailRow,
|
||||
WordOccurrenceRow,
|
||||
} from './types';
|
||||
import { fromDbTimestamp, toDbTimestamp } from './query-shared';
|
||||
import { nowMs } from './time';
|
||||
import {
|
||||
areLexicalDailyRollupsReady,
|
||||
getLexicalDailyRollups,
|
||||
localEpochDaySql,
|
||||
} from './lexical-rollups';
|
||||
import { isVocabularyStatsRowVisible } from './vocabulary-visibility';
|
||||
|
||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
|
||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
|
||||
const VOCABULARY_CHART_LIMIT = 12;
|
||||
const VOCABULARY_CHART_PAGE_SIZE = 100;
|
||||
const EXCLUSION_ALIAS_BATCH_SIZE = 300;
|
||||
const VOCABULARY_SUMMARY_SCAN_BATCH_SIZE = 5_000;
|
||||
const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
|
||||
const SENTENCE_SEARCH_MAX_LIMIT = 100;
|
||||
const KANJI_PATTERN = /\p{Script=Han}/gu;
|
||||
|
||||
export interface VocabularyChartData {
|
||||
ready: boolean;
|
||||
topWords: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||
topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||
newWordsTimeline: Array<{ epochDay: number; wordCount: number }>;
|
||||
newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>;
|
||||
}
|
||||
|
||||
function resolveSentenceSearchLimit(limit: number): number {
|
||||
if (!Number.isFinite(limit)) return SENTENCE_SEARCH_DEFAULT_LIMIT;
|
||||
const normalized = Math.floor(limit);
|
||||
@@ -73,33 +90,6 @@ function uniqueKanji(text: string): string[] {
|
||||
return Array.from(new Set(text.match(KANJI_PATTERN) ?? []));
|
||||
}
|
||||
|
||||
function toVocabularyToken(row: VocabularyStatsRow): MergedToken {
|
||||
const partOfSpeech =
|
||||
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
|
||||
? (row.partOfSpeech as PartOfSpeech)
|
||||
: PartOfSpeech.other;
|
||||
|
||||
return {
|
||||
surface: row.word,
|
||||
reading: row.reading ?? '',
|
||||
headword: row.headword,
|
||||
startPos: 0,
|
||||
endPos: row.word.length,
|
||||
partOfSpeech,
|
||||
pos1: row.pos1 ?? '',
|
||||
pos2: row.pos2 ?? '',
|
||||
pos3: row.pos3 ?? '',
|
||||
frequencyRank: row.frequencyRank ?? undefined,
|
||||
isMerged: false,
|
||||
isKnown: false,
|
||||
isNPlusOneTarget: false,
|
||||
};
|
||||
}
|
||||
|
||||
function isVocabularyStatsRowVisible(row: VocabularyStatsRow): boolean {
|
||||
return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
|
||||
}
|
||||
|
||||
export function getVocabularyStats(
|
||||
db: DatabaseSync,
|
||||
limit = 100,
|
||||
@@ -153,6 +143,198 @@ export function getVocabularyStats(
|
||||
return visibleRows.slice(0, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Chart data is intentionally independent of the paginated vocabulary tables.
|
||||
* Top words use the frequency index; new-word history reads permanent daily
|
||||
* lexical rollups rather than loading every vocabulary row into the dashboard.
|
||||
*/
|
||||
export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData {
|
||||
const ready = areLexicalDailyRollupsReady(db);
|
||||
const excludedAliases = new Set(
|
||||
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
|
||||
);
|
||||
const isExcluded = (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>): boolean =>
|
||||
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias));
|
||||
const topWords = getTopVocabularyChartWords(db, isExcluded);
|
||||
const rollups = ready ? getLexicalDailyRollups(db) : [];
|
||||
const timeline = new Map(rollups.map((row) => [row.epochDay, { ...row }]));
|
||||
if (excludedAliases.size > 0 && ready) {
|
||||
const aliases = [...excludedAliases];
|
||||
const excludedRows = new Map<
|
||||
number,
|
||||
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
|
||||
wordId: number;
|
||||
epochDay: number;
|
||||
}
|
||||
>();
|
||||
for (let offset = 0; offset < aliases.length; offset += EXCLUSION_ALIAS_BATCH_SIZE) {
|
||||
const batch = aliases.slice(offset, offset + EXCLUSION_ALIAS_BATCH_SIZE);
|
||||
const placeholders = batch.map(() => '?').join(', ');
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id AS wordId, headword, word, reading, pos2,
|
||||
${localEpochDaySql('first_seen')} AS epochDay
|
||||
FROM imm_words
|
||||
WHERE vocabulary_visible = 1
|
||||
AND (headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}))
|
||||
`,
|
||||
)
|
||||
.all(...batch, ...batch, ...batch) as Array<
|
||||
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
|
||||
wordId: number;
|
||||
epochDay: number;
|
||||
}
|
||||
>;
|
||||
for (const row of rows) excludedRows.set(row.wordId, row);
|
||||
}
|
||||
for (const word of excludedRows.values()) {
|
||||
if (!isExcluded(word)) continue;
|
||||
const rollup = timeline.get(word.epochDay);
|
||||
if (!rollup) continue;
|
||||
rollup.wordCount -= 1;
|
||||
if (word.pos2 !== '固有名詞') rollup.wordCountWithoutNames -= 1;
|
||||
}
|
||||
}
|
||||
return {
|
||||
ready,
|
||||
topWords: topWords.all.map((word) => ({
|
||||
wordId: word.wordId,
|
||||
headword: vocabularyDisplayHeadword(word),
|
||||
frequency: word.frequency,
|
||||
})),
|
||||
topWordsWithoutNames: topWords.withoutNames.map((word) => ({
|
||||
wordId: word.wordId,
|
||||
headword: vocabularyDisplayHeadword(word),
|
||||
frequency: word.frequency,
|
||||
})),
|
||||
newWordsTimeline: [...timeline.values()]
|
||||
.filter((row) => row.wordCount > 0)
|
||||
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCount })),
|
||||
newWordsTimelineWithoutNames: [...timeline.values()]
|
||||
.filter((row) => row.wordCountWithoutNames > 0)
|
||||
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCountWithoutNames })),
|
||||
};
|
||||
}
|
||||
|
||||
function getTopVocabularyChartWords(
|
||||
db: DatabaseSync,
|
||||
isExcluded: (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>) => boolean,
|
||||
): { all: VocabularyStatsRow[]; withoutNames: VocabularyStatsRow[] } {
|
||||
const stmt = db.prepare(`
|
||||
SELECT id AS wordId, headword, word, reading,
|
||||
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
|
||||
frequency, frequency_rank AS frequencyRank,
|
||||
first_seen AS firstSeen, last_seen AS lastSeen,
|
||||
0 AS animeCount
|
||||
FROM imm_words
|
||||
ORDER BY frequency DESC, id
|
||||
LIMIT ? OFFSET ?
|
||||
`);
|
||||
const all: VocabularyStatsRow[] = [];
|
||||
const withoutNames: VocabularyStatsRow[] = [];
|
||||
let offset = 0;
|
||||
|
||||
while (all.length < VOCABULARY_CHART_LIMIT || withoutNames.length < VOCABULARY_CHART_LIMIT) {
|
||||
const page = stmt.all(VOCABULARY_CHART_PAGE_SIZE, offset) as VocabularyStatsRow[];
|
||||
if (page.length === 0) break;
|
||||
for (const word of page) {
|
||||
if (!isVocabularyStatsRowVisible(word) || isExcluded(word)) continue;
|
||||
if (all.length < VOCABULARY_CHART_LIMIT) all.push(word);
|
||||
if (word.pos2 !== '固有名詞' && withoutNames.length < VOCABULARY_CHART_LIMIT) {
|
||||
withoutNames.push(word);
|
||||
}
|
||||
}
|
||||
offset += page.length;
|
||||
}
|
||||
|
||||
return { all, withoutNames };
|
||||
}
|
||||
|
||||
function excludedVocabularyAliases(
|
||||
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
|
||||
): string[] {
|
||||
const aliases = [word.headword?.trim() ?? '', word.word?.trim() ?? ''].filter(Boolean);
|
||||
if (aliases.length === 0) aliases.push(word.reading?.trim() ?? '');
|
||||
return [...new Set(aliases)];
|
||||
}
|
||||
|
||||
function vocabularyDisplayHeadword(
|
||||
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
|
||||
): string {
|
||||
return word.headword?.trim() || word.word?.trim() || word.reading?.trim() || '';
|
||||
}
|
||||
|
||||
function timestampSeconds(timestamp: number): number {
|
||||
return timestamp < 10_000_000_000 ? timestamp : Math.floor(timestamp / 1000);
|
||||
}
|
||||
|
||||
export function getVocabularySummary(
|
||||
db: DatabaseSync,
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
nowMs: number = Date.now(),
|
||||
scanBatchSize: number = VOCABULARY_SUMMARY_SCAN_BATCH_SIZE,
|
||||
): VocabularyStatsSummary {
|
||||
// Visibility and exclusion rules live in JS, so rows are scanned in id-keyed
|
||||
// batches to keep memory bounded on large vocabularies.
|
||||
const scanStmt = db.prepare(`
|
||||
SELECT id AS wordId, headword, word, reading,
|
||||
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
|
||||
frequency, frequency_rank AS frequencyRank,
|
||||
first_seen AS firstSeen, last_seen AS lastSeen,
|
||||
0 AS animeCount
|
||||
FROM imm_words
|
||||
WHERE id > ?
|
||||
ORDER BY id
|
||||
LIMIT ?
|
||||
`);
|
||||
const excludedAliases = new Set(
|
||||
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
|
||||
);
|
||||
const weekAgoSec = nowMs / 1000 - 7 * 86_400;
|
||||
const summary: VocabularyStatsSummary = {
|
||||
uniqueWords: 0,
|
||||
uniqueWordsWithoutNames: 0,
|
||||
uniqueKanji: (db.prepare('SELECT COUNT(*) AS count FROM imm_kanji').get() as { count: number })
|
||||
.count,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: knownWords ? 0 : null,
|
||||
knownWordCountWithoutNames: knownWords ? 0 : null,
|
||||
};
|
||||
|
||||
let lastId = Number.MIN_SAFE_INTEGER;
|
||||
for (;;) {
|
||||
const words = scanStmt.all(lastId, scanBatchSize) as VocabularyStatsRow[];
|
||||
if (words.length === 0) break;
|
||||
lastId = words[words.length - 1]!.wordId;
|
||||
for (const word of words) {
|
||||
if (
|
||||
!isVocabularyStatsRowVisible(word) ||
|
||||
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const isName = word.pos2 === '固有名詞';
|
||||
const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec;
|
||||
const isKnown = knownWords?.has(vocabularyDisplayHeadword(word)) ?? false;
|
||||
summary.uniqueWords += 1;
|
||||
if (!isName) summary.uniqueWordsWithoutNames += 1;
|
||||
if (isNewThisWeek) {
|
||||
summary.newThisWeek += 1;
|
||||
if (!isName) summary.newThisWeekWithoutNames += 1;
|
||||
}
|
||||
if (isKnown) {
|
||||
summary.knownWordCount! += 1;
|
||||
if (!isName) summary.knownWordCountWithoutNames! += 1;
|
||||
}
|
||||
}
|
||||
if (words.length < scanBatchSize) break;
|
||||
}
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
export function getStatsExcludedWords(db: DatabaseSync): StatsExcludedWordRow[] {
|
||||
return db
|
||||
.prepare(
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
toDbTimestamp,
|
||||
} from './query-shared';
|
||||
import { getDailyRollups, getMonthlyRollups } from './query-sessions';
|
||||
import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
|
||||
|
||||
type TrendRange = '7d' | '30d' | '90d' | '365d' | 'all';
|
||||
type TrendGroupBy = 'day' | 'month';
|
||||
@@ -660,6 +661,16 @@ function buildNewWordsPerDay(
|
||||
cutoffMs: string | null,
|
||||
axis: number[] | null,
|
||||
): TrendChartPoint[] {
|
||||
if (areLexicalDailyRollupsReady(db)) {
|
||||
// A trend range is defined in calendar buckets, so the rollup includes the
|
||||
// complete local cutoff day rather than applying a time-of-day boundary.
|
||||
const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
|
||||
const rows = getLexicalDailyRollups(db).filter(
|
||||
(row) => cutoffDay === null || row.epochDay >= cutoffDay,
|
||||
);
|
||||
return fillAxisPoints(axis, new Map(rows.map((row) => [row.epochDay, row.wordCount])));
|
||||
}
|
||||
|
||||
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
||||
const prepared = db.prepare(`
|
||||
SELECT
|
||||
@@ -691,6 +702,18 @@ function buildNewWordsPerMonth(
|
||||
cutoffMs: string | null,
|
||||
axis: number[] | null,
|
||||
): TrendChartPoint[] {
|
||||
if (areLexicalDailyRollupsReady(db)) {
|
||||
const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
|
||||
const byMonth = new Map<number, number>();
|
||||
for (const row of getLexicalDailyRollups(db)) {
|
||||
if (cutoffDay !== null && row.epochDay < cutoffDay) continue;
|
||||
const { year, month } = dayPartsFromEpochDay(row.epochDay);
|
||||
const monthKey = year * 100 + month;
|
||||
byMonth.set(monthKey, (byMonth.get(monthKey) ?? 0) + row.wordCount);
|
||||
}
|
||||
return fillAxisPoints(axis, byMonth);
|
||||
}
|
||||
|
||||
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
||||
const prepared = db.prepare(`
|
||||
SELECT
|
||||
|
||||
@@ -4,6 +4,7 @@ import { parseMediaInfo } from '../../../jimaku/utils';
|
||||
import { normalizeTitleIdentity } from '../../utils/title-normalization';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { nowMs } from './time';
|
||||
import { ensureLexicalDailyRollupTables, markLexicalDailyRollupsReady } from './lexical-rollups';
|
||||
import { SCHEMA_VERSION } from './types';
|
||||
import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types';
|
||||
import { toDbMs, toDbTimestamp } from './query-shared';
|
||||
@@ -890,11 +891,11 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
VALUES ('last_rollup_sample_ms', 0)
|
||||
ON CONFLICT(state_key) DO NOTHING
|
||||
`);
|
||||
|
||||
const currentVersion = db
|
||||
.prepare('SELECT schema_version FROM imm_schema_version ORDER BY schema_version DESC LIMIT 1')
|
||||
.get() as { schema_version: number } | null;
|
||||
if (currentVersion?.schema_version === SCHEMA_VERSION) {
|
||||
ensureLexicalDailyRollupTables(db);
|
||||
ensureLifetimeSummaryTables(db);
|
||||
ensureStatsExcludedWordsTable(db);
|
||||
ensureAnimeMergeTables(db);
|
||||
@@ -1068,6 +1069,7 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
last_seen REAL,
|
||||
frequency INTEGER,
|
||||
frequency_rank INTEGER,
|
||||
vocabulary_visible INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1)),
|
||||
UNIQUE(headword, word, reading)
|
||||
);
|
||||
`);
|
||||
@@ -1451,8 +1453,18 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
addColumnIfMissing(db, 'imm_sessions', 'ended_media_ms', 'INTEGER');
|
||||
}
|
||||
|
||||
if (currentVersion?.schema_version && currentVersion.schema_version < 23) {
|
||||
addColumnIfMissing(
|
||||
db,
|
||||
'imm_words',
|
||||
'vocabulary_visible',
|
||||
'INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1))',
|
||||
);
|
||||
}
|
||||
|
||||
migrateSessionEventTimestampsToText(db);
|
||||
|
||||
ensureLexicalDailyRollupTables(db);
|
||||
ensureLifetimeSummaryTables(db);
|
||||
ensureStatsExcludedWordsTable(db);
|
||||
|
||||
@@ -1585,6 +1597,12 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
VALUES (${SCHEMA_VERSION}, ${toDbTimestamp(nowMs())})
|
||||
ON CONFLICT DO NOTHING
|
||||
`);
|
||||
|
||||
// A new database has no history to materialize. Upgrades are populated by the
|
||||
// background worker so startup never scans the existing vocabulary table.
|
||||
if (!currentVersion) {
|
||||
markLexicalDailyRollupsReady(db);
|
||||
}
|
||||
}
|
||||
|
||||
export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPreparedStatements {
|
||||
@@ -1617,9 +1635,10 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
|
||||
`),
|
||||
wordUpsertStmt: db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency, frequency_rank
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen,
|
||||
frequency, frequency_rank, vocabulary_visible
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, 1
|
||||
)
|
||||
ON CONFLICT(headword, word, reading) DO UPDATE SET
|
||||
frequency = COALESCE(frequency, 0) + 1,
|
||||
@@ -1632,6 +1651,7 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
|
||||
pos1 = COALESCE(NULLIF(imm_words.pos1, ''), excluded.pos1),
|
||||
pos2 = COALESCE(NULLIF(imm_words.pos2, ''), excluded.pos2),
|
||||
pos3 = COALESCE(NULLIF(imm_words.pos3, ''), excluded.pos3),
|
||||
vocabulary_visible = 1,
|
||||
first_seen = MIN(COALESCE(first_seen, excluded.first_seen), excluded.first_seen),
|
||||
last_seen = MAX(COALESCE(last_seen, excluded.last_seen), excluded.last_seen),
|
||||
frequency_rank = CASE
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export const SCHEMA_VERSION = 21;
|
||||
export const SCHEMA_VERSION = 23;
|
||||
export const DEFAULT_QUEUE_CAP = 1_000;
|
||||
export const DEFAULT_BATCH_SIZE = 25;
|
||||
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
|
||||
@@ -306,6 +306,16 @@ export interface VocabularyStatsRow {
|
||||
lastSeen: number;
|
||||
}
|
||||
|
||||
export interface VocabularyStatsSummary {
|
||||
uniqueWords: number;
|
||||
uniqueWordsWithoutNames: number;
|
||||
uniqueKanji: number;
|
||||
newThisWeek: number;
|
||||
newThisWeekWithoutNames: number;
|
||||
knownWordCount: number | null;
|
||||
knownWordCountWithoutNames: number | null;
|
||||
}
|
||||
|
||||
export interface StatsExcludedWordRow {
|
||||
headword: string;
|
||||
word: string;
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
resolveVocabularySummaryWorkerPath,
|
||||
VocabularySummaryWorkerRuntime,
|
||||
} from './vocabulary-summary-worker-runtime';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas, ensureSchema } from './storage';
|
||||
|
||||
test('vocabulary summary worker reads the database from a separate connection', async () => {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-vocabulary-summary-worker-'));
|
||||
const dbPath = path.join(tempDir, 'immersion.sqlite');
|
||||
const runtime = new VocabularySummaryWorkerRuntime();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
applyPragmas(db);
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES ('猫', '猫', 'ねこ', 'noun', '名詞', '一般', '', 1, 1, 1)
|
||||
`,
|
||||
).run();
|
||||
db.close();
|
||||
|
||||
const summary = await runtime.run(dbPath, new Set(['猫']));
|
||||
|
||||
assert.equal(summary.uniqueWords, 1);
|
||||
assert.equal(summary.knownWordCount, 1);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
try {
|
||||
db.close();
|
||||
} catch {
|
||||
// The worker needs the setup connection closed before it starts.
|
||||
}
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary summary worker module resolves in the current layout', () => {
|
||||
const workerPath = resolveVocabularySummaryWorkerPath();
|
||||
assert.ok(workerPath, 'expected the vocabulary summary worker module to resolve');
|
||||
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
|
||||
});
|
||||
|
||||
test('vocabulary summary worker never falls back to the caller thread', async () => {
|
||||
const runtime = new VocabularySummaryWorkerRuntime({
|
||||
resolveWorkerPath: () => null,
|
||||
warn: () => {},
|
||||
});
|
||||
|
||||
try {
|
||||
await assert.rejects(
|
||||
runtime.run('/tmp/subminer-summary-worker-not-used.sqlite', null),
|
||||
/worker unavailable/i,
|
||||
);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary summary worker times out when it never responds', async () => {
|
||||
let terminated = false;
|
||||
const runtime = new VocabularySummaryWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/fake-worker.js',
|
||||
createWorker: async () => ({
|
||||
once() {
|
||||
return this;
|
||||
},
|
||||
terminate: async () => {
|
||||
terminated = true;
|
||||
return 0;
|
||||
},
|
||||
}),
|
||||
timeoutMs: 1,
|
||||
warn: () => {},
|
||||
} as never);
|
||||
|
||||
try {
|
||||
const outcome = await Promise.race([
|
||||
runtime.run('/tmp/not-used.sqlite', null).then(
|
||||
() => 'resolved',
|
||||
(error: unknown) => String(error),
|
||||
),
|
||||
new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
|
||||
]);
|
||||
|
||||
assert.match(outcome, /timed out/);
|
||||
assert.equal(terminated, true);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createLogger } from '../../../logger';
|
||||
import type { VocabularyStatsSummary } from './types';
|
||||
|
||||
interface VocabularySummaryWorkerResponse {
|
||||
summary?: VocabularyStatsSummary;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
interface VocabularySummaryWorkerHandle {
|
||||
once(event: 'message', listener: (message: VocabularySummaryWorkerResponse) => void): this;
|
||||
once(event: 'error', listener: (error: Error) => void): this;
|
||||
once(event: 'exit', listener: (code: number) => void): this;
|
||||
terminate(): Promise<number>;
|
||||
}
|
||||
|
||||
interface VocabularySummaryWorkerRuntimeOptions {
|
||||
resolveWorkerPath?: () => string | null;
|
||||
createWorker?: (
|
||||
workerPath: string,
|
||||
workerData: { dbPath: string; knownWords: string[] | null },
|
||||
) => Promise<VocabularySummaryWorkerHandle>;
|
||||
timeoutMs?: number;
|
||||
warn?: (message: string, ...meta: unknown[]) => void;
|
||||
}
|
||||
|
||||
export type RunVocabularySummaryTask = (
|
||||
dbPath: string,
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
) => Promise<VocabularyStatsSummary>;
|
||||
|
||||
export function resolveVocabularySummaryWorkerPath(): string | null {
|
||||
const fileName = __filename.endsWith('.ts')
|
||||
? 'vocabulary-summary-worker-thread.ts'
|
||||
: 'vocabulary-summary-worker-thread.js';
|
||||
const workerPath = path.join(__dirname, fileName);
|
||||
return fs.existsSync(workerPath) ? workerPath : null;
|
||||
}
|
||||
|
||||
const logger = createLogger('main:immersion-tracker:vocabulary-summary-worker');
|
||||
const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
|
||||
|
||||
export class VocabularySummaryWorkerRuntime {
|
||||
private readonly activeWorkers = new Set<VocabularySummaryWorkerHandle>();
|
||||
private destroyed = false;
|
||||
|
||||
constructor(private readonly options: VocabularySummaryWorkerRuntimeOptions = {}) {}
|
||||
|
||||
async run(
|
||||
dbPath: string,
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
): Promise<VocabularyStatsSummary> {
|
||||
if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
|
||||
const workerData = { dbPath, knownWords: knownWords ? [...knownWords] : null };
|
||||
let worker: VocabularySummaryWorkerHandle;
|
||||
try {
|
||||
const workerPath = (this.options.resolveWorkerPath ?? resolveVocabularySummaryWorkerPath)();
|
||||
if (!workerPath) throw new Error('Emitted vocabulary summary worker module was not found');
|
||||
const createWorker =
|
||||
this.options.createWorker ??
|
||||
(async (resolvedPath, data) => {
|
||||
const { Worker } = await import('node:worker_threads');
|
||||
return new Worker(resolvedPath, { workerData: data });
|
||||
});
|
||||
worker = await createWorker(workerPath, workerData);
|
||||
} catch (error) {
|
||||
if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
|
||||
(this.options.warn ?? logger.warn)(
|
||||
'Vocabulary summary worker unavailable; refusing to scan vocabulary on the current thread',
|
||||
error,
|
||||
);
|
||||
throw new Error('Vocabulary summary worker unavailable');
|
||||
}
|
||||
|
||||
if (this.destroyed) {
|
||||
await worker.terminate().catch(() => undefined);
|
||||
throw new Error('Vocabulary summary worker is shut down');
|
||||
}
|
||||
|
||||
return new Promise<VocabularyStatsSummary>((resolve, reject) => {
|
||||
let settled = false;
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
this.activeWorkers.add(worker);
|
||||
const settle = (result: VocabularyStatsSummary | Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timeout) clearTimeout(timeout);
|
||||
this.activeWorkers.delete(worker);
|
||||
void worker.terminate().catch(() => undefined);
|
||||
if (result instanceof Error) reject(result);
|
||||
else resolve(result);
|
||||
};
|
||||
timeout = setTimeout(
|
||||
() => settle(new Error('Vocabulary summary worker timed out')),
|
||||
this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
worker.once('message', (message) => {
|
||||
if (message.summary) {
|
||||
settle(message.summary);
|
||||
return;
|
||||
}
|
||||
settle(
|
||||
new Error(
|
||||
`Vocabulary summary failed: ${String(message.error ?? 'unknown worker error')}`,
|
||||
),
|
||||
);
|
||||
});
|
||||
worker.once('error', (error) => settle(error));
|
||||
worker.once('exit', (code) => {
|
||||
if (!settled) {
|
||||
settle(
|
||||
new Error(
|
||||
code === 0
|
||||
? 'Vocabulary summary worker exited without a response'
|
||||
: `Vocabulary summary worker exited with code ${code}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
for (const worker of this.activeWorkers) {
|
||||
void worker.terminate().catch(() => undefined);
|
||||
}
|
||||
this.activeWorkers.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
import { executeVocabularySummaryTask } from './vocabulary-summary-worker';
|
||||
|
||||
interface VocabularySummaryWorkerData {
|
||||
dbPath: string;
|
||||
knownWords: string[] | null;
|
||||
}
|
||||
|
||||
if (!parentPort) throw new Error('vocabulary summary worker missing parent port');
|
||||
|
||||
const request = workerData as VocabularySummaryWorkerData;
|
||||
|
||||
try {
|
||||
parentPort.postMessage({
|
||||
summary: executeVocabularySummaryTask(request.dbPath, request.knownWords),
|
||||
});
|
||||
} catch (error) {
|
||||
parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { getVocabularySummary } from './query-lexical';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas } from './storage';
|
||||
import type { VocabularyStatsSummary } from './types';
|
||||
|
||||
export function executeVocabularySummaryTask(
|
||||
dbPath: string,
|
||||
knownWords: string[] | null,
|
||||
): VocabularyStatsSummary {
|
||||
const db = new Database(dbPath);
|
||||
try {
|
||||
applyPragmas(db);
|
||||
return getVocabularySummary(db, knownWords ? new Set(knownWords) : null);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { PartOfSpeech, type MergedToken } from '../../../types';
|
||||
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
|
||||
|
||||
export interface VocabularyVisibilityRow {
|
||||
word: string | null;
|
||||
headword: string | null;
|
||||
reading?: string | null;
|
||||
partOfSpeech?: string | null;
|
||||
pos1?: string | null;
|
||||
pos2?: string | null;
|
||||
pos3?: string | null;
|
||||
frequencyRank?: number | null;
|
||||
}
|
||||
|
||||
function toVocabularyToken(row: VocabularyVisibilityRow): MergedToken {
|
||||
const word = row.word ?? '';
|
||||
const headword = row.headword ?? word;
|
||||
const partOfSpeech =
|
||||
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
|
||||
? (row.partOfSpeech as PartOfSpeech)
|
||||
: PartOfSpeech.other;
|
||||
|
||||
return {
|
||||
surface: word,
|
||||
reading: row.reading ?? '',
|
||||
headword,
|
||||
startPos: 0,
|
||||
endPos: word.length,
|
||||
partOfSpeech,
|
||||
pos1: row.pos1 ?? '',
|
||||
pos2: row.pos2 ?? '',
|
||||
pos3: row.pos3 ?? '',
|
||||
frequencyRank: row.frequencyRank ?? undefined,
|
||||
isMerged: false,
|
||||
isKnown: false,
|
||||
isNPlusOneTarget: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function isVocabularyStatsRowVisible(row: VocabularyVisibilityRow): boolean {
|
||||
if (!(row.word?.trim() || row.headword?.trim())) return false;
|
||||
return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
parseExcludedWordsBody,
|
||||
parseIntQuery,
|
||||
parsePositiveIdList,
|
||||
loadKnownWordsSet,
|
||||
} from './route-support.js';
|
||||
|
||||
export function registerStatsLibraryRoutes(
|
||||
@@ -31,6 +32,17 @@ export function registerStatsLibraryRoutes(
|
||||
return c.json(statsJson('vocabulary', vocab));
|
||||
});
|
||||
|
||||
app.get('/api/stats/vocabulary/summary', async (c) => {
|
||||
const summary = await tracker.getVocabularySummary(
|
||||
loadKnownWordsSet(options?.knownWordCachePath),
|
||||
);
|
||||
return c.json(statsJson('vocabularySummary', summary));
|
||||
});
|
||||
|
||||
app.get('/api/stats/vocabulary/charts', async (c) => {
|
||||
return c.json(statsJson('vocabularyCharts', await tracker.getVocabularyChartData()));
|
||||
});
|
||||
|
||||
app.get('/api/stats/excluded-words', async (c) => {
|
||||
return c.json(statsJson('excludedWords', await tracker.getStatsExcludedWords()));
|
||||
});
|
||||
|
||||
@@ -89,6 +89,7 @@ const WORD_COPY_COLUMNS = [
|
||||
'last_seen',
|
||||
'frequency',
|
||||
'frequency_rank',
|
||||
'vocabulary_visible',
|
||||
] as const;
|
||||
|
||||
export function mergeAnime(
|
||||
|
||||
Reference in New Issue
Block a user