mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-08-18 12:18:29 -07:00
fix(stats): report complete vocabulary totals and new-word history (#202)
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
type: fixed
|
||||
area: stats
|
||||
|
||||
- Fixed Vocabulary totals and charts counting only the first browsing page instead of all tracked vocabulary, without delaying the rest of the page.
|
||||
- New-word history now uses permanent daily lexical rollups that apply the same vocabulary filters as the totals and normalize legacy second/millisecond timestamps; versioned background rebuilds repair existing history without dropping playback writes.
|
||||
- Calendar-day chart labels now preserve the recorded local date in time zones west of UTC.
|
||||
- Vocabulary summary cards and charts refresh automatically after the word exclusion list changes, and failed or unfinished loads use bounded retries before showing an inline error with a Retry control.
|
||||
- Rapid exclusion edits no longer race each other; writes are sent in order so a slower earlier save cannot overwrite a newer list.
|
||||
@@ -82,7 +82,7 @@ Expandable session history with new-word activity, cumulative totals, and pause/
|
||||
|
||||
#### Vocabulary
|
||||
|
||||
Top repeated words (click a bar to open the word), new-word timeline, cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons.
|
||||
The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The word and kanji tables load first while those complete totals calculate separately. Top Repeated Words and New Words by Day use complete tracking history rather than the table's browsing page. New-word history is maintained as a permanent daily lexical rollup using the same token-visibility rules as the totals, including normalization of older timestamps stored in either seconds or milliseconds and retroactive corrections when tracked material is removed or reprocessed. On the first launch after an applicable upgrade, that history is version-rebuilt in the background and the chart refreshes when it is ready; if it remains unavailable, polling stops and an inline Retry control appears. The cards and charts also refresh automatically after the word exclusion list changes. The rest of the tab includes cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons.
|
||||
|
||||

|
||||
|
||||
@@ -182,6 +182,7 @@ In practice:
|
||||
- Anime and episode pages keep lifetime totals from summary tables while session drill-down still reads retained sessions directly. With the current defaults, both are kept forever.
|
||||
- Trends can read the full available history because daily/monthly rollups are also kept forever by default.
|
||||
- Vocabulary and kanji totals are cumulative and not bounded by the raw session retention knobs.
|
||||
- New-word charts use their own permanent lexical daily rollups, which are not pruned by activity-rollup retention.
|
||||
|
||||
## Storage / Performance Model
|
||||
|
||||
@@ -351,6 +352,7 @@ Rollup tables:
|
||||
|
||||
- `imm_daily_rollups`
|
||||
- `imm_monthly_rollups`
|
||||
- `imm_lexical_daily_rollups` - permanent first-discovery counts for vocabulary and kanji chart history
|
||||
- `imm_rollup_state` - incremental rollup progress bookkeeping
|
||||
|
||||
Vocabulary tables:
|
||||
|
||||
@@ -23,7 +23,9 @@ Trend charts now consume one chart-oriented backend payload from `/api/stats/tre
|
||||
- lookup rate trends
|
||||
- watch-time by day-of-week/hour
|
||||
- vocabulary-backed:
|
||||
- new-words trend
|
||||
- new-words trend reads permanent daily lexical rollups
|
||||
- rollup rows count only vocabulary-visible tokens and normalize mixed legacy timestamp units
|
||||
- a persisted rollup version invalidates stale materializations and triggers an atomic background rebuild
|
||||
|
||||
## Metric Semantics
|
||||
|
||||
|
||||
@@ -18,6 +18,9 @@ export function createImmersionDbFixture(dbPath: string): void {
|
||||
db.prepare(
|
||||
`INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('last_rollup_sample_ms', 0)`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('lexical_daily_rollups_version', 0)`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`INSERT INTO imm_lifetime_global(global_id, CREATED_DATE, LAST_UPDATE_DATE) VALUES (1, ?, ?)`,
|
||||
).run(String(Date.now()), String(Date.now()));
|
||||
|
||||
@@ -108,6 +108,36 @@ test('fixture schema stays aligned with production sync-touched tables and index
|
||||
}
|
||||
});
|
||||
|
||||
test('fixture leaves lexical rollups pending when their table is absent', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-fixture-rollup-state-'));
|
||||
const fixturePath = path.join(dir, 'fixture.sqlite');
|
||||
try {
|
||||
createImmersionDbFixture(fixturePath);
|
||||
const db = new BunDatabase(fixturePath, { readonly: true });
|
||||
try {
|
||||
const state = db
|
||||
.query<{ state_value: string }>(
|
||||
`SELECT state_value FROM imm_rollup_state
|
||||
WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||
)
|
||||
.get();
|
||||
const rollupTable = db
|
||||
.query<{ name: string }>(
|
||||
`SELECT name FROM sqlite_schema
|
||||
WHERE type = 'table' AND name = 'imm_lexical_daily_rollups'`,
|
||||
)
|
||||
.get();
|
||||
|
||||
assert.equal(state?.state_value, '0');
|
||||
assert.equal(rollupTable, null);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('fixture session inserts enforce foreign keys', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-fixture-foreign-keys-'));
|
||||
const fixturePath = path.join(dir, 'fixture.sqlite');
|
||||
|
||||
@@ -154,6 +154,7 @@ export const IMMERSION_DB_FIXTURE_DDL = `
|
||||
last_seen REAL,
|
||||
frequency INTEGER,
|
||||
frequency_rank INTEGER,
|
||||
vocabulary_visible INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1)),
|
||||
UNIQUE(headword, word, reading)
|
||||
);
|
||||
CREATE TABLE imm_kanji(
|
||||
|
||||
@@ -284,6 +284,22 @@ function createMockTracker(
|
||||
getSessionTimeline: async () => [],
|
||||
getSessionEvents: async () => [],
|
||||
getVocabularyStats: async () => VOCABULARY_STATS,
|
||||
getVocabularySummary: async () => ({
|
||||
uniqueWords: 501,
|
||||
uniqueWordsWithoutNames: 500,
|
||||
uniqueKanji: 201,
|
||||
newThisWeek: 7,
|
||||
newThisWeekWithoutNames: 6,
|
||||
knownWordCount: 250,
|
||||
knownWordCountWithoutNames: 249,
|
||||
}),
|
||||
getVocabularyChartData: async () => ({
|
||||
ready: true,
|
||||
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
}),
|
||||
getStatsExcludedWords: async () => [],
|
||||
replaceStatsExcludedWords: async () => {},
|
||||
getKanjiStats: async () => KANJI_STATS,
|
||||
@@ -711,6 +727,38 @@ describe('stats server API routes', () => {
|
||||
assert.equal(body[0].headword, 'する');
|
||||
});
|
||||
|
||||
it('GET /api/stats/vocabulary/summary returns database-wide card totals', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
|
||||
const res = await app.request('/api/stats/vocabulary/summary');
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(await res.json(), {
|
||||
uniqueWords: 501,
|
||||
uniqueWordsWithoutNames: 500,
|
||||
uniqueKanji: 201,
|
||||
newThisWeek: 7,
|
||||
newThisWeekWithoutNames: 6,
|
||||
knownWordCount: 250,
|
||||
knownWordCountWithoutNames: 249,
|
||||
});
|
||||
});
|
||||
|
||||
it('GET /api/stats/vocabulary/charts returns complete chart datasets', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
|
||||
const res = await app.request('/api/stats/vocabulary/charts');
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.deepEqual(await res.json(), {
|
||||
ready: true,
|
||||
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
|
||||
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
|
||||
});
|
||||
});
|
||||
|
||||
it('GET /api/stats/kanji returns kanji frequency data', async () => {
|
||||
const app = createStatsApp(createMockTracker());
|
||||
const res = await app.request('/api/stats/kanji');
|
||||
|
||||
@@ -559,6 +559,165 @@ test('fresh tracker DB creates lifetime summary tables', async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('fresh tracker DB skips lexical rollup backfill work', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let backfillRuns = 0;
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor({ dbPath }, {
|
||||
runLexicalRollupBackfillTask: async () => {
|
||||
backfillRuns += 1;
|
||||
},
|
||||
} as never);
|
||||
|
||||
assert.equal(backfillRuns, 0);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('tracker starts the injected lexical rollup backfill when it is pending', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let backfillRuns = 0;
|
||||
|
||||
try {
|
||||
const setupDb = new Database(dbPath);
|
||||
const { ensureSchema } = await import('./immersion-tracker/storage');
|
||||
ensureSchema(setupDb);
|
||||
setupDb
|
||||
.prepare(
|
||||
`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||
)
|
||||
.run();
|
||||
setupDb.close();
|
||||
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor({ dbPath }, {
|
||||
runLexicalRollupBackfillTask: async () => {
|
||||
backfillRuns += 1;
|
||||
},
|
||||
} as never);
|
||||
|
||||
assert.equal(backfillRuns, 1);
|
||||
await waitForCondition(
|
||||
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
|
||||
);
|
||||
assert.equal(
|
||||
(tracker as unknown as { preserveWriteQueueUntilDrained: boolean })
|
||||
.preserveWriteQueueUntilDrained,
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('tracker queues playback writes until lexical rollup backfill settles', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let startBackfill = (): void => {};
|
||||
let releaseBackfill = (): void => {};
|
||||
let markBackfillStarted = (): void => {};
|
||||
const backfillStartGate = new Promise<void>((resolve) => {
|
||||
startBackfill = resolve;
|
||||
});
|
||||
const heldBackfill = new Promise<void>((resolve) => {
|
||||
releaseBackfill = resolve;
|
||||
});
|
||||
const backfillStarted = new Promise<void>((resolve) => {
|
||||
markBackfillStarted = resolve;
|
||||
});
|
||||
|
||||
try {
|
||||
const setupDb = new Database(dbPath);
|
||||
const { ensureSchema } = await import('./immersion-tracker/storage');
|
||||
ensureSchema(setupDb);
|
||||
setupDb
|
||||
.prepare(
|
||||
`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||
)
|
||||
.run();
|
||||
setupDb.close();
|
||||
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath, policy: { queueCap: 100 } },
|
||||
{
|
||||
runLexicalRollupBackfillTask: async (workerDbPath) => {
|
||||
await backfillStartGate;
|
||||
const workerDb = new Database(workerDbPath);
|
||||
try {
|
||||
workerDb.exec('BEGIN IMMEDIATE');
|
||||
markBackfillStarted();
|
||||
await heldBackfill;
|
||||
workerDb.exec('COMMIT');
|
||||
} catch (error) {
|
||||
try {
|
||||
workerDb.exec('ROLLBACK');
|
||||
} catch {
|
||||
// Preserve the original worker failure.
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
workerDb.close();
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
tracker.handleMediaChange('https://example.com/backfill-test.mp4', 'Backfill Test');
|
||||
startBackfill();
|
||||
await backfillStarted;
|
||||
for (let index = 0; index < 125; index += 1) tracker.recordCardsMined(1);
|
||||
|
||||
const privateApi = tracker as unknown as {
|
||||
db: DatabaseSync;
|
||||
queue: unknown[];
|
||||
droppedWriteCount: number;
|
||||
flushNow: () => void;
|
||||
writeLock: { locked: boolean };
|
||||
};
|
||||
assert.equal(privateApi.writeLock.locked, true);
|
||||
privateApi.flushNow();
|
||||
|
||||
assert.ok(privateApi.queue.length > 100, 'the protected queue may grow past its normal cap');
|
||||
assert.equal(privateApi.droppedWriteCount, 0, 'backfill must not discard playback writes');
|
||||
assert.equal(
|
||||
(
|
||||
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
|
||||
total: number;
|
||||
}
|
||||
).total,
|
||||
0,
|
||||
);
|
||||
|
||||
releaseBackfill();
|
||||
await waitForCondition(() => privateApi.queue.length === 0, 5_000);
|
||||
assert.equal(
|
||||
(
|
||||
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
|
||||
total: number;
|
||||
}
|
||||
).total,
|
||||
125,
|
||||
);
|
||||
} finally {
|
||||
releaseBackfill();
|
||||
if (tracker) {
|
||||
await waitForCondition(
|
||||
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
|
||||
5_000,
|
||||
);
|
||||
}
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('startup backfills lifetime summaries when retained sessions exist but summary tables are empty', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
@@ -4909,3 +5068,149 @@ test('ensureAnimeCoverArt fetches art via the latest video of the anime', async
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary coalesces concurrent requests into one worker task', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let taskRuns = 0;
|
||||
let releaseTask: (() => void) | null = null;
|
||||
const seenKnownWords: Array<ReadonlySet<string> | null> = [];
|
||||
const summary = {
|
||||
uniqueWords: 1,
|
||||
uniqueWordsWithoutNames: 1,
|
||||
uniqueKanji: 0,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: null,
|
||||
knownWordCountWithoutNames: null,
|
||||
};
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runVocabularySummaryTask: async (_dbPath, knownWords) => {
|
||||
taskRuns += 1;
|
||||
seenKnownWords.push(knownWords);
|
||||
await new Promise<void>((resolve) => {
|
||||
releaseTask = resolve;
|
||||
});
|
||||
return summary;
|
||||
},
|
||||
destroyVocabularySummaryRunner: () => {},
|
||||
},
|
||||
);
|
||||
|
||||
const knownWordsSnapshot = new Set(['猫']);
|
||||
const first = tracker.getVocabularySummary(knownWordsSnapshot);
|
||||
const second = tracker.getVocabularySummary(knownWordsSnapshot);
|
||||
await waitForCondition(() => releaseTask !== null);
|
||||
let release = releaseTask as (() => void) | null;
|
||||
assert.ok(release);
|
||||
release();
|
||||
assert.deepEqual(await first, summary);
|
||||
assert.equal(await second, await first);
|
||||
assert.equal(taskRuns, 1);
|
||||
assert.deepEqual(seenKnownWords, [knownWordsSnapshot]);
|
||||
|
||||
releaseTask = null;
|
||||
const third = tracker.getVocabularySummary(null);
|
||||
await waitForCondition(() => releaseTask !== null);
|
||||
release = releaseTask as (() => void) | null;
|
||||
assert.ok(release);
|
||||
release();
|
||||
assert.deepEqual(await third, summary);
|
||||
assert.equal(taskRuns, 2);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary coalesces equivalent known-word snapshots by value', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
let taskRuns = 0;
|
||||
const releases: Array<() => void> = [];
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runVocabularySummaryTask: async () => {
|
||||
taskRuns += 1;
|
||||
await new Promise<void>((resolve) => releases.push(resolve));
|
||||
return {
|
||||
uniqueWords: 2,
|
||||
uniqueWordsWithoutNames: 2,
|
||||
uniqueKanji: 2,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: 2,
|
||||
knownWordCountWithoutNames: 2,
|
||||
};
|
||||
},
|
||||
destroyVocabularySummaryRunner: () => {},
|
||||
},
|
||||
);
|
||||
|
||||
const first = tracker.getVocabularySummary(new Set(['猫', '犬']));
|
||||
const second = tracker.getVocabularySummary(new Set(['犬', '猫']));
|
||||
await waitForCondition(() => releases.length > 0);
|
||||
const observedTaskRuns = taskRuns;
|
||||
for (const release of releases) release();
|
||||
await Promise.all([first, second]);
|
||||
|
||||
assert.equal(observedTaskRuns, 1);
|
||||
|
||||
const third = tracker.getVocabularySummary(new Set(['猫', '犬']));
|
||||
await waitForCondition(() => releases.length === 2);
|
||||
releases[1]!();
|
||||
await third;
|
||||
assert.equal(taskRuns, 2, 'a settled snapshot must be evicted from the in-flight map');
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary keeps different known-word snapshots independent', async () => {
|
||||
const dbPath = makeDbPath();
|
||||
let tracker: ImmersionTrackerService | null = null;
|
||||
const releases: Array<() => void> = [];
|
||||
|
||||
try {
|
||||
const Ctor = await loadTrackerCtor();
|
||||
tracker = new Ctor(
|
||||
{ dbPath },
|
||||
{
|
||||
runVocabularySummaryTask: async (_dbPath, knownWords) => {
|
||||
await new Promise<void>((resolve) => releases.push(resolve));
|
||||
return {
|
||||
uniqueWords: 1,
|
||||
uniqueWordsWithoutNames: 1,
|
||||
uniqueKanji: 0,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: knownWords?.size ?? null,
|
||||
knownWordCountWithoutNames: knownWords?.size ?? null,
|
||||
};
|
||||
},
|
||||
destroyVocabularySummaryRunner: () => {},
|
||||
},
|
||||
);
|
||||
|
||||
const withoutKnownWords = tracker.getVocabularySummary(null);
|
||||
const withKnownWords = tracker.getVocabularySummary(new Set(['猫']));
|
||||
await waitForCondition(() => releases.length === 2);
|
||||
for (const release of releases) release();
|
||||
|
||||
assert.equal((await withoutKnownWords).knownWordCount, null);
|
||||
assert.equal((await withKnownWords).knownWordCount, 1);
|
||||
} finally {
|
||||
tracker?.destroy();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -58,6 +58,7 @@ import {
|
||||
getSessionEvents,
|
||||
getSimilarWords,
|
||||
getStatsExcludedWords,
|
||||
getVocabularyChartData,
|
||||
getVocabularyStats,
|
||||
replaceStatsExcludedWords,
|
||||
searchSubtitleSentences,
|
||||
@@ -96,6 +97,12 @@ import {
|
||||
DeleteMaintenanceWorkerRuntime,
|
||||
type RunDeleteMaintenanceTask,
|
||||
} from './immersion-tracker/delete-maintenance-worker-runtime';
|
||||
import {
|
||||
VocabularySummaryWorkerRuntime,
|
||||
type RunVocabularySummaryTask,
|
||||
} from './immersion-tracker/vocabulary-summary-worker-runtime';
|
||||
import { LexicalRollupWorkerRuntime } from './immersion-tracker/lexical-rollup-worker-runtime';
|
||||
import { areLexicalDailyRollupsReady } from './immersion-tracker/lexical-rollups';
|
||||
import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
|
||||
import {
|
||||
cleanupDuplicateSubtitleLines,
|
||||
@@ -185,6 +192,7 @@ import {
|
||||
type StatsExcludedWordRow,
|
||||
type StreakCalendarRow,
|
||||
type VocabularyCleanupSummary,
|
||||
type VocabularyStatsSummary,
|
||||
type WatchTimePerAnimeRow,
|
||||
type WordAnimeAppearanceRow,
|
||||
type WordDetailRow,
|
||||
@@ -405,13 +413,24 @@ export class ImmersionTrackerService {
|
||||
private readonly monthlyRollupRetentionMs: number;
|
||||
private readonly vacuumIntervalMs: number;
|
||||
private readonly dbPath: string;
|
||||
private readonly writeLock = { locked: false };
|
||||
private readonly writeLock = {
|
||||
locked: false,
|
||||
reasons: new Set<'flush' | 'delete-maintenance' | 'lexical-rollup-backfill'>(),
|
||||
};
|
||||
private readonly destroyDeleteMaintenanceRunner: () => void;
|
||||
private readonly runVocabularySummaryTask: (
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
) => Promise<VocabularyStatsSummary>;
|
||||
private readonly vocabularySummariesInFlight = new Map<string, Promise<VocabularyStatsSummary>>();
|
||||
private readonly destroyVocabularySummaryRunner: () => void;
|
||||
private readonly runLexicalRollupBackfillTask: () => Promise<void>;
|
||||
private readonly destroyLexicalRollupBackfillRunner: () => void;
|
||||
private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
|
||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private flushScheduled = false;
|
||||
private droppedWriteCount = 0;
|
||||
private preserveWriteQueueUntilDrained = false;
|
||||
private lastVacuumMs = 0;
|
||||
private isDestroyed = false;
|
||||
private sessionState: SessionState | null = null;
|
||||
@@ -434,6 +453,10 @@ export class ImmersionTrackerService {
|
||||
dependencies: {
|
||||
runDeleteMaintenanceTask?: RunDeleteMaintenanceTask;
|
||||
destroyDeleteMaintenanceRunner?: () => void;
|
||||
runVocabularySummaryTask?: RunVocabularySummaryTask;
|
||||
destroyVocabularySummaryRunner?: () => void;
|
||||
runLexicalRollupBackfillTask?: (dbPath: string) => Promise<void>;
|
||||
destroyLexicalRollupBackfillRunner?: () => void;
|
||||
} = {},
|
||||
) {
|
||||
this.dbPath = options.dbPath;
|
||||
@@ -453,13 +476,34 @@ export class ImmersionTrackerService {
|
||||
runTask: (task) => runDeleteMaintenanceTask(this.dbPath, task),
|
||||
onBusy: () => {
|
||||
this.requireWriteQueueDrained('delete maintenance');
|
||||
this.writeLock.locked = true;
|
||||
this.setWriteLock('delete-maintenance', true);
|
||||
},
|
||||
onIdle: () => {
|
||||
this.writeLock.locked = false;
|
||||
this.setWriteLock('delete-maintenance', false);
|
||||
if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
|
||||
},
|
||||
});
|
||||
if (dependencies.runVocabularySummaryTask) {
|
||||
this.runVocabularySummaryTask = (knownWords) =>
|
||||
dependencies.runVocabularySummaryTask!(this.dbPath, knownWords);
|
||||
this.destroyVocabularySummaryRunner =
|
||||
dependencies.destroyVocabularySummaryRunner ?? (() => {});
|
||||
} else {
|
||||
const vocabularySummaryRuntime = new VocabularySummaryWorkerRuntime();
|
||||
this.runVocabularySummaryTask = (knownWords) =>
|
||||
vocabularySummaryRuntime.run(this.dbPath, knownWords);
|
||||
this.destroyVocabularySummaryRunner = () => vocabularySummaryRuntime.destroy();
|
||||
}
|
||||
if (dependencies.runLexicalRollupBackfillTask) {
|
||||
this.runLexicalRollupBackfillTask = () =>
|
||||
dependencies.runLexicalRollupBackfillTask!(this.dbPath);
|
||||
this.destroyLexicalRollupBackfillRunner =
|
||||
dependencies.destroyLexicalRollupBackfillRunner ?? (() => {});
|
||||
} else {
|
||||
const lexicalRollupRuntime = new LexicalRollupWorkerRuntime();
|
||||
this.runLexicalRollupBackfillTask = () => lexicalRollupRuntime.run(this.dbPath);
|
||||
this.destroyLexicalRollupBackfillRunner = () => lexicalRollupRuntime.destroy();
|
||||
}
|
||||
const parentDir = path.dirname(this.dbPath);
|
||||
if (!fs.existsSync(parentDir)) {
|
||||
fs.mkdirSync(parentDir, { recursive: true });
|
||||
@@ -547,6 +591,7 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
}
|
||||
this.preparedStatements = createTrackerPreparedStatements(this.db);
|
||||
if (!areLexicalDailyRollupsReady(this.db)) this.startLexicalRollupBackfill();
|
||||
this.scheduleMaintenance();
|
||||
this.scheduleFlush();
|
||||
}
|
||||
@@ -565,6 +610,8 @@ export class ImmersionTrackerService {
|
||||
this.isDestroyed = true;
|
||||
this.deleteMaintenanceScheduler.destroy();
|
||||
this.destroyDeleteMaintenanceRunner();
|
||||
this.destroyVocabularySummaryRunner();
|
||||
this.destroyLexicalRollupBackfillRunner();
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
@@ -634,6 +681,25 @@ export class ImmersionTrackerService {
|
||||
return getVocabularyStats(this.db, limit, excludePos);
|
||||
}
|
||||
|
||||
async getVocabularySummary(knownWords: ReadonlySet<string> | null) {
|
||||
const key = knownWords ? JSON.stringify([...knownWords].sort()) : 'null';
|
||||
const inFlight = this.vocabularySummariesInFlight.get(key);
|
||||
if (inFlight) return inFlight;
|
||||
const task = this.runVocabularySummaryTask(knownWords);
|
||||
this.vocabularySummariesInFlight.set(key, task);
|
||||
try {
|
||||
return await task;
|
||||
} finally {
|
||||
if (this.vocabularySummariesInFlight.get(key) === task) {
|
||||
this.vocabularySummariesInFlight.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async getVocabularyChartData() {
|
||||
return getVocabularyChartData(this.db);
|
||||
}
|
||||
|
||||
async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {
|
||||
return getStatsExcludedWords(this.db);
|
||||
}
|
||||
@@ -910,6 +976,33 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
}
|
||||
|
||||
private setWriteLock(
|
||||
reason: 'flush' | 'delete-maintenance' | 'lexical-rollup-backfill',
|
||||
active: boolean,
|
||||
): void {
|
||||
if (active) this.writeLock.reasons.add(reason);
|
||||
else this.writeLock.reasons.delete(reason);
|
||||
this.writeLock.locked = this.writeLock.reasons.size > 0;
|
||||
}
|
||||
|
||||
private startLexicalRollupBackfill(): void {
|
||||
this.requireWriteQueueDrained('lexical rollup backfill');
|
||||
this.preserveWriteQueueUntilDrained = true;
|
||||
this.setWriteLock('lexical-rollup-backfill', true);
|
||||
void this.runLexicalRollupBackfillTask()
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
'Lexical daily rollup backfill failed; it will retry on next startup',
|
||||
error,
|
||||
);
|
||||
})
|
||||
.finally(() => {
|
||||
this.setWriteLock('lexical-rollup-backfill', false);
|
||||
if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
|
||||
else if (!this.isDestroyed) this.scheduleFlush(0);
|
||||
});
|
||||
}
|
||||
|
||||
async reassignAnimeAnilist(
|
||||
animeId: number,
|
||||
info: {
|
||||
@@ -1906,7 +1999,12 @@ export class ImmersionTrackerService {
|
||||
|
||||
private recordWrite(write: QueuedWrite): void {
|
||||
if (this.isDestroyed) return;
|
||||
const { dropped } = enqueueWrite(this.queue, write, this.queueCap);
|
||||
// A lexical migration owns the database write lock, so dropping the oldest
|
||||
// entry cannot relieve pressure: nothing can flush until the worker exits.
|
||||
// Preserve that finite startup burst and drain it as soon as the lock lifts.
|
||||
const { dropped } = this.preserveWriteQueueUntilDrained
|
||||
? (this.queue.push(write), { dropped: 0 })
|
||||
: enqueueWrite(this.queue, write, this.queueCap);
|
||||
if (dropped > 0) {
|
||||
this.droppedWriteCount += dropped;
|
||||
this.logger.warn(`Immersion tracker queue overflow; dropped ${dropped} oldest writes`);
|
||||
@@ -1954,6 +2052,7 @@ export class ImmersionTrackerService {
|
||||
private flushNow(): void {
|
||||
if (this.writeLock.locked || this.isDestroyed) return;
|
||||
if (this.queue.length === 0) {
|
||||
this.preserveWriteQueueUntilDrained = false;
|
||||
this.flushScheduled = false;
|
||||
return;
|
||||
}
|
||||
@@ -1965,7 +2064,7 @@ export class ImmersionTrackerService {
|
||||
}
|
||||
|
||||
const batch = this.queue.splice(0, Math.min(this.batchSize, this.queue.length));
|
||||
this.writeLock.locked = true;
|
||||
this.setWriteLock('flush', true);
|
||||
try {
|
||||
this.db.exec('BEGIN IMMEDIATE');
|
||||
for (const write of batch) {
|
||||
@@ -1977,8 +2076,9 @@ export class ImmersionTrackerService {
|
||||
this.queue.unshift(...batch);
|
||||
this.logger.warn('Immersion tracker flush failed, retrying later', error as Error);
|
||||
} finally {
|
||||
this.writeLock.locked = false;
|
||||
this.setWriteLock('flush', false);
|
||||
this.flushScheduled = false;
|
||||
if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
|
||||
if (this.queue.length > 0) {
|
||||
this.scheduleFlush(this.flushIntervalMs);
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
getKanjiOccurrences,
|
||||
getSessionSummaries,
|
||||
getVocabularyStats,
|
||||
getVocabularySummary,
|
||||
getKanjiStats,
|
||||
getSessionEvents,
|
||||
getSessionTimeline,
|
||||
@@ -1875,6 +1876,115 @@ test('getVocabularyStats returns rows ordered by frequency descending', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary counts every tracked vocabulary row instead of a display page', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const nowSec = Math.floor(Date.now() / 1000);
|
||||
const insertWord = db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', ?, ?, 1)
|
||||
`);
|
||||
const insertKanji = db.prepare(`
|
||||
INSERT INTO imm_kanji (kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)
|
||||
`);
|
||||
|
||||
for (let index = 0; index < 501; index += 1) {
|
||||
insertWord.run(`単語${index}`, `単語${index}`, nowSec - 8 * 86_400, nowSec - 8 * 86_400);
|
||||
}
|
||||
for (let index = 0; index < 201; index += 1) {
|
||||
insertKanji.run(
|
||||
String.fromCodePoint(0x4e00 + index),
|
||||
nowSec - 8 * 86_400,
|
||||
nowSec - 8 * 86_400,
|
||||
);
|
||||
}
|
||||
insertWord.run('今週', '今週', nowSec - 86_400, nowSec - 86_400);
|
||||
|
||||
assert.deepEqual(getVocabularySummary(db, new Set(['単語0', '今週']), nowSec * 1000), {
|
||||
uniqueWords: 502,
|
||||
uniqueWordsWithoutNames: 502,
|
||||
uniqueKanji: 201,
|
||||
newThisWeek: 1,
|
||||
newThisWeekWithoutNames: 1,
|
||||
knownWordCount: 2,
|
||||
knownWordCountWithoutNames: 2,
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary applies vocabulary exclusions and Hide Names totals', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, '', 'noun', '名詞', ?, '', 1, 1, 1)
|
||||
`);
|
||||
insertWord.run('猫', '猫', '一般');
|
||||
insertWord.run('太郎', '太郎', '固有名詞');
|
||||
insertWord.run('東京', '東京都', '一般');
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_stats_excluded_words (headword, word, reading)
|
||||
VALUES ('東京', '東京', '')
|
||||
`,
|
||||
).run();
|
||||
|
||||
assert.deepEqual(getVocabularySummary(db, new Set(['猫', '太郎', '東京']), 9 * 86_400_000), {
|
||||
uniqueWords: 2,
|
||||
uniqueWordsWithoutNames: 1,
|
||||
uniqueKanji: 0,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: 2,
|
||||
knownWordCountWithoutNames: 1,
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularySummary counts identically across id-keyed scan batches', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', 1, 1, 1)
|
||||
`);
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
insertWord.run(`単語${index}`, `単語${index}`);
|
||||
}
|
||||
|
||||
const fullScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000);
|
||||
const batchedScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000, 2);
|
||||
|
||||
assert.equal(fullScan.uniqueWords, 5);
|
||||
assert.deepEqual(batchedScan, fullScan);
|
||||
} finally {
|
||||
db.close();
|
||||
cleanupDbPath(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = openTestDb(dbPath);
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
LexicalRollupWorkerRuntime,
|
||||
resolveLexicalRollupWorkerPath,
|
||||
} from './lexical-rollup-worker-runtime';
|
||||
import { areLexicalDailyRollupsReady } from './lexical-rollups';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas, ensureSchema } from './storage';
|
||||
|
||||
test('lexical rollup worker backfills without using the tracker connection', async () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-runtime-'));
|
||||
const dbPath = path.join(directory, 'immersion.sqlite');
|
||||
const runtime = new LexicalRollupWorkerRuntime();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
applyPragmas(db);
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES ('鳥', '鳥', 'とり', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||
db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
|
||||
'lexical_daily_rollups_version',
|
||||
);
|
||||
db.close();
|
||||
|
||||
await runtime.run(dbPath);
|
||||
|
||||
const checkDb = new Database(dbPath);
|
||||
try {
|
||||
assert.equal(areLexicalDailyRollupsReady(checkDb), true);
|
||||
} finally {
|
||||
checkDb.close();
|
||||
}
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
try {
|
||||
db.close();
|
||||
} catch {
|
||||
// Closed before the worker starts.
|
||||
}
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup worker module resolves in the current layout', () => {
|
||||
const workerPath = resolveLexicalRollupWorkerPath();
|
||||
assert.ok(workerPath, 'expected the lexical rollup worker module to resolve');
|
||||
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
|
||||
});
|
||||
|
||||
test('lexical rollup worker leaves a backfill pending when no worker can start', async () => {
|
||||
const runtime = new LexicalRollupWorkerRuntime({
|
||||
resolveWorkerPath: () => null,
|
||||
warn: () => {},
|
||||
} as never);
|
||||
|
||||
try {
|
||||
await assert.doesNotReject(runtime.run('/tmp/not-used.sqlite'));
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup worker absorbs termination failures after settling', async () => {
|
||||
let sendMessage: ((message: { ok: boolean }) => void) | null = null;
|
||||
const runtime = new LexicalRollupWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/fake-worker.js',
|
||||
createWorker: async () => ({
|
||||
once(event: string, listener: (value: never) => void) {
|
||||
if (event === 'message') sendMessage = listener as (message: { ok: boolean }) => void;
|
||||
return this;
|
||||
},
|
||||
terminate: async () => {
|
||||
throw new Error('termination failed');
|
||||
},
|
||||
}),
|
||||
warn: () => {},
|
||||
} as never);
|
||||
|
||||
const unhandled: unknown[] = [];
|
||||
const captureUnhandled = (reason: unknown) => unhandled.push(reason);
|
||||
process.on('unhandledRejection', captureUnhandled);
|
||||
try {
|
||||
const task = runtime.run('/tmp/not-used.sqlite');
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
const notify = sendMessage as ((message: { ok: boolean }) => void) | null;
|
||||
assert.ok(notify);
|
||||
notify({ ok: true });
|
||||
await task;
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.deepEqual(unhandled, []);
|
||||
} finally {
|
||||
process.off('unhandledRejection', captureUnhandled);
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup worker times out when it never responds', async () => {
|
||||
let terminated = false;
|
||||
const runtime = new LexicalRollupWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/fake-worker.js',
|
||||
createWorker: async () => ({
|
||||
once() {
|
||||
return this;
|
||||
},
|
||||
terminate: async () => {
|
||||
terminated = true;
|
||||
return 0;
|
||||
},
|
||||
}),
|
||||
timeoutMs: 1,
|
||||
warn: () => {},
|
||||
} as never);
|
||||
|
||||
try {
|
||||
const outcome = await Promise.race([
|
||||
runtime.run('/tmp/not-used.sqlite').then(
|
||||
() => 'resolved',
|
||||
(error: unknown) => String(error),
|
||||
),
|
||||
new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
|
||||
]);
|
||||
|
||||
assert.match(outcome, /timed out/);
|
||||
assert.equal(terminated, true);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createLogger } from '../../../logger';
|
||||
|
||||
interface WorkerResponse {
|
||||
ok?: boolean;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
interface WorkerHandle {
|
||||
once(event: 'message', listener: (message: WorkerResponse) => void): this;
|
||||
once(event: 'error', listener: (error: Error) => void): this;
|
||||
once(event: 'exit', listener: (code: number) => void): this;
|
||||
terminate(): Promise<number>;
|
||||
}
|
||||
|
||||
interface LexicalRollupWorkerRuntimeOptions {
|
||||
resolveWorkerPath?: () => string | null;
|
||||
createWorker?: (workerPath: string, workerData: { dbPath: string }) => Promise<WorkerHandle>;
|
||||
timeoutMs?: number;
|
||||
warn?: (message: string, ...meta: unknown[]) => void;
|
||||
}
|
||||
|
||||
const logger = createLogger('main:immersion-tracker:lexical-rollup-worker');
|
||||
const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
|
||||
|
||||
export function resolveLexicalRollupWorkerPath(): string | null {
|
||||
const fileName = __filename.endsWith('.ts')
|
||||
? 'lexical-rollup-worker-thread.ts'
|
||||
: 'lexical-rollup-worker-thread.js';
|
||||
const workerPath = path.join(__dirname, fileName);
|
||||
return fs.existsSync(workerPath) ? workerPath : null;
|
||||
}
|
||||
|
||||
export class LexicalRollupWorkerRuntime {
|
||||
private readonly activeWorkers = new Set<WorkerHandle>();
|
||||
private destroyed = false;
|
||||
|
||||
constructor(private readonly options: LexicalRollupWorkerRuntimeOptions = {}) {}
|
||||
|
||||
async run(dbPath: string): Promise<void> {
|
||||
if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
|
||||
let worker: WorkerHandle;
|
||||
try {
|
||||
const workerPath = (this.options.resolveWorkerPath ?? resolveLexicalRollupWorkerPath)();
|
||||
if (!workerPath) throw new Error('Emitted lexical rollup worker module was not found');
|
||||
const createWorker =
|
||||
this.options.createWorker ??
|
||||
(async (resolvedPath, workerData) => {
|
||||
const { Worker } = await import('node:worker_threads');
|
||||
return new Worker(resolvedPath, { workerData });
|
||||
});
|
||||
worker = await createWorker(workerPath, { dbPath });
|
||||
} catch (error) {
|
||||
if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
|
||||
(this.options.warn ?? logger.warn)(
|
||||
'Lexical rollup worker unavailable; leaving backfill pending for a later startup',
|
||||
error,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.destroyed) {
|
||||
await worker.terminate().catch(() => undefined);
|
||||
throw new Error('Lexical rollup worker is shut down');
|
||||
}
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
let settled = false;
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
this.activeWorkers.add(worker);
|
||||
const settle = (error?: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timeout) clearTimeout(timeout);
|
||||
this.activeWorkers.delete(worker);
|
||||
void worker.terminate().catch(() => undefined);
|
||||
if (error) reject(error);
|
||||
else resolve();
|
||||
};
|
||||
timeout = setTimeout(
|
||||
() => settle(new Error('Lexical rollup worker timed out')),
|
||||
this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
|
||||
);
|
||||
worker.once('message', (message) => {
|
||||
if (message.ok) settle();
|
||||
else
|
||||
settle(
|
||||
new Error(
|
||||
`Lexical rollup backfill failed: ${String(message.error ?? 'unknown error')}`,
|
||||
),
|
||||
);
|
||||
});
|
||||
worker.once('error', (error) => settle(error));
|
||||
worker.once('exit', (code) => {
|
||||
if (!settled) {
|
||||
settle(
|
||||
new Error(
|
||||
code === 0
|
||||
? 'Lexical rollup worker exited without a response'
|
||||
: `Lexical rollup worker exited with code ${code}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
for (const worker of this.activeWorkers) {
|
||||
void worker.terminate().catch(() => undefined);
|
||||
}
|
||||
this.activeWorkers.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
|
||||
|
||||
if (!parentPort) throw new Error('lexical rollup worker missing parent port');
|
||||
|
||||
try {
|
||||
executeLexicalRollupBackfillTask((workerData as { dbPath: string }).dbPath);
|
||||
parentPort.postMessage({ ok: true });
|
||||
} catch (error) {
|
||||
parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
|
||||
import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
|
||||
import { Database } from './sqlite';
|
||||
import { ensureSchema } from './storage';
|
||||
|
||||
test('lexical rollup backfill materializes pre-existing vocabulary off the caller DB connection', () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-worker-'));
|
||||
const dbPath = path.join(directory, 'immersion.sqlite');
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||
).run('犬', '犬', 'いぬ', 1_700_000_000, 1_700_000_000);
|
||||
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||
db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
|
||||
'lexical_daily_rollups_version',
|
||||
);
|
||||
|
||||
executeLexicalRollupBackfillTask(dbPath);
|
||||
|
||||
assert.equal(areLexicalDailyRollupsReady(db), true);
|
||||
assert.equal(getLexicalDailyRollups(db)[0]?.wordCount, 1);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
import { areLexicalDailyRollupsReady, rebuildLexicalDailyRollups } from './lexical-rollups';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas } from './storage';
|
||||
|
||||
export function executeLexicalRollupBackfillTask(dbPath: string): void {
|
||||
const db = new Database(dbPath);
|
||||
try {
|
||||
applyPragmas(db);
|
||||
if (!areLexicalDailyRollupsReady(db)) {
|
||||
rebuildLexicalDailyRollups(db);
|
||||
}
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
areLexicalDailyRollupsReady,
|
||||
getLexicalDailyRollups,
|
||||
rebuildLexicalDailyRollups,
|
||||
} from './lexical-rollups';
|
||||
import { getTrendsDashboard } from './query-trends';
|
||||
import {
|
||||
getVocabularyChartData,
|
||||
getVocabularySummary,
|
||||
replaceStatsExcludedWords,
|
||||
} from './query-lexical';
|
||||
import { Database } from './sqlite';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { ensureSchema } from './storage';
|
||||
|
||||
function makeDbPath(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollups-'));
|
||||
return path.join(dir, 'immersion.sqlite');
|
||||
}
|
||||
|
||||
test('lexical daily rollups follow first-seen corrections and deletions', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const firstDay = 19_500;
|
||||
const correctedDay = firstDay + 2;
|
||||
const firstSeen = firstDay * 86_400 + 43_200;
|
||||
const correctedSeen = correctedDay * 86_400 + 43_200;
|
||||
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||
).run('猫', '猫', 'ねこ', firstSeen, firstSeen);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)`,
|
||||
).run('猫', firstSeen, firstSeen);
|
||||
|
||||
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||
{ epochDay: firstDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 1 },
|
||||
]);
|
||||
|
||||
db.prepare(`UPDATE imm_words SET first_seen = ? WHERE headword = ?`).run(correctedSeen, '猫');
|
||||
db.prepare(`DELETE FROM imm_kanji WHERE kanji = ?`).run('猫');
|
||||
|
||||
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||
{ epochDay: correctedDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
|
||||
]);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical daily rollups normalize second and millisecond timestamps', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const epochDay = 19_500;
|
||||
const timestampSeconds = epochDay * 86_400 + 43_200;
|
||||
const timestampMilliseconds = timestampSeconds * 1_000;
|
||||
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||
).run('猫', '猫', 'ねこ', timestampSeconds, timestampSeconds);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, ?, ?, 1)`,
|
||||
).run('犬', '犬', 'いぬ', timestampMilliseconds, timestampMilliseconds);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)`,
|
||||
).run('猫', timestampSeconds, timestampSeconds);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, ?, 1)`,
|
||||
).run('犬', timestampMilliseconds, timestampMilliseconds);
|
||||
|
||||
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||
{ epochDay, wordCount: 2, wordCountWithoutNames: 2, kanjiCount: 2 },
|
||||
]);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup rebuild excludes rows hidden by vocabulary persistence rules', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const epochDay = 19_500;
|
||||
const firstSeen = epochDay * 86_400 + 43_200;
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(
|
||||
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||
).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(
|
||||
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||
).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
|
||||
|
||||
rebuildLexicalDailyRollups(db);
|
||||
|
||||
assert.deepEqual(getLexicalDailyRollups(db), [
|
||||
{ epochDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
|
||||
]);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup rebuild tolerates nullable legacy vocabulary text', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (NULL, NULL, NULL, 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (NULL, '猫', 'ねこ', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
|
||||
assert.doesNotThrow(() => rebuildLexicalDailyRollups(db));
|
||||
assert.equal(areLexicalDailyRollupsReady(db), true);
|
||||
assert.equal(getVocabularySummary(db, null).uniqueWords, 1);
|
||||
assert.equal(getVocabularySummary(db, new Set(['猫'])).knownWordCount, 1);
|
||||
assert.equal(getVocabularyChartData(db).topWords[0]?.headword, '猫');
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup rebuild scans vocabulary visibility in bounded id batches', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
const expectedBatchSize = 5_000;
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, '', 1700000000, 1700000000, 1)`,
|
||||
);
|
||||
db.exec('BEGIN');
|
||||
for (let index = 0; index <= expectedBatchSize; index += 1) {
|
||||
insertWord.run(`語${index}`, `語${index}`);
|
||||
}
|
||||
db.exec('COMMIT');
|
||||
|
||||
const scanPageSizes: number[] = [];
|
||||
const instrumentedDb: DatabaseSync = {
|
||||
prepare(source) {
|
||||
const statement = db.prepare(source);
|
||||
if (!source.includes('WHERE id > ?') || !source.includes('ORDER BY id')) {
|
||||
return statement;
|
||||
}
|
||||
return {
|
||||
run: (...params) => statement.run(...params),
|
||||
get: (...params) => statement.get(...params),
|
||||
all: (...params) => {
|
||||
const rows = statement.all(...params);
|
||||
scanPageSizes.push(rows.length);
|
||||
return rows;
|
||||
},
|
||||
};
|
||||
},
|
||||
exec(source) {
|
||||
db.exec(source);
|
||||
return instrumentedDb;
|
||||
},
|
||||
close() {
|
||||
return instrumentedDb;
|
||||
},
|
||||
};
|
||||
|
||||
rebuildLexicalDailyRollups(instrumentedDb);
|
||||
|
||||
assert.deepEqual(scanPageSizes, [expectedBatchSize, 1]);
|
||||
assert.equal(getVocabularySummary(db, null).uniqueWords, expectedBatchSize + 1);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('chart exclusions do not subtract vocabulary rows already hidden from the rollup', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const epochDay = 19_500;
|
||||
const firstSeen = epochDay * 86_400 + 43_200;
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(
|
||||
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||
).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(
|
||||
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
|
||||
).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
|
||||
rebuildLexicalDailyRollups(db);
|
||||
replaceStatsExcludedWords(db, [{ headword: 'は', word: 'は', reading: 'は' }]);
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.deepEqual(charts.newWordsTimeline, [{ epochDay, wordCount: 1 }]);
|
||||
assert.deepEqual(charts.newWordsTimelineWithoutNames, [{ epochDay, wordCount: 1 }]);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('legacy lexical rollup readiness does not satisfy the current rollup version', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_rollup_state(state_key, state_value)
|
||||
VALUES ('lexical_daily_rollups_ready', '1')
|
||||
ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`DELETE FROM imm_rollup_state WHERE state_key = 'lexical_daily_rollups_version'`,
|
||||
).run();
|
||||
|
||||
assert.equal(areLexicalDailyRollupsReady(db), false);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('imm_words persists vocabulary visibility for rollup maintenance', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const columns = db.prepare(`PRAGMA table_info(imm_words)`).all() as Array<{ name: string }>;
|
||||
|
||||
assert.equal(
|
||||
columns.some((column) => column.name === 'vocabulary_visible'),
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary charts use complete top-word and lexical rollup data', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, '', 1700000000, 1700000000, ?)`,
|
||||
);
|
||||
for (let index = 0; index < 501; index += 1) {
|
||||
insertWord.run(`語${index}`, `語${index}`, index === 500 ? 10_000 : 1);
|
||||
}
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.equal(charts.topWords[0]?.headword, '語500');
|
||||
assert.equal(charts.topWords[0]?.frequency, 10_000);
|
||||
assert.equal(charts.newWordsTimeline[0]?.wordCount, 501);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary charts find full top-word sets beyond excluded and name rows', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
const insertWord = db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, pos2, first_seen, last_seen, frequency)
|
||||
VALUES (?, ?, '', ?, 1700000000, 1700000000, ?)`,
|
||||
);
|
||||
const exclusions = [];
|
||||
for (let index = 0; index < 100; index += 1) {
|
||||
const headword = `語${index}`;
|
||||
insertWord.run(
|
||||
headword,
|
||||
headword,
|
||||
index < 80 && index >= 60 ? '固有名詞' : '一般',
|
||||
100 - index,
|
||||
);
|
||||
if (index < 60) exclusions.push({ headword, word: headword, reading: '' });
|
||||
}
|
||||
replaceStatsExcludedWords(db, exclusions);
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.equal(charts.topWords.length, 12);
|
||||
assert.equal(charts.topWords[0]?.headword, '語60');
|
||||
assert.equal(charts.topWordsWithoutNames.length, 12);
|
||||
assert.equal(charts.topWordsWithoutNames[0]?.headword, '語80');
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary charts handle exclusion lists above one SQLite variable batch', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES ('語0', '語0', '', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
const exclusions = Array.from({ length: 10_923 }, (_, index) => ({
|
||||
headword: `語${index}`,
|
||||
word: `語${index}`,
|
||||
reading: '',
|
||||
}));
|
||||
replaceStatsExcludedWords(db, exclusions);
|
||||
|
||||
const charts = getVocabularyChartData(db);
|
||||
|
||||
assert.deepEqual(charts.topWords, []);
|
||||
assert.deepEqual(charts.newWordsTimeline, []);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('lexical rollup rebuild preserves the original error when rollback also fails', () => {
|
||||
const originalError = new Error('rebuild failed');
|
||||
const db = {
|
||||
exec(sql: string) {
|
||||
if (sql === 'BEGIN IMMEDIATE') return;
|
||||
if (sql === 'ROLLBACK') throw new Error('rollback failed');
|
||||
throw originalError;
|
||||
},
|
||||
prepare() {
|
||||
return { all: () => [], run: () => undefined };
|
||||
},
|
||||
} as unknown as DatabaseSync;
|
||||
|
||||
assert.throws(() => rebuildLexicalDailyRollups(db), originalError);
|
||||
});
|
||||
|
||||
test('trends read historical new-word buckets from lexical rollups', () => {
|
||||
const dbPath = makeDbPath();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
|
||||
VALUES ('海', '海', 'うみ', 1700000000, 1700000000, 1)`,
|
||||
).run();
|
||||
db.prepare(`UPDATE imm_lexical_daily_rollups SET word_count = 9`).run();
|
||||
|
||||
const dashboard = getTrendsDashboard(db, 'all', 'day', false);
|
||||
|
||||
assert.equal(dashboard.progress.newWords[0]?.value, 9);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,226 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility';
|
||||
|
||||
export interface LexicalDailyRollup {
|
||||
epochDay: number;
|
||||
wordCount: number;
|
||||
wordCountWithoutNames: number;
|
||||
kanjiCount: number;
|
||||
}
|
||||
|
||||
const LOCAL_EPOCH_DAY_SQL = `
|
||||
CAST(julianday(
|
||||
CASE
|
||||
WHEN ABS(CAST(%VALUE% AS REAL)) >= 10000000000 THEN CAST(%VALUE% AS REAL) / 1000
|
||||
ELSE CAST(%VALUE% AS REAL)
|
||||
END,
|
||||
'unixepoch', 'localtime'
|
||||
) - 2440587.5 AS INTEGER)
|
||||
`;
|
||||
|
||||
const LEXICAL_DAILY_ROLLUP_VERSION = '2';
|
||||
const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
|
||||
const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
|
||||
|
||||
export function localEpochDaySql(value: string): string {
|
||||
return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
|
||||
}
|
||||
|
||||
function createWordRollupTriggers(db: DatabaseSync): void {
|
||||
const dayForNew = localEpochDaySql('NEW.first_seen');
|
||||
const dayForOld = localEpochDaySql('OLD.first_seen');
|
||||
|
||||
db.exec(`
|
||||
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_insert;
|
||||
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_delete;
|
||||
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_first_seen_update;
|
||||
|
||||
CREATE TRIGGER imm_words_lexical_rollup_insert
|
||||
AFTER INSERT ON imm_words
|
||||
WHEN NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
VALUES (${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0)
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||
word_count = word_count + 1,
|
||||
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER imm_words_lexical_rollup_delete
|
||||
AFTER DELETE ON imm_words
|
||||
WHEN OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
VALUES (${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0)
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||
word_count = word_count - 1,
|
||||
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||
DELETE FROM imm_lexical_daily_rollups
|
||||
WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER imm_words_lexical_rollup_first_seen_update
|
||||
AFTER UPDATE OF first_seen, pos2, vocabulary_visible ON imm_words
|
||||
WHEN OLD.first_seen IS NOT NEW.first_seen
|
||||
OR OLD.pos2 IS NOT NEW.pos2
|
||||
OR OLD.vocabulary_visible IS NOT NEW.vocabulary_visible
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0
|
||||
WHERE OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||
word_count = word_count - 1,
|
||||
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0
|
||||
WHERE NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET
|
||||
word_count = word_count + 1,
|
||||
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
|
||||
DELETE FROM imm_lexical_daily_rollups
|
||||
WHERE word_count = 0 AND kanji_count = 0;
|
||||
END;
|
||||
`);
|
||||
}
|
||||
|
||||
function createKanjiRollupTriggers(db: DatabaseSync): void {
|
||||
const dayForNew = localEpochDaySql('NEW.first_seen');
|
||||
const dayForOld = localEpochDaySql('OLD.first_seen');
|
||||
db.exec(`
|
||||
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_insert;
|
||||
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_delete;
|
||||
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_first_seen_update;
|
||||
|
||||
CREATE TRIGGER imm_kanji_lexical_rollup_insert
|
||||
AFTER INSERT ON imm_kanji WHEN NEW.first_seen IS NOT NULL
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
VALUES (${dayForNew}, 0, 0, 1)
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
|
||||
END;
|
||||
CREATE TRIGGER imm_kanji_lexical_rollup_delete
|
||||
AFTER DELETE ON imm_kanji WHEN OLD.first_seen IS NOT NULL
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
VALUES (${dayForOld}, 0, 0, -1)
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
|
||||
DELETE FROM imm_lexical_daily_rollups
|
||||
WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
|
||||
END;
|
||||
CREATE TRIGGER imm_kanji_lexical_rollup_first_seen_update
|
||||
AFTER UPDATE OF first_seen ON imm_kanji WHEN OLD.first_seen IS NOT NEW.first_seen
|
||||
BEGIN
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${dayForOld}, 0, 0, -1 WHERE OLD.first_seen IS NOT NULL
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${dayForNew}, 0, 0, 1 WHERE NEW.first_seen IS NOT NULL
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
|
||||
DELETE FROM imm_lexical_daily_rollups WHERE word_count = 0 AND kanji_count = 0;
|
||||
END;
|
||||
`);
|
||||
}
|
||||
|
||||
export function ensureLexicalDailyRollupTables(db: DatabaseSync): void {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS imm_lexical_daily_rollups(
|
||||
epoch_day INTEGER PRIMARY KEY,
|
||||
word_count INTEGER NOT NULL DEFAULT 0,
|
||||
word_count_without_names INTEGER NOT NULL DEFAULT 0,
|
||||
kanji_count INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
INSERT INTO imm_rollup_state(state_key, state_value)
|
||||
VALUES ('${LEXICAL_DAILY_ROLLUP_VERSION_KEY}', '0')
|
||||
ON CONFLICT(state_key) DO NOTHING;
|
||||
`);
|
||||
createWordRollupTriggers(db);
|
||||
createKanjiRollupTriggers(db);
|
||||
}
|
||||
|
||||
export function areLexicalDailyRollupsReady(db: DatabaseSync): boolean {
|
||||
const row = db
|
||||
.prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
|
||||
.get(LEXICAL_DAILY_ROLLUP_VERSION_KEY) as { value: string } | null;
|
||||
return row?.value === LEXICAL_DAILY_ROLLUP_VERSION;
|
||||
}
|
||||
|
||||
export function markLexicalDailyRollupsReady(db: DatabaseSync): void {
|
||||
db.prepare(
|
||||
`INSERT INTO imm_rollup_state(state_key, state_value)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
|
||||
).run(LEXICAL_DAILY_ROLLUP_VERSION_KEY, LEXICAL_DAILY_ROLLUP_VERSION);
|
||||
}
|
||||
|
||||
/** Rebuild from the first-seen source of truth; run off the UI/main DB thread. */
|
||||
export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
|
||||
let transactionStarted = false;
|
||||
try {
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
transactionStarted = true;
|
||||
const scanVocabulary = db.prepare(
|
||||
`SELECT id, word, headword, reading, part_of_speech AS partOfSpeech,
|
||||
pos1, pos2, pos3, frequency_rank AS frequencyRank
|
||||
FROM imm_words
|
||||
WHERE id > ?
|
||||
ORDER BY id
|
||||
LIMIT ?`,
|
||||
);
|
||||
const updateVisibility = db.prepare(
|
||||
`UPDATE imm_words SET vocabulary_visible = ? WHERE id = ? AND vocabulary_visible IS NOT ?`,
|
||||
);
|
||||
let lastId = Number.MIN_SAFE_INTEGER;
|
||||
for (;;) {
|
||||
const vocabularyRows = scanVocabulary.all(
|
||||
lastId,
|
||||
VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE,
|
||||
) as Array<VocabularyVisibilityRow & { id: number }>;
|
||||
if (vocabularyRows.length === 0) break;
|
||||
for (const row of vocabularyRows) {
|
||||
const visible = isVocabularyStatsRowVisible(row) ? 1 : 0;
|
||||
updateVisibility.run(visible, row.id, visible);
|
||||
}
|
||||
lastId = vocabularyRows[vocabularyRows.length - 1]!.id;
|
||||
if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break;
|
||||
}
|
||||
db.exec('DELETE FROM imm_lexical_daily_rollups');
|
||||
db.exec(`
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${localEpochDaySql('first_seen')}, COUNT(*),
|
||||
SUM(CASE WHEN pos2 = '固有名詞' THEN 0 ELSE 1 END), 0
|
||||
FROM imm_words
|
||||
WHERE first_seen IS NOT NULL AND vocabulary_visible = 1
|
||||
GROUP BY ${localEpochDaySql('first_seen')};
|
||||
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
|
||||
SELECT ${localEpochDaySql('first_seen')}, 0, 0, COUNT(*)
|
||||
FROM imm_kanji
|
||||
WHERE first_seen IS NOT NULL
|
||||
GROUP BY ${localEpochDaySql('first_seen')}
|
||||
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + excluded.kanji_count;
|
||||
`);
|
||||
markLexicalDailyRollupsReady(db);
|
||||
db.exec('COMMIT');
|
||||
} catch (error) {
|
||||
if (transactionStarted) {
|
||||
try {
|
||||
db.exec('ROLLBACK');
|
||||
} catch {
|
||||
// Preserve the rebuild failure; it is the actionable cause.
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function getLexicalDailyRollups(db: DatabaseSync): LexicalDailyRollup[] {
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT epoch_day AS epochDay, word_count AS wordCount,
|
||||
word_count_without_names AS wordCountWithoutNames, kanji_count AS kanjiCount
|
||||
FROM imm_lexical_daily_rollups
|
||||
ORDER BY epoch_day ASC
|
||||
`,
|
||||
)
|
||||
.all() as LexicalDailyRollup[];
|
||||
}
|
||||
@@ -1,6 +1,4 @@
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { PartOfSpeech, type MergedToken } from '../../../types';
|
||||
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
|
||||
import type {
|
||||
KanjiAnimeAppearanceRow,
|
||||
KanjiDetailRow,
|
||||
@@ -13,19 +11,38 @@ import type {
|
||||
SimilarWordRow,
|
||||
StatsExcludedWordRow,
|
||||
VocabularyStatsRow,
|
||||
VocabularyStatsSummary,
|
||||
WordAnimeAppearanceRow,
|
||||
WordDetailRow,
|
||||
WordOccurrenceRow,
|
||||
} from './types';
|
||||
import { fromDbTimestamp, toDbTimestamp } from './query-shared';
|
||||
import { nowMs } from './time';
|
||||
import {
|
||||
areLexicalDailyRollupsReady,
|
||||
getLexicalDailyRollups,
|
||||
localEpochDaySql,
|
||||
} from './lexical-rollups';
|
||||
import { isVocabularyStatsRowVisible } from './vocabulary-visibility';
|
||||
|
||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
|
||||
const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
|
||||
const VOCABULARY_CHART_LIMIT = 12;
|
||||
const VOCABULARY_CHART_PAGE_SIZE = 100;
|
||||
const EXCLUSION_ALIAS_BATCH_SIZE = 300;
|
||||
const VOCABULARY_SUMMARY_SCAN_BATCH_SIZE = 5_000;
|
||||
const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
|
||||
const SENTENCE_SEARCH_MAX_LIMIT = 100;
|
||||
const KANJI_PATTERN = /\p{Script=Han}/gu;
|
||||
|
||||
export interface VocabularyChartData {
|
||||
ready: boolean;
|
||||
topWords: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||
topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||
newWordsTimeline: Array<{ epochDay: number; wordCount: number }>;
|
||||
newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>;
|
||||
}
|
||||
|
||||
function resolveSentenceSearchLimit(limit: number): number {
|
||||
if (!Number.isFinite(limit)) return SENTENCE_SEARCH_DEFAULT_LIMIT;
|
||||
const normalized = Math.floor(limit);
|
||||
@@ -73,33 +90,6 @@ function uniqueKanji(text: string): string[] {
|
||||
return Array.from(new Set(text.match(KANJI_PATTERN) ?? []));
|
||||
}
|
||||
|
||||
function toVocabularyToken(row: VocabularyStatsRow): MergedToken {
|
||||
const partOfSpeech =
|
||||
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
|
||||
? (row.partOfSpeech as PartOfSpeech)
|
||||
: PartOfSpeech.other;
|
||||
|
||||
return {
|
||||
surface: row.word,
|
||||
reading: row.reading ?? '',
|
||||
headword: row.headword,
|
||||
startPos: 0,
|
||||
endPos: row.word.length,
|
||||
partOfSpeech,
|
||||
pos1: row.pos1 ?? '',
|
||||
pos2: row.pos2 ?? '',
|
||||
pos3: row.pos3 ?? '',
|
||||
frequencyRank: row.frequencyRank ?? undefined,
|
||||
isMerged: false,
|
||||
isKnown: false,
|
||||
isNPlusOneTarget: false,
|
||||
};
|
||||
}
|
||||
|
||||
function isVocabularyStatsRowVisible(row: VocabularyStatsRow): boolean {
|
||||
return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
|
||||
}
|
||||
|
||||
export function getVocabularyStats(
|
||||
db: DatabaseSync,
|
||||
limit = 100,
|
||||
@@ -153,6 +143,198 @@ export function getVocabularyStats(
|
||||
return visibleRows.slice(0, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Chart data is intentionally independent of the paginated vocabulary tables.
|
||||
* Top words use the frequency index; new-word history reads permanent daily
|
||||
* lexical rollups rather than loading every vocabulary row into the dashboard.
|
||||
*/
|
||||
export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData {
|
||||
const ready = areLexicalDailyRollupsReady(db);
|
||||
const excludedAliases = new Set(
|
||||
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
|
||||
);
|
||||
const isExcluded = (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>): boolean =>
|
||||
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias));
|
||||
const topWords = getTopVocabularyChartWords(db, isExcluded);
|
||||
const rollups = ready ? getLexicalDailyRollups(db) : [];
|
||||
const timeline = new Map(rollups.map((row) => [row.epochDay, { ...row }]));
|
||||
if (excludedAliases.size > 0 && ready) {
|
||||
const aliases = [...excludedAliases];
|
||||
const excludedRows = new Map<
|
||||
number,
|
||||
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
|
||||
wordId: number;
|
||||
epochDay: number;
|
||||
}
|
||||
>();
|
||||
for (let offset = 0; offset < aliases.length; offset += EXCLUSION_ALIAS_BATCH_SIZE) {
|
||||
const batch = aliases.slice(offset, offset + EXCLUSION_ALIAS_BATCH_SIZE);
|
||||
const placeholders = batch.map(() => '?').join(', ');
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id AS wordId, headword, word, reading, pos2,
|
||||
${localEpochDaySql('first_seen')} AS epochDay
|
||||
FROM imm_words
|
||||
WHERE vocabulary_visible = 1
|
||||
AND (headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}))
|
||||
`,
|
||||
)
|
||||
.all(...batch, ...batch, ...batch) as Array<
|
||||
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
|
||||
wordId: number;
|
||||
epochDay: number;
|
||||
}
|
||||
>;
|
||||
for (const row of rows) excludedRows.set(row.wordId, row);
|
||||
}
|
||||
for (const word of excludedRows.values()) {
|
||||
if (!isExcluded(word)) continue;
|
||||
const rollup = timeline.get(word.epochDay);
|
||||
if (!rollup) continue;
|
||||
rollup.wordCount -= 1;
|
||||
if (word.pos2 !== '固有名詞') rollup.wordCountWithoutNames -= 1;
|
||||
}
|
||||
}
|
||||
return {
|
||||
ready,
|
||||
topWords: topWords.all.map((word) => ({
|
||||
wordId: word.wordId,
|
||||
headword: vocabularyDisplayHeadword(word),
|
||||
frequency: word.frequency,
|
||||
})),
|
||||
topWordsWithoutNames: topWords.withoutNames.map((word) => ({
|
||||
wordId: word.wordId,
|
||||
headword: vocabularyDisplayHeadword(word),
|
||||
frequency: word.frequency,
|
||||
})),
|
||||
newWordsTimeline: [...timeline.values()]
|
||||
.filter((row) => row.wordCount > 0)
|
||||
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCount })),
|
||||
newWordsTimelineWithoutNames: [...timeline.values()]
|
||||
.filter((row) => row.wordCountWithoutNames > 0)
|
||||
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCountWithoutNames })),
|
||||
};
|
||||
}
|
||||
|
||||
function getTopVocabularyChartWords(
|
||||
db: DatabaseSync,
|
||||
isExcluded: (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>) => boolean,
|
||||
): { all: VocabularyStatsRow[]; withoutNames: VocabularyStatsRow[] } {
|
||||
const stmt = db.prepare(`
|
||||
SELECT id AS wordId, headword, word, reading,
|
||||
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
|
||||
frequency, frequency_rank AS frequencyRank,
|
||||
first_seen AS firstSeen, last_seen AS lastSeen,
|
||||
0 AS animeCount
|
||||
FROM imm_words
|
||||
ORDER BY frequency DESC, id
|
||||
LIMIT ? OFFSET ?
|
||||
`);
|
||||
const all: VocabularyStatsRow[] = [];
|
||||
const withoutNames: VocabularyStatsRow[] = [];
|
||||
let offset = 0;
|
||||
|
||||
while (all.length < VOCABULARY_CHART_LIMIT || withoutNames.length < VOCABULARY_CHART_LIMIT) {
|
||||
const page = stmt.all(VOCABULARY_CHART_PAGE_SIZE, offset) as VocabularyStatsRow[];
|
||||
if (page.length === 0) break;
|
||||
for (const word of page) {
|
||||
if (!isVocabularyStatsRowVisible(word) || isExcluded(word)) continue;
|
||||
if (all.length < VOCABULARY_CHART_LIMIT) all.push(word);
|
||||
if (word.pos2 !== '固有名詞' && withoutNames.length < VOCABULARY_CHART_LIMIT) {
|
||||
withoutNames.push(word);
|
||||
}
|
||||
}
|
||||
offset += page.length;
|
||||
}
|
||||
|
||||
return { all, withoutNames };
|
||||
}
|
||||
|
||||
function excludedVocabularyAliases(
|
||||
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
|
||||
): string[] {
|
||||
const aliases = [word.headword?.trim() ?? '', word.word?.trim() ?? ''].filter(Boolean);
|
||||
if (aliases.length === 0) aliases.push(word.reading?.trim() ?? '');
|
||||
return [...new Set(aliases)];
|
||||
}
|
||||
|
||||
function vocabularyDisplayHeadword(
|
||||
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
|
||||
): string {
|
||||
return word.headword?.trim() || word.word?.trim() || word.reading?.trim() || '';
|
||||
}
|
||||
|
||||
function timestampSeconds(timestamp: number): number {
|
||||
return timestamp < 10_000_000_000 ? timestamp : Math.floor(timestamp / 1000);
|
||||
}
|
||||
|
||||
export function getVocabularySummary(
|
||||
db: DatabaseSync,
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
nowMs: number = Date.now(),
|
||||
scanBatchSize: number = VOCABULARY_SUMMARY_SCAN_BATCH_SIZE,
|
||||
): VocabularyStatsSummary {
|
||||
// Visibility and exclusion rules live in JS, so rows are scanned in id-keyed
|
||||
// batches to keep memory bounded on large vocabularies.
|
||||
const scanStmt = db.prepare(`
|
||||
SELECT id AS wordId, headword, word, reading,
|
||||
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
|
||||
frequency, frequency_rank AS frequencyRank,
|
||||
first_seen AS firstSeen, last_seen AS lastSeen,
|
||||
0 AS animeCount
|
||||
FROM imm_words
|
||||
WHERE id > ?
|
||||
ORDER BY id
|
||||
LIMIT ?
|
||||
`);
|
||||
const excludedAliases = new Set(
|
||||
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
|
||||
);
|
||||
const weekAgoSec = nowMs / 1000 - 7 * 86_400;
|
||||
const summary: VocabularyStatsSummary = {
|
||||
uniqueWords: 0,
|
||||
uniqueWordsWithoutNames: 0,
|
||||
uniqueKanji: (db.prepare('SELECT COUNT(*) AS count FROM imm_kanji').get() as { count: number })
|
||||
.count,
|
||||
newThisWeek: 0,
|
||||
newThisWeekWithoutNames: 0,
|
||||
knownWordCount: knownWords ? 0 : null,
|
||||
knownWordCountWithoutNames: knownWords ? 0 : null,
|
||||
};
|
||||
|
||||
let lastId = Number.MIN_SAFE_INTEGER;
|
||||
for (;;) {
|
||||
const words = scanStmt.all(lastId, scanBatchSize) as VocabularyStatsRow[];
|
||||
if (words.length === 0) break;
|
||||
lastId = words[words.length - 1]!.wordId;
|
||||
for (const word of words) {
|
||||
if (
|
||||
!isVocabularyStatsRowVisible(word) ||
|
||||
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const isName = word.pos2 === '固有名詞';
|
||||
const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec;
|
||||
const isKnown = knownWords?.has(vocabularyDisplayHeadword(word)) ?? false;
|
||||
summary.uniqueWords += 1;
|
||||
if (!isName) summary.uniqueWordsWithoutNames += 1;
|
||||
if (isNewThisWeek) {
|
||||
summary.newThisWeek += 1;
|
||||
if (!isName) summary.newThisWeekWithoutNames += 1;
|
||||
}
|
||||
if (isKnown) {
|
||||
summary.knownWordCount! += 1;
|
||||
if (!isName) summary.knownWordCountWithoutNames! += 1;
|
||||
}
|
||||
}
|
||||
if (words.length < scanBatchSize) break;
|
||||
}
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
export function getStatsExcludedWords(db: DatabaseSync): StatsExcludedWordRow[] {
|
||||
return db
|
||||
.prepare(
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
toDbTimestamp,
|
||||
} from './query-shared';
|
||||
import { getDailyRollups, getMonthlyRollups } from './query-sessions';
|
||||
import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
|
||||
|
||||
type TrendRange = '7d' | '30d' | '90d' | '365d' | 'all';
|
||||
type TrendGroupBy = 'day' | 'month';
|
||||
@@ -660,6 +661,16 @@ function buildNewWordsPerDay(
|
||||
cutoffMs: string | null,
|
||||
axis: number[] | null,
|
||||
): TrendChartPoint[] {
|
||||
if (areLexicalDailyRollupsReady(db)) {
|
||||
// A trend range is defined in calendar buckets, so the rollup includes the
|
||||
// complete local cutoff day rather than applying a time-of-day boundary.
|
||||
const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
|
||||
const rows = getLexicalDailyRollups(db).filter(
|
||||
(row) => cutoffDay === null || row.epochDay >= cutoffDay,
|
||||
);
|
||||
return fillAxisPoints(axis, new Map(rows.map((row) => [row.epochDay, row.wordCount])));
|
||||
}
|
||||
|
||||
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
||||
const prepared = db.prepare(`
|
||||
SELECT
|
||||
@@ -691,6 +702,18 @@ function buildNewWordsPerMonth(
|
||||
cutoffMs: string | null,
|
||||
axis: number[] | null,
|
||||
): TrendChartPoint[] {
|
||||
if (areLexicalDailyRollupsReady(db)) {
|
||||
const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
|
||||
const byMonth = new Map<number, number>();
|
||||
for (const row of getLexicalDailyRollups(db)) {
|
||||
if (cutoffDay !== null && row.epochDay < cutoffDay) continue;
|
||||
const { year, month } = dayPartsFromEpochDay(row.epochDay);
|
||||
const monthKey = year * 100 + month;
|
||||
byMonth.set(monthKey, (byMonth.get(monthKey) ?? 0) + row.wordCount);
|
||||
}
|
||||
return fillAxisPoints(axis, byMonth);
|
||||
}
|
||||
|
||||
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
|
||||
const prepared = db.prepare(`
|
||||
SELECT
|
||||
|
||||
@@ -4,6 +4,7 @@ import { parseMediaInfo } from '../../../jimaku/utils';
|
||||
import { normalizeTitleIdentity } from '../../utils/title-normalization';
|
||||
import type { DatabaseSync } from './sqlite';
|
||||
import { nowMs } from './time';
|
||||
import { ensureLexicalDailyRollupTables, markLexicalDailyRollupsReady } from './lexical-rollups';
|
||||
import { SCHEMA_VERSION } from './types';
|
||||
import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types';
|
||||
import { toDbMs, toDbTimestamp } from './query-shared';
|
||||
@@ -890,11 +891,11 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
VALUES ('last_rollup_sample_ms', 0)
|
||||
ON CONFLICT(state_key) DO NOTHING
|
||||
`);
|
||||
|
||||
const currentVersion = db
|
||||
.prepare('SELECT schema_version FROM imm_schema_version ORDER BY schema_version DESC LIMIT 1')
|
||||
.get() as { schema_version: number } | null;
|
||||
if (currentVersion?.schema_version === SCHEMA_VERSION) {
|
||||
ensureLexicalDailyRollupTables(db);
|
||||
ensureLifetimeSummaryTables(db);
|
||||
ensureStatsExcludedWordsTable(db);
|
||||
ensureAnimeMergeTables(db);
|
||||
@@ -1068,6 +1069,7 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
last_seen REAL,
|
||||
frequency INTEGER,
|
||||
frequency_rank INTEGER,
|
||||
vocabulary_visible INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1)),
|
||||
UNIQUE(headword, word, reading)
|
||||
);
|
||||
`);
|
||||
@@ -1451,8 +1453,18 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
addColumnIfMissing(db, 'imm_sessions', 'ended_media_ms', 'INTEGER');
|
||||
}
|
||||
|
||||
if (currentVersion?.schema_version && currentVersion.schema_version < 23) {
|
||||
addColumnIfMissing(
|
||||
db,
|
||||
'imm_words',
|
||||
'vocabulary_visible',
|
||||
'INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1))',
|
||||
);
|
||||
}
|
||||
|
||||
migrateSessionEventTimestampsToText(db);
|
||||
|
||||
ensureLexicalDailyRollupTables(db);
|
||||
ensureLifetimeSummaryTables(db);
|
||||
ensureStatsExcludedWordsTable(db);
|
||||
|
||||
@@ -1585,6 +1597,12 @@ export function ensureSchema(db: DatabaseSync): void {
|
||||
VALUES (${SCHEMA_VERSION}, ${toDbTimestamp(nowMs())})
|
||||
ON CONFLICT DO NOTHING
|
||||
`);
|
||||
|
||||
// A new database has no history to materialize. Upgrades are populated by the
|
||||
// background worker so startup never scans the existing vocabulary table.
|
||||
if (!currentVersion) {
|
||||
markLexicalDailyRollupsReady(db);
|
||||
}
|
||||
}
|
||||
|
||||
export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPreparedStatements {
|
||||
@@ -1617,9 +1635,10 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
|
||||
`),
|
||||
wordUpsertStmt: db.prepare(`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency, frequency_rank
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen,
|
||||
frequency, frequency_rank, vocabulary_visible
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, 1
|
||||
)
|
||||
ON CONFLICT(headword, word, reading) DO UPDATE SET
|
||||
frequency = COALESCE(frequency, 0) + 1,
|
||||
@@ -1632,6 +1651,7 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
|
||||
pos1 = COALESCE(NULLIF(imm_words.pos1, ''), excluded.pos1),
|
||||
pos2 = COALESCE(NULLIF(imm_words.pos2, ''), excluded.pos2),
|
||||
pos3 = COALESCE(NULLIF(imm_words.pos3, ''), excluded.pos3),
|
||||
vocabulary_visible = 1,
|
||||
first_seen = MIN(COALESCE(first_seen, excluded.first_seen), excluded.first_seen),
|
||||
last_seen = MAX(COALESCE(last_seen, excluded.last_seen), excluded.last_seen),
|
||||
frequency_rank = CASE
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export const SCHEMA_VERSION = 21;
|
||||
export const SCHEMA_VERSION = 23;
|
||||
export const DEFAULT_QUEUE_CAP = 1_000;
|
||||
export const DEFAULT_BATCH_SIZE = 25;
|
||||
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
|
||||
@@ -306,6 +306,16 @@ export interface VocabularyStatsRow {
|
||||
lastSeen: number;
|
||||
}
|
||||
|
||||
export interface VocabularyStatsSummary {
|
||||
uniqueWords: number;
|
||||
uniqueWordsWithoutNames: number;
|
||||
uniqueKanji: number;
|
||||
newThisWeek: number;
|
||||
newThisWeekWithoutNames: number;
|
||||
knownWordCount: number | null;
|
||||
knownWordCountWithoutNames: number | null;
|
||||
}
|
||||
|
||||
export interface StatsExcludedWordRow {
|
||||
headword: string;
|
||||
word: string;
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
resolveVocabularySummaryWorkerPath,
|
||||
VocabularySummaryWorkerRuntime,
|
||||
} from './vocabulary-summary-worker-runtime';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas, ensureSchema } from './storage';
|
||||
|
||||
test('vocabulary summary worker reads the database from a separate connection', async () => {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-vocabulary-summary-worker-'));
|
||||
const dbPath = path.join(tempDir, 'immersion.sqlite');
|
||||
const runtime = new VocabularySummaryWorkerRuntime();
|
||||
const db = new Database(dbPath);
|
||||
|
||||
try {
|
||||
applyPragmas(db);
|
||||
ensureSchema(db);
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO imm_words (
|
||||
headword, word, reading, part_of_speech, pos1, pos2, pos3,
|
||||
first_seen, last_seen, frequency
|
||||
) VALUES ('猫', '猫', 'ねこ', 'noun', '名詞', '一般', '', 1, 1, 1)
|
||||
`,
|
||||
).run();
|
||||
db.close();
|
||||
|
||||
const summary = await runtime.run(dbPath, new Set(['猫']));
|
||||
|
||||
assert.equal(summary.uniqueWords, 1);
|
||||
assert.equal(summary.knownWordCount, 1);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
try {
|
||||
db.close();
|
||||
} catch {
|
||||
// The worker needs the setup connection closed before it starts.
|
||||
}
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary summary worker module resolves in the current layout', () => {
|
||||
const workerPath = resolveVocabularySummaryWorkerPath();
|
||||
assert.ok(workerPath, 'expected the vocabulary summary worker module to resolve');
|
||||
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
|
||||
});
|
||||
|
||||
test('vocabulary summary worker never falls back to the caller thread', async () => {
|
||||
const runtime = new VocabularySummaryWorkerRuntime({
|
||||
resolveWorkerPath: () => null,
|
||||
warn: () => {},
|
||||
});
|
||||
|
||||
try {
|
||||
await assert.rejects(
|
||||
runtime.run('/tmp/subminer-summary-worker-not-used.sqlite', null),
|
||||
/worker unavailable/i,
|
||||
);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('vocabulary summary worker times out when it never responds', async () => {
|
||||
let terminated = false;
|
||||
const runtime = new VocabularySummaryWorkerRuntime({
|
||||
resolveWorkerPath: () => '/tmp/fake-worker.js',
|
||||
createWorker: async () => ({
|
||||
once() {
|
||||
return this;
|
||||
},
|
||||
terminate: async () => {
|
||||
terminated = true;
|
||||
return 0;
|
||||
},
|
||||
}),
|
||||
timeoutMs: 1,
|
||||
warn: () => {},
|
||||
} as never);
|
||||
|
||||
try {
|
||||
const outcome = await Promise.race([
|
||||
runtime.run('/tmp/not-used.sqlite', null).then(
|
||||
() => 'resolved',
|
||||
(error: unknown) => String(error),
|
||||
),
|
||||
new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
|
||||
]);
|
||||
|
||||
assert.match(outcome, /timed out/);
|
||||
assert.equal(terminated, true);
|
||||
} finally {
|
||||
runtime.destroy();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createLogger } from '../../../logger';
|
||||
import type { VocabularyStatsSummary } from './types';
|
||||
|
||||
interface VocabularySummaryWorkerResponse {
|
||||
summary?: VocabularyStatsSummary;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
interface VocabularySummaryWorkerHandle {
|
||||
once(event: 'message', listener: (message: VocabularySummaryWorkerResponse) => void): this;
|
||||
once(event: 'error', listener: (error: Error) => void): this;
|
||||
once(event: 'exit', listener: (code: number) => void): this;
|
||||
terminate(): Promise<number>;
|
||||
}
|
||||
|
||||
interface VocabularySummaryWorkerRuntimeOptions {
|
||||
resolveWorkerPath?: () => string | null;
|
||||
createWorker?: (
|
||||
workerPath: string,
|
||||
workerData: { dbPath: string; knownWords: string[] | null },
|
||||
) => Promise<VocabularySummaryWorkerHandle>;
|
||||
timeoutMs?: number;
|
||||
warn?: (message: string, ...meta: unknown[]) => void;
|
||||
}
|
||||
|
||||
export type RunVocabularySummaryTask = (
|
||||
dbPath: string,
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
) => Promise<VocabularyStatsSummary>;
|
||||
|
||||
export function resolveVocabularySummaryWorkerPath(): string | null {
|
||||
const fileName = __filename.endsWith('.ts')
|
||||
? 'vocabulary-summary-worker-thread.ts'
|
||||
: 'vocabulary-summary-worker-thread.js';
|
||||
const workerPath = path.join(__dirname, fileName);
|
||||
return fs.existsSync(workerPath) ? workerPath : null;
|
||||
}
|
||||
|
||||
const logger = createLogger('main:immersion-tracker:vocabulary-summary-worker');
|
||||
const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
|
||||
|
||||
export class VocabularySummaryWorkerRuntime {
|
||||
private readonly activeWorkers = new Set<VocabularySummaryWorkerHandle>();
|
||||
private destroyed = false;
|
||||
|
||||
constructor(private readonly options: VocabularySummaryWorkerRuntimeOptions = {}) {}
|
||||
|
||||
async run(
|
||||
dbPath: string,
|
||||
knownWords: ReadonlySet<string> | null,
|
||||
): Promise<VocabularyStatsSummary> {
|
||||
if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
|
||||
const workerData = { dbPath, knownWords: knownWords ? [...knownWords] : null };
|
||||
let worker: VocabularySummaryWorkerHandle;
|
||||
try {
|
||||
const workerPath = (this.options.resolveWorkerPath ?? resolveVocabularySummaryWorkerPath)();
|
||||
if (!workerPath) throw new Error('Emitted vocabulary summary worker module was not found');
|
||||
const createWorker =
|
||||
this.options.createWorker ??
|
||||
(async (resolvedPath, data) => {
|
||||
const { Worker } = await import('node:worker_threads');
|
||||
return new Worker(resolvedPath, { workerData: data });
|
||||
});
|
||||
worker = await createWorker(workerPath, workerData);
|
||||
} catch (error) {
|
||||
if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
|
||||
(this.options.warn ?? logger.warn)(
|
||||
'Vocabulary summary worker unavailable; refusing to scan vocabulary on the current thread',
|
||||
error,
|
||||
);
|
||||
throw new Error('Vocabulary summary worker unavailable');
|
||||
}
|
||||
|
||||
if (this.destroyed) {
|
||||
await worker.terminate().catch(() => undefined);
|
||||
throw new Error('Vocabulary summary worker is shut down');
|
||||
}
|
||||
|
||||
return new Promise<VocabularyStatsSummary>((resolve, reject) => {
|
||||
let settled = false;
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null;
|
||||
this.activeWorkers.add(worker);
|
||||
const settle = (result: VocabularyStatsSummary | Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timeout) clearTimeout(timeout);
|
||||
this.activeWorkers.delete(worker);
|
||||
void worker.terminate().catch(() => undefined);
|
||||
if (result instanceof Error) reject(result);
|
||||
else resolve(result);
|
||||
};
|
||||
timeout = setTimeout(
|
||||
() => settle(new Error('Vocabulary summary worker timed out')),
|
||||
this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
worker.once('message', (message) => {
|
||||
if (message.summary) {
|
||||
settle(message.summary);
|
||||
return;
|
||||
}
|
||||
settle(
|
||||
new Error(
|
||||
`Vocabulary summary failed: ${String(message.error ?? 'unknown worker error')}`,
|
||||
),
|
||||
);
|
||||
});
|
||||
worker.once('error', (error) => settle(error));
|
||||
worker.once('exit', (code) => {
|
||||
if (!settled) {
|
||||
settle(
|
||||
new Error(
|
||||
code === 0
|
||||
? 'Vocabulary summary worker exited without a response'
|
||||
: `Vocabulary summary worker exited with code ${code}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
for (const worker of this.activeWorkers) {
|
||||
void worker.terminate().catch(() => undefined);
|
||||
}
|
||||
this.activeWorkers.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
import { executeVocabularySummaryTask } from './vocabulary-summary-worker';
|
||||
|
||||
interface VocabularySummaryWorkerData {
|
||||
dbPath: string;
|
||||
knownWords: string[] | null;
|
||||
}
|
||||
|
||||
if (!parentPort) throw new Error('vocabulary summary worker missing parent port');
|
||||
|
||||
const request = workerData as VocabularySummaryWorkerData;
|
||||
|
||||
try {
|
||||
parentPort.postMessage({
|
||||
summary: executeVocabularySummaryTask(request.dbPath, request.knownWords),
|
||||
});
|
||||
} catch (error) {
|
||||
parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { getVocabularySummary } from './query-lexical';
|
||||
import { Database } from './sqlite';
|
||||
import { applyPragmas } from './storage';
|
||||
import type { VocabularyStatsSummary } from './types';
|
||||
|
||||
export function executeVocabularySummaryTask(
|
||||
dbPath: string,
|
||||
knownWords: string[] | null,
|
||||
): VocabularyStatsSummary {
|
||||
const db = new Database(dbPath);
|
||||
try {
|
||||
applyPragmas(db);
|
||||
return getVocabularySummary(db, knownWords ? new Set(knownWords) : null);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { PartOfSpeech, type MergedToken } from '../../../types';
|
||||
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
|
||||
|
||||
export interface VocabularyVisibilityRow {
|
||||
word: string | null;
|
||||
headword: string | null;
|
||||
reading?: string | null;
|
||||
partOfSpeech?: string | null;
|
||||
pos1?: string | null;
|
||||
pos2?: string | null;
|
||||
pos3?: string | null;
|
||||
frequencyRank?: number | null;
|
||||
}
|
||||
|
||||
function toVocabularyToken(row: VocabularyVisibilityRow): MergedToken {
|
||||
const word = row.word ?? '';
|
||||
const headword = row.headword ?? word;
|
||||
const partOfSpeech =
|
||||
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
|
||||
? (row.partOfSpeech as PartOfSpeech)
|
||||
: PartOfSpeech.other;
|
||||
|
||||
return {
|
||||
surface: word,
|
||||
reading: row.reading ?? '',
|
||||
headword,
|
||||
startPos: 0,
|
||||
endPos: word.length,
|
||||
partOfSpeech,
|
||||
pos1: row.pos1 ?? '',
|
||||
pos2: row.pos2 ?? '',
|
||||
pos3: row.pos3 ?? '',
|
||||
frequencyRank: row.frequencyRank ?? undefined,
|
||||
isMerged: false,
|
||||
isKnown: false,
|
||||
isNPlusOneTarget: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function isVocabularyStatsRowVisible(row: VocabularyVisibilityRow): boolean {
|
||||
if (!(row.word?.trim() || row.headword?.trim())) return false;
|
||||
return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
parseExcludedWordsBody,
|
||||
parseIntQuery,
|
||||
parsePositiveIdList,
|
||||
loadKnownWordsSet,
|
||||
} from './route-support.js';
|
||||
|
||||
export function registerStatsLibraryRoutes(
|
||||
@@ -31,6 +32,17 @@ export function registerStatsLibraryRoutes(
|
||||
return c.json(statsJson('vocabulary', vocab));
|
||||
});
|
||||
|
||||
app.get('/api/stats/vocabulary/summary', async (c) => {
|
||||
const summary = await tracker.getVocabularySummary(
|
||||
loadKnownWordsSet(options?.knownWordCachePath),
|
||||
);
|
||||
return c.json(statsJson('vocabularySummary', summary));
|
||||
});
|
||||
|
||||
app.get('/api/stats/vocabulary/charts', async (c) => {
|
||||
return c.json(statsJson('vocabularyCharts', await tracker.getVocabularyChartData()));
|
||||
});
|
||||
|
||||
app.get('/api/stats/excluded-words', async (c) => {
|
||||
return c.json(statsJson('excludedWords', await tracker.getStatsExcludedWords()));
|
||||
});
|
||||
|
||||
@@ -89,6 +89,7 @@ const WORD_COPY_COLUMNS = [
|
||||
'last_seen',
|
||||
'frequency',
|
||||
'frequency_rank',
|
||||
'vocabulary_visible',
|
||||
] as const;
|
||||
|
||||
export function mergeAnime(
|
||||
|
||||
@@ -50,6 +50,24 @@ export interface StatsKnownWordsSummary {
|
||||
knownWordCount: number;
|
||||
}
|
||||
|
||||
export interface StatsVocabularySummary {
|
||||
uniqueWords: number;
|
||||
uniqueWordsWithoutNames: number;
|
||||
uniqueKanji: number;
|
||||
newThisWeek: number;
|
||||
newThisWeekWithoutNames: number;
|
||||
knownWordCount: number | null;
|
||||
knownWordCountWithoutNames: number | null;
|
||||
}
|
||||
|
||||
export interface StatsVocabularyCharts {
|
||||
ready: boolean;
|
||||
topWords: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||
topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>;
|
||||
newWordsTimeline: Array<{ epochDay: number; wordCount: number }>;
|
||||
newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>;
|
||||
}
|
||||
|
||||
export interface StatsAnilistSearchResult {
|
||||
id: number;
|
||||
episodes: number | null;
|
||||
@@ -164,6 +182,8 @@ export interface StatsJsonResponseMap {
|
||||
sessionEvents: SessionEvent[];
|
||||
sessionKnownWordsTimeline: StatsSessionKnownWordsTimelinePoint[];
|
||||
vocabulary: VocabularyEntry[];
|
||||
vocabularySummary: StatsVocabularySummary;
|
||||
vocabularyCharts: StatsVocabularyCharts;
|
||||
excludedWords: StatsExcludedWord[];
|
||||
setExcludedWords: StatsOkResponse;
|
||||
duplicateLineCleanup: StatsDuplicateLineCleanupResult;
|
||||
@@ -222,6 +242,8 @@ export interface StatsHttpClient {
|
||||
getSessionEvents: (id: number, limit?: number, eventTypes?: number[]) => Promise<SessionEvent[]>;
|
||||
getSessionKnownWordsTimeline: (id: number) => Promise<StatsSessionKnownWordsTimelinePoint[]>;
|
||||
getVocabulary: (limit?: number) => Promise<VocabularyEntry[]>;
|
||||
getVocabularySummary: () => Promise<StatsVocabularySummary>;
|
||||
getVocabularyCharts: () => Promise<StatsVocabularyCharts>;
|
||||
getExcludedWords: () => Promise<StatsExcludedWord[]>;
|
||||
setExcludedWords: (words: StatsExcludedWord[]) => Promise<void>;
|
||||
cleanupDuplicateLines: (
|
||||
|
||||
@@ -6,11 +6,10 @@ import { KanjiBreakdown } from './KanjiBreakdown';
|
||||
import { KanjiDetailPanel } from './KanjiDetailPanel';
|
||||
import { ExclusionManager } from './ExclusionManager';
|
||||
import { DuplicateLineCleanup } from './DuplicateLineCleanup';
|
||||
import { formatNumber } from '../../lib/formatters';
|
||||
import { epochDayToDate, formatNumber } from '../../lib/formatters';
|
||||
import { TrendChart } from '../trends/TrendChart';
|
||||
import { FrequencyRankTable } from './FrequencyRankTable';
|
||||
import { CrossAnimeWordsTable } from './CrossAnimeWordsTable';
|
||||
import { buildVocabularySummary } from '../../lib/dashboard-data';
|
||||
import type { ExcludedWord } from '../../hooks/useExcludedWords';
|
||||
import type { KanjiEntry, VocabularyEntry } from '../../types/stats';
|
||||
|
||||
@@ -35,7 +34,18 @@ export function VocabularyTab({
|
||||
onRemoveExclusion,
|
||||
onClearExclusions,
|
||||
}: VocabularyTabProps) {
|
||||
const { words, kanji, knownWords, loading, error, reload } = useVocabulary();
|
||||
const {
|
||||
words,
|
||||
kanji,
|
||||
knownWords,
|
||||
summary,
|
||||
charts,
|
||||
loading,
|
||||
error,
|
||||
aggregatesError,
|
||||
refreshAggregates,
|
||||
reload,
|
||||
} = useVocabulary();
|
||||
const [selectedKanjiId, setSelectedKanjiId] = useState<number | null>(null);
|
||||
const [hideNames, setHideNames] = useState(false);
|
||||
const [showExclusionManager, setShowExclusionManager] = useState(false);
|
||||
@@ -48,19 +58,26 @@ export function VocabularyTab({
|
||||
if (excluded.length > 0) result = result.filter((w) => !isExcluded(w));
|
||||
return result;
|
||||
}, [words, hideNames, excluded, isExcluded]);
|
||||
const summary = useMemo(
|
||||
() => buildVocabularySummary(filteredWords, kanji),
|
||||
[filteredWords, kanji],
|
||||
const chartData = useMemo(
|
||||
() => ({
|
||||
topWords: ((hideNames ? charts?.topWordsWithoutNames : charts?.topWords) ?? []).map(
|
||||
(word) => ({
|
||||
label: word.headword,
|
||||
value: word.frequency,
|
||||
}),
|
||||
),
|
||||
newWordsTimeline: (
|
||||
(hideNames ? charts?.newWordsTimelineWithoutNames : charts?.newWordsTimeline) ?? []
|
||||
).map((point) => ({
|
||||
label: epochDayToDate(point.epochDay).toLocaleDateString(undefined, {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
}),
|
||||
value: point.wordCount,
|
||||
})),
|
||||
}),
|
||||
[charts, hideNames],
|
||||
);
|
||||
const knownWordCount = useMemo(() => {
|
||||
if (knownWords.size === 0) return 0;
|
||||
|
||||
let count = 0;
|
||||
for (const w of filteredWords) {
|
||||
if (knownWords.has(w.headword)) count += 1;
|
||||
}
|
||||
return count;
|
||||
}, [filteredWords, knownWords]);
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
@@ -82,7 +99,9 @@ export function VocabularyTab({
|
||||
};
|
||||
|
||||
const handleBarClick = (headword: string): void => {
|
||||
const match = filteredWords.find((w) => w.headword === headword);
|
||||
const match = (hideNames ? charts?.topWordsWithoutNames : charts?.topWords)?.find(
|
||||
(word) => word.headword === headword,
|
||||
);
|
||||
if (match) onOpenWordDetail?.(match.wordId);
|
||||
};
|
||||
|
||||
@@ -90,33 +109,60 @@ export function VocabularyTab({
|
||||
setSelectedKanjiId(entry.kanjiId);
|
||||
};
|
||||
|
||||
const displayedSummary = hideNames
|
||||
? {
|
||||
uniqueWords: summary?.uniqueWordsWithoutNames ?? 0,
|
||||
newThisWeek: summary?.newThisWeekWithoutNames ?? 0,
|
||||
knownWordCount: summary?.knownWordCountWithoutNames ?? null,
|
||||
}
|
||||
: {
|
||||
uniqueWords: summary?.uniqueWords ?? 0,
|
||||
newThisWeek: summary?.newThisWeek ?? 0,
|
||||
knownWordCount: summary?.knownWordCount ?? null,
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div className="grid grid-cols-2 xl:grid-cols-4 gap-3">
|
||||
<StatCard
|
||||
label="Unique Words"
|
||||
value={formatNumber(summary.uniqueWords)}
|
||||
value={summary ? formatNumber(displayedSummary.uniqueWords) : '…'}
|
||||
color="text-ctp-blue"
|
||||
/>
|
||||
{knownWords.size > 0 && (
|
||||
{displayedSummary.knownWordCount !== null ? (
|
||||
<StatCard
|
||||
label="Known Words"
|
||||
value={`${formatNumber(knownWordCount)} (${summary.uniqueWords > 0 ? Math.round((knownWordCount / summary.uniqueWords) * 100) : 0}%)`}
|
||||
value={`${formatNumber(displayedSummary.knownWordCount)} (${displayedSummary.uniqueWords > 0 ? Math.round((displayedSummary.knownWordCount / displayedSummary.uniqueWords) * 100) : 0}%)`}
|
||||
color="text-ctp-green"
|
||||
/>
|
||||
)}
|
||||
) : knownWords.size > 0 ? (
|
||||
<StatCard label="Known Words" value="…" color="text-ctp-green" />
|
||||
) : null}
|
||||
<StatCard
|
||||
label="Unique Kanji"
|
||||
value={formatNumber(summary.uniqueKanji)}
|
||||
value={summary ? formatNumber(summary.uniqueKanji) : '…'}
|
||||
color="text-ctp-teal"
|
||||
/>
|
||||
<StatCard
|
||||
label="New This Week"
|
||||
value={`+${formatNumber(summary.newThisWeek)}`}
|
||||
value={summary ? `+${formatNumber(displayedSummary.newThisWeek)}` : '…'}
|
||||
color="text-ctp-mauve"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{aggregatesError && (
|
||||
<p className="text-xs text-ctp-red" role="alert">
|
||||
{aggregatesError}{' '}
|
||||
<button
|
||||
type="button"
|
||||
onClick={refreshAggregates}
|
||||
className="underline hover:text-ctp-text"
|
||||
>
|
||||
Retry
|
||||
</button>
|
||||
</p>
|
||||
)}
|
||||
|
||||
<div className="flex items-center justify-end gap-3">
|
||||
{hasNames && (
|
||||
<button
|
||||
@@ -154,19 +200,25 @@ export function VocabularyTab({
|
||||
<div className="grid grid-cols-1 xl:grid-cols-2 gap-4">
|
||||
<TrendChart
|
||||
title="Top Repeated Words"
|
||||
data={summary.topWords}
|
||||
data={chartData.topWords}
|
||||
color="#8aadf4"
|
||||
type="bar"
|
||||
onBarClick={handleBarClick}
|
||||
/>
|
||||
<TrendChart
|
||||
title="New Words by Day"
|
||||
data={summary.newWordsTimeline}
|
||||
data={chartData.newWordsTimeline}
|
||||
color="#c6a0f6"
|
||||
type="line"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{charts && !charts.ready && (
|
||||
<p className="text-xs text-ctp-overlay1" role="status">
|
||||
Building vocabulary history in the background…
|
||||
</p>
|
||||
)}
|
||||
|
||||
<FrequencyRankTable
|
||||
words={filteredWords}
|
||||
knownWords={knownWords}
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
initializeExcludedWordsStore,
|
||||
resetExcludedWordsStoreForTests,
|
||||
setExcludedWords,
|
||||
subscribeExcludedWordsServerSync,
|
||||
} from './useExcludedWords';
|
||||
import { BASE_URL } from '../lib/api-client';
|
||||
|
||||
@@ -199,3 +200,91 @@ test('initializeExcludedWordsStore retries after transient database load failure
|
||||
resetExcludedWordsStoreForTests();
|
||||
}
|
||||
});
|
||||
|
||||
test('a failing server-sync listener neither rolls back the write nor blocks other listeners', async () => {
|
||||
resetExcludedWordsStoreForTests();
|
||||
const { values: storage, restore } = installLocalStorage();
|
||||
const originalFetch = globalThis.fetch;
|
||||
const originalConsoleError = console.error;
|
||||
console.error = () => {};
|
||||
globalThis.fetch = (async () =>
|
||||
new Response(JSON.stringify({ ok: true }), { status: 200 })) as typeof globalThis.fetch;
|
||||
const notified: string[] = [];
|
||||
const unsubscribeFirst = subscribeExcludedWordsServerSync(() => {
|
||||
notified.push('first');
|
||||
throw new Error('listener exploded');
|
||||
});
|
||||
const unsubscribeSecond = subscribeExcludedWordsServerSync(() => {
|
||||
notified.push('second');
|
||||
});
|
||||
|
||||
try {
|
||||
const rows = [{ headword: 'する', word: 'する', reading: 'する' }];
|
||||
await assert.doesNotReject(() => setExcludedWords(rows));
|
||||
|
||||
assert.deepEqual(notified, ['first', 'second']);
|
||||
assert.deepEqual(getExcludedWordsSnapshot(), rows);
|
||||
assert.equal(storage.get(STORAGE_KEY), JSON.stringify(rows));
|
||||
} finally {
|
||||
unsubscribeFirst();
|
||||
unsubscribeSecond();
|
||||
globalThis.fetch = originalFetch;
|
||||
console.error = originalConsoleError;
|
||||
restore();
|
||||
resetExcludedWordsStoreForTests();
|
||||
}
|
||||
});
|
||||
|
||||
test('overlapping writes serialize so an older list cannot overwrite a newer edit', async () => {
|
||||
resetExcludedWordsStoreForTests();
|
||||
const { restore } = installLocalStorage();
|
||||
const originalFetch = globalThis.fetch;
|
||||
const sentBodies: string[] = [];
|
||||
let releaseFirst: (() => void) | null = null;
|
||||
globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
sentBodies.push(String(init?.body ?? ''));
|
||||
if (sentBodies.length === 1) {
|
||||
await new Promise<void>((resolve) => {
|
||||
releaseFirst = resolve;
|
||||
});
|
||||
}
|
||||
return new Response(JSON.stringify({ ok: true }), { status: 200 });
|
||||
}) as typeof globalThis.fetch;
|
||||
const syncs: string[] = [];
|
||||
const unsubscribe = subscribeExcludedWordsServerSync(() => {
|
||||
syncs.push(JSON.stringify(getExcludedWordsSnapshot()));
|
||||
});
|
||||
|
||||
try {
|
||||
const first = [{ headword: '猫', word: '猫', reading: 'ねこ' }];
|
||||
const second = [...first, { headword: '犬', word: '犬', reading: 'いぬ' }];
|
||||
const third = [...second, { headword: '鳥', word: '鳥', reading: 'とり' }];
|
||||
|
||||
// The first write reaches the network before the later edits are made.
|
||||
const firstWrite = setExcludedWords(first);
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
const secondWrite = setExcludedWords(second);
|
||||
const thirdWrite = setExcludedWords(third);
|
||||
|
||||
const release = releaseFirst as (() => void) | null;
|
||||
assert.ok(release, 'expected the first write to be in flight');
|
||||
release();
|
||||
await Promise.all([firstWrite, secondWrite, thirdWrite]);
|
||||
|
||||
// The in-flight write finishes first, the superseded middle write is
|
||||
// dropped, and the newest list is the last thing the server is told.
|
||||
assert.deepEqual(sentBodies, [
|
||||
JSON.stringify({ words: first }),
|
||||
JSON.stringify({ words: third }),
|
||||
]);
|
||||
assert.deepEqual(getExcludedWordsSnapshot(), third);
|
||||
// Only the final revision notifies: the first write's acknowledgement was
|
||||
// already obsolete, so it must not trigger an aggregate recomputation.
|
||||
assert.deepEqual(syncs, [JSON.stringify(third)]);
|
||||
} finally {
|
||||
unsubscribe();
|
||||
globalThis.fetch = originalFetch;
|
||||
restore();
|
||||
resetExcludedWordsStoreForTests();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -44,6 +44,32 @@ let cachedKeys: Set<string> | null = null;
|
||||
let initialized: Promise<void> | null = null;
|
||||
let revision = 0;
|
||||
const listeners = new Set<() => void>();
|
||||
// Fires only after the stats server acknowledged an exclusion write, so
|
||||
// subscribers can refetch server-computed aggregates without racing the POST.
|
||||
const serverSyncListeners = new Set<() => void>();
|
||||
|
||||
export function subscribeExcludedWordsServerSync(fn: () => void): () => void {
|
||||
serverSyncListeners.add(fn);
|
||||
return () => {
|
||||
serverSyncListeners.delete(fn);
|
||||
};
|
||||
}
|
||||
|
||||
function notifyServerSync(): void {
|
||||
// Listener failures are their own concern: one must not roll back a write
|
||||
// that already succeeded, nor stop the remaining listeners from running.
|
||||
for (const fn of serverSyncListeners) {
|
||||
try {
|
||||
fn();
|
||||
} catch (error) {
|
||||
console.error('Excluded words server-sync listener failed', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Full-list writes are serialized so a slow earlier request cannot land after a
|
||||
// newer one and overwrite it with a stale list.
|
||||
let writeChain: Promise<void> = Promise.resolve();
|
||||
|
||||
function readLocalStorage(): ExcludedWord[] {
|
||||
if (typeof localStorage === 'undefined') return [];
|
||||
@@ -102,16 +128,27 @@ export async function setExcludedWords(words: ExcludedWord[]): Promise<void> {
|
||||
const normalized = dedupeExcludedWords(words);
|
||||
revision = writeRevision;
|
||||
applyWords(normalized);
|
||||
try {
|
||||
await apiClient.setExcludedWords(normalized);
|
||||
} catch (error) {
|
||||
if (revision === writeRevision) {
|
||||
revision = previousRevision;
|
||||
applyWords(previousWords);
|
||||
const write = writeChain.then(async () => {
|
||||
// A newer edit already superseded this list and carries the newest state,
|
||||
// so sending this one would push a stale list to the server.
|
||||
if (revision !== writeRevision) return;
|
||||
try {
|
||||
await apiClient.setExcludedWords(normalized);
|
||||
} catch (error) {
|
||||
if (revision === writeRevision) {
|
||||
revision = previousRevision;
|
||||
applyWords(previousWords);
|
||||
}
|
||||
console.error('Failed to persist excluded words to stats database', error);
|
||||
throw error;
|
||||
}
|
||||
console.error('Failed to persist excluded words to stats database', error);
|
||||
throw error;
|
||||
}
|
||||
// A newer edit arrived while this write was in flight, so the server state
|
||||
// this acknowledges is already obsolete. Its own acknowledgement notifies
|
||||
// with the newest list; skipping here avoids a wasted aggregate scan.
|
||||
if (revision === writeRevision) notifyServerSync();
|
||||
});
|
||||
writeChain = write.catch(() => {});
|
||||
return write;
|
||||
}
|
||||
|
||||
export function initializeExcludedWordsStore(): Promise<void> {
|
||||
@@ -155,6 +192,8 @@ export function resetExcludedWordsStoreForTests(): void {
|
||||
initialized = null;
|
||||
revision = 0;
|
||||
listeners.clear();
|
||||
serverSyncListeners.clear();
|
||||
writeChain = Promise.resolve();
|
||||
}
|
||||
|
||||
function subscribe(fn: () => void): () => void {
|
||||
|
||||
@@ -0,0 +1,447 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { Window } from 'happy-dom';
|
||||
import { act } from 'react';
|
||||
import { createRoot, type Root } from 'react-dom/client';
|
||||
import { apiClient } from '../lib/api-client';
|
||||
import { resetExcludedWordsStoreForTests, setExcludedWords } from './useExcludedWords';
|
||||
import { useVocabulary } from './useVocabulary';
|
||||
import type { StatsVocabularyCharts, StatsVocabularySummary } from '../types/stats';
|
||||
|
||||
type VocabularyState = ReturnType<typeof useVocabulary>;
|
||||
|
||||
function installDom(): () => void {
|
||||
const previousWindow = Object.getOwnPropertyDescriptor(globalThis, 'window');
|
||||
const previousDocument = Object.getOwnPropertyDescriptor(globalThis, 'document');
|
||||
const previousHTMLElement = Object.getOwnPropertyDescriptor(globalThis, 'HTMLElement');
|
||||
const previousIsReactActEnvironment = Object.getOwnPropertyDescriptor(
|
||||
globalThis,
|
||||
'IS_REACT_ACT_ENVIRONMENT',
|
||||
);
|
||||
const window = new Window();
|
||||
|
||||
Object.defineProperty(globalThis, 'window', { value: window, configurable: true });
|
||||
Object.defineProperty(globalThis, 'document', { value: window.document, configurable: true });
|
||||
Object.defineProperty(globalThis, 'HTMLElement', {
|
||||
value: window.HTMLElement,
|
||||
configurable: true,
|
||||
});
|
||||
Object.defineProperty(globalThis, 'IS_REACT_ACT_ENVIRONMENT', {
|
||||
value: true,
|
||||
configurable: true,
|
||||
writable: true,
|
||||
});
|
||||
|
||||
return () => {
|
||||
const restoreProperty = (name: string, descriptor: PropertyDescriptor | undefined) => {
|
||||
if (descriptor) Object.defineProperty(globalThis, name, descriptor);
|
||||
else Reflect.deleteProperty(globalThis, name);
|
||||
};
|
||||
restoreProperty('window', previousWindow);
|
||||
restoreProperty('document', previousDocument);
|
||||
restoreProperty('HTMLElement', previousHTMLElement);
|
||||
restoreProperty('IS_REACT_ACT_ENVIRONMENT', previousIsReactActEnvironment);
|
||||
};
|
||||
}
|
||||
|
||||
test('DOM harness restores the original global property descriptors', () => {
|
||||
const propertyNames = ['window', 'document', 'HTMLElement', 'IS_REACT_ACT_ENVIRONMENT'] as const;
|
||||
const before = propertyNames.map((name) => Object.getOwnPropertyDescriptor(globalThis, name));
|
||||
|
||||
const restore = installDom();
|
||||
restore();
|
||||
|
||||
const after = propertyNames.map((name) => Object.getOwnPropertyDescriptor(globalThis, name));
|
||||
assert.deepEqual(after, before);
|
||||
});
|
||||
|
||||
function installLocalStorage(): () => void {
|
||||
const previous = Object.getOwnPropertyDescriptor(globalThis, 'localStorage');
|
||||
const values = new Map<string, string>();
|
||||
Object.defineProperty(globalThis, 'localStorage', {
|
||||
configurable: true,
|
||||
value: {
|
||||
getItem: (key: string) => values.get(key) ?? null,
|
||||
setItem: (key: string, value: string) => values.set(key, value),
|
||||
removeItem: (key: string) => values.delete(key),
|
||||
},
|
||||
});
|
||||
return () => {
|
||||
if (previous) Object.defineProperty(globalThis, 'localStorage', previous);
|
||||
else delete (globalThis as { localStorage?: unknown }).localStorage;
|
||||
};
|
||||
}
|
||||
|
||||
interface FakeClock {
|
||||
tick: (ms: number) => void;
|
||||
restore: () => void;
|
||||
}
|
||||
|
||||
/** Bun's `node:test` shim has no `mock.timers`, so the retry clock is faked here. */
|
||||
function installFakeTimers(): FakeClock {
|
||||
const originalSetTimeout = globalThis.setTimeout;
|
||||
const originalClearTimeout = globalThis.clearTimeout;
|
||||
const timers = new Map<number, { at: number; fn: () => void }>();
|
||||
let now = 0;
|
||||
let nextId = 1;
|
||||
|
||||
globalThis.setTimeout = ((fn: () => void, delay = 0) => {
|
||||
const id = nextId;
|
||||
nextId += 1;
|
||||
timers.set(id, { at: now + delay, fn });
|
||||
return id;
|
||||
}) as unknown as typeof globalThis.setTimeout;
|
||||
globalThis.clearTimeout = ((id: number) => {
|
||||
timers.delete(id);
|
||||
}) as unknown as typeof globalThis.clearTimeout;
|
||||
|
||||
return {
|
||||
tick: (ms: number) => {
|
||||
now += ms;
|
||||
const due = [...timers.entries()]
|
||||
.filter(([, timer]) => timer.at <= now)
|
||||
.sort(([, a], [, b]) => a.at - b.at);
|
||||
for (const [id, timer] of due) {
|
||||
timers.delete(id);
|
||||
timer.fn();
|
||||
}
|
||||
},
|
||||
restore: () => {
|
||||
globalThis.setTimeout = originalSetTimeout;
|
||||
globalThis.clearTimeout = originalClearTimeout;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function summaryFixture(): StatsVocabularySummary {
|
||||
return {
|
||||
uniqueWords: 42,
|
||||
uniqueWordsWithoutNames: 40,
|
||||
uniqueKanji: 7,
|
||||
newThisWeek: 3,
|
||||
newThisWeekWithoutNames: 2,
|
||||
knownWordCount: 10,
|
||||
knownWordCountWithoutNames: 9,
|
||||
};
|
||||
}
|
||||
|
||||
function chartsFixture(overrides: Partial<StatsVocabularyCharts> = {}): StatsVocabularyCharts {
|
||||
return {
|
||||
ready: true,
|
||||
topWords: [{ wordId: 1, headword: '猫', frequency: 5 }],
|
||||
topWordsWithoutNames: [{ wordId: 1, headword: '猫', frequency: 5 }],
|
||||
newWordsTimeline: [{ epochDay: 20_000, wordCount: 4 }],
|
||||
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 4 }],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
state: () => VocabularyState;
|
||||
flush: () => Promise<void>;
|
||||
tick: (ms: number) => Promise<void>;
|
||||
unmount: () => Promise<void>;
|
||||
teardown: () => Promise<void>;
|
||||
}
|
||||
|
||||
async function mountHook(): Promise<Harness> {
|
||||
const uninstallDom = installDom();
|
||||
const uninstallLocalStorage = installLocalStorage();
|
||||
const clock = installFakeTimers();
|
||||
|
||||
let latest: VocabularyState | null = null;
|
||||
function Probe() {
|
||||
latest = useVocabulary();
|
||||
return null;
|
||||
}
|
||||
|
||||
const container = document.createElement('div');
|
||||
document.body.append(container);
|
||||
let root: Root | null = createRoot(container);
|
||||
await act(async () => {
|
||||
root!.render(<Probe />);
|
||||
});
|
||||
|
||||
const flush = async (): Promise<void> => {
|
||||
// Drain promise callbacks without advancing the mocked clock.
|
||||
await act(async () => {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
state: () => {
|
||||
assert.ok(latest, 'expected the hook to have rendered');
|
||||
return latest;
|
||||
},
|
||||
flush,
|
||||
tick: async (ms: number) => {
|
||||
await act(async () => {
|
||||
clock.tick(ms);
|
||||
});
|
||||
await flush();
|
||||
},
|
||||
unmount: async () => {
|
||||
await act(async () => {
|
||||
root?.unmount();
|
||||
root = null;
|
||||
});
|
||||
},
|
||||
teardown: async () => {
|
||||
await act(async () => {
|
||||
root?.unmount();
|
||||
root = null;
|
||||
});
|
||||
clock.restore();
|
||||
// React's scheduler can still have deferred work queued; let it drain on
|
||||
// a real timer while the DOM globals it reads are still installed.
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
uninstallLocalStorage();
|
||||
uninstallDom();
|
||||
resetExcludedWordsStoreForTests();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function stubVocabularyClient(overrides: {
|
||||
getVocabularySummary: () => Promise<StatsVocabularySummary>;
|
||||
getVocabularyCharts: () => Promise<StatsVocabularyCharts>;
|
||||
}): () => void {
|
||||
const original = {
|
||||
getVocabulary: apiClient.getVocabulary,
|
||||
getKanji: apiClient.getKanji,
|
||||
getKnownWords: apiClient.getKnownWords,
|
||||
getVocabularySummary: apiClient.getVocabularySummary,
|
||||
getVocabularyCharts: apiClient.getVocabularyCharts,
|
||||
setExcludedWords: apiClient.setExcludedWords,
|
||||
};
|
||||
apiClient.getVocabulary = async () => [];
|
||||
apiClient.getKanji = async () => [];
|
||||
apiClient.getKnownWords = async () => [];
|
||||
apiClient.setExcludedWords = async () => {};
|
||||
apiClient.getVocabularySummary = overrides.getVocabularySummary;
|
||||
apiClient.getVocabularyCharts = overrides.getVocabularyCharts;
|
||||
return () => Object.assign(apiClient, original);
|
||||
}
|
||||
|
||||
test('aggregate failures retry with backoff, then surface an error that Retry clears', async () => {
|
||||
const originalConsoleError = console.error;
|
||||
console.error = () => {};
|
||||
let summaryCalls = 0;
|
||||
let failSummary = true;
|
||||
const restoreClient = stubVocabularyClient({
|
||||
getVocabularySummary: async () => {
|
||||
summaryCalls += 1;
|
||||
if (failSummary) throw new Error('summary unavailable');
|
||||
return summaryFixture();
|
||||
},
|
||||
getVocabularyCharts: async () => chartsFixture(),
|
||||
});
|
||||
const harness = await mountHook();
|
||||
|
||||
try {
|
||||
await harness.flush();
|
||||
assert.equal(summaryCalls, 1);
|
||||
assert.equal(harness.state().aggregatesError, null, 'no error until retries are exhausted');
|
||||
|
||||
// Backoff is 1s, 2s, 4s, 8s across the remaining four attempts.
|
||||
for (const delayMs of [1_000, 2_000, 4_000, 8_000]) {
|
||||
await harness.tick(delayMs);
|
||||
}
|
||||
assert.equal(summaryCalls, 5, 'retries are bounded at the attempt limit');
|
||||
assert.match(harness.state().aggregatesError ?? '', /totals failed to load/i);
|
||||
|
||||
// Nothing further is scheduled once the limit is reached.
|
||||
await harness.tick(60_000);
|
||||
assert.equal(summaryCalls, 5);
|
||||
|
||||
failSummary = false;
|
||||
await act(async () => {
|
||||
harness.state().refreshAggregates();
|
||||
});
|
||||
await harness.flush();
|
||||
|
||||
assert.equal(summaryCalls, 6);
|
||||
assert.equal(harness.state().aggregatesError, null);
|
||||
assert.deepEqual(harness.state().summary, summaryFixture());
|
||||
} finally {
|
||||
await harness.teardown();
|
||||
restoreClient();
|
||||
console.error = originalConsoleError;
|
||||
}
|
||||
});
|
||||
|
||||
test('charts poll while the backfill is pending and stop once it is ready', async () => {
|
||||
let chartCalls = 0;
|
||||
const restoreClient = stubVocabularyClient({
|
||||
getVocabularySummary: async () => summaryFixture(),
|
||||
getVocabularyCharts: async () => {
|
||||
chartCalls += 1;
|
||||
return chartsFixture({ ready: chartCalls >= 3 });
|
||||
},
|
||||
});
|
||||
const harness = await mountHook();
|
||||
|
||||
try {
|
||||
await harness.flush();
|
||||
assert.equal(chartCalls, 1);
|
||||
assert.equal(harness.state().charts?.ready, false);
|
||||
|
||||
await harness.tick(1_000);
|
||||
assert.equal(chartCalls, 2);
|
||||
await harness.tick(1_000);
|
||||
assert.equal(chartCalls, 3);
|
||||
assert.equal(harness.state().charts?.ready, true);
|
||||
|
||||
// A ready result ends the poll.
|
||||
await harness.tick(60_000);
|
||||
assert.equal(chartCalls, 3);
|
||||
} finally {
|
||||
await harness.teardown();
|
||||
restoreClient();
|
||||
}
|
||||
});
|
||||
|
||||
test('chart backfill polling stops and surfaces Retry when readiness never arrives', async () => {
|
||||
let chartCalls = 0;
|
||||
const restoreClient = stubVocabularyClient({
|
||||
getVocabularySummary: async () => summaryFixture(),
|
||||
getVocabularyCharts: async () => {
|
||||
chartCalls += 1;
|
||||
return chartsFixture({ ready: false });
|
||||
},
|
||||
});
|
||||
const harness = await mountHook();
|
||||
|
||||
try {
|
||||
await harness.flush();
|
||||
for (let poll = 0; poll < 65; poll += 1) await harness.tick(5_000);
|
||||
|
||||
assert.equal(chartCalls, 60, 'a failed backfill must not poll for the lifetime of the tab');
|
||||
assert.match(harness.state().aggregatesError ?? '', /still building/i);
|
||||
|
||||
await act(async () => {
|
||||
harness.state().refreshAggregates();
|
||||
});
|
||||
await harness.flush();
|
||||
assert.equal(chartCalls, 61, 'Retry starts one fresh bounded polling cycle');
|
||||
assert.equal(harness.state().aggregatesError, null);
|
||||
} finally {
|
||||
await harness.teardown();
|
||||
restoreClient();
|
||||
}
|
||||
});
|
||||
|
||||
test('aggregates refetch after an exclusion edit is acknowledged by the server', async () => {
|
||||
let summaryCalls = 0;
|
||||
let chartCalls = 0;
|
||||
const restoreClient = stubVocabularyClient({
|
||||
getVocabularySummary: async () => {
|
||||
summaryCalls += 1;
|
||||
return summaryFixture();
|
||||
},
|
||||
getVocabularyCharts: async () => {
|
||||
chartCalls += 1;
|
||||
return chartsFixture();
|
||||
},
|
||||
});
|
||||
const harness = await mountHook();
|
||||
|
||||
try {
|
||||
await harness.flush();
|
||||
assert.equal(summaryCalls, 1);
|
||||
assert.equal(chartCalls, 1);
|
||||
|
||||
await act(async () => {
|
||||
await setExcludedWords([{ headword: '猫', word: '猫', reading: 'ねこ' }]);
|
||||
});
|
||||
await harness.flush();
|
||||
|
||||
assert.equal(summaryCalls, 2, 'totals must not keep counting the excluded word');
|
||||
assert.equal(chartCalls, 2);
|
||||
} finally {
|
||||
await harness.teardown();
|
||||
restoreClient();
|
||||
}
|
||||
});
|
||||
|
||||
test('pending retries are cancelled when the tab unmounts', async () => {
|
||||
const originalConsoleError = console.error;
|
||||
console.error = () => {};
|
||||
let summaryCalls = 0;
|
||||
const restoreClient = stubVocabularyClient({
|
||||
getVocabularySummary: async () => {
|
||||
summaryCalls += 1;
|
||||
throw new Error('summary unavailable');
|
||||
},
|
||||
getVocabularyCharts: async () => chartsFixture(),
|
||||
});
|
||||
const harness = await mountHook();
|
||||
|
||||
try {
|
||||
await harness.flush();
|
||||
assert.equal(summaryCalls, 1);
|
||||
|
||||
await harness.unmount();
|
||||
await harness.tick(60_000);
|
||||
|
||||
assert.equal(summaryCalls, 1, 'no retry may run after unmount');
|
||||
} finally {
|
||||
await harness.teardown();
|
||||
restoreClient();
|
||||
console.error = originalConsoleError;
|
||||
}
|
||||
});
|
||||
|
||||
test('a slow response from a superseded refresh cannot replace the newest aggregates', async () => {
|
||||
let summaryCalls = 0;
|
||||
let releaseSuperseded: (() => void) | null = null;
|
||||
const restoreClient = stubVocabularyClient({
|
||||
getVocabularySummary: async () => {
|
||||
summaryCalls += 1;
|
||||
const call = summaryCalls;
|
||||
// The second call is the one that gets superseded while still in flight.
|
||||
if (call === 2) {
|
||||
await new Promise<void>((resolve) => {
|
||||
releaseSuperseded = resolve;
|
||||
});
|
||||
}
|
||||
return { ...summaryFixture(), uniqueWords: call };
|
||||
},
|
||||
getVocabularyCharts: async () => chartsFixture(),
|
||||
});
|
||||
const harness = await mountHook();
|
||||
|
||||
try {
|
||||
await harness.flush();
|
||||
assert.equal(harness.state().summary?.uniqueWords, 1);
|
||||
|
||||
// First refresh stalls, then a second refresh supersedes it and resolves.
|
||||
await act(async () => {
|
||||
harness.state().refreshAggregates();
|
||||
});
|
||||
await harness.flush();
|
||||
await act(async () => {
|
||||
harness.state().refreshAggregates();
|
||||
});
|
||||
await harness.flush();
|
||||
|
||||
assert.equal(summaryCalls, 3);
|
||||
assert.equal(harness.state().summary?.uniqueWords, 3);
|
||||
|
||||
const release = releaseSuperseded as (() => void) | null;
|
||||
assert.ok(release, 'expected the superseded request to still be in flight');
|
||||
release();
|
||||
await harness.flush();
|
||||
|
||||
assert.equal(
|
||||
harness.state().summary?.uniqueWords,
|
||||
3,
|
||||
'the superseded response must not overwrite the newest totals',
|
||||
);
|
||||
} finally {
|
||||
await harness.teardown();
|
||||
restoreClient();
|
||||
}
|
||||
});
|
||||
@@ -1,16 +1,46 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import { getStatsClient } from './useStatsApi';
|
||||
import type { VocabularyEntry, KanjiEntry } from '../types/stats';
|
||||
import { subscribeExcludedWordsServerSync } from './useExcludedWords';
|
||||
import type {
|
||||
VocabularyEntry,
|
||||
KanjiEntry,
|
||||
StatsVocabularyCharts,
|
||||
StatsVocabularySummary,
|
||||
} from '../types/stats';
|
||||
|
||||
const AGGREGATE_RETRY_BASE_MS = 1_000;
|
||||
const AGGREGATE_RETRY_MAX_MS = 30_000;
|
||||
const AGGREGATE_RETRY_LIMIT = 5;
|
||||
const CHART_BACKFILL_POLL_MS = 1_000;
|
||||
const CHART_BACKFILL_SLOW_POLL_MS = 5_000;
|
||||
const CHART_BACKFILL_FAST_POLLS = 30;
|
||||
const CHART_BACKFILL_POLL_LIMIT = 60;
|
||||
|
||||
function aggregateRetryDelayMs(attempt: number): number {
|
||||
return Math.min(AGGREGATE_RETRY_BASE_MS * 2 ** attempt, AGGREGATE_RETRY_MAX_MS);
|
||||
}
|
||||
|
||||
export function useVocabulary() {
|
||||
const [words, setWords] = useState<VocabularyEntry[]>([]);
|
||||
const [kanji, setKanji] = useState<KanjiEntry[]>([]);
|
||||
const [knownWords, setKnownWords] = useState<Set<string>>(new Set());
|
||||
const [summary, setSummary] = useState<StatsVocabularySummary | null>(null);
|
||||
const [charts, setCharts] = useState<StatsVocabularyCharts | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [aggregatesError, setAggregatesError] = useState<string | null>(null);
|
||||
// Bumped by `reload` after maintenance rewrites the vocabulary tables.
|
||||
const [reloadToken, setReloadToken] = useState(0);
|
||||
const reload = useCallback(() => setReloadToken((token) => token + 1), []);
|
||||
// Bumped independently when only the server-computed summary/charts are
|
||||
// stale, e.g. after the exclusion list changes on the server.
|
||||
const [aggregatesToken, setAggregatesToken] = useState(0);
|
||||
const refreshAggregates = useCallback(() => setAggregatesToken((token) => token + 1), []);
|
||||
const reload = useCallback(() => {
|
||||
setReloadToken((token) => token + 1);
|
||||
setAggregatesToken((token) => token + 1);
|
||||
}, []);
|
||||
|
||||
useEffect(() => subscribeExcludedWordsServerSync(refreshAggregates), [refreshAggregates]);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
@@ -51,5 +81,88 @@ export function useVocabulary() {
|
||||
};
|
||||
}, [reloadToken]);
|
||||
|
||||
return { words, kanji, knownWords, loading, error, reload };
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
setAggregatesError(null);
|
||||
const client = getStatsClient();
|
||||
const timers = new Set<ReturnType<typeof setTimeout>>();
|
||||
const schedule = (fn: () => void, delayMs: number): void => {
|
||||
const timer = setTimeout(() => {
|
||||
timers.delete(timer);
|
||||
fn();
|
||||
}, delayMs);
|
||||
timers.add(timer);
|
||||
};
|
||||
|
||||
const loadSummary = (attempt: number): void => {
|
||||
void client
|
||||
.getVocabularySummary()
|
||||
.then((nextSummary) => {
|
||||
if (!cancelled) setSummary(nextSummary);
|
||||
})
|
||||
.catch((summaryError: unknown) => {
|
||||
console.error('Failed to load vocabulary summary', summaryError);
|
||||
if (cancelled) return;
|
||||
if (attempt + 1 < AGGREGATE_RETRY_LIMIT) {
|
||||
schedule(() => loadSummary(attempt + 1), aggregateRetryDelayMs(attempt));
|
||||
} else {
|
||||
setAggregatesError((previous) => previous ?? 'Vocabulary totals failed to load.');
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const loadCharts = (attempt: number, readyPolls: number): void => {
|
||||
void client
|
||||
.getVocabularyCharts()
|
||||
.then((nextCharts) => {
|
||||
if (cancelled) return;
|
||||
setCharts(nextCharts);
|
||||
if (!nextCharts.ready) {
|
||||
const completedPolls = readyPolls + 1;
|
||||
if (completedPolls < CHART_BACKFILL_POLL_LIMIT) {
|
||||
schedule(
|
||||
() => loadCharts(0, completedPolls),
|
||||
completedPolls < CHART_BACKFILL_FAST_POLLS
|
||||
? CHART_BACKFILL_POLL_MS
|
||||
: CHART_BACKFILL_SLOW_POLL_MS,
|
||||
);
|
||||
} else {
|
||||
setAggregatesError(
|
||||
(previous) =>
|
||||
previous ?? 'Vocabulary charts are still building. Retry to check again.',
|
||||
);
|
||||
}
|
||||
}
|
||||
})
|
||||
.catch((chartError: unknown) => {
|
||||
console.error('Failed to load vocabulary charts', chartError);
|
||||
if (cancelled) return;
|
||||
if (attempt + 1 < AGGREGATE_RETRY_LIMIT) {
|
||||
schedule(() => loadCharts(attempt + 1, readyPolls), aggregateRetryDelayMs(attempt));
|
||||
} else {
|
||||
setAggregatesError((previous) => previous ?? 'Vocabulary charts failed to load.');
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
loadSummary(0);
|
||||
loadCharts(0, 0);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
for (const timer of timers) clearTimeout(timer);
|
||||
};
|
||||
}, [aggregatesToken]);
|
||||
|
||||
return {
|
||||
words,
|
||||
kanji,
|
||||
knownWords,
|
||||
summary,
|
||||
charts,
|
||||
loading,
|
||||
error,
|
||||
aggregatesError,
|
||||
refreshAggregates,
|
||||
reload,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -100,6 +100,8 @@ export const apiClient = {
|
||||
getSessionKnownWordsTimeline: (id: number) =>
|
||||
fetchJson('sessionKnownWordsTimeline', `/api/stats/sessions/${id}/known-words-timeline`),
|
||||
getVocabulary: (limit = 100) => fetchJson('vocabulary', `/api/stats/vocabulary?limit=${limit}`),
|
||||
getVocabularySummary: () => fetchJson('vocabularySummary', '/api/stats/vocabulary/summary'),
|
||||
getVocabularyCharts: () => fetchJson('vocabularyCharts', '/api/stats/vocabulary/charts'),
|
||||
getExcludedWords: () => fetchJson('excludedWords', '/api/stats/excluded-words'),
|
||||
setExcludedWords: async (words: StatsExcludedWord[]): Promise<void> => {
|
||||
await fetchResponse('/api/stats/excluded-words', {
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
|
||||
import { epochMsFromDbTimestamp, formatRelativeDate, formatSessionDayLabel } from './formatters';
|
||||
import {
|
||||
epochDayToDate,
|
||||
epochMsFromDbTimestamp,
|
||||
formatRelativeDate,
|
||||
formatSessionDayLabel,
|
||||
} from './formatters';
|
||||
|
||||
const FIXED_NOW = new Date(2026, 2, 16, 12, 0, 0).getTime();
|
||||
|
||||
@@ -108,6 +113,19 @@ test('epochMsFromDbTimestamp keeps ms timestamps as-is', () => {
|
||||
assert.equal(epochMsFromDbTimestamp(1_700_000_000_000), 1_700_000_000_000);
|
||||
});
|
||||
|
||||
test('epochDayToDate preserves the calendar day west of UTC', () => {
|
||||
const previousTimezone = process.env.TZ;
|
||||
process.env.TZ = 'America/Los_Angeles';
|
||||
try {
|
||||
const epochDay = Math.floor(Date.UTC(2026, 2, 16) / 86_400_000);
|
||||
const date = epochDayToDate(epochDay);
|
||||
assert.deepEqual([date.getFullYear(), date.getMonth(), date.getDate()], [2026, 2, 16]);
|
||||
} finally {
|
||||
if (previousTimezone === undefined) delete process.env.TZ;
|
||||
else process.env.TZ = previousTimezone;
|
||||
}
|
||||
});
|
||||
|
||||
test('formatSessionDayLabel formats today and yesterday', () => {
|
||||
withFixedNow((now) => {
|
||||
const oneDayMs = 24 * 60 * 60_000;
|
||||
|
||||
@@ -38,7 +38,8 @@ export function formatRelativeDate(ms: number): string {
|
||||
}
|
||||
|
||||
export function epochDayToDate(epochDay: number): Date {
|
||||
return new Date(epochDay * 86_400_000);
|
||||
const utcDate = new Date(epochDay * 86_400_000);
|
||||
return new Date(utcDate.getUTCFullYear(), utcDate.getUTCMonth(), utcDate.getUTCDate());
|
||||
}
|
||||
|
||||
export function localDayFromMs(ms: number): number {
|
||||
|
||||
@@ -6,6 +6,7 @@ import { fileURLToPath } from 'node:url';
|
||||
const VOCABULARY_TAB_PATH = fileURLToPath(
|
||||
new URL('../components/vocabulary/VocabularyTab.tsx', import.meta.url),
|
||||
);
|
||||
const VOCABULARY_HOOK_PATH = fileURLToPath(new URL('../hooks/useVocabulary.ts', import.meta.url));
|
||||
|
||||
test('VocabularyTab declares all hooks before loading and error early returns', () => {
|
||||
const source = fs.readFileSync(VOCABULARY_TAB_PATH, 'utf8');
|
||||
@@ -20,15 +21,32 @@ test('VocabularyTab declares all hooks before loading and error early returns',
|
||||
assert.deepEqual(hooksAfterLoadingGuard ?? [], []);
|
||||
});
|
||||
|
||||
test('VocabularyTab memoizes summary and known-word aggregate calculations', () => {
|
||||
test('VocabularyTab uses uncapped server-side data for its charts and card totals', () => {
|
||||
const source = fs.readFileSync(VOCABULARY_TAB_PATH, 'utf8');
|
||||
|
||||
assert.match(source, /\} = useVocabulary\(\);/);
|
||||
assert.match(source, /charts\?\.topWordsWithoutNames/);
|
||||
assert.match(source, /charts\?\.newWordsTimelineWithoutNames/);
|
||||
assert.doesNotMatch(source, /buildVocabularySummary\(/);
|
||||
assert.match(source, /uniqueWords: summary\?\.uniqueWordsWithoutNames \?\? 0/);
|
||||
assert.match(source, /uniqueWords: summary\?\.uniqueWords \?\? 0/);
|
||||
assert.match(source, /value=\{summary \? formatNumber\(summary\.uniqueKanji\) : '…'\}/);
|
||||
});
|
||||
|
||||
test('VocabularyTab surfaces aggregate failures with a retry control', () => {
|
||||
const source = fs.readFileSync(VOCABULARY_TAB_PATH, 'utf8');
|
||||
|
||||
assert.match(source, /aggregatesError/);
|
||||
assert.match(source, /onClick=\{refreshAggregates\}/);
|
||||
});
|
||||
|
||||
test('useVocabulary loads exact card totals without holding up the vocabulary tables', () => {
|
||||
const source = fs.readFileSync(VOCABULARY_HOOK_PATH, 'utf8');
|
||||
|
||||
assert.match(
|
||||
source,
|
||||
/const summary = useMemo\([\s\S]*buildVocabularySummary\(filteredWords, kanji\)[\s\S]*\[filteredWords, kanji\][\s\S]*\);/,
|
||||
);
|
||||
assert.match(
|
||||
source,
|
||||
/const knownWordCount = useMemo\(\(\) => \{[\s\S]*for \(const w of filteredWords\) \{[\s\S]*knownWords\.has\(w\.headword\)[\s\S]*\}\s*return count;\s*\}, \[filteredWords, knownWords\]\);/,
|
||||
/Promise\.allSettled\(\[\s*client\.getVocabulary\(500\),\s*client\.getKanji\(200\),\s*client\.getKnownWords\(\),?\s*\]\)/,
|
||||
);
|
||||
assert.match(source, /client\s*\.getVocabularySummary\(\)\s*\.then\(/);
|
||||
assert.match(source, /client\s*\.getVocabularyCharts\(\)/);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user