From 3b8988655719781a8f11865e7c88e23d6c6f6cf0 Mon Sep 17 00:00:00 2001 From: sudacode Date: Mon, 17 Aug 2026 23:49:04 -0700 Subject: [PATCH] fix(stats): version lexical rollups and preserve backfill writes - Persist vocabulary_visible on imm_words and filter rollups/triggers by it, matching the totals' exclusion rules - Normalize legacy second/millisecond timestamps when bucketing rollup rows into local days - Replace the boolean rollup-ready flag with a version key so schema changes trigger an atomic rebuild - Preserve queued playback writes during rollup backfill instead of dropping them under queue pressure - Time out the lexical rollup worker instead of hanging forever on no response - Key concurrent vocabulary summary requests by their known-words snapshot so they no longer share stale state - Bound stats dashboard retries for unfinished vocabulary loads before showing the inline Retry control --- changes/stats-vocabulary-summary-totals.md | 4 +- docs-site/immersion-tracking.md | 2 +- docs/architecture/stats-trends-data-flow.md | 4 +- launcher/test-support/immersion-db-fixture.ts | 3 + launcher/test-support/immersion-db-schema.ts | 1 + .../services/__tests__/stats-server.test.ts | 15 ++ .../immersion-tracker-service.test.ts | 78 ++++++-- .../services/immersion-tracker-service.ts | 29 ++- .../lexical-rollup-worker-runtime.test.ts | 35 +++- .../lexical-rollup-worker-runtime.ts | 8 + .../lexical-rollup-worker.test.ts | 2 +- .../immersion-tracker/lexical-rollups.test.ts | 178 +++++++++++++++++- .../immersion-tracker/lexical-rollups.ts | 78 +++++--- .../immersion-tracker/query-lexical.ts | 49 ++--- .../services/immersion-tracker/storage.ts | 16 +- src/core/services/immersion-tracker/types.ts | 2 +- .../vocabulary-visibility.ts | 43 +++++ src/core/services/stats-sync/merge-catalog.ts | 1 + stats/src/hooks/useVocabulary.test.tsx | 30 +++ stats/src/hooks/useVocabulary.ts | 21 ++- 20 files changed, 506 insertions(+), 93 deletions(-) create mode 100644 src/core/services/immersion-tracker/vocabulary-visibility.ts diff --git a/changes/stats-vocabulary-summary-totals.md b/changes/stats-vocabulary-summary-totals.md index 7c1b7f76..bbd71508 100644 --- a/changes/stats-vocabulary-summary-totals.md +++ b/changes/stats-vocabulary-summary-totals.md @@ -2,7 +2,7 @@ type: fixed area: stats - Fixed Vocabulary totals and charts counting only the first browsing page instead of all tracked vocabulary, without delaying the rest of the page. -- New-word history now uses permanent daily lexical rollups, backfilled in the background and repaired when tracked material is removed or reprocessed; playback writes queue safely during the one-time rebuild and resume afterward. +- New-word history now uses permanent daily lexical rollups that apply the same vocabulary filters as the totals and normalize legacy second/millisecond timestamps; versioned background rebuilds repair existing history without dropping playback writes. - Calendar-day chart labels now preserve the recorded local date in time zones west of UTC. -- Vocabulary summary cards and charts refresh automatically after the word exclusion list changes, and failed loads retry with backoff before showing an inline error with a Retry control. +- Vocabulary summary cards and charts refresh automatically after the word exclusion list changes, and failed or unfinished loads use bounded retries before showing an inline error with a Retry control. - Rapid exclusion edits no longer race each other; writes are sent in order so a slower earlier save cannot overwrite a newer list. diff --git a/docs-site/immersion-tracking.md b/docs-site/immersion-tracking.md index 291f4617..d650a14f 100644 --- a/docs-site/immersion-tracking.md +++ b/docs-site/immersion-tracking.md @@ -82,7 +82,7 @@ Expandable session history with new-word activity, cumulative totals, and pause/ #### Vocabulary -The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The word and kanji tables load first while those complete totals calculate separately. Top Repeated Words and New Words by Day use complete tracking history rather than the table's browsing page; new-word history is maintained as a permanent daily lexical rollup, including retroactive corrections when tracked material is removed or reprocessed. On the first launch after upgrading, that history is built in the background and the chart refreshes when it is ready. The cards and charts also refresh automatically after the word exclusion list changes. The rest of the tab includes cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons. +The summary cards show all unique vocabulary and kanji recorded in the local tracking database; **New This Week** is the only weekly figure and uses a rolling seven-day window. The word and kanji tables load first while those complete totals calculate separately. Top Repeated Words and New Words by Day use complete tracking history rather than the table's browsing page. New-word history is maintained as a permanent daily lexical rollup using the same token-visibility rules as the totals, including normalization of older timestamps stored in either seconds or milliseconds and retroactive corrections when tracked material is removed or reprocessed. On the first launch after an applicable upgrade, that history is version-rebuilt in the background and the chart refreshes when it is ready; if it remains unavailable, polling stops and an inline Retry control appears. The cards and charts also refresh automatically after the word exclusion list changes. The rest of the tab includes cross-title and frequency rank tables with Hide Known / Hide Kana filters, kanji breakdown, word exclusion list, and click-through occurrence drilldown with Mine Word / Mine Sentence / Mine Audio buttons. ![Stats Vocabulary](/screenshots/stats-vocabulary.png) diff --git a/docs/architecture/stats-trends-data-flow.md b/docs/architecture/stats-trends-data-flow.md index 8c7d27bb..d0a6624b 100644 --- a/docs/architecture/stats-trends-data-flow.md +++ b/docs/architecture/stats-trends-data-flow.md @@ -23,7 +23,9 @@ Trend charts now consume one chart-oriented backend payload from `/api/stats/tre - lookup rate trends - watch-time by day-of-week/hour - vocabulary-backed: - - new-words trend + - new-words trend reads permanent daily lexical rollups + - rollup rows count only vocabulary-visible tokens and normalize mixed legacy timestamp units + - a persisted rollup version invalidates stale materializations and triggers an atomic background rebuild ## Metric Semantics diff --git a/launcher/test-support/immersion-db-fixture.ts b/launcher/test-support/immersion-db-fixture.ts index 3edbaa14..28910b2d 100644 --- a/launcher/test-support/immersion-db-fixture.ts +++ b/launcher/test-support/immersion-db-fixture.ts @@ -18,6 +18,9 @@ export function createImmersionDbFixture(dbPath: string): void { db.prepare( `INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('last_rollup_sample_ms', 0)`, ).run(); + db.prepare( + `INSERT INTO imm_rollup_state(state_key, state_value) VALUES ('lexical_daily_rollups_version', 2)`, + ).run(); db.prepare( `INSERT INTO imm_lifetime_global(global_id, CREATED_DATE, LAST_UPDATE_DATE) VALUES (1, ?, ?)`, ).run(String(Date.now()), String(Date.now())); diff --git a/launcher/test-support/immersion-db-schema.ts b/launcher/test-support/immersion-db-schema.ts index 11977633..c99e2cf4 100644 --- a/launcher/test-support/immersion-db-schema.ts +++ b/launcher/test-support/immersion-db-schema.ts @@ -154,6 +154,7 @@ export const IMMERSION_DB_FIXTURE_DDL = ` last_seen REAL, frequency INTEGER, frequency_rank INTEGER, + vocabulary_visible INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1)), UNIQUE(headword, word, reading) ); CREATE TABLE imm_kanji( diff --git a/src/core/services/__tests__/stats-server.test.ts b/src/core/services/__tests__/stats-server.test.ts index 0dff5209..bc9f3674 100644 --- a/src/core/services/__tests__/stats-server.test.ts +++ b/src/core/services/__tests__/stats-server.test.ts @@ -744,6 +744,21 @@ describe('stats server API routes', () => { }); }); + it('GET /api/stats/vocabulary/charts returns complete chart datasets', async () => { + const app = createStatsApp(createMockTracker()); + + const res = await app.request('/api/stats/vocabulary/charts'); + + assert.equal(res.status, 200); + assert.deepEqual(await res.json(), { + ready: true, + topWords: [{ wordId: 1, headword: 'する', frequency: 50 }], + topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }], + newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }], + newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }], + }); + }); + it('GET /api/stats/kanji returns kanji frequency data', async () => { const app = createStatsApp(createMockTracker()); const res = await app.request('/api/stats/kanji'); diff --git a/src/core/services/immersion-tracker-service.test.ts b/src/core/services/immersion-tracker-service.test.ts index 4c154e09..5d46e161 100644 --- a/src/core/services/immersion-tracker-service.test.ts +++ b/src/core/services/immersion-tracker-service.test.ts @@ -590,7 +590,7 @@ test('tracker starts the injected lexical rollup backfill when it is pending', a ensureSchema(setupDb); setupDb .prepare( - `UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_ready'`, + `UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`, ) .run(); setupDb.close(); @@ -603,6 +603,14 @@ test('tracker starts the injected lexical rollup backfill when it is pending', a } as never); assert.equal(backfillRuns, 1); + await waitForCondition( + () => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked, + ); + assert.equal( + (tracker as unknown as { preserveWriteQueueUntilDrained: boolean }) + .preserveWriteQueueUntilDrained, + false, + ); } finally { tracker?.destroy(); cleanupDbPath(dbPath); @@ -631,14 +639,14 @@ test('tracker queues playback writes until lexical rollup backfill settles', asy ensureSchema(setupDb); setupDb .prepare( - `UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_ready'`, + `UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`, ) .run(); setupDb.close(); const Ctor = await loadTrackerCtor(); tracker = new Ctor( - { dbPath }, + { dbPath, policy: { queueCap: 100 } }, { runLexicalRollupBackfillTask: async (workerDbPath) => { await backfillStartGate; @@ -664,18 +672,20 @@ test('tracker queues playback writes until lexical rollup backfill settles', asy tracker.handleMediaChange('https://example.com/backfill-test.mp4', 'Backfill Test'); startBackfill(); await backfillStarted; - tracker.recordCardsMined(1); + for (let index = 0; index < 125; index += 1) tracker.recordCardsMined(1); const privateApi = tracker as unknown as { db: DatabaseSync; queue: unknown[]; + droppedWriteCount: number; flushNow: () => void; writeLock: { locked: boolean }; }; assert.equal(privateApi.writeLock.locked, true); privateApi.flushNow(); - assert.ok(privateApi.queue.length > 0); + assert.ok(privateApi.queue.length > 100, 'the protected queue may grow past its normal cap'); + assert.equal(privateApi.droppedWriteCount, 0, 'backfill must not discard playback writes'); assert.equal( ( privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as { @@ -686,17 +696,23 @@ test('tracker queues playback writes until lexical rollup backfill settles', asy ); releaseBackfill(); - await waitForCondition(() => privateApi.queue.length === 0); + await waitForCondition(() => privateApi.queue.length === 0, 5_000); assert.equal( ( privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as { total: number; } ).total, - 1, + 125, ); } finally { releaseBackfill(); + if (tracker) { + await waitForCondition( + () => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked, + 5_000, + ); + } tracker?.destroy(); cleanupDbPath(dbPath); } @@ -5086,8 +5102,9 @@ test('getVocabularySummary coalesces concurrent requests into one worker task', }, ); - const first = tracker.getVocabularySummary(null); - const second = tracker.getVocabularySummary(new Set(['猫'])); + const knownWordsSnapshot = new Set(['猫']); + const first = tracker.getVocabularySummary(knownWordsSnapshot); + const second = tracker.getVocabularySummary(knownWordsSnapshot); await waitForCondition(() => releaseTask !== null); let release = releaseTask as (() => void) | null; assert.ok(release); @@ -5095,9 +5112,7 @@ test('getVocabularySummary coalesces concurrent requests into one worker task', assert.deepEqual(await first, summary); assert.equal(await second, await first); assert.equal(taskRuns, 1); - // The coalesced caller's known-words set must not replace the snapshot the - // in-flight scan already started with. - assert.deepEqual(seenKnownWords, [null]); + assert.deepEqual(seenKnownWords, [knownWordsSnapshot]); releaseTask = null; const third = tracker.getVocabularySummary(null); @@ -5112,3 +5127,42 @@ test('getVocabularySummary coalesces concurrent requests into one worker task', cleanupDbPath(dbPath); } }); + +test('getVocabularySummary keeps different known-word snapshots independent', async () => { + const dbPath = makeDbPath(); + let tracker: ImmersionTrackerService | null = null; + const releases: Array<() => void> = []; + + try { + const Ctor = await loadTrackerCtor(); + tracker = new Ctor( + { dbPath }, + { + runVocabularySummaryTask: async (_dbPath, knownWords) => { + await new Promise((resolve) => releases.push(resolve)); + return { + uniqueWords: 1, + uniqueWordsWithoutNames: 1, + uniqueKanji: 0, + newThisWeek: 0, + newThisWeekWithoutNames: 0, + knownWordCount: knownWords?.size ?? null, + knownWordCountWithoutNames: knownWords?.size ?? null, + }; + }, + destroyVocabularySummaryRunner: () => {}, + }, + ); + + const withoutKnownWords = tracker.getVocabularySummary(null); + const withKnownWords = tracker.getVocabularySummary(new Set(['猫'])); + await waitForCondition(() => releases.length === 2); + for (const release of releases) release(); + + assert.equal((await withoutKnownWords).knownWordCount, null); + assert.equal((await withKnownWords).knownWordCount, 1); + } finally { + tracker?.destroy(); + cleanupDbPath(dbPath); + } +}); diff --git a/src/core/services/immersion-tracker-service.ts b/src/core/services/immersion-tracker-service.ts index 60d54ae1..15217e93 100644 --- a/src/core/services/immersion-tracker-service.ts +++ b/src/core/services/immersion-tracker-service.ts @@ -421,7 +421,10 @@ export class ImmersionTrackerService { private readonly runVocabularySummaryTask: ( knownWords: ReadonlySet | null, ) => Promise; - private vocabularySummaryInFlight: Promise | null = null; + private readonly vocabularySummariesInFlight = new Map< + ReadonlySet | null, + Promise + >(); private readonly destroyVocabularySummaryRunner: () => void; private readonly runLexicalRollupBackfillTask: () => Promise; private readonly destroyLexicalRollupBackfillRunner: () => void; @@ -430,6 +433,7 @@ export class ImmersionTrackerService { private maintenanceTimer: ReturnType | null = null; private flushScheduled = false; private droppedWriteCount = 0; + private preserveWriteQueueUntilDrained = false; private lastVacuumMs = 0; private isDestroyed = false; private sessionState: SessionState | null = null; @@ -681,16 +685,16 @@ export class ImmersionTrackerService { } async getVocabularySummary(knownWords: ReadonlySet | null) { - // Concurrent dashboard refreshes share one worker scan; the coalesced - // callers accept the first caller's known-words snapshot. - const inFlight = this.vocabularySummaryInFlight; + const inFlight = this.vocabularySummariesInFlight.get(knownWords); if (inFlight) return inFlight; const task = this.runVocabularySummaryTask(knownWords); - this.vocabularySummaryInFlight = task; + this.vocabularySummariesInFlight.set(knownWords, task); try { return await task; } finally { - if (this.vocabularySummaryInFlight === task) this.vocabularySummaryInFlight = null; + if (this.vocabularySummariesInFlight.get(knownWords) === task) { + this.vocabularySummariesInFlight.delete(knownWords); + } } } @@ -985,6 +989,7 @@ export class ImmersionTrackerService { private startLexicalRollupBackfill(): void { this.requireWriteQueueDrained('lexical rollup backfill'); + this.preserveWriteQueueUntilDrained = true; this.setWriteLock('lexical-rollup-backfill', true); void this.runLexicalRollupBackfillTask() .catch((error: unknown) => { @@ -995,7 +1000,8 @@ export class ImmersionTrackerService { }) .finally(() => { this.setWriteLock('lexical-rollup-backfill', false); - if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0); + if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false; + else if (!this.isDestroyed) this.scheduleFlush(0); }); } @@ -1995,7 +2001,12 @@ export class ImmersionTrackerService { private recordWrite(write: QueuedWrite): void { if (this.isDestroyed) return; - const { dropped } = enqueueWrite(this.queue, write, this.queueCap); + // A lexical migration owns the database write lock, so dropping the oldest + // entry cannot relieve pressure: nothing can flush until the worker exits. + // Preserve that finite startup burst and drain it as soon as the lock lifts. + const { dropped } = this.preserveWriteQueueUntilDrained + ? (this.queue.push(write), { dropped: 0 }) + : enqueueWrite(this.queue, write, this.queueCap); if (dropped > 0) { this.droppedWriteCount += dropped; this.logger.warn(`Immersion tracker queue overflow; dropped ${dropped} oldest writes`); @@ -2043,6 +2054,7 @@ export class ImmersionTrackerService { private flushNow(): void { if (this.writeLock.locked || this.isDestroyed) return; if (this.queue.length === 0) { + this.preserveWriteQueueUntilDrained = false; this.flushScheduled = false; return; } @@ -2068,6 +2080,7 @@ export class ImmersionTrackerService { } finally { this.setWriteLock('flush', false); this.flushScheduled = false; + if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false; if (this.queue.length > 0) { this.scheduleFlush(this.flushIntervalMs); } diff --git a/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.test.ts b/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.test.ts index 63c4d318..6ae22046 100644 --- a/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.test.ts +++ b/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.test.ts @@ -26,7 +26,7 @@ test('lexical rollup worker backfills without using the tracker connection', asy ).run(); db.exec('DELETE FROM imm_lexical_daily_rollups'); db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run( - 'lexical_daily_rollups_ready', + 'lexical_daily_rollups_version', ); db.close(); @@ -101,3 +101,36 @@ test('lexical rollup worker absorbs termination failures after settling', async runtime.destroy(); } }); + +test('lexical rollup worker times out when it never responds', async () => { + let terminated = false; + const runtime = new LexicalRollupWorkerRuntime({ + resolveWorkerPath: () => '/tmp/fake-worker.js', + createWorker: async () => ({ + once() { + return this; + }, + terminate: async () => { + terminated = true; + return 0; + }, + }), + timeoutMs: 1, + warn: () => {}, + } as never); + + try { + const outcome = await Promise.race([ + runtime.run('/tmp/not-used.sqlite').then( + () => 'resolved', + (error: unknown) => String(error), + ), + new Promise((resolve) => setTimeout(() => resolve('still pending'), 50)), + ]); + + assert.match(outcome, /timed out/); + assert.equal(terminated, true); + } finally { + runtime.destroy(); + } +}); diff --git a/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.ts b/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.ts index cdc880d7..30c5c45f 100644 --- a/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.ts +++ b/src/core/services/immersion-tracker/lexical-rollup-worker-runtime.ts @@ -17,10 +17,12 @@ interface WorkerHandle { interface LexicalRollupWorkerRuntimeOptions { resolveWorkerPath?: () => string | null; createWorker?: (workerPath: string, workerData: { dbPath: string }) => Promise; + timeoutMs?: number; warn?: (message: string, ...meta: unknown[]) => void; } const logger = createLogger('main:immersion-tracker:lexical-rollup-worker'); +const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000; export function resolveLexicalRollupWorkerPath(): string | null { const fileName = __filename.endsWith('.ts') @@ -65,15 +67,21 @@ export class LexicalRollupWorkerRuntime { return new Promise((resolve, reject) => { let settled = false; + let timeout: ReturnType | null = null; this.activeWorkers.add(worker); const settle = (error?: Error) => { if (settled) return; settled = true; + if (timeout) clearTimeout(timeout); this.activeWorkers.delete(worker); void worker.terminate().catch(() => undefined); if (error) reject(error); else resolve(); }; + timeout = setTimeout( + () => settle(new Error('Lexical rollup worker timed out')), + this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS, + ); worker.once('message', (message) => { if (message.ok) settle(); else diff --git a/src/core/services/immersion-tracker/lexical-rollup-worker.test.ts b/src/core/services/immersion-tracker/lexical-rollup-worker.test.ts index b3bdcb15..ee38f4f1 100644 --- a/src/core/services/immersion-tracker/lexical-rollup-worker.test.ts +++ b/src/core/services/immersion-tracker/lexical-rollup-worker.test.ts @@ -21,7 +21,7 @@ test('lexical rollup backfill materializes pre-existing vocabulary off the calle ).run('犬', '犬', 'いぬ', 1_700_000_000, 1_700_000_000); db.exec('DELETE FROM imm_lexical_daily_rollups'); db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run( - 'lexical_daily_rollups_ready', + 'lexical_daily_rollups_version', ); executeLexicalRollupBackfillTask(dbPath); diff --git a/src/core/services/immersion-tracker/lexical-rollups.test.ts b/src/core/services/immersion-tracker/lexical-rollups.test.ts index a2a030c3..09279d5e 100644 --- a/src/core/services/immersion-tracker/lexical-rollups.test.ts +++ b/src/core/services/immersion-tracker/lexical-rollups.test.ts @@ -3,9 +3,17 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import test from 'node:test'; -import { getLexicalDailyRollups, rebuildLexicalDailyRollups } from './lexical-rollups'; +import { + areLexicalDailyRollupsReady, + getLexicalDailyRollups, + rebuildLexicalDailyRollups, +} from './lexical-rollups'; import { getTrendsDashboard } from './query-trends'; -import { getVocabularyChartData, replaceStatsExcludedWords } from './query-lexical'; +import { + getVocabularyChartData, + getVocabularySummary, + replaceStatsExcludedWords, +} from './query-lexical'; import { Database } from './sqlite'; import type { DatabaseSync } from './sqlite'; import { ensureSchema } from './storage'; @@ -51,6 +59,169 @@ test('lexical daily rollups follow first-seen corrections and deletions', () => } }); +test('lexical daily rollups normalize second and millisecond timestamps', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const epochDay = 19_500; + const timestampSeconds = epochDay * 86_400 + 43_200; + const timestampMilliseconds = timestampSeconds * 1_000; + + db.prepare( + `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency) + VALUES (?, ?, ?, ?, ?, 1)`, + ).run('猫', '猫', 'ねこ', timestampSeconds, timestampSeconds); + db.prepare( + `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency) + VALUES (?, ?, ?, ?, ?, 1)`, + ).run('犬', '犬', 'いぬ', timestampMilliseconds, timestampMilliseconds); + db.prepare( + `INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency) + VALUES (?, ?, ?, 1)`, + ).run('猫', timestampSeconds, timestampSeconds); + db.prepare( + `INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency) + VALUES (?, ?, ?, 1)`, + ).run('犬', timestampMilliseconds, timestampMilliseconds); + + assert.deepEqual(getLexicalDailyRollups(db), [ + { epochDay, wordCount: 2, wordCountWithoutNames: 2, kanjiCount: 2 }, + ]); + } finally { + db.close(); + fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }); + } +}); + +test('lexical rollup rebuild excludes rows hidden by vocabulary persistence rules', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const epochDay = 19_500; + const firstSeen = epochDay * 86_400 + 43_200; + db.prepare( + `INSERT INTO imm_words( + headword, word, reading, part_of_speech, first_seen, last_seen, frequency + ) VALUES (?, ?, ?, ?, ?, ?, 1)`, + ).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen); + db.prepare( + `INSERT INTO imm_words( + headword, word, reading, part_of_speech, first_seen, last_seen, frequency + ) VALUES (?, ?, ?, ?, ?, ?, 1)`, + ).run('は', 'は', 'は', 'particle', firstSeen, firstSeen); + + rebuildLexicalDailyRollups(db); + + assert.deepEqual(getLexicalDailyRollups(db), [ + { epochDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 }, + ]); + } finally { + db.close(); + fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }); + } +}); + +test('lexical rollup rebuild tolerates nullable legacy vocabulary text', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + db.prepare( + `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency) + VALUES (NULL, NULL, NULL, 1700000000, 1700000000, 1)`, + ).run(); + db.prepare( + `INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency) + VALUES (NULL, '猫', 'ねこ', 1700000000, 1700000000, 1)`, + ).run(); + + assert.doesNotThrow(() => rebuildLexicalDailyRollups(db)); + assert.equal(areLexicalDailyRollupsReady(db), true); + assert.equal(getVocabularySummary(db, null).uniqueWords, 1); + assert.equal(getVocabularySummary(db, new Set(['猫'])).knownWordCount, 1); + assert.equal(getVocabularyChartData(db).topWords[0]?.headword, '猫'); + } finally { + db.close(); + fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }); + } +}); + +test('chart exclusions do not subtract vocabulary rows already hidden from the rollup', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const epochDay = 19_500; + const firstSeen = epochDay * 86_400 + 43_200; + db.prepare( + `INSERT INTO imm_words( + headword, word, reading, part_of_speech, first_seen, last_seen, frequency + ) VALUES (?, ?, ?, ?, ?, ?, 1)`, + ).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen); + db.prepare( + `INSERT INTO imm_words( + headword, word, reading, part_of_speech, first_seen, last_seen, frequency + ) VALUES (?, ?, ?, ?, ?, ?, 1)`, + ).run('は', 'は', 'は', 'particle', firstSeen, firstSeen); + rebuildLexicalDailyRollups(db); + replaceStatsExcludedWords(db, [{ headword: 'は', word: 'は', reading: 'は' }]); + + const charts = getVocabularyChartData(db); + + assert.deepEqual(charts.newWordsTimeline, [{ epochDay, wordCount: 1 }]); + assert.deepEqual(charts.newWordsTimelineWithoutNames, [{ epochDay, wordCount: 1 }]); + } finally { + db.close(); + fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }); + } +}); + +test('legacy lexical rollup readiness does not satisfy the current rollup version', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + db.prepare( + `INSERT INTO imm_rollup_state(state_key, state_value) + VALUES ('lexical_daily_rollups_ready', '1') + ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`, + ).run(); + db.prepare( + `DELETE FROM imm_rollup_state WHERE state_key = 'lexical_daily_rollups_version'`, + ).run(); + + assert.equal(areLexicalDailyRollupsReady(db), false); + } finally { + db.close(); + fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }); + } +}); + +test('imm_words persists vocabulary visibility for rollup maintenance', () => { + const dbPath = makeDbPath(); + const db = new Database(dbPath); + + try { + ensureSchema(db); + const columns = db.prepare(`PRAGMA table_info(imm_words)`).all() as Array<{ name: string }>; + + assert.equal( + columns.some((column) => column.name === 'vocabulary_visible'), + true, + ); + } finally { + db.close(); + fs.rmSync(path.dirname(dbPath), { recursive: true, force: true }); + } +}); + test('vocabulary charts use complete top-word and lexical rollup data', () => { const dbPath = makeDbPath(); const db = new Database(dbPath); @@ -146,6 +317,9 @@ test('lexical rollup rebuild preserves the original error when rollback also fai if (sql === 'ROLLBACK') throw new Error('rollback failed'); throw originalError; }, + prepare() { + return { all: () => [], run: () => undefined }; + }, } as unknown as DatabaseSync; assert.throws(() => rebuildLexicalDailyRollups(db), originalError); diff --git a/src/core/services/immersion-tracker/lexical-rollups.ts b/src/core/services/immersion-tracker/lexical-rollups.ts index 78372975..fd6e29d3 100644 --- a/src/core/services/immersion-tracker/lexical-rollups.ts +++ b/src/core/services/immersion-tracker/lexical-rollups.ts @@ -1,4 +1,5 @@ import type { DatabaseSync } from './sqlite'; +import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility'; export interface LexicalDailyRollup { epochDay: number; @@ -8,11 +9,20 @@ export interface LexicalDailyRollup { } const LOCAL_EPOCH_DAY_SQL = ` - CAST(julianday(CAST(%VALUE% AS REAL), 'unixepoch', 'localtime') - 2440587.5 AS INTEGER) + CAST(julianday( + CASE + WHEN ABS(CAST(%VALUE% AS REAL)) >= 10000000000 THEN CAST(%VALUE% AS REAL) / 1000 + ELSE CAST(%VALUE% AS REAL) + END, + 'unixepoch', 'localtime' + ) - 2440587.5 AS INTEGER) `; +const LEXICAL_DAILY_ROLLUP_VERSION = '2'; +const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version'; + export function localEpochDaySql(value: string): string { - return LOCAL_EPOCH_DAY_SQL.replace('%VALUE%', value); + return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value); } function createWordRollupTriggers(db: DatabaseSync): void { @@ -20,9 +30,13 @@ function createWordRollupTriggers(db: DatabaseSync): void { const dayForOld = localEpochDaySql('OLD.first_seen'); db.exec(` - CREATE TRIGGER IF NOT EXISTS imm_words_lexical_rollup_insert + DROP TRIGGER IF EXISTS imm_words_lexical_rollup_insert; + DROP TRIGGER IF EXISTS imm_words_lexical_rollup_delete; + DROP TRIGGER IF EXISTS imm_words_lexical_rollup_first_seen_update; + + CREATE TRIGGER imm_words_lexical_rollup_insert AFTER INSERT ON imm_words - WHEN NEW.first_seen IS NOT NULL + WHEN NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1 BEGIN INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) VALUES (${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0) @@ -31,9 +45,9 @@ function createWordRollupTriggers(db: DatabaseSync): void { word_count_without_names = word_count_without_names + excluded.word_count_without_names; END; - CREATE TRIGGER IF NOT EXISTS imm_words_lexical_rollup_delete + CREATE TRIGGER imm_words_lexical_rollup_delete AFTER DELETE ON imm_words - WHEN OLD.first_seen IS NOT NULL + WHEN OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1 BEGIN INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) VALUES (${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0) @@ -44,19 +58,21 @@ function createWordRollupTriggers(db: DatabaseSync): void { WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0; END; - CREATE TRIGGER IF NOT EXISTS imm_words_lexical_rollup_first_seen_update - AFTER UPDATE OF first_seen, pos2 ON imm_words - WHEN OLD.first_seen IS NOT NEW.first_seen OR OLD.pos2 IS NOT NEW.pos2 + CREATE TRIGGER imm_words_lexical_rollup_first_seen_update + AFTER UPDATE OF first_seen, pos2, vocabulary_visible ON imm_words + WHEN OLD.first_seen IS NOT NEW.first_seen + OR OLD.pos2 IS NOT NEW.pos2 + OR OLD.vocabulary_visible IS NOT NEW.vocabulary_visible BEGIN INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) SELECT ${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0 - WHERE OLD.first_seen IS NOT NULL + WHERE OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1 ON CONFLICT(epoch_day) DO UPDATE SET word_count = word_count - 1, word_count_without_names = word_count_without_names + excluded.word_count_without_names; INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) SELECT ${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0 - WHERE NEW.first_seen IS NOT NULL + WHERE NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1 ON CONFLICT(epoch_day) DO UPDATE SET word_count = word_count + 1, word_count_without_names = word_count_without_names + excluded.word_count_without_names; @@ -70,14 +86,18 @@ function createKanjiRollupTriggers(db: DatabaseSync): void { const dayForNew = localEpochDaySql('NEW.first_seen'); const dayForOld = localEpochDaySql('OLD.first_seen'); db.exec(` - CREATE TRIGGER IF NOT EXISTS imm_kanji_lexical_rollup_insert + DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_insert; + DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_delete; + DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_first_seen_update; + + CREATE TRIGGER imm_kanji_lexical_rollup_insert AFTER INSERT ON imm_kanji WHEN NEW.first_seen IS NOT NULL BEGIN INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) VALUES (${dayForNew}, 0, 0, 1) ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1; END; - CREATE TRIGGER IF NOT EXISTS imm_kanji_lexical_rollup_delete + CREATE TRIGGER imm_kanji_lexical_rollup_delete AFTER DELETE ON imm_kanji WHEN OLD.first_seen IS NOT NULL BEGIN INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) @@ -86,7 +106,7 @@ function createKanjiRollupTriggers(db: DatabaseSync): void { DELETE FROM imm_lexical_daily_rollups WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0; END; - CREATE TRIGGER IF NOT EXISTS imm_kanji_lexical_rollup_first_seen_update + CREATE TRIGGER imm_kanji_lexical_rollup_first_seen_update AFTER UPDATE OF first_seen ON imm_kanji WHEN OLD.first_seen IS NOT NEW.first_seen BEGIN INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) @@ -109,7 +129,7 @@ export function ensureLexicalDailyRollupTables(db: DatabaseSync): void { kanji_count INTEGER NOT NULL DEFAULT 0 ); INSERT INTO imm_rollup_state(state_key, state_value) - VALUES ('lexical_daily_rollups_ready', '0') + VALUES ('${LEXICAL_DAILY_ROLLUP_VERSION_KEY}', '0') ON CONFLICT(state_key) DO NOTHING; `); createWordRollupTriggers(db); @@ -119,14 +139,16 @@ export function ensureLexicalDailyRollupTables(db: DatabaseSync): void { export function areLexicalDailyRollupsReady(db: DatabaseSync): boolean { const row = db .prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`) - .get('lexical_daily_rollups_ready') as { value: string } | null; - return row?.value === '1'; + .get(LEXICAL_DAILY_ROLLUP_VERSION_KEY) as { value: string } | null; + return row?.value === LEXICAL_DAILY_ROLLUP_VERSION; } export function markLexicalDailyRollupsReady(db: DatabaseSync): void { - db.prepare(`UPDATE imm_rollup_state SET state_value = '1' WHERE state_key = ?`).run( - 'lexical_daily_rollups_ready', - ); + db.prepare( + `INSERT INTO imm_rollup_state(state_key, state_value) + VALUES (?, ?) + ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`, + ).run(LEXICAL_DAILY_ROLLUP_VERSION_KEY, LEXICAL_DAILY_ROLLUP_VERSION); } /** Rebuild from the first-seen source of truth; run off the UI/main DB thread. */ @@ -135,13 +157,27 @@ export function rebuildLexicalDailyRollups(db: DatabaseSync): void { try { db.exec('BEGIN IMMEDIATE'); transactionStarted = true; + const vocabularyRows = db + .prepare( + `SELECT id, word, headword, reading, part_of_speech AS partOfSpeech, + pos1, pos2, pos3, frequency_rank AS frequencyRank + FROM imm_words`, + ) + .all() as Array; + const updateVisibility = db.prepare( + `UPDATE imm_words SET vocabulary_visible = ? WHERE id = ? AND vocabulary_visible IS NOT ?`, + ); + for (const row of vocabularyRows) { + const visible = isVocabularyStatsRowVisible(row) ? 1 : 0; + updateVisibility.run(visible, row.id, visible); + } db.exec('DELETE FROM imm_lexical_daily_rollups'); db.exec(` INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) SELECT ${localEpochDaySql('first_seen')}, COUNT(*), SUM(CASE WHEN pos2 = '固有名詞' THEN 0 ELSE 1 END), 0 FROM imm_words - WHERE first_seen IS NOT NULL + WHERE first_seen IS NOT NULL AND vocabulary_visible = 1 GROUP BY ${localEpochDaySql('first_seen')}; INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count) SELECT ${localEpochDaySql('first_seen')}, 0, 0, COUNT(*) diff --git a/src/core/services/immersion-tracker/query-lexical.ts b/src/core/services/immersion-tracker/query-lexical.ts index 9eacb333..2f7bf553 100644 --- a/src/core/services/immersion-tracker/query-lexical.ts +++ b/src/core/services/immersion-tracker/query-lexical.ts @@ -1,6 +1,4 @@ import type { DatabaseSync } from './sqlite'; -import { PartOfSpeech, type MergedToken } from '../../../types'; -import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage'; import type { KanjiAnimeAppearanceRow, KanjiDetailRow, @@ -25,6 +23,7 @@ import { getLexicalDailyRollups, localEpochDaySql, } from './lexical-rollups'; +import { isVocabularyStatsRowVisible } from './vocabulary-visibility'; const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4; const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100; @@ -91,33 +90,6 @@ function uniqueKanji(text: string): string[] { return Array.from(new Set(text.match(KANJI_PATTERN) ?? [])); } -function toVocabularyToken(row: VocabularyStatsRow): MergedToken { - const partOfSpeech = - row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech) - ? (row.partOfSpeech as PartOfSpeech) - : PartOfSpeech.other; - - return { - surface: row.word, - reading: row.reading ?? '', - headword: row.headword, - startPos: 0, - endPos: row.word.length, - partOfSpeech, - pos1: row.pos1 ?? '', - pos2: row.pos2 ?? '', - pos3: row.pos3 ?? '', - frequencyRank: row.frequencyRank ?? undefined, - isMerged: false, - isKnown: false, - isNPlusOneTarget: false, - }; -} - -function isVocabularyStatsRowVisible(row: VocabularyStatsRow): boolean { - return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row)); -} - export function getVocabularyStats( db: DatabaseSync, limit = 100, @@ -204,7 +176,8 @@ export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData { SELECT id AS wordId, headword, word, reading, pos2, ${localEpochDaySql('first_seen')} AS epochDay FROM imm_words - WHERE headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}) + WHERE vocabulary_visible = 1 + AND (headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders})) `, ) .all(...batch, ...batch, ...batch) as Array< @@ -227,12 +200,12 @@ export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData { ready, topWords: topWords.all.map((word) => ({ wordId: word.wordId, - headword: word.headword, + headword: vocabularyDisplayHeadword(word), frequency: word.frequency, })), topWordsWithoutNames: topWords.withoutNames.map((word) => ({ wordId: word.wordId, - headword: word.headword, + headword: vocabularyDisplayHeadword(word), frequency: word.frequency, })), newWordsTimeline: [...timeline.values()] @@ -281,11 +254,17 @@ function getTopVocabularyChartWords( function excludedVocabularyAliases( word: Pick, ): string[] { - const aliases = [word.headword.trim(), word.word.trim()].filter(Boolean); - if (aliases.length === 0) aliases.push(word.reading.trim()); + const aliases = [word.headword?.trim() ?? '', word.word?.trim() ?? ''].filter(Boolean); + if (aliases.length === 0) aliases.push(word.reading?.trim() ?? ''); return [...new Set(aliases)]; } +function vocabularyDisplayHeadword( + word: Pick, +): string { + return word.headword?.trim() || word.word?.trim() || word.reading?.trim() || ''; +} + function timestampSeconds(timestamp: number): number { return timestamp < 10_000_000_000 ? timestamp : Math.floor(timestamp / 1000); } @@ -338,7 +317,7 @@ export function getVocabularySummary( } const isName = word.pos2 === '固有名詞'; const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec; - const isKnown = knownWords?.has(word.headword) ?? false; + const isKnown = knownWords?.has(vocabularyDisplayHeadword(word)) ?? false; summary.uniqueWords += 1; if (!isName) summary.uniqueWordsWithoutNames += 1; if (isNewThisWeek) { diff --git a/src/core/services/immersion-tracker/storage.ts b/src/core/services/immersion-tracker/storage.ts index 1d11baf7..c55de8dc 100644 --- a/src/core/services/immersion-tracker/storage.ts +++ b/src/core/services/immersion-tracker/storage.ts @@ -1069,6 +1069,7 @@ export function ensureSchema(db: DatabaseSync): void { last_seen REAL, frequency INTEGER, frequency_rank INTEGER, + vocabulary_visible INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1)), UNIQUE(headword, word, reading) ); `); @@ -1452,6 +1453,15 @@ export function ensureSchema(db: DatabaseSync): void { addColumnIfMissing(db, 'imm_sessions', 'ended_media_ms', 'INTEGER'); } + if (currentVersion?.schema_version && currentVersion.schema_version < 23) { + addColumnIfMissing( + db, + 'imm_words', + 'vocabulary_visible', + 'INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1))', + ); + } + migrateSessionEventTimestampsToText(db); ensureLexicalDailyRollupTables(db); @@ -1625,9 +1635,10 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar `), wordUpsertStmt: db.prepare(` INSERT INTO imm_words ( - headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency, frequency_rank + headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, + frequency, frequency_rank, vocabulary_visible ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, 1 ) ON CONFLICT(headword, word, reading) DO UPDATE SET frequency = COALESCE(frequency, 0) + 1, @@ -1640,6 +1651,7 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar pos1 = COALESCE(NULLIF(imm_words.pos1, ''), excluded.pos1), pos2 = COALESCE(NULLIF(imm_words.pos2, ''), excluded.pos2), pos3 = COALESCE(NULLIF(imm_words.pos3, ''), excluded.pos3), + vocabulary_visible = 1, first_seen = MIN(COALESCE(first_seen, excluded.first_seen), excluded.first_seen), last_seen = MAX(COALESCE(last_seen, excluded.last_seen), excluded.last_seen), frequency_rank = CASE diff --git a/src/core/services/immersion-tracker/types.ts b/src/core/services/immersion-tracker/types.ts index e28caa37..18d3d59a 100644 --- a/src/core/services/immersion-tracker/types.ts +++ b/src/core/services/immersion-tracker/types.ts @@ -1,4 +1,4 @@ -export const SCHEMA_VERSION = 22; +export const SCHEMA_VERSION = 23; export const DEFAULT_QUEUE_CAP = 1_000; export const DEFAULT_BATCH_SIZE = 25; export const DEFAULT_FLUSH_INTERVAL_MS = 500; diff --git a/src/core/services/immersion-tracker/vocabulary-visibility.ts b/src/core/services/immersion-tracker/vocabulary-visibility.ts new file mode 100644 index 00000000..ba939512 --- /dev/null +++ b/src/core/services/immersion-tracker/vocabulary-visibility.ts @@ -0,0 +1,43 @@ +import { PartOfSpeech, type MergedToken } from '../../../types'; +import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage'; + +export interface VocabularyVisibilityRow { + word: string | null; + headword: string | null; + reading?: string | null; + partOfSpeech?: string | null; + pos1?: string | null; + pos2?: string | null; + pos3?: string | null; + frequencyRank?: number | null; +} + +function toVocabularyToken(row: VocabularyVisibilityRow): MergedToken { + const word = row.word ?? ''; + const headword = row.headword ?? word; + const partOfSpeech = + row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech) + ? (row.partOfSpeech as PartOfSpeech) + : PartOfSpeech.other; + + return { + surface: word, + reading: row.reading ?? '', + headword, + startPos: 0, + endPos: word.length, + partOfSpeech, + pos1: row.pos1 ?? '', + pos2: row.pos2 ?? '', + pos3: row.pos3 ?? '', + frequencyRank: row.frequencyRank ?? undefined, + isMerged: false, + isKnown: false, + isNPlusOneTarget: false, + }; +} + +export function isVocabularyStatsRowVisible(row: VocabularyVisibilityRow): boolean { + if (!(row.word?.trim() || row.headword?.trim())) return false; + return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row)); +} diff --git a/src/core/services/stats-sync/merge-catalog.ts b/src/core/services/stats-sync/merge-catalog.ts index 9b8c9b54..a58b8d62 100644 --- a/src/core/services/stats-sync/merge-catalog.ts +++ b/src/core/services/stats-sync/merge-catalog.ts @@ -89,6 +89,7 @@ const WORD_COPY_COLUMNS = [ 'last_seen', 'frequency', 'frequency_rank', + 'vocabulary_visible', ] as const; export function mergeAnime( diff --git a/stats/src/hooks/useVocabulary.test.tsx b/stats/src/hooks/useVocabulary.test.tsx index 1010e1b8..ac0b0218 100644 --- a/stats/src/hooks/useVocabulary.test.tsx +++ b/stats/src/hooks/useVocabulary.test.tsx @@ -285,6 +285,36 @@ test('charts poll while the backfill is pending and stop once it is ready', asyn } }); +test('chart backfill polling stops and surfaces Retry when readiness never arrives', async () => { + let chartCalls = 0; + const restoreClient = stubVocabularyClient({ + getVocabularySummary: async () => summaryFixture(), + getVocabularyCharts: async () => { + chartCalls += 1; + return chartsFixture({ ready: false }); + }, + }); + const harness = await mountHook(); + + try { + await harness.flush(); + for (let poll = 0; poll < 65; poll += 1) await harness.tick(5_000); + + assert.equal(chartCalls, 60, 'a failed backfill must not poll for the lifetime of the tab'); + assert.match(harness.state().aggregatesError ?? '', /still building/i); + + await act(async () => { + harness.state().refreshAggregates(); + }); + await harness.flush(); + assert.equal(chartCalls, 61, 'Retry starts one fresh bounded polling cycle'); + assert.equal(harness.state().aggregatesError, null); + } finally { + await harness.teardown(); + restoreClient(); + } +}); + test('aggregates refetch after an exclusion edit is acknowledged by the server', async () => { let summaryCalls = 0; let chartCalls = 0; diff --git a/stats/src/hooks/useVocabulary.ts b/stats/src/hooks/useVocabulary.ts index 9a03fef3..bc1eb3f1 100644 --- a/stats/src/hooks/useVocabulary.ts +++ b/stats/src/hooks/useVocabulary.ts @@ -14,6 +14,7 @@ const AGGREGATE_RETRY_LIMIT = 5; const CHART_BACKFILL_POLL_MS = 1_000; const CHART_BACKFILL_SLOW_POLL_MS = 5_000; const CHART_BACKFILL_FAST_POLLS = 30; +const CHART_BACKFILL_POLL_LIMIT = 60; function aggregateRetryDelayMs(attempt: number): number { return Math.min(AGGREGATE_RETRY_BASE_MS * 2 ** attempt, AGGREGATE_RETRY_MAX_MS); @@ -117,12 +118,20 @@ export function useVocabulary() { if (cancelled) return; setCharts(nextCharts); if (!nextCharts.ready) { - schedule( - () => loadCharts(0, readyPolls + 1), - readyPolls < CHART_BACKFILL_FAST_POLLS - ? CHART_BACKFILL_POLL_MS - : CHART_BACKFILL_SLOW_POLL_MS, - ); + const completedPolls = readyPolls + 1; + if (completedPolls < CHART_BACKFILL_POLL_LIMIT) { + schedule( + () => loadCharts(0, completedPolls), + completedPolls < CHART_BACKFILL_FAST_POLLS + ? CHART_BACKFILL_POLL_MS + : CHART_BACKFILL_SLOW_POLL_MS, + ); + } else { + setAggregatesError( + (previous) => + previous ?? 'Vocabulary charts are still building. Retry to check again.', + ); + } } }) .catch((chartError: unknown) => {