refactor(anki): parse the known-word cache in one shared module

The cache manager and the stats server each carried a hand-written parser
for the same file. When the format went to v4 for maturity tiers only the
manager was updated, so loadKnownWordsSet fell through to null and every
session reported 0 known words from an intact cache.

Move the state union, its parser, and a knownWordsFromState accessor into
known-word-cache-format and have both readers go through it. The stats
server no longer mentions a version at all, and both version switches close
with assertNever, so adding a V5 is a compile error at every consumer
instead of a silently empty result. An unparseable cache now warns rather
than reading as "no known words".

Two stats fixtures omitted refreshedAtMs/scope and only passed because the
stats reader was laxer than the manager; they now carry the fields every
cache the app writes has always had.

changes/fix-stats-known-word-cache-format.md

@@ -0,0 +1,5 @@
+type: fixed
+area: stats
+
+- Stats reported 0 known words for every session after the known-word cache gained maturity tiers. The stats server carried its own copy of the cache parser that only recognized versions up to 3, so the new v4 file was read as "no cache" rather than as a format it should understand.
+- The cache format, its parser, and the derived known-word set now live in one module that both the cache manager and the stats server read, and the version dispatch ends in an exhaustive check so a future format bump fails the build instead of silently reporting zero. A cache that exists but does not parse now logs a warning rather than passing for an empty one.

src/anki-integration/known-word-cache-format.test.ts

@@ -0,0 +1,76 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import {
+  KnownWordCacheState,
+  knownWordsFromState,
+  parseKnownWordCacheState,
+} from './known-word-cache-format';
+
+function parseOrThrow(value: unknown): KnownWordCacheState {
+  const parsed = parseKnownWordCacheState(value);
+  assert.ok(parsed, 'expected the payload to parse');
+  return parsed;
+}
+
+const BASE = { refreshedAtMs: 1, scope: 'deck:test' };
+
+test('known-word cache format reads words from every version the union covers', () => {
+  assert.deepEqual(
+    knownWordsFromState(parseOrThrow({ ...BASE, version: 1, words: ['する'] })),
+    new Set(['する']),
+  );
+
+  assert.deepEqual(
+    knownWordsFromState(
+      parseOrThrow({ ...BASE, version: 2, words: ['する'], notes: { '1': ['する'] } }),
+    ),
+    new Set(['する']),
+  );
+
+  assert.deepEqual(
+    knownWordsFromState(
+      parseOrThrow({
+        ...BASE,
+        version: 3,
+        notes: { '1': [{ word: 'する', reading: 'する' }], '2': [{ word: '猫', reading: null }] },
+      }),
+    ),
+    new Set(['する', '猫']),
+  );
+
+  // v4 only adds `tiers`; the words a reader sees must not change.
+  assert.deepEqual(
+    knownWordsFromState(
+      parseOrThrow({
+        ...BASE,
+        version: 4,
+        notes: { '1': [{ word: 'する', reading: 'する' }], '2': [{ word: '猫', reading: null }] },
+        tiers: { '1': 'mature', '2': 'young' },
+      }),
+    ),
+    new Set(['する', '猫']),
+  );
+});
+
+test('known-word cache format rejects payloads that are not a known cache state', () => {
+  const notes = { '1': [{ word: 'する', reading: null }] };
+
+  assert.equal(parseKnownWordCacheState(null), null);
+  assert.equal(parseKnownWordCacheState('{}'), null);
+
+  // An unknown version must not be read as an empty cache.
+  assert.equal(parseKnownWordCacheState({ ...BASE, version: 99, notes }), null);
+
+  assert.equal(
+    parseKnownWordCacheState({ version: 4, scope: 'deck:test', notes, tiers: {} }),
+    null,
+  );
+  assert.equal(parseKnownWordCacheState({ version: 4, refreshedAtMs: 1, notes, tiers: {} }), null);
+
+  // v4 without its tiers map is a v3 payload mislabelled as v4.
+  assert.equal(parseKnownWordCacheState({ ...BASE, version: 4, notes }), null);
+
+  // v3 carries entry objects, not the bare strings v2 used.
+  assert.equal(parseKnownWordCacheState({ ...BASE, version: 3, notes: { '1': ['する'] } }), null);
+});

src/anki-integration/known-word-cache-format.ts

@@ -0,0 +1,141 @@
+// On-disk shape of the known-word cache, plus the only parser for it.
+//
+// Two processes read this file: the cache manager (which rebuilds its indexes
+// from it) and the stats server (which counts known words). They used to carry
+// separate hand-written parsers, so bumping the format to v4 for maturity tiers
+// left the stats server silently reporting zero known words. Everything that
+// touches the format now goes through here, and the version dispatch below ends
+// in assertNever so adding a V5 to the union fails the build at every consumer
+// instead of degrading to an empty result at runtime.
+
+import type { KnownWordMaturityTier } from '../types/subtitle';
+import type { KnownWordEntry } from './known-word-entries';
+
+export interface KnownWordCacheStateV1 {
+  readonly version: 1;
+  readonly refreshedAtMs: number;
+  readonly scope: string;
+  readonly words: string[];
+}
+
+export interface KnownWordCacheStateV2 {
+  readonly version: 2;
+  readonly refreshedAtMs: number;
+  readonly scope: string;
+  readonly words: string[];
+  readonly notes: Record<string, string[]>;
+}
+
+export interface KnownWordCacheStateV3 {
+  readonly version: 3;
+  readonly refreshedAtMs: number;
+  readonly scope: string;
+  readonly notes: Record<string, KnownWordEntry[]>;
+}
+
+export interface KnownWordCacheStateV4 {
+  readonly version: 4;
+  readonly refreshedAtMs: number;
+  readonly scope: string;
+  readonly notes: Record<string, KnownWordEntry[]>;
+  readonly tiers: Record<string, KnownWordMaturityTier>;
+}
+
+export type KnownWordCacheState =
+  | KnownWordCacheStateV1
+  | KnownWordCacheStateV2
+  | KnownWordCacheStateV3
+  | KnownWordCacheStateV4;
+
+// Version written by persistKnownWordCacheState. Readers accept every version
+// in the union above; only the writer pins one.
+export type CurrentKnownWordCacheState = KnownWordCacheStateV4;
+
+// Exported so every consumer that switches on `version` can close its dispatch
+// the same way: a new member of the union becomes a type error at each call
+// site rather than a case that silently falls through.
+export function assertNever(value: never): never {
+  throw new Error(`Unhandled known-word cache state: ${JSON.stringify(value)}`);
+}
+
+function isEntryRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function isKnownWordEntry(entry: unknown): boolean {
+  if (!isEntryRecord(entry)) return false;
+  const candidate = entry as Partial<KnownWordEntry>;
+  return (
+    typeof candidate.word === 'string' &&
+    (candidate.reading === null || typeof candidate.reading === 'string')
+  );
+}
+
+// Returns the narrowed state, or null when the payload is not a cache state we
+// recognize. Per-entry values that are merely unusable (an unknown maturity
+// tier, a non-numeric note id) are dropped by callers at load time rather than
+// rejecting the whole file.
+export function parseKnownWordCacheState(value: unknown): KnownWordCacheState | null {
+  if (!isEntryRecord(value)) return null;
+  const candidate = value;
+  if (
+    candidate.version !== 1 &&
+    candidate.version !== 2 &&
+    candidate.version !== 3 &&
+    candidate.version !== 4
+  ) {
+    return null;
+  }
+  if (typeof candidate.refreshedAtMs !== 'number') return null;
+  if (typeof candidate.scope !== 'string') return null;
+
+  if (candidate.version === 1 || candidate.version === 2) {
+    if (!Array.isArray(candidate.words)) return null;
+    if (!candidate.words.every((entry: unknown) => typeof entry === 'string')) return null;
+  }
+
+  if (candidate.version === 4) {
+    // Per-tier values are sanitized entry-by-entry at load time.
+    if (!isEntryRecord(candidate.tiers)) return null;
+  }
+
+  if (candidate.version === 2 || candidate.version === 3 || candidate.version === 4) {
+    if (!isEntryRecord(candidate.notes)) return null;
+    const isValidNoteEntry =
+      candidate.version === 2
+        ? (entry: unknown): boolean => typeof entry === 'string'
+        : isKnownWordEntry;
+    if (
+      !Object.values(candidate.notes).every(
+        (noteEntries) => Array.isArray(noteEntries) && noteEntries.every(isValidNoteEntry),
+      )
+    ) {
+      return null;
+    }
+  }
+
+  return candidate as unknown as KnownWordCacheState;
+}
+
+// Every word the cache considers known, flattened across notes. Consumers that
+// only need membership (the stats server) use this instead of walking the
+// version-specific layout themselves.
+export function knownWordsFromState(state: KnownWordCacheState): Set<string> {
+  switch (state.version) {
+    case 1:
+    case 2:
+      return new Set(state.words);
+    case 3:
+    case 4: {
+      const words = new Set<string>();
+      for (const entries of Object.values(state.notes)) {
+        for (const entry of entries) {
+          if (entry.word) words.add(entry.word);
+        }
+      }
+      return words;
+    }
+    default:
+      return assertNever(state);
+  }
+}

src/anki-integration/known-word-cache.ts

@@ -15,6 +15,11 @@ import {
   maxKnownWordMaturityTier,
   sanitizeKnownWordMaturityTier,
 } from './known-word-maturity';
+import {
+  CurrentKnownWordCacheState,
+  assertNever,
+  parseKnownWordCacheState,
+} from './known-word-cache-format';
 import {
   DEFAULT_KNOWN_WORD_READING_FIELDS,
   KnownWordEntry,
@@ -94,42 +99,6 @@ export interface KnownWordCacheNoteInfo {
   fields: Record<string, { value: string }>;
 }
 
-interface KnownWordCacheStateV1 {
-  readonly version: 1;
-  readonly refreshedAtMs: number;
-  readonly scope: string;
-  readonly words: string[];
-}
-
-interface KnownWordCacheStateV2 {
-  readonly version: 2;
-  readonly refreshedAtMs: number;
-  readonly scope: string;
-  readonly words: string[];
-  readonly notes: Record<string, string[]>;
-}
-
-interface KnownWordCacheStateV3 {
-  readonly version: 3;
-  readonly refreshedAtMs: number;
-  readonly scope: string;
-  readonly notes: Record<string, KnownWordEntry[]>;
-}
-
-interface KnownWordCacheStateV4 {
-  readonly version: 4;
-  readonly refreshedAtMs: number;
-  readonly scope: string;
-  readonly notes: Record<string, KnownWordEntry[]>;
-  readonly tiers: Record<string, KnownWordMaturityTier>;
-}
-
-type KnownWordCacheState =
-  | KnownWordCacheStateV1
-  | KnownWordCacheStateV2
-  | KnownWordCacheStateV3
-  | KnownWordCacheStateV4;
-
 const NO_READING_KEY = '';
 
 interface KnownWordCacheClient {
@@ -842,8 +811,8 @@ export class KnownWordCacheManager {
         return;
       }
 
-      const parsed = JSON.parse(raw) as unknown;
-      if (!this.isKnownWordCacheStateValid(parsed)) {
+      const parsed = parseKnownWordCacheState(JSON.parse(raw) as unknown);
+      if (!parsed) {
         this.clearInMemoryState();
         this.knownWordsStateKey = this.getKnownWordCacheStateKey();
         return;
@@ -856,7 +825,36 @@ export class KnownWordCacheManager {
       }
 
       this.clearInMemoryState();
-      if (parsed.version === 3 || parsed.version === 4) {
+      switch (parsed.version) {
+        case 1:
+          // v1 has no per-note snapshots to convert; refetch from Anki.
+          this.knownWordsStateKey = this.getKnownWordCacheStateKey();
+          return;
+        case 2:
+          // Older states have no readings; load them reading-less (fail-open,
+          // matching the old behavior) but leave the cache marked stale so the
+          // next refresh upgrades entries with readings from Anki.
+          for (const [noteIdKey, words] of Object.entries(parsed.notes)) {
+            const noteId = Number.parseInt(noteIdKey, 10);
+            if (!Number.isInteger(noteId) || noteId <= 0) {
+              continue;
+            }
+            const normalizedEntries = normalizeKnownWordEntryList(
+              words.map((word) => ({
+                word: this.normalizeKnownWordForLookup(word),
+                reading: null,
+              })),
+            );
+            if (normalizedEntries.length === 0) {
+              continue;
+            }
+            this.noteEntriesById.set(noteId, normalizedEntries);
+            this.addEntriesToIndexes(noteId, normalizedEntries);
+          }
+          this.knownWordsStateKey = parsed.scope;
+          return;
+        case 3:
+        case 4:
           for (const [noteIdKey, entries] of Object.entries(parsed.notes)) {
             const noteId = Number.parseInt(noteIdKey, 10);
             if (!Number.isInteger(noteId) || noteId <= 0) {
@@ -881,32 +879,9 @@ export class KnownWordCacheManager {
           this.knownWordsLastRefreshedAtMs = parsed.refreshedAtMs;
           this.knownWordsStateKey = parsed.scope;
           return;
+        default:
+          assertNever(parsed);
       }
-
-      if (parsed.version === 2) {
-        // Older states have no readings; load them reading-less (fail-open,
-        // matching the old behavior) but leave the cache marked stale so the
-        // next refresh upgrades entries with readings from Anki.
-        for (const [noteIdKey, words] of Object.entries(parsed.notes)) {
-          const noteId = Number.parseInt(noteIdKey, 10);
-          if (!Number.isInteger(noteId) || noteId <= 0) {
-            continue;
-          }
-          const normalizedEntries = normalizeKnownWordEntryList(
-            words.map((word) => ({ word: this.normalizeKnownWordForLookup(word), reading: null })),
-          );
-          if (normalizedEntries.length === 0) {
-            continue;
-          }
-          this.noteEntriesById.set(noteId, normalizedEntries);
-          this.addEntriesToIndexes(noteId, normalizedEntries);
-        }
-        this.knownWordsStateKey = parsed.scope;
-        return;
-      }
-
-      // v1 has no per-note snapshots to convert; refetch from Anki.
-      this.knownWordsStateKey = this.getKnownWordCacheStateKey();
     } catch (error) {
       log.warn('Failed to load known-word cache state:', (error as Error).message);
       this.clearInMemoryState();
@@ -928,7 +903,7 @@ export class KnownWordCacheManager {
         }
       }
 
-      const state: KnownWordCacheStateV4 = {
+      const state: CurrentKnownWordCacheState = {
         version: 4,
         refreshedAtMs: this.knownWordsLastRefreshedAtMs,
         scope: this.knownWordsStateKey,
@@ -941,63 +916,6 @@ export class KnownWordCacheManager {
     }
   }
 
-  private isKnownWordCacheStateValid(value: unknown): value is KnownWordCacheState {
-    if (typeof value !== 'object' || value === null) return false;
-    const candidate = value as Record<string, unknown>;
-    if (
-      candidate.version !== 1 &&
-      candidate.version !== 2 &&
-      candidate.version !== 3 &&
-      candidate.version !== 4
-    ) {
-      return false;
-    }
-    if (typeof candidate.refreshedAtMs !== 'number') return false;
-    if (typeof candidate.scope !== 'string') return false;
-    if (candidate.version === 1 || candidate.version === 2) {
-      if (!Array.isArray(candidate.words)) return false;
-      if (!candidate.words.every((entry: unknown) => typeof entry === 'string')) {
-        return false;
-      }
-    }
-    if (candidate.version === 4) {
-      // Per-tier values are sanitized entry-by-entry at load time.
-      if (
-        typeof candidate.tiers !== 'object' ||
-        candidate.tiers === null ||
-        Array.isArray(candidate.tiers)
-      ) {
-        return false;
-      }
-    }
-    if (candidate.version === 2 || candidate.version === 3 || candidate.version === 4) {
-      if (
-        typeof candidate.notes !== 'object' ||
-        candidate.notes === null ||
-        Array.isArray(candidate.notes)
-      ) {
-        return false;
-      }
-      const isValidNoteEntry =
-        candidate.version === 2
-          ? (entry: unknown): boolean => typeof entry === 'string'
-          : (entry: unknown): boolean =>
-              typeof entry === 'object' &&
-              entry !== null &&
-              typeof (entry as KnownWordEntry).word === 'string' &&
-              ((entry as KnownWordEntry).reading === null ||
-                typeof (entry as KnownWordEntry).reading === 'string');
-      if (
-        !Object.values(candidate.notes as Record<string, unknown>).every(
-          (noteEntries) => Array.isArray(noteEntries) && noteEntries.every(isValidNoteEntry),
-        )
-      ) {
-        return false;
-      }
-    }
-    return true;
-  }
-
   private extractKnownWordEntriesFromNoteInfo(
     noteInfo: KnownWordCacheNoteInfo,
     preferredFields = this.getConfiguredFields(),

src/core/services/__tests__/stats-server.test.ts

@@ -497,6 +497,8 @@ describe('stats server API routes', () => {
         cachePath,
         JSON.stringify({
           version: 1,
+          refreshedAtMs: 1,
+          scope: 'deck:test',
           words: ['する'],
         }),
       );
@@ -561,6 +563,69 @@ describe('stats server API routes', () => {
     });
   });
 
+  it('GET /api/stats/sessions enriches known-word metrics from a v4 maturity cache', async () => {
+    await withTempDir(async (dir) => {
+      const cachePath = path.join(dir, 'known-words.json');
+      fs.writeFileSync(
+        cachePath,
+        JSON.stringify({
+          version: 4,
+          refreshedAtMs: 1,
+          scope: 'deck:test',
+          notes: {
+            '101': [{ word: 'する', reading: 'する' }],
+            '102': [{ word: '猫', reading: null }],
+          },
+          tiers: { '101': 'mature', '102': 'young' },
+        }),
+      );
+
+      const app = createStatsApp(
+        createMockTracker({
+          getSessionWordsByLine: async (sessionId: number) =>
+            sessionId === 1
+              ? [
+                  { lineIndex: 1, headword: 'する', occurrenceCount: 2 },
+                  { lineIndex: 2, headword: '未知', occurrenceCount: 1 },
+                ]
+              : [],
+        }),
+        { knownWordCachePath: cachePath },
+      );
+
+      const res = await app.request('/api/stats/sessions?limit=5');
+      assert.equal(res.status, 200);
+      const body = await res.json();
+      const first = body[0];
+      assert.equal(first.knownWordsSeen, 2);
+      assert.equal(first.knownWordRate, 66.7);
+    });
+  });
+
+  it('GET /api/stats/sessions reports no known words when the cache format is unrecognized', async () => {
+    await withTempDir(async (dir) => {
+      const cachePath = path.join(dir, 'known-words.json');
+      fs.writeFileSync(
+        cachePath,
+        JSON.stringify({ version: 99, refreshedAtMs: 1, scope: 'deck:test', notes: {} }),
+      );
+
+      const app = createStatsApp(
+        createMockTracker({
+          getSessionWordsByLine: async () => [
+            { lineIndex: 1, headword: 'する', occurrenceCount: 2 },
+          ],
+        }),
+        { knownWordCachePath: cachePath },
+      );
+
+      const res = await app.request('/api/stats/sessions?limit=5');
+      assert.equal(res.status, 200);
+      const body = await res.json();
+      assert.equal(body[0].knownWordsSeen, 0);
+    });
+  });
+
   it('GET /api/stats/sessions/:id/events forwards event type filters to the tracker', async () => {
     let seenSessionId = 0;
     let seenLimit = 0;
@@ -609,6 +674,8 @@ describe('stats server API routes', () => {
         cachePath,
         JSON.stringify({
           version: 1,
+          refreshedAtMs: 1,
+          scope: 'deck:test',
           words: ['知る', '猫'],
         }),
       );

src/core/services/stats-server/route-support.ts

@@ -6,11 +6,18 @@ import {
   getConfiguredWordFieldName,
   getPreferredNoteFieldValue,
 } from '../../../anki-field-config.js';
+import {
+  knownWordsFromState,
+  parseKnownWordCacheState,
+} from '../../../anki-integration/known-word-cache-format.js';
+import { createLogger } from '../../../logger.js';
 import type { AnkiConnectConfig } from '../../../types.js';
 import type { StatsExcludedWord } from '../../../types/stats-wire.js';
 import type { ImmersionTrackerService } from '../immersion-tracker-service.js';
 import { splitSentenceSearchTerms } from '../immersion-tracker/query-lexical.js';
 
+const statsKnownWordsLogger = createLogger('stats:known-words');
+
 const STATS_STATIC_CONTENT_TYPES: Record<string, string> = {
   '.css': 'text/css; charset=utf-8',
   '.gif': 'image/gif',
@@ -84,24 +91,14 @@ export function parseExcludedWordsBody(body: unknown): StatsExcludedWord[] | nul
 export function loadKnownWordsSet(cachePath: string | undefined): Set<string> | null {
   if (!cachePath || !existsSync(cachePath)) return null;
   try {
-    const raw = JSON.parse(readFileSync(cachePath, 'utf-8')) as {
-      version?: number;
-      words?: string[];
-      notes?: Record<string, Array<{ word?: unknown; reading?: unknown }>>;
-    };
-    if ((raw.version === 1 || raw.version === 2) && Array.isArray(raw.words)) {
-      return new Set(raw.words);
-    }
-    if (raw.version === 3 && raw.notes && typeof raw.notes === 'object') {
-      const words = new Set<string>();
-      for (const entries of Object.values(raw.notes)) {
-        if (!Array.isArray(entries)) continue;
-        for (const entry of entries) {
-          if (entry && typeof entry.word === 'string' && entry.word) words.add(entry.word);
-        }
-      }
-      return words;
+    const state = parseKnownWordCacheState(JSON.parse(readFileSync(cachePath, 'utf-8')) as unknown);
+    if (!state) {
+      // A cache that exists but does not parse is a format mismatch, not an
+      // empty collection; say so instead of reporting zero known words.
+      statsKnownWordsLogger.warn(`Unrecognized known-word cache format at ${cachePath}`);
+      return null;
     }
+    return knownWordsFromState(state);
   } catch {
     // Treat an unreadable cache as unavailable.
   }