Anki maturity-based known-word highlighting (#172)

src/core/services/__tests__/stats-server.test.ts

@@ -497,6 +497,8 @@ describe('stats server API routes', () => {
         cachePath,
         JSON.stringify({
           version: 1,
+          refreshedAtMs: 1,
+          scope: 'deck:test',
           words: ['する'],
         }),
       );
@@ -561,6 +563,69 @@ describe('stats server API routes', () => {
     });
   });
 
+  it('GET /api/stats/sessions enriches known-word metrics from a v4 maturity cache', async () => {
+    await withTempDir(async (dir) => {
+      const cachePath = path.join(dir, 'known-words.json');
+      fs.writeFileSync(
+        cachePath,
+        JSON.stringify({
+          version: 4,
+          refreshedAtMs: 1,
+          scope: 'deck:test',
+          notes: {
+            '101': [{ word: 'する', reading: 'する' }],
+            '102': [{ word: '猫', reading: null }],
+          },
+          tiers: { '101': 'mature', '102': 'young' },
+        }),
+      );
+
+      const app = createStatsApp(
+        createMockTracker({
+          getSessionWordsByLine: async (sessionId: number) =>
+            sessionId === 1
+              ? [
+                  { lineIndex: 1, headword: 'する', occurrenceCount: 2 },
+                  { lineIndex: 2, headword: '未知', occurrenceCount: 1 },
+                ]
+              : [],
+        }),
+        { knownWordCachePath: cachePath },
+      );
+
+      const res = await app.request('/api/stats/sessions?limit=5');
+      assert.equal(res.status, 200);
+      const body = await res.json();
+      const first = body[0];
+      assert.equal(first.knownWordsSeen, 2);
+      assert.equal(first.knownWordRate, 66.7);
+    });
+  });
+
+  it('GET /api/stats/sessions reports no known words when the cache format is unrecognized', async () => {
+    await withTempDir(async (dir) => {
+      const cachePath = path.join(dir, 'known-words.json');
+      fs.writeFileSync(
+        cachePath,
+        JSON.stringify({ version: 99, refreshedAtMs: 1, scope: 'deck:test', notes: {} }),
+      );
+
+      const app = createStatsApp(
+        createMockTracker({
+          getSessionWordsByLine: async () => [
+            { lineIndex: 1, headword: 'する', occurrenceCount: 2 },
+          ],
+        }),
+        { knownWordCachePath: cachePath },
+      );
+
+      const res = await app.request('/api/stats/sessions?limit=5');
+      assert.equal(res.status, 200);
+      const body = await res.json();
+      assert.equal(body[0].knownWordsSeen, 0);
+    });
+  });
+
   it('GET /api/stats/sessions/:id/events forwards event type filters to the tracker', async () => {
     let seenSessionId = 0;
     let seenLimit = 0;
@@ -609,6 +674,8 @@ describe('stats server API routes', () => {
         cachePath,
         JSON.stringify({
           version: 1,
+          refreshedAtMs: 1,
+          scope: 'deck:test',
           words: ['知る', '猫'],
         }),
       );

src/core/services/stats-server/route-support.ts

@@ -6,11 +6,18 @@ import {
   getConfiguredWordFieldName,
   getPreferredNoteFieldValue,
 } from '../../../anki-field-config.js';
+import {
+  knownWordsFromState,
+  parseKnownWordCacheState,
+} from '../../../anki-integration/known-word-cache-format.js';
+import { createLogger } from '../../../logger.js';
 import type { AnkiConnectConfig } from '../../../types.js';
 import type { StatsExcludedWord } from '../../../types/stats-wire.js';
 import type { ImmersionTrackerService } from '../immersion-tracker-service.js';
 import { splitSentenceSearchTerms } from '../immersion-tracker/query-lexical.js';
 
+const statsKnownWordsLogger = createLogger('stats:known-words');
+
 const STATS_STATIC_CONTENT_TYPES: Record<string, string> = {
   '.css': 'text/css; charset=utf-8',
   '.gif': 'image/gif',
@@ -84,24 +91,14 @@ export function parseExcludedWordsBody(body: unknown): StatsExcludedWord[] | nul
 export function loadKnownWordsSet(cachePath: string | undefined): Set<string> | null {
   if (!cachePath || !existsSync(cachePath)) return null;
   try {
-    const raw = JSON.parse(readFileSync(cachePath, 'utf-8')) as {
-      version?: number;
-      words?: string[];
-      notes?: Record<string, Array<{ word?: unknown; reading?: unknown }>>;
-    };
-    if ((raw.version === 1 || raw.version === 2) && Array.isArray(raw.words)) {
-      return new Set(raw.words);
-    }
-    if (raw.version === 3 && raw.notes && typeof raw.notes === 'object') {
-      const words = new Set<string>();
-      for (const entries of Object.values(raw.notes)) {
-        if (!Array.isArray(entries)) continue;
-        for (const entry of entries) {
-          if (entry && typeof entry.word === 'string' && entry.word) words.add(entry.word);
-        }
-      }
-      return words;
+    const state = parseKnownWordCacheState(JSON.parse(readFileSync(cachePath, 'utf-8')) as unknown);
+    if (!state) {
+      // A cache that exists but does not parse is a format mismatch, not an
+      // empty collection; say so instead of reporting zero known words.
+      statsKnownWordsLogger.warn(`Unrecognized known-word cache format at ${cachePath}`);
+      return null;
     }
+    return knownWordsFromState(state);
   } catch {
     // Treat an unreadable cache as unavailable.
   }

src/core/services/tokenizer.ts

@@ -10,6 +10,7 @@ import {
   Token,
   FrequencyDictionaryLookup,
   JlptLevel,
+  KnownWordMaturityTier,
   PartOfSpeech,
 } from '../../types';
 import {
@@ -43,6 +44,12 @@ export type KnownWordLookupFn = (
   options?: { allowReadingOnlyMatch?: boolean },
 ) => boolean;
 
+export type KnownWordTierLookupFn = (
+  text: string,
+  reading?: string,
+  options?: { allowReadingOnlyMatch?: boolean },
+) => KnownWordMaturityTier | null;
+
 export interface TokenizerServiceDeps {
   getYomitanExt: () => Extension | null;
   getYomitanSession?: () => Session | null;
@@ -53,6 +60,7 @@ export interface TokenizerServiceDeps {
   getYomitanParserInitPromise: () => Promise<boolean> | null;
   setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
   isKnownWord: KnownWordLookupFn;
+  getKnownWordTier?: KnownWordTierLookupFn;
   getKnownWordMatchMode: () => NPlusOneMatchMode;
   getKnownWordsEnabled?: () => boolean;
   getJlptLevel: (text: string) => JlptLevel | null;
@@ -88,6 +96,7 @@ export interface TokenizerDepsRuntimeOptions {
   getYomitanParserInitPromise: () => Promise<boolean> | null;
   setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
   isKnownWord: KnownWordLookupFn;
+  getKnownWordTier?: KnownWordTierLookupFn;
   getKnownWordMatchMode: () => NPlusOneMatchMode;
   getKnownWordsEnabled?: () => boolean;
   getJlptLevel: (text: string) => JlptLevel | null;
@@ -204,6 +213,11 @@ async function applyAnnotationStage(
     tokens,
     {
       isKnownWord: getKnownWordLookup(deps, options),
+      // Maturity tiers only refine known-word rendering, so they follow the
+      // known-word toggle rather than the N+1 gate.
+      ...(options.knownWordsEnabled && deps.getKnownWordTier
+        ? { getKnownWordTier: deps.getKnownWordTier }
+        : {}),
       knownWordMatchMode: deps.getKnownWordMatchMode(),
       getJlptLevel: deps.getJlptLevel,
     },
@@ -242,6 +256,7 @@ export function createTokenizerDepsRuntime(
     getYomitanParserInitPromise: options.getYomitanParserInitPromise,
     setYomitanParserInitPromise: options.setYomitanParserInitPromise,
     isKnownWord: options.isKnownWord,
+    getKnownWordTier: options.getKnownWordTier,
     getKnownWordMatchMode: options.getKnownWordMatchMode,
     getKnownWordsEnabled: options.getKnownWordsEnabled,
     getJlptLevel: options.getJlptLevel,

src/core/services/tokenizer/annotation-stage-maturity.test.ts

@@ -0,0 +1,130 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { KnownWordMaturityTier, MergedToken, PartOfSpeech } from '../../../types';
+import { annotateTokens, AnnotationStageDeps } from './annotation-stage';
+
+function makeToken(overrides: Partial<MergedToken> = {}): MergedToken {
+  return {
+    surface: '猫',
+    reading: 'ネコ',
+    headword: '猫',
+    startPos: 0,
+    endPos: 1,
+    partOfSpeech: PartOfSpeech.noun,
+    isMerged: false,
+    isKnown: false,
+    isNPlusOneTarget: false,
+    ...overrides,
+  };
+}
+
+function makeDeps(overrides: Partial<AnnotationStageDeps> = {}): AnnotationStageDeps {
+  return {
+    isKnownWord: () => false,
+    knownWordMatchMode: 'headword',
+    getJlptLevel: () => null,
+    ...overrides,
+  };
+}
+
+test('annotateTokens attaches maturity tier to known tokens', () => {
+  const tokens = [makeToken({ surface: '食べた', headword: '食べる', reading: 'タベタ' })];
+  const result = annotateTokens(
+    tokens,
+    makeDeps({
+      isKnownWord: (text) => text === '食べる',
+      getKnownWordTier: (text) => (text === '食べる' ? 'mature' : null),
+    }),
+  );
+
+  assert.equal(result[0]?.isKnown, true);
+  assert.equal(result[0]?.knownMaturity, 'mature');
+});
+
+test('annotateTokens leaves maturity undefined without a tier lookup dep', () => {
+  const tokens = [makeToken()];
+  const result = annotateTokens(tokens, makeDeps({ isKnownWord: () => true }));
+
+  assert.equal(result[0]?.isKnown, true);
+  assert.equal(result[0]?.knownMaturity, undefined);
+});
+
+test('annotateTokens leaves maturity undefined when the tier lookup has no data', () => {
+  const tokens = [makeToken()];
+  const result = annotateTokens(
+    tokens,
+    makeDeps({ isKnownWord: () => true, getKnownWordTier: () => null }),
+  );
+
+  assert.equal(result[0]?.isKnown, true);
+  assert.equal(result[0]?.knownMaturity, undefined);
+});
+
+test('annotateTokens never attaches maturity to unknown tokens', () => {
+  const tokens = [makeToken()];
+  const result = annotateTokens(
+    tokens,
+    makeDeps({ isKnownWord: () => false, getKnownWordTier: () => 'mature' }),
+  );
+
+  assert.equal(result[0]?.isKnown, false);
+  assert.equal(result[0]?.knownMaturity, undefined);
+});
+
+test('annotateTokens resolves maturity through the kana reading fallback', () => {
+  // Token 大体 with a card mined in kana (だいたい): known status comes from
+  // the reading fallback, so the tier must follow the same path.
+  const tierByText = new Map<string, KnownWordMaturityTier>([['だいたい', 'young']]);
+  const seenTierLookups: Array<{
+    text: string;
+    reading: string | undefined;
+    allowReadingOnlyMatch: boolean | undefined;
+  }> = [];
+  const tokens = [makeToken({ surface: '大体', headword: '大体', reading: 'だいたい', endPos: 2 })];
+
+  const result = annotateTokens(
+    tokens,
+    makeDeps({
+      isKnownWord: (text) => text === 'だいたい',
+      getKnownWordTier: (text, reading, options) => {
+        seenTierLookups.push({
+          text,
+          reading,
+          allowReadingOnlyMatch: options?.allowReadingOnlyMatch,
+        });
+        return tierByText.get(text) ?? null;
+      },
+    }),
+  );
+
+  assert.equal(result[0]?.isKnown, true);
+  assert.equal(result[0]?.knownMaturity, 'young');
+  // The fallback lookup must opt out of reading-only matching, exactly like
+  // the boolean known-status fallback.
+  const fallbackLookup = seenTierLookups.find((lookup) => lookup.text === 'だいたい');
+  assert.equal(fallbackLookup?.allowReadingOnlyMatch, false);
+});
+
+test('annotateTokens keeps maturity on POS-excluded known tokens', () => {
+  // Known-word annotations survive the POS noise filter; the tier must too.
+  const tokens = [makeToken({ surface: 'の', headword: 'の', reading: 'ノ', pos1: '助詞' })];
+  const result = annotateTokens(
+    tokens,
+    makeDeps({ isKnownWord: () => true, getKnownWordTier: () => 'learning' }),
+  );
+
+  assert.equal(result[0]?.isKnown, true);
+  assert.equal(result[0]?.knownMaturity, 'learning');
+});
+
+test('annotateTokens strips maturity when known-word annotation is disabled', () => {
+  const tokens = [makeToken({ knownMaturity: 'mature' })];
+  const result = annotateTokens(
+    tokens,
+    makeDeps({ isKnownWord: () => true, getKnownWordTier: () => 'mature' }),
+    { knownWordsEnabled: false },
+  );
+
+  assert.equal(result[0]?.isKnown, false);
+  assert.equal(result[0]?.knownMaturity, undefined);
+});

src/core/services/tokenizer/annotation-stage.ts

@@ -7,7 +7,13 @@ import {
   DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG,
   resolveAnnotationPos2ExclusionSet,
 } from '../../../token-pos2-exclusions';
-import { JlptLevel, MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types';
+import {
+  JlptLevel,
+  KnownWordMaturityTier,
+  MergedToken,
+  NPlusOneMatchMode,
+  PartOfSpeech,
+} from '../../../types';
 import { shouldIgnoreJlptByTerm, shouldIgnoreJlptForMecabPos1 } from '../jlpt-token-filter';
 import {
   shouldExcludeTokenFromSubtitleAnnotations as sharedShouldExcludeTokenFromSubtitleAnnotations,
@@ -36,6 +42,11 @@ export interface AnnotationStageDeps {
     reading?: string,
     options?: { allowReadingOnlyMatch?: boolean },
   ) => boolean;
+  getKnownWordTier?: (
+    text: string,
+    reading?: string,
+    options?: { allowReadingOnlyMatch?: boolean },
+  ) => KnownWordMaturityTier | null;
   knownWordMatchMode: NPlusOneMatchMode;
   getJlptLevel: (text: string) => JlptLevel | null;
 }
@@ -52,7 +63,7 @@ export interface AnnotationStageOptions {
   sourceText?: string;
 }
 
-function resolveKnownWordText(
+export function resolveKnownWordText(
   surface: string,
   headword: string,
   matchMode: NPlusOneMatchMode,
@@ -560,7 +571,7 @@ function isCompleteReadingForSurface(surface: string, reading: string): boolean
 // (see isCompleteReadingForSurface); undefined otherwise. Shared so the
 // known-word reading disambiguation and the reading fallback stay in sync if the
 // validity rule changes.
-function resolveCompleteTokenReading(token: MergedToken): string | undefined {
+export function resolveCompleteTokenReading(token: MergedToken): string | undefined {
   const normalizedReading = token.reading.trim();
   if (!normalizedReading || !isCompleteReadingForSurface(token.surface, normalizedReading)) {
     return undefined;
@@ -573,7 +584,7 @@ function resolveCompleteTokenReading(token: MergedToken): string | undefined {
 // inflected surface's reading does not match the dictionary form's reading,
 // and partial furigana readings (see isCompleteReadingForSurface) would cause
 // false negatives. Undefined falls back to text-only matching (fail-open).
-function resolveKnownWordReadingForMatch(
+export function resolveKnownWordReadingForMatch(
   token: MergedToken,
   knownWordMatchMode: NPlusOneMatchMode,
 ): string | undefined {
@@ -616,6 +627,30 @@ function computeTokenKnownStatus(
   );
 }
 
+// Maturity tier for a token already confirmed known, following the same
+// primary + kana-fallback lookup sequence as computeTokenKnownStatus so the
+// tier always describes a note the boolean match could have come from.
+function computeTokenKnownMaturity(
+  token: MergedToken,
+  getKnownWordTier: NonNullable<AnnotationStageDeps['getKnownWordTier']>,
+  knownWordMatchMode: NPlusOneMatchMode,
+): KnownWordMaturityTier | undefined {
+  const matchText = resolveKnownWordText(token.surface, token.headword, knownWordMatchMode);
+  const matchReading = resolveKnownWordReadingForMatch(token, knownWordMatchMode);
+  const primaryTier = matchText ? getKnownWordTier(matchText, matchReading) : null;
+  if (primaryTier) {
+    return primaryTier;
+  }
+
+  const fallbackReading = resolveCompleteTokenReading(token);
+  if (!fallbackReading || fallbackReading === matchText.trim()) {
+    return undefined;
+  }
+  return (
+    getKnownWordTier(fallbackReading, undefined, { allowReadingOnlyMatch: false }) ?? undefined
+  );
+}
+
 function filterTokenFrequencyRank(
   token: MergedToken,
   pos1Exclusions: ReadonlySet<string>,
@@ -677,6 +712,11 @@ export function annotateTokens(
       : false;
     nPlusOneKnownStatuses[index] = isKnownForMatching;
 
+    const knownMaturity =
+      knownWordsEnabled && isKnownForMatching && deps.getKnownWordTier
+        ? computeTokenKnownMaturity(token, deps.getKnownWordTier, deps.knownWordMatchMode)
+        : undefined;
+
     const prioritizedNameMatch = nameMatchEnabled && token.isNameMatch === true;
 
     // A confirmed character-name match must survive the POS noise filter:
@@ -696,6 +736,7 @@ export function annotateTokens(
       return {
         ...strippedToken,
         isKnown: knownWordsEnabled ? isKnownForMatching : false,
+        knownMaturity,
       };
     }
 
@@ -712,6 +753,7 @@ export function annotateTokens(
     return {
       ...token,
       isKnown: knownWordsEnabled ? isKnownForMatching : false,
+      knownMaturity,
       isNPlusOneTarget: nPlusOneEnabled && !prioritizedNameMatch ? token.isNPlusOneTarget : false,
       frequencyRank,
       jlptLevel,