Anki maturity-based known-word highlighting (#172)

src/core/services/tokenizer/annotation-stage-maturity.test.ts

@@ -0,0 +1,130 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { KnownWordMaturityTier, MergedToken, PartOfSpeech } from '../../../types';
+import { annotateTokens, AnnotationStageDeps } from './annotation-stage';
+
+function makeToken(overrides: Partial<MergedToken> = {}): MergedToken {
+  return {
+    surface: '猫',
+    reading: 'ネコ',
+    headword: '猫',
+    startPos: 0,
+    endPos: 1,
+    partOfSpeech: PartOfSpeech.noun,
+    isMerged: false,
+    isKnown: false,
+    isNPlusOneTarget: false,
+    ...overrides,
+  };
+}
+
+function makeDeps(overrides: Partial<AnnotationStageDeps> = {}): AnnotationStageDeps {
+  return {
+    isKnownWord: () => false,
+    knownWordMatchMode: 'headword',
+    getJlptLevel: () => null,
+    ...overrides,
+  };
+}
+
+test('annotateTokens attaches maturity tier to known tokens', () => {
+  const tokens = [makeToken({ surface: '食べた', headword: '食べる', reading: 'タベタ' })];
+  const result = annotateTokens(
+    tokens,
+    makeDeps({
+      isKnownWord: (text) => text === '食べる',
+      getKnownWordTier: (text) => (text === '食べる' ? 'mature' : null),
+    }),
+  );
+
+  assert.equal(result[0]?.isKnown, true);
+  assert.equal(result[0]?.knownMaturity, 'mature');
+});
+
+test('annotateTokens leaves maturity undefined without a tier lookup dep', () => {
+  const tokens = [makeToken()];
+  const result = annotateTokens(tokens, makeDeps({ isKnownWord: () => true }));
+
+  assert.equal(result[0]?.isKnown, true);
+  assert.equal(result[0]?.knownMaturity, undefined);
+});
+
+test('annotateTokens leaves maturity undefined when the tier lookup has no data', () => {
+  const tokens = [makeToken()];
+  const result = annotateTokens(
+    tokens,
+    makeDeps({ isKnownWord: () => true, getKnownWordTier: () => null }),
+  );
+
+  assert.equal(result[0]?.isKnown, true);
+  assert.equal(result[0]?.knownMaturity, undefined);
+});
+
+test('annotateTokens never attaches maturity to unknown tokens', () => {
+  const tokens = [makeToken()];
+  const result = annotateTokens(
+    tokens,
+    makeDeps({ isKnownWord: () => false, getKnownWordTier: () => 'mature' }),
+  );
+
+  assert.equal(result[0]?.isKnown, false);
+  assert.equal(result[0]?.knownMaturity, undefined);
+});
+
+test('annotateTokens resolves maturity through the kana reading fallback', () => {
+  // Token 大体 with a card mined in kana (だいたい): known status comes from
+  // the reading fallback, so the tier must follow the same path.
+  const tierByText = new Map<string, KnownWordMaturityTier>([['だいたい', 'young']]);
+  const seenTierLookups: Array<{
+    text: string;
+    reading: string | undefined;
+    allowReadingOnlyMatch: boolean | undefined;
+  }> = [];
+  const tokens = [makeToken({ surface: '大体', headword: '大体', reading: 'だいたい', endPos: 2 })];
+
+  const result = annotateTokens(
+    tokens,
+    makeDeps({
+      isKnownWord: (text) => text === 'だいたい',
+      getKnownWordTier: (text, reading, options) => {
+        seenTierLookups.push({
+          text,
+          reading,
+          allowReadingOnlyMatch: options?.allowReadingOnlyMatch,
+        });
+        return tierByText.get(text) ?? null;
+      },
+    }),
+  );
+
+  assert.equal(result[0]?.isKnown, true);
+  assert.equal(result[0]?.knownMaturity, 'young');
+  // The fallback lookup must opt out of reading-only matching, exactly like
+  // the boolean known-status fallback.
+  const fallbackLookup = seenTierLookups.find((lookup) => lookup.text === 'だいたい');
+  assert.equal(fallbackLookup?.allowReadingOnlyMatch, false);
+});
+
+test('annotateTokens keeps maturity on POS-excluded known tokens', () => {
+  // Known-word annotations survive the POS noise filter; the tier must too.
+  const tokens = [makeToken({ surface: 'の', headword: 'の', reading: 'ノ', pos1: '助詞' })];
+  const result = annotateTokens(
+    tokens,
+    makeDeps({ isKnownWord: () => true, getKnownWordTier: () => 'learning' }),
+  );
+
+  assert.equal(result[0]?.isKnown, true);
+  assert.equal(result[0]?.knownMaturity, 'learning');
+});
+
+test('annotateTokens strips maturity when known-word annotation is disabled', () => {
+  const tokens = [makeToken({ knownMaturity: 'mature' })];
+  const result = annotateTokens(
+    tokens,
+    makeDeps({ isKnownWord: () => true, getKnownWordTier: () => 'mature' }),
+    { knownWordsEnabled: false },
+  );
+
+  assert.equal(result[0]?.isKnown, false);
+  assert.equal(result[0]?.knownMaturity, undefined);
+});

src/core/services/tokenizer/annotation-stage.ts

@@ -7,7 +7,13 @@ import {
   DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG,
   resolveAnnotationPos2ExclusionSet,
 } from '../../../token-pos2-exclusions';
-import { JlptLevel, MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types';
+import {
+  JlptLevel,
+  KnownWordMaturityTier,
+  MergedToken,
+  NPlusOneMatchMode,
+  PartOfSpeech,
+} from '../../../types';
 import { shouldIgnoreJlptByTerm, shouldIgnoreJlptForMecabPos1 } from '../jlpt-token-filter';
 import {
   shouldExcludeTokenFromSubtitleAnnotations as sharedShouldExcludeTokenFromSubtitleAnnotations,
@@ -36,6 +42,11 @@ export interface AnnotationStageDeps {
     reading?: string,
     options?: { allowReadingOnlyMatch?: boolean },
   ) => boolean;
+  getKnownWordTier?: (
+    text: string,
+    reading?: string,
+    options?: { allowReadingOnlyMatch?: boolean },
+  ) => KnownWordMaturityTier | null;
   knownWordMatchMode: NPlusOneMatchMode;
   getJlptLevel: (text: string) => JlptLevel | null;
 }
@@ -52,7 +63,7 @@ export interface AnnotationStageOptions {
   sourceText?: string;
 }
 
-function resolveKnownWordText(
+export function resolveKnownWordText(
   surface: string,
   headword: string,
   matchMode: NPlusOneMatchMode,
@@ -560,7 +571,7 @@ function isCompleteReadingForSurface(surface: string, reading: string): boolean
 // (see isCompleteReadingForSurface); undefined otherwise. Shared so the
 // known-word reading disambiguation and the reading fallback stay in sync if the
 // validity rule changes.
-function resolveCompleteTokenReading(token: MergedToken): string | undefined {
+export function resolveCompleteTokenReading(token: MergedToken): string | undefined {
   const normalizedReading = token.reading.trim();
   if (!normalizedReading || !isCompleteReadingForSurface(token.surface, normalizedReading)) {
     return undefined;
@@ -573,7 +584,7 @@ function resolveCompleteTokenReading(token: MergedToken): string | undefined {
 // inflected surface's reading does not match the dictionary form's reading,
 // and partial furigana readings (see isCompleteReadingForSurface) would cause
 // false negatives. Undefined falls back to text-only matching (fail-open).
-function resolveKnownWordReadingForMatch(
+export function resolveKnownWordReadingForMatch(
   token: MergedToken,
   knownWordMatchMode: NPlusOneMatchMode,
 ): string | undefined {
@@ -616,6 +627,30 @@ function computeTokenKnownStatus(
   );
 }
 
+// Maturity tier for a token already confirmed known, following the same
+// primary + kana-fallback lookup sequence as computeTokenKnownStatus so the
+// tier always describes a note the boolean match could have come from.
+function computeTokenKnownMaturity(
+  token: MergedToken,
+  getKnownWordTier: NonNullable<AnnotationStageDeps['getKnownWordTier']>,
+  knownWordMatchMode: NPlusOneMatchMode,
+): KnownWordMaturityTier | undefined {
+  const matchText = resolveKnownWordText(token.surface, token.headword, knownWordMatchMode);
+  const matchReading = resolveKnownWordReadingForMatch(token, knownWordMatchMode);
+  const primaryTier = matchText ? getKnownWordTier(matchText, matchReading) : null;
+  if (primaryTier) {
+    return primaryTier;
+  }
+
+  const fallbackReading = resolveCompleteTokenReading(token);
+  if (!fallbackReading || fallbackReading === matchText.trim()) {
+    return undefined;
+  }
+  return (
+    getKnownWordTier(fallbackReading, undefined, { allowReadingOnlyMatch: false }) ?? undefined
+  );
+}
+
 function filterTokenFrequencyRank(
   token: MergedToken,
   pos1Exclusions: ReadonlySet<string>,
@@ -677,6 +712,11 @@ export function annotateTokens(
       : false;
     nPlusOneKnownStatuses[index] = isKnownForMatching;
 
+    const knownMaturity =
+      knownWordsEnabled && isKnownForMatching && deps.getKnownWordTier
+        ? computeTokenKnownMaturity(token, deps.getKnownWordTier, deps.knownWordMatchMode)
+        : undefined;
+
     const prioritizedNameMatch = nameMatchEnabled && token.isNameMatch === true;
 
     // A confirmed character-name match must survive the POS noise filter:
@@ -696,6 +736,7 @@ export function annotateTokens(
       return {
         ...strippedToken,
         isKnown: knownWordsEnabled ? isKnownForMatching : false,
+        knownMaturity,
       };
     }
 
@@ -712,6 +753,7 @@ export function annotateTokens(
     return {
       ...token,
       isKnown: knownWordsEnabled ? isKnownForMatching : false,
+      knownMaturity,
       isNPlusOneTarget: nPlusOneEnabled && !prioritizedNameMatch ? token.isNPlusOneTarget : false,
       frequencyRank,
       jlptLevel,