fix(tokenizer): prevent grammar tokens from borrowing known-word highlight via unrelated readings (#147)

changes/particle-known-word-highlight.md

@@ -0,0 +1,5 @@
+type: fixed
+area: overlay
+
+- Fixed single-kana grammar tokens counting as known words by borrowing the reading of an unrelated Anki note (よ in 全然いいよ matched a card read よ such as 夜, standalone え matched 絵), which painted them with the known-word highlight. Reading-only known-word matching now requires at least two kana; single-kana cards still match by their word field, and tokens genuinely present in the known-words cache (e.g. です) keep their highlight.
+- Standalone suffix tokens (MeCab pos2 接尾, e.g. さん, れる) are now excluded from JLPT/frequency/N+1 annotations by default, matching how particles and interjections are treated. Cache-backed known-word highlighting still applies; override via the pos2 exclusion config if you want them annotated.

src/anki-integration/known-word-cache.test.ts

@@ -734,6 +734,49 @@ test('KnownWordCacheManager disambiguates known words by note reading', async ()
   }
 });
 
+test('KnownWordCacheManager does not match single-kana text by reading alone', async () => {
+  const config: AnkiConnectConfig = {
+    fields: {
+      word: 'Word',
+    },
+    knownWords: {
+      highlightEnabled: true,
+    },
+  };
+  const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
+
+  try {
+    clientState.findNotesResult = [1, 2];
+    clientState.notesInfoResult = [
+      {
+        noteId: 1,
+        fields: {
+          Word: { value: '夜' },
+          'Word Reading': { value: 'よ' },
+        },
+      },
+      {
+        noteId: 2,
+        fields: {
+          Word: { value: 'え' },
+        },
+      },
+    ];
+
+    await manager.refresh(true);
+
+    // よ must not count as known just because 夜 is read よ.
+    assert.equal(manager.isKnownWord('よ'), false);
+    assert.equal(manager.isKnownWord('ヨ'), false);
+    assert.equal(manager.isKnownWord('夜'), true);
+    assert.equal(manager.isKnownWord('夜', 'よ'), true);
+    // A literal single-kana word entry still matches via the word map.
+    assert.equal(manager.isKnownWord('え'), true);
+  } finally {
+    cleanup();
+  }
+});
+
 test('KnownWordCacheManager probes reading fields even with per-deck word fields configured', async () => {
   const config: AnkiConnectConfig = {
     fields: {

src/anki-integration/known-word-cache.ts

@@ -163,7 +163,14 @@ export class KnownWordCacheManager {
       );
     }
 
-    return this.readingCounts.has(convertKatakanaToHiragana(normalized));
+    // Reading-only fallback, except for single-kana text: particles and
+    // interjections (よ, ね, え…) would otherwise borrow the reading of an
+    // unrelated note (夜「よ」, 絵「え」) and count as known.
+    const hiragana = convertKatakanaToHiragana(normalized);
+    if ([...hiragana].length === 1) {
+      return false;
+    }
+    return this.readingCounts.has(hiragana);
   }
 
   refresh(force = false): Promise<void> {

src/core/services/tokenizer/annotation-stage.test.ts

@@ -1722,6 +1722,36 @@ test('annotateTokens keeps known status while clearing other annotations for sta
   }
 });
 
+test('annotateTokens excludes standalone noun-suffix tokens from annotations while keeping cache-backed known status', () => {
+  const tokens = [
+    makeToken({
+      surface: 'さん',
+      headword: 'さん',
+      reading: 'サン',
+      partOfSpeech: PartOfSpeech.noun,
+      pos1: '名詞',
+      pos2: '接尾',
+      startPos: 0,
+      endPos: 2,
+      frequencyRank: 33,
+    }),
+  ];
+
+  const result = annotateTokens(
+    tokens,
+    makeDeps({
+      isKnownWord: (text) => text === 'さん',
+      getJlptLevel: (text) => (text === 'さん' ? 'N5' : null),
+    }),
+    { minSentenceWordsForNPlusOne: 1 },
+  );
+
+  assert.equal(result[0]?.isKnown, true);
+  assert.equal(result[0]?.isNPlusOneTarget, false);
+  assert.equal(result[0]?.frequencyRank, undefined);
+  assert.equal(result[0]?.jlptLevel, undefined);
+});
+
 test('annotateTokens keeps known status while clearing other annotations for auxiliary-only te-kureru helper spans', () => {
   const tokens = [
     makeToken({

src/token-pos2-exclusions.ts

@@ -3,6 +3,7 @@ import { normalizePos1ExclusionList } from './token-pos1-exclusions';
 
 export const DEFAULT_ANNOTATION_POS2_EXCLUSION_DEFAULTS = Object.freeze([
   '非自立',
+  '接尾',
 ]) as readonly string[];
 
 export const DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG: ResolvedTokenPos2ExclusionConfig = {