diff --git a/changes/particle-known-word-highlight.md b/changes/particle-known-word-highlight.md new file mode 100644 index 00000000..6be0ee61 --- /dev/null +++ b/changes/particle-known-word-highlight.md @@ -0,0 +1,5 @@ +type: fixed +area: overlay + +- Fixed single-kana grammar tokens counting as known words by borrowing the reading of an unrelated Anki note (よ in 全然いいよ matched a card read よ such as 夜, standalone え matched 絵), which painted them with the known-word highlight. Reading-only known-word matching now requires at least two kana; single-kana cards still match by their word field, and tokens genuinely present in the known-words cache (e.g. です) keep their highlight. +- Standalone suffix tokens (MeCab pos2 接尾, e.g. さん, れる) are now excluded from JLPT/frequency/N+1 annotations by default, matching how particles and interjections are treated. Cache-backed known-word highlighting still applies; override via the pos2 exclusion config if you want them annotated. diff --git a/src/anki-integration/known-word-cache.test.ts b/src/anki-integration/known-word-cache.test.ts index 4dd0578c..1672aac3 100644 --- a/src/anki-integration/known-word-cache.test.ts +++ b/src/anki-integration/known-word-cache.test.ts @@ -734,6 +734,49 @@ test('KnownWordCacheManager disambiguates known words by note reading', async () } }); +test('KnownWordCacheManager does not match single-kana text by reading alone', async () => { + const config: AnkiConnectConfig = { + fields: { + word: 'Word', + }, + knownWords: { + highlightEnabled: true, + }, + }; + const { manager, clientState, cleanup } = createKnownWordCacheHarness(config); + + try { + clientState.findNotesResult = [1, 2]; + clientState.notesInfoResult = [ + { + noteId: 1, + fields: { + Word: { value: '夜' }, + 'Word Reading': { value: 'よ' }, + }, + }, + { + noteId: 2, + fields: { + Word: { value: 'え' }, + }, + }, + ]; + + await manager.refresh(true); + + // よ must not count as known just because 夜 is read よ. + assert.equal(manager.isKnownWord('よ'), false); + assert.equal(manager.isKnownWord('ヨ'), false); + assert.equal(manager.isKnownWord('夜'), true); + assert.equal(manager.isKnownWord('夜', 'よ'), true); + // A literal single-kana word entry still matches via the word map. + assert.equal(manager.isKnownWord('え'), true); + } finally { + cleanup(); + } +}); + test('KnownWordCacheManager probes reading fields even with per-deck word fields configured', async () => { const config: AnkiConnectConfig = { fields: { diff --git a/src/anki-integration/known-word-cache.ts b/src/anki-integration/known-word-cache.ts index bd2c211e..3f6bd003 100644 --- a/src/anki-integration/known-word-cache.ts +++ b/src/anki-integration/known-word-cache.ts @@ -163,7 +163,14 @@ export class KnownWordCacheManager { ); } - return this.readingCounts.has(convertKatakanaToHiragana(normalized)); + // Reading-only fallback, except for single-kana text: particles and + // interjections (よ, ね, え…) would otherwise borrow the reading of an + // unrelated note (夜「よ」, 絵「え」) and count as known. + const hiragana = convertKatakanaToHiragana(normalized); + if ([...hiragana].length === 1) { + return false; + } + return this.readingCounts.has(hiragana); } refresh(force = false): Promise { diff --git a/src/core/services/tokenizer/annotation-stage.test.ts b/src/core/services/tokenizer/annotation-stage.test.ts index b37815c9..bf46d2eb 100644 --- a/src/core/services/tokenizer/annotation-stage.test.ts +++ b/src/core/services/tokenizer/annotation-stage.test.ts @@ -1722,6 +1722,36 @@ test('annotateTokens keeps known status while clearing other annotations for sta } }); +test('annotateTokens excludes standalone noun-suffix tokens from annotations while keeping cache-backed known status', () => { + const tokens = [ + makeToken({ + surface: 'さん', + headword: 'さん', + reading: 'サン', + partOfSpeech: PartOfSpeech.noun, + pos1: '名詞', + pos2: '接尾', + startPos: 0, + endPos: 2, + frequencyRank: 33, + }), + ]; + + const result = annotateTokens( + tokens, + makeDeps({ + isKnownWord: (text) => text === 'さん', + getJlptLevel: (text) => (text === 'さん' ? 'N5' : null), + }), + { minSentenceWordsForNPlusOne: 1 }, + ); + + assert.equal(result[0]?.isKnown, true); + assert.equal(result[0]?.isNPlusOneTarget, false); + assert.equal(result[0]?.frequencyRank, undefined); + assert.equal(result[0]?.jlptLevel, undefined); +}); + test('annotateTokens keeps known status while clearing other annotations for auxiliary-only te-kureru helper spans', () => { const tokens = [ makeToken({ diff --git a/src/token-pos2-exclusions.ts b/src/token-pos2-exclusions.ts index 01b14398..100ae285 100644 --- a/src/token-pos2-exclusions.ts +++ b/src/token-pos2-exclusions.ts @@ -3,6 +3,7 @@ import { normalizePos1ExclusionList } from './token-pos1-exclusions'; export const DEFAULT_ANNOTATION_POS2_EXCLUSION_DEFAULTS = Object.freeze([ '非自立', + '接尾', ]) as readonly string[]; export const DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG: ResolvedTokenPos2ExclusionConfig = {