From 38ddb29aa023ae93004e4d42fda905737ef36bdb Mon Sep 17 00:00:00 2001 From: sudacode Date: Tue, 7 Jul 2026 00:13:10 -0700 Subject: [PATCH] feat(anki): reading-aware known-word matching (cache v3) (#142) --- changes/known-word-reading-aware-matching.md | 4 + config.example.jsonc | 2 +- docs-site/public/config.example.jsonc | 2 +- src/anki-integration.ts | 4 +- src/anki-integration/known-word-cache.test.ts | 258 ++++++++++++++- src/anki-integration/known-word-cache.ts | 307 ++++++++++++------ src/anki-integration/known-word-entries.ts | 113 +++++++ .../definitions/options-integrations.ts | 2 +- src/core/services/tokenizer.ts | 7 +- .../tokenizer/annotation-stage.test.ts | 64 ++++ .../services/tokenizer/annotation-stage.ts | 51 ++- .../tokenizer/yomitan-parser-runtime.test.ts | 5 + .../tokenizer/yomitan-parser-runtime.ts | 3 + src/main.ts | 2 +- .../subtitle-tokenization-main-deps.ts | 4 +- src/types/subtitle.ts | 2 + 16 files changed, 701 insertions(+), 129 deletions(-) create mode 100644 changes/known-word-reading-aware-matching.md create mode 100644 src/anki-integration/known-word-entries.ts diff --git a/changes/known-word-reading-aware-matching.md b/changes/known-word-reading-aware-matching.md new file mode 100644 index 00000000..398ec904 --- /dev/null +++ b/changes/known-word-reading-aware-matching.md @@ -0,0 +1,4 @@ +type: fixed +area: overlay + +- Fixed words being highlighted green as known when a same-spelled Anki card taught a different reading (e.g. とこ parsed as 床 "bed" matching a known 床/ゆか "floor" card). The known-word cache now stores each card's word together with its reading and only matches when the token's reading agrees; cards without a reading field keep matching in any reading as before. diff --git a/config.example.jsonc b/config.example.jsonc index cfd0d18e..95ebe7ae 100644 --- a/config.example.jsonc +++ b/config.example.jsonc @@ -569,7 +569,7 @@ "refreshMinutes": 1440, // Minutes between known-word cache refreshes. "addMinedWordsImmediately": true, // Immediately append newly mined card words into the known-word cache. Values: true | false "matchMode": "headword", // Known-word matching strategy for subtitle annotations. Cache matches always receive known-word highlighting even when POS filters suppress other annotation types. Values: headword | surface - "decks": {} // Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. + "decks": {} // Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. Reading fields (Reading, Word Reading, ExpressionReading) are always probed so cached words match only in the reading their note teaches; words from notes without readings match in any reading. }, // Known words setting. "behavior": { "overwriteAudio": true, // When updating an existing card, overwrite the audio field instead of skipping it. Values: true | false diff --git a/docs-site/public/config.example.jsonc b/docs-site/public/config.example.jsonc index cfd0d18e..95ebe7ae 100644 --- a/docs-site/public/config.example.jsonc +++ b/docs-site/public/config.example.jsonc @@ -569,7 +569,7 @@ "refreshMinutes": 1440, // Minutes between known-word cache refreshes. "addMinedWordsImmediately": true, // Immediately append newly mined card words into the known-word cache. Values: true | false "matchMode": "headword", // Known-word matching strategy for subtitle annotations. Cache matches always receive known-word highlighting even when POS filters suppress other annotation types. Values: headword | surface - "decks": {} // Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. + "decks": {} // Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. Reading fields (Reading, Word Reading, ExpressionReading) are always probed so cached words match only in the reading their note teaches; words from notes without readings match in any reading. }, // Known words setting. "behavior": { "overwriteAudio": true, // When updating an existing card, overwrite the audio field instead of skipping it. Values: true | false diff --git a/src/anki-integration.ts b/src/anki-integration.ts index a8fbf60f..390c6cd4 100644 --- a/src/anki-integration.ts +++ b/src/anki-integration.ts @@ -703,8 +703,8 @@ export class AnkiIntegration { }); } - isKnownWord(text: string): boolean { - return this.knownWordCache.isKnownWord(text); + isKnownWord(text: string, reading?: string): boolean { + return this.knownWordCache.isKnownWord(text, reading); } getKnownWordMatchMode(): NPlusOneMatchMode { diff --git a/src/anki-integration/known-word-cache.test.ts b/src/anki-integration/known-word-cache.test.ts index ddd3a4e0..4dd0578c 100644 --- a/src/anki-integration/known-word-cache.test.ts +++ b/src/anki-integration/known-word-cache.test.ts @@ -108,6 +108,55 @@ test('KnownWordCacheManager startLifecycle keeps fresh persisted cache without i assert.equal(manager.isKnownWord('猫'), true); assert.equal(calls.findNotes, 0); assert.equal(calls.notesInfo, 0); + // v2 states carry no readings, so they load usable but stale to trigger a + // prompt upgrade refresh. + assert.equal( + ( + manager as unknown as { + getMsUntilNextRefresh: () => number; + } + ).getMsUntilNextRefresh(), + 0, + ); + } finally { + Date.now = originalDateNow; + manager.stopLifecycle(); + cleanup(); + } +}); + +test('KnownWordCacheManager startLifecycle keeps fresh v3 persisted cache without immediate refresh', async () => { + const config: AnkiConnectConfig = { + knownWords: { + highlightEnabled: true, + refreshMinutes: 60, + }, + }; + const { manager, calls, statePath, cleanup } = createKnownWordCacheHarness(config); + const originalDateNow = Date.now; + + try { + Date.now = () => 120_000; + fs.writeFileSync( + statePath, + JSON.stringify({ + version: 3, + refreshedAtMs: 120_000, + scope: '{"refreshMinutes":60,"scope":"all","fieldsWord":""}', + notes: { + '1': [{ word: '猫', reading: 'ねこ' }], + }, + }), + 'utf-8', + ); + + manager.startLifecycle(); + + assert.equal(manager.isKnownWord('猫'), true); + assert.equal(manager.isKnownWord('猫', 'ねこ'), true); + assert.equal(manager.isKnownWord('猫', 'びょう'), false); + assert.equal(calls.findNotes, 0); + assert.equal(calls.notesInfo, 0); assert.equal( ( manager as unknown as { @@ -263,13 +312,11 @@ test('KnownWordCacheManager refresh incrementally reconciles deleted and edited const persisted = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as { version: number; - words: string[]; - notes?: Record; + notes?: Record>; }; - assert.equal(persisted.version, 2); - assert.deepEqual(persisted.words.sort(), ['鳥']); + assert.equal(persisted.version, 3); assert.deepEqual(persisted.notes, { - '1': ['鳥'], + '1': [{ word: '鳥', reading: null }], }); } finally { cleanup(); @@ -392,10 +439,10 @@ test('KnownWordCacheManager preserves cache state key captured before refresh wo const persisted = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as { scope: string; - words: string[]; + notes: Record>; }; assert.equal(persisted.scope, '{"refreshMinutes":1,"scope":"all","fieldsWord":"Word"}'); - assert.deepEqual(persisted.words, ['猫']); + assert.deepEqual(persisted.notes, { '1': [{ word: '猫', reading: null }] }); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } @@ -648,3 +695,200 @@ test('KnownWordCacheManager skips immediate append when addMinedWordsImmediately cleanup(); } }); + +test('KnownWordCacheManager disambiguates known words by note reading', async () => { + const config: AnkiConnectConfig = { + fields: { + word: 'Word', + }, + knownWords: { + highlightEnabled: true, + }, + }; + const { manager, clientState, cleanup } = createKnownWordCacheHarness(config); + + try { + clientState.findNotesResult = [1]; + clientState.notesInfoResult = [ + { + noteId: 1, + fields: { + Word: { value: '床' }, + 'Word Reading': { value: 'ゆか' }, + }, + }, + ]; + + await manager.refresh(true); + + assert.equal(manager.isKnownWord('床'), true); + assert.equal(manager.isKnownWord('床', 'ゆか'), true); + assert.equal(manager.isKnownWord('床', 'ユカ'), true); + // Same spelling, different word (床/とこ "bed") must not match. + assert.equal(manager.isKnownWord('床', 'とこ'), false); + // Note readings stay matchable as kana words. + assert.equal(manager.isKnownWord('ゆか'), true); + assert.equal(manager.isKnownWord('とこ'), false); + } finally { + cleanup(); + } +}); + +test('KnownWordCacheManager probes reading fields even with per-deck word fields configured', async () => { + const config: AnkiConnectConfig = { + fields: { + word: 'Expression', + }, + knownWords: { + highlightEnabled: true, + decks: { + 'Kaishi 1.5k': ['Word'], + }, + }, + }; + const { manager, clientState, cleanup } = createKnownWordCacheHarness(config); + + try { + clientState.findNotesByQuery.set('deck:"Kaishi 1.5k"', [1]); + clientState.notesInfoResult = [ + { + noteId: 1, + fields: { + Word: { value: '床' }, + 'Word Reading': { value: 'ゆか' }, + }, + }, + ]; + + await manager.refresh(true); + + assert.equal(manager.isKnownWord('床', 'ゆか'), true); + assert.equal(manager.isKnownWord('床', 'とこ'), false); + } finally { + cleanup(); + } +}); + +test('KnownWordCacheManager matches words without readings in any reading', async () => { + const config: AnkiConnectConfig = { + fields: { + word: 'Word', + }, + knownWords: { + highlightEnabled: true, + }, + }; + const { manager, clientState, cleanup } = createKnownWordCacheHarness(config); + + try { + clientState.findNotesResult = [1]; + clientState.notesInfoResult = [ + { + noteId: 1, + fields: { + Word: { value: '床' }, + }, + }, + ]; + + await manager.refresh(true); + + assert.equal(manager.isKnownWord('床'), true); + assert.equal(manager.isKnownWord('床', 'とこ'), true); + } finally { + cleanup(); + } +}); + +test('KnownWordCacheManager extracts word and reading from furigana word fields', async () => { + const config: AnkiConnectConfig = { + fields: { + word: 'Word', + }, + knownWords: { + highlightEnabled: true, + }, + }; + const { manager, clientState, cleanup } = createKnownWordCacheHarness(config); + + try { + clientState.findNotesResult = [1]; + clientState.notesInfoResult = [ + { + noteId: 1, + fields: { + Word: { value: 'お 決[き]まり' }, + }, + }, + ]; + + await manager.refresh(true); + + assert.equal(manager.isKnownWord('お決まり'), true); + assert.equal(manager.isKnownWord('お決まり', 'おきまり'), true); + assert.equal(manager.isKnownWord('お決まり', 'おさだまり'), false); + } finally { + cleanup(); + } +}); + +test('KnownWordCacheManager treats non-kana reading fields as words', async () => { + const config: AnkiConnectConfig = { + fields: { + word: 'Word', + }, + knownWords: { + highlightEnabled: true, + }, + }; + const { manager, clientState, cleanup } = createKnownWordCacheHarness(config); + + try { + clientState.findNotesResult = [1]; + clientState.notesInfoResult = [ + { + noteId: 1, + fields: { + Reading: { value: '漢字' }, + }, + }, + ]; + + await manager.refresh(true); + + assert.equal(manager.isKnownWord('漢字'), true); + } finally { + cleanup(); + } +}); + +test('KnownWordCacheManager keeps kana-only reading notes matchable', async () => { + const config: AnkiConnectConfig = { + fields: { + word: 'Word', + }, + knownWords: { + highlightEnabled: true, + }, + }; + const { manager, clientState, cleanup } = createKnownWordCacheHarness(config); + + try { + clientState.findNotesResult = [1]; + clientState.notesInfoResult = [ + { + noteId: 1, + fields: { + Reading: { value: 'たべる' }, + }, + }, + ]; + + await manager.refresh(true); + + assert.equal(manager.isKnownWord('たべる'), true); + assert.equal(manager.isKnownWord('タベル'), true); + } finally { + cleanup(); + } +}); diff --git a/src/anki-integration/known-word-cache.ts b/src/anki-integration/known-word-cache.ts index b5bb5f9a..bd2c211e 100644 --- a/src/anki-integration/known-word-cache.ts +++ b/src/anki-integration/known-word-cache.ts @@ -5,6 +5,16 @@ import { DEFAULT_ANKI_CONNECT_CONFIG } from '../config'; import { getConfiguredWordFieldName } from '../anki-field-config'; import { AnkiConnectConfig } from '../types/anki'; import { createLogger } from '../logger'; +import { + DEFAULT_KNOWN_WORD_READING_FIELDS, + KnownWordEntry, + convertKatakanaToHiragana, + isReadingFieldName, + knownWordEntryListsEqual, + normalizeKnownReadingForLookup, + normalizeKnownWordEntryList, + parseFuriganaAnnotatedText, +} from './known-word-entries'; const log = createLogger('anki').child('integration.known-word-cache'); @@ -79,7 +89,16 @@ interface KnownWordCacheStateV2 { readonly notes: Record; } -type KnownWordCacheState = KnownWordCacheStateV1 | KnownWordCacheStateV2; +interface KnownWordCacheStateV3 { + readonly version: 3; + readonly refreshedAtMs: number; + readonly scope: string; + readonly notes: Record; +} + +type KnownWordCacheState = KnownWordCacheStateV1 | KnownWordCacheStateV2 | KnownWordCacheStateV3; + +const NO_READING_KEY = ''; interface KnownWordCacheClient { findNotes: ( @@ -106,9 +125,12 @@ type KnownWordQueryScope = { export class KnownWordCacheManager { private knownWordsLastRefreshedAtMs = 0; private knownWordsStateKey = ''; - private knownWords: Set = new Set(); - private wordReferenceCounts = new Map(); - private noteWordsById = new Map(); + // word → (hiragana reading | NO_READING_KEY → note count). NO_READING_KEY + // entries fail open: the word matches regardless of the token's reading. + private wordReadingCounts = new Map>(); + // hiragana reading → note count, so kana tokens still match by reading alone. + private readingCounts = new Map(); + private noteEntriesById = new Map(); private knownWordsRefreshTimer: ReturnType | null = null; private knownWordsRefreshTimeout: ReturnType | null = null; private isRefreshingKnownWords = false; @@ -120,13 +142,28 @@ export class KnownWordCacheManager { ); } - isKnownWord(text: string): boolean { + isKnownWord(text: string, reading?: string): boolean { if (!this.isKnownWordCacheEnabled()) { return false; } const normalized = this.normalizeKnownWordForLookup(text); - return normalized.length > 0 ? this.knownWords.has(normalized) : false; + if (normalized.length === 0) { + return false; + } + + const knownReadings = this.wordReadingCounts.get(normalized); + if (knownReadings && knownReadings.size > 0) { + const normalizedReading = + typeof reading === 'string' ? normalizeKnownReadingForLookup(reading) : ''; + return ( + normalizedReading.length === 0 || + knownReadings.has(NO_READING_KEY) || + knownReadings.has(normalizedReading) + ); + } + + return this.readingCounts.has(convertKatakanaToHiragana(normalized)); } refresh(force = false): Promise { @@ -173,7 +210,7 @@ export class KnownWordCacheManager { let didMutateCache = false; const currentStateKey = this.getKnownWordCacheStateKey(); if (this.knownWordsStateKey && this.knownWordsStateKey !== currentStateKey) { - didMutateCache = this.knownWords.size > 0 || this.noteWordsById.size > 0; + didMutateCache = this.wordReadingCounts.size > 0 || this.noteEntriesById.size > 0; this.clearKnownWordCacheState(); } if (!this.knownWordsStateKey) { @@ -185,8 +222,8 @@ export class KnownWordCacheManager { return didMutateCache; } - const nextWords = this.extractNormalizedKnownWordsFromNoteInfo(noteInfo, preferredFields); - const changed = this.replaceNoteSnapshot(noteInfo.noteId, nextWords); + const nextEntries = this.extractKnownWordEntriesFromNoteInfo(noteInfo, preferredFields); + const changed = this.replaceNoteSnapshot(noteInfo.noteId, nextEntries); if (!changed) { return didMutateCache; } @@ -198,7 +235,7 @@ export class KnownWordCacheManager { log.info( 'Known-word cache updated in-session', `noteId=${noteInfo.noteId}`, - `wordCount=${nextWords.length}`, + `wordCount=${nextEntries.length}`, `scope=${getKnownWordCacheScopeForConfig(this.deps.getConfig())}`, ); return true; @@ -236,11 +273,11 @@ export class KnownWordCacheManager { const noteFieldsById = await this.fetchKnownWordNoteFieldsById(); const currentNoteIds = Array.from(noteFieldsById.keys()).sort((a, b) => a - b); - if (this.noteWordsById.size === 0) { + if (this.noteEntriesById.size === 0) { await this.rebuildFromCurrentNotes(currentNoteIds, noteFieldsById); } else { const currentNoteIdSet = new Set(currentNoteIds); - for (const noteId of Array.from(this.noteWordsById.keys())) { + for (const noteId of Array.from(this.noteEntriesById.keys())) { if (!currentNoteIdSet.has(noteId)) { this.removeNoteSnapshot(noteId); } @@ -251,7 +288,7 @@ export class KnownWordCacheManager { for (const noteInfo of noteInfos) { this.replaceNoteSnapshot( noteInfo.noteId, - this.extractNormalizedKnownWordsFromNoteInfo( + this.extractKnownWordEntriesFromNoteInfo( noteInfo, noteFieldsById.get(noteInfo.noteId), ), @@ -266,7 +303,7 @@ export class KnownWordCacheManager { log.info( 'Known-word cache refreshed', `noteCount=${currentNoteIds.length}`, - `wordCount=${this.knownWords.size}`, + `wordCount=${this.wordReadingCounts.size}`, ); } catch (error) { log.warn('Failed to refresh known-word cache:', (error as Error).message); @@ -291,7 +328,13 @@ export class KnownWordCacheManager { private getDefaultKnownWordFields(): string[] { const configuredWordField = getConfiguredWordFieldName(this.deps.getConfig()); - return [...new Set([configuredWordField, 'Word', 'Reading', 'Word Reading'])]; + return this.withDefaultReadingFields([configuredWordField, 'Word']); + } + + // Reading fields are always probed (even when a deck configures explicit + // word fields) so entries can carry the reading their note teaches. + private withDefaultReadingFields(fields: string[]): string[] { + return [...new Set([...fields, ...DEFAULT_KNOWN_WORD_READING_FIELDS])]; } private getKnownWordDecks(): string[] { @@ -337,7 +380,9 @@ export class KnownWordCacheManager { .filter((field) => field.length > 0), ), ]; - return normalizedFields.length > 0 ? normalizedFields : this.getDefaultKnownWordFields(); + return normalizedFields.length > 0 + ? this.withDefaultReadingFields(normalizedFields) + : this.getDefaultKnownWordFields(); } const deckFields = selectedDeckEntry[1]; @@ -351,7 +396,7 @@ export class KnownWordCacheManager { ), ]; if (normalizedFields.length > 0) { - return normalizedFields; + return this.withDefaultReadingFields(normalizedFields); } } @@ -382,7 +427,10 @@ export class KnownWordCacheManager { : []; scopes.push({ query: `deck:"${escapeAnkiSearchValue(trimmedDeckName)}"`, - fields: normalizedFields.length > 0 ? normalizedFields : this.getDefaultKnownWordFields(), + fields: + normalizedFields.length > 0 + ? this.withDefaultReadingFields(normalizedFields) + : this.getDefaultKnownWordFields(), }); } if (scopes.length > 0) { @@ -490,7 +538,7 @@ export class KnownWordCacheManager { for (const noteInfo of noteInfos) { this.replaceNoteSnapshot( noteInfo.noteId, - this.extractNormalizedKnownWordsFromNoteInfo(noteInfo, noteFieldsById.get(noteInfo.noteId)), + this.extractKnownWordEntriesFromNoteInfo(noteInfo, noteFieldsById.get(noteInfo.noteId)), ); } } @@ -519,56 +567,77 @@ export class KnownWordCacheManager { return noteInfos; } - private replaceNoteSnapshot(noteId: number, nextWords: string[]): boolean { - const normalizedWords = normalizeKnownWordList(nextWords); - const previousWords = this.noteWordsById.get(noteId) ?? []; - if (knownWordListsEqual(previousWords, normalizedWords)) { + private replaceNoteSnapshot(noteId: number, nextEntries: KnownWordEntry[]): boolean { + const normalizedEntries = normalizeKnownWordEntryList(nextEntries); + const previousEntries = this.noteEntriesById.get(noteId) ?? []; + if (knownWordEntryListsEqual(previousEntries, normalizedEntries)) { return false; } - this.removeWordsFromCounts(previousWords); - if (normalizedWords.length > 0) { - this.noteWordsById.set(noteId, normalizedWords); - this.addWordsToCounts(normalizedWords); + this.removeEntriesFromCounts(previousEntries); + if (normalizedEntries.length > 0) { + this.noteEntriesById.set(noteId, normalizedEntries); + this.addEntriesToCounts(normalizedEntries); } else { - this.noteWordsById.delete(noteId); + this.noteEntriesById.delete(noteId); } return true; } private removeNoteSnapshot(noteId: number): void { - const previousWords = this.noteWordsById.get(noteId); - if (!previousWords) { + const previousEntries = this.noteEntriesById.get(noteId); + if (!previousEntries) { return; } - this.noteWordsById.delete(noteId); - this.removeWordsFromCounts(previousWords); + this.noteEntriesById.delete(noteId); + this.removeEntriesFromCounts(previousEntries); } - private addWordsToCounts(words: string[]): void { - for (const word of words) { - const nextCount = (this.wordReferenceCounts.get(word) ?? 0) + 1; - this.wordReferenceCounts.set(word, nextCount); - this.knownWords.add(word); + private addEntriesToCounts(entries: KnownWordEntry[]): void { + for (const entry of entries) { + const readingKey = entry.reading ?? NO_READING_KEY; + let readings = this.wordReadingCounts.get(entry.word); + if (!readings) { + readings = new Map(); + this.wordReadingCounts.set(entry.word, readings); + } + readings.set(readingKey, (readings.get(readingKey) ?? 0) + 1); + if (entry.reading) { + this.readingCounts.set(entry.reading, (this.readingCounts.get(entry.reading) ?? 0) + 1); + } } } - private removeWordsFromCounts(words: string[]): void { - for (const word of words) { - const nextCount = (this.wordReferenceCounts.get(word) ?? 0) - 1; - if (nextCount > 0) { - this.wordReferenceCounts.set(word, nextCount); - } else { - this.wordReferenceCounts.delete(word); - this.knownWords.delete(word); + private removeEntriesFromCounts(entries: KnownWordEntry[]): void { + for (const entry of entries) { + const readingKey = entry.reading ?? NO_READING_KEY; + const readings = this.wordReadingCounts.get(entry.word); + if (readings) { + const nextCount = (readings.get(readingKey) ?? 0) - 1; + if (nextCount > 0) { + readings.set(readingKey, nextCount); + } else { + readings.delete(readingKey); + if (readings.size === 0) { + this.wordReadingCounts.delete(entry.word); + } + } + } + if (entry.reading) { + const nextReadingCount = (this.readingCounts.get(entry.reading) ?? 0) - 1; + if (nextReadingCount > 0) { + this.readingCounts.set(entry.reading, nextReadingCount); + } else { + this.readingCounts.delete(entry.reading); + } } } } private clearInMemoryState(): void { - this.knownWords = new Set(); - this.wordReferenceCounts = new Map(); - this.noteWordsById = new Map(); + this.wordReadingCounts = new Map(); + this.readingCounts = new Map(); + this.noteEntriesById = new Map(); this.knownWordsLastRefreshedAtMs = 0; } @@ -601,32 +670,48 @@ export class KnownWordCacheManager { } this.clearInMemoryState(); + if (parsed.version === 3) { + for (const [noteIdKey, entries] of Object.entries(parsed.notes)) { + const noteId = Number.parseInt(noteIdKey, 10); + if (!Number.isInteger(noteId) || noteId <= 0) { + continue; + } + const normalizedEntries = normalizeKnownWordEntryList(entries); + if (normalizedEntries.length === 0) { + continue; + } + this.noteEntriesById.set(noteId, normalizedEntries); + this.addEntriesToCounts(normalizedEntries); + } + this.knownWordsLastRefreshedAtMs = parsed.refreshedAtMs; + this.knownWordsStateKey = parsed.scope; + return; + } + if (parsed.version === 2) { + // Older states have no readings; load them reading-less (fail-open, + // matching the old behavior) but leave the cache marked stale so the + // next refresh upgrades entries with readings from Anki. for (const [noteIdKey, words] of Object.entries(parsed.notes)) { const noteId = Number.parseInt(noteIdKey, 10); if (!Number.isInteger(noteId) || noteId <= 0) { continue; } - const normalizedWords = normalizeKnownWordList(words); - if (normalizedWords.length === 0) { + const normalizedEntries = normalizeKnownWordEntryList( + words.map((word) => ({ word: this.normalizeKnownWordForLookup(word), reading: null })), + ); + if (normalizedEntries.length === 0) { continue; } - this.noteWordsById.set(noteId, normalizedWords); - this.addWordsToCounts(normalizedWords); - } - } else { - for (const value of parsed.words) { - const normalized = this.normalizeKnownWordForLookup(value); - if (!normalized) { - continue; - } - this.knownWords.add(normalized); - this.wordReferenceCounts.set(normalized, 1); + this.noteEntriesById.set(noteId, normalizedEntries); + this.addEntriesToCounts(normalizedEntries); } + this.knownWordsStateKey = parsed.scope; + return; } - this.knownWordsLastRefreshedAtMs = parsed.refreshedAtMs; - this.knownWordsStateKey = parsed.scope; + // v1 has no per-note snapshots to convert; refetch from Anki. + this.knownWordsStateKey = this.getKnownWordCacheStateKey(); } catch (error) { log.warn('Failed to load known-word cache state:', (error as Error).message); this.clearInMemoryState(); @@ -636,18 +721,17 @@ export class KnownWordCacheManager { private persistKnownWordCacheState(): void { try { - const notes: Record = {}; - for (const [noteId, words] of this.noteWordsById.entries()) { - if (words.length > 0) { - notes[String(noteId)] = words; + const notes: Record = {}; + for (const [noteId, entries] of this.noteEntriesById.entries()) { + if (entries.length > 0) { + notes[String(noteId)] = entries; } } - const state: KnownWordCacheStateV2 = { - version: 2, + const state: KnownWordCacheStateV3 = { + version: 3, refreshedAtMs: this.knownWordsLastRefreshedAtMs, scope: this.knownWordsStateKey, - words: Array.from(this.knownWords), notes, }; fs.writeFileSync(this.statePath, JSON.stringify(state), 'utf-8'); @@ -659,14 +743,18 @@ export class KnownWordCacheManager { private isKnownWordCacheStateValid(value: unknown): value is KnownWordCacheState { if (typeof value !== 'object' || value === null) return false; const candidate = value as Record; - if (candidate.version !== 1 && candidate.version !== 2) return false; - if (typeof candidate.refreshedAtMs !== 'number') return false; - if (typeof candidate.scope !== 'string') return false; - if (!Array.isArray(candidate.words)) return false; - if (!candidate.words.every((entry: unknown) => typeof entry === 'string')) { + if (candidate.version !== 1 && candidate.version !== 2 && candidate.version !== 3) { return false; } - if (candidate.version === 2) { + if (typeof candidate.refreshedAtMs !== 'number') return false; + if (typeof candidate.scope !== 'string') return false; + if (candidate.version !== 3) { + if (!Array.isArray(candidate.words)) return false; + if (!candidate.words.every((entry: unknown) => typeof entry === 'string')) { + return false; + } + } + if (candidate.version === 2 || candidate.version === 3) { if ( typeof candidate.notes !== 'object' || candidate.notes === null || @@ -674,10 +762,18 @@ export class KnownWordCacheManager { ) { return false; } + const isValidNoteEntry = + candidate.version === 2 + ? (entry: unknown): boolean => typeof entry === 'string' + : (entry: unknown): boolean => + typeof entry === 'object' && + entry !== null && + typeof (entry as KnownWordEntry).word === 'string' && + ((entry as KnownWordEntry).reading === null || + typeof (entry as KnownWordEntry).reading === 'string'); if ( !Object.values(candidate.notes as Record).every( - (entry) => - Array.isArray(entry) && entry.every((word: unknown) => typeof word === 'string'), + (noteEntries) => Array.isArray(noteEntries) && noteEntries.every(isValidNoteEntry), ) ) { return false; @@ -686,11 +782,12 @@ export class KnownWordCacheManager { return true; } - private extractNormalizedKnownWordsFromNoteInfo( + private extractKnownWordEntriesFromNoteInfo( noteInfo: KnownWordCacheNoteInfo, preferredFields = this.getConfiguredFields(), - ): string[] { - const words: string[] = []; + ): KnownWordEntry[] { + const wordValues: string[] = []; + let noteReading: string | null = null; for (const preferredField of preferredFields) { const fieldName = resolveFieldName(Object.keys(noteInfo.fields), preferredField); if (!fieldName) continue; @@ -698,12 +795,36 @@ export class KnownWordCacheManager { const raw = noteInfo.fields[fieldName]?.value; if (!raw) continue; - const normalized = this.normalizeKnownWordForLookup(raw); - if (normalized) { - words.push(normalized); + const cleaned = this.normalizeRawKnownWordValue(raw); + if (!cleaned) continue; + + if (isReadingFieldName(preferredField)) { + const normalizedReading = normalizeKnownReadingForLookup(cleaned); + if (normalizedReading) { + noteReading ??= normalizedReading; + continue; + } + // Non-kana content in a reading field: treat it as a word so decks + // with repurposed reading fields keep matching (fail-open). } + wordValues.push(cleaned); } - return normalizeKnownWordList(words); + + const entries: KnownWordEntry[] = []; + for (const value of wordValues) { + const parsed = parseFuriganaAnnotatedText(value); + const word = parsed.text.trim().toLowerCase(); + if (!word) continue; + const inlineReading = parsed.reading ? normalizeKnownReadingForLookup(parsed.reading) : ''; + entries.push({ word, reading: inlineReading || noteReading }); + } + + // Kana-only notes (reading field but no word field) stay matchable. + if (entries.length === 0 && noteReading) { + entries.push({ word: noteReading, reading: noteReading }); + } + + return normalizeKnownWordEntryList(entries); } private normalizeRawKnownWordValue(value: string): string { @@ -718,22 +839,6 @@ export class KnownWordCacheManager { } } -function normalizeKnownWordList(words: string[]): string[] { - return [...new Set(words.map((word) => word.trim()).filter((word) => word.length > 0))].sort(); -} - -function knownWordListsEqual(left: string[], right: string[]): boolean { - if (left.length !== right.length) { - return false; - } - for (let index = 0; index < left.length; index += 1) { - if (left[index] !== right[index]) { - return false; - } - } - return true; -} - function resolveFieldName(availableFieldNames: string[], preferredName: string): string | null { const exact = availableFieldNames.find((name) => name === preferredName); if (exact) return exact; diff --git a/src/anki-integration/known-word-entries.ts b/src/anki-integration/known-word-entries.ts new file mode 100644 index 00000000..24296610 --- /dev/null +++ b/src/anki-integration/known-word-entries.ts @@ -0,0 +1,113 @@ +// Known-word cache entries pair a word with the reading its Anki note teaches, +// so spelling collisions across readings (e.g. 床/ゆか vs 床/とこ) don't mark +// unrelated words as known. reading === null means the note carries no usable +// reading and the word matches in any reading (fail-open). +export interface KnownWordEntry { + word: string; + reading: string | null; +} + +const KATAKANA_TO_HIRAGANA_OFFSET = 0x60; +const KATAKANA_CODEPOINT_START = 0x30a1; +const KATAKANA_CODEPOINT_END = 0x30f6; +const FURIGANA_SEGMENT_PATTERN = /([^\s　\[\]]*)\[([^\]]*)\]/g; +const FURIGANA_BRACKET_PATTERN = /\[[^\]]*\]/g; +const WHITESPACE_PATTERN = /[\s　]+/g; + +// Reading-bearing field names probed on every known-word note, in addition to +// any configured word fields (covers Kaishi's "Word Reading" and Lapis's +// "ExpressionReading" note types). +export const DEFAULT_KNOWN_WORD_READING_FIELDS = [ + 'Reading', + 'Word Reading', + 'ExpressionReading', + 'Expression Reading', +]; + +export function isReadingFieldName(fieldName: string): boolean { + return /reading/i.test(fieldName); +} + +export function convertKatakanaToHiragana(text: string): string { + let converted = ''; + for (const char of text) { + const code = char.codePointAt(0); + if (code !== undefined && code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END) { + converted += String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET); + continue; + } + converted += char; + } + return converted; +} + +function isHiraganaReadingChar(char: string): boolean { + const code = char.codePointAt(0); + if (code === undefined) { + return false; + } + return (code >= 0x3041 && code <= 0x309f) || code === 0x30fc; +} + +// Splits Anki furigana syntax (`床[とこ]`, `お 決[き]まり`) into base text and +// reading. Values without brackets pass through with reading null. +export function parseFuriganaAnnotatedText(value: string): { + text: string; + reading: string | null; +} { + if (!value.includes('[')) { + return { text: value, reading: null }; + } + const text = value.replace(FURIGANA_BRACKET_PATTERN, '').replace(WHITESPACE_PATTERN, ''); + const reading = value.replace(FURIGANA_SEGMENT_PATTERN, '$2').replace(WHITESPACE_PATTERN, ''); + return { text, reading: reading.length > 0 ? reading : null }; +} + +// Returns the hiragana-normalized reading, or '' when the value is not a +// plausible kana reading (callers fall back to text-only matching then). +export function normalizeKnownReadingForLookup(value: string): string { + const parsed = parseFuriganaAnnotatedText(value.trim()); + const candidate = (parsed.reading ?? parsed.text).trim(); + if (!candidate) { + return ''; + } + const hiragana = convertKatakanaToHiragana(candidate); + for (const char of hiragana) { + if (!isHiraganaReadingChar(char)) { + return ''; + } + } + return hiragana; +} + +export function makeKnownWordEntryKey(entry: KnownWordEntry): string { + return `${entry.word}\u0000${entry.reading ?? ''}`; +} + +export function normalizeKnownWordEntryList(entries: KnownWordEntry[]): KnownWordEntry[] { + const byKey = new Map(); + for (const entry of entries) { + const word = entry.word.trim(); + if (!word) { + continue; + } + const reading = entry.reading?.trim() || null; + const normalized: KnownWordEntry = { word, reading }; + byKey.set(makeKnownWordEntryKey(normalized), normalized); + } + return [...byKey.values()].sort((left, right) => + makeKnownWordEntryKey(left).localeCompare(makeKnownWordEntryKey(right)), + ); +} + +export function knownWordEntryListsEqual(left: KnownWordEntry[], right: KnownWordEntry[]): boolean { + if (left.length !== right.length) { + return false; + } + for (let index = 0; index < left.length; index += 1) { + if (makeKnownWordEntryKey(left[index]!) !== makeKnownWordEntryKey(right[index]!)) { + return false; + } + } + return true; +} diff --git a/src/config/definitions/options-integrations.ts b/src/config/definitions/options-integrations.ts index 571f968e..2ec03b4a 100644 --- a/src/config/definitions/options-integrations.ts +++ b/src/config/definitions/options-integrations.ts @@ -317,7 +317,7 @@ export function buildIntegrationConfigOptionRegistry( kind: 'object', defaultValue: defaultConfig.ankiConnect.knownWords.decks, description: - 'Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }.', + 'Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. Reading fields (Reading, Word Reading, ExpressionReading) are always probed so cached words match only in the reading their note teaches; words from notes without readings match in any reading.', }, { path: 'ankiConnect.isKiku.fieldGrouping', diff --git a/src/core/services/tokenizer.ts b/src/core/services/tokenizer.ts index a6477317..0b6424cd 100644 --- a/src/core/services/tokenizer.ts +++ b/src/core/services/tokenizer.ts @@ -42,7 +42,7 @@ export interface TokenizerServiceDeps { setYomitanParserReadyPromise: (promise: Promise | null) => void; getYomitanParserInitPromise: () => Promise | null; setYomitanParserInitPromise: (promise: Promise | null) => void; - isKnownWord: (text: string) => boolean; + isKnownWord: (text: string, reading?: string) => boolean; getKnownWordMatchMode: () => NPlusOneMatchMode; getKnownWordsEnabled?: () => boolean; getJlptLevel: (text: string) => JlptLevel | null; @@ -77,7 +77,7 @@ export interface TokenizerDepsRuntimeOptions { setYomitanParserReadyPromise: (promise: Promise | null) => void; getYomitanParserInitPromise: () => Promise | null; setYomitanParserInitPromise: (promise: Promise | null) => void; - isKnownWord: (text: string) => boolean; + isKnownWord: (text: string, reading?: string) => boolean; getKnownWordMatchMode: () => NPlusOneMatchMode; getKnownWordsEnabled?: () => boolean; getJlptLevel: (text: string) => JlptLevel | null; @@ -129,7 +129,7 @@ const INVISIBLE_SEPARATOR_PATTERN = /[\u200b\u2060\ufeff]/g; function getKnownWordLookup( deps: TokenizerServiceDeps, options: TokenizerAnnotationOptions, -): (text: string) => boolean { +): (text: string, reading?: string) => boolean { if (!options.knownWordsEnabled && !options.nPlusOneEnabled) { return () => false; } @@ -723,6 +723,7 @@ async function parseWithYomitanInternalParser( surface: token.surface, reading: token.reading, headword: token.headword, + headwordReading: token.headwordReading, startPos: token.startPos, endPos: token.endPos, partOfSpeech: posMetadata.partOfSpeech, diff --git a/src/core/services/tokenizer/annotation-stage.test.ts b/src/core/services/tokenizer/annotation-stage.test.ts index d971c2c8..4fbb9ad0 100644 --- a/src/core/services/tokenizer/annotation-stage.test.ts +++ b/src/core/services/tokenizer/annotation-stage.test.ts @@ -56,6 +56,70 @@ test('annotateTokens known-word match mode uses headword vs surface', () => { assert.equal(surfaceResult[0]?.isKnown, false); }); +test('annotateTokens passes dictionary-form reading so spelling collisions stay unknown', () => { + // とこ (colloquial ところ) resolves to headword 床/とこ; a known 床/ゆか card + // must not mark it known (#138 regression). + const cache = new Map([['床', 'ゆか']]); + const isKnownWord = (text: string, reading?: string): boolean => { + if (!cache.has(text)) { + return false; + } + return reading === undefined || cache.get(text) === reading; + }; + const tokens = [ + makeToken({ + surface: 'とこ', + headword: '床', + reading: 'とこ', + headwordReading: 'とこ', + endPos: 2, + }), + ]; + + const result = annotateTokens(tokens, makeDeps({ isKnownWord })); + + assert.equal(result[0]?.isKnown, false); +}); + +test('annotateTokens keeps inflected known words matched via headword reading', () => { + const isKnownWord = (text: string, reading?: string): boolean => + text === '行く' && (reading === undefined || reading === 'いく'); + const tokens = [ + makeToken({ + surface: '行きたい', + headword: '行く', + reading: 'いきたい', + headwordReading: 'いく', + partOfSpeech: PartOfSpeech.verb, + endPos: 4, + }), + ]; + + const result = annotateTokens(tokens, makeDeps({ isKnownWord })); + + assert.equal(result[0]?.isKnown, true); +}); + +test('annotateTokens omits reading for headword match when token lacks headword reading and is inflected', () => { + // MeCab tokens have no dictionary-form reading; the surface reading of an + // inflected form must not be compared against the note's dictionary reading. + const isKnownWord = (text: string, reading?: string): boolean => + text === '食べる' && reading === undefined; + const tokens = [ + makeToken({ + surface: '食べた', + headword: '食べる', + reading: 'タベタ', + partOfSpeech: PartOfSpeech.verb, + endPos: 3, + }), + ]; + + const result = annotateTokens(tokens, makeDeps({ isKnownWord })); + + assert.equal(result[0]?.isKnown, true); +}); + test('annotateTokens marks known words when N+1 is disabled', () => { const tokens = [ makeToken({ surface: '私', headword: '私', startPos: 0, endPos: 1 }), diff --git a/src/core/services/tokenizer/annotation-stage.ts b/src/core/services/tokenizer/annotation-stage.ts index ad3e6cb9..abebda5e 100644 --- a/src/core/services/tokenizer/annotation-stage.ts +++ b/src/core/services/tokenizer/annotation-stage.ts @@ -25,7 +25,7 @@ const jlptLevelLookupCaches = new WeakMap< >(); export interface AnnotationStageDeps { - isKnownWord: (text: string) => boolean; + isKnownWord: (text: string, reading?: string) => boolean; knownWordMatchMode: NPlusOneMatchMode; getJlptLevel: (text: string) => JlptLevel | null; } @@ -661,26 +661,57 @@ function isCompleteReadingForSurface(surface: string, reading: string): boolean return true; } +// Returns the token's trimmed reading only when it plausibly covers the surface +// (see isCompleteReadingForSurface); undefined otherwise. Shared so the +// known-word reading disambiguation and the reading fallback stay in sync if the +// validity rule changes. +function resolveCompleteTokenReading(token: MergedToken): string | undefined { + const normalizedReading = token.reading.trim(); + if (!normalizedReading || !isCompleteReadingForSurface(token.surface, normalizedReading)) { + return undefined; + } + return normalizedReading; +} + +// Reading to disambiguate the known-word text match, or undefined when the +// token has no reading that describes the match text: in headword mode an +// inflected surface's reading does not match the dictionary form's reading, +// and partial furigana readings (see isCompleteReadingForSurface) would cause +// false negatives. Undefined falls back to text-only matching (fail-open). +function resolveKnownWordReadingForMatch( + token: MergedToken, + knownWordMatchMode: NPlusOneMatchMode, +): string | undefined { + if (knownWordMatchMode === 'headword') { + const headwordReading = token.headwordReading?.trim(); + if (headwordReading) { + return headwordReading; + } + if (token.surface !== token.headword) { + return undefined; + } + } + + return resolveCompleteTokenReading(token); +} + function computeTokenKnownStatus( token: MergedToken, - isKnownWord: (text: string) => boolean, + isKnownWord: (text: string, reading?: string) => boolean, knownWordMatchMode: NPlusOneMatchMode, ): boolean { const matchText = resolveKnownWordText(token.surface, token.headword, knownWordMatchMode); - if (token.isKnown || (matchText ? isKnownWord(matchText) : false)) { + const matchReading = resolveKnownWordReadingForMatch(token, knownWordMatchMode); + if (token.isKnown || (matchText ? isKnownWord(matchText, matchReading) : false)) { return true; } - const normalizedReading = token.reading.trim(); - if (!normalizedReading) { + const fallbackReading = resolveCompleteTokenReading(token); + if (!fallbackReading) { return false; } - if (!isCompleteReadingForSurface(token.surface, normalizedReading)) { - return false; - } - - return normalizedReading !== matchText.trim() && isKnownWord(normalizedReading); + return fallbackReading !== matchText.trim() && isKnownWord(fallbackReading); } function filterTokenFrequencyRank( diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts index a2b4784a..83ee68e4 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.test.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.test.ts @@ -966,6 +966,7 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF surface: '潜み', reading: 'ひそみ', headword: '潜む', + headwordReading: 'ひそむ', startPos: 0, endPos: 2, isNameMatch: false, @@ -1032,6 +1033,7 @@ test('requestYomitanScanTokens emits complete readings for kanji-kana compounds' surface: '待ち合わせてる', reading: 'まちあわせてる', headword: '待ち合わせる', + headwordReading: 'まちあわせる', startPos: 0, endPos: 7, isNameMatch: false, @@ -1139,6 +1141,7 @@ test('requestYomitanScanTokens uses frequency from later exact-match entry when surface: '者', reading: 'もの', headword: '者', + headwordReading: 'もの', startPos: 0, endPos: 1, isNameMatch: false, @@ -1240,6 +1243,7 @@ test('requestYomitanScanTokens can use frequency from later exact secondary-matc surface: '者', reading: 'もの', headword: '者', + headwordReading: 'もの', startPos: 0, endPos: 1, isNameMatch: false, @@ -1340,6 +1344,7 @@ test('requestYomitanScanTokens uses exact frequency entry when selected reading surface: '第二', reading: 'だいに', headword: '第二', + headwordReading: 'だいに', startPos: 0, endPos: 2, isNameMatch: false, diff --git a/src/core/services/tokenizer/yomitan-parser-runtime.ts b/src/core/services/tokenizer/yomitan-parser-runtime.ts index 8497e610..cc6cc908 100644 --- a/src/core/services/tokenizer/yomitan-parser-runtime.ts +++ b/src/core/services/tokenizer/yomitan-parser-runtime.ts @@ -51,6 +51,7 @@ export interface YomitanScanToken { surface: string; reading: string; headword: string; + headwordReading?: string; startPos: number; endPos: number; isNameMatch?: boolean; @@ -92,6 +93,7 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] { typeof entry.surface === 'string' && typeof entry.reading === 'string' && typeof entry.headword === 'string' && + (entry.headwordReading === undefined || typeof entry.headwordReading === 'string') && typeof entry.startPos === 'number' && typeof entry.endPos === 'number' && (entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean') && @@ -1318,6 +1320,7 @@ ${YOMITAN_SCANNING_HELPERS} surface: segments.map((segment) => segment.text).join("") || source, reading: segments.map(getSegmentReadingContribution).join(""), headword: preferredHeadword.term, + headwordReading: reading || undefined, startPos: i, endPos: i + originalTextLength, isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true, diff --git a/src/main.ts b/src/main.ts index cd87b624..d3375ae5 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4638,7 +4638,7 @@ const { setYomitanParserInitPromise: (promise) => { appState.yomitanParserInitPromise = promise; }, - isKnownWord: (text) => Boolean(appState.ankiIntegration?.isKnownWord(text)), + isKnownWord: (text, reading) => Boolean(appState.ankiIntegration?.isKnownWord(text, reading)), recordLookup: (hit) => { ensureImmersionTrackerStarted(); appState.immersionTracker?.recordLookup(hit); diff --git a/src/main/runtime/subtitle-tokenization-main-deps.ts b/src/main/runtime/subtitle-tokenization-main-deps.ts index 28209a37..8af13587 100644 --- a/src/main/runtime/subtitle-tokenization-main-deps.ts +++ b/src/main/runtime/subtitle-tokenization-main-deps.ts @@ -37,8 +37,8 @@ export function createBuildTokenizerDepsMainHandler(deps: TokenizerMainDeps) { getYomitanParserInitPromise: () => deps.getYomitanParserInitPromise(), setYomitanParserInitPromise: (promise: Promise | null) => deps.setYomitanParserInitPromise(promise), - isKnownWord: (text: string) => { - const hit = deps.isKnownWord(text); + isKnownWord: (text: string, reading?: string) => { + const hit = deps.isKnownWord(text, reading); deps.recordLookup(hit); return hit; }, diff --git a/src/types/subtitle.ts b/src/types/subtitle.ts index 2fb6c4c9..2e775f34 100644 --- a/src/types/subtitle.ts +++ b/src/types/subtitle.ts @@ -29,6 +29,8 @@ export interface MergedToken { surface: string; reading: string; headword: string; + /** Dictionary-form reading of headword (kana), when the parser provides it. */ + headwordReading?: string; startPos: number; endPos: number; partOfSpeech: PartOfSpeech;