feat(anki): reading-aware known-word matching (cache v3) (#142)

This commit is contained in:
2026-07-07 00:13:10 -07:00
committed by GitHub
parent 8b9a70c5a6
commit 38ddb29aa0
16 changed files with 701 additions and 129 deletions
@@ -0,0 +1,4 @@
type: fixed
area: overlay
- Fixed words being highlighted green as known when a same-spelled Anki card taught a different reading (e.g. とこ parsed as 床 "bed" matching a known 床/ゆか "floor" card). The known-word cache now stores each card's word together with its reading and only matches when the token's reading agrees; cards without a reading field keep matching in any reading as before.
+1 -1
View File
@@ -569,7 +569,7 @@
"refreshMinutes": 1440, // Minutes between known-word cache refreshes. "refreshMinutes": 1440, // Minutes between known-word cache refreshes.
"addMinedWordsImmediately": true, // Immediately append newly mined card words into the known-word cache. Values: true | false "addMinedWordsImmediately": true, // Immediately append newly mined card words into the known-word cache. Values: true | false
"matchMode": "headword", // Known-word matching strategy for subtitle annotations. Cache matches always receive known-word highlighting even when POS filters suppress other annotation types. Values: headword | surface "matchMode": "headword", // Known-word matching strategy for subtitle annotations. Cache matches always receive known-word highlighting even when POS filters suppress other annotation types. Values: headword | surface
"decks": {} // Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. "decks": {} // Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. Reading fields (Reading, Word Reading, ExpressionReading) are always probed so cached words match only in the reading their note teaches; words from notes without readings match in any reading.
}, // Known words setting. }, // Known words setting.
"behavior": { "behavior": {
"overwriteAudio": true, // When updating an existing card, overwrite the audio field instead of skipping it. Values: true | false "overwriteAudio": true, // When updating an existing card, overwrite the audio field instead of skipping it. Values: true | false
+1 -1
View File
@@ -569,7 +569,7 @@
"refreshMinutes": 1440, // Minutes between known-word cache refreshes. "refreshMinutes": 1440, // Minutes between known-word cache refreshes.
"addMinedWordsImmediately": true, // Immediately append newly mined card words into the known-word cache. Values: true | false "addMinedWordsImmediately": true, // Immediately append newly mined card words into the known-word cache. Values: true | false
"matchMode": "headword", // Known-word matching strategy for subtitle annotations. Cache matches always receive known-word highlighting even when POS filters suppress other annotation types. Values: headword | surface "matchMode": "headword", // Known-word matching strategy for subtitle annotations. Cache matches always receive known-word highlighting even when POS filters suppress other annotation types. Values: headword | surface
"decks": {} // Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. "decks": {} // Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. Reading fields (Reading, Word Reading, ExpressionReading) are always probed so cached words match only in the reading their note teaches; words from notes without readings match in any reading.
}, // Known words setting. }, // Known words setting.
"behavior": { "behavior": {
"overwriteAudio": true, // When updating an existing card, overwrite the audio field instead of skipping it. Values: true | false "overwriteAudio": true, // When updating an existing card, overwrite the audio field instead of skipping it. Values: true | false
+2 -2
View File
@@ -703,8 +703,8 @@ export class AnkiIntegration {
}); });
} }
isKnownWord(text: string): boolean { isKnownWord(text: string, reading?: string): boolean {
return this.knownWordCache.isKnownWord(text); return this.knownWordCache.isKnownWord(text, reading);
} }
getKnownWordMatchMode(): NPlusOneMatchMode { getKnownWordMatchMode(): NPlusOneMatchMode {
+251 -7
View File
@@ -108,6 +108,55 @@ test('KnownWordCacheManager startLifecycle keeps fresh persisted cache without i
assert.equal(manager.isKnownWord('猫'), true); assert.equal(manager.isKnownWord('猫'), true);
assert.equal(calls.findNotes, 0); assert.equal(calls.findNotes, 0);
assert.equal(calls.notesInfo, 0); assert.equal(calls.notesInfo, 0);
// v2 states carry no readings, so they load usable but stale to trigger a
// prompt upgrade refresh.
assert.equal(
(
manager as unknown as {
getMsUntilNextRefresh: () => number;
}
).getMsUntilNextRefresh(),
0,
);
} finally {
Date.now = originalDateNow;
manager.stopLifecycle();
cleanup();
}
});
test('KnownWordCacheManager startLifecycle keeps fresh v3 persisted cache without immediate refresh', async () => {
const config: AnkiConnectConfig = {
knownWords: {
highlightEnabled: true,
refreshMinutes: 60,
},
};
const { manager, calls, statePath, cleanup } = createKnownWordCacheHarness(config);
const originalDateNow = Date.now;
try {
Date.now = () => 120_000;
fs.writeFileSync(
statePath,
JSON.stringify({
version: 3,
refreshedAtMs: 120_000,
scope: '{"refreshMinutes":60,"scope":"all","fieldsWord":""}',
notes: {
'1': [{ word: '猫', reading: 'ねこ' }],
},
}),
'utf-8',
);
manager.startLifecycle();
assert.equal(manager.isKnownWord('猫'), true);
assert.equal(manager.isKnownWord('猫', 'ねこ'), true);
assert.equal(manager.isKnownWord('猫', 'びょう'), false);
assert.equal(calls.findNotes, 0);
assert.equal(calls.notesInfo, 0);
assert.equal( assert.equal(
( (
manager as unknown as { manager as unknown as {
@@ -263,13 +312,11 @@ test('KnownWordCacheManager refresh incrementally reconciles deleted and edited
const persisted = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as { const persisted = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as {
version: number; version: number;
words: string[]; notes?: Record<string, Array<{ word: string; reading: string | null }>>;
notes?: Record<string, string[]>;
}; };
assert.equal(persisted.version, 2); assert.equal(persisted.version, 3);
assert.deepEqual(persisted.words.sort(), ['鳥']);
assert.deepEqual(persisted.notes, { assert.deepEqual(persisted.notes, {
'1': ['鳥'], '1': [{ word: '鳥', reading: null }],
}); });
} finally { } finally {
cleanup(); cleanup();
@@ -392,10 +439,10 @@ test('KnownWordCacheManager preserves cache state key captured before refresh wo
const persisted = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as { const persisted = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as {
scope: string; scope: string;
words: string[]; notes: Record<string, Array<{ word: string; reading: string | null }>>;
}; };
assert.equal(persisted.scope, '{"refreshMinutes":1,"scope":"all","fieldsWord":"Word"}'); assert.equal(persisted.scope, '{"refreshMinutes":1,"scope":"all","fieldsWord":"Word"}');
assert.deepEqual(persisted.words, ['猫']); assert.deepEqual(persisted.notes, { '1': [{ word: '猫', reading: null }] });
} finally { } finally {
fs.rmSync(stateDir, { recursive: true, force: true }); fs.rmSync(stateDir, { recursive: true, force: true });
} }
@@ -648,3 +695,200 @@ test('KnownWordCacheManager skips immediate append when addMinedWordsImmediately
cleanup(); cleanup();
} }
}); });
test('KnownWordCacheManager disambiguates known words by note reading', async () => {
const config: AnkiConnectConfig = {
fields: {
word: 'Word',
},
knownWords: {
highlightEnabled: true,
},
};
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
try {
clientState.findNotesResult = [1];
clientState.notesInfoResult = [
{
noteId: 1,
fields: {
Word: { value: '床' },
'Word Reading': { value: 'ゆか' },
},
},
];
await manager.refresh(true);
assert.equal(manager.isKnownWord('床'), true);
assert.equal(manager.isKnownWord('床', 'ゆか'), true);
assert.equal(manager.isKnownWord('床', 'ユカ'), true);
// Same spelling, different word (床/とこ "bed") must not match.
assert.equal(manager.isKnownWord('床', 'とこ'), false);
// Note readings stay matchable as kana words.
assert.equal(manager.isKnownWord('ゆか'), true);
assert.equal(manager.isKnownWord('とこ'), false);
} finally {
cleanup();
}
});
test('KnownWordCacheManager probes reading fields even with per-deck word fields configured', async () => {
const config: AnkiConnectConfig = {
fields: {
word: 'Expression',
},
knownWords: {
highlightEnabled: true,
decks: {
'Kaishi 1.5k': ['Word'],
},
},
};
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
try {
clientState.findNotesByQuery.set('deck:"Kaishi 1.5k"', [1]);
clientState.notesInfoResult = [
{
noteId: 1,
fields: {
Word: { value: '床' },
'Word Reading': { value: 'ゆか' },
},
},
];
await manager.refresh(true);
assert.equal(manager.isKnownWord('床', 'ゆか'), true);
assert.equal(manager.isKnownWord('床', 'とこ'), false);
} finally {
cleanup();
}
});
test('KnownWordCacheManager matches words without readings in any reading', async () => {
const config: AnkiConnectConfig = {
fields: {
word: 'Word',
},
knownWords: {
highlightEnabled: true,
},
};
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
try {
clientState.findNotesResult = [1];
clientState.notesInfoResult = [
{
noteId: 1,
fields: {
Word: { value: '床' },
},
},
];
await manager.refresh(true);
assert.equal(manager.isKnownWord('床'), true);
assert.equal(manager.isKnownWord('床', 'とこ'), true);
} finally {
cleanup();
}
});
test('KnownWordCacheManager extracts word and reading from furigana word fields', async () => {
const config: AnkiConnectConfig = {
fields: {
word: 'Word',
},
knownWords: {
highlightEnabled: true,
},
};
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
try {
clientState.findNotesResult = [1];
clientState.notesInfoResult = [
{
noteId: 1,
fields: {
Word: { value: 'お 決[き]まり' },
},
},
];
await manager.refresh(true);
assert.equal(manager.isKnownWord('お決まり'), true);
assert.equal(manager.isKnownWord('お決まり', 'おきまり'), true);
assert.equal(manager.isKnownWord('お決まり', 'おさだまり'), false);
} finally {
cleanup();
}
});
test('KnownWordCacheManager treats non-kana reading fields as words', async () => {
const config: AnkiConnectConfig = {
fields: {
word: 'Word',
},
knownWords: {
highlightEnabled: true,
},
};
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
try {
clientState.findNotesResult = [1];
clientState.notesInfoResult = [
{
noteId: 1,
fields: {
Reading: { value: '漢字' },
},
},
];
await manager.refresh(true);
assert.equal(manager.isKnownWord('漢字'), true);
} finally {
cleanup();
}
});
test('KnownWordCacheManager keeps kana-only reading notes matchable', async () => {
const config: AnkiConnectConfig = {
fields: {
word: 'Word',
},
knownWords: {
highlightEnabled: true,
},
};
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
try {
clientState.findNotesResult = [1];
clientState.notesInfoResult = [
{
noteId: 1,
fields: {
Reading: { value: 'たべる' },
},
},
];
await manager.refresh(true);
assert.equal(manager.isKnownWord('たべる'), true);
assert.equal(manager.isKnownWord('タベル'), true);
} finally {
cleanup();
}
});
+206 -101
View File
@@ -5,6 +5,16 @@ import { DEFAULT_ANKI_CONNECT_CONFIG } from '../config';
import { getConfiguredWordFieldName } from '../anki-field-config'; import { getConfiguredWordFieldName } from '../anki-field-config';
import { AnkiConnectConfig } from '../types/anki'; import { AnkiConnectConfig } from '../types/anki';
import { createLogger } from '../logger'; import { createLogger } from '../logger';
import {
DEFAULT_KNOWN_WORD_READING_FIELDS,
KnownWordEntry,
convertKatakanaToHiragana,
isReadingFieldName,
knownWordEntryListsEqual,
normalizeKnownReadingForLookup,
normalizeKnownWordEntryList,
parseFuriganaAnnotatedText,
} from './known-word-entries';
const log = createLogger('anki').child('integration.known-word-cache'); const log = createLogger('anki').child('integration.known-word-cache');
@@ -79,7 +89,16 @@ interface KnownWordCacheStateV2 {
readonly notes: Record<string, string[]>; readonly notes: Record<string, string[]>;
} }
type KnownWordCacheState = KnownWordCacheStateV1 | KnownWordCacheStateV2; interface KnownWordCacheStateV3 {
readonly version: 3;
readonly refreshedAtMs: number;
readonly scope: string;
readonly notes: Record<string, KnownWordEntry[]>;
}
type KnownWordCacheState = KnownWordCacheStateV1 | KnownWordCacheStateV2 | KnownWordCacheStateV3;
const NO_READING_KEY = '';
interface KnownWordCacheClient { interface KnownWordCacheClient {
findNotes: ( findNotes: (
@@ -106,9 +125,12 @@ type KnownWordQueryScope = {
export class KnownWordCacheManager { export class KnownWordCacheManager {
private knownWordsLastRefreshedAtMs = 0; private knownWordsLastRefreshedAtMs = 0;
private knownWordsStateKey = ''; private knownWordsStateKey = '';
private knownWords: Set<string> = new Set(); // word → (hiragana reading | NO_READING_KEY → note count). NO_READING_KEY
private wordReferenceCounts = new Map<string, number>(); // entries fail open: the word matches regardless of the token's reading.
private noteWordsById = new Map<number, string[]>(); private wordReadingCounts = new Map<string, Map<string, number>>();
// hiragana reading → note count, so kana tokens still match by reading alone.
private readingCounts = new Map<string, number>();
private noteEntriesById = new Map<number, KnownWordEntry[]>();
private knownWordsRefreshTimer: ReturnType<typeof setInterval> | null = null; private knownWordsRefreshTimer: ReturnType<typeof setInterval> | null = null;
private knownWordsRefreshTimeout: ReturnType<typeof setTimeout> | null = null; private knownWordsRefreshTimeout: ReturnType<typeof setTimeout> | null = null;
private isRefreshingKnownWords = false; private isRefreshingKnownWords = false;
@@ -120,13 +142,28 @@ export class KnownWordCacheManager {
); );
} }
isKnownWord(text: string): boolean { isKnownWord(text: string, reading?: string): boolean {
if (!this.isKnownWordCacheEnabled()) { if (!this.isKnownWordCacheEnabled()) {
return false; return false;
} }
const normalized = this.normalizeKnownWordForLookup(text); const normalized = this.normalizeKnownWordForLookup(text);
return normalized.length > 0 ? this.knownWords.has(normalized) : false; if (normalized.length === 0) {
return false;
}
const knownReadings = this.wordReadingCounts.get(normalized);
if (knownReadings && knownReadings.size > 0) {
const normalizedReading =
typeof reading === 'string' ? normalizeKnownReadingForLookup(reading) : '';
return (
normalizedReading.length === 0 ||
knownReadings.has(NO_READING_KEY) ||
knownReadings.has(normalizedReading)
);
}
return this.readingCounts.has(convertKatakanaToHiragana(normalized));
} }
refresh(force = false): Promise<void> { refresh(force = false): Promise<void> {
@@ -173,7 +210,7 @@ export class KnownWordCacheManager {
let didMutateCache = false; let didMutateCache = false;
const currentStateKey = this.getKnownWordCacheStateKey(); const currentStateKey = this.getKnownWordCacheStateKey();
if (this.knownWordsStateKey && this.knownWordsStateKey !== currentStateKey) { if (this.knownWordsStateKey && this.knownWordsStateKey !== currentStateKey) {
didMutateCache = this.knownWords.size > 0 || this.noteWordsById.size > 0; didMutateCache = this.wordReadingCounts.size > 0 || this.noteEntriesById.size > 0;
this.clearKnownWordCacheState(); this.clearKnownWordCacheState();
} }
if (!this.knownWordsStateKey) { if (!this.knownWordsStateKey) {
@@ -185,8 +222,8 @@ export class KnownWordCacheManager {
return didMutateCache; return didMutateCache;
} }
const nextWords = this.extractNormalizedKnownWordsFromNoteInfo(noteInfo, preferredFields); const nextEntries = this.extractKnownWordEntriesFromNoteInfo(noteInfo, preferredFields);
const changed = this.replaceNoteSnapshot(noteInfo.noteId, nextWords); const changed = this.replaceNoteSnapshot(noteInfo.noteId, nextEntries);
if (!changed) { if (!changed) {
return didMutateCache; return didMutateCache;
} }
@@ -198,7 +235,7 @@ export class KnownWordCacheManager {
log.info( log.info(
'Known-word cache updated in-session', 'Known-word cache updated in-session',
`noteId=${noteInfo.noteId}`, `noteId=${noteInfo.noteId}`,
`wordCount=${nextWords.length}`, `wordCount=${nextEntries.length}`,
`scope=${getKnownWordCacheScopeForConfig(this.deps.getConfig())}`, `scope=${getKnownWordCacheScopeForConfig(this.deps.getConfig())}`,
); );
return true; return true;
@@ -236,11 +273,11 @@ export class KnownWordCacheManager {
const noteFieldsById = await this.fetchKnownWordNoteFieldsById(); const noteFieldsById = await this.fetchKnownWordNoteFieldsById();
const currentNoteIds = Array.from(noteFieldsById.keys()).sort((a, b) => a - b); const currentNoteIds = Array.from(noteFieldsById.keys()).sort((a, b) => a - b);
if (this.noteWordsById.size === 0) { if (this.noteEntriesById.size === 0) {
await this.rebuildFromCurrentNotes(currentNoteIds, noteFieldsById); await this.rebuildFromCurrentNotes(currentNoteIds, noteFieldsById);
} else { } else {
const currentNoteIdSet = new Set(currentNoteIds); const currentNoteIdSet = new Set(currentNoteIds);
for (const noteId of Array.from(this.noteWordsById.keys())) { for (const noteId of Array.from(this.noteEntriesById.keys())) {
if (!currentNoteIdSet.has(noteId)) { if (!currentNoteIdSet.has(noteId)) {
this.removeNoteSnapshot(noteId); this.removeNoteSnapshot(noteId);
} }
@@ -251,7 +288,7 @@ export class KnownWordCacheManager {
for (const noteInfo of noteInfos) { for (const noteInfo of noteInfos) {
this.replaceNoteSnapshot( this.replaceNoteSnapshot(
noteInfo.noteId, noteInfo.noteId,
this.extractNormalizedKnownWordsFromNoteInfo( this.extractKnownWordEntriesFromNoteInfo(
noteInfo, noteInfo,
noteFieldsById.get(noteInfo.noteId), noteFieldsById.get(noteInfo.noteId),
), ),
@@ -266,7 +303,7 @@ export class KnownWordCacheManager {
log.info( log.info(
'Known-word cache refreshed', 'Known-word cache refreshed',
`noteCount=${currentNoteIds.length}`, `noteCount=${currentNoteIds.length}`,
`wordCount=${this.knownWords.size}`, `wordCount=${this.wordReadingCounts.size}`,
); );
} catch (error) { } catch (error) {
log.warn('Failed to refresh known-word cache:', (error as Error).message); log.warn('Failed to refresh known-word cache:', (error as Error).message);
@@ -291,7 +328,13 @@ export class KnownWordCacheManager {
private getDefaultKnownWordFields(): string[] { private getDefaultKnownWordFields(): string[] {
const configuredWordField = getConfiguredWordFieldName(this.deps.getConfig()); const configuredWordField = getConfiguredWordFieldName(this.deps.getConfig());
return [...new Set([configuredWordField, 'Word', 'Reading', 'Word Reading'])]; return this.withDefaultReadingFields([configuredWordField, 'Word']);
}
// Reading fields are always probed (even when a deck configures explicit
// word fields) so entries can carry the reading their note teaches.
private withDefaultReadingFields(fields: string[]): string[] {
return [...new Set([...fields, ...DEFAULT_KNOWN_WORD_READING_FIELDS])];
} }
private getKnownWordDecks(): string[] { private getKnownWordDecks(): string[] {
@@ -337,7 +380,9 @@ export class KnownWordCacheManager {
.filter((field) => field.length > 0), .filter((field) => field.length > 0),
), ),
]; ];
return normalizedFields.length > 0 ? normalizedFields : this.getDefaultKnownWordFields(); return normalizedFields.length > 0
? this.withDefaultReadingFields(normalizedFields)
: this.getDefaultKnownWordFields();
} }
const deckFields = selectedDeckEntry[1]; const deckFields = selectedDeckEntry[1];
@@ -351,7 +396,7 @@ export class KnownWordCacheManager {
), ),
]; ];
if (normalizedFields.length > 0) { if (normalizedFields.length > 0) {
return normalizedFields; return this.withDefaultReadingFields(normalizedFields);
} }
} }
@@ -382,7 +427,10 @@ export class KnownWordCacheManager {
: []; : [];
scopes.push({ scopes.push({
query: `deck:"${escapeAnkiSearchValue(trimmedDeckName)}"`, query: `deck:"${escapeAnkiSearchValue(trimmedDeckName)}"`,
fields: normalizedFields.length > 0 ? normalizedFields : this.getDefaultKnownWordFields(), fields:
normalizedFields.length > 0
? this.withDefaultReadingFields(normalizedFields)
: this.getDefaultKnownWordFields(),
}); });
} }
if (scopes.length > 0) { if (scopes.length > 0) {
@@ -490,7 +538,7 @@ export class KnownWordCacheManager {
for (const noteInfo of noteInfos) { for (const noteInfo of noteInfos) {
this.replaceNoteSnapshot( this.replaceNoteSnapshot(
noteInfo.noteId, noteInfo.noteId,
this.extractNormalizedKnownWordsFromNoteInfo(noteInfo, noteFieldsById.get(noteInfo.noteId)), this.extractKnownWordEntriesFromNoteInfo(noteInfo, noteFieldsById.get(noteInfo.noteId)),
); );
} }
} }
@@ -519,56 +567,77 @@ export class KnownWordCacheManager {
return noteInfos; return noteInfos;
} }
private replaceNoteSnapshot(noteId: number, nextWords: string[]): boolean { private replaceNoteSnapshot(noteId: number, nextEntries: KnownWordEntry[]): boolean {
const normalizedWords = normalizeKnownWordList(nextWords); const normalizedEntries = normalizeKnownWordEntryList(nextEntries);
const previousWords = this.noteWordsById.get(noteId) ?? []; const previousEntries = this.noteEntriesById.get(noteId) ?? [];
if (knownWordListsEqual(previousWords, normalizedWords)) { if (knownWordEntryListsEqual(previousEntries, normalizedEntries)) {
return false; return false;
} }
this.removeWordsFromCounts(previousWords); this.removeEntriesFromCounts(previousEntries);
if (normalizedWords.length > 0) { if (normalizedEntries.length > 0) {
this.noteWordsById.set(noteId, normalizedWords); this.noteEntriesById.set(noteId, normalizedEntries);
this.addWordsToCounts(normalizedWords); this.addEntriesToCounts(normalizedEntries);
} else { } else {
this.noteWordsById.delete(noteId); this.noteEntriesById.delete(noteId);
} }
return true; return true;
} }
private removeNoteSnapshot(noteId: number): void { private removeNoteSnapshot(noteId: number): void {
const previousWords = this.noteWordsById.get(noteId); const previousEntries = this.noteEntriesById.get(noteId);
if (!previousWords) { if (!previousEntries) {
return; return;
} }
this.noteWordsById.delete(noteId); this.noteEntriesById.delete(noteId);
this.removeWordsFromCounts(previousWords); this.removeEntriesFromCounts(previousEntries);
} }
private addWordsToCounts(words: string[]): void { private addEntriesToCounts(entries: KnownWordEntry[]): void {
for (const word of words) { for (const entry of entries) {
const nextCount = (this.wordReferenceCounts.get(word) ?? 0) + 1; const readingKey = entry.reading ?? NO_READING_KEY;
this.wordReferenceCounts.set(word, nextCount); let readings = this.wordReadingCounts.get(entry.word);
this.knownWords.add(word); if (!readings) {
readings = new Map();
this.wordReadingCounts.set(entry.word, readings);
}
readings.set(readingKey, (readings.get(readingKey) ?? 0) + 1);
if (entry.reading) {
this.readingCounts.set(entry.reading, (this.readingCounts.get(entry.reading) ?? 0) + 1);
}
} }
} }
private removeWordsFromCounts(words: string[]): void { private removeEntriesFromCounts(entries: KnownWordEntry[]): void {
for (const word of words) { for (const entry of entries) {
const nextCount = (this.wordReferenceCounts.get(word) ?? 0) - 1; const readingKey = entry.reading ?? NO_READING_KEY;
if (nextCount > 0) { const readings = this.wordReadingCounts.get(entry.word);
this.wordReferenceCounts.set(word, nextCount); if (readings) {
} else { const nextCount = (readings.get(readingKey) ?? 0) - 1;
this.wordReferenceCounts.delete(word); if (nextCount > 0) {
this.knownWords.delete(word); readings.set(readingKey, nextCount);
} else {
readings.delete(readingKey);
if (readings.size === 0) {
this.wordReadingCounts.delete(entry.word);
}
}
}
if (entry.reading) {
const nextReadingCount = (this.readingCounts.get(entry.reading) ?? 0) - 1;
if (nextReadingCount > 0) {
this.readingCounts.set(entry.reading, nextReadingCount);
} else {
this.readingCounts.delete(entry.reading);
}
} }
} }
} }
private clearInMemoryState(): void { private clearInMemoryState(): void {
this.knownWords = new Set(); this.wordReadingCounts = new Map();
this.wordReferenceCounts = new Map(); this.readingCounts = new Map();
this.noteWordsById = new Map(); this.noteEntriesById = new Map();
this.knownWordsLastRefreshedAtMs = 0; this.knownWordsLastRefreshedAtMs = 0;
} }
@@ -601,32 +670,48 @@ export class KnownWordCacheManager {
} }
this.clearInMemoryState(); this.clearInMemoryState();
if (parsed.version === 3) {
for (const [noteIdKey, entries] of Object.entries(parsed.notes)) {
const noteId = Number.parseInt(noteIdKey, 10);
if (!Number.isInteger(noteId) || noteId <= 0) {
continue;
}
const normalizedEntries = normalizeKnownWordEntryList(entries);
if (normalizedEntries.length === 0) {
continue;
}
this.noteEntriesById.set(noteId, normalizedEntries);
this.addEntriesToCounts(normalizedEntries);
}
this.knownWordsLastRefreshedAtMs = parsed.refreshedAtMs;
this.knownWordsStateKey = parsed.scope;
return;
}
if (parsed.version === 2) { if (parsed.version === 2) {
// Older states have no readings; load them reading-less (fail-open,
// matching the old behavior) but leave the cache marked stale so the
// next refresh upgrades entries with readings from Anki.
for (const [noteIdKey, words] of Object.entries(parsed.notes)) { for (const [noteIdKey, words] of Object.entries(parsed.notes)) {
const noteId = Number.parseInt(noteIdKey, 10); const noteId = Number.parseInt(noteIdKey, 10);
if (!Number.isInteger(noteId) || noteId <= 0) { if (!Number.isInteger(noteId) || noteId <= 0) {
continue; continue;
} }
const normalizedWords = normalizeKnownWordList(words); const normalizedEntries = normalizeKnownWordEntryList(
if (normalizedWords.length === 0) { words.map((word) => ({ word: this.normalizeKnownWordForLookup(word), reading: null })),
);
if (normalizedEntries.length === 0) {
continue; continue;
} }
this.noteWordsById.set(noteId, normalizedWords); this.noteEntriesById.set(noteId, normalizedEntries);
this.addWordsToCounts(normalizedWords); this.addEntriesToCounts(normalizedEntries);
}
} else {
for (const value of parsed.words) {
const normalized = this.normalizeKnownWordForLookup(value);
if (!normalized) {
continue;
}
this.knownWords.add(normalized);
this.wordReferenceCounts.set(normalized, 1);
} }
this.knownWordsStateKey = parsed.scope;
return;
} }
this.knownWordsLastRefreshedAtMs = parsed.refreshedAtMs; // v1 has no per-note snapshots to convert; refetch from Anki.
this.knownWordsStateKey = parsed.scope; this.knownWordsStateKey = this.getKnownWordCacheStateKey();
} catch (error) { } catch (error) {
log.warn('Failed to load known-word cache state:', (error as Error).message); log.warn('Failed to load known-word cache state:', (error as Error).message);
this.clearInMemoryState(); this.clearInMemoryState();
@@ -636,18 +721,17 @@ export class KnownWordCacheManager {
private persistKnownWordCacheState(): void { private persistKnownWordCacheState(): void {
try { try {
const notes: Record<string, string[]> = {}; const notes: Record<string, KnownWordEntry[]> = {};
for (const [noteId, words] of this.noteWordsById.entries()) { for (const [noteId, entries] of this.noteEntriesById.entries()) {
if (words.length > 0) { if (entries.length > 0) {
notes[String(noteId)] = words; notes[String(noteId)] = entries;
} }
} }
const state: KnownWordCacheStateV2 = { const state: KnownWordCacheStateV3 = {
version: 2, version: 3,
refreshedAtMs: this.knownWordsLastRefreshedAtMs, refreshedAtMs: this.knownWordsLastRefreshedAtMs,
scope: this.knownWordsStateKey, scope: this.knownWordsStateKey,
words: Array.from(this.knownWords),
notes, notes,
}; };
fs.writeFileSync(this.statePath, JSON.stringify(state), 'utf-8'); fs.writeFileSync(this.statePath, JSON.stringify(state), 'utf-8');
@@ -659,14 +743,18 @@ export class KnownWordCacheManager {
private isKnownWordCacheStateValid(value: unknown): value is KnownWordCacheState { private isKnownWordCacheStateValid(value: unknown): value is KnownWordCacheState {
if (typeof value !== 'object' || value === null) return false; if (typeof value !== 'object' || value === null) return false;
const candidate = value as Record<string, unknown>; const candidate = value as Record<string, unknown>;
if (candidate.version !== 1 && candidate.version !== 2) return false; if (candidate.version !== 1 && candidate.version !== 2 && candidate.version !== 3) {
if (typeof candidate.refreshedAtMs !== 'number') return false;
if (typeof candidate.scope !== 'string') return false;
if (!Array.isArray(candidate.words)) return false;
if (!candidate.words.every((entry: unknown) => typeof entry === 'string')) {
return false; return false;
} }
if (candidate.version === 2) { if (typeof candidate.refreshedAtMs !== 'number') return false;
if (typeof candidate.scope !== 'string') return false;
if (candidate.version !== 3) {
if (!Array.isArray(candidate.words)) return false;
if (!candidate.words.every((entry: unknown) => typeof entry === 'string')) {
return false;
}
}
if (candidate.version === 2 || candidate.version === 3) {
if ( if (
typeof candidate.notes !== 'object' || typeof candidate.notes !== 'object' ||
candidate.notes === null || candidate.notes === null ||
@@ -674,10 +762,18 @@ export class KnownWordCacheManager {
) { ) {
return false; return false;
} }
const isValidNoteEntry =
candidate.version === 2
? (entry: unknown): boolean => typeof entry === 'string'
: (entry: unknown): boolean =>
typeof entry === 'object' &&
entry !== null &&
typeof (entry as KnownWordEntry).word === 'string' &&
((entry as KnownWordEntry).reading === null ||
typeof (entry as KnownWordEntry).reading === 'string');
if ( if (
!Object.values(candidate.notes as Record<string, unknown>).every( !Object.values(candidate.notes as Record<string, unknown>).every(
(entry) => (noteEntries) => Array.isArray(noteEntries) && noteEntries.every(isValidNoteEntry),
Array.isArray(entry) && entry.every((word: unknown) => typeof word === 'string'),
) )
) { ) {
return false; return false;
@@ -686,11 +782,12 @@ export class KnownWordCacheManager {
return true; return true;
} }
private extractNormalizedKnownWordsFromNoteInfo( private extractKnownWordEntriesFromNoteInfo(
noteInfo: KnownWordCacheNoteInfo, noteInfo: KnownWordCacheNoteInfo,
preferredFields = this.getConfiguredFields(), preferredFields = this.getConfiguredFields(),
): string[] { ): KnownWordEntry[] {
const words: string[] = []; const wordValues: string[] = [];
let noteReading: string | null = null;
for (const preferredField of preferredFields) { for (const preferredField of preferredFields) {
const fieldName = resolveFieldName(Object.keys(noteInfo.fields), preferredField); const fieldName = resolveFieldName(Object.keys(noteInfo.fields), preferredField);
if (!fieldName) continue; if (!fieldName) continue;
@@ -698,12 +795,36 @@ export class KnownWordCacheManager {
const raw = noteInfo.fields[fieldName]?.value; const raw = noteInfo.fields[fieldName]?.value;
if (!raw) continue; if (!raw) continue;
const normalized = this.normalizeKnownWordForLookup(raw); const cleaned = this.normalizeRawKnownWordValue(raw);
if (normalized) { if (!cleaned) continue;
words.push(normalized);
if (isReadingFieldName(preferredField)) {
const normalizedReading = normalizeKnownReadingForLookup(cleaned);
if (normalizedReading) {
noteReading ??= normalizedReading;
continue;
}
// Non-kana content in a reading field: treat it as a word so decks
// with repurposed reading fields keep matching (fail-open).
} }
wordValues.push(cleaned);
} }
return normalizeKnownWordList(words);
const entries: KnownWordEntry[] = [];
for (const value of wordValues) {
const parsed = parseFuriganaAnnotatedText(value);
const word = parsed.text.trim().toLowerCase();
if (!word) continue;
const inlineReading = parsed.reading ? normalizeKnownReadingForLookup(parsed.reading) : '';
entries.push({ word, reading: inlineReading || noteReading });
}
// Kana-only notes (reading field but no word field) stay matchable.
if (entries.length === 0 && noteReading) {
entries.push({ word: noteReading, reading: noteReading });
}
return normalizeKnownWordEntryList(entries);
} }
private normalizeRawKnownWordValue(value: string): string { private normalizeRawKnownWordValue(value: string): string {
@@ -718,22 +839,6 @@ export class KnownWordCacheManager {
} }
} }
function normalizeKnownWordList(words: string[]): string[] {
return [...new Set(words.map((word) => word.trim()).filter((word) => word.length > 0))].sort();
}
function knownWordListsEqual(left: string[], right: string[]): boolean {
if (left.length !== right.length) {
return false;
}
for (let index = 0; index < left.length; index += 1) {
if (left[index] !== right[index]) {
return false;
}
}
return true;
}
function resolveFieldName(availableFieldNames: string[], preferredName: string): string | null { function resolveFieldName(availableFieldNames: string[], preferredName: string): string | null {
const exact = availableFieldNames.find((name) => name === preferredName); const exact = availableFieldNames.find((name) => name === preferredName);
if (exact) return exact; if (exact) return exact;
+113
View File
@@ -0,0 +1,113 @@
// Known-word cache entries pair a word with the reading its Anki note teaches,
// so spelling collisions across readings (e.g. 床/ゆか vs 床/とこ) don't mark
// unrelated words as known. reading === null means the note carries no usable
// reading and the word matches in any reading (fail-open).
export interface KnownWordEntry {
word: string;
reading: string | null;
}
const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
const KATAKANA_CODEPOINT_START = 0x30a1;
const KATAKANA_CODEPOINT_END = 0x30f6;
const FURIGANA_SEGMENT_PATTERN = /([^\s \[\]]*)\[([^\]]*)\]/g;
const FURIGANA_BRACKET_PATTERN = /\[[^\]]*\]/g;
const WHITESPACE_PATTERN = /[\s ]+/g;
// Reading-bearing field names probed on every known-word note, in addition to
// any configured word fields (covers Kaishi's "Word Reading" and Lapis's
// "ExpressionReading" note types).
export const DEFAULT_KNOWN_WORD_READING_FIELDS = [
'Reading',
'Word Reading',
'ExpressionReading',
'Expression Reading',
];
export function isReadingFieldName(fieldName: string): boolean {
return /reading/i.test(fieldName);
}
export function convertKatakanaToHiragana(text: string): string {
let converted = '';
for (const char of text) {
const code = char.codePointAt(0);
if (code !== undefined && code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END) {
converted += String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET);
continue;
}
converted += char;
}
return converted;
}
function isHiraganaReadingChar(char: string): boolean {
const code = char.codePointAt(0);
if (code === undefined) {
return false;
}
return (code >= 0x3041 && code <= 0x309f) || code === 0x30fc;
}
// Splits Anki furigana syntax (`床[とこ]`, `お 決[き]まり`) into base text and
// reading. Values without brackets pass through with reading null.
export function parseFuriganaAnnotatedText(value: string): {
text: string;
reading: string | null;
} {
if (!value.includes('[')) {
return { text: value, reading: null };
}
const text = value.replace(FURIGANA_BRACKET_PATTERN, '').replace(WHITESPACE_PATTERN, '');
const reading = value.replace(FURIGANA_SEGMENT_PATTERN, '$2').replace(WHITESPACE_PATTERN, '');
return { text, reading: reading.length > 0 ? reading : null };
}
// Returns the hiragana-normalized reading, or '' when the value is not a
// plausible kana reading (callers fall back to text-only matching then).
export function normalizeKnownReadingForLookup(value: string): string {
const parsed = parseFuriganaAnnotatedText(value.trim());
const candidate = (parsed.reading ?? parsed.text).trim();
if (!candidate) {
return '';
}
const hiragana = convertKatakanaToHiragana(candidate);
for (const char of hiragana) {
if (!isHiraganaReadingChar(char)) {
return '';
}
}
return hiragana;
}
export function makeKnownWordEntryKey(entry: KnownWordEntry): string {
return `${entry.word}\u0000${entry.reading ?? ''}`;
}
export function normalizeKnownWordEntryList(entries: KnownWordEntry[]): KnownWordEntry[] {
const byKey = new Map<string, KnownWordEntry>();
for (const entry of entries) {
const word = entry.word.trim();
if (!word) {
continue;
}
const reading = entry.reading?.trim() || null;
const normalized: KnownWordEntry = { word, reading };
byKey.set(makeKnownWordEntryKey(normalized), normalized);
}
return [...byKey.values()].sort((left, right) =>
makeKnownWordEntryKey(left).localeCompare(makeKnownWordEntryKey(right)),
);
}
export function knownWordEntryListsEqual(left: KnownWordEntry[], right: KnownWordEntry[]): boolean {
if (left.length !== right.length) {
return false;
}
for (let index = 0; index < left.length; index += 1) {
if (makeKnownWordEntryKey(left[index]!) !== makeKnownWordEntryKey(right[index]!)) {
return false;
}
}
return true;
}
@@ -317,7 +317,7 @@ export function buildIntegrationConfigOptionRegistry(
kind: 'object', kind: 'object',
defaultValue: defaultConfig.ankiConnect.knownWords.decks, defaultValue: defaultConfig.ankiConnect.knownWords.decks,
description: description:
'Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }.', 'Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. Reading fields (Reading, Word Reading, ExpressionReading) are always probed so cached words match only in the reading their note teaches; words from notes without readings match in any reading.',
}, },
{ {
path: 'ankiConnect.isKiku.fieldGrouping', path: 'ankiConnect.isKiku.fieldGrouping',
+4 -3
View File
@@ -42,7 +42,7 @@ export interface TokenizerServiceDeps {
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void; setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
getYomitanParserInitPromise: () => Promise<boolean> | null; getYomitanParserInitPromise: () => Promise<boolean> | null;
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void; setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
isKnownWord: (text: string) => boolean; isKnownWord: (text: string, reading?: string) => boolean;
getKnownWordMatchMode: () => NPlusOneMatchMode; getKnownWordMatchMode: () => NPlusOneMatchMode;
getKnownWordsEnabled?: () => boolean; getKnownWordsEnabled?: () => boolean;
getJlptLevel: (text: string) => JlptLevel | null; getJlptLevel: (text: string) => JlptLevel | null;
@@ -77,7 +77,7 @@ export interface TokenizerDepsRuntimeOptions {
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void; setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
getYomitanParserInitPromise: () => Promise<boolean> | null; getYomitanParserInitPromise: () => Promise<boolean> | null;
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void; setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
isKnownWord: (text: string) => boolean; isKnownWord: (text: string, reading?: string) => boolean;
getKnownWordMatchMode: () => NPlusOneMatchMode; getKnownWordMatchMode: () => NPlusOneMatchMode;
getKnownWordsEnabled?: () => boolean; getKnownWordsEnabled?: () => boolean;
getJlptLevel: (text: string) => JlptLevel | null; getJlptLevel: (text: string) => JlptLevel | null;
@@ -129,7 +129,7 @@ const INVISIBLE_SEPARATOR_PATTERN = /[\u200b\u2060\ufeff]/g;
function getKnownWordLookup( function getKnownWordLookup(
deps: TokenizerServiceDeps, deps: TokenizerServiceDeps,
options: TokenizerAnnotationOptions, options: TokenizerAnnotationOptions,
): (text: string) => boolean { ): (text: string, reading?: string) => boolean {
if (!options.knownWordsEnabled && !options.nPlusOneEnabled) { if (!options.knownWordsEnabled && !options.nPlusOneEnabled) {
return () => false; return () => false;
} }
@@ -723,6 +723,7 @@ async function parseWithYomitanInternalParser(
surface: token.surface, surface: token.surface,
reading: token.reading, reading: token.reading,
headword: token.headword, headword: token.headword,
headwordReading: token.headwordReading,
startPos: token.startPos, startPos: token.startPos,
endPos: token.endPos, endPos: token.endPos,
partOfSpeech: posMetadata.partOfSpeech, partOfSpeech: posMetadata.partOfSpeech,
@@ -56,6 +56,70 @@ test('annotateTokens known-word match mode uses headword vs surface', () => {
assert.equal(surfaceResult[0]?.isKnown, false); assert.equal(surfaceResult[0]?.isKnown, false);
}); });
test('annotateTokens passes dictionary-form reading so spelling collisions stay unknown', () => {
// とこ (colloquial ところ) resolves to headword 床/とこ; a known 床/ゆか card
// must not mark it known (#138 regression).
const cache = new Map([['床', 'ゆか']]);
const isKnownWord = (text: string, reading?: string): boolean => {
if (!cache.has(text)) {
return false;
}
return reading === undefined || cache.get(text) === reading;
};
const tokens = [
makeToken({
surface: 'とこ',
headword: '床',
reading: 'とこ',
headwordReading: 'とこ',
endPos: 2,
}),
];
const result = annotateTokens(tokens, makeDeps({ isKnownWord }));
assert.equal(result[0]?.isKnown, false);
});
test('annotateTokens keeps inflected known words matched via headword reading', () => {
const isKnownWord = (text: string, reading?: string): boolean =>
text === '行く' && (reading === undefined || reading === 'いく');
const tokens = [
makeToken({
surface: '行きたい',
headword: '行く',
reading: 'いきたい',
headwordReading: 'いく',
partOfSpeech: PartOfSpeech.verb,
endPos: 4,
}),
];
const result = annotateTokens(tokens, makeDeps({ isKnownWord }));
assert.equal(result[0]?.isKnown, true);
});
test('annotateTokens omits reading for headword match when token lacks headword reading and is inflected', () => {
// MeCab tokens have no dictionary-form reading; the surface reading of an
// inflected form must not be compared against the note's dictionary reading.
const isKnownWord = (text: string, reading?: string): boolean =>
text === '食べる' && reading === undefined;
const tokens = [
makeToken({
surface: '食べた',
headword: '食べる',
reading: 'タベタ',
partOfSpeech: PartOfSpeech.verb,
endPos: 3,
}),
];
const result = annotateTokens(tokens, makeDeps({ isKnownWord }));
assert.equal(result[0]?.isKnown, true);
});
test('annotateTokens marks known words when N+1 is disabled', () => { test('annotateTokens marks known words when N+1 is disabled', () => {
const tokens = [ const tokens = [
makeToken({ surface: '私', headword: '私', startPos: 0, endPos: 1 }), makeToken({ surface: '私', headword: '私', startPos: 0, endPos: 1 }),
+41 -10
View File
@@ -25,7 +25,7 @@ const jlptLevelLookupCaches = new WeakMap<
>(); >();
export interface AnnotationStageDeps { export interface AnnotationStageDeps {
isKnownWord: (text: string) => boolean; isKnownWord: (text: string, reading?: string) => boolean;
knownWordMatchMode: NPlusOneMatchMode; knownWordMatchMode: NPlusOneMatchMode;
getJlptLevel: (text: string) => JlptLevel | null; getJlptLevel: (text: string) => JlptLevel | null;
} }
@@ -661,26 +661,57 @@ function isCompleteReadingForSurface(surface: string, reading: string): boolean
return true; return true;
} }
// Returns the token's trimmed reading only when it plausibly covers the surface
// (see isCompleteReadingForSurface); undefined otherwise. Shared so the
// known-word reading disambiguation and the reading fallback stay in sync if the
// validity rule changes.
function resolveCompleteTokenReading(token: MergedToken): string | undefined {
const normalizedReading = token.reading.trim();
if (!normalizedReading || !isCompleteReadingForSurface(token.surface, normalizedReading)) {
return undefined;
}
return normalizedReading;
}
// Reading to disambiguate the known-word text match, or undefined when the
// token has no reading that describes the match text: in headword mode an
// inflected surface's reading does not match the dictionary form's reading,
// and partial furigana readings (see isCompleteReadingForSurface) would cause
// false negatives. Undefined falls back to text-only matching (fail-open).
function resolveKnownWordReadingForMatch(
token: MergedToken,
knownWordMatchMode: NPlusOneMatchMode,
): string | undefined {
if (knownWordMatchMode === 'headword') {
const headwordReading = token.headwordReading?.trim();
if (headwordReading) {
return headwordReading;
}
if (token.surface !== token.headword) {
return undefined;
}
}
return resolveCompleteTokenReading(token);
}
function computeTokenKnownStatus( function computeTokenKnownStatus(
token: MergedToken, token: MergedToken,
isKnownWord: (text: string) => boolean, isKnownWord: (text: string, reading?: string) => boolean,
knownWordMatchMode: NPlusOneMatchMode, knownWordMatchMode: NPlusOneMatchMode,
): boolean { ): boolean {
const matchText = resolveKnownWordText(token.surface, token.headword, knownWordMatchMode); const matchText = resolveKnownWordText(token.surface, token.headword, knownWordMatchMode);
if (token.isKnown || (matchText ? isKnownWord(matchText) : false)) { const matchReading = resolveKnownWordReadingForMatch(token, knownWordMatchMode);
if (token.isKnown || (matchText ? isKnownWord(matchText, matchReading) : false)) {
return true; return true;
} }
const normalizedReading = token.reading.trim(); const fallbackReading = resolveCompleteTokenReading(token);
if (!normalizedReading) { if (!fallbackReading) {
return false; return false;
} }
if (!isCompleteReadingForSurface(token.surface, normalizedReading)) { return fallbackReading !== matchText.trim() && isKnownWord(fallbackReading);
return false;
}
return normalizedReading !== matchText.trim() && isKnownWord(normalizedReading);
} }
function filterTokenFrequencyRank( function filterTokenFrequencyRank(
@@ -966,6 +966,7 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF
surface: '潜み', surface: '潜み',
reading: 'ひそみ', reading: 'ひそみ',
headword: '潜む', headword: '潜む',
headwordReading: 'ひそむ',
startPos: 0, startPos: 0,
endPos: 2, endPos: 2,
isNameMatch: false, isNameMatch: false,
@@ -1032,6 +1033,7 @@ test('requestYomitanScanTokens emits complete readings for kanji-kana compounds'
surface: '待ち合わせてる', surface: '待ち合わせてる',
reading: 'まちあわせてる', reading: 'まちあわせてる',
headword: '待ち合わせる', headword: '待ち合わせる',
headwordReading: 'まちあわせる',
startPos: 0, startPos: 0,
endPos: 7, endPos: 7,
isNameMatch: false, isNameMatch: false,
@@ -1139,6 +1141,7 @@ test('requestYomitanScanTokens uses frequency from later exact-match entry when
surface: '者', surface: '者',
reading: 'もの', reading: 'もの',
headword: '者', headword: '者',
headwordReading: 'もの',
startPos: 0, startPos: 0,
endPos: 1, endPos: 1,
isNameMatch: false, isNameMatch: false,
@@ -1240,6 +1243,7 @@ test('requestYomitanScanTokens can use frequency from later exact secondary-matc
surface: '者', surface: '者',
reading: 'もの', reading: 'もの',
headword: '者', headword: '者',
headwordReading: 'もの',
startPos: 0, startPos: 0,
endPos: 1, endPos: 1,
isNameMatch: false, isNameMatch: false,
@@ -1340,6 +1344,7 @@ test('requestYomitanScanTokens uses exact frequency entry when selected reading
surface: '第二', surface: '第二',
reading: 'だいに', reading: 'だいに',
headword: '第二', headword: '第二',
headwordReading: 'だいに',
startPos: 0, startPos: 0,
endPos: 2, endPos: 2,
isNameMatch: false, isNameMatch: false,
@@ -51,6 +51,7 @@ export interface YomitanScanToken {
surface: string; surface: string;
reading: string; reading: string;
headword: string; headword: string;
headwordReading?: string;
startPos: number; startPos: number;
endPos: number; endPos: number;
isNameMatch?: boolean; isNameMatch?: boolean;
@@ -92,6 +93,7 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
typeof entry.surface === 'string' && typeof entry.surface === 'string' &&
typeof entry.reading === 'string' && typeof entry.reading === 'string' &&
typeof entry.headword === 'string' && typeof entry.headword === 'string' &&
(entry.headwordReading === undefined || typeof entry.headwordReading === 'string') &&
typeof entry.startPos === 'number' && typeof entry.startPos === 'number' &&
typeof entry.endPos === 'number' && typeof entry.endPos === 'number' &&
(entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean') && (entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean') &&
@@ -1318,6 +1320,7 @@ ${YOMITAN_SCANNING_HELPERS}
surface: segments.map((segment) => segment.text).join("") || source, surface: segments.map((segment) => segment.text).join("") || source,
reading: segments.map(getSegmentReadingContribution).join(""), reading: segments.map(getSegmentReadingContribution).join(""),
headword: preferredHeadword.term, headword: preferredHeadword.term,
headwordReading: reading || undefined,
startPos: i, startPos: i,
endPos: i + originalTextLength, endPos: i + originalTextLength,
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true, isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
+1 -1
View File
@@ -4638,7 +4638,7 @@ const {
setYomitanParserInitPromise: (promise) => { setYomitanParserInitPromise: (promise) => {
appState.yomitanParserInitPromise = promise; appState.yomitanParserInitPromise = promise;
}, },
isKnownWord: (text) => Boolean(appState.ankiIntegration?.isKnownWord(text)), isKnownWord: (text, reading) => Boolean(appState.ankiIntegration?.isKnownWord(text, reading)),
recordLookup: (hit) => { recordLookup: (hit) => {
ensureImmersionTrackerStarted(); ensureImmersionTrackerStarted();
appState.immersionTracker?.recordLookup(hit); appState.immersionTracker?.recordLookup(hit);
@@ -37,8 +37,8 @@ export function createBuildTokenizerDepsMainHandler(deps: TokenizerMainDeps) {
getYomitanParserInitPromise: () => deps.getYomitanParserInitPromise(), getYomitanParserInitPromise: () => deps.getYomitanParserInitPromise(),
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => setYomitanParserInitPromise: (promise: Promise<boolean> | null) =>
deps.setYomitanParserInitPromise(promise), deps.setYomitanParserInitPromise(promise),
isKnownWord: (text: string) => { isKnownWord: (text: string, reading?: string) => {
const hit = deps.isKnownWord(text); const hit = deps.isKnownWord(text, reading);
deps.recordLookup(hit); deps.recordLookup(hit);
return hit; return hit;
}, },
+2
View File
@@ -29,6 +29,8 @@ export interface MergedToken {
surface: string; surface: string;
reading: string; reading: string;
headword: string; headword: string;
/** Dictionary-form reading of headword (kana), when the parser provides it. */
headwordReading?: string;
startPos: number; startPos: number;
endPos: number; endPos: number;
partOfSpeech: PartOfSpeech; partOfSpeech: PartOfSpeech;