mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-07 13:08:54 -07:00
feat(anki): reading-aware known-word matching (cache v3) (#142)
This commit is contained in:
@@ -0,0 +1,4 @@
|
|||||||
|
type: fixed
|
||||||
|
area: overlay
|
||||||
|
|
||||||
|
- Fixed words being highlighted green as known when a same-spelled Anki card taught a different reading (e.g. とこ parsed as 床 "bed" matching a known 床/ゆか "floor" card). The known-word cache now stores each card's word together with its reading and only matches when the token's reading agrees; cards without a reading field keep matching in any reading as before.
|
||||||
@@ -569,7 +569,7 @@
|
|||||||
"refreshMinutes": 1440, // Minutes between known-word cache refreshes.
|
"refreshMinutes": 1440, // Minutes between known-word cache refreshes.
|
||||||
"addMinedWordsImmediately": true, // Immediately append newly mined card words into the known-word cache. Values: true | false
|
"addMinedWordsImmediately": true, // Immediately append newly mined card words into the known-word cache. Values: true | false
|
||||||
"matchMode": "headword", // Known-word matching strategy for subtitle annotations. Cache matches always receive known-word highlighting even when POS filters suppress other annotation types. Values: headword | surface
|
"matchMode": "headword", // Known-word matching strategy for subtitle annotations. Cache matches always receive known-word highlighting even when POS filters suppress other annotation types. Values: headword | surface
|
||||||
"decks": {} // Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }.
|
"decks": {} // Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. Reading fields (Reading, Word Reading, ExpressionReading) are always probed so cached words match only in the reading their note teaches; words from notes without readings match in any reading.
|
||||||
}, // Known words setting.
|
}, // Known words setting.
|
||||||
"behavior": {
|
"behavior": {
|
||||||
"overwriteAudio": true, // When updating an existing card, overwrite the audio field instead of skipping it. Values: true | false
|
"overwriteAudio": true, // When updating an existing card, overwrite the audio field instead of skipping it. Values: true | false
|
||||||
|
|||||||
@@ -569,7 +569,7 @@
|
|||||||
"refreshMinutes": 1440, // Minutes between known-word cache refreshes.
|
"refreshMinutes": 1440, // Minutes between known-word cache refreshes.
|
||||||
"addMinedWordsImmediately": true, // Immediately append newly mined card words into the known-word cache. Values: true | false
|
"addMinedWordsImmediately": true, // Immediately append newly mined card words into the known-word cache. Values: true | false
|
||||||
"matchMode": "headword", // Known-word matching strategy for subtitle annotations. Cache matches always receive known-word highlighting even when POS filters suppress other annotation types. Values: headword | surface
|
"matchMode": "headword", // Known-word matching strategy for subtitle annotations. Cache matches always receive known-word highlighting even when POS filters suppress other annotation types. Values: headword | surface
|
||||||
"decks": {} // Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }.
|
"decks": {} // Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. Reading fields (Reading, Word Reading, ExpressionReading) are always probed so cached words match only in the reading their note teaches; words from notes without readings match in any reading.
|
||||||
}, // Known words setting.
|
}, // Known words setting.
|
||||||
"behavior": {
|
"behavior": {
|
||||||
"overwriteAudio": true, // When updating an existing card, overwrite the audio field instead of skipping it. Values: true | false
|
"overwriteAudio": true, // When updating an existing card, overwrite the audio field instead of skipping it. Values: true | false
|
||||||
|
|||||||
@@ -703,8 +703,8 @@ export class AnkiIntegration {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
isKnownWord(text: string): boolean {
|
isKnownWord(text: string, reading?: string): boolean {
|
||||||
return this.knownWordCache.isKnownWord(text);
|
return this.knownWordCache.isKnownWord(text, reading);
|
||||||
}
|
}
|
||||||
|
|
||||||
getKnownWordMatchMode(): NPlusOneMatchMode {
|
getKnownWordMatchMode(): NPlusOneMatchMode {
|
||||||
|
|||||||
@@ -108,6 +108,55 @@ test('KnownWordCacheManager startLifecycle keeps fresh persisted cache without i
|
|||||||
assert.equal(manager.isKnownWord('猫'), true);
|
assert.equal(manager.isKnownWord('猫'), true);
|
||||||
assert.equal(calls.findNotes, 0);
|
assert.equal(calls.findNotes, 0);
|
||||||
assert.equal(calls.notesInfo, 0);
|
assert.equal(calls.notesInfo, 0);
|
||||||
|
// v2 states carry no readings, so they load usable but stale to trigger a
|
||||||
|
// prompt upgrade refresh.
|
||||||
|
assert.equal(
|
||||||
|
(
|
||||||
|
manager as unknown as {
|
||||||
|
getMsUntilNextRefresh: () => number;
|
||||||
|
}
|
||||||
|
).getMsUntilNextRefresh(),
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
Date.now = originalDateNow;
|
||||||
|
manager.stopLifecycle();
|
||||||
|
cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('KnownWordCacheManager startLifecycle keeps fresh v3 persisted cache without immediate refresh', async () => {
|
||||||
|
const config: AnkiConnectConfig = {
|
||||||
|
knownWords: {
|
||||||
|
highlightEnabled: true,
|
||||||
|
refreshMinutes: 60,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const { manager, calls, statePath, cleanup } = createKnownWordCacheHarness(config);
|
||||||
|
const originalDateNow = Date.now;
|
||||||
|
|
||||||
|
try {
|
||||||
|
Date.now = () => 120_000;
|
||||||
|
fs.writeFileSync(
|
||||||
|
statePath,
|
||||||
|
JSON.stringify({
|
||||||
|
version: 3,
|
||||||
|
refreshedAtMs: 120_000,
|
||||||
|
scope: '{"refreshMinutes":60,"scope":"all","fieldsWord":""}',
|
||||||
|
notes: {
|
||||||
|
'1': [{ word: '猫', reading: 'ねこ' }],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
'utf-8',
|
||||||
|
);
|
||||||
|
|
||||||
|
manager.startLifecycle();
|
||||||
|
|
||||||
|
assert.equal(manager.isKnownWord('猫'), true);
|
||||||
|
assert.equal(manager.isKnownWord('猫', 'ねこ'), true);
|
||||||
|
assert.equal(manager.isKnownWord('猫', 'びょう'), false);
|
||||||
|
assert.equal(calls.findNotes, 0);
|
||||||
|
assert.equal(calls.notesInfo, 0);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
(
|
(
|
||||||
manager as unknown as {
|
manager as unknown as {
|
||||||
@@ -263,13 +312,11 @@ test('KnownWordCacheManager refresh incrementally reconciles deleted and edited
|
|||||||
|
|
||||||
const persisted = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as {
|
const persisted = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as {
|
||||||
version: number;
|
version: number;
|
||||||
words: string[];
|
notes?: Record<string, Array<{ word: string; reading: string | null }>>;
|
||||||
notes?: Record<string, string[]>;
|
|
||||||
};
|
};
|
||||||
assert.equal(persisted.version, 2);
|
assert.equal(persisted.version, 3);
|
||||||
assert.deepEqual(persisted.words.sort(), ['鳥']);
|
|
||||||
assert.deepEqual(persisted.notes, {
|
assert.deepEqual(persisted.notes, {
|
||||||
'1': ['鳥'],
|
'1': [{ word: '鳥', reading: null }],
|
||||||
});
|
});
|
||||||
} finally {
|
} finally {
|
||||||
cleanup();
|
cleanup();
|
||||||
@@ -392,10 +439,10 @@ test('KnownWordCacheManager preserves cache state key captured before refresh wo
|
|||||||
|
|
||||||
const persisted = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as {
|
const persisted = JSON.parse(fs.readFileSync(statePath, 'utf-8')) as {
|
||||||
scope: string;
|
scope: string;
|
||||||
words: string[];
|
notes: Record<string, Array<{ word: string; reading: string | null }>>;
|
||||||
};
|
};
|
||||||
assert.equal(persisted.scope, '{"refreshMinutes":1,"scope":"all","fieldsWord":"Word"}');
|
assert.equal(persisted.scope, '{"refreshMinutes":1,"scope":"all","fieldsWord":"Word"}');
|
||||||
assert.deepEqual(persisted.words, ['猫']);
|
assert.deepEqual(persisted.notes, { '1': [{ word: '猫', reading: null }] });
|
||||||
} finally {
|
} finally {
|
||||||
fs.rmSync(stateDir, { recursive: true, force: true });
|
fs.rmSync(stateDir, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
@@ -648,3 +695,200 @@ test('KnownWordCacheManager skips immediate append when addMinedWordsImmediately
|
|||||||
cleanup();
|
cleanup();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('KnownWordCacheManager disambiguates known words by note reading', async () => {
|
||||||
|
const config: AnkiConnectConfig = {
|
||||||
|
fields: {
|
||||||
|
word: 'Word',
|
||||||
|
},
|
||||||
|
knownWords: {
|
||||||
|
highlightEnabled: true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
|
||||||
|
|
||||||
|
try {
|
||||||
|
clientState.findNotesResult = [1];
|
||||||
|
clientState.notesInfoResult = [
|
||||||
|
{
|
||||||
|
noteId: 1,
|
||||||
|
fields: {
|
||||||
|
Word: { value: '床' },
|
||||||
|
'Word Reading': { value: 'ゆか' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
await manager.refresh(true);
|
||||||
|
|
||||||
|
assert.equal(manager.isKnownWord('床'), true);
|
||||||
|
assert.equal(manager.isKnownWord('床', 'ゆか'), true);
|
||||||
|
assert.equal(manager.isKnownWord('床', 'ユカ'), true);
|
||||||
|
// Same spelling, different word (床/とこ "bed") must not match.
|
||||||
|
assert.equal(manager.isKnownWord('床', 'とこ'), false);
|
||||||
|
// Note readings stay matchable as kana words.
|
||||||
|
assert.equal(manager.isKnownWord('ゆか'), true);
|
||||||
|
assert.equal(manager.isKnownWord('とこ'), false);
|
||||||
|
} finally {
|
||||||
|
cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('KnownWordCacheManager probes reading fields even with per-deck word fields configured', async () => {
|
||||||
|
const config: AnkiConnectConfig = {
|
||||||
|
fields: {
|
||||||
|
word: 'Expression',
|
||||||
|
},
|
||||||
|
knownWords: {
|
||||||
|
highlightEnabled: true,
|
||||||
|
decks: {
|
||||||
|
'Kaishi 1.5k': ['Word'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
|
||||||
|
|
||||||
|
try {
|
||||||
|
clientState.findNotesByQuery.set('deck:"Kaishi 1.5k"', [1]);
|
||||||
|
clientState.notesInfoResult = [
|
||||||
|
{
|
||||||
|
noteId: 1,
|
||||||
|
fields: {
|
||||||
|
Word: { value: '床' },
|
||||||
|
'Word Reading': { value: 'ゆか' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
await manager.refresh(true);
|
||||||
|
|
||||||
|
assert.equal(manager.isKnownWord('床', 'ゆか'), true);
|
||||||
|
assert.equal(manager.isKnownWord('床', 'とこ'), false);
|
||||||
|
} finally {
|
||||||
|
cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('KnownWordCacheManager matches words without readings in any reading', async () => {
|
||||||
|
const config: AnkiConnectConfig = {
|
||||||
|
fields: {
|
||||||
|
word: 'Word',
|
||||||
|
},
|
||||||
|
knownWords: {
|
||||||
|
highlightEnabled: true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
|
||||||
|
|
||||||
|
try {
|
||||||
|
clientState.findNotesResult = [1];
|
||||||
|
clientState.notesInfoResult = [
|
||||||
|
{
|
||||||
|
noteId: 1,
|
||||||
|
fields: {
|
||||||
|
Word: { value: '床' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
await manager.refresh(true);
|
||||||
|
|
||||||
|
assert.equal(manager.isKnownWord('床'), true);
|
||||||
|
assert.equal(manager.isKnownWord('床', 'とこ'), true);
|
||||||
|
} finally {
|
||||||
|
cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('KnownWordCacheManager extracts word and reading from furigana word fields', async () => {
|
||||||
|
const config: AnkiConnectConfig = {
|
||||||
|
fields: {
|
||||||
|
word: 'Word',
|
||||||
|
},
|
||||||
|
knownWords: {
|
||||||
|
highlightEnabled: true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
|
||||||
|
|
||||||
|
try {
|
||||||
|
clientState.findNotesResult = [1];
|
||||||
|
clientState.notesInfoResult = [
|
||||||
|
{
|
||||||
|
noteId: 1,
|
||||||
|
fields: {
|
||||||
|
Word: { value: 'お 決[き]まり' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
await manager.refresh(true);
|
||||||
|
|
||||||
|
assert.equal(manager.isKnownWord('お決まり'), true);
|
||||||
|
assert.equal(manager.isKnownWord('お決まり', 'おきまり'), true);
|
||||||
|
assert.equal(manager.isKnownWord('お決まり', 'おさだまり'), false);
|
||||||
|
} finally {
|
||||||
|
cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('KnownWordCacheManager treats non-kana reading fields as words', async () => {
|
||||||
|
const config: AnkiConnectConfig = {
|
||||||
|
fields: {
|
||||||
|
word: 'Word',
|
||||||
|
},
|
||||||
|
knownWords: {
|
||||||
|
highlightEnabled: true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
|
||||||
|
|
||||||
|
try {
|
||||||
|
clientState.findNotesResult = [1];
|
||||||
|
clientState.notesInfoResult = [
|
||||||
|
{
|
||||||
|
noteId: 1,
|
||||||
|
fields: {
|
||||||
|
Reading: { value: '漢字' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
await manager.refresh(true);
|
||||||
|
|
||||||
|
assert.equal(manager.isKnownWord('漢字'), true);
|
||||||
|
} finally {
|
||||||
|
cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('KnownWordCacheManager keeps kana-only reading notes matchable', async () => {
|
||||||
|
const config: AnkiConnectConfig = {
|
||||||
|
fields: {
|
||||||
|
word: 'Word',
|
||||||
|
},
|
||||||
|
knownWords: {
|
||||||
|
highlightEnabled: true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const { manager, clientState, cleanup } = createKnownWordCacheHarness(config);
|
||||||
|
|
||||||
|
try {
|
||||||
|
clientState.findNotesResult = [1];
|
||||||
|
clientState.notesInfoResult = [
|
||||||
|
{
|
||||||
|
noteId: 1,
|
||||||
|
fields: {
|
||||||
|
Reading: { value: 'たべる' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
await manager.refresh(true);
|
||||||
|
|
||||||
|
assert.equal(manager.isKnownWord('たべる'), true);
|
||||||
|
assert.equal(manager.isKnownWord('タベル'), true);
|
||||||
|
} finally {
|
||||||
|
cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|||||||
@@ -5,6 +5,16 @@ import { DEFAULT_ANKI_CONNECT_CONFIG } from '../config';
|
|||||||
import { getConfiguredWordFieldName } from '../anki-field-config';
|
import { getConfiguredWordFieldName } from '../anki-field-config';
|
||||||
import { AnkiConnectConfig } from '../types/anki';
|
import { AnkiConnectConfig } from '../types/anki';
|
||||||
import { createLogger } from '../logger';
|
import { createLogger } from '../logger';
|
||||||
|
import {
|
||||||
|
DEFAULT_KNOWN_WORD_READING_FIELDS,
|
||||||
|
KnownWordEntry,
|
||||||
|
convertKatakanaToHiragana,
|
||||||
|
isReadingFieldName,
|
||||||
|
knownWordEntryListsEqual,
|
||||||
|
normalizeKnownReadingForLookup,
|
||||||
|
normalizeKnownWordEntryList,
|
||||||
|
parseFuriganaAnnotatedText,
|
||||||
|
} from './known-word-entries';
|
||||||
|
|
||||||
const log = createLogger('anki').child('integration.known-word-cache');
|
const log = createLogger('anki').child('integration.known-word-cache');
|
||||||
|
|
||||||
@@ -79,7 +89,16 @@ interface KnownWordCacheStateV2 {
|
|||||||
readonly notes: Record<string, string[]>;
|
readonly notes: Record<string, string[]>;
|
||||||
}
|
}
|
||||||
|
|
||||||
type KnownWordCacheState = KnownWordCacheStateV1 | KnownWordCacheStateV2;
|
interface KnownWordCacheStateV3 {
|
||||||
|
readonly version: 3;
|
||||||
|
readonly refreshedAtMs: number;
|
||||||
|
readonly scope: string;
|
||||||
|
readonly notes: Record<string, KnownWordEntry[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
type KnownWordCacheState = KnownWordCacheStateV1 | KnownWordCacheStateV2 | KnownWordCacheStateV3;
|
||||||
|
|
||||||
|
const NO_READING_KEY = '';
|
||||||
|
|
||||||
interface KnownWordCacheClient {
|
interface KnownWordCacheClient {
|
||||||
findNotes: (
|
findNotes: (
|
||||||
@@ -106,9 +125,12 @@ type KnownWordQueryScope = {
|
|||||||
export class KnownWordCacheManager {
|
export class KnownWordCacheManager {
|
||||||
private knownWordsLastRefreshedAtMs = 0;
|
private knownWordsLastRefreshedAtMs = 0;
|
||||||
private knownWordsStateKey = '';
|
private knownWordsStateKey = '';
|
||||||
private knownWords: Set<string> = new Set();
|
// word → (hiragana reading | NO_READING_KEY → note count). NO_READING_KEY
|
||||||
private wordReferenceCounts = new Map<string, number>();
|
// entries fail open: the word matches regardless of the token's reading.
|
||||||
private noteWordsById = new Map<number, string[]>();
|
private wordReadingCounts = new Map<string, Map<string, number>>();
|
||||||
|
// hiragana reading → note count, so kana tokens still match by reading alone.
|
||||||
|
private readingCounts = new Map<string, number>();
|
||||||
|
private noteEntriesById = new Map<number, KnownWordEntry[]>();
|
||||||
private knownWordsRefreshTimer: ReturnType<typeof setInterval> | null = null;
|
private knownWordsRefreshTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
private knownWordsRefreshTimeout: ReturnType<typeof setTimeout> | null = null;
|
private knownWordsRefreshTimeout: ReturnType<typeof setTimeout> | null = null;
|
||||||
private isRefreshingKnownWords = false;
|
private isRefreshingKnownWords = false;
|
||||||
@@ -120,13 +142,28 @@ export class KnownWordCacheManager {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
isKnownWord(text: string): boolean {
|
isKnownWord(text: string, reading?: string): boolean {
|
||||||
if (!this.isKnownWordCacheEnabled()) {
|
if (!this.isKnownWordCacheEnabled()) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
const normalized = this.normalizeKnownWordForLookup(text);
|
const normalized = this.normalizeKnownWordForLookup(text);
|
||||||
return normalized.length > 0 ? this.knownWords.has(normalized) : false;
|
if (normalized.length === 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const knownReadings = this.wordReadingCounts.get(normalized);
|
||||||
|
if (knownReadings && knownReadings.size > 0) {
|
||||||
|
const normalizedReading =
|
||||||
|
typeof reading === 'string' ? normalizeKnownReadingForLookup(reading) : '';
|
||||||
|
return (
|
||||||
|
normalizedReading.length === 0 ||
|
||||||
|
knownReadings.has(NO_READING_KEY) ||
|
||||||
|
knownReadings.has(normalizedReading)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.readingCounts.has(convertKatakanaToHiragana(normalized));
|
||||||
}
|
}
|
||||||
|
|
||||||
refresh(force = false): Promise<void> {
|
refresh(force = false): Promise<void> {
|
||||||
@@ -173,7 +210,7 @@ export class KnownWordCacheManager {
|
|||||||
let didMutateCache = false;
|
let didMutateCache = false;
|
||||||
const currentStateKey = this.getKnownWordCacheStateKey();
|
const currentStateKey = this.getKnownWordCacheStateKey();
|
||||||
if (this.knownWordsStateKey && this.knownWordsStateKey !== currentStateKey) {
|
if (this.knownWordsStateKey && this.knownWordsStateKey !== currentStateKey) {
|
||||||
didMutateCache = this.knownWords.size > 0 || this.noteWordsById.size > 0;
|
didMutateCache = this.wordReadingCounts.size > 0 || this.noteEntriesById.size > 0;
|
||||||
this.clearKnownWordCacheState();
|
this.clearKnownWordCacheState();
|
||||||
}
|
}
|
||||||
if (!this.knownWordsStateKey) {
|
if (!this.knownWordsStateKey) {
|
||||||
@@ -185,8 +222,8 @@ export class KnownWordCacheManager {
|
|||||||
return didMutateCache;
|
return didMutateCache;
|
||||||
}
|
}
|
||||||
|
|
||||||
const nextWords = this.extractNormalizedKnownWordsFromNoteInfo(noteInfo, preferredFields);
|
const nextEntries = this.extractKnownWordEntriesFromNoteInfo(noteInfo, preferredFields);
|
||||||
const changed = this.replaceNoteSnapshot(noteInfo.noteId, nextWords);
|
const changed = this.replaceNoteSnapshot(noteInfo.noteId, nextEntries);
|
||||||
if (!changed) {
|
if (!changed) {
|
||||||
return didMutateCache;
|
return didMutateCache;
|
||||||
}
|
}
|
||||||
@@ -198,7 +235,7 @@ export class KnownWordCacheManager {
|
|||||||
log.info(
|
log.info(
|
||||||
'Known-word cache updated in-session',
|
'Known-word cache updated in-session',
|
||||||
`noteId=${noteInfo.noteId}`,
|
`noteId=${noteInfo.noteId}`,
|
||||||
`wordCount=${nextWords.length}`,
|
`wordCount=${nextEntries.length}`,
|
||||||
`scope=${getKnownWordCacheScopeForConfig(this.deps.getConfig())}`,
|
`scope=${getKnownWordCacheScopeForConfig(this.deps.getConfig())}`,
|
||||||
);
|
);
|
||||||
return true;
|
return true;
|
||||||
@@ -236,11 +273,11 @@ export class KnownWordCacheManager {
|
|||||||
const noteFieldsById = await this.fetchKnownWordNoteFieldsById();
|
const noteFieldsById = await this.fetchKnownWordNoteFieldsById();
|
||||||
const currentNoteIds = Array.from(noteFieldsById.keys()).sort((a, b) => a - b);
|
const currentNoteIds = Array.from(noteFieldsById.keys()).sort((a, b) => a - b);
|
||||||
|
|
||||||
if (this.noteWordsById.size === 0) {
|
if (this.noteEntriesById.size === 0) {
|
||||||
await this.rebuildFromCurrentNotes(currentNoteIds, noteFieldsById);
|
await this.rebuildFromCurrentNotes(currentNoteIds, noteFieldsById);
|
||||||
} else {
|
} else {
|
||||||
const currentNoteIdSet = new Set(currentNoteIds);
|
const currentNoteIdSet = new Set(currentNoteIds);
|
||||||
for (const noteId of Array.from(this.noteWordsById.keys())) {
|
for (const noteId of Array.from(this.noteEntriesById.keys())) {
|
||||||
if (!currentNoteIdSet.has(noteId)) {
|
if (!currentNoteIdSet.has(noteId)) {
|
||||||
this.removeNoteSnapshot(noteId);
|
this.removeNoteSnapshot(noteId);
|
||||||
}
|
}
|
||||||
@@ -251,7 +288,7 @@ export class KnownWordCacheManager {
|
|||||||
for (const noteInfo of noteInfos) {
|
for (const noteInfo of noteInfos) {
|
||||||
this.replaceNoteSnapshot(
|
this.replaceNoteSnapshot(
|
||||||
noteInfo.noteId,
|
noteInfo.noteId,
|
||||||
this.extractNormalizedKnownWordsFromNoteInfo(
|
this.extractKnownWordEntriesFromNoteInfo(
|
||||||
noteInfo,
|
noteInfo,
|
||||||
noteFieldsById.get(noteInfo.noteId),
|
noteFieldsById.get(noteInfo.noteId),
|
||||||
),
|
),
|
||||||
@@ -266,7 +303,7 @@ export class KnownWordCacheManager {
|
|||||||
log.info(
|
log.info(
|
||||||
'Known-word cache refreshed',
|
'Known-word cache refreshed',
|
||||||
`noteCount=${currentNoteIds.length}`,
|
`noteCount=${currentNoteIds.length}`,
|
||||||
`wordCount=${this.knownWords.size}`,
|
`wordCount=${this.wordReadingCounts.size}`,
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
log.warn('Failed to refresh known-word cache:', (error as Error).message);
|
log.warn('Failed to refresh known-word cache:', (error as Error).message);
|
||||||
@@ -291,7 +328,13 @@ export class KnownWordCacheManager {
|
|||||||
|
|
||||||
private getDefaultKnownWordFields(): string[] {
|
private getDefaultKnownWordFields(): string[] {
|
||||||
const configuredWordField = getConfiguredWordFieldName(this.deps.getConfig());
|
const configuredWordField = getConfiguredWordFieldName(this.deps.getConfig());
|
||||||
return [...new Set([configuredWordField, 'Word', 'Reading', 'Word Reading'])];
|
return this.withDefaultReadingFields([configuredWordField, 'Word']);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reading fields are always probed (even when a deck configures explicit
|
||||||
|
// word fields) so entries can carry the reading their note teaches.
|
||||||
|
private withDefaultReadingFields(fields: string[]): string[] {
|
||||||
|
return [...new Set([...fields, ...DEFAULT_KNOWN_WORD_READING_FIELDS])];
|
||||||
}
|
}
|
||||||
|
|
||||||
private getKnownWordDecks(): string[] {
|
private getKnownWordDecks(): string[] {
|
||||||
@@ -337,7 +380,9 @@ export class KnownWordCacheManager {
|
|||||||
.filter((field) => field.length > 0),
|
.filter((field) => field.length > 0),
|
||||||
),
|
),
|
||||||
];
|
];
|
||||||
return normalizedFields.length > 0 ? normalizedFields : this.getDefaultKnownWordFields();
|
return normalizedFields.length > 0
|
||||||
|
? this.withDefaultReadingFields(normalizedFields)
|
||||||
|
: this.getDefaultKnownWordFields();
|
||||||
}
|
}
|
||||||
|
|
||||||
const deckFields = selectedDeckEntry[1];
|
const deckFields = selectedDeckEntry[1];
|
||||||
@@ -351,7 +396,7 @@ export class KnownWordCacheManager {
|
|||||||
),
|
),
|
||||||
];
|
];
|
||||||
if (normalizedFields.length > 0) {
|
if (normalizedFields.length > 0) {
|
||||||
return normalizedFields;
|
return this.withDefaultReadingFields(normalizedFields);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -382,7 +427,10 @@ export class KnownWordCacheManager {
|
|||||||
: [];
|
: [];
|
||||||
scopes.push({
|
scopes.push({
|
||||||
query: `deck:"${escapeAnkiSearchValue(trimmedDeckName)}"`,
|
query: `deck:"${escapeAnkiSearchValue(trimmedDeckName)}"`,
|
||||||
fields: normalizedFields.length > 0 ? normalizedFields : this.getDefaultKnownWordFields(),
|
fields:
|
||||||
|
normalizedFields.length > 0
|
||||||
|
? this.withDefaultReadingFields(normalizedFields)
|
||||||
|
: this.getDefaultKnownWordFields(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (scopes.length > 0) {
|
if (scopes.length > 0) {
|
||||||
@@ -490,7 +538,7 @@ export class KnownWordCacheManager {
|
|||||||
for (const noteInfo of noteInfos) {
|
for (const noteInfo of noteInfos) {
|
||||||
this.replaceNoteSnapshot(
|
this.replaceNoteSnapshot(
|
||||||
noteInfo.noteId,
|
noteInfo.noteId,
|
||||||
this.extractNormalizedKnownWordsFromNoteInfo(noteInfo, noteFieldsById.get(noteInfo.noteId)),
|
this.extractKnownWordEntriesFromNoteInfo(noteInfo, noteFieldsById.get(noteInfo.noteId)),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -519,56 +567,77 @@ export class KnownWordCacheManager {
|
|||||||
return noteInfos;
|
return noteInfos;
|
||||||
}
|
}
|
||||||
|
|
||||||
private replaceNoteSnapshot(noteId: number, nextWords: string[]): boolean {
|
private replaceNoteSnapshot(noteId: number, nextEntries: KnownWordEntry[]): boolean {
|
||||||
const normalizedWords = normalizeKnownWordList(nextWords);
|
const normalizedEntries = normalizeKnownWordEntryList(nextEntries);
|
||||||
const previousWords = this.noteWordsById.get(noteId) ?? [];
|
const previousEntries = this.noteEntriesById.get(noteId) ?? [];
|
||||||
if (knownWordListsEqual(previousWords, normalizedWords)) {
|
if (knownWordEntryListsEqual(previousEntries, normalizedEntries)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.removeWordsFromCounts(previousWords);
|
this.removeEntriesFromCounts(previousEntries);
|
||||||
if (normalizedWords.length > 0) {
|
if (normalizedEntries.length > 0) {
|
||||||
this.noteWordsById.set(noteId, normalizedWords);
|
this.noteEntriesById.set(noteId, normalizedEntries);
|
||||||
this.addWordsToCounts(normalizedWords);
|
this.addEntriesToCounts(normalizedEntries);
|
||||||
} else {
|
} else {
|
||||||
this.noteWordsById.delete(noteId);
|
this.noteEntriesById.delete(noteId);
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private removeNoteSnapshot(noteId: number): void {
|
private removeNoteSnapshot(noteId: number): void {
|
||||||
const previousWords = this.noteWordsById.get(noteId);
|
const previousEntries = this.noteEntriesById.get(noteId);
|
||||||
if (!previousWords) {
|
if (!previousEntries) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.noteWordsById.delete(noteId);
|
this.noteEntriesById.delete(noteId);
|
||||||
this.removeWordsFromCounts(previousWords);
|
this.removeEntriesFromCounts(previousEntries);
|
||||||
}
|
}
|
||||||
|
|
||||||
private addWordsToCounts(words: string[]): void {
|
private addEntriesToCounts(entries: KnownWordEntry[]): void {
|
||||||
for (const word of words) {
|
for (const entry of entries) {
|
||||||
const nextCount = (this.wordReferenceCounts.get(word) ?? 0) + 1;
|
const readingKey = entry.reading ?? NO_READING_KEY;
|
||||||
this.wordReferenceCounts.set(word, nextCount);
|
let readings = this.wordReadingCounts.get(entry.word);
|
||||||
this.knownWords.add(word);
|
if (!readings) {
|
||||||
|
readings = new Map();
|
||||||
|
this.wordReadingCounts.set(entry.word, readings);
|
||||||
|
}
|
||||||
|
readings.set(readingKey, (readings.get(readingKey) ?? 0) + 1);
|
||||||
|
if (entry.reading) {
|
||||||
|
this.readingCounts.set(entry.reading, (this.readingCounts.get(entry.reading) ?? 0) + 1);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private removeWordsFromCounts(words: string[]): void {
|
private removeEntriesFromCounts(entries: KnownWordEntry[]): void {
|
||||||
for (const word of words) {
|
for (const entry of entries) {
|
||||||
const nextCount = (this.wordReferenceCounts.get(word) ?? 0) - 1;
|
const readingKey = entry.reading ?? NO_READING_KEY;
|
||||||
if (nextCount > 0) {
|
const readings = this.wordReadingCounts.get(entry.word);
|
||||||
this.wordReferenceCounts.set(word, nextCount);
|
if (readings) {
|
||||||
} else {
|
const nextCount = (readings.get(readingKey) ?? 0) - 1;
|
||||||
this.wordReferenceCounts.delete(word);
|
if (nextCount > 0) {
|
||||||
this.knownWords.delete(word);
|
readings.set(readingKey, nextCount);
|
||||||
|
} else {
|
||||||
|
readings.delete(readingKey);
|
||||||
|
if (readings.size === 0) {
|
||||||
|
this.wordReadingCounts.delete(entry.word);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (entry.reading) {
|
||||||
|
const nextReadingCount = (this.readingCounts.get(entry.reading) ?? 0) - 1;
|
||||||
|
if (nextReadingCount > 0) {
|
||||||
|
this.readingCounts.set(entry.reading, nextReadingCount);
|
||||||
|
} else {
|
||||||
|
this.readingCounts.delete(entry.reading);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private clearInMemoryState(): void {
|
private clearInMemoryState(): void {
|
||||||
this.knownWords = new Set();
|
this.wordReadingCounts = new Map();
|
||||||
this.wordReferenceCounts = new Map();
|
this.readingCounts = new Map();
|
||||||
this.noteWordsById = new Map();
|
this.noteEntriesById = new Map();
|
||||||
this.knownWordsLastRefreshedAtMs = 0;
|
this.knownWordsLastRefreshedAtMs = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -601,32 +670,48 @@ export class KnownWordCacheManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
this.clearInMemoryState();
|
this.clearInMemoryState();
|
||||||
|
if (parsed.version === 3) {
|
||||||
|
for (const [noteIdKey, entries] of Object.entries(parsed.notes)) {
|
||||||
|
const noteId = Number.parseInt(noteIdKey, 10);
|
||||||
|
if (!Number.isInteger(noteId) || noteId <= 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const normalizedEntries = normalizeKnownWordEntryList(entries);
|
||||||
|
if (normalizedEntries.length === 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
this.noteEntriesById.set(noteId, normalizedEntries);
|
||||||
|
this.addEntriesToCounts(normalizedEntries);
|
||||||
|
}
|
||||||
|
this.knownWordsLastRefreshedAtMs = parsed.refreshedAtMs;
|
||||||
|
this.knownWordsStateKey = parsed.scope;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (parsed.version === 2) {
|
if (parsed.version === 2) {
|
||||||
|
// Older states have no readings; load them reading-less (fail-open,
|
||||||
|
// matching the old behavior) but leave the cache marked stale so the
|
||||||
|
// next refresh upgrades entries with readings from Anki.
|
||||||
for (const [noteIdKey, words] of Object.entries(parsed.notes)) {
|
for (const [noteIdKey, words] of Object.entries(parsed.notes)) {
|
||||||
const noteId = Number.parseInt(noteIdKey, 10);
|
const noteId = Number.parseInt(noteIdKey, 10);
|
||||||
if (!Number.isInteger(noteId) || noteId <= 0) {
|
if (!Number.isInteger(noteId) || noteId <= 0) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const normalizedWords = normalizeKnownWordList(words);
|
const normalizedEntries = normalizeKnownWordEntryList(
|
||||||
if (normalizedWords.length === 0) {
|
words.map((word) => ({ word: this.normalizeKnownWordForLookup(word), reading: null })),
|
||||||
|
);
|
||||||
|
if (normalizedEntries.length === 0) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
this.noteWordsById.set(noteId, normalizedWords);
|
this.noteEntriesById.set(noteId, normalizedEntries);
|
||||||
this.addWordsToCounts(normalizedWords);
|
this.addEntriesToCounts(normalizedEntries);
|
||||||
}
|
|
||||||
} else {
|
|
||||||
for (const value of parsed.words) {
|
|
||||||
const normalized = this.normalizeKnownWordForLookup(value);
|
|
||||||
if (!normalized) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
this.knownWords.add(normalized);
|
|
||||||
this.wordReferenceCounts.set(normalized, 1);
|
|
||||||
}
|
}
|
||||||
|
this.knownWordsStateKey = parsed.scope;
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.knownWordsLastRefreshedAtMs = parsed.refreshedAtMs;
|
// v1 has no per-note snapshots to convert; refetch from Anki.
|
||||||
this.knownWordsStateKey = parsed.scope;
|
this.knownWordsStateKey = this.getKnownWordCacheStateKey();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
log.warn('Failed to load known-word cache state:', (error as Error).message);
|
log.warn('Failed to load known-word cache state:', (error as Error).message);
|
||||||
this.clearInMemoryState();
|
this.clearInMemoryState();
|
||||||
@@ -636,18 +721,17 @@ export class KnownWordCacheManager {
|
|||||||
|
|
||||||
private persistKnownWordCacheState(): void {
|
private persistKnownWordCacheState(): void {
|
||||||
try {
|
try {
|
||||||
const notes: Record<string, string[]> = {};
|
const notes: Record<string, KnownWordEntry[]> = {};
|
||||||
for (const [noteId, words] of this.noteWordsById.entries()) {
|
for (const [noteId, entries] of this.noteEntriesById.entries()) {
|
||||||
if (words.length > 0) {
|
if (entries.length > 0) {
|
||||||
notes[String(noteId)] = words;
|
notes[String(noteId)] = entries;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const state: KnownWordCacheStateV2 = {
|
const state: KnownWordCacheStateV3 = {
|
||||||
version: 2,
|
version: 3,
|
||||||
refreshedAtMs: this.knownWordsLastRefreshedAtMs,
|
refreshedAtMs: this.knownWordsLastRefreshedAtMs,
|
||||||
scope: this.knownWordsStateKey,
|
scope: this.knownWordsStateKey,
|
||||||
words: Array.from(this.knownWords),
|
|
||||||
notes,
|
notes,
|
||||||
};
|
};
|
||||||
fs.writeFileSync(this.statePath, JSON.stringify(state), 'utf-8');
|
fs.writeFileSync(this.statePath, JSON.stringify(state), 'utf-8');
|
||||||
@@ -659,14 +743,18 @@ export class KnownWordCacheManager {
|
|||||||
private isKnownWordCacheStateValid(value: unknown): value is KnownWordCacheState {
|
private isKnownWordCacheStateValid(value: unknown): value is KnownWordCacheState {
|
||||||
if (typeof value !== 'object' || value === null) return false;
|
if (typeof value !== 'object' || value === null) return false;
|
||||||
const candidate = value as Record<string, unknown>;
|
const candidate = value as Record<string, unknown>;
|
||||||
if (candidate.version !== 1 && candidate.version !== 2) return false;
|
if (candidate.version !== 1 && candidate.version !== 2 && candidate.version !== 3) {
|
||||||
if (typeof candidate.refreshedAtMs !== 'number') return false;
|
|
||||||
if (typeof candidate.scope !== 'string') return false;
|
|
||||||
if (!Array.isArray(candidate.words)) return false;
|
|
||||||
if (!candidate.words.every((entry: unknown) => typeof entry === 'string')) {
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (candidate.version === 2) {
|
if (typeof candidate.refreshedAtMs !== 'number') return false;
|
||||||
|
if (typeof candidate.scope !== 'string') return false;
|
||||||
|
if (candidate.version !== 3) {
|
||||||
|
if (!Array.isArray(candidate.words)) return false;
|
||||||
|
if (!candidate.words.every((entry: unknown) => typeof entry === 'string')) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (candidate.version === 2 || candidate.version === 3) {
|
||||||
if (
|
if (
|
||||||
typeof candidate.notes !== 'object' ||
|
typeof candidate.notes !== 'object' ||
|
||||||
candidate.notes === null ||
|
candidate.notes === null ||
|
||||||
@@ -674,10 +762,18 @@ export class KnownWordCacheManager {
|
|||||||
) {
|
) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
const isValidNoteEntry =
|
||||||
|
candidate.version === 2
|
||||||
|
? (entry: unknown): boolean => typeof entry === 'string'
|
||||||
|
: (entry: unknown): boolean =>
|
||||||
|
typeof entry === 'object' &&
|
||||||
|
entry !== null &&
|
||||||
|
typeof (entry as KnownWordEntry).word === 'string' &&
|
||||||
|
((entry as KnownWordEntry).reading === null ||
|
||||||
|
typeof (entry as KnownWordEntry).reading === 'string');
|
||||||
if (
|
if (
|
||||||
!Object.values(candidate.notes as Record<string, unknown>).every(
|
!Object.values(candidate.notes as Record<string, unknown>).every(
|
||||||
(entry) =>
|
(noteEntries) => Array.isArray(noteEntries) && noteEntries.every(isValidNoteEntry),
|
||||||
Array.isArray(entry) && entry.every((word: unknown) => typeof word === 'string'),
|
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
return false;
|
return false;
|
||||||
@@ -686,11 +782,12 @@ export class KnownWordCacheManager {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private extractNormalizedKnownWordsFromNoteInfo(
|
private extractKnownWordEntriesFromNoteInfo(
|
||||||
noteInfo: KnownWordCacheNoteInfo,
|
noteInfo: KnownWordCacheNoteInfo,
|
||||||
preferredFields = this.getConfiguredFields(),
|
preferredFields = this.getConfiguredFields(),
|
||||||
): string[] {
|
): KnownWordEntry[] {
|
||||||
const words: string[] = [];
|
const wordValues: string[] = [];
|
||||||
|
let noteReading: string | null = null;
|
||||||
for (const preferredField of preferredFields) {
|
for (const preferredField of preferredFields) {
|
||||||
const fieldName = resolveFieldName(Object.keys(noteInfo.fields), preferredField);
|
const fieldName = resolveFieldName(Object.keys(noteInfo.fields), preferredField);
|
||||||
if (!fieldName) continue;
|
if (!fieldName) continue;
|
||||||
@@ -698,12 +795,36 @@ export class KnownWordCacheManager {
|
|||||||
const raw = noteInfo.fields[fieldName]?.value;
|
const raw = noteInfo.fields[fieldName]?.value;
|
||||||
if (!raw) continue;
|
if (!raw) continue;
|
||||||
|
|
||||||
const normalized = this.normalizeKnownWordForLookup(raw);
|
const cleaned = this.normalizeRawKnownWordValue(raw);
|
||||||
if (normalized) {
|
if (!cleaned) continue;
|
||||||
words.push(normalized);
|
|
||||||
|
if (isReadingFieldName(preferredField)) {
|
||||||
|
const normalizedReading = normalizeKnownReadingForLookup(cleaned);
|
||||||
|
if (normalizedReading) {
|
||||||
|
noteReading ??= normalizedReading;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// Non-kana content in a reading field: treat it as a word so decks
|
||||||
|
// with repurposed reading fields keep matching (fail-open).
|
||||||
}
|
}
|
||||||
|
wordValues.push(cleaned);
|
||||||
}
|
}
|
||||||
return normalizeKnownWordList(words);
|
|
||||||
|
const entries: KnownWordEntry[] = [];
|
||||||
|
for (const value of wordValues) {
|
||||||
|
const parsed = parseFuriganaAnnotatedText(value);
|
||||||
|
const word = parsed.text.trim().toLowerCase();
|
||||||
|
if (!word) continue;
|
||||||
|
const inlineReading = parsed.reading ? normalizeKnownReadingForLookup(parsed.reading) : '';
|
||||||
|
entries.push({ word, reading: inlineReading || noteReading });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kana-only notes (reading field but no word field) stay matchable.
|
||||||
|
if (entries.length === 0 && noteReading) {
|
||||||
|
entries.push({ word: noteReading, reading: noteReading });
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalizeKnownWordEntryList(entries);
|
||||||
}
|
}
|
||||||
|
|
||||||
private normalizeRawKnownWordValue(value: string): string {
|
private normalizeRawKnownWordValue(value: string): string {
|
||||||
@@ -718,22 +839,6 @@ export class KnownWordCacheManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeKnownWordList(words: string[]): string[] {
|
|
||||||
return [...new Set(words.map((word) => word.trim()).filter((word) => word.length > 0))].sort();
|
|
||||||
}
|
|
||||||
|
|
||||||
function knownWordListsEqual(left: string[], right: string[]): boolean {
|
|
||||||
if (left.length !== right.length) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
for (let index = 0; index < left.length; index += 1) {
|
|
||||||
if (left[index] !== right[index]) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveFieldName(availableFieldNames: string[], preferredName: string): string | null {
|
function resolveFieldName(availableFieldNames: string[], preferredName: string): string | null {
|
||||||
const exact = availableFieldNames.find((name) => name === preferredName);
|
const exact = availableFieldNames.find((name) => name === preferredName);
|
||||||
if (exact) return exact;
|
if (exact) return exact;
|
||||||
|
|||||||
@@ -0,0 +1,113 @@
|
|||||||
|
// Known-word cache entries pair a word with the reading its Anki note teaches,
|
||||||
|
// so spelling collisions across readings (e.g. 床/ゆか vs 床/とこ) don't mark
|
||||||
|
// unrelated words as known. reading === null means the note carries no usable
|
||||||
|
// reading and the word matches in any reading (fail-open).
|
||||||
|
export interface KnownWordEntry {
|
||||||
|
word: string;
|
||||||
|
reading: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
|
||||||
|
const KATAKANA_CODEPOINT_START = 0x30a1;
|
||||||
|
const KATAKANA_CODEPOINT_END = 0x30f6;
|
||||||
|
const FURIGANA_SEGMENT_PATTERN = /([^\s \[\]]*)\[([^\]]*)\]/g;
|
||||||
|
const FURIGANA_BRACKET_PATTERN = /\[[^\]]*\]/g;
|
||||||
|
const WHITESPACE_PATTERN = /[\s ]+/g;
|
||||||
|
|
||||||
|
// Reading-bearing field names probed on every known-word note, in addition to
|
||||||
|
// any configured word fields (covers Kaishi's "Word Reading" and Lapis's
|
||||||
|
// "ExpressionReading" note types).
|
||||||
|
export const DEFAULT_KNOWN_WORD_READING_FIELDS = [
|
||||||
|
'Reading',
|
||||||
|
'Word Reading',
|
||||||
|
'ExpressionReading',
|
||||||
|
'Expression Reading',
|
||||||
|
];
|
||||||
|
|
||||||
|
export function isReadingFieldName(fieldName: string): boolean {
|
||||||
|
return /reading/i.test(fieldName);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function convertKatakanaToHiragana(text: string): string {
|
||||||
|
let converted = '';
|
||||||
|
for (const char of text) {
|
||||||
|
const code = char.codePointAt(0);
|
||||||
|
if (code !== undefined && code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END) {
|
||||||
|
converted += String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
converted += char;
|
||||||
|
}
|
||||||
|
return converted;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isHiraganaReadingChar(char: string): boolean {
|
||||||
|
const code = char.codePointAt(0);
|
||||||
|
if (code === undefined) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return (code >= 0x3041 && code <= 0x309f) || code === 0x30fc;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Splits Anki furigana syntax (`床[とこ]`, `お 決[き]まり`) into base text and
|
||||||
|
// reading. Values without brackets pass through with reading null.
|
||||||
|
export function parseFuriganaAnnotatedText(value: string): {
|
||||||
|
text: string;
|
||||||
|
reading: string | null;
|
||||||
|
} {
|
||||||
|
if (!value.includes('[')) {
|
||||||
|
return { text: value, reading: null };
|
||||||
|
}
|
||||||
|
const text = value.replace(FURIGANA_BRACKET_PATTERN, '').replace(WHITESPACE_PATTERN, '');
|
||||||
|
const reading = value.replace(FURIGANA_SEGMENT_PATTERN, '$2').replace(WHITESPACE_PATTERN, '');
|
||||||
|
return { text, reading: reading.length > 0 ? reading : null };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns the hiragana-normalized reading, or '' when the value is not a
|
||||||
|
// plausible kana reading (callers fall back to text-only matching then).
|
||||||
|
export function normalizeKnownReadingForLookup(value: string): string {
|
||||||
|
const parsed = parseFuriganaAnnotatedText(value.trim());
|
||||||
|
const candidate = (parsed.reading ?? parsed.text).trim();
|
||||||
|
if (!candidate) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
const hiragana = convertKatakanaToHiragana(candidate);
|
||||||
|
for (const char of hiragana) {
|
||||||
|
if (!isHiraganaReadingChar(char)) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return hiragana;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function makeKnownWordEntryKey(entry: KnownWordEntry): string {
|
||||||
|
return `${entry.word}\u0000${entry.reading ?? ''}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeKnownWordEntryList(entries: KnownWordEntry[]): KnownWordEntry[] {
|
||||||
|
const byKey = new Map<string, KnownWordEntry>();
|
||||||
|
for (const entry of entries) {
|
||||||
|
const word = entry.word.trim();
|
||||||
|
if (!word) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const reading = entry.reading?.trim() || null;
|
||||||
|
const normalized: KnownWordEntry = { word, reading };
|
||||||
|
byKey.set(makeKnownWordEntryKey(normalized), normalized);
|
||||||
|
}
|
||||||
|
return [...byKey.values()].sort((left, right) =>
|
||||||
|
makeKnownWordEntryKey(left).localeCompare(makeKnownWordEntryKey(right)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function knownWordEntryListsEqual(left: KnownWordEntry[], right: KnownWordEntry[]): boolean {
|
||||||
|
if (left.length !== right.length) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (let index = 0; index < left.length; index += 1) {
|
||||||
|
if (makeKnownWordEntryKey(left[index]!) !== makeKnownWordEntryKey(right[index]!)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
@@ -317,7 +317,7 @@ export function buildIntegrationConfigOptionRegistry(
|
|||||||
kind: 'object',
|
kind: 'object',
|
||||||
defaultValue: defaultConfig.ankiConnect.knownWords.decks,
|
defaultValue: defaultConfig.ankiConnect.knownWords.decks,
|
||||||
description:
|
description:
|
||||||
'Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }.',
|
'Decks and expression/word fields for known-word cache. Object mapping deck names to arrays of field names to extract, e.g. { "Kaishi 1.5k": ["Word"] }. Reading fields (Reading, Word Reading, ExpressionReading) are always probed so cached words match only in the reading their note teaches; words from notes without readings match in any reading.',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
path: 'ankiConnect.isKiku.fieldGrouping',
|
path: 'ankiConnect.isKiku.fieldGrouping',
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ export interface TokenizerServiceDeps {
|
|||||||
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
|
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
|
||||||
getYomitanParserInitPromise: () => Promise<boolean> | null;
|
getYomitanParserInitPromise: () => Promise<boolean> | null;
|
||||||
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
|
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
|
||||||
isKnownWord: (text: string) => boolean;
|
isKnownWord: (text: string, reading?: string) => boolean;
|
||||||
getKnownWordMatchMode: () => NPlusOneMatchMode;
|
getKnownWordMatchMode: () => NPlusOneMatchMode;
|
||||||
getKnownWordsEnabled?: () => boolean;
|
getKnownWordsEnabled?: () => boolean;
|
||||||
getJlptLevel: (text: string) => JlptLevel | null;
|
getJlptLevel: (text: string) => JlptLevel | null;
|
||||||
@@ -77,7 +77,7 @@ export interface TokenizerDepsRuntimeOptions {
|
|||||||
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
|
setYomitanParserReadyPromise: (promise: Promise<void> | null) => void;
|
||||||
getYomitanParserInitPromise: () => Promise<boolean> | null;
|
getYomitanParserInitPromise: () => Promise<boolean> | null;
|
||||||
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
|
setYomitanParserInitPromise: (promise: Promise<boolean> | null) => void;
|
||||||
isKnownWord: (text: string) => boolean;
|
isKnownWord: (text: string, reading?: string) => boolean;
|
||||||
getKnownWordMatchMode: () => NPlusOneMatchMode;
|
getKnownWordMatchMode: () => NPlusOneMatchMode;
|
||||||
getKnownWordsEnabled?: () => boolean;
|
getKnownWordsEnabled?: () => boolean;
|
||||||
getJlptLevel: (text: string) => JlptLevel | null;
|
getJlptLevel: (text: string) => JlptLevel | null;
|
||||||
@@ -129,7 +129,7 @@ const INVISIBLE_SEPARATOR_PATTERN = /[\u200b\u2060\ufeff]/g;
|
|||||||
function getKnownWordLookup(
|
function getKnownWordLookup(
|
||||||
deps: TokenizerServiceDeps,
|
deps: TokenizerServiceDeps,
|
||||||
options: TokenizerAnnotationOptions,
|
options: TokenizerAnnotationOptions,
|
||||||
): (text: string) => boolean {
|
): (text: string, reading?: string) => boolean {
|
||||||
if (!options.knownWordsEnabled && !options.nPlusOneEnabled) {
|
if (!options.knownWordsEnabled && !options.nPlusOneEnabled) {
|
||||||
return () => false;
|
return () => false;
|
||||||
}
|
}
|
||||||
@@ -723,6 +723,7 @@ async function parseWithYomitanInternalParser(
|
|||||||
surface: token.surface,
|
surface: token.surface,
|
||||||
reading: token.reading,
|
reading: token.reading,
|
||||||
headword: token.headword,
|
headword: token.headword,
|
||||||
|
headwordReading: token.headwordReading,
|
||||||
startPos: token.startPos,
|
startPos: token.startPos,
|
||||||
endPos: token.endPos,
|
endPos: token.endPos,
|
||||||
partOfSpeech: posMetadata.partOfSpeech,
|
partOfSpeech: posMetadata.partOfSpeech,
|
||||||
|
|||||||
@@ -56,6 +56,70 @@ test('annotateTokens known-word match mode uses headword vs surface', () => {
|
|||||||
assert.equal(surfaceResult[0]?.isKnown, false);
|
assert.equal(surfaceResult[0]?.isKnown, false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('annotateTokens passes dictionary-form reading so spelling collisions stay unknown', () => {
|
||||||
|
// とこ (colloquial ところ) resolves to headword 床/とこ; a known 床/ゆか card
|
||||||
|
// must not mark it known (#138 regression).
|
||||||
|
const cache = new Map([['床', 'ゆか']]);
|
||||||
|
const isKnownWord = (text: string, reading?: string): boolean => {
|
||||||
|
if (!cache.has(text)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return reading === undefined || cache.get(text) === reading;
|
||||||
|
};
|
||||||
|
const tokens = [
|
||||||
|
makeToken({
|
||||||
|
surface: 'とこ',
|
||||||
|
headword: '床',
|
||||||
|
reading: 'とこ',
|
||||||
|
headwordReading: 'とこ',
|
||||||
|
endPos: 2,
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = annotateTokens(tokens, makeDeps({ isKnownWord }));
|
||||||
|
|
||||||
|
assert.equal(result[0]?.isKnown, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('annotateTokens keeps inflected known words matched via headword reading', () => {
|
||||||
|
const isKnownWord = (text: string, reading?: string): boolean =>
|
||||||
|
text === '行く' && (reading === undefined || reading === 'いく');
|
||||||
|
const tokens = [
|
||||||
|
makeToken({
|
||||||
|
surface: '行きたい',
|
||||||
|
headword: '行く',
|
||||||
|
reading: 'いきたい',
|
||||||
|
headwordReading: 'いく',
|
||||||
|
partOfSpeech: PartOfSpeech.verb,
|
||||||
|
endPos: 4,
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = annotateTokens(tokens, makeDeps({ isKnownWord }));
|
||||||
|
|
||||||
|
assert.equal(result[0]?.isKnown, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('annotateTokens omits reading for headword match when token lacks headword reading and is inflected', () => {
|
||||||
|
// MeCab tokens have no dictionary-form reading; the surface reading of an
|
||||||
|
// inflected form must not be compared against the note's dictionary reading.
|
||||||
|
const isKnownWord = (text: string, reading?: string): boolean =>
|
||||||
|
text === '食べる' && reading === undefined;
|
||||||
|
const tokens = [
|
||||||
|
makeToken({
|
||||||
|
surface: '食べた',
|
||||||
|
headword: '食べる',
|
||||||
|
reading: 'タベタ',
|
||||||
|
partOfSpeech: PartOfSpeech.verb,
|
||||||
|
endPos: 3,
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
const result = annotateTokens(tokens, makeDeps({ isKnownWord }));
|
||||||
|
|
||||||
|
assert.equal(result[0]?.isKnown, true);
|
||||||
|
});
|
||||||
|
|
||||||
test('annotateTokens marks known words when N+1 is disabled', () => {
|
test('annotateTokens marks known words when N+1 is disabled', () => {
|
||||||
const tokens = [
|
const tokens = [
|
||||||
makeToken({ surface: '私', headword: '私', startPos: 0, endPos: 1 }),
|
makeToken({ surface: '私', headword: '私', startPos: 0, endPos: 1 }),
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ const jlptLevelLookupCaches = new WeakMap<
|
|||||||
>();
|
>();
|
||||||
|
|
||||||
export interface AnnotationStageDeps {
|
export interface AnnotationStageDeps {
|
||||||
isKnownWord: (text: string) => boolean;
|
isKnownWord: (text: string, reading?: string) => boolean;
|
||||||
knownWordMatchMode: NPlusOneMatchMode;
|
knownWordMatchMode: NPlusOneMatchMode;
|
||||||
getJlptLevel: (text: string) => JlptLevel | null;
|
getJlptLevel: (text: string) => JlptLevel | null;
|
||||||
}
|
}
|
||||||
@@ -661,26 +661,57 @@ function isCompleteReadingForSurface(surface: string, reading: string): boolean
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Returns the token's trimmed reading only when it plausibly covers the surface
|
||||||
|
// (see isCompleteReadingForSurface); undefined otherwise. Shared so the
|
||||||
|
// known-word reading disambiguation and the reading fallback stay in sync if the
|
||||||
|
// validity rule changes.
|
||||||
|
function resolveCompleteTokenReading(token: MergedToken): string | undefined {
|
||||||
|
const normalizedReading = token.reading.trim();
|
||||||
|
if (!normalizedReading || !isCompleteReadingForSurface(token.surface, normalizedReading)) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
return normalizedReading;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reading to disambiguate the known-word text match, or undefined when the
|
||||||
|
// token has no reading that describes the match text: in headword mode an
|
||||||
|
// inflected surface's reading does not match the dictionary form's reading,
|
||||||
|
// and partial furigana readings (see isCompleteReadingForSurface) would cause
|
||||||
|
// false negatives. Undefined falls back to text-only matching (fail-open).
|
||||||
|
function resolveKnownWordReadingForMatch(
|
||||||
|
token: MergedToken,
|
||||||
|
knownWordMatchMode: NPlusOneMatchMode,
|
||||||
|
): string | undefined {
|
||||||
|
if (knownWordMatchMode === 'headword') {
|
||||||
|
const headwordReading = token.headwordReading?.trim();
|
||||||
|
if (headwordReading) {
|
||||||
|
return headwordReading;
|
||||||
|
}
|
||||||
|
if (token.surface !== token.headword) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return resolveCompleteTokenReading(token);
|
||||||
|
}
|
||||||
|
|
||||||
function computeTokenKnownStatus(
|
function computeTokenKnownStatus(
|
||||||
token: MergedToken,
|
token: MergedToken,
|
||||||
isKnownWord: (text: string) => boolean,
|
isKnownWord: (text: string, reading?: string) => boolean,
|
||||||
knownWordMatchMode: NPlusOneMatchMode,
|
knownWordMatchMode: NPlusOneMatchMode,
|
||||||
): boolean {
|
): boolean {
|
||||||
const matchText = resolveKnownWordText(token.surface, token.headword, knownWordMatchMode);
|
const matchText = resolveKnownWordText(token.surface, token.headword, knownWordMatchMode);
|
||||||
if (token.isKnown || (matchText ? isKnownWord(matchText) : false)) {
|
const matchReading = resolveKnownWordReadingForMatch(token, knownWordMatchMode);
|
||||||
|
if (token.isKnown || (matchText ? isKnownWord(matchText, matchReading) : false)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
const normalizedReading = token.reading.trim();
|
const fallbackReading = resolveCompleteTokenReading(token);
|
||||||
if (!normalizedReading) {
|
if (!fallbackReading) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isCompleteReadingForSurface(token.surface, normalizedReading)) {
|
return fallbackReading !== matchText.trim() && isKnownWord(fallbackReading);
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return normalizedReading !== matchText.trim() && isKnownWord(normalizedReading);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function filterTokenFrequencyRank(
|
function filterTokenFrequencyRank(
|
||||||
|
|||||||
@@ -966,6 +966,7 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF
|
|||||||
surface: '潜み',
|
surface: '潜み',
|
||||||
reading: 'ひそみ',
|
reading: 'ひそみ',
|
||||||
headword: '潜む',
|
headword: '潜む',
|
||||||
|
headwordReading: 'ひそむ',
|
||||||
startPos: 0,
|
startPos: 0,
|
||||||
endPos: 2,
|
endPos: 2,
|
||||||
isNameMatch: false,
|
isNameMatch: false,
|
||||||
@@ -1032,6 +1033,7 @@ test('requestYomitanScanTokens emits complete readings for kanji-kana compounds'
|
|||||||
surface: '待ち合わせてる',
|
surface: '待ち合わせてる',
|
||||||
reading: 'まちあわせてる',
|
reading: 'まちあわせてる',
|
||||||
headword: '待ち合わせる',
|
headword: '待ち合わせる',
|
||||||
|
headwordReading: 'まちあわせる',
|
||||||
startPos: 0,
|
startPos: 0,
|
||||||
endPos: 7,
|
endPos: 7,
|
||||||
isNameMatch: false,
|
isNameMatch: false,
|
||||||
@@ -1139,6 +1141,7 @@ test('requestYomitanScanTokens uses frequency from later exact-match entry when
|
|||||||
surface: '者',
|
surface: '者',
|
||||||
reading: 'もの',
|
reading: 'もの',
|
||||||
headword: '者',
|
headword: '者',
|
||||||
|
headwordReading: 'もの',
|
||||||
startPos: 0,
|
startPos: 0,
|
||||||
endPos: 1,
|
endPos: 1,
|
||||||
isNameMatch: false,
|
isNameMatch: false,
|
||||||
@@ -1240,6 +1243,7 @@ test('requestYomitanScanTokens can use frequency from later exact secondary-matc
|
|||||||
surface: '者',
|
surface: '者',
|
||||||
reading: 'もの',
|
reading: 'もの',
|
||||||
headword: '者',
|
headword: '者',
|
||||||
|
headwordReading: 'もの',
|
||||||
startPos: 0,
|
startPos: 0,
|
||||||
endPos: 1,
|
endPos: 1,
|
||||||
isNameMatch: false,
|
isNameMatch: false,
|
||||||
@@ -1340,6 +1344,7 @@ test('requestYomitanScanTokens uses exact frequency entry when selected reading
|
|||||||
surface: '第二',
|
surface: '第二',
|
||||||
reading: 'だいに',
|
reading: 'だいに',
|
||||||
headword: '第二',
|
headword: '第二',
|
||||||
|
headwordReading: 'だいに',
|
||||||
startPos: 0,
|
startPos: 0,
|
||||||
endPos: 2,
|
endPos: 2,
|
||||||
isNameMatch: false,
|
isNameMatch: false,
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ export interface YomitanScanToken {
|
|||||||
surface: string;
|
surface: string;
|
||||||
reading: string;
|
reading: string;
|
||||||
headword: string;
|
headword: string;
|
||||||
|
headwordReading?: string;
|
||||||
startPos: number;
|
startPos: number;
|
||||||
endPos: number;
|
endPos: number;
|
||||||
isNameMatch?: boolean;
|
isNameMatch?: boolean;
|
||||||
@@ -92,6 +93,7 @@ function isScanTokenArray(value: unknown): value is YomitanScanToken[] {
|
|||||||
typeof entry.surface === 'string' &&
|
typeof entry.surface === 'string' &&
|
||||||
typeof entry.reading === 'string' &&
|
typeof entry.reading === 'string' &&
|
||||||
typeof entry.headword === 'string' &&
|
typeof entry.headword === 'string' &&
|
||||||
|
(entry.headwordReading === undefined || typeof entry.headwordReading === 'string') &&
|
||||||
typeof entry.startPos === 'number' &&
|
typeof entry.startPos === 'number' &&
|
||||||
typeof entry.endPos === 'number' &&
|
typeof entry.endPos === 'number' &&
|
||||||
(entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean') &&
|
(entry.isNameMatch === undefined || typeof entry.isNameMatch === 'boolean') &&
|
||||||
@@ -1318,6 +1320,7 @@ ${YOMITAN_SCANNING_HELPERS}
|
|||||||
surface: segments.map((segment) => segment.text).join("") || source,
|
surface: segments.map((segment) => segment.text).join("") || source,
|
||||||
reading: segments.map(getSegmentReadingContribution).join(""),
|
reading: segments.map(getSegmentReadingContribution).join(""),
|
||||||
headword: preferredHeadword.term,
|
headword: preferredHeadword.term,
|
||||||
|
headwordReading: reading || undefined,
|
||||||
startPos: i,
|
startPos: i,
|
||||||
endPos: i + originalTextLength,
|
endPos: i + originalTextLength,
|
||||||
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
|
isNameMatch: includeNameMatchMetadata && preferredHeadword.isNameMatch === true,
|
||||||
|
|||||||
+1
-1
@@ -4638,7 +4638,7 @@ const {
|
|||||||
setYomitanParserInitPromise: (promise) => {
|
setYomitanParserInitPromise: (promise) => {
|
||||||
appState.yomitanParserInitPromise = promise;
|
appState.yomitanParserInitPromise = promise;
|
||||||
},
|
},
|
||||||
isKnownWord: (text) => Boolean(appState.ankiIntegration?.isKnownWord(text)),
|
isKnownWord: (text, reading) => Boolean(appState.ankiIntegration?.isKnownWord(text, reading)),
|
||||||
recordLookup: (hit) => {
|
recordLookup: (hit) => {
|
||||||
ensureImmersionTrackerStarted();
|
ensureImmersionTrackerStarted();
|
||||||
appState.immersionTracker?.recordLookup(hit);
|
appState.immersionTracker?.recordLookup(hit);
|
||||||
|
|||||||
@@ -37,8 +37,8 @@ export function createBuildTokenizerDepsMainHandler(deps: TokenizerMainDeps) {
|
|||||||
getYomitanParserInitPromise: () => deps.getYomitanParserInitPromise(),
|
getYomitanParserInitPromise: () => deps.getYomitanParserInitPromise(),
|
||||||
setYomitanParserInitPromise: (promise: Promise<boolean> | null) =>
|
setYomitanParserInitPromise: (promise: Promise<boolean> | null) =>
|
||||||
deps.setYomitanParserInitPromise(promise),
|
deps.setYomitanParserInitPromise(promise),
|
||||||
isKnownWord: (text: string) => {
|
isKnownWord: (text: string, reading?: string) => {
|
||||||
const hit = deps.isKnownWord(text);
|
const hit = deps.isKnownWord(text, reading);
|
||||||
deps.recordLookup(hit);
|
deps.recordLookup(hit);
|
||||||
return hit;
|
return hit;
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -29,6 +29,8 @@ export interface MergedToken {
|
|||||||
surface: string;
|
surface: string;
|
||||||
reading: string;
|
reading: string;
|
||||||
headword: string;
|
headword: string;
|
||||||
|
/** Dictionary-form reading of headword (kana), when the parser provides it. */
|
||||||
|
headwordReading?: string;
|
||||||
startPos: number;
|
startPos: number;
|
||||||
endPos: number;
|
endPos: number;
|
||||||
partOfSpeech: PartOfSpeech;
|
partOfSpeech: PartOfSpeech;
|
||||||
|
|||||||
Reference in New Issue
Block a user