fix(tokenizer): stop partial furigana readings from marking words known (#137)

This commit is contained in:
2026-07-05 17:02:44 -07:00
committed by GitHub
parent eef4500599
commit b14f977e33
5 changed files with 150 additions and 2 deletions
@@ -964,7 +964,7 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF
assert.deepEqual(result, [
{
surface: '潜み',
reading: 'ひそ',
reading: 'ひそ',
headword: '潜む',
startPos: 0,
endPos: 2,
@@ -974,6 +974,72 @@ test('requestYomitanScanTokens extracts best frequency rank from selected termsF
]);
});
test('requestYomitanScanTokens emits complete readings for kanji-kana compounds', async () => {
let scannerScript = '';
const deps = createDeps(async (script) => {
if (script.includes('termsFind')) {
scannerScript = script;
return [];
}
if (script.includes('optionsGetFull')) {
return {
profileCurrent: 0,
profiles: [
{
options: {
scanning: { length: 40 },
dictionaries: [{ name: 'JPDBv2㋕', enabled: true, id: 0 }],
},
},
],
};
}
return null;
});
await requestYomitanScanTokens('待ち合わせてる', deps, {
error: () => undefined,
});
const result = await runInjectedYomitanScript(scannerScript, (action, params) => {
if (action !== 'termsFind') {
throw new Error(`unexpected action: ${action}`);
}
const text = (params as { text?: string } | undefined)?.text ?? '';
if (!text.startsWith('待ち合わせてる')) {
return { originalTextLength: 0, dictionaryEntries: [] };
}
return {
originalTextLength: 7,
dictionaryEntries: [
{
headwords: [
{
term: '待ち合わせる',
reading: 'まちあわせる',
sources: [{ originalText: '待ち合わせてる', isPrimary: true, matchType: 'exact' }],
},
],
},
],
};
});
assert.deepEqual(result, [
{
surface: '待ち合わせてる',
reading: 'まちあわせてる',
headword: '待ち合わせる',
startPos: 0,
endPos: 7,
isNameMatch: false,
frequencyRank: undefined,
},
]);
});
test('requestYomitanScanTokens uses frequency from later exact-match entry when first exact entry has none', async () => {
let scannerScript = '';
const deps = createDeps(async (script) => {