fix(overlay): remove content adverbs from annotation stop-word list

- 確かに and やはり no longer excluded from frequency/JLPT highlighting and vocab stats
- Stop-word list now covers only interjections, pronouns, and grammar fragments
This commit is contained in:
2026-07-08 20:15:45 -07:00
parent 925413adfe
commit 8b21a2bca8
3 changed files with 32 additions and 11 deletions
@@ -728,15 +728,6 @@ test('shouldExcludeTokenFromVocabularyPersistence excludes common frequency stop
pos2: '代名詞|副助詞/並立助詞/終助詞',
pos3: '一般|*',
}),
makeToken({
surface: '確かに',
headword: '確かに',
reading: 'たしかに',
partOfSpeech: PartOfSpeech.other,
pos1: '名詞|助詞',
pos2: '形容動詞語幹|副詞化',
pos3: '*',
}),
makeToken({
surface: 'あなた',
headword: '貴方',
@@ -753,6 +744,34 @@ test('shouldExcludeTokenFromVocabularyPersistence excludes common frequency stop
}
});
test('content adverbs are not excluded from annotations or vocabulary persistence', () => {
const tokens = [
makeToken({
surface: '確かに',
headword: '確かに',
reading: 'たしかに',
partOfSpeech: PartOfSpeech.other,
pos1: '名詞|助詞',
pos2: '形容動詞語幹|副詞化',
pos3: '*',
}),
makeToken({
surface: 'やはり',
headword: 'やはり',
reading: 'ヤハリ',
partOfSpeech: PartOfSpeech.other,
pos1: '副詞',
pos2: '一般',
pos3: '*',
}),
];
for (const token of tokens) {
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false, token.surface);
assert.equal(shouldExcludeTokenFromVocabularyPersistence(token), false, token.surface);
}
});
test('shouldExcludeTokenFromSubtitleAnnotations excludes standalone して grammar helper fragments', () => {
const token = makeToken({
surface: 'して',
@@ -47,7 +47,6 @@ export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
'へえ',
'ふう',
'ほう',
'やはり',
'何か',
'何だ',
'何も',
@@ -55,7 +54,6 @@ export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
'有る',
'在る',
'様',
'確かに',
'誰も',
'貴方',
'もんか',