diff --git a/changes/content-adverb-annotation-stop-list.md b/changes/content-adverb-annotation-stop-list.md new file mode 100644 index 00000000..72b63fca --- /dev/null +++ b/changes/content-adverb-annotation-stop-list.md @@ -0,0 +1,4 @@ +type: fixed +area: overlay + +- Removed 確かに and やはり from the annotation stop-word list so they get frequency/JLPT highlighting again and count toward vocabulary stats; the list now only covers interjections, pronouns, and grammar fragments. diff --git a/src/core/services/tokenizer/annotation-stage.test.ts b/src/core/services/tokenizer/annotation-stage.test.ts index bf46d2eb..ccd58bd3 100644 --- a/src/core/services/tokenizer/annotation-stage.test.ts +++ b/src/core/services/tokenizer/annotation-stage.test.ts @@ -728,15 +728,6 @@ test('shouldExcludeTokenFromVocabularyPersistence excludes common frequency stop pos2: '代名詞|副助詞/並立助詞/終助詞', pos3: '一般|*', }), - makeToken({ - surface: '確かに', - headword: '確かに', - reading: 'たしかに', - partOfSpeech: PartOfSpeech.other, - pos1: '名詞|助詞', - pos2: '形容動詞語幹|副詞化', - pos3: '*', - }), makeToken({ surface: 'あなた', headword: '貴方', @@ -753,6 +744,34 @@ test('shouldExcludeTokenFromVocabularyPersistence excludes common frequency stop } }); +test('content adverbs are not excluded from annotations or vocabulary persistence', () => { + const tokens = [ + makeToken({ + surface: '確かに', + headword: '確かに', + reading: 'たしかに', + partOfSpeech: PartOfSpeech.other, + pos1: '名詞|助詞', + pos2: '形容動詞語幹|副詞化', + pos3: '*', + }), + makeToken({ + surface: 'やはり', + headword: 'やはり', + reading: 'ヤハリ', + partOfSpeech: PartOfSpeech.other, + pos1: '副詞', + pos2: '一般', + pos3: '*', + }), + ]; + + for (const token of tokens) { + assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false, token.surface); + assert.equal(shouldExcludeTokenFromVocabularyPersistence(token), false, token.surface); + } +}); + test('shouldExcludeTokenFromSubtitleAnnotations excludes standalone して grammar helper fragments', () => { const token = makeToken({ surface: 'して', diff --git a/src/core/services/tokenizer/subtitle-annotation-filter.ts b/src/core/services/tokenizer/subtitle-annotation-filter.ts index 3e8d3b08..153ab17e 100644 --- a/src/core/services/tokenizer/subtitle-annotation-filter.ts +++ b/src/core/services/tokenizer/subtitle-annotation-filter.ts @@ -47,7 +47,6 @@ export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([ 'へえ', 'ふう', 'ほう', - 'やはり', '何か', '何だ', '何も', @@ -55,7 +54,6 @@ export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([ '有る', '在る', '様', - '確かに', '誰も', '貴方', 'もんか',