mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-27 04:49:49 -07:00
fix(overlay): remove content adverbs from annotation stop-word list
- 確かに and やはり no longer excluded from frequency/JLPT highlighting and vocab stats - Stop-word list now covers only interjections, pronouns, and grammar fragments
This commit is contained in:
@@ -728,15 +728,6 @@ test('shouldExcludeTokenFromVocabularyPersistence excludes common frequency stop
|
||||
pos2: '代名詞|副助詞/並立助詞/終助詞',
|
||||
pos3: '一般|*',
|
||||
}),
|
||||
makeToken({
|
||||
surface: '確かに',
|
||||
headword: '確かに',
|
||||
reading: 'たしかに',
|
||||
partOfSpeech: PartOfSpeech.other,
|
||||
pos1: '名詞|助詞',
|
||||
pos2: '形容動詞語幹|副詞化',
|
||||
pos3: '*',
|
||||
}),
|
||||
makeToken({
|
||||
surface: 'あなた',
|
||||
headword: '貴方',
|
||||
@@ -753,6 +744,34 @@ test('shouldExcludeTokenFromVocabularyPersistence excludes common frequency stop
|
||||
}
|
||||
});
|
||||
|
||||
test('content adverbs are not excluded from annotations or vocabulary persistence', () => {
|
||||
const tokens = [
|
||||
makeToken({
|
||||
surface: '確かに',
|
||||
headword: '確かに',
|
||||
reading: 'たしかに',
|
||||
partOfSpeech: PartOfSpeech.other,
|
||||
pos1: '名詞|助詞',
|
||||
pos2: '形容動詞語幹|副詞化',
|
||||
pos3: '*',
|
||||
}),
|
||||
makeToken({
|
||||
surface: 'やはり',
|
||||
headword: 'やはり',
|
||||
reading: 'ヤハリ',
|
||||
partOfSpeech: PartOfSpeech.other,
|
||||
pos1: '副詞',
|
||||
pos2: '一般',
|
||||
pos3: '*',
|
||||
}),
|
||||
];
|
||||
|
||||
for (const token of tokens) {
|
||||
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false, token.surface);
|
||||
assert.equal(shouldExcludeTokenFromVocabularyPersistence(token), false, token.surface);
|
||||
}
|
||||
});
|
||||
|
||||
test('shouldExcludeTokenFromSubtitleAnnotations excludes standalone して grammar helper fragments', () => {
|
||||
const token = makeToken({
|
||||
surface: 'して',
|
||||
|
||||
@@ -47,7 +47,6 @@ export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
|
||||
'へえ',
|
||||
'ふう',
|
||||
'ほう',
|
||||
'やはり',
|
||||
'何か',
|
||||
'何だ',
|
||||
'何も',
|
||||
@@ -55,7 +54,6 @@ export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
|
||||
'有る',
|
||||
'在る',
|
||||
'様',
|
||||
'確かに',
|
||||
'誰も',
|
||||
'貴方',
|
||||
'もんか',
|
||||
|
||||
Reference in New Issue
Block a user