mirror of
https://github.com/ksyasuda/SubMiner.git
synced 2026-07-27 16:49:51 -07:00
fix(overlay): remove content adverbs from annotation stop-word list
- 確かに and やはり no longer excluded from frequency/JLPT highlighting and vocab stats - Stop-word list now covers only interjections, pronouns, and grammar fragments
This commit is contained in:
@@ -0,0 +1,4 @@
|
|||||||
|
type: fixed
|
||||||
|
area: overlay
|
||||||
|
|
||||||
|
- Removed 確かに and やはり from the annotation stop-word list so they get frequency/JLPT highlighting again and count toward vocabulary stats; the list now only covers interjections, pronouns, and grammar fragments.
|
||||||
@@ -728,15 +728,6 @@ test('shouldExcludeTokenFromVocabularyPersistence excludes common frequency stop
|
|||||||
pos2: '代名詞|副助詞/並立助詞/終助詞',
|
pos2: '代名詞|副助詞/並立助詞/終助詞',
|
||||||
pos3: '一般|*',
|
pos3: '一般|*',
|
||||||
}),
|
}),
|
||||||
makeToken({
|
|
||||||
surface: '確かに',
|
|
||||||
headword: '確かに',
|
|
||||||
reading: 'たしかに',
|
|
||||||
partOfSpeech: PartOfSpeech.other,
|
|
||||||
pos1: '名詞|助詞',
|
|
||||||
pos2: '形容動詞語幹|副詞化',
|
|
||||||
pos3: '*',
|
|
||||||
}),
|
|
||||||
makeToken({
|
makeToken({
|
||||||
surface: 'あなた',
|
surface: 'あなた',
|
||||||
headword: '貴方',
|
headword: '貴方',
|
||||||
@@ -753,6 +744,34 @@ test('shouldExcludeTokenFromVocabularyPersistence excludes common frequency stop
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('content adverbs are not excluded from annotations or vocabulary persistence', () => {
|
||||||
|
const tokens = [
|
||||||
|
makeToken({
|
||||||
|
surface: '確かに',
|
||||||
|
headword: '確かに',
|
||||||
|
reading: 'たしかに',
|
||||||
|
partOfSpeech: PartOfSpeech.other,
|
||||||
|
pos1: '名詞|助詞',
|
||||||
|
pos2: '形容動詞語幹|副詞化',
|
||||||
|
pos3: '*',
|
||||||
|
}),
|
||||||
|
makeToken({
|
||||||
|
surface: 'やはり',
|
||||||
|
headword: 'やはり',
|
||||||
|
reading: 'ヤハリ',
|
||||||
|
partOfSpeech: PartOfSpeech.other,
|
||||||
|
pos1: '副詞',
|
||||||
|
pos2: '一般',
|
||||||
|
pos3: '*',
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const token of tokens) {
|
||||||
|
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), false, token.surface);
|
||||||
|
assert.equal(shouldExcludeTokenFromVocabularyPersistence(token), false, token.surface);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('shouldExcludeTokenFromSubtitleAnnotations excludes standalone して grammar helper fragments', () => {
|
test('shouldExcludeTokenFromSubtitleAnnotations excludes standalone して grammar helper fragments', () => {
|
||||||
const token = makeToken({
|
const token = makeToken({
|
||||||
surface: 'して',
|
surface: 'して',
|
||||||
|
|||||||
@@ -47,7 +47,6 @@ export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
|
|||||||
'へえ',
|
'へえ',
|
||||||
'ふう',
|
'ふう',
|
||||||
'ほう',
|
'ほう',
|
||||||
'やはり',
|
|
||||||
'何か',
|
'何か',
|
||||||
'何だ',
|
'何だ',
|
||||||
'何も',
|
'何も',
|
||||||
@@ -55,7 +54,6 @@ export const SUBTITLE_ANNOTATION_EXCLUDED_TERMS = new Set([
|
|||||||
'有る',
|
'有る',
|
||||||
'在る',
|
'在る',
|
||||||
'様',
|
'様',
|
||||||
'確かに',
|
|
||||||
'誰も',
|
'誰も',
|
||||||
'貴方',
|
'貴方',
|
||||||
'もんか',
|
'もんか',
|
||||||
|
|||||||
Reference in New Issue
Block a user