Fix tokenizer annotations for explanatory contrast ending (#33)

This commit is contained in:
2026-03-23 09:25:17 -07:00
committed by GitHub
parent 0317c7f011
commit c17f0a4080
5 changed files with 184 additions and 0 deletions
@@ -246,6 +246,18 @@ test('shouldExcludeTokenFromSubtitleAnnotations excludes explanatory pondering e
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
});
test('shouldExcludeTokenFromSubtitleAnnotations excludes explanatory contrast endings', () => {
const token = makeToken({
surface: 'んですけど',
headword: 'ん',
reading: 'ンデスケド',
pos1: '名詞|助動詞|助詞',
pos2: '非自立',
});
assert.equal(shouldExcludeTokenFromSubtitleAnnotations(token), true);
});
test('shouldExcludeTokenFromSubtitleAnnotations excludes auxiliary-stem そうだ grammar tails', () => {
const token = makeToken({
surface: 'そうだ',
@@ -46,6 +46,7 @@ const SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_TRAILING_PARTICLES = [
'ね',
'よ',
'な',
'けど',
'よね',
'かな',
'かね',
@@ -41,6 +41,7 @@ const SUBTITLE_ANNOTATION_EXCLUDED_EXPLANATORY_ENDING_TRAILING_PARTICLES = [
'ね',
'よ',
'な',
'けど',
'よね',
'かな',
'かね',