refactor(tokenizer): consolidate kana/POS helpers into token-classification (#161)

src/core/services/tokenizer.ts

@@ -25,6 +25,7 @@ import {
   requestYomitanTermFrequencies,
 } from './tokenizer/yomitan-parser-runtime';
 import type { YomitanTermFrequency } from './tokenizer/yomitan-parser-runtime';
+import { isKanaChar } from './tokenizer/token-classification';
 
 const logger = createLogger('main:tokenizer');
 
@@ -322,20 +323,6 @@ function normalizeFrequencyLookupText(rawText: string): string {
   return rawText.trim().toLowerCase();
 }
 
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
 function getTrailingKanaSuffix(surface: string): string {
   const chars = Array.from(surface);
   let splitIndex = chars.length;

src/core/services/tokenizer/annotation-stage.test.ts

@@ -1563,6 +1563,7 @@ test('annotateTokens keeps frequency for unknown non-independent kanji noun toke
   );
 
   assert.equal(result[0]?.isKnown, false);
+  assert.equal(result[0]?.isNPlusOneTarget, true);
   assert.equal(result[0]?.frequencyRank, 718);
   assert.equal(result[0]?.jlptLevel, 'N4');
 });

src/core/services/tokenizer/annotation-stage.ts

@@ -10,14 +10,19 @@ import {
 import { JlptLevel, MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types';
 import { shouldIgnoreJlptByTerm, shouldIgnoreJlptForMecabPos1 } from '../jlpt-token-filter';
 import {
-  isKanjiNonIndependentNounToken,
   shouldExcludeTokenFromSubtitleAnnotations as sharedShouldExcludeTokenFromSubtitleAnnotations,
   stripSubtitleAnnotationMetadata as sharedStripSubtitleAnnotationMetadata,
 } from './subtitle-annotation-filter';
+import {
+  isKanaChar,
+  isKanaOnlyText,
+  isPosTagExcluded,
+  isTokenPos2Excluded,
+  normalizeKana,
+  normalizePosTag,
+  splitPosTag,
+} from './token-classification';
 
-const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
-const KATAKANA_CODEPOINT_START = 0x30a1;
-const KATAKANA_CODEPOINT_END = 0x30f6;
 const JLPT_LEVEL_LOOKUP_CACHE_LIMIT = 2048;
 
 const jlptLevelLookupCaches = new WeakMap<
@@ -55,30 +60,6 @@ function resolveKnownWordText(
   return matchMode === 'surface' ? surface : headword;
 }
 
-function normalizePos1Tag(pos1: string | undefined): string {
-  return typeof pos1 === 'string' ? pos1.trim() : '';
-}
-
-function splitNormalizedTagParts(normalizedTag: string): string[] {
-  if (!normalizedTag) {
-    return [];
-  }
-
-  return normalizedTag
-    .split('|')
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
-}
-
-function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<string>): boolean {
-  const parts = splitNormalizedTagParts(normalizedTag);
-  if (parts.length === 0) {
-    return false;
-  }
-
-  return parts.every((part) => exclusions.has(part));
-}
-
 function resolvePos1Exclusions(options: AnnotationStageOptions): ReadonlySet<string> {
   if (options.pos1Exclusions) {
     return options.pos1Exclusions;
@@ -95,10 +76,6 @@ function resolvePos2Exclusions(options: AnnotationStageOptions): ReadonlySet<str
   return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
 }
 
-function normalizePos2Tag(pos2: string | undefined): string {
-  return typeof pos2 === 'string' ? pos2.trim() : '';
-}
-
 function isExcludedComponent(
   pos1: string | undefined,
   pos2: string | undefined,
@@ -117,12 +94,12 @@ function shouldAllowContentLedMergedTokenFrequency(
   pos1Exclusions: ReadonlySet<string>,
   pos2Exclusions: ReadonlySet<string>,
 ): boolean {
-  const pos1Parts = splitNormalizedTagParts(normalizedPos1);
+  const pos1Parts = splitPosTag(normalizedPos1);
   if (pos1Parts.length < 2) {
     return false;
   }
 
-  const pos2Parts = splitNormalizedTagParts(normalizedPos2);
+  const pos2Parts = splitPosTag(normalizedPos2);
   if (isExcludedComponent(pos1Parts[0], pos2Parts[0], pos1Exclusions, pos2Exclusions)) {
     return false;
   }
@@ -144,8 +121,8 @@ function shouldAllowOrdinalPrefixNounFrequency(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
-  const pos2Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos2));
+  const pos1Parts = splitPosTag(token.pos1);
+  const pos2Parts = splitPosTag(token.pos2);
   return (
     pos1Parts.length >= 2 &&
     pos1Parts[0] === '接頭詞' &&
@@ -166,8 +143,8 @@ function shouldAllowHonorificPrefixNounFrequency(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
-  const pos2Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos2));
+  const pos1Parts = splitPosTag(token.pos1);
+  const pos2Parts = splitPosTag(token.pos2);
   return (
     pos1Parts.length >= 2 &&
     pos1Parts[0] === '接頭詞' &&
@@ -182,12 +159,12 @@ function shouldAllowDeterminerLedNounFrequency(
   pos1Exclusions: ReadonlySet<string>,
   pos2Exclusions: ReadonlySet<string>,
 ): boolean {
-  const pos1Parts = splitNormalizedTagParts(normalizedPos1);
+  const pos1Parts = splitPosTag(normalizedPos1);
   if (pos1Parts.length < 2 || pos1Parts[0] !== '連体詞') {
     return false;
   }
 
-  const pos2Parts = splitNormalizedTagParts(normalizedPos2);
+  const pos2Parts = splitPosTag(normalizedPos2);
   if (!isExcludedComponent(pos1Parts[0], pos2Parts[0], pos1Exclusions, pos2Exclusions)) {
     return false;
   }
@@ -218,9 +195,9 @@ function isFrequencyExcludedByPos(
     return true;
   }
 
-  const normalizedPos1 = normalizePos1Tag(token.pos1);
+  const normalizedPos1 = normalizePosTag(token.pos1);
   const hasPos1 = normalizedPos1.length > 0;
-  const normalizedPos2 = normalizePos2Tag(token.pos2);
+  const normalizedPos2 = normalizePosTag(token.pos2);
   const hasPos2 = normalizedPos2.length > 0;
   const allowContentLedMergedToken = shouldAllowContentLedMergedTokenFrequency(
     normalizedPos1,
@@ -238,7 +215,7 @@ function isFrequencyExcludedByPos(
   const allowHonorificPrefixNounToken = shouldAllowHonorificPrefixNounFrequency(token);
 
   if (
-    isExcludedByTagSet(normalizedPos1, pos1Exclusions) &&
+    isPosTagExcluded(normalizedPos1, pos1Exclusions) &&
     !allowContentLedMergedToken &&
     !allowDeterminerLedNounToken &&
     !allowOrdinalPrefixNounToken &&
@@ -248,7 +225,7 @@ function isFrequencyExcludedByPos(
   }
 
   if (
-    isExcludedByTagSet(normalizedPos2, pos2Exclusions) &&
+    isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions) &&
     !allowContentLedMergedToken &&
     !allowDeterminerLedNounToken &&
     !allowOrdinalPrefixNounToken &&
@@ -280,8 +257,7 @@ export function shouldExcludeTokenFromVocabularyPersistence(
 
   return (
     sharedShouldExcludeTokenFromSubtitleAnnotations(token, { pos1Exclusions, pos2Exclusions }) ||
-    (isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) &&
-      !isKanjiNonIndependentNounToken(token, pos1Exclusions))
+    isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions)
   );
 }
 
@@ -332,45 +308,6 @@ function resolveJlptLookupText(token: MergedToken): string {
   return token.surface;
 }
 
-function normalizeJlptTextForExclusion(text: string): string {
-  const raw = text.trim();
-  if (!raw) {
-    return '';
-  }
-
-  let normalized = '';
-  for (const char of raw) {
-    const code = char.codePointAt(0);
-    if (code === undefined) {
-      continue;
-    }
-
-    if (code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END) {
-      normalized += String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET);
-      continue;
-    }
-
-    normalized += char;
-  }
-
-  return normalized;
-}
-
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
 function isRepeatedKanaSfx(text: string): boolean {
   const normalized = text.trim();
   if (!normalized) {
@@ -406,7 +343,7 @@ function isRepeatedKanaSfx(text: string): boolean {
 }
 
 function isTrailingSmallTsuKanaSfx(text: string): boolean {
-  const normalized = normalizeJlptTextForExclusion(text);
+  const normalized = normalizeKana(text);
   if (!normalized) {
     return false;
   }
@@ -424,7 +361,7 @@ function isTrailingSmallTsuKanaSfx(text: string): boolean {
 }
 
 function isReduplicatedKanaSfx(text: string): boolean {
-  const normalized = normalizeJlptTextForExclusion(text);
+  const normalized = normalizeKana(text);
   if (!normalized) {
     return false;
   }
@@ -443,7 +380,7 @@ function isReduplicatedKanaSfx(text: string): boolean {
 }
 
 function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
-  const normalized = normalizeJlptTextForExclusion(text);
+  const normalized = normalizeKana(text);
   if (!normalized) {
     return false;
   }
@@ -460,7 +397,7 @@ function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
 }
 
 function hasAdjacentKanaRepeat(text: string): boolean {
-  const normalized = normalizeJlptTextForExclusion(text);
+  const normalized = normalizeKana(text);
   if (!normalized) {
     return false;
   }
@@ -490,7 +427,7 @@ function isLikelyFrequencyNoiseToken(token: MergedToken): boolean {
       continue;
     }
 
-    const normalizedCandidate = normalizeJlptTextForExclusion(trimmedCandidate);
+    const normalizedCandidate = normalizeKana(trimmedCandidate);
     if (!normalizedCandidate) {
       continue;
     }
@@ -528,19 +465,6 @@ function isSingleKanaFrequencyNoiseToken(text: string | undefined): boolean {
   return chars.length === 1 && isKanaChar(chars[0]!);
 }
 
-function isKanaOnlyText(text: string | undefined): boolean {
-  if (typeof text !== 'string') {
-    return false;
-  }
-
-  const normalized = text.trim();
-  if (!normalized) {
-    return false;
-  }
-
-  return [...normalized].every(isKanaChar);
-}
-
 function isKanaOnlyMixedFunctionContentToken(
   token: MergedToken,
   pos1Exclusions: ReadonlySet<string>,
@@ -549,7 +473,7 @@ function isKanaOnlyMixedFunctionContentToken(
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   const hasMixedFunctionContentParts =
     pos1Parts.length >= 2 &&
     pos1Parts.some((part) => pos1Exclusions.has(part)) &&
@@ -558,8 +482,8 @@ function isKanaOnlyMixedFunctionContentToken(
     return false;
   }
 
-  const normalizedReading = normalizeJlptTextForExclusion(token.reading);
-  const normalizedHeadwordReading = normalizeJlptTextForExclusion(token.headwordReading ?? '');
+  const normalizedReading = normalizeKana(token.reading);
+  const normalizedHeadwordReading = normalizeKana(token.headwordReading ?? '');
   return (
     !normalizedReading ||
     !normalizedHeadwordReading ||
@@ -577,7 +501,7 @@ function isJlptEligibleToken(token: MergedToken): boolean {
   );
 
   for (const candidate of candidates) {
-    const normalizedCandidate = normalizeJlptTextForExclusion(candidate);
+    const normalizedCandidate = normalizeKana(candidate);
     if (!normalizedCandidate) {
       continue;
     }
@@ -612,8 +536,8 @@ export function stripSubtitleAnnotationMetadata(
 // at least as many characters as the surface, with the surface's kana appearing
 // in order within the reading.
 function isCompleteReadingForSurface(surface: string, reading: string): boolean {
-  const surfaceChars = [...normalizeJlptTextForExclusion(surface)];
-  const readingChars = [...normalizeJlptTextForExclusion(reading)];
+  const surfaceChars = [...normalizeKana(surface)];
+  const readingChars = [...normalizeKana(reading)];
   if (readingChars.length < surfaceChars.length) {
     return false;
   }
@@ -697,10 +621,7 @@ function filterTokenFrequencyRank(
   pos1Exclusions: ReadonlySet<string>,
   pos2Exclusions: ReadonlySet<string>,
 ): number | undefined {
-  if (
-    isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) &&
-    !isKanjiNonIndependentNounToken(token, pos1Exclusions)
-  ) {
+  if (isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions)) {
     return undefined;
   }
 

src/core/services/tokenizer/parser-selection-stage.ts

@@ -1,5 +1,6 @@
 import { MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types';
 import { isStandaloneGrammarEndingText } from './grammar-ending';
+import { isKanaChar, isKanaOnlyText } from './token-classification';
 
 interface YomitanParseHeadword {
   term?: unknown;
@@ -41,21 +42,6 @@ function resolveKnownWordText(
   return matchMode === 'surface' ? surface : headword;
 }
 
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
 function isYomitanParseLine(value: unknown): value is YomitanParseLine {
   if (!Array.isArray(value)) {
     return false;
@@ -138,10 +124,6 @@ function selectMergedHeadword(
   return firstHeadword;
 }
 
-function isKanaOnlyText(text: string): boolean {
-  return text.length > 0 && Array.from(text).every((char) => isKanaChar(char));
-}
-
 function isStandaloneGrammarEndingSegment(segment: YomitanParseSegment): boolean {
   const surface = segment.text?.trim() ?? '';
   const headword = extractYomitanHeadword(segment).trim();

src/core/services/tokenizer/part-of-speech.ts

@@ -1,8 +1,5 @@
 import { PartOfSpeech } from '../../../types';
-
-function normalizePosTag(value: string | null | undefined): string {
-  return typeof value === 'string' ? value.trim() : '';
-}
+import { normalizePosTag, splitPosTag } from './token-classification';
 
 export function isPartOfSpeechValue(value: unknown): value is PartOfSpeech {
   return typeof value === 'string' && Object.values(PartOfSpeech).includes(value as PartOfSpeech);
@@ -35,10 +32,7 @@ export function deriveStoredPartOfSpeech(input: {
   partOfSpeech?: string | null;
   pos1?: string | null;
 }): PartOfSpeech {
-  const pos1Parts = normalizePosTag(input.pos1)
-    .split('|')
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
+  const pos1Parts = splitPosTag(input.pos1);
 
   if (pos1Parts.length > 0) {
     const derivedParts = [...new Set(pos1Parts.map((part) => mapMecabPos1ToPartOfSpeech(part)))];

src/core/services/tokenizer/subtitle-annotation-filter.ts

@@ -6,13 +6,16 @@ import {
   DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG,
   resolveAnnotationPos2ExclusionSet,
 } from '../../../token-pos2-exclusions';
-import { MergedToken, PartOfSpeech } from '../../../types';
+import { MergedToken } from '../../../types';
 import { shouldIgnoreJlptByTerm } from '../jlpt-token-filter';
 import { isSubtitleGrammarEndingText } from './grammar-ending';
-
-const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
-const KATAKANA_CODEPOINT_START = 0x30a1;
-const KATAKANA_CODEPOINT_END = 0x30f6;
+import {
+  isContentTokenByPos,
+  isKanaChar,
+  isKanaOnlyText,
+  normalizeKana,
+  splitPosTag,
+} from './token-classification';
 
 const STANDALONE_GRAMMAR_PARTICLE_PHRASES = ['たって', 'だって'] as const;
 const STANDALONE_GRAMMAR_PARTICLE_PHRASES_SET: ReadonlySet<string> = new Set(
@@ -97,30 +100,6 @@ export interface SubtitleAnnotationFilterOptions {
   pos2Exclusions?: ReadonlySet<string>;
 }
 
-function normalizePosTag(pos: string | undefined): string {
-  return typeof pos === 'string' ? pos.trim() : '';
-}
-
-function splitNormalizedTagParts(normalizedTag: string): string[] {
-  if (!normalizedTag) {
-    return [];
-  }
-
-  return normalizedTag
-    .split('|')
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
-}
-
-function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<string>): boolean {
-  const parts = splitNormalizedTagParts(normalizedTag);
-  if (parts.length === 0) {
-    return false;
-  }
-
-  return parts.every((part) => exclusions.has(part));
-}
-
 function resolvePos1Exclusions(options: SubtitleAnnotationFilterOptions = {}): ReadonlySet<string> {
   if (options.pos1Exclusions) {
     return options.pos1Exclusions;
@@ -137,85 +116,6 @@ function resolvePos2Exclusions(options: SubtitleAnnotationFilterOptions = {}): R
   return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
 }
 
-function hasKanjiChar(text: string): boolean {
-  for (const char of text) {
-    const code = char.codePointAt(0);
-    if (code === undefined) {
-      continue;
-    }
-    if (
-      (code >= 0x3400 && code <= 0x4dbf) ||
-      (code >= 0x4e00 && code <= 0x9fff) ||
-      (code >= 0xf900 && code <= 0xfaff)
-    ) {
-      return true;
-    }
-  }
-  return false;
-}
-
-// Kanji-bearing non-independent nouns (日, 方, 上, …) are real vocabulary that
-// Yomitan segments as standalone tokens; MeCab's 非自立 tag exists to suppress
-// kana grammar nouns (こと, もの, とき) and must not hide these.
-export function isKanjiNonIndependentNounToken(
-  token: MergedToken,
-  pos1Exclusions: ReadonlySet<string>,
-): boolean {
-  if (pos1Exclusions.has('名詞')) {
-    return false;
-  }
-
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
-  if (pos1Parts.length !== 1 || pos2Parts.length !== 1) {
-    return false;
-  }
-  if (pos1Parts[0] !== '名詞' || pos2Parts[0] !== '非自立') {
-    return false;
-  }
-
-  return hasKanjiChar(token.surface) || hasKanjiChar(token.headword);
-}
-
-function normalizeKana(text: string): string {
-  const raw = text.trim();
-  if (!raw) {
-    return '';
-  }
-
-  let normalized = '';
-  for (const char of raw) {
-    const code = char.codePointAt(0);
-    if (code === undefined) {
-      continue;
-    }
-
-    if (code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END) {
-      normalized += String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET);
-      continue;
-    }
-
-    normalized += char;
-  }
-
-  return normalized;
-}
-
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
 function isTrailingSmallTsuKanaSfx(text: string): boolean {
   const normalized = normalizeKana(text);
   if (!normalized) {
@@ -286,7 +186,7 @@ function isExcludedTrailingParticleMergedToken(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   if (pos1Parts.length < 2) {
     return false;
   }
@@ -300,7 +200,7 @@ function isExcludedTrailingParticleMergedToken(token: MergedToken): boolean {
 }
 
 function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean {
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   if (
     pos1Parts.length === 0 ||
     !pos1Parts.every((part) => AUXILIARY_STEM_GRAMMAR_TAIL_POS1.has(part))
@@ -308,7 +208,7 @@ function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean {
     return false;
   }
 
-  const pos3Parts = splitNormalizedTagParts(normalizePosTag(token.pos3));
+  const pos3Parts = splitPosTag(token.pos3);
   return pos3Parts.includes('助動詞語幹');
 }
 
@@ -324,12 +224,12 @@ function isKanaOnlyNonIndependentNounHelperMerge(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   if (pos1Parts.length < 2 || pos1Parts[0] !== '名詞') {
     return false;
   }
 
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
+  const pos2Parts = splitPosTag(token.pos2);
   if (pos2Parts[0] !== '非自立') {
     return false;
   }
@@ -337,16 +237,11 @@ function isKanaOnlyNonIndependentNounHelperMerge(token: MergedToken): boolean {
   return pos1Parts.slice(1).every((part) => NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1.has(part));
 }
 
-function isKanaOnlyText(text: string): boolean {
-  const normalized = normalizeKana(text);
-  return normalized.length > 0 && [...normalized].every(isKanaChar);
-}
-
 function isLexicalKureruVerb(token: MergedToken): boolean {
   const normalizedSurface = normalizeKana(token.surface);
   const normalizedHeadword = normalizeKana(token.headword);
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
+  const pos1Parts = splitPosTag(token.pos1);
+  const pos2Parts = splitPosTag(token.pos2);
   return (
     normalizedSurface === 'くれ' &&
     normalizedHeadword === 'くれる' &&
@@ -363,7 +258,7 @@ function isStandaloneAuxiliaryInflectionFragment(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   if (pos1Parts.length === 0) {
     return false;
   }
@@ -372,7 +267,7 @@ function isStandaloneAuxiliaryInflectionFragment(token: MergedToken): boolean {
     return true;
   }
 
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
+  const pos2Parts = splitPosTag(token.pos2);
   return (
     pos1Parts[0] === '動詞' &&
     pos2Parts[0] === '接尾' &&
@@ -387,7 +282,7 @@ function isAuxiliaryOnlyHelperSpan(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   if (
     pos1Parts.length === 0 ||
     !pos1Parts.every((part) => AUXILIARY_HELPER_SPAN_POS1.has(part)) ||
@@ -397,7 +292,7 @@ function isAuxiliaryOnlyHelperSpan(token: MergedToken): boolean {
     return false;
   }
 
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
+  const pos2Parts = splitPosTag(token.pos2);
   return !pos2Parts.some((part) => LEXICAL_VERB_POS2.has(part));
 }
 
@@ -408,7 +303,7 @@ function isStandaloneSuruTeGrammarHelper(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   return (
     isKanaOnlyText(normalizedSurface) && (pos1Parts.length === 0 || pos1Parts.includes('動詞'))
   );
@@ -483,29 +378,7 @@ export function shouldExcludeTokenFromSubtitleAnnotations(
 
   const pos1Exclusions = resolvePos1Exclusions(options);
   const pos2Exclusions = resolvePos2Exclusions(options);
-  const normalizedPos1 = normalizePosTag(token.pos1);
-  const normalizedPos2 = normalizePosTag(token.pos2);
-  const hasPos1 = normalizedPos1.length > 0;
-  const hasPos2 = normalizedPos2.length > 0;
-
-  if (isExcludedByTagSet(normalizedPos1, pos1Exclusions)) {
-    return true;
-  }
-
-  if (
-    isExcludedByTagSet(normalizedPos2, pos2Exclusions) &&
-    !isKanjiNonIndependentNounToken(token, pos1Exclusions)
-  ) {
-    return true;
-  }
-
-  if (
-    !hasPos1 &&
-    !hasPos2 &&
-    (token.partOfSpeech === PartOfSpeech.particle ||
-      token.partOfSpeech === PartOfSpeech.bound_auxiliary ||
-      token.partOfSpeech === PartOfSpeech.symbol)
-  ) {
+  if (!isContentTokenByPos(token, pos1Exclusions, pos2Exclusions)) {
     return true;
   }
 

src/core/services/tokenizer/token-classification.test.ts

@@ -0,0 +1,57 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { MergedToken, PartOfSpeech } from '../../../types';
+import {
+  isContentTokenByPos,
+  isKanaCandidateIgnorableChar,
+  isKanaCandidateText,
+  isKanaChar,
+  isKanaOnlyText,
+  isTokenPos2Excluded,
+} from './token-classification';
+
+const POS1_EXCLUSIONS = new Set(['助詞']);
+const POS2_EXCLUSIONS = new Set(['非自立']);
+
+function makeNoun(surface: string): MergedToken {
+  return {
+    surface,
+    reading: surface,
+    headword: surface,
+    startPos: 0,
+    endPos: surface.length,
+    partOfSpeech: PartOfSpeech.noun,
+    pos1: '名詞',
+    pos2: '非自立',
+    isMerged: false,
+    isKnown: false,
+    isNPlusOneTarget: false,
+  };
+}
+
+test('kana classification excludes the katakana-hiragana double hyphen', () => {
+  assert.equal(isKanaChar('゠'), false);
+  assert.equal(isKanaOnlyText('゠'), false);
+});
+
+test('POS classification keeps kanji non-independent nouns as content', () => {
+  const token = makeNoun('日');
+
+  assert.equal(isTokenPos2Excluded(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), false);
+  assert.equal(isContentTokenByPos(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), true);
+});
+
+test('POS classification excludes kana non-independent nouns', () => {
+  const token = makeNoun('こと');
+
+  assert.equal(isTokenPos2Excluded(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), true);
+  assert.equal(isContentTokenByPos(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), false);
+});
+
+test('kana candidate classification allows punctuation around kana only', () => {
+  assert.equal(isKanaCandidateIgnorableChar('！'), true);
+  assert.equal(isKanaCandidateIgnorableChar('猫'), false);
+  assert.equal(isKanaCandidateText('「かな！？」'), true);
+  assert.equal(isKanaCandidateText('「！？」'), false);
+  assert.equal(isKanaCandidateText('かな猫'), false);
+});

src/core/services/tokenizer/token-classification.ts

@@ -0,0 +1,167 @@
+import { MergedToken, PartOfSpeech } from '../../../types';
+
+const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
+const KATAKANA_CODEPOINT_START = 0x30a1;
+const KATAKANA_CODEPOINT_END = 0x30f6;
+
+export function normalizeKana(text: string): string {
+  const raw = text.trim();
+  if (!raw) {
+    return '';
+  }
+
+  let normalized = '';
+  for (const char of raw) {
+    const code = char.codePointAt(0);
+    if (code === undefined) {
+      continue;
+    }
+
+    normalized +=
+      code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END
+        ? String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET)
+        : char;
+  }
+
+  return normalized;
+}
+
+export function isKanaChar(char: string): boolean {
+  const code = char.codePointAt(0);
+  if (code === undefined) {
+    return false;
+  }
+
+  return (
+    (code >= 0x3041 && code <= 0x3096) ||
+    (code >= 0x309b && code <= 0x309f) ||
+    code === 0x30fc ||
+    (code >= 0x30a1 && code <= 0x30fa) ||
+    (code >= 0x30fd && code <= 0x30ff)
+  );
+}
+
+export function isKanaOnlyText(text: string | null | undefined): boolean {
+  if (typeof text !== 'string') {
+    return false;
+  }
+
+  const normalized = normalizeKana(text);
+  return normalized.length > 0 && [...normalized].every(isKanaChar);
+}
+
+export function isKanaCandidateIgnorableChar(char: string): boolean {
+  return /^[\s.,!?;:()[\]{}"'`、。！？…‥・「」『』（）［］｛｝〈〉《》【】―-]$/u.test(char);
+}
+
+export function isKanaCandidateText(text: string): boolean {
+  const normalized = text.trim();
+  if (normalized.length === 0) {
+    return false;
+  }
+
+  let hasKana = false;
+  for (const char of normalized) {
+    if (isKanaChar(char)) {
+      hasKana = true;
+      continue;
+    }
+    if (!isKanaCandidateIgnorableChar(char)) {
+      return false;
+    }
+  }
+
+  return hasKana;
+}
+
+export function normalizePosTag(value: string | null | undefined): string {
+  return typeof value === 'string' ? value.trim() : '';
+}
+
+export function splitPosTag(value: string | null | undefined): string[] {
+  const normalized = normalizePosTag(value);
+  if (!normalized) {
+    return [];
+  }
+
+  return normalized
+    .split('|')
+    .map((part) => part.trim())
+    .filter((part) => part.length > 0);
+}
+
+export function isPosTagExcluded(
+  value: string | null | undefined,
+  exclusions: ReadonlySet<string>,
+): boolean {
+  const parts = splitPosTag(value);
+  return parts.length > 0 && parts.every((part) => exclusions.has(part));
+}
+
+function hasKanjiChar(text: string): boolean {
+  for (const char of text) {
+    const code = char.codePointAt(0);
+    if (
+      code !== undefined &&
+      ((code >= 0x3400 && code <= 0x4dbf) ||
+        (code >= 0x4e00 && code <= 0x9fff) ||
+        (code >= 0xf900 && code <= 0xfaff))
+    ) {
+      return true;
+    }
+  }
+  return false;
+}
+
+// MeCab's 非自立 tag suppresses kana grammar nouns (こと, もの, とき), but
+// Yomitan can segment kanji-bearing nouns (日, 方, 上, …) as real vocabulary.
+function isKanjiNonIndependentNounToken(
+  token: MergedToken,
+  pos1Exclusions: ReadonlySet<string>,
+): boolean {
+  if (pos1Exclusions.has('名詞')) {
+    return false;
+  }
+
+  const pos1Parts = splitPosTag(token.pos1);
+  const pos2Parts = splitPosTag(token.pos2);
+  return (
+    pos1Parts.length === 1 &&
+    pos1Parts[0] === '名詞' &&
+    pos2Parts.length === 1 &&
+    pos2Parts[0] === '非自立' &&
+    (hasKanjiChar(token.surface) || hasKanjiChar(token.headword))
+  );
+}
+
+export function isTokenPos2Excluded(
+  token: MergedToken,
+  pos1Exclusions: ReadonlySet<string>,
+  pos2Exclusions: ReadonlySet<string>,
+): boolean {
+  return (
+    isPosTagExcluded(token.pos2, pos2Exclusions) &&
+    !isKanjiNonIndependentNounToken(token, pos1Exclusions)
+  );
+}
+
+export function isContentTokenByPos(
+  token: MergedToken,
+  pos1Exclusions: ReadonlySet<string>,
+  pos2Exclusions: ReadonlySet<string>,
+): boolean {
+  if (
+    isPosTagExcluded(token.pos1, pos1Exclusions) ||
+    isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions)
+  ) {
+    return false;
+  }
+
+  if (splitPosTag(token.pos1).length > 0 || splitPosTag(token.pos2).length > 0) {
+    return true;
+  }
+
+  return ![PartOfSpeech.particle, PartOfSpeech.bound_auxiliary, PartOfSpeech.symbol].includes(
+    token.partOfSpeech,
+  );
+}

src/token-merger.ts

@@ -20,6 +20,7 @@ import { PartOfSpeech, Token, MergedToken } from './types';
 import { DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG } from './token-pos1-exclusions';
 import { DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG } from './token-pos2-exclusions';
 import { shouldExcludeTokenFromSubtitleAnnotations } from './core/services/tokenizer/subtitle-annotation-filter';
+import { isKanaCandidateText } from './core/services/tokenizer/token-classification';
 
 export function isNoun(tok: Token): boolean {
   return tok.partOfSpeech === PartOfSpeech.noun;
@@ -261,67 +262,6 @@ const SENTENCE_BOUNDARY_SURFACES = new Set(['。', '？', '！', '?', '!', '…'
 const N_PLUS_ONE_IGNORED_POS1 = new Set(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG.defaults);
 const N_PLUS_ONE_IGNORED_POS2 = new Set(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG.defaults);
 
-function normalizePos1Tag(pos1: string | undefined): string {
-  return typeof pos1 === 'string' ? pos1.trim() : '';
-}
-
-function normalizePos2Tag(pos2: string | undefined): string {
-  return typeof pos2 === 'string' ? pos2.trim() : '';
-}
-
-function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<string>): boolean {
-  if (!normalizedTag) {
-    return false;
-  }
-  const parts = normalizedTag
-    .split('|')
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
-  if (parts.length === 0) {
-    return false;
-  }
-  return parts.every((part) => exclusions.has(part));
-}
-
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
-function isKanaCandidateIgnorableChar(char: string): boolean {
-  return /^[\s.,!?;:()[\]{}"'`、。！？…‥・「」『』（）［］｛｝〈〉《》【】―-]$/u.test(char);
-}
-
-function isKanaOnlyText(text: string): boolean {
-  const normalized = text.trim();
-  if (normalized.length === 0) {
-    return false;
-  }
-
-  let hasKana = false;
-  for (const char of normalized) {
-    if (isKanaChar(char)) {
-      hasKana = true;
-      continue;
-    }
-    if (!isKanaCandidateIgnorableChar(char)) {
-      return false;
-    }
-  }
-
-  return hasKana;
-}
-
 function normalizeSourceTextForTokenOffsets(sourceText: string | undefined): string | undefined {
   return typeof sourceText === 'string' ? sourceText.replace(/\r?\n/g, ' ').trim() : undefined;
 }
@@ -334,7 +274,7 @@ export function isNPlusOneCandidateToken(
   if (token.isKnown) {
     return false;
   }
-  if (isKanaOnlyText(token.surface)) {
+  if (isKanaCandidateText(token.surface)) {
     return false;
   }
   return isNPlusOneWordCountToken(token, pos1Exclusions, pos2Exclusions);
@@ -349,28 +289,6 @@ function isNPlusOneWordCountToken(
     return false;
   }
 
-  const normalizedPos1 = normalizePos1Tag(token.pos1);
-  const hasPos1 = normalizedPos1.length > 0;
-  if (isExcludedByTagSet(normalizedPos1, pos1Exclusions)) {
-    return false;
-  }
-
-  const normalizedPos2 = normalizePos2Tag(token.pos2);
-  const hasPos2 = normalizedPos2.length > 0;
-  if (isExcludedByTagSet(normalizedPos2, pos2Exclusions)) {
-    return false;
-  }
-
-  if (
-    !hasPos1 &&
-    !hasPos2 &&
-    (token.partOfSpeech === PartOfSpeech.particle ||
-      token.partOfSpeech === PartOfSpeech.bound_auxiliary ||
-      token.partOfSpeech === PartOfSpeech.symbol)
-  ) {
-    return false;
-  }
-
   if (token.partOfSpeech === PartOfSpeech.noun && token.pos2 === '固有名詞') {
     return false;
   }