refactor(tokenizer): consolidate kana/POS helpers into token-classification (#161)

changes/kana-pos-content-classification.md

@@ -0,0 +1,4 @@
+type: fixed
+area: overlay
+
+- Kept kanji vocabulary tagged `名詞/非自立` eligible for N+1 highlighting, consistent with frequency, JLPT, and vocabulary persistence.

docs-site/subtitle-annotations.md

@@ -10,6 +10,8 @@ SubMiner's primary tokenizer is Yomitan itself - subtitle text is tokenized base
 
 Before any of those layers render, SubMiner strips annotation metadata from tokens that are usually just subtitle glue or annotation noise. Standalone particles, auxiliaries, adnominals, common explanatory endings like `んです` / `のだ`, merged trailing quote-particle forms like `...って`, auxiliary-stem grammar tails like `そうだ` (MeCab POS3 `助動詞語幹`), repeated kana interjections, and similar non-lexical helper tokens remain hoverable in the subtitle text, but they render as plain tokens without known-word, N+1, frequency, JLPT, or name-match annotation styling.
 
+Kanji vocabulary that MeCab labels `名詞/非自立`, such as `日` or `以外`, remains content for every annotation layer. The `非自立` exclusion only suppresses kana grammar nouns such as `こと` and `もの`.
+
 ## N+1 Word Highlighting
 
 N+1 highlighting identifies sentences where you know every word except one, making them ideal mining targets. When enabled, SubMiner builds a local cache of your known vocabulary from Anki and highlights tokens accordingly.
@@ -25,7 +27,7 @@ N+1 highlighting identifies sentences where you know every word except one, maki
 **Key settings:**
 
 | Option                                    | Default      | Description                                              |
-| ----------------------------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| ----------------------------------------- | ------------ | -------------------------------------------------------- |
 | `ankiConnect.knownWords.highlightEnabled` | `false`      | Enable known-word cache lookups used by N+1 highlighting |
 | `ankiConnect.knownWords.refreshMinutes`   | `1440`       | Minutes between Anki cache refreshes                     |
 | `ankiConnect.knownWords.decks`            | `{}`         | Deck→fields map for known-word cache queries             |

src/core/services/tokenizer.ts

@@ -25,6 +25,7 @@ import {
   requestYomitanTermFrequencies,
 } from './tokenizer/yomitan-parser-runtime';
 import type { YomitanTermFrequency } from './tokenizer/yomitan-parser-runtime';
+import { isKanaChar } from './tokenizer/token-classification';
 
 const logger = createLogger('main:tokenizer');
 
@@ -322,20 +323,6 @@ function normalizeFrequencyLookupText(rawText: string): string {
   return rawText.trim().toLowerCase();
 }
 
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
 function getTrailingKanaSuffix(surface: string): string {
   const chars = Array.from(surface);
   let splitIndex = chars.length;

src/core/services/tokenizer/annotation-stage.test.ts

@@ -1563,6 +1563,7 @@ test('annotateTokens keeps frequency for unknown non-independent kanji noun toke
   );
 
   assert.equal(result[0]?.isKnown, false);
+  assert.equal(result[0]?.isNPlusOneTarget, true);
   assert.equal(result[0]?.frequencyRank, 718);
   assert.equal(result[0]?.jlptLevel, 'N4');
 });

src/core/services/tokenizer/annotation-stage.ts

@@ -10,14 +10,19 @@ import {
 import { JlptLevel, MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types';
 import { shouldIgnoreJlptByTerm, shouldIgnoreJlptForMecabPos1 } from '../jlpt-token-filter';
 import {
-  isKanjiNonIndependentNounToken,
   shouldExcludeTokenFromSubtitleAnnotations as sharedShouldExcludeTokenFromSubtitleAnnotations,
   stripSubtitleAnnotationMetadata as sharedStripSubtitleAnnotationMetadata,
 } from './subtitle-annotation-filter';
+import {
+  isKanaChar,
+  isKanaOnlyText,
+  isPosTagExcluded,
+  isTokenPos2Excluded,
+  normalizeKana,
+  normalizePosTag,
+  splitPosTag,
+} from './token-classification';
 
-const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
-const KATAKANA_CODEPOINT_START = 0x30a1;
-const KATAKANA_CODEPOINT_END = 0x30f6;
 const JLPT_LEVEL_LOOKUP_CACHE_LIMIT = 2048;
 
 const jlptLevelLookupCaches = new WeakMap<
@@ -55,30 +60,6 @@ function resolveKnownWordText(
   return matchMode === 'surface' ? surface : headword;
 }
 
-function normalizePos1Tag(pos1: string | undefined): string {
-  return typeof pos1 === 'string' ? pos1.trim() : '';
-}
-
-function splitNormalizedTagParts(normalizedTag: string): string[] {
-  if (!normalizedTag) {
-    return [];
-  }
-
-  return normalizedTag
-    .split('|')
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
-}
-
-function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<string>): boolean {
-  const parts = splitNormalizedTagParts(normalizedTag);
-  if (parts.length === 0) {
-    return false;
-  }
-
-  return parts.every((part) => exclusions.has(part));
-}
-
 function resolvePos1Exclusions(options: AnnotationStageOptions): ReadonlySet<string> {
   if (options.pos1Exclusions) {
     return options.pos1Exclusions;
@@ -95,10 +76,6 @@ function resolvePos2Exclusions(options: AnnotationStageOptions): ReadonlySet<str
   return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
 }
 
-function normalizePos2Tag(pos2: string | undefined): string {
-  return typeof pos2 === 'string' ? pos2.trim() : '';
-}
-
 function isExcludedComponent(
   pos1: string | undefined,
   pos2: string | undefined,
@@ -117,12 +94,12 @@ function shouldAllowContentLedMergedTokenFrequency(
   pos1Exclusions: ReadonlySet<string>,
   pos2Exclusions: ReadonlySet<string>,
 ): boolean {
-  const pos1Parts = splitNormalizedTagParts(normalizedPos1);
+  const pos1Parts = splitPosTag(normalizedPos1);
   if (pos1Parts.length < 2) {
     return false;
   }
 
-  const pos2Parts = splitNormalizedTagParts(normalizedPos2);
+  const pos2Parts = splitPosTag(normalizedPos2);
   if (isExcludedComponent(pos1Parts[0], pos2Parts[0], pos1Exclusions, pos2Exclusions)) {
     return false;
   }
@@ -144,8 +121,8 @@ function shouldAllowOrdinalPrefixNounFrequency(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
-  const pos2Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos2));
+  const pos2Parts = splitPosTag(token.pos2);
   return (
     pos1Parts.length >= 2 &&
     pos1Parts[0] === '接頭詞' &&
@@ -166,8 +143,8 @@ function shouldAllowHonorificPrefixNounFrequency(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
-  const pos2Parts = splitNormalizedTagParts(normalizePos2Tag(token.pos2));
+  const pos2Parts = splitPosTag(token.pos2);
   return (
     pos1Parts.length >= 2 &&
     pos1Parts[0] === '接頭詞' &&
@@ -182,12 +159,12 @@ function shouldAllowDeterminerLedNounFrequency(
   pos1Exclusions: ReadonlySet<string>,
   pos2Exclusions: ReadonlySet<string>,
 ): boolean {
-  const pos1Parts = splitNormalizedTagParts(normalizedPos1);
+  const pos1Parts = splitPosTag(normalizedPos1);
   if (pos1Parts.length < 2 || pos1Parts[0] !== '連体詞') {
     return false;
   }
 
-  const pos2Parts = splitNormalizedTagParts(normalizedPos2);
+  const pos2Parts = splitPosTag(normalizedPos2);
   if (!isExcludedComponent(pos1Parts[0], pos2Parts[0], pos1Exclusions, pos2Exclusions)) {
     return false;
   }
@@ -218,9 +195,9 @@ function isFrequencyExcludedByPos(
     return true;
   }
 
-  const normalizedPos1 = normalizePos1Tag(token.pos1);
+  const normalizedPos1 = normalizePosTag(token.pos1);
   const hasPos1 = normalizedPos1.length > 0;
-  const normalizedPos2 = normalizePos2Tag(token.pos2);
+  const normalizedPos2 = normalizePosTag(token.pos2);
   const hasPos2 = normalizedPos2.length > 0;
   const allowContentLedMergedToken = shouldAllowContentLedMergedTokenFrequency(
     normalizedPos1,
@@ -238,7 +215,7 @@ function isFrequencyExcludedByPos(
   const allowHonorificPrefixNounToken = shouldAllowHonorificPrefixNounFrequency(token);
 
   if (
-    isExcludedByTagSet(normalizedPos1, pos1Exclusions) &&
+    isPosTagExcluded(normalizedPos1, pos1Exclusions) &&
     !allowContentLedMergedToken &&
     !allowDeterminerLedNounToken &&
     !allowOrdinalPrefixNounToken &&
@@ -248,7 +225,7 @@ function isFrequencyExcludedByPos(
   }
 
   if (
-    isExcludedByTagSet(normalizedPos2, pos2Exclusions) &&
+    isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions) &&
     !allowContentLedMergedToken &&
     !allowDeterminerLedNounToken &&
     !allowOrdinalPrefixNounToken &&
@@ -280,8 +257,7 @@ export function shouldExcludeTokenFromVocabularyPersistence(
 
   return (
     sharedShouldExcludeTokenFromSubtitleAnnotations(token, { pos1Exclusions, pos2Exclusions }) ||
-    (isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) &&
+    isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions)
-      !isKanjiNonIndependentNounToken(token, pos1Exclusions))
   );
 }
 
@@ -332,45 +308,6 @@ function resolveJlptLookupText(token: MergedToken): string {
   return token.surface;
 }
 
-function normalizeJlptTextForExclusion(text: string): string {
-  const raw = text.trim();
-  if (!raw) {
-    return '';
-  }
-
-  let normalized = '';
-  for (const char of raw) {
-    const code = char.codePointAt(0);
-    if (code === undefined) {
-      continue;
-    }
-
-    if (code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END) {
-      normalized += String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET);
-      continue;
-    }
-
-    normalized += char;
-  }
-
-  return normalized;
-}
-
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
 function isRepeatedKanaSfx(text: string): boolean {
   const normalized = text.trim();
   if (!normalized) {
@@ -406,7 +343,7 @@ function isRepeatedKanaSfx(text: string): boolean {
 }
 
 function isTrailingSmallTsuKanaSfx(text: string): boolean {
-  const normalized = normalizeJlptTextForExclusion(text);
+  const normalized = normalizeKana(text);
   if (!normalized) {
     return false;
   }
@@ -424,7 +361,7 @@ function isTrailingSmallTsuKanaSfx(text: string): boolean {
 }
 
 function isReduplicatedKanaSfx(text: string): boolean {
-  const normalized = normalizeJlptTextForExclusion(text);
+  const normalized = normalizeKana(text);
   if (!normalized) {
     return false;
   }
@@ -443,7 +380,7 @@ function isReduplicatedKanaSfx(text: string): boolean {
 }
 
 function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
-  const normalized = normalizeJlptTextForExclusion(text);
+  const normalized = normalizeKana(text);
   if (!normalized) {
     return false;
   }
@@ -460,7 +397,7 @@ function isReduplicatedKanaSfxWithOptionalTrailingTo(text: string): boolean {
 }
 
 function hasAdjacentKanaRepeat(text: string): boolean {
-  const normalized = normalizeJlptTextForExclusion(text);
+  const normalized = normalizeKana(text);
   if (!normalized) {
     return false;
   }
@@ -490,7 +427,7 @@ function isLikelyFrequencyNoiseToken(token: MergedToken): boolean {
       continue;
     }
 
-    const normalizedCandidate = normalizeJlptTextForExclusion(trimmedCandidate);
+    const normalizedCandidate = normalizeKana(trimmedCandidate);
     if (!normalizedCandidate) {
       continue;
     }
@@ -528,19 +465,6 @@ function isSingleKanaFrequencyNoiseToken(text: string | undefined): boolean {
   return chars.length === 1 && isKanaChar(chars[0]!);
 }
 
-function isKanaOnlyText(text: string | undefined): boolean {
-  if (typeof text !== 'string') {
-    return false;
-  }
-
-  const normalized = text.trim();
-  if (!normalized) {
-    return false;
-  }
-
-  return [...normalized].every(isKanaChar);
-}
-
 function isKanaOnlyMixedFunctionContentToken(
   token: MergedToken,
   pos1Exclusions: ReadonlySet<string>,
@@ -549,7 +473,7 @@ function isKanaOnlyMixedFunctionContentToken(
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   const hasMixedFunctionContentParts =
     pos1Parts.length >= 2 &&
     pos1Parts.some((part) => pos1Exclusions.has(part)) &&
@@ -558,8 +482,8 @@ function isKanaOnlyMixedFunctionContentToken(
     return false;
   }
 
-  const normalizedReading = normalizeJlptTextForExclusion(token.reading);
+  const normalizedReading = normalizeKana(token.reading);
-  const normalizedHeadwordReading = normalizeJlptTextForExclusion(token.headwordReading ?? '');
+  const normalizedHeadwordReading = normalizeKana(token.headwordReading ?? '');
   return (
     !normalizedReading ||
     !normalizedHeadwordReading ||
@@ -577,7 +501,7 @@ function isJlptEligibleToken(token: MergedToken): boolean {
   );
 
   for (const candidate of candidates) {
-    const normalizedCandidate = normalizeJlptTextForExclusion(candidate);
+    const normalizedCandidate = normalizeKana(candidate);
     if (!normalizedCandidate) {
       continue;
     }
@@ -612,8 +536,8 @@ export function stripSubtitleAnnotationMetadata(
 // at least as many characters as the surface, with the surface's kana appearing
 // in order within the reading.
 function isCompleteReadingForSurface(surface: string, reading: string): boolean {
-  const surfaceChars = [...normalizeJlptTextForExclusion(surface)];
+  const surfaceChars = [...normalizeKana(surface)];
-  const readingChars = [...normalizeJlptTextForExclusion(reading)];
+  const readingChars = [...normalizeKana(reading)];
   if (readingChars.length < surfaceChars.length) {
     return false;
   }
@@ -697,10 +621,7 @@ function filterTokenFrequencyRank(
   pos1Exclusions: ReadonlySet<string>,
   pos2Exclusions: ReadonlySet<string>,
 ): number | undefined {
-  if (
+  if (isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions)) {
-    isFrequencyExcludedByPos(token, pos1Exclusions, pos2Exclusions) &&
-    !isKanjiNonIndependentNounToken(token, pos1Exclusions)
-  ) {
     return undefined;
   }
 

src/core/services/tokenizer/parser-selection-stage.ts

@@ -1,5 +1,6 @@
 import { MergedToken, NPlusOneMatchMode, PartOfSpeech } from '../../../types';
 import { isStandaloneGrammarEndingText } from './grammar-ending';
+import { isKanaChar, isKanaOnlyText } from './token-classification';
 
 interface YomitanParseHeadword {
   term?: unknown;
@@ -41,21 +42,6 @@ function resolveKnownWordText(
   return matchMode === 'surface' ? surface : headword;
 }
 
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
 function isYomitanParseLine(value: unknown): value is YomitanParseLine {
   if (!Array.isArray(value)) {
     return false;
@@ -138,10 +124,6 @@ function selectMergedHeadword(
   return firstHeadword;
 }
 
-function isKanaOnlyText(text: string): boolean {
-  return text.length > 0 && Array.from(text).every((char) => isKanaChar(char));
-}
-
 function isStandaloneGrammarEndingSegment(segment: YomitanParseSegment): boolean {
   const surface = segment.text?.trim() ?? '';
   const headword = extractYomitanHeadword(segment).trim();

src/core/services/tokenizer/part-of-speech.ts

@@ -1,8 +1,5 @@
 import { PartOfSpeech } from '../../../types';
-
+import { normalizePosTag, splitPosTag } from './token-classification';
-function normalizePosTag(value: string | null | undefined): string {
-  return typeof value === 'string' ? value.trim() : '';
-}
 
 export function isPartOfSpeechValue(value: unknown): value is PartOfSpeech {
   return typeof value === 'string' && Object.values(PartOfSpeech).includes(value as PartOfSpeech);
@@ -35,10 +32,7 @@ export function deriveStoredPartOfSpeech(input: {
   partOfSpeech?: string | null;
   pos1?: string | null;
 }): PartOfSpeech {
-  const pos1Parts = normalizePosTag(input.pos1)
+  const pos1Parts = splitPosTag(input.pos1);
-    .split('|')
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
 
   if (pos1Parts.length > 0) {
     const derivedParts = [...new Set(pos1Parts.map((part) => mapMecabPos1ToPartOfSpeech(part)))];

src/core/services/tokenizer/subtitle-annotation-filter.ts

@@ -6,13 +6,16 @@ import {
   DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG,
   resolveAnnotationPos2ExclusionSet,
 } from '../../../token-pos2-exclusions';
-import { MergedToken, PartOfSpeech } from '../../../types';
+import { MergedToken } from '../../../types';
 import { shouldIgnoreJlptByTerm } from '../jlpt-token-filter';
 import { isSubtitleGrammarEndingText } from './grammar-ending';
-
+import {
-const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
+  isContentTokenByPos,
-const KATAKANA_CODEPOINT_START = 0x30a1;
+  isKanaChar,
-const KATAKANA_CODEPOINT_END = 0x30f6;
+  isKanaOnlyText,
+  normalizeKana,
+  splitPosTag,
+} from './token-classification';
 
 const STANDALONE_GRAMMAR_PARTICLE_PHRASES = ['たって', 'だって'] as const;
 const STANDALONE_GRAMMAR_PARTICLE_PHRASES_SET: ReadonlySet<string> = new Set(
@@ -97,30 +100,6 @@ export interface SubtitleAnnotationFilterOptions {
   pos2Exclusions?: ReadonlySet<string>;
 }
 
-function normalizePosTag(pos: string | undefined): string {
-  return typeof pos === 'string' ? pos.trim() : '';
-}
-
-function splitNormalizedTagParts(normalizedTag: string): string[] {
-  if (!normalizedTag) {
-    return [];
-  }
-
-  return normalizedTag
-    .split('|')
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
-}
-
-function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<string>): boolean {
-  const parts = splitNormalizedTagParts(normalizedTag);
-  if (parts.length === 0) {
-    return false;
-  }
-
-  return parts.every((part) => exclusions.has(part));
-}
-
 function resolvePos1Exclusions(options: SubtitleAnnotationFilterOptions = {}): ReadonlySet<string> {
   if (options.pos1Exclusions) {
     return options.pos1Exclusions;
@@ -137,85 +116,6 @@ function resolvePos2Exclusions(options: SubtitleAnnotationFilterOptions = {}): R
   return resolveAnnotationPos2ExclusionSet(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG);
 }
 
-function hasKanjiChar(text: string): boolean {
-  for (const char of text) {
-    const code = char.codePointAt(0);
-    if (code === undefined) {
-      continue;
-    }
-    if (
-      (code >= 0x3400 && code <= 0x4dbf) ||
-      (code >= 0x4e00 && code <= 0x9fff) ||
-      (code >= 0xf900 && code <= 0xfaff)
-    ) {
-      return true;
-    }
-  }
-  return false;
-}
-
-// Kanji-bearing non-independent nouns (日, 方, 上, …) are real vocabulary that
-// Yomitan segments as standalone tokens; MeCab's 非自立 tag exists to suppress
-// kana grammar nouns (こと, もの, とき) and must not hide these.
-export function isKanjiNonIndependentNounToken(
-  token: MergedToken,
-  pos1Exclusions: ReadonlySet<string>,
-): boolean {
-  if (pos1Exclusions.has('名詞')) {
-    return false;
-  }
-
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
-  if (pos1Parts.length !== 1 || pos2Parts.length !== 1) {
-    return false;
-  }
-  if (pos1Parts[0] !== '名詞' || pos2Parts[0] !== '非自立') {
-    return false;
-  }
-
-  return hasKanjiChar(token.surface) || hasKanjiChar(token.headword);
-}
-
-function normalizeKana(text: string): string {
-  const raw = text.trim();
-  if (!raw) {
-    return '';
-  }
-
-  let normalized = '';
-  for (const char of raw) {
-    const code = char.codePointAt(0);
-    if (code === undefined) {
-      continue;
-    }
-
-    if (code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END) {
-      normalized += String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET);
-      continue;
-    }
-
-    normalized += char;
-  }
-
-  return normalized;
-}
-
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
 function isTrailingSmallTsuKanaSfx(text: string): boolean {
   const normalized = normalizeKana(text);
   if (!normalized) {
@@ -286,7 +186,7 @@ function isExcludedTrailingParticleMergedToken(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   if (pos1Parts.length < 2) {
     return false;
   }
@@ -300,7 +200,7 @@ function isExcludedTrailingParticleMergedToken(token: MergedToken): boolean {
 }
 
 function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean {
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   if (
     pos1Parts.length === 0 ||
     !pos1Parts.every((part) => AUXILIARY_STEM_GRAMMAR_TAIL_POS1.has(part))
@@ -308,7 +208,7 @@ function isAuxiliaryStemGrammarTailToken(token: MergedToken): boolean {
     return false;
   }
 
-  const pos3Parts = splitNormalizedTagParts(normalizePosTag(token.pos3));
+  const pos3Parts = splitPosTag(token.pos3);
   return pos3Parts.includes('助動詞語幹');
 }
 
@@ -324,12 +224,12 @@ function isKanaOnlyNonIndependentNounHelperMerge(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   if (pos1Parts.length < 2 || pos1Parts[0] !== '名詞') {
     return false;
   }
 
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
+  const pos2Parts = splitPosTag(token.pos2);
   if (pos2Parts[0] !== '非自立') {
     return false;
   }
@@ -337,16 +237,11 @@ function isKanaOnlyNonIndependentNounHelperMerge(token: MergedToken): boolean {
   return pos1Parts.slice(1).every((part) => NON_INDEPENDENT_NOUN_HELPER_TAIL_POS1.has(part));
 }
 
-function isKanaOnlyText(text: string): boolean {
-  const normalized = normalizeKana(text);
-  return normalized.length > 0 && [...normalized].every(isKanaChar);
-}
-
 function isLexicalKureruVerb(token: MergedToken): boolean {
   const normalizedSurface = normalizeKana(token.surface);
   const normalizedHeadword = normalizeKana(token.headword);
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
+  const pos2Parts = splitPosTag(token.pos2);
   return (
     normalizedSurface === 'くれ' &&
     normalizedHeadword === 'くれる' &&
@@ -363,7 +258,7 @@ function isStandaloneAuxiliaryInflectionFragment(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   if (pos1Parts.length === 0) {
     return false;
   }
@@ -372,7 +267,7 @@ function isStandaloneAuxiliaryInflectionFragment(token: MergedToken): boolean {
     return true;
   }
 
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
+  const pos2Parts = splitPosTag(token.pos2);
   return (
     pos1Parts[0] === '動詞' &&
     pos2Parts[0] === '接尾' &&
@@ -387,7 +282,7 @@ function isAuxiliaryOnlyHelperSpan(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   if (
     pos1Parts.length === 0 ||
     !pos1Parts.every((part) => AUXILIARY_HELPER_SPAN_POS1.has(part)) ||
@@ -397,7 +292,7 @@ function isAuxiliaryOnlyHelperSpan(token: MergedToken): boolean {
     return false;
   }
 
-  const pos2Parts = splitNormalizedTagParts(normalizePosTag(token.pos2));
+  const pos2Parts = splitPosTag(token.pos2);
   return !pos2Parts.some((part) => LEXICAL_VERB_POS2.has(part));
 }
 
@@ -408,7 +303,7 @@ function isStandaloneSuruTeGrammarHelper(token: MergedToken): boolean {
     return false;
   }
 
-  const pos1Parts = splitNormalizedTagParts(normalizePosTag(token.pos1));
+  const pos1Parts = splitPosTag(token.pos1);
   return (
     isKanaOnlyText(normalizedSurface) && (pos1Parts.length === 0 || pos1Parts.includes('動詞'))
   );
@@ -483,29 +378,7 @@ export function shouldExcludeTokenFromSubtitleAnnotations(
 
   const pos1Exclusions = resolvePos1Exclusions(options);
   const pos2Exclusions = resolvePos2Exclusions(options);
-  const normalizedPos1 = normalizePosTag(token.pos1);
+  if (!isContentTokenByPos(token, pos1Exclusions, pos2Exclusions)) {
-  const normalizedPos2 = normalizePosTag(token.pos2);
-  const hasPos1 = normalizedPos1.length > 0;
-  const hasPos2 = normalizedPos2.length > 0;
-
-  if (isExcludedByTagSet(normalizedPos1, pos1Exclusions)) {
-    return true;
-  }
-
-  if (
-    isExcludedByTagSet(normalizedPos2, pos2Exclusions) &&
-    !isKanjiNonIndependentNounToken(token, pos1Exclusions)
-  ) {
-    return true;
-  }
-
-  if (
-    !hasPos1 &&
-    !hasPos2 &&
-    (token.partOfSpeech === PartOfSpeech.particle ||
-      token.partOfSpeech === PartOfSpeech.bound_auxiliary ||
-      token.partOfSpeech === PartOfSpeech.symbol)
-  ) {
     return true;
   }
 

src/core/services/tokenizer/token-classification.test.ts

@@ -0,0 +1,57 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { MergedToken, PartOfSpeech } from '../../../types';
+import {
+  isContentTokenByPos,
+  isKanaCandidateIgnorableChar,
+  isKanaCandidateText,
+  isKanaChar,
+  isKanaOnlyText,
+  isTokenPos2Excluded,
+} from './token-classification';
+
+const POS1_EXCLUSIONS = new Set(['助詞']);
+const POS2_EXCLUSIONS = new Set(['非自立']);
+
+function makeNoun(surface: string): MergedToken {
+  return {
+    surface,
+    reading: surface,
+    headword: surface,
+    startPos: 0,
+    endPos: surface.length,
+    partOfSpeech: PartOfSpeech.noun,
+    pos1: '名詞',
+    pos2: '非自立',
+    isMerged: false,
+    isKnown: false,
+    isNPlusOneTarget: false,
+  };
+}
+
+test('kana classification excludes the katakana-hiragana double hyphen', () => {
+  assert.equal(isKanaChar('゠'), false);
+  assert.equal(isKanaOnlyText('゠'), false);
+});
+
+test('POS classification keeps kanji non-independent nouns as content', () => {
+  const token = makeNoun('日');
+
+  assert.equal(isTokenPos2Excluded(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), false);
+  assert.equal(isContentTokenByPos(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), true);
+});
+
+test('POS classification excludes kana non-independent nouns', () => {
+  const token = makeNoun('こと');
+
+  assert.equal(isTokenPos2Excluded(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), true);
+  assert.equal(isContentTokenByPos(token, POS1_EXCLUSIONS, POS2_EXCLUSIONS), false);
+});
+
+test('kana candidate classification allows punctuation around kana only', () => {
+  assert.equal(isKanaCandidateIgnorableChar('！'), true);
+  assert.equal(isKanaCandidateIgnorableChar('猫'), false);
+  assert.equal(isKanaCandidateText('「かな！？」'), true);
+  assert.equal(isKanaCandidateText('「！？」'), false);
+  assert.equal(isKanaCandidateText('かな猫'), false);
+});

src/core/services/tokenizer/token-classification.ts

@@ -0,0 +1,167 @@
+import { MergedToken, PartOfSpeech } from '../../../types';
+
+const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
+const KATAKANA_CODEPOINT_START = 0x30a1;
+const KATAKANA_CODEPOINT_END = 0x30f6;
+
+export function normalizeKana(text: string): string {
+  const raw = text.trim();
+  if (!raw) {
+    return '';
+  }
+
+  let normalized = '';
+  for (const char of raw) {
+    const code = char.codePointAt(0);
+    if (code === undefined) {
+      continue;
+    }
+
+    normalized +=
+      code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END
+        ? String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET)
+        : char;
+  }
+
+  return normalized;
+}
+
+export function isKanaChar(char: string): boolean {
+  const code = char.codePointAt(0);
+  if (code === undefined) {
+    return false;
+  }
+
+  return (
+    (code >= 0x3041 && code <= 0x3096) ||
+    (code >= 0x309b && code <= 0x309f) ||
+    code === 0x30fc ||
+    (code >= 0x30a1 && code <= 0x30fa) ||
+    (code >= 0x30fd && code <= 0x30ff)
+  );
+}
+
+export function isKanaOnlyText(text: string | null | undefined): boolean {
+  if (typeof text !== 'string') {
+    return false;
+  }
+
+  const normalized = normalizeKana(text);
+  return normalized.length > 0 && [...normalized].every(isKanaChar);
+}
+
+export function isKanaCandidateIgnorableChar(char: string): boolean {
+  return /^[\s.,!?;:()[\]{}"'`、。！？…‥・「」『』（）［］｛｝〈〉《》【】―-]$/u.test(char);
+}
+
+export function isKanaCandidateText(text: string): boolean {
+  const normalized = text.trim();
+  if (normalized.length === 0) {
+    return false;
+  }
+
+  let hasKana = false;
+  for (const char of normalized) {
+    if (isKanaChar(char)) {
+      hasKana = true;
+      continue;
+    }
+    if (!isKanaCandidateIgnorableChar(char)) {
+      return false;
+    }
+  }
+
+  return hasKana;
+}
+
+export function normalizePosTag(value: string | null | undefined): string {
+  return typeof value === 'string' ? value.trim() : '';
+}
+
+export function splitPosTag(value: string | null | undefined): string[] {
+  const normalized = normalizePosTag(value);
+  if (!normalized) {
+    return [];
+  }
+
+  return normalized
+    .split('|')
+    .map((part) => part.trim())
+    .filter((part) => part.length > 0);
+}
+
+export function isPosTagExcluded(
+  value: string | null | undefined,
+  exclusions: ReadonlySet<string>,
+): boolean {
+  const parts = splitPosTag(value);
+  return parts.length > 0 && parts.every((part) => exclusions.has(part));
+}
+
+function hasKanjiChar(text: string): boolean {
+  for (const char of text) {
+    const code = char.codePointAt(0);
+    if (
+      code !== undefined &&
+      ((code >= 0x3400 && code <= 0x4dbf) ||
+        (code >= 0x4e00 && code <= 0x9fff) ||
+        (code >= 0xf900 && code <= 0xfaff))
+    ) {
+      return true;
+    }
+  }
+  return false;
+}
+
+// MeCab's 非自立 tag suppresses kana grammar nouns (こと, もの, とき), but
+// Yomitan can segment kanji-bearing nouns (日, 方, 上, …) as real vocabulary.
+function isKanjiNonIndependentNounToken(
+  token: MergedToken,
+  pos1Exclusions: ReadonlySet<string>,
+): boolean {
+  if (pos1Exclusions.has('名詞')) {
+    return false;
+  }
+
+  const pos1Parts = splitPosTag(token.pos1);
+  const pos2Parts = splitPosTag(token.pos2);
+  return (
+    pos1Parts.length === 1 &&
+    pos1Parts[0] === '名詞' &&
+    pos2Parts.length === 1 &&
+    pos2Parts[0] === '非自立' &&
+    (hasKanjiChar(token.surface) || hasKanjiChar(token.headword))
+  );
+}
+
+export function isTokenPos2Excluded(
+  token: MergedToken,
+  pos1Exclusions: ReadonlySet<string>,
+  pos2Exclusions: ReadonlySet<string>,
+): boolean {
+  return (
+    isPosTagExcluded(token.pos2, pos2Exclusions) &&
+    !isKanjiNonIndependentNounToken(token, pos1Exclusions)
+  );
+}
+
+export function isContentTokenByPos(
+  token: MergedToken,
+  pos1Exclusions: ReadonlySet<string>,
+  pos2Exclusions: ReadonlySet<string>,
+): boolean {
+  if (
+    isPosTagExcluded(token.pos1, pos1Exclusions) ||
+    isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions)
+  ) {
+    return false;
+  }
+
+  if (splitPosTag(token.pos1).length > 0 || splitPosTag(token.pos2).length > 0) {
+    return true;
+  }
+
+  return ![PartOfSpeech.particle, PartOfSpeech.bound_auxiliary, PartOfSpeech.symbol].includes(
+    token.partOfSpeech,
+  );
+}

src/token-merger.ts

@@ -20,6 +20,7 @@ import { PartOfSpeech, Token, MergedToken } from './types';
 import { DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG } from './token-pos1-exclusions';
 import { DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG } from './token-pos2-exclusions';
 import { shouldExcludeTokenFromSubtitleAnnotations } from './core/services/tokenizer/subtitle-annotation-filter';
+import { isKanaCandidateText } from './core/services/tokenizer/token-classification';
 
 export function isNoun(tok: Token): boolean {
   return tok.partOfSpeech === PartOfSpeech.noun;
@@ -261,67 +262,6 @@ const SENTENCE_BOUNDARY_SURFACES = new Set(['。', '？', '！', '?', '!', '…'
 const N_PLUS_ONE_IGNORED_POS1 = new Set(DEFAULT_ANNOTATION_POS1_EXCLUSION_CONFIG.defaults);
 const N_PLUS_ONE_IGNORED_POS2 = new Set(DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG.defaults);
 
-function normalizePos1Tag(pos1: string | undefined): string {
-  return typeof pos1 === 'string' ? pos1.trim() : '';
-}
-
-function normalizePos2Tag(pos2: string | undefined): string {
-  return typeof pos2 === 'string' ? pos2.trim() : '';
-}
-
-function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<string>): boolean {
-  if (!normalizedTag) {
-    return false;
-  }
-  const parts = normalizedTag
-    .split('|')
-    .map((part) => part.trim())
-    .filter((part) => part.length > 0);
-  if (parts.length === 0) {
-    return false;
-  }
-  return parts.every((part) => exclusions.has(part));
-}
-
-function isKanaChar(char: string): boolean {
-  const code = char.codePointAt(0);
-  if (code === undefined) {
-    return false;
-  }
-
-  return (
-    (code >= 0x3041 && code <= 0x3096) ||
-    (code >= 0x309b && code <= 0x309f) ||
-    code === 0x30fc ||
-    (code >= 0x30a0 && code <= 0x30fa) ||
-    (code >= 0x30fd && code <= 0x30ff)
-  );
-}
-
-function isKanaCandidateIgnorableChar(char: string): boolean {
-  return /^[\s.,!?;:()[\]{}"'`、。！？…‥・「」『』（）［］｛｝〈〉《》【】―-]$/u.test(char);
-}
-
-function isKanaOnlyText(text: string): boolean {
-  const normalized = text.trim();
-  if (normalized.length === 0) {
-    return false;
-  }
-
-  let hasKana = false;
-  for (const char of normalized) {
-    if (isKanaChar(char)) {
-      hasKana = true;
-      continue;
-    }
-    if (!isKanaCandidateIgnorableChar(char)) {
-      return false;
-    }
-  }
-
-  return hasKana;
-}
-
 function normalizeSourceTextForTokenOffsets(sourceText: string | undefined): string | undefined {
   return typeof sourceText === 'string' ? sourceText.replace(/\r?\n/g, ' ').trim() : undefined;
 }
@@ -334,7 +274,7 @@ export function isNPlusOneCandidateToken(
   if (token.isKnown) {
     return false;
   }
-  if (isKanaOnlyText(token.surface)) {
+  if (isKanaCandidateText(token.surface)) {
     return false;
   }
   return isNPlusOneWordCountToken(token, pos1Exclusions, pos2Exclusions);
@@ -349,28 +289,6 @@ function isNPlusOneWordCountToken(
     return false;
   }
 
-  const normalizedPos1 = normalizePos1Tag(token.pos1);
-  const hasPos1 = normalizedPos1.length > 0;
-  if (isExcludedByTagSet(normalizedPos1, pos1Exclusions)) {
-    return false;
-  }
-
-  const normalizedPos2 = normalizePos2Tag(token.pos2);
-  const hasPos2 = normalizedPos2.length > 0;
-  if (isExcludedByTagSet(normalizedPos2, pos2Exclusions)) {
-    return false;
-  }
-
-  if (
-    !hasPos1 &&
-    !hasPos2 &&
-    (token.partOfSpeech === PartOfSpeech.particle ||
-      token.partOfSpeech === PartOfSpeech.bound_auxiliary ||
-      token.partOfSpeech === PartOfSpeech.symbol)
-  ) {
-    return false;
-  }
-
   if (token.partOfSpeech === PartOfSpeech.noun && token.pos2 === '固有名詞') {
     return false;
   }