import { MergedToken, PartOfSpeech } from '../../../types'; const KATAKANA_TO_HIRAGANA_OFFSET = 0x60; const KATAKANA_CODEPOINT_START = 0x30a1; const KATAKANA_CODEPOINT_END = 0x30f6; export function normalizeKana(text: string): string { const raw = text.trim(); if (!raw) { return ''; } let normalized = ''; for (const char of raw) { const code = char.codePointAt(0); if (code === undefined) { continue; } normalized += code >= KATAKANA_CODEPOINT_START && code <= KATAKANA_CODEPOINT_END ? String.fromCodePoint(code - KATAKANA_TO_HIRAGANA_OFFSET) : char; } return normalized; } export function isKanaChar(char: string): boolean { const code = char.codePointAt(0); if (code === undefined) { return false; } return ( (code >= 0x3041 && code <= 0x3096) || (code >= 0x309b && code <= 0x309f) || code === 0x30fc || (code >= 0x30a1 && code <= 0x30fa) || (code >= 0x30fd && code <= 0x30ff) ); } export function isKanaOnlyText(text: string | null | undefined): boolean { if (typeof text !== 'string') { return false; } const normalized = normalizeKana(text); return normalized.length > 0 && [...normalized].every(isKanaChar); } export function isKanaCandidateIgnorableChar(char: string): boolean { return /^[\s.,!?;:()[\]{}"'`、。!?…‥・「」『』()[]{}〈〉《》【】―-]$/u.test(char); } export function isKanaCandidateText(text: string): boolean { const normalized = text.trim(); if (normalized.length === 0) { return false; } let hasKana = false; for (const char of normalized) { if (isKanaChar(char)) { hasKana = true; continue; } if (!isKanaCandidateIgnorableChar(char)) { return false; } } return hasKana; } export function normalizePosTag(value: string | null | undefined): string { return typeof value === 'string' ? value.trim() : ''; } export function splitPosTag(value: string | null | undefined): string[] { const normalized = normalizePosTag(value); if (!normalized) { return []; } return normalized .split('|') .map((part) => part.trim()) .filter((part) => part.length > 0); } export function isPosTagExcluded( value: string | null | undefined, exclusions: ReadonlySet, ): boolean { const parts = splitPosTag(value); return parts.length > 0 && parts.every((part) => exclusions.has(part)); } function hasKanjiChar(text: string): boolean { for (const char of text) { const code = char.codePointAt(0); if ( code !== undefined && ((code >= 0x3400 && code <= 0x4dbf) || (code >= 0x4e00 && code <= 0x9fff) || (code >= 0xf900 && code <= 0xfaff) || (code >= 0x20000 && code <= 0x2fa1f) || (code >= 0x30000 && code <= 0x323af)) ) { return true; } } return false; } // MeCab's 非自立 tag suppresses kana grammar nouns (こと, もの, とき), but // Yomitan can segment kanji-bearing nouns (日, 方, 上, …) as real vocabulary. export function isKanjiNonIndependentNounToken( token: MergedToken, pos1Exclusions: ReadonlySet, ): boolean { if (pos1Exclusions.has('名詞')) { return false; } const pos1Parts = splitPosTag(token.pos1); const pos2Parts = splitPosTag(token.pos2); return ( pos1Parts.length === 1 && pos1Parts[0] === '名詞' && pos2Parts.length === 1 && pos2Parts[0] === '非自立' && (hasKanjiChar(token.surface) || hasKanjiChar(token.headword)) ); } export function isTokenPos2Excluded( token: MergedToken, pos1Exclusions: ReadonlySet, pos2Exclusions: ReadonlySet, ): boolean { return ( isPosTagExcluded(token.pos2, pos2Exclusions) && !isKanjiNonIndependentNounToken(token, pos1Exclusions) ); } export function isContentTokenByPos( token: MergedToken, pos1Exclusions: ReadonlySet, pos2Exclusions: ReadonlySet, ): boolean { if ( isPosTagExcluded(token.pos1, pos1Exclusions) || isTokenPos2Excluded(token, pos1Exclusions, pos2Exclusions) ) { return false; } if (splitPosTag(token.pos1).length > 0 || splitPosTag(token.pos2).length > 0) { return true; } return ![PartOfSpeech.particle, PartOfSpeech.bound_auxiliary, PartOfSpeech.symbol].includes( token.partOfSpeech, ); }