fix: Kiku field grouping, frequency particles, sidebar media, Yomitan popup visibility (#91)

This commit is contained in:
2026-05-27 01:40:48 -07:00
committed by GitHub
parent efe50ed1e4
commit 1dcfed86ab
52 changed files with 1695 additions and 368 deletions
@@ -133,3 +133,129 @@ test('createFieldGroupingOverlayRuntime callback restores hidden visible overlay
assert.equal(visible, false);
assert.deepEqual(visibilityTransitions, [true, false]);
});
async function settleWithinMicrotasks<T>(
promise: Promise<T>,
attempts = 10,
): Promise<T | 'timeout'> {
let settled = false;
let settledValue: T | undefined;
void promise.then((value) => {
settled = true;
settledValue = value;
});
for (let i = 0; i < attempts; i += 1) {
await Promise.resolve();
if (settled) {
return settledValue as T;
}
}
return 'timeout';
}
test('createFieldGroupingOverlayRuntime callback cancels and cleans up when kiku modal never acknowledges open', async () => {
let resolver: ((choice: KikuFieldGroupingChoice) => void) | null = null;
const sends: Array<{
channel: string;
payload: unknown;
restoreOnModalClose?: string;
preferModalWindow?: boolean;
}> = [];
const waitCalls: Array<{ modal: string; timeoutMs: number }> = [];
const warnings: string[] = [];
const closed: string[] = [];
const originalSetTimeout = globalThis.setTimeout;
globalThis.setTimeout = (() => 0) as unknown as typeof globalThis.setTimeout;
try {
const runtime = createFieldGroupingOverlayRuntime<'kiku'>({
getMainWindow: () => null,
getVisibleOverlayVisible: () => true,
setVisibleOverlayVisible: () => {},
getResolver: () => resolver,
setResolver: (nextResolver) => {
resolver = nextResolver;
},
getRestoreVisibleOverlayOnModalClose: () => new Set<'kiku'>(),
sendToVisibleOverlay: (channel, payload, runtimeOptions) => {
sends.push({
channel,
payload,
restoreOnModalClose: runtimeOptions?.restoreOnModalClose,
preferModalWindow: runtimeOptions?.preferModalWindow,
});
return true;
},
waitForModalOpen: async (modal, timeoutMs) => {
waitCalls.push({ modal, timeoutMs });
return false;
},
handleOverlayModalClosed: (modal) => {
closed.push(modal);
},
logWarn: (message) => {
warnings.push(message);
},
});
const request = {
original: {
noteId: 1,
expression: 'a',
sentencePreview: 'a',
hasAudio: false,
hasImage: false,
isOriginal: true,
},
duplicate: {
noteId: 2,
expression: 'b',
sentencePreview: 'b',
hasAudio: false,
hasImage: false,
isOriginal: false,
},
};
const pendingChoice = runtime.createFieldGroupingCallback()(request);
const result = await settleWithinMicrotasks(pendingChoice);
assert.notEqual(result, 'timeout');
assert.deepEqual(result, {
keepNoteId: 0,
deleteNoteId: 0,
deleteDuplicate: true,
cancelled: true,
});
assert.equal(resolver, null);
assert.deepEqual(
sends.map(({ channel, restoreOnModalClose, preferModalWindow }) => ({
channel,
restoreOnModalClose,
preferModalWindow,
})),
[
{
channel: 'kiku:field-grouping-request',
restoreOnModalClose: 'kiku',
preferModalWindow: true,
},
{
channel: 'kiku:field-grouping-request',
restoreOnModalClose: 'kiku',
preferModalWindow: true,
},
],
);
assert.deepEqual(waitCalls, [
{ modal: 'kiku', timeoutMs: 1500 },
{ modal: 'kiku', timeoutMs: 1500 },
]);
assert.deepEqual(warnings, [
'Kiku field grouping modal did not acknowledge modal open on first attempt; retrying dedicated modal window.',
]);
assert.deepEqual(closed, ['kiku']);
} finally {
globalThis.setTimeout = originalSetTimeout;
}
});
+48 -3
View File
@@ -8,6 +8,10 @@ interface WindowLike {
};
}
const KIKU_FIELD_GROUPING_MODAL_OPEN_TIMEOUT_MS = 1500;
const KIKU_FIELD_GROUPING_MODAL_RETRY_WARNING =
'Kiku field grouping modal did not acknowledge modal open on first attempt; retrying dedicated modal window.';
export interface FieldGroupingOverlayRuntimeOptions<T extends string> {
getMainWindow: () => WindowLike | null;
getVisibleOverlayVisible: () => boolean;
@@ -15,10 +19,13 @@ export interface FieldGroupingOverlayRuntimeOptions<T extends string> {
getResolver: () => ((choice: KikuFieldGroupingChoice) => void) | null;
setResolver: (resolver: ((choice: KikuFieldGroupingChoice) => void) | null) => void;
getRestoreVisibleOverlayOnModalClose: () => Set<T>;
waitForModalOpen?: (modal: T, timeoutMs: number) => Promise<boolean>;
handleOverlayModalClosed?: (modal: T) => void;
logWarn?: (message: string) => void;
sendToVisibleOverlay?: (
channel: string,
payload?: unknown,
runtimeOptions?: { restoreOnModalClose?: T },
runtimeOptions?: { restoreOnModalClose?: T; preferModalWindow?: boolean },
) => boolean;
}
@@ -28,7 +35,7 @@ export function createFieldGroupingOverlayRuntime<T extends string>(
sendToVisibleOverlay: (
channel: string,
payload?: unknown,
runtimeOptions?: { restoreOnModalClose?: T },
runtimeOptions?: { restoreOnModalClose?: T; preferModalWindow?: boolean },
) => boolean;
createFieldGroupingCallback: () => (
data: KikuFieldGroupingRequestData,
@@ -37,7 +44,7 @@ export function createFieldGroupingOverlayRuntime<T extends string>(
const sendToVisibleOverlay = (
channel: string,
payload?: unknown,
runtimeOptions?: { restoreOnModalClose?: T },
runtimeOptions?: { restoreOnModalClose?: T; preferModalWindow?: boolean },
): boolean => {
if (options.sendToVisibleOverlay) {
const wasVisible = options.getVisibleOverlayVisible();
@@ -58,6 +65,43 @@ export function createFieldGroupingOverlayRuntime<T extends string>(
});
};
const sendKikuFieldGroupingRequest = async (
data: KikuFieldGroupingRequestData,
): Promise<boolean> => {
const kikuModal = 'kiku' as T;
const sendOpen = (): boolean =>
sendToVisibleOverlay('kiku:field-grouping-request', data, {
restoreOnModalClose: kikuModal,
preferModalWindow: true,
});
if (!options.waitForModalOpen) {
return sendOpen();
}
if (!sendOpen()) {
return false;
}
if (await options.waitForModalOpen(kikuModal, KIKU_FIELD_GROUPING_MODAL_OPEN_TIMEOUT_MS)) {
return true;
}
options.logWarn?.(KIKU_FIELD_GROUPING_MODAL_RETRY_WARNING);
if (!sendOpen()) {
options.handleOverlayModalClosed?.(kikuModal);
return false;
}
const opened = await options.waitForModalOpen(
kikuModal,
KIKU_FIELD_GROUPING_MODAL_OPEN_TIMEOUT_MS,
);
if (!opened) {
options.handleOverlayModalClosed?.(kikuModal);
}
return opened;
};
const createFieldGroupingCallback = (): ((
data: KikuFieldGroupingRequestData,
) => Promise<KikuFieldGroupingChoice>) => {
@@ -67,6 +111,7 @@ export function createFieldGroupingOverlayRuntime<T extends string>(
getResolver: options.getResolver,
setResolver: options.setResolver,
sendToVisibleOverlay,
sendKikuFieldGroupingRequest,
});
};
+32 -14
View File
@@ -5,7 +5,7 @@ export function createFieldGroupingCallback(options: {
setVisibleOverlayVisible: (visible: boolean) => void;
getResolver: () => ((choice: KikuFieldGroupingChoice) => void) | null;
setResolver: (resolver: ((choice: KikuFieldGroupingChoice) => void) | null) => void;
sendRequestToVisibleOverlay: (data: KikuFieldGroupingRequestData) => boolean;
sendRequestToVisibleOverlay: (data: KikuFieldGroupingRequestData) => boolean | Promise<boolean>;
}): (data: KikuFieldGroupingRequestData) => Promise<KikuFieldGroupingChoice> {
return async (data: KikuFieldGroupingRequestData): Promise<KikuFieldGroupingChoice> => {
return new Promise((resolve) => {
@@ -21,10 +21,15 @@ export function createFieldGroupingCallback(options: {
const previousVisibleOverlay = options.getVisibleOverlayVisible();
let settled = false;
let timeout: ReturnType<typeof setTimeout> | null = null;
const finish = (choice: KikuFieldGroupingChoice): void => {
if (settled) return;
settled = true;
if (timeout !== null) {
clearTimeout(timeout);
timeout = null;
}
if (options.getResolver() === finish) {
options.setResolver(null);
}
@@ -36,25 +41,38 @@ export function createFieldGroupingCallback(options: {
};
options.setResolver(finish);
if (!options.sendRequestToVisibleOverlay(data)) {
finish({
keepNoteId: 0,
deleteNoteId: 0,
deleteDuplicate: true,
cancelled: true,
});
return;
}
setTimeout(() => {
if (!settled) {
void Promise.resolve(options.sendRequestToVisibleOverlay(data)).then(
(sent) => {
if (settled) return;
if (!sent) {
finish({
keepNoteId: 0,
deleteNoteId: 0,
deleteDuplicate: true,
cancelled: true,
});
return;
}
timeout = setTimeout(() => {
if (!settled) {
finish({
keepNoteId: 0,
deleteNoteId: 0,
deleteDuplicate: true,
cancelled: true,
});
}
}, 90000);
},
() => {
finish({
keepNoteId: 0,
deleteNoteId: 0,
deleteDuplicate: true,
cancelled: true,
});
}
}, 90000);
},
);
});
};
}
+77
View File
@@ -630,6 +630,83 @@ test('registerIpcHandlers forwards yomitan lookup tracking commands to immersion
assert.deepEqual(calls, ['lookup']);
});
test('registerIpcHandlers forwards valid subtitle sidebar mining context', () => {
const { registrar, handlers } = createFakeIpcRegistrar();
const contexts: unknown[] = [];
const deps = createRegisterIpcDeps() as IpcServiceDeps & {
recordSubtitleMiningContext: (context: unknown | null) => void;
};
deps.recordSubtitleMiningContext = (context) => {
contexts.push(context);
};
registerIpcHandlers(deps, registrar);
const handler = handlers.on.get(IPC_CHANNELS.command.recordYomitanLookup);
assert.equal(typeof handler, 'function');
handler?.(
{},
{
source: 'subtitle-sidebar',
text: 'sidebar previous line',
startTime: 10,
endTime: 12,
capturedAtMs: 123,
},
);
assert.deepEqual(contexts, [
{
source: 'subtitle-sidebar',
text: 'sidebar previous line',
startTime: 10,
endTime: 12,
capturedAtMs: 123,
},
]);
});
test('registerIpcHandlers records yomitan lookup when subtitle context recording fails', () => {
const { registrar, handlers } = createFakeIpcRegistrar();
const calls: string[] = [];
const warnings: unknown[][] = [];
const originalWarn = console.warn;
console.warn = (...args: unknown[]) => {
warnings.push(args);
};
const deps = createRegisterIpcDeps({
immersionTracker: createFakeImmersionTracker({
recordYomitanLookup: () => {
calls.push('lookup');
},
}),
}) as IpcServiceDeps & {
recordSubtitleMiningContext: (context: unknown | null) => void;
};
deps.recordSubtitleMiningContext = () => {
throw new Error('context write failed');
};
try {
registerIpcHandlers(deps, registrar);
const handler = handlers.on.get(IPC_CHANNELS.command.recordYomitanLookup);
assert.equal(typeof handler, 'function');
assert.doesNotThrow(() => {
handler?.({}, { source: 'subtitle-sidebar', text: 'line', startTime: 1, endTime: 2 });
});
assert.deepEqual(calls, ['lookup']);
assert.equal(warnings.length, 1);
assert.equal(warnings[0]?.[0], 'Failed to record subtitle mining context:');
assert.equal(warnings[0]?.[1], 'context write failed');
} finally {
console.warn = originalWarn;
}
});
test('registerIpcHandlers returns empty stats overview shape without a tracker', async () => {
const { registrar, handlers } = createFakeIpcRegistrar();
registerIpcHandlers(createRegisterIpcDeps(), registrar);
+50 -1
View File
@@ -9,6 +9,7 @@ import type {
ResolvedControllerConfig,
RuntimeOptionId,
RuntimeOptionValue,
SubtitleMiningContext,
SubtitleSidebarSnapshot,
SubtitlePosition,
SubsyncManualRunRequest,
@@ -95,6 +96,7 @@ export interface IpcServiceDeps {
getAnilistQueueStatus: () => unknown;
retryAnilistQueueNow: () => Promise<{ ok: boolean; message: string }>;
runAnilistPostWatchUpdateOnManualMark?: () => Promise<void>;
recordSubtitleMiningContext?: (context: SubtitleMiningContext | null) => void;
getCharacterDictionarySelection?: (searchTitle?: string) => Promise<unknown>;
setCharacterDictionarySelection?: (
mediaId: number,
@@ -175,6 +177,43 @@ interface IpcMainRegistrar {
handle: (channel: string, listener: (event: unknown, ...args: unknown[]) => unknown) => void;
}
function parseSubtitleMiningContext(payload: unknown): SubtitleMiningContext | null {
if (!payload || typeof payload !== 'object') {
return null;
}
const record = payload as Record<string, unknown>;
const source = record.source;
const text = record.text;
const startTime = record.startTime;
const endTime = record.endTime;
const capturedAtMs = record.capturedAtMs;
if (
source !== 'subtitle-sidebar' ||
typeof text !== 'string' ||
text.trim().length === 0 ||
typeof startTime !== 'number' ||
typeof endTime !== 'number' ||
!Number.isFinite(startTime) ||
!Number.isFinite(endTime) ||
endTime <= startTime
) {
return null;
}
const parsed: SubtitleMiningContext = {
source: 'subtitle-sidebar',
text,
startTime,
endTime,
};
if (typeof capturedAtMs === 'number' && Number.isFinite(capturedAtMs)) {
parsed.capturedAtMs = capturedAtMs;
}
return parsed;
}
export interface IpcDepsRuntimeOptions {
getMainWindow: () => WindowLike | null;
getVisibleOverlayVisibility: () => boolean;
@@ -230,6 +269,7 @@ export interface IpcDepsRuntimeOptions {
getAnilistQueueStatus: () => unknown;
retryAnilistQueueNow: () => Promise<{ ok: boolean; message: string }>;
runAnilistPostWatchUpdateOnManualMark?: () => Promise<void>;
recordSubtitleMiningContext?: (context: SubtitleMiningContext | null) => void;
getCharacterDictionarySelection?: (searchTitle?: string) => Promise<unknown>;
setCharacterDictionarySelection?: (
mediaId: number,
@@ -257,6 +297,7 @@ export function createIpcDepsRuntime(options: IpcDepsRuntimeOptions): IpcService
onOverlayModalOpened: options.onOverlayModalOpened,
onOverlayMouseInteractionChanged: options.onOverlayMouseInteractionChanged,
openYomitanSettings: options.openYomitanSettings,
recordSubtitleMiningContext: options.recordSubtitleMiningContext,
quitApp: options.quitApp,
toggleDevTools: () => {
const mainWindow = options.getMainWindow();
@@ -423,7 +464,15 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar
deps.openYomitanSettings();
});
ipc.on(IPC_CHANNELS.command.recordYomitanLookup, () => {
ipc.on(IPC_CHANNELS.command.recordYomitanLookup, (_event: unknown, payload: unknown) => {
try {
deps.recordSubtitleMiningContext?.(parseSubtitleMiningContext(payload));
} catch (error) {
console.warn(
'Failed to record subtitle mining context:',
error instanceof Error ? error.message : String(error),
);
}
deps.immersionTracker?.recordYomitanLookup();
});
+64
View File
@@ -124,6 +124,70 @@ test('mineSentenceCard creates sentence card from mpv subtitle state', async ()
]);
});
test('mineSentenceCard refreshes secondary subtitle text before creating card', async () => {
const created: Array<{ sentence: string; secondarySub?: string }> = [];
const requestedProperties: string[] = [];
await mineSentenceCard({
ankiIntegration: {
updateLastAddedFromClipboard: async () => {},
triggerFieldGroupingForLastAddedCard: async () => {},
markLastCardAsAudioCard: async () => {},
createSentenceCard: async (sentence, _startTime, _endTime, secondarySub) => {
created.push({ sentence, secondarySub });
return true;
},
},
mpvClient: {
connected: true,
currentSubText: '日本語字幕',
currentSubStart: 10,
currentSubEnd: 12,
currentSecondarySubText: '日本語字幕',
requestProperty: async (name: string) => {
requestedProperties.push(name);
return name === 'secondary-sub-text' ? 'English subtitle' : null;
},
},
showMpvOsd: () => {},
});
assert.deepEqual(requestedProperties, ['secondary-sub-text']);
assert.deepEqual(created, [{ sentence: '日本語字幕', secondarySub: 'English subtitle' }]);
});
test('mineSentenceCard does not fall back to stale cached secondary subtitle after successful refresh', async () => {
const created: Array<{ sentence: string; secondarySub?: string }> = [];
await mineSentenceCard({
ankiIntegration: {
updateLastAddedFromClipboard: async () => {},
triggerFieldGroupingForLastAddedCard: async () => {},
markLastCardAsAudioCard: async () => {},
createSentenceCard: async (sentence, _startTime, _endTime, secondarySub) => {
created.push({ sentence, secondarySub });
return true;
},
},
mpvClient: {
connected: true,
currentSubText: '日本語字幕',
currentSubStart: 10,
currentSubEnd: 12,
currentSecondarySubText: 'stale cached subtitle',
requestProperty: async (name: string) => {
if (name === 'secondary-sub-text') {
return '';
}
return null;
},
},
showMpvOsd: () => {},
});
assert.deepEqual(created, [{ sentence: '日本語字幕', secondarySub: undefined }]);
});
test('handleMultiCopyDigit copies available history and reports truncation', () => {
const osd: string[] = [];
const copied: string[] = [];
+29 -1
View File
@@ -25,6 +25,7 @@ interface MpvClientLike {
currentSubStart: number;
currentSubEnd: number;
currentSecondarySubText?: string;
requestProperty?: (name: string) => Promise<unknown>;
}
export function handleMultiCopyDigit(
@@ -95,6 +96,32 @@ function getSecondarySubTextForMinedBlocks(
return getCurrentSecondarySubText();
}
function normalizeSecondarySubText(text: unknown, primaryText: string): string | undefined {
if (typeof text !== 'string') {
return undefined;
}
const trimmed = text.trim();
if (!trimmed || trimmed === primaryText.trim()) {
return undefined;
}
return trimmed;
}
async function getCurrentSecondarySubTextForSentenceCard(
mpvClient: MpvClientLike,
): Promise<string | undefined> {
const primaryText = mpvClient.currentSubText;
if (mpvClient.requestProperty) {
try {
const latestSecondaryText = await mpvClient.requestProperty('secondary-sub-text');
return normalizeSecondarySubText(latestSecondaryText, primaryText);
} catch {
// Fall back to the cached secondary subtitle below.
}
}
return normalizeSecondarySubText(mpvClient.currentSecondarySubText, primaryText);
}
export async function updateLastCardFromClipboard(deps: {
ankiIntegration: AnkiIntegrationLike | null;
readClipboardText: () => string;
@@ -141,11 +168,12 @@ export async function mineSentenceCard(deps: {
return false;
}
const secondarySubText = await getCurrentSecondarySubTextForSentenceCard(mpvClient);
return await anki.createSentenceCard(
mpvClient.currentSubText,
mpvClient.currentSubStart,
mpvClient.currentSubEnd,
mpvClient.currentSecondarySubText || undefined,
secondarySubText,
);
}
+7 -4
View File
@@ -62,8 +62,9 @@ export function createFieldGroupingCallbackRuntime<T extends string>(options: {
sendToVisibleOverlay: (
channel: string,
payload?: unknown,
runtimeOptions?: { restoreOnModalClose?: T },
runtimeOptions?: { restoreOnModalClose?: T; preferModalWindow?: boolean },
) => boolean;
sendKikuFieldGroupingRequest?: (data: KikuFieldGroupingRequestData) => Promise<boolean>;
}): (data: KikuFieldGroupingRequestData) => Promise<KikuFieldGroupingChoice> {
return createFieldGroupingCallback({
getVisibleOverlayVisible: options.getVisibleOverlayVisible,
@@ -71,8 +72,10 @@ export function createFieldGroupingCallbackRuntime<T extends string>(options: {
getResolver: options.getResolver,
setResolver: options.setResolver,
sendRequestToVisibleOverlay: (data) =>
options.sendToVisibleOverlay('kiku:field-grouping-request', data, {
restoreOnModalClose: 'kiku' as T,
}),
options.sendKikuFieldGroupingRequest
? options.sendKikuFieldGroupingRequest(data)
: options.sendToVisibleOverlay('kiku:field-grouping-request', data, {
restoreOnModalClose: 'kiku' as T,
}),
});
}
+60
View File
@@ -25,6 +25,7 @@ interface YomitanTokenInput {
surface: string;
reading?: string;
headword?: string;
frequencyRank?: number;
isNameMatch?: boolean;
wordClasses?: string[];
}
@@ -57,6 +58,7 @@ function makeDepsFromYomitanTokens(
startPos,
endPos,
isNameMatch: token.isNameMatch ?? false,
frequencyRank: token.frequencyRank,
wordClasses: token.wordClasses,
};
});
@@ -4279,6 +4281,64 @@ test('tokenizeSubtitle keeps frequency for content-led merged token with trailin
assert.equal(result.tokens?.[0]?.frequencyRank, 5468);
});
test('tokenizeSubtitle keeps Yomitan frequency for noun-particle-noun compounds', async () => {
const result = await tokenizeSubtitle(
'目の前',
makeDepsFromYomitanTokens(
[{ surface: '目の前', reading: 'めのまえ', headword: '目の前', frequencyRank: 581 }],
{
getFrequencyDictionaryEnabled: () => true,
tokenizeWithMecab: async () => [
{
headword: '目',
surface: '目',
reading: 'メ',
startPos: 0,
endPos: 1,
partOfSpeech: PartOfSpeech.noun,
pos1: '名詞',
pos2: '一般',
isMerged: false,
isKnown: false,
isNPlusOneTarget: false,
},
{
headword: 'の',
surface: 'の',
reading: '',
startPos: 1,
endPos: 2,
partOfSpeech: PartOfSpeech.particle,
pos1: '助詞',
pos2: '連体化',
isMerged: false,
isKnown: false,
isNPlusOneTarget: false,
},
{
headword: '前',
surface: '前',
reading: 'マエ',
startPos: 2,
endPos: 3,
partOfSpeech: PartOfSpeech.noun,
pos1: '名詞',
pos2: '副詞可能',
isMerged: false,
isKnown: false,
isNPlusOneTarget: false,
},
],
},
),
);
assert.equal(result.tokens?.length, 1);
assert.equal(result.tokens?.[0]?.surface, '目の前');
assert.equal(result.tokens?.[0]?.pos1, '名詞|助詞');
assert.equal(result.tokens?.[0]?.frequencyRank, 581);
});
test('tokenizeSubtitle keeps frequency for ordinal prefix-noun tokens', async () => {
const result = await tokenizeSubtitle(
'第二走者',
@@ -70,9 +70,8 @@ function isExcludedByTagSet(normalizedTag: string, exclusions: ReadonlySet<strin
if (parts.length === 0) {
return false;
}
// Frequency highlighting should be conservative: if any merged component is excluded,
// skip highlighting the whole token to avoid noisy merged fragments.
return parts.some((part) => exclusions.has(part));
return parts.every((part) => exclusions.has(part));
}
function resolvePos1Exclusions(options: AnnotationStageOptions): ReadonlySet<string> {
@@ -227,6 +226,10 @@ function isFrequencyExcludedByPos(
return true;
}
if (isKanaOnlyMixedFunctionContentToken(token, pos1Exclusions)) {
return true;
}
const normalizedPos1 = normalizePos1Tag(token.pos1);
const hasPos1 = normalizedPos1.length > 0;
const normalizedPos2 = normalizePos2Tag(token.pos2);
@@ -564,6 +567,35 @@ function isSingleKanaFrequencyNoiseToken(text: string | undefined): boolean {
return chars.length === 1 && isKanaChar(chars[0]!);
}
function isKanaOnlyText(text: string | undefined): boolean {
if (typeof text !== 'string') {
return false;
}
const normalized = text.trim();
if (!normalized) {
return false;
}
return [...normalized].every(isKanaChar);
}
function isKanaOnlyMixedFunctionContentToken(
token: MergedToken,
pos1Exclusions: ReadonlySet<string>,
): boolean {
if (!isKanaOnlyText(token.surface)) {
return false;
}
const pos1Parts = splitNormalizedTagParts(normalizePos1Tag(token.pos1));
return (
pos1Parts.length >= 2 &&
pos1Parts.some((part) => pos1Exclusions.has(part)) &&
pos1Parts.some((part) => !pos1Exclusions.has(part))
);
}
function isJlptEligibleToken(token: MergedToken): boolean {
if (token.pos1 && shouldIgnoreJlptForMecabPos1(token.pos1)) {
return false;