feat(release): prepare v0.19.5 with subtitle pipeline improvements

- Rework ASS, YouTube, overlapping, and secondary subtitle handling
- Keep overlay, mining, sidebar, and stats aligned with visible cues
- Harden release-note validation and package Linux thumbnail support
This commit is contained in:
2026-09-02 09:27:06 -07:00
278 changed files with 16719 additions and 1837 deletions
+84 -3
View File
@@ -11,10 +11,12 @@ import type { MediaInput } from './media-input';
import { AnkiConnectConfig } from './types';
type TestOverlayNotificationPayload = {
id?: string;
title: string;
body?: string;
image?: string;
variant?: string;
persistent?: boolean;
actions?: Array<{ id: string; label: string; noteId?: number }>;
};
@@ -153,6 +155,7 @@ function createFieldGroupingMergeCollaborator(options?: {
getEffectiveSentenceCardConfig: () => ({
sentenceField: 'Sentence',
audioField: 'SentenceAudio',
fieldGroupingProvider: 'kiku' as const,
}),
getCurrentSubtitleText: () => options?.currentSubtitleText,
resolveFieldName,
@@ -606,6 +609,7 @@ test('AnkiIntegration applies ready YouTube cache media to every queued note id'
const integration = new AnkiIntegration(
{
fields: {
audio: 'ExpressionAudio',
image: 'Picture',
},
media: {
@@ -659,7 +663,7 @@ test('AnkiIntegration applies ready YouTube cache media to every queued note id'
noteIds.map((noteId) => ({
noteId,
fields: {
SentenceAudio: { value: '' },
ExpressionAudio: { value: '' },
Picture: { value: '' },
},
})),
@@ -944,7 +948,7 @@ test('AnkiIntegration queues YouTube media updates against recovered source URLs
noteInfo: {
noteId: 404,
fields: {
SentenceAudio: { value: '' },
ExpressionAudio: { value: '' },
Picture: { value: '' },
},
},
@@ -956,7 +960,8 @@ test('AnkiIntegration queues YouTube media updates against recovered source URLs
assert.equal(queued, true);
assert.equal(updatedNotes.length, 1);
assert.equal(updatedNotes[0]?.noteId, 404);
assert.match(updatedNotes[0]?.fields.SentenceAudio ?? '', /^\[sound:audio_/);
assert.match(updatedNotes[0]?.fields.ExpressionAudio ?? '', /^\[sound:audio_/);
assert.equal(updatedNotes[0]?.fields.SentenceAudio, undefined);
assert.match(updatedNotes[0]?.fields.Picture ?? '', /^<img src="image_/);
assert.equal(storedMedia.length, 2);
assert.deepEqual(audioVolumeScales, [0.3 ** 3]);
@@ -1182,6 +1187,82 @@ test('AnkiIntegration embeds generated notification image on overlay mined-card
assert.deepEqual(cleanupPaths, [notificationIconPath]);
});
test('AnkiIntegration keeps overlay card-update progress visible until the terminal notification', async () => {
const overlayNotifications: TestOverlayNotificationPayload[] = [];
const integration = new AnkiIntegration(
{
behavior: {
notificationType: 'overlay',
},
},
{} as never,
{} as never,
undefined,
undefined,
undefined,
undefined,
{},
undefined,
(payload) => {
overlayNotifications.push(payload);
},
);
const updateNotifications = integration as unknown as {
beginUpdateProgress: (message: string) => void;
showNotification: (noteId: number, label: string | number) => Promise<void>;
};
updateNotifications.beginUpdateProgress('Updating card');
await updateNotifications.showNotification(42, '食べる');
assert.deepEqual(
overlayNotifications.map(({ id, variant, persistent }) => ({ id, variant, persistent })),
[
{ id: 'anki-update-progress', variant: 'progress', persistent: true },
{ id: 'anki-update-progress', variant: 'success', persistent: false },
],
);
});
test('AnkiIntegration dismisses persistent overlay update progress when no terminal notification replaces it', () => {
const overlayNotifications: TestOverlayNotificationPayload[] = [];
const dismissedIds: string[] = [];
const integration = new AnkiIntegration(
{
behavior: {
notificationType: 'overlay',
},
},
{} as never,
{} as never,
undefined,
undefined,
undefined,
undefined,
{},
undefined,
(payload) => {
overlayNotifications.push(payload);
},
undefined,
undefined,
undefined,
(id) => {
dismissedIds.push(id);
},
);
const updateNotifications = integration as unknown as {
beginUpdateProgress: (message: string) => void;
endUpdateProgress: () => void;
};
updateNotifications.beginUpdateProgress('Updating card');
updateNotifications.endUpdateProgress();
assert.equal(overlayNotifications[0]?.persistent, true);
assert.deepEqual(dismissedIds, ['anki-update-progress']);
});
test('AnkiIntegration keeps overlay notification image when temp icon write fails', async () => {
const desktopNotifications: Array<{ title: string; body?: string; icon?: string }> = [];
const overlayNotifications: TestOverlayNotificationPayload[] = [];
+54 -16
View File
@@ -218,6 +218,8 @@ export class AnkiIntegration {
null;
private overlayNotificationCallback: ((payload: OverlayNotificationPayload) => void) | null =
null;
private overlayNotificationDismissCallback: ((id: string) => void) | null = null;
private overlayUpdateProgressActive = false;
private updateInProgress = false;
private uiFeedbackState: UiFeedbackState = createUiFeedbackState();
private parseWarningKeys = new Set<string>();
@@ -265,6 +267,7 @@ export class AnkiIntegration {
getCachedMediaPath?: MediaGenerationInputResolverOptions['getCachedMediaPath'],
shouldRequireRemoteMediaCache?: () => boolean,
getYoutubeMediaSourceUrl?: () => Promise<string | null | undefined> | string | null | undefined,
overlayNotificationDismissCallback?: (id: string) => void,
) {
this.config = normalizeAnkiIntegrationConfig(config);
this.aiConfig = { ...aiConfig };
@@ -280,6 +283,7 @@ export class AnkiIntegration {
this.getCachedMediaPath = getCachedMediaPath ?? null;
this.shouldRequireRemoteMediaCache = shouldRequireRemoteMediaCache ?? null;
this.getYoutubeMediaSourceUrl = getYoutubeMediaSourceUrl ?? null;
this.overlayNotificationDismissCallback = overlayNotificationDismissCallback ?? null;
this.pendingYoutubeMediaQueue = this.createPendingYoutubeMediaQueue();
this.knownWordCache = this.createKnownWordCache(knownWordCacheStatePath);
this.pollingRunner = this.createPollingRunner();
@@ -379,8 +383,6 @@ export class AnkiIntegration {
getCachedMediaPath: this.getCachedMediaPath,
shouldRequireRemoteMediaCache: () => this.shouldRequireRemoteMediaCache?.() === true,
getSubtitleMediaRange: (context) => this.getSubtitleMediaRange(context),
getResolvedSentenceAudioFieldName: (noteInfo) =>
this.getResolvedSentenceAudioFieldName(noteInfo),
resolveConfiguredFieldName: (noteInfo, ...preferredNames) =>
this.resolveConfiguredFieldName(noteInfo, ...preferredNames),
mergeFieldValue: (existing, newValue, overwrite) =>
@@ -657,8 +659,6 @@ export class AnkiIntegration {
this.setCardTypeFields(updatedFields, availableFieldNames, cardKind),
resolveConfiguredFieldName: (noteInfo, ...preferredNames) =>
this.resolveConfiguredFieldName(noteInfo, ...preferredNames),
getResolvedSentenceAudioFieldName: (noteInfo) =>
this.getResolvedSentenceAudioFieldName(noteInfo),
getAnimatedImageLeadInSeconds: (noteInfo) => this.getAnimatedImageLeadInSeconds(noteInfo),
mergeFieldValue: (existing, newValue, overwrite) =>
this.mergeFieldValue(existing, newValue, overwrite),
@@ -835,6 +835,19 @@ export class AnkiIntegration {
};
}
private getSenrenConfig(): {
enabled: boolean;
fieldGrouping?: 'auto' | 'manual' | 'disabled';
deleteDuplicateInAuto?: boolean;
} {
const senren = this.config.isSenren;
return {
enabled: senren?.enabled === true,
fieldGrouping: senren?.fieldGrouping,
deleteDuplicateInAuto: senren?.deleteDuplicateInAuto,
};
}
private getEffectiveSentenceCardConfig(): {
model?: string;
sentenceField: string;
@@ -843,10 +856,27 @@ export class AnkiIntegration {
kikuEnabled: boolean;
kikuFieldGrouping: 'auto' | 'manual' | 'disabled';
kikuDeleteDuplicateInAuto: boolean;
senrenEnabled: boolean;
fieldGroupingProvider: 'kiku' | 'senren' | null;
fieldGroupingMode: 'auto' | 'manual' | 'disabled';
fieldGroupingDeleteDuplicateInAuto: boolean;
wordCardKind: WordCardKind;
} {
const lapis = this.getLapisConfig();
const kiku = this.getKikuConfig();
const senren = this.getSenrenConfig();
const kikuFieldGrouping = (kiku.fieldGrouping || 'disabled') as 'auto' | 'manual' | 'disabled';
const senrenFieldGrouping = (senren.fieldGrouping || 'auto') as 'auto' | 'manual' | 'disabled';
// Kiku and Senren are mutually exclusive; config resolution enforces it, and
// Kiku wins here too in case a runtime patch re-enables both.
const fieldGroupingProvider = kiku.enabled ? 'kiku' : senren.enabled ? 'senren' : null;
const fieldGroupingMode =
fieldGroupingProvider === 'kiku'
? kikuFieldGrouping
: fieldGroupingProvider === 'senren'
? senrenFieldGrouping
: 'disabled';
return {
model: lapis.sentenceCardModel,
@@ -854,8 +884,15 @@ export class AnkiIntegration {
audioField: 'SentenceAudio',
lapisEnabled: lapis.enabled,
kikuEnabled: kiku.enabled,
kikuFieldGrouping: (kiku.fieldGrouping || 'disabled') as 'auto' | 'manual' | 'disabled',
kikuFieldGrouping,
kikuDeleteDuplicateInAuto: kiku.deleteDuplicateInAuto !== false,
senrenEnabled: senren.enabled,
fieldGroupingProvider,
fieldGroupingMode,
fieldGroupingDeleteDuplicateInAuto:
fieldGroupingProvider === 'senren'
? senren.deleteDuplicateInAuto !== false
: kiku.deleteDuplicateInAuto !== false,
wordCardKind: resolveWordCardKindSetting(this.config.lapisKiku?.wordCardKind),
};
}
@@ -874,7 +911,7 @@ export class AnkiIntegration {
private async processNewCard(
noteId: number,
options?: { skipKikuFieldGrouping?: boolean },
options?: { skipFieldGrouping?: boolean },
): Promise<void> {
await this.noteUpdateWorkflow.execute(noteId, options);
}
@@ -1203,12 +1240,13 @@ export class AnkiIntegration {
private beginUpdateProgress(initialMessage: string): void {
if (!this.shouldUseOsdNotifications()) {
if (this.shouldUseOverlayNotifications()) {
this.overlayUpdateProgressActive = true;
this.overlayNotificationCallback?.({
id: 'anki-update-progress',
title: 'Anki update',
body: initialMessage,
variant: 'progress',
persistent: false,
persistent: true,
});
}
return;
@@ -1220,6 +1258,10 @@ export class AnkiIntegration {
private endUpdateProgress(): void {
if (!this.shouldUseOsdNotifications()) {
if (this.overlayUpdateProgressActive) {
this.overlayUpdateProgressActive = false;
this.overlayNotificationDismissCallback?.('anki-update-progress');
}
return;
}
endUpdateProgress(this.uiFeedbackState, (timer) => {
@@ -1243,18 +1285,20 @@ export class AnkiIntegration {
if (!this.shouldUseOsdNotifications()) {
this.updateInProgress = true;
if (this.shouldUseOverlayNotifications()) {
this.overlayUpdateProgressActive = true;
this.overlayNotificationCallback?.({
id: 'anki-update-progress',
title: 'Anki update',
body: initialMessage,
variant: 'progress',
persistent: false,
persistent: true,
});
}
try {
return await action();
} finally {
this.updateInProgress = false;
this.endUpdateProgress();
}
}
return withUpdateProgress(
@@ -1353,6 +1397,7 @@ export class AnkiIntegration {
: undefined;
if (shouldShowOverlayNotification && this.overlayNotificationCallback) {
this.overlayUpdateProgressActive = false;
this.overlayNotificationCallback({
id: 'anki-update-progress',
title: 'Anki Card Updated',
@@ -1496,7 +1541,7 @@ export class AnkiIntegration {
trackedDuplicateNoteIdsBeforeCreate: Set<number>,
): boolean {
const sentenceCardConfig = this.getEffectiveSentenceCardConfig();
if (!sentenceCardConfig.kikuEnabled || sentenceCardConfig.kikuFieldGrouping === 'disabled') {
if (sentenceCardConfig.fieldGroupingMode === 'disabled') {
return false;
}
@@ -1555,13 +1600,6 @@ export class AnkiIntegration {
return sentenceCardConfig.audioField || 'SentenceAudio';
}
private getResolvedSentenceAudioFieldName(noteInfo: NoteInfo): string | null {
return (
this.resolveNoteFieldName(noteInfo, this.getPreferredSentenceAudioFieldName()) ||
this.resolveConfiguredFieldName(noteInfo, this.config.fields?.audio)
);
}
private getConfiguredWordFieldName(): string {
return getConfiguredWordFieldName(this.config);
}
@@ -124,8 +124,7 @@ function createManualUpdateService(overrides: Partial<CardCreationDeps> = {}): {
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: false,
kikuFieldGrouping: 'disabled',
kikuDeleteDuplicateInAuto: false,
fieldGroupingMode: 'disabled',
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
@@ -143,7 +142,7 @@ function createManualUpdateService(overrides: Partial<CardCreationDeps> = {}): {
};
}
test('manual clipboard subtitle update replaces sentence audio without touching expression audio', async () => {
test('manual clipboard subtitle update replaces audio in the configured field', async () => {
const { service, updatedFields, mergeCalls, storedMedia } = createManualUpdateService();
await service.updateLastAddedFromClipboard('字幕');
@@ -151,14 +150,142 @@ test('manual clipboard subtitle update replaces sentence audio without touching
assert.equal(updatedFields.length, 1);
assert.equal(storedMedia.length, 1);
const audioValue = `[sound:${storedMedia[0]}]`;
assert.equal(updatedFields[0]?.SentenceAudio, audioValue);
assert.equal('ExpressionAudio' in updatedFields[0]!, false);
assert.equal(updatedFields[0]?.ExpressionAudio, audioValue);
assert.equal('SentenceAudio' in updatedFields[0]!, false);
assert.deepEqual(
mergeCalls.map((call) => call.overwrite),
[true],
);
});
test('manual clipboard word-card update uses configured fields with Lapis and Kiku enabled', async () => {
const { service, updatedFields } = createManualUpdateService({
getConfig: () =>
({
deck: 'Mining',
fields: {
word: 'Expression',
sentence: 'Context',
audio: 'ContextAudio',
},
media: {
generateAudio: true,
generateImage: false,
maxMediaDuration: 30,
},
behavior: {
overwriteAudio: false,
overwriteImage: false,
},
ai: false,
}) as AnkiConnectConfig,
client: {
addNote: async () => 0,
addTags: async () => undefined,
notesInfo: async () => [
{
noteId: 42,
fields: {
Expression: { value: '単語' },
Sentence: { value: '' },
SentenceAudio: { value: '' },
Context: { value: '' },
ContextAudio: { value: '' },
},
},
],
updateNoteFields: async (_noteId, fields) => {
updatedFields.push(fields);
},
storeMediaFile: async () => undefined,
findNotes: async () => [42],
retrieveMediaFile: async () => '',
},
getEffectiveSentenceCardConfig: () => ({
model: 'Sentence',
sentenceField: 'Sentence',
audioField: 'SentenceAudio',
lapisEnabled: true,
kikuEnabled: true,
fieldGroupingMode: 'disabled',
}),
});
await service.updateLastAddedFromClipboard('字幕');
assert.equal(updatedFields.length, 1);
assert.match(updatedFields[0]?.ContextAudio ?? '', /^\[sound:audio_\d+\.mp3\]$/);
assert.deepEqual(Object.keys(updatedFields[0] ?? {}).sort(), ['Context', 'ContextAudio']);
assert.equal(updatedFields[0]?.Context, '字幕');
});
test('audio-card action keeps Lapis and Kiku sentence fields', async () => {
const { service, updatedFields } = createManualUpdateService({
getConfig: () =>
({
deck: 'Mining',
fields: {
word: 'Expression',
sentence: 'Context',
audio: 'ContextAudio',
},
media: {
generateAudio: true,
generateImage: false,
maxMediaDuration: 30,
},
behavior: {},
ai: false,
}) as AnkiConnectConfig,
getMpvClient: () =>
({
currentVideoPath: '/video.mp4',
currentAudioStreamIndex: 0,
currentSubText: '字幕',
currentSubStart: 12,
currentSubEnd: 14,
}) as never,
client: {
addNote: async () => 0,
addTags: async () => undefined,
notesInfo: async () => [
{
noteId: 42,
fields: {
Expression: { value: '単語' },
Sentence: { value: '' },
SentenceAudio: { value: '' },
Context: { value: '' },
ContextAudio: { value: '' },
},
},
],
updateNoteFields: async (_noteId, fields) => {
updatedFields.push(fields);
},
storeMediaFile: async () => undefined,
findNotes: async () => [42],
retrieveMediaFile: async () => '',
},
getEffectiveSentenceCardConfig: () => ({
model: 'Sentence',
sentenceField: 'Sentence',
audioField: 'SentenceAudio',
lapisEnabled: true,
kikuEnabled: true,
fieldGroupingMode: 'disabled',
}),
});
await service.markLastCardAsAudioCard();
assert.equal(updatedFields.length, 1);
assert.equal(updatedFields[0]?.Sentence, '字幕');
assert.match(updatedFields[0]?.SentenceAudio ?? '', /^\[sound:audio_\d+\.mp3\]$/);
assert.equal('Context' in (updatedFields[0] ?? {}), false);
assert.equal('ContextAudio' in (updatedFields[0] ?? {}), false);
});
test('manual clipboard subtitle update marks Kiku word cards as word-and-sentence cards when enabled', async () => {
const { service, updatedFields } = createManualUpdateService({
getConfig: () =>
@@ -208,8 +335,7 @@ test('manual clipboard subtitle update marks Kiku word cards as word-and-sentenc
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: true,
kikuFieldGrouping: 'disabled',
kikuDeleteDuplicateInAuto: false,
fieldGroupingMode: 'disabled',
}),
setCardTypeFields,
});
@@ -225,7 +351,7 @@ test('manual clipboard subtitle update marks Kiku word cards as word-and-sentenc
});
});
test('manual clipboard subtitle update skips audio when sentence audio field is missing', async () => {
test('manual clipboard subtitle update uses configured audio when SentenceAudio is missing', async () => {
const { service, updatedFields, mergeCalls, storedMedia } = createManualUpdateService({
client: {
addNote: async () => 0,
@@ -255,8 +381,9 @@ test('manual clipboard subtitle update skips audio when sentence audio field is
assert.equal(storedMedia.length, 1);
assert.equal(updatedFields.length, 1);
assert.deepEqual(updatedFields[0], { Sentence: '字幕' });
assert.equal(mergeCalls.length, 0);
assert.match(updatedFields[0]?.ExpressionAudio ?? '', /^\[sound:audio_\d+\.mp3\]$/);
assert.equal(updatedFields[0]?.Sentence, '字幕');
assert.equal(mergeCalls.length, 1);
});
test('manual clipboard subtitle update uses resolved mpv stream URLs for remote media', async () => {
@@ -117,8 +117,7 @@ test('sentence card writes generated audio only to sentence audio field', async
audioField: 'SentenceAudio',
lapisEnabled: true,
kikuEnabled: false,
kikuFieldGrouping: 'disabled',
kikuDeleteDuplicateInAuto: false,
fieldGroupingMode: 'disabled',
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
+8 -16
View File
@@ -69,8 +69,7 @@ test('CardCreationService counts locally created sentence cards', async () => {
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: false,
kikuFieldGrouping: 'disabled',
kikuDeleteDuplicateInAuto: false,
fieldGroupingMode: 'disabled',
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
@@ -168,8 +167,7 @@ test('CardCreationService keeps updating after trackLastAddedNoteId throws', asy
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: false,
kikuFieldGrouping: 'disabled',
kikuDeleteDuplicateInAuto: false,
fieldGroupingMode: 'disabled',
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
@@ -267,8 +265,7 @@ test('CardCreationService keeps updating after recordCardsMinedCallback throws',
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: false,
kikuFieldGrouping: 'disabled',
kikuDeleteDuplicateInAuto: false,
fieldGroupingMode: 'disabled',
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
@@ -387,8 +384,7 @@ test('CardCreationService uses stream-open-filename for remote media generation'
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: false,
kikuFieldGrouping: 'disabled',
kikuDeleteDuplicateInAuto: false,
fieldGroupingMode: 'disabled',
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
@@ -490,8 +486,7 @@ test('CardCreationService does not use mpv stream indexes for ready cached YouTu
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: false,
kikuFieldGrouping: 'disabled',
kikuDeleteDuplicateInAuto: false,
fieldGroupingMode: 'disabled',
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
@@ -629,8 +624,7 @@ test('CardCreationService queues YouTube media when required cache is not ready'
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: false,
kikuFieldGrouping: 'disabled',
kikuDeleteDuplicateInAuto: false,
fieldGroupingMode: 'disabled',
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
@@ -728,8 +722,7 @@ test('CardCreationService tracks pre-add duplicate note ids for kiku sentence ca
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: true,
kikuFieldGrouping: 'manual',
kikuDeleteDuplicateInAuto: false,
fieldGroupingMode: 'manual',
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
@@ -817,8 +810,7 @@ test('CardCreationService does not track duplicate ids when pre-add lookup retur
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: true,
kikuFieldGrouping: 'manual',
kikuDeleteDuplicateInAuto: false,
fieldGroupingMode: 'manual',
}),
getFallbackDurationSeconds: () => 10,
appendKnownWordsFromNoteInfo: () => undefined,
+13 -25
View File
@@ -132,8 +132,7 @@ interface CardCreationDeps {
audioField: string;
lapisEnabled: boolean;
kikuEnabled: boolean;
kikuFieldGrouping: 'auto' | 'manual' | 'disabled';
kikuDeleteDuplicateInAuto: boolean;
fieldGroupingMode: 'auto' | 'manual' | 'disabled';
wordCardKind?: WordCardKind;
};
getFallbackDurationSeconds: () => number;
@@ -260,9 +259,16 @@ export class CardCreationService {
fields,
this.deps.getConfig(),
);
const sentenceAudioField = this.getResolvedSentenceOnlyAudioFieldName(noteInfo);
const config = this.deps.getConfig();
const sentenceAudioField = this.deps.resolveConfiguredFieldName(
noteInfo,
config.fields?.audio ?? DEFAULT_ANKI_CONNECT_CONFIG.fields.audio,
);
const sentenceCardConfig = this.deps.getEffectiveSentenceCardConfig();
const sentenceField = sentenceCardConfig.sentenceField;
const sentenceField = this.deps.resolveConfiguredFieldName(
noteInfo,
config.fields?.sentence ?? DEFAULT_ANKI_CONNECT_CONFIG.fields.sentence,
);
const sentence = blocks.join(' ');
const updatedFields: Record<string, string> = {};
@@ -284,7 +290,6 @@ export class CardCreationService {
`Clipboard update: timing range ${rangeStart.toFixed(2)}s - ${rangeEnd.toFixed(2)}s`,
);
const config = this.deps.getConfig();
const generateAudio = shouldGenerateAudio(config);
const generateImage = shouldGenerateImage(config);
const mediaResolverOptions = this.getMediaResolverOptions();
@@ -457,13 +462,13 @@ export class CardCreationService {
this.deps.setCardTypeFields(updatedFields, Object.keys(noteInfo.fields), 'audio');
const sentenceField = this.deps.getConfig().fields?.sentence;
const sentenceCardConfig = this.deps.getEffectiveSentenceCardConfig();
const sentenceField = sentenceCardConfig.sentenceField;
if (sentenceField) {
const processedSentence = this.deps.processSentence(mpvClient.currentSubText, fields);
updatedFields[sentenceField] = processedSentence;
}
const sentenceCardConfig = this.deps.getEffectiveSentenceCardConfig();
const audioFieldName = sentenceCardConfig.audioField;
try {
const audioFilename = this.generateAudioFilename();
@@ -632,8 +637,7 @@ export class CardCreationService {
).trim();
let duplicateNoteIds: number[] = [];
if (
sentenceCardConfig.kikuEnabled &&
sentenceCardConfig.kikuFieldGrouping !== 'disabled' &&
sentenceCardConfig.fieldGroupingMode !== 'disabled' &&
pendingExpressionText &&
this.deps.findDuplicateNoteIds
) {
@@ -806,22 +810,6 @@ export class CardCreationService {
}
}
private getResolvedSentenceAudioFieldName(noteInfo: CardCreationNoteInfo): string | null {
return (
this.deps.resolveNoteFieldName(
noteInfo,
this.deps.getEffectiveSentenceCardConfig().audioField || 'SentenceAudio',
) || this.deps.resolveConfiguredFieldName(noteInfo, this.deps.getConfig().fields?.audio)
);
}
private getResolvedSentenceOnlyAudioFieldName(noteInfo: CardCreationNoteInfo): string | null {
return this.deps.resolveNoteFieldName(
noteInfo,
this.deps.getEffectiveSentenceCardConfig().audioField || 'SentenceAudio',
);
}
private createPendingNoteInfo(fields: Record<string, string>): CardCreationNoteInfo {
return {
noteId: -1,
@@ -26,6 +26,7 @@ function createCollaborator(
miscInfoValue?: string;
};
warnings?: Array<{ fieldName: string; reason: string; detail?: string }>;
fieldGroupingProvider?: 'kiku' | 'senren' | null;
} = {},
) {
const warnings = options.warnings ?? [];
@@ -46,6 +47,8 @@ function createCollaborator(
getEffectiveSentenceCardConfig: () => ({
sentenceField: 'Sentence',
audioField: 'SentenceAudio',
fieldGroupingProvider:
options.fieldGroupingProvider === undefined ? 'kiku' : options.fieldGroupingProvider,
}),
getCurrentSubtitleText: () => options.currentSubtitleText,
resolveFieldName,
@@ -251,7 +254,218 @@ test('computeFieldGroupingMergedFields uses generated media only when includeGen
assert.equal(withMedia.MiscInfo, '<span data-group-id="11">generated misc</span>');
});
test('computeFieldGroupingMergedFields clears SentenceFurigana when either note lacks it', async () => {
test('computeFieldGroupingMergedFields merges Senren notes into scene-switching markup', async () => {
const { collaborator } = createCollaborator({ fieldGroupingProvider: 'senren' });
const merged = await collaborator.computeFieldGroupingMergedFields(
300,
200,
makeNote(300, {
word: '語',
sentence: '<span class="group">前<span class="highlight">語</span>後</span>',
sentenceAudio: '[sound:original.opus]',
picture: '<img src="original.webp">',
miscInfo: '<span class="group">Show EP1 (0:01:00)</span>',
}),
makeNote(200, {
word: '語',
sentence: '<span class="group">次<span class="highlight">語</span>文</span>',
sentenceAudio: '[sound:new.opus]',
picture: '<img src="new.webp">',
miscInfo: 'Show EP2 (0:02:00)',
}),
false,
);
assert.equal(
merged.sentence,
'<span class="group">前<span class="highlight">語</span>後</span>' +
'<span class="group2">次<span class="highlight">語</span>文</span>',
);
assert.equal(merged.sentenceAudio, '[sound:original.opus][sound:new.opus]');
assert.equal(merged.picture, '<img src="original.webp"><img src="new.webp">');
assert.equal(
merged.miscInfo,
'<span class="group">Show EP1 (0:01:00)</span><span class="group2">Show EP2 (0:02:00)</span>',
);
});
test('Senren merge warns for invalid source audio when kept audio is empty', async () => {
const warnings: Array<{ fieldName: string; reason: string; detail?: string }> = [];
const { collaborator } = createCollaborator({
fieldGroupingProvider: 'senren',
warnings,
});
const merged = await collaborator.computeFieldGroupingMergedFields(
300,
200,
makeNote(300, { SentenceAudio: '' }),
makeNote(200, { SentenceAudio: 'invalid audio' }),
false,
);
assert.equal(merged.SentenceAudio, 'invalid audio');
assert.deepEqual(warnings, [
{
fieldName: 'SentenceAudio',
reason: 'missing-sound-tag',
detail: undefined,
},
]);
});
test('Senren merge wraps ungrouped legacy content and preserves numbered groups', async () => {
const { collaborator } = createCollaborator({ fieldGroupingProvider: 'senren' });
const merged = await collaborator.computeFieldGroupingMergedFields(
300,
200,
makeNote(300, {
sentence: 'plain legacy sentence',
sentenceAudio: '[sound:a.opus][sound:b.opus]',
miscInfo: '<span class="group2">pinned</span> stray text',
}),
makeNote(200, {
sentence: '<span class="group">new sentence</span>',
sentenceAudio: '[sound:c.opus]',
miscInfo: '<span class="group">new misc</span>',
}),
false,
);
assert.equal(
merged.sentence,
'<span class="group">plain legacy sentence</span><span class="group3">new sentence</span>',
);
assert.equal(merged.sentenceAudio, '[sound:a.opus][sound:b.opus][sound:c.opus]');
assert.equal(
merged.miscInfo,
'<span class="group2">pinned</span><span class="group">stray text</span>' +
'<span class="group3">new misc</span>',
);
});
test('Senren merge rebases numbered groups from an appended source note', async () => {
const { collaborator } = createCollaborator({ fieldGroupingProvider: 'senren' });
const merged = await collaborator.computeFieldGroupingMergedFields(
300,
200,
makeNote(300, {
sentenceAudio: '[sound:keep-a.opus][sound:keep-b.opus]',
miscInfo: '<span class="group">keep one</span><span class="group">keep two</span>',
}),
makeNote(200, {
sentenceAudio: '[sound:source-a.opus][sound:source-b.opus]',
miscInfo: '<span class="group2">source two</span>',
}),
false,
);
assert.equal(
merged.sentenceAudio,
'[sound:keep-a.opus][sound:keep-b.opus][sound:source-a.opus][sound:source-b.opus]',
);
assert.equal(
merged.miscInfo,
'<span class="group">keep one</span><span class="group">keep two</span>' +
'<span class="group4">source two</span>',
);
});
test('Senren merge rebases plain source groups after empty and sparse kept fields', async () => {
const { collaborator } = createCollaborator({ fieldGroupingProvider: 'senren' });
const merged = await collaborator.computeFieldGroupingMergedFields(
300,
200,
makeNote(300, {
sentenceAudio: '[sound:keep-a.opus][sound:keep-b.opus]',
sentence: '',
miscInfo: '<span class="group">keep first</span>',
}),
makeNote(200, {
sentenceAudio: '[sound:source-a.opus][sound:source-b.opus]',
sentence: '<span class="group">source first</span>',
miscInfo: '<span class="group">source first</span><span class="group2">source second</span>',
}),
false,
);
assert.equal(merged.sentence, '<span class="group3">source first</span>');
assert.equal(
merged.miscInfo,
'<span class="group">keep first</span><span class="group3">source first</span>' +
'<span class="group4">source second</span>',
);
});
test('Senren merge keeps ungrouped text in place around an existing group span', async () => {
const { collaborator } = createCollaborator({ fieldGroupingProvider: 'senren' });
const merged = await collaborator.computeFieldGroupingMergedFields(
300,
200,
makeNote(300, {
miscInfo: 'leading<span class="group">middle</span>trailing',
sentenceAudio: '[sound:a.opus][sound:b.opus][sound:c.opus]',
}),
makeNote(200, {
miscInfo: '<span class="group">appended</span>',
sentenceAudio: '[sound:d.opus]',
}),
false,
);
// Order must follow the source field, and the two ungrouped runs must stay separate.
assert.equal(
merged.miscInfo,
'<span class="group">leading</span><span class="group">middle</span>' +
'<span class="group">trailing</span><span class="group4">appended</span>',
);
assert.equal(merged.sentenceAudio, '[sound:a.opus][sound:b.opus][sound:c.opus][sound:d.opus]');
});
test('Senren merge closes unclosed group spans so later scenes stay siblings', async () => {
const { collaborator } = createCollaborator({ fieldGroupingProvider: 'senren' });
const merged = await collaborator.computeFieldGroupingMergedFields(
300,
200,
makeNote(300, { miscInfo: '<span class="group">a<span class="highlight">b' }),
makeNote(200, { miscInfo: '<span class="group">next</span>' }),
false,
);
assert.equal(
merged.miscInfo,
'<span class="group">a<span class="highlight">b</span></span><span class="group">next</span>',
);
const openTags = merged.miscInfo!.match(/<span\b/g)?.length ?? 0;
const closeTags = merged.miscInfo!.match(/<\/span>/g)?.length ?? 0;
assert.equal(openTags, closeTags);
});
test('Senren merge closes unclosed trailing markup before appending later scenes', async () => {
const { collaborator } = createCollaborator({ fieldGroupingProvider: 'senren' });
const merged = await collaborator.computeFieldGroupingMergedFields(
300,
200,
makeNote(300, { miscInfo: 'leading<span class="highlight">tail' }),
makeNote(200, { miscInfo: '<span class="group">next</span>' }),
false,
);
assert.equal(
merged.miscInfo,
'<span class="group">leading<span class="highlight">tail</span></span>' +
'<span class="group">next</span>',
);
});
test('Kiku merge clears SentenceFurigana when either note lacks it', async () => {
const { collaborator } = createCollaborator();
const merged = await collaborator.computeFieldGroupingMergedFields(
@@ -268,3 +482,21 @@ test('computeFieldGroupingMergedFields clears SentenceFurigana when either note
assert.equal(merged.SentenceFurigana, '');
});
test('Senren merge keeps duplicate SentenceFurigana when the kept field is empty', async () => {
const { collaborator } = createCollaborator({ fieldGroupingProvider: 'senren' });
const merged = await collaborator.computeFieldGroupingMergedFields(
300,
200,
makeNote(300, {
SentenceFurigana: '',
}),
makeNote(200, {
SentenceFurigana: 'duplicate furigana',
}),
false,
);
assert.equal(merged.SentenceFurigana, '<span class="group">duplicate furigana</span>');
});
+154 -1
View File
@@ -19,6 +19,7 @@ interface FieldGroupingMergeDeps {
getEffectiveSentenceCardConfig: () => {
sentenceField: string;
audioField: string;
fieldGroupingProvider: 'kiku' | 'senren' | null;
};
getCurrentSubtitleText: () => string | undefined;
resolveFieldName: (availableFieldNames: string[], preferredName: string) => string | null;
@@ -78,6 +79,13 @@ export class FieldGroupingMergeCollaborator {
const configuredWordField = getConfiguredWordFieldName(config);
const groupableFields = this.getGroupableFieldNames();
const keepFieldNames = Object.keys(keepNoteInfo.fields);
const sentenceCardConfig = this.deps.getEffectiveSentenceCardConfig();
const senrenSourceSceneOffset =
sentenceCardConfig.fieldGroupingProvider === 'senren'
? this.countSenrenAudioScenes(
this.getResolvedFieldValue(keepNoteInfo, sentenceCardConfig.audioField),
)
: 0;
const sourceFields: Record<string, string> = {};
const resolvedKeepFieldByPreferred = new Map<string, string>();
for (const preferredFieldName of groupableFields) {
@@ -154,14 +162,18 @@ export class FieldGroupingMergeCollaborator {
if (!existingValue.trim() && !newValue.trim()) continue;
if (keepFieldNormalized === 'sentencefurigana') {
const hasBothValues = existingValue.trim().length > 0 && newValue.trim().length > 0;
const usesSenrenGrouping =
this.deps.getEffectiveSentenceCardConfig().fieldGroupingProvider === 'senren';
mergedFields[keepFieldName] =
existingValue.trim() && newValue.trim()
hasBothValues || usesSenrenGrouping
? this.applyFieldGrouping(
existingValue,
newValue,
keepNoteId,
deleteNoteId,
keepFieldName,
senrenSourceSceneOffset,
)
: '';
continue;
@@ -174,6 +186,7 @@ export class FieldGroupingMergeCollaborator {
keepNoteId,
deleteNoteId,
keepFieldName,
senrenSourceSceneOffset,
);
} else if (existingValue.trim() && newValue.trim()) {
mergedFields[keepFieldName] = this.applyFieldGrouping(
@@ -182,6 +195,7 @@ export class FieldGroupingMergeCollaborator {
keepNoteId,
deleteNoteId,
keepFieldName,
senrenSourceSceneOffset,
);
} else {
if (!newValue.trim()) continue;
@@ -342,13 +356,152 @@ export class FieldGroupingMergeCollaborator {
return [...entries].sort((a, b) => b.groupId - a.groupId);
}
private isSentenceAudioField(fieldName: string): boolean {
const normalized = fieldName.toLowerCase();
const audioField = (
this.deps.getEffectiveSentenceCardConfig().audioField || 'sentenceaudio'
).toLowerCase();
return normalized === 'sentenceaudio' || normalized === audioField;
}
private isSenrenGroupOpenTag(openTag: string): boolean {
const classMatch =
openTag.match(/class\s*=\s*"([^"]*)"/i) || openTag.match(/class\s*=\s*'([^']*)'/i);
if (!classMatch) return false;
// Senren's templates match class tokens case-sensitively (/^group\d*$/).
return classMatch[1]!.split(/\s+/).some((token) => /^group\d*$/.test(token));
}
private countSenrenAudioScenes(value: string): number {
const soundEntries = value.match(/\[sound:[^\]]+\]/g)?.length ?? 0;
if (soundEntries > 0) return soundEntries;
return this.parseSenrenSceneEntries(value).length;
}
private rebaseSenrenGroup(entry: string, sceneOffset: number, sourceEntryIndex: number): string {
if (sceneOffset <= 0) return entry;
return entry.replace(
/^(\s*<span\b[^>]*?\bclass\s*=\s*)(["'])([^"']*)\2/i,
(_match: string, prefix: string, quote: string, rawClasses: string) => {
const classes = rawClasses
.split(/(\s+)/)
.map((classToken) => {
if (classToken === 'group') {
return `group${sceneOffset + sourceEntryIndex + 1}`;
}
const groupMatch = classToken.match(/^group(\d+)$/);
if (!groupMatch) return classToken;
const targetScene = Number(groupMatch[1]);
if (!Number.isSafeInteger(targetScene) || targetScene <= 0) return classToken;
return `group${targetScene + sceneOffset}`;
})
.join('');
return `${prefix}${quote}${classes}${quote}`;
},
);
}
/**
* Splits a Senren field into ordered scene entries. Top-level
* `<span class="group">`/`"groupN"` spans are kept verbatim (nested markup like
* `<span class="highlight">` included); ungrouped runs are wrapped in a group
* span at their original position, because Senren discards anything outside a
* group span once scene switching activates.
*/
private parseSenrenSceneEntries(value: string): string[] {
const tokenRegex = /<span\b[^>]*>|<\/span>/gi;
const entries: string[] = [];
const pushUngrouped = (raw: string): void => {
const text = raw.replace(/<br\s*\/?>/gi, ' ').trim();
if (text) entries.push(`<span class="group">${text}</span>`);
};
let cursor = 0;
let depth = 0;
let entryStart = -1;
let match;
while ((match = tokenRegex.exec(value)) !== null) {
const token = match[0]!;
if (token[1] !== '/') {
if (depth === 0 && this.isSenrenGroupOpenTag(token)) {
pushUngrouped(value.slice(cursor, match.index));
entryStart = match.index;
cursor = match.index;
}
depth += 1;
} else {
depth = Math.max(0, depth - 1);
if (depth === 0 && entryStart !== -1) {
const end = match.index + token.length;
entries.push(value.slice(entryStart, end));
entryStart = -1;
cursor = end;
}
}
}
if (entryStart !== -1) {
// Unclosed group span: close every span still open (the group and any nested
// markup) so the following scenes are siblings rather than nested inside it.
entries.push(`${value.slice(entryStart)}${'</span>'.repeat(depth)}`);
} else {
pushUngrouped(`${value.slice(cursor)}${'</span>'.repeat(depth)}`);
}
return entries;
}
/**
* Merges two notes' field values in Senren's scene-switching format. Scenes are
* appended in order (existing first, never resorted) so indices stay aligned
* across sentence/picture/miscInfo with the sentenceAudio entries, which alone
* drive Senren's scene count.
*/
private applySenrenFieldGrouping(
existingValue: string,
newValue: string,
fieldName: string,
sourceSceneOffset: number,
): string {
if (this.isPictureField(fieldName)) {
const tags = [...this.extractImageTags(existingValue), ...this.extractImageTags(newValue)];
if (tags.length === 0) return existingValue || newValue;
return tags.join('');
}
if (this.isSentenceAudioField(fieldName)) {
const existing = existingValue.trim();
const added = newValue.trim();
if (added && !/\[sound:[^\]]+\]/.test(added)) {
this.deps.warnFieldParseOnce(fieldName, 'missing-sound-tag');
}
if (!existing || !added) return existing || added;
return existing + added;
}
const sourceEntries = this.parseSenrenSceneEntries(newValue).map((entry, sourceEntryIndex) =>
this.rebaseSenrenGroup(entry, sourceSceneOffset, sourceEntryIndex),
);
const merged = [...this.parseSenrenSceneEntries(existingValue), ...sourceEntries];
if (merged.length === 0) return existingValue || newValue;
return merged.join('');
}
private applyFieldGrouping(
existingValue: string,
newValue: string,
keepGroupId: number,
sourceGroupId: number,
fieldName: string,
senrenSourceSceneOffset: number,
): string {
if (this.deps.getEffectiveSentenceCardConfig().fieldGroupingProvider === 'senren') {
return this.applySenrenFieldGrouping(
existingValue,
newValue,
fieldName,
senrenSourceSceneOffset,
);
}
if (this.shouldUseStrictSpanGrouping(fieldName)) {
if (this.isPictureField(fieldName)) {
const keepEntries = this.parsePictureEntries(existingValue, keepGroupId);
@@ -71,7 +71,7 @@ function createWorkflowHarness() {
getEffectiveSentenceCardConfig: () => ({
sentenceField: 'Sentence',
audioField: 'SentenceAudio',
kikuDeleteDuplicateInAuto: true,
fieldGroupingDeleteDuplicateInAuto: true,
}),
getCurrentSubtitleText: () => 'subtitle-text',
getFieldGroupingCallback: (): FieldGroupingCallback | null => {
@@ -24,7 +24,7 @@ export interface FieldGroupingWorkflowDeps {
getEffectiveSentenceCardConfig: () => {
sentenceField: string;
audioField: string;
kikuDeleteDuplicateInAuto: boolean;
fieldGroupingDeleteDuplicateInAuto: boolean;
};
getCurrentSubtitleText: () => string | undefined;
getFieldGroupingCallback:
@@ -75,7 +75,7 @@ export class FieldGroupingWorkflow {
originalNoteId,
newNoteId,
this.getExpression(newNoteInfo),
sentenceCardConfig.kikuDeleteDuplicateInAuto,
sentenceCardConfig.fieldGroupingDeleteDuplicateInAuto,
);
} catch (error) {
this.deps.logError('Field grouping auto merge failed:', (error as Error).message);
+14 -18
View File
@@ -21,14 +21,14 @@ function createHarness(
manualHandled?: boolean;
expression?: string | null;
currentSentenceImageField?: string | undefined;
onProcessNewCard?: (noteId: number, options?: { skipKikuFieldGrouping?: boolean }) => void;
onProcessNewCard?: (noteId: number, options?: { skipFieldGrouping?: boolean }) => void;
} = {},
) {
const calls: string[] = [];
const findNotesQueries: Array<{ query: string; maxRetries?: number }> = [];
const noteInfoRequests: number[][] = [];
const duplicateRequests: Array<{ expression: string; excludeNoteId: number }> = [];
const processCalls: Array<{ noteId: number; options?: { skipKikuFieldGrouping?: boolean } }> = [];
const processCalls: Array<{ noteId: number; options?: { skipFieldGrouping?: boolean } }> = [];
const autoCalls: Array<{ originalNoteId: number; newNoteId: number; expression: string }> = [];
const manualCalls: Array<{ originalNoteId: number; newNoteId: number; expression: string }> = [];
@@ -46,9 +46,8 @@ function createHarness(
sentenceField: 'Sentence',
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: options.kikuEnabled ?? true,
kikuFieldGrouping: options.kikuFieldGrouping ?? 'auto',
kikuDeleteDuplicateInAuto: true,
fieldGroupingProvider: (options.kikuEnabled ?? true) ? ('kiku' as const) : null,
fieldGroupingMode: options.kikuFieldGrouping ?? 'auto',
}),
isUpdateInProgress: () => false,
getDeck: options.deck ? () => options.deck : undefined,
@@ -134,7 +133,7 @@ test('triggerFieldGroupingForLastAddedCard stops when kiku mode is disabled', as
await harness.service.triggerFieldGroupingForLastAddedCard();
assert.deepEqual(harness.calls, ['osd:Kiku mode is not enabled']);
assert.deepEqual(harness.calls, ['osd:Field grouping requires Kiku or Senren mode']);
assert.equal(harness.findNotesQueries.length, 0);
});
@@ -143,7 +142,7 @@ test('triggerFieldGroupingForLastAddedCard stops when field grouping is disabled
await harness.service.triggerFieldGroupingForLastAddedCard();
assert.deepEqual(harness.calls, ['osd:Kiku field grouping is disabled']);
assert.deepEqual(harness.calls, ['osd:Field grouping is disabled']);
assert.equal(harness.findNotesQueries.length, 0);
});
@@ -155,9 +154,8 @@ test('triggerFieldGroupingForLastAddedCard stops when an update is already in pr
sentenceField: 'Sentence',
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: true,
kikuFieldGrouping: 'auto',
kikuDeleteDuplicateInAuto: true,
fieldGroupingProvider: 'kiku' as const,
fieldGroupingMode: 'auto' as const,
}),
isUpdateInProgress: () => true,
withUpdateProgress: async () => {
@@ -266,7 +264,7 @@ test('triggerFieldGroupingForLastAddedCard prefers tracked duplicate note ids be
});
test('triggerFieldGroupingForLastAddedCard refreshes the card when configured fields are missing', async () => {
const processCalls: Array<{ noteId: number; options?: { skipKikuFieldGrouping?: boolean } }> = [];
const processCalls: Array<{ noteId: number; options?: { skipFieldGrouping?: boolean } }> = [];
const harness = createHarness({
noteIds: [11],
notesInfo: [
@@ -298,7 +296,7 @@ test('triggerFieldGroupingForLastAddedCard refreshes the card when configured fi
await harness.service.triggerFieldGroupingForLastAddedCard();
assert.deepEqual(processCalls, [{ noteId: 11, options: { skipKikuFieldGrouping: true } }]);
assert.deepEqual(processCalls, [{ noteId: 11, options: { skipFieldGrouping: true } }]);
assert.deepEqual(harness.manualCalls, []);
});
@@ -352,9 +350,8 @@ test('buildFieldGroupingPreview returns merged compact and full previews', async
sentenceField: 'Sentence',
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: true,
kikuFieldGrouping: 'auto',
kikuDeleteDuplicateInAuto: true,
fieldGroupingProvider: 'kiku' as const,
fieldGroupingMode: 'auto' as const,
}),
isUpdateInProgress: () => false,
withUpdateProgress: async (_message, action) => action(),
@@ -417,9 +414,8 @@ test('buildFieldGroupingPreview reports missing notes cleanly', async () => {
sentenceField: 'Sentence',
audioField: 'SentenceAudio',
lapisEnabled: false,
kikuEnabled: true,
kikuFieldGrouping: 'auto',
kikuDeleteDuplicateInAuto: true,
fieldGroupingProvider: 'kiku' as const,
fieldGroupingMode: 'auto' as const,
}),
isUpdateInProgress: () => false,
withUpdateProgress: async (_message, action) => action(),
+9 -10
View File
@@ -20,9 +20,8 @@ interface FieldGroupingDeps {
sentenceField: string;
audioField: string;
lapisEnabled: boolean;
kikuEnabled: boolean;
kikuFieldGrouping: 'auto' | 'manual' | 'disabled';
kikuDeleteDuplicateInAuto: boolean;
fieldGroupingProvider: 'kiku' | 'senren' | null;
fieldGroupingMode: 'auto' | 'manual' | 'disabled';
};
isUpdateInProgress: () => boolean;
getDeck?: () => string | undefined;
@@ -46,7 +45,7 @@ interface FieldGroupingDeps {
noteInfo: FieldGroupingNoteInfo,
configuredFieldNames: (string | undefined)[],
) => boolean;
processNewCard: (noteId: number, options?: { skipKikuFieldGrouping?: boolean }) => Promise<void>;
processNewCard: (noteId: number, options?: { skipFieldGrouping?: boolean }) => Promise<void>;
getSentenceCardImageFieldName: () => string | undefined;
resolveFieldName: (availableFieldNames: string[], preferredName: string) => string | null;
computeFieldGroupingMergedFields: (
@@ -76,12 +75,12 @@ export class FieldGroupingService {
async triggerFieldGroupingForLastAddedCard(): Promise<void> {
const sentenceCardConfig = this.deps.getEffectiveSentenceCardConfig();
if (!sentenceCardConfig.kikuEnabled) {
this.deps.showOsdNotification('Kiku mode is not enabled');
if (sentenceCardConfig.fieldGroupingProvider === null) {
this.deps.showOsdNotification('Field grouping requires Kiku or Senren mode');
return;
}
if (sentenceCardConfig.kikuFieldGrouping === 'disabled') {
this.deps.showOsdNotification('Kiku field grouping is disabled');
if (sentenceCardConfig.fieldGroupingMode === 'disabled') {
this.deps.showOsdNotification('Field grouping is disabled');
return;
}
@@ -134,7 +133,7 @@ export class FieldGroupingService {
])
) {
await this.deps.processNewCard(noteId, {
skipKikuFieldGrouping: true,
skipFieldGrouping: true,
});
}
@@ -147,7 +146,7 @@ export class FieldGroupingService {
const noteInfo = refreshedInfo[0]!;
if (sentenceCardConfig.kikuFieldGrouping === 'auto') {
if (sentenceCardConfig.fieldGroupingMode === 'auto') {
await this.deps.handleFieldGroupingAuto(
duplicateNoteId,
noteId,
@@ -58,7 +58,7 @@ function createWorkflowHarness() {
sentenceField: 'Sentence',
lapisEnabled: false,
kikuEnabled: false,
kikuFieldGrouping: 'disabled' as const,
fieldGroupingMode: 'disabled' as const,
}),
appendKnownWordsFromNoteInfo: (_noteInfo: NoteUpdateWorkflowNoteInfo) => undefined,
extractFields: (fields: Record<string, { value: string }>) => {
@@ -80,7 +80,6 @@ function createWorkflowHarness() {
const names = Object.keys(noteInfo.fields);
return names.find((name) => name.toLowerCase() === preferred.toLowerCase()) ?? null;
},
getResolvedSentenceAudioFieldName: () => null,
getAnimatedImageLeadInSeconds: async () => 0,
mergeFieldValue: (_existing: string, next: string, _overwrite: boolean) => next,
generateAudioFilename: () => 'audio_1.mp3',
@@ -120,6 +119,49 @@ test('NoteUpdateWorkflow updates sentence field and emits notification', async (
assert.equal(harness.notifications.length, 1);
});
test('NoteUpdateWorkflow uses configured fields for word-card enrichment with Lapis and Kiku enabled', async () => {
const harness = createWorkflowHarness();
harness.deps.getConfig = () => ({
fields: {
sentence: 'Context',
audio: 'ContextAudio',
},
media: {
generateAudio: true,
generateImage: false,
},
behavior: {},
});
harness.deps.getEffectiveSentenceCardConfig = () => ({
sentenceField: 'Sentence',
lapisEnabled: true,
kikuEnabled: true,
fieldGroupingMode: 'disabled',
});
harness.deps.client.notesInfo = async () =>
[
{
noteId: 42,
fields: {
Expression: { value: 'taberu' },
Sentence: { value: '' },
SentenceAudio: { value: '' },
Context: { value: '' },
ContextAudio: { value: '' },
},
},
] satisfies NoteUpdateWorkflowNoteInfo[];
harness.deps.generateAudio = async () => Buffer.from('audio');
await harness.workflow.execute(42);
assert.equal(harness.updates.length, 1);
assert.deepEqual(harness.updates[0]?.fields, {
Context: 'subtitle-text',
ContextAudio: '[sound:audio_1.mp3]',
});
});
test('NoteUpdateWorkflow updates sentence furigana when highlight processor changes it', async () => {
const harness = createWorkflowHarness();
harness.deps.client.notesInfo = async () =>
@@ -151,7 +193,7 @@ test('NoteUpdateWorkflow marks enriched Kiku word cards as word-and-sentence car
sentenceField: 'Sentence',
lapisEnabled: false,
kikuEnabled: true,
kikuFieldGrouping: 'manual',
fieldGroupingMode: 'manual',
});
harness.deps.client.notesInfo = async () =>
[
@@ -184,7 +226,7 @@ test('NoteUpdateWorkflow marks the configured word card kind instead of word-and
sentenceField: 'Sentence',
lapisEnabled: false,
kikuEnabled: true,
kikuFieldGrouping: 'manual',
fieldGroupingMode: 'manual',
wordCardKind: 'click',
});
harness.deps.client.notesInfo = async () =>
@@ -220,7 +262,7 @@ test('NoteUpdateWorkflow leaves card type flags alone when the word card kind is
sentenceField: 'Sentence',
lapisEnabled: false,
kikuEnabled: true,
kikuFieldGrouping: 'manual',
fieldGroupingMode: 'manual',
wordCardKind: 'none',
});
harness.deps.client.notesInfo = async () =>
@@ -275,7 +317,7 @@ test('NoteUpdateWorkflow preserves explicit sentence card type during sentence e
sentenceField: 'Sentence',
lapisEnabled: true,
kikuEnabled: false,
kikuFieldGrouping: 'disabled',
fieldGroupingMode: 'disabled',
});
harness.deps.client.notesInfo = async () =>
[
@@ -318,7 +360,7 @@ test('NoteUpdateWorkflow updates note before auto field grouping merge', async (
sentenceField: 'Sentence',
lapisEnabled: false,
kikuEnabled: true,
kikuFieldGrouping: 'auto',
fieldGroupingMode: 'auto',
});
harness.deps.findDuplicateNote = async () => 99;
harness.deps.client.notesInfo = async () => {
@@ -432,6 +474,7 @@ test('NoteUpdateWorkflow uses subtitle sidebar context for sentence media timing
harness.deps.getConfig = () => ({
fields: {
sentence: 'Sentence',
audio: 'SentenceAudio',
image: 'Picture',
miscInfo: 'MiscInfo',
},
@@ -444,7 +487,6 @@ test('NoteUpdateWorkflow uses subtitle sidebar context for sentence media timing
});
harness.deps.getCurrentSubtitleText = () => 'current primary line';
harness.deps.getCurrentSubtitleStart = () => 20;
harness.deps.getResolvedSentenceAudioFieldName = () => 'SentenceAudio';
harness.deps.generateAudio = async (context?: SubtitleMiningContext) => {
audioContext = context ?? null;
return Buffer.from('audio');
@@ -501,6 +543,7 @@ test('NoteUpdateWorkflow snapshots one media range for audio and image without a
harness.deps.getConfig = () => ({
fields: {
sentence: 'Sentence',
audio: 'SentenceAudio',
image: 'Picture',
miscInfo: 'MiscInfo',
},
@@ -511,7 +554,6 @@ test('NoteUpdateWorkflow snapshots one media range for audio and image without a
},
behavior: {},
});
harness.deps.getResolvedSentenceAudioFieldName = () => 'SentenceAudio';
harness.deps.captureSubtitleMediaContext = () => {
captureCalls += 1;
return capturedContext;
+15 -12
View File
@@ -19,6 +19,7 @@ export interface NoteUpdateWorkflowDeps {
fields?: {
word?: string;
sentence?: string;
audio?: string;
image?: string;
miscInfo?: string;
};
@@ -39,7 +40,7 @@ export interface NoteUpdateWorkflowDeps {
sentenceField: string;
lapisEnabled: boolean;
kikuEnabled: boolean;
kikuFieldGrouping: 'auto' | 'manual' | 'disabled';
fieldGroupingMode: 'auto' | 'manual' | 'disabled';
wordCardKind?: WordCardKind;
};
appendKnownWordsFromNoteInfo: (noteInfo: NoteUpdateWorkflowNoteInfo) => void;
@@ -75,7 +76,6 @@ export interface NoteUpdateWorkflowDeps {
noteInfo: NoteUpdateWorkflowNoteInfo,
...preferredNames: (string | undefined)[]
) => string | null;
getResolvedSentenceAudioFieldName: (noteInfo: NoteUpdateWorkflowNoteInfo) => string | null;
getAnimatedImageLeadInSeconds: (noteInfo: NoteUpdateWorkflowNoteInfo) => Promise<number>;
mergeFieldValue: (existing: string, newValue: string, overwrite: boolean) => string;
generateAudioFilename: () => string;
@@ -160,7 +160,7 @@ export class NoteUpdateWorkflow {
return null;
}
async execute(noteId: number, options?: { skipKikuFieldGrouping?: boolean }): Promise<void> {
async execute(noteId: number, options?: { skipFieldGrouping?: boolean }): Promise<void> {
this.deps.beginUpdateProgress('Updating card');
try {
const notesInfoResult = await this.deps.client.notesInfo([noteId]);
@@ -187,9 +187,7 @@ export class NoteUpdateWorkflow {
const sentenceCardConfig = this.deps.getEffectiveSentenceCardConfig();
const shouldRunFieldGrouping =
!options?.skipKikuFieldGrouping &&
sentenceCardConfig.kikuEnabled &&
sentenceCardConfig.kikuFieldGrouping !== 'disabled';
!options?.skipFieldGrouping && sentenceCardConfig.fieldGroupingMode !== 'disabled';
let duplicateNoteId: number | null = null;
if (shouldRunFieldGrouping && hasExpressionText) {
duplicateNoteId = await this.deps.findDuplicateNote(expressionText, noteId, noteInfo);
@@ -198,11 +196,13 @@ export class NoteUpdateWorkflow {
const updatedFields: Record<string, string> = {};
let updatePerformed = false;
let miscInfoFilename: string | null = null;
const sentenceField = sentenceCardConfig.sentenceField;
const configuredSentenceField =
config.fields?.sentence ?? DEFAULT_ANKI_CONNECT_CONFIG.fields.sentence;
const sentenceField = this.deps.resolveConfiguredFieldName(noteInfo, configuredSentenceField);
const subtitleMiningContext = this.consumeMatchingSubtitleMiningContext(
fields,
sentenceField,
config.fields?.sentence,
sentenceField ?? configuredSentenceField,
configuredSentenceField,
);
// Audio and image generation run sequentially and audio extraction can take tens of
// seconds, so resolve the clip range exactly once up front; reading live mpv sub
@@ -258,7 +258,10 @@ export class NoteUpdateWorkflow {
if (audioBuffer) {
await this.deps.client.storeMediaFile(audioFilename, audioBuffer);
const sentenceAudioField = this.deps.getResolvedSentenceAudioFieldName(noteInfo);
const sentenceAudioField = this.deps.resolveConfiguredFieldName(
noteInfo,
config.fields?.audio ?? DEFAULT_ANKI_CONNECT_CONFIG.fields.audio,
);
if (sentenceAudioField) {
const existingAudio = noteInfo.fields[sentenceAudioField]?.value || '';
updatedFields[sentenceAudioField] = this.deps.mergeFieldValue(
@@ -345,7 +348,7 @@ export class NoteUpdateWorkflow {
noteInfoForGrouping = refreshedInfo[0]!;
}
if (sentenceCardConfig.kikuFieldGrouping === 'auto') {
if (sentenceCardConfig.fieldGroupingMode === 'auto') {
await this.deps.handleFieldGroupingAuto(
duplicateNoteId,
noteId,
@@ -354,7 +357,7 @@ export class NoteUpdateWorkflow {
);
return;
}
if (sentenceCardConfig.kikuFieldGrouping === 'manual') {
if (sentenceCardConfig.fieldGroupingMode === 'manual') {
await this.deps.handleFieldGroupingManual(
duplicateNoteId,
noteId,
@@ -31,7 +31,6 @@ function createDeps(
getCachedMediaPath: async () => null,
shouldRequireRemoteMediaCache: () => true,
getSubtitleMediaRange: () => ({ startTime: 1, endTime: 2 }),
getResolvedSentenceAudioFieldName: () => 'SentenceAudio',
resolveConfiguredFieldName: () => 'Picture',
mergeFieldValue: (_existing, newValue) => newValue,
getAnimatedImageLeadInSeconds: async () => 0,
@@ -133,7 +132,7 @@ test('PendingYoutubeMediaQueue defaults missing media flags to enabled when queu
noteIds.map((noteId) => ({
noteId,
fields: {
SentenceAudio: { value: '' },
ExpressionAudio: { value: '' },
Picture: { value: '' },
},
})),
@@ -144,13 +143,16 @@ test('PendingYoutubeMediaQueue defaults missing media flags to enabled when queu
storedMedia.push(filename);
},
},
getConfig: () => ({ media: {}, fields: { image: 'Picture' } }) as AnkiConnectConfig,
getConfig: () =>
({ media: {}, fields: { audio: 'ExpressionAudio', image: 'Picture' } }) as AnkiConnectConfig,
resolveConfiguredFieldName: (noteInfo, ...preferredNames) =>
preferredNames.find((name) => name && name in noteInfo.fields) ?? null,
});
const queue = new PendingYoutubeMediaQueue(deps);
const queued = await queue.queueFromNote({
noteId: 42,
noteInfo: { noteId: 42, fields: {} },
noteInfo: { noteId: 42, fields: { ExpressionAudio: { value: '' } } },
label: 'demo',
});
await queue.handleReady('https://youtu.be/abc123', '/tmp/media.mkv');
@@ -158,7 +160,8 @@ test('PendingYoutubeMediaQueue defaults missing media flags to enabled when queu
assert.equal(queued, true);
assert.equal(updatedNotes.length, 1);
assert.equal(storedMedia.length, 2);
assert.match(updatedNotes[0]?.fields.SentenceAudio ?? '', /^\[sound:audio\.mp3\]$/);
assert.match(updatedNotes[0]?.fields.ExpressionAudio ?? '', /^\[sound:audio\.mp3\]$/);
assert.equal('SentenceAudio' in (updatedNotes[0]?.fields ?? {}), false);
assert.match(updatedNotes[0]?.fields.Picture ?? '', /^<img src="image\.webp">$/);
});
@@ -39,7 +39,6 @@ export interface PendingYoutubeMediaQueueDeps {
startTime: number;
endTime: number;
};
getResolvedSentenceAudioFieldName: (noteInfo: PendingYoutubeMediaNoteInfo) => string | null;
resolveConfiguredFieldName: (
noteInfo: PendingYoutubeMediaNoteInfo,
...preferredNames: (string | undefined)[]
@@ -136,7 +135,7 @@ export class PendingYoutubeMediaQueue {
startTime: mediaRange.startTime,
endTime: mediaRange.endTime,
label: job.label,
audioFieldName: this.deps.getResolvedSentenceAudioFieldName(job.noteInfo) ?? undefined,
audioFieldName: this.resolveConfiguredAudioFieldName(job.noteInfo) ?? undefined,
imageFieldName:
this.deps.resolveConfiguredFieldName(
job.noteInfo,
@@ -247,6 +246,14 @@ export class PendingYoutubeMediaQueue {
return matched;
}
private resolveConfiguredAudioFieldName(noteInfo: PendingYoutubeMediaNoteInfo): string | null {
const config = this.deps.getConfig();
return this.deps.resolveConfiguredFieldName(
noteInfo,
config.fields?.audio ?? DEFAULT_ANKI_CONNECT_CONFIG.fields.audio,
);
}
private async applyUpdate(
job: PendingYoutubeMediaUpdate,
cachedPath: string,
@@ -283,7 +290,7 @@ export class PendingYoutubeMediaQueue {
if (audioBuffer) {
await this.deps.client.storeMediaFile(audioFilename, audioBuffer);
const audioField =
job.audioFieldName || this.deps.getResolvedSentenceAudioFieldName(noteInfo) || null;
job.audioFieldName || this.resolveConfiguredAudioFieldName(noteInfo) || null;
if (audioField) {
const existingAudio = noteInfo.fields[audioField]?.value || '';
mediaFields[audioField] = this.deps.mergeFieldValue(
+8
View File
@@ -116,6 +116,10 @@ export function normalizeAnkiIntegrationConfig(config: AnkiConnectConfig): AnkiC
...DEFAULT_ANKI_CONNECT_CONFIG.isKiku,
...(config.isKiku ?? {}),
},
isSenren: {
...DEFAULT_ANKI_CONNECT_CONFIG.isSenren,
...(config.isSenren ?? {}),
},
lapisKiku: {
...DEFAULT_ANKI_CONNECT_CONFIG.lapisKiku,
...(config.lapisKiku ?? {}),
@@ -209,6 +213,10 @@ export class AnkiIntegrationRuntime {
patch.isKiku !== undefined
? { ...this.config.isKiku, ...patch.isKiku }
: this.config.isKiku,
isSenren:
patch.isSenren !== undefined
? { ...this.config.isSenren, ...patch.isSenren }
: this.config.isSenren,
lapisKiku:
patch.lapisKiku !== undefined
? { ...this.config.lapisKiku, ...patch.lapisKiku }
+42
View File
@@ -2188,6 +2188,7 @@ test('runtime options registry is centralized', () => {
'subtitle.annotation.frequency',
'anki.nPlusOneMatchMode',
'anki.kikuFieldGrouping',
'anki.senrenFieldGrouping',
]);
});
@@ -2775,6 +2776,47 @@ test('accepts a Kiku/Lapis word card kind and warns on an unknown one', () => {
);
});
test('forces Senren off when Kiku is also enabled and validates Senren fieldGrouping', () => {
const dir = makeTempDir();
fs.writeFileSync(
path.join(dir, 'config.jsonc'),
`{
"ankiConnect": {
"isKiku": { "enabled": true },
"isSenren": { "enabled": true }
}
}`,
'utf-8',
);
const service = new ConfigService(dir);
assert.equal(service.getConfig().ankiConnect.isKiku.enabled, true);
assert.equal(service.getConfig().ankiConnect.isSenren.enabled, false);
assert.ok(
service.getWarnings().some((warning) => warning.path === 'ankiConnect.isSenren.enabled'),
);
const senrenOnlyDir = makeTempDir();
fs.writeFileSync(
path.join(senrenOnlyDir, 'config.jsonc'),
`{
"ankiConnect": {
"isSenren": { "enabled": true, "fieldGrouping": "sometimes" }
}
}`,
'utf-8',
);
const senrenOnlyService = new ConfigService(senrenOnlyDir);
assert.equal(senrenOnlyService.getConfig().ankiConnect.isSenren.enabled, true);
assert.equal(senrenOnlyService.getConfig().ankiConnect.isSenren.fieldGrouping, 'auto');
assert.ok(
senrenOnlyService
.getWarnings()
.some((warning) => warning.path === 'ankiConnect.isSenren.fieldGrouping'),
);
});
test('accepts valid ankiConnect knownWords deck object', () => {
const dir = makeTempDir();
fs.writeFileSync(
@@ -92,6 +92,11 @@ export const INTEGRATIONS_DEFAULT_CONFIG: Pick<
fieldGrouping: 'disabled',
deleteDuplicateInAuto: true,
},
isSenren: {
enabled: false,
fieldGrouping: 'auto',
deleteDuplicateInAuto: true,
},
lapisKiku: {
wordCardKind: 'word-and-sentence',
},
@@ -363,6 +363,28 @@ export function buildIntegrationConfigOptionRegistry(
description:
'When Kiku field grouping is "auto", delete the duplicate source card after grouping completes.',
},
{
path: 'ankiConnect.isSenren.fieldGrouping',
kind: 'enum',
enumValues: ['auto', 'manual', 'disabled'],
defaultValue: defaultConfig.ankiConnect.isSenren.fieldGrouping,
description: 'Senren duplicate-card field grouping mode (scene switching).',
runtime: runtimeOptionById.get('anki.senrenFieldGrouping'),
},
{
path: 'ankiConnect.isSenren.enabled',
kind: 'boolean',
defaultValue: defaultConfig.ankiConnect.isSenren.enabled,
description:
'Enable Senren-specific duplicate handling (scene-switching field grouping, including miscInfo grouping). Mutually exclusive with isKiku.enabled.',
},
{
path: 'ankiConnect.isSenren.deleteDuplicateInAuto',
kind: 'boolean',
defaultValue: defaultConfig.ankiConnect.isSenren.deleteDuplicateInAuto,
description:
'When Senren field grouping is "auto", delete the duplicate source card after grouping completes.',
},
{
path: 'ankiConnect.isLapis.enabled',
kind: 'boolean',
+17
View File
@@ -124,5 +124,22 @@ export function buildRuntimeOptionRegistry(
},
}),
},
{
id: 'anki.senrenFieldGrouping',
path: 'ankiConnect.isSenren.fieldGrouping',
label: 'Senren Field Grouping',
scope: 'ankiConnect',
valueType: 'enum',
allowedValues: ['auto', 'manual', 'disabled'],
defaultValue: 'auto',
requiresRestart: false,
formatValueForOsd: (value) => String(value),
toAnkiPatch: (value) => ({
isSenren: {
fieldGrouping:
value === 'auto' || value === 'manual' || value === 'disabled' ? value : 'auto',
},
}),
},
];
}
+1 -1
View File
@@ -135,7 +135,7 @@ const INTEGRATION_TEMPLATE_SECTIONS: ConfigTemplateSection[] = [
title: 'AnkiConnect Integration',
description: ['Automatic Anki updates and media generation options.'],
notes: [
'Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running.',
'Hot-reload: ankiConnect.ai.enabled, media.normalizeAudio/mirrorMpvVolume, knownWords, nPlusOne, fields.word/audio/image/sentence/miscInfo, behavior.autoUpdateNewCards, isLapis.sentenceCardModel, isKiku.fieldGrouping, isSenren.fieldGrouping, and lapisKiku.wordCardKind update live while SubMiner is running.',
'Shared AI provider transport settings are read from top-level ai and typically require restart.',
'Most other AnkiConnect settings still require restart.',
],
+2
View File
@@ -1,6 +1,7 @@
import type { ResolveContext } from './context';
import { initializeAnkiConnectResolution } from './anki-connect/initialize';
import { applyAnkiKikuResolution } from './anki-connect/kiku';
import { applyAnkiSenrenResolution } from './anki-connect/senren';
import { applyAnkiLapisKikuResolution } from './anki-connect/lapis-kiku';
import { applyAnkiKnownWordsResolution } from './anki-connect/known-words';
import { applyAnkiLegacyResolution } from './anki-connect/legacy';
@@ -23,5 +24,6 @@ export function applyAnkiConnectResolution(context: ResolveContext): void {
applyAnkiLegacyResolution(context, ankiConnect, behavior, fields, media, metadata);
applyAnkiKnownWordsResolution(context, ankiConnect, behavior);
applyAnkiKikuResolution(context);
applyAnkiSenrenResolution(context);
applyAnkiLapisKikuResolution(context, ankiConnect);
}
@@ -77,6 +77,12 @@ export function initializeAnkiConnectResolution(
? (ankiConnect.isKiku as (typeof context.resolved)['ankiConnect']['isKiku'])
: {}),
},
isSenren: {
...context.resolved.ankiConnect.isSenren,
...(isObject(ankiConnect.isSenren)
? (ankiConnect.isSenren as (typeof context.resolved)['ankiConnect']['isSenren'])
: {}),
},
lapisKiku: {
...context.resolved.ankiConnect.lapisKiku,
},
+34
View File
@@ -0,0 +1,34 @@
import { DEFAULT_CONFIG } from '../../definitions';
import type { ResolveContext } from '../context';
export function applyAnkiSenrenResolution(context: ResolveContext): void {
if (
context.resolved.ankiConnect.isSenren.fieldGrouping !== 'auto' &&
context.resolved.ankiConnect.isSenren.fieldGrouping !== 'manual' &&
context.resolved.ankiConnect.isSenren.fieldGrouping !== 'disabled'
) {
context.warn(
'ankiConnect.isSenren.fieldGrouping',
context.resolved.ankiConnect.isSenren.fieldGrouping,
DEFAULT_CONFIG.ankiConnect.isSenren.fieldGrouping,
'Expected auto, manual, or disabled.',
);
context.resolved.ankiConnect.isSenren.fieldGrouping =
DEFAULT_CONFIG.ankiConnect.isSenren.fieldGrouping;
}
// Kiku and Senren field grouping write incompatible markup into the same note
// fields, so only one may be active; Kiku wins to preserve pre-existing setups.
if (
context.resolved.ankiConnect.isSenren.enabled === true &&
context.resolved.ankiConnect.isKiku.enabled === true
) {
context.warn(
'ankiConnect.isSenren.enabled',
true,
false,
'Kiku and Senren are mutually exclusive; disable isKiku.enabled to use Senren field grouping.',
);
context.resolved.ankiConnect.isSenren.enabled = false;
}
}
+6 -3
View File
@@ -298,10 +298,12 @@ test('settings registry puts feature toggles first, then other toggles alphabeti
];
assert.equal(miningSections[0], 'AnkiConnect');
const kikuLapis = fields.filter((candidate) => candidate.section === 'Kiku/Lapis Features');
const kikuLapis = fields.filter(
(candidate) => candidate.section === 'Kiku/Lapis/Senren Features',
);
assert.deepEqual(
kikuLapis.slice(0, 2).map((candidate) => candidate.configPath),
['ankiConnect.isLapis.enabled', 'ankiConnect.isKiku.enabled'],
kikuLapis.slice(0, 3).map((candidate) => candidate.configPath),
['ankiConnect.isLapis.enabled', 'ankiConnect.isKiku.enabled', 'ankiConnect.isSenren.enabled'],
);
});
@@ -366,6 +368,7 @@ test('settings registry marks safe live config paths as hot-reloadable', () => {
'ankiConnect.fields.miscInfo',
'ankiConnect.isLapis.sentenceCardModel',
'ankiConnect.isKiku.fieldGrouping',
'ankiConnect.isSenren.fieldGrouping',
]) {
assert.equal(field(path).restartBehavior, 'hot-reload', path);
}
+9 -3
View File
@@ -131,7 +131,7 @@ const SECTION_ORDER = new Map<string, number>(
'AnkiConnect',
'Note Fields',
'Media Capture',
'Kiku/Lapis Features',
'Kiku/Lapis/Senren Features',
'Anki AI',
'AnkiConnect Proxy',
'Jimaku',
@@ -163,6 +163,7 @@ const PATH_ORDER = new Map<string, number>(
'ankiConnect.proxy.enabled',
'ankiConnect.isLapis.enabled',
'ankiConnect.isKiku.enabled',
'ankiConnect.isSenren.enabled',
'subtitleStyle.knownWordColor',
'ankiConnect.knownWords.matureThresholdDays',
'subtitleStyle.knownWordMaturityColors.new',
@@ -221,6 +222,7 @@ const LABEL_OVERRIDES: Record<string, string> = {
'ankiConnect.nPlusOne.enabled': 'Enabled',
'ankiConnect.isLapis.enabled': 'Enable Lapis Features',
'ankiConnect.isKiku.enabled': 'Enable Kiku Features',
'ankiConnect.isSenren.enabled': 'Enable Senren Features',
'ankiConnect.lapisKiku.wordCardKind': 'Word Card Type',
'stats.toggleKey': 'Toggle Stats Overlay',
'shortcuts.openCharacterDictionaryManager': 'Open Character Dictionary Manager',
@@ -251,7 +253,9 @@ const DESCRIPTION_OVERRIDES: Record<string, string> = {
'ankiConnect.pollingRate':
'Polling interval in milliseconds. Ignored while the local AnkiConnect proxy is enabled because push-based enrichment is used instead.',
'ankiConnect.isKiku.enabled':
'Enable Kiku-specific mining behavior. Kiku supersedes Lapis: Lapis features still work, and Kiku adds duplicate handling and field grouping.',
'Enable Kiku-specific mining behavior. Kiku supersedes Lapis: Lapis features still work, and Kiku adds duplicate handling and field grouping. Mutually exclusive with Senren.',
'ankiConnect.isSenren.enabled':
'Enable Senren-specific duplicate handling: field grouping merges duplicates into Senren scene-switching markup (including miscInfo grouping). Mutually exclusive with Kiku; only one can be enabled at a time.',
'ankiConnect.isLapis.enabled':
'Enable Lapis-specific mining behavior and sentence-card model targeting. When Kiku is enabled, Lapis features still work and Kiku-specific features are added on top.',
'ankiConnect.isLapis.sentenceCardModel':
@@ -407,9 +411,10 @@ function categoryAndSection(path: string): { category: ConfigSettingsCategory; s
if (
path.startsWith('ankiConnect.isKiku.') ||
path.startsWith('ankiConnect.isLapis.') ||
path.startsWith('ankiConnect.isSenren.') ||
path.startsWith('ankiConnect.lapisKiku.')
) {
return { category: 'mining-anki', section: 'Kiku/Lapis Features' };
return { category: 'mining-anki', section: 'Kiku/Lapis/Senren Features' };
}
if (path.startsWith('ankiConnect.ai.')) {
return { category: 'mining-anki', section: 'Anki AI' };
@@ -709,6 +714,7 @@ function restartBehaviorForPath(path: string): ConfigSettingsRestartBehavior {
path === 'ankiConnect.fields.miscInfo' ||
path === 'ankiConnect.isLapis.sentenceCardModel' ||
path === 'ankiConnect.isKiku.fieldGrouping' ||
path === 'ankiConnect.isSenren.fieldGrouping' ||
path === 'ankiConnect.lapisKiku.wordCardKind' ||
path === 'mpv.aniskipEnabled' ||
path === 'mpv.aniskipButtonKey' ||
@@ -284,6 +284,22 @@ function createMockTracker(
getSessionTimeline: async () => [],
getSessionEvents: async () => [],
getVocabularyStats: async () => VOCABULARY_STATS,
getVocabularySummary: async () => ({
uniqueWords: 501,
uniqueWordsWithoutNames: 500,
uniqueKanji: 201,
newThisWeek: 7,
newThisWeekWithoutNames: 6,
knownWordCount: 250,
knownWordCountWithoutNames: 249,
}),
getVocabularyChartData: async () => ({
ready: true,
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
}),
getStatsExcludedWords: async () => [],
replaceStatsExcludedWords: async () => {},
getKanjiStats: async () => KANJI_STATS,
@@ -711,6 +727,38 @@ describe('stats server API routes', () => {
assert.equal(body[0].headword, 'する');
});
it('GET /api/stats/vocabulary/summary returns database-wide card totals', async () => {
const app = createStatsApp(createMockTracker());
const res = await app.request('/api/stats/vocabulary/summary');
assert.equal(res.status, 200);
assert.deepEqual(await res.json(), {
uniqueWords: 501,
uniqueWordsWithoutNames: 500,
uniqueKanji: 201,
newThisWeek: 7,
newThisWeekWithoutNames: 6,
knownWordCount: 250,
knownWordCountWithoutNames: 249,
});
});
it('GET /api/stats/vocabulary/charts returns complete chart datasets', async () => {
const app = createStatsApp(createMockTracker());
const res = await app.request('/api/stats/vocabulary/charts');
assert.equal(res.status, 200);
assert.deepEqual(await res.json(), {
ready: true,
topWords: [{ wordId: 1, headword: 'する', frequency: 50 }],
topWordsWithoutNames: [{ wordId: 1, headword: 'する', frequency: 50 }],
newWordsTimeline: [{ epochDay: 20_000, wordCount: 3 }],
newWordsTimelineWithoutNames: [{ epochDay: 20_000, wordCount: 3 }],
});
});
it('GET /api/stats/kanji returns kanji frequency data', async () => {
const app = createStatsApp(createMockTracker());
const res = await app.request('/api/stats/kanji');
+2
View File
@@ -65,6 +65,7 @@ export interface AnkiJimakuIpcRuntimeOptions {
getYoutubeMediaSourceUrl?: () => Promise<string | null | undefined> | string | null | undefined;
showDesktopNotification: (title: string, options: { body?: string; icon?: string }) => void;
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
dismissOverlayNotification?: (id: string) => void;
createFieldGroupingCallback: () => (
data: KikuFieldGroupingRequestData,
) => Promise<KikuFieldGroupingChoice>;
@@ -166,6 +167,7 @@ export function registerAnkiJimakuIpcRuntime(
options.getCachedMediaPath,
options.shouldRequireRemoteMediaCache,
options.getYoutubeMediaSourceUrl,
options.dismissOverlayNotification,
);
integration.start();
options.setAnkiIntegration(integration);
+40
View File
@@ -10,6 +10,8 @@ import {
isAssTemporalCommand,
normalizePlainSubtitleText,
parseAssEffectField,
removeLiveGlyphFragmentLines,
removeAssControlDebrisLines,
} from './ass-text';
test('assToPlainText drops vector drawing runs', () => {
@@ -74,6 +76,14 @@ test('assToPlainText normalizes CRLF before converting', () => {
assert.equal(assToPlainText('一行目\r\n二行目'), '一行目\n二行目');
});
test('removeAssControlDebrisLines drops malformed spacer resets without eating dialogue', () => {
assert.equal(
removeAssControlDebrisLines('Visible line\n\\\n{\\fr0\n\\{\\frz287.5'),
'Visible line',
);
assert.equal(removeAssControlDebrisLines('本文{\\pos(1,2)'), '本文{\\pos(1,2)');
});
test('normalizePlainSubtitleText settles whitespace without decoding ASS', () => {
// A brace reaching this layer is literal text mpv chose to show, not markup.
assert.equal(normalizePlainSubtitleText('本文{\\pos(1,2)'), '本文{\\pos(1,2)');
@@ -193,3 +203,33 @@ test('isAnimatedAssEffectKind covers the stock animated effects only', () => {
assert.equal(isAnimatedAssEffectKind('other'), false);
assert.equal(isAnimatedAssEffectKind('none'), false);
});
test('removeLiveGlyphFragmentLines drops a per-glyph typesetting wall and its syllable', () => {
const wall = [...'wansdumretoikhI'].join('\n');
assert.equal(removeLiveGlyphFragmentLines(`${wall}\ntai`), '');
});
test('removeLiveGlyphFragmentLines keeps concurrent dialogue beside a glyph wall', () => {
const wall = [...'wansdumretoikhI'].join('\n');
assert.equal(removeLiveGlyphFragmentLines(`${wall}\nそれよりも ノート…`), 'それよりも ノート…');
});
test('removeLiveGlyphFragmentLines leaves ordinary short lines alone', () => {
const text = 'え\nはい。\nそうだな';
assert.equal(removeLiveGlyphFragmentLines(text), text);
});
test('normalizePlainSubtitleText folds cue-boundary blank lines for text consumers', () => {
// The display layer splits on the blank line before normalizing; everyone else --
// tokenizer, cache key, dedup gate, mined sentence -- wants the plain line form.
assert.equal(
normalizePlainSubtitleText('\u4e00\u884c\u76ee\n\n\u4e8c\u884c\u76ee'),
'\u4e00\u884c\u76ee\n\u4e8c\u884c\u76ee',
);
assert.equal(
normalizePlainSubtitleText('\u4e00\u884c\u76ee\n\n\u4e8c\u884c\u76ee', {
collapseLineBreaks: true,
}),
'\u4e00\u884c\u76ee \u4e8c\u884c\u76ee',
);
});
+39
View File
@@ -91,6 +91,41 @@ export function assToPlainText(text: string, lineBreak: AssLineBreak = '\n'): st
return resolveWhitespaceEscapes(stripAssMarkup(text.replace(/\r\n/g, '\n')), lineBreak);
}
const MALFORMED_ASS_ROTATION_RESET = /^\\?\{\\(?:fr|frx|fry|frz|fax|fay)[-+.0-9]*$/u;
/**
* Drop non-rendering spacer events left as literal text by a malformed, unclosed ASS
* rotation reset. These events otherwise become repeated `\\` or `{\\fr0` subtitle
* lines after mpv-compatible decoding.
*/
export function removeAssControlDebrisLines(text: string): string {
return text
.split('\n')
.filter((line) => {
const compact = line.replace(/\s+/gu, '');
return compact !== '\\' && !MALFORMED_ASS_ROTATION_RESET.test(compact);
})
.join('\n');
}
const MIN_GLYPH_BURST_LINES = 6;
const MAX_GLYPH_BURST_COMPANION_GLYPHS = 3;
/**
* Per-glyph karaoke typesetting flattened into live text becomes a wall of
* single-character lines plus the short syllable currently being typed. No authored
* subtitle stacks this many one-glyph lines at once, so when the wall is present drop
* it and its short companion fragments while keeping any concurrent dialogue line.
*/
export function removeLiveGlyphFragmentLines(text: string): string {
const lines = text.split('\n');
const singleGlyphLines = lines.filter((line) => [...line.trim()].length === 1).length;
if (singleGlyphLines < MIN_GLYPH_BURST_LINES) return text;
return lines
.filter((line) => [...line.trim()].length > MAX_GLYPH_BURST_COMPANION_GLYPHS)
.join('\n');
}
export interface NormalizePlainSubtitleTextOptions {
/** Fold every line break into a single space. */
collapseLineBreaks?: boolean;
@@ -118,6 +153,10 @@ export function normalizePlainSubtitleText(
);
if (collapseLineBreaks) {
normalized = normalized.replace(/\n/g, ' ').replace(/\s+/g, ' ');
} else {
// Simultaneous cues reach the display layer separated by a blank line; every other
// consumer wants the plain one-break-per-line form.
normalized = normalized.replace(/\n{2,}/g, '\n');
}
return trim ? normalized.trim() : normalized;
+1
View File
@@ -85,6 +85,7 @@ const HOT_RELOAD_EXACT_OR_PREFIX_PATHS = [
'ankiConnect.fields.miscInfo',
'ankiConnect.isLapis.sentenceCardModel',
'ankiConnect.isKiku.fieldGrouping',
'ankiConnect.isSenren.fieldGrouping',
'ankiConnect.lapisKiku.wordCardKind',
] as const;
@@ -559,6 +559,241 @@ test('fresh tracker DB creates lifetime summary tables', async () => {
}
});
test('fresh tracker DB skips lexical rollup backfill work', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let backfillRuns = 0;
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath }, {
runLexicalRollupBackfillTask: async () => {
backfillRuns += 1;
},
} as never);
assert.equal(backfillRuns, 0);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('tracker starts the injected lexical rollup backfill when it is pending', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let backfillRuns = 0;
try {
const setupDb = new Database(dbPath);
const { ensureSchema } = await import('./immersion-tracker/storage');
ensureSchema(setupDb);
setupDb
.prepare(
`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
)
.run();
setupDb.close();
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath }, {
runLexicalRollupBackfillTask: async () => {
backfillRuns += 1;
},
} as never);
assert.equal(backfillRuns, 1);
await waitForCondition(
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
);
assert.equal(
(tracker as unknown as { preserveWriteQueueUntilDrained: boolean })
.preserveWriteQueueUntilDrained,
false,
);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('tracker runs startup session-rollup maintenance before lexical backfill locks writes', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let releaseBackfill = (): void => {};
const heldBackfill = new Promise<void>((resolve) => {
releaseBackfill = resolve;
});
try {
const startedAtMs = trackerNowMs() - 60_000;
const endedAtMs = trackerNowMs();
const setupDb = new Database(dbPath);
const { ensureSchema } = await import('./immersion-tracker/storage');
ensureSchema(setupDb);
setupDb.exec(`
INSERT INTO imm_videos (
video_id, video_key, canonical_title, source_type, duration_ms, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (1, 'local:/tmp/rollup-recovery.mkv', 'Rollup Recovery', 1, 0, '1', '1');
INSERT INTO imm_sessions (
session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status,
active_watched_ms, lines_seen, tokens_seen, cards_mined, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (
1, 'rollup-recovery', 1, '${startedAtMs}', '${endedAtMs}', 2,
60000, 10, 20, 2, '${startedAtMs}', '${endedAtMs}'
);
INSERT INTO imm_session_telemetry (
session_id, sample_ms, total_watched_ms, active_watched_ms, lines_seen,
tokens_seen, cards_mined, lookup_count, lookup_hits, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (
1, '${endedAtMs}', 60000, 60000, 10, 20, 2, 0, 0,
'${endedAtMs}', '${endedAtMs}'
);
DELETE FROM imm_daily_rollups;
DELETE FROM imm_monthly_rollups;
UPDATE imm_rollup_state SET state_value = '0';
`);
setupDb.close();
const Ctor = await loadTrackerCtor();
tracker = new Ctor({ dbPath }, {
runLexicalRollupBackfillTask: async () => heldBackfill,
} as never);
const privateApi = tracker as unknown as {
db: DatabaseSync;
writeLock: { locked: boolean };
};
assert.equal(privateApi.writeLock.locked, true);
assert.equal(
(
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_daily_rollups').get() as {
total: number;
}
).total,
1,
);
assert.equal(
(
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_monthly_rollups').get() as {
total: number;
}
).total,
1,
);
} finally {
releaseBackfill();
if (tracker) {
await waitForCondition(
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
);
}
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('tracker queues playback writes until lexical rollup backfill settles', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let startBackfill = (): void => {};
let releaseBackfill = (): void => {};
let markBackfillStarted = (): void => {};
const backfillStartGate = new Promise<void>((resolve) => {
startBackfill = resolve;
});
const heldBackfill = new Promise<void>((resolve) => {
releaseBackfill = resolve;
});
const backfillStarted = new Promise<void>((resolve) => {
markBackfillStarted = resolve;
});
try {
const setupDb = new Database(dbPath);
const { ensureSchema } = await import('./immersion-tracker/storage');
ensureSchema(setupDb);
setupDb
.prepare(
`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = 'lexical_daily_rollups_version'`,
)
.run();
setupDb.close();
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath, policy: { queueCap: 100 } },
{
runLexicalRollupBackfillTask: async (workerDbPath) => {
await backfillStartGate;
const workerDb = new Database(workerDbPath);
try {
workerDb.exec('BEGIN IMMEDIATE');
markBackfillStarted();
await heldBackfill;
workerDb.exec('COMMIT');
} catch (error) {
try {
workerDb.exec('ROLLBACK');
} catch {
// Preserve the original worker failure.
}
throw error;
} finally {
workerDb.close();
}
},
},
);
tracker.handleMediaChange('https://example.com/backfill-test.mp4', 'Backfill Test');
startBackfill();
await backfillStarted;
for (let index = 0; index < 125; index += 1) tracker.recordCardsMined(1);
const privateApi = tracker as unknown as {
db: DatabaseSync;
queue: unknown[];
droppedWriteCount: number;
flushNow: () => void;
writeLock: { locked: boolean };
};
assert.equal(privateApi.writeLock.locked, true);
privateApi.flushNow();
assert.ok(privateApi.queue.length > 100, 'the protected queue may grow past its normal cap');
assert.equal(privateApi.droppedWriteCount, 0, 'backfill must not discard playback writes');
assert.equal(
(
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
total: number;
}
).total,
0,
);
releaseBackfill();
await waitForCondition(() => privateApi.queue.length === 0, 5_000);
assert.equal(
(
privateApi.db.prepare('SELECT COUNT(*) AS total FROM imm_session_events').get() as {
total: number;
}
).total,
125,
);
} finally {
releaseBackfill();
if (tracker) {
await waitForCondition(
() => !(tracker as unknown as { writeLock: { locked: boolean } }).writeLock.locked,
5_000,
);
}
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('startup backfills lifetime summaries when retained sessions exist but summary tables are empty', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
@@ -5022,3 +5257,149 @@ test('anime browser streams group by series instead of by the proxy file extensi
cleanupDbPath(dbPath);
}
});
test('getVocabularySummary coalesces concurrent requests into one worker task', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let taskRuns = 0;
let releaseTask: (() => void) | null = null;
const seenKnownWords: Array<ReadonlySet<string> | null> = [];
const summary = {
uniqueWords: 1,
uniqueWordsWithoutNames: 1,
uniqueKanji: 0,
newThisWeek: 0,
newThisWeekWithoutNames: 0,
knownWordCount: null,
knownWordCountWithoutNames: null,
};
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath },
{
runVocabularySummaryTask: async (_dbPath, knownWords) => {
taskRuns += 1;
seenKnownWords.push(knownWords);
await new Promise<void>((resolve) => {
releaseTask = resolve;
});
return summary;
},
destroyVocabularySummaryRunner: () => {},
},
);
const knownWordsSnapshot = new Set(['猫']);
const first = tracker.getVocabularySummary(knownWordsSnapshot);
const second = tracker.getVocabularySummary(knownWordsSnapshot);
await waitForCondition(() => releaseTask !== null);
let release = releaseTask as (() => void) | null;
assert.ok(release);
release();
assert.deepEqual(await first, summary);
assert.equal(await second, await first);
assert.equal(taskRuns, 1);
assert.deepEqual(seenKnownWords, [knownWordsSnapshot]);
releaseTask = null;
const third = tracker.getVocabularySummary(null);
await waitForCondition(() => releaseTask !== null);
release = releaseTask as (() => void) | null;
assert.ok(release);
release();
assert.deepEqual(await third, summary);
assert.equal(taskRuns, 2);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('getVocabularySummary coalesces equivalent known-word snapshots by value', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
let taskRuns = 0;
const releases: Array<() => void> = [];
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath },
{
runVocabularySummaryTask: async () => {
taskRuns += 1;
await new Promise<void>((resolve) => releases.push(resolve));
return {
uniqueWords: 2,
uniqueWordsWithoutNames: 2,
uniqueKanji: 2,
newThisWeek: 0,
newThisWeekWithoutNames: 0,
knownWordCount: 2,
knownWordCountWithoutNames: 2,
};
},
destroyVocabularySummaryRunner: () => {},
},
);
const first = tracker.getVocabularySummary(new Set(['猫', '犬']));
const second = tracker.getVocabularySummary(new Set(['犬', '猫']));
await waitForCondition(() => releases.length > 0);
const observedTaskRuns = taskRuns;
for (const release of releases) release();
await Promise.all([first, second]);
assert.equal(observedTaskRuns, 1);
const third = tracker.getVocabularySummary(new Set(['猫', '犬']));
await waitForCondition(() => releases.length === 2);
releases[1]!();
await third;
assert.equal(taskRuns, 2, 'a settled snapshot must be evicted from the in-flight map');
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
test('getVocabularySummary keeps different known-word snapshots independent', async () => {
const dbPath = makeDbPath();
let tracker: ImmersionTrackerService | null = null;
const releases: Array<() => void> = [];
try {
const Ctor = await loadTrackerCtor();
tracker = new Ctor(
{ dbPath },
{
runVocabularySummaryTask: async (_dbPath, knownWords) => {
await new Promise<void>((resolve) => releases.push(resolve));
return {
uniqueWords: 1,
uniqueWordsWithoutNames: 1,
uniqueKanji: 0,
newThisWeek: 0,
newThisWeekWithoutNames: 0,
knownWordCount: knownWords?.size ?? null,
knownWordCountWithoutNames: knownWords?.size ?? null,
};
},
destroyVocabularySummaryRunner: () => {},
},
);
const withoutKnownWords = tracker.getVocabularySummary(null);
const withKnownWords = tracker.getVocabularySummary(new Set(['猫']));
await waitForCondition(() => releases.length === 2);
for (const release of releases) release();
assert.equal((await withoutKnownWords).knownWordCount, null);
assert.equal((await withKnownWords).knownWordCount, 1);
} finally {
tracker?.destroy();
cleanupDbPath(dbPath);
}
});
+106 -6
View File
@@ -59,6 +59,7 @@ import {
getSessionEvents,
getSimilarWords,
getStatsExcludedWords,
getVocabularyChartData,
getVocabularyStats,
replaceStatsExcludedWords,
searchSubtitleSentences,
@@ -97,6 +98,12 @@ import {
DeleteMaintenanceWorkerRuntime,
type RunDeleteMaintenanceTask,
} from './immersion-tracker/delete-maintenance-worker-runtime';
import {
VocabularySummaryWorkerRuntime,
type RunVocabularySummaryTask,
} from './immersion-tracker/vocabulary-summary-worker-runtime';
import { LexicalRollupWorkerRuntime } from './immersion-tracker/lexical-rollup-worker-runtime';
import { areLexicalDailyRollupsReady } from './immersion-tracker/lexical-rollups';
import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
import {
cleanupDuplicateSubtitleLines,
@@ -190,6 +197,7 @@ import {
type StatsExcludedWordRow,
type StreakCalendarRow,
type VocabularyCleanupSummary,
type VocabularyStatsSummary,
type WatchTimePerAnimeRow,
type WordAnimeAppearanceRow,
type WordDetailRow,
@@ -444,13 +452,24 @@ export class ImmersionTrackerService {
private readonly monthlyRollupRetentionMs: number;
private readonly vacuumIntervalMs: number;
private readonly dbPath: string;
private readonly writeLock = { locked: false };
private readonly writeLock = {
locked: false,
reasons: new Set<'flush' | 'delete-maintenance' | 'lexical-rollup-backfill'>(),
};
private readonly destroyDeleteMaintenanceRunner: () => void;
private readonly runVocabularySummaryTask: (
knownWords: ReadonlySet<string> | null,
) => Promise<VocabularyStatsSummary>;
private readonly vocabularySummariesInFlight = new Map<string, Promise<VocabularyStatsSummary>>();
private readonly destroyVocabularySummaryRunner: () => void;
private readonly runLexicalRollupBackfillTask: () => Promise<void>;
private readonly destroyLexicalRollupBackfillRunner: () => void;
private readonly deleteMaintenanceScheduler: DeleteMaintenanceScheduler;
private flushTimer: ReturnType<typeof setTimeout> | null = null;
private maintenanceTimer: ReturnType<typeof setInterval> | null = null;
private flushScheduled = false;
private droppedWriteCount = 0;
private preserveWriteQueueUntilDrained = false;
private lastVacuumMs = 0;
private isDestroyed = false;
private sessionState: SessionState | null = null;
@@ -473,6 +492,10 @@ export class ImmersionTrackerService {
dependencies: {
runDeleteMaintenanceTask?: RunDeleteMaintenanceTask;
destroyDeleteMaintenanceRunner?: () => void;
runVocabularySummaryTask?: RunVocabularySummaryTask;
destroyVocabularySummaryRunner?: () => void;
runLexicalRollupBackfillTask?: (dbPath: string) => Promise<void>;
destroyLexicalRollupBackfillRunner?: () => void;
} = {},
) {
this.dbPath = options.dbPath;
@@ -492,13 +515,34 @@ export class ImmersionTrackerService {
runTask: (task) => runDeleteMaintenanceTask(this.dbPath, task),
onBusy: () => {
this.requireWriteQueueDrained('delete maintenance');
this.writeLock.locked = true;
this.setWriteLock('delete-maintenance', true);
},
onIdle: () => {
this.writeLock.locked = false;
this.setWriteLock('delete-maintenance', false);
if (!this.isDestroyed && this.queue.length > 0) this.scheduleFlush(0);
},
});
if (dependencies.runVocabularySummaryTask) {
this.runVocabularySummaryTask = (knownWords) =>
dependencies.runVocabularySummaryTask!(this.dbPath, knownWords);
this.destroyVocabularySummaryRunner =
dependencies.destroyVocabularySummaryRunner ?? (() => {});
} else {
const vocabularySummaryRuntime = new VocabularySummaryWorkerRuntime();
this.runVocabularySummaryTask = (knownWords) =>
vocabularySummaryRuntime.run(this.dbPath, knownWords);
this.destroyVocabularySummaryRunner = () => vocabularySummaryRuntime.destroy();
}
if (dependencies.runLexicalRollupBackfillTask) {
this.runLexicalRollupBackfillTask = () =>
dependencies.runLexicalRollupBackfillTask!(this.dbPath);
this.destroyLexicalRollupBackfillRunner =
dependencies.destroyLexicalRollupBackfillRunner ?? (() => {});
} else {
const lexicalRollupRuntime = new LexicalRollupWorkerRuntime();
this.runLexicalRollupBackfillTask = () => lexicalRollupRuntime.run(this.dbPath);
this.destroyLexicalRollupBackfillRunner = () => lexicalRollupRuntime.destroy();
}
const parentDir = path.dirname(this.dbPath);
if (!fs.existsSync(parentDir)) {
fs.mkdirSync(parentDir, { recursive: true });
@@ -587,6 +631,7 @@ export class ImmersionTrackerService {
}
this.preparedStatements = createTrackerPreparedStatements(this.db);
this.scheduleMaintenance();
if (!areLexicalDailyRollupsReady(this.db)) this.startLexicalRollupBackfill();
this.scheduleFlush();
}
@@ -604,6 +649,8 @@ export class ImmersionTrackerService {
this.isDestroyed = true;
this.deleteMaintenanceScheduler.destroy();
this.destroyDeleteMaintenanceRunner();
this.destroyVocabularySummaryRunner();
this.destroyLexicalRollupBackfillRunner();
this.db.close();
}
@@ -673,6 +720,25 @@ export class ImmersionTrackerService {
return getVocabularyStats(this.db, limit, excludePos);
}
async getVocabularySummary(knownWords: ReadonlySet<string> | null) {
const key = knownWords ? JSON.stringify([...knownWords].sort()) : 'null';
const inFlight = this.vocabularySummariesInFlight.get(key);
if (inFlight) return inFlight;
const task = this.runVocabularySummaryTask(knownWords);
this.vocabularySummariesInFlight.set(key, task);
try {
return await task;
} finally {
if (this.vocabularySummariesInFlight.get(key) === task) {
this.vocabularySummariesInFlight.delete(key);
}
}
}
async getVocabularyChartData() {
return getVocabularyChartData(this.db);
}
async getStatsExcludedWords(): Promise<StatsExcludedWordRow[]> {
return getStatsExcludedWords(this.db);
}
@@ -1032,6 +1098,33 @@ export class ImmersionTrackerService {
}
}
private setWriteLock(
reason: 'flush' | 'delete-maintenance' | 'lexical-rollup-backfill',
active: boolean,
): void {
if (active) this.writeLock.reasons.add(reason);
else this.writeLock.reasons.delete(reason);
this.writeLock.locked = this.writeLock.reasons.size > 0;
}
private startLexicalRollupBackfill(): void {
this.requireWriteQueueDrained('lexical rollup backfill');
this.preserveWriteQueueUntilDrained = true;
this.setWriteLock('lexical-rollup-backfill', true);
void this.runLexicalRollupBackfillTask()
.catch((error: unknown) => {
this.logger.warn(
'Lexical daily rollup backfill failed; it will retry on next startup',
error,
);
})
.finally(() => {
this.setWriteLock('lexical-rollup-backfill', false);
if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
else if (!this.isDestroyed) this.scheduleFlush(0);
});
}
async reassignAnimeAnilist(
animeId: number,
info: {
@@ -2102,7 +2195,12 @@ export class ImmersionTrackerService {
private recordWrite(write: QueuedWrite): void {
if (this.isDestroyed) return;
const { dropped } = enqueueWrite(this.queue, write, this.queueCap);
// A lexical migration owns the database write lock, so dropping the oldest
// entry cannot relieve pressure: nothing can flush until the worker exits.
// Preserve that finite startup burst and drain it as soon as the lock lifts.
const { dropped } = this.preserveWriteQueueUntilDrained
? (this.queue.push(write), { dropped: 0 })
: enqueueWrite(this.queue, write, this.queueCap);
if (dropped > 0) {
this.droppedWriteCount += dropped;
this.logger.warn(`Immersion tracker queue overflow; dropped ${dropped} oldest writes`);
@@ -2150,6 +2248,7 @@ export class ImmersionTrackerService {
private flushNow(): void {
if (this.writeLock.locked || this.isDestroyed) return;
if (this.queue.length === 0) {
this.preserveWriteQueueUntilDrained = false;
this.flushScheduled = false;
return;
}
@@ -2161,7 +2260,7 @@ export class ImmersionTrackerService {
}
const batch = this.queue.splice(0, Math.min(this.batchSize, this.queue.length));
this.writeLock.locked = true;
this.setWriteLock('flush', true);
try {
this.db.exec('BEGIN IMMEDIATE');
for (const write of batch) {
@@ -2173,8 +2272,9 @@ export class ImmersionTrackerService {
this.queue.unshift(...batch);
this.logger.warn('Immersion tracker flush failed, retrying later', error as Error);
} finally {
this.writeLock.locked = false;
this.setWriteLock('flush', false);
this.flushScheduled = false;
if (this.queue.length === 0) this.preserveWriteQueueUntilDrained = false;
if (this.queue.length > 0) {
this.scheduleFlush(this.flushIntervalMs);
}
@@ -1,7 +1,4 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { Database } from '../sqlite.js';
import type { DatabaseSync } from '../sqlite.js';
@@ -21,17 +18,6 @@ interface SeedLine {
createdMs?: number;
}
function makeDbPath(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-duplicate-line-test-'));
return path.join(dir, 'immersion.sqlite');
}
function cleanupDbPath(dbPath: string): void {
const dir = path.dirname(dbPath);
if (!fs.existsSync(dir)) return;
fs.rmSync(dir, { recursive: true, force: true });
}
/** One episode, two sessions of it, and one word occurrence per seeded line. */
function seed(db: DatabaseSync, lines: SeedLine[]): void {
db.exec(`
@@ -82,12 +68,16 @@ function seed(db: DatabaseSync, lines: SeedLine[]): void {
`);
}
function createDb(lines: SeedLine[]): { db: DatabaseSync; dbPath: string } {
const dbPath = makeDbPath();
const db = new Database(dbPath);
/**
* These tests exercise the cleanup SQL, not durability. A fresh on-disk database per
* test pays a schema-creation fsync that is cheap on a local NVMe but slow enough on CI
* runners to blow the 5s per-test timeout, so the database stays in memory.
*/
function createDb(lines: SeedLine[]): { db: DatabaseSync } {
const db = new Database(':memory:');
ensureSchema(db);
seed(db, lines);
return { db, dbPath };
return { db };
}
/** A typeset line mpv reported once per animation frame. */
@@ -119,7 +109,7 @@ function wordFrequency(db: DatabaseSync): number {
}
test('a karaoke burst collapses to one line and gives back its word counts', () => {
const { db, dbPath } = createDb([
const { db } = createDb([
...karaokeFrames(1, '飛び上がる', 10_000, 40, 40),
{ session: 1, text: 'おはよう', startMs: 20_000, endMs: 22_000 },
]);
@@ -148,7 +138,6 @@ test('a karaoke burst collapses to one line and gives back its word counts', ()
assert.equal(summary.samples[0]!.videoTitle, 'Ep 1');
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
@@ -160,7 +149,7 @@ test('ordinary repeated dialogue survives', () => {
startMs: 5_000 + index * 800,
endMs: 5_000 + (index + 1) * 800,
}));
const { db, dbPath } = createDb(lines);
const { db } = createDb(lines);
try {
const summary = cleanupDuplicateSubtitleLines(db);
@@ -171,14 +160,13 @@ test('ordinary repeated dialogue survives', () => {
assert.equal(wordFrequency(db), 6);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a long run of quarter-second frames is still a burst', () => {
// Between the timing-only bound (0.1s) and the animation-frame bound (0.3s): heavier
// typesetting lands here, and the run length is what makes it conclusive.
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
try {
const summary = cleanupDuplicateSubtitleLines(db);
@@ -189,12 +177,11 @@ test('a long run of quarter-second frames is still a burst', () => {
assert.equal(wordFrequency(db), 1);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a qualifying short-frame burst may end with one long hold frame', () => {
const { db, dbPath } = createDb([
const { db } = createDb([
...karaokeFrames(1, '飛び上がる', 10_000, 8, 40),
{ session: 1, text: '飛び上がる', startMs: 10_320, endMs: 12_320 },
]);
@@ -208,12 +195,11 @@ test('a qualifying short-frame burst may end with one long hold frame', () => {
assert.equal(wordFrequency(db), 1);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a long event before the final frame prevents burst cleanup', () => {
const { db, dbPath } = createDb([
const { db } = createDb([
...karaokeFrames(1, '飛び上がる', 10_000, 5, 40),
{ session: 1, text: '飛び上がる', startMs: 10_200, endMs: 12_200 },
{ session: 1, text: '飛び上がる', startMs: 12_200, endMs: 12_240 },
@@ -226,12 +212,11 @@ test('a long event before the final frame prevents burst cleanup', () => {
assert.equal(countLines(db), 7);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a run of frames longer than the animation bound survives', () => {
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 400));
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 400));
try {
const summary = cleanupDuplicateSubtitleLines(db);
@@ -240,7 +225,6 @@ test('a run of frames longer than the animation bound survives', () => {
assert.equal(countLines(db), 6);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
@@ -248,7 +232,7 @@ test('the four-frame residue the live gate stores is cleaned up', () => {
// The streaming gate records the first four frames of a burst before the run is long
// enough to recognise. Four contiguous identical events under the strict timing-only
// bound are that residue, and no real dialogue.
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 40));
const { db } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 40));
try {
const summary = cleanupDuplicateSubtitleLines(db);
@@ -259,14 +243,13 @@ test('the four-frame residue the live gate stores is cleaned up', () => {
assert.equal(wordFrequency(db), 1);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a four-frame run above the strict frame bound survives', () => {
// Long enough per event to be plausible dialogue; only a five-event run may use the
// looser animation-frame bound.
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 250));
const { db } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 250));
try {
const summary = cleanupDuplicateSubtitleLines(db);
@@ -275,14 +258,13 @@ test('a four-frame run above the strict frame bound survives', () => {
assert.equal(countLines(db), 4);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('an explicit minRunLength raises the bar', () => {
// Five quarter-second frames qualify under the defaults; a cautious run asking for six
// leaves them alone. Above the strict bound, so the residue rule stays out of it.
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 5, 250));
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 5, 250));
try {
const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
@@ -293,12 +275,11 @@ test('an explicit minRunLength raises the bar', () => {
assert.equal(countLines(db), 5);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('an explicit maxFrameSeconds tightens the frame bound', () => {
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
try {
const summary = cleanupDuplicateSubtitleLines(db, { maxFrameSeconds: 0.2 });
@@ -307,13 +288,12 @@ test('an explicit maxFrameSeconds tightens the frame bound', () => {
assert.equal(countLines(db), 6);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a non-finite maxFrameSeconds falls back to the default bound', () => {
// Six normal-beat lines: Infinity must not turn every event into a "short frame".
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 800));
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 800));
try {
const summary = cleanupDuplicateSubtitleLines(db, { maxFrameSeconds: Infinity });
@@ -322,12 +302,11 @@ test('a non-finite maxFrameSeconds falls back to the default bound', () => {
assert.equal(countLines(db), 6);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('sampleLimit zero removes bursts but reports no samples', () => {
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
try {
const summary = cleanupDuplicateSubtitleLines(db, { sampleLimit: 0 });
@@ -337,12 +316,11 @@ test('sampleLimit zero removes bursts but reports no samples', () => {
assert.equal(countLines(db), 1);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a short run below every threshold survives', () => {
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 3, 40));
const { db } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 3, 40));
try {
const summary = cleanupDuplicateSubtitleLines(db);
@@ -351,7 +329,6 @@ test('a short run below every threshold survives', () => {
assert.equal(countLines(db), 3);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
@@ -361,7 +338,7 @@ test('interleaved dual-line karaoke collapses each line to one row', () => {
const kanji = karaokeFrames(1, '飛び上がる', 10_000, 20, 60);
const romaji = karaokeFrames(1, 'tobiagaru', 10_001, 20, 60);
const interleaved = [...kanji, ...romaji].sort((a, b) => a.startMs - b.startMs);
const { db, dbPath } = createDb(interleaved);
const { db } = createDb(interleaved);
try {
const summary = cleanupDuplicateSubtitleLines(db);
@@ -372,12 +349,11 @@ test('interleaved dual-line karaoke collapses each line to one row', () => {
assert.equal(wordFrequency(db), 2);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('the same line in a rewatch session is never merged into the first watch', () => {
const { db, dbPath } = createDb([
const { db } = createDb([
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
...karaokeFrames(2, '飛び上がる', 10_000, 6, 40),
]);
@@ -392,12 +368,11 @@ test('the same line in a rewatch session is never merged into the first watch',
assert.equal(wordFrequency(db), 2);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a gap between runs splits them', () => {
const { db, dbPath } = createDb([
const { db } = createDb([
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
...karaokeFrames(1, '飛び上がる', 60_000, 6, 40),
]);
@@ -409,12 +384,11 @@ test('a gap between runs splits them', () => {
assert.equal(countLines(db), 2);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('a dry run reports what an apply would do and writes nothing', () => {
const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
const { db } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
try {
const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
@@ -430,14 +404,13 @@ test('a dry run reports what an apply would do and writes nothing', () => {
assert.equal(countLines(db), 1);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('the lookback window leaves older bursts alone', () => {
const recentMs = BASE_MS;
const oldMs = BASE_MS - 40 * DAY_MS;
const { db, dbPath } = createDb([
const { db } = createDb([
...karaokeFrames(1, '飛び上がる', 10_000, 6, 40).map((line) => ({
...line,
createdMs: oldMs,
@@ -462,6 +435,5 @@ test('the lookback window leaves older bursts alone', () => {
} finally {
globalThis.__subminerTestNowMs = undefined;
db.close();
cleanupDbPath(dbPath);
}
});
@@ -31,6 +31,7 @@ import {
getKanjiOccurrences,
getSessionSummaries,
getVocabularyStats,
getVocabularySummary,
getKanjiStats,
getSessionEvents,
getSessionTimeline,
@@ -1875,6 +1876,115 @@ test('getVocabularyStats returns rows ordered by frequency descending', () => {
}
});
test('getVocabularySummary counts every tracked vocabulary row instead of a display page', () => {
const dbPath = makeDbPath();
const db = openTestDb(dbPath);
try {
ensureSchema(db);
const nowSec = Math.floor(Date.now() / 1000);
const insertWord = db.prepare(`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3,
first_seen, last_seen, frequency
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', ?, ?, 1)
`);
const insertKanji = db.prepare(`
INSERT INTO imm_kanji (kanji, first_seen, last_seen, frequency)
VALUES (?, ?, ?, 1)
`);
for (let index = 0; index < 501; index += 1) {
insertWord.run(`単語${index}`, `単語${index}`, nowSec - 8 * 86_400, nowSec - 8 * 86_400);
}
for (let index = 0; index < 201; index += 1) {
insertKanji.run(
String.fromCodePoint(0x4e00 + index),
nowSec - 8 * 86_400,
nowSec - 8 * 86_400,
);
}
insertWord.run('今週', '今週', nowSec - 86_400, nowSec - 86_400);
assert.deepEqual(getVocabularySummary(db, new Set(['単語0', '今週']), nowSec * 1000), {
uniqueWords: 502,
uniqueWordsWithoutNames: 502,
uniqueKanji: 201,
newThisWeek: 1,
newThisWeekWithoutNames: 1,
knownWordCount: 2,
knownWordCountWithoutNames: 2,
});
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getVocabularySummary applies vocabulary exclusions and Hide Names totals', () => {
const dbPath = makeDbPath();
const db = openTestDb(dbPath);
try {
ensureSchema(db);
const insertWord = db.prepare(`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3,
first_seen, last_seen, frequency
) VALUES (?, ?, '', 'noun', '名詞', ?, '', 1, 1, 1)
`);
insertWord.run('猫', '猫', '一般');
insertWord.run('太郎', '太郎', '固有名詞');
insertWord.run('東京', '東京都', '一般');
db.prepare(
`
INSERT INTO imm_stats_excluded_words (headword, word, reading)
VALUES ('東京', '東京', '')
`,
).run();
assert.deepEqual(getVocabularySummary(db, new Set(['猫', '太郎', '東京']), 9 * 86_400_000), {
uniqueWords: 2,
uniqueWordsWithoutNames: 1,
uniqueKanji: 0,
newThisWeek: 0,
newThisWeekWithoutNames: 0,
knownWordCount: 2,
knownWordCountWithoutNames: 1,
});
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getVocabularySummary counts identically across id-keyed scan batches', () => {
const dbPath = makeDbPath();
const db = openTestDb(dbPath);
try {
ensureSchema(db);
const insertWord = db.prepare(`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3,
first_seen, last_seen, frequency
) VALUES (?, ?, '', 'noun', '名詞', '一般', '', 1, 1, 1)
`);
for (let index = 0; index < 5; index += 1) {
insertWord.run(`単語${index}`, `単語${index}`);
}
const fullScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000);
const batchedScan = getVocabularySummary(db, new Set(['単語0']), 9 * 86_400_000, 2);
assert.equal(fullScan.uniqueWords, 5);
assert.deepEqual(batchedScan, fullScan);
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('getVocabularyStats filters rows that fail tokenizer vocabulary rules', () => {
const dbPath = makeDbPath();
const db = openTestDb(dbPath);
@@ -0,0 +1,136 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import {
LexicalRollupWorkerRuntime,
resolveLexicalRollupWorkerPath,
} from './lexical-rollup-worker-runtime';
import { areLexicalDailyRollupsReady } from './lexical-rollups';
import { Database } from './sqlite';
import { applyPragmas, ensureSchema } from './storage';
test('lexical rollup worker backfills without using the tracker connection', async () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-runtime-'));
const dbPath = path.join(directory, 'immersion.sqlite');
const runtime = new LexicalRollupWorkerRuntime();
const db = new Database(dbPath);
try {
applyPragmas(db);
ensureSchema(db);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES ('鳥', '鳥', 'とり', 1700000000, 1700000000, 1)`,
).run();
db.exec('DELETE FROM imm_lexical_daily_rollups');
db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
'lexical_daily_rollups_version',
);
db.close();
await runtime.run(dbPath);
const checkDb = new Database(dbPath);
try {
assert.equal(areLexicalDailyRollupsReady(checkDb), true);
} finally {
checkDb.close();
}
} finally {
runtime.destroy();
try {
db.close();
} catch {
// Closed before the worker starts.
}
fs.rmSync(directory, { recursive: true, force: true });
}
});
test('lexical rollup worker module resolves in the current layout', () => {
const workerPath = resolveLexicalRollupWorkerPath();
assert.ok(workerPath, 'expected the lexical rollup worker module to resolve');
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
});
test('lexical rollup worker leaves a backfill pending when no worker can start', async () => {
const runtime = new LexicalRollupWorkerRuntime({
resolveWorkerPath: () => null,
warn: () => {},
} as never);
try {
await assert.doesNotReject(runtime.run('/tmp/not-used.sqlite'));
} finally {
runtime.destroy();
}
});
test('lexical rollup worker absorbs termination failures after settling', async () => {
let sendMessage: ((message: { ok: boolean }) => void) | null = null;
const runtime = new LexicalRollupWorkerRuntime({
resolveWorkerPath: () => '/tmp/fake-worker.js',
createWorker: async () => ({
once(event: string, listener: (value: never) => void) {
if (event === 'message') sendMessage = listener as (message: { ok: boolean }) => void;
return this;
},
terminate: async () => {
throw new Error('termination failed');
},
}),
warn: () => {},
} as never);
const unhandled: unknown[] = [];
const captureUnhandled = (reason: unknown) => unhandled.push(reason);
process.on('unhandledRejection', captureUnhandled);
try {
const task = runtime.run('/tmp/not-used.sqlite');
await new Promise((resolve) => setImmediate(resolve));
const notify = sendMessage as ((message: { ok: boolean }) => void) | null;
assert.ok(notify);
notify({ ok: true });
await task;
await new Promise((resolve) => setImmediate(resolve));
assert.deepEqual(unhandled, []);
} finally {
process.off('unhandledRejection', captureUnhandled);
runtime.destroy();
}
});
test('lexical rollup worker times out when it never responds', async () => {
let terminated = false;
const runtime = new LexicalRollupWorkerRuntime({
resolveWorkerPath: () => '/tmp/fake-worker.js',
createWorker: async () => ({
once() {
return this;
},
terminate: async () => {
terminated = true;
return 0;
},
}),
timeoutMs: 1,
warn: () => {},
} as never);
try {
const outcome = await Promise.race([
runtime.run('/tmp/not-used.sqlite').then(
() => 'resolved',
(error: unknown) => String(error),
),
new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
]);
assert.match(outcome, /timed out/);
assert.equal(terminated, true);
} finally {
runtime.destroy();
}
});
@@ -0,0 +1,117 @@
import fs from 'node:fs';
import path from 'node:path';
import { createLogger } from '../../../logger';
interface WorkerResponse {
ok?: boolean;
error?: unknown;
}
interface WorkerHandle {
once(event: 'message', listener: (message: WorkerResponse) => void): this;
once(event: 'error', listener: (error: Error) => void): this;
once(event: 'exit', listener: (code: number) => void): this;
terminate(): Promise<number>;
}
interface LexicalRollupWorkerRuntimeOptions {
resolveWorkerPath?: () => string | null;
createWorker?: (workerPath: string, workerData: { dbPath: string }) => Promise<WorkerHandle>;
timeoutMs?: number;
warn?: (message: string, ...meta: unknown[]) => void;
}
const logger = createLogger('main:immersion-tracker:lexical-rollup-worker');
const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
export function resolveLexicalRollupWorkerPath(): string | null {
const fileName = __filename.endsWith('.ts')
? 'lexical-rollup-worker-thread.ts'
: 'lexical-rollup-worker-thread.js';
const workerPath = path.join(__dirname, fileName);
return fs.existsSync(workerPath) ? workerPath : null;
}
export class LexicalRollupWorkerRuntime {
private readonly activeWorkers = new Set<WorkerHandle>();
private destroyed = false;
constructor(private readonly options: LexicalRollupWorkerRuntimeOptions = {}) {}
async run(dbPath: string): Promise<void> {
if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
let worker: WorkerHandle;
try {
const workerPath = (this.options.resolveWorkerPath ?? resolveLexicalRollupWorkerPath)();
if (!workerPath) throw new Error('Emitted lexical rollup worker module was not found');
const createWorker =
this.options.createWorker ??
(async (resolvedPath, workerData) => {
const { Worker } = await import('node:worker_threads');
return new Worker(resolvedPath, { workerData });
});
worker = await createWorker(workerPath, { dbPath });
} catch (error) {
if (this.destroyed) throw new Error('Lexical rollup worker is shut down');
(this.options.warn ?? logger.warn)(
'Lexical rollup worker unavailable; leaving backfill pending for a later startup',
error,
);
return;
}
if (this.destroyed) {
await worker.terminate().catch(() => undefined);
throw new Error('Lexical rollup worker is shut down');
}
return new Promise<void>((resolve, reject) => {
let settled = false;
let timeout: ReturnType<typeof setTimeout> | null = null;
this.activeWorkers.add(worker);
const settle = (error?: Error) => {
if (settled) return;
settled = true;
if (timeout) clearTimeout(timeout);
this.activeWorkers.delete(worker);
void worker.terminate().catch(() => undefined);
if (error) reject(error);
else resolve();
};
timeout = setTimeout(
() => settle(new Error('Lexical rollup worker timed out')),
this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
);
worker.once('message', (message) => {
if (message.ok) settle();
else
settle(
new Error(
`Lexical rollup backfill failed: ${String(message.error ?? 'unknown error')}`,
),
);
});
worker.once('error', (error) => settle(error));
worker.once('exit', (code) => {
if (!settled) {
settle(
new Error(
code === 0
? 'Lexical rollup worker exited without a response'
: `Lexical rollup worker exited with code ${code}`,
),
);
}
});
});
}
destroy(): void {
if (this.destroyed) return;
this.destroyed = true;
for (const worker of this.activeWorkers) {
void worker.terminate().catch(() => undefined);
}
this.activeWorkers.clear();
}
}
@@ -0,0 +1,11 @@
import { parentPort, workerData } from 'node:worker_threads';
import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
if (!parentPort) throw new Error('lexical rollup worker missing parent port');
try {
executeLexicalRollupBackfillTask((workerData as { dbPath: string }).dbPath);
parentPort.postMessage({ ok: true });
} catch (error) {
parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
}
@@ -0,0 +1,35 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
import { executeLexicalRollupBackfillTask } from './lexical-rollup-worker';
import { Database } from './sqlite';
import { ensureSchema } from './storage';
test('lexical rollup backfill materializes pre-existing vocabulary off the caller DB connection', () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollup-worker-'));
const dbPath = path.join(directory, 'immersion.sqlite');
const db = new Database(dbPath);
try {
ensureSchema(db);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, ?, ?, ?, 1)`,
).run('犬', '犬', 'いぬ', 1_700_000_000, 1_700_000_000);
db.exec('DELETE FROM imm_lexical_daily_rollups');
db.prepare(`UPDATE imm_rollup_state SET state_value = '0' WHERE state_key = ?`).run(
'lexical_daily_rollups_version',
);
executeLexicalRollupBackfillTask(dbPath);
assert.equal(areLexicalDailyRollupsReady(db), true);
assert.equal(getLexicalDailyRollups(db)[0]?.wordCount, 1);
} finally {
db.close();
fs.rmSync(directory, { recursive: true, force: true });
}
});
@@ -0,0 +1,15 @@
import { areLexicalDailyRollupsReady, rebuildLexicalDailyRollups } from './lexical-rollups';
import { Database } from './sqlite';
import { applyPragmas } from './storage';
export function executeLexicalRollupBackfillTask(dbPath: string): void {
const db = new Database(dbPath);
try {
applyPragmas(db);
if (!areLexicalDailyRollupsReady(db)) {
rebuildLexicalDailyRollups(db);
}
} finally {
db.close();
}
}
@@ -0,0 +1,421 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import {
areLexicalDailyRollupsReady,
getLexicalDailyRollups,
rebuildLexicalDailyRollups,
} from './lexical-rollups';
import { getTrendsDashboard } from './query-trends';
import {
getVocabularyChartData,
getVocabularySummary,
replaceStatsExcludedWords,
} from './query-lexical';
import { Database } from './sqlite';
import type { DatabaseSync } from './sqlite';
import { ensureSchema } from './storage';
function makeDbPath(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-lexical-rollups-'));
return path.join(dir, 'immersion.sqlite');
}
test('lexical daily rollups follow first-seen corrections and deletions', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const firstDay = 19_500;
const correctedDay = firstDay + 2;
const firstSeen = firstDay * 86_400 + 43_200;
const correctedSeen = correctedDay * 86_400 + 43_200;
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, ?, ?, ?, 1)`,
).run('猫', '猫', 'ねこ', firstSeen, firstSeen);
db.prepare(
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
VALUES (?, ?, ?, 1)`,
).run('猫', firstSeen, firstSeen);
assert.deepEqual(getLexicalDailyRollups(db), [
{ epochDay: firstDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 1 },
]);
db.prepare(`UPDATE imm_words SET first_seen = ? WHERE headword = ?`).run(correctedSeen, '猫');
db.prepare(`DELETE FROM imm_kanji WHERE kanji = ?`).run('猫');
assert.deepEqual(getLexicalDailyRollups(db), [
{ epochDay: correctedDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
]);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('lexical daily rollups normalize second and millisecond timestamps', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const epochDay = 19_500;
const timestampSeconds = epochDay * 86_400 + 43_200;
const timestampMilliseconds = timestampSeconds * 1_000;
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, ?, ?, ?, 1)`,
).run('猫', '猫', 'ねこ', timestampSeconds, timestampSeconds);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, ?, ?, ?, 1)`,
).run('犬', '犬', 'いぬ', timestampMilliseconds, timestampMilliseconds);
db.prepare(
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
VALUES (?, ?, ?, 1)`,
).run('猫', timestampSeconds, timestampSeconds);
db.prepare(
`INSERT INTO imm_kanji(kanji, first_seen, last_seen, frequency)
VALUES (?, ?, ?, 1)`,
).run('犬', timestampMilliseconds, timestampMilliseconds);
assert.deepEqual(getLexicalDailyRollups(db), [
{ epochDay, wordCount: 2, wordCountWithoutNames: 2, kanjiCount: 2 },
]);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('lexical rollup rebuild excludes rows hidden by vocabulary persistence rules', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const epochDay = 19_500;
const firstSeen = epochDay * 86_400 + 43_200;
db.prepare(
`INSERT INTO imm_words(
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
db.prepare(
`INSERT INTO imm_words(
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
rebuildLexicalDailyRollups(db);
assert.deepEqual(getLexicalDailyRollups(db), [
{ epochDay, wordCount: 1, wordCountWithoutNames: 1, kanjiCount: 0 },
]);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('lexical rollup rebuild tolerates nullable legacy vocabulary text', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (NULL, NULL, NULL, 1700000000, 1700000000, 1)`,
).run();
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (NULL, '猫', 'ねこ', 1700000000, 1700000000, 1)`,
).run();
assert.doesNotThrow(() => rebuildLexicalDailyRollups(db));
assert.equal(areLexicalDailyRollupsReady(db), true);
assert.equal(getVocabularySummary(db, null).uniqueWords, 1);
assert.equal(getVocabularySummary(db, new Set(['猫'])).knownWordCount, 1);
assert.equal(getVocabularyChartData(db).topWords[0]?.headword, '猫');
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('lexical rollup rebuild scans vocabulary visibility in bounded id batches', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
const expectedBatchSize = 5_000;
try {
ensureSchema(db);
const insertWord = db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, '', 1700000000, 1700000000, 1)`,
);
db.exec('BEGIN');
for (let index = 0; index <= expectedBatchSize; index += 1) {
insertWord.run(`${index}`, `${index}`);
}
db.exec('COMMIT');
const scanPageSizes: number[] = [];
const instrumentedDb: DatabaseSync = {
prepare(source) {
const statement = db.prepare(source);
if (!source.includes('WHERE id > ?') || !source.includes('ORDER BY id')) {
return statement;
}
return {
run: (...params) => statement.run(...params),
get: (...params) => statement.get(...params),
all: (...params) => {
const rows = statement.all(...params);
scanPageSizes.push(rows.length);
return rows;
},
};
},
exec(source) {
db.exec(source);
return instrumentedDb;
},
close() {
return instrumentedDb;
},
};
rebuildLexicalDailyRollups(instrumentedDb);
assert.deepEqual(scanPageSizes, [expectedBatchSize, 1]);
assert.equal(getVocabularySummary(db, null).uniqueWords, expectedBatchSize + 1);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('chart exclusions do not subtract vocabulary rows already hidden from the rollup', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const epochDay = 19_500;
const firstSeen = epochDay * 86_400 + 43_200;
db.prepare(
`INSERT INTO imm_words(
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
).run('猫', '猫', 'ねこ', 'noun', firstSeen, firstSeen);
db.prepare(
`INSERT INTO imm_words(
headword, word, reading, part_of_speech, first_seen, last_seen, frequency
) VALUES (?, ?, ?, ?, ?, ?, 1)`,
).run('は', 'は', 'は', 'particle', firstSeen, firstSeen);
rebuildLexicalDailyRollups(db);
replaceStatsExcludedWords(db, [{ headword: 'は', word: 'は', reading: 'は' }]);
const charts = getVocabularyChartData(db);
assert.deepEqual(charts.newWordsTimeline, [{ epochDay, wordCount: 1 }]);
assert.deepEqual(charts.newWordsTimelineWithoutNames, [{ epochDay, wordCount: 1 }]);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('legacy lexical rollup readiness does not satisfy the current rollup version', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
db.prepare(
`INSERT INTO imm_rollup_state(state_key, state_value)
VALUES ('lexical_daily_rollups_ready', '1')
ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
).run();
db.prepare(
`DELETE FROM imm_rollup_state WHERE state_key = 'lexical_daily_rollups_version'`,
).run();
assert.equal(areLexicalDailyRollupsReady(db), false);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('current lexical rollup readiness accepts legacy integer state storage', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
db.exec(`
CREATE TABLE imm_rollup_state(
state_key TEXT PRIMARY KEY,
state_value INTEGER NOT NULL
);
INSERT INTO imm_rollup_state(state_key, state_value)
VALUES ('lexical_daily_rollups_version', 2);
`);
assert.equal(areLexicalDailyRollupsReady(db), true);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('imm_words persists vocabulary visibility for rollup maintenance', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const columns = db.prepare(`PRAGMA table_info(imm_words)`).all() as Array<{ name: string }>;
assert.equal(
columns.some((column) => column.name === 'vocabulary_visible'),
true,
);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('vocabulary charts use complete top-word and lexical rollup data', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const insertWord = db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES (?, ?, '', 1700000000, 1700000000, ?)`,
);
for (let index = 0; index < 501; index += 1) {
insertWord.run(`${index}`, `${index}`, index === 500 ? 10_000 : 1);
}
const charts = getVocabularyChartData(db);
assert.equal(charts.topWords[0]?.headword, '語500');
assert.equal(charts.topWords[0]?.frequency, 10_000);
assert.equal(charts.newWordsTimeline[0]?.wordCount, 501);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('vocabulary charts find full top-word sets beyond excluded and name rows', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
const insertWord = db.prepare(
`INSERT INTO imm_words(headword, word, reading, pos2, first_seen, last_seen, frequency)
VALUES (?, ?, '', ?, 1700000000, 1700000000, ?)`,
);
const exclusions = [];
for (let index = 0; index < 100; index += 1) {
const headword = `${index}`;
insertWord.run(
headword,
headword,
index < 80 && index >= 60 ? '固有名詞' : '一般',
100 - index,
);
if (index < 60) exclusions.push({ headword, word: headword, reading: '' });
}
replaceStatsExcludedWords(db, exclusions);
const charts = getVocabularyChartData(db);
assert.equal(charts.topWords.length, 12);
assert.equal(charts.topWords[0]?.headword, '語60');
assert.equal(charts.topWordsWithoutNames.length, 12);
assert.equal(charts.topWordsWithoutNames[0]?.headword, '語80');
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('vocabulary charts handle exclusion lists above one SQLite variable batch', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES ('語0', '語0', '', 1700000000, 1700000000, 1)`,
).run();
const exclusions = Array.from({ length: 10_923 }, (_, index) => ({
headword: `${index}`,
word: `${index}`,
reading: '',
}));
replaceStatsExcludedWords(db, exclusions);
const charts = getVocabularyChartData(db);
assert.deepEqual(charts.topWords, []);
assert.deepEqual(charts.newWordsTimeline, []);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
test('lexical rollup rebuild preserves the original error when rollback also fails', () => {
const originalError = new Error('rebuild failed');
const db = {
exec(sql: string) {
if (sql === 'BEGIN IMMEDIATE') return;
if (sql === 'ROLLBACK') throw new Error('rollback failed');
throw originalError;
},
prepare() {
return { all: () => [], run: () => undefined };
},
} as unknown as DatabaseSync;
assert.throws(() => rebuildLexicalDailyRollups(db), originalError);
});
test('trends read historical new-word buckets from lexical rollups', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
db.prepare(
`INSERT INTO imm_words(headword, word, reading, first_seen, last_seen, frequency)
VALUES ('海', '海', 'うみ', 1700000000, 1700000000, 1)`,
).run();
db.prepare(`UPDATE imm_lexical_daily_rollups SET word_count = 9`).run();
const dashboard = getTrendsDashboard(db, 'all', 'day', false);
assert.equal(dashboard.progress.newWords[0]?.value, 9);
} finally {
db.close();
fs.rmSync(path.dirname(dbPath), { recursive: true, force: true });
}
});
@@ -0,0 +1,229 @@
import type { DatabaseSync } from './sqlite';
import { isVocabularyStatsRowVisible, type VocabularyVisibilityRow } from './vocabulary-visibility';
export interface LexicalDailyRollup {
epochDay: number;
wordCount: number;
wordCountWithoutNames: number;
kanjiCount: number;
}
const LOCAL_EPOCH_DAY_SQL = `
CAST(julianday(
CASE
WHEN ABS(CAST(%VALUE% AS REAL)) >= 10000000000 THEN CAST(%VALUE% AS REAL) / 1000
ELSE CAST(%VALUE% AS REAL)
END,
'unixepoch', 'localtime'
) - 2440587.5 AS INTEGER)
`;
const LEXICAL_DAILY_ROLLUP_VERSION = '2';
const LEXICAL_DAILY_ROLLUP_VERSION_KEY = 'lexical_daily_rollups_version';
const VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE = 5_000;
export function localEpochDaySql(value: string): string {
return LOCAL_EPOCH_DAY_SQL.replaceAll('%VALUE%', value);
}
function createWordRollupTriggers(db: DatabaseSync): void {
const dayForNew = localEpochDaySql('NEW.first_seen');
const dayForOld = localEpochDaySql('OLD.first_seen');
db.exec(`
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_insert;
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_delete;
DROP TRIGGER IF EXISTS imm_words_lexical_rollup_first_seen_update;
CREATE TRIGGER imm_words_lexical_rollup_insert
AFTER INSERT ON imm_words
WHEN NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
BEGIN
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
VALUES (${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0)
ON CONFLICT(epoch_day) DO UPDATE SET
word_count = word_count + 1,
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
END;
CREATE TRIGGER imm_words_lexical_rollup_delete
AFTER DELETE ON imm_words
WHEN OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
BEGIN
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
VALUES (${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0)
ON CONFLICT(epoch_day) DO UPDATE SET
word_count = word_count - 1,
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
DELETE FROM imm_lexical_daily_rollups
WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
END;
CREATE TRIGGER imm_words_lexical_rollup_first_seen_update
AFTER UPDATE OF first_seen, pos2, vocabulary_visible ON imm_words
WHEN OLD.first_seen IS NOT NEW.first_seen
OR OLD.pos2 IS NOT NEW.pos2
OR OLD.vocabulary_visible IS NOT NEW.vocabulary_visible
BEGIN
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
SELECT ${dayForOld}, -1, CASE WHEN OLD.pos2 = '固有名詞' THEN 0 ELSE -1 END, 0
WHERE OLD.first_seen IS NOT NULL AND OLD.vocabulary_visible = 1
ON CONFLICT(epoch_day) DO UPDATE SET
word_count = word_count - 1,
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
SELECT ${dayForNew}, 1, CASE WHEN NEW.pos2 = '固有名詞' THEN 0 ELSE 1 END, 0
WHERE NEW.first_seen IS NOT NULL AND NEW.vocabulary_visible = 1
ON CONFLICT(epoch_day) DO UPDATE SET
word_count = word_count + 1,
word_count_without_names = word_count_without_names + excluded.word_count_without_names;
DELETE FROM imm_lexical_daily_rollups
WHERE word_count = 0 AND kanji_count = 0;
END;
`);
}
function createKanjiRollupTriggers(db: DatabaseSync): void {
const dayForNew = localEpochDaySql('NEW.first_seen');
const dayForOld = localEpochDaySql('OLD.first_seen');
db.exec(`
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_insert;
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_delete;
DROP TRIGGER IF EXISTS imm_kanji_lexical_rollup_first_seen_update;
CREATE TRIGGER imm_kanji_lexical_rollup_insert
AFTER INSERT ON imm_kanji WHEN NEW.first_seen IS NOT NULL
BEGIN
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
VALUES (${dayForNew}, 0, 0, 1)
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
END;
CREATE TRIGGER imm_kanji_lexical_rollup_delete
AFTER DELETE ON imm_kanji WHEN OLD.first_seen IS NOT NULL
BEGIN
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
VALUES (${dayForOld}, 0, 0, -1)
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
DELETE FROM imm_lexical_daily_rollups
WHERE epoch_day = ${dayForOld} AND word_count = 0 AND kanji_count = 0;
END;
CREATE TRIGGER imm_kanji_lexical_rollup_first_seen_update
AFTER UPDATE OF first_seen ON imm_kanji WHEN OLD.first_seen IS NOT NEW.first_seen
BEGIN
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
SELECT ${dayForOld}, 0, 0, -1 WHERE OLD.first_seen IS NOT NULL
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count - 1;
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
SELECT ${dayForNew}, 0, 0, 1 WHERE NEW.first_seen IS NOT NULL
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + 1;
DELETE FROM imm_lexical_daily_rollups WHERE word_count = 0 AND kanji_count = 0;
END;
`);
}
export function ensureLexicalDailyRollupTables(db: DatabaseSync): void {
db.exec(`
CREATE TABLE IF NOT EXISTS imm_lexical_daily_rollups(
epoch_day INTEGER PRIMARY KEY,
word_count INTEGER NOT NULL DEFAULT 0,
word_count_without_names INTEGER NOT NULL DEFAULT 0,
kanji_count INTEGER NOT NULL DEFAULT 0
);
INSERT INTO imm_rollup_state(state_key, state_value)
VALUES ('${LEXICAL_DAILY_ROLLUP_VERSION_KEY}', '0')
ON CONFLICT(state_key) DO NOTHING;
`);
createWordRollupTriggers(db);
createKanjiRollupTriggers(db);
}
export function areLexicalDailyRollupsReady(db: DatabaseSync): boolean {
const row = db
.prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
.get(LEXICAL_DAILY_ROLLUP_VERSION_KEY) as { value: string | number } | undefined;
// Older databases created this column with INTEGER affinity, while current
// databases use TEXT. SQLite returns the same persisted version with a
// different JS type depending on that legacy schema.
return row !== undefined && String(row.value) === LEXICAL_DAILY_ROLLUP_VERSION;
}
export function markLexicalDailyRollupsReady(db: DatabaseSync): void {
db.prepare(
`INSERT INTO imm_rollup_state(state_key, state_value)
VALUES (?, ?)
ON CONFLICT(state_key) DO UPDATE SET state_value = excluded.state_value`,
).run(LEXICAL_DAILY_ROLLUP_VERSION_KEY, LEXICAL_DAILY_ROLLUP_VERSION);
}
/** Rebuild from the first-seen source of truth; run off the UI/main DB thread. */
export function rebuildLexicalDailyRollups(db: DatabaseSync): void {
let transactionStarted = false;
try {
db.exec('BEGIN IMMEDIATE');
transactionStarted = true;
const scanVocabulary = db.prepare(
`SELECT id, word, headword, reading, part_of_speech AS partOfSpeech,
pos1, pos2, pos3, frequency_rank AS frequencyRank
FROM imm_words
WHERE id > ?
ORDER BY id
LIMIT ?`,
);
const updateVisibility = db.prepare(
`UPDATE imm_words SET vocabulary_visible = ? WHERE id = ? AND vocabulary_visible IS NOT ?`,
);
let lastId = Number.MIN_SAFE_INTEGER;
for (;;) {
const vocabularyRows = scanVocabulary.all(
lastId,
VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE,
) as Array<VocabularyVisibilityRow & { id: number }>;
if (vocabularyRows.length === 0) break;
for (const row of vocabularyRows) {
const visible = isVocabularyStatsRowVisible(row) ? 1 : 0;
updateVisibility.run(visible, row.id, visible);
}
lastId = vocabularyRows[vocabularyRows.length - 1]!.id;
if (vocabularyRows.length < VOCABULARY_VISIBILITY_SCAN_BATCH_SIZE) break;
}
db.exec('DELETE FROM imm_lexical_daily_rollups');
db.exec(`
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
SELECT ${localEpochDaySql('first_seen')}, COUNT(*),
SUM(CASE WHEN pos2 = '固有名詞' THEN 0 ELSE 1 END), 0
FROM imm_words
WHERE first_seen IS NOT NULL AND vocabulary_visible = 1
GROUP BY ${localEpochDaySql('first_seen')};
INSERT INTO imm_lexical_daily_rollups(epoch_day, word_count, word_count_without_names, kanji_count)
SELECT ${localEpochDaySql('first_seen')}, 0, 0, COUNT(*)
FROM imm_kanji
WHERE first_seen IS NOT NULL
GROUP BY ${localEpochDaySql('first_seen')}
ON CONFLICT(epoch_day) DO UPDATE SET kanji_count = kanji_count + excluded.kanji_count;
`);
markLexicalDailyRollupsReady(db);
db.exec('COMMIT');
} catch (error) {
if (transactionStarted) {
try {
db.exec('ROLLBACK');
} catch {
// Preserve the rebuild failure; it is the actionable cause.
}
}
throw error;
}
}
export function getLexicalDailyRollups(db: DatabaseSync): LexicalDailyRollup[] {
return db
.prepare(
`
SELECT epoch_day AS epochDay, word_count AS wordCount,
word_count_without_names AS wordCountWithoutNames, kanji_count AS kanjiCount
FROM imm_lexical_daily_rollups
ORDER BY epoch_day ASC
`,
)
.all() as LexicalDailyRollup[];
}
@@ -1,6 +1,4 @@
import type { DatabaseSync } from './sqlite';
import { PartOfSpeech, type MergedToken } from '../../../types';
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
import type {
KanjiAnimeAppearanceRow,
KanjiDetailRow,
@@ -13,19 +11,38 @@ import type {
SimilarWordRow,
StatsExcludedWordRow,
VocabularyStatsRow,
VocabularyStatsSummary,
WordAnimeAppearanceRow,
WordDetailRow,
WordOccurrenceRow,
} from './types';
import { fromDbTimestamp, toDbTimestamp } from './query-shared';
import { nowMs } from './time';
import {
areLexicalDailyRollupsReady,
getLexicalDailyRollups,
localEpochDaySql,
} from './lexical-rollups';
import { isVocabularyStatsRowVisible } from './vocabulary-visibility';
const VOCABULARY_STATS_FILTER_OVERSAMPLE_FACTOR = 4;
const VOCABULARY_STATS_FILTER_OVERSAMPLE_MIN = 100;
const VOCABULARY_CHART_LIMIT = 12;
const VOCABULARY_CHART_PAGE_SIZE = 100;
const EXCLUSION_ALIAS_BATCH_SIZE = 300;
const VOCABULARY_SUMMARY_SCAN_BATCH_SIZE = 5_000;
const SENTENCE_SEARCH_DEFAULT_LIMIT = 50;
const SENTENCE_SEARCH_MAX_LIMIT = 100;
const KANJI_PATTERN = /\p{Script=Han}/gu;
export interface VocabularyChartData {
ready: boolean;
topWords: Array<{ wordId: number; headword: string; frequency: number }>;
topWordsWithoutNames: Array<{ wordId: number; headword: string; frequency: number }>;
newWordsTimeline: Array<{ epochDay: number; wordCount: number }>;
newWordsTimelineWithoutNames: Array<{ epochDay: number; wordCount: number }>;
}
function resolveSentenceSearchLimit(limit: number): number {
if (!Number.isFinite(limit)) return SENTENCE_SEARCH_DEFAULT_LIMIT;
const normalized = Math.floor(limit);
@@ -73,33 +90,6 @@ function uniqueKanji(text: string): string[] {
return Array.from(new Set(text.match(KANJI_PATTERN) ?? []));
}
function toVocabularyToken(row: VocabularyStatsRow): MergedToken {
const partOfSpeech =
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
? (row.partOfSpeech as PartOfSpeech)
: PartOfSpeech.other;
return {
surface: row.word,
reading: row.reading ?? '',
headword: row.headword,
startPos: 0,
endPos: row.word.length,
partOfSpeech,
pos1: row.pos1 ?? '',
pos2: row.pos2 ?? '',
pos3: row.pos3 ?? '',
frequencyRank: row.frequencyRank ?? undefined,
isMerged: false,
isKnown: false,
isNPlusOneTarget: false,
};
}
function isVocabularyStatsRowVisible(row: VocabularyStatsRow): boolean {
return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
}
export function getVocabularyStats(
db: DatabaseSync,
limit = 100,
@@ -153,6 +143,198 @@ export function getVocabularyStats(
return visibleRows.slice(0, limit);
}
/**
* Chart data is intentionally independent of the paginated vocabulary tables.
* Top words use the frequency index; new-word history reads permanent daily
* lexical rollups rather than loading every vocabulary row into the dashboard.
*/
export function getVocabularyChartData(db: DatabaseSync): VocabularyChartData {
const ready = areLexicalDailyRollupsReady(db);
const excludedAliases = new Set(
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
);
const isExcluded = (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>): boolean =>
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias));
const topWords = getTopVocabularyChartWords(db, isExcluded);
const rollups = ready ? getLexicalDailyRollups(db) : [];
const timeline = new Map(rollups.map((row) => [row.epochDay, { ...row }]));
if (excludedAliases.size > 0 && ready) {
const aliases = [...excludedAliases];
const excludedRows = new Map<
number,
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
wordId: number;
epochDay: number;
}
>();
for (let offset = 0; offset < aliases.length; offset += EXCLUSION_ALIAS_BATCH_SIZE) {
const batch = aliases.slice(offset, offset + EXCLUSION_ALIAS_BATCH_SIZE);
const placeholders = batch.map(() => '?').join(', ');
const rows = db
.prepare(
`
SELECT id AS wordId, headword, word, reading, pos2,
${localEpochDaySql('first_seen')} AS epochDay
FROM imm_words
WHERE vocabulary_visible = 1
AND (headword IN (${placeholders}) OR word IN (${placeholders}) OR reading IN (${placeholders}))
`,
)
.all(...batch, ...batch, ...batch) as Array<
Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading' | 'pos2'> & {
wordId: number;
epochDay: number;
}
>;
for (const row of rows) excludedRows.set(row.wordId, row);
}
for (const word of excludedRows.values()) {
if (!isExcluded(word)) continue;
const rollup = timeline.get(word.epochDay);
if (!rollup) continue;
rollup.wordCount -= 1;
if (word.pos2 !== '固有名詞') rollup.wordCountWithoutNames -= 1;
}
}
return {
ready,
topWords: topWords.all.map((word) => ({
wordId: word.wordId,
headword: vocabularyDisplayHeadword(word),
frequency: word.frequency,
})),
topWordsWithoutNames: topWords.withoutNames.map((word) => ({
wordId: word.wordId,
headword: vocabularyDisplayHeadword(word),
frequency: word.frequency,
})),
newWordsTimeline: [...timeline.values()]
.filter((row) => row.wordCount > 0)
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCount })),
newWordsTimelineWithoutNames: [...timeline.values()]
.filter((row) => row.wordCountWithoutNames > 0)
.map((row) => ({ epochDay: row.epochDay, wordCount: row.wordCountWithoutNames })),
};
}
function getTopVocabularyChartWords(
db: DatabaseSync,
isExcluded: (word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>) => boolean,
): { all: VocabularyStatsRow[]; withoutNames: VocabularyStatsRow[] } {
const stmt = db.prepare(`
SELECT id AS wordId, headword, word, reading,
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
frequency, frequency_rank AS frequencyRank,
first_seen AS firstSeen, last_seen AS lastSeen,
0 AS animeCount
FROM imm_words
ORDER BY frequency DESC, id
LIMIT ? OFFSET ?
`);
const all: VocabularyStatsRow[] = [];
const withoutNames: VocabularyStatsRow[] = [];
let offset = 0;
while (all.length < VOCABULARY_CHART_LIMIT || withoutNames.length < VOCABULARY_CHART_LIMIT) {
const page = stmt.all(VOCABULARY_CHART_PAGE_SIZE, offset) as VocabularyStatsRow[];
if (page.length === 0) break;
for (const word of page) {
if (!isVocabularyStatsRowVisible(word) || isExcluded(word)) continue;
if (all.length < VOCABULARY_CHART_LIMIT) all.push(word);
if (word.pos2 !== '固有名詞' && withoutNames.length < VOCABULARY_CHART_LIMIT) {
withoutNames.push(word);
}
}
offset += page.length;
}
return { all, withoutNames };
}
function excludedVocabularyAliases(
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
): string[] {
const aliases = [word.headword?.trim() ?? '', word.word?.trim() ?? ''].filter(Boolean);
if (aliases.length === 0) aliases.push(word.reading?.trim() ?? '');
return [...new Set(aliases)];
}
function vocabularyDisplayHeadword(
word: Pick<VocabularyStatsRow, 'headword' | 'word' | 'reading'>,
): string {
return word.headword?.trim() || word.word?.trim() || word.reading?.trim() || '';
}
function timestampSeconds(timestamp: number): number {
return timestamp < 10_000_000_000 ? timestamp : Math.floor(timestamp / 1000);
}
export function getVocabularySummary(
db: DatabaseSync,
knownWords: ReadonlySet<string> | null,
nowMs: number = Date.now(),
scanBatchSize: number = VOCABULARY_SUMMARY_SCAN_BATCH_SIZE,
): VocabularyStatsSummary {
// Visibility and exclusion rules live in JS, so rows are scanned in id-keyed
// batches to keep memory bounded on large vocabularies.
const scanStmt = db.prepare(`
SELECT id AS wordId, headword, word, reading,
part_of_speech AS partOfSpeech, pos1, pos2, pos3,
frequency, frequency_rank AS frequencyRank,
first_seen AS firstSeen, last_seen AS lastSeen,
0 AS animeCount
FROM imm_words
WHERE id > ?
ORDER BY id
LIMIT ?
`);
const excludedAliases = new Set(
getStatsExcludedWords(db).flatMap((word) => excludedVocabularyAliases(word)),
);
const weekAgoSec = nowMs / 1000 - 7 * 86_400;
const summary: VocabularyStatsSummary = {
uniqueWords: 0,
uniqueWordsWithoutNames: 0,
uniqueKanji: (db.prepare('SELECT COUNT(*) AS count FROM imm_kanji').get() as { count: number })
.count,
newThisWeek: 0,
newThisWeekWithoutNames: 0,
knownWordCount: knownWords ? 0 : null,
knownWordCountWithoutNames: knownWords ? 0 : null,
};
let lastId = Number.MIN_SAFE_INTEGER;
for (;;) {
const words = scanStmt.all(lastId, scanBatchSize) as VocabularyStatsRow[];
if (words.length === 0) break;
lastId = words[words.length - 1]!.wordId;
for (const word of words) {
if (
!isVocabularyStatsRowVisible(word) ||
excludedVocabularyAliases(word).some((alias) => excludedAliases.has(alias))
) {
continue;
}
const isName = word.pos2 === '固有名詞';
const isNewThisWeek = timestampSeconds(fromDbTimestamp(word.firstSeen) ?? 0) >= weekAgoSec;
const isKnown = knownWords?.has(vocabularyDisplayHeadword(word)) ?? false;
summary.uniqueWords += 1;
if (!isName) summary.uniqueWordsWithoutNames += 1;
if (isNewThisWeek) {
summary.newThisWeek += 1;
if (!isName) summary.newThisWeekWithoutNames += 1;
}
if (isKnown) {
summary.knownWordCount! += 1;
if (!isName) summary.knownWordCountWithoutNames! += 1;
}
}
if (words.length < scanBatchSize) break;
}
return summary;
}
export function getStatsExcludedWords(db: DatabaseSync): StatsExcludedWordRow[] {
return db
.prepare(
@@ -13,6 +13,7 @@ import {
toDbTimestamp,
} from './query-shared';
import { getDailyRollups, getMonthlyRollups } from './query-sessions';
import { areLexicalDailyRollupsReady, getLexicalDailyRollups } from './lexical-rollups';
type TrendRange = '7d' | '30d' | '90d' | '365d' | 'all';
type TrendGroupBy = 'day' | 'month';
@@ -660,6 +661,16 @@ function buildNewWordsPerDay(
cutoffMs: string | null,
axis: number[] | null,
): TrendChartPoint[] {
if (areLexicalDailyRollupsReady(db)) {
// A trend range is defined in calendar buckets, so the rollup includes the
// complete local cutoff day rather than applying a time-of-day boundary.
const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
const rows = getLexicalDailyRollups(db).filter(
(row) => cutoffDay === null || row.epochDay >= cutoffDay,
);
return fillAxisPoints(axis, new Map(rows.map((row) => [row.epochDay, row.wordCount])));
}
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
const prepared = db.prepare(`
SELECT
@@ -691,6 +702,18 @@ function buildNewWordsPerMonth(
cutoffMs: string | null,
axis: number[] | null,
): TrendChartPoint[] {
if (areLexicalDailyRollupsReady(db)) {
const cutoffDay = cutoffMs === null ? null : getLocalEpochDay(db, cutoffMs);
const byMonth = new Map<number, number>();
for (const row of getLexicalDailyRollups(db)) {
if (cutoffDay !== null && row.epochDay < cutoffDay) continue;
const { year, month } = dayPartsFromEpochDay(row.epochDay);
const monthKey = year * 100 + month;
byMonth.set(monthKey, (byMonth.get(monthKey) ?? 0) + row.wordCount);
}
return fillAxisPoints(axis, byMonth);
}
const whereClause = cutoffMs === null ? '' : 'AND first_seen >= ?';
const prepared = db.prepare(`
SELECT
@@ -3,7 +3,7 @@ import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { Database } from './sqlite';
import { Database, type DatabaseSync } from './sqlite';
import { getStatsExcludedWords, replaceStatsExcludedWords } from './query-lexical';
import { finalizeSessionRecord, startSessionRecord } from './session';
import {
@@ -87,6 +87,29 @@ test('applyPragmas sets the SQLite tuning defaults used by immersion tracking',
}
});
test('applyPragmas installs the busy timeout before WAL negotiation', () => {
const statements: string[] = [];
const db: DatabaseSync = {
exec(source) {
statements.push(source);
return db;
},
prepare() {
throw new Error('not used');
},
close() {
return db;
},
};
applyPragmas(db);
assert.deepEqual(statements.slice(0, 2), [
'PRAGMA busy_timeout = 2500',
'PRAGMA journal_mode = WAL',
]);
});
test('ensureSchema creates immersion core tables', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
@@ -184,6 +207,51 @@ test('ensureSchema adds manual assignment locks when upgrading the previous sche
}
});
test('ensureSchema preserves durable session rollups across unrelated schema upgrades', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
try {
ensureSchema(db);
db.exec(`
INSERT INTO imm_videos (
video_id, video_key, canonical_title, source_type, duration_ms, CREATED_DATE, LAST_UPDATE_DATE
) VALUES (1, 'local:/tmp/preserved.mkv', 'Preserved', 1, 0, '1', '1');
INSERT INTO imm_daily_rollups (
rollup_day, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards
) VALUES (20000, 1, 2, 30, 40, 50, 3);
INSERT INTO imm_monthly_rollups (
rollup_month, video_id, total_sessions, total_active_min, total_lines_seen,
total_tokens_seen, total_cards
) VALUES (202410, 1, 2, 30, 40, 50, 3);
UPDATE imm_rollup_state
SET state_value = '123'
WHERE state_key = 'last_rollup_sample_ms';
UPDATE imm_schema_version SET schema_version = 21;
`);
ensureSchema(db);
const daily = db
.prepare('SELECT total_sessions AS totalSessions FROM imm_daily_rollups')
.get() as { totalSessions: number } | null;
const monthly = db
.prepare('SELECT total_sessions AS totalSessions FROM imm_monthly_rollups')
.get() as { totalSessions: number } | null;
const rollupState = db
.prepare(`SELECT state_value AS value FROM imm_rollup_state WHERE state_key = ?`)
.get('last_rollup_sample_ms') as { value: string } | null;
assert.equal(daily?.totalSessions, 2);
assert.equal(monthly?.totalSessions, 2);
assert.equal(rollupState?.value, '123');
} finally {
db.close();
cleanupDbPath(dbPath);
}
});
test('stats excluded words are replaced and read from sqlite storage', () => {
const dbPath = makeDbPath();
const db = new Database(dbPath);
+29 -11
View File
@@ -4,6 +4,7 @@ import { parseMediaInfo } from '../../../jimaku/utils';
import { normalizeTitleIdentity } from '../../utils/title-normalization';
import type { DatabaseSync } from './sqlite';
import { nowMs } from './time';
import { ensureLexicalDailyRollupTables, markLexicalDailyRollupsReady } from './lexical-rollups';
import { SCHEMA_VERSION } from './types';
import type { QueuedWrite, VideoMetadata, YoutubeVideoMetadata } from './types';
import { toDbMs, toDbTimestamp } from './query-shared';
@@ -314,10 +315,12 @@ function migrateSessionEventTimestampsToText(db: DatabaseSync): void {
}
export function applyPragmas(db: DatabaseSync): void {
// Install the wait policy before WAL negotiation, which can briefly contend with
// another connection closing or checkpointing the same database.
db.exec('PRAGMA busy_timeout = 2500');
db.exec('PRAGMA journal_mode = WAL');
db.exec('PRAGMA synchronous = NORMAL');
db.exec('PRAGMA foreign_keys = ON');
db.exec('PRAGMA busy_timeout = 2500');
db.exec(`PRAGMA journal_size_limit = ${WAL_JOURNAL_SIZE_LIMIT_BYTES}`);
}
@@ -890,11 +893,11 @@ export function ensureSchema(db: DatabaseSync): void {
VALUES ('last_rollup_sample_ms', 0)
ON CONFLICT(state_key) DO NOTHING
`);
const currentVersion = db
.prepare('SELECT schema_version FROM imm_schema_version ORDER BY schema_version DESC LIMIT 1')
.get() as { schema_version: number } | null;
if (currentVersion?.schema_version === SCHEMA_VERSION) {
ensureLexicalDailyRollupTables(db);
ensureLifetimeSummaryTables(db);
ensureStatsExcludedWordsTable(db);
ensureAnimeMergeTables(db);
@@ -1068,6 +1071,7 @@ export function ensureSchema(db: DatabaseSync): void {
last_seen REAL,
frequency INTEGER,
frequency_rank INTEGER,
vocabulary_visible INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1)),
UNIQUE(headword, word, reading)
);
`);
@@ -1451,8 +1455,18 @@ export function ensureSchema(db: DatabaseSync): void {
addColumnIfMissing(db, 'imm_sessions', 'ended_media_ms', 'INTEGER');
}
if (currentVersion?.schema_version && currentVersion.schema_version < 23) {
addColumnIfMissing(
db,
'imm_words',
'vocabulary_visible',
'INTEGER NOT NULL DEFAULT 1 CHECK(vocabulary_visible IN (0, 1))',
);
}
migrateSessionEventTimestampsToText(db);
ensureLexicalDailyRollupTables(db);
ensureLifetimeSummaryTables(db);
ensureStatsExcludedWordsTable(db);
@@ -1572,19 +1586,21 @@ export function ensureSchema(db: DatabaseSync): void {
ON imm_youtube_videos(youtube_video_id)
`);
if (currentVersion?.schema_version && currentVersion.schema_version < SCHEMA_VERSION) {
db.exec('DELETE FROM imm_daily_rollups');
db.exec('DELETE FROM imm_monthly_rollups');
db.exec(
`UPDATE imm_rollup_state SET state_value = 0 WHERE state_key = 'last_rollup_sample_ms'`,
);
}
// Session rollups intentionally outlive raw session and telemetry retention.
// Preserve them across unrelated schema upgrades because deleted historical
// buckets cannot be rebuilt after their source rows have been pruned.
db.exec(`
INSERT INTO imm_schema_version(schema_version, applied_at_ms)
VALUES (${SCHEMA_VERSION}, ${toDbTimestamp(nowMs())})
ON CONFLICT DO NOTHING
`);
// A new database has no history to materialize. Upgrades are populated by the
// background worker so startup never scans the existing vocabulary table.
if (!currentVersion) {
markLexicalDailyRollupsReady(db);
}
}
export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPreparedStatements {
@@ -1617,9 +1633,10 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
`),
wordUpsertStmt: db.prepare(`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency, frequency_rank
headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen,
frequency, frequency_rank, vocabulary_visible
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?
?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, 1
)
ON CONFLICT(headword, word, reading) DO UPDATE SET
frequency = COALESCE(frequency, 0) + 1,
@@ -1632,6 +1649,7 @@ export function createTrackerPreparedStatements(db: DatabaseSync): TrackerPrepar
pos1 = COALESCE(NULLIF(imm_words.pos1, ''), excluded.pos1),
pos2 = COALESCE(NULLIF(imm_words.pos2, ''), excluded.pos2),
pos3 = COALESCE(NULLIF(imm_words.pos3, ''), excluded.pos3),
vocabulary_visible = 1,
first_seen = MIN(COALESCE(first_seen, excluded.first_seen), excluded.first_seen),
last_seen = MAX(COALESCE(last_seen, excluded.last_seen), excluded.last_seen),
frequency_rank = CASE
+11 -1
View File
@@ -1,4 +1,4 @@
export const SCHEMA_VERSION = 21;
export const SCHEMA_VERSION = 23;
export const DEFAULT_QUEUE_CAP = 1_000;
export const DEFAULT_BATCH_SIZE = 25;
export const DEFAULT_FLUSH_INTERVAL_MS = 500;
@@ -306,6 +306,16 @@ export interface VocabularyStatsRow {
lastSeen: number;
}
export interface VocabularyStatsSummary {
uniqueWords: number;
uniqueWordsWithoutNames: number;
uniqueKanji: number;
newThisWeek: number;
newThisWeekWithoutNames: number;
knownWordCount: number | null;
knownWordCountWithoutNames: number | null;
}
export interface StatsExcludedWordRow {
headword: string;
word: string;
@@ -0,0 +1,100 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import {
resolveVocabularySummaryWorkerPath,
VocabularySummaryWorkerRuntime,
} from './vocabulary-summary-worker-runtime';
import { Database } from './sqlite';
import { applyPragmas, ensureSchema } from './storage';
test('vocabulary summary worker reads the database from a separate connection', async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-vocabulary-summary-worker-'));
const dbPath = path.join(tempDir, 'immersion.sqlite');
const runtime = new VocabularySummaryWorkerRuntime();
const db = new Database(dbPath);
try {
applyPragmas(db);
ensureSchema(db);
db.prepare(
`
INSERT INTO imm_words (
headword, word, reading, part_of_speech, pos1, pos2, pos3,
first_seen, last_seen, frequency
) VALUES ('猫', '猫', 'ねこ', 'noun', '名詞', '一般', '', 1, 1, 1)
`,
).run();
db.close();
const summary = await runtime.run(dbPath, new Set(['猫']));
assert.equal(summary.uniqueWords, 1);
assert.equal(summary.knownWordCount, 1);
} finally {
runtime.destroy();
try {
db.close();
} catch {
// The worker needs the setup connection closed before it starts.
}
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
test('vocabulary summary worker module resolves in the current layout', () => {
const workerPath = resolveVocabularySummaryWorkerPath();
assert.ok(workerPath, 'expected the vocabulary summary worker module to resolve');
assert.ok(workerPath.endsWith(__filename.endsWith('.ts') ? '.ts' : '.js'));
});
test('vocabulary summary worker never falls back to the caller thread', async () => {
const runtime = new VocabularySummaryWorkerRuntime({
resolveWorkerPath: () => null,
warn: () => {},
});
try {
await assert.rejects(
runtime.run('/tmp/subminer-summary-worker-not-used.sqlite', null),
/worker unavailable/i,
);
} finally {
runtime.destroy();
}
});
test('vocabulary summary worker times out when it never responds', async () => {
let terminated = false;
const runtime = new VocabularySummaryWorkerRuntime({
resolveWorkerPath: () => '/tmp/fake-worker.js',
createWorker: async () => ({
once() {
return this;
},
terminate: async () => {
terminated = true;
return 0;
},
}),
timeoutMs: 1,
warn: () => {},
} as never);
try {
const outcome = await Promise.race([
runtime.run('/tmp/not-used.sqlite', null).then(
() => 'resolved',
(error: unknown) => String(error),
),
new Promise<string>((resolve) => setTimeout(() => resolve('still pending'), 50)),
]);
assert.match(outcome, /timed out/);
assert.equal(terminated, true);
} finally {
runtime.destroy();
}
});
@@ -0,0 +1,133 @@
import fs from 'node:fs';
import path from 'node:path';
import { createLogger } from '../../../logger';
import type { VocabularyStatsSummary } from './types';
interface VocabularySummaryWorkerResponse {
summary?: VocabularyStatsSummary;
error?: unknown;
}
interface VocabularySummaryWorkerHandle {
once(event: 'message', listener: (message: VocabularySummaryWorkerResponse) => void): this;
once(event: 'error', listener: (error: Error) => void): this;
once(event: 'exit', listener: (code: number) => void): this;
terminate(): Promise<number>;
}
interface VocabularySummaryWorkerRuntimeOptions {
resolveWorkerPath?: () => string | null;
createWorker?: (
workerPath: string,
workerData: { dbPath: string; knownWords: string[] | null },
) => Promise<VocabularySummaryWorkerHandle>;
timeoutMs?: number;
warn?: (message: string, ...meta: unknown[]) => void;
}
export type RunVocabularySummaryTask = (
dbPath: string,
knownWords: ReadonlySet<string> | null,
) => Promise<VocabularyStatsSummary>;
export function resolveVocabularySummaryWorkerPath(): string | null {
const fileName = __filename.endsWith('.ts')
? 'vocabulary-summary-worker-thread.ts'
: 'vocabulary-summary-worker-thread.js';
const workerPath = path.join(__dirname, fileName);
return fs.existsSync(workerPath) ? workerPath : null;
}
const logger = createLogger('main:immersion-tracker:vocabulary-summary-worker');
const DEFAULT_WORKER_TIMEOUT_MS = 5 * 60 * 1_000;
export class VocabularySummaryWorkerRuntime {
private readonly activeWorkers = new Set<VocabularySummaryWorkerHandle>();
private destroyed = false;
constructor(private readonly options: VocabularySummaryWorkerRuntimeOptions = {}) {}
async run(
dbPath: string,
knownWords: ReadonlySet<string> | null,
): Promise<VocabularyStatsSummary> {
if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
const workerData = { dbPath, knownWords: knownWords ? [...knownWords] : null };
let worker: VocabularySummaryWorkerHandle;
try {
const workerPath = (this.options.resolveWorkerPath ?? resolveVocabularySummaryWorkerPath)();
if (!workerPath) throw new Error('Emitted vocabulary summary worker module was not found');
const createWorker =
this.options.createWorker ??
(async (resolvedPath, data) => {
const { Worker } = await import('node:worker_threads');
return new Worker(resolvedPath, { workerData: data });
});
worker = await createWorker(workerPath, workerData);
} catch (error) {
if (this.destroyed) throw new Error('Vocabulary summary worker is shut down');
(this.options.warn ?? logger.warn)(
'Vocabulary summary worker unavailable; refusing to scan vocabulary on the current thread',
error,
);
throw new Error('Vocabulary summary worker unavailable');
}
if (this.destroyed) {
await worker.terminate().catch(() => undefined);
throw new Error('Vocabulary summary worker is shut down');
}
return new Promise<VocabularyStatsSummary>((resolve, reject) => {
let settled = false;
let timeout: ReturnType<typeof setTimeout> | null = null;
this.activeWorkers.add(worker);
const settle = (result: VocabularyStatsSummary | Error) => {
if (settled) return;
settled = true;
if (timeout) clearTimeout(timeout);
this.activeWorkers.delete(worker);
void worker.terminate().catch(() => undefined);
if (result instanceof Error) reject(result);
else resolve(result);
};
timeout = setTimeout(
() => settle(new Error('Vocabulary summary worker timed out')),
this.options.timeoutMs ?? DEFAULT_WORKER_TIMEOUT_MS,
);
worker.once('message', (message) => {
if (message.summary) {
settle(message.summary);
return;
}
settle(
new Error(
`Vocabulary summary failed: ${String(message.error ?? 'unknown worker error')}`,
),
);
});
worker.once('error', (error) => settle(error));
worker.once('exit', (code) => {
if (!settled) {
settle(
new Error(
code === 0
? 'Vocabulary summary worker exited without a response'
: `Vocabulary summary worker exited with code ${code}`,
),
);
}
});
});
}
destroy(): void {
if (this.destroyed) return;
this.destroyed = true;
for (const worker of this.activeWorkers) {
void worker.terminate().catch(() => undefined);
}
this.activeWorkers.clear();
}
}
@@ -0,0 +1,19 @@
import { parentPort, workerData } from 'node:worker_threads';
import { executeVocabularySummaryTask } from './vocabulary-summary-worker';
interface VocabularySummaryWorkerData {
dbPath: string;
knownWords: string[] | null;
}
if (!parentPort) throw new Error('vocabulary summary worker missing parent port');
const request = workerData as VocabularySummaryWorkerData;
try {
parentPort.postMessage({
summary: executeVocabularySummaryTask(request.dbPath, request.knownWords),
});
} catch (error) {
parentPort.postMessage({ error: error instanceof Error ? error.message : String(error) });
}
@@ -0,0 +1,17 @@
import { getVocabularySummary } from './query-lexical';
import { Database } from './sqlite';
import { applyPragmas } from './storage';
import type { VocabularyStatsSummary } from './types';
export function executeVocabularySummaryTask(
dbPath: string,
knownWords: string[] | null,
): VocabularyStatsSummary {
const db = new Database(dbPath);
try {
applyPragmas(db);
return getVocabularySummary(db, knownWords ? new Set(knownWords) : null);
} finally {
db.close();
}
}
@@ -0,0 +1,43 @@
import { PartOfSpeech, type MergedToken } from '../../../types';
import { shouldExcludeTokenFromVocabularyPersistence } from '../tokenizer/annotation-stage';
export interface VocabularyVisibilityRow {
word: string | null;
headword: string | null;
reading?: string | null;
partOfSpeech?: string | null;
pos1?: string | null;
pos2?: string | null;
pos3?: string | null;
frequencyRank?: number | null;
}
function toVocabularyToken(row: VocabularyVisibilityRow): MergedToken {
const word = row.word ?? '';
const headword = row.headword ?? word;
const partOfSpeech =
row.partOfSpeech && Object.values(PartOfSpeech).includes(row.partOfSpeech as PartOfSpeech)
? (row.partOfSpeech as PartOfSpeech)
: PartOfSpeech.other;
return {
surface: word,
reading: row.reading ?? '',
headword,
startPos: 0,
endPos: word.length,
partOfSpeech,
pos1: row.pos1 ?? '',
pos2: row.pos2 ?? '',
pos3: row.pos3 ?? '',
frequencyRank: row.frequencyRank ?? undefined,
isMerged: false,
isKnown: false,
isNPlusOneTarget: false,
};
}
export function isVocabularyStatsRowVisible(row: VocabularyVisibilityRow): boolean {
if (!(row.word?.trim() || row.headword?.trim())) return false;
return !shouldExcludeTokenFromVocabularyPersistence(toVocabularyToken(row));
}
+5 -6
View File
@@ -50,6 +50,11 @@ export {
} from './tokenizer/yomitan-parser-runtime';
export { syncYomitanDefaultAnkiServer } from './tokenizer/yomitan-parser-runtime';
export { createSubtitleProcessingController } from './subtitle-processing-controller';
export {
resolveSanitizedSubtitleSeekCommand,
subtitleCueListSeekTime,
subtitleCueSeekTime,
} from './subtitle-cue-navigation';
export { createFrequencyDictionaryLookup } from './frequency-dictionary';
export { createJlptVocabularyLookup } from './jlpt-vocab';
export {
@@ -126,12 +131,6 @@ export {
resolvePlaybackPlan as resolveJellyfinPlaybackPlanRuntime,
ticksToSeconds as jellyfinTicksToSecondsRuntime,
} from './jellyfin';
export { loadJellyfinSubtitleDelay, saveJellyfinSubtitleDelay } from './jellyfin-subtitle-delay';
export {
estimateSubtitleTimingOffset,
type SubtitleTimingOffsetOptions,
type SubtitleTimingOffsetResult,
} from './subtitle-timing-offset';
export { buildJellyfinTimelinePayload, JellyfinRemoteSessionService } from './jellyfin-remote';
export {
broadcastRuntimeOptionsChangedRuntime,
+8 -1
View File
@@ -34,6 +34,7 @@ import {
parseSubsyncManualRunRequest,
parseYoutubePickerResolveRequest,
} from '../../shared/ipc/validators';
import { applyOverlayClickThrough } from './overlay-click-through';
const { ipcMain } = electron;
@@ -442,7 +443,13 @@ export function registerIpcHandlers(deps: IpcServiceDeps, ipc: IpcMainRegistrar
const senderWindow =
electron.BrowserWindow?.fromWebContents((event as IpcMainEvent).sender) ?? null;
if (senderWindow && !senderWindow.isDestroyed()) {
senderWindow.setIgnoreMouseEvents(ignore, parsedOptions);
// Route forwarding requests through the platform-aware helper so Windows never
// installs Electron's global mouse hook (see overlay-click-through.ts).
if (ignore && parsedOptions?.forward) {
applyOverlayClickThrough(senderWindow);
} else {
senderWindow.setIgnoreMouseEvents(ignore, parsedOptions);
}
}
deps.onOverlayMouseInteractionChanged?.(!ignore, senderWindow);
},
@@ -1,54 +0,0 @@
import assert from 'node:assert/strict';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import test from 'node:test';
import { loadJellyfinSubtitleDelay, saveJellyfinSubtitleDelay } from './jellyfin-subtitle-delay';
function statePath(name: string): string {
return path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-jellyfin-delay-')), name);
}
test('jellyfin subtitle delay store saves and loads delay by item and stream', () => {
const filePath = statePath('delays.json');
assert.equal(
saveJellyfinSubtitleDelay({
filePath,
itemId: 'episode-1',
streamIndex: 3,
delaySeconds: 1.25,
}),
true,
);
assert.equal(loadJellyfinSubtitleDelay({ filePath, itemId: 'episode-1', streamIndex: 3 }), 1.25);
assert.equal(loadJellyfinSubtitleDelay({ filePath, itemId: 'episode-1', streamIndex: 4 }), null);
});
test('jellyfin subtitle delay store preserves other stream delays when updating one stream', () => {
const filePath = statePath('delays.json');
saveJellyfinSubtitleDelay({ filePath, itemId: 'episode-1', streamIndex: 3, delaySeconds: 1.25 });
saveJellyfinSubtitleDelay({ filePath, itemId: 'episode-1', streamIndex: 4, delaySeconds: -0.5 });
saveJellyfinSubtitleDelay({ filePath, itemId: 'episode-1', streamIndex: 3, delaySeconds: 2 });
assert.equal(loadJellyfinSubtitleDelay({ filePath, itemId: 'episode-1', streamIndex: 3 }), 2);
assert.equal(loadJellyfinSubtitleDelay({ filePath, itemId: 'episode-1', streamIndex: 4 }), -0.5);
});
test('jellyfin subtitle delay store ignores invalid files and values', () => {
const filePath = statePath('delays.json');
fs.writeFileSync(filePath, '{');
assert.equal(loadJellyfinSubtitleDelay({ filePath, itemId: 'episode-1', streamIndex: 3 }), null);
assert.equal(
saveJellyfinSubtitleDelay({
filePath,
itemId: 'episode-1',
streamIndex: 3,
delaySeconds: Number.NaN,
}),
false,
);
});
@@ -1,66 +0,0 @@
import * as fs from 'fs';
import * as path from 'path';
type JellyfinSubtitleDelayStore = {
version?: unknown;
delays?: unknown;
};
type JellyfinSubtitleDelayParams = {
filePath: string;
itemId: string;
streamIndex: number;
};
type SaveJellyfinSubtitleDelayParams = JellyfinSubtitleDelayParams & {
delaySeconds: number;
};
function storeKey(itemId: string, streamIndex: number): string {
return JSON.stringify([itemId, streamIndex]);
}
function readDelayMap(filePath: string): Record<string, number> {
try {
if (!fs.existsSync(filePath)) return {};
const parsed = JSON.parse(fs.readFileSync(filePath, 'utf-8')) as JellyfinSubtitleDelayStore;
if (
!parsed ||
typeof parsed !== 'object' ||
!parsed.delays ||
typeof parsed.delays !== 'object'
) {
return {};
}
const delays: Record<string, number> = {};
for (const [key, value] of Object.entries(parsed.delays as Record<string, unknown>)) {
if (typeof value === 'number' && Number.isFinite(value)) {
delays[key] = value;
}
}
return delays;
} catch {
return {};
}
}
export function loadJellyfinSubtitleDelay(params: JellyfinSubtitleDelayParams): number | null {
const delay = readDelayMap(params.filePath)[storeKey(params.itemId, params.streamIndex)];
return typeof delay === 'number' && Number.isFinite(delay) ? delay : null;
}
export function saveJellyfinSubtitleDelay(params: SaveJellyfinSubtitleDelayParams): boolean {
if (!Number.isFinite(params.delaySeconds)) return false;
try {
const delays = readDelayMap(params.filePath);
delays[storeKey(params.itemId, params.streamIndex)] = params.delaySeconds;
const dir = path.dirname(params.filePath);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
fs.writeFileSync(params.filePath, JSON.stringify({ version: 1, delays }, null, 2));
return true;
} catch {
return false;
}
}
+121 -39
View File
@@ -125,9 +125,83 @@ test('mineSentenceCard creates sentence card from mpv subtitle state', async ()
]);
});
test('mineSentenceCard refreshes secondary subtitle text before creating card', async () => {
test('mineSentenceCard prefers a canonical primary subtitle snapshot', async () => {
const created: Array<{
sentence: string;
startTime: number;
endTime: number;
secondarySub?: string;
}> = [];
await mineSentenceCard({
ankiIntegration: {
updateLastAddedFromClipboard: async () => {},
triggerFieldGroupingForLastAddedCard: async () => {},
markLastCardAsAudioCard: async () => {},
createSentenceCard: async (sentence, startTime, endTime, secondarySub) => {
created.push({ sentence, startTime, endTime, secondarySub });
return true;
},
},
mpvClient: {
connected: true,
currentSubText: '今今今手手手',
currentSubStart: 11.4,
currentSubEnd: 11.8,
currentSecondarySubText: 'English subtitle',
},
primarySubtitle: {
text: '今 手にある物差しでは',
startTime: 11.13,
endTime: 13.83,
},
showMpvOsd: () => {},
});
assert.deepEqual(created, [
{
sentence: '今 手にある物差しでは',
startTime: 11.13,
endTime: 13.83,
secondarySub: 'English subtitle',
},
]);
});
test('mineSentenceCard uses normalized secondary subtitle state instead of raw mpv text', async () => {
const created: Array<{ sentence: string; secondarySub?: string }> = [];
let requestedRawSecondaryText = false;
await mineSentenceCard({
ankiIntegration: {
updateLastAddedFromClipboard: async () => {},
triggerFieldGroupingForLastAddedCard: async () => {},
markLastCardAsAudioCard: async () => {},
createSentenceCard: async (sentence, _startTime, _endTime, secondarySub) => {
created.push({ sentence, secondarySub });
return true;
},
},
mpvClient: {
connected: true,
currentSubText: '日本語字幕',
currentSubStart: 10,
currentSubEnd: 12,
currentSecondarySubText: 'Your\nmosaic',
requestProperty: async () => {
requestedRawSecondaryText = true;
return 'Your\nYour\nYour\nYour\nmosaic';
},
},
showMpvOsd: () => {},
});
assert.equal(requestedRawSecondaryText, false);
assert.deepEqual(created, [{ sentence: '日本語字幕', secondarySub: 'Your\nmosaic' }]);
});
test('mineSentenceCard omits normalized secondary text that matches the primary subtitle', async () => {
const created: Array<{ sentence: string; secondarySub?: string }> = [];
const requestedProperties: string[] = [];
await mineSentenceCard({
ankiIntegration: {
@@ -145,43 +219,6 @@ test('mineSentenceCard refreshes secondary subtitle text before creating card',
currentSubStart: 10,
currentSubEnd: 12,
currentSecondarySubText: '日本語字幕',
requestProperty: async (name: string) => {
requestedProperties.push(name);
return name === 'secondary-sub-text' ? 'English subtitle' : null;
},
},
showMpvOsd: () => {},
});
assert.deepEqual(requestedProperties, ['secondary-sub-text']);
assert.deepEqual(created, [{ sentence: '日本語字幕', secondarySub: 'English subtitle' }]);
});
test('mineSentenceCard does not fall back to stale cached secondary subtitle after successful refresh', async () => {
const created: Array<{ sentence: string; secondarySub?: string }> = [];
await mineSentenceCard({
ankiIntegration: {
updateLastAddedFromClipboard: async () => {},
triggerFieldGroupingForLastAddedCard: async () => {},
markLastCardAsAudioCard: async () => {},
createSentenceCard: async (sentence, _startTime, _endTime, secondarySub) => {
created.push({ sentence, secondarySub });
return true;
},
},
mpvClient: {
connected: true,
currentSubText: '日本語字幕',
currentSubStart: 10,
currentSubEnd: 12,
currentSecondarySubText: 'stale cached subtitle',
requestProperty: async (name: string) => {
if (name === 'secondary-sub-text') {
return '';
}
return null;
},
},
showMpvOsd: () => {},
});
@@ -207,6 +244,35 @@ test('handleMultiCopyDigit copies available history and reports truncation', ()
assert.equal(osd.at(-1), 'Only 2 lines available, copied 2');
});
test('handleMultiCopyDigit copies backward from the current subtitle after a backward seek', () => {
const copied: string[] = [];
const tracker = new SubtitleTimingTracker();
try {
tracker.recordSubtitle('A', 1, 2);
tracker.recordSubtitle('B', 3, 4);
tracker.recordSubtitle('C', 5, 6);
tracker.recordSubtitle('B', 3, 4);
const deps = {
subtitleTimingTracker: tracker,
writeClipboardText: (text: string) => copied.push(text),
showMpvOsd: () => {},
};
handleMultiCopyDigit(1, deps);
handleMultiCopyDigit(2, deps);
assert.deepEqual(copied, ['B', 'A\n\nB']);
assert.deepEqual(tracker.getRecentEntries(2), [
{ displayText: 'A', startTime: 1, endTime: 2, secondaryText: undefined },
{ displayText: 'B', startTime: 3, endTime: 4, secondaryText: undefined },
]);
} finally {
tracker.destroy();
}
});
test('handleMineSentenceDigit reports async create failures', async () => {
const osd: string[] = [];
const logs: Array<{ message: string; err: unknown }> = [];
@@ -307,6 +373,22 @@ test('handleMineSentenceDigit keeps per-entry timings when subtitle text repeats
}
});
test('subtitle timing history preserves adjacent repeated text with distinct timings', () => {
const tracker = new SubtitleTimingTracker();
try {
tracker.recordSubtitle('same', 1, 2);
tracker.recordSubtitle('same', 3, 4);
assert.deepEqual(tracker.getRecentEntries(2), [
{ displayText: 'same', startTime: 1, endTime: 2, secondaryText: undefined },
{ displayText: 'same', startTime: 3, endTime: 4, secondaryText: undefined },
]);
} finally {
tracker.destroy();
}
});
test('handleMineSentenceDigit joins per-entry secondary subtitles when available', async () => {
const created: Array<{ sentence: string; secondarySub?: string }> = [];
const tracker = new SubtitleTimingTracker();
+10 -16
View File
@@ -129,18 +129,10 @@ function normalizeSecondarySubText(text: unknown, primaryText: string): string |
return trimmed;
}
async function getCurrentSecondarySubTextForSentenceCard(
function getCurrentSecondarySubTextForSentenceCard(
mpvClient: MpvClientLike,
): Promise<string | undefined> {
const primaryText = mpvClient.currentSubText;
if (mpvClient.requestProperty) {
try {
const latestSecondaryText = await mpvClient.requestProperty('secondary-sub-text');
return normalizeSecondarySubText(latestSecondaryText, primaryText);
} catch {
// Fall back to the cached secondary subtitle below.
}
}
primaryText: string,
): string | undefined {
return normalizeSecondarySubText(mpvClient.currentSecondarySubText, primaryText);
}
@@ -175,6 +167,7 @@ export async function markLastCardAsAudioCard(deps: {
export async function mineSentenceCard(deps: {
ankiIntegration: AnkiIntegrationLike | null;
mpvClient: MpvClientLike | null;
primarySubtitle?: Pick<SubtitleMiningContext, 'text' | 'startTime' | 'endTime'>;
showMpvOsd: (text: string) => void;
}): Promise<boolean> {
const anki = requireAnkiIntegration(deps.ankiIntegration, deps.showMpvOsd);
@@ -185,16 +178,17 @@ export async function mineSentenceCard(deps: {
deps.showMpvOsd('MPV not connected');
return false;
}
if (!mpvClient.currentSubText) {
const primaryText = deps.primarySubtitle?.text ?? mpvClient.currentSubText;
if (!primaryText) {
deps.showMpvOsd('No current subtitle');
return false;
}
const secondarySubText = await getCurrentSecondarySubTextForSentenceCard(mpvClient);
const secondarySubText = getCurrentSecondarySubTextForSentenceCard(mpvClient, primaryText);
return await anki.createSentenceCard(
mpvClient.currentSubText,
mpvClient.currentSubStart,
mpvClient.currentSubEnd,
primaryText,
deps.primarySubtitle?.startTime ?? mpvClient.currentSubStart,
deps.primarySubtitle?.endTime ?? mpvClient.currentSubEnd,
secondarySubText,
);
}
+2
View File
@@ -65,6 +65,8 @@ const MPV_SUBTITLE_PROPERTY_OBSERVATIONS: string[] = [
'secondary-sub-visibility',
'sub-visibility',
'sid',
'secondary-sid',
'secondary-sub-delay',
'track-list',
];
+67 -1
View File
@@ -63,6 +63,8 @@ function createDeps(overrides: Partial<MpvProtocolHandleMessageDeps> = {}): {
emitSubtitleTiming: (payload) => state.events.push(payload),
emitSecondarySubtitleChange: (payload) => state.events.push(payload),
emitSubtitleTrackChange: (payload) => state.events.push(payload),
emitSecondarySubtitleTrackChange: (payload) => state.events.push(payload),
emitSecondarySubtitleDelayChange: (payload) => state.events.push(payload),
emitSubtitleTrackListChange: (payload) => state.events.push(payload),
getCurrentSubText: () => state.subText,
setCurrentSubText: (text) => {
@@ -81,6 +83,7 @@ function createDeps(overrides: Partial<MpvProtocolHandleMessageDeps> = {}): {
state.secondarySubText = text;
},
resolvePendingRequest: () => false,
shouldEnforceSecondarySubVisibilityHidden: () => true,
setSecondarySubVisibility: () => {},
syncCurrentAudioStreamIndex: () => {},
setCurrentAudioTrackId: () => {},
@@ -158,12 +161,57 @@ test('dispatchMpvProtocolMessage emits subtitle track changes', async () => {
});
await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sid', data: '3' }, deps);
await dispatchMpvProtocolMessage(
{ event: 'property-change', name: 'secondary-sid', data: '4' },
deps,
);
await dispatchMpvProtocolMessage(
{ event: 'property-change', name: 'secondary-sub-delay', data: '0.5' },
deps,
);
await dispatchMpvProtocolMessage(
{ event: 'property-change', name: 'track-list', data: [{ type: 'sub', id: 3 }] },
deps,
);
assert.deepEqual(state.events, [{ sid: 3 }, { trackList: [{ type: 'sub', id: 3 }] }]);
assert.deepEqual(state.events, [
{ sid: 3 },
{ sid: 4 },
{ delay: 0.5 },
{ trackList: [{ type: 'sub', id: 3 }] },
]);
});
test('dispatchMpvProtocolMessage rejects decimal subtitle track IDs', async () => {
const { deps, state } = createDeps();
await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sid', data: '4.5' }, deps);
await dispatchMpvProtocolMessage(
{ event: 'property-change', name: 'secondary-sid', data: '4.5' },
deps,
);
await dispatchMpvProtocolMessage({ event: 'property-change', name: 'sid', data: 4.5 }, deps);
await dispatchMpvProtocolMessage(
{ event: 'property-change', name: 'secondary-sid', data: 4.5 },
deps,
);
assert.deepEqual(state.events, [{ sid: null }, { sid: null }, { sid: null }, { sid: null }]);
});
test('dispatchMpvProtocolMessage hides native secondary subtitles after a track change', async () => {
const visibilityChanges: boolean[] = [];
const { deps, state } = createDeps({
setSecondarySubVisibility: (visible) => visibilityChanges.push(visible),
});
await dispatchMpvProtocolMessage(
{ event: 'property-change', name: 'secondary-sid', data: '4' },
deps,
);
assert.deepEqual(visibilityChanges, [false]);
assert.deepEqual(state.events, [{ sid: 4 }]);
});
test('dispatchMpvProtocolMessage enforces sub-visibility hidden when overlay suppression is enabled', async () => {
@@ -207,6 +255,24 @@ test('dispatchMpvProtocolMessage skips sub-visibility suppression when overlay i
assert.equal(state.commands.length, 0);
});
test('dispatchMpvProtocolMessage corrects native secondary subtitle visibility', async () => {
const visibilityChanges: boolean[] = [];
const { deps } = createDeps({
setSecondarySubVisibility: (visible) => visibilityChanges.push(visible),
});
await dispatchMpvProtocolMessage(
{ event: 'property-change', name: 'secondary-sub-visibility', data: 'yes' },
deps,
);
await dispatchMpvProtocolMessage(
{ event: 'property-change', name: 'secondary-sub-visibility', data: 'no' },
deps,
);
assert.deepEqual(visibilityChanges, [false]);
});
test('dispatchMpvProtocolMessage sets secondary subtitle track based on track list response', async () => {
const { deps, state } = createDeps();
+30 -1
View File
@@ -56,6 +56,8 @@ export interface MpvProtocolHandleMessageDeps {
emitSubtitleTiming: (payload: { text: string; start: number; end: number }) => void;
emitSecondarySubtitleChange: (payload: { text: string }) => void;
emitSubtitleTrackChange: (payload: { sid: number | null }) => void;
emitSecondarySubtitleTrackChange: (payload: { sid: number | null }) => void;
emitSecondarySubtitleDelayChange: (payload: { delay: number }) => void;
emitSubtitleTrackListChange: (payload: { trackList: unknown[] | null }) => void;
getCurrentSubText: () => string;
setCurrentSubText: (text: string) => void;
@@ -72,6 +74,7 @@ export interface MpvProtocolHandleMessageDeps {
emitSubtitleMetricsChange: (payload: Partial<MpvSubtitleRenderMetrics>) => void;
setCurrentSecondarySubText: (text: string) => void;
resolvePendingRequest: (requestId: number, message: MpvMessage) => boolean;
shouldEnforceSecondarySubVisibilityHidden: () => boolean;
setSecondarySubVisibility: (visible: boolean) => void;
syncCurrentAudioStreamIndex: () => void;
setCurrentAudioTrackId: (value: number | null) => void;
@@ -284,7 +287,28 @@ export async function dispatchMpvProtocolMessage(
: typeof msg.data === 'string'
? Number(msg.data)
: null;
deps.emitSubtitleTrackChange({ sid: sid !== null && Number.isFinite(sid) ? sid : null });
deps.emitSubtitleTrackChange({ sid: sid !== null && Number.isInteger(sid) ? sid : null });
} else if (msg.name === 'secondary-sid') {
if (deps.shouldEnforceSecondarySubVisibilityHidden()) {
deps.setSecondarySubVisibility(false);
}
const sid =
typeof msg.data === 'number'
? msg.data
: typeof msg.data === 'string'
? Number(msg.data)
: null;
deps.emitSecondarySubtitleTrackChange({
sid: sid !== null && Number.isInteger(sid) ? sid : null,
});
} else if (msg.name === 'secondary-sub-delay') {
const delay =
typeof msg.data === 'number'
? msg.data
: typeof msg.data === 'string'
? Number(msg.data)
: 0;
deps.emitSecondarySubtitleDelayChange({ delay: Number.isFinite(delay) ? delay : 0 });
} else if (msg.name === 'track-list') {
deps.emitSubtitleTrackListChange({
trackList: Array.isArray(msg.data) ? (msg.data as unknown[]) : null,
@@ -358,6 +382,11 @@ export async function dispatchMpvProtocolMessage(
if (deps.isVisibleOverlayVisible() && asBoolean(msg.data, false)) {
deps.sendCommand({ command: ['set_property', 'sub-visibility', false] });
}
} else if (msg.name === 'secondary-sub-visibility') {
const visible = parseVisibilityProperty(msg.data);
if (deps.shouldEnforceSecondarySubVisibilityHidden() && visible === true) {
deps.setSecondarySubVisibility(false);
}
} else if (msg.name === 'sub-use-margins') {
deps.emitSubtitleMetricsChange({
subUseMargins: asBoolean(msg.data, deps.getSubtitleMetrics().subUseMargins),
+78 -1
View File
@@ -38,7 +38,15 @@ class ManualCloseSocket extends FakeSocket {
}
}
const wait = () => new Promise((resolve) => setTimeout(resolve, 0));
class HangingSocket extends FakeSocket {
override connect(path: string): void {
this.connectedPaths.push(path);
// Never emits 'connect', 'error', or 'close' on its own: models a named
// pipe dial that stalls indefinitely.
}
}
const wait = (ms = 0) => new Promise((resolve) => setTimeout(resolve, ms));
test('getMpvReconnectDelay follows existing reconnect ramp', () => {
assert.equal(getMpvReconnectDelay(0, true), 1000);
@@ -232,6 +240,75 @@ test('MpvSocketTransport.shutdown clears socket and lifecycle flags', async () =
assert.deepEqual(events, []);
});
test('MpvSocketTransport aborts a hung connect after the timeout and allows a fresh dial', async () => {
const events: string[] = [];
const errors: Error[] = [];
const sockets: HangingSocket[] = [];
const transport = new MpvSocketTransport({
socketPath: '/tmp/mpv.sock',
connectTimeoutMs: 5,
onConnect: () => {
events.push('connect');
},
onData: () => {},
onError: (error) => {
events.push('error');
errors.push(error);
},
onClose: () => {
events.push('close');
},
socketFactory: () => {
const socket = new HangingSocket();
sockets.push(socket);
return socket as unknown as net.Socket;
},
});
transport.connect();
assert.equal(transport.isConnecting, true);
await wait(20);
assert.deepEqual(events, ['error', 'close']);
assert.match(errors[0]!.message, /connect timed out/);
assert.equal(sockets[0]!.destroyed, true);
assert.equal(transport.isConnecting, false);
assert.equal(transport.isConnected, false);
transport.connect();
assert.equal(transport.isConnecting, true);
assert.equal(sockets.length, 2);
assert.equal(sockets[1]!.connectedPaths.at(0), '/tmp/mpv.sock');
transport.shutdown();
});
test('MpvSocketTransport does not fire the connect timeout after a successful connect', async () => {
const events: string[] = [];
const transport = new MpvSocketTransport({
socketPath: '/tmp/mpv.sock',
connectTimeoutMs: 5,
onConnect: () => {
events.push('connect');
},
onData: () => {},
onError: () => {
events.push('error');
},
onClose: () => {
events.push('close');
},
socketFactory: () => new FakeSocket() as unknown as net.Socket,
});
transport.connect();
await wait(20);
assert.deepEqual(events, ['connect']);
assert.equal(transport.isConnected, true);
});
test('MpvSocketTransport ignores stale socket events after shutdown and reconnect', async () => {
const events: string[] = [];
const sockets: ManualCloseSocket[] = [];
+36
View File
@@ -62,6 +62,8 @@ interface MpvSocketTransportEvents {
onClose: () => void;
}
export const MPV_CONNECT_TIMEOUT_MS = 5000;
export interface MpvSocketTransportOptions {
socketPath: string;
onConnect: () => void;
@@ -69,13 +71,16 @@ export interface MpvSocketTransportOptions {
onError: (error: Error) => void;
onClose: () => void;
socketFactory?: () => net.Socket;
connectTimeoutMs?: number;
}
export class MpvSocketTransport {
private socketPath: string;
private readonly callbacks: MpvSocketTransportEvents;
private readonly socketFactory: () => net.Socket;
private readonly connectTimeoutMs: number;
private socketRef: net.Socket | null = null;
private connectTimer: ReturnType<typeof setTimeout> | null = null;
public socket: net.Socket | null = null;
public connected = false;
public connecting = false;
@@ -83,6 +88,7 @@ export class MpvSocketTransport {
constructor(options: MpvSocketTransportOptions) {
this.socketPath = options.socketPath;
this.socketFactory = options.socketFactory ?? (() => new net.Socket());
this.connectTimeoutMs = options.connectTimeoutMs ?? MPV_CONNECT_TIMEOUT_MS;
this.callbacks = {
onConnect: options.onConnect,
onData: options.onData,
@@ -91,6 +97,31 @@ export class MpvSocketTransport {
};
}
private clearConnectTimeout(): void {
if (this.connectTimer) {
clearTimeout(this.connectTimer);
this.connectTimer = null;
}
}
// A named-pipe/socket dial that neither connects nor errors would otherwise
// latch `connecting` forever and silently block every future connect().
private armConnectTimeout(socket: net.Socket): void {
this.clearConnectTimeout();
this.connectTimer = setTimeout(() => {
this.connectTimer = null;
if (this.socketRef !== socket || this.connected) return;
this.connecting = false;
this.callbacks.onError(
new Error(`MPV IPC connect timed out after ${this.connectTimeoutMs}ms: ${this.socketPath}`),
);
// Destroying the socket emits 'close', which drives the normal
// disconnect path (including reconnect scheduling) upstream.
socket.destroy();
}, this.connectTimeoutMs);
this.connectTimer.unref?.();
}
setSocketPath(socketPath: string): void {
this.socketPath = socketPath;
}
@@ -111,6 +142,7 @@ export class MpvSocketTransport {
socket.on('connect', () => {
if (this.socketRef !== socket) return;
this.clearConnectTimeout();
this.connected = true;
this.connecting = false;
this.callbacks.onConnect();
@@ -123,6 +155,7 @@ export class MpvSocketTransport {
socket.on('error', (error: Error) => {
if (this.socketRef !== socket) return;
this.clearConnectTimeout();
this.connected = false;
this.connecting = false;
this.callbacks.onError(error);
@@ -130,12 +163,14 @@ export class MpvSocketTransport {
socket.on('close', () => {
if (this.socketRef !== socket) return;
this.clearConnectTimeout();
this.connected = false;
this.connecting = false;
this.callbacks.onClose();
});
socket.connect(this.socketPath);
this.armConnectTimeout(socket);
}
send(payload: MpvSocketMessagePayload): boolean {
@@ -149,6 +184,7 @@ export class MpvSocketTransport {
}
shutdown(): void {
this.clearConnectTimeout();
const socket = this.socketRef;
this.socketRef = null;
this.socket = null;
+179 -1
View File
@@ -1,5 +1,6 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { EventEmitter } from 'node:events';
import {
MpvIpcClient,
MpvIpcClientDeps,
@@ -23,6 +24,18 @@ function makeDeps(overrides: Partial<MpvIpcClientProtocolDeps> = {}): MpvIpcClie
};
}
const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
async function waitFor(predicate: () => boolean, timeoutMs = 2000): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (!predicate()) {
if (Date.now() >= deadline) {
throw new Error('Timed out waiting for MPV retry connection');
}
await wait(10);
}
}
function captureWarnLogs(run: () => void): string[] {
const originalWarn = console.warn;
const originalLogLevel = process.env.SUBMINER_LOG_LEVEL;
@@ -639,7 +652,7 @@ test('MpvIpcClient captures and disables secondary subtitle visibility on reques
]);
});
test('MpvIpcClient restorePreviousSecondarySubVisibility restores and clears tracked value', async () => {
test('MpvIpcClient restores secondary subtitle visibility and relinquishes suppression', async () => {
const commands: unknown[] = [];
const client = new MpvIpcClient('/tmp/mpv.sock', makeDeps());
const previous: boolean[] = [];
@@ -658,6 +671,12 @@ test('MpvIpcClient restorePreviousSecondarySubVisibility restores and clears tra
});
client.restorePreviousSecondarySubVisibility();
await invokeHandleMessage(client, {
event: 'property-change',
name: 'secondary-sub-visibility',
data: 'yes',
});
assert.equal(previous[0], true);
assert.equal(previous.length, 1);
assert.deepEqual(commands, [
@@ -669,8 +688,53 @@ test('MpvIpcClient restorePreviousSecondarySubVisibility restores and clears tra
},
]);
await invokeHandleMessage(client, {
event: 'property-change',
name: 'secondary-sub-visibility',
data: 'yes',
});
assert.equal(commands.length, 2);
client.restorePreviousSecondarySubVisibility();
assert.equal(commands.length, 2);
const callbacks = (client as any).transport.callbacks;
callbacks.onConnect();
commands.length = 0;
await invokeHandleMessage(client, {
event: 'property-change',
name: 'secondary-sub-visibility',
data: 'yes',
});
assert.deepEqual(commands, [{ command: ['set_property', 'secondary-sub-visibility', 'no'] }]);
});
test('MpvIpcClient keeps secondary subtitle suppression when restoration send fails', async () => {
const commands: unknown[] = [];
const client = new MpvIpcClient('/tmp/mpv.sock', makeDeps());
(client as any).send = (payload: unknown) => {
commands.push(payload);
return false;
};
await invokeHandleMessage(client, {
request_id: MPV_REQUEST_ID_SECONDARY_SUB_VISIBILITY,
data: 'yes',
});
client.restorePreviousSecondarySubVisibility();
await invokeHandleMessage(client, {
event: 'property-change',
name: 'secondary-sid',
data: 4,
});
assert.deepEqual(commands, [
{ command: ['set_property', 'secondary-sub-visibility', 'no'] },
{ command: ['set_property', 'secondary-sub-visibility', 'yes'] },
{ command: ['set_property', 'secondary-sub-visibility', 'no'] },
]);
});
test('MpvIpcClient updates current audio stream index from track list', async () => {
@@ -756,3 +820,117 @@ test('MpvIpcClient playNextSubtitle still auto-pauses at end while already playi
assert.equal((client as any).pendingPauseAtSubEnd, true);
assert.deepEqual(commands, [{ command: ['sub-seek', 1] }]);
});
class HangingTestSocket extends EventEmitter {
public connectedPaths: string[] = [];
public destroyed = false;
connect(path: string): void {
this.connectedPaths.push(path);
// Never resolves: models a stalled named-pipe dial.
}
write(): boolean {
return true;
}
destroy(): void {
this.destroyed = true;
}
}
class RetryTestSocket extends EventEmitter {
public connectedPaths: string[] = [];
public destroyed = false;
constructor(private readonly shouldConnect: boolean) {
super();
}
connect(path: string): void {
this.connectedPaths.push(path);
if (this.shouldConnect) {
setTimeout(() => this.emit('connect'), 0);
}
}
write(): boolean {
return true;
}
destroy(): void {
if (this.destroyed) return;
this.destroyed = true;
this.emit('close');
}
}
test('MpvIpcClient automatically retries the same socket path after a connect timeout', async () => {
const sockets: RetryTestSocket[] = [];
let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
const originalLogLevel = process.env.SUBMINER_LOG_LEVEL;
const client = new MpvIpcClient(
'/tmp/mpv.sock',
makeDeps({
connectTimeoutMs: 5,
getReconnectTimer: () => reconnectTimer,
setReconnectTimer: (timer) => {
reconnectTimer = timer;
},
socketFactory: () => {
const socket = new RetryTestSocket(sockets.length > 0);
sockets.push(socket);
return socket as unknown as import('node:net').Socket;
},
}),
);
process.env.SUBMINER_LOG_LEVEL = 'error';
try {
client.connect();
await waitFor(() => client.connected);
assert.equal(sockets.length, 2);
assert.equal(sockets[0]!.destroyed, true);
assert.equal(sockets[0]!.connectedPaths.at(0), '/tmp/mpv.sock');
assert.equal(sockets[1]!.connectedPaths.at(0), '/tmp/mpv.sock');
assert.equal(client.connected, true);
} finally {
if (originalLogLevel === undefined) {
delete process.env.SUBMINER_LOG_LEVEL;
} else {
process.env.SUBMINER_LOG_LEVEL = originalLogLevel;
}
if (reconnectTimer) clearTimeout(reconnectTimer);
(client as any).transport.shutdown();
}
});
test('MpvIpcClient.setSocketPath aborts an in-flight connect so the next dial targets the new path', () => {
const sockets: HangingTestSocket[] = [];
const client = new MpvIpcClient(
'/tmp/mpv-old.sock',
makeDeps({
socketFactory: () => {
const socket = new HangingTestSocket();
sockets.push(socket);
return socket as unknown as import('node:net').Socket;
},
}),
);
client.connect();
assert.equal(sockets.length, 1);
assert.equal(sockets[0]!.connectedPaths.at(0), '/tmp/mpv-old.sock');
assert.equal((client as any).connecting, true);
client.setSocketPath('/tmp/mpv-new.sock');
assert.equal((client as any).connecting, false);
assert.equal(sockets[0]!.destroyed, true);
client.connect();
assert.equal(sockets.length, 2);
assert.equal(sockets[1]!.connectedPaths.at(0), '/tmp/mpv-new.sock');
(client as any).transport.shutdown();
});
+31 -2
View File
@@ -9,7 +9,11 @@ import {
splitMpvMessagesFromBuffer,
} from './mpv-protocol';
import { requestMpvInitialState, subscribeToMpvProperties } from './mpv-properties';
import { scheduleMpvReconnect, MpvSocketTransport } from './mpv-transport';
import {
scheduleMpvReconnect,
MpvSocketTransport,
MpvSocketTransportOptions,
} from './mpv-transport';
import { createLogger } from '../../logger';
const logger = createLogger('main:mpv');
@@ -110,6 +114,8 @@ export interface MpvIpcClientProtocolDeps {
shouldAutoLoadSecondarySubTrack?: (path: string) => boolean;
shouldQuitOnMpvShutdown?: () => boolean;
requestAppQuit?: () => void;
socketFactory?: MpvSocketTransportOptions['socketFactory'];
connectTimeoutMs?: number;
}
export interface MpvIpcClientDeps extends MpvIpcClientProtocolDeps {}
@@ -125,6 +131,8 @@ export interface MpvIpcClientEventMap {
'fullscreen-change': { fullscreen: boolean };
'secondary-subtitle-change': { text: string };
'subtitle-track-change': { sid: number | null };
'secondary-subtitle-track-change': { sid: number | null };
'secondary-subtitle-delay-change': { delay: number };
'subtitle-track-list-change': { trackList: unknown[] | null };
'media-path-change': { path: string };
'media-title-change': { title: string | null };
@@ -177,6 +185,7 @@ export class MpvIpcClient implements MpvClient {
osdDimensions: null,
};
private previousSecondarySubVisibility: boolean | null = null;
private enforceSecondarySubVisibilityHidden = true;
private playbackPaused: boolean | null = null;
private pauseAtTime: number | null = null;
private pendingPauseAtSubEnd = false;
@@ -189,7 +198,10 @@ export class MpvIpcClient implements MpvClient {
this.transport = new MpvSocketTransport({
socketPath,
socketFactory: deps.socketFactory,
connectTimeoutMs: deps.connectTimeoutMs,
onConnect: () => {
this.enforceSecondarySubVisibilityHidden = true;
this.connected = true;
this.connecting = false;
this.socket = this.transport.getSocket();
@@ -290,6 +302,14 @@ export class MpvIpcClient implements MpvClient {
previousSocketPath: this.socketPath,
socketPath,
});
if (this.connecting && !this.connected) {
// Abort the in-flight dial to the old path; otherwise the connecting
// latch turns every later connect() into a no-op while we hang on a
// stale socket.
logger.debug('Aborting in-flight MPV IPC connect for socket path change.');
this.transport.shutdown();
this.connecting = false;
}
}
this.socketPath = socketPath;
this.transport.setSocketPath(socketPath);
@@ -423,6 +443,12 @@ export class MpvIpcClient implements MpvClient {
emitSubtitleTrackChange: (payload) => {
this.emit('subtitle-track-change', payload);
},
emitSecondarySubtitleTrackChange: (payload) => {
this.emit('secondary-subtitle-track-change', payload);
},
emitSecondarySubtitleDelayChange: (payload) => {
this.emit('secondary-subtitle-delay-change', payload);
},
emitSubtitleTrackListChange: (payload) => {
this.emit('subtitle-track-list-change', payload);
},
@@ -453,6 +479,7 @@ export class MpvIpcClient implements MpvClient {
},
resolvePendingRequest: (requestId: number, message: MpvMessage) =>
this.tryResolvePendingRequest(requestId, message),
shouldEnforceSecondarySubVisibilityHidden: () => this.enforceSecondarySubVisibilityHidden,
setSecondarySubVisibility: (visible: boolean) => this.setSecondarySubVisibility(visible),
syncCurrentAudioStreamIndex: () => {
this.syncCurrentAudioStreamIndex();
@@ -627,9 +654,11 @@ export class MpvIpcClient implements MpvClient {
restorePreviousSecondarySubVisibility(): void {
const previous = this.previousSecondarySubVisibility;
if (previous === null) return;
this.send({
const restored = this.send({
command: ['set_property', 'secondary-sub-visibility', previous ? 'yes' : 'no'],
});
if (!restored) return;
this.enforceSecondarySubVisibilityHidden = false;
this.previousSecondarySubVisibility = null;
}
@@ -0,0 +1,21 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { applyOverlayClickThrough } from './overlay-click-through';
test('applyOverlayClickThrough requests forwarding only off Windows', () => {
const calls: Array<{ ignore: boolean; forward: boolean }> = [];
const window = {
setIgnoreMouseEvents: (ignore: boolean, options?: { forward?: boolean }) => {
calls.push({ ignore, forward: options?.forward === true });
},
};
applyOverlayClickThrough(window, true);
applyOverlayClickThrough(window, false);
assert.deepEqual(calls, [
{ ignore: true, forward: false },
{ ignore: true, forward: true },
]);
});
@@ -0,0 +1,27 @@
type ClickThroughWindow = {
setIgnoreMouseEvents: (ignore: boolean, options?: { forward?: boolean }) => void;
};
/**
* Puts an overlay window into click-through mode. Forwarded mouse-move ({ forward: true }) is
* what lets renderer hover tracking wake a click-through overlay, but on Windows Electron
* implements it with a global WH_MOUSE_LL hook whose callback runs on the main-process message
* loop, so any main-thread stall delays mouse input system-wide (electron/electron#10183).
* Windows instead wakes the overlay via the main-process cursor poll
* (tickWindowsOverlayPointerInteraction), so no forwarding is requested there. macOS still
* needs forwarding for renderer hover tracking; Linux ignores the flag entirely
* (electron/electron#16777).
*
* Pass isWindowsPlatform when the caller already carries a platform flag (tests simulate
* platforms through it); otherwise the real process.platform decides.
*/
export function applyOverlayClickThrough(
window: ClickThroughWindow,
isWindowsPlatform?: boolean,
): void {
if (isWindowsPlatform ?? process.platform === 'win32') {
window.setIgnoreMouseEvents(true);
} else {
window.setIgnoreMouseEvents(true, { forward: true });
}
}
@@ -21,6 +21,7 @@ type CreateAnkiIntegrationArgs = {
mpvClient: { send?: (payload: { command: string[] }) => void };
showDesktopNotification: (title: string, options: { body?: string; icon?: string }) => void;
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
dismissOverlayNotification?: (id: string) => void;
createFieldGroupingCallback: () => (
data: KikuFieldGroupingRequestData,
) => Promise<KikuFieldGroupingChoice>;
@@ -74,6 +75,7 @@ function createDefaultAnkiIntegration(args: CreateAnkiIntegrationArgs): AnkiInte
args.getCachedMediaPath,
args.shouldRequireRemoteMediaCache,
args.getYoutubeMediaSourceUrl,
args.dismissOverlayNotification,
);
}
@@ -137,6 +139,7 @@ export function initializeOverlayRuntime(
setAnkiIntegration: (integration: unknown | null) => void;
showDesktopNotification: (title: string, options: { body?: string; icon?: string }) => void;
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
dismissOverlayNotification?: (id: string) => void;
createFieldGroupingCallback: () => (
data: KikuFieldGroupingRequestData,
) => Promise<KikuFieldGroupingChoice>;
@@ -177,6 +180,7 @@ export function initializeOverlayAnkiIntegration(options: {
setAnkiIntegration: (integration: unknown | null) => void;
showDesktopNotification: (title: string, options: { body?: string; icon?: string }) => void;
showOverlayNotification?: (payload: OverlayNotificationPayload) => void;
dismissOverlayNotification?: (id: string) => void;
createFieldGroupingCallback: () => (
data: KikuFieldGroupingRequestData,
) => Promise<KikuFieldGroupingChoice>;
@@ -219,6 +223,7 @@ export function initializeOverlayAnkiIntegration(options: {
mpvClient,
showDesktopNotification: options.showDesktopNotification,
showOverlayNotification: options.showOverlayNotification,
dismissOverlayNotification: options.dismissOverlayNotification,
createFieldGroupingCallback: options.createFieldGroupingCallback,
knownWordCacheStatePath: options.getKnownWordCacheStatePath(),
...(options.getCachedMediaPath ? { getCachedMediaPath: options.getCachedMediaPath } : {}),
+7 -7
View File
@@ -848,7 +848,7 @@ test('Windows visible overlay stays click-through and binds to mpv while tracked
} as never);
assert.ok(calls.includes('opacity:0'));
assert.ok(calls.includes('mouse-ignore:true:forward'));
assert.ok(calls.includes('mouse-ignore:true:plain'));
assert.ok(calls.includes('show-inactive'));
assert.ok(calls.includes('sync-windows-z-order'));
assert.ok(!calls.includes('move-top'));
@@ -1060,7 +1060,7 @@ test('tracked Windows overlay refresh rebinds while already visible', () => {
isWindowsPlatform: true,
} as never);
assert.ok(calls.includes('mouse-ignore:true:forward'));
assert.ok(calls.includes('mouse-ignore:true:plain'));
assert.ok(calls.includes('sync-windows-z-order'));
assert.ok(!calls.includes('move-top'));
assert.ok(!calls.includes('show'));
@@ -1134,7 +1134,7 @@ test('forced passthrough still reapplies while visible on Windows', () => {
forceMousePassthrough: true,
} as never);
assert.ok(calls.includes('mouse-ignore:true:forward'));
assert.ok(calls.includes('mouse-ignore:true:plain'));
assert.ok(!calls.includes('always-on-top:false'));
assert.ok(!calls.includes('move-top'));
assert.ok(calls.includes('sync-windows-z-order'));
@@ -1339,7 +1339,7 @@ test('tracked Windows overlay rebinds without hiding when tracker focus changes'
assert.ok(!calls.includes('always-on-top:false'));
assert.ok(!calls.includes('move-top'));
assert.ok(calls.includes('mouse-ignore:true:forward'));
assert.ok(calls.includes('mouse-ignore:true:plain'));
assert.ok(calls.includes('sync-windows-z-order'));
assert.ok(!calls.includes('ensure-level'));
assert.ok(!calls.includes('enforce-order'));
@@ -1489,7 +1489,7 @@ test('tracked Windows overlay reshows click-through even if focus state is stale
isWindowsPlatform: true,
} as never);
assert.ok(calls.includes('mouse-ignore:true:forward'));
assert.ok(calls.includes('mouse-ignore:true:plain'));
assert.ok(calls.includes('show-inactive'));
assert.ok(!calls.includes('show'));
});
@@ -1532,7 +1532,7 @@ test('tracked Windows overlay binds above mpv even when tracker focus lags', ()
assert.ok(!calls.includes('always-on-top:false'));
assert.ok(!calls.includes('move-top'));
assert.ok(calls.includes('mouse-ignore:true:forward'));
assert.ok(calls.includes('mouse-ignore:true:plain'));
assert.ok(calls.includes('sync-windows-z-order'));
assert.ok(!calls.includes('ensure-level'));
});
@@ -2193,7 +2193,7 @@ test('Windows preserves visible overlay and rebinds to mpv while tracker transie
assert.ok(!calls.includes('show'));
assert.ok(!calls.includes('always-on-top:false'));
assert.ok(!calls.includes('move-top'));
assert.ok(calls.includes('mouse-ignore:true:forward'));
assert.ok(calls.includes('mouse-ignore:true:plain'));
assert.ok(calls.includes('sync-windows-z-order'));
assert.ok(!calls.includes('ensure-level'));
assert.ok(calls.includes('sync-shortcuts'));
+5 -4
View File
@@ -1,6 +1,7 @@
import type { BrowserWindow } from 'electron';
import { BaseWindowTracker } from '../../window-trackers';
import { WindowGeometry } from '../../types';
import { applyOverlayClickThrough } from './overlay-click-through';
import { OVERLAY_WINDOW_CONTENT_READY_FLAG } from './overlay-window-flags';
const WINDOWS_OVERLAY_REVEAL_DELAY_MS = 48;
@@ -117,7 +118,7 @@ export function updateVisibleOverlayVisibility(args: {
clearPendingWindowsOverlayReveal(mainWindow);
setOverlayWindowOpacity(mainWindow, 0);
}
mainWindow.setIgnoreMouseEvents(true, { forward: true });
applyOverlayClickThrough(mainWindow, args.isWindowsPlatform);
releaseOverlayWindowLevel(mainWindow);
mainWindow.hide();
args.syncOverlayShortcuts();
@@ -215,7 +216,7 @@ export function updateVisibleOverlayVisibility(args: {
shouldPreserveWindowsOverlayDuringFocusHandoff ||
(hasWindowsForegroundProcessSignal && windowsForegroundProcessName === 'mpv');
if (shouldIgnoreMouseEvents) {
mainWindow.setIgnoreMouseEvents(true, { forward: true });
applyOverlayClickThrough(mainWindow, args.isWindowsPlatform);
} else {
mainWindow.setIgnoreMouseEvents(false);
}
@@ -263,7 +264,7 @@ export function updateVisibleOverlayVisibility(args: {
if (hasNonNativeInputRegion) {
mainWindow.setIgnoreMouseEvents(false);
} else {
mainWindow.setIgnoreMouseEvents(true, { forward: true });
applyOverlayClickThrough(mainWindow, args.isWindowsPlatform);
}
if (args.isWindowsPlatform) {
scheduleWindowsOverlayReveal(
@@ -424,7 +425,7 @@ export function updateVisibleOverlayVisibility(args: {
return;
}
args.setTrackerNotReadyWarningShown(false);
mainWindow.setIgnoreMouseEvents(true, { forward: true });
applyOverlayClickThrough(mainWindow, args.isWindowsPlatform);
releaseOverlayWindowLevel(mainWindow);
mainWindow.hide();
args.syncOverlayShortcuts();
@@ -0,0 +1,15 @@
const MIN_FLATTENED_DUPLICATE_LENGTH = 16;
const TERMINAL_SENTENCE_PUNCTUATION = /[.!?]+$/gu;
/**
* Identifies long lines that become duplicates when positioned ASS events are
* flattened into the secondary subtitle bar. Short dialogue stays distinct.
*/
export function flattenedSecondarySubtitleLineIdentity(text: string): string | null {
const identity = text
.normalize('NFKC')
.replace(/\s+/gu, '')
.replace(TERMINAL_SENTENCE_PUNCTUATION, '');
return identity.length >= MIN_FLATTENED_DUPLICATE_LENGTH ? identity : null;
}
@@ -10,6 +10,7 @@ import {
parseExcludedWordsBody,
parseIntQuery,
parsePositiveIdList,
loadKnownWordsSet,
} from './route-support.js';
export function registerStatsLibraryRoutes(
@@ -31,6 +32,17 @@ export function registerStatsLibraryRoutes(
return c.json(statsJson('vocabulary', vocab));
});
app.get('/api/stats/vocabulary/summary', async (c) => {
const summary = await tracker.getVocabularySummary(
loadKnownWordsSet(options?.knownWordCachePath),
);
return c.json(statsJson('vocabularySummary', summary));
});
app.get('/api/stats/vocabulary/charts', async (c) => {
return c.json(statsJson('vocabularyCharts', await tracker.getVocabularyChartData()));
});
app.get('/api/stats/excluded-words', async (c) => {
return c.json(statsJson('excludedWords', await tracker.getStatsExcludedWords()));
});
@@ -89,6 +89,7 @@ const WORD_COPY_COLUMNS = [
'last_seen',
'frequency',
'frequency_rank',
'vocabulary_visible',
] as const;
export function mergeAnime(
+182 -8
View File
@@ -27,17 +27,188 @@ function cueKey(cue: SubtitleCue): string {
/**
* Identical text over an identical span is redundant however it was authored -- most
* often a layered ASS event stacking a shadow copy under the visible one.
* often a layered ASS event stacking a shadow copy under the visible one. When one of
* the duplicates is a recovered canonical cue, that copy survives: dropping it would
* strip the `source` marker and animation envelope the live overlay substitutes on.
*/
function collapseExactDuplicates(cues: AnnotatedSubtitleCue[]): AnnotatedSubtitleCue[] {
const seen = new Set<string>();
return cues.filter((cue) => {
const survivorByKey = new Map<string, AnnotatedSubtitleCue>();
const keysInOrder: string[] = [];
for (const cue of cues) {
const key = cueKey(cue);
if (seen.has(key)) {
return false;
const existing = survivorByKey.get(key);
if (!existing) {
survivorByKey.set(key, cue);
keysInOrder.push(key);
} else if (!existing.source && cue.source) {
survivorByKey.set(key, cue);
}
seen.add(key);
}
return keysInOrder.map((key) => survivorByKey.get(key)!);
}
const SPATIAL_ASS_OVERRIDE_COMMANDS = new Set([
'a',
'an',
'clip',
'iclip',
'move',
'org',
'pbo',
'pos',
'q',
]);
interface RepeatedPhaseRun {
cues: AnnotatedSubtitleCue[];
indices: number[];
}
// A changing override signature alone is weak: two ordinary repeats restyled with
// different colors look identical to a phase pair. Real phase redraws carry a styling
// stack over a full lyric line, and they exist to move a color/highlight boundary
// *within* the line -- so every event also has an override block after visible text
// began. An ordinary restyled repeat carries only a leading block and stays separate.
const MIN_PHASE_EVIDENCE_OVERRIDES = 2;
const MIN_PHASE_TEXT_LENGTH = 4;
function hasMidLineOverrideBlock(rawText: string): boolean {
let sawVisibleText = false;
for (let i = 0; i < rawText.length; i += 1) {
if (rawText[i] === '{') {
const close = rawText.indexOf('}', i);
if (close === -1) {
// Unclosed brace renders as literal text; nothing after it is markup.
return false;
}
if (sawVisibleText) {
return true;
}
i = close;
} else if (!/\s/.test(rawText[i]!)) {
sawVisibleText = true;
}
}
return false;
}
function assStyleKey(cue: AnnotatedSubtitleCue): string {
return `${cue.style}\0${cue.name}\0${cue.layer}`;
}
function spatialOverrideSignature(cue: AnnotatedSubtitleCue): string {
return cue.overrides
.filter((command) => SPATIAL_ASS_OVERRIDE_COMMANDS.has(command.name.toLowerCase()))
.map((command) => `${command.name.toLowerCase()}(${command.args})`)
.join('|');
}
function hasStableSpatialOverrides(run: readonly AnnotatedSubtitleCue[]): boolean {
const firstSignature = spatialOverrideSignature(run[0]!);
return run.every((cue) => spatialOverrideSignature(cue) === firstSignature);
}
function hasDirectPhaseEvidence(run: readonly AnnotatedSubtitleCue[]): boolean {
// Phases redraw one authored line in place. Whatever the animation evidence, a run
// whose spatial placement changes is separate authored occurrences -- two flush
// same-text `\move` signs at different coordinates must never merge.
if (!hasStableSpatialOverrides(run)) {
return false;
}
if (run.every((cue) => hasAssTemporalOverride(cue.overrides))) {
return true;
}
if (run.every((cue) => isAnimatedAssEffectKind(cue.effectKind))) {
return true;
}
const [first] = run;
return (
first!.text.replace(/\s+/gu, '').length >= MIN_PHASE_TEXT_LENGTH &&
run.every(
(cue) =>
cue.overrides.length >= MIN_PHASE_EVIDENCE_OVERRIDES &&
hasMidLineOverrideBlock(cue.rawText),
) &&
run.some((cue) => cue.overrideSignature !== first!.overrideSignature)
);
}
function collectRepeatedPhaseRuns(cues: AnnotatedSubtitleCue[]): RepeatedPhaseRun[] {
const runs: RepeatedPhaseRun[] = [];
let start = 0;
while (start < cues.length) {
const first = cues[start]!;
const styleKey = assStyleKey(first);
let end = start;
while (end + 1 < cues.length) {
const current = cues[end]!;
const next = cues[end + 1]!;
const isFlush =
Math.abs(next.startTime - current.endTime) <= DUPLICATE_CUE_GAP_TOLERANCE_SECONDS;
if (
first.source !== undefined ||
next.source !== undefined ||
next.text !== first.text ||
assStyleKey(next) !== styleKey ||
!isFlush
) {
break;
}
end += 1;
}
if (end > start) {
const indices = Array.from({ length: end - start + 1 }, (_, offset) => start + offset);
runs.push({
cues: indices.map((index) => cues[index]!),
indices,
});
}
start = end + 1;
}
return runs;
}
/**
* Some karaoke scripts redraw one complete lyric for each color/highlight phase. These
* events last far longer than animation frames, but are still one sidebar/history line.
* The events must prove themselves through direct animation metadata or changing
* non-spatial overrides. Plain repeated dialogue and separately positioned signs stay
* intact.
*/
function collapseAnimatedStylePhases(cues: AnnotatedSubtitleCue[]): AnnotatedSubtitleCue[] {
const runs = collectRepeatedPhaseRuns(cues);
if (runs.length === 0) {
return cues;
}
const dropped = new Set<number>();
const extendedEnd = new Map<number, number>();
for (const run of runs) {
if (!hasDirectPhaseEvidence(run.cues)) {
continue;
}
const [firstIndex, ...remainingIndices] = run.indices;
for (const index of remainingIndices) {
dropped.add(index);
}
extendedEnd.set(firstIndex!, Math.max(...run.cues.map((cue) => cue.endTime)));
}
if (dropped.size === 0) {
return cues;
}
return cues.flatMap((cue, index) => {
if (dropped.has(index)) {
return [];
}
const endTime = extendedEnd.get(index);
return endTime !== undefined ? [{ ...cue, endTime }] : [cue];
});
}
@@ -52,7 +223,7 @@ function countFramesShorterThan(run: AnnotatedSubtitleCue[], maxSeconds: number)
* anything wrapped in `\t(...)`), an animated `Effect` column, or a value that actually
* changes from event to event, which is how per-frame typesetting is authored.
*/
export function hasAssAnimationEvidence(run: AnnotatedSubtitleCue[]): boolean {
export function hasAssAnimationEvidence(run: readonly AnnotatedSubtitleCue[]): boolean {
if (run.every((cue) => hasAssTemporalOverride(cue.overrides))) {
return true;
}
@@ -176,5 +347,8 @@ export function mergeDuplicateCues(
cues: AnnotatedSubtitleCue[],
format: SubtitleSourceFormat,
): AnnotatedSubtitleCue[] {
return collapseAnimationBursts(collapseExactDuplicates(cues), format);
const exactDeduplicated = collapseExactDuplicates(cues);
const phaseDeduplicated =
format === 'ass' ? collapseAnimatedStylePhases(exactDeduplicated) : exactDeduplicated;
return collapseAnimationBursts(phaseDeduplicated, format);
}
@@ -0,0 +1,108 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
resolveSanitizedSubtitleSeekCommand,
subtitleCueListSeekTime,
subtitleCueSeekTime,
} from './subtitle-cue-navigation';
test('next subtitle navigation skips generated ASS events and seeks to the next sanitized cue', () => {
const cues = [
{
startTime: 10,
endTime: 13,
text: 'first lyric',
source: 'canonical-ass' as const,
animationStartTime: 9.7,
animationEndTime: 13.4,
},
{
startTime: 13,
endTime: 16,
text: 'second lyric',
source: 'canonical-ass' as const,
animationStartTime: 12.7,
animationEndTime: 16.4,
},
];
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.2), [
'seek',
13.08,
'absolute+exact',
]);
});
test('next subtitle navigation treats simultaneous sanitized cues as one line boundary', () => {
const cues = [
{ startTime: 10, endTime: 13, text: 'romaji' },
{ startTime: 10.02, endTime: 13, text: 'English' },
{ startTime: 13, endTime: 16, text: 'next romaji' },
{ startTime: 13.02, endTime: 16, text: 'Next English' },
];
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.1), [
'seek',
13.08,
'absolute+exact',
]);
});
test('next subtitle navigation advances past the latest overlapping lyric', () => {
const cues = [
{ startTime: 10, endTime: 14, text: 'exiting lyric' },
{ startTime: 13, endTime: 16, text: 'current lyric' },
{ startTime: 16, endTime: 19, text: 'next lyric' },
];
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 13.2), [
'seek',
16.08,
'absolute+exact',
]);
});
test('previous subtitle navigation leaves the current cue and seeks to the prior cue', () => {
const cues = [
{ startTime: 10, endTime: 12, text: 'first line' },
{ startTime: 13, endTime: 16, text: 'current line' },
];
assert.deepEqual(resolveSanitizedSubtitleSeekCommand(['sub-seek', -1], cues, 14.5), [
'seek',
10.08,
'absolute+exact',
]);
});
test('subtitle navigation falls back when no sanitized destination exists', () => {
const cues = [{ startTime: 10, endTime: 13, text: 'only line' }];
assert.equal(resolveSanitizedSubtitleSeekCommand(['sub-seek', 1], cues, 10.2), null);
assert.equal(resolveSanitizedSubtitleSeekCommand(['seek', 5], cues, 10.2), null);
});
test('sidebar cue seeks share the boundary-safe sanitized cue timestamp', () => {
assert.equal(subtitleCueSeekTime({ startTime: 1, endTime: 2, text: 'line' }), 1.08);
assert.equal(subtitleCueSeekTime({ startTime: 1, endTime: 1.04, text: 'short' }), 1.03);
});
test('sidebar cue selection clears an overlapping previous lyric', () => {
const cues = [
{ startTime: 1, endTime: 3.4, text: 'previous lyric' },
{ startTime: 3, endTime: 5, text: 'selected lyric' },
];
assert.equal(subtitleCueListSeekTime(cues, cues[1]!), 3.48);
});
test('sidebar cue selection remains inside a short cue when overlap cannot be cleared', () => {
const cues = [
{ startTime: 1, endTime: 3.4, text: 'previous lyric' },
{ startTime: 3, endTime: 3.2, text: 'selected lyric' },
];
const seekTime = subtitleCueListSeekTime(cues, cues[1]!);
assert.ok(seekTime >= 3.19);
assert.ok(seekTime < cues[1]!.endTime);
});
@@ -0,0 +1,128 @@
import type { SubtitleCue } from './subtitle-cue-parser';
const CUE_START_GROUP_TOLERANCE_SECONDS = 0.05;
const CUE_BOUNDARY_SEEK_OFFSET_SECONDS = 0.08;
const CUE_END_GUARD_SECONDS = 0.01;
type CueGroup = {
startTime: number;
endTime: number;
cue: SubtitleCue;
};
function isValidCue(cue: SubtitleCue): boolean {
return (
Number.isFinite(cue.startTime) && Number.isFinite(cue.endTime) && cue.endTime > cue.startTime
);
}
function groupCueBoundaries(cues: readonly SubtitleCue[]): CueGroup[] {
const sorted = cues.filter(isValidCue).sort((left, right) => {
return left.startTime - right.startTime || left.endTime - right.endTime;
});
const groups: CueGroup[] = [];
for (const cue of sorted) {
const current = groups.at(-1);
if (current && cue.startTime - current.startTime <= CUE_START_GROUP_TOLERANCE_SECONDS) {
current.endTime = Math.max(current.endTime, cue.endTime);
continue;
}
groups.push({ startTime: cue.startTime, endTime: cue.endTime, cue });
}
return groups;
}
/** A small offset avoids asking mpv to render exactly on a subtitle boundary. */
export function subtitleCueSeekTime(cue: SubtitleCue): number {
return Math.max(
cue.startTime,
Math.min(cue.endTime - CUE_END_GUARD_SECONDS, cue.startTime + CUE_BOUNDARY_SEEK_OFFSET_SECONDS),
);
}
/**
* Choose a stable point inside a selected cue. Karaoke lines can overlap while the
* previous line animates out, so a sidebar selection should clear that overlap when
* the selected cue has enough time remaining.
*/
export function subtitleCueListSeekTime(
cues: readonly SubtitleCue[],
selectedCue: SubtitleCue,
): number {
const groups = groupCueBoundaries(cues);
const selectedGroupIndex = groups.findIndex(
(group) =>
selectedCue.startTime >= group.startTime &&
selectedCue.startTime - group.startTime <= CUE_START_GROUP_TOLERANCE_SECONDS,
);
const previousGroupEndTime =
selectedGroupIndex > 0 ? groups[selectedGroupIndex - 1]?.endTime : undefined;
if (previousGroupEndTime === undefined || previousGroupEndTime <= selectedCue.startTime) {
return subtitleCueSeekTime(selectedCue);
}
return Math.max(
selectedCue.startTime,
Math.min(
selectedCue.endTime - CUE_END_GUARD_SECONDS,
previousGroupEndTime + CUE_BOUNDARY_SEEK_OFFSET_SECONDS,
),
);
}
/**
* Translate mpv subtitle-line navigation onto parsed cues. Generated ASS karaoke can
* contain hundreds of subtitle events for one visible line, while the parsed list has
* already collapsed those events into the authored lines the user expects to navigate.
*/
export function resolveSanitizedSubtitleSeekCommand(
command: readonly (string | number)[],
cues: readonly SubtitleCue[],
currentTimeSec: number,
): (string | number)[] | null {
if (
command.length < 2 ||
command[0] !== 'sub-seek' ||
(command[1] !== -1 && command[1] !== 1) ||
!Number.isFinite(currentTimeSec)
) {
return null;
}
const groups = groupCueBoundaries(cues);
if (groups.length === 0) {
return null;
}
let activeIndex = -1;
for (const [index, group] of groups.entries()) {
if (group.startTime <= currentTimeSec && group.endTime > currentTimeSec) {
activeIndex = index;
}
}
let destination: CueGroup | undefined;
if (command[1] === 1) {
destination =
activeIndex >= 0
? groups[activeIndex + 1]
: groups.find((group) => group.startTime > currentTimeSec);
} else if (activeIndex >= 0) {
destination = groups[activeIndex - 1];
} else {
for (let index = groups.length - 1; index >= 0; index -= 1) {
const group = groups[index]!;
if (group.startTime < currentTimeSec) {
destination = group;
break;
}
}
}
if (!destination) {
return null;
}
return ['seek', subtitleCueSeekTime(destination.cue), 'absolute+exact'];
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -134,7 +134,7 @@ export function createSubtitleProcessingController(
try {
const cachedTokenized = getCachedTokenization(text);
if (cachedTokenized) {
output = cachedTokenized;
output = { ...cachedTokenized, text };
} else {
// Cache miss: show the plain line on time; the tokenized payload
// upgrades it once ready. Skipped on refreshes of an already
@@ -266,7 +266,7 @@ export function createSubtitleProcessingController(
lastEmittedText = text;
lastEmittedGeneration = cacheGeneration;
lastPlainEmittedText = null;
return cached;
return { ...cached, text };
},
hasCachedSubtitle: (text: string) => {
const cacheKey = normalizeSubtitleCacheKey(text);
@@ -1,73 +0,0 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { estimateSubtitleTimingOffset } from './subtitle-timing-offset';
function cue(startTime: number) {
return { startTime, endTime: startTime + 1, text: `cue ${startTime}` };
}
test('estimate subtitle timing offset detects a late Jellyfin subtitle timeline', () => {
const primary = [
34.935, 36.937, 41.441, 45.279, 48.115, 52.286, 54.955, 59.793, 63.63, 67.634, 76.643, 80.814,
87.988, 90.991, 94.094, 97.097,
].map(cue);
const reference = [
3.46, 9.48, 13.61, 21.4, 28.16, 32.06, 35.93, 45.1, 56.57, 59.68, 62.44, 65.56,
].map(cue);
const result = estimateSubtitleTimingOffset(primary, reference);
assert.ok(result);
assert.ok(result.offsetSeconds > -32);
assert.ok(result.offsetSeconds < -31);
assert.ok(result.matchCount >= 8);
assert.ok(result.meanErrorSeconds <= 0.75);
});
test('estimate subtitle timing offset favors the early episode timeline', () => {
const primary = [
34.935, 36.937, 41.441, 45.279, 48.115, 52.286, 54.955, 59.793, 63.63, 67.634, 76.643, 80.814,
87.988, 90.991, 94.094, 97.097, 207.974, 212.579, 222.422, 228.095, 232.432, 238.271, 244.778,
246.78, 249.282, 251.284, 253.62, 256.289, 259.626, 262.129, 264.965, 267.634, 270.303, 274.407,
277.077, 280.08, 284.084, 288.421, 291.925, 295.262, 298.431, 301.101, 306.773, 308.942,
312.946, 316.283, 321.621, 326.626, 331.131, 336.069, 340.407, 343.41, 351.418, 355.422,
357.924, 362.429, 365.432, 370.604, 373.273, 377.944, 381.114, 384.618, 387.621, 390.957,
396.73, 399.232, 401.568, 403.57, 405.572, 407.574, 409.743, 412.746, 418.752, 425.258, 427.26,
435.602, 440.44, 442.942, 445.445, 449.783,
].map(cue);
const reference = [
3.46, 9.48, 13.61, 21.4, 28.16, 32.06, 35.93, 45.1, 56.57, 59.68, 62.44, 65.56, 165.77, 172.81,
176.1, 177.27, 186.33, 191.33, 195.78, 201.83, 212.9, 214.09, 216.73, 220.2, 222.91, 225.65,
232.8, 237.92, 242.23, 243.28, 247.53, 252.04, 255.9, 258.86, 262.09, 264.43, 276.07, 278.01,
280.98, 285.67, 289.89, 294.57, 300, 303.56, 308.58, 316.37, 318.38, 319.86, 325.38, 328.82,
333.68, 335.26, 336.82, 340.11, 342.11, 344.36, 346.39, 347.53, 350.92, 370.18, 372.88, 376.43,
388.2, 390.57, 403.96, 406.36, 409.72, 413.78, 425.55, 432.76, 435.03, 438.06, 443.73, 448.31,
450.57, 457.62, 463.41, 465.85, 473.79, 480.59,
].map(cue);
const result = estimateSubtitleTimingOffset(primary, reference);
assert.ok(result);
assert.ok(result.offsetSeconds > -32);
assert.ok(result.offsetSeconds < -31);
});
test('estimate subtitle timing offset ignores subtitle timelines that are already aligned', () => {
const starts = [1, 5, 9, 14, 20, 25, 31, 38];
const result = estimateSubtitleTimingOffset(
starts.map(cue),
starts.map((start) => cue(start + 0.04)),
);
assert.equal(result, null);
});
test('estimate subtitle timing offset rejects weak timeline matches', () => {
const primary = [10, 20, 30, 40, 50, 60, 70, 80].map(cue);
const reference = [1, 2, 3, 4, 5, 6, 7, 8].map(cue);
const result = estimateSubtitleTimingOffset(primary, reference);
assert.equal(result, null);
});
-153
View File
@@ -1,153 +0,0 @@
import type { SubtitleCue } from './subtitle-cue-parser';
export type SubtitleTimingOffsetResult = {
offsetSeconds: number;
matchCount: number;
meanErrorSeconds: number;
maxErrorSeconds: number;
};
export type SubtitleTimingOffsetOptions = {
maxCueCount?: number;
maxOffsetSeconds?: number;
matchThresholdSeconds?: number;
maxMeanErrorSeconds?: number;
minMatchCount?: number;
minMatchRatio?: number;
minUsefulOffsetSeconds?: number;
};
type OffsetScore = SubtitleTimingOffsetResult;
const DEFAULT_MAX_CUE_COUNT = 60;
const DEFAULT_MAX_OFFSET_SECONDS = 180;
const DEFAULT_MATCH_THRESHOLD_SECONDS = 1;
const DEFAULT_MAX_MEAN_ERROR_SECONDS = 0.75;
const DEFAULT_MIN_MATCH_COUNT = 8;
const DEFAULT_MIN_MATCH_RATIO = 0.25;
const DEFAULT_MIN_USEFUL_OFFSET_SECONDS = 0.25;
function normalizeCueStarts(cues: SubtitleCue[], maxCueCount: number): number[] {
const starts = cues
.map((cue) => cue.startTime)
.filter((start) => Number.isFinite(start) && start >= 0)
.sort((a, b) => a - b);
const deduped: number[] = [];
for (const start of starts) {
const previous = deduped[deduped.length - 1];
if (previous === undefined || Math.abs(start - previous) > 0.05) {
deduped.push(start);
}
if (deduped.length >= maxCueCount) {
break;
}
}
return deduped;
}
function roundToMillis(value: number): number {
return Math.round(value * 1000) / 1000;
}
function scoreOffset(
primaryStarts: number[],
referenceStarts: number[],
offsetSeconds: number,
matchThresholdSeconds: number,
): OffsetScore {
let primaryIndex = 0;
let referenceIndex = 0;
let matchCount = 0;
let totalErrorSeconds = 0;
let maxErrorSeconds = 0;
while (primaryIndex < primaryStarts.length && referenceIndex < referenceStarts.length) {
const shiftedPrimary = primaryStarts[primaryIndex]! + offsetSeconds;
const reference = referenceStarts[referenceIndex]!;
const errorSeconds = Math.abs(shiftedPrimary - reference);
if (errorSeconds <= matchThresholdSeconds) {
matchCount += 1;
totalErrorSeconds += errorSeconds;
maxErrorSeconds = Math.max(maxErrorSeconds, errorSeconds);
primaryIndex += 1;
referenceIndex += 1;
continue;
}
if (shiftedPrimary < reference) {
primaryIndex += 1;
} else {
referenceIndex += 1;
}
}
return {
offsetSeconds,
matchCount,
meanErrorSeconds: matchCount > 0 ? totalErrorSeconds / matchCount : Number.POSITIVE_INFINITY,
maxErrorSeconds,
};
}
function isBetterScore(next: OffsetScore, current: OffsetScore | null): boolean {
if (current === null) return true;
if (next.matchCount !== current.matchCount) return next.matchCount > current.matchCount;
if (next.meanErrorSeconds !== current.meanErrorSeconds) {
return next.meanErrorSeconds < current.meanErrorSeconds;
}
return Math.abs(next.offsetSeconds) < Math.abs(current.offsetSeconds);
}
export function estimateSubtitleTimingOffset(
primaryCues: SubtitleCue[],
referenceCues: SubtitleCue[],
options: SubtitleTimingOffsetOptions = {},
): SubtitleTimingOffsetResult | null {
const maxCueCount = options.maxCueCount ?? DEFAULT_MAX_CUE_COUNT;
const maxOffsetSeconds = options.maxOffsetSeconds ?? DEFAULT_MAX_OFFSET_SECONDS;
const matchThresholdSeconds = options.matchThresholdSeconds ?? DEFAULT_MATCH_THRESHOLD_SECONDS;
const maxMeanErrorSeconds = options.maxMeanErrorSeconds ?? DEFAULT_MAX_MEAN_ERROR_SECONDS;
const minMatchCount = options.minMatchCount ?? DEFAULT_MIN_MATCH_COUNT;
const minMatchRatio = options.minMatchRatio ?? DEFAULT_MIN_MATCH_RATIO;
const minUsefulOffsetSeconds =
options.minUsefulOffsetSeconds ?? DEFAULT_MIN_USEFUL_OFFSET_SECONDS;
const primaryStarts = normalizeCueStarts(primaryCues, maxCueCount);
const referenceStarts = normalizeCueStarts(referenceCues, maxCueCount);
const comparableCueCount = Math.min(primaryStarts.length, referenceStarts.length);
if (comparableCueCount < minMatchCount) {
return null;
}
const candidates = new Set<number>();
for (const primaryStart of primaryStarts) {
for (const referenceStart of referenceStarts) {
const offsetSeconds = roundToMillis(referenceStart - primaryStart);
if (Math.abs(offsetSeconds) <= maxOffsetSeconds) {
candidates.add(offsetSeconds);
}
}
}
let best: OffsetScore | null = null;
for (const offsetSeconds of candidates) {
if (Math.abs(offsetSeconds) < minUsefulOffsetSeconds) {
continue;
}
const score = scoreOffset(primaryStarts, referenceStarts, offsetSeconds, matchThresholdSeconds);
if (score.matchCount < minMatchCount) {
continue;
}
if (score.matchCount / comparableCueCount < minMatchRatio) {
continue;
}
if (score.meanErrorSeconds > maxMeanErrorSeconds) {
continue;
}
if (isBetterScore(score, best)) {
best = score;
}
}
return best;
}
+17
View File
@@ -84,6 +84,17 @@ function createDeferred<T>() {
};
}
test('tokenizeSubtitle keeps the blank line separating simultaneous cues', async () => {
// The tokenized payload's text drives display; folding the cue boundary would merge
// two speakers back onto one line the moment tokenization upgrades the plain emit.
const result = await tokenizeSubtitle(
'\u4e00\u884c\u76ee\n\n\u4e8c\u884c\u76ee',
makeDeps({ getYomitanExt: () => null }),
);
assert.equal(result.text, '\u4e00\u884c\u76ee\n\n\u4e8c\u884c\u76ee');
});
test('tokenizeSubtitle splits same-line grammar endings before applying annotations', async () => {
const result = await tokenizeSubtitle(
'猫です',
@@ -1682,6 +1693,12 @@ test('tokenizeSubtitle normalizes newlines before Yomitan parse request', async
assert.equal(result.tokens, null);
});
test('tokenizeSubtitle preserves CRLF boundaries between simultaneous cues', async () => {
const result = await tokenizeSubtitle('a\r\n\r\nb', makeDeps());
assert.deepEqual(result, { text: 'a\n\nb', tokens: null });
});
test('tokenizeSubtitle collapses zero-width separators before Yomitan parse request', async () => {
let parseInput = '';
const result = await tokenizeSubtitle(
+9 -1
View File
@@ -887,7 +887,15 @@ export async function tokenizeSubtitle(
text: string,
deps: TokenizerServiceDeps,
): Promise<SubtitleData> {
const displayText = normalizePlainSubtitleText(text);
// Normalize per cue group: the blank line separating simultaneous cues is display
// structure the payload text must keep, or the tokenized upgrade re-merges lines the
// provisional plain emit already showed apart.
const displayText = text
.replace(/\r\n/g, '\n')
.split(/\n{2,}/)
.map((part) => normalizePlainSubtitleText(part))
.filter(Boolean)
.join('\n\n');
// ASS decoding already happened upstream (cue parser for files, mpv for live text), so
// all this drops is whitespace -- but a whitespace-only line still normalizes to empty.
+112
View File
@@ -39,6 +39,118 @@ test('convertYoutubeTimedTextToVtt does not swallow text after zero-length overl
);
});
test('convertYoutubeTimedTextToVtt extends rolling captions to the next window event', () => {
// Real-world shape of YouTube's sentence-level auto captions: window-append
// filler rows (a="1", sometimes without d) mark the display timeline, while
// long text rows carry a placeholder d="3000" far shorter than the speech.
const result = convertYoutubeTimedTextToVtt(
[
'<timedtext><body>',
'<p t="98550" d="3010" w="1" a="1">\n</p>',
'<p t="98560" d="3000" w="1"><s ac="0">ありがとうって言えないよね。こんなんじゃ。</s></p>',
'<p t="106950" w="1" a="1">\n</p>',
'<p t="106960" d="3799" w="1"><s ac="0">私だったら無理だよ。</s></p>',
'</body></timedtext>',
].join('\n'),
);
assert.equal(
result,
[
'WEBVTT',
'',
'00:01:38.560 --> 00:01:46.950',
'ありがとうって言えないよね。こんなんじゃ。',
'',
'00:01:46.960 --> 00:01:50.759',
'私だったら無理だよ。',
'',
].join('\n'),
);
});
test('convertYoutubeTimedTextToVtt pages oversized two-row rolling captions', () => {
const text =
'あの西に結構こう山田がスーパーアプローチしてるんだけど西気づかないからちょっとこっちも気づかない感じでこう接してあげようかなて思ってんだけどあの唇巻き込んじゃうしあの思ってることも全部縁に出ちゃって自分であちゃったって言っちゃうタイプなんで結構なんかこうドライなんだけどそこがおもろいよねみたいな';
const result = convertYoutubeTimedTextToVtt(
[
'<timedtext format="3">',
'<head>',
'<ws id="1" mh="2" ju="0" sd="3"/>',
'<wp id="1" ap="6" ah="20" av="100" rc="2" cc="40"/>',
'</head>',
'<body>',
'<w t="0" id="1" wp="1" ws="1"/>',
`<p t="60440" d="3000" w="1"><s ac="0">${text}</s></p>`,
'<p t="72695" w="1" a="1">\n</p>',
'</body>',
'</timedtext>',
].join('\n'),
);
const cues = result
.trim()
.split(/\n\n/)
.filter((block) => block.includes('-->'));
const cueText = cues.map((cue) => cue.split('\n').slice(1).join('\n'));
assert.equal(cues.length, 2);
assert.deepEqual(
cues.map((cue) => cue.split('\n')[0]),
['00:01:00.440 --> 00:01:07.064', '00:01:07.064 --> 00:01:12.695'],
);
assert.ok(cueText.every((page) => [...page].length <= 80));
assert.equal(cueText.join(''), text);
});
test('convertYoutubeTimedTextToVtt leaves pop-on captions intact', () => {
const result = convertYoutubeTimedTextToVtt(
[
'<timedtext format="3">',
'<head>',
'<ws id="1" mh="0"/>',
'<wp id="1" rc="2" cc="4"/>',
'</head>',
'<body>',
'<w t="0" id="1" wp="1" ws="1"/>',
'<p t="1000" d="3000" w="1">abcdefghijklmnopqrst</p>',
'</body>',
'</timedtext>',
].join('\n'),
);
assert.equal(
result,
['WEBVTT', '', '00:00:01.000 --> 00:00:04.000', 'abcdefghijklmnopqrst', ''].join('\n'),
);
});
test('convertYoutubeTimedTextToVtt keeps explicit 3000ms sound-cue durations in rolling documents', () => {
const result = convertYoutubeTimedTextToVtt(
[
'<timedtext><body>',
'<p t="20305" d="3000" w="1">[音楽]</p>',
'<p t="26269" w="1" a="1">\n</p>',
'<p t="26279" d="3000" w="1"><s ac="0">じゃあ、君からお願いします。</s></p>',
'</body></timedtext>',
].join('\n'),
);
assert.equal(
result,
[
'WEBVTT',
'',
'00:00:20.305 --> 00:00:23.305',
'[音楽]',
'',
'00:00:26.279 --> 00:00:29.279',
'じゃあ、君からお願いします。',
'',
].join('\n'),
);
});
test('normalizeYoutubeAutoVtt strips cumulative rolling-caption prefixes', () => {
const result = normalizeYoutubeAutoVtt(
[
+228 -12
View File
@@ -2,9 +2,31 @@ interface YoutubeTimedTextRow {
startMs: number;
durationMs: number;
text: string;
isGenerated: boolean;
rollingWindow: YoutubeRollingWindow | null;
}
interface YoutubeRollingWindow {
rowCount: number;
columnCount: number;
}
interface YoutubeTimedTextWindowDefinitions {
rollingStyleIds: Set<string>;
positions: Map<string, YoutubeRollingWindow>;
windows: Map<string, YoutubeRollingWindow>;
}
interface YoutubeTimedTextDocument {
rows: YoutubeTimedTextRow[];
// Start times of every <p> event, including empty window-append fillers.
// Rolling speech rows with a 3000ms placeholder display until the next event.
eventStartsMs: number[];
hasRollingWindowEvents: boolean;
}
const YOUTUBE_TIMEDTEXT_EXTENSIONS = new Set(['srv1', 'srv2', 'srv3', 'ytsrv3']);
const YOUTUBE_ROLLING_PLACEHOLDER_DURATION_MS = 3_000;
function decodeNumericEntity(match: string, codePoint: number): string {
if (
@@ -39,27 +61,129 @@ function parseAttributeMap(raw: string): Map<string, string> {
return attrs;
}
function extractYoutubeTimedTextRows(xml: string): YoutubeTimedTextRow[] {
function parsePositiveInteger(value: string | undefined): number | null {
if (value === undefined) {
return null;
}
const parsed = Number(value);
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : null;
}
function extractYoutubeTimedTextWindowDefinitions(xml: string): YoutubeTimedTextWindowDefinitions {
const rollingStyleIds = new Set<string>();
for (const match of xml.matchAll(/<ws\b([^>]*)\/?\s*>/g)) {
const attrs = parseAttributeMap(match[1] ?? '');
const id = attrs.get('id');
if (id !== undefined && attrs.get('mh') === '2') {
rollingStyleIds.add(id);
}
}
const positions = new Map<string, YoutubeRollingWindow>();
for (const match of xml.matchAll(/<wp\b([^>]*)\/?\s*>/g)) {
const attrs = parseAttributeMap(match[1] ?? '');
const id = attrs.get('id');
const rowCount = parsePositiveInteger(attrs.get('rc'));
const columnCount = parsePositiveInteger(attrs.get('cc'));
if (id !== undefined && rowCount !== null && columnCount !== null) {
positions.set(id, { rowCount, columnCount });
}
}
const windows = new Map<string, YoutubeRollingWindow>();
for (const match of xml.matchAll(/<w\b([^>]*)\/?\s*>/g)) {
const attrs = parseAttributeMap(match[1] ?? '');
const id = attrs.get('id');
const styleId = attrs.get('ws');
const positionId = attrs.get('wp');
const position = positionId === undefined ? undefined : positions.get(positionId);
if (
id !== undefined &&
styleId !== undefined &&
rollingStyleIds.has(styleId) &&
position !== undefined
) {
windows.set(id, position);
}
}
return { rollingStyleIds, positions, windows };
}
function resolveRollingWindow(
attrs: Map<string, string>,
definitions: YoutubeTimedTextWindowDefinitions,
): YoutubeRollingWindow | null {
const windowId = attrs.get('w');
if (windowId !== undefined) {
return definitions.windows.get(windowId) ?? null;
}
const styleId = attrs.get('ws');
const positionId = attrs.get('wp');
if (
styleId === undefined ||
positionId === undefined ||
!definitions.rollingStyleIds.has(styleId)
) {
return null;
}
return definitions.positions.get(positionId) ?? null;
}
function extractYoutubeTimedTextDocument(xml: string): YoutubeTimedTextDocument {
const rows: YoutubeTimedTextRow[] = [];
const eventStartsMs: number[] = [];
let hasRollingWindowEvents = false;
const windowDefinitions = extractYoutubeTimedTextWindowDefinitions(xml);
for (const match of xml.matchAll(/<p\b([^>]*)>([\s\S]*?)<\/p>/g)) {
const attrs = parseAttributeMap(match[1] ?? '');
const startMs = Number(attrs.get('t'));
if (!Number.isFinite(startMs)) {
continue;
}
eventStartsMs.push(startMs);
if (attrs.get('a') === '1') {
hasRollingWindowEvents = true;
}
const durationMs = Number(attrs.get('d'));
if (!Number.isFinite(startMs) || !Number.isFinite(durationMs)) {
if (!Number.isFinite(durationMs)) {
continue;
}
const inner = (match[2] ?? '').replace(/<br\s*\/?>/gi, '\n').replace(/<[^>]+>/g, '');
const rawInner = match[2] ?? '';
const inner = rawInner.replace(/<br\s*\/?>/gi, '\n').replace(/<[^>]+>/g, '');
const text = decodeHtmlEntities(inner).trim();
if (!text) {
continue;
}
rows.push({ startMs, durationMs, text });
rows.push({
startMs,
durationMs,
text,
isGenerated: /<s\b/.test(rawInner),
rollingWindow: resolveRollingWindow(attrs, windowDefinitions),
});
}
return rows;
eventStartsMs.sort((a, b) => a - b);
return { rows, eventStartsMs, hasRollingWindowEvents };
}
function findNextEventStartMs(eventStartsMs: number[], afterMs: number): number | undefined {
for (const startMs of eventStartsMs) {
if (startMs > afterMs) {
return startMs;
}
}
return undefined;
}
function isGeneratedRollingCue(row: YoutubeTimedTextRow, hasRollingWindowEvents: boolean): boolean {
return row.isGenerated && (row.rollingWindow !== null || hasRollingWindowEvents);
}
function formatVttTimestamp(ms: number): string {
@@ -71,6 +195,79 @@ function formatVttTimestamp(ms: number): string {
return `${String(hours).padStart(2, '0')}:${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')}.${String(millis).padStart(3, '0')}`;
}
const ROLLING_PAGE_BREAK_PATTERN = /[\s!?]/u;
// VTT cannot carry SRV3's row and column limits. Page only roll-up windows so
// the overlay keeps their bounded presentation without changing authored cues.
function splitRollingCaptionIntoPages(text: string, rollingWindow: YoutubeRollingWindow): string[] {
const pageCapacity = rollingWindow.rowCount * rollingWindow.columnCount;
const characters = [...text];
if (
!Number.isSafeInteger(pageCapacity) ||
pageCapacity <= 0 ||
characters.length <= pageCapacity
) {
return [text];
}
const pages: string[] = [];
let pageStart = 0;
while (pageStart < characters.length) {
let pageEnd = Math.min(pageStart + pageCapacity, characters.length);
if (pageEnd < characters.length) {
const earliestNaturalBreak = pageStart + Math.ceil(pageCapacity * 0.6);
for (let index = pageEnd - 1; index >= earliestNaturalBreak; index -= 1) {
if (ROLLING_PAGE_BREAK_PATTERN.test(characters[index]!)) {
pageEnd = index + 1;
break;
}
}
}
pages.push(characters.slice(pageStart, pageEnd).join(''));
pageStart = pageEnd;
}
return pages;
}
interface TimedCaptionPage {
startMs: number;
endMs: number;
text: string;
}
function timeCaptionPages(input: {
text: string;
pages: string[];
startMs: number;
endMs: number;
}): TimedCaptionPage[] {
const durationMs = input.endMs - input.startMs;
if (input.pages.length === 1 || durationMs < input.pages.length) {
return [{ startMs: input.startMs, endMs: input.endMs, text: input.text }];
}
const totalCharacters = [...input.text].length;
const timedPages: TimedCaptionPage[] = [];
let consumedCharacters = 0;
let pageStartMs = input.startMs;
// Automatic captions often omit span offsets, so distribute the known cue
// duration by page length while guaranteeing every page at least one ms.
for (let index = 0; index < input.pages.length; index += 1) {
const page = input.pages[index]!;
consumedCharacters += [...page].length;
const remainingPages = input.pages.length - index - 1;
const proportionalEndMs =
input.startMs + Math.round((durationMs * consumedCharacters) / totalCharacters);
const pageEndMs =
remainingPages === 0
? input.endMs
: Math.min(Math.max(proportionalEndMs, pageStartMs + 1), input.endMs - remainingPages);
timedPages.push({ startMs: pageStartMs, endMs: pageEndMs, text: page });
pageStartMs = pageEndMs;
}
return timedPages;
}
export function isYoutubeTimedTextExtension(value: string | undefined): boolean {
if (!value) {
return false;
@@ -79,7 +276,7 @@ export function isYoutubeTimedTextExtension(value: string | undefined): boolean
}
export function convertYoutubeTimedTextToVtt(xml: string): string {
const rows = extractYoutubeTimedTextRows(xml);
const { rows, eventStartsMs, hasRollingWindowEvents } = extractYoutubeTimedTextDocument(xml);
if (rows.length === 0) {
return 'WEBVTT\n';
}
@@ -90,10 +287,19 @@ export function convertYoutubeTimedTextToVtt(xml: string): string {
const row = rows[index]!;
const nextRow = rows[index + 1];
const unclampedEnd = row.startMs + row.durationMs;
// YouTube uses exactly 3000ms as a placeholder for generated rolling speech.
// Plain-text cues can explicitly use the same duration and must keep it.
const nextEventStart =
isGeneratedRollingCue(row, hasRollingWindowEvents) &&
row.durationMs === YOUTUBE_ROLLING_PLACEHOLDER_DURATION_MS
? findNextEventStartMs(eventStartsMs, row.startMs)
: undefined;
const clampedEnd =
nextRow && unclampedEnd > nextRow.startMs
? Math.max(row.startMs, nextRow.startMs - 1)
: unclampedEnd;
nextEventStart !== undefined
? nextEventStart
: nextRow && unclampedEnd > nextRow.startMs
? Math.max(row.startMs, nextRow.startMs - 1)
: unclampedEnd;
if (clampedEnd <= row.startMs) {
continue;
}
@@ -106,9 +312,19 @@ export function convertYoutubeTimedTextToVtt(xml: string): string {
if (!text) {
continue;
}
blocks.push(
`${formatVttTimestamp(row.startMs)} --> ${formatVttTimestamp(clampedEnd)}\n${text}`,
);
const pages = row.rollingWindow
? splitRollingCaptionIntoPages(text, row.rollingWindow)
: [text];
for (const page of timeCaptionPages({
text,
pages,
startMs: row.startMs,
endMs: clampedEnd,
})) {
blocks.push(
`${formatVttTimestamp(page.startMs)} --> ${formatVttTimestamp(page.endMs)}\n${page.text}`,
);
}
}
return `WEBVTT\n\n${blocks.join('\n\n')}\n`;
+17 -1
View File
@@ -1,6 +1,22 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { createNotifySendReplacer, resolveDefaultNotificationIconPath } from './notification';
import {
buildNotifySendEnv,
createNotifySendReplacer,
resolveDefaultNotificationIconPath,
} from './notification';
test('notify-send child environment drops the AppImage library-path override', () => {
const env = buildNotifySendEnv({
LD_LIBRARY_PATH: '/tmp/.mount_SubMinXXXXXX/usr/lib',
DBUS_SESSION_BUS_ADDRESS: 'unix:path=/run/user/1000/bus',
HOME: '/home/user',
});
assert.equal(env.LD_LIBRARY_PATH, undefined);
assert.equal(env.DBUS_SESSION_BUS_ADDRESS, 'unix:path=/run/user/1000/bus');
assert.equal(env.HOME, '/home/user');
});
test('default notification icon resolves packaged SubMiner asset when no per-notification icon is provided', () => {
const path = resolveDefaultNotificationIconPath({
+12 -1
View File
@@ -203,8 +203,19 @@ export function createNotifySendReplacer(
};
}
/**
* Electron AppImages export `LD_LIBRARY_PATH=<mount>/usr/lib`, whose bundled libnotify predates the
* symbols the system notify-send links against, so an inherited environment kills the child with a
* symbol lookup error before it can send anything. A system binary resolves its own libraries fine,
* so the override is dropped entirely rather than filtered.
*/
export function buildNotifySendEnv(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
const { LD_LIBRARY_PATH: _dropped, ...rest } = env;
return rest;
}
const showLinuxReplaceableNotification = createNotifySendReplacer((args, callback) =>
execFile('notify-send', args, { timeout: 5_000 }, (error, stdout) =>
execFile('notify-send', args, { timeout: 5_000, env: buildNotifySendEnv() }, (error, stdout) =>
callback(error, stdout ?? ''),
),
);
+125 -42
View File
@@ -236,6 +236,7 @@ import {
createCycleSecondarySubModeRuntimeHandler,
} from './main/runtime/domains/mpv';
import { buildSubtitleTrackDiagnostics } from './main/runtime/mpv-track-diagnostics';
import { resolveCanonicalPrimarySubtitle } from './main/runtime/primary-subtitle-text';
import {
createBuildCopyCurrentSubtitleMainDepsHandler,
createBuildHandleMineSentenceDigitMainDepsHandler,
@@ -302,7 +303,6 @@ import {
listJellyfinItemsRuntime,
listJellyfinLibrariesRuntime,
listJellyfinSubtitleTracksRuntime,
loadJellyfinSubtitleDelay,
loadSubtitlePosition as loadSubtitlePositionCore,
loadYomitanExtension as loadYomitanExtensionCore,
markLastCardAsAudioCard as markLastCardAsAudioCardCore,
@@ -312,9 +312,9 @@ import {
promoteSettingsWindowAboveOverlay,
registerGlobalShortcuts as registerGlobalShortcutsCore,
replayCurrentSubtitleRuntime,
resolveSanitizedSubtitleSeekCommand,
resolveJellyfinPlaybackPlanRuntime,
runStartupBootstrapRuntime,
saveJellyfinSubtitleDelay,
saveSubtitlePosition as saveSubtitlePositionCore,
clearYomitanParserCachesForWindow,
getYomitanCurrentAnkiDeckName as getYomitanCurrentAnkiDeckNameCore,
@@ -332,6 +332,7 @@ import {
acquireYoutubeSubtitleTrack,
acquireYoutubeSubtitleTracks,
} from './core/services/youtube/generate';
import { applyOverlayClickThrough } from './core/services/overlay-click-through';
import { createYoutubeMediaCacheService } from './core/services/youtube/media-cache';
import { resolveYoutubePlaybackUrl } from './core/services/youtube/playback-resolve';
import { probeYoutubeTracks } from './core/services/youtube/track-probe';
@@ -534,6 +535,7 @@ import {
createRefreshSubtitlePrefetchFromActiveTrackHandler,
createResolveActiveSubtitleSidebarSourceHandler,
} from './main/runtime/subtitle-prefetch-runtime';
import { createSecondarySubtitleTrackController } from './main/runtime/secondary-subtitle-track';
import {
createCreateAnilistSetupWindowHandler,
createCreateConfigSettingsWindowHandler,
@@ -599,9 +601,10 @@ import {
import { buildSubtitleSidebarSourceKey } from './main/runtime/subtitle-prefetch-source';
import { createSubtitlePrefetchInitController } from './main/runtime/subtitle-prefetch-init';
import {
createCachedInternalSubtitleTrackExtractor,
loadSubtitleSourceText,
extractInternalSubtitleTrackToTempFile,
} from './main/runtime/internal-subtitle-extraction';
import { createRemoteMediaPathDetector } from './main/runtime/network-media-path';
import { applyCharacterDictionarySelection } from './main/character-dictionary-selection';
import { getSubsyncConfig } from './subsync/utils';
@@ -687,7 +690,6 @@ function spawnManagedMpvProcess(args: string[]): ReturnType<typeof spawn> {
}
let activeJellyfinRemotePlayback: ActiveJellyfinRemotePlaybackState | null = null;
let activeJellyfinSubtitleDelayKey: { itemId: string; streamIndex: number } | null = null;
let jellyfinRemoteLastProgressAtMs = 0;
let jellyfinMpvAutoLaunchInFlight: Promise<boolean> | null = null;
let backgroundWarmupsStarted = false;
@@ -1820,10 +1822,42 @@ async function openYoutubeTrackPickerFromPlayback(): Promise<void> {
let appTray: Tray | null = null;
let tokenizeSubtitleDeferred: ((text: string) => Promise<SubtitleData>) | null = null;
function withCurrentSubtitleTiming(payload: SubtitleData): SubtitleData {
const canonical = resolveCanonicalPrimarySubtitle({
liveText: payload.text,
currentTimeSec: Number(appState.mpvClient?.currentTimePos),
cues: appState.activeParsedSubtitleCues,
});
return {
...payload,
startTime: appState.mpvClient?.currentSubStart ?? null,
endTime: appState.mpvClient?.currentSubEnd ?? null,
startTime: canonical?.startTime ?? appState.mpvClient?.currentSubStart ?? null,
endTime: canonical?.endTime ?? appState.mpvClient?.currentSubEnd ?? null,
};
}
function captureCurrentPrimarySubtitleMiningContext(): SubtitleMiningContext | null {
const canonical = resolveCanonicalPrimarySubtitle({
liveText: appState.mpvClient?.currentSubText ?? '',
currentTimeSec: Number(appState.mpvClient?.currentTimePos),
cues: appState.activeParsedSubtitleCues,
});
// Same validity bar as the live capture path: an unusable canonical span must fall
// back rather than hand mining an empty line or an inverted range.
const canonicalText = canonical?.text.trim();
if (
!canonical ||
!canonicalText ||
!Number.isFinite(canonical.startTime) ||
!Number.isFinite(canonical.endTime) ||
canonical.endTime <= canonical.startTime
) {
return captureLiveSubtitleMiningContext(appState.mpvClient);
}
return {
source: 'overlay',
text: canonicalText,
startTime: canonical.startTime,
endTime: canonical.endTime,
capturedAtMs: Date.now(),
};
}
function emitSubtitlePayload(payload: SubtitleData, options?: { resumePrefetch?: boolean }): void {
@@ -1938,6 +1972,31 @@ let linuxVisibleOverlayOwnerBindingKey: string | null = null;
let linuxVisibleOverlayWindowModeSwitchToken = 0;
let subtitleSidebarRequestedOpen = false;
const SEEK_THRESHOLD_SECONDS = 3;
const EXPLICIT_SEEK_INTENT_TTL_MS = 2000;
let explicitSeekIntentExpiresAtMs = 0;
function isExplicitMpvSeekCommand(command: readonly (string | number)[]): boolean {
return command[0] === 'seek' || command[0] === 'sub-seek';
}
function sendRendererMpvCommand(rawCommand: (string | number)[]): void {
const command =
resolveSanitizedSubtitleSeekCommand(
rawCommand,
appState.activeParsedSubtitleCues,
appState.mpvClient?.currentTimePos ?? Number.NaN,
) ?? rawCommand;
if (isExplicitMpvSeekCommand(command)) {
explicitSeekIntentExpiresAtMs = Date.now() + EXPLICIT_SEEK_INTENT_TTL_MS;
}
sendMpvCommandRuntime(appState.mpvClient, command);
}
function consumeExplicitSeekIntent(): boolean {
const pending = explicitSeekIntentExpiresAtMs >= Date.now();
explicitSeekIntentExpiresAtMs = 0;
return pending;
}
const autoplaySubtitlePrimingRuntime = createAutoplaySubtitlePrimingRuntime({
getCurrentMediaPath: () => appState.currentMediaPath,
@@ -1957,7 +2016,7 @@ const autoplaySubtitlePrimingRuntime = createAutoplaySubtitlePrimingRuntime({
getLastObservedTimePos: () => lastObservedTimePos,
getVisibleOverlayVisible: () => overlayManager.getVisibleOverlayVisible(),
emitSecondarySubtitle: (text) => {
overlayManager.broadcastToOverlayWindows('secondary-subtitle:set', text);
secondarySubtitleTrackController.handleLiveText(text);
},
initSubtitlePrefetch: (sourcePath, currentTimePos, sourceKey) =>
subtitlePrefetchInitController.initSubtitlePrefetch(sourcePath, currentTimePos, sourceKey),
@@ -2008,13 +2067,33 @@ const subtitlePrefetchInitController = createSubtitlePrefetchInitController({
}
},
});
const cachedInternalSubtitleTrackExtractor = createCachedInternalSubtitleTrackExtractor();
const detectRemoteMediaPath = createRemoteMediaPathDetector();
const resolveActiveSubtitleSidebarSourceHandler = createResolveActiveSubtitleSidebarSourceHandler({
getFfmpegPath: () => configService.getConfig().subsync.ffmpeg_path.trim() || 'ffmpeg',
extractInternalSubtitleTrack: (ffmpegPath, videoPath, track) =>
extractInternalSubtitleTrackToTempFile(ffmpegPath, videoPath, track),
cachedInternalSubtitleTrackExtractor.extract(ffmpegPath, videoPath, track),
logDebug: (message) => logger.debug(message),
});
const secondarySubtitleTrackController = createSecondarySubtitleTrackController({
getMpvClient: () => appState.mpvClient,
getCurrentTimePos: () => appState.mpvClient?.currentTimePos ?? lastObservedTimePos,
resolveSubtitleSource: (input) => resolveActiveSubtitleSidebarSourceHandler(input),
loadSubtitleSourceText,
parseSubtitleCues: (content, filename) => parseSubtitleCues(content, filename),
setCurrentSecondaryText: (text) => {
if (appState.mpvClient) {
appState.mpvClient.currentSecondarySubText = text;
}
},
broadcastSecondaryText: (text) => {
overlayManager.broadcastToOverlayWindows('secondary-subtitle:set', text);
},
logDebug: (message) => logger.debug(message),
logWarn: (message, error) => logger.warn(message, error),
});
const refreshSubtitlePrefetchFromActiveTrackHandler =
createRefreshSubtitlePrefetchFromActiveTrackHandler({
getMpvClient: () => appState.mpvClient,
@@ -2022,8 +2101,8 @@ const refreshSubtitlePrefetchFromActiveTrackHandler =
// Remote media has no extractable on-disk track to fall back to, so a transient
// resolve miss (sid briefly 'no', a cycle onto an embedded stream track) would
// otherwise drop a working cue list for the rest of the episode.
shouldKeepExistingCuesOnMissingSource: (videoPath) =>
isYoutubeMediaPath(videoPath) || isRemoteMediaPath(videoPath),
shouldKeepExistingCuesOnMissingSource: async (videoPath) =>
isYoutubeMediaPath(videoPath) || (await detectRemoteMediaPath(videoPath)),
subtitlePrefetchInitController,
resolveActiveSubtitleSidebarSource: (input) => resolveActiveSubtitleSidebarSourceHandler(input),
logDebug: (message) => logger.debug(message),
@@ -2426,7 +2505,6 @@ const fieldGroupingOverlayRuntime = createFieldGroupingOverlayRuntime<OverlayHos
const createFieldGroupingCallback = fieldGroupingOverlayRuntime.createFieldGroupingCallback;
const SUBTITLE_POSITIONS_DIR = path.join(CONFIG_DIR, 'subtitle-positions');
const JELLYFIN_SUBTITLE_DELAYS_PATH = path.join(CONFIG_DIR, 'jellyfin-subtitle-delays.json');
/**
* What the anime browser resolved for the stream that is playing. Consulted by
@@ -2627,6 +2705,12 @@ const characterDictionaryAutoSyncRuntime = createCharacterDictionaryAutoSyncRunt
const characterDictionaryImageLookup = createCharacterDictionaryImageLookup({
userDataPath: USER_DATA_PATH,
getCurrentMediaId: () => characterDictionaryAutoSyncRuntime.getCurrentMediaId(),
onIndexReady: () => refreshCurrentSubtitleAnnotations(),
onIndexReadyError: (error) =>
logger.warn(
'Failed to refresh subtitle annotations after character portrait index became ready.',
error,
),
});
// Lets the Yomitan scan runtime skip name lookups at positions where no
@@ -3096,23 +3180,6 @@ const {
wait: (ms) => new Promise<void>((resolve) => setTimeout(resolve, ms)),
cacheSubtitleTrack: (track) => jellyfinSubtitleCacheIo.cacheSubtitleTrack(track),
cleanupCachedSubtitles: (dirs) => jellyfinSubtitleCacheIo.cleanupCachedSubtitles(dirs),
getSavedSubtitleDelay: (itemId, streamIndex) =>
loadJellyfinSubtitleDelay({
filePath: JELLYFIN_SUBTITLE_DELAYS_PATH,
itemId,
streamIndex,
}),
setActiveSubtitleDelayKey: (key) => {
activeJellyfinSubtitleDelayKey = key;
},
loadSubtitleSourceText,
saveSubtitleDelay: (itemId, streamIndex, delaySeconds) =>
saveJellyfinSubtitleDelay({
filePath: JELLYFIN_SUBTITLE_DELAYS_PATH,
itemId,
streamIndex,
delaySeconds,
}),
initSubtitlePrefetch: (sourcePath) =>
subtitlePrefetchRuntime.refreshSubtitleSidebarFromSource(sourcePath),
logDebug: (message, error) => {
@@ -3178,7 +3245,6 @@ const {
getActivePlayback: () => activeJellyfinRemotePlayback,
clearActivePlayback: () => {
activeJellyfinRemotePlayback = null;
activeJellyfinSubtitleDelayKey = null;
},
getSession: () => appState.jellyfinRemoteSession,
getNow: () => Date.now(),
@@ -4051,6 +4117,7 @@ const {
appState.yomitanSettingsWindow = null;
},
stopJellyfinRemoteSession: () => stopJellyfinRemoteSession(),
cleanupInternalSubtitleTrackCache: () => cachedInternalSubtitleTrackExtractor.clear(),
cleanupYoutubeSubtitleTempDirs: () => youtubeFlowRuntime.cleanupSubtitleTempDirs(),
cleanupYoutubeMediaCache: () => youtubeMediaCache.cleanup(),
cleanupJellyfinSubtitleCache: () => cleanupJellyfinSubtitleCache(),
@@ -4168,7 +4235,7 @@ const recordTrackedCardsMined = (count: number, noteIds?: number[]): void => {
ensureImmersionTrackerStarted();
appState.immersionTracker?.recordCardsMined(count, noteIds);
};
const refreshCurrentSubtitleAfterKnownWordUpdate = (): void => {
function refreshCurrentSubtitleAnnotations(): void {
const hasCurrentSubtitle = appState.currentSubText.trim().length > 0;
if (hasCurrentSubtitle) {
subtitlePrefetchService?.pause();
@@ -4179,7 +4246,7 @@ const refreshCurrentSubtitleAfterKnownWordUpdate = (): void => {
// Idle controller: no settle is coming to release the pause above.
subtitlePrefetchService?.resume();
}
};
}
let hasAttemptedImmersionTrackerStartup = false;
const ensureImmersionTrackerStarted = (): void => {
if (hasAttemptedImmersionTrackerStartup || appState.immersionTracker) {
@@ -4556,6 +4623,7 @@ const {
onMpvConnected: () => {
maybeStartOverlayLoadingOsd();
flushQueuedMpvOsdNotifications();
secondarySubtitleTrackController.scheduleRefresh(0);
if (appState.sessionBindingsInitialized) {
sendMpvCommandRuntime(appState.mpvClient, [
'script-message',
@@ -4574,6 +4642,9 @@ const {
broadcastToOverlayWindows: (channel, payload) => {
overlayManager.broadcastToOverlayWindows(channel, payload);
},
onSecondarySubtitleChange: (text) => {
secondarySubtitleTrackController.handleLiveText(text);
},
getImmediateSubtitlePayload: (text) => subtitleProcessingController.consumeCachedSubtitle(text),
emitImmediateSubtitle: (payload) => {
emitSubtitlePayload(payload);
@@ -4607,6 +4678,8 @@ const {
appState.activeParsedSubtitleMediaPath,
);
if ((normalizedPath || null) !== previousPath) {
cachedInternalSubtitleTrackExtractor.clear();
secondarySubtitleTrackController.reset();
const resetSubtitlePayload = { text: '', tokens: null };
const frequencyDictionary = configService.getConfig().subtitleStyle.frequencyDictionary;
const frequencyOptions = {
@@ -4624,7 +4697,6 @@ const {
appState.activeParsedSubtitleSource = null;
appState.activeParsedSubtitleMediaPath = null;
}
activeJellyfinSubtitleDelayKey = null;
overlayManager.broadcastToOverlayWindows('subtitle:set', resetSubtitlePayload);
subtitleWsService.broadcast(resetSubtitlePayload, frequencyOptions);
annotationSubtitleWsService.broadcast(resetSubtitlePayload, frequencyOptions);
@@ -4641,6 +4713,7 @@ const {
void youtubeMediaCachePlaybackRuntime.handleMediaPathChange(path);
if (path) {
ensureImmersionTrackerStarted();
secondarySubtitleTrackController.scheduleRefresh();
void subtitlePrefetchRuntime.refreshSubtitlePrefetchFromActiveTrack();
// Retry after a short delay because MPV can populate track-list after path.
subtitlePrefetchRuntime.scheduleSubtitlePrefetchRefresh(500);
@@ -4689,12 +4762,14 @@ const {
reportJellyfinRemoteProgress: (forceImmediate) => {
void reportJellyfinRemoteProgress(forceImmediate);
},
consumeExplicitSeek: () => consumeExplicitSeekIntent(),
onTimePosUpdate: (time) => {
const delta = time - lastObservedTimePos;
if (subtitlePrefetchService && (delta > SEEK_THRESHOLD_SECONDS || delta < 0)) {
subtitlePrefetchService.onSeek(time);
}
lastObservedTimePos = time;
secondarySubtitleTrackController.handleTimePos(time);
},
onFullscreenChange: (fullscreen) => {
cancelLinuxMpvFullscreenOverlayRefreshBurst = updateLinuxMpvFullscreenOverlayRefreshBurst(
@@ -4722,6 +4797,13 @@ const {
autoplaySubtitlePrimingRuntime.scheduleSubtitlePrefetchRefresh();
youtubePrimarySubtitleNotificationRuntime.handleSubtitleTrackChange(sid);
},
onSecondarySubtitleTrackChange: () => {
secondarySubtitleTrackController.handleTrackChange();
secondarySubtitleTrackController.scheduleRefresh(0);
},
onSecondarySubtitleDelayChange: (delay) => {
secondarySubtitleTrackController.handleDelayChange(delay);
},
onSubtitleTrackListChange: (trackList) => {
const diagnostics = buildSubtitleTrackDiagnostics(
lastObservedPrimarySubtitleTrackId,
@@ -4735,6 +4817,7 @@ const {
logger.info('[mpv-subtitles] subtitle track list updated', diagnostics);
}
managedLocalSubtitleSelectionRuntime.handleSubtitleTrackListChange(trackList);
secondarySubtitleTrackController.scheduleRefresh(0);
autoplaySubtitlePrimingRuntime.scheduleSubtitlePrefetchRefresh();
youtubePrimarySubtitleNotificationRuntime.handleSubtitleTrackListChange(trackList);
},
@@ -5187,9 +5270,7 @@ function initializeOverlayRuntime(): void {
overlayModalRuntime.primeModalWindow();
}
appState.ankiIntegration?.setRecordCardsMinedCallback(recordTrackedCardsMined);
appState.ankiIntegration?.setKnownWordCacheUpdatedCallback(
refreshCurrentSubtitleAfterKnownWordUpdate,
);
appState.ankiIntegration?.setKnownWordCacheUpdatedCallback(refreshCurrentSubtitleAnnotations);
appState.ankiIntegration?.setSubtitleMiningContextConsumer(consumePendingSubtitleMiningContext);
syncOverlayMpvSubtitleSuppression();
}
@@ -5404,6 +5485,7 @@ const markLastCardAsAudioCardHandler = createMarkLastCardAsAudioCardHandler(
const buildMineSentenceCardMainDepsHandler = createBuildMineSentenceCardMainDepsHandler({
getAnkiIntegration: () => appState.ankiIntegration,
getMpvClient: () => appState.mpvClient,
getPrimarySubtitle: () => captureCurrentPrimarySubtitleMiningContext(),
showMpvOsd: (text) => overlayNotificationsRuntime.showConfiguredStatusNotification(text),
mineSentenceCardCore,
recordCardsMined: (count, noteIds) => {
@@ -5588,8 +5670,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
showPlaybackFeedback: (text: string) => showConfiguredPlaybackFeedback(text),
replayCurrentSubtitle: () => replayCurrentSubtitleRuntime(appState.mpvClient),
playNextSubtitle: () => playNextSubtitleRuntime(appState.mpvClient),
sendMpvCommand: (rawCommand: (string | number)[]) =>
sendMpvCommandRuntime(appState.mpvClient, rawCommand),
sendMpvCommand: (rawCommand: (string | number)[]) => sendRendererMpvCommand(rawCommand),
getMpvClient: () => appState.mpvClient,
isMpvConnected: () => Boolean(appState.mpvClient && appState.mpvClient.connected),
hasRuntimeOptionsManager: () => appState.runtimeOptionsManager !== null,
@@ -5645,7 +5726,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
senderWindow === modalWindow &&
!senderWindow.isDestroyed()
) {
senderWindow.setIgnoreMouseEvents(true, { forward: true });
applyOverlayClickThrough(senderWindow);
senderWindow.hide();
}
handleOverlayModalClosedHandler(modal);
@@ -5719,9 +5800,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
// live mpv sub timings at lookup time so media generation clips the mined line even
// when extraction finishes long after playback has moved on.
recordSubtitleMiningContext: (context) =>
recordSubtitleMiningContext(
context ?? captureLiveSubtitleMiningContext(appState.mpvClient),
),
recordSubtitleMiningContext(context ?? captureCurrentPrimarySubtitleMiningContext()),
quitApp: () => requestAppQuit(),
toggleVisibleOverlay: () => toggleVisibleOverlay(),
tokenizeCurrentSubtitle: async () => {
@@ -5985,7 +6064,7 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
appState.ankiIntegration = integration;
appState.ankiIntegration?.setRecordCardsMinedCallback(recordTrackedCardsMined);
appState.ankiIntegration?.setKnownWordCacheUpdatedCallback(
refreshCurrentSubtitleAfterKnownWordUpdate,
refreshCurrentSubtitleAnnotations,
);
appState.ankiIntegration?.setSubtitleMiningContextConsumer(
consumePendingSubtitleMiningContext,
@@ -5999,6 +6078,8 @@ const { registerIpcRuntimeHandlers } = composeIpcRuntimeHandlers({
showDesktopNotification,
showOverlayNotification: (payload) =>
overlayNotificationsRuntime.showOverlayNotification(payload),
dismissOverlayNotification: (id) =>
overlayNotificationsRuntime.dismissOverlayNotification(id),
createFieldGroupingCallback: () => createFieldGroupingCallback(),
broadcastRuntimeOptionsChanged: () =>
overlayVisibilityComposer.broadcastRuntimeOptionsChanged(),
@@ -6496,6 +6577,8 @@ const { initializeOverlayRuntime: initializeOverlayRuntimeHandler } =
showDesktopNotification,
showOverlayNotification: (payload) =>
overlayNotificationsRuntime.showOverlayNotification(payload),
dismissOverlayNotification: (id) =>
overlayNotificationsRuntime.dismissOverlayNotification(id),
createFieldGroupingCallback: () => createFieldGroupingCallback(),
getKnownWordCacheStatePath: () => path.join(USER_DATA_PATH, 'known-words-cache.json'),
getCachedMediaPath: (currentVideoPath, kind) =>
+22 -17
View File
@@ -244,13 +244,13 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
};
};
const findCachedSnapshotForSeriesKey = (
const findCachedSnapshotForSeriesKey = async (
seriesKey: string,
fallbackSeriesKey?: string,
): CharacterDictionarySnapshot | null => {
): Promise<CharacterDictionarySnapshot | null> => {
const acceptedKeys = new Set([seriesKey, fallbackSeriesKey].filter(Boolean));
return (
readCachedSnapshots(outputDir).find((snapshot) => {
(await readCachedSnapshots(outputDir)).find((snapshot) => {
const snapshotSeriesKey = buildCharacterDictionarySeriesKey({
mediaPath: null,
mediaTitle: snapshot.mediaTitle,
@@ -293,7 +293,9 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
const cachedResolution = readCachedMediaResolution(outputDir, seriesKey);
if (cachedResolution) {
const cachedSnapshot = readSnapshot(getSnapshotPath(outputDir, cachedResolution.mediaId));
const cachedSnapshot = await readSnapshot(
getSnapshotPath(outputDir, cachedResolution.mediaId),
);
if (cachedSnapshot) {
deps.logInfo?.(
`[dictionary] cached AniList match: ${cachedSnapshot.mediaTitle} -> AniList ${cachedSnapshot.mediaId}`,
@@ -305,7 +307,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
}
}
const cachedSnapshot = findCachedSnapshotForSeriesKey(seriesKey, unscopedSeriesKey);
const cachedSnapshot = await findCachedSnapshotForSeriesKey(seriesKey, unscopedSeriesKey);
if (cachedSnapshot) {
writeCachedMediaResolution(outputDir, {
seriesKey,
@@ -348,7 +350,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
progress?: CharacterDictionarySnapshotProgressCallbacks,
): Promise<CharacterDictionarySnapshotResult> => {
const snapshotPath = getSnapshotPath(outputDir, mediaId);
const cachedSnapshot = readSnapshot(snapshotPath);
const cachedSnapshot = await readSnapshot(snapshotPath);
const refreshReason = cachedSnapshot ? getCachedSnapshotRefreshReason(cachedSnapshot) : null;
if (cachedSnapshot && refreshReason === null) {
deps.logInfo?.(`[dictionary] snapshot hit for AniList ${mediaId}`);
@@ -448,7 +450,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
}
const nameSplitTokenizerAvailable = isNameSplitTokenizerAvailable();
const resolvedNameSplits = nameSplitTokenizerAvailable
const nameSplitResolution = nameSplitTokenizerAvailable
? await resolveJapaneseNameSplits(
characters,
deps.tokenizeJapaneseName!,
@@ -464,8 +466,8 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
},
)
: undefined;
const nameSplitSource =
resolvedNameSplits && resolvedNameSplits.size > 0 ? 'mecab' : 'heuristic';
const resolvedNameSplits = nameSplitResolution?.splits;
const nameSplitSource = nameSplitResolution?.kind === 'complete' ? 'mecab' : 'heuristic';
progress?.onGenerateProgress?.({
mediaId,
@@ -485,7 +487,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
resolvedNameSplits,
nameSplitSource,
);
writeSnapshot(snapshotPath, snapshot);
await writeSnapshot(snapshotPath, snapshot);
deps.logInfo?.(
`[dictionary] stored snapshot for AniList ${mediaId}: ${snapshot.entryCount} terms`,
);
@@ -526,19 +528,22 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
const snapshotResults = await Promise.all(
normalizedMediaIds.map((mediaId) => getOrCreateSnapshot(mediaId)),
);
const snapshots = snapshotResults.map(({ mediaId }) => {
const snapshot = readSnapshot(getSnapshotPath(outputDir, mediaId));
// Sequential on purpose: each snapshot parse is a chunk of main-thread work, so reading them
// one at a time keeps the event loop breathing between files.
const snapshots: CharacterDictionarySnapshot[] = [];
for (const { mediaId } of snapshotResults) {
const snapshot = await readSnapshot(getSnapshotPath(outputDir, mediaId));
if (!snapshot) {
throw new Error(`Missing character dictionary snapshot for AniList ${mediaId}.`);
}
return snapshot;
});
snapshots.push(snapshot);
}
const revision = buildMergedRevision(normalizedMediaIds, snapshots);
const description =
snapshots.length === 1
? `Character names from ${snapshots[0]!.mediaTitle}`
: `Character names from ${snapshots.length} recent anime`;
const { zipPath, entryCount } = buildDictionaryZip(
const { zipPath, entryCount } = await buildDictionaryZip(
getMergedZipPath(outputDir),
CHARACTER_DICTIONARY_MERGED_TITLE,
description,
@@ -633,7 +638,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
resolvedMedia.title,
waitForAniListRequestSlot,
);
const storedSnapshot = readSnapshot(getSnapshotPath(outputDir, resolvedMedia.id));
const storedSnapshot = await readSnapshot(getSnapshotPath(outputDir, resolvedMedia.id));
if (!storedSnapshot) {
throw new Error(`Snapshot missing after generation for AniList ${resolvedMedia.id}.`);
}
@@ -642,7 +647,7 @@ export function createCharacterDictionaryRuntimeService(deps: CharacterDictionar
const description = `Character names from ${storedSnapshot.mediaTitle} [AniList media ID ${resolvedMedia.id}]`;
const zipPath = path.join(outputDir, `anilist-${resolvedMedia.id}.zip`);
deps.logInfo?.(`[dictionary] building ZIP for AniList ${resolvedMedia.id}`);
buildDictionaryZip(
await buildDictionaryZip(
zipPath,
dictionaryTitle,
description,
@@ -3,6 +3,7 @@ import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import test from 'node:test';
import { isDeepStrictEqual } from 'node:util';
import { getSnapshotPath, readSnapshot, writeSnapshot } from './cache';
import { CHARACTER_DICTIONARY_FORMAT_VERSION } from './constants';
import type { CharacterDictionarySnapshot } from './types';
@@ -29,17 +30,72 @@ function createSnapshot(): CharacterDictionarySnapshot {
};
}
test('writeSnapshot persists and readSnapshot restores current-format snapshots', () => {
test('writeSnapshot persists and readSnapshot restores current-format snapshots', async () => {
const outputDir = makeTempDir();
const snapshotPath = getSnapshotPath(outputDir, 130298);
const snapshot = createSnapshot();
writeSnapshot(snapshotPath, snapshot);
await writeSnapshot(snapshotPath, snapshot);
assert.deepEqual(readSnapshot(snapshotPath), { ...snapshot, nameSplitSource: 'heuristic' });
assert.deepEqual(await readSnapshot(snapshotPath), { ...snapshot, nameSplitSource: 'heuristic' });
});
test('readSnapshot preserves the mecab name-split source and defaults missing values to heuristic', () => {
// A manual generate and an auto-sync can both land on the same media, so two writes for one
// snapshot can overlap. They must not stream into a shared temp file and interleave into a
// half-and-half snapshot.
test('concurrent writeSnapshot calls for the same media leave one complete snapshot', async () => {
const outputDir = makeTempDir();
const snapshotPath = getSnapshotPath(outputDir, 130298);
const base = createSnapshot();
// Distinct titles, lengths, and term text so the surviving file can be pinned to exactly one
// writer rather than merely "a snapshot that parses". A shared temp file is caught by the
// losing writers failing to rename; interleaved content is only caught when the timing happens
// to leave a mix, which is why the assertion checks identity rather than shape.
const variants: CharacterDictionarySnapshot[] = ['alpha', 'beta', 'gamma'].map((label, index) => {
const entryCount = 400 + index * 100;
return {
...base,
mediaTitle: `${base.mediaTitle} ${label}`,
entryCount,
termEntries: Array.from({ length: entryCount }, (_entry, entryIndex) => [
`${label}${entryIndex}`,
'なまえ',
'name primary',
'',
75,
[`${label} character ${entryIndex} `.repeat(600)],
0,
'',
]) as CharacterDictionarySnapshot['termEntries'],
};
});
await Promise.all(variants.map((variant) => writeSnapshot(snapshotPath, variant)));
const restored = await readSnapshot(snapshotPath);
const expected = variants.map((variant) => ({
...variant,
nameSplitSource: 'heuristic' as const,
}));
const matches = expected.filter((candidate) => isDeepStrictEqual(restored, candidate));
assert.equal(
matches.length,
1,
`expected exactly one writer's complete snapshot to survive, got ${
restored === null
? 'an unreadable file'
: `entryCount=${restored.entryCount}, terms=${restored.termEntries.length}, title=${restored.mediaTitle}`
}`,
);
// Every writer cleaned up after itself, so no temp files are left behind.
const leftovers = fs
.readdirSync(path.dirname(snapshotPath))
.filter((name) => name.includes('.tmp-'));
assert.deepEqual(leftovers, []);
});
test('readSnapshot preserves the mecab name-split source and defaults missing values to heuristic', async () => {
const outputDir = makeTempDir();
const snapshotPath = getSnapshotPath(outputDir, 130298);
const snapshot: CharacterDictionarySnapshot = {
@@ -47,12 +103,12 @@ test('readSnapshot preserves the mecab name-split source and defaults missing va
nameSplitSource: 'mecab',
};
writeSnapshot(snapshotPath, snapshot);
await writeSnapshot(snapshotPath, snapshot);
assert.equal(readSnapshot(snapshotPath)?.nameSplitSource, 'mecab');
assert.equal((await readSnapshot(snapshotPath))?.nameSplitSource, 'mecab');
});
test('readSnapshot ignores snapshots written with an older format version', () => {
test('readSnapshot ignores snapshots written with an older format version', async () => {
const outputDir = makeTempDir();
const snapshotPath = getSnapshotPath(outputDir, 130298);
const staleSnapshot = {
@@ -63,10 +119,10 @@ test('readSnapshot ignores snapshots written with an older format version', () =
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
fs.writeFileSync(snapshotPath, JSON.stringify(staleSnapshot), 'utf8');
assert.equal(readSnapshot(snapshotPath), null);
assert.equal(await readSnapshot(snapshotPath), null);
});
test('readSnapshot ignores v15 snapshots with stale romanized character-name entries', () => {
test('readSnapshot ignores v15 snapshots with stale romanized character-name entries', async () => {
const outputDir = makeTempDir();
const snapshotPath = getSnapshotPath(outputDir, 130298);
const staleSnapshot = {
@@ -78,5 +134,5 @@ test('readSnapshot ignores v15 snapshots with stale romanized character-name ent
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
fs.writeFileSync(snapshotPath, JSON.stringify(staleSnapshot), 'utf8');
assert.equal(readSnapshot(snapshotPath), null);
assert.equal(await readSnapshot(snapshotPath), null);
});
+83 -10
View File
@@ -102,24 +102,42 @@ export function writeCachedMediaResolution(
writeMediaResolutionEntries(outputDir, [...remaining, normalized]);
}
export function readCachedSnapshots(outputDir: string): CharacterDictionarySnapshot[] {
/**
* Snapshots for long series run to hundreds of MB each, so everything here reads them off the main
* thread's critical path: file IO is async and only the unavoidable JSON.parse runs on the loop,
* one file at a time. Reading the whole directory synchronously used to block the process for
* multiple seconds, long enough for the compositor to declare the app unresponsive mid-playback.
*/
export async function readCachedSnapshots(
outputDir: string,
): Promise<CharacterDictionarySnapshot[]> {
let entries: fs.Dirent[] = [];
try {
entries = fs.readdirSync(getSnapshotsDir(outputDir), { withFileTypes: true });
entries = await fs.promises.readdir(getSnapshotsDir(outputDir), { withFileTypes: true });
} catch {
return [];
}
return entries
const names = entries
.filter((entry) => entry.isFile() && /^anilist-\d+\.json$/.test(entry.name))
.sort((left, right) => left.name.localeCompare(right.name))
.map((entry) => readSnapshot(path.join(getSnapshotsDir(outputDir), entry.name)))
.filter((snapshot): snapshot is CharacterDictionarySnapshot => snapshot !== null);
.map((entry) => entry.name)
.sort((left, right) => left.localeCompare(right));
const snapshots: CharacterDictionarySnapshot[] = [];
for (const name of names) {
const snapshot = await readSnapshot(path.join(getSnapshotsDir(outputDir), name));
if (snapshot) {
snapshots.push(snapshot);
}
}
return snapshots;
}
export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot | null {
export async function readSnapshot(
snapshotPath: string,
): Promise<CharacterDictionarySnapshot | null> {
try {
const raw = fs.readFileSync(snapshotPath, 'utf8');
const raw = await fs.promises.readFile(snapshotPath, 'utf8');
const parsed = JSON.parse(raw) as Partial<CharacterDictionarySnapshot>;
if (!parsed || typeof parsed !== 'object') {
return null;
@@ -150,9 +168,64 @@ export function readSnapshot(snapshotPath: string): CharacterDictionarySnapshot
}
}
export function writeSnapshot(snapshotPath: string, snapshot: CharacterDictionarySnapshot): void {
// Flushing in a few-MB batches keeps each stringify-and-write slice short; a single
// JSON.stringify of a large snapshot blocks the event loop for seconds.
const SNAPSHOT_WRITE_FLUSH_BYTES = 4 * 1024 * 1024;
// Distinguishes concurrent writes of the same snapshot within one process; the pid alone only
// separates processes, so two overlapping writers would otherwise stream into the same temp file.
let snapshotWriteSequence = 0;
/**
* Streams the snapshot to disk piece by piece instead of stringifying it in one shot, then renames
* the finished file into place so a crash mid-write (or two concurrent writers for the same media)
* can never leave a torn file where a snapshot used to be.
*/
export async function writeSnapshot(
snapshotPath: string,
snapshot: CharacterDictionarySnapshot,
): Promise<void> {
ensureDir(path.dirname(snapshotPath));
fs.writeFileSync(snapshotPath, JSON.stringify(snapshot, null, 2), 'utf8');
snapshotWriteSequence += 1;
const tempPath = `${snapshotPath}.tmp-${process.pid}-${snapshotWriteSequence}`;
const handle = await fs.promises.open(tempPath, 'w');
try {
let buffered: string[] = [];
let bufferedBytes = 0;
const push = async (chunk: string): Promise<void> => {
buffered.push(chunk);
bufferedBytes += chunk.length;
if (bufferedBytes >= SNAPSHOT_WRITE_FLUSH_BYTES) {
const joined = buffered.join('');
buffered = [];
bufferedBytes = 0;
await handle.write(joined, null, 'utf8');
}
};
const writeArray = async (key: string, items: readonly unknown[]): Promise<void> => {
await push(`,${JSON.stringify(key)}:[`);
for (let i = 0; i < items.length; i += 1) {
await push(`${i > 0 ? ',' : ''}${JSON.stringify(items[i])}`);
}
await push(']');
};
const { termEntries, images, ...scalars } = snapshot;
const head = JSON.stringify(scalars);
await push(head.slice(0, -1));
await writeArray('termEntries', termEntries);
await writeArray('images', images);
await push('}');
if (buffered.length > 0) {
await handle.write(buffered.join(''), null, 'utf8');
}
} catch (error) {
await handle.close();
await fs.promises.rm(tempPath, { force: true });
throw error;
}
await handle.close();
await fs.promises.rename(tempPath, snapshotPath);
}
export function buildMergedRevision(
@@ -11,6 +11,22 @@ import {
} from './image-lookup';
import type { CharacterDictionarySnapshot } from './types';
// Lookup indexes rebuild in the background while gets serve stale data, so tests poll until the
// refresh they triggered has landed.
async function waitForRefresh<T>(probe: () => T | null | undefined): Promise<T> {
const deadline = Date.now() + 5000;
for (;;) {
const value = probe();
if (value !== null && value !== undefined) {
return value;
}
if (Date.now() > deadline) {
throw new Error('timed out waiting for background snapshot refresh');
}
await new Promise((resolve) => setTimeout(resolve, 5));
}
}
const PNG_1X1_BASE64 =
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+nmX8AAAAASUVORK5CYII=';
@@ -18,7 +34,7 @@ function makeTempDir(): string {
return fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-character-image-lookup-'));
}
test('buildCharacterNameImageIndexFromSnapshots maps name terms to character portrait data URLs', () => {
test('buildCharacterNameImageIndexFromSnapshots maps name terms to character portrait data URLs', async () => {
const outputDir = makeTempDir();
const snapshot: CharacterDictionarySnapshot = {
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
@@ -75,9 +91,9 @@ test('buildCharacterNameImageIndexFromSnapshots maps name terms to character por
{ path: 'img/m130298-va456.png', dataBase64: 'BBBB' },
],
};
writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
const index = buildCharacterNameImageIndexFromSnapshots(outputDir);
const index = await buildCharacterNameImageIndexFromSnapshots(outputDir);
assert.deepEqual(index.get('アレクシア'), {
src: 'data:image/png;base64,AAAA',
@@ -85,7 +101,7 @@ test('buildCharacterNameImageIndexFromSnapshots maps name terms to character por
});
});
test('buildCharacterNameImageIndexFromSnapshots sniffs image MIME from bytes before path extension', () => {
test('buildCharacterNameImageIndexFromSnapshots sniffs image MIME from bytes before path extension', async () => {
const outputDir = makeTempDir();
const snapshot: CharacterDictionarySnapshot = {
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
@@ -116,14 +132,14 @@ test('buildCharacterNameImageIndexFromSnapshots sniffs image MIME from bytes bef
],
images: [{ path: 'img/m130298-c123.jpg', dataBase64: PNG_1X1_BASE64 }],
};
writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
const index = buildCharacterNameImageIndexFromSnapshots(outputDir);
const index = await buildCharacterNameImageIndexFromSnapshots(outputDir);
assert.equal(index.get('アレクシア')?.src, `data:image/png;base64,${PNG_1X1_BASE64}`);
});
test('createCharacterDictionaryImageLookup can scope duplicate names to the current media', () => {
test('createCharacterDictionaryImageLookup can scope duplicate names to the current media', async () => {
const outputDir = makeTempDir();
const towerSnapshot: CharacterDictionarySnapshot = {
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
@@ -173,15 +189,76 @@ test('createCharacterDictionaryImageLookup can scope duplicate names to the curr
],
images: [{ path: 'img/m21202-c2.png', dataBase64: 'KONOSUBA' }],
};
writeSnapshot(getSnapshotPath(outputDir, towerSnapshot.mediaId), towerSnapshot);
writeSnapshot(getSnapshotPath(outputDir, konosubaSnapshot.mediaId), konosubaSnapshot);
await writeSnapshot(getSnapshotPath(outputDir, towerSnapshot.mediaId), towerSnapshot);
await writeSnapshot(getSnapshotPath(outputDir, konosubaSnapshot.mediaId), konosubaSnapshot);
const lookup = createCharacterDictionaryImageLookup({ outputDir });
assert.equal(lookup.get('カズ', 21202)?.alt, 'Kazuma');
const scoped = await waitForRefresh(() => lookup.get('カズ', 21202));
assert.equal(scoped.alt, 'Kazuma');
});
test('createCharacterDictionaryImageLookup does not fall back globally on scoped miss', () => {
test('createCharacterDictionaryImageLookup reports and retries a failed index-ready callback', async () => {
const outputDir = makeTempDir();
const snapshot: CharacterDictionarySnapshot = {
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
mediaId: 21858,
mediaTitle: 'Little Witch Academia',
entryCount: 1,
updatedAt: 1_700_000_000_000,
termEntries: [
[
'ダイアナ',
'だいあな',
'name primary',
'',
75,
[
{
type: 'structured-content',
content: {
tag: 'img',
path: 'img/m21858-c81709.png',
alt: 'ダイアナ・キャベンディッシュ',
},
},
],
0,
'',
],
],
images: [{ path: 'img/m21858-c81709.png', dataBase64: PNG_1X1_BASE64 }],
};
await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
const callbackError = new Error('annotation refresh failed');
const reportingError = new Error('error reporter failed');
let readyCount = 0;
const reportedErrors: unknown[] = [];
const lookup = createCharacterDictionaryImageLookup({
outputDir,
onIndexReady: () => {
readyCount += 1;
if (readyCount === 1) {
throw callbackError;
}
},
onIndexReadyError: (error) => {
reportedErrors.push(error);
throw reportingError;
},
});
assert.equal(lookup.get('ダイアナ', snapshot.mediaId), null);
await waitForRefresh(() => (reportedErrors.length === 1 ? true : null));
assert.ok(lookup.get('ダイアナ', snapshot.mediaId));
assert.equal(readyCount, 2);
assert.deepEqual(reportedErrors, [callbackError]);
lookup.get('ダイアナ', snapshot.mediaId);
assert.equal(readyCount, 2);
});
test('createCharacterDictionaryImageLookup does not fall back globally on scoped miss', async () => {
const outputDir = makeTempDir();
const snapshot: CharacterDictionarySnapshot = {
formatVersion: CHARACTER_DICTIONARY_FORMAT_VERSION,
@@ -208,10 +285,11 @@ test('createCharacterDictionaryImageLookup does not fall back globally on scoped
],
images: [{ path: 'img/m115230-c1.png', dataBase64: 'TOWER' }],
};
writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
await writeSnapshot(getSnapshotPath(outputDir, snapshot.mediaId), snapshot);
const lookup = createCharacterDictionaryImageLookup({ outputDir });
const unscoped = await waitForRefresh(() => lookup.get('カズ'));
assert.equal(unscoped.alt, 'Kaz');
assert.equal(lookup.get('カズ', 21202), null);
assert.equal(lookup.get('カズ')?.alt, 'Kaz');
});
@@ -204,11 +204,11 @@ function getSnapshotDirectorySignature(outputDir: string): string {
return parts.sort().join('|');
}
export function buildCharacterNameImageIndexFromSnapshots(
export async function buildCharacterNameImageIndexFromSnapshots(
outputDir: string,
): Map<string, CharacterNameImage> {
): Promise<Map<string, CharacterNameImage>> {
const index = new Map<string, CharacterNameImage>();
for (const snapshot of readCachedSnapshots(outputDir)) {
for (const snapshot of await readCachedSnapshots(outputDir)) {
appendSnapshotImages(index, snapshot);
}
return index;
@@ -218,6 +218,8 @@ export function createCharacterDictionaryImageLookup(deps: {
userDataPath?: string;
outputDir?: string;
getCurrentMediaId?: () => number | null | undefined;
onIndexReady?: () => void;
onIndexReadyError?: (error: unknown) => void;
}): {
get: (term: string, mediaId?: number | null) => CharacterNameImage | null;
invalidate: () => void;
@@ -228,7 +230,30 @@ export function createCharacterDictionaryImageLookup(deps: {
let signature: string | null = null;
let index = new Map<string, CharacterNameImage>();
let indexByMediaId = new Map<number, Map<string, CharacterNameImage>>();
let refreshInFlight = false;
let indexReadyDeliveryPending = false;
function deliverIndexReadyIfPending(): void {
if (!indexReadyDeliveryPending || !deps.onIndexReady) {
return;
}
indexReadyDeliveryPending = false;
try {
deps.onIndexReady();
} catch (error) {
indexReadyDeliveryPending = true;
try {
deps.onIndexReadyError?.(error);
} catch {
// Error reporting must not reject the detached index refresh task.
}
}
}
// Rebuilding means re-reading every cached snapshot (potentially GBs of JSON), which used to run
// synchronously inside a lookup and froze the whole app right after a snapshot changed. Lookups
// now serve the previous index while a single background rebuild catches up; the swap is atomic
// and the signature only advances once the rebuild it belongs to has landed.
function refreshIfNeeded(): void {
if (!outputDir) {
index = new Map<string, CharacterNameImage>();
@@ -236,21 +261,34 @@ export function createCharacterDictionaryImageLookup(deps: {
signature = '';
return;
}
deliverIndexReadyIfPending();
const nextSignature = getSnapshotDirectorySignature(outputDir);
if (nextSignature === signature) {
if (nextSignature === signature || refreshInFlight) {
return;
}
signature = nextSignature;
index = new Map<string, CharacterNameImage>();
indexByMediaId = new Map<number, Map<string, CharacterNameImage>>();
for (const snapshot of readCachedSnapshots(outputDir)) {
appendSnapshotImages(index, snapshot);
const mediaIndex = new Map<string, CharacterNameImage>();
appendSnapshotImages(mediaIndex, snapshot);
if (mediaIndex.size > 0) {
indexByMediaId.set(snapshot.mediaId, mediaIndex);
refreshInFlight = true;
void (async () => {
try {
const snapshots = await readCachedSnapshots(outputDir);
const nextIndex = new Map<string, CharacterNameImage>();
const nextIndexByMediaId = new Map<number, Map<string, CharacterNameImage>>();
for (const snapshot of snapshots) {
appendSnapshotImages(nextIndex, snapshot);
const mediaIndex = new Map<string, CharacterNameImage>();
appendSnapshotImages(mediaIndex, snapshot);
if (mediaIndex.size > 0) {
nextIndexByMediaId.set(snapshot.mediaId, mediaIndex);
}
}
index = nextIndex;
indexByMediaId = nextIndexByMediaId;
signature = nextSignature;
indexReadyDeliveryPending = deps.onIndexReady !== undefined;
deliverIndexReadyIfPending();
} finally {
refreshInFlight = false;
}
}
})();
}
return {
@@ -32,17 +32,33 @@ function writeSnapshot(outputDir: string, mediaId: number, entries: Array<[strin
);
}
function withTempDir<T>(run: (dir: string) => T): T {
async function withTempDir<T>(run: (dir: string) => Promise<T> | T): Promise<T> {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-name-candidates-'));
try {
return run(dir);
return await run(dir);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
}
test('collects terms and readings for the current media', () => {
withTempDir((dir) => {
// The snapshot index rebuilds in the background while lookups serve stale data, so tests poll the
// probe until the refresh they triggered has landed.
async function waitForRefresh<T>(probe: () => T | null | undefined): Promise<T> {
const deadline = Date.now() + 5000;
for (;;) {
const value = probe();
if (value !== null && value !== undefined) {
return value;
}
if (Date.now() > deadline) {
throw new Error('timed out waiting for background snapshot refresh');
}
await new Promise((resolve) => setTimeout(resolve, 5));
}
}
test('collects terms and readings for the current media', async () => {
await withTempDir(async (dir) => {
writeSnapshot(dir, 1, [
['ミナト', 'みなと'],
['湊', 'みなと'],
@@ -53,17 +69,16 @@ test('collects terms and readings for the current media', () => {
outputDir: dir,
getCurrentMediaId: () => 1,
});
const candidates = lookup.get();
const candidates = await waitForRefresh(() => lookup.get());
assert.ok(candidates);
assert.deepEqual([...candidates.forms].sort(), ['みなと', 'ミナト', '湊'].sort());
// Deduplicated: both entries share the みなと reading.
assert.equal(candidates.forms.length, 3);
});
});
test('returns null without a media scope so the scanner stays exhaustive', () => {
withTempDir((dir) => {
test('returns null without a media scope so the scanner stays exhaustive', async () => {
await withTempDir(async (dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
const lookup = createCharacterNameCandidateLookup({
@@ -71,12 +86,14 @@ test('returns null without a media scope so the scanner stays exhaustive', () =>
getCurrentMediaId: () => null,
});
// The explicitly-scoped probe proves the index has loaded before the unscoped case is judged.
await waitForRefresh(() => lookup.get(1));
assert.equal(lookup.get(), null);
});
});
test('returns null for a media with no cached snapshot', () => {
withTempDir((dir) => {
test('returns null for a media with no cached snapshot', async () => {
await withTempDir(async (dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
const lookup = createCharacterNameCandidateLookup({
@@ -84,29 +101,31 @@ test('returns null for a media with no cached snapshot', () => {
getCurrentMediaId: () => 999,
});
await waitForRefresh(() => lookup.get(1));
assert.equal(lookup.get(), null);
});
});
test('key changes when the snapshot content changes', () => {
withTempDir((dir) => {
test('key changes when the snapshot content changes', async () => {
await withTempDir(async (dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
const lookup = createCharacterNameCandidateLookup({
outputDir: dir,
getCurrentMediaId: () => 1,
});
const first = lookup.get();
const first = await waitForRefresh(() => lookup.get());
writeSnapshot(dir, 1, [
['ミナト', 'みなと'],
['アクア', 'あくあ'],
]);
lookup.invalidate();
const second = lookup.get();
const second = await waitForRefresh(() => {
const candidates = lookup.get();
return candidates && candidates.forms.length === 4 ? candidates : null;
});
assert.ok(first && second);
assert.notEqual(first.key, second.key);
assert.equal(second.forms.length, 4);
});
});
@@ -114,8 +133,8 @@ test('key changes when the snapshot content changes', () => {
// directory every call. Asserted behaviorally: an unannounced on-disk change is
// invisible until the recheck interval elapses, which can only be true if the
// filesystem is not consulted per lookup.
test('does not re-read the snapshot directory on every lookup', () => {
withTempDir((dir) => {
test('does not re-read the snapshot directory on every lookup', async () => {
await withTempDir(async (dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
let nowMs = 1_000_000;
const lookup = createCharacterNameCandidateLookup({
@@ -124,6 +143,7 @@ test('does not re-read the snapshot directory on every lookup', () => {
now: () => nowMs,
});
await waitForRefresh(() => lookup.get());
assert.equal(lookup.get()?.forms.length, 2);
writeSnapshot(dir, 1, [
@@ -135,12 +155,16 @@ test('does not re-read the snapshot directory on every lookup', () => {
assert.equal(lookup.get()?.forms.length, 2, 'expected the cached list within the interval');
nowMs += 10_000;
assert.equal(lookup.get()?.forms.length, 4, 'expected a refresh past the interval');
const refreshed = await waitForRefresh(() => {
const candidates = lookup.get();
return candidates && candidates.forms.length === 4 ? candidates : null;
});
assert.equal(refreshed.forms.length, 4, 'expected a refresh past the interval');
});
});
test('invalidate picks up a snapshot change immediately', () => {
withTempDir((dir) => {
test('invalidate picks up a snapshot change on the next refresh', async () => {
await withTempDir(async (dir) => {
writeSnapshot(dir, 1, [['ミナト', 'みなと']]);
let nowMs = 1_000_000;
const lookup = createCharacterNameCandidateLookup({
@@ -149,6 +173,7 @@ test('invalidate picks up a snapshot change immediately', () => {
now: () => nowMs,
});
await waitForRefresh(() => lookup.get());
assert.equal(lookup.get()?.forms.length, 2);
writeSnapshot(dir, 1, [
@@ -158,6 +183,10 @@ test('invalidate picks up a snapshot change immediately', () => {
nowMs += 1;
lookup.invalidate();
assert.equal(lookup.get()?.forms.length, 4);
const refreshed = await waitForRefresh(() => {
const candidates = lookup.get();
return candidates && candidates.forms.length === 4 ? candidates : null;
});
assert.equal(refreshed.forms.length, 4);
});
});
@@ -98,7 +98,12 @@ export function createCharacterNameCandidateLookup(deps: {
let signature: string | null = null;
let lastSignatureCheckAtMs = 0;
let formsByMediaId = new Map<number, string[]>();
let refreshInFlight = false;
// Same stale-while-revalidate shape as the image lookup: the rebuild re-reads every cached
// snapshot, so it runs in the background while lookups keep serving the previous forms. The
// signature only advances once its rebuild has landed, so a failed or superseded rebuild is
// retried on the next signature check.
function refreshIfNeeded(): void {
if (!outputDir) {
formsByMediaId = new Map<number, string[]>();
@@ -114,17 +119,26 @@ export function createCharacterNameCandidateLookup(deps: {
}
lastSignatureCheckAtMs = nowMs;
const nextSignature = getSnapshotDirectorySignature(outputDir);
if (nextSignature === signature) {
if (nextSignature === signature || refreshInFlight) {
return;
}
signature = nextSignature;
formsByMediaId = new Map<number, string[]>();
for (const snapshot of readCachedSnapshots(outputDir)) {
const forms = collectSnapshotNameForms(snapshot);
if (forms.length > 0) {
formsByMediaId.set(snapshot.mediaId, forms);
refreshInFlight = true;
void (async () => {
try {
const snapshots = await readCachedSnapshots(outputDir);
const nextFormsByMediaId = new Map<number, string[]>();
for (const snapshot of snapshots) {
const forms = collectSnapshotNameForms(snapshot);
if (forms.length > 0) {
nextFormsByMediaId.set(snapshot.mediaId, forms);
}
}
formsByMediaId = nextFormsByMediaId;
signature = nextSignature;
} finally {
refreshInFlight = false;
}
}
})();
}
return {
@@ -43,7 +43,8 @@ test('resolveJapaneseNameSplits splits a single-kanji surname via person-name PO
}),
);
assert.deepEqual(splits.get('東紫乃'), { family: '東', given: '紫乃' });
assert.equal(splits.kind, 'complete');
assert.deepEqual(splits.splits.get('東紫乃'), { family: '東', given: '紫乃' });
});
test('resolveJapaneseNameSplits corrects a hint-length-misleading surname boundary', async () => {
@@ -64,7 +65,8 @@ test('resolveJapaneseNameSplits corrects a hint-length-misleading surname bounda
}),
);
assert.deepEqual(splits.get('渡辺真奈美'), { family: '渡辺', given: '真奈美' });
assert.equal(splits.kind, 'complete');
assert.deepEqual(splits.splits.get('渡辺真奈美'), { family: '渡辺', given: '真奈美' });
});
test('resolveJapaneseNameSplits falls back to hint readings when POS tags are generic', async () => {
@@ -85,7 +87,8 @@ test('resolveJapaneseNameSplits falls back to hint readings when POS tags are ge
}),
);
assert.deepEqual(splits.get('鈴木みゆ'), { family: '鈴木', given: 'みゆ' });
assert.equal(splits.kind, 'complete');
assert.deepEqual(splits.splits.get('鈴木みゆ'), { family: '鈴木', given: 'みゆ' });
});
test('resolveJapaneseNameSplits skips names whose tokens do not reconstruct the name', async () => {
@@ -96,7 +99,8 @@ test('resolveJapaneseNameSplits skips names whose tokens do not reconstruct the
}),
);
assert.equal(splits.size, 0);
assert.equal(splits.kind, 'complete');
assert.equal(splits.splits.size, 0);
});
test('resolveJapaneseNameSplits skips ambiguous or untagged segmentations', async () => {
@@ -117,7 +121,8 @@ test('resolveJapaneseNameSplits skips ambiguous or untagged segmentations', asyn
}),
);
assert.equal(splits.size, 0);
assert.equal(splits.kind, 'complete');
assert.equal(splits.splits.size, 0);
});
test('resolveJapaneseNameSplits survives tokenizer failures', async () => {
@@ -130,7 +135,8 @@ test('resolveJapaneseNameSplits survives tokenizer failures', async () => {
(message) => warnings.push(message),
);
assert.equal(splits.size, 0);
assert.equal(splits.kind, 'incomplete');
assert.equal(splits.splits.size, 0);
assert.equal(warnings.length, 1);
assert.match(warnings[0]!, /mecab unavailable/);
});
@@ -7,6 +7,10 @@ import type {
ResolvedNameSplit,
} from './types';
export type JapaneseNameSplitResolution =
| { kind: 'complete'; splits: Map<string, ResolvedNameSplit> }
| { kind: 'incomplete'; splits: Map<string, ResolvedNameSplit> };
const NAME_SEPARATOR_PATTERN = /[\s ・・·•]/;
function joinSurfaces(tokens: NameSplitToken[]): string {
@@ -87,8 +91,9 @@ export async function resolveJapaneseNameSplits(
tokenize: NameSplitTokenizer,
logWarn?: (message: string) => void,
onCharacterResolved?: (completed: number, total: number) => void,
): Promise<Map<string, ResolvedNameSplit>> {
): Promise<JapaneseNameSplitResolution> {
const splits = new Map<string, ResolvedNameSplit>();
let tokenizerFailed = false;
let resolvedCharacters = 0;
for (const character of characters) {
const familyHintReading = buildReadingFromHint(character.lastNameHint?.trim() || '');
@@ -99,12 +104,17 @@ export async function resolveJapaneseNameSplits(
try {
tokens = await tokenize(name);
} catch (err) {
tokenizerFailed = true;
logWarn?.(
`[dictionary] name split tokenization failed for "${name}": ${(err as Error).message}`,
);
continue;
}
if (!tokens || tokens.length < 2 || joinSurfaces(tokens) !== name) continue;
if (!tokens) {
tokenizerFailed = true;
continue;
}
if (tokens.length < 2 || joinSurfaces(tokens) !== name) continue;
const splitIndex =
splitIndexFromPersonNamePos(tokens) ??
splitIndexFromHintReadings(tokens, familyHintReading, givenHintReading);
@@ -118,5 +128,5 @@ export async function resolveJapaneseNameSplits(
resolvedCharacters += 1;
onCharacterResolved?.(resolvedCharacters, characters.length);
}
return splits;
return tokenizerFailed ? { kind: 'incomplete', splits } : { kind: 'complete', splits };
}
@@ -7,7 +7,7 @@ import test from 'node:test';
import { createCharacterDictionaryRuntimeService } from '../character-dictionary-runtime';
import { getSnapshotPath, writeSnapshot } from './cache';
import { CHARACTER_DICTIONARY_FORMAT_VERSION } from './constants';
import type { CharacterDictionarySnapshot } from './types';
import type { CharacterDictionarySnapshot, NameSplitTokenizer } from './types';
const GRAPHQL_URL = 'https://graphql.anilist.co';
const PNG_1X1 = Buffer.from(
@@ -34,7 +34,7 @@ function createSnapshotWithoutImages(): CharacterDictionarySnapshot {
test('generateForCurrentMedia refreshes same-version snapshots missing images when inline images are enabled', async () => {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
await writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
const originalFetch = globalThis.fetch;
const fetchUrls: string[] = [];
@@ -121,10 +121,15 @@ test('generateForCurrentMedia refreshes same-version snapshots missing images wh
}
});
test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable', async () => {
async function runNameSplitRefreshScenario(tokenizeJapaneseName: NameSplitTokenizer): Promise<{
characterPageRequests: number;
firstResultFromCache: boolean;
refreshedNameSplitSource: CharacterDictionarySnapshot['nameSplitSource'];
secondResultFromCache: boolean;
}> {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), {
await writeSnapshot(getSnapshotPath(outputDir, 130298), {
...createSnapshotWithoutImages(),
nameSplitSource: 'heuristic',
});
@@ -172,7 +177,6 @@ test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable'
}) as typeof globalThis.fetch;
try {
let tokenizerCalls = 0;
const runtime = createCharacterDictionaryRuntimeService({
userDataPath,
getCurrentMediaPath: () => '/tmp/eminence-s01e05.mkv',
@@ -185,35 +189,60 @@ test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable'
source: 'fallback',
}),
getNameMatchImagesEnabled: () => false,
tokenizeJapaneseName: async () => {
tokenizerCalls += 1;
return null;
},
tokenizeJapaneseName,
getJapaneseNameTokenizerAvailable: () => true,
now: () => 1_700_000_000_500,
});
const result = await runtime.generateForCurrentMedia();
const firstResult = await runtime.generateForCurrentMedia();
const refreshedSnapshot = JSON.parse(
fs.readFileSync(getSnapshotPath(outputDir, 130298), 'utf8'),
) as CharacterDictionarySnapshot;
const secondResult = await runtime.generateForCurrentMedia();
assert.equal(result.fromCache, false);
assert.equal(refreshedSnapshot.nameSplitSource, 'heuristic');
const retriedResult = await runtime.generateForCurrentMedia();
assert.equal(retriedResult.fromCache, false);
assert.equal(characterPageRequests, 2);
assert.equal(tokenizerCalls, 2);
return {
characterPageRequests,
firstResultFromCache: firstResult.fromCache,
refreshedNameSplitSource: refreshedSnapshot.nameSplitSource,
secondResultFromCache: secondResult.fromCache,
};
} finally {
globalThis.fetch = originalFetch;
}
}
test('generateForCurrentMedia keeps failed MeCab name split refreshes retryable', async () => {
let tokenizerCalls = 0;
const result = await runNameSplitRefreshScenario(async () => {
tokenizerCalls += 1;
return null;
});
assert.equal(result.firstResultFromCache, false);
assert.equal(result.refreshedNameSplitSource, 'heuristic');
assert.equal(result.secondResultFromCache, false);
assert.equal(result.characterPageRequests, 2);
assert.equal(tokenizerCalls, 2);
});
test('generateForCurrentMedia caches completed MeCab refreshes with no resolved splits', async () => {
let tokenizerCalls = 0;
const result = await runNameSplitRefreshScenario(async () => {
tokenizerCalls += 1;
return [];
});
assert.equal(result.firstResultFromCache, false);
assert.equal(result.refreshedNameSplitSource, 'mecab');
assert.equal(result.secondResultFromCache, true);
assert.equal(result.characterPageRequests, 1);
assert.equal(tokenizerCalls, 1);
});
test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is available', async () => {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), {
await writeSnapshot(getSnapshotPath(outputDir, 130298), {
...createSnapshotWithoutImages(),
nameSplitSource: 'mecab',
});
@@ -253,7 +282,7 @@ test('generateForCurrentMedia keeps mecab-split snapshots when MeCab is availabl
test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is unavailable', async () => {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), {
await writeSnapshot(getSnapshotPath(outputDir, 130298), {
...createSnapshotWithoutImages(),
nameSplitSource: 'heuristic',
});
@@ -293,7 +322,7 @@ test('generateForCurrentMedia keeps heuristic-split snapshots while MeCab is una
test('generateForCurrentMedia keeps same-version snapshots without images when inline images are disabled', async () => {
const userDataPath = makeTempDir();
const outputDir = path.join(userDataPath, 'character-dictionaries');
writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
await writeSnapshot(getSnapshotPath(outputDir, 130298), createSnapshotWithoutImages());
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (input: string | URL | Request) => {
@@ -42,7 +42,7 @@ function readStoredZipEntries(zipPath: string): Map<string, Buffer> {
return entries;
}
test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', () => {
test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', async () => {
const tempDir = makeTempDir();
const outputPath = path.join(tempDir, 'dictionary.zip');
const termEntries: CharacterDictionaryTermEntry[] = [
@@ -62,7 +62,7 @@ test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', ()
);
}) as typeof Buffer.concat;
const result = buildDictionaryZip(
const result = await buildDictionaryZip(
outputPath,
'Dictionary Title',
'Dictionary Description',
@@ -106,11 +106,11 @@ test('buildDictionaryZip writes a valid stored zip without fs.writeFileSync', ()
}
});
test('readDictionaryZipRevision reads the built revision and rejects foreign archives', () => {
test('readDictionaryZipRevision reads the built revision and rejects foreign archives', async () => {
const dir = makeTempDir();
try {
const zipPath = path.join(dir, 'merged.zip');
buildDictionaryZip(
await buildDictionaryZip(
zipPath,
'SubMiner Character Dictionary',
'Character names',

Some files were not shown because too many files have changed in this diff Show More