feat(subtitles): generate Jellyfin subtitles and widen halfwidth kana

- Whisper subtitle generation now works for Jellyfin playback. It reads the item's original file through toJellyfinOriginalStreamUrl and writes to a reused cache folder. For transcoded playback it uses Jellyfin's audio index
- generateJapaneseSubtitles accepts http(s) streams when an output path is set; subprocess errors redact URL tokens
- Add widenHalfwidthKana/narrowFullwidthKana and widen subtitle text at ingest (mpv sub-text, parsed cues, live primary text), so halfwidth and fullwidth duplicates merge into one cue
- Stats queries widen older stored lines, sentence search matches katakana in either width, and stats mining widens sentence/word
- Update Jellyfin, subtitle generation, and architecture docs; add changelog fragments

src/core/services/__tests__/stats-server.test.ts

@@ -1935,6 +1935,27 @@ describe('stats server API routes', () => {
     });
   });
 
+  it('POST /api/stats/mine-card widens halfwidth katakana from older stats lines', async () => {
+    await withMiningEnv({}, async (env) => {
+      const app = createStatsApp(createMockTracker(), {
+        addYomitanNote: async () => 777,
+        ankiConnectConfig: ankiConfig(env, { fields: { sentence: 'Sentence' } }),
+      });
+
+      const { status, body } = await mineCard(app, 'word', {
+        sourcePath: env.sourcePath,
+        sentence: 'ｱﾝﾀがｱｰﾀﾙたん？',
+        word: 'ｱﾝﾀ',
+      });
+
+      assert.equal(status, 200, JSON.stringify(body));
+      assert.equal(
+        sentNote(env.requests, 'updateNoteFields').fields.Sentence,
+        '<b>アンタ</b>がアータルたん？',
+      );
+    });
+  });
+
   it('POST /api/stats/mine-card leaves word card selection text to Yomitan glossary', async () => {
     await withMiningEnv({}, async (env) => {
       const app = createStatsApp(createMockTracker(), {

src/core/services/immersion-tracker/__tests__/query.test.ts

@@ -3936,6 +3936,83 @@ test('getWordOccurrences maps a normalized word back to anime, video, and subtit
   }
 });
 
+test('getWordOccurrences widens halfwidth katakana in lines recorded before ingest widening', () => {
+  const dbPath = makeDbPath();
+  const db = openTestDb(dbPath);
+
+  try {
+    ensureSchema(db);
+    const videoId = getOrCreateVideoRecord(db, 'local:/tmp/kamikatsu-s01e08.mkv', {
+      canonicalTitle: 'Episode 8',
+      sourcePath: '/tmp/KamiKatsu S01E08.mkv',
+      sourceUrl: null,
+      sourceType: SOURCE_TYPE_LOCAL,
+    });
+    const { sessionId } = startSessionRecord(db, videoId, 1_000_000);
+    const wordResult = db
+      .prepare(
+        `INSERT INTO imm_words (
+          headword, word, reading, part_of_speech, pos1, pos2, pos3, first_seen, last_seen, frequency
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      )
+      .run('アンタ', 'ｱﾝﾀ', 'あんた', 'noun', '名詞', '代名詞', '', 1_000, 1_000, 1);
+    const lineResult = db
+      .prepare(
+        `INSERT INTO imm_subtitle_lines (
+          session_id, event_id, video_id, anime_id, line_index, segment_start_ms, segment_end_ms, text, CREATED_DATE, LAST_UPDATE_DATE
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      )
+      .run(sessionId, null, videoId, null, 1, 0, 1000, 'ｱﾝﾀがｱｰﾀﾙたん？', 1_000, 1_000);
+    db.prepare(
+      `INSERT INTO imm_word_line_occurrences (line_id, word_id, occurrence_count)
+       VALUES (?, ?, ?)`,
+    ).run(Number(lineResult.lastInsertRowid), Number(wordResult.lastInsertRowid), 1);
+
+    const rows = getWordOccurrences(db, 'アンタ', 'ｱﾝﾀ', 'あんた', 10);
+
+    assert.deepEqual(
+      rows.map((row) => row.text),
+      ['アンタがアータルたん？'],
+    );
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('searchSubtitleSentences matches halfwidth and fullwidth katakana in either direction', () => {
+  const dbPath = makeDbPath();
+  const db = openTestDb(dbPath);
+
+  try {
+    ensureSchema(db);
+    const videoId = getOrCreateVideoRecord(db, 'local:/tmp/kamikatsu-s01e08.mkv', {
+      canonicalTitle: 'Episode 8',
+      sourcePath: '/tmp/KamiKatsu S01E08.mkv',
+      sourceUrl: null,
+      sourceType: SOURCE_TYPE_LOCAL,
+    });
+    const { sessionId } = startSessionRecord(db, videoId, 1_000_000);
+    const insertLine = db.prepare(
+      `INSERT INTO imm_subtitle_lines (
+        session_id, event_id, video_id, anime_id, line_index, segment_start_ms, segment_end_ms, text, CREATED_DATE, LAST_UPDATE_DATE
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    );
+    insertLine.run(sessionId, null, videoId, null, 1, 0, 1000, 'ｱﾝﾀがｱｰﾀﾙたん？', 1_000, 1_000);
+    insertLine.run(sessionId, null, videoId, null, 2, 1000, 2000, 'アンタ、ダキニ', 2_000, 2_000);
+
+    for (const query of ['アンタ', 'ｱﾝﾀ']) {
+      assert.deepEqual(
+        searchSubtitleSentences(db, query, 10).map((row) => row.text),
+        ['アンタ、ダキニ', 'アンタがアータルたん？'],
+      );
+    }
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
 test('searchSubtitleSentences searches known subtitle lines and returns media context', () => {
   const dbPath = makeDbPath();
   const db = openTestDb(dbPath);

src/core/services/immersion-tracker/query-lexical.ts

@@ -18,6 +18,7 @@ import type {
 } from './types';
 import { fromDbTimestamp, toDbTimestamp } from './query-shared';
 import { nowMs } from './time';
+import { narrowFullwidthKana, widenHalfwidthKana } from '../../utils/halfwidth-kana';
 import {
   areLexicalDailyRollupsReady,
   getLexicalDailyRollups,
@@ -382,6 +383,13 @@ export function getKanjiStats(db: DatabaseSync, limit = 100): KanjiStatsRow[] {
   return stmt.all(limit) as KanjiStatsRow[];
 }
 
+// Lines recorded before subtitle text was widened at ingest can still hold halfwidth
+// katakana; widen them on the way out so the dashboard and cards mined from it match
+// newer rows. Display-only: word identity columns stay as stored since they are keys.
+function widenLineText<T extends { text: string }>(row: T): T {
+  return { ...row, text: widenHalfwidthKana(row.text) };
+}
+
 export function getWordOccurrences(
   db: DatabaseSync,
   headword: string,
@@ -390,7 +398,7 @@ export function getWordOccurrences(
   limit = 100,
   offset = 0,
 ): WordOccurrenceRow[] {
-  return db
+  const rows = db
     .prepare(
       `
         SELECT
@@ -418,6 +426,7 @@ export function getWordOccurrences(
       `,
     )
     .all(headword, word, reading, limit, offset) as unknown as WordOccurrenceRow[];
+  return rows.map(widenLineText);
 }
 
 export function getKanjiOccurrences(
@@ -426,7 +435,7 @@ export function getKanjiOccurrences(
   limit = 100,
   offset = 0,
 ): KanjiOccurrenceRow[] {
-  return db
+  const rows = db
     .prepare(
       `
         SELECT
@@ -454,6 +463,7 @@ export function getKanjiOccurrences(
       `,
     )
     .all(kanji, limit, offset) as unknown as KanjiOccurrenceRow[];
+  return rows.map(widenLineText);
 }
 
 export function searchSubtitleSentences(
@@ -470,6 +480,14 @@ export function searchSubtitleSentences(
   const params: string[] = [];
   for (const term of terms) {
     const likeTerm = `%${escapeLikeTerm(term)}%`;
+    // Lines recorded before ingest widening can hold halfwidth katakana, so subtitle
+    // text is matched against the term's widened and narrowed forms as well.
+    const widenedTerm = widenHalfwidthKana(term);
+    const textLikeTerms = uniqueNonEmptyTerms([
+      term,
+      widenedTerm,
+      narrowFullwidthKana(widenedTerm),
+    ]).map((variant) => `%${escapeLikeTerm(variant)}%`);
     const headwords = getHeadwordCandidatesForSentenceSearchTerm(term, options);
     const headwordClause =
       headwords.length > 0
@@ -485,16 +503,16 @@ export function searchSubtitleSentences(
         : '';
     clauses.push(`
       (
-        l.text LIKE ? ESCAPE '\\'
+        ${textLikeTerms.map(() => "l.text LIKE ? ESCAPE '\\'").join(' OR ')}
         OR v.canonical_title LIKE ? ESCAPE '\\'
         OR COALESCE(a.canonical_title, '') LIKE ? ESCAPE '\\'
         ${headwordClause}
       )
     `);
-    params.push(likeTerm, likeTerm, likeTerm, ...headwords);
+    params.push(...textLikeTerms, likeTerm, likeTerm, ...headwords);
   }
 
-  return db
+  const rows = db
     .prepare(
       `
         SELECT
@@ -518,6 +536,7 @@ export function searchSubtitleSentences(
       `,
     )
     .all(...params, resolvedLimit) as unknown as SentenceSearchResultRow[];
+  return rows.map(widenLineText);
 }
 
 export function getSessionEvents(

src/core/services/index.ts

@@ -130,6 +130,7 @@ export {
   listSubtitleTracks as listJellyfinSubtitleTracksRuntime,
   resolvePlaybackPlan as resolveJellyfinPlaybackPlanRuntime,
   ticksToSeconds as jellyfinTicksToSecondsRuntime,
+  toJellyfinOriginalStreamUrl,
 } from './jellyfin';
 export { JellyfinRemoteSessionService } from './jellyfin-remote';
 export {

src/core/services/jellyfin.test.ts

@@ -7,6 +7,7 @@ import {
   listSubtitleTracks,
   resolvePlaybackPlan,
   ticksToSeconds,
+  toJellyfinOriginalStreamUrl,
 } from './jellyfin';
 
 const clientInfo = {
@@ -862,3 +863,26 @@ test('resolvePlaybackPlan replaces a legacy api_key on the server transcoding ur
     globalThis.fetch = originalFetch;
   }
 });
+
+test('toJellyfinOriginalStreamUrl maps direct and transcoded playback to the original file', () => {
+  assert.equal(
+    toJellyfinOriginalStreamUrl(
+      'https://media.example/jf/videos/item-1/master.m3u8?ApiKey=token&MediaSourceId=src&VideoCodec=h264&StartTimeTicks=9',
+      'item-1',
+    ),
+    'https://media.example/jf/Videos/item-1/stream?static=true&ApiKey=token&MediaSourceId=src',
+  );
+  assert.equal(
+    toJellyfinOriginalStreamUrl(
+      'http://jf.local:8096/Videos/item-1/stream?static=true&api_key=legacy&AudioStreamIndex=2',
+      'item-1',
+    ),
+    'http://jf.local:8096/Videos/item-1/stream?static=true&ApiKey=legacy',
+  );
+  assert.equal(
+    toJellyfinOriginalStreamUrl('https://jf/Videos/other/stream?ApiKey=token', 'item-1'),
+    null,
+  );
+  assert.equal(toJellyfinOriginalStreamUrl('https://jf/Videos/item-1/stream', 'item-1'), null);
+  assert.equal(toJellyfinOriginalStreamUrl('/media/item-1.mkv', 'item-1'), null);
+});

src/core/services/jellyfin.ts

@@ -254,6 +254,29 @@ function createDirectPlayUrl(
   return `${session.serverUrl}/Videos/${itemId}/stream?${query.toString()}`;
 }
 
+/**
+ * Maps a playing stream URL of `itemId`, direct or transcoded, to its original file. Returns null
+ * when the URL is not that item's stream or carries no access token.
+ */
+export function toJellyfinOriginalStreamUrl(playbackUrl: string, itemId: string): string | null {
+  let url: URL;
+  try {
+    url = new URL(playbackUrl);
+  } catch {
+    return null;
+  }
+  const at = url.pathname.toLowerCase().indexOf(`/videos/${itemId.toLowerCase()}/`);
+  if (!/^https?:$/.test(url.protocol) || !itemId || at < 0) return null;
+  const param = (pattern: RegExp) =>
+    [...url.searchParams].find(([key, value]) => pattern.test(key) && value)?.[1];
+  const accessToken = param(/^api_?key$/i);
+  if (!accessToken) return null;
+  const query = new URLSearchParams({ static: 'true', ApiKey: accessToken });
+  const mediaSourceId = param(/^mediasourceid$/i);
+  if (mediaSourceId) query.set('MediaSourceId', mediaSourceId);
+  return `${url.origin}${url.pathname.slice(0, at)}/Videos/${itemId}/stream?${query.toString()}`;
+}
+
 function createTranscodeUrl(
   session: JellyfinAuthSession,
   itemId: string,

src/core/services/mpv-protocol.test.ts

@@ -131,6 +131,18 @@ test('dispatchMpvProtocolMessage emits subtitle text on property change', async
   assert.deepEqual(state.events, [{ text: '字幕', isOverlayVisible: false }]);
 });
 
+test('dispatchMpvProtocolMessage widens halfwidth katakana in subtitle text', async () => {
+  const { deps, state } = createDeps();
+
+  await dispatchMpvProtocolMessage(
+    { event: 'property-change', name: 'sub-text', data: 'ｱﾝﾀがｱｰﾀﾙたん？' },
+    deps,
+  );
+
+  assert.equal(state.subText, 'アンタがアータルたん？');
+  assert.deepEqual(state.events, [{ text: 'アンタがアータルたん？', isOverlayVisible: false }]);
+});
+
 test('dispatchMpvProtocolMessage emits subtitle track changes', async () => {
   const { deps, state } = createDeps({
     emitSubtitleTrackChange: (payload) => state.events.push(payload),

src/core/services/mpv-protocol.ts

@@ -1,5 +1,6 @@
 import { MpvSubtitleRenderMetrics } from '../../types';
 import { sanitizeMediaTitle } from '../../shared/media-identity';
+import { widenHalfwidthKana } from '../utils/halfwidth-kana';
 
 export type MpvMessage = {
   event?: string;
@@ -244,7 +245,7 @@ export async function dispatchMpvProtocolMessage(
 ): Promise<void> {
   if (msg.event === 'property-change') {
     if (msg.name === 'sub-text') {
-      const nextSubText = (msg.data as string) || '';
+      const nextSubText = widenHalfwidthKana((msg.data as string) || '');
       const overlayVisible = deps.isVisibleOverlayVisible();
       deps.emitSubtitleChange({
         text: nextSubText,
@@ -458,7 +459,7 @@ export async function dispatchMpvProtocolMessage(
         }>,
       );
     } else if (msg.request_id === MPV_REQUEST_ID_SUBTEXT) {
-      const nextSubText = (msg.data as string) || '';
+      const nextSubText = widenHalfwidthKana((msg.data as string) || '');
       deps.setCurrentSubText(nextSubText);
       deps.emitSubtitleChange({
         text: nextSubText,

src/core/services/stats-server/mining-routes.ts

@@ -8,6 +8,7 @@ import { clampMediaEndTime } from '../../../anki-integration/media-duration.js';
 import { MediaGenerator } from '../../../media-generator.js';
 import { statsJson } from '../../../types/stats-http-contract.js';
 import { STATS_MINING_TAG } from '../../../shared/anki-source.js';
+import { widenHalfwidthKana } from '../../utils/halfwidth-kana.js';
 import {
   resolveRetimedSecondarySubtitleTextFromSidecar,
   resolveSecondarySubtitleTextFromSidecar,
@@ -36,8 +37,11 @@ export function registerStatsMiningRoutes(app: Hono, options?: StatsMiningRouteO
     const sourcePath = typeof body?.sourcePath === 'string' ? body.sourcePath.trim() : '';
     const startMs = typeof body?.startMs === 'number' ? body.startMs : NaN;
     const endMs = typeof body?.endMs === 'number' ? body.endMs : NaN;
-    const sentence = typeof body?.sentence === 'string' ? body.sentence.trim() : '';
-    const word = typeof body?.word === 'string' ? body.word.trim() : '';
+    // Older stats rows can carry halfwidth katakana; cards get the fullwidth form that
+    // live mining produces, and the word is widened alongside so highlighting still hits.
+    const sentence =
+      typeof body?.sentence === 'string' ? widenHalfwidthKana(body.sentence.trim()) : '';
+    const word = typeof body?.word === 'string' ? widenHalfwidthKana(body.word.trim()) : '';
     const bodySecondaryText =
       typeof body?.secondaryText === 'string' ? body.secondaryText.trim() : '';
     const videoTitle = typeof body?.videoTitle === 'string' ? body.videoTitle.trim() : '';

src/core/services/subtitle-cue-parser.test.ts

@@ -55,6 +55,36 @@ test('parseSrtCues strips HTML-like markup while preserving line breaks', () =>
   assert.equal(cues[0]!.text, 'これは\nテストです');
 });
 
+test('parseSubtitleCues widens halfwidth katakana in cue text', () => {
+  const cues = parseSubtitleCues(
+    '1\n00:00:01,000 --> 00:00:02,000\n(ﾄｰｶ)ﾀﾞｷﾆお姉ちゃん\n',
+    'ep.srt',
+  );
+
+  assert.equal(cues[0]?.text, '(トーカ)ダキニお姉ちゃん');
+});
+
+test('parseSubtitleCues merges cues that differ only in halfwidth katakana', () => {
+  const cues = parseSubtitleCues(
+    [
+      '1',
+      '00:00:01,000 --> 00:00:02,000',
+      '山ｶﾞｸ',
+      '',
+      '2',
+      '00:00:01,000 --> 00:00:02,000',
+      '山ガク',
+      '',
+    ].join('\n'),
+    'ep.srt',
+  );
+
+  assert.deepEqual(
+    cues.map((cue) => cue.text),
+    ['山ガク'],
+  );
+});
+
 test('parseSrtCues handles hours in timestamps', () => {
   const content = ['1', '01:30:00,000 --> 01:30:05,000', 'テスト', ''].join('\n');
 

src/core/services/subtitle-cue-parser.ts

@@ -9,6 +9,7 @@ import {
   type AssOverrideCommand,
 } from './ass-text';
 import { hasAssAnimationEvidence, mergeDuplicateCues } from './subtitle-cue-dedup';
+import { widenHalfwidthKana } from '../utils/halfwidth-kana';
 
 /** Vertical third of the screen a cue is authored to occupy. */
 export type AssVerticalBand = 'top' | 'middle' | 'bottom';
@@ -104,7 +105,7 @@ function decodeSubtitleCueText(text: string): string {
 }
 
 function sanitizeSubtitleCueText(text: string): string {
-  return decodeSubtitleCueText(text).trim();
+  return widenHalfwidthKana(decodeSubtitleCueText(text)).trim();
 }
 
 function sanitizeAssCueText(text: string): string {
@@ -2655,10 +2656,16 @@ function parseAnnotatedAssCues(content: string): AnnotatedSubtitleCue[] {
     removeAssFontTextureEvents(parseAnnotatedAssEvents(content, placement)),
     placement.playResY,
   );
-  return mergeAssCaptionRows(
+  const cues = mergeAssCaptionRows(
     recoverFragmentOnlyAssLines(recoverCanonicalAssEvents(events)),
     placement.playResY,
   );
+  // Widened after reconstruction, which works glyph by glyph, but before duplicate
+  // merging, so ｶﾞｸ and ガク authored for the same span collapse into one cue.
+  for (const cue of cues) {
+    cue.text = widenHalfwidthKana(cue.text);
+  }
+  return cues;
 }
 
 export function parseAssCues(content: string): SubtitleCue[] {

src/core/services/subtitle-generation-process.ts

@@ -3,6 +3,11 @@ import { expandSubtitleGenerationPath } from './subtitle-generation-files';
 
 const OUTPUT_LIMIT = 64 * 1024;
 
+// ffmpeg echoes input URLs in its errors; streamed media carries access tokens in the query.
+function redactUrlCredentials(text: string): string {
+  return text.replace(/([?&](?:api_?key|token|access_token)=)[^&#\s'"]+/gi, '$1<redacted>');
+}
+
 // Keep partial lines between chunks: ffmpeg and whisper both report progress on stderr.
 export function runSubtitleGenerationProcess(input: {
   command: string;
@@ -63,7 +68,11 @@ export function runSubtitleGenerationProcess(input: {
       cleanup();
       if (input.signal?.aborted) reject(new Error('Subtitle generation cancelled.'));
       else if (code !== 0) {
-        reject(new Error(`${input.command} exited with status ${code}: ${stderr.trim()}`));
+        reject(
+          new Error(
+            `${input.command} exited with status ${code}: ${redactUrlCredentials(stderr.trim())}`,
+          ),
+        );
       } else resolve(stdout);
     });
   });

src/core/services/subtitle-generation.test.ts

@@ -378,7 +378,22 @@ test('empty executable paths find tools on PATH and explicit overrides take prec
     }
   }));
 
-test('generation rejects remote media, missing models, and missing tools before starting a subprocess', () =>
+test('generation reads an HTTP stream as-is and saves to the requested output', () =>
+  fixture(async (directory) => {
+    const input = await generationFixture(directory);
+    const mediaPath = 'https://jellyfin.example/Videos/item/stream?static=true&ApiKey=secret';
+    const outputPath = path.join(directory, 'out', 'stream.ja.srt');
+    await mkdir(path.dirname(outputPath));
+    assert.equal(await generateJapaneseSubtitles({ ...input, mediaPath, outputPath }), outputPath);
+    const calls: string[][] = (await readFile(input.callsPath, 'utf8'))
+      .trim()
+      .split('\n')
+      .map((line) => JSON.parse(line));
+    assert.equal(calls[0]?.at(-1), mediaPath);
+    assert.equal(calls[1]?.[calls[1].indexOf('-i') + 1], mediaPath);
+  }));
+
+test('generation rejects remote media without an output path, missing models, and missing tools before starting a subprocess', () =>
   fixture(async (directory) => {
     const input = await generationFixture(directory);
     await assert.rejects(
@@ -473,6 +488,17 @@ test('process cancellation terminates work and bounds diagnostic output', () =>
         error.message.length < 66000 &&
         error.message.includes('status 7'),
     );
+    const unauthorized = await executable(
+      directory,
+      'unauthorized',
+      "process.stderr.write('https://host/Videos/1/stream?ApiKey=se+cr/et==&static=true: Server returned 401');process.exitCode=1;",
+    );
+    await assert.rejects(
+      runSubtitleGenerationProcess({ command: unauthorized, args: [] }),
+      (error: unknown) =>
+        error instanceof Error &&
+        error.message.includes('?ApiKey=<redacted>&static=true: Server returned 401'),
+    );
   }));
 
 test('download uses the pinned model revision and removes files that fail integrity', () =>

src/core/services/subtitle-generation.ts

@@ -191,6 +191,7 @@ export async function requireSubtitleGenerationSetup(
 export async function generateJapaneseSubtitles(input: {
   config: SubtitleGenerationConfig;
   modelDirectory: string;
+  /** A local file, or an http(s) stream when `outputPath` is set. */
   mediaPath: string;
   audioStreamIndex?: number;
   references?: readonly SubtitleGenerationReference[];
@@ -201,10 +202,12 @@ export async function generateJapaneseSubtitles(input: {
   signal?: AbortSignal;
 }): Promise<string> {
   input.signal?.throwIfAborted();
-  if (/^[a-z][a-z\d+.-]*:\/\//i.test(input.mediaPath))
+  // HTTP streams are read by ffmpeg directly but have no directory to save beside.
+  const remote = /^https?:\/\//i.test(input.mediaPath);
+  if (remote ? !input.outputPath : /^[a-z][a-z\d+.-]*:\/\//i.test(input.mediaPath))
     throw new Error('Subtitle generation requires a local media file.');
-  const mediaPath = path.resolve(input.mediaPath);
-  if (!(await stat(mediaPath)).isFile())
+  const mediaPath = remote ? input.mediaPath : path.resolve(input.mediaPath);
+  if (!remote && !(await stat(mediaPath)).isFile())
     throw new Error('Subtitle generation requires a local media file.');
   if (input.outputPath) await ensureAvailableOutput(path.resolve(input.outputPath));
   await ensureWritableDirectory(

src/core/services/tokenizer/token-classification.ts

@@ -1,23 +1,12 @@
 import { MergedToken, PartOfSpeech } from '../../../types';
+import { widenHalfwidthKana } from '../../utils/halfwidth-kana';
 
 const KATAKANA_TO_HIRAGANA_OFFSET = 0x60;
 const KATAKANA_CODEPOINT_START = 0x30a1;
 const KATAKANA_CODEPOINT_END = 0x30f6;
 
-// No `u` flag: the range is entirely BMP so it changes nothing here, and
-// Bun's unicode-mode matcher mis-handles this class next to certain ligatures.
-const HALFWIDTH_KANA_RUN = /[\uff66-\uff9f]+/g;
-
-// NFKC over the halfwidth kana only, never the whole string: it composes the
-// voiced pairs (ｶ + ﾞ) into single characters so ｶﾞｸ compares equal to ガク
-// instead of counting one character longer than the word it spells, but run
-// over everything it would also rewrite unrelated text (① → 1, ㍑ → リットル).
-function composeHalfwidthKana(text: string): string {
-  return text.replace(HALFWIDTH_KANA_RUN, (run) => run.normalize('NFKC'));
-}
-
 export function normalizeKana(text: string): string {
-  const raw = composeHalfwidthKana(text).trim();
+  const raw = widenHalfwidthKana(text).trim();
   if (!raw) {
     return '';
   }

src/core/utils/halfwidth-kana.test.ts

@@ -0,0 +1,24 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { narrowFullwidthKana, widenHalfwidthKana } from './halfwidth-kana';
+
+test('widenHalfwidthKana widens halfwidth katakana and composes voiced marks', () => {
+  assert.equal(
+    widenHalfwidthKana('(ﾄｰｶ)ﾀﾞｷﾆお姉ちゃん ｱﾝﾀがｱｰﾀﾙたん｡'),
+    '(トーカ)ダキニお姉ちゃん アンタがアータルたん。',
+  );
+});
+
+test('widenHalfwidthKana leaves other compatibility characters alone', () => {
+  assert.equal(widenHalfwidthKana('見て見てっ！①㍑ＡＢ'), '見て見てっ！①㍑ＡＢ');
+});
+
+test('widenHalfwidthKana folds a lone halfwidth voiced mark into the kana before it', () => {
+  assert.equal(widenHalfwidthKana('山カﾞク ハﾟン'), '山ガク パン');
+  assert.equal(widenHalfwidthKana('アﾞ'), 'ア\u3099');
+});
+
+test('narrowFullwidthKana inverts widening and leaves other text alone', () => {
+  assert.equal(narrowFullwidthKana('ダキニお姉ちゃん アンタ。ヴ'), 'ﾀﾞｷﾆお姉ちゃん ｱﾝﾀ｡ｳﾞ');
+  assert.equal(narrowFullwidthKana('漢字ひらがな！'), '漢字ひらがな！');
+});

src/core/utils/halfwidth-kana.ts

@@ -0,0 +1,49 @@
+// No `u` flag: the range is entirely BMP so it changes nothing here, and
+// Bun's unicode-mode matcher mis-handles this class next to certain ligatures.
+// The optional leading character is captured so a run opening with a voiced
+// mark (カﾞ) can fold the mark into the fullwidth kana before it.
+const HALFWIDTH_KANA_RUN = /([^｡-ﾟ]?)([｡-ﾟ]+)/g;
+const LEADING_COMBINING_VOICED_MARK = /^[゙゚]/;
+
+/**
+ * Rewrites halfwidth katakana and its punctuation (ｱﾝﾀ, ﾀﾞｷﾆ, ｡｢｣､･) to the fullwidth
+ * forms. Broadcast-derived subtitles write katakana halfwidth to save space; subtitle
+ * text is normalized with this where it enters the app (live mpv text, parsed cues) so
+ * display, lookups, mining, and stats all see one form.
+ *
+ * NFKC runs over the halfwidth runs only, never the whole string: it composes the voiced
+ * pairs (ｶ + ﾞ) into single characters, but over everything it would also rewrite
+ * unrelated text (① → 1, ！ → !, ㍑ → リットル).
+ */
+export function widenHalfwidthKana(text: string): string {
+  return text.replace(HALFWIDTH_KANA_RUN, (_match, previous: string, run: string) => {
+    const widened = run.normalize('NFKC');
+    if (!previous || !LEADING_COMBINING_VOICED_MARK.test(widened)) {
+      return previous + widened;
+    }
+    // A lone ﾞ widens to a combining mark; NFC composes it with the kana before it
+    // (カ + ゛ → ガ) and leaves pairs that have no composed form untouched.
+    return (previous + widened[0]).normalize('NFC') + widened.slice(1);
+  });
+}
+
+// Inverse of widenHalfwidthKana per character, derived from it so the two stay in sync.
+// Voiced pairs (ｶﾞ) are included because they widen to a single character (ガ).
+const HALFWIDTH_BY_FULLWIDTH = new Map<string, string>();
+for (let code = 0xff61; code <= 0xff9f; code += 1) {
+  const halfwidth = String.fromCharCode(code);
+  for (const form of [halfwidth, `${halfwidth}ﾞ`, `${halfwidth}ﾟ`]) {
+    const widened = widenHalfwidthKana(form);
+    if (widened.length === 1 && !HALFWIDTH_BY_FULLWIDTH.has(widened)) {
+      HALFWIDTH_BY_FULLWIDTH.set(widened, form);
+    }
+  }
+}
+
+/**
+ * Rewrites fullwidth katakana and punctuation to halfwidth forms (アンタ → ｱﾝﾀ). Used to
+ * search text stored before subtitle ingest widened it; display paths should widen instead.
+ */
+export function narrowFullwidthKana(text: string): string {
+  return Array.from(text, (char) => HALFWIDTH_BY_FULLWIDTH.get(char) ?? char).join('');
+}