fix(subtitles): collapse duplicate ASS events and decode text once (#186)

src/core/services/tokenizer.ts

@@ -27,6 +27,7 @@ import {
 } from './tokenizer/yomitan-parser-runtime';
 import type { YomitanTermFrequency } from './tokenizer/yomitan-parser-runtime';
 import { isKanaChar } from './tokenizer/token-classification';
+import { normalizePlainSubtitleText } from './ass-text';
 
 const logger = createLogger('main:tokenizer');
 
@@ -886,14 +887,14 @@ export async function tokenizeSubtitle(
   text: string,
   deps: TokenizerServiceDeps,
 ): Promise<SubtitleData> {
-  const displayText = text
-    .replace(/\r\n/g, '\n')
-    .replace(/\\N/g, '\n')
-    .replace(/\\n/g, '\n')
-    .trim();
+  const displayText = normalizePlainSubtitleText(text);
 
+  // ASS decoding already happened upstream (cue parser for files, mpv for live text), so
+  // all this drops is whitespace -- but a whitespace-only line still normalizes to empty.
+  // Return the normalized form anyway: handing back the original would put a blank line
+  // into application state as if it were subtitle text.
   if (!displayText) {
-    return { text, tokens: null };
+    return { text: displayText, tokens: null };
   }
 
   const tokenizeText = displayText