refactor: remove dead registry and pipeline modules

src/core/services/tokenizer.test.ts

@@ -1235,6 +1235,30 @@ test('tokenizeSubtitle normalizes newlines before Yomitan parse request', async
   assert.equal(result.tokens, null);
 });
 
+test('tokenizeSubtitle collapses zero-width separators before Yomitan parse request', async () => {
+  let parseInput = '';
+  const result = await tokenizeSubtitle(
+    'キリキリと\u200bかかってこい\nこのヘナチョコ冒険者どもめが！',
+    makeDeps({
+      getYomitanExt: () => ({ id: 'dummy-ext' }) as any,
+      getYomitanParserWindow: () =>
+        ({
+          isDestroyed: () => false,
+          webContents: {
+            executeJavaScript: async (script: string) => {
+              parseInput = script;
+              return null;
+            },
+          },
+        }) as unknown as Electron.BrowserWindow,
+    }),
+  );
+
+  assert.match(parseInput, /キリキリと かかってこい このヘナチョコ冒険者どもめが！/);
+  assert.equal(result.text, 'キリキリと\u200bかかってこい\nこのヘナチョコ冒険者どもめが！');
+  assert.equal(result.tokens, null);
+});
+
 test('tokenizeSubtitle returns null tokens when Yomitan parsing is unavailable', async () => {
   const result = await tokenizeSubtitle('猫です', makeDeps());
 

src/core/services/tokenizer.ts

@@ -106,6 +106,7 @@ const DEFAULT_ANNOTATION_POS1_EXCLUSIONS = resolveAnnotationPos1ExclusionSet(
 const DEFAULT_ANNOTATION_POS2_EXCLUSIONS = resolveAnnotationPos2ExclusionSet(
   DEFAULT_ANNOTATION_POS2_EXCLUSION_CONFIG,
 );
+const INVISIBLE_SEPARATOR_PATTERN = /[\u200b\u2060\ufeff]/g;
 
 function getKnownWordLookup(
   deps: TokenizerServiceDeps,
@@ -563,7 +564,11 @@ export async function tokenizeSubtitle(
     return { text, tokens: null };
   }
 
-  const tokenizeText = displayText.replace(/\n/g, ' ').replace(/\s+/g, ' ').trim();
+  const tokenizeText = displayText
+    .replace(INVISIBLE_SEPARATOR_PATTERN, ' ')
+    .replace(/\n/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
   const annotationOptions = getAnnotationOptions(deps);
 
   const yomitanTokens = await parseWithYomitanInternalParser(tokenizeText, deps, annotationOptions);