feat(core): add module scaffolding and provider registries

src/subtitle/stages/merge.ts

@@ -0,0 +1,12 @@
+import { TokenMergerProvider } from "../../token-mergers";
+import { MergedToken, Token } from "../../types";
+
+export function mergeStage(
+  mergerProvider: TokenMergerProvider | null,
+  tokens: Token[] | null,
+): MergedToken[] | null {
+  if (!mergerProvider || !tokens || tokens.length === 0) {
+    return null;
+  }
+  return mergerProvider.merge(tokens);
+}

src/subtitle/stages/normalize.ts

@@ -0,0 +1,14 @@
+export function normalizeDisplayText(text: string): string {
+  return text
+    .replace(/\r\n/g, "\n")
+    .replace(/\\N/g, "\n")
+    .replace(/\\n/g, "\n")
+    .trim();
+}
+
+export function normalizeTokenizerInput(displayText: string): string {
+  return displayText
+    .replace(/\n/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}

src/subtitle/stages/tokenize.ts

@@ -0,0 +1,12 @@
+import { TokenizerProvider } from "../../tokenizers";
+import { Token } from "../../types";
+
+export async function tokenizeStage(
+  tokenizerProvider: TokenizerProvider | null,
+  input: string,
+): Promise<Token[] | null> {
+  if (!tokenizerProvider || !input) {
+    return null;
+  }
+  return tokenizerProvider.tokenize(input);
+}