fix(overlay): show plain subtitle line immediately on tokenization cache miss (#184)

src/core/services/subtitle-processing-controller.test.ts

@@ -7,18 +7,138 @@ function flushMicrotasks(): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, 0));
 }
 
-test('subtitle processing emits tokenized payload when tokenization succeeds', async () => {
+test('subtitle processing emits plain payload immediately on cache miss, then tokenized payload', async () => {
   const emitted: SubtitleData[] = [];
   const controller = createSubtitleProcessingController({
     tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
     emitSubtitle: (payload) => emitted.push(payload),
   });
 
+  controller.onSubtitleChange('字幕');
+  assert.deepEqual(emitted, [{ text: '字幕', tokens: null }]);
+  await flushMicrotasks();
+  assert.deepEqual(emitted, [
+    { text: '字幕', tokens: null },
+    { text: '字幕', tokens: [] },
+  ]);
+});
+
+test('cache invalidation during pending tokenization does not re-emit the plain payload', async () => {
+  const emitted: SubtitleData[] = [];
+  const resolvers: Array<(value: SubtitleData | null) => void> = [];
+  const controller = createSubtitleProcessingController({
+    tokenizeSubtitle: async (text) =>
+      await new Promise<SubtitleData | null>((resolve) => {
+        resolvers.push(() => resolve({ text, tokens: [{ value: resolvers.length } as never] }));
+      }),
+    emitSubtitle: (payload) => emitted.push(payload),
+  });
+
+  controller.onSubtitleChange('行');
+  assert.deepEqual(emitted, [{ text: '行', tokens: null }]);
+
+  controller.invalidateTokenizationCache();
+  resolvers[0]?.({ text: '行', tokens: [] });
+  await flushMicrotasks();
+  // Retry for the new generation is now pending; still no duplicate plain emit.
+  assert.deepEqual(emitted, [{ text: '行', tokens: null }]);
+
+  resolvers[1]?.({ text: '行', tokens: [] });
+  await flushMicrotasks();
+  assert.deepEqual(emitted, [
+    { text: '行', tokens: null },
+    { text: '行', tokens: [{ value: 2 } as never] },
+  ]);
+});
+
+test('failed refresh does not downgrade an already emitted tokenized subtitle', async () => {
+  const emitted: SubtitleData[] = [];
+  let tokenizeCalls = 0;
+  const controller = createSubtitleProcessingController({
+    tokenizeSubtitle: async (text) => {
+      tokenizeCalls += 1;
+      if (tokenizeCalls > 1) {
+        throw new Error('tokenizer gone');
+      }
+      return { text, tokens: [] };
+    },
+    emitSubtitle: (payload) => emitted.push(payload),
+  });
+
+  controller.onSubtitleChange('行');
+  await flushMicrotasks();
+  controller.invalidateTokenizationCache();
+  controller.refreshCurrentSubtitle();
+  await flushMicrotasks();
+
+  assert.equal(tokenizeCalls, 2);
+  assert.deepEqual(emitted, [
+    { text: '行', tokens: null },
+    { text: '行', tokens: [] },
+  ]);
+});
+
+test('null-tokenization refresh does not downgrade an already emitted tokenized subtitle', async () => {
+  const emitted: SubtitleData[] = [];
+  let tokenizeCalls = 0;
+  const controller = createSubtitleProcessingController({
+    tokenizeSubtitle: async (text) => {
+      tokenizeCalls += 1;
+      return tokenizeCalls > 1 ? null : { text, tokens: [] };
+    },
+    emitSubtitle: (payload) => emitted.push(payload),
+  });
+
+  controller.onSubtitleChange('行');
+  await flushMicrotasks();
+  controller.invalidateTokenizationCache();
+  controller.refreshCurrentSubtitle();
+  await flushMicrotasks();
+
+  assert.equal(tokenizeCalls, 2);
+  assert.deepEqual(emitted, [
+    { text: '行', tokens: null },
+    { text: '行', tokens: [] },
+  ]);
+});
+
+test('subtitle processing does not emit plain payload for cached lines', async () => {
+  const emitted: SubtitleData[] = [];
+  const controller = createSubtitleProcessingController({
+    tokenizeSubtitle: async (text) => ({ text, tokens: [] }),
+    emitSubtitle: (payload) => emitted.push(payload),
+  });
+
+  controller.preCacheTokenization('字幕', { text: '字幕', tokens: [] });
   controller.onSubtitleChange('字幕');
   await flushMicrotasks();
   assert.deepEqual(emitted, [{ text: '字幕', tokens: [] }]);
 });
 
+test('subtitle processing shows plain line while tokenization is still pending', async () => {
+  const emitted: SubtitleData[] = [];
+  let resolveTokenization: ((value: SubtitleData | null) => void) | undefined;
+  const controller = createSubtitleProcessingController({
+    tokenizeSubtitle: async (text) =>
+      await new Promise<SubtitleData | null>((resolve) => {
+        resolveTokenization = () => resolve({ text, tokens: [] });
+      }),
+    emitSubtitle: (payload) => emitted.push(payload),
+  });
+
+  controller.onSubtitleChange('遅い行');
+  await flushMicrotasks();
+  assert.deepEqual(emitted, [{ text: '遅い行', tokens: null }]);
+
+  assert.ok(resolveTokenization);
+  resolveTokenization({ text: '遅い行', tokens: [] });
+  await flushMicrotasks();
+  assert.deepEqual(emitted, [
+    { text: '遅い行', tokens: null },
+    { text: '遅い行', tokens: [] },
+  ]);
+});
+
 test('subtitle processing drops stale tokenization and delivers latest subtitle only once', async () => {
   const emitted: SubtitleData[] = [];
   let firstResolve: ((value: SubtitleData | null) => void) | undefined;
@@ -41,7 +161,11 @@ test('subtitle processing drops stale tokenization and delivers latest subtitle
   await flushMicrotasks();
   await flushMicrotasks();
 
-  assert.deepEqual(emitted, [{ text: 'second', tokens: [] }]);
+  assert.deepEqual(emitted, [
+    { text: 'first', tokens: null },
+    { text: 'second', tokens: null },
+    { text: 'second', tokens: [] },
+  ]);
 });
 
 test('subtitle processing skips duplicate subtitle emission', async () => {
@@ -60,7 +184,10 @@ test('subtitle processing skips duplicate subtitle emission', async () => {
   controller.onSubtitleChange('same');
   await flushMicrotasks();
 
-  assert.equal(emitted.length, 1);
+  assert.deepEqual(emitted, [
+    { text: 'same', tokens: null },
+    { text: 'same', tokens: [] },
+  ]);
   assert.equal(tokenizeCalls, 1);
 });
 
@@ -84,7 +211,9 @@ test('subtitle processing reuses cached tokenization for repeated subtitle text'
 
   assert.equal(tokenizeCalls, 2);
   assert.deepEqual(emitted, [
+    { text: 'first', tokens: null },
     { text: 'first', tokens: [] },
+    { text: 'second', tokens: null },
     { text: 'second', tokens: [] },
     { text: 'first', tokens: [] },
   ]);
@@ -100,7 +229,48 @@ test('subtitle processing falls back to plain subtitle when tokenization returns
   controller.onSubtitleChange('fallback');
   await flushMicrotasks();
 
+  assert.deepEqual(
+    emitted,
+    [{ text: 'fallback', tokens: null }],
+    'plain payload should not be re-emitted when tokenization yields nothing new',
+  );
+});
+
+test('null tokenization is not cached and a later cue retries tokenization', async () => {
+  const emitted: SubtitleData[] = [];
+  const callsByText = new Map<string, number>();
+  let failNext = true;
+  const controller = createSubtitleProcessingController({
+    tokenizeSubtitle: async (text) => {
+      callsByText.set(text, (callsByText.get(text) ?? 0) + 1);
+      if (text === 'fallback' && failNext) {
+        failNext = false;
+        return null;
+      }
+      return { text, tokens: [] };
+    },
+    emitSubtitle: (payload) => emitted.push(payload),
+  });
+
+  controller.onSubtitleChange('fallback');
+  await flushMicrotasks();
+
+  assert.equal(callsByText.get('fallback'), 1);
+  assert.equal(
+    controller.hasCachedSubtitle('fallback'),
+    false,
+    'plain fallback must not be cached when tokenization yields nothing',
+  );
   assert.deepEqual(emitted, [{ text: 'fallback', tokens: null }]);
+
+  controller.onSubtitleChange('other');
+  await flushMicrotasks();
+  controller.onSubtitleChange('fallback');
+  await flushMicrotasks();
+
+  assert.equal(callsByText.get('fallback'), 2, 'later cue should retry tokenization');
+  assert.equal(controller.hasCachedSubtitle('fallback'), true);
+  assert.deepEqual(emitted.at(-1), { text: 'fallback', tokens: [] });
 });
 
 test('subtitle processing ignores duplicate current subtitle refresh without cache invalidation', async () => {
@@ -120,7 +290,10 @@ test('subtitle processing ignores duplicate current subtitle refresh without cac
   await flushMicrotasks();
 
   assert.equal(tokenizeCalls, 1);
-  assert.deepEqual(emitted, [{ text: 'same', tokens: [] }]);
+  assert.deepEqual(emitted, [
+    { text: 'same', tokens: null },
+    { text: 'same', tokens: [] },
+  ]);
 });
 
 test('subtitle processing coalesces refresh requests while current subtitle is processing', async () => {
@@ -146,7 +319,10 @@ test('subtitle processing coalesces refresh requests while current subtitle is p
   await flushMicrotasks();
 
   assert.equal(tokenizeCalls, 1);
-  assert.deepEqual(emitted, [{ text: 'same', tokens: [] }]);
+  assert.deepEqual(emitted, [
+    { text: 'same', tokens: null },
+    { text: 'same', tokens: [] },
+  ]);
 });
 
 test('subtitle processing refresh re-tokenizes after cache invalidation', async () => {
@@ -168,6 +344,7 @@ test('subtitle processing refresh re-tokenizes after cache invalidation', async
 
   assert.equal(tokenizeCalls, 2);
   assert.deepEqual(emitted, [
+    { text: 'same', tokens: null },
     { text: 'same', tokens: [{ value: 1 } as never] },
     { text: 'same', tokens: [{ value: 2 } as never] },
   ]);
@@ -183,7 +360,10 @@ test('subtitle processing refresh can use explicit text override', async () => {
   controller.refreshCurrentSubtitle('initial');
   await flushMicrotasks();
 
-  assert.deepEqual(emitted, [{ text: 'initial', tokens: [] }]);
+  assert.deepEqual(emitted, [
+    { text: 'initial', tokens: null },
+    { text: 'initial', tokens: [] },
+  ]);
 });
 
 test('subtitle processing cache invalidation only affects future subtitle events', async () => {
@@ -205,10 +385,10 @@ test('subtitle processing cache invalidation only affects future subtitle events
   await flushMicrotasks();
 
   assert.equal(callsByText.get('same'), 1);
-  assert.equal(emitted.length, 3);
+  assert.equal(emitted.length, 5);
 
   controller.invalidateTokenizationCache();
-  assert.equal(emitted.length, 3);
+  assert.equal(emitted.length, 5);
 
   controller.onSubtitleChange('different');
   await flushMicrotasks();