fix(stats): stop counting duplicate typeset subtitle lines (#191)

src/core/services/__tests__/stats-server.test.ts

@@ -1032,6 +1032,180 @@ describe('stats server API routes', () => {
     ]);
   });
 
+  it('POST /api/stats/maintenance/duplicate-lines forwards the window and dry-run flag', async () => {
+    let seenOptions: unknown = null;
+    const summary = {
+      dryRun: true,
+      lookbackDays: 30,
+      scannedLines: 900,
+      burstGroups: 2,
+      removedLines: 180,
+      removedWordOccurrences: 540,
+      removedKanjiOccurrences: 120,
+      samples: [],
+    };
+    const app = createStatsApp(
+      createMockTracker({
+        cleanupDuplicateSubtitleLines: async (options: unknown) => {
+          seenOptions = options;
+          return summary;
+        },
+      }),
+    );
+
+    const res = await app.request('/api/stats/maintenance/duplicate-lines', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ dryRun: true, lookbackDays: 30 }),
+    });
+
+    assert.equal(res.status, 200);
+    assert.deepEqual(await res.json(), summary);
+    assert.deepEqual(seenOptions, { dryRun: true, lookbackDays: 30 });
+  });
+
+  it('POST /api/stats/maintenance/duplicate-lines rejects cross-origin simple requests', async () => {
+    let cleanupCalls = 0;
+    const app = createStatsApp(
+      createMockTracker({
+        cleanupDuplicateSubtitleLines: async () => {
+          cleanupCalls += 1;
+          throw new Error('cleanup must not run');
+        },
+      }),
+    );
+
+    const res = await app.request('/api/stats/maintenance/duplicate-lines', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'text/plain',
+        Origin: 'https://attacker.example',
+      },
+      body: JSON.stringify({ dryRun: false, lookbackDays: null }),
+    });
+
+    assert.equal(res.status, 415);
+    assert.equal(cleanupCalls, 0);
+  });
+
+  it('POST /api/stats/maintenance/duplicate-lines rejects a window shorter than a day', async () => {
+    let cleanupCalls = 0;
+    const app = createStatsApp(
+      createMockTracker({
+        cleanupDuplicateSubtitleLines: async () => {
+          cleanupCalls += 1;
+          return {
+            dryRun: true,
+            lookbackDays: null,
+            scannedLines: 0,
+            burstGroups: 0,
+            removedLines: 0,
+            removedWordOccurrences: 0,
+            removedKanjiOccurrences: 0,
+            samples: [],
+          };
+        },
+      }),
+    );
+
+    const res = await app.request('/api/stats/maintenance/duplicate-lines', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ dryRun: true, lookbackDays: 0.5 }),
+    });
+
+    assert.equal(res.status, 400);
+    assert.equal(cleanupCalls, 0);
+  });
+
+  it('POST /api/stats/maintenance/duplicate-lines floors a fractional multi-day window', async () => {
+    let seenOptions: unknown = null;
+    const app = createStatsApp(
+      createMockTracker({
+        cleanupDuplicateSubtitleLines: async (options: unknown) => {
+          seenOptions = options;
+          return {
+            dryRun: true,
+            lookbackDays: 1,
+            scannedLines: 0,
+            burstGroups: 0,
+            removedLines: 0,
+            removedWordOccurrences: 0,
+            removedKanjiOccurrences: 0,
+            samples: [],
+          };
+        },
+      }),
+    );
+
+    const res = await app.request('/api/stats/maintenance/duplicate-lines', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ dryRun: true, lookbackDays: 1.5 }),
+    });
+
+    assert.equal(res.status, 200);
+    assert.deepEqual(seenOptions, { dryRun: true, lookbackDays: 1 });
+  });
+
+  it('POST /api/stats/maintenance/duplicate-lines accepts an explicit empty object for all history', async () => {
+    let seenOptions: unknown = null;
+    const app = createStatsApp(
+      createMockTracker({
+        cleanupDuplicateSubtitleLines: async (options: unknown) => {
+          seenOptions = options;
+          return {
+            dryRun: false,
+            lookbackDays: null,
+            scannedLines: 0,
+            burstGroups: 0,
+            removedLines: 0,
+            removedWordOccurrences: 0,
+            removedKanjiOccurrences: 0,
+            samples: [],
+          };
+        },
+      }),
+    );
+
+    const res = await app.request('/api/stats/maintenance/duplicate-lines', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: '{}',
+    });
+
+    assert.equal(res.status, 200);
+    assert.deepEqual(seenOptions, { dryRun: false, lookbackDays: null });
+  });
+
+  for (const malformed of [
+    { name: 'a missing body', body: undefined },
+    { name: 'malformed JSON', body: '{' },
+    { name: 'JSON null', body: 'null' },
+    { name: 'a JSON array', body: '[]' },
+  ]) {
+    it(`POST /api/stats/maintenance/duplicate-lines rejects ${malformed.name}`, async () => {
+      let cleanupCalls = 0;
+      const app = createStatsApp(
+        createMockTracker({
+          cleanupDuplicateSubtitleLines: async () => {
+            cleanupCalls += 1;
+            throw new Error('cleanup must not run');
+          },
+        }),
+      );
+
+      const res = await app.request('/api/stats/maintenance/duplicate-lines', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: malformed.body,
+      });
+
+      assert.equal(res.status, 400);
+      assert.equal(cleanupCalls, 0);
+    });
+  }
+
   it('PUT /api/stats/excluded-words rejects malformed rows', async () => {
     const app = createStatsApp(createMockTracker());
 

src/core/services/immersion-tracker-service.ts

@@ -92,6 +92,11 @@ import {
   type RunDeleteMaintenanceTask,
 } from './immersion-tracker/delete-maintenance-worker-runtime';
 import { DeleteMaintenanceScheduler } from './immersion-tracker/delete-maintenance-scheduler';
+import {
+  cleanupDuplicateSubtitleLines,
+  type DuplicateSubtitleLineCleanupOptions,
+  type DuplicateSubtitleLineCleanupSummary,
+} from './immersion-tracker/duplicate-line-cleanup';
 import { repairJellyfinStreamVideoLinks } from './immersion-tracker/jellyfin-link-repair';
 import {
   repairLegacySeasonlessAnimeRows,
@@ -630,6 +635,18 @@ export class ImmersionTrackerService {
     });
   }
 
+  /**
+   * Collapse animation bursts that earlier versions recorded frame by frame. The whole
+   * queue is drained first so a burst still waiting to be written is scanned as stored
+   * rows rather than surviving the cleanup and landing a moment after it.
+   */
+  async cleanupDuplicateSubtitleLines(
+    options: DuplicateSubtitleLineCleanupOptions = {},
+  ): Promise<DuplicateSubtitleLineCleanupSummary> {
+    this.drainQueue();
+    return cleanupDuplicateSubtitleLines(this.db, options);
+  }
+
   async rebuildLifetimeSummaries(): Promise<LifetimeRebuildSummary> {
     this.flushTelemetry(true);
     this.flushNow();
@@ -1849,6 +1866,24 @@ export class ImmersionTrackerService {
     }
   }
 
+  /**
+   * Write out everything queued, not just the next batch.
+   *
+   * `flushNow` writes at most `batchSize` entries and does nothing at all while the write
+   * lock is held, so a maintenance pass that runs straight after it can still be reading
+   * a database that is missing rows. Each pass has to shrink the queue to continue: a
+   * failed flush puts its batch back, and looping on that would never finish.
+   */
+  private drainQueue(): void {
+    while (this.queue.length > 0) {
+      const pendingBefore = this.queue.length;
+      this.flushNow();
+      if (this.queue.length >= pendingBefore) {
+        return;
+      }
+    }
+  }
+
   private flushSingle(write: QueuedWrite): void {
     executeQueuedWrite(write, this.preparedStatements);
   }

src/core/services/immersion-tracker/__tests__/duplicate-line-cleanup.test.ts

@@ -0,0 +1,467 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+import { Database } from '../sqlite.js';
+import type { DatabaseSync } from '../sqlite.js';
+import { ensureSchema } from '../storage.js';
+import { cleanupDuplicateSubtitleLines } from '../duplicate-line-cleanup.js';
+
+const DAY_MS = 86_400_000;
+const BASE_MS = 1_700_000_000_000;
+const WORD_ID = 1;
+
+interface SeedLine {
+  session: number;
+  text: string;
+  startMs: number;
+  endMs: number;
+  /** Recording wall-clock, i.e. what the lookback window filters on. */
+  createdMs?: number;
+}
+
+function makeDbPath(): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'subminer-duplicate-line-test-'));
+  return path.join(dir, 'immersion.sqlite');
+}
+
+function cleanupDbPath(dbPath: string): void {
+  const dir = path.dirname(dbPath);
+  if (!fs.existsSync(dir)) return;
+  fs.rmSync(dir, { recursive: true, force: true });
+}
+
+/** One episode, two sessions of it, and one word occurrence per seeded line. */
+function seed(db: DatabaseSync, lines: SeedLine[]): void {
+  db.exec(`
+    INSERT INTO imm_anime(anime_id, normalized_title_key, canonical_title, CREATED_DATE, LAST_UPDATE_DATE)
+      VALUES (1, 'show', 'Show', ${BASE_MS}, ${BASE_MS});
+    INSERT INTO imm_videos(video_id, video_key, anime_id, canonical_title, source_type, watched, duration_ms, CREATED_DATE, LAST_UPDATE_DATE)
+      VALUES (1, 'v1', 1, 'Ep 1', 1, 1, 1440000, ${BASE_MS}, ${BASE_MS});
+    INSERT INTO imm_sessions(session_id, session_uuid, video_id, started_at_ms, ended_at_ms, status, CREATED_DATE, LAST_UPDATE_DATE)
+      VALUES (1, 's1', 1, '${BASE_MS}', '${BASE_MS + 1000}', 2, ${BASE_MS}, ${BASE_MS}),
+             (2, 's2', 1, '${BASE_MS + DAY_MS}', '${BASE_MS + DAY_MS + 1000}', 2, ${BASE_MS}, ${BASE_MS});
+    INSERT INTO imm_words(id, headword, word, reading, part_of_speech, pos1, first_seen, last_seen, frequency)
+      VALUES (${WORD_ID}, '飛び上がる', '飛び上がる', '', 'verb', '動詞', ${Math.floor(BASE_MS / 1000)}, ${Math.floor(BASE_MS / 1000)}, 0);
+  `);
+
+  const insertLine = db.prepare(
+    `INSERT INTO imm_subtitle_lines(
+       line_id, session_id, video_id, anime_id, line_index,
+       segment_start_ms, segment_end_ms, text, CREATED_DATE, LAST_UPDATE_DATE)
+     VALUES (?, ?, 1, 1, ?, ?, ?, ?, ?, ?)`,
+  );
+  const insertOccurrence = db.prepare(
+    `INSERT INTO imm_word_line_occurrences(line_id, word_id, occurrence_count, seen_ms)
+     VALUES (?, ?, 1, ?)`,
+  );
+
+  lines.forEach((line, index) => {
+    const lineId = index + 1;
+    const lineIndex = index + 1;
+    const createdMs = line.createdMs ?? BASE_MS;
+    insertLine.run(
+      lineId,
+      line.session,
+      lineIndex,
+      line.startMs,
+      line.endMs,
+      line.text,
+      createdMs,
+      createdMs,
+    );
+    insertOccurrence.run(lineId, WORD_ID, createdMs);
+  });
+
+  db.exec(`
+    UPDATE imm_words SET frequency = (
+      SELECT COALESCE(SUM(o.occurrence_count), 0)
+      FROM imm_word_line_occurrences o WHERE o.word_id = imm_words.id
+    )
+  `);
+}
+
+function createDb(lines: SeedLine[]): { db: DatabaseSync; dbPath: string } {
+  const dbPath = makeDbPath();
+  const db = new Database(dbPath);
+  ensureSchema(db);
+  seed(db, lines);
+  return { db, dbPath };
+}
+
+/** A typeset line mpv reported once per animation frame. */
+function karaokeFrames(
+  session: number,
+  text: string,
+  startMs: number,
+  frames: number,
+  frameMs: number,
+): SeedLine[] {
+  return Array.from({ length: frames }, (_, index) => ({
+    session,
+    text,
+    startMs: startMs + index * frameMs,
+    endMs: startMs + (index + 1) * frameMs,
+  }));
+}
+
+function countLines(db: DatabaseSync): number {
+  return (db.prepare('SELECT COUNT(*) AS total FROM imm_subtitle_lines').get() as { total: number })
+    .total;
+}
+
+function wordFrequency(db: DatabaseSync): number {
+  const row = db.prepare('SELECT frequency FROM imm_words WHERE id = ?').get(WORD_ID) as {
+    frequency: number;
+  } | null;
+  return row?.frequency ?? 0;
+}
+
+test('a karaoke burst collapses to one line and gives back its word counts', () => {
+  const { db, dbPath } = createDb([
+    ...karaokeFrames(1, '飛び上がる', 10_000, 40, 40),
+    { session: 1, text: 'おはよう', startMs: 20_000, endMs: 22_000 },
+  ]);
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db);
+
+    assert.equal(summary.burstGroups, 1);
+    assert.equal(summary.removedLines, 39);
+    assert.equal(summary.removedWordOccurrences, 39);
+    assert.equal(countLines(db), 2);
+    assert.equal(wordFrequency(db), 2);
+
+    // The surviving line covers the whole run, the way the parsed cue would.
+    const kept = db
+      .prepare(
+        'SELECT segment_start_ms AS startMs, segment_end_ms AS endMs FROM imm_subtitle_lines WHERE line_id = 1',
+      )
+      .get() as { startMs: number; endMs: number };
+    assert.equal(kept.startMs, 10_000);
+    assert.equal(kept.endMs, 10_000 + 40 * 40);
+
+    assert.equal(summary.samples.length, 1);
+    assert.equal(summary.samples[0]!.text, '飛び上がる');
+    assert.equal(summary.samples[0]!.frames, 40);
+    assert.equal(summary.samples[0]!.videoTitle, 'Ep 1');
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('ordinary repeated dialogue survives', () => {
+  // Six contiguous `飛び上がる`, each held for a normal beat rather than a frame.
+  const lines = Array.from({ length: 6 }, (_, index) => ({
+    session: 1,
+    text: '飛び上がる',
+    startMs: 5_000 + index * 800,
+    endMs: 5_000 + (index + 1) * 800,
+  }));
+  const { db, dbPath } = createDb(lines);
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db);
+
+    assert.equal(summary.burstGroups, 0);
+    assert.equal(summary.removedLines, 0);
+    assert.equal(countLines(db), 6);
+    assert.equal(wordFrequency(db), 6);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('a long run of quarter-second frames is still a burst', () => {
+  // Between the timing-only bound (0.1s) and the animation-frame bound (0.3s): heavier
+  // typesetting lands here, and the run length is what makes it conclusive.
+  const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db);
+
+    assert.equal(summary.burstGroups, 1);
+    assert.equal(summary.removedLines, 5);
+    assert.equal(countLines(db), 1);
+    assert.equal(wordFrequency(db), 1);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('a qualifying short-frame burst may end with one long hold frame', () => {
+  const { db, dbPath } = createDb([
+    ...karaokeFrames(1, '飛び上がる', 10_000, 8, 40),
+    { session: 1, text: '飛び上がる', startMs: 10_320, endMs: 12_320 },
+  ]);
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db);
+
+    assert.equal(summary.burstGroups, 1);
+    assert.equal(summary.removedLines, 8);
+    assert.equal(countLines(db), 1);
+    assert.equal(wordFrequency(db), 1);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('a long event before the final frame prevents burst cleanup', () => {
+  const { db, dbPath } = createDb([
+    ...karaokeFrames(1, '飛び上がる', 10_000, 5, 40),
+    { session: 1, text: '飛び上がる', startMs: 10_200, endMs: 12_200 },
+    { session: 1, text: '飛び上がる', startMs: 12_200, endMs: 12_240 },
+  ]);
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db);
+
+    assert.equal(summary.burstGroups, 0);
+    assert.equal(countLines(db), 7);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('a run of frames longer than the animation bound survives', () => {
+  const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 400));
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db);
+
+    assert.equal(summary.burstGroups, 0);
+    assert.equal(countLines(db), 6);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('the four-frame residue the live gate stores is cleaned up', () => {
+  // The streaming gate records the first four frames of a burst before the run is long
+  // enough to recognise. Four contiguous identical events under the strict timing-only
+  // bound are that residue, and no real dialogue.
+  const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 40));
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db);
+
+    assert.equal(summary.burstGroups, 1);
+    assert.equal(summary.removedLines, 3);
+    assert.equal(countLines(db), 1);
+    assert.equal(wordFrequency(db), 1);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('a four-frame run above the strict frame bound survives', () => {
+  // Long enough per event to be plausible dialogue; only a five-event run may use the
+  // looser animation-frame bound.
+  const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 4, 250));
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db);
+
+    assert.equal(summary.burstGroups, 0);
+    assert.equal(countLines(db), 4);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('an explicit minRunLength raises the bar', () => {
+  // Five quarter-second frames qualify under the defaults; a cautious run asking for six
+  // leaves them alone. Above the strict bound, so the residue rule stays out of it.
+  const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 5, 250));
+
+  try {
+    const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
+    assert.equal(preview.burstGroups, 1);
+
+    const summary = cleanupDuplicateSubtitleLines(db, { minRunLength: 6 });
+    assert.equal(summary.burstGroups, 0);
+    assert.equal(countLines(db), 5);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('an explicit maxFrameSeconds tightens the frame bound', () => {
+  const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 250));
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db, { maxFrameSeconds: 0.2 });
+
+    assert.equal(summary.burstGroups, 0);
+    assert.equal(countLines(db), 6);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('a non-finite maxFrameSeconds falls back to the default bound', () => {
+  // Six normal-beat lines: Infinity must not turn every event into a "short frame".
+  const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 6, 800));
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db, { maxFrameSeconds: Infinity });
+
+    assert.equal(summary.burstGroups, 0);
+    assert.equal(countLines(db), 6);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('sampleLimit zero removes bursts but reports no samples', () => {
+  const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db, { sampleLimit: 0 });
+
+    assert.equal(summary.removedLines, 39);
+    assert.deepEqual(summary.samples, []);
+    assert.equal(countLines(db), 1);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('a short run below every threshold survives', () => {
+  const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 1_000, 3, 40));
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db);
+
+    assert.equal(summary.burstGroups, 0);
+    assert.equal(countLines(db), 3);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('interleaved dual-line karaoke collapses each line to one row', () => {
+  // Kanji and romaji lines frame-flipped together, the way fansub OPs are typeset. The
+  // rows arrive interleaved in time order; each text must still chain into its own run.
+  const kanji = karaokeFrames(1, '飛び上がる', 10_000, 20, 60);
+  const romaji = karaokeFrames(1, 'tobiagaru', 10_001, 20, 60);
+  const interleaved = [...kanji, ...romaji].sort((a, b) => a.startMs - b.startMs);
+  const { db, dbPath } = createDb(interleaved);
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db);
+
+    assert.equal(summary.burstGroups, 2);
+    assert.equal(summary.removedLines, 38);
+    assert.equal(countLines(db), 2);
+    assert.equal(wordFrequency(db), 2);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('the same line in a rewatch session is never merged into the first watch', () => {
+  const { db, dbPath } = createDb([
+    ...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
+    ...karaokeFrames(2, '飛び上がる', 10_000, 6, 40),
+  ]);
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db);
+
+    assert.equal(summary.burstGroups, 2);
+    assert.equal(summary.removedLines, 10);
+    // One surviving line per session, not one across both.
+    assert.equal(countLines(db), 2);
+    assert.equal(wordFrequency(db), 2);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('a gap between runs splits them', () => {
+  const { db, dbPath } = createDb([
+    ...karaokeFrames(1, '飛び上がる', 10_000, 6, 40),
+    ...karaokeFrames(1, '飛び上がる', 60_000, 6, 40),
+  ]);
+
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db);
+
+    assert.equal(summary.burstGroups, 2);
+    assert.equal(countLines(db), 2);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('a dry run reports what an apply would do and writes nothing', () => {
+  const { db, dbPath } = createDb(karaokeFrames(1, '飛び上がる', 10_000, 40, 40));
+
+  try {
+    const preview = cleanupDuplicateSubtitleLines(db, { dryRun: true });
+
+    assert.equal(preview.dryRun, true);
+    assert.equal(preview.removedLines, 39);
+    assert.equal(countLines(db), 40);
+    assert.equal(wordFrequency(db), 40);
+
+    const applied = cleanupDuplicateSubtitleLines(db);
+    assert.equal(applied.removedLines, preview.removedLines);
+    assert.equal(applied.removedWordOccurrences, preview.removedWordOccurrences);
+    assert.equal(countLines(db), 1);
+  } finally {
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});
+
+test('the lookback window leaves older bursts alone', () => {
+  const recentMs = BASE_MS;
+  const oldMs = BASE_MS - 40 * DAY_MS;
+  const { db, dbPath } = createDb([
+    ...karaokeFrames(1, '飛び上がる', 10_000, 6, 40).map((line) => ({
+      ...line,
+      createdMs: oldMs,
+    })),
+    ...karaokeFrames(2, '飛び上がる', 10_000, 6, 40).map((line) => ({
+      ...line,
+      createdMs: recentMs,
+    })),
+  ]);
+
+  globalThis.__subminerTestNowMs = BASE_MS;
+  try {
+    const summary = cleanupDuplicateSubtitleLines(db, { lookbackDays: 30 });
+
+    assert.equal(summary.lookbackDays, 30);
+    assert.equal(summary.scannedLines, 6);
+    assert.equal(summary.burstGroups, 1);
+    assert.equal(summary.removedLines, 5);
+    // Six untouched old frames plus the one surviving recent line.
+    assert.equal(countLines(db), 7);
+    assert.equal(wordFrequency(db), 7);
+  } finally {
+    globalThis.__subminerTestNowMs = undefined;
+    db.close();
+    cleanupDbPath(dbPath);
+  }
+});

src/core/services/immersion-tracker/duplicate-line-cleanup.ts

@@ -0,0 +1,423 @@
+/*
+ * Retroactive removal of animation-burst subtitle lines from the stats database.
+ *
+ * Before the live ingest gate existed, a karaoke OP recorded one line -- and one count
+ * for every word in it -- per animation frame, which is enough to put an OP lyric at the
+ * top of "Top Repeated Words" for good. This module finds those runs in what is already
+ * stored and takes them back down to one line.
+ *
+ * Only timing is available here: the stored text has been stripped of ASS markup, so the
+ * authoring evidence the file-level parser uses (`\t`, `\move`, karaoke timing, a
+ * changing override signature) is long gone. What is left is a run of identical,
+ * contiguous, short-lived lines inside a single session.
+ *
+ * The run has to be as long as the timing-only rule in `subtitle-cue-dedup` demands, but
+ * its short frames may be as long as the animation-frame bound rather than the much
+ * tighter timing-only one. A qualifying run may end with one longer hold, which is a
+ * common karaoke shape. Five or more repeats of the same text, each ending where the next
+ * begins, is already conclusive on its own -- no dialogue does that -- and the tighter
+ * bound would walk straight past the heavier typesetting that motivated this, where
+ * frames sit nearer a quarter of a second. Both bounds are options, so a cautious run can
+ * ask for more, and a dry run always reports before anything is removed.
+ *
+ * Scope: subtitle lines, their word/kanji occurrences, and the `imm_words`/`imm_kanji`
+ * aggregates those occurrences feed. Session telemetry (`lines_seen`, `tokens_seen`) and
+ * the rollups derived from it are left alone; they are cumulative samples taken at record
+ * time, and for sessions whose raw rows have since been pruned they cannot be recomputed.
+ */
+
+import type { DatabaseSync } from './sqlite';
+import {
+  ANIMATION_FRAME_MAX_SECONDS,
+  DUPLICATE_CUE_GAP_TOLERANCE_SECONDS,
+  MIN_STREAM_RESIDUE_FRAMES,
+  MIN_TIMING_ONLY_FRAMES,
+  TIMING_ONLY_FRAME_MAX_SECONDS,
+} from '../subtitle-burst-constants';
+import {
+  applyLexicalRemovals,
+  makePlaceholders,
+  planLexicalRemovalsForLines,
+  toDbTimestamp,
+} from './query-shared';
+import { nowMs } from './time';
+
+const MS_PER_DAY = 86_400_000;
+/** SQLite caps bound parameters per statement; stay well under it. */
+const ID_BATCH_SIZE = 400;
+const DEFAULT_SAMPLE_LIMIT = 20;
+
+export interface DuplicateSubtitleLineCleanupOptions {
+  /** Only consider lines recorded within this many days. Null or omitted = all history. */
+  lookbackDays?: number | null;
+  /** Measure without writing. */
+  dryRun?: boolean;
+  /** Identical contiguous lines needed before a run counts as an animation. */
+  minRunLength?: number;
+  /** Longest a single event may last and still look like an animation frame. */
+  maxFrameSeconds?: number;
+  /** How many of the largest runs to describe in the summary. */
+  sampleLimit?: number;
+}
+
+export interface DuplicateSubtitleLineBurst {
+  sessionId: number;
+  videoId: number;
+  text: string;
+  /** Kept line, extended to cover the whole run. */
+  keptLineId: number;
+  removedLineIds: number[];
+  startMs: number;
+  endMs: number;
+}
+
+export interface DuplicateSubtitleLineSample {
+  videoId: number;
+  videoTitle: string | null;
+  text: string;
+  frames: number;
+  removedLines: number;
+  startMs: number;
+  endMs: number;
+}
+
+export interface DuplicateSubtitleLineCleanupSummary {
+  dryRun: boolean;
+  lookbackDays: number | null;
+  scannedLines: number;
+  burstGroups: number;
+  removedLines: number;
+  removedWordOccurrences: number;
+  removedKanjiOccurrences: number;
+  samples: DuplicateSubtitleLineSample[];
+}
+
+export interface StoredSubtitleLineRow {
+  lineId: number;
+  sessionId: number;
+  videoId: number;
+  text: string;
+  startMs: number;
+  endMs: number;
+}
+
+interface ResolvedBounds {
+  lookbackDays: number | null;
+  minRunLength: number;
+  maxFrameMs: number;
+  /** Shorter runs qualify only when every event sits under this much stricter bound. */
+  residueMinRunLength: number;
+  strictFrameMs: number;
+  gapToleranceMs: number;
+  sampleLimit: number;
+}
+
+function resolveBounds(options: DuplicateSubtitleLineCleanupOptions): ResolvedBounds {
+  const lookbackDays =
+    typeof options.lookbackDays === 'number' && Number.isFinite(options.lookbackDays)
+      ? Math.max(1, Math.floor(options.lookbackDays))
+      : null;
+  const minRunLength =
+    typeof options.minRunLength === 'number' && Number.isFinite(options.minRunLength)
+      ? Math.max(2, Math.floor(options.minRunLength))
+      : MIN_TIMING_ONLY_FRAMES;
+  const maxFrameSeconds =
+    typeof options.maxFrameSeconds === 'number' &&
+    Number.isFinite(options.maxFrameSeconds) &&
+    options.maxFrameSeconds > 0
+      ? options.maxFrameSeconds
+      : ANIMATION_FRAME_MAX_SECONDS;
+  const sampleLimit =
+    typeof options.sampleLimit === 'number' && options.sampleLimit >= 0
+      ? Math.floor(options.sampleLimit)
+      : DEFAULT_SAMPLE_LIMIT;
+  return {
+    lookbackDays,
+    minRunLength,
+    maxFrameMs: Math.round(maxFrameSeconds * 1000),
+    residueMinRunLength: Math.max(MIN_STREAM_RESIDUE_FRAMES, minRunLength - 1),
+    strictFrameMs: Math.round(TIMING_ONLY_FRAME_MAX_SECONDS * 1000),
+    gapToleranceMs: Math.round(DUPLICATE_CUE_GAP_TOLERANCE_SECONDS * 1000),
+    sampleLimit,
+  };
+}
+
+/**
+ * `CREATED_DATE` holds epoch milliseconds on rows this app wrote, but older and synced
+ * rows can carry seconds, so normalize before comparing against the cutoff.
+ */
+const CREATED_MS_SQL = `
+  CASE
+    WHEN sl.CREATED_DATE < 10000000000 THEN sl.CREATED_DATE * 1000
+    ELSE sl.CREATED_DATE
+  END`;
+
+function readCandidateLines(db: DatabaseSync, bounds: ResolvedBounds): StoredSubtitleLineRow[] {
+  const scope =
+    bounds.lookbackDays === null
+      ? ''
+      : `AND sl.CREATED_DATE IS NOT NULL AND ${CREATED_MS_SQL} >= ?`;
+  const params = bounds.lookbackDays === null ? [] : [nowMs() - bounds.lookbackDays * MS_PER_DAY];
+  return db
+    .prepare(
+      `SELECT
+         sl.line_id AS lineId,
+         sl.session_id AS sessionId,
+         sl.video_id AS videoId,
+         sl.text AS text,
+         sl.segment_start_ms AS startMs,
+         sl.segment_end_ms AS endMs
+       FROM imm_subtitle_lines sl
+       WHERE sl.segment_start_ms IS NOT NULL
+         AND sl.segment_end_ms IS NOT NULL
+         ${scope}
+       ORDER BY sl.session_id, sl.video_id, sl.segment_start_ms, sl.line_id`,
+    )
+    .all(...params) as StoredSubtitleLineRow[];
+}
+
+function isBurst(run: StoredSubtitleLineRow[], bounds: ResolvedBounds): boolean {
+  const isShortFrame = (row: StoredSubtitleLineRow): boolean =>
+    row.endMs - row.startMs <= bounds.maxFrameMs;
+  // The residue the live gate leaves behind: it records the first frames of a burst
+  // before the run is long enough to recognise, so one frame fewer than the timing-only
+  // minimum, every one under the strict timing-only bound. No dialogue holds identical
+  // sub-tenth-second lines back to back that many times.
+  if (
+    run.length >= bounds.residueMinRunLength &&
+    run.every((row) => row.endMs - row.startMs <= bounds.strictFrameMs)
+  ) {
+    return true;
+  }
+  if (run.length < bounds.minRunLength) {
+    return false;
+  }
+  if (run.every(isShortFrame)) {
+    return true;
+  }
+  // Karaoke commonly finishes its short animation frames with one long hold. Only the
+  // final event may exceed the frame bound, and the short frames before it must already
+  // meet the minimum run length on their own.
+  return (
+    run.length - 1 >= bounds.minRunLength &&
+    run.slice(0, -1).every(isShortFrame) &&
+    !isShortFrame(run[run.length - 1]!)
+  );
+}
+
+function toBurst(run: StoredSubtitleLineRow[]): DuplicateSubtitleLineBurst {
+  const [first] = run;
+  return {
+    sessionId: first!.sessionId,
+    videoId: first!.videoId,
+    text: first!.text,
+    keptLineId: first!.lineId,
+    removedLineIds: run.slice(1).map((row) => row.lineId),
+    startMs: first!.startMs,
+    endMs: run.reduce((latest, row) => Math.max(latest, row.endMs), first!.endMs),
+  };
+}
+
+/**
+ * Group stored lines into animation runs.
+ *
+ * Rows are bucketed per (session, video, text) before chaining, the way the file-level
+ * dedup buckets cues: dual-line karaoke interleaves two texts frame by frame, and
+ * chaining across the interleave would break every run at length one.
+ *
+ * Runs never cross a session, which is what keeps a rewatch intact: the same episode
+ * watched twice stores the same line twice, and those two belong to different sessions.
+ */
+export function findDuplicateSubtitleLineBursts(
+  rows: readonly StoredSubtitleLineRow[],
+  options: DuplicateSubtitleLineCleanupOptions = {},
+): DuplicateSubtitleLineBurst[] {
+  const bounds = resolveBounds(options);
+
+  // Insertion order preserves the query's startMs ordering within each bucket.
+  const rowsByKey = new Map<string, StoredSubtitleLineRow[]>();
+  for (const row of rows) {
+    const key = `${row.sessionId}|${row.videoId}|${row.text}`;
+    const bucket = rowsByKey.get(key);
+    if (bucket) {
+      bucket.push(row);
+    } else {
+      rowsByKey.set(key, [row]);
+    }
+  }
+
+  const bursts: DuplicateSubtitleLineBurst[] = [];
+  for (const bucket of rowsByKey.values()) {
+    if (bucket.length < 2) {
+      continue;
+    }
+
+    let run: StoredSubtitleLineRow[] = [];
+    let chainEndMs = 0;
+
+    const closeRun = (): void => {
+      if (run.length > 1 && isBurst(run, bounds)) {
+        bursts.push(toBurst(run));
+      }
+      run = [];
+    };
+
+    for (const row of bucket) {
+      if (run.length > 0 && row.startMs <= chainEndMs + bounds.gapToleranceMs) {
+        run.push(row);
+        chainEndMs = Math.max(chainEndMs, row.endMs);
+        continue;
+      }
+      closeRun();
+      run = [row];
+      chainEndMs = row.endMs;
+    }
+    closeRun();
+  }
+
+  return bursts;
+}
+
+function chunk<T>(values: T[], size: number): T[][] {
+  const chunks: T[][] = [];
+  for (let i = 0; i < values.length; i += size) {
+    chunks.push(values.slice(i, i + size));
+  }
+  return chunks;
+}
+
+function buildSamples(
+  db: DatabaseSync,
+  bursts: DuplicateSubtitleLineBurst[],
+  sampleLimit: number,
+): DuplicateSubtitleLineSample[] {
+  if (sampleLimit === 0 || bursts.length === 0) {
+    return [];
+  }
+  const largest = [...bursts]
+    .sort((a, b) => b.removedLineIds.length - a.removedLineIds.length)
+    .slice(0, sampleLimit);
+  const videoIds = [...new Set(largest.map((burst) => burst.videoId))];
+  const titles = new Map<number, string>();
+  for (const batch of chunk(videoIds, ID_BATCH_SIZE)) {
+    const rows = db
+      .prepare(
+        `SELECT video_id AS videoId, canonical_title AS title
+         FROM imm_videos
+         WHERE video_id IN (${makePlaceholders(batch)})`,
+      )
+      .all(...batch) as Array<{ videoId: number; title: string | null }>;
+    for (const row of rows) {
+      if (row.title) titles.set(row.videoId, row.title);
+    }
+  }
+
+  return largest.map((burst) => ({
+    videoId: burst.videoId,
+    videoTitle: titles.get(burst.videoId) ?? null,
+    text: burst.text,
+    frames: burst.removedLineIds.length + 1,
+    removedLines: burst.removedLineIds.length,
+    startMs: burst.startMs,
+    endMs: burst.endMs,
+  }));
+}
+
+function sumRemovedOccurrences(
+  db: DatabaseSync,
+  table: 'imm_word_line_occurrences' | 'imm_kanji_line_occurrences',
+  lineIds: number[],
+): number {
+  let total = 0;
+  for (const batch of chunk(lineIds, ID_BATCH_SIZE)) {
+    const row = db
+      .prepare(
+        `SELECT COALESCE(SUM(occurrence_count), 0) AS total
+         FROM ${table}
+         WHERE line_id IN (${makePlaceholders(batch)})`,
+      )
+      .get(...batch) as { total: number } | null;
+    total += row?.total ?? 0;
+  }
+  return total;
+}
+
+function applyBursts(db: DatabaseSync, bursts: DuplicateSubtitleLineBurst[]): void {
+  const removedLineIds = bursts.flatMap((burst) => burst.removedLineIds);
+  const currentMs = toDbTimestamp(nowMs());
+
+  db.exec('BEGIN IMMEDIATE');
+  try {
+    for (const batch of chunk(removedLineIds, ID_BATCH_SIZE)) {
+      const placeholders = makePlaceholders(batch);
+      // Measured before the delete, applied after it: `applyLexicalRemovals` checks the
+      // surviving occurrences to decide whether a zeroed count really means the word is
+      // gone, so the rows it inspects have to be the post-delete ones.
+      const plan = planLexicalRemovalsForLines(db, batch);
+      db.prepare(`DELETE FROM imm_word_line_occurrences WHERE line_id IN (${placeholders})`).run(
+        ...batch,
+      );
+      db.prepare(`DELETE FROM imm_kanji_line_occurrences WHERE line_id IN (${placeholders})`).run(
+        ...batch,
+      );
+      db.prepare(`DELETE FROM imm_subtitle_lines WHERE line_id IN (${placeholders})`).run(...batch);
+      applyLexicalRemovals(db, plan);
+    }
+
+    const extendStmt = db.prepare(
+      `UPDATE imm_subtitle_lines
+       SET segment_end_ms = ?, LAST_UPDATE_DATE = ?
+       WHERE line_id = ? AND (segment_end_ms IS NULL OR segment_end_ms < ?)`,
+    );
+    for (const burst of bursts) {
+      extendStmt.run(burst.endMs, currentMs, burst.keptLineId, burst.endMs);
+    }
+    db.exec('COMMIT');
+  } catch (error) {
+    try {
+      db.exec('ROLLBACK');
+    } catch {
+      // Surface the transaction failure, not the rollback's.
+    }
+    throw error;
+  }
+}
+
+/**
+ * Collapse stored animation bursts down to one line each.
+ *
+ * A dry run measures exactly what an apply would remove, using the same scan, so the
+ * numbers shown in a confirmation prompt are the numbers that will happen.
+ */
+export function cleanupDuplicateSubtitleLines(
+  db: DatabaseSync,
+  options: DuplicateSubtitleLineCleanupOptions = {},
+): DuplicateSubtitleLineCleanupSummary {
+  const bounds = resolveBounds(options);
+  const dryRun = options.dryRun === true;
+  const rows = readCandidateLines(db, bounds);
+  const bursts = findDuplicateSubtitleLineBursts(rows, options);
+  const removedLineIds = bursts.flatMap((burst) => burst.removedLineIds);
+
+  const summary: DuplicateSubtitleLineCleanupSummary = {
+    dryRun,
+    lookbackDays: bounds.lookbackDays,
+    scannedLines: rows.length,
+    burstGroups: bursts.length,
+    removedLines: removedLineIds.length,
+    removedWordOccurrences: sumRemovedOccurrences(db, 'imm_word_line_occurrences', removedLineIds),
+    removedKanjiOccurrences: sumRemovedOccurrences(
+      db,
+      'imm_kanji_line_occurrences',
+      removedLineIds,
+    ),
+    samples: buildSamples(db, bursts, bounds.sampleLimit),
+  };
+
+  if (dryRun || removedLineIds.length === 0) {
+    return summary;
+  }
+
+  applyBursts(db, bursts);
+  return summary;
+}

src/core/services/immersion-tracker/query-shared.ts

@@ -276,6 +276,19 @@ export function planLexicalRemovalsForSessions(
   return planLexicalRemovals(db, `sl.session_id IN (${makePlaceholders(sessionIds)})`, sessionIds);
 }
 
+/**
+ * Measure what deleting these individual subtitle lines removes from the vocabulary
+ * tables. Used by the duplicate-line cleanup, which drops animation frames out of the
+ * middle of sessions that otherwise stay intact.
+ */
+export function planLexicalRemovalsForLines(
+  db: DatabaseSync,
+  lineIds: number[],
+): LexicalRemovalPlan {
+  if (lineIds.length === 0) return EMPTY_LEXICAL_REMOVAL_PLAN;
+  return planLexicalRemovals(db, `sl.line_id IN (${makePlaceholders(lineIds)})`, lineIds);
+}
+
 /** Measure what deleting these videos removes from the vocabulary tables. */
 export function planLexicalRemovalsForVideos(
   db: DatabaseSync,

src/core/services/stats-server/library-routes.ts

@@ -5,6 +5,7 @@ import {
   buildSentenceSearchOptions,
   enrichSessionsWithKnownWordMetrics,
   parseBooleanQuery,
+  parseDuplicateLineCleanupBody,
   parseExcludedWordsBody,
   parseIntQuery,
 } from './route-support.js';
@@ -40,6 +41,19 @@ export function registerStatsLibraryRoutes(
     return c.json(statsJson('setExcludedWords', { ok: true }));
   });
 
+  // Collapse animation bursts older versions recorded frame by frame. `dryRun` measures
+  // the same scan without writing, so the confirmation the user sees is the real cost.
+  app.post('/api/stats/maintenance/duplicate-lines', async (c) => {
+    const contentType = c.req.header('content-type')?.split(';', 1)[0]?.trim().toLowerCase();
+    if (contentType !== 'application/json') return c.body(null, 415);
+    const body = await c.req.json().catch(() => null);
+    const options = parseDuplicateLineCleanupBody(body);
+    if (!options) return c.body(null, 400);
+    const { dryRun, lookbackDays } = options;
+    const result = await tracker.cleanupDuplicateSubtitleLines({ dryRun, lookbackDays });
+    return c.json(statsJson('duplicateLineCleanup', result));
+  });
+
   app.get('/api/stats/vocabulary/occurrences', async (c) => {
     const headword = (c.req.query('headword') ?? '').trim();
     const word = (c.req.query('word') ?? '').trim();

src/core/services/stats-server/route-support.ts

@@ -88,6 +88,35 @@ export function parseExcludedWordsBody(body: unknown): StatsExcludedWord[] | nul
   return words;
 }
 
+/**
+ * Read a duplicate-line cleanup request. An explicit object with no lookback scans all
+ * history. Invalid bodies and invalid windows are rejected instead of broadening scope.
+ */
+export function parseDuplicateLineCleanupBody(body: unknown): {
+  dryRun: boolean;
+  lookbackDays: number | null;
+} | null {
+  if (!body || typeof body !== 'object' || Array.isArray(body)) {
+    return null;
+  }
+  const source = body as Record<string, unknown>;
+  if (source.dryRun !== undefined && typeof source.dryRun !== 'boolean') {
+    return null;
+  }
+  const rawLookback = source.lookbackDays;
+  if (
+    rawLookback !== undefined &&
+    rawLookback !== null &&
+    (typeof rawLookback !== 'number' || !Number.isFinite(rawLookback) || rawLookback < 1)
+  ) {
+    return null;
+  }
+  return {
+    dryRun: source.dryRun === true,
+    lookbackDays: typeof rawLookback === 'number' ? Math.floor(rawLookback) : null,
+  };
+}
+
 export function loadKnownWordsSet(cachePath: string | undefined): Set<string> | null {
   if (!cachePath || !existsSync(cachePath)) return null;
   try {

src/core/services/subtitle-burst-constants.ts

@@ -0,0 +1,48 @@
+/*
+ * Thresholds that decide when a run of repeated subtitle events is one animation.
+ *
+ * Three consumers have to agree on these numbers or the same karaoke line is one cue in
+ * the sidebar and two hundred in the stats: the file-level cue dedup
+ * (`subtitle-cue-dedup`), the live gate that decides what immersion stats record
+ * (`subtitle-line-dedup-gate`), and the retroactive database cleanup
+ * (`immersion-tracker/duplicate-line-cleanup`).
+ */
+
+/**
+ * Back-to-back frames of the same animation are authored flush against each other; a
+ * tiny tolerance absorbs the centisecond rounding of the ASS timestamp format.
+ */
+export const DUPLICATE_CUE_GAP_TOLERANCE_SECONDS = 0.05;
+
+/**
+ * A burst is a *sequence*. Two adjacent events are two events, not an animation --
+ * characters do repeat each other, and a repeated line can legitimately be short.
+ */
+export const MIN_BURST_EVENTS = 3;
+
+/**
+ * Real dialogue holds on screen for about a second, so a run with a couple of much
+ * shorter events among them looks like frames. Used only alongside authoring evidence.
+ */
+export const ANIMATION_FRAME_MAX_SECONDS = 0.3;
+
+/** A karaoke run usually ends on a long "hold" frame, so not every event is short. */
+export const MIN_TAGGED_BURST_FRAMES = 2;
+
+/**
+ * SRT and VTT carry no authoring metadata at all, so timing is the only signal available
+ * -- which makes it the easiest one to get wrong. ASS->SRT conversion leaves frames at
+ * ~0.04s, well under any real utterance, and a burst leaves many of them behind. Both
+ * bounds are deliberately far stricter than the ASS path: a run of ordinary short lines
+ * (`えっ` traded between characters) must not clear them.
+ */
+export const TIMING_ONLY_FRAME_MAX_SECONDS = 0.1;
+export const MIN_TIMING_ONLY_FRAMES = 5;
+
+/**
+ * The live gate can only recognise a streaming burst from the inside, so it records the
+ * first `MIN_TIMING_ONLY_FRAMES - 1` frames before it starts dropping. That stored
+ * residue is one frame short of the timing-only minimum, and the retroactive cleanup
+ * accepts it only when every event also sits under the strict timing-only frame bound.
+ */
+export const MIN_STREAM_RESIDUE_FRAMES = MIN_TIMING_ONLY_FRAMES - 1;

src/core/services/subtitle-cue-dedup.ts

@@ -7,31 +7,20 @@
  */
 
 import { hasAssTemporalOverride, isAnimatedAssEffectKind } from './ass-text';
+import {
+  ANIMATION_FRAME_MAX_SECONDS,
+  DUPLICATE_CUE_GAP_TOLERANCE_SECONDS,
+  MIN_BURST_EVENTS,
+  MIN_TAGGED_BURST_FRAMES,
+  MIN_TIMING_ONLY_FRAMES,
+  TIMING_ONLY_FRAME_MAX_SECONDS,
+} from './subtitle-burst-constants';
 import type {
   AnnotatedSubtitleCue,
   SubtitleCue,
   SubtitleSourceFormat,
 } from './subtitle-cue-parser';
 
-// Back-to-back frames of the same animation are authored flush against each other; a
-// tiny tolerance absorbs the centisecond rounding of the ASS timestamp format.
-const DUPLICATE_CUE_GAP_TOLERANCE_SECONDS = 0.05;
-// A burst is a *sequence*. Two adjacent events are two events, not an animation --
-// characters do repeat each other, and a repeated line can legitimately be short.
-const MIN_BURST_EVENTS = 3;
-// Real dialogue holds on screen for about a second, so a run with a couple of much
-// shorter events among them looks like frames. Used only alongside authoring evidence.
-const ANIMATION_FRAME_MAX_SECONDS = 0.3;
-// A karaoke run usually ends on a long "hold" frame, so not every event is short.
-const MIN_TAGGED_BURST_FRAMES = 2;
-// SRT and VTT carry no authoring metadata at all, so timing is the only signal available
-// -- which makes it the easiest one to get wrong. ASS->SRT conversion leaves frames at
-// ~0.04s, well under any real utterance, and a burst leaves many of them behind. Both
-// bounds are deliberately far stricter than the ASS path: a run of ordinary short lines
-// (`えっ` traded between characters) must not clear them.
-const TIMING_ONLY_FRAME_MAX_SECONDS = 0.1;
-const MIN_TIMING_ONLY_FRAMES = 5;
-
 function cueKey(cue: SubtitleCue): string {
   return `${cue.startTime}|${cue.endTime}|${cue.text}`;
 }

src/core/services/subtitle-line-dedup-gate.test.ts

@@ -0,0 +1,204 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { createSubtitleLineDedupGate } from './subtitle-line-dedup-gate';
+import type { SubtitleCue } from '../../types';
+
+function karaokeFrames(text: string, start: number, frames: number, frameSeconds: number) {
+  return Array.from({ length: frames }, (_, index) => ({
+    text,
+    startSec: start + index * frameSeconds,
+    endSec: start + (index + 1) * frameSeconds,
+  }));
+}
+
+test('parsed cues drop the frames the sidebar already collapsed', () => {
+  // What `mergeDuplicateCues` leaves behind for a karaoke run: one cue over the run.
+  const cues: SubtitleCue[] = [
+    { startTime: 10, endTime: 14, text: '飛び上がる' },
+    { startTime: 14, endTime: 16, text: 'もしも' },
+  ];
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
+
+  const recorded = karaokeFrames('飛び上がる', 10, 40, 0.04).filter((sample) =>
+    gate.shouldRecord(sample),
+  );
+
+  assert.equal(recorded.length, 1);
+  assert.equal(recorded[0]!.startSec, 10);
+  assert.equal(gate.shouldRecord({ text: 'もしも', startSec: 14, endSec: 16 }), true);
+});
+
+test('parsed cues keep separate lines that merely repeat', () => {
+  const cues: SubtitleCue[] = [
+    { startTime: 3, endTime: 3.4, text: 'えっ' },
+    { startTime: 3.4, endTime: 3.9, text: 'えっ' },
+    { startTime: 3.9, endTime: 4.5, text: 'えっ' },
+  ];
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
+
+  const recorded = cues.filter((cue) =>
+    gate.shouldRecord({ text: cue.text, startSec: cue.startTime, endSec: cue.endTime }),
+  );
+
+  assert.equal(recorded.length, 3);
+});
+
+test('parsed cues outrank the streaming heuristic for short repeated cues', () => {
+  // Long enough to trip the timing-only rule, but the parser saw these with full
+  // lookahead and kept them, so every one of them is a line the sidebar shows.
+  const cues: SubtitleCue[] = Array.from({ length: 8 }, (_, index) => ({
+    startTime: 3 + index * 0.08,
+    endTime: 3 + (index + 1) * 0.08,
+    text: 'えっ',
+  }));
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
+
+  const recorded = cues.filter((cue) =>
+    gate.shouldRecord({ text: cue.text, startSec: cue.startTime, endSec: cue.endTime }),
+  );
+
+  assert.equal(recorded.length, 8);
+});
+
+test('parsed cues preserve legitimately separate cues only 40ms apart', () => {
+  const cues: SubtitleCue[] = Array.from({ length: 8 }, (_, index) => ({
+    startTime: 3 + index * 0.04,
+    endTime: 3 + (index + 1) * 0.04,
+    text: 'えっ',
+  }));
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
+
+  const recorded = cues.filter((cue) =>
+    gate.shouldRecord({ text: cue.text, startSec: cue.startTime, endSec: cue.endTime }),
+  );
+
+  assert.equal(recorded.length, 8);
+});
+
+test('a line whose timing does not match any cue still records', () => {
+  // A shifted track, an embedded sub nobody parsed: no match, no drop.
+  const cues: SubtitleCue[] = [{ startTime: 10, endTime: 14, text: '飛び上がる' }];
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
+
+  assert.equal(gate.shouldRecord({ text: '飛び上がる', startSec: 42, endSec: 44 }), true);
+});
+
+test('shifted parsed text falls back to streaming burst detection', () => {
+  const cues: SubtitleCue[] = [{ startTime: 10, endTime: 14, text: '飛び上がる' }];
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
+
+  const recorded = karaokeFrames('飛び上がる', 42, 40, 0.04).filter((sample) =>
+    gate.shouldRecord(sample),
+  );
+
+  assert.equal(recorded.length, 4);
+});
+
+test('replacing the parsed cue source forgets a streaming run', () => {
+  let cues: SubtitleCue[] = [];
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
+
+  karaokeFrames('飛び上がる', 42, 20, 0.04).forEach((sample) => gate.shouldRecord(sample));
+  // A new source publishes its own cue list; the old run must not carry over.
+  cues = [{ startTime: 100, endTime: 104, text: 'もしも' }];
+
+  assert.equal(gate.shouldRecord({ text: '飛び上がる', startSec: 42.8, endSec: 42.84 }), true);
+});
+
+test('without parsed cues a long run of identical short frames stops recording', () => {
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => null });
+
+  const recorded = karaokeFrames('ひとしずく', 0, 200, 0.04).filter((sample) =>
+    gate.shouldRecord(sample),
+  );
+
+  assert.equal(recorded.length, 4);
+});
+
+test('without parsed cues interleaved dual-line karaoke stops recording per line', () => {
+  // Fansub OPs typically run two typeset lines at once -- kanji and romaji -- and mpv
+  // reports their frames interleaved. Each line must build its own run.
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => null });
+
+  const samples = Array.from({ length: 40 }, (_, index) => {
+    const start = 10 + Math.floor(index / 2) * 0.06;
+    return index % 2 === 0
+      ? { text: '歌詞', startSec: start, endSec: start + 0.06 }
+      : { text: 'kashi', startSec: start + 0.001, endSec: start + 0.061 };
+  });
+  const recorded = samples.filter((sample) => gate.shouldRecord(sample));
+
+  assert.equal(recorded.filter((sample) => sample.text === '歌詞').length, 4);
+  assert.equal(recorded.filter((sample) => sample.text === 'kashi').length, 4);
+});
+
+test('interleaved dialogue between two speakers keeps recording', () => {
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => null });
+
+  // Two characters trading normal-length lines back and forth.
+  const samples = Array.from({ length: 12 }, (_, index) => {
+    const start = 5 + index * 0.7;
+    return {
+      text: index % 2 === 0 ? 'えっ' : 'なに',
+      startSec: start,
+      endSec: start + 0.7,
+    };
+  });
+  const recorded = samples.filter((sample) => gate.shouldRecord(sample));
+
+  assert.equal(recorded.length, 12);
+});
+
+test('without parsed cues ordinary repeated dialogue keeps recording', () => {
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => null });
+
+  // Six contiguous `えっ`, each held for a normal beat rather than an animation frame.
+  const recorded = karaokeFrames('えっ', 0, 6, 0.6).filter((sample) => gate.shouldRecord(sample));
+
+  assert.equal(recorded.length, 6);
+});
+
+test('the same event offered twice does not advance the run', () => {
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => null });
+
+  // mpv fires the timing handler once for `sub-start` and once for `sub-end`.
+  for (let i = 0; i < 8; i += 1) {
+    assert.equal(gate.shouldRecord({ text: '待って', startSec: 5, endSec: 5.05 }), true);
+  }
+});
+
+test('a gap between frames starts a new run', () => {
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => null });
+
+  const first = karaokeFrames('もし', 0, 6, 0.04).filter((sample) => gate.shouldRecord(sample));
+  const second = karaokeFrames('もし', 30, 6, 0.04).filter((sample) => gate.shouldRecord(sample));
+
+  assert.equal(first.length, 4);
+  assert.equal(second.length, 4);
+});
+
+test('reset forgets the streaming run', () => {
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => null });
+
+  karaokeFrames('もし', 0, 20, 0.04).forEach((sample) => gate.shouldRecord(sample));
+  gate.reset();
+
+  assert.equal(gate.shouldRecord({ text: 'もし', startSec: 0.8, endSec: 0.84 }), true);
+});
+
+test('reset ignores stale parsed cues until the source publishes a new cue list', () => {
+  let cues: SubtitleCue[] = [{ startTime: 10, endTime: 14, text: '飛び上がる' }];
+  const gate = createSubtitleLineDedupGate({ getParsedCues: () => cues });
+
+  assert.equal(gate.shouldRecord({ text: '飛び上がる', startSec: 10, endSec: 10.04 }), true);
+  gate.reset();
+
+  const recordedWithStaleCues = karaokeFrames('飛び上がる', 10.04, 8, 0.04).filter((sample) =>
+    gate.shouldRecord(sample),
+  );
+  assert.equal(recordedWithStaleCues.length, 4);
+
+  cues = [{ startTime: 20, endTime: 24, text: '飛び上がる' }];
+  assert.equal(gate.shouldRecord({ text: '飛び上がる', startSec: 20, endSec: 20.04 }), true);
+  assert.equal(gate.shouldRecord({ text: '飛び上がる', startSec: 20.04, endSec: 20.08 }), false);
+});

src/core/services/subtitle-line-dedup-gate.ts

@@ -0,0 +1,241 @@
+/*
+ * Decides which live mpv subtitle lines reach the immersion stats.
+ *
+ * The sidebar reads a parsed subtitle file, so it can collapse an animation burst with
+ * full lookahead (`subtitle-cue-dedup`). Stats are fed from mpv's `sub-start`/`sub-end`
+ * properties instead -- one event per animation frame, each with its own start time --
+ * so without a gate a karaoke OP counts its lyrics once per frame and buries every real
+ * word in the vocabulary charts.
+ *
+ * Two layers, in order:
+ *
+ * 1. When the active source has been parsed, its cue list has *already* been collapsed.
+ *    A live line that lands inside a surviving cue of the same text, but after that
+ *    cue's start, is a frame the sidebar merged away, so stats drop it too. This is the
+ *    layer that keeps the two views consistent by construction.
+ * 2. Otherwise (embedded track nobody parsed, a source whose timings mpv has shifted)
+ *    fall back to timing alone. No authoring metadata is available live -- mpv delivers
+ *    `sub-text-ass` after `sub-start`/`sub-end`, so any ASS text read here belongs to the
+ *    previous event -- which puts this layer in the same position as the SRT path in
+ *    `subtitle-cue-dedup`, and it uses that path's deliberately strict bounds.
+ */
+
+import { normalizePlainSubtitleText } from './ass-text';
+import {
+  DUPLICATE_CUE_GAP_TOLERANCE_SECONDS,
+  MIN_TIMING_ONLY_FRAMES,
+  TIMING_ONLY_FRAME_MAX_SECONDS,
+} from './subtitle-burst-constants';
+import type { SubtitleCue } from './subtitle-cue-parser';
+
+export interface SubtitleLineSample {
+  text: string;
+  startSec: number;
+  endSec: number;
+}
+
+export interface SubtitleLineDedupGateDeps {
+  /** Cues for the active source, already collapsed by the parser. */
+  getParsedCues: () => readonly SubtitleCue[] | null | undefined;
+}
+
+export interface SubtitleLineDedupGate {
+  /** False when this line is an animation frame of a line already recorded. */
+  shouldRecord: (sample: SubtitleLineSample) => boolean;
+  /** Forget run state and ignore the current cue list until its source is replaced. */
+  reset: () => void;
+}
+
+interface CueSpan {
+  startTime: number;
+  endTime: number;
+}
+
+interface StreamingRunState {
+  startMs: number;
+  chainEndSec: number;
+  /** Contiguous identical short frames seen so far, including the recorded first one. */
+  frames: number;
+}
+
+/**
+ * Dual-line karaoke interleaves two texts frame by frame, so runs are tracked per text.
+ * Dead runs are pruned as playback moves past them; the cap only matters after a
+ * backward seek leaves runs whose ends sit ahead of the new position.
+ */
+const MAX_ACTIVE_STREAMING_RUNS = 32;
+
+/** Exact cue identity, separate from the looser tolerance used to chain adjacent frames. */
+const CUE_START_IDENTITY_TOLERANCE_SECONDS = 0.005;
+
+function normalizeLineText(text: string): string {
+  return normalizePlainSubtitleText(text, { collapseLineBreaks: true });
+}
+
+function buildSpansByText(cues: readonly SubtitleCue[]): Map<string, CueSpan[]> {
+  const spansByText = new Map<string, CueSpan[]>();
+  for (const cue of cues) {
+    const key = normalizeLineText(cue.text);
+    if (!key) continue;
+    const span = { startTime: cue.startTime, endTime: cue.endTime };
+    const existing = spansByText.get(key);
+    if (existing) {
+      existing.push(span);
+    } else {
+      spansByText.set(key, [span]);
+    }
+  }
+  return spansByText;
+}
+
+/**
+ * A frame the parser merged away: the same text, starting inside a surviving cue but
+ * after it began.
+ *
+ * Starting a cue always wins over falling inside one. The first frame of a collapsed run
+ * starts *at* the merged cue, and a line the parser deliberately kept separate -- three
+ * characters trading `えっ` back to back -- begins exactly where the one before it ends.
+ */
+function isMergedAwayFrame(spans: readonly CueSpan[], startSec: number): boolean | null {
+  const coveringSpans = spans.filter(
+    (span) =>
+      startSec >= span.startTime - CUE_START_IDENTITY_TOLERANCE_SECONDS &&
+      startSec <= span.endTime + CUE_START_IDENTITY_TOLERANCE_SECONDS,
+  );
+  if (coveringSpans.length === 0) {
+    return null;
+  }
+  const startsOwnCue = spans.some(
+    (span) => Math.abs(startSec - span.startTime) <= CUE_START_IDENTITY_TOLERANCE_SECONDS,
+  );
+  if (startsOwnCue) {
+    return false;
+  }
+  return coveringSpans.some(
+    (span) =>
+      startSec > span.startTime + CUE_START_IDENTITY_TOLERANCE_SECONDS &&
+      startSec <= span.endTime + CUE_START_IDENTITY_TOLERANCE_SECONDS,
+  );
+}
+
+export function createSubtitleLineDedupGate(
+  deps: SubtitleLineDedupGateDeps,
+): SubtitleLineDedupGate {
+  let indexedCues: readonly SubtitleCue[] | null | undefined;
+  let ignoredCuesAfterReset: readonly SubtitleCue[] | null | undefined;
+  let spansByText: Map<string, CueSpan[]> = new Map();
+  const runs = new Map<string, StreamingRunState>();
+
+  const lookupSpans = (text: string): CueSpan[] | null => {
+    const cues = deps.getParsedCues() ?? null;
+    if (ignoredCuesAfterReset !== undefined) {
+      if (cues === ignoredCuesAfterReset) {
+        return null;
+      }
+      ignoredCuesAfterReset = undefined;
+    }
+    if (cues !== indexedCues) {
+      indexedCues = cues;
+      spansByText = cues?.length ? buildSpansByText(cues) : new Map();
+      runs.clear();
+    }
+    return spansByText.get(text) ?? null;
+  };
+
+  /**
+   * A run this sample cannot continue is a run no later sample can continue either --
+   * continuation needs a start inside the running end plus tolerance, and starts only
+   * move forward outside of seeks.
+   */
+  const pruneDeadRuns = (startSec: number): void => {
+    for (const [text, state] of runs) {
+      if (state.chainEndSec + DUPLICATE_CUE_GAP_TOLERANCE_SECONDS < startSec) {
+        runs.delete(text);
+      }
+    }
+  };
+
+  /**
+   * Timing-only burst detection over a stream. Without lookahead the run can only be
+   * recognised from the inside, so the first frames of a burst are recorded and the rest
+   * dropped -- an OP costs a handful of counted lines instead of several hundred.
+   */
+  const advanceStreamingRun = (text: string, sample: SubtitleLineSample): boolean => {
+    pruneDeadRuns(sample.startSec);
+    const startMs = Math.round(sample.startSec * 1000);
+    const run = runs.get(text);
+    // mpv reports `sub-start` and `sub-end` separately, so one event can be offered
+    // twice. The same start is the same frame, never the next one in a run.
+    if (run && run.startMs === startMs) {
+      run.chainEndSec = Math.max(run.chainEndSec, sample.endSec);
+      return run.frames < MIN_TIMING_ONLY_FRAMES;
+    }
+
+    const isShortFrame = sample.endSec - sample.startSec < TIMING_ONLY_FRAME_MAX_SECONDS;
+    // Frames are authored flush against each other, but typesetters do overlap them, so
+    // the chain only requires forward progress that stays inside the running end.
+    const continuesRun =
+      run !== undefined &&
+      isShortFrame &&
+      startMs > run.startMs &&
+      sample.startSec <= run.chainEndSec + DUPLICATE_CUE_GAP_TOLERANCE_SECONDS;
+
+    if (continuesRun && run) {
+      run.startMs = startMs;
+      run.chainEndSec = Math.max(run.chainEndSec, sample.endSec);
+      run.frames += 1;
+      return run.frames < MIN_TIMING_ONLY_FRAMES;
+    }
+
+    const fresh: StreamingRunState = {
+      startMs,
+      chainEndSec: sample.endSec,
+      frames: isShortFrame ? 1 : 0,
+    };
+    runs.set(text, fresh);
+    if (runs.size > MAX_ACTIVE_STREAMING_RUNS) {
+      let oldestText: string | undefined;
+      let oldestEnd = Infinity;
+      for (const [runText, state] of runs) {
+        if (runText !== text && state.chainEndSec < oldestEnd) {
+          oldestEnd = state.chainEndSec;
+          oldestText = runText;
+        }
+      }
+      if (oldestText !== undefined) {
+        runs.delete(oldestText);
+      }
+    }
+    return fresh.frames < MIN_TIMING_ONLY_FRAMES;
+  };
+
+  return {
+    shouldRecord: (sample) => {
+      const text = normalizeLineText(sample.text);
+      if (!text) {
+        return true;
+      }
+
+      // The parsed cue list has the final say wherever it covers this line. Falling
+      // through to the streaming heuristic would let it drop cues the parser looked at
+      // with full lookahead and deliberately kept apart, which is the disagreement
+      // between sidebar and stats this gate exists to prevent.
+      const spans = lookupSpans(text);
+      if (spans) {
+        const mergedAway = isMergedAwayFrame(spans, sample.startSec);
+        if (mergedAway !== null) {
+          runs.delete(text);
+          return !mergedAway;
+        }
+      }
+
+      return advanceStreamingRun(text, sample);
+    },
+    reset: () => {
+      runs.clear();
+      ignoredCuesAfterReset = deps.getParsedCues() ?? null;
+      indexedCues = undefined;
+      spansByText = new Map();
+    },
+  };
+}