fix(stats): stop counting duplicate typeset subtitle lines (#191)

stats/src/components/vocabulary/DuplicateLineCleanup.test.tsx

@@ -0,0 +1,254 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { Window } from 'happy-dom';
+import { act } from 'react';
+import { createRoot } from 'react-dom/client';
+import { apiClient } from '../../lib/api-client';
+import type { StatsDuplicateLineCleanupResult } from '../../types/stats';
+import { DuplicateLineCleanup } from './DuplicateLineCleanup';
+
+interface TestWindow extends Window {
+  IS_REACT_ACT_ENVIRONMENT?: boolean;
+}
+
+function installDom(): () => void {
+  const previousWindow = globalThis.window;
+  const previousDocument = globalThis.document;
+  const previousHTMLElement = globalThis.HTMLElement;
+  const previousISReactActEnvironment = (
+    globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT?: boolean }
+  ).IS_REACT_ACT_ENVIRONMENT;
+  const window = new Window() as TestWindow;
+
+  Object.defineProperty(globalThis, 'window', { value: window, configurable: true });
+  Object.defineProperty(globalThis, 'document', { value: window.document, configurable: true });
+  Object.defineProperty(globalThis, 'HTMLElement', {
+    value: window.HTMLElement,
+    configurable: true,
+  });
+  (
+    globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT?: boolean }
+  ).IS_REACT_ACT_ENVIRONMENT = true;
+
+  return () => {
+    Object.defineProperty(globalThis, 'window', { value: previousWindow, configurable: true });
+    Object.defineProperty(globalThis, 'document', { value: previousDocument, configurable: true });
+    Object.defineProperty(globalThis, 'HTMLElement', {
+      value: previousHTMLElement,
+      configurable: true,
+    });
+    (
+      globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT?: boolean }
+    ).IS_REACT_ACT_ENVIRONMENT = previousISReactActEnvironment;
+  };
+}
+
+function findButton(container: Element, label: string): HTMLButtonElement {
+  const match = [...container.querySelectorAll('button')].find(
+    (button) => (button.textContent ?? '').trim() === label,
+  );
+  assert.ok(match, `expected a "${label}" button`);
+  return match as unknown as HTMLButtonElement;
+}
+
+/** The backdrop stays clickable during an apply, so it reaches the guard in `close`. */
+function findBackdrop(container: Element): HTMLButtonElement {
+  const match = container.querySelector('button[aria-label="Close duplicate line cleanup"]');
+  assert.ok(match, 'expected the backdrop close button');
+  return match as unknown as HTMLButtonElement;
+}
+
+function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
+  let resolve!: (value: T) => void;
+  const promise = new Promise<T>((done) => {
+    resolve = done;
+  });
+  return { promise, resolve };
+}
+
+function summary(
+  overrides: Partial<StatsDuplicateLineCleanupResult> = {},
+): StatsDuplicateLineCleanupResult {
+  return {
+    dryRun: false,
+    lookbackDays: 30,
+    scannedLines: 900,
+    burstGroups: 2,
+    removedLines: 180,
+    removedWordOccurrences: 540,
+    removedKanjiOccurrences: 120,
+    samples: [],
+    ...overrides,
+  };
+}
+
+interface Harness {
+  container: Element;
+  cleanedCalls: () => number;
+  closedCalls: () => number;
+  teardown: () => void;
+}
+
+async function mount(cleanup: (typeof apiClient)['cleanupDuplicateLines']): Promise<Harness> {
+  const uninstallDom = installDom();
+  const originalCleanup = apiClient.cleanupDuplicateLines;
+  apiClient.cleanupDuplicateLines = cleanup;
+
+  let cleaned = 0;
+  let closed = 0;
+  const container = document.createElement('div');
+  document.body.append(container);
+  const root = createRoot(container);
+
+  await act(async () => {
+    root.render(
+      <DuplicateLineCleanup
+        onClose={() => {
+          closed += 1;
+        }}
+        onCleaned={() => {
+          cleaned += 1;
+        }}
+      />,
+    );
+  });
+
+  return {
+    container,
+    cleanedCalls: () => cleaned,
+    closedCalls: () => closed,
+    teardown: () => {
+      apiClient.cleanupDuplicateLines = originalCleanup;
+      uninstallDom();
+    },
+  };
+}
+
+test('a reload is still owed after a later scan replaces the applied result', async () => {
+  const harness = await mount(async ({ dryRun } = {}) => summary({ dryRun: dryRun === true }));
+
+  try {
+    await act(async () => {
+      findButton(harness.container, 'Scan').click();
+    });
+    await act(async () => {
+      findButton(harness.container, 'Clean Up').click();
+    });
+    assert.equal(harness.cleanedCalls(), 0, 'reload must wait for the result to be read');
+
+    // The follow-up scan clears the applied summary, but the rows are already gone.
+    await act(async () => {
+      findButton(harness.container, 'Scan').click();
+    });
+    await act(async () => {
+      findButton(harness.container, 'Close').click();
+    });
+
+    assert.equal(harness.cleanedCalls(), 1);
+    assert.equal(harness.closedCalls(), 1);
+  } finally {
+    harness.teardown();
+  }
+});
+
+test('a reload is still owed after the lookback window changes', async () => {
+  const harness = await mount(async ({ dryRun } = {}) => summary({ dryRun: dryRun === true }));
+
+  try {
+    await act(async () => {
+      findButton(harness.container, 'Scan').click();
+    });
+    await act(async () => {
+      findButton(harness.container, 'Clean Up').click();
+    });
+    await act(async () => {
+      findButton(harness.container, '7 days').click();
+    });
+    await act(async () => {
+      findButton(harness.container, 'Close').click();
+    });
+
+    assert.equal(harness.cleanedCalls(), 1);
+  } finally {
+    harness.teardown();
+  }
+});
+
+test('closing is refused while an apply is in flight', async () => {
+  const pending = deferred<StatsDuplicateLineCleanupResult>();
+  const harness = await mount(async ({ dryRun } = {}) =>
+    dryRun === true ? summary({ dryRun: true }) : pending.promise,
+  );
+
+  try {
+    await act(async () => {
+      findButton(harness.container, 'Scan').click();
+    });
+    await act(async () => {
+      findButton(harness.container, 'Clean Up').click();
+    });
+
+    assert.equal(findButton(harness.container, 'Close').disabled, true);
+    // The backdrop is never disabled, so this is the path that has to be refused.
+    await act(async () => {
+      findBackdrop(harness.container).click();
+    });
+    assert.equal(harness.closedCalls(), 0, 'the modal must stay open mid-apply');
+    assert.equal(harness.cleanedCalls(), 0);
+
+    await act(async () => {
+      pending.resolve(summary({ removedLines: 12 }));
+      await pending.promise;
+    });
+    await act(async () => {
+      findButton(harness.container, 'Close').click();
+    });
+
+    assert.equal(harness.closedCalls(), 1);
+    assert.equal(harness.cleanedCalls(), 1);
+  } finally {
+    harness.teardown();
+  }
+});
+
+test('a scan on its own owes no reload', async () => {
+  const harness = await mount(async ({ dryRun } = {}) => summary({ dryRun: dryRun === true }));
+
+  try {
+    await act(async () => {
+      findButton(harness.container, 'Scan').click();
+    });
+    await act(async () => {
+      findButton(harness.container, 'Close').click();
+    });
+
+    assert.equal(harness.cleanedCalls(), 0);
+    assert.equal(harness.closedCalls(), 1);
+  } finally {
+    harness.teardown();
+  }
+});
+
+test('an apply that removes nothing owes no reload', async () => {
+  // The scan saw work to do, but by the time it ran another cleanup had taken it.
+  const harness = await mount(async ({ dryRun } = {}) =>
+    dryRun === true ? summary({ dryRun: true }) : summary({ burstGroups: 0, removedLines: 0 }),
+  );
+
+  try {
+    await act(async () => {
+      findButton(harness.container, 'Scan').click();
+    });
+    await act(async () => {
+      findButton(harness.container, 'Clean Up').click();
+    });
+    await act(async () => {
+      findButton(harness.container, 'Close').click();
+    });
+
+    assert.equal(harness.cleanedCalls(), 0);
+    assert.equal(harness.closedCalls(), 1);
+  } finally {
+    harness.teardown();
+  }
+});

stats/src/components/vocabulary/DuplicateLineCleanup.tsx

@@ -0,0 +1,202 @@
+import { useCallback, useState } from 'react';
+import { getStatsClient } from '../../hooks/useStatsApi';
+import { formatNumber } from '../../lib/formatters';
+import type { StatsDuplicateLineCleanupResult } from '../../types/stats';
+
+interface DuplicateLineCleanupProps {
+  onClose: () => void;
+  /** Called after rows are actually removed, so the charts can reload. */
+  onCleaned: () => void;
+}
+
+const LOOKBACK_CHOICES: Array<{ label: string; days: number | null }> = [
+  { label: '7 days', days: 7 },
+  { label: '30 days', days: 30 },
+  { label: '90 days', days: 90 },
+  { label: '1 year', days: 365 },
+  { label: 'All time', days: null },
+];
+
+function formatTimecode(ms: number): string {
+  const totalSeconds = Math.max(0, Math.floor(ms / 1000));
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  return `${minutes}:${String(seconds).padStart(2, '0')}`;
+}
+
+export function DuplicateLineCleanup({ onClose, onCleaned }: DuplicateLineCleanupProps) {
+  const [lookbackDays, setLookbackDays] = useState<number | null>(30);
+  const [preview, setPreview] = useState<StatsDuplicateLineCleanupResult | null>(null);
+  const [applied, setApplied] = useState<StatsDuplicateLineCleanupResult | null>(null);
+  const [busy, setBusy] = useState<'scan' | 'apply' | null>(null);
+  const [error, setError] = useState<string | null>(null);
+  // Survives everything the displayed result does not: another scan, a different window.
+  // Rows are gone from the moment an apply succeeds, so the reload is owed until it runs.
+  const [needsReload, setNeedsReload] = useState(false);
+
+  const run = useCallback(
+    async (dryRun: boolean) => {
+      setBusy(dryRun ? 'scan' : 'apply');
+      setError(null);
+      try {
+        const result = await getStatsClient().cleanupDuplicateLines({ dryRun, lookbackDays });
+        if (dryRun) {
+          setPreview(result);
+          setApplied(null);
+        } else {
+          setApplied(result);
+          setPreview(null);
+          if (result.removedLines > 0) {
+            setNeedsReload(true);
+          }
+        }
+      } catch (cause) {
+        setError(cause instanceof Error ? cause.message : String(cause));
+      } finally {
+        setBusy(null);
+      }
+    },
+    [lookbackDays],
+  );
+
+  // Reloading the vocabulary tables unmounts this modal along with the rest of the tab,
+  // so it waits for the user to close: they get to read what was removed first. Closing
+  // is refused mid-apply, which would drop the reload on the floor along with the report.
+  const close = useCallback(() => {
+    if (busy === 'apply') {
+      return;
+    }
+    if (needsReload) {
+      onCleaned();
+    }
+    onClose();
+  }, [busy, needsReload, onCleaned, onClose]);
+
+  const result = applied ?? preview;
+  const nothingToDo = preview !== null && preview.removedLines === 0;
+
+  return (
+    <div className="fixed inset-0 z-50">
+      <button
+        type="button"
+        aria-label="Close duplicate line cleanup"
+        className="absolute inset-0 bg-ctp-crust/70 backdrop-blur-[2px]"
+        onClick={close}
+      />
+      <div className="absolute inset-x-0 top-1/2 mx-auto max-w-xl -translate-y-1/2 rounded-xl border border-ctp-surface1 bg-ctp-mantle shadow-2xl">
+        <div className="flex items-center justify-between border-b border-ctp-surface1 px-5 py-4">
+          <h2 className="text-sm font-semibold text-ctp-text">Duplicate Lines</h2>
+          <button
+            type="button"
+            disabled={busy === 'apply'}
+            className="rounded-md border border-ctp-surface2 px-3 py-1.5 text-xs font-medium text-ctp-subtext0 transition hover:border-ctp-blue hover:text-ctp-blue disabled:opacity-50"
+            onClick={close}
+          >
+            Close
+          </button>
+        </div>
+
+        <div className="space-y-4 px-5 py-4">
+          <p className="text-xs leading-relaxed text-ctp-subtext0">
+            Typeset subtitles — karaoke openings, animated signs — are authored as one event per
+            animation frame, and older versions counted every frame as its own line. This finds
+            those runs and collapses each one back to a single line, giving back the word and kanji
+            counts they inflated. Ordinary repeated dialogue is left alone.
+          </p>
+
+          <div>
+            <div className="mb-2 text-xs font-medium text-ctp-subtext1">Look back over</div>
+            <div className="flex flex-wrap gap-2">
+              {LOOKBACK_CHOICES.map((choice) => (
+                <button
+                  key={choice.label}
+                  type="button"
+                  disabled={busy !== null}
+                  onClick={() => {
+                    setLookbackDays(choice.days);
+                    setPreview(null);
+                    setApplied(null);
+                  }}
+                  className={`rounded-lg border px-3 py-1.5 text-xs transition disabled:opacity-50 ${
+                    lookbackDays === choice.days
+                      ? 'border-ctp-blue/50 bg-ctp-surface2 text-ctp-text'
+                      : 'border-ctp-surface1 bg-ctp-surface0 text-ctp-overlay2 hover:text-ctp-subtext0'
+                  }`}
+                >
+                  {choice.label}
+                </button>
+              ))}
+            </div>
+          </div>
+
+          {error && (
+            <div className="rounded-lg border border-ctp-red/30 bg-ctp-red/10 px-3 py-2 text-xs text-ctp-red">
+              {error}
+            </div>
+          )}
+
+          {result && (
+            <div className="rounded-lg bg-ctp-surface0 px-4 py-3">
+              <div className="text-sm text-ctp-text">
+                {applied
+                  ? `Removed ${formatNumber(applied.removedLines)} repeated lines`
+                  : nothingToDo
+                    ? 'No animation bursts found in this window'
+                    : `Found ${formatNumber(preview!.burstGroups)} bursts covering ${formatNumber(preview!.removedLines)} extra lines`}
+              </div>
+              <div className="mt-1 text-xs text-ctp-overlay2">
+                {formatNumber(result.scannedLines)} lines scanned ·{' '}
+                {formatNumber(result.removedWordOccurrences)} word counts ·{' '}
+                {formatNumber(result.removedKanjiOccurrences)} kanji counts
+                {applied ? ' removed' : ' would be removed'}
+              </div>
+
+              {result.samples.length > 0 && (
+                <div className="mt-3 max-h-52 space-y-1.5 overflow-y-auto">
+                  {result.samples.map((sample) => (
+                    <div
+                      key={`${sample.videoId}:${sample.startMs}:${sample.text}`}
+                      className="flex items-center justify-between gap-3 rounded-md bg-ctp-mantle px-3 py-1.5"
+                    >
+                      <div className="min-w-0">
+                        <div className="truncate text-xs text-ctp-text">{sample.text}</div>
+                        <div className="truncate text-[11px] text-ctp-overlay1">
+                          {sample.videoTitle ?? `Video ${sample.videoId}`} ·{' '}
+                          {formatTimecode(sample.startMs)}
+                        </div>
+                      </div>
+                      <span className="shrink-0 text-xs text-ctp-peach">×{sample.frames}</span>
+                    </div>
+                  ))}
+                </div>
+              )}
+            </div>
+          )}
+
+          <div className="flex items-center justify-end gap-2">
+            <button
+              type="button"
+              disabled={busy !== null}
+              onClick={() => void run(true)}
+              className="rounded-md border border-ctp-surface2 px-3 py-1.5 text-xs font-medium text-ctp-subtext0 transition hover:border-ctp-blue hover:text-ctp-blue disabled:opacity-50"
+            >
+              {busy === 'scan' ? 'Scanning…' : 'Scan'}
+            </button>
+            <button
+              type="button"
+              disabled={busy !== null || preview === null || nothingToDo}
+              onClick={() => void run(false)}
+              className="rounded-md border border-ctp-red/30 px-3 py-1.5 text-xs font-medium text-ctp-red transition hover:bg-ctp-red/10 disabled:opacity-40"
+            >
+              {busy === 'apply' ? 'Cleaning…' : 'Clean Up'}
+            </button>
+          </div>
+          <p className="text-[11px] text-ctp-overlay1">
+            Scan first: cleanup removes rows and cannot be undone. Session watch time and lines-seen
+            totals are left untouched.
+          </p>
+        </div>
+      </div>
+    </div>
+  );
+}

stats/src/components/vocabulary/VocabularyTab.tsx

@@ -5,6 +5,7 @@ import { WordList } from './WordList';
 import { KanjiBreakdown } from './KanjiBreakdown';
 import { KanjiDetailPanel } from './KanjiDetailPanel';
 import { ExclusionManager } from './ExclusionManager';
+import { DuplicateLineCleanup } from './DuplicateLineCleanup';
 import { formatNumber } from '../../lib/formatters';
 import { TrendChart } from '../trends/TrendChart';
 import { FrequencyRankTable } from './FrequencyRankTable';
@@ -34,10 +35,11 @@ export function VocabularyTab({
   onRemoveExclusion,
   onClearExclusions,
 }: VocabularyTabProps) {
-  const { words, kanji, knownWords, loading, error } = useVocabulary();
+  const { words, kanji, knownWords, loading, error, reload } = useVocabulary();
   const [selectedKanjiId, setSelectedKanjiId] = useState<number | null>(null);
   const [hideNames, setHideNames] = useState(false);
   const [showExclusionManager, setShowExclusionManager] = useState(false);
+  const [showDuplicateLineCleanup, setShowDuplicateLineCleanup] = useState(false);
 
   const hasNames = useMemo(() => words.some(isProperNoun), [words]);
   const filteredWords = useMemo(() => {
@@ -129,6 +131,13 @@ export function VocabularyTab({
             Hide Names
           </button>
         )}
+        <button
+          type="button"
+          onClick={() => setShowDuplicateLineCleanup(true)}
+          className="shrink-0 rounded-lg border border-ctp-surface1 bg-ctp-surface0 px-3 py-2 text-xs text-ctp-overlay2 transition-colors hover:text-ctp-subtext0"
+        >
+          Duplicates
+        </button>
         <button
           type="button"
           onClick={() => setShowExclusionManager(true)}
@@ -193,6 +202,13 @@ export function VocabularyTab({
           onClose={() => setShowExclusionManager(false)}
         />
       )}
+
+      {showDuplicateLineCleanup && (
+        <DuplicateLineCleanup
+          onClose={() => setShowDuplicateLineCleanup(false)}
+          onCleaned={reload}
+        />
+      )}
     </div>
   );
 }

stats/src/hooks/useVocabulary.ts

@@ -1,4 +1,4 @@
-import { useState, useEffect } from 'react';
+import { useState, useEffect, useCallback } from 'react';
 import { getStatsClient } from './useStatsApi';
 import type { VocabularyEntry, KanjiEntry } from '../types/stats';
 
@@ -8,6 +8,9 @@ export function useVocabulary() {
   const [knownWords, setKnownWords] = useState<Set<string>>(new Set());
   const [loading, setLoading] = useState(true);
   const [error, setError] = useState<string | null>(null);
+  // Bumped by `reload` after maintenance rewrites the vocabulary tables.
+  const [reloadToken, setReloadToken] = useState(0);
+  const reload = useCallback(() => setReloadToken((token) => token + 1), []);
 
   useEffect(() => {
     let cancelled = false;
@@ -46,7 +49,7 @@ export function useVocabulary() {
     return () => {
       cancelled = true;
     };
-  }, []);
+  }, [reloadToken]);
 
-  return { words, kanji, knownWords, loading, error };
+  return { words, kanji, knownWords, loading, error, reload };
 }

stats/src/lib/api-client.ts

@@ -6,6 +6,8 @@ import type {
   StatsAnkiNotesInfoRequest,
   StatsCoverImagesRequest,
   StatsDeleteSessionsRequest,
+  StatsDuplicateLineCleanupRequest,
+  StatsDuplicateLineCleanupResult,
   StatsExcludedWordsRequest,
   StatsHttpClient,
   StatsJsonResponseMap,
@@ -102,6 +104,19 @@ export const apiClient = {
       body: JSON.stringify({ words } satisfies StatsExcludedWordsRequest),
     });
   },
+  cleanupDuplicateLines: async (
+    options: StatsDuplicateLineCleanupRequest = {},
+  ): Promise<StatsDuplicateLineCleanupResult> => {
+    const res = await fetchResponse('/api/stats/maintenance/duplicate-lines', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        dryRun: options.dryRun === true,
+        lookbackDays: options.lookbackDays ?? null,
+      } satisfies StatsDuplicateLineCleanupRequest),
+    });
+    return res.json() as Promise<StatsDuplicateLineCleanupResult>;
+  },
   getWordOccurrences: (headword: string, word: string, reading: string, limit = 50, offset = 0) =>
     fetchJson(
       'wordOccurrences',