fix(controller): pick up macOS presses while mpv is in front (#321)

src/main.ts

@@ -35,6 +35,11 @@ import { createDiscordRpcClient } from './main/runtime/discord-rpc-client.js';
 import { startAppControlServer } from './main/runtime/app-control-server';
 import { getAppControlSocketPath } from './shared/app-control';
 import { focusMacOSOverlayWindow } from './main/runtime/macos-overlay-window-focus';
+import {
+  handleBackgroundControllerPress,
+  resolveMacOSControllerWatchCommand,
+  startMacOSControllerWatch,
+} from './main/runtime/macos-controller-watch';
 import { restoreMacOSMpvFocusAfterModalClose } from './main/runtime/macos-modal-focus-handoff';
 import {
   getDefaultPasswordStore,
@@ -2417,3 +2422,37 @@ registerSubtitleGenerationIpc({
   openModal: () => openSubtitleGenerationModal(overlayModals.createOverlayHostedModalOpenDeps()),
 });
 app.on('before-quit', () => subtitleGenerationRuntime.cancel());
+
+if (process.platform === 'darwin') {
+  void app.whenReady().then(() => {
+    const command = resolveMacOSControllerWatchCommand({
+      resourcesPath: process.resourcesPath,
+      mainDirname: __dirname,
+    });
+    if (!command) {
+      logger.warn('macOS controller watch helper not found; controller needs overlay focus');
+      return;
+    }
+    const controllerWatch = startMacOSControllerWatch({
+      command,
+      logger,
+      onPress: (press) =>
+        handleBackgroundControllerPress(press, {
+          isControllerEnabled: () => configService.getConfig().controller.enabled,
+          isMpvFocused: () => appState.windowTracker?.isTargetWindowFocused() ?? false,
+          getModalWindow: () => overlayManager.getModalWindow(),
+          getMainWindow: () => overlayManager.getMainWindow(),
+          focusWindow: (window) =>
+            focusMacOSOverlayWindow({
+              platform: process.platform,
+              getOverlayWindow: () => window,
+              stealAppFocus: () => app.focus({ steal: true }),
+              warn: (message, details) => logger.warn(message, details),
+            }),
+          replayPress: (window, index) =>
+            window.webContents.send(IPC_CHANNELS.event.controllerBackgroundPress, index),
+        }),
+    });
+    app.on('will-quit', () => controllerWatch.stop());
+  });
+}

src/main/runtime/macos-controller-watch.test.ts

@@ -0,0 +1,96 @@
+import assert from 'node:assert/strict';
+import path from 'node:path';
+import test from 'node:test';
+import {
+  handleBackgroundControllerPress,
+  parseControllerWatchLine,
+  resolveMacOSControllerWatchCommand,
+} from './macos-controller-watch';
+
+test('parseControllerWatchLine reads press lines only', () => {
+  assert.deepEqual(parseControllerWatchLine('press 7 2\n'), { buttonIndex: 7, controllerCount: 2 });
+  assert.equal(parseControllerWatchLine('press 7'), null);
+  assert.equal(parseControllerWatchLine('release 7 1'), null);
+});
+
+function fakeFs(files: Record<string, number>) {
+  return {
+    existsSync: (filePath: string) => filePath in files,
+    statSync: (filePath: string) => ({ mtimeMs: files[filePath] }),
+  } as unknown as Parameters<typeof resolveMacOSControllerWatchCommand>[1];
+}
+
+test('resolveMacOSControllerWatchCommand prefers the packaged binary', () => {
+  const packaged = path.join('/res', 'scripts', 'macos-controller-watch');
+  assert.deepEqual(
+    resolveMacOSControllerWatchCommand(
+      { resourcesPath: '/res', mainDirname: '/repo/dist' },
+      fakeFs({ [packaged]: 1 }),
+    ),
+    { command: packaged, args: [] },
+  );
+});
+
+test('resolveMacOSControllerWatchCommand runs the source when the build is stale', () => {
+  const built = path.join('/repo/dist', 'scripts', 'macos-controller-watch');
+  const source = path.join('/repo', 'scripts', 'macos-controller-watch.swift');
+  const options = { resourcesPath: undefined, mainDirname: '/repo/dist' };
+
+  assert.deepEqual(
+    resolveMacOSControllerWatchCommand(options, fakeFs({ [built]: 2, [source]: 1 })),
+    {
+      command: built,
+      args: [],
+    },
+  );
+  assert.deepEqual(
+    resolveMacOSControllerWatchCommand(options, fakeFs({ [built]: 1, [source]: 2 })),
+    {
+      command: 'swift',
+      args: [source],
+    },
+  );
+  assert.equal(resolveMacOSControllerWatchCommand(options, fakeFs({})), null);
+});
+
+function window(name: string, visible = true) {
+  return { name, isDestroyed: () => false, isVisible: () => visible };
+}
+
+function run(overrides: {
+  controllerCount?: number;
+  enabled?: boolean;
+  mpvFocused?: boolean;
+  modal?: ReturnType<typeof window> | null;
+  main?: ReturnType<typeof window> | null;
+}): string[] {
+  const calls: string[] = [];
+  const press = { buttonIndex: 4, controllerCount: overrides.controllerCount ?? 1 };
+  handleBackgroundControllerPress(press, {
+    isControllerEnabled: () => overrides.enabled ?? true,
+    isMpvFocused: () => overrides.mpvFocused ?? true,
+    getModalWindow: () => overrides.modal ?? null,
+    getMainWindow: () => (overrides.main === undefined ? window('main') : overrides.main),
+    focusWindow: (target) => calls.push(`focus:${target.name}`),
+    replayPress: (target, index) => calls.push(`replay:${target.name}:${index}`),
+  });
+  return calls;
+}
+
+test('a press while mpv is in front focuses the overlay and replays it', () => {
+  assert.deepEqual(run({}), ['focus:main', 'replay:main:4']);
+});
+
+test('with several pads connected a press focuses the overlay without replaying', () => {
+  assert.deepEqual(run({ controllerCount: 2 }), ['focus:main']);
+});
+
+test('an open modal window takes the press instead of the main overlay', () => {
+  assert.deepEqual(run({ modal: window('modal') }), ['focus:modal', 'replay:modal:4']);
+});
+
+test('presses are ignored unless mpv is in front, the controller is on and an overlay shows', () => {
+  assert.deepEqual(run({ mpvFocused: false }), []);
+  assert.deepEqual(run({ enabled: false }), []);
+  assert.deepEqual(run({ main: window('main', false) }), []);
+});

src/main/runtime/macos-controller-watch.ts

@@ -0,0 +1,120 @@
+import { spawn as spawnProcess } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as readline from 'node:readline';
+import { isCompiledMacOSHelperCurrent } from '../../window-trackers/macos-tracker';
+
+// macOS only delivers GameController input to the frontmost app, so the overlay's Gamepad API
+// sees nothing while mpv is in front. `scripts/macos-controller-watch.swift` opts in to
+// background events and reports each button press; SubMiner then focuses the overlay (after
+// which Chromium reads the pad itself) and replays that first press there.
+
+const HELPER_NAME = 'macos-controller-watch';
+
+export type MacOSControllerWatchCommand = { command: string; args: string[] };
+
+/**
+ * Finds the helper: the packaged binary in Resources, a compiled build in dist/scripts that is
+ * newer than its source, or the Swift source run through `swift` in a dev checkout.
+ */
+export function resolveMacOSControllerWatchCommand(
+  options: {
+    resourcesPath: string | undefined;
+    /** Directory holding the built main process (dist/ in a checkout). */
+    mainDirname: string;
+  },
+  helperFs: Pick<typeof fs, 'existsSync' | 'statSync'> = fs,
+): MacOSControllerWatchCommand | null {
+  const sourcePath = path.join(options.mainDirname, '..', 'scripts', `${HELPER_NAME}.swift`);
+  if (options.resourcesPath) {
+    const packagedPath = path.join(options.resourcesPath, 'scripts', HELPER_NAME);
+    if (helperFs.existsSync(packagedPath)) return { command: packagedPath, args: [] };
+  }
+  const builtPath = path.join(options.mainDirname, 'scripts', HELPER_NAME);
+  if (isCompiledMacOSHelperCurrent(builtPath, sourcePath, helperFs)) {
+    return { command: builtPath, args: [] };
+  }
+  if (helperFs.existsSync(sourcePath)) return { command: 'swift', args: [sourcePath] };
+  return null;
+}
+
+export type ControllerWatchPress = { buttonIndex: number; controllerCount: number };
+
+/** Parses a helper line ("press <standard-mapping button index> <connected pads>"). */
+export function parseControllerWatchLine(line: string): ControllerWatchPress | null {
+  const match = /^press (\d+) (\d+)$/.exec(line.trim());
+  return match ? { buttonIndex: Number(match[1]), controllerCount: Number(match[2]) } : null;
+}
+
+type ControllerWatchLogger = {
+  info: (message: string, ...args: unknown[]) => void;
+  warn: (message: string, ...args: unknown[]) => void;
+};
+
+/** Runs the helper until `stop` is called; the helper also exits when SubMiner does. */
+export function startMacOSControllerWatch(options: {
+  command: MacOSControllerWatchCommand;
+  onPress: (press: ControllerWatchPress) => void;
+  logger: ControllerWatchLogger;
+  spawn?: typeof spawnProcess;
+}): { stop: () => void } {
+  const spawn = options.spawn ?? spawnProcess;
+  const child = spawn(options.command.command, options.command.args, {
+    stdio: ['pipe', 'pipe', 'pipe'],
+  });
+  let stopped = false;
+
+  readline.createInterface({ input: child.stdout }).on('line', (line) => {
+    const press = parseControllerWatchLine(line);
+    if (press) options.onPress(press);
+  });
+  readline.createInterface({ input: child.stderr }).on('line', (line) => {
+    options.logger.warn(`[${HELPER_NAME}] ${line}`);
+  });
+  child.on('error', (error) => {
+    options.logger.warn(`Failed to start ${HELPER_NAME}`, error);
+  });
+  child.on('exit', (code, signal) => {
+    if (!stopped) options.logger.warn(`${HELPER_NAME} exited`, { code, signal });
+  });
+  options.logger.info(`Started ${HELPER_NAME}: ${options.command.command}`);
+
+  return {
+    stop: () => {
+      stopped = true;
+      child.stdin?.end();
+      child.kill();
+    },
+  };
+}
+
+type FocusableWindow = {
+  isDestroyed: () => boolean;
+  isVisible: () => boolean;
+};
+
+/**
+ * A press the overlay missed because mpv was in front: focus the overlay window that owns the
+ * controller (a visible modal, else the main overlay) and replay the press in it. Presses made
+ * while another app is in front are left alone, so a pad used elsewhere is not hijacked. With
+ * several pads connected the press only focuses: it might not be from the pad the overlay reads.
+ */
+export function handleBackgroundControllerPress<TWindow extends FocusableWindow>(
+  press: ControllerWatchPress,
+  deps: {
+    isControllerEnabled: () => boolean;
+    isMpvFocused: () => boolean;
+    getModalWindow: () => TWindow | null;
+    getMainWindow: () => TWindow | null;
+    focusWindow: (window: TWindow) => void;
+    replayPress: (window: TWindow, buttonIndex: number) => void;
+  },
+): void {
+  if (!deps.isControllerEnabled() || !deps.isMpvFocused()) return;
+  const target = [deps.getModalWindow(), deps.getMainWindow()].find(
+    (window): window is TWindow => window !== null && !window.isDestroyed() && window.isVisible(),
+  );
+  if (!target) return;
+  deps.focusWindow(target);
+  if (press.controllerCount === 1) deps.replayPress(target, press.buttonIndex);
+}

src/preload.ts

@@ -197,6 +197,10 @@ const onPlaybackPauseChangedEvent = createQueuedIpcListenerWithPayload<boolean>(
 const onControllerConfigChangedEvent = createQueuedIpcListener(
   IPC_CHANNELS.event.controllerConfigChanged,
 );
+const onControllerBackgroundPressEvent = createQueuedIpcListenerWithPayload<number>(
+  IPC_CHANNELS.event.controllerBackgroundPress,
+  (payload) => (typeof payload === 'number' && Number.isInteger(payload) ? payload : -1),
+);
 const onKeyboardModeToggleRequestedEvent = createQueuedIpcListener(
   IPC_CHANNELS.event.keyboardModeToggleRequested,
 );
@@ -509,6 +513,7 @@ const electronAPI: ElectronAPI = {
   onKeyboardModeToggleRequested: onKeyboardModeToggleRequestedEvent,
   onControllerSuspendedChanged: onControllerSuspendedChangedEvent,
   onControllerConfigChanged: onControllerConfigChangedEvent,
+  onControllerBackgroundPress: onControllerBackgroundPressEvent,
   onPlaybackPauseChanged: onPlaybackPauseChangedEvent,
   onLookupWindowToggleRequested: onLookupWindowToggleRequestedEvent,
   getPlaylistBrowserSnapshot: (): Promise<PlaylistBrowserSnapshot> =>

src/renderer/handlers/controller-press-replay.test.ts

@@ -0,0 +1,107 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { createControllerPressReplay } from './controller-press-replay';
+import type { GamepadLike } from './gamepad-controller';
+
+function pad(pressed: number[] = [], mapping = 'standard'): GamepadLike {
+  return {
+    id: 'pad',
+    index: 0,
+    connected: true,
+    mapping,
+    axes: [0, 0, 0, 0],
+    buttons: Array.from({ length: 17 }, (_, index) => ({
+      value: pressed.includes(index) ? 1 : 0,
+      pressed: pressed.includes(index),
+    })),
+  };
+}
+
+const pressedIndices = (gamepad: GamepadLike | null | undefined): number[] =>
+  gamepad?.buttons.flatMap((button, index) => (button.pressed ? [index] : [])) ?? [];
+
+test('a queued press reads as pressed for exactly one poll', () => {
+  const replay = createControllerPressReplay();
+  replay.queue(0, 1000);
+
+  assert.deepEqual(pressedIndices(replay.apply([pad()], 1010)[0]), [0]);
+  assert.deepEqual(pressedIndices(replay.apply([pad()], 1026)[0]), []);
+});
+
+test('a press the Gamepad API already reported is not replayed', () => {
+  const replay = createControllerPressReplay();
+  replay.apply([pad([1])], 990);
+  replay.queue(1, 1000);
+
+  assert.deepEqual(pressedIndices(replay.apply([pad()], 1010)[0]), []);
+});
+
+test('a press seen after the replay was queued is not replayed again', () => {
+  const replay = createControllerPressReplay();
+  replay.queue(2, 1000);
+
+  assert.deepEqual(pressedIndices(replay.apply([pad([2])], 1010)[0]), [2]);
+  assert.deepEqual(pressedIndices(replay.apply([pad()], 1026)[0]), []);
+});
+
+test('an older press of the same button does not block the replay', () => {
+  const replay = createControllerPressReplay();
+  replay.apply([pad([3])], 0);
+  replay.queue(3, 5000);
+
+  assert.deepEqual(pressedIndices(replay.apply([pad()], 5010)[0]), [3]);
+});
+
+test('a replayed native pad keeps its fields', () => {
+  // Like the browser's Gamepad, these fields are prototype getters that object spread skips.
+  class NativePad implements GamepadLike {
+    get id() {
+      return 'pad';
+    }
+    get index() {
+      return 2;
+    }
+    get connected() {
+      return true;
+    }
+    get mapping() {
+      return 'standard';
+    }
+    get timestamp() {
+      return 900;
+    }
+    get axes() {
+      return [0.5, 0];
+    }
+    get buttons() {
+      return pad().buttons;
+    }
+  }
+  const replay = createControllerPressReplay();
+  replay.queue(0, 1000);
+
+  const replayed = replay.apply([new NativePad()], 1010)[0];
+  assert.deepEqual(pressedIndices(replayed), [0]);
+  assert.equal(replayed?.id, 'pad');
+  assert.equal(replayed?.index, 2);
+  assert.equal(replayed?.connected, true);
+  assert.equal(replayed?.mapping, 'standard');
+  assert.equal(replayed?.timestamp, 900);
+  assert.deepEqual(replayed?.axes, [0.5, 0]);
+});
+
+test('nothing is replayed while several pads are connected', () => {
+  const replay = createControllerPressReplay();
+  replay.queue(0, 1000);
+
+  const pads = replay.apply([pad(), { ...pad(), index: 1 }], 1010);
+  assert.deepEqual(pads.map(pressedIndices), [[], []]);
+  assert.deepEqual(pressedIndices(replay.apply([pad()], 1026)[0]), []);
+});
+
+test('pads without the standard mapping are left alone', () => {
+  const replay = createControllerPressReplay();
+  replay.queue(0, 1000);
+
+  assert.deepEqual(pressedIndices(replay.apply([pad([], '')], 1010)[0]), []);
+});

src/renderer/handlers/controller-press-replay.ts

@@ -0,0 +1,75 @@
+import type { GamepadLike } from './gamepad-controller.js';
+
+/**
+ * How far back a press the Gamepad API reported itself still counts as the same press the
+ * main process is replaying. Covers the helper -> main -> renderer IPC hop.
+ */
+const SAME_PRESS_WINDOW_MS = 500;
+
+/**
+ * Replays macOS background presses (see `src/main/runtime/macos-controller-watch.ts`) into the
+ * controller poll. A queued button reads as pressed for one poll, so the controller sees a
+ * normal press edge; the next poll reads the real pad again, so a button still held does not
+ * fire twice. A press the Gamepad API already reported is dropped.
+ *
+ * Only a lone standard-mapping pad gets the replay: the press carries no pad identity, so with
+ * several visible it could run an action on the wrong one. With none visible yet, or a pad
+ * mapped differently, the press only served to focus the overlay.
+ */
+export function createControllerPressReplay() {
+  const pendingPresses: Array<{ buttonIndex: number; queuedAt: number }> = [];
+  const lastRealPressAt = new Map<number, number>();
+
+  function recordRealPresses(gamepads: GamepadLike[], now: number): void {
+    for (const gamepad of gamepads) {
+      gamepad.buttons.forEach((button, buttonIndex) => {
+        if (button.pressed) lastRealPressAt.set(buttonIndex, now);
+      });
+    }
+  }
+
+  function withPressed(gamepad: GamepadLike, buttonIndices: number[]): GamepadLike {
+    const buttons = gamepad.buttons.map((button, index) =>
+      buttonIndices.includes(index) ? { value: 1, pressed: true, touched: true } : button,
+    );
+    // Native Gamepad fields are prototype getters, so object spread would drop them.
+    return {
+      id: gamepad.id,
+      index: gamepad.index,
+      connected: gamepad.connected,
+      mapping: gamepad.mapping,
+      timestamp: gamepad.timestamp,
+      axes: gamepad.axes,
+      buttons,
+    };
+  }
+
+  return {
+    queue(buttonIndex: number, now: number): void {
+      pendingPresses.push({ buttonIndex, queuedAt: now });
+    },
+
+    /** Called once per poll with the Gamepad API's pads; returns the pads to read. */
+    apply(gamepads: Array<GamepadLike | null>, now: number): Array<GamepadLike | null> {
+      const connected = gamepads.filter((gamepad): gamepad is GamepadLike =>
+        Boolean(gamepad?.connected),
+      );
+      recordRealPresses(connected, now);
+      if (pendingPresses.length === 0) return gamepads;
+
+      const replayed = pendingPresses
+        .filter(({ buttonIndex, queuedAt }) => {
+          const seenAt = lastRealPressAt.get(buttonIndex);
+          return seenAt === undefined || seenAt < queuedAt - SAME_PRESS_WINDOW_MS;
+        })
+        .map(({ buttonIndex }) => buttonIndex);
+      pendingPresses.length = 0;
+      const onlyPad = connected.length === 1 ? connected[0] : undefined;
+      if (replayed.length === 0 || onlyPad?.mapping !== 'standard') return gamepads;
+
+      return gamepads.map((gamepad) =>
+        gamepad === onlyPad ? withPressed(onlyPad, replayed) : gamepad,
+      );
+    },
+  };
+}

src/renderer/renderer.ts

@@ -29,6 +29,7 @@ import type {
 import type { OverlayHostedModal } from '../shared/ipc/contracts';
 import { createKeyboardHandlers } from './handlers/keyboard.js';
 import { createGamepadController } from './handlers/gamepad-controller.js';
+import { createControllerPressReplay } from './handlers/controller-press-replay.js';
 import { createMouseHandlers } from './handlers/mouse.js';
 import { createControllerStatusIndicator } from './controller-status-indicator.js';
 import { createControllerDebugModal } from './modals/controller-debug.js';
@@ -394,6 +395,7 @@ let lastSubtitlePreview = '';
 let lastSecondarySubtitlePreview = '';
 let overlayErrorToastTimeout: ReturnType<typeof setTimeout> | null = null;
 let controllerAnimationFrameId: number | null = null;
+const controllerPressReplay = createControllerPressReplay();
 
 function truncateForErrorLog(text: string): string {
   const normalized = text.replace(/\s+/g, ' ').trim();
@@ -528,7 +530,8 @@ function startControllerPolling(): void {
   }
 
   const gamepadController = createGamepadController({
-    getGamepads: () => Array.from(navigator.getGamepads?.() ?? []),
+    getGamepads: () =>
+      controllerPressReplay.apply(Array.from(navigator.getGamepads?.() ?? []), performance.now()),
     getConfig: () =>
       withAltCSelectedController(
         ctx.state.controllerConfig ?? {
@@ -862,6 +865,11 @@ function registerKeyboardCommandHandlers(): void {
     keyboardModeAutoPause.handlePlaybackPauseChanged(paused);
   });
 
+  // macOS: replay a press made while mpv was in front; main has just focused this window.
+  window.electronAPI.onControllerBackgroundPress((buttonIndex) => {
+    if (buttonIndex >= 0) controllerPressReplay.queue(buttonIndex, performance.now());
+  });
+
   // Alt+C saves in its own window; pick up remaps and the button style without a restart.
   window.electronAPI.onControllerConfigChanged(() => {
     window.electronAPI

src/shared/ipc/contracts.ts

@@ -162,6 +162,7 @@ export const IPC_CHANNELS = {
     keyboardModeToggleRequested: 'keyboard-mode-toggle:requested',
     controllerSuspendedChanged: 'controller:suspended-changed',
     controllerConfigChanged: 'controller:config-changed',
+    controllerBackgroundPress: 'controller:background-press',
     playbackPauseChanged: 'mpv:pause-changed',
     lookupWindowToggleRequested: 'lookup-window-toggle:requested',
     sessionHelpOpen: 'session-help:open',

src/types/runtime.ts

@@ -695,6 +695,8 @@ export interface ElectronAPI {
   onControllerSuspendedChanged: (callback: (suspended: boolean) => void) => void;
   /** Alt+C saved controller settings; reload them with `getControllerConfig`. */
   onControllerConfigChanged: (callback: () => void) => void;
+  /** macOS: a button pressed while mpv was in front, which the Gamepad API never saw. */
+  onControllerBackgroundPress: (callback: (buttonIndex: number) => void) => void;
   /** mpv's pause property changed, from any source. */
   onPlaybackPauseChanged: (callback: (paused: boolean) => void) => void;
   onLookupWindowToggleRequested: (callback: () => void) => void;