fix: unwrap mpv youtube streams for anki media mining

src/anki-integration.ts

@@ -58,6 +58,7 @@ import { NoteUpdateWorkflow } from './anki-integration/note-update-workflow';
 import { FieldGroupingWorkflow } from './anki-integration/field-grouping-workflow';
 import { resolveAnimatedImageLeadInSeconds } from './anki-integration/animated-image-sync';
 import { AnkiIntegrationRuntime, normalizeAnkiIntegrationConfig } from './anki-integration/runtime';
+import { resolveMediaGenerationInputPath } from './anki-integration/media-source';
 
 const log = createLogger('anki').child('integration');
 
@@ -597,6 +598,10 @@ export class AnkiIntegration {
     this.runtime.start();
   }
 
+  waitUntilReady(): Promise<void> {
+    return this.runtime.waitUntilReady();
+  }
+
   stop(): void {
     this.runtime.stop();
   }
@@ -647,7 +652,10 @@ export class AnkiIntegration {
       return null;
     }
 
-    const videoPath = mpvClient.currentVideoPath;
+    const videoPath = await resolveMediaGenerationInputPath(mpvClient, 'audio');
+    if (!videoPath) {
+      return null;
+    }
     let startTime = mpvClient.currentSubStart;
     let endTime = mpvClient.currentSubEnd;
 
@@ -672,7 +680,10 @@ export class AnkiIntegration {
       return null;
     }
 
-    const videoPath = this.mpvClient.currentVideoPath;
+    const videoPath = await resolveMediaGenerationInputPath(this.mpvClient, 'video');
+    if (!videoPath) {
+      return null;
+    }
     const timestamp = this.mpvClient.currentTimePos || 0;
 
     if (this.config.media?.imageType === 'avif') {
@@ -946,8 +957,15 @@ export class AnkiIntegration {
       if (this.mpvClient && this.mpvClient.currentVideoPath) {
         try {
           const timestamp = this.mpvClient.currentTimePos || 0;
+          const notificationIconSource = await resolveMediaGenerationInputPath(
+            this.mpvClient,
+            'video',
+          );
+          if (!notificationIconSource) {
+            throw new Error('No media source available for notification icon');
+          }
           const iconBuffer = await this.mediaGenerator.generateNotificationIcon(
-            this.mpvClient.currentVideoPath,
+            notificationIconSource,
             timestamp,
           );
           if (iconBuffer && iconBuffer.length > 0) {

src/anki-integration/card-creation.test.ts

@@ -283,3 +283,117 @@ test('CardCreationService keeps updating after recordCardsMinedCallback throws',
   assert.equal(calls.notesInfo, 1);
   assert.equal(calls.updateNoteFields, 1);
 });
+
+test('CardCreationService uses stream-open-filename for remote media generation', async () => {
+  const audioPaths: string[] = [];
+  const imagePaths: string[] = [];
+  const edlSource = [
+    'edl://!new_stream;!no_clip;!no_chapters;%70%https://audio.example/videoplayback?mime=audio%2Fwebm',
+    '!new_stream;!no_clip;!no_chapters;%69%https://video.example/videoplayback?mime=video%2Fmp4',
+    '!global_tags,title=test',
+  ].join(';');
+
+  const service = new CardCreationService({
+    getConfig: () =>
+      ({
+        deck: 'Mining',
+        fields: {
+          sentence: 'Sentence',
+          audio: 'SentenceAudio',
+          image: 'Picture',
+        },
+        media: {
+          generateAudio: true,
+          generateImage: true,
+          imageFormat: 'jpg',
+        },
+        behavior: {},
+        ai: false,
+      }) as AnkiConnectConfig,
+    getAiConfig: () => ({}),
+    getTimingTracker: () => ({}) as never,
+    getMpvClient: () =>
+      ({
+        currentVideoPath: 'https://www.youtube.com/watch?v=abc123',
+        currentSubText: '字幕',
+        currentSubStart: 1,
+        currentSubEnd: 2,
+        currentTimePos: 1.5,
+        currentAudioStreamIndex: 0,
+        requestProperty: async (name: string) => {
+          assert.equal(name, 'stream-open-filename');
+          return edlSource;
+        },
+      }) as never,
+    client: {
+      addNote: async () => 42,
+      addTags: async () => undefined,
+      notesInfo: async () => [
+        {
+          noteId: 42,
+          fields: {
+            Sentence: { value: '' },
+            SentenceAudio: { value: '' },
+            Picture: { value: '' },
+          },
+        },
+      ],
+      updateNoteFields: async () => undefined,
+      storeMediaFile: async () => undefined,
+      findNotes: async () => [],
+      retrieveMediaFile: async () => '',
+    },
+    mediaGenerator: {
+      generateAudio: async (path) => {
+        audioPaths.push(path);
+        return Buffer.from('audio');
+      },
+      generateScreenshot: async (path) => {
+        imagePaths.push(path);
+        return Buffer.from('image');
+      },
+      generateAnimatedImage: async () => null,
+    },
+    showOsdNotification: () => undefined,
+    showUpdateResult: () => undefined,
+    showStatusNotification: () => undefined,
+    showNotification: async () => undefined,
+    beginUpdateProgress: () => undefined,
+    endUpdateProgress: () => undefined,
+    withUpdateProgress: async (_message, action) => action(),
+    resolveConfiguredFieldName: (noteInfo, preferredName) => {
+      if (!preferredName) return null;
+      return Object.keys(noteInfo.fields).find((field) => field === preferredName) ?? null;
+    },
+    resolveNoteFieldName: (noteInfo, preferredName) => {
+      if (!preferredName) return null;
+      return Object.keys(noteInfo.fields).find((field) => field === preferredName) ?? null;
+    },
+    getAnimatedImageLeadInSeconds: async () => 0,
+    extractFields: () => ({}),
+    processSentence: (sentence) => sentence,
+    setCardTypeFields: () => undefined,
+    mergeFieldValue: (_existing, newValue) => newValue,
+    formatMiscInfoPattern: () => '',
+    getEffectiveSentenceCardConfig: () => ({
+      model: 'Sentence',
+      sentenceField: 'Sentence',
+      audioField: 'SentenceAudio',
+      lapisEnabled: false,
+      kikuEnabled: false,
+      kikuFieldGrouping: 'disabled',
+      kikuDeleteDuplicateInAuto: false,
+    }),
+    getFallbackDurationSeconds: () => 10,
+    appendKnownWordsFromNoteInfo: () => undefined,
+    isUpdateInProgress: () => false,
+    setUpdateInProgress: () => undefined,
+    trackLastAddedNoteId: () => undefined,
+  });
+
+  const created = await service.createSentenceCard('テスト', 0, 1);
+
+  assert.equal(created, true);
+  assert.deepEqual(audioPaths, ['https://audio.example/videoplayback?mime=audio%2Fwebm']);
+  assert.deepEqual(imagePaths, ['https://video.example/videoplayback?mime=video%2Fmp4']);
+});

src/anki-integration/card-creation.ts

@@ -8,6 +8,7 @@ import { createLogger } from '../logger';
 import { SubtitleTimingTracker } from '../subtitle-timing-tracker';
 import { MpvClient } from '../types';
 import { resolveSentenceBackText } from './ai';
+import { resolveMediaGenerationInputPath } from './media-source';
 
 const log = createLogger('anki').child('integration.card-creation');
 
@@ -501,7 +502,12 @@ export class CardCreationService {
     this.deps.showOsdNotification('Creating sentence card...');
     try {
       return await this.deps.withUpdateProgress('Creating sentence card', async () => {
-        const videoPath = mpvClient.currentVideoPath;
+        const videoPath = await resolveMediaGenerationInputPath(mpvClient, 'video');
+        const audioSourcePath = await resolveMediaGenerationInputPath(mpvClient, 'audio');
+        if (!videoPath) {
+          this.deps.showOsdNotification('No video loaded');
+          return false;
+        }
         const fields: Record<string, string> = {};
         const errors: string[] = [];
         let miscInfoFilename: string | null = null;
@@ -605,7 +611,9 @@ export class CardCreationService {
 
         try {
           const audioFilename = this.generateAudioFilename();
-          const audioBuffer = await this.mediaGenerateAudio(videoPath, startTime, endTime);
+          const audioBuffer = audioSourcePath
+            ? await this.mediaGenerateAudio(audioSourcePath, startTime, endTime)
+            : null;
 
           if (audioBuffer) {
             await this.deps.client.storeMediaFile(audioFilename, audioBuffer);

src/anki-integration/media-source.test.ts

@@ -0,0 +1,64 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { resolveMediaGenerationInputPath } from './media-source';
+
+test('resolveMediaGenerationInputPath keeps local file paths', async () => {
+  const result = await resolveMediaGenerationInputPath({
+    currentVideoPath: '/tmp/video.mkv',
+  });
+
+  assert.equal(result, '/tmp/video.mkv');
+});
+
+test('resolveMediaGenerationInputPath prefers stream-open-filename for remote media', async () => {
+  const requests: string[] = [];
+
+  const result = await resolveMediaGenerationInputPath({
+    currentVideoPath: 'https://www.youtube.com/watch?v=abc123',
+    requestProperty: async (name: string) => {
+      requests.push(name);
+      return 'https://rr1---sn.example.googlevideo.com/videoplayback?id=123';
+    },
+  });
+
+  assert.equal(result, 'https://rr1---sn.example.googlevideo.com/videoplayback?id=123');
+  assert.deepEqual(requests, ['stream-open-filename']);
+});
+
+test('resolveMediaGenerationInputPath unwraps mpv edl source for audio and video', async () => {
+  const edlSource = [
+    'edl://!new_stream;!no_clip;!no_chapters;%70%https://audio.example/videoplayback?mime=audio%2Fwebm',
+    '!new_stream;!no_clip;!no_chapters;%69%https://video.example/videoplayback?mime=video%2Fmp4',
+    '!global_tags,title=test',
+  ].join(';');
+
+  const audioResult = await resolveMediaGenerationInputPath(
+    {
+      currentVideoPath: 'https://www.youtube.com/watch?v=abc123',
+      requestProperty: async () => edlSource,
+    },
+    'audio',
+  );
+  const videoResult = await resolveMediaGenerationInputPath(
+    {
+      currentVideoPath: 'https://www.youtube.com/watch?v=abc123',
+      requestProperty: async () => edlSource,
+    },
+    'video',
+  );
+
+  assert.equal(audioResult, 'https://audio.example/videoplayback?mime=audio%2Fwebm');
+  assert.equal(videoResult, 'https://video.example/videoplayback?mime=video%2Fmp4');
+});
+
+test('resolveMediaGenerationInputPath falls back to currentVideoPath when stream-open-filename fails', async () => {
+  const result = await resolveMediaGenerationInputPath({
+    currentVideoPath: 'https://www.youtube.com/watch?v=abc123',
+    requestProperty: async () => {
+      throw new Error('property unavailable');
+    },
+  });
+
+  assert.equal(result, 'https://www.youtube.com/watch?v=abc123');
+});

src/anki-integration/media-source.ts

@@ -0,0 +1,82 @@
+import { isRemoteMediaPath } from '../jimaku/utils';
+import type { MpvClient } from '../types';
+
+export type MediaGenerationKind = 'audio' | 'video';
+
+function trimToNonEmptyString(value: unknown): string | null {
+  if (typeof value !== 'string') {
+    return null;
+  }
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+function extractUrlsFromMpvEdlSource(source: string): string[] {
+  const matches = source.matchAll(/%\d+%(https?:\/\/.*?)(?=;!new_stream|;!global_tags|$)/gms);
+  return [...matches]
+    .map((match) => trimToNonEmptyString(match[1]))
+    .filter((value): value is string => value !== null);
+}
+
+function classifyMediaUrl(url: string): MediaGenerationKind | null {
+  try {
+    const mime = new URL(url).searchParams.get('mime')?.toLowerCase() ?? '';
+    if (mime.startsWith('audio/')) {
+      return 'audio';
+    }
+    if (mime.startsWith('video/')) {
+      return 'video';
+    }
+  } catch {
+    // Ignore malformed URLs and fall back to stream order.
+  }
+
+  return null;
+}
+
+function resolvePreferredUrlFromMpvEdlSource(
+  source: string,
+  kind: MediaGenerationKind,
+): string | null {
+  const urls = extractUrlsFromMpvEdlSource(source);
+  if (urls.length === 0) {
+    return null;
+  }
+
+  const typedMatch = urls.find((url) => classifyMediaUrl(url) === kind);
+  if (typedMatch) {
+    return typedMatch;
+  }
+
+  return kind === 'audio' ? urls[0] ?? null : urls[urls.length - 1] ?? null;
+}
+
+export async function resolveMediaGenerationInputPath(
+  mpvClient: Pick<MpvClient, 'currentVideoPath' | 'requestProperty'> | null | undefined,
+  kind: MediaGenerationKind = 'video',
+): Promise<string | null> {
+  const currentVideoPath = trimToNonEmptyString(mpvClient?.currentVideoPath);
+  if (!currentVideoPath) {
+    return null;
+  }
+
+  if (!isRemoteMediaPath(currentVideoPath) || !mpvClient?.requestProperty) {
+    return currentVideoPath;
+  }
+
+  try {
+    const streamOpenFilename = trimToNonEmptyString(
+      await mpvClient.requestProperty('stream-open-filename'),
+    );
+    if (streamOpenFilename?.startsWith('edl://')) {
+      return resolvePreferredUrlFromMpvEdlSource(streamOpenFilename, kind) ?? streamOpenFilename;
+    }
+    if (streamOpenFilename) {
+      return streamOpenFilename;
+    }
+  } catch {
+    // Fall back to the current path when mpv does not expose a resolved stream URL.
+  }
+
+  return currentVideoPath;
+}