From a3038afa023b4a609fb62dffec21f4da3e780078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 7 May 2025 11:39:36 -0700 Subject: [PATCH 1/2] DailyTransport: fix multiple audio/video sources --- CHANGELOG.md | 3 +++ src/pipecat/transports/services/daily.py | 17 +++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 319dce632..16da15420 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `DailyTransport` issue that was causing issues when multiple audio or + video sources where being captured. + - Fixed a `UltravoxSTTService` issue that would cause the service to generate all tokens as one word. diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 5d00e76bc..9118b2107 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -700,7 +700,7 @@ class DailyTransportClient(EventHandler): await self.update_subscriptions(participant_settings={participant_id: media}) - self._audio_renderers[participant_id] = {audio_source: callback} + self._audio_renderers.setdefault(participant_id, {})[audio_source] = callback self._client.set_audio_renderer( participant_id, @@ -724,7 +724,7 @@ class DailyTransportClient(EventHandler): await self.update_subscriptions(participant_settings={participant_id: media}) - self._video_renderers[participant_id] = {video_source: callback} + self._video_renderers.setdefault(participant_id, {})[video_source] = callback self._client.set_video_renderer( participant_id, @@ -1061,12 +1061,13 @@ class DailyInputTransport(BaseInputTransport): video_source: str = "camera", color_format: str = "RGB", ): - self._video_renderers[participant_id] = { - video_source: { - "framerate": framerate, - "timestamp": 0, - "render_next_frame": [], - } + if participant_id not in self._video_renderers: + self._video_renderers[participant_id] = {} + + self._video_renderers[participant_id][video_source] = { + "framerate": framerate, + "timestamp": 0, + "render_next_frame": [], } await self._client.capture_participant_video( From ed00f7d071973ac64bce555b0ff20aff323eede4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 7 May 2025 11:42:16 -0700 Subject: [PATCH 2/2] add video_source field to UserImageRequestFrame --- CHANGELOG.md | 3 +++ src/pipecat/frames/frames.py | 3 ++- src/pipecat/services/llm_service.py | 2 ++ src/pipecat/transports/services/daily.py | 3 ++- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16da15420..3d2132d8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `UserImageRequestFrame.video_source` field has been added to request an image + from the desired video source. + - Added support for the AWS Nova Sonic speech-to-speech model with the new `AWSNovaSonicLLMService`. See https://docs.aws.amazon.com/nova/latest/userguide/speech.html. diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 05f5b666d..8d3f38459 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -715,9 +715,10 @@ class UserImageRequestFrame(SystemFrame): context: Optional[Any] = None function_name: Optional[str] = None tool_call_id: Optional[str] = None + video_source: Optional[str] = None def __str__(self): - return f"{self.name}(user: {self.user_id}, function: {self.function_name}, request: {self.tool_call_id})" + return f"{self.name}(user: {self.user_id}, video_source: {self.video_source}, function: {self.function_name}, request: {self.tool_call_id})" @dataclass diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index 15b2bd6e5..21b62325d 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -190,6 +190,7 @@ class LLMService(AIService): function_name: Optional[str] = None, tool_call_id: Optional[str] = None, text_content: Optional[str] = None, + video_source: Optional[str] = None, ): await self.push_frame( UserImageRequestFrame( @@ -197,6 +198,7 @@ class LLMService(AIService): function_name=function_name, tool_call_id=tool_call_id, context=text_content, + video_source=video_source, ), FrameDirection.UPSTREAM, ) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 9118b2107..f1a514d0e 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -1076,7 +1076,8 @@ class DailyInputTransport(BaseInputTransport): async def request_participant_image(self, frame: UserImageRequestFrame): if frame.user_id in self._video_renderers: - self._video_renderers[frame.user_id]["render_next_frame"].append(frame) + video_source = frame.video_source if frame.video_source else "camera" + self._video_renderers[frame.user_id][video_source]["render_next_frame"].append(frame) async def _on_participant_video_frame( self, participant_id: str, video_frame: VideoFrame, video_source: str