From ed00f7d071973ac64bce555b0ff20aff323eede4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 7 May 2025 11:42:16 -0700 Subject: [PATCH] add video_source field to UserImageRequestFrame --- CHANGELOG.md | 3 +++ src/pipecat/frames/frames.py | 3 ++- src/pipecat/services/llm_service.py | 2 ++ src/pipecat/transports/services/daily.py | 3 ++- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16da15420..3d2132d8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `UserImageRequestFrame.video_source` field has been added to request an image + from the desired video source. + - Added support for the AWS Nova Sonic speech-to-speech model with the new `AWSNovaSonicLLMService`. See https://docs.aws.amazon.com/nova/latest/userguide/speech.html. diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 05f5b666d..8d3f38459 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -715,9 +715,10 @@ class UserImageRequestFrame(SystemFrame): context: Optional[Any] = None function_name: Optional[str] = None tool_call_id: Optional[str] = None + video_source: Optional[str] = None def __str__(self): - return f"{self.name}(user: {self.user_id}, function: {self.function_name}, request: {self.tool_call_id})" + return f"{self.name}(user: {self.user_id}, video_source: {self.video_source}, function: {self.function_name}, request: {self.tool_call_id})" @dataclass diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index 15b2bd6e5..21b62325d 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -190,6 +190,7 @@ class LLMService(AIService): function_name: Optional[str] = None, tool_call_id: Optional[str] = None, text_content: Optional[str] = None, + video_source: Optional[str] = None, ): await self.push_frame( UserImageRequestFrame( @@ -197,6 +198,7 @@ class LLMService(AIService): function_name=function_name, tool_call_id=tool_call_id, context=text_content, + video_source=video_source, ), FrameDirection.UPSTREAM, ) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 9118b2107..f1a514d0e 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -1076,7 +1076,8 @@ class DailyInputTransport(BaseInputTransport): async def request_participant_image(self, frame: UserImageRequestFrame): if frame.user_id in self._video_renderers: - self._video_renderers[frame.user_id]["render_next_frame"].append(frame) + video_source = frame.video_source if frame.video_source else "camera" + self._video_renderers[frame.user_id][video_source]["render_next_frame"].append(frame) async def _on_participant_video_frame( self, participant_id: str, video_frame: VideoFrame, video_source: str