diff --git a/changelog/4370.added.2.md b/changelog/4370.added.2.md new file mode 100644 index 000000000..355a2ce40 --- /dev/null +++ b/changelog/4370.added.2.md @@ -0,0 +1,12 @@ +- Added `camera_out_send_settings` to `DailyParams`. This dict is passed verbatim to the Daily client's camera publishing settings, allowing applications to fully control encoding, codec, bitrate, and framerate. + + ```python + params = DailyParams( + camera_out_send_settings={ + "maxQuality": "high", + "encodings": { + "high": {"maxBitrate": 2_000_000, "maxFramerate": 30} + }, + }, + ) + ``` diff --git a/changelog/4370.added.md b/changelog/4370.added.md new file mode 100644 index 000000000..620ba721e --- /dev/null +++ b/changelog/4370.added.md @@ -0,0 +1,16 @@ +- Added support for Daily's built-in `screenVideo` destination in `DailyTransport`. When `"screenVideo"` is included in `video_out_destinations` transport parameter, a dedicated screen video track is created at join time and frames with `transport_destination="screenVideo"` are routed to it. + + ```python + params = DailyParams( + video_out_enabled=True, + video_out_is_live=True, + video_out_width=1280, + video_out_height=720, + video_out_destinations=["screenVideo"] + ) + + ... + + frame = OutputImageRawFrame(...) + frame.transport_destination = "screenVideo" + ``` diff --git a/changelog/4370.deprecated.md b/changelog/4370.deprecated.md new file mode 100644 index 000000000..6fea1c970 --- /dev/null +++ b/changelog/4370.deprecated.md @@ -0,0 +1 @@ +- Deprecated `TransportParams.video_out_bitrate` for the Daily transport. Use `DailyParams.camera_out_send_settings` instead to configure camera publishing encodings (bitrate, framerate, codec, etc.). diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 4e946bbb1..d485129f4 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -90,6 +90,18 @@ class BaseOutputTransport(FrameProcessor): # it. self._media_senders: dict[Any, BaseOutputTransport.MediaSender] = {} + if params.video_out_bitrate is not None: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Transport parameter `video_out_bitrate` is deprecated and will be removed in a future " + "version. Use provider specific settings instead.", + DeprecationWarning, + stacklevel=2, + ) + @property def sample_rate(self) -> int: """Get the current audio sample rate. diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py index 829fc6fe1..457844930 100644 --- a/src/pipecat/transports/base_transport.py +++ b/src/pipecat/transports/base_transport.py @@ -47,7 +47,7 @@ class TransportParams(BaseModel): video_out_is_live: Enable real-time video output streaming. video_out_width: Video output width in pixels. video_out_height: Video output height in pixels. - video_out_bitrate: Video output bitrate in bits per second. + video_out_bitrate: [DEPRECATED] Video output bitrate in bits per second. video_out_framerate: Video output frame rate in FPS. video_out_color_format: Video output color format string. video_out_codec: Preferred video codec for output (e.g., 'VP8', 'H264', 'H265'). @@ -76,7 +76,7 @@ class TransportParams(BaseModel): video_out_is_live: bool = False video_out_width: int = 1024 video_out_height: int = 768 - video_out_bitrate: int = 800000 + video_out_bitrate: int | None = None video_out_framerate: int = 30 video_out_color_format: str = "RGB" video_out_codec: str | None = None diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index 7084f1d15..2d30ceecb 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -325,6 +325,7 @@ class DailyParams(TransportParams): api_key: Daily API authentication key. audio_in_user_tracks: Receive users' audio in separate tracks camera_out_enabled: Whether to enable the main camera output track. + camera_out_send_settings: Camera output track publishing settings. custom_audio_track_params: Per-destination configuration for custom audio tracks. custom_video_track_params: Per-destination configuration for custom video tracks. dialin_settings: Optional settings for dial-in functionality. @@ -337,6 +338,7 @@ class DailyParams(TransportParams): api_key: str = "" audio_in_user_tracks: bool = True camera_out_enabled: bool = True + camera_out_send_settings: dict[str, Any] | None = None custom_audio_track_params: Mapping[str, DailyCustomAudioTrackParams] | None = None custom_video_track_params: Mapping[str, DailyCustomVideoTrackParams] | None = None dialin_settings: DailyDialinSettings | None = None @@ -555,6 +557,7 @@ class DailyTransportClient(EventHandler): self._camera_track: DailyVideoTrack | None = None self._microphone_track: DailyAudioTrack | None = None self._custom_audio_tracks: dict[str, DailyAudioTrack] = {} + # Custom video tracks will also include `screenVideo`. self._custom_video_tracks: dict[str, DailyVideoTrack] = {} def _speaker_name(self): @@ -666,19 +669,19 @@ class DailyTransportClient(EventHandler): self._client.update_publishing(publishing) async def register_video_destination(self, destination: str): - """Register a custom video destination for multi-track output. + """Register a video destination for multi-track output. + + Built-in destination ("camera") is configured at join time so it's + skipped here. Args: destination: The destination identifier to register. + """ - params = (self._params.custom_video_track_params or {}).get(destination) - self._custom_video_tracks[destination] = await self.add_custom_video_track( - destination, params=params - ) - publishing: dict[str, Any] = {"customVideo": {destination: True}} - if params and params.send_settings: - publishing["customVideo"][destination] = {"sendSettings": params.send_settings} - self._client.update_publishing(publishing) + if destination == "screenVideo": + await self._register_screen_video_destination() + else: + await self._register_custom_video_destination(destination) async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: """Write an audio frame to the appropriate audio track. @@ -719,7 +722,7 @@ class DailyTransportClient(EventHandler): """ destination = frame.transport_destination video_source: CustomVideoSource | None = None - if not destination and self._camera_track: + if (not destination or destination == "camera") and self._camera_track: video_source = self._camera_track.source elif destination and destination in self._custom_video_tracks: track = self._custom_video_tracks[destination] @@ -795,7 +798,11 @@ class DailyTransportClient(EventHandler): self._callback_task_handler(self._video_queue), f"{self}::video_callback_task", ) - if self._params.video_out_enabled and not self._camera_track: + if ( + self._params.video_out_enabled + and self._params.camera_out_enabled + and not self._camera_track + ): video_source = CustomVideoSource( self._params.video_out_width, self._params.video_out_height, @@ -804,7 +811,11 @@ class DailyTransportClient(EventHandler): video_track = CustomVideoTrack(video_source) self._camera_track = DailyVideoTrack(source=video_source, track=video_track) - if self._params.audio_out_enabled and not self._microphone_track: + if ( + self._params.audio_out_enabled + and self._params.microphone_out_enabled + and not self._microphone_track + ): logger.debug( f"Creating custom audio source, auto silence {self._params.audio_out_auto_silence}" ) @@ -899,28 +910,19 @@ class DailyTransportClient(EventHandler): }, "publishing": { "camera": { - "sendSettings": { - "maxQuality": "low", - **( - {"preferredCodec": self._params.video_out_codec} - if self._params.video_out_codec - else {} - ), - "encodings": { - "low": { - "maxBitrate": self._params.video_out_bitrate, - "maxFramerate": self._params.video_out_framerate, - } - }, - } + "isPublishing": camera_enabled, + "sendSettings": self._params.camera_out_send_settings + if self._params.camera_out_send_settings + else {}, }, "microphone": { + "isPublishing": microphone_enabled, "sendSettings": { "channelConfig": "stereo" if self._params.audio_out_channels == 2 else "mono", "bitrate": self._params.audio_out_bitrate, - } + }, }, }, }, @@ -1317,23 +1319,17 @@ class DailyTransportClient(EventHandler): """ future = self._get_event_loop().create_future() - width = params.width if params else self._params.video_out_width - height = params.height if params else self._params.video_out_height - color_format = params.color_format if params else self._params.video_out_color_format - - video_source = CustomVideoSource(width, height, color_format) - - video_track = CustomVideoTrack(video_source) + video_track = self._create_video_track(params) self._client.add_custom_video_track( track_name=track_name, - video_track=video_track, + video_track=video_track.track, completion=completion_callback(future), ) await future - return DailyVideoTrack(source=video_source, track=video_track) + return video_track async def remove_custom_video_track(self, track_name: str) -> CallClientError | None: """Remove a custom video track. @@ -1344,6 +1340,8 @@ class DailyTransportClient(EventHandler): Returns: error: An error description or None. """ + if track_name == "screenVideo": + return future = self._get_event_loop().create_future() self._client.remove_custom_video_track( track_name=track_name, @@ -1424,6 +1422,56 @@ class DailyTransportClient(EventHandler): ) return await future + async def _create_video_track( + self, + params: DailyCustomVideoTrackParams | None = None, + ) -> DailyVideoTrack: + """Create a video track for the given parameters.""" + future = self._get_event_loop().create_future() + + width = params.width if params else self._params.video_out_width + height = params.height if params else self._params.video_out_height + color_format = params.color_format if params else self._params.video_out_color_format + + video_source = CustomVideoSource(width, height, color_format) + + video_track = CustomVideoTrack(video_source) + + return DailyVideoTrack(source=video_source, track=video_track) + + async def _register_screen_video_destination(self): + """Register screen video destination track.""" + params = (self._params.custom_video_track_params or {}).get("screenVideo") + + video_track = await self._create_video_track(params) + self._custom_video_tracks["screenVideo"] = video_track + + # screenVideo inpupts settings. + inputs: dict[str, Any] = { + "screenVideo": { + "isEnabled": True, + "settings": {"customTrack": {"id": video_track.track.id}}, + } + } + self._client.update_inputs(inputs) + + # screenVideo publishing settings. + publishing: dict[str, Any] = {"screenVideo": True} + if params and params.send_settings: + publishing["screenVideo"] = {"sendSettings": params.send_settings} + self._client.update_publishing(publishing) + + async def _register_custom_video_destination(self, destination: str): + """Register a custom video destination for multi-track output.""" + params = (self._params.custom_video_track_params or {}).get(destination) + self._custom_video_tracks[destination] = await self.add_custom_video_track( + destination, params=params + ) + publishing: dict[str, Any] = {"customVideo": {destination: True}} + if params and params.send_settings: + publishing["customVideo"][destination] = {"sendSettings": params.send_settings} + self._client.update_publishing(publishing) + # # # Daily (EventHandler)