diff --git a/changelog/4104.added.md b/changelog/4104.added.md new file mode 100644 index 000000000..31c6f3873 --- /dev/null +++ b/changelog/4104.added.md @@ -0,0 +1 @@ +- Added `audio_out_auto_silence` parameter to `TransportParams` (defaults to `True`). When set to `False`, the transport waits for audio data instead of inserting silence when the output queue is empty, which is useful for scenarios that require uninterrupted audio playback without artificial gaps. diff --git a/examples/foundational/21a-tavus-video-service.py b/examples/foundational/21a-tavus-video-service.py index 6e03a2418..5658951b5 100644 --- a/examples/foundational/21a-tavus-video-service.py +++ b/examples/foundational/21a-tavus-video-service.py @@ -42,6 +42,7 @@ transport_params = { video_out_is_live=True, video_out_width=1280, video_out_height=720, + audio_out_auto_silence=False, ), "webrtc": lambda: TransportParams( audio_in_enabled=True, @@ -50,6 +51,7 @@ transport_params = { video_out_is_live=True, video_out_width=1280, video_out_height=720, + audio_out_auto_silence=False, ), } diff --git a/pyproject.toml b/pyproject.toml index 91afcc794..03fe4ebd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ azure = [ "azure-cognitiveservices-speech>=1.47.0,<2"] cartesia = [ "pipecat-ai[websockets-base]" ] camb = [ "camb-sdk>=1.5.4,<2" ] cerebras = [] -daily = [ "daily-python~=0.25.0" ] +daily = [ "daily-python~=0.27.0" ] deepgram = [ "deepgram-sdk>=6.0.1,<7", "pipecat-ai[websockets-base]" ] deepseek = [] elevenlabs = [ "pipecat-ai[websockets-base]" ] diff --git a/src/pipecat/services/tavus/video.py b/src/pipecat/services/tavus/video.py index 9043710c8..a41bee5b4 100644 --- a/src/pipecat/services/tavus/video.py +++ b/src/pipecat/services/tavus/video.py @@ -214,7 +214,9 @@ class TavusVideoService(AIService): await super().start(frame) await self._client.start(frame) if self._transport_destination: - await self._client.register_audio_destination(self._transport_destination) + await self._client.register_audio_destination( + self._transport_destination, auto_silence=False + ) await self._create_send_task() async def stop(self, frame: EndFrame): diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py index a78e1046c..5b6c8673c 100644 --- a/src/pipecat/transports/base_transport.py +++ b/src/pipecat/transports/base_transport.py @@ -84,6 +84,8 @@ class TransportParams(BaseModel): audio_out_mixer: Audio mixer instance or destination mapping. audio_out_destinations: List of audio output destination identifiers. audio_out_end_silence_secs: How much silence to send after an EndFrame (0 for no silence). + audio_out_auto_silence: Insert silence frames when the audio output queue is empty. + When False, the transport will wait for audio data instead of inserting silence. audio_in_enabled: Enable audio input streaming. audio_in_sample_rate: Input audio sample rate in Hz. audio_in_channels: Number of input audio channels. @@ -144,6 +146,7 @@ class TransportParams(BaseModel): audio_out_mixer: Optional[BaseAudioMixer | Mapping[Optional[str], BaseAudioMixer]] = None audio_out_destinations: List[str] = Field(default_factory=list) audio_out_end_silence_secs: int = 2 + audio_out_auto_silence: bool = True audio_in_enabled: bool = False audio_in_sample_rate: Optional[int] = None audio_in_channels: int = 1 diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index 2798f6d5e..8ef94ae40 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -676,15 +676,19 @@ class DailyTransportClient(EventHandler): await asyncio.sleep(0.01) return None - async def register_audio_destination(self, destination: str): + async def register_audio_destination( + self, destination: str, auto_silence: Optional[bool] = True + ): """Register a custom audio destination for multi-track output. Args: destination: The destination identifier to register. + auto_silence: If True, the audio source inserts silence when no audio is available. + If False, the source waits for audio data. Defaults to True. """ params = (self._params.custom_audio_track_params or {}).get(destination) self._custom_audio_tracks[destination] = await self.add_custom_audio_track( - destination, params=params + destination, params=params, auto_silence=auto_silence ) publishing: Dict[str, Any] = {"customAudio": {destination: True}} if params and params.send_settings: @@ -831,7 +835,14 @@ class DailyTransportClient(EventHandler): self._camera_track = DailyVideoTrack(source=video_source, track=video_track) if self._params.audio_out_enabled and not self._microphone_track: - audio_source = CustomAudioSource(self._out_sample_rate, self._params.audio_out_channels) + logger.debug( + f"Creating custom audio source, auto silence {self._params.audio_out_auto_silence}" + ) + audio_source = CustomAudioSource( + self._out_sample_rate, + self._params.audio_out_channels, + self._params.audio_out_auto_silence, + ) audio_track = CustomAudioTrack(audio_source) self._microphone_track = DailyAudioTrack(source=audio_source, track=audio_track) @@ -1269,12 +1280,15 @@ class DailyTransportClient(EventHandler): self, track_name: str, params: Optional[DailyCustomAudioTrackParams] = None, + auto_silence: Optional[bool] = True, ) -> DailyAudioTrack: """Add a custom audio track for multi-stream output. Args: track_name: Name for the custom audio track. params: Optional per-track configuration for sample rate, channels, and sendSettings. + auto_silence: If True, the audio source inserts silence when no audio is available. + If False, the source waits for audio data. Defaults to True. Returns: The created DailyAudioTrack instance. @@ -1284,7 +1298,7 @@ class DailyTransportClient(EventHandler): sample_rate = params.sample_rate if params and params.sample_rate else self._out_sample_rate channels = params.channels if params else 1 - audio_source = CustomAudioSource(sample_rate, channels) + audio_source = CustomAudioSource(sample_rate, channels, auto_silence) audio_track = CustomAudioTrack(audio_source) diff --git a/src/pipecat/transports/smallwebrtc/transport.py b/src/pipecat/transports/smallwebrtc/transport.py index 36f883278..76ea34464 100644 --- a/src/pipecat/transports/smallwebrtc/transport.py +++ b/src/pipecat/transports/smallwebrtc/transport.py @@ -79,14 +79,17 @@ class RawAudioTrack(AudioStreamTrack): supporting queued audio data with proper synchronization. """ - def __init__(self, sample_rate): + def __init__(self, sample_rate: int, auto_silence: bool = True): """Initialize the raw audio track. Args: sample_rate: The audio sample rate in Hz. + auto_silence: If True, emit silence when the queue is empty. If False, + wait until audio data is available. """ super().__init__() self._sample_rate = sample_rate + self._auto_silence = auto_silence self._samples_per_10ms = sample_rate * 10 // 1000 self._bytes_per_10ms = self._samples_per_10ms * 2 # 16-bit (2 bytes per sample) self._timestamp = 0 @@ -123,7 +126,8 @@ class RawAudioTrack(AudioStreamTrack): """Return the next audio frame for WebRTC transmission. Returns: - An AudioFrame containing the next audio data or silence. + An AudioFrame containing the next audio data, or silence if the queue is empty + and ``auto_silence`` is True. """ # Compute required wait time for synchronization if self._timestamp > 0: @@ -131,12 +135,19 @@ class RawAudioTrack(AudioStreamTrack): if wait > 0: await asyncio.sleep(wait) - if self._chunk_queue: + if not self._chunk_queue: + if self._auto_silence: + chunk = bytes(self._bytes_per_10ms) + else: + while not self._chunk_queue: + await asyncio.sleep(0.005) + chunk, future = self._chunk_queue.popleft() + if future and not future.done(): + future.set_result(True) + else: chunk, future = self._chunk_queue.popleft() if future and not future.done(): future.set_result(True) - else: - chunk = bytes(self._bytes_per_10ms) # silence # Convert the byte data to an ndarray of int16 samples samples = np.frombuffer(chunk, dtype=np.int16) @@ -484,7 +495,10 @@ class SmallWebRTCClient: self._video_input_track = self._webrtc_connection.video_input_track() self._screen_video_track = self._webrtc_connection.screen_video_input_track() if self._params.audio_out_enabled: - self._audio_output_track = RawAudioTrack(sample_rate=self._out_sample_rate) + self._audio_output_track = RawAudioTrack( + sample_rate=self._out_sample_rate, + auto_silence=self._params.audio_out_auto_silence, + ) self._webrtc_connection.replace_audio_track(self._audio_output_track) if self._params.video_out_enabled: diff --git a/src/pipecat/transports/tavus/transport.py b/src/pipecat/transports/tavus/transport.py index 872e6eefc..426b7f72c 100644 --- a/src/pipecat/transports/tavus/transport.py +++ b/src/pipecat/transports/tavus/transport.py @@ -417,16 +417,20 @@ class TavusTransportClient: return False return await self._client.write_audio_frame(frame) - async def register_audio_destination(self, destination: str): + async def register_audio_destination( + self, destination: str, auto_silence: Optional[bool] = True + ): """Register an audio destination for output. Args: destination: The destination identifier to register. + auto_silence: If True, the audio source inserts silence when no audio is available. + If False, the source waits for audio data. Defaults to True. """ if not self._client: return - await self._client.register_audio_destination(destination) + await self._client.register_audio_destination(destination, auto_silence=auto_silence) class TavusInputTransport(BaseInputTransport): diff --git a/uv.lock b/uv.lock index 1263097a8..96b469aea 100644 --- a/uv.lock +++ b/uv.lock @@ -1402,13 +1402,13 @@ wheels = [ [[package]] name = "daily-python" -version = "0.25.0" +version = "0.27.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/58/074c6fca866fa13006b880eab521985d39300ea0d1df75a60d6dac4b2d47/daily_python-0.25.0-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:bff92b598863201bdffeea17819b7418d5c03a7ffc665a02be0333237fb3e4be", size = 13312157, upload-time = "2026-03-17T00:10:03.273Z" }, - { url = "https://files.pythonhosted.org/packages/ca/d1/0b623e8e6d06713e8d193335db1e5ec46760af276f08d8b378a0a8e4696b/daily_python-0.25.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:733c70e77bb1c8933a7fa779722592a109617b2d098c192399c2bfffcb210847", size = 11831685, upload-time = "2026-03-17T00:10:05.36Z" }, - { url = "https://files.pythonhosted.org/packages/e4/36/45c3a59e92a37e3a51dbe2603f9e855408ecdb48f1b4cd396de83839e6f9/daily_python-0.25.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a6da5f8c99ccbdb59042a4c5e48f9a85f68a32f65f16b3778da5f319fd6d7e17", size = 13865923, upload-time = "2026-03-17T00:10:07.391Z" }, - { url = "https://files.pythonhosted.org/packages/eb/5e/4574dedb8faa6a578079c89825f17c07a0bd4e1b4c2d2161966e62a6c6aa/daily_python-0.25.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0258ce43b92682379ecfcee1fa276799ccf41db25410f86395ae9927c2661cba", size = 14439736, upload-time = "2026-03-17T00:10:09.51Z" }, + { url = "https://files.pythonhosted.org/packages/6a/0c/57a46f95b58548503b4633ca1067e25cee401c6274b61446b340a891483d/daily_python-0.27.0-cp37-abi3-macosx_10_15_x86_64.whl", hash = "sha256:1a8143e33b024ee9bfa25646bb3263d41e11e2b99344cd214f9306ea47554d56", size = 13310124, upload-time = "2026-03-23T20:19:56.234Z" }, + { url = "https://files.pythonhosted.org/packages/7e/4a/e691c1d3f4682fc564e59781cea1fff6c5b850112dcdd78bc69c930f8bed/daily_python-0.27.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:dfc724d788b47c2d522fba7b37e38aaba00fd5a04ebbf964e5d52503ccadc1ff", size = 11834682, upload-time = "2026-03-23T20:19:58.487Z" }, + { url = "https://files.pythonhosted.org/packages/dd/ac/00adc5a83dc0474ab4ae9404157ea8435c99c52f41270e93b8a64b1e7ea8/daily_python-0.27.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:386671b415a5c31f5ffc9e473128ec228abc77d8cefd43cdc3ca0b5ca416122b", size = 13866110, upload-time = "2026-03-23T20:20:00.438Z" }, + { url = "https://files.pythonhosted.org/packages/f3/0a/f9162dccde2d0ad9dea1cfe43f02fc0abf7f535cd4172984855d66c27bc7/daily_python-0.27.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:740be060cc173bfb42b1f788f2c34f46a32769fe06ffc23931768e004b5dea85", size = 14412100, upload-time = "2026-03-23T20:20:02.304Z" }, ] [[package]] @@ -4810,7 +4810,7 @@ requires-dist = [ { name = "azure-cognitiveservices-speech", marker = "extra == 'azure'", specifier = ">=1.47.0,<2" }, { name = "camb-sdk", marker = "extra == 'camb'", specifier = ">=1.5.4,<2" }, { name = "coremltools", marker = "extra == 'local-smart-turn'", specifier = ">=8.0" }, - { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.25.0" }, + { name = "daily-python", marker = "extra == 'daily'", specifier = "~=0.27.0" }, { name = "deepgram-sdk", marker = "extra == 'deepgram'", specifier = ">=6.0.1,<7" }, { name = "docstring-parser", specifier = ">=0.16,<1" }, { name = "einops", marker = "extra == 'moondream'", specifier = "~=0.8.0" },