Merge pull request #4104 from pipecat-ai/filipi/audio_issue

Allow defining whether to insert silence in the output transport.

src/pipecat/services/tavus/video.py

@@ -214,7 +214,9 @@ class TavusVideoService(AIService):
         await super().start(frame)
         await self._client.start(frame)
         if self._transport_destination:
-            await self._client.register_audio_destination(self._transport_destination)
+            await self._client.register_audio_destination(
+                self._transport_destination, auto_silence=False
+            )
         await self._create_send_task()
 
     async def stop(self, frame: EndFrame):

src/pipecat/transports/base_transport.py

@@ -84,6 +84,8 @@ class TransportParams(BaseModel):
         audio_out_mixer: Audio mixer instance or destination mapping.
         audio_out_destinations: List of audio output destination identifiers.
         audio_out_end_silence_secs: How much silence to send after an EndFrame (0 for no silence).
+        audio_out_auto_silence: Insert silence frames when the audio output queue is empty.
+            When False, the transport will wait for audio data instead of inserting silence.
         audio_in_enabled: Enable audio input streaming.
         audio_in_sample_rate: Input audio sample rate in Hz.
         audio_in_channels: Number of input audio channels.
@@ -144,6 +146,7 @@ class TransportParams(BaseModel):
     audio_out_mixer: Optional[BaseAudioMixer | Mapping[Optional[str], BaseAudioMixer]] = None
     audio_out_destinations: List[str] = Field(default_factory=list)
     audio_out_end_silence_secs: int = 2
+    audio_out_auto_silence: bool = True
     audio_in_enabled: bool = False
     audio_in_sample_rate: Optional[int] = None
     audio_in_channels: int = 1

src/pipecat/transports/daily/transport.py

@@ -676,15 +676,19 @@ class DailyTransportClient(EventHandler):
             await asyncio.sleep(0.01)
             return None
 
-    async def register_audio_destination(self, destination: str):
+    async def register_audio_destination(
+        self, destination: str, auto_silence: Optional[bool] = True
+    ):
         """Register a custom audio destination for multi-track output.
 
         Args:
             destination: The destination identifier to register.
+            auto_silence: If True, the audio source inserts silence when no audio is available.
+                If False, the source waits for audio data. Defaults to True.
         """
         params = (self._params.custom_audio_track_params or {}).get(destination)
         self._custom_audio_tracks[destination] = await self.add_custom_audio_track(
-            destination, params=params
+            destination, params=params, auto_silence=auto_silence
         )
         publishing: Dict[str, Any] = {"customAudio": {destination: True}}
         if params and params.send_settings:
@@ -831,7 +835,14 @@ class DailyTransportClient(EventHandler):
             self._camera_track = DailyVideoTrack(source=video_source, track=video_track)
 
         if self._params.audio_out_enabled and not self._microphone_track:
-            audio_source = CustomAudioSource(self._out_sample_rate, self._params.audio_out_channels)
+            logger.debug(
+                f"Creating custom audio source, auto silence {self._params.audio_out_auto_silence}"
+            )
+            audio_source = CustomAudioSource(
+                self._out_sample_rate,
+                self._params.audio_out_channels,
+                self._params.audio_out_auto_silence,
+            )
             audio_track = CustomAudioTrack(audio_source)
             self._microphone_track = DailyAudioTrack(source=audio_source, track=audio_track)
 
@@ -1269,12 +1280,15 @@ class DailyTransportClient(EventHandler):
         self,
         track_name: str,
         params: Optional[DailyCustomAudioTrackParams] = None,
+        auto_silence: Optional[bool] = True,
     ) -> DailyAudioTrack:
         """Add a custom audio track for multi-stream output.
 
         Args:
             track_name: Name for the custom audio track.
             params: Optional per-track configuration for sample rate, channels, and sendSettings.
+            auto_silence: If True, the audio source inserts silence when no audio is available.
+                If False, the source waits for audio data. Defaults to True.
 
         Returns:
             The created DailyAudioTrack instance.
@@ -1284,7 +1298,7 @@ class DailyTransportClient(EventHandler):
         sample_rate = params.sample_rate if params and params.sample_rate else self._out_sample_rate
         channels = params.channels if params else 1
 
-        audio_source = CustomAudioSource(sample_rate, channels)
+        audio_source = CustomAudioSource(sample_rate, channels, auto_silence)
 
         audio_track = CustomAudioTrack(audio_source)
 

src/pipecat/transports/smallwebrtc/transport.py

@@ -79,14 +79,17 @@ class RawAudioTrack(AudioStreamTrack):
     supporting queued audio data with proper synchronization.
     """
 
-    def __init__(self, sample_rate):
+    def __init__(self, sample_rate: int, auto_silence: bool = True):
         """Initialize the raw audio track.
 
         Args:
             sample_rate: The audio sample rate in Hz.
+            auto_silence: If True, emit silence when the queue is empty. If False,
+                wait until audio data is available.
         """
         super().__init__()
         self._sample_rate = sample_rate
+        self._auto_silence = auto_silence
         self._samples_per_10ms = sample_rate * 10 // 1000
         self._bytes_per_10ms = self._samples_per_10ms * 2  # 16-bit (2 bytes per sample)
         self._timestamp = 0
@@ -123,7 +126,8 @@ class RawAudioTrack(AudioStreamTrack):
         """Return the next audio frame for WebRTC transmission.
 
         Returns:
-            An AudioFrame containing the next audio data or silence.
+            An AudioFrame containing the next audio data, or silence if the queue is empty
+            and ``auto_silence`` is True.
         """
         # Compute required wait time for synchronization
         if self._timestamp > 0:
@@ -131,12 +135,19 @@ class RawAudioTrack(AudioStreamTrack):
             if wait > 0:
                 await asyncio.sleep(wait)
 
-        if self._chunk_queue:
+        if not self._chunk_queue:
+            if self._auto_silence:
+                chunk = bytes(self._bytes_per_10ms)
+            else:
+                while not self._chunk_queue:
+                    await asyncio.sleep(0.005)
+                chunk, future = self._chunk_queue.popleft()
+                if future and not future.done():
+                    future.set_result(True)
+        else:
             chunk, future = self._chunk_queue.popleft()
             if future and not future.done():
                 future.set_result(True)
-        else:
-            chunk = bytes(self._bytes_per_10ms)  # silence
 
         # Convert the byte data to an ndarray of int16 samples
         samples = np.frombuffer(chunk, dtype=np.int16)
@@ -484,7 +495,10 @@ class SmallWebRTCClient:
         self._video_input_track = self._webrtc_connection.video_input_track()
         self._screen_video_track = self._webrtc_connection.screen_video_input_track()
         if self._params.audio_out_enabled:
-            self._audio_output_track = RawAudioTrack(sample_rate=self._out_sample_rate)
+            self._audio_output_track = RawAudioTrack(
+                sample_rate=self._out_sample_rate,
+                auto_silence=self._params.audio_out_auto_silence,
+            )
             self._webrtc_connection.replace_audio_track(self._audio_output_track)
 
         if self._params.video_out_enabled:

src/pipecat/transports/tavus/transport.py

@@ -417,16 +417,20 @@ class TavusTransportClient:
             return False
         return await self._client.write_audio_frame(frame)
 
-    async def register_audio_destination(self, destination: str):
+    async def register_audio_destination(
+        self, destination: str, auto_silence: Optional[bool] = True
+    ):
         """Register an audio destination for output.
 
         Args:
             destination: The destination identifier to register.
+            auto_silence: If True, the audio source inserts silence when no audio is available.
+                If False, the source waits for audio data. Defaults to True.
         """
         if not self._client:
             return
 
-        await self._client.register_audio_destination(destination)
+        await self._client.register_audio_destination(destination, auto_silence=auto_silence)
 
 
 class TavusInputTransport(BaseInputTransport):