Merge pull request #1979 from pipecat-ai/aleix/buffer-tts-before-playback

buffer audio from TTS service before pushing frames

CHANGELOG.md

@@ -20,6 +20,9 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 
 ### Fixed
 
+- Fixed an issue with various TTS services that would cause audio glitches at
+  the start of every bot turn.
+
 - Fixed an `AssemblyAISTTService` issue that could cause unexpected behavior
   when yielding empty `Frame()`s.
 

src/pipecat/services/aws/tts.py

@@ -253,7 +253,8 @@ class AWSPollyTTSService(TTSService):
 
             yield TTSStartedFrame()
 
-            CHUNK_SIZE = 1024
+            CHUNK_SIZE = self.chunk_size
+
             for i in range(0, len(audio_data), CHUNK_SIZE):
                 chunk = audio_data[i : i + CHUNK_SIZE]
                 if len(chunk) > 0:

src/pipecat/services/google/tts.py

@@ -362,8 +362,8 @@ class GoogleHttpTTSService(TTSService):
             # Skip the first 44 bytes to remove the WAV header
             audio_content = response.audio_content[44:]
 
-            # Read and yield audio data in chunks
+            CHUNK_SIZE = self.chunk_size
-            CHUNK_SIZE = 1024
+
             for i in range(0, len(audio_content), CHUNK_SIZE):
                 chunk = audio_content[i : i + CHUNK_SIZE]
                 if not chunk:
@@ -505,9 +505,10 @@ class GoogleTTSService(TTSService):
             yield TTSStartedFrame()
 
             audio_buffer = b""
-            CHUNK_SIZE = 1024
             first_chunk_for_ttfb = False
 
+            CHUNK_SIZE = self.chunk_size
+
             async for response in streaming_responses:
                 chunk = response.audio_content
                 if not chunk:

src/pipecat/services/minimax/tts.py

@@ -227,7 +227,8 @@ class MiniMaxHttpTTSService(TTSService):
 
                 # Process the streaming response
                 buffer = bytearray()
-                CHUNK_SIZE = 1024
+
+                CHUNK_SIZE = self.chunk_size
 
                 async for chunk in response.content.iter_chunked(CHUNK_SIZE):
                     if not chunk:
@@ -279,10 +280,8 @@ class MiniMaxHttpTTSService(TTSService):
                                         await self.stop_ttfb_metrics()
                                         yield TTSAudioRawFrame(
                                             audio=audio_chunk,
-                                            sample_rate=self._settings["audio_setting"][
+                                            sample_rate=self.sample_rate,
-                                                "sample_rate"
+                                            num_channels=1,
-                                            ],
-                                            num_channels=self._settings["audio_setting"]["channel"],
                                         )
                                 except ValueError as e:
                                     logger.error(f"Error converting hex to binary: {e}")

src/pipecat/services/openai/tts.py

@@ -125,7 +125,7 @@ class OpenAITTSService(TTSService):
 
                 await self.start_tts_usage_metrics(text)
 
-                CHUNK_SIZE = 1024
+                CHUNK_SIZE = self.chunk_size
 
                 yield TTSStartedFrame()
                 async for chunk in r.iter_bytes(CHUNK_SIZE):

src/pipecat/services/piper/tts.py

@@ -85,8 +85,7 @@ class PiperTTSService(TTSService):
 
                 await self.start_tts_usage_metrics(text)
 
-                # Process the streaming response
+                CHUNK_SIZE = self.chunk_size
-                CHUNK_SIZE = 1024
 
                 yield TTSStartedFrame()
                 async for chunk in response.content.iter_chunked(CHUNK_SIZE):

src/pipecat/services/rime/tts.py

@@ -430,8 +430,7 @@ class RimeHttpTTSService(TTSService):
 
                 yield TTSStartedFrame()
 
-                # Process the streaming response
+                CHUNK_SIZE = self.chunk_size
-                CHUNK_SIZE = 1024
 
                 async for chunk in response.content.iter_chunked(CHUNK_SIZE):
                     if need_to_strip_wav_header and chunk.startswith(b"RIFF"):

src/pipecat/services/tts_service.py

@@ -106,6 +106,19 @@ class TTSService(AIService):
     def sample_rate(self) -> int:
         return self._sample_rate
 
+    @property
+    def chunk_size(self) -> int:
+        """This property indicates how much audio we download (from TTS services
+        that require chunking) before we start pushing the first audio
+        frame. This will make sure we download the rest of the audio while audio
+        is being played without causing audio glitches (specially at the
+        beginning). Of course, this will also depend on how fast the TTS service
+        generates bytes.
+
+        """
+        CHUNK_SECONDS = 0.5
+        return int(self.sample_rate * CHUNK_SECONDS * 2)  # 2 bytes/sample
+
     async def set_model(self, model: str):
         self.set_model_name(model)
 

src/pipecat/services/xtts/tts.py

@@ -152,7 +152,7 @@ class XTTSService(TTSService):
 
             yield TTSStartedFrame()
 
-            CHUNK_SIZE = 1024
+            CHUNK_SIZE = self.chunk_size
 
             buffer = bytearray()
             async for chunk in r.content.iter_chunked(CHUNK_SIZE):

tests/test_piper_tts.py

@@ -47,8 +47,9 @@ async def test_run_piper_tts_success(aiohttp_client):
 
         # Write out some chunked byte data
         # In reality, you’d return WAV data or similar
-        data_chunk_1 = b"\x00\x01\x02\x03" * 1024  # 4096 bytes, 04 TTSAudioRawFrame
+        CHUNK_SIZE = 24000
-        data_chunk_2 = b"\x04\x05\x06\x07" * 1024  # another chunk
+        data_chunk_1 = b"\x00\x01\x02\x03" * CHUNK_SIZE  # 4xTTSAudioRawFrame
+        data_chunk_2 = b"\x04\x05\x06\x07" * CHUNK_SIZE  # another chunk
         await resp.write(data_chunk_1)
         await asyncio.sleep(0.01)  # simulate async chunk delay
         await resp.write(data_chunk_2)