From 901dd041f084af30bae412a8ba20484f9d81cb62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 8 Jun 2025 18:31:54 -0700 Subject: [PATCH] buffer audio from TTS service before pushing frames --- CHANGELOG.md | 3 +++ src/pipecat/services/aws/tts.py | 3 ++- src/pipecat/services/google/tts.py | 7 ++++--- src/pipecat/services/minimax/tts.py | 9 ++++----- src/pipecat/services/openai/tts.py | 2 +- src/pipecat/services/piper/tts.py | 3 +-- src/pipecat/services/rime/tts.py | 3 +-- src/pipecat/services/tts_service.py | 13 +++++++++++++ src/pipecat/services/xtts/tts.py | 2 +- tests/test_piper_tts.py | 5 +++-- 10 files changed, 33 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62fea69f2..6f8930f44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) ### Fixed +- Fixed an issue with various TTS services that would cause audio glitches at + the start of every bot turn. + - Fixed an `AssemblyAISTTService` issue that could cause unexpected behavior when yielding empty `Frame()`s. diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index a0719f227..0159096d1 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -253,7 +253,8 @@ class AWSPollyTTSService(TTSService): yield TTSStartedFrame() - CHUNK_SIZE = 1024 + CHUNK_SIZE = self.chunk_size + for i in range(0, len(audio_data), CHUNK_SIZE): chunk = audio_data[i : i + CHUNK_SIZE] if len(chunk) > 0: diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index e28f9fadb..6e57b7b8d 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -362,8 +362,8 @@ class GoogleHttpTTSService(TTSService): # Skip the first 44 bytes to remove the WAV header audio_content = response.audio_content[44:] - # Read and yield audio data in chunks - CHUNK_SIZE = 1024 + CHUNK_SIZE = self.chunk_size + for i in range(0, len(audio_content), CHUNK_SIZE): chunk = audio_content[i : i + CHUNK_SIZE] if not chunk: @@ -505,9 +505,10 @@ class GoogleTTSService(TTSService): yield TTSStartedFrame() audio_buffer = b"" - CHUNK_SIZE = 1024 first_chunk_for_ttfb = False + CHUNK_SIZE = self.chunk_size + async for response in streaming_responses: chunk = response.audio_content if not chunk: diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 932996751..86f954755 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -227,7 +227,8 @@ class MiniMaxHttpTTSService(TTSService): # Process the streaming response buffer = bytearray() - CHUNK_SIZE = 1024 + + CHUNK_SIZE = self.chunk_size async for chunk in response.content.iter_chunked(CHUNK_SIZE): if not chunk: @@ -279,10 +280,8 @@ class MiniMaxHttpTTSService(TTSService): await self.stop_ttfb_metrics() yield TTSAudioRawFrame( audio=audio_chunk, - sample_rate=self._settings["audio_setting"][ - "sample_rate" - ], - num_channels=self._settings["audio_setting"]["channel"], + sample_rate=self.sample_rate, + num_channels=1, ) except ValueError as e: logger.error(f"Error converting hex to binary: {e}") diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index 61fb3e77c..946d5e396 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -125,7 +125,7 @@ class OpenAITTSService(TTSService): await self.start_tts_usage_metrics(text) - CHUNK_SIZE = 1024 + CHUNK_SIZE = self.chunk_size yield TTSStartedFrame() async for chunk in r.iter_bytes(CHUNK_SIZE): diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py index 4b009b76e..65caa3650 100644 --- a/src/pipecat/services/piper/tts.py +++ b/src/pipecat/services/piper/tts.py @@ -85,8 +85,7 @@ class PiperTTSService(TTSService): await self.start_tts_usage_metrics(text) - # Process the streaming response - CHUNK_SIZE = 1024 + CHUNK_SIZE = self.chunk_size yield TTSStartedFrame() async for chunk in response.content.iter_chunked(CHUNK_SIZE): diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index b3f46961f..669f72a99 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -430,8 +430,7 @@ class RimeHttpTTSService(TTSService): yield TTSStartedFrame() - # Process the streaming response - CHUNK_SIZE = 1024 + CHUNK_SIZE = self.chunk_size async for chunk in response.content.iter_chunked(CHUNK_SIZE): if need_to_strip_wav_header and chunk.startswith(b"RIFF"): diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 0bdcd0d1c..904f603a9 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -106,6 +106,19 @@ class TTSService(AIService): def sample_rate(self) -> int: return self._sample_rate + @property + def chunk_size(self) -> int: + """This property indicates how much audio we download (from TTS services + that require chunking) before we start pushing the first audio + frame. This will make sure we download the rest of the audio while audio + is being played without causing audio glitches (specially at the + beginning). Of course, this will also depend on how fast the TTS service + generates bytes. + + """ + CHUNK_SECONDS = 0.5 + return int(self.sample_rate * CHUNK_SECONDS * 2) # 2 bytes/sample + async def set_model(self, model: str): self.set_model_name(model) diff --git a/src/pipecat/services/xtts/tts.py b/src/pipecat/services/xtts/tts.py index 18528f0ea..2111e4d72 100644 --- a/src/pipecat/services/xtts/tts.py +++ b/src/pipecat/services/xtts/tts.py @@ -152,7 +152,7 @@ class XTTSService(TTSService): yield TTSStartedFrame() - CHUNK_SIZE = 1024 + CHUNK_SIZE = self.chunk_size buffer = bytearray() async for chunk in r.content.iter_chunked(CHUNK_SIZE): diff --git a/tests/test_piper_tts.py b/tests/test_piper_tts.py index 673ea087a..75893f93f 100644 --- a/tests/test_piper_tts.py +++ b/tests/test_piper_tts.py @@ -47,8 +47,9 @@ async def test_run_piper_tts_success(aiohttp_client): # Write out some chunked byte data # In reality, you’d return WAV data or similar - data_chunk_1 = b"\x00\x01\x02\x03" * 1024 # 4096 bytes, 04 TTSAudioRawFrame - data_chunk_2 = b"\x04\x05\x06\x07" * 1024 # another chunk + CHUNK_SIZE = 24000 + data_chunk_1 = b"\x00\x01\x02\x03" * CHUNK_SIZE # 4xTTSAudioRawFrame + data_chunk_2 = b"\x04\x05\x06\x07" * CHUNK_SIZE # another chunk await resp.write(data_chunk_1) await asyncio.sleep(0.01) # simulate async chunk delay await resp.write(data_chunk_2)