Merge pull request #663 from pipecat-ai/aleix/audio-resampling-with-resampy

audio: use resamply for audio resampling

examples/foundational/07m-interruptible-aws.py

@@ -40,7 +40,6 @@ async def main():
             "Respond bot",
             DailyParams(
                 audio_out_enabled=True,
-                audio_out_sample_rate=16000,
                 vad_enabled=True,
                 vad_analyzer=SileroVADAnalyzer(),
                 vad_audio_passthrough=True,

pyproject.toml

@@ -28,7 +28,7 @@ dependencies = [
     "protobuf~=4.25.4",
     "pydantic~=2.8.2",
     "pyloudnorm~=0.1.1",
-    "scipy~=1.14.1",
+    "resampy~=0.4.3",
 ]
 
 [project.urls]

src/pipecat/audio/utils.py

@@ -7,15 +7,14 @@
 import audioop
 import numpy as np
 import pyloudnorm as pyln
-from scipy import signal
+import resampy
 
 
 def resample_audio(audio: bytes, original_rate: int, target_rate: int) -> bytes:
     if original_rate == target_rate:
         return audio
     audio_data = np.frombuffer(audio, dtype=np.int16)
-    num_samples = int(len(audio) * target_rate / original_rate)
-    resampled_audio = signal.resample(audio_data, num_samples)
+    resampled_audio = resampy.resample(audio_data, original_rate, target_rate)
     return resampled_audio.astype(np.int16).tobytes()
 
 

src/pipecat/services/aws.py

@@ -9,6 +9,7 @@ from typing import AsyncGenerator, Optional
 from loguru import logger
 from pydantic import BaseModel
 
+from pipecat.audio.utils import resample_audio
 from pipecat.frames.frames import (
     ErrorFrame,
     Frame,
@@ -45,7 +46,7 @@ class AWSTTSService(TTSService):
         aws_access_key_id: str,
         region: str,
         voice_id: str = "Joanna",
-        sample_rate: int = 16000,
+        sample_rate: int = 24000,
         params: InputParams = InputParams(),
         **kwargs,
     ):
@@ -178,7 +179,8 @@ class AWSTTSService(TTSService):
                 "OutputFormat": "pcm",
                 "VoiceId": self._voice_id,
                 "Engine": self._settings["engine"],
-                "SampleRate": str(self._settings["sample_rate"]),
+                # AWS only supports 8000 and 16000 for PCM. We select 16000.
+                "SampleRate": "16000",
             }
 
             # Filter out None values
@@ -198,7 +200,8 @@ class AWSTTSService(TTSService):
                         chunk = audio_data[i : i + chunk_size]
                         if len(chunk) > 0:
                             await self.stop_ttfb_metrics()
-                            frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
+                            resampled = resample_audio(chunk, 16000, self._settings["sample_rate"])
+                            frame = TTSAudioRawFrame(resampled, self._settings["sample_rate"], 1)
                             yield frame
 
             yield TTSStoppedFrame()

src/pipecat/services/xtts.py

@@ -151,19 +151,19 @@ class XTTSService(TTSService):
             async for chunk in r.content.iter_chunked(1024):
                 if len(chunk) > 0:
                     await self.stop_ttfb_metrics()
-                    # Append new chunk to the buffer
+                    # Append new chunk to the buffer.
                     buffer.extend(chunk)
 
-                    # Check if buffer has enough data for processing
+                    # Check if buffer has enough data for processing.
                     while (
                         len(buffer) >= 48000
                     ):  # Assuming at least 0.5 seconds of audio data at 24000 Hz
-                        # Process the buffer up to a safe size for resampling
+                        # Process the buffer up to a safe size for resampling.
                         process_data = buffer[:48000]
-                        # Remove processed data from buffer
+                        # Remove processed data from buffer.
                         buffer = buffer[48000:]
 
-                        # Resample the audio from 24000 Hz
+                        # XTTS uses 24000 so we need to resample to our desired rate.
                         resampled_audio = resample_audio(
                             bytes(process_data), 24000, self._sample_rate
                         )
@@ -171,7 +171,7 @@ class XTTSService(TTSService):
                         frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1)
                         yield frame
 
-            # Process any remaining data in the buffer
+            # Process any remaining data in the buffer.
             if len(buffer) > 0:
                 resampled_audio = resample_audio(bytes(buffer), 24000, self._sample_rate)
                 frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1)

test-requirements.txt

@@ -22,7 +22,7 @@ pydantic~=2.8.2
 pyloudnorm~=0.1.1
 pyht~=0.1.4
 python-dotenv~=1.0.1
-scipy~=1.14.1
+resampy~=0.4.3
 silero-vad~=5.1
 together~=1.2.7
 transformers~=4.44.0