From 778b05a252a8e5a79816980ee06fc0edb5f29e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 25 Oct 2024 09:22:22 -0700 Subject: [PATCH] audio: use resamply for audio resampling --- examples/foundational/07m-interruptible-aws.py | 1 - pyproject.toml | 2 +- src/pipecat/audio/utils.py | 5 ++--- src/pipecat/services/aws.py | 9 ++++++--- src/pipecat/services/xtts.py | 12 ++++++------ test-requirements.txt | 2 +- 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/examples/foundational/07m-interruptible-aws.py b/examples/foundational/07m-interruptible-aws.py index 298c8bbd5..41d1f0530 100644 --- a/examples/foundational/07m-interruptible-aws.py +++ b/examples/foundational/07m-interruptible-aws.py @@ -40,7 +40,6 @@ async def main(): "Respond bot", DailyParams( audio_out_enabled=True, - audio_out_sample_rate=16000, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), vad_audio_passthrough=True, diff --git a/pyproject.toml b/pyproject.toml index 61de9bc3c..1d5630db3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "protobuf~=4.25.4", "pydantic~=2.8.2", "pyloudnorm~=0.1.1", - "scipy~=1.14.1", + "resampy~=0.4.3", ] [project.urls] diff --git a/src/pipecat/audio/utils.py b/src/pipecat/audio/utils.py index 42555661a..f2260c57b 100644 --- a/src/pipecat/audio/utils.py +++ b/src/pipecat/audio/utils.py @@ -7,15 +7,14 @@ import audioop import numpy as np import pyloudnorm as pyln -from scipy import signal +import resampy def resample_audio(audio: bytes, original_rate: int, target_rate: int) -> bytes: if original_rate == target_rate: return audio audio_data = np.frombuffer(audio, dtype=np.int16) - num_samples = int(len(audio) * target_rate / original_rate) - resampled_audio = signal.resample(audio_data, num_samples) + resampled_audio = resampy.resample(audio_data, original_rate, target_rate) return resampled_audio.astype(np.int16).tobytes() diff --git a/src/pipecat/services/aws.py b/src/pipecat/services/aws.py index e54cb43a1..f42e9de3d 100644 --- a/src/pipecat/services/aws.py +++ b/src/pipecat/services/aws.py @@ -9,6 +9,7 @@ from typing import AsyncGenerator, Optional from loguru import logger from pydantic import BaseModel +from pipecat.audio.utils import resample_audio from pipecat.frames.frames import ( ErrorFrame, Frame, @@ -45,7 +46,7 @@ class AWSTTSService(TTSService): aws_access_key_id: str, region: str, voice_id: str = "Joanna", - sample_rate: int = 16000, + sample_rate: int = 24000, params: InputParams = InputParams(), **kwargs, ): @@ -178,7 +179,8 @@ class AWSTTSService(TTSService): "OutputFormat": "pcm", "VoiceId": self._voice_id, "Engine": self._settings["engine"], - "SampleRate": str(self._settings["sample_rate"]), + # AWS only supports 8000 and 16000 for PCM. We select 16000. + "SampleRate": "16000", } # Filter out None values @@ -198,7 +200,8 @@ class AWSTTSService(TTSService): chunk = audio_data[i : i + chunk_size] if len(chunk) > 0: await self.stop_ttfb_metrics() - frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1) + resampled = resample_audio(chunk, 16000, self._settings["sample_rate"]) + frame = TTSAudioRawFrame(resampled, self._settings["sample_rate"], 1) yield frame yield TTSStoppedFrame() diff --git a/src/pipecat/services/xtts.py b/src/pipecat/services/xtts.py index 61cc7d87a..bcdb132fa 100644 --- a/src/pipecat/services/xtts.py +++ b/src/pipecat/services/xtts.py @@ -151,19 +151,19 @@ class XTTSService(TTSService): async for chunk in r.content.iter_chunked(1024): if len(chunk) > 0: await self.stop_ttfb_metrics() - # Append new chunk to the buffer + # Append new chunk to the buffer. buffer.extend(chunk) - # Check if buffer has enough data for processing + # Check if buffer has enough data for processing. while ( len(buffer) >= 48000 ): # Assuming at least 0.5 seconds of audio data at 24000 Hz - # Process the buffer up to a safe size for resampling + # Process the buffer up to a safe size for resampling. process_data = buffer[:48000] - # Remove processed data from buffer + # Remove processed data from buffer. buffer = buffer[48000:] - # Resample the audio from 24000 Hz + # XTTS uses 24000 so we need to resample to our desired rate. resampled_audio = resample_audio( bytes(process_data), 24000, self._sample_rate ) @@ -171,7 +171,7 @@ class XTTSService(TTSService): frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1) yield frame - # Process any remaining data in the buffer + # Process any remaining data in the buffer. if len(buffer) > 0: resampled_audio = resample_audio(bytes(buffer), 24000, self._sample_rate) frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1) diff --git a/test-requirements.txt b/test-requirements.txt index b2008606d..62be00385 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,7 +22,7 @@ pydantic~=2.8.2 pyloudnorm~=0.1.1 pyht~=0.1.4 python-dotenv~=1.0.1 -scipy~=1.14.1 +resampy~=0.4.3 silero-vad~=5.1 together~=1.2.7 transformers~=4.44.0