Merge pull request #359 from pipecat-ai/aleix/twilio-elevenlabs-sample-rates

twilio and elevenlabs sample rates

CHANGELOG.md

@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- `ElevenLabsTTSService` can now specify ElevenLabs input parameters such as
+  `output_format`.
+
+- `TwilioFrameSerializer` can now specify Twilio's and Pipecat's desired sample
+  rates to use.
+
 - Added new `on_participant_updated` event to `DailyTransport`.
 
 - Added `DailyRESTHelper.delete_room_by_name()`.

src/pipecat/serializers/twilio.py

@@ -7,18 +7,25 @@
 import base64
 import json
 
+from pydantic import BaseModel
+
 from pipecat.frames.frames import AudioRawFrame, Frame
 from pipecat.serializers.base_serializer import FrameSerializer
-from pipecat.utils.audio import ulaw_8000_to_pcm_16000, pcm_16000_to_ulaw_8000
+from pipecat.utils.audio import ulaw_to_pcm, pcm_to_ulaw
 
 
 class TwilioFrameSerializer(FrameSerializer):
+    class InputParams(BaseModel):
+        twilio_sample_rate: int = 8000
+        sample_rate: int = 16000
+
     SERIALIZABLE_TYPES = {
         AudioRawFrame: "audio",
     }
 
-    def __init__(self, stream_sid: str):
+    def __init__(self, stream_sid: str, params: InputParams = InputParams()):
         self._stream_sid = stream_sid
+        self._params = params
 
     def serialize(self, frame: Frame) -> str | bytes | None:
         if not isinstance(frame, AudioRawFrame):
@@ -26,7 +33,7 @@ class TwilioFrameSerializer(FrameSerializer):
 
         data = frame.audio
 
-        serialized_data = pcm_16000_to_ulaw_8000(data)
+        serialized_data = pcm_to_ulaw(data, frame.sample_rate, self._params.twilio_sample_rate)
         payload = base64.b64encode(serialized_data).decode("utf-8")
         answer = {
             "event": "media",
@@ -47,6 +54,12 @@ class TwilioFrameSerializer(FrameSerializer):
             payload_base64 = message["media"]["payload"]
             payload = base64.b64decode(payload_base64)
 
-            deserialized_data = ulaw_8000_to_pcm_16000(payload)
+            deserialized_data = ulaw_to_pcm(
-            audio_frame = AudioRawFrame(audio=deserialized_data, num_channels=1, sample_rate=16000)
+                payload,
+                self._params.twilio_sample_rate,
+                self._params.sample_rate)
+            audio_frame = AudioRawFrame(
+                audio=deserialized_data,
+                num_channels=1,
+                sample_rate=self._params.sample_rate)
             return audio_frame

src/pipecat/services/elevenlabs.py

@@ -6,7 +6,8 @@
 
 import aiohttp
 
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Literal
+from pydantic import BaseModel
 
 from pipecat.frames.frames import AudioRawFrame, ErrorFrame, Frame, MetricsFrame
 from pipecat.services.ai_services import TTSService
@@ -15,6 +16,8 @@ from loguru import logger
 
 
 class ElevenLabsTTSService(TTSService):
+    class InputParams(BaseModel):
+        output_format: Literal["pcm_16000", "pcm_22050", "pcm_24000", "pcm_44100"] = "pcm_16000"
 
     def __init__(
             self,
@@ -23,12 +26,14 @@ class ElevenLabsTTSService(TTSService):
             voice_id: str,
             aiohttp_session: aiohttp.ClientSession,
             model: str = "eleven_turbo_v2_5",
+            params: InputParams = InputParams(),
             **kwargs):
         super().__init__(**kwargs)
 
         self._api_key = api_key
         self._voice_id = voice_id
         self._model = model
+        self._params = params
         self._aiohttp_session = aiohttp_session
 
     def can_generate_metrics(self) -> bool:
@@ -46,8 +51,8 @@ class ElevenLabsTTSService(TTSService):
         payload = {"text": text, "model_id": self._model}
 
         querystring = {
-            "output_format": "pcm_16000",
+            "output_format": self._params.output_format
-            "optimize_streaming_latency": 2}
+        }
 
         headers = {
             "xi-api-key": self._api_key,

src/pipecat/utils/audio.py

@@ -34,21 +34,21 @@ def exp_smoothing(value: float, prev_value: float, factor: float) -> float:
     return prev_value + factor * (value - prev_value)
 
 
-def ulaw_8000_to_pcm_16000(ulaw_8000_bytes):
+def ulaw_to_pcm(ulaw_bytes: bytes, in_sample_rate: int, out_sample_rate: int):
     # Convert μ-law to PCM
-    pcm_8000_bytes = audioop.ulaw2lin(ulaw_8000_bytes, 2)
+    in_pcm_bytes = audioop.ulaw2lin(ulaw_bytes, 2)
 
-    # Resample from 8000 Hz to 16000 Hz
+    # Resample
-    pcm_16000_bytes = audioop.ratecv(pcm_8000_bytes, 2, 1, 8000, 16000, None)[0]
+    out_pcm_bytes = audioop.ratecv(in_pcm_bytes, 2, 1, in_sample_rate, out_sample_rate, None)[0]
 
-    return pcm_16000_bytes
+    return out_pcm_bytes
 
 
-def pcm_16000_to_ulaw_8000(pcm_16000_bytes):
+def pcm_to_ulaw(pcm_bytes: bytes, in_sample_rate: int, out_sample_rate: int):
-    # Resample from 16000 Hz to 8000 Hz
+    # Resample
-    pcm_8000_bytes = audioop.ratecv(pcm_16000_bytes, 2, 1, 16000, 8000, None)[0]
+    in_pcm_bytes = audioop.ratecv(pcm_bytes, 2, 1, in_sample_rate, out_sample_rate, None)[0]
 
     # Convert PCM to μ-law
-    ulaw_8000_bytes = audioop.lin2ulaw(pcm_8000_bytes, 2)
+    ulaw_bytes = audioop.lin2ulaw(in_pcm_bytes, 2)
 
-    return ulaw_8000_bytes
+    return ulaw_bytes