From 6a7ab6b8ac888fdf39b0868f326ad3cfc44d1eab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 8 Aug 2024 22:28:21 -0700 Subject: [PATCH 1/2] serializers(twilio): allow specifying input and output sample rates --- CHANGELOG.md | 3 +++ src/pipecat/serializers/twilio.py | 23 ++++++++++++++++++----- src/pipecat/utils/audio.py | 20 ++++++++++---------- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dba1c500d..b9d727a9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `TwilioFrameSerializer` can now specify Twilio's and Pipecat's desired sample + rates to use. + - Added new `on_participant_updated` event to `DailyTransport`. - Added `DailyRESTHelper.delete_room_by_name()`. diff --git a/src/pipecat/serializers/twilio.py b/src/pipecat/serializers/twilio.py index 713f43a86..8836fcd6d 100644 --- a/src/pipecat/serializers/twilio.py +++ b/src/pipecat/serializers/twilio.py @@ -7,18 +7,25 @@ import base64 import json +from pydantic import BaseModel + from pipecat.frames.frames import AudioRawFrame, Frame from pipecat.serializers.base_serializer import FrameSerializer -from pipecat.utils.audio import ulaw_8000_to_pcm_16000, pcm_16000_to_ulaw_8000 +from pipecat.utils.audio import ulaw_to_pcm, pcm_to_ulaw class TwilioFrameSerializer(FrameSerializer): + class InputParams(BaseModel): + twilio_sample_rate: int = 8000 + sample_rate: int = 16000 + SERIALIZABLE_TYPES = { AudioRawFrame: "audio", } - def __init__(self, stream_sid: str): + def __init__(self, stream_sid: str, params: InputParams = InputParams()): self._stream_sid = stream_sid + self._params = params def serialize(self, frame: Frame) -> str | bytes | None: if not isinstance(frame, AudioRawFrame): @@ -26,7 +33,7 @@ class TwilioFrameSerializer(FrameSerializer): data = frame.audio - serialized_data = pcm_16000_to_ulaw_8000(data) + serialized_data = pcm_to_ulaw(data, frame.sample_rate, self._params.twilio_sample_rate) payload = base64.b64encode(serialized_data).decode("utf-8") answer = { "event": "media", @@ -47,6 +54,12 @@ class TwilioFrameSerializer(FrameSerializer): payload_base64 = message["media"]["payload"] payload = base64.b64decode(payload_base64) - deserialized_data = ulaw_8000_to_pcm_16000(payload) - audio_frame = AudioRawFrame(audio=deserialized_data, num_channels=1, sample_rate=16000) + deserialized_data = ulaw_to_pcm( + payload, + self._params.twilio_sample_rate, + self._params.sample_rate) + audio_frame = AudioRawFrame( + audio=deserialized_data, + num_channels=1, + sample_rate=self._params.sample_rate) return audio_frame diff --git a/src/pipecat/utils/audio.py b/src/pipecat/utils/audio.py index a3fc1b0ab..0764c6abb 100644 --- a/src/pipecat/utils/audio.py +++ b/src/pipecat/utils/audio.py @@ -34,21 +34,21 @@ def exp_smoothing(value: float, prev_value: float, factor: float) -> float: return prev_value + factor * (value - prev_value) -def ulaw_8000_to_pcm_16000(ulaw_8000_bytes): +def ulaw_to_pcm(ulaw_bytes: bytes, in_sample_rate: int, out_sample_rate: int): # Convert μ-law to PCM - pcm_8000_bytes = audioop.ulaw2lin(ulaw_8000_bytes, 2) + in_pcm_bytes = audioop.ulaw2lin(ulaw_bytes, 2) - # Resample from 8000 Hz to 16000 Hz - pcm_16000_bytes = audioop.ratecv(pcm_8000_bytes, 2, 1, 8000, 16000, None)[0] + # Resample + out_pcm_bytes = audioop.ratecv(in_pcm_bytes, 2, 1, in_sample_rate, out_sample_rate, None)[0] - return pcm_16000_bytes + return out_pcm_bytes -def pcm_16000_to_ulaw_8000(pcm_16000_bytes): - # Resample from 16000 Hz to 8000 Hz - pcm_8000_bytes = audioop.ratecv(pcm_16000_bytes, 2, 1, 16000, 8000, None)[0] +def pcm_to_ulaw(pcm_bytes: bytes, in_sample_rate: int, out_sample_rate: int): + # Resample + in_pcm_bytes = audioop.ratecv(pcm_bytes, 2, 1, in_sample_rate, out_sample_rate, None)[0] # Convert PCM to μ-law - ulaw_8000_bytes = audioop.lin2ulaw(pcm_8000_bytes, 2) + ulaw_bytes = audioop.lin2ulaw(in_pcm_bytes, 2) - return ulaw_8000_bytes + return ulaw_bytes From d255e954d6aaa885bed86330e74c3653e451c24e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 8 Aug 2024 22:29:40 -0700 Subject: [PATCH 2/2] services(elevenlabs): allow specifying output_format --- CHANGELOG.md | 3 +++ src/pipecat/services/elevenlabs.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9d727a9b..63dbda5bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `ElevenLabsTTSService` can now specify ElevenLabs input parameters such as + `output_format`. + - `TwilioFrameSerializer` can now specify Twilio's and Pipecat's desired sample rates to use. diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index cc63d189a..0de629034 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -6,7 +6,8 @@ import aiohttp -from typing import AsyncGenerator +from typing import AsyncGenerator, Literal +from pydantic import BaseModel from pipecat.frames.frames import AudioRawFrame, ErrorFrame, Frame, MetricsFrame from pipecat.services.ai_services import TTSService @@ -15,6 +16,8 @@ from loguru import logger class ElevenLabsTTSService(TTSService): + class InputParams(BaseModel): + output_format: Literal["pcm_16000", "pcm_22050", "pcm_24000", "pcm_44100"] = "pcm_16000" def __init__( self, @@ -23,12 +26,14 @@ class ElevenLabsTTSService(TTSService): voice_id: str, aiohttp_session: aiohttp.ClientSession, model: str = "eleven_turbo_v2_5", + params: InputParams = InputParams(), **kwargs): super().__init__(**kwargs) self._api_key = api_key self._voice_id = voice_id self._model = model + self._params = params self._aiohttp_session = aiohttp_session def can_generate_metrics(self) -> bool: @@ -46,8 +51,8 @@ class ElevenLabsTTSService(TTSService): payload = {"text": text, "model_id": self._model} querystring = { - "output_format": "pcm_16000", - "optimize_streaming_latency": 2} + "output_format": self._params.output_format + } headers = { "xi-api-key": self._api_key,