diff --git a/CHANGELOG.md b/CHANGELOG.md index 475301ba4..c45b8598f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 grained control of what media subscriptions you want for each participant in a room. +### Changed + +- Changed default output sample rate to 24000. This changes all TTS service to + output to 24000 and also the default output transport sample rate. This + improves audio quality at the cost of some extra bandwidth. + ## [0.0.47] - 2024-10-22 ### Added diff --git a/examples/foundational/01b-livekit-audio.py b/examples/foundational/01b-livekit-audio.py index a463adcf4..9b3efc603 100644 --- a/examples/foundational/01b-livekit-audio.py +++ b/examples/foundational/01b-livekit-audio.py @@ -81,7 +81,7 @@ async def main(): url=url, token=token, room_name=room_name, - params=LiveKitParams(audio_out_enabled=True, audio_out_sample_rate=16000), + params=LiveKitParams(audio_out_enabled=True), ) tts = CartesiaTTSService( diff --git a/examples/foundational/07e-interruptible-playht.py b/examples/foundational/07e-interruptible-playht.py index cd0c0505d..f93bf6580 100644 --- a/examples/foundational/07e-interruptible-playht.py +++ b/examples/foundational/07e-interruptible-playht.py @@ -40,7 +40,6 @@ async def main(): "Respond bot", DailyParams( audio_out_enabled=True, - audio_out_sample_rate=16000, transcription_enabled=True, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), diff --git a/examples/foundational/07f-interruptible-azure.py b/examples/foundational/07f-interruptible-azure.py index 1de3a5b0d..c605de431 100644 --- a/examples/foundational/07f-interruptible-azure.py +++ b/examples/foundational/07f-interruptible-azure.py @@ -41,7 +41,6 @@ async def main(): "Respond bot", DailyParams( audio_out_enabled=True, - audio_out_sample_rate=16000, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), vad_audio_passthrough=True, diff --git a/examples/foundational/09-mirror.py b/examples/foundational/09-mirror.py index ff71c60d6..50c3a5b18 100644 --- a/examples/foundational/09-mirror.py +++ b/examples/foundational/09-mirror.py @@ -63,6 +63,7 @@ async def main(): "Test", DailyParams( audio_in_enabled=True, + audio_in_sample_rate=24000, audio_out_enabled=True, camera_out_enabled=True, camera_out_is_live=True, diff --git a/examples/foundational/09a-local-mirror.py b/examples/foundational/09a-local-mirror.py index c3c66569b..3d5ce347c 100644 --- a/examples/foundational/09a-local-mirror.py +++ b/examples/foundational/09a-local-mirror.py @@ -65,7 +65,7 @@ async def main(): tk_root.title("Local Mirror") daily_transport = DailyTransport( - room_url, token, "Test", DailyParams(audio_in_enabled=True) + room_url, token, "Test", DailyParams(audio_in_enabled=True, audio_in_sample_rate=24000) ) tk_transport = TkLocalTransport( diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py index e11c02f49..ede473b31 100644 --- a/examples/foundational/12c-describe-video-anthropic.py +++ b/examples/foundational/12c-describe-video-anthropic.py @@ -78,9 +78,6 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady - params=CartesiaTTSService.InputParams( - sample_rate=16000, - ), ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index f3310366a..c2af63ad8 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -10,7 +10,7 @@ import os import sys from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMMessagesFrame +from pipecat.frames.frames import LLMMessagesFrame, TTSUpdateSettingsFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.runner import PipelineRunner @@ -19,7 +19,6 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService -from pipecat.services.whisper import Model, WhisperSTTService from pipecat.transports.services.daily import DailyParams, DailyTransport from openai.types.chat import ChatCompletionToolParam @@ -61,16 +60,14 @@ async def main(): token, "Pipecat", DailyParams( - audio_in_enabled=True, audio_out_enabled=True, + transcription_enabled=True, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), vad_audio_passthrough=True, ), ) - stt = WhisperSTTService(model=Model.LARGE) - english_tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady @@ -116,7 +113,6 @@ async def main(): pipeline = Pipeline( [ transport.input(), # Transport user input - stt, # STT context_aggregator.user(), # User responses llm, # LLM ParallelPipeline( # TTS (bot will speak the chosen language) diff --git a/src/pipecat/audio/utils.py b/src/pipecat/audio/utils.py index 6675a07ab..42555661a 100644 --- a/src/pipecat/audio/utils.py +++ b/src/pipecat/audio/utils.py @@ -11,6 +11,8 @@ from scipy import signal def resample_audio(audio: bytes, original_rate: int, target_rate: int) -> bytes: + if original_rate == target_rate: + return audio audio_data = np.frombuffer(audio, dtype=np.int16) num_samples = int(len(audio) * target_rate / original_rate) resampled_audio = signal.resample(audio_data, num_samples) diff --git a/src/pipecat/audio/vad/silero.py b/src/pipecat/audio/vad/silero.py index 84b825275..1da0fb12d 100644 --- a/src/pipecat/audio/vad/silero.py +++ b/src/pipecat/audio/vad/silero.py @@ -52,7 +52,7 @@ class SileroOnnxModel: if sr not in self.sample_rates: raise ValueError( - f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)" + f"Supported sampling rates: {self.sample_rates} (or multiple of 16000)" ) if sr / np.shape(x)[1] > 31.25: raise ValueError("Input audio chunk is too short") diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 22ceffb7b..62d52f76e 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -205,7 +205,7 @@ class TTSService(AIService): # if push_stop_frames is True, wait for this idle period before pushing TTSStoppedFrame stop_frame_timeout_s: float = 1.0, # TTS output sample rate - sample_rate: int = 16000, + sample_rate: int = 24000, text_filter: Optional[BaseTextFilter] = None, **kwargs, ): @@ -514,7 +514,7 @@ class SegmentedSTTService(STTService): min_volume: float = 0.6, max_silence_secs: float = 0.3, max_buffer_secs: float = 1.5, - sample_rate: int = 16000, + sample_rate: int = 24000, num_channels: int = 1, **kwargs, ): diff --git a/src/pipecat/services/azure.py b/src/pipecat/services/azure.py index b7d4f0d4c..56dde30c1 100644 --- a/src/pipecat/services/azure.py +++ b/src/pipecat/services/azure.py @@ -25,8 +25,14 @@ from pipecat.frames.frames import ( TTSStoppedFrame, URLImageRawFrame, ) +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.ai_services import ImageGenService, STTService, TTSService -from pipecat.services.openai import BaseOpenAILLMService +from pipecat.services.openai import ( + BaseOpenAILLMService, + OpenAIAssistantContextAggregator, + OpenAIContextAggregatorPair, + OpenAIUserContextAggregator, +) from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 @@ -38,6 +44,7 @@ try: SpeechConfig, SpeechRecognizer, SpeechSynthesizer, + SpeechSynthesisOutputFormat, ) from azure.cognitiveservices.speech.audio import ( AudioStreamFormat, @@ -70,6 +77,33 @@ class AzureLLMService(BaseOpenAILLMService): api_version=self._api_version, ) + @staticmethod + def create_context_aggregator( + context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True + ) -> OpenAIContextAggregatorPair: + user = OpenAIUserContextAggregator(context) + assistant = OpenAIAssistantContextAggregator( + user, expect_stripped_words=assistant_expect_stripped_words + ) + return OpenAIContextAggregatorPair(_user=user, _assistant=assistant) + + +def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputFormat: + match sample_rate: + case 8000: + return SpeechSynthesisOutputFormat.Raw8Khz16BitMonoPcm + case 16000: + return SpeechSynthesisOutputFormat.Raw16Khz16BitMonoPcm + case 22050: + return SpeechSynthesisOutputFormat.Raw22050Hz16BitMonoPcm + case 24000: + return SpeechSynthesisOutputFormat.Raw24Khz16BitMonoPcm + case 44100: + return SpeechSynthesisOutputFormat.Raw44100Hz16BitMonoPcm + case 48000: + return SpeechSynthesisOutputFormat.Raw48Khz16BitMonoPcm + return SpeechSynthesisOutputFormat.Raw16Khz16BitMonoPcm + class AzureTTSService(TTSService): class InputParams(BaseModel): @@ -88,13 +122,15 @@ class AzureTTSService(TTSService): api_key: str, region: str, voice="en-US-SaraNeural", - sample_rate: int = 16000, + sample_rate: int = 24000, params: InputParams = InputParams(), **kwargs, ): super().__init__(sample_rate=sample_rate, **kwargs) speech_config = SpeechConfig(subscription=api_key, region=region) + speech_config.set_speech_synthesis_output_format(sample_rate_to_output_format(sample_rate)) + self._speech_synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=None) self._settings = { @@ -283,7 +319,7 @@ class AzureSTTService(STTService): api_key: str, region: str, language=Language.EN_US, - sample_rate=16000, + sample_rate=24000, channels=1, **kwargs, ): diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 5ce0774de..35f91b4cd 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -80,7 +80,7 @@ class CartesiaTTSService(WordTTSService): cartesia_version: str = "2024-06-10", url: str = "wss://api.cartesia.ai/tts/websocket", model: str = "sonic-english", - sample_rate: int = 16000, + sample_rate: int = 24000, encoding: str = "pcm_s16le", container: str = "raw", params: InputParams = InputParams(), @@ -99,6 +99,7 @@ class CartesiaTTSService(WordTTSService): super().__init__( aggregate_sentences=True, push_text_frames=False, + sample_rate=sample_rate, **kwargs, ) @@ -298,13 +299,13 @@ class CartesiaHttpTTSService(TTSService): voice_id: str, model: str = "sonic-english", base_url: str = "https://api.cartesia.ai", - sample_rate: int = 16000, + sample_rate: int = 24000, encoding: str = "pcm_s16le", container: str = "raw", params: InputParams = InputParams(), **kwargs, ): - super().__init__(**kwargs) + super().__init__(sample_rate=sample_rate, **kwargs) self._api_key = api_key self._settings = { diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index ff694217d..d298e3e9b 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -51,11 +51,11 @@ class DeepgramTTSService(TTSService): *, api_key: str, voice: str = "aura-helios-en", - sample_rate: int = 16000, + sample_rate: int = 24000, encoding: str = "linear16", **kwargs, ): - super().__init__(**kwargs) + super().__init__(sample_rate=sample_rate, **kwargs) self._settings = { "sample_rate": sample_rate, diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index ebbfc2e19..b4dee5fa3 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -99,7 +99,7 @@ class ElevenLabsTTSService(WordTTSService): voice_id: str, model: str = "eleven_turbo_v2_5", url: str = "wss://api.elevenlabs.io", - output_format: ElevenLabsOutputFormat = "pcm_16000", + output_format: ElevenLabsOutputFormat = "pcm_24000", params: InputParams = InputParams(), **kwargs, ): diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 6e7cd68f3..bfc398296 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -115,7 +115,7 @@ class PlayHTTTSService(TTSService): user_id: str, voice_url: str, voice_engine: str = "PlayHT3.0-mini", - sample_rate: int = 16000, + sample_rate: int = 24000, output_format: str = "wav", params: InputParams = InputParams(), **kwargs, @@ -310,7 +310,7 @@ class PlayHTHttpTTSService(TTSService): user_id: str, voice_url: str, voice_engine: str = "PlayHT3.0-mini", - sample_rate: int = 16000, + sample_rate: int = 24000, params: InputParams = InputParams(), **kwargs, ): diff --git a/src/pipecat/services/xtts.py b/src/pipecat/services/xtts.py index 1c444f9f1..61cc7d87a 100644 --- a/src/pipecat/services/xtts.py +++ b/src/pipecat/services/xtts.py @@ -39,9 +39,10 @@ class XTTSService(TTSService): language: Language, base_url: str, aiohttp_session: aiohttp.ClientSession, + sample_rate: int = 24000, **kwargs, ): - super().__init__(**kwargs) + super().__init__(sample_rate=sample_rate, **kwargs) self._settings = { "language": self.language_to_service_language(language), @@ -162,16 +163,18 @@ class XTTSService(TTSService): # Remove processed data from buffer buffer = buffer[48000:] - # Resample the audio from 24000 Hz to 16000 Hz - resampled_audio = resample_audio(bytes(process_data), 24000, 16000) + # Resample the audio from 24000 Hz + resampled_audio = resample_audio( + bytes(process_data), 24000, self._sample_rate + ) # Create the frame with the resampled audio - frame = TTSAudioRawFrame(resampled_audio, 16000, 1) + frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1) yield frame # Process any remaining data in the buffer if len(buffer) > 0: - resampled_audio = resample_audio(bytes(buffer), 24000, 16000) - frame = TTSAudioRawFrame(resampled_audio, 16000, 1) + resampled_audio = resample_audio(bytes(buffer), 24000, self._sample_rate) + frame = TTSAudioRawFrame(resampled_audio, self._sample_rate, 1) yield frame yield TTSStoppedFrame() diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py index d2f98c26c..2ca2b6470 100644 --- a/src/pipecat/transports/base_transport.py +++ b/src/pipecat/transports/base_transport.py @@ -30,7 +30,7 @@ class TransportParams(BaseModel): camera_out_color_format: str = "RGB" audio_out_enabled: bool = False audio_out_is_live: bool = False - audio_out_sample_rate: int = 16000 + audio_out_sample_rate: int = 24000 audio_out_channels: int = 1 audio_out_bitrate: int = 96000 audio_in_enabled: bool = False diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index 3c0aca146..6b6b55c54 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -11,6 +11,7 @@ from typing import Any, Awaitable, Callable, List from pydantic import BaseModel from pipecat.audio.utils import resample_audio +from pipecat.audio.vad.vad_analyzer import VADAnalyzer from pipecat.frames.frames import ( AudioRawFrame, CancelFrame,