From c9d0af9ee05620d496606607232bf226d8bf9401 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 19 May 2025 09:43:24 -0400 Subject: [PATCH 1/5] Deprecate emotion, add new speed parameter --- src/pipecat/services/cartesia/tts.py | 46 +++++++++++++++++++--------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index bc7b1f121..39f577ec3 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -7,6 +7,7 @@ import base64 import json import uuid +import warnings from typing import AsyncGenerator, List, Optional, Union from loguru import logger @@ -151,10 +152,13 @@ class CartesiaTTSService(AudioContextWordTTSService): voice_config["mode"] = "id" voice_config["id"] = self._voice_id - if self._settings["speed"] or self._settings["emotion"]: + if self._settings["emotion"]: + warnings.warn( + "The 'emotion' parameter in __experimental_controls is deprecated and will be removed in a future version.", + DeprecationWarning, + stacklevel=2, + ) voice_config["__experimental_controls"] = {} - if self._settings["speed"]: - voice_config["__experimental_controls"]["speed"] = self._settings["speed"] if self._settings["emotion"]: voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"] @@ -169,6 +173,10 @@ class CartesiaTTSService(AudioContextWordTTSService): "add_timestamps": add_timestamps, "use_original_timestamps": False if self.model_name == "sonic" else True, } + + if self._settings["speed"]: + msg["speed"] = self._settings["speed"] + return json.dumps(msg) async def start(self, frame: StartFrame): @@ -368,24 +376,32 @@ class CartesiaHttpTTSService(TTSService): try: voice_controls = None - if self._settings["speed"] or self._settings["emotion"]: + if self._settings["emotion"]: + warnings.warn( + "The 'emotion' parameter in _experimental_voice_controls is deprecated and will be removed in a future version.", + DeprecationWarning, + stacklevel=2, + ) voice_controls = {} - if self._settings["speed"]: - voice_controls["speed"] = self._settings["speed"] if self._settings["emotion"]: voice_controls["emotion"] = self._settings["emotion"] await self.start_ttfb_metrics() - output = await self._client.tts.sse( - model_id=self._model_name, - transcript=text, - voice_id=self._voice_id, - output_format=self._settings["output_format"], - language=self._settings["language"], - stream=False, - _experimental_voice_controls=voice_controls, - ) + kwargs = { + "model_id": self._model_name, + "transcript": text, + "voice_id": self._voice_id, + "output_format": self._settings["output_format"], + "language": self._settings["language"], + "stream": False, + "_experimental_voice_controls": voice_controls, + } + + if self._settings["speed"]: + kwargs["speed"] = self._settings["speed"] + + output = await self._client.tts.sse(**kwargs) await self.start_tts_usage_metrics(text) From 682f8e4d4587afb4b2d9de2147944497b91fdedc Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 19 May 2025 11:10:03 -0400 Subject: [PATCH 2/5] Bump the cartesia_version for CartesiaTTSService, and cartesia package for CartesiaHttpTTSService --- pyproject.toml | 2 +- src/pipecat/services/cartesia/tts.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 78bd78773..b8db368f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ assemblyai = [ "assemblyai~=0.37.0" ] aws = [ "boto3~=1.37.16", "websockets~=13.1" ] aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.0.2" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] -cartesia = [ "cartesia~=1.4.0", "websockets~=13.1" ] +cartesia = [ "cartesia~=2.0.3", "websockets~=13.1" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.18.2" ] diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 39f577ec3..92762683d 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -84,7 +84,7 @@ class CartesiaTTSService(AudioContextWordTTSService): *, api_key: str, voice_id: str, - cartesia_version: str = "2024-06-10", + cartesia_version: str = "2025-04-16", url: str = "wss://api.cartesia.ai/tts/websocket", model: str = "sonic-2", sample_rate: Optional[int] = None, From bd09ccd6087f3f4f91d1a750f44753aa3d6a1645 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 19 May 2025 11:31:28 -0400 Subject: [PATCH 3/5] Update CartesiaHttpTTSService to work with the new cartesia 2.0 client --- src/pipecat/services/cartesia/tts.py | 56 ++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 92762683d..5d37a583f 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -326,6 +326,7 @@ class CartesiaHttpTTSService(TTSService): voice_id: str, model: str = "sonic-2", base_url: str = "https://api.cartesia.ai", + cartesia_version: str = "2024-11-13", sample_rate: Optional[int] = None, encoding: str = "pcm_s16le", container: str = "raw", @@ -335,6 +336,8 @@ class CartesiaHttpTTSService(TTSService): super().__init__(sample_rate=sample_rate, **kwargs) self._api_key = api_key + self._base_url = base_url + self._cartesia_version = cartesia_version self._settings = { "output_format": { "container": container, @@ -350,7 +353,10 @@ class CartesiaHttpTTSService(TTSService): self.set_voice(voice_id) self.set_model_name(model) - self._client = AsyncCartesia(api_key=api_key, base_url=base_url) + self._client = AsyncCartesia( + api_key=api_key, + base_url=base_url, + ) def can_generate_metrics(self) -> bool: return True @@ -375,45 +381,63 @@ class CartesiaHttpTTSService(TTSService): logger.debug(f"{self}: Generating TTS [{text}]") try: - voice_controls = None + voice_config = {"mode": "id", "id": self._voice_id} + if self._settings["emotion"]: warnings.warn( - "The 'emotion' parameter in _experimental_voice_controls is deprecated and will be removed in a future version.", + "The 'emotion' parameter in voice.__experimental_controls is deprecated and will be removed in a future version.", DeprecationWarning, stacklevel=2, ) - voice_controls = {} - if self._settings["emotion"]: - voice_controls["emotion"] = self._settings["emotion"] + voice_config["__experimental_controls"] = {"emotion": self._settings["emotion"]} await self.start_ttfb_metrics() - kwargs = { + payload = { "model_id": self._model_name, "transcript": text, - "voice_id": self._voice_id, + "voice": voice_config, "output_format": self._settings["output_format"], "language": self._settings["language"], - "stream": False, - "_experimental_voice_controls": voice_controls, } if self._settings["speed"]: - kwargs["speed"] = self._settings["speed"] - - output = await self._client.tts.sse(**kwargs) - - await self.start_tts_usage_metrics(text) + payload["speed"] = self._settings["speed"] yield TTSStartedFrame() + session = await self._client._get_session() + + headers = { + "Cartesia-Version": self._cartesia_version, + "X-API-Key": self._api_key, + "Content-Type": "application/json", + } + + url = f"{self._base_url}/tts/bytes" + + async with session.post(url, json=payload, headers=headers) as response: + if response.status != 200: + error_text = await response.text() + logger.error(f"Cartesia API error: {error_text}") + await self.push_error(ErrorFrame(f"Cartesia API error: {error_text}")) + raise Exception(f"Cartesia API returned status {response.status}: {error_text}") + + audio_data = await response.read() + + await self.start_tts_usage_metrics(text) + frame = TTSAudioRawFrame( - audio=output["audio"], sample_rate=self.sample_rate, num_channels=1 + audio=audio_data, + sample_rate=self.sample_rate, + num_channels=1, ) yield frame + except Exception as e: logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(f"Error generating TTS: {e}")) finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() From 916b9d6c6d6f6d7a360a30b310fa5ea2cba7ac4c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 19 May 2025 11:31:47 -0400 Subject: [PATCH 4/5] Add an example for CartesiaHttpTTSService --- .../07-interruptible-cartesia-http.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 examples/foundational/07-interruptible-cartesia-http.py diff --git a/examples/foundational/07-interruptible-cartesia-http.py b/examples/foundational/07-interruptible-cartesia-http.py new file mode 100644 index 000000000..957938df2 --- /dev/null +++ b/examples/foundational/07-interruptible-cartesia-http.py @@ -0,0 +1,105 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia.tts import CartesiaHttpTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection + +load_dotenv(override=True) + + +async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + logger.info(f"Starting bot") + + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaHttpTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) + + +if __name__ == "__main__": + from run import main + + main() From 8e4e03541cd1eb03251852d70ecdb209cfe60488 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 19 May 2025 11:47:10 -0400 Subject: [PATCH 5/5] Update CHANGELOG --- CHANGELOG.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b83d77d54..264584f49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added additional languages to `LmntTTSService`. Languages include: `hi`, `id`, - `it`, `ja`, `nl`, `pl`, `ru`, `sv`, `th`, `tr`, `uk`, `vi`. + `it`, `ja`, `nl`, `pl`, `ru`, `sv`, `th`, `tr`, `uk`, `vi`. - Added a `model` parameter to the `LmntTTSService` constructor, allowing switching between LMNT models. @@ -59,12 +59,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- The API version for `CartesiaTTSService` and `CartesiaHttpTTSService` has + been updated. Also, the `cartesia` dependency has been updated to 2.x. + +- `CartesiaTTSService` and `CartesiaHttpTTSService` now support Cartesia's new + `speed` parameter which accepts values of `slow`, `normal`, and `fast`. + - `GeminiMultimodalLiveLLMService` now uses the user transcription and usage metrics provided by Gemini Live. - `GoogleLLMService` has been updated to use `google-genai` instead of the deprecated `google-generativeai`. +### Deprecated + +- In `CartesiaTTSService` and `CartesiaHttpTTSService`, `emotion` has been + deprecated by Cartesia. Pipecat is following suit and deprecating `emotion` + as well. + ### Removed - Since `GeminiMultimodalLiveLLMService` now transcribes it's own audio, the