From 4f30a48ecd292913aa1b4c56367af0828abe6851 Mon Sep 17 00:00:00 2001 From: mattie ruth backman Date: Mon, 17 Nov 2025 16:52:48 -0500 Subject: [PATCH] Rime and Cartesia TTS Updates: `CartesiaTTSService`: - Modified use of custom default text_aggregator to avoid deprecation warnings and push users towards use of transformers or the `LLMTextProcessor` - Added convenience methods for taking advantage of Cartesia's SSML tags: spell, emotion, pauses, volume, and speed. `RimeTTSService`: - Modified use of custom default text_aggregator to avoid deprecation warnings and push users towards use of transformers or the `LLMTextProcessor` - Added convenience methods for taking advantage of Rime's customization options: spell, pauses, pronunciations, and inline speed control. --- CHANGELOG.md | 12 ++++ src/pipecat/services/cartesia/tts.py | 104 ++++++++++++++++++++++++++- src/pipecat/services/rime/tts.py | 45 +++++++++++- 3 files changed, 157 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07d922632..8a54c5d1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -169,6 +169,18 @@ Croatian, Hungarian, Malay, Norwegian, Nynorsk, Slovak, Slovenian, Swedish, and fly. `remove_text_transformer` has also been added to support removing a registered transform callback. + - Updated `CartesiaTTSService`: + - Modified use of custom default text_aggregator to avoid deprecation warnings and push users + towards use of transformers or the `LLMTextProcessor` + - Added convenience methods for taking advantage of Cartesia's SSML tags: spell, emotion, + pauses, volume, and speed. + + - Updated `RimeTTSService`: + - Modified use of custom default text_aggregator to avoid deprecation warnings and push users + towards use of transformers or the `LLMTextProcessor` + - Added convenience methods for taking advantage of Rime's customization options: spell, + pauses, pronunciations, and inline speed control. + ### Deprecated - The `api_key` parameter in `GeminiTTSService` is deprecated. Use diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index f8881200c..d42802cf2 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -10,7 +10,8 @@ import base64 import json import uuid import warnings -from typing import AsyncGenerator, List, Literal, Optional, Union +from enum import Enum +from typing import AsyncGenerator, List, Literal, Optional from loguru import logger from pydantic import BaseModel, Field @@ -125,6 +126,72 @@ def language_to_cartesia_language(language: Language) -> Optional[str]: return resolve_language(language, LANGUAGE_MAP, use_base_code=True) +class CartesiaEmotion(str, Enum): + """Predefined Emotions supported by Cartesia.""" + + # Primary emotions supported by Cartesia + NEUTRAL = "neutral" + ANGRY = "angry" + EXCITED = "excited" + CONTENT = "content" + SAD = "sad" + SCARED = "scared" + # Additional emotions supported by Cartesia + HAPPY = "happy" + ENTHUSIASTIC = "enthusiastic" + ELATED = "elated" + EUPHORIC = "euphoric" + TRIUMPHANT = "triumphant" + AMAZED = "amazed" + SURPRISED = "surprised" + FLIRTATIOUS = "flirtatious" + JOKING_COMEDIC = "joking/comedic" + CURIOUS = "curious" + PEACEFUL = "peaceful" + SERENE = "serene" + CALM = "calm" + GRATEFUL = "grateful" + AFFECTIONATE = "affectionate" + TRUST = "trust" + SYMPATHETIC = "sympathetic" + ANTICIPATION = "anticipation" + MYSTERIOUS = "mysterious" + MAD = "mad" + OUTRAGED = "outraged" + FRUSTRATED = "frustrated" + AGITATED = "agitated" + THREATENED = "threatened" + DISGUSTED = "disgusted" + CONTEMPT = "contempt" + ENVIOUS = "envious" + SARCASTIC = "sarcastic" + IRONIC = "ironic" + DEJECTED = "dejected" + MELANCHOLIC = "melancholic" + DISAPPOINTED = "disappointed" + HURT = "hurt" + GUILTY = "guilty" + BORED = "bored" + TIRED = "tired" + REJECTED = "rejected" + NOSTALGIC = "nostalgic" + WISTFUL = "wistful" + APOLOGETIC = "apologetic" + HESITANT = "hesitant" + INSECURE = "insecure" + CONFUSED = "confused" + RESIGNED = "resigned" + ANXIOUS = "anxious" + PANICKED = "panicked" + ALARMED = "alarmed" + PROUD = "proud" + CONFIDENT = "confident" + DISTANT = "distant" + SKEPTICAL = "skeptical" + CONTEMPLATIVE = "contemplative" + DETERMINED = "determined" + + class CartesiaTTSService(AudioContextWordTTSService): """Cartesia TTS service with WebSocket streaming and word timestamps. @@ -182,6 +249,10 @@ class CartesiaTTSService(AudioContextWordTTSService): container: Audio container format. params: Additional input parameters for voice customization. text_aggregator: Custom text aggregator for processing input text. + + .. deprecated:: 0.0.95 + Use an LLMTextProcessor before the TTSService for custom text aggregation. + aggregate_sentences: Whether to aggregate sentences within the TTSService. **kwargs: Additional arguments passed to the parent service. """ @@ -200,10 +271,18 @@ class CartesiaTTSService(AudioContextWordTTSService): push_text_frames=False, pause_frame_processing=True, sample_rate=sample_rate, - text_aggregator=text_aggregator or SkipTagsAggregator([("", "")]), + text_aggregator=text_aggregator, **kwargs, ) + if not text_aggregator: + # Always skip tags added for spelled-out text + # Note: This is primarily to support backwards compatibility. + # The preferred way of taking advantage of Cartesia SSML Tags is + # to use an LLMTextProcessor and/or a text_transformer to identify + # and insert these tags for the purpose of the TTS service alone. + self._text_aggregator = SkipTagsAggregator([("", "")]) + params = params or CartesiaTTSService.InputParams() self._api_key = api_key @@ -257,6 +336,27 @@ class CartesiaTTSService(AudioContextWordTTSService): """ return language_to_cartesia_language(language) + # A set of Cartesia-specific helpers for text transformations + def SPELL(text: str) -> str: + """Wrap text in Cartesia spell tag.""" + return f"{text}" + + def EMOTION_TAG(emotion: CartesiaEmotion) -> str: + """Convenience method to create an emotion tag.""" + return f'' + + def PAUSE_TAG(seconds: float) -> str: + """Convenience method to create a pause tag.""" + return f'' + + def VOLUME_TAG(volume: float) -> str: + """Convenience method to create a volume tag.""" + return f'' + + def SPEED_TAG(speed: float) -> str: + """Convenience method to create a speed tag.""" + return f'' + def _is_cjk_language(self, language: str) -> bool: """Check if the given language is CJK (Chinese, Japanese, Korean). diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 7b62f20fa..c9f461350 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -113,6 +113,10 @@ class RimeTTSService(AudioContextWordTTSService): sample_rate: Audio sample rate in Hz. params: Additional configuration parameters. text_aggregator: Custom text aggregator for processing input text. + + .. deprecated:: 0.0.95 + Use an LLMTextProcessor before the TTSService for custom text aggregation. + aggregate_sentences: Whether to aggregate sentences within the TTSService. **kwargs: Additional arguments passed to parent class. """ @@ -123,10 +127,17 @@ class RimeTTSService(AudioContextWordTTSService): push_stop_frames=True, pause_frame_processing=True, sample_rate=sample_rate, - text_aggregator=text_aggregator or SkipTagsAggregator([("spell(", ")")]), **kwargs, ) + if not text_aggregator: + # Always skip tags added for spelled-out text + # Note: This is primarily to support backwards compatibility. + # The preferred way of taking advantage of Rime spelling is + # to use an LLMTextProcessor and/or a text_transformer to identify + # and insert these tags for the purpose of the TTS service alone. + self._text_aggregator = SkipTagsAggregator([("spell(", ")")]) + params = params or RimeTTSService.InputParams() # Store service configuration @@ -152,6 +163,7 @@ class RimeTTSService(AudioContextWordTTSService): self._context_id = None # Tracks current turn self._receive_task = None self._cumulative_time = 0 # Accumulates time across messages + self._extra_msg_fields = {} # Extra fields for next message def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -181,6 +193,31 @@ class RimeTTSService(AudioContextWordTTSService): self._model = model await super().set_model(model) + # A set of Rime-specific helpers for text transformations + def SPELL(text: str) -> str: + """Wrap text in Rime spell function.""" + return f"spell({text})" + + def PAUSE_TAG(seconds: float) -> str: + """Convenience method to create a pause tag.""" + return f"<{seconds * 1000}>" + + def PRONOUNCE(self, text: str, word: str, phoneme: str) -> str: + """Convenience method to support Rime's custom pronunciations feature. + + https://docs.rime.ai/api-reference/custom-pronunciation + """ + self._extra_msg_fields["phonemizeBetweenBrackets"] = True + return text.replace(word, f"{phoneme}") + + def INLINE_SPEED(self, text: str, speed: float) -> str: + """Convenience method to support inline speeds.""" + if not self._extra_msg_fields: + self._extra_msg_fields = {} + speed_vals = self._extra_msg_fields.get("inlineSpeedAlpha", "").split(",") + self._extra_msg_fields["inlineSpeedAlpha"] = ",".join(speed_vals + [str(speed)]) + return f"[{text}]" + async def _update_settings(self, settings: Mapping[str, Any]): """Update service settings and reconnect if voice changed.""" prev_voice = self._voice_id @@ -193,7 +230,11 @@ class RimeTTSService(AudioContextWordTTSService): def _build_msg(self, text: str = "") -> dict: """Build JSON message for Rime API.""" - return {"text": text, "contextId": self._context_id} + msg = {"text": text, "contextId": self._context_id} + if self._extra_msg_fields: + msg |= self._extra_msg_fields + self._extra_msg_fields = {} + return msg def _build_clear_msg(self) -> dict: """Build clear operation message."""