From 2784b0f438339a9e98248060c20c28616b29b13f Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 19 Nov 2025 03:24:57 +0530 Subject: [PATCH 01/25] Add RimeNonJsonTTSService for non-JSON WebSocket API support This commit introduces the RimeNonJsonTTSService class, enabling Text-to-Speech synthesis over WebSocket endpoints that require plain text messages. The service includes configuration parameters for language, segmentation, and audio settings, and handles WebSocket connections for raw audio byte transmission. Limitations include the lack of support for word-level timestamps and context IDs. --- src/pipecat/services/rime/tts.py | 245 ++++++++++++++++++++++++++++++- 1 file changed, 244 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 7b62f20fa..f747a62bf 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -17,6 +17,7 @@ from typing import Any, AsyncGenerator, Mapping, Optional import aiohttp from loguru import logger +from pipecat.transcriptions import language from pydantic import BaseModel from pipecat.frames.frames import ( @@ -31,7 +32,11 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.tts_service import AudioContextWordTTSService, TTSService +from pipecat.services.tts_service import ( + AudioContextWordTTSService, + InterruptibleTTSService, + TTSService, +) from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.text.base_text_aggregator import BaseTextAggregator from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator @@ -574,3 +579,241 @@ class RimeHttpTTSService(TTSService): finally: await self.stop_ttfb_metrics() yield TTSStoppedFrame() + + +class RimeNonJsonTTSService(InterruptibleTTSService): + """ + Pipecat TTS service for Rime's non-JSON WebSocket API. + + This service enables Text-to-Speech synthesis over WebSocket endpoints that require plain text (not JSON) messages and return raw audio bytes. It is designed for use with TTS models like Arcana, which currently do not support JSON-based WebSocket protocols (though this may change in the future). + + Limitations: + - Does not support word-level timestamps or context IDs. + - Intended specifically for integrations where the TTS provider only accepts and returns non-JSON messages. + - Arcana and similar models may add JSON WebSocket support in the future; this service focuses on the current plain text protocol. + """ + + class InputParams(BaseModel): + """Configuration parameters for Rime WebSocket TTS service. + + Parameters: + language: Language for synthesis. Defaults to English. + segment: Text segmentation mode ("immediate", "bySentence", "never"). + repetition_penalty: Token repetition penalty (1.0-2.0). + temperature: Sampling temperature (0.0-1.0). + top_p: Cumulative probability threshold (0.0-1.0). + extra: Additional parameters to pass to the API (for future compatibility). + """ + + language: Optional[Language] = None + segment: Optional[str] = None + repetition_penalty: Optional[float] = None + temperature: Optional[float] = None + top_p: Optional[float] = None + extra: Optional[dict[str, Any]] = None + + def __init__( + self, + *, + api_key: str, + voice_id: str, + url: str = "wss://users.rime.ai/ws", + model: str = "arcana", + audio_format: str = "pcm", + sample_rate: Optional[int] = 24000, + params: Optional[InputParams] = None, + aggregate_sentences: Optional[bool] = True, + **kwargs, + ): + """ + Initialize Rime Non-JSON WebSocket TTS service. + Args: + api_key: Rime API key for authentication. + voice_id: ID of the voice to use. + url: Rime websocket API endpoint. + model: Model ID to use for synthesis. + audio_format: Audio format to use. + sample_rate: Audio sample rate in Hz. + params: Additional configuration parameters. + aggregate_sentences: Whether to aggregate sentences within the TTSService. + **kwargs: Additional arguments passed to parent class. + """ + super().__init__( + sample_rate=sample_rate, + aggregate_sentences=aggregate_sentences, + push_stop_frames=True, + pause_frame_processing=True, + **kwargs, + ) + params = params or RimeNonJsonTTSService.InputParams() + + self._settings = { + "speaker": voice_id, + "modelId": model, + "audioFormat": audio_format, + "samplingRate": sample_rate, + } + + if params.language: + self._settings["lang"] = self.language_to_service_language(params.language) + if params.segment is not None: + self._settings["segment"] = params.segment + if params.repetition_penalty is not None: + self._settings["repetition_penalty"] = params.repetition_penalty + if params.temperature is not None: + self._settings["temperature"] = params.temperature + if params.top_p is not None: + self._settings["top_p"] = params.top_p + # Add any extra parameters for future compatibility + if params.extra: + self._settings.update(params.extra) + + self._started = False + self._receive_task = None + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + Returns: + True, as Rime Non-JSON WebSocket service supports metrics generation. + """ + return True + + def language_to_service_language(self, language: Language) -> str: + """Convert pipecat Language enum to Rime language code. + + Args: + language: The Language enum value to convert. + + Returns: + Three-letter Rime language code (e.g., 'eng' for English). + Falls back to the language's base code with a warning if not in the verified list. + """ + return language_to_rime_language(language) + + async def start(self, frame: StartFrame): + """Start the Rime Non-JSON WebSocket TTS service. + Args: + frame: The start frame containing initialization parameters. + """ + await super().start(frame) + self._settings["samplingRate"] = self.sample_rate + await self._connect() + + async def stop(self, frame: EndFrame): + """Stop the service and close connection.""" + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + """Cancel current operation and clean up.""" + await super().cancel(frame) + await self._disconnect() + + async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame downstream with special handling for stop conditions. + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ + await super().push_frame(frame, direction) + if isinstance(frame, (TTSStoppedFrame, InterruptionFrame)): + self._started = False + + async def _connect(self): + """Establish WebSocket connection and start receive task.""" + await self._connect_websocket() + if self._websocket and not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) + + async def _disconnect(self): + """Close WebSocket connection and clean up tasks.""" + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + await self._disconnect_websocket() + + async def _connect_websocket(self): + """Establish WebSocket connection to Rime None json websocket.""" + try: + if self._websocket and self._websocket.state is State.OPEN: + return + # Build URL with query parameters (only non-None values) + params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None) + url = f"{self._url}?{params}" + headers = {"Authorization": f"Bearer {self._api_key}"} + self._websocket = await websocket_connect(url, additional_headers=headers) + await self._call_event_handler("on_connected") + except Exception as e: + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") + + async def _disconnect_websocket(self): + """Close WebSocket connection and clean up state.""" + try: + await self.stop_all_metrics() + if self._websocket: + # Send EOS command to gracefully close + await self._websocket.send("") + await self._websocket.close() + logger.debug("Disconnected from Rime None json websocket") + except Exception as e: + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + finally: + self._started = False + self._websocket = None + await self._call_event_handler("on_disconnected") + + def _get_websocket(self): + """Get active WebSocket connection or raise exception.""" + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + + async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection): + """Handle interruption by clearing buffer.""" + await super()._handle_interruption(frame, direction) + await self.stop_all_metrics() + if self._websocket: + # Send CLEAR command to clear buffer + await self._websocket.send("") + + async def flush_audio(self): + """Flush any pending audio synthesis.""" + if not self._websocket: + return + + logger.trace(f"{self}: flushing audio") + await self._websocket.send("") + + async def _receive_messages(self): + """Process incoming WebSocket messages (raw audio bytes).""" + async for message in self._get_websocket(): + try: + # Rime Arcana sends raw audio bytes directly (not JSON) + if isinstance(message, bytes): + await self.stop_ttfb_metrics() + + frame = TTSAudioRawFrame( + audio=message, + sample_rate=self.sample_rate, + num_channels=1, + ) + await self.push_frame(frame) + except Exception as e: + logger.error(f"{self} exception: {e}") + await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + + @traced_tts + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """ + Generate speech from text using Rime's streaming API. + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ + logger.debug(f"{self}: Generating TTS [{text}]") \ No newline at end of file From 0dcb65bd5613271b939b8b19696206d387c68793 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 19 Nov 2025 03:34:58 +0530 Subject: [PATCH 02/25] add run tts methos for rimeNonJsonTTs --- src/pipecat/services/rime/tts.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index f747a62bf..926e07681 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -816,4 +816,27 @@ class RimeNonJsonTTSService(InterruptibleTTSService): Yields: Frame: Audio frames containing the synthesized speech. """ - logger.debug(f"{self}: Generating TTS [{text}]") \ No newline at end of file + logger.debug(f"{self}: Generating TTS [{text}]") + try: + if not self._websocket or self._websocket.state is State.CLOSED: + await self._connect() + try: + if not self._started: + await self.start_ttfb_metrics() + yield TTSStartedFrame() + self._started = True + # Send bare text (not JSON) + await self._get_websocket().send(text) + await self.start_tts_usage_metrics(text) + + except Exception as e: + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") + yield TTSStoppedFrame() + await self._disconnect() + await self._connect() + return + yield None + except Exception as e: + logger.error(f"{self} exception: {e}") + yield ErrorFrame(error=f"{self} error: {e}") From 7bbb5be910119b3cf375f2b214beae9e7a992b75 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 19 Nov 2025 03:35:54 +0530 Subject: [PATCH 03/25] format fix --- src/pipecat/services/rime/tts.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 926e07681..1213fb41a 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -17,7 +17,6 @@ from typing import Any, AsyncGenerator, Mapping, Optional import aiohttp from loguru import logger -from pipecat.transcriptions import language from pydantic import BaseModel from pipecat.frames.frames import ( @@ -37,6 +36,7 @@ from pipecat.services.tts_service import ( InterruptibleTTSService, TTSService, ) +from pipecat.transcriptions import language from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.text.base_text_aggregator import BaseTextAggregator from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator @@ -582,8 +582,7 @@ class RimeHttpTTSService(TTSService): class RimeNonJsonTTSService(InterruptibleTTSService): - """ - Pipecat TTS service for Rime's non-JSON WebSocket API. + """Pipecat TTS service for Rime's non-JSON WebSocket API. This service enables Text-to-Speech synthesis over WebSocket endpoints that require plain text (not JSON) messages and return raw audio bytes. It is designed for use with TTS models like Arcana, which currently do not support JSON-based WebSocket protocols (though this may change in the future). @@ -625,8 +624,8 @@ class RimeNonJsonTTSService(InterruptibleTTSService): aggregate_sentences: Optional[bool] = True, **kwargs, ): - """ - Initialize Rime Non-JSON WebSocket TTS service. + """Initialize Rime Non-JSON WebSocket TTS service. + Args: api_key: Rime API key for authentication. voice_id: ID of the voice to use. @@ -673,6 +672,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService): def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. + Returns: True, as Rime Non-JSON WebSocket service supports metrics generation. """ @@ -692,6 +692,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService): async def start(self, frame: StartFrame): """Start the Rime Non-JSON WebSocket TTS service. + Args: frame: The start frame containing initialization parameters. """ @@ -711,6 +712,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService): async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): """Push a frame downstream with special handling for stop conditions. + Args: frame: The frame to push. direction: The direction to push the frame. @@ -808,8 +810,8 @@ class RimeNonJsonTTSService(InterruptibleTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """ - Generate speech from text using Rime's streaming API. + """Generate speech from text using Rime's streaming API. + Args: text: The text to synthesize into speech. From 66c58f815515d0b4b24aa7ce4ddd79ff53d32dcd Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 19 Nov 2025 03:40:59 +0530 Subject: [PATCH 04/25] fix --- src/pipecat/services/rime/tts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 1213fb41a..3fa1b549a 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -645,7 +645,10 @@ class RimeNonJsonTTSService(InterruptibleTTSService): **kwargs, ) params = params or RimeNonJsonTTSService.InputParams() - + self._api_key = api_key + self._url = url + self._voice_id = voice_id + self._model = model self._settings = { "speaker": voice_id, "modelId": model, From 88daad524eaf8f8651206953f2d80e9f9450c2db Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 19 Nov 2025 03:43:49 +0530 Subject: [PATCH 05/25] Refactor whitespace in RimeNonJsonTTSService to improve code readability --- src/pipecat/services/rime/tts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 3fa1b549a..b1d5c1ada 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -645,10 +645,10 @@ class RimeNonJsonTTSService(InterruptibleTTSService): **kwargs, ) params = params or RimeNonJsonTTSService.InputParams() - self._api_key = api_key + self._api_key = api_key self._url = url - self._voice_id = voice_id - self._model = model + self._voice_id = voice_id + self._model = model self._settings = { "speaker": voice_id, "modelId": model, From 43931911669ebc498e94b1b4b62295c16c090925 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 19 Nov 2025 04:53:21 +0530 Subject: [PATCH 06/25] Add method to update settings in RimeNonJsonTTSService --- src/pipecat/services/rime/tts.py | 58 ++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index b1d5c1ada..251cd3bd6 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -845,3 +845,61 @@ class RimeNonJsonTTSService(InterruptibleTTSService): except Exception as e: logger.error(f"{self} exception: {e}") yield ErrorFrame(error=f"{self} error: {e}") + + async def _update_settings(self, settings: Mapping[str, Any]): + """Update service settings and reconnect if necessary. + + Since all settings are WebSocket URL query parameters, + any setting change requires reconnecting to apply the new values. + """ + needs_reconnect = False + + # Track previous values from self._settings only + prev_settings = self._settings.copy() + + # Let parent class handle standard settings (voice, model, language) + await super()._update_settings(settings) + + # Check if voice changed and update settings dict + if "voice" in settings or "voice_id" in settings: + self._settings["speaker"] = self._voice_id + if prev_settings.get("speaker") != self._voice_id: + logger.info(f"Switching TTS voice to: [{self._voice_id}]") + needs_reconnect = True + + # Check if model changed and update settings dict + if "model" in settings: + self._settings["modelId"] = self._model + if prev_settings.get("modelId") != self._model: + logger.info(f"Switching TTS model to: [{self._model}]") + needs_reconnect = True + + # Handle language explicitly + if "language" in settings: + new_lang = self.language_to_service_language(settings["language"]) + if new_lang and new_lang != prev_settings.get("lang"): + logger.info(f"Updating language to: [{new_lang}]") + self._settings["lang"] = new_lang + needs_reconnect = True + + # Check other parameters + for key in ["segment", "repetition_penalty", "temperature", "top_p"]: + if key in settings and settings[key] != prev_settings.get(key): + logger.info(f"Updating {key} to: [{settings[key]}]") + self._settings[key] = settings[key] + needs_reconnect = True + + # Handle extra parameters + for key, value in settings.items(): + if key not in ["voice", "voice_id", "model", "language", "segment", + "repetition_penalty", "temperature", "top_p", "audio_format"]: + if value != prev_settings.get(key): + logger.info(f"Updating extra parameter {key} to: [{value}]") + self._settings[key] = value + needs_reconnect = True + + # Reconnect if any setting changed + if needs_reconnect: + logger.debug("Settings changed, reconnecting WebSocket with new parameters") + await self._disconnect() + await self._connect() From 860d9c4f2900a66001a46e4036ba18f0546ce739 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 19 Nov 2025 04:53:27 +0530 Subject: [PATCH 07/25] Refactor _update_settings method in RimeNonJsonTTSService for improved readability and maintainability --- src/pipecat/services/rime/tts.py | 123 +++++++++++++++++-------------- 1 file changed, 66 insertions(+), 57 deletions(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 251cd3bd6..db36d524e 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -845,61 +845,70 @@ class RimeNonJsonTTSService(InterruptibleTTSService): except Exception as e: logger.error(f"{self} exception: {e}") yield ErrorFrame(error=f"{self} error: {e}") - - async def _update_settings(self, settings: Mapping[str, Any]): - """Update service settings and reconnect if necessary. - - Since all settings are WebSocket URL query parameters, - any setting change requires reconnecting to apply the new values. - """ - needs_reconnect = False - - # Track previous values from self._settings only - prev_settings = self._settings.copy() - - # Let parent class handle standard settings (voice, model, language) - await super()._update_settings(settings) - - # Check if voice changed and update settings dict - if "voice" in settings or "voice_id" in settings: - self._settings["speaker"] = self._voice_id - if prev_settings.get("speaker") != self._voice_id: - logger.info(f"Switching TTS voice to: [{self._voice_id}]") - needs_reconnect = True - - # Check if model changed and update settings dict - if "model" in settings: - self._settings["modelId"] = self._model - if prev_settings.get("modelId") != self._model: - logger.info(f"Switching TTS model to: [{self._model}]") - needs_reconnect = True - - # Handle language explicitly - if "language" in settings: - new_lang = self.language_to_service_language(settings["language"]) - if new_lang and new_lang != prev_settings.get("lang"): - logger.info(f"Updating language to: [{new_lang}]") - self._settings["lang"] = new_lang - needs_reconnect = True - - # Check other parameters - for key in ["segment", "repetition_penalty", "temperature", "top_p"]: - if key in settings and settings[key] != prev_settings.get(key): - logger.info(f"Updating {key} to: [{settings[key]}]") - self._settings[key] = settings[key] - needs_reconnect = True - - # Handle extra parameters - for key, value in settings.items(): - if key not in ["voice", "voice_id", "model", "language", "segment", - "repetition_penalty", "temperature", "top_p", "audio_format"]: - if value != prev_settings.get(key): - logger.info(f"Updating extra parameter {key} to: [{value}]") - self._settings[key] = value - needs_reconnect = True - - # Reconnect if any setting changed - if needs_reconnect: - logger.debug("Settings changed, reconnecting WebSocket with new parameters") - await self._disconnect() + + async def _update_settings(self, settings: Mapping[str, Any]): + """Update service settings and reconnect if necessary. + + Since all settings are WebSocket URL query parameters, + any setting change requires reconnecting to apply the new values. + """ + needs_reconnect = False + + # Track previous values from self._settings only + prev_settings = self._settings.copy() + + # Let parent class handle standard settings (voice, model, language) + await super()._update_settings(settings) + + # Check if voice changed and update settings dict + if "voice" in settings or "voice_id" in settings: + self._settings["speaker"] = self._voice_id + if prev_settings.get("speaker") != self._voice_id: + logger.info(f"Switching TTS voice to: [{self._voice_id}]") + needs_reconnect = True + + # Check if model changed and update settings dict + if "model" in settings: + self._settings["modelId"] = self._model + if prev_settings.get("modelId") != self._model: + logger.info(f"Switching TTS model to: [{self._model}]") + needs_reconnect = True + + # Handle language explicitly + if "language" in settings: + new_lang = self.language_to_service_language(settings["language"]) + if new_lang and new_lang != prev_settings.get("lang"): + logger.info(f"Updating language to: [{new_lang}]") + self._settings["lang"] = new_lang + needs_reconnect = True + + # Check other parameters + for key in ["segment", "repetition_penalty", "temperature", "top_p"]: + if key in settings and settings[key] != prev_settings.get(key): + logger.info(f"Updating {key} to: [{settings[key]}]") + self._settings[key] = settings[key] + needs_reconnect = True + + # Handle extra parameters + for key, value in settings.items(): + if key not in [ + "voice", + "voice_id", + "model", + "language", + "segment", + "repetition_penalty", + "temperature", + "top_p", + "audio_format", + ]: + if value != prev_settings.get(key): + logger.info(f"Updating extra parameter {key} to: [{value}]") + self._settings[key] = value + needs_reconnect = True + + # Reconnect if any setting changed + if needs_reconnect: + logger.debug("Settings changed, reconnecting WebSocket with new parameters") + await self._disconnect() await self._connect() From afeef94900d66353b4e6082a145dae457c0d3299 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 19 Nov 2025 04:55:14 +0530 Subject: [PATCH 08/25] Remove unused audio_format parameter from extra settings in RimeNonJsonTTSService --- src/pipecat/services/rime/tts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index db36d524e..a11329309 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -900,7 +900,6 @@ class RimeNonJsonTTSService(InterruptibleTTSService): "repetition_penalty", "temperature", "top_p", - "audio_format", ]: if value != prev_settings.get(key): logger.info(f"Updating extra parameter {key} to: [{value}]") From 1d9696e6149f0111719271c7ee8541996fca089c Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 19 Nov 2025 22:19:00 +0530 Subject: [PATCH 09/25] Add audio flushing after sending text in RimeNonJsonTTSService This update ensures that audio is flushed immediately after sending bare text to the WebSocket, improving the responsiveness of the Text-to-Speech service. --- src/pipecat/services/rime/tts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index a11329309..075627d17 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -833,6 +833,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService): # Send bare text (not JSON) await self._get_websocket().send(text) await self.start_tts_usage_metrics(text) + await self.flush_audio() except Exception as e: logger.error(f"{self} exception: {e}") From da671cd23240bc6825ed9e28bc1272c0e121bc3a Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 19 Nov 2025 22:19:36 +0530 Subject: [PATCH 10/25] Fix whitespace inconsistency in audio flushing method of RimeNonJsonTTSService --- src/pipecat/services/rime/tts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 075627d17..6dfaadf45 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -833,7 +833,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService): # Send bare text (not JSON) await self._get_websocket().send(text) await self.start_tts_usage_metrics(text) - await self.flush_audio() + await self.flush_audio() except Exception as e: logger.error(f"{self} exception: {e}") From de4e9c54f6fa2fade9c2075bdd4d6b18b269d3dc Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 19 Nov 2025 22:44:50 +0530 Subject: [PATCH 11/25] Increase WebSocket max size limit in RimeNonJsonTTSService to enhance data handling capacity --- src/pipecat/services/rime/tts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 6dfaadf45..56d255e99 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -746,7 +746,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService): params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None) url = f"{self._url}?{params}" headers = {"Authorization": f"Bearer {self._api_key}"} - self._websocket = await websocket_connect(url, additional_headers=headers) + self._websocket = await websocket_connect(url, additional_headers=headers, max_size=1024 * 1024 * 16) await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} exception: {e}") From cc861d6b70a350b196330a7e031609ad5a8c2054 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 19 Nov 2025 22:46:36 +0530 Subject: [PATCH 12/25] Refactor WebSocket connection code in RimeNonJsonTTSService for improved readability --- src/pipecat/services/rime/tts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 56d255e99..5c17ae664 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -746,7 +746,9 @@ class RimeNonJsonTTSService(InterruptibleTTSService): params = "&".join(f"{k}={v}" for k, v in self._settings.items() if v is not None) url = f"{self._url}?{params}" headers = {"Authorization": f"Bearer {self._api_key}"} - self._websocket = await websocket_connect(url, additional_headers=headers, max_size=1024 * 1024 * 16) + self._websocket = await websocket_connect( + url, additional_headers=headers, max_size=1024 * 1024 * 16 + ) await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} exception: {e}") From 0707141998d89658f00aedb6668111b21d1f2e94 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Thu, 20 Nov 2025 01:36:35 +0530 Subject: [PATCH 13/25] fix --- src/pipecat/services/rime/tts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 5c17ae664..924ca3986 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -835,7 +835,6 @@ class RimeNonJsonTTSService(InterruptibleTTSService): # Send bare text (not JSON) await self._get_websocket().send(text) await self.start_tts_usage_metrics(text) - await self.flush_audio() except Exception as e: logger.error(f"{self} exception: {e}") From e5fb643cf571afa50d877ef9e28d153460fabaa6 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Mon, 8 Dec 2025 19:45:13 +0530 Subject: [PATCH 14/25] Improve docstring formatting in RimeNonJsonTTSService for better readability --- src/pipecat/services/rime/tts.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 924ca3986..e1932ed1d 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -584,12 +584,18 @@ class RimeHttpTTSService(TTSService): class RimeNonJsonTTSService(InterruptibleTTSService): """Pipecat TTS service for Rime's non-JSON WebSocket API. - This service enables Text-to-Speech synthesis over WebSocket endpoints that require plain text (not JSON) messages and return raw audio bytes. It is designed for use with TTS models like Arcana, which currently do not support JSON-based WebSocket protocols (though this may change in the future). + This service enables Text-to-Speech synthesis over WebSocket endpoints + that require plain text (not JSON) messages and return raw audio bytes. + It is designed for use with TTS models like Arcana, which currently do + not support JSON-based WebSocket protocols (though this may change in + the future). Limitations: - Does not support word-level timestamps or context IDs. - - Intended specifically for integrations where the TTS provider only accepts and returns non-JSON messages. - - Arcana and similar models may add JSON WebSocket support in the future; this service focuses on the current plain text protocol. + - Intended specifically for integrations where the TTS provider only + accepts and returns non-JSON messages. + - Arcana and similar models may add JSON WebSocket support in the + future; this service focuses on the current plain text protocol. """ class InputParams(BaseModel): From 12093fcffcfca4a945a7d91fe04622adc7695c52 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Mon, 8 Dec 2025 19:50:38 +0530 Subject: [PATCH 15/25] Update default sample_rate parameter in RimeNonJsonTTSService to None for flexibility --- src/pipecat/services/rime/tts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index e1932ed1d..8b81d5b86 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -625,7 +625,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService): url: str = "wss://users.rime.ai/ws", model: str = "arcana", audio_format: str = "pcm", - sample_rate: Optional[int] = 24000, + sample_rate: Optional[int] = None, params: Optional[InputParams] = None, aggregate_sentences: Optional[bool] = True, **kwargs, From 5bfea84bd5c365e593b46e71e8f9d076beefd9ab Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Mon, 8 Dec 2025 20:30:46 +0530 Subject: [PATCH 16/25] Refactor RimeNonJsonTTSService to extend WebsocketTTSService, enhancing WebSocket functionality and improving code clarity --- src/pipecat/services/rime/tts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 8b81d5b86..ed8afae59 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -33,8 +33,8 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection from pipecat.services.tts_service import ( AudioContextWordTTSService, - InterruptibleTTSService, TTSService, + WebsocketTTSService, ) from pipecat.transcriptions import language from pipecat.transcriptions.language import Language, resolve_language @@ -581,7 +581,7 @@ class RimeHttpTTSService(TTSService): yield TTSStoppedFrame() -class RimeNonJsonTTSService(InterruptibleTTSService): +class RimeNonJsonTTSService(WebsocketTTSService): """Pipecat TTS service for Rime's non-JSON WebSocket API. This service enables Text-to-Speech synthesis over WebSocket endpoints @@ -770,7 +770,7 @@ class RimeNonJsonTTSService(InterruptibleTTSService): # Send EOS command to gracefully close await self._websocket.send("") await self._websocket.close() - logger.debug("Disconnected from Rime None json websocket") + logger.debug("Disconnected from Rime non json websocket") except Exception as e: logger.error(f"{self} exception: {e}") await self.push_error(ErrorFrame(error=f"{self} error: {e}")) From f4e33fc8dde23a729ba5dfc7f13a1dd30c85f0d7 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Mon, 8 Dec 2025 20:32:13 +0530 Subject: [PATCH 17/25] Update docstrings in RimeNonJsonTTSService for clarity and consistency, specifying 'Non-JSON' in relevant descriptions. --- src/pipecat/services/rime/tts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index ed8afae59..d424491b9 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -599,9 +599,9 @@ class RimeNonJsonTTSService(WebsocketTTSService): """ class InputParams(BaseModel): - """Configuration parameters for Rime WebSocket TTS service. + """Configuration parameters for Rime Non-JSON WebSocket TTS service. - Parameters: + Args: language: Language for synthesis. Defaults to English. segment: Text segmentation mode ("immediate", "bySentence", "never"). repetition_penalty: Token repetition penalty (1.0-2.0). @@ -744,7 +744,7 @@ class RimeNonJsonTTSService(WebsocketTTSService): await self._disconnect_websocket() async def _connect_websocket(self): - """Establish WebSocket connection to Rime None json websocket.""" + """Establish WebSocket connection to Rime non-JSON websocket.""" try: if self._websocket and self._websocket.state is State.OPEN: return @@ -770,7 +770,7 @@ class RimeNonJsonTTSService(WebsocketTTSService): # Send EOS command to gracefully close await self._websocket.send("") await self._websocket.close() - logger.debug("Disconnected from Rime non json websocket") + logger.debug("Disconnected from Rime non-JSON websocket") except Exception as e: logger.error(f"{self} exception: {e}") await self.push_error(ErrorFrame(error=f"{self} error: {e}")) From 329b8ac4261d6c6bbe41d641761dfb754c062272 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Mon, 8 Dec 2025 21:06:48 +0530 Subject: [PATCH 18/25] Refactor error handling in RimeNonJsonTTSService to provide a more generic error message, improving clarity in error reporting. --- src/pipecat/services/rime/tts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index df71668b7..d29cbcdff 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -885,8 +885,7 @@ class RimeNonJsonTTSService(WebsocketTTSService): return yield None except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") async def _update_settings(self, settings: Mapping[str, Any]): """Update service settings and reconnect if necessary. From 924831089ca65fbf5dcdaf7882a0c1edae8ddc3e Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Mon, 8 Dec 2025 21:17:01 +0530 Subject: [PATCH 19/25] Enhance error handling in RimeNonJsonTTSService by standardizing error messages for improved clarity and consistency in reporting. --- src/pipecat/services/rime/tts.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index d29cbcdff..ddf9b35e9 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -850,8 +850,7 @@ class RimeNonJsonTTSService(WebsocketTTSService): ) await self.push_frame(frame) except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Error: {e}", exception=e) @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: @@ -877,8 +876,7 @@ class RimeNonJsonTTSService(WebsocketTTSService): await self.start_tts_usage_metrics(text) except Exception as e: - logger.error(f"{self} exception: {e}") - yield ErrorFrame(error=f"{self} error: {e}") + yield ErrorFrame(error=f"Unknown error occurred: {e}") yield TTSStoppedFrame() await self._disconnect() await self._connect() From 0b4d984be6af969bcb9d08121714b652b8f0366f Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Mon, 8 Dec 2025 21:24:30 +0530 Subject: [PATCH 20/25] Standardize error handling in RimeNonJsonTTSService by replacing specific error messages with a generic "Unknown error occurred" format, enhancing consistency in error reporting. --- src/pipecat/services/rime/tts.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index ddf9b35e9..2c41667e9 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -791,8 +791,7 @@ class RimeNonJsonTTSService(WebsocketTTSService): ) await self._call_event_handler("on_connected") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -806,8 +805,7 @@ class RimeNonJsonTTSService(WebsocketTTSService): await self._websocket.close() logger.debug("Disconnected from Rime non-JSON websocket") except Exception as e: - logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e) finally: self._started = False self._websocket = None From 99f89351fa8b7d81b459c2859e818909a84bb794 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Mon, 8 Dec 2025 21:32:50 +0530 Subject: [PATCH 21/25] Add support for non-JSON streaming mode in RimeTTSService, enabling both JSON and raw audio WebSocket streaming for enhanced performance and flexibility. --- changelog/3085.added.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog/3085.added.md diff --git a/changelog/3085.added.md b/changelog/3085.added.md new file mode 100644 index 000000000..d46455c2d --- /dev/null +++ b/changelog/3085.added.md @@ -0,0 +1,4 @@ +- Added support for non-JSON streaming mode in `RimeTTSService`. The service now + supports both JSON and non-JSON (raw audio) WebSocket streaming modes for + improved performance and flexibility. + From 4fcb099fd7988582c681d3f33724aa3947e0a697 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Tue, 9 Dec 2025 02:43:57 +0530 Subject: [PATCH 22/25] Add RimeNonJsonTTSService to support non-JSON streaming mode, enabling WebSocket streaming for the Arcana model. --- changelog/3085.added.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/changelog/3085.added.md b/changelog/3085.added.md index d46455c2d..c1effe44b 100644 --- a/changelog/3085.added.md +++ b/changelog/3085.added.md @@ -1,4 +1,2 @@ -- Added support for non-JSON streaming mode in `RimeTTSService`. The service now - supports both JSON and non-JSON (raw audio) WebSocket streaming modes for - improved performance and flexibility. +- Added `RimeNonJsonTTSService` which supports non-JSON streaming mode. This new class supports websocket streaming for the Arcana model. From 6ca117a3c1ba9324826e8b3190dd7794ccb818a0 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Tue, 9 Dec 2025 02:45:17 +0530 Subject: [PATCH 23/25] Remove unused import of 'language' in tts.py to clean up the code and improve readability. --- src/pipecat/services/rime/tts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 2c41667e9..a25ea8f36 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -36,7 +36,6 @@ from pipecat.services.tts_service import ( TTSService, WebsocketTTSService, ) -from pipecat.transcriptions import language from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.text.base_text_aggregator import BaseTextAggregator from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator From 2a6a0d83db7d87f14b9e9ed719fa7b8229e69149 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Tue, 9 Dec 2025 02:49:37 +0530 Subject: [PATCH 24/25] Update docstring in RimeNonJsonTTSService to clarify the focus on the current plain text protocol and note potential future support for JSON WebSocket. --- src/pipecat/services/rime/tts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index a25ea8f36..b3082272f 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -627,8 +627,10 @@ class RimeNonJsonTTSService(WebsocketTTSService): - Does not support word-level timestamps or context IDs. - Intended specifically for integrations where the TTS provider only accepts and returns non-JSON messages. + + Note: - Arcana and similar models may add JSON WebSocket support in the - future; this service focuses on the current plain text protocol. + future. This service focuses on the current plain text protocol. """ class InputParams(BaseModel): From 455579ffcc4cf3d5827ddebd9f56af196bd22766 Mon Sep 17 00:00:00 2001 From: Gokul Js Date: Wed, 10 Dec 2025 04:56:52 +0530 Subject: [PATCH 25/25] Refactor RimeNonJsonTTSService to extend InterruptibleTTSService, removing dependency on WebsocketTTSService and streamlining audio interruption handling. --- src/pipecat/services/rime/tts.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index b3082272f..7f95ffd21 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -33,8 +33,8 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection from pipecat.services.tts_service import ( AudioContextWordTTSService, + InterruptibleTTSService, TTSService, - WebsocketTTSService, ) from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.text.base_text_aggregator import BaseTextAggregator @@ -614,7 +614,7 @@ class RimeHttpTTSService(TTSService): yield TTSStoppedFrame() -class RimeNonJsonTTSService(WebsocketTTSService): +class RimeNonJsonTTSService(InterruptibleTTSService): """Pipecat TTS service for Rime's non-JSON WebSocket API. This service enables Text-to-Speech synthesis over WebSocket endpoints @@ -818,14 +818,6 @@ class RimeNonJsonTTSService(WebsocketTTSService): return self._websocket raise Exception("Websocket not connected") - async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection): - """Handle interruption by clearing buffer.""" - await super()._handle_interruption(frame, direction) - await self.stop_all_metrics() - if self._websocket: - # Send CLEAR command to clear buffer - await self._websocket.send("") - async def flush_audio(self): """Flush any pending audio synthesis.""" if not self._websocket: