cartesia tts support

src/pipecat/services/cartesia.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+from cartesia.tts import AsyncCartesiaTTS
+
+import time
+from typing import AsyncGenerator
+
+from pipecat.frames.frames import AudioRawFrame, ErrorFrame, Frame
+from pipecat.services.ai_services import TTSService
+
+from loguru import logger
+
+
+class CartesiaTTSService(TTSService):
+
+    def __init__(
+            self,
+            *,
+            api_key: str,
+            voice_name: str,
+            **kwargs):
+        super().__init__(**kwargs)
+
+        self._api_key = api_key
+        self._voice_name = voice_name
+
+        self._client = None
+
+    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+        logger.debug(f"Transcribing text: [{text}]")
+
+        try:
+            if self._client is None:
+                self._client = AsyncCartesiaTTS(api_key=self._api_key)
+                voices = self._client.get_voices()
+                self._voice_id = voices[self._voice_name]["id"]
+                self._voice = self._client.get_voice_embedding(voice_id=self._voice_id)
+
+            chunk_generator = await self._client.generate(
+                transcript=text, voice=self._voice, stream=True,
+                model_id="upbeat-moon", data_rtype='array', output_format='pcm_16000',
+                # a chunk_time of 0.1 seems to be the default. there are small audio pops/gaps which
+                # we need to debug
+                chunk_time=0.1
+            )
+
+            async for chunk in chunk_generator:
+                # print(f"")
+                frame = AudioRawFrame(chunk['audio'], 16000, 1)
+                yield frame
+        except Exception as e:
+            logger.error(f"Exception {e}")

src/pipecat/services/deepgram.py

@@ -5,6 +5,7 @@
 #
 
 import aiohttp
+import json
 
 from typing import AsyncGenerator
 
@@ -32,17 +33,21 @@ class DeepgramTTSService(TTSService):
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
         logger.info(f"Running Deepgram TTS for {text}")
         base_url = "https://api.deepgram.com/v1/speak"
-        request_url = f"{base_url}?model={self._voice}&encoding=linear16&container=none&sample_rate=16000"
+        request_url = f"{base_url}?model={
+            self._voice}&encoding=linear16&container=none&sample_rate=16000"
         headers = {"authorization": f"token {self._api_key}"}
         body = {"text": text}
 
-        async with self._aiohttp_session.post(request_url, headers=headers, json=body) as r:
-            if r.status != 200:
-                text = await r.text()
-                logger.error(f"Error getting audio (status: {r.status}, error: {text})")
-                yield ErrorFrame(f"Error getting audio (status: {r.status}, error: {text})")
-                return
+        try:
+            async with self._aiohttp_session.post(request_url, headers=headers, json=body) as r:
+                if r.status != 200:
+                    text = await r.text()
+                    logger.error(f"Error getting audio (status: {r.status}, error: {text})")
+                    yield ErrorFrame(f"Error getting audio (status: {r.status}, error: {text})")
+                    return
 
-            async for data in r.content:
-                frame = AudioRawFrame(audio=data, sample_rate=16000, num_channels=1)
-                yield frame
+                async for data in r.content:
+                    frame = AudioRawFrame(audio=data, sample_rate=16000, num_channels=1)
+                    yield frame
+        except Exception as e:
+            logger.error(f"Exception {e}")

src/pipecat/services/elevenlabs.py

@@ -8,7 +8,7 @@ import aiohttp
 
 from typing import AsyncGenerator
 
-from pipecat.frames.frames import AudioRawFrame, ErrorFrame, Frame, TTSStartedFrame, TTSStoppedFrame, TextFrame
+from pipecat.frames.frames import AudioRawFrame, ErrorFrame, Frame
 from pipecat.services.ai_services import TTSService
 
 from loguru import logger

src/pipecat/services/google.py

@@ -86,9 +86,18 @@ class GoogleLLMService(LLMService):
             logger.debug(f"Google LLM TTFB: {time.time() - start_time}")
 
             async for chunk in self._async_generator_wrapper(response):
-                await self.push_frame(LLMResponseStartFrame())
-                await self.push_frame(TextFrame(chunk.text))
-                await self.push_frame(LLMResponseEndFrame())
+                try:
+                    text = chunk.text
+                    await self.push_frame(LLMResponseStartFrame())
+                    await self.push_frame(TextFrame(text))
+                    await self.push_frame(LLMResponseEndFrame())
+                except Exception as e:
+                    # Google LLMs seem to flag safety issues a lot!
+                    if chunk.candidates[0].finish_reason == 3:
+                        logger.debug(
+                            f"LLM refused to generate content for safety reasons - {messages}.")
+                    else:
+                        logger.error(f"Error {e}")
 
         except Exception as e:
             logger.error(f"Exception: {e}")