From 3931cb32355d5ac74eeb82e52b6f6f3427b710e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= <aleix@daily.co>
Date: Thu, 22 Aug 2024 22:44:11 -0700
Subject: [PATCH] services(cartesia): allow setting language and language
 voices

---
 src/pipecat/services/ai_services.py | 18 +++++++++++++++++-
 src/pipecat/services/cartesia.py    | 26 ++++++++++++++++++++++++--
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py
index d4a3c021e..8208986ef 100644
--- a/src/pipecat/services/ai_services.py
+++ b/src/pipecat/services/ai_services.py
@@ -9,7 +9,7 @@ import io
 import wave
 
 from abc import abstractmethod
-from typing import AsyncGenerator, Optional
+from typing import AsyncGenerator, Mapping, Optional
 
 from pipecat.frames.frames import (
     AudioRawFrame,
@@ -20,16 +20,20 @@ from pipecat.frames.frames import (
     LLMFullResponseEndFrame,
     StartFrame,
     StartInterruptionFrame,
+    TTSLanguageUpdateFrame,
+    TTSLanguageVoicesUpdateFrame,
     TTSSpeakFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
     TTSVoiceUpdateFrame,
     TextFrame,
+    TranscriptionFrame,
     UserImageRequestFrame,
     VisionImageRawFrame
 )
 from pipecat.processors.async_frame_processor import AsyncFrameProcessor
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.transcriptions.languages import Language
 from pipecat.utils.audio import calculate_audio_volume
 from pipecat.utils.string import match_endofsentence
 from pipecat.utils.utils import exp_smoothing
@@ -177,6 +181,14 @@ class TTSService(AIService):
     async def set_voice(self, voice: str):
         pass
 
+    @abstractmethod
+    async def set_language(self, language: Language):
+        pass
+
+    @abstractmethod
+    async def set_language_voices(self, voices: Mapping[Language, str]):
+        pass
+
     # Converts the text to audio.
     @abstractmethod
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
@@ -235,6 +247,10 @@ class TTSService(AIService):
             await self._push_tts_frames(frame.text, False)
         elif isinstance(frame, TTSVoiceUpdateFrame):
             await self.set_voice(frame.voice)
+        elif isinstance(frame, TTSLanguageUpdateFrame):
+            await self.set_language(frame.language)
+        elif isinstance(frame, TTSLanguageVoicesUpdateFrame):
+            await self.set_language_voices(frame.voices)
         else:
             await self.push_frame(frame, direction)
 
diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py
index 735d12b5b..7c54c7679 100644
--- a/src/pipecat/services/cartesia.py
+++ b/src/pipecat/services/cartesia.py
@@ -10,9 +10,8 @@ import base64
 import asyncio
 import time
 
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Mapping
 
-from pipecat.processors.frame_processor import FrameDirection
 from pipecat.frames.frames import (
     CancelFrame,
     ErrorFrame,
@@ -26,6 +25,8 @@ from pipecat.frames.frames import (
     TextFrame,
     LLMFullResponseEndFrame
 )
+from pipecat.processors.frame_processor import FrameDirection
+from pipecat.transcriptions.languages import Language
 from pipecat.services.ai_services import TTSService
 
 from loguru import logger
@@ -40,6 +41,15 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+def language_to_cartesia_language(language: Language) -> str | None:
+    match language:
+        case Language.EN:
+            return "en"
+        case Language.ES:
+            return "es"
+    return None
+
+
 class CartesiaTTSService(TTSService):
 
     def __init__(
@@ -79,6 +89,7 @@ class CartesiaTTSService(TTSService):
             "sample_rate": sample_rate,
         }
         self._language = language
+        self._language_voices = {}
 
         self._websocket = None
         self._context_id = None
@@ -94,6 +105,17 @@ class CartesiaTTSService(TTSService):
         logger.debug(f"Switching TTS voice to: [{voice}]")
         self._voice_id = voice
 
+    async def set_language(self, language: Language):
+        cartesia_language = language_to_cartesia_language(language)
+        if cartesia_language and language in self._language_voices:
+            logger.debug(f"Switching TTS language to: [{language}]")
+            self._language = cartesia_language
+            await self.set_voice(self._language_voices[language])
+
+    async def set_language_voices(self, voices: Mapping[Language, str]):
+        logger.debug(f"Setting TTS language voices to: {voices}")
+        self._language_voices = voices
+
     async def start(self, frame: StartFrame):
         await super().start(frame)
         await self._connect()