From 7796a272ce96e00714416f9725b5fad6f2f2b644 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 3 Oct 2024 22:51:15 -0400 Subject: [PATCH 1/2] Improve language handling for TTS services --- src/pipecat/services/ai_services.py | 5 +- src/pipecat/services/aws.py | 125 +++++++++--------- src/pipecat/services/azure.py | 190 ++++++++++++++-------------- src/pipecat/services/cartesia.py | 33 +++-- src/pipecat/services/elevenlabs.py | 145 ++++++++++----------- src/pipecat/services/gladia.py | 161 +++++++++++------------ src/pipecat/services/google.py | 177 +++++++++++++------------- src/pipecat/services/lmnt.py | 54 ++++---- src/pipecat/services/xtts.py | 93 +++++++------- 9 files changed, 499 insertions(+), 484 deletions(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index c7892b78b..90ce29e76 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -204,6 +204,9 @@ class TTSService(AIService): async def flush_audio(self): pass + def language_to_service_language(self, language: Language) -> str | None: + return Language(language) + # Converts the text to audio. @abstractmethod async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: @@ -234,7 +237,7 @@ class TTSService(AIService): logger.debug(f"Updating TTS setting {key} to: [{value}]") self._settings[key] = value if key == "language": - self._settings[key] = Language(value) + self._settings[key] = self.language_to_service_language(value) elif key == "model": self.set_model_name(value) elif key == "voice": diff --git a/src/pipecat/services/aws.py b/src/pipecat/services/aws.py index 0ae72b968..e54cb43a1 100644 --- a/src/pipecat/services/aws.py +++ b/src/pipecat/services/aws.py @@ -30,67 +30,6 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_aws_language(language: Language) -> str | None: - match language: - case Language.CA: - return "ca-ES" - case Language.ZH: - return "cmn-CN" - case Language.DA: - return "da-DK" - case Language.NL: - return "nl-NL" - case Language.NL_BE: - return "nl-BE" - case Language.EN: - return "en-US" - case Language.EN_US: - return "en-US" - case Language.EN_AU: - return "en-AU" - case Language.EN_GB: - return "en-GB" - case Language.EN_NZ: - return "en-NZ" - case Language.EN_IN: - return "en-IN" - case Language.FI: - return "fi-FI" - case Language.FR: - return "fr-FR" - case Language.FR_CA: - return "fr-CA" - case Language.DE: - return "de-DE" - case Language.HI: - return "hi-IN" - case Language.IT: - return "it-IT" - case Language.JA: - return "ja-JP" - case Language.KO: - return "ko-KR" - case Language.NO: - return "nb-NO" - case Language.PL: - return "pl-PL" - case Language.PT: - return "pt-PT" - case Language.PT_BR: - return "pt-BR" - case Language.RO: - return "ro-RO" - case Language.RU: - return "ru-RU" - case Language.ES: - return "es-ES" - case Language.SV: - return "sv-SE" - case Language.TR: - return "tr-TR" - return None - - class AWSTTSService(TTSService): class InputParams(BaseModel): engine: Optional[str] = None @@ -121,7 +60,9 @@ class AWSTTSService(TTSService): self._settings = { "sample_rate": sample_rate, "engine": params.engine, - "language": params.language if params.language else Language.EN, + "language": self.language_to_service_language(params.language) + if params.language + else Language.EN, "pitch": params.pitch, "rate": params.rate, "volume": params.volume, @@ -132,10 +73,68 @@ class AWSTTSService(TTSService): def can_generate_metrics(self) -> bool: return True + def language_to_service_language(self, language: Language) -> str | None: + match language: + case Language.CA: + return "ca-ES" + case Language.ZH: + return "cmn-CN" + case Language.DA: + return "da-DK" + case Language.NL: + return "nl-NL" + case Language.NL_BE: + return "nl-BE" + case Language.EN | Language.EN_US: + return "en-US" + case Language.EN_AU: + return "en-AU" + case Language.EN_GB: + return "en-GB" + case Language.EN_NZ: + return "en-NZ" + case Language.EN_IN: + return "en-IN" + case Language.FI: + return "fi-FI" + case Language.FR: + return "fr-FR" + case Language.FR_CA: + return "fr-CA" + case Language.DE: + return "de-DE" + case Language.HI: + return "hi-IN" + case Language.IT: + return "it-IT" + case Language.JA: + return "ja-JP" + case Language.KO: + return "ko-KR" + case Language.NO: + return "nb-NO" + case Language.PL: + return "pl-PL" + case Language.PT: + return "pt-PT" + case Language.PT_BR: + return "pt-BR" + case Language.RO: + return "ro-RO" + case Language.RU: + return "ru-RU" + case Language.ES: + return "es-ES" + case Language.SV: + return "sv-SE" + case Language.TR: + return "tr-TR" + return None + def _construct_ssml(self, text: str) -> str: ssml = "" - language = language_to_aws_language(self._settings["language"]) + language = self._settings["language"] ssml += f"" prosody_attrs = [] diff --git a/src/pipecat/services/azure.py b/src/pipecat/services/azure.py index 190a297a0..b7d4f0d4c 100644 --- a/src/pipecat/services/azure.py +++ b/src/pipecat/services/azure.py @@ -27,7 +27,6 @@ from pipecat.frames.frames import ( ) from pipecat.services.ai_services import ImageGenService, STTService, TTSService from pipecat.services.openai import BaseOpenAILLMService -from pipecat.transcriptions import language from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 @@ -72,101 +71,10 @@ class AzureLLMService(BaseOpenAILLMService): ) -def language_to_azure_language(language: Language) -> str | None: - match language: - case Language.BG: - return "bg-BG" - case Language.CA: - return "ca-ES" - case Language.ZH: - return "zh-CN" - case Language.ZH_TW: - return "zh-TW" - case Language.CS: - return "cs-CZ" - case Language.DA: - return "da-DK" - case Language.NL: - return "nl-NL" - case Language.EN: - return "en-US" - case Language.EN_US: - return "en-US" - case Language.EN_AU: - return "en-AU" - case Language.EN_GB: - return "en-GB" - case Language.EN_NZ: - return "en-NZ" - case Language.EN_IN: - return "en-IN" - case Language.ET: - return "et-EE" - case Language.FI: - return "fi-FI" - case Language.NL_BE: - return "nl-BE" - case Language.FR: - return "fr-FR" - case Language.FR_CA: - return "fr-CA" - case Language.DE: - return "de-DE" - case Language.DE_CH: - return "de-CH" - case Language.EL: - return "el-GR" - case Language.HI: - return "hi-IN" - case Language.HU: - return "hu-HU" - case Language.ID: - return "id-ID" - case Language.IT: - return "it-IT" - case Language.JA: - return "ja-JP" - case Language.KO: - return "ko-KR" - case Language.LV: - return "lv-LV" - case Language.LT: - return "lt-LT" - case Language.MS: - return "ms-MY" - case Language.NO: - return "nb-NO" - case Language.PL: - return "pl-PL" - case Language.PT: - return "pt-PT" - case Language.PT_BR: - return "pt-BR" - case Language.RO: - return "ro-RO" - case Language.RU: - return "ru-RU" - case Language.SK: - return "sk-SK" - case Language.ES: - return "es-ES" - case Language.SV: - return "sv-SE" - case Language.TH: - return "th-TH" - case Language.TR: - return "tr-TR" - case Language.UK: - return "uk-UA" - case Language.VI: - return "vi-VN" - return None - - class AzureTTSService(TTSService): class InputParams(BaseModel): emphasis: Optional[str] = None - language: Optional[Language] = Language.EN + language: Optional[Language] = Language.EN_US pitch: Optional[str] = None rate: Optional[str] = "1.05" role: Optional[str] = None @@ -192,7 +100,9 @@ class AzureTTSService(TTSService): self._settings = { "sample_rate": sample_rate, "emphasis": params.emphasis, - "language": params.language if params.language else Language.EN, + "language": self.language_to_service_language(params.language) + if params.language + else Language.EN_US, "pitch": params.pitch, "rate": params.rate, "role": params.role, @@ -206,8 +116,96 @@ class AzureTTSService(TTSService): def can_generate_metrics(self) -> bool: return True + def language_to_service_language(self, language: Language) -> str | None: + match language: + case Language.BG: + return "bg-BG" + case Language.CA: + return "ca-ES" + case Language.ZH: + return "zh-CN" + case Language.ZH_TW: + return "zh-TW" + case Language.CS: + return "cs-CZ" + case Language.DA: + return "da-DK" + case Language.NL: + return "nl-NL" + case Language.EN | Language.EN_US: + return "en-US" + case Language.EN_AU: + return "en-AU" + case Language.EN_GB: + return "en-GB" + case Language.EN_NZ: + return "en-NZ" + case Language.EN_IN: + return "en-IN" + case Language.ET: + return "et-EE" + case Language.FI: + return "fi-FI" + case Language.NL_BE: + return "nl-BE" + case Language.FR: + return "fr-FR" + case Language.FR_CA: + return "fr-CA" + case Language.DE: + return "de-DE" + case Language.DE_CH: + return "de-CH" + case Language.EL: + return "el-GR" + case Language.HI: + return "hi-IN" + case Language.HU: + return "hu-HU" + case Language.ID: + return "id-ID" + case Language.IT: + return "it-IT" + case Language.JA: + return "ja-JP" + case Language.KO: + return "ko-KR" + case Language.LV: + return "lv-LV" + case Language.LT: + return "lt-LT" + case Language.MS: + return "ms-MY" + case Language.NO: + return "nb-NO" + case Language.PL: + return "pl-PL" + case Language.PT: + return "pt-PT" + case Language.PT_BR: + return "pt-BR" + case Language.RO: + return "ro-RO" + case Language.RU: + return "ru-RU" + case Language.SK: + return "sk-SK" + case Language.ES: + return "es-ES" + case Language.SV: + return "sv-SE" + case Language.TH: + return "th-TH" + case Language.TR: + return "tr-TR" + case Language.UK: + return "uk-UA" + case Language.VI: + return "vi-VN" + return None + def _construct_ssml(self, text: str) -> str: - language = language_to_azure_language(self._settings["language"]) + language = self._settings["language"] ssml = ( f" str | None: match language: case Language.DE: return "de" - case Language.EN: + case ( + Language.EN + | Language.EN_US + | Language.EN_GB + | Language.EN_AU + | Language.EN_NZ + | Language.EN_IN + ): return "en" case Language.ES: return "es" - case Language.FR: + case Language.FR | Language.FR_CA: return "fr" case Language.JA: return "ja" - case Language.PT: + case Language.PT | Language.PT_BR: return "pt" - case Language.ZH: + case Language.ZH | Language.ZH_TW: return "zh" return None @@ -106,7 +113,9 @@ class CartesiaTTSService(WordTTSService): "encoding": params.encoding, "sample_rate": params.sample_rate, }, - "language": params.language if params.language else Language.EN, + "language": self.language_to_service_language(params.language) + if params.language + else Language.EN, "speed": params.speed, "emotion": params.emotion, } @@ -125,6 +134,9 @@ class CartesiaTTSService(WordTTSService): await super().set_model(model) logger.debug(f"Switching TTS model to: [{model}]") + def language_to_service_language(self, language: Language) -> str | None: + return language_to_cartesia_language(language) + def _build_msg( self, text: str = "", continue_transcript: bool = True, add_timestamps: bool = True ): @@ -146,7 +158,7 @@ class CartesiaTTSService(WordTTSService): "model_id": self.model_name, "voice": voice_config, "output_format": self._settings["output_format"], - "language": language_to_cartesia_language(self._settings["language"]), + "language": self._settings["language"], "add_timestamps": add_timestamps, } return json.dumps(msg) @@ -303,7 +315,9 @@ class CartesiaHttpTTSService(TTSService): "encoding": params.encoding, "sample_rate": params.sample_rate, }, - "language": params.language if params.language else Language.EN, + "language": self.language_to_service_language(params.language) + if params.language + else Language.EN, "speed": params.speed, "emotion": params.emotion, } @@ -315,6 +329,9 @@ class CartesiaHttpTTSService(TTSService): def can_generate_metrics(self) -> bool: return True + def language_to_service_language(self, language: Language) -> str | None: + return language_to_cartesia_language(language) + async def stop(self, frame: EndFrame): await super().stop(frame) await self._client.close() @@ -343,7 +360,7 @@ class CartesiaHttpTTSService(TTSService): transcript=text, voice_id=self._voice_id, output_format=self._settings["output_format"], - language=language_to_cartesia_language(self._settings["language"]), + language=self._settings["language"], stream=False, _experimental_voice_controls=voice_controls, ) diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 95ac5c1f1..4e79efe2c 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -50,76 +50,6 @@ def sample_rate_from_output_format(output_format: str) -> int: return 16000 -def language_to_elevenlabs_language(language: Language) -> str | None: - match language: - case Language.BG: - return "bg" - case Language.ZH: - return "zh" - case Language.CS: - return "cs" - case Language.DA: - return "da" - case Language.NL: - return "nl" - case ( - Language.EN - | Language.EN_US - | Language.EN_AU - | Language.EN_GB - | Language.EN_NZ - | Language.EN_IN - ): - return "en" - case Language.FI: - return "fi" - case Language.FR | Language.FR_CA: - return "fr" - case Language.DE | Language.DE_CH: - return "de" - case Language.EL: - return "el" - case Language.HI: - return "hi" - case Language.HU: - return "hu" - case Language.ID: - return "id" - case Language.IT: - return "it" - case Language.JA: - return "ja" - case Language.KO: - return "ko" - case Language.MS: - return "ms" - case Language.NO: - return "no" - case Language.PL: - return "pl" - case Language.PT: - return "pt-PT" - case Language.PT_BR: - return "pt-BR" - case Language.RO: - return "ro" - case Language.RU: - return "ru" - case Language.SK: - return "sk" - case Language.ES: - return "es" - case Language.SV: - return "sv" - case Language.TR: - return "tr" - case Language.UK: - return "uk" - case Language.VI: - return "vi" - return None - - def calculate_word_times( alignment_info: Mapping[str, Any], cumulative_time: float ) -> List[Tuple[str, float]]: @@ -198,7 +128,9 @@ class ElevenLabsTTSService(WordTTSService): self._url = url self._settings = { "sample_rate": sample_rate_from_output_format(params.output_format), - "language": params.language if params.language else Language.EN, + "language": self.language_to_service_language(params.language) + if params.language + else Language.EN, "output_format": params.output_format, "optimize_streaming_latency": params.optimize_streaming_latency, "stability": params.stability, @@ -220,6 +152,75 @@ class ElevenLabsTTSService(WordTTSService): def can_generate_metrics(self) -> bool: return True + def language_to_service_language(self, language: Language) -> str | None: + match language: + case Language.BG: + return "bg" + case Language.ZH: + return "zh" + case Language.CS: + return "cs" + case Language.DA: + return "da" + case Language.NL: + return "nl" + case ( + Language.EN + | Language.EN_US + | Language.EN_AU + | Language.EN_GB + | Language.EN_NZ + | Language.EN_IN + ): + return "en" + case Language.FI: + return "fi" + case Language.FR | Language.FR_CA: + return "fr" + case Language.DE | Language.DE_CH: + return "de" + case Language.EL: + return "el" + case Language.HI: + return "hi" + case Language.HU: + return "hu" + case Language.ID: + return "id" + case Language.IT: + return "it" + case Language.JA: + return "ja" + case Language.KO: + return "ko" + case Language.MS: + return "ms" + case Language.NO: + return "no" + case Language.PL: + return "pl" + case Language.PT: + return "pt-PT" + case Language.PT_BR: + return "pt-BR" + case Language.RO: + return "ro" + case Language.RU: + return "ru" + case Language.SK: + return "sk" + case Language.ES: + return "es" + case Language.SV: + return "sv" + case Language.TR: + return "tr" + case Language.UK: + return "uk" + case Language.VI: + return "vi" + return None + def _set_voice_settings(self): voice_settings = {} if ( @@ -293,7 +294,7 @@ class ElevenLabsTTSService(WordTTSService): url += f"&optimize_streaming_latency={self._settings['optimize_streaming_latency']}" # Language can only be used with the 'eleven_turbo_v2_5' model - language = language_to_elevenlabs_language(self._settings["language"]) + language = self._settings["language"] if model == "eleven_turbo_v2_5": url += f"&language_code={language}" else: diff --git a/src/pipecat/services/gladia.py b/src/pipecat/services/gladia.py index e95329040..4cebca056 100644 --- a/src/pipecat/services/gladia.py +++ b/src/pipecat/services/gladia.py @@ -34,84 +34,6 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_gladia_language(language: Language) -> str | None: - match language: - case Language.BG: - return "bulgarian" - case Language.CA: - return "catalan" - case Language.ZH: - return "chinese" - case Language.CS: - return "czech" - case Language.DA: - return "danish" - case Language.NL: - return "dutch" - case ( - Language.EN - | Language.EN_US - | Language.EN_AU - | Language.EN_GB - | Language.EN_NZ - | Language.EN_IN - ): - return "english" - case Language.ET: - return "estonian" - case Language.FI: - return "finnish" - case Language.FR | Language.FR_CA: - return "french" - case Language.DE | Language.DE_CH: - return "german" - case Language.EL: - return "greek" - case Language.HI: - return "hindi" - case Language.HU: - return "hungarian" - case Language.ID: - return "indonesian" - case Language.IT: - return "italian" - case Language.JA: - return "japanese" - case Language.KO: - return "korean" - case Language.LV: - return "latvian" - case Language.LT: - return "lithuanian" - case Language.MS: - return "malay" - case Language.NO: - return "norwegian" - case Language.PL: - return "polish" - case Language.PT | Language.PT_BR: - return "portuguese" - case Language.RO: - return "romanian" - case Language.RU: - return "russian" - case Language.SK: - return "slovak" - case Language.ES: - return "spanish" - case Language.SV: - return "slovenian" - case Language.TH: - return "thai" - case Language.TR: - return "turkish" - case Language.UK: - return "ukrainian" - case Language.VI: - return "vietnamese" - return None - - class GladiaSTTService(STTService): class InputParams(BaseModel): sample_rate: Optional[int] = 16000 @@ -135,13 +57,92 @@ class GladiaSTTService(STTService): self._url = url self._settings = { "sample_rate": params.sample_rate, - "language": params.language if params.language else Language.EN, + "language": self.language_to_service_language(params.language) + if params.language + else Language.EN, "transcription_hint": params.transcription_hint, "endpointing": params.endpointing, "prosody": params.prosody, } self._confidence = confidence + def language_to_service_language(self, language: Language) -> str | None: + match language: + case Language.BG: + return "bulgarian" + case Language.CA: + return "catalan" + case Language.ZH: + return "chinese" + case Language.CS: + return "czech" + case Language.DA: + return "danish" + case Language.NL: + return "dutch" + case ( + Language.EN + | Language.EN_US + | Language.EN_AU + | Language.EN_GB + | Language.EN_NZ + | Language.EN_IN + ): + return "english" + case Language.ET: + return "estonian" + case Language.FI: + return "finnish" + case Language.FR | Language.FR_CA: + return "french" + case Language.DE | Language.DE_CH: + return "german" + case Language.EL: + return "greek" + case Language.HI: + return "hindi" + case Language.HU: + return "hungarian" + case Language.ID: + return "indonesian" + case Language.IT: + return "italian" + case Language.JA: + return "japanese" + case Language.KO: + return "korean" + case Language.LV: + return "latvian" + case Language.LT: + return "lithuanian" + case Language.MS: + return "malay" + case Language.NO: + return "norwegian" + case Language.PL: + return "polish" + case Language.PT | Language.PT_BR: + return "portuguese" + case Language.RO: + return "romanian" + case Language.RU: + return "russian" + case Language.SK: + return "slovak" + case Language.ES: + return "spanish" + case Language.SV: + return "slovenian" + case Language.TH: + return "thai" + case Language.TR: + return "turkish" + case Language.UK: + return "ukrainian" + case Language.VI: + return "vietnamese" + return None + async def start(self, frame: StartFrame): await super().start(frame) self._websocket = await websockets.connect(self._url) @@ -169,7 +170,7 @@ class GladiaSTTService(STTService): "model_type": "fast", "language_behaviour": "manual", "sample_rate": self._settings["sample_rate"], - "language": language_to_gladia_language(self._settings["language"]), + "language": self._settings["language"], "transcription_hint": self._settings["transcription_hint"], "endpointing": self._settings["endpointing"], "prosody": self._settings["prosody"], diff --git a/src/pipecat/services/google.py b/src/pipecat/services/google.py index 93c2f2d62..8fc5151a3 100644 --- a/src/pipecat/services/google.py +++ b/src/pipecat/services/google.py @@ -146,93 +146,6 @@ class GoogleLLMService(LLMService): await self._process_context(context) -def language_to_google_language(language: Language) -> str | None: - match language: - case Language.BG: - return "bg-BG" - case Language.CA: - return "ca-ES" - case Language.ZH: - return "cmn-CN" - case Language.ZH_TW: - return "cmn-TW" - case Language.CS: - return "cs-CZ" - case Language.DA: - return "da-DK" - case Language.NL: - return "nl-NL" - case Language.EN: - return "en-US" - case Language.EN_US: - return "en-US" - case Language.EN_AU: - return "en-AU" - case Language.EN_GB: - return "en-GB" - case Language.EN_IN: - return "en-IN" - case Language.ET: - return "et-EE" - case Language.FI: - return "fi-FI" - case Language.NL_BE: - return "nl-BE" - case Language.FR: - return "fr-FR" - case Language.FR_CA: - return "fr-CA" - case Language.DE: - return "de-DE" - case Language.EL: - return "el-GR" - case Language.HI: - return "hi-IN" - case Language.HU: - return "hu-HU" - case Language.ID: - return "id-ID" - case Language.IT: - return "it-IT" - case Language.JA: - return "ja-JP" - case Language.KO: - return "ko-KR" - case Language.LV: - return "lv-LV" - case Language.LT: - return "lt-LT" - case Language.MS: - return "ms-MY" - case Language.NO: - return "nb-NO" - case Language.PL: - return "pl-PL" - case Language.PT: - return "pt-PT" - case Language.PT_BR: - return "pt-BR" - case Language.RO: - return "ro-RO" - case Language.RU: - return "ru-RU" - case Language.SK: - return "sk-SK" - case Language.ES: - return "es-ES" - case Language.SV: - return "sv-SE" - case Language.TH: - return "th-TH" - case Language.TR: - return "tr-TR" - case Language.UK: - return "uk-UA" - case Language.VI: - return "vi-VN" - return None - - class GoogleTTSService(TTSService): class InputParams(BaseModel): pitch: Optional[str] = None @@ -261,7 +174,9 @@ class GoogleTTSService(TTSService): "rate": params.rate, "volume": params.volume, "emphasis": params.emphasis, - "language": params.language if params.language else Language.EN, + "language": self.language_to_service_language(params.language) + if params.language + else Language.EN, "gender": params.gender, "google_style": params.google_style, } @@ -291,13 +206,97 @@ class GoogleTTSService(TTSService): def can_generate_metrics(self) -> bool: return True + def language_to_service_language(self, language: Language) -> str | None: + match language: + case Language.BG: + return "bg-BG" + case Language.CA: + return "ca-ES" + case Language.ZH: + return "cmn-CN" + case Language.ZH_TW: + return "cmn-TW" + case Language.CS: + return "cs-CZ" + case Language.DA: + return "da-DK" + case Language.NL: + return "nl-NL" + case Language.EN | Language.EN_US: + return "en-US" + case Language.EN_AU: + return "en-AU" + case Language.EN_GB: + return "en-GB" + case Language.EN_IN: + return "en-IN" + case Language.ET: + return "et-EE" + case Language.FI: + return "fi-FI" + case Language.NL_BE: + return "nl-BE" + case Language.FR: + return "fr-FR" + case Language.FR_CA: + return "fr-CA" + case Language.DE: + return "de-DE" + case Language.EL: + return "el-GR" + case Language.HI: + return "hi-IN" + case Language.HU: + return "hu-HU" + case Language.ID: + return "id-ID" + case Language.IT: + return "it-IT" + case Language.JA: + return "ja-JP" + case Language.KO: + return "ko-KR" + case Language.LV: + return "lv-LV" + case Language.LT: + return "lt-LT" + case Language.MS: + return "ms-MY" + case Language.NO: + return "nb-NO" + case Language.PL: + return "pl-PL" + case Language.PT: + return "pt-PT" + case Language.PT_BR: + return "pt-BR" + case Language.RO: + return "ro-RO" + case Language.RU: + return "ru-RU" + case Language.SK: + return "sk-SK" + case Language.ES: + return "es-ES" + case Language.SV: + return "sv-SE" + case Language.TH: + return "th-TH" + case Language.TR: + return "tr-TR" + case Language.UK: + return "uk-UA" + case Language.VI: + return "vi-VN" + return None + def _construct_ssml(self, text: str) -> str: ssml = "" # Voice tag voice_attrs = [f"name='{self._voice_id}'"] - language = language_to_google_language(self._settings["language"]) + language = self._settings["language"] voice_attrs.append(f"language='{language}'") if self._settings["gender"]: diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index 8f7fe6528..d41bacf98 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -35,32 +35,6 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_lmnt_language(language: Language) -> str | None: - match language: - case Language.DE: - return "de" - case ( - Language.EN - | Language.EN_US - | Language.EN_AU - | Language.EN_GB - | Language.EN_NZ - | Language.EN_IN - ): - return "en" - case Language.ES: - return "es" - case Language.FR | Language.FR_CA: - return "fr" - case Language.PT | Language.PT_BR: - return "pt" - case Language.ZH | Language.ZH_TW: - return "zh" - case Language.KO: - return "ko" - return None - - class LmntTTSService(TTSService): def __init__( self, @@ -82,7 +56,7 @@ class LmntTTSService(TTSService): "encoding": "pcm_s16le", "sample_rate": sample_rate, }, - "language": language, + "language": self.language_to_service_language(language), } self.set_voice(voice_id) @@ -97,6 +71,31 @@ class LmntTTSService(TTSService): def can_generate_metrics(self) -> bool: return True + def language_to_service_language(self, language: Language) -> str | None: + match language: + case Language.DE: + return "de" + case ( + Language.EN + | Language.EN_US + | Language.EN_AU + | Language.EN_GB + | Language.EN_NZ + | Language.EN_IN + ): + return "en" + case Language.ES: + return "es" + case Language.FR | Language.FR_CA: + return "fr" + case Language.PT | Language.PT_BR: + return "pt" + case Language.ZH | Language.ZH_TW: + return "zh" + case Language.KO: + return "ko" + return None + async def start(self, frame: StartFrame): await super().start(frame) await self._connect() @@ -121,6 +120,7 @@ class LmntTTSService(TTSService): self._voice_id, format="raw", sample_rate=self._settings["output_format"]["sample_rate"], + language=self._settings["language"], ) self._receive_task = self.get_event_loop().create_task(self._receive_task_handler()) except Exception as e: diff --git a/src/pipecat/services/xtts.py b/src/pipecat/services/xtts.py index 2f842c896..6c1408553 100644 --- a/src/pipecat/services/xtts.py +++ b/src/pipecat/services/xtts.py @@ -37,50 +37,6 @@ except ModuleNotFoundError as e: # https://github.com/coqui-ai/xtts-streaming-server -def language_to_xtts_language(language: Language) -> str | None: - match language: - case Language.CS: - return "cs" - case Language.DE: - return "de" - case ( - Language.EN - | Language.EN_US - | Language.EN_AU - | Language.EN_GB - | Language.EN_NZ - | Language.EN_IN - ): - return "en" - case Language.ES: - return "es" - case Language.FR: - return "fr" - case Language.HI: - return "hi" - case Language.HU: - return "hu" - case Language.IT: - return "it" - case Language.JA: - return "ja" - case Language.KO: - return "ko" - case Language.NL: - return "nl" - case Language.PL: - return "pl" - case Language.PT | Language.PT_BR: - return "pt" - case Language.RU: - return "ru" - case Language.TR: - return "tr" - case Language.ZH: - return "zh-cn" - return None - - class XTTSService(TTSService): def __init__( self, @@ -94,7 +50,7 @@ class XTTSService(TTSService): super().__init__(**kwargs) self._settings = { - "language": language, + "language": self.language_to_service_language(language), "base_url": base_url, } self.set_voice(voice_id) @@ -104,6 +60,49 @@ class XTTSService(TTSService): def can_generate_metrics(self) -> bool: return True + def language_to_service_language(self, language: Language) -> str | None: + match language: + case Language.CS: + return "cs" + case Language.DE: + return "de" + case ( + Language.EN + | Language.EN_US + | Language.EN_AU + | Language.EN_GB + | Language.EN_NZ + | Language.EN_IN + ): + return "en" + case Language.ES: + return "es" + case Language.FR: + return "fr" + case Language.HI: + return "hi" + case Language.HU: + return "hu" + case Language.IT: + return "it" + case Language.JA: + return "ja" + case Language.KO: + return "ko" + case Language.NL: + return "nl" + case Language.PL: + return "pl" + case Language.PT | Language.PT_BR: + return "pt" + case Language.RU: + return "ru" + case Language.TR: + return "tr" + case Language.ZH: + return "zh-cn" + return None + async def start(self, frame: StartFrame): await super().start(frame) async with self._aiohttp_session.get(self._settings["base_url"] + "/studio_speakers") as r: @@ -131,11 +130,9 @@ class XTTSService(TTSService): url = self._settings["base_url"] + "/tts_stream" - language = language_to_xtts_language(self._settings["language"]) - payload = { "text": text.replace(".", "").replace("*", ""), - "language": language, + "language": self._settings["language"], "speaker_embedding": embeddings["speaker_embedding"], "gpt_cond_latent": embeddings["gpt_cond_latent"], "add_wav_header": False, From c49b31e6ad253d7ac71764c0d738ecd1fbebe962 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 3 Oct 2024 23:13:59 -0400 Subject: [PATCH 2/2] Add CHANGELOG entry --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d679c2b16..f5d2d40df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.0.43] - 2024-10-03 +### Changed + +- For TTS services, convert inputted languages to match each service's language + format + ### Fixed - Fixed an issue where changing a language with the Deepgram STT service wouldn't apply the change. This was fixed by disconnecting and reconnecting - when the language changes + when the language changes. ## [0.0.42] - 2024-10-02