From 51270a96c5ed922e0e575d0fa8323289e76c080d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 22 Aug 2024 22:40:42 -0700 Subject: [PATCH 01/17] frames: add language to transcription frames --- src/pipecat/frames/frames.py | 7 +++++-- src/pipecat/transcriptions/language.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 src/pipecat/transcriptions/language.py diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index e3b2aab37..1fe13024a 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -8,6 +8,7 @@ from typing import Any, List, Mapping, Optional, Tuple from dataclasses import dataclass, field +from pipecat.transcriptions.languages import Language from pipecat.utils.utils import obj_count, obj_id from pipecat.vad.vad_analyzer import VADParams @@ -131,9 +132,10 @@ class TranscriptionFrame(TextFrame): """ user_id: str timestamp: str + language: Language | None = None def __str__(self): - return f"{self.name}(user: {self.user_id}, text: {self.text}, timestamp: {self.timestamp})" + return f"{self.name}(user: {self.user_id}, text: {self.text}, language: {self.language}, timestamp: {self.timestamp})" @dataclass @@ -142,9 +144,10 @@ class InterimTranscriptionFrame(TextFrame): the transport's receive queue when a participant speaks.""" user_id: str timestamp: str + language: Language | None = None def __str__(self): - return f"{self.name}(user: {self.user_id}, text: {self.text}, timestamp: {self.timestamp})" + return f"{self.name}(user: {self.user_id}, text: {self.text}, language: {self.language}, timestamp: {self.timestamp})" @dataclass diff --git a/src/pipecat/transcriptions/language.py b/src/pipecat/transcriptions/language.py new file mode 100644 index 000000000..2a8e3c3ac --- /dev/null +++ b/src/pipecat/transcriptions/language.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import sys + +from enum import Enum + +if sys.version_info < (3, 11): + class StrEnum(str, Enum): + def __new__(cls, value): + obj = str.__new__(cls, value) + obj._value_ = value + return obj +else: + from enum import StrEnum + + +class Language(StrEnum): + EN = "en" + ES = "es" From c0cdabf61de600389611b9d198b2c3ebf0c55acf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 22 Aug 2024 22:40:53 -0700 Subject: [PATCH 02/17] frames: adde TTSLanguageUpdateFrame and TTSLanguageVoicesUpdateFrame --- src/pipecat/frames/frames.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 1fe13024a..a6b3e22ae 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -443,6 +443,22 @@ class TTSVoiceUpdateFrame(ControlFrame): voice: str +@dataclass +class TTSLanguageUpdateFrame(ControlFrame): + """A control frame containing a request to update to a new TTS language. + """ + language: Language + + +@dataclass +class TTSLanguageVoicesUpdateFrame(ControlFrame): + """A control frame containing a mapping between a language and the desired + voice for that language. + + """ + voices: Mapping[Language, str] + + @dataclass class FunctionCallInProgressFrame(SystemFrame): """A frame signaling that a function call is in progress. From 38cd86ad52d5790c23c049a47f8b166d52902134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 22 Aug 2024 22:43:30 -0700 Subject: [PATCH 03/17] services: added language to transcription frames --- src/pipecat/services/deepgram.py | 26 ++++++++++++++++++++---- src/pipecat/transports/services/daily.py | 19 +++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 035fdd25c..30a572d38 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -22,6 +22,7 @@ from pipecat.frames.frames import ( TranscriptionFrame) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AsyncAIService, TTSService +from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 from loguru import logger @@ -30,10 +31,12 @@ from loguru import logger # See .env.example for Deepgram configuration needed try: from deepgram import ( + AsyncListenWebSocketClient, DeepgramClient, DeepgramClientOptions, LiveTranscriptionEvents, LiveOptions, + LiveResultResponse ) except ModuleNotFoundError as e: logger.error(f"Exception: {e}") @@ -42,6 +45,15 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +def deepgram_language_to_language(language: str) -> Language | None: + match language: + case "en": + return Language.EN + case "es": + return Language.ES + return None + + class DeepgramTTSService(TTSService): def __init__( @@ -128,7 +140,7 @@ class DeepgramSTTService(AsyncAIService): self._client = DeepgramClient( api_key, config=DeepgramClientOptions(url=url, options={"keepalive": "true"})) - self._connection = self._client.listen.asynclive.v("1") + self._connection: AsyncListenWebSocketClient = self._client.listen.asyncwebsocket.v("1") self._connection.on(LiveTranscriptionEvents.Transcript, self._on_message) async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -157,11 +169,17 @@ class DeepgramSTTService(AsyncAIService): await self._connection.finish() async def _on_message(self, *args, **kwargs): - result = kwargs["result"] + result: LiveResultResponse = kwargs["result"] + if len(result.channel.alternatives) == 0: + return is_final = result.is_final transcript = result.channel.alternatives[0].transcript + language = None + if result.channel.alternatives[0].languages: + language = result.channel.alternatives[0].languages[0] + language = deepgram_language_to_language(language) if len(transcript) > 0: if is_final: - await self.queue_frame(TranscriptionFrame(transcript, "", time_now_iso8601())) + await self.queue_frame(TranscriptionFrame(transcript, "", time_now_iso8601(), language)) else: - await self.queue_frame(InterimTranscriptionFrame(transcript, "", time_now_iso8601())) + await self.queue_frame(InterimTranscriptionFrame(transcript, "", time_now_iso8601(), language)) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 98bc3cc3c..188ae65ce 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -36,6 +36,7 @@ from pipecat.frames.frames import ( UserImageRawFrame, UserImageRequestFrame) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.transcriptions.language import Language from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams @@ -746,6 +747,15 @@ class DailyOutputTransport(BaseOutputTransport): await self._client.write_frame_to_camera(frame) +def daily_language_to_language(language: str) -> Language | None: + match language: + case "en": + return Language.EN + case "es": + return Language.ES + return None + + class DailyTransport(BaseTransport): def __init__( @@ -950,11 +960,16 @@ class DailyTransport(BaseTransport): text = message["text"] timestamp = message["timestamp"] is_final = message["rawResponse"]["is_final"] + try: + language = message["rawResponse"]["channel"]["alternatives"][0]["languages"][0] + language = daily_language_to_language(language) + except KeyError: + language = None if is_final: - frame = TranscriptionFrame(text, participant_id, timestamp) + frame = TranscriptionFrame(text, participant_id, timestamp, language) logger.debug(f"Transcription (from: {participant_id}): [{text}]") else: - frame = InterimTranscriptionFrame(text, participant_id, timestamp) + frame = InterimTranscriptionFrame(text, participant_id, timestamp, language) if self._input: await self._input.push_transcription_frame(frame) From 3931cb32355d5ac74eeb82e52b6f6f3427b710e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 22 Aug 2024 22:44:11 -0700 Subject: [PATCH 04/17] services(cartesia): allow setting language and language voices --- src/pipecat/services/ai_services.py | 18 +++++++++++++++++- src/pipecat/services/cartesia.py | 26 ++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index d4a3c021e..8208986ef 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -9,7 +9,7 @@ import io import wave from abc import abstractmethod -from typing import AsyncGenerator, Optional +from typing import AsyncGenerator, Mapping, Optional from pipecat.frames.frames import ( AudioRawFrame, @@ -20,16 +20,20 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame, StartFrame, StartInterruptionFrame, + TTSLanguageUpdateFrame, + TTSLanguageVoicesUpdateFrame, TTSSpeakFrame, TTSStartedFrame, TTSStoppedFrame, TTSVoiceUpdateFrame, TextFrame, + TranscriptionFrame, UserImageRequestFrame, VisionImageRawFrame ) from pipecat.processors.async_frame_processor import AsyncFrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.transcriptions.languages import Language from pipecat.utils.audio import calculate_audio_volume from pipecat.utils.string import match_endofsentence from pipecat.utils.utils import exp_smoothing @@ -177,6 +181,14 @@ class TTSService(AIService): async def set_voice(self, voice: str): pass + @abstractmethod + async def set_language(self, language: Language): + pass + + @abstractmethod + async def set_language_voices(self, voices: Mapping[Language, str]): + pass + # Converts the text to audio. @abstractmethod async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: @@ -235,6 +247,10 @@ class TTSService(AIService): await self._push_tts_frames(frame.text, False) elif isinstance(frame, TTSVoiceUpdateFrame): await self.set_voice(frame.voice) + elif isinstance(frame, TTSLanguageUpdateFrame): + await self.set_language(frame.language) + elif isinstance(frame, TTSLanguageVoicesUpdateFrame): + await self.set_language_voices(frame.voices) else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 735d12b5b..7c54c7679 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -10,9 +10,8 @@ import base64 import asyncio import time -from typing import AsyncGenerator +from typing import AsyncGenerator, Mapping -from pipecat.processors.frame_processor import FrameDirection from pipecat.frames.frames import ( CancelFrame, ErrorFrame, @@ -26,6 +25,8 @@ from pipecat.frames.frames import ( TextFrame, LLMFullResponseEndFrame ) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.transcriptions.languages import Language from pipecat.services.ai_services import TTSService from loguru import logger @@ -40,6 +41,15 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +def language_to_cartesia_language(language: Language) -> str | None: + match language: + case Language.EN: + return "en" + case Language.ES: + return "es" + return None + + class CartesiaTTSService(TTSService): def __init__( @@ -79,6 +89,7 @@ class CartesiaTTSService(TTSService): "sample_rate": sample_rate, } self._language = language + self._language_voices = {} self._websocket = None self._context_id = None @@ -94,6 +105,17 @@ class CartesiaTTSService(TTSService): logger.debug(f"Switching TTS voice to: [{voice}]") self._voice_id = voice + async def set_language(self, language: Language): + cartesia_language = language_to_cartesia_language(language) + if cartesia_language and language in self._language_voices: + logger.debug(f"Switching TTS language to: [{language}]") + self._language = cartesia_language + await self.set_voice(self._language_voices[language]) + + async def set_language_voices(self, voices: Mapping[Language, str]): + logger.debug(f"Setting TTS language voices to: {voices}") + self._language_voices = voices + async def start(self, frame: StartFrame): await super().start(frame) await self._connect() From 6629b853c58c02537b5223cb7266ab9b520565c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 26 Aug 2024 11:10:42 -0700 Subject: [PATCH 05/17] services(deepgram): inherit from STTService instead of AsyncAIService --- src/pipecat/services/deepgram.py | 69 ++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 30a572d38..6179bb56d 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -16,12 +16,10 @@ from pipecat.frames.frames import ( Frame, InterimTranscriptionFrame, StartFrame, - SystemFrame, TTSStartedFrame, TTSStoppedFrame, TranscriptionFrame) -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import AsyncAIService, TTSService +from pipecat.services.ai_services import STTService, TTSService from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 @@ -45,15 +43,6 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def deepgram_language_to_language(language: str) -> Language | None: - match language: - case "en": - return Language.EN - case "es": - return Language.ES - return None - - class DeepgramTTSService(TTSService): def __init__( @@ -119,7 +108,7 @@ class DeepgramTTSService(TTSService): logger.exception(f"{self} exception: {e}") -class DeepgramSTTService(AsyncAIService): +class DeepgramSTTService(STTService): def __init__(self, *, api_key: str, @@ -132,6 +121,8 @@ class DeepgramSTTService(AsyncAIService): channels=1, interim_results=True, smart_format=True, + punctuate=True, + profanity_filter=True, ), **kwargs): super().__init__(**kwargs) @@ -143,30 +134,46 @@ class DeepgramSTTService(AsyncAIService): self._connection: AsyncListenWebSocketClient = self._client.listen.asyncwebsocket.v("1") self._connection.on(LiveTranscriptionEvents.Transcript, self._on_message) - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) + async def set_model(self, model: str): + logger.debug(f"Switching STT model to: [{model}]") + self._live_options.model = model + await self._disconnect() + await self._connect() - if isinstance(frame, SystemFrame): - await self.push_frame(frame, direction) - elif isinstance(frame, AudioRawFrame): - await self._connection.send(frame.audio) - else: - await self.queue_frame(frame, direction) + async def set_language(self, language: Language): + logger.debug(f"Switching STT language to: [{language}]") + self._live_options.language = language.value + await self._disconnect() + await self._connect() async def start(self, frame: StartFrame): await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + await super().cancel(frame) + await self._disconnect() + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + await self.start_processing_metrics() + await self._connection.send(audio) + yield None + await self.stop_processing_metrics() + + async def _connect(self): if await self._connection.start(self._live_options): logger.debug(f"{self}: Connected to Deepgram") else: logger.error(f"{self}: Unable to connect to Deepgram") - async def stop(self, frame: EndFrame): - await super().stop(frame) - await self._connection.finish() - - async def cancel(self, frame: CancelFrame): - await super().cancel(frame) - await self._connection.finish() + async def _disconnect(self): + if self._connection.is_connected: + await self._connection.finish() + logger.debug(f"{self}: Disconnected from Deepgram") async def _on_message(self, *args, **kwargs): result: LiveResultResponse = kwargs["result"] @@ -177,9 +184,9 @@ class DeepgramSTTService(AsyncAIService): language = None if result.channel.alternatives[0].languages: language = result.channel.alternatives[0].languages[0] - language = deepgram_language_to_language(language) + language = Language(language) if len(transcript) > 0: if is_final: - await self.queue_frame(TranscriptionFrame(transcript, "", time_now_iso8601(), language)) + await self.push_frame(TranscriptionFrame(transcript, "", time_now_iso8601(), language)) else: - await self.queue_frame(InterimTranscriptionFrame(transcript, "", time_now_iso8601(), language)) + await self.push_frame(InterimTranscriptionFrame(transcript, "", time_now_iso8601(), language)) From 568d9dc0a356e866618ebe67e56b844f6e5e5125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 26 Aug 2024 11:11:07 -0700 Subject: [PATCH 06/17] services(whisper): inherit from SegmentedSTTService --- src/pipecat/services/whisper.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/whisper.py b/src/pipecat/services/whisper.py index 59ce51a46..04f357a94 100644 --- a/src/pipecat/services/whisper.py +++ b/src/pipecat/services/whisper.py @@ -14,7 +14,7 @@ from typing import AsyncGenerator import numpy as np from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame -from pipecat.services.ai_services import STTService +from pipecat.services.ai_services import SegmentedSTTService from pipecat.utils.time import time_now_iso8601 from loguru import logger @@ -38,7 +38,7 @@ class Model(Enum): DISTIL_MEDIUM_EN = "Systran/faster-distil-whisper-medium.en" -class WhisperSTTService(STTService): +class WhisperSTTService(SegmentedSTTService): """Class to transcribe audio with a locally-downloaded Whisper model""" def __init__(self, @@ -77,6 +77,7 @@ class WhisperSTTService(STTService): yield ErrorFrame("Whisper model not available") return + await self.start_processing_metrics() await self.start_ttfb_metrics() # Divide by 32768 because we have signed 16-bit data. @@ -88,7 +89,9 @@ class WhisperSTTService(STTService): if segment.no_speech_prob < self._no_speech_prob: text += f"{segment.text} " + await self.stop_ttfb_metrics() + await self.stop_processing_metrics() + if text: - await self.stop_ttfb_metrics() logger.debug(f"Transcription: [{text}]") yield TranscriptionFrame(text, "", time_now_iso8601()) From a253606d50d05e15462b28ae285e79127a1d50f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 26 Aug 2024 11:11:28 -0700 Subject: [PATCH 07/17] services(daily): on_joined now returns all data not only participant --- src/pipecat/transports/services/daily.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 188ae65ce..18d8fb49c 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -306,7 +306,7 @@ class DailyTransportClient(EventHandler): if self._token and self._params.transcription_enabled: await self._start_transcription() - await self._callbacks.on_joined(data["participants"]["local"]) + await self._callbacks.on_joined(data) else: error_msg = f"Error joining {self._room_url}: {error}" logger.error(error_msg) @@ -874,8 +874,8 @@ class DailyTransport(BaseTransport): self._input.capture_participant_video( participant_id, framerate, video_source, color_format) - async def _on_joined(self, participant): - await self._call_event_handler("on_joined", participant) + async def _on_joined(self, data): + await self._call_event_handler("on_joined", data) async def _on_left(self): await self._call_event_handler("on_left") From fd3fdacdeee522aa9d260ec1b3398c005f4d0820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 26 Aug 2024 11:12:27 -0700 Subject: [PATCH 08/17] transcriptions: added more languages --- src/pipecat/services/cartesia.py | 12 ++++++- src/pipecat/transcriptions/language.py | 45 ++++++++++++++++++++++-- src/pipecat/transports/services/daily.py | 11 +----- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 7c54c7679..17bf34d15 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -26,7 +26,7 @@ from pipecat.frames.frames import ( LLMFullResponseEndFrame ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.transcriptions.languages import Language +from pipecat.transcriptions.language import Language from pipecat.services.ai_services import TTSService from loguru import logger @@ -43,10 +43,20 @@ except ModuleNotFoundError as e: def language_to_cartesia_language(language: Language) -> str | None: match language: + case Language.DE: + return "de" case Language.EN: return "en" case Language.ES: return "es" + case Language.FR: + return "fr" + case Language.JA: + return "ja" + case Language.PT: + return "pt" + case Language.ZH: + return "zh" return None diff --git a/src/pipecat/transcriptions/language.py b/src/pipecat/transcriptions/language.py index 2a8e3c3ac..f9e98104b 100644 --- a/src/pipecat/transcriptions/language.py +++ b/src/pipecat/transcriptions/language.py @@ -19,5 +19,46 @@ else: class Language(StrEnum): - EN = "en" - ES = "es" + BG = "bg" # Bulgarian + CA = "ca" # Catalan + ZH = "zh" # Chinese simplified + ZH_TW = "zh-TW" # Chinese traditional + CS = "cs" # Czech + DA = "da" # Danish + NL = "nl" # Dutch + EN = "en" # English + EN_US = "en-US" # English (USA) + EN_AU = "en-AU" # English (Australia) + EN_GB = "en-GB" # English (Great Britain) + EN_NZ = "en-NZ" # English (New Zealand) + EN_IN = "en-IN" # English (India) + ET = "et" # Estonian + FI = "fi" # Finnish + NL_BE = "nl-BE" # Flemmish + FR = "fr" # French + FR_CA = "fr-CA" # French (Canada) + DE = "de" # German + DE_CH = "de-CH" # German (Switzerland) + EL = "el" # Greek + HI = "hi" # Hindi + HU = "hu" # Hungarian + ID = "id" # Indonesian + IT = "it" # Italian + JA = "ja" # Japanese + KO = "ko" # Korean + LV = "lv" # Latvian + LT = "lt" # Lithuanian + MS = "ms" # Malay + NO = "no" # Norwegian + PL = "pl" # Polish + PT = "pt" # Portuguese + PT_BR = "pt-BR" # Portuguese (Brazil) + RO = "ro" # Romanian + RU = "ru" # Russian + SK = "sk" # Slovak + ES = "es" # Spanish + SV = "sv" # Swedish + TH = "th" # Thai + TR = "tr" # Turkish + UK = "uk" # Ukrainian + VI = "vi" # Vietnamese diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 18d8fb49c..bb6032fa4 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -747,15 +747,6 @@ class DailyOutputTransport(BaseOutputTransport): await self._client.write_frame_to_camera(frame) -def daily_language_to_language(language: str) -> Language | None: - match language: - case "en": - return Language.EN - case "es": - return Language.ES - return None - - class DailyTransport(BaseTransport): def __init__( @@ -962,7 +953,7 @@ class DailyTransport(BaseTransport): is_final = message["rawResponse"]["is_final"] try: language = message["rawResponse"]["channel"]["alternatives"][0]["languages"][0] - language = daily_language_to_language(language) + language = Language(language) except KeyError: language = None if is_final: From 4e0ece17b6fd4607b69dacc4a1565e6cc6a67e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 26 Aug 2024 11:12:51 -0700 Subject: [PATCH 09/17] services: added support for setting STT model and language --- src/pipecat/frames/frames.py | 29 ++++++-- src/pipecat/services/ai_services.py | 104 +++++++++++++++++----------- src/pipecat/services/cartesia.py | 19 +++-- 3 files changed, 95 insertions(+), 57 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index a6b3e22ae..884e1e9eb 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -8,7 +8,7 @@ from typing import Any, List, Mapping, Optional, Tuple from dataclasses import dataclass, field -from pipecat.transcriptions.languages import Language +from pipecat.transcriptions.language import Language from pipecat.utils.utils import obj_count, obj_id from pipecat.vad.vad_analyzer import VADParams @@ -436,6 +436,13 @@ class LLMModelUpdateFrame(ControlFrame): model: str +@dataclass +class TTSModelUpdateFrame(ControlFrame): + """A control frame containing a request to update the TTS model. + """ + model: str + + @dataclass class TTSVoiceUpdateFrame(ControlFrame): """A control frame containing a request to update to a new TTS voice. @@ -445,18 +452,26 @@ class TTSVoiceUpdateFrame(ControlFrame): @dataclass class TTSLanguageUpdateFrame(ControlFrame): - """A control frame containing a request to update to a new TTS language. + """A control frame containing a request to update to a new TTS language and + optional voice. + """ language: Language + voice: str | None = None @dataclass -class TTSLanguageVoicesUpdateFrame(ControlFrame): - """A control frame containing a mapping between a language and the desired - voice for that language. - +class STTModelUpdateFrame(ControlFrame): + """A control frame containing a request to update the STT model. """ - voices: Mapping[Language, str] + model: str + + +@dataclass +class STTLanguageUpdateFrame(ControlFrame): + """A control frame containing a request to update to STT language. + """ + language: Language @dataclass diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 8208986ef..50b73e85b 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -18,22 +18,23 @@ from pipecat.frames.frames import ( ErrorFrame, Frame, LLMFullResponseEndFrame, + STTLanguageUpdateFrame, + STTModelUpdateFrame, StartFrame, StartInterruptionFrame, TTSLanguageUpdateFrame, - TTSLanguageVoicesUpdateFrame, + TTSModelUpdateFrame, TTSSpeakFrame, TTSStartedFrame, TTSStoppedFrame, TTSVoiceUpdateFrame, TextFrame, - TranscriptionFrame, UserImageRequestFrame, VisionImageRawFrame ) from pipecat.processors.async_frame_processor import AsyncFrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.transcriptions.languages import Language +from pipecat.transcriptions.language import Language from pipecat.utils.audio import calculate_audio_volume from pipecat.utils.string import match_endofsentence from pipecat.utils.utils import exp_smoothing @@ -177,16 +178,16 @@ class TTSService(AIService): self._stop_frame_queue: asyncio.Queue = asyncio.Queue() self._current_sentence: str = "" + @abstractmethod + async def set_model(self, model: str): + pass + @abstractmethod async def set_voice(self, voice: str): pass @abstractmethod - async def set_language(self, language: Language): - pass - - @abstractmethod - async def set_language_voices(self, voices: Mapping[Language, str]): + async def set_language(self, language: Language, voice: str | None): pass # Converts the text to audio. @@ -245,12 +246,12 @@ class TTSService(AIService): await self.push_frame(frame, direction) elif isinstance(frame, TTSSpeakFrame): await self._push_tts_frames(frame.text, False) + elif isinstance(frame, TTSModelUpdateFrame): + await self.set_model(frame.model) elif isinstance(frame, TTSVoiceUpdateFrame): await self.set_voice(frame.voice) elif isinstance(frame, TTSLanguageUpdateFrame): - await self.set_language(frame.language) - elif isinstance(frame, TTSLanguageVoicesUpdateFrame): - await self.set_language_voices(frame.voices) + await self.set_language(frame.language, frame.voice) else: await self.push_frame(frame, direction) @@ -305,6 +306,47 @@ class TTSService(AIService): class STTService(AIService): """STTService is a base class for speech-to-text services.""" + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @abstractmethod + async def set_model(self, model: str): + pass + + @abstractmethod + async def set_language(self, language: Language): + pass + + @abstractmethod + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Returns transcript as a string""" + pass + + async def process_audio_frame(self, frame: AudioRawFrame): + await self.process_generator(self.run_stt(frame.audio)) + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Processes a frame of audio data, either buffering or transcribing it.""" + await super().process_frame(frame, direction) + + if isinstance(frame, AudioRawFrame): + # In this service we accumulate audio internally and at the end we + # push a TextFrame. We don't really want to push audio frames down. + await self.process_audio_frame(frame) + elif isinstance(frame, STTModelUpdateFrame): + await self.set_model(frame.model) + elif isinstance(frame, STTLanguageUpdateFrame): + await self.set_language(frame.language) + else: + await self.push_frame(frame, direction) + + +class SegmentedSTTService(STTService): + """SegmentedSTTService is an STTService that will detect speech and will run + speech-to-text on speech segments only, instead of a continous stream. + + """ + def __init__(self, *, min_volume: float = 0.6, @@ -325,24 +367,7 @@ class STTService(AIService): self._smoothing_factor = 0.2 self._prev_volume = 0 - @abstractmethod - async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Returns transcript as a string""" - pass - - def _new_wave(self): - content = io.BytesIO() - ww = wave.open(content, "wb") - ww.setsampwidth(2) - ww.setnchannels(self._num_channels) - ww.setframerate(self._sample_rate) - return (content, ww) - - def _get_smoothed_volume(self, frame: AudioRawFrame) -> float: - volume = calculate_audio_volume(frame.audio, frame.sample_rate) - return exp_smoothing(volume, self._prev_volume, self._smoothing_factor) - - async def _append_audio(self, frame: AudioRawFrame): + async def process_audio_frame(self, frame: AudioRawFrame): # Try to filter out empty background noise volume = self._get_smoothed_volume(frame) if volume >= self._min_volume: @@ -362,9 +387,7 @@ class STTService(AIService): self._silence_num_frames = 0 self._wave.close() self._content.seek(0) - await self.start_processing_metrics() await self.process_generator(self.run_stt(self._content.read())) - await self.stop_processing_metrics() (self._content, self._wave) = self._new_wave() async def stop(self, frame: EndFrame): @@ -373,16 +396,17 @@ class STTService(AIService): async def cancel(self, frame: CancelFrame): self._wave.close() - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Processes a frame of audio data, either buffering or transcribing it.""" - await super().process_frame(frame, direction) + def _new_wave(self): + content = io.BytesIO() + ww = wave.open(content, "wb") + ww.setsampwidth(2) + ww.setnchannels(self._num_channels) + ww.setframerate(self._sample_rate) + return (content, ww) - if isinstance(frame, AudioRawFrame): - # In this service we accumulate audio internally and at the end we - # push a TextFrame. We don't really want to push audio frames down. - await self._append_audio(frame) - else: - await self.push_frame(frame, direction) + def _get_smoothed_volume(self, frame: AudioRawFrame) -> float: + volume = calculate_audio_volume(frame.audio, frame.sample_rate) + return exp_smoothing(volume, self._prev_volume, self._smoothing_factor) class ImageGenService(AIService): diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 17bf34d15..b267d14c1 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -99,7 +99,6 @@ class CartesiaTTSService(TTSService): "sample_rate": sample_rate, } self._language = language - self._language_voices = {} self._websocket = None self._context_id = None @@ -111,20 +110,20 @@ class CartesiaTTSService(TTSService): def can_generate_metrics(self) -> bool: return True + async def set_model(self, model: str): + logger.debug(f"Switching TTS model to: [{model}]") + self._model_id = model + async def set_voice(self, voice: str): logger.debug(f"Switching TTS voice to: [{voice}]") self._voice_id = voice - async def set_language(self, language: Language): + async def set_language(self, language: Language, voice: str | None): + logger.debug(f"Switching TTS language to: [{language}]") cartesia_language = language_to_cartesia_language(language) - if cartesia_language and language in self._language_voices: - logger.debug(f"Switching TTS language to: [{language}]") - self._language = cartesia_language - await self.set_voice(self._language_voices[language]) - - async def set_language_voices(self, voices: Mapping[Language, str]): - logger.debug(f"Setting TTS language voices to: {voices}") - self._language_voices = voices + self._language = cartesia_language + if voice: + self._voice_id = voice async def start(self, frame: StartFrame): await super().start(frame) From b3b39626e174a89cf3d78e5b3655060d107716b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 26 Aug 2024 11:26:10 -0700 Subject: [PATCH 10/17] services: allow switching STT language and mdoel at the same time --- src/pipecat/frames/frames.py | 5 ++++- src/pipecat/services/ai_services.py | 4 ++-- src/pipecat/services/cartesia.py | 1 + src/pipecat/services/deepgram.py | 5 ++++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 884e1e9eb..1f94c75f0 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -462,9 +462,12 @@ class TTSLanguageUpdateFrame(ControlFrame): @dataclass class STTModelUpdateFrame(ControlFrame): - """A control frame containing a request to update the STT model. + """A control frame containing a request to update the STT model and optional + language. + """ model: str + language: Language | None = None @dataclass diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 50b73e85b..2048be046 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -310,7 +310,7 @@ class STTService(AIService): super().__init__(**kwargs) @abstractmethod - async def set_model(self, model: str): + async def set_model(self, model: str, language: Language | None): pass @abstractmethod @@ -334,7 +334,7 @@ class STTService(AIService): # push a TextFrame. We don't really want to push audio frames down. await self.process_audio_frame(frame) elif isinstance(frame, STTModelUpdateFrame): - await self.set_model(frame.model) + await self.set_model(frame.model, frame.language) elif isinstance(frame, STTLanguageUpdateFrame): await self.set_language(frame.language) else: diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index b267d14c1..899fb3561 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -123,6 +123,7 @@ class CartesiaTTSService(TTSService): cartesia_language = language_to_cartesia_language(language) self._language = cartesia_language if voice: + logger.debug(f"Switching TTS voice to: [{voice}]") self._voice_id = voice async def start(self, frame: StartFrame): diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 6179bb56d..098392cd6 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -134,9 +134,12 @@ class DeepgramSTTService(STTService): self._connection: AsyncListenWebSocketClient = self._client.listen.asyncwebsocket.v("1") self._connection.on(LiveTranscriptionEvents.Transcript, self._on_message) - async def set_model(self, model: str): + async def set_model(self, model: str, language: Language | None): logger.debug(f"Switching STT model to: [{model}]") self._live_options.model = model + if language: + logger.debug(f"Switching STT language to: [{language}]") + self._live_options.language = language.value await self._disconnect() await self._connect() From fedfc366f64dc45b9149be05f6fa9811e49317b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 26 Aug 2024 11:34:48 -0700 Subject: [PATCH 11/17] services(deepgram): fix strenum values --- src/pipecat/services/deepgram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 098392cd6..d8eb5f7c6 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -139,13 +139,13 @@ class DeepgramSTTService(STTService): self._live_options.model = model if language: logger.debug(f"Switching STT language to: [{language}]") - self._live_options.language = language.value + self._live_options.language = language await self._disconnect() await self._connect() async def set_language(self, language: Language): logger.debug(f"Switching STT language to: [{language}]") - self._live_options.language = language.value + self._live_options.language = language await self._disconnect() await self._connect() From d7c967997753a29c4a4c3d1fee31e523cdc9cef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 26 Aug 2024 12:08:53 -0700 Subject: [PATCH 12/17] services: allow TTSModelUpdateFrame to also update language and voice --- src/pipecat/frames/frames.py | 2 ++ src/pipecat/services/ai_services.py | 2 +- src/pipecat/services/cartesia.py | 9 ++++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 1f94c75f0..87f2d599d 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -441,6 +441,8 @@ class TTSModelUpdateFrame(ControlFrame): """A control frame containing a request to update the TTS model. """ model: str + voice: str | None = None + language: Language | None = None @dataclass diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 2048be046..a48afe78a 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -179,7 +179,7 @@ class TTSService(AIService): self._current_sentence: str = "" @abstractmethod - async def set_model(self, model: str): + async def set_model(self, model: str, voice: str | None, language: Language | None): pass @abstractmethod diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 899fb3561..bc7d76dde 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -110,9 +110,16 @@ class CartesiaTTSService(TTSService): def can_generate_metrics(self) -> bool: return True - async def set_model(self, model: str): + async def set_model(self, model: str, voice: str | None, language: Language | None): logger.debug(f"Switching TTS model to: [{model}]") self._model_id = model + if language: + logger.debug(f"Switching TTS language to: [{language}]") + cartesia_language = language_to_cartesia_language(language) + self._language = cartesia_language + if voice: + logger.debug(f"Switching TTS voice to: [{voice}]") + self._voice_id = voice async def set_voice(self, voice: str): logger.debug(f"Switching TTS voice to: [{voice}]") From 864768635a3a9a374822a5d1e0821343856b46e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 26 Aug 2024 12:14:36 -0700 Subject: [PATCH 13/17] services: add voice and language to set_model() --- src/pipecat/services/ai_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index a48afe78a..4f5415586 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -247,7 +247,7 @@ class TTSService(AIService): elif isinstance(frame, TTSSpeakFrame): await self._push_tts_frames(frame.text, False) elif isinstance(frame, TTSModelUpdateFrame): - await self.set_model(frame.model) + await self.set_model(frame.model, frame.voice, frame.language) elif isinstance(frame, TTSVoiceUpdateFrame): await self.set_voice(frame.voice) elif isinstance(frame, TTSLanguageUpdateFrame): From ae6fbb3146fd3090ba8dca8c613777eb2ed14dbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 26 Aug 2024 14:30:57 -0700 Subject: [PATCH 14/17] services: just set model, voice, language independently --- src/pipecat/frames/frames.py | 4 ---- src/pipecat/services/ai_services.py | 12 ++++++------ src/pipecat/services/cartesia.py | 17 +++-------------- src/pipecat/services/deepgram.py | 5 +---- 4 files changed, 10 insertions(+), 28 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 87f2d599d..13c2f53f1 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -441,8 +441,6 @@ class TTSModelUpdateFrame(ControlFrame): """A control frame containing a request to update the TTS model. """ model: str - voice: str | None = None - language: Language | None = None @dataclass @@ -459,7 +457,6 @@ class TTSLanguageUpdateFrame(ControlFrame): """ language: Language - voice: str | None = None @dataclass @@ -469,7 +466,6 @@ class STTModelUpdateFrame(ControlFrame): """ model: str - language: Language | None = None @dataclass diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 4f5415586..c10f3e46e 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -179,7 +179,7 @@ class TTSService(AIService): self._current_sentence: str = "" @abstractmethod - async def set_model(self, model: str, voice: str | None, language: Language | None): + async def set_model(self, model: str): pass @abstractmethod @@ -187,7 +187,7 @@ class TTSService(AIService): pass @abstractmethod - async def set_language(self, language: Language, voice: str | None): + async def set_language(self, language: Language): pass # Converts the text to audio. @@ -247,11 +247,11 @@ class TTSService(AIService): elif isinstance(frame, TTSSpeakFrame): await self._push_tts_frames(frame.text, False) elif isinstance(frame, TTSModelUpdateFrame): - await self.set_model(frame.model, frame.voice, frame.language) + await self.set_model(frame.model) elif isinstance(frame, TTSVoiceUpdateFrame): await self.set_voice(frame.voice) elif isinstance(frame, TTSLanguageUpdateFrame): - await self.set_language(frame.language, frame.voice) + await self.set_language(frame.language) else: await self.push_frame(frame, direction) @@ -310,7 +310,7 @@ class STTService(AIService): super().__init__(**kwargs) @abstractmethod - async def set_model(self, model: str, language: Language | None): + async def set_model(self, model: str): pass @abstractmethod @@ -334,7 +334,7 @@ class STTService(AIService): # push a TextFrame. We don't really want to push audio frames down. await self.process_audio_frame(frame) elif isinstance(frame, STTModelUpdateFrame): - await self.set_model(frame.model, frame.language) + await self.set_model(frame.model) elif isinstance(frame, STTLanguageUpdateFrame): await self.set_language(frame.language) else: diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index bc7d76dde..e3541ccea 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -110,28 +110,17 @@ class CartesiaTTSService(TTSService): def can_generate_metrics(self) -> bool: return True - async def set_model(self, model: str, voice: str | None, language: Language | None): + async def set_model(self, model: str): logger.debug(f"Switching TTS model to: [{model}]") self._model_id = model - if language: - logger.debug(f"Switching TTS language to: [{language}]") - cartesia_language = language_to_cartesia_language(language) - self._language = cartesia_language - if voice: - logger.debug(f"Switching TTS voice to: [{voice}]") - self._voice_id = voice async def set_voice(self, voice: str): logger.debug(f"Switching TTS voice to: [{voice}]") self._voice_id = voice - async def set_language(self, language: Language, voice: str | None): + async def set_language(self, language: Language): logger.debug(f"Switching TTS language to: [{language}]") - cartesia_language = language_to_cartesia_language(language) - self._language = cartesia_language - if voice: - logger.debug(f"Switching TTS voice to: [{voice}]") - self._voice_id = voice + self._language = language_to_cartesia_language(language) async def start(self, frame: StartFrame): await super().start(frame) diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index d8eb5f7c6..d899d4bdb 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -134,12 +134,9 @@ class DeepgramSTTService(STTService): self._connection: AsyncListenWebSocketClient = self._client.listen.asyncwebsocket.v("1") self._connection.on(LiveTranscriptionEvents.Transcript, self._on_message) - async def set_model(self, model: str, language: Language | None): + async def set_model(self, model: str): logger.debug(f"Switching STT model to: [{model}]") self._live_options.model = model - if language: - logger.debug(f"Switching STT language to: [{language}]") - self._live_options.language = language await self._disconnect() await self._connect() From 5f32fb125d4351f5357d1ac33e0687240a8c7033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 26 Aug 2024 15:06:01 -0700 Subject: [PATCH 15/17] updated CHANGELOG.md --- CHANGELOG.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dc4273be..7acae5c82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,30 @@ All notable changes to **pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Added `TTSModelUpdateFrame`, `TTSLanguageUpdateFrame`, `STTModelUpdateFrame`, + and `STTLanguageUpdateFrame` frames to allow you to switch models, language + and voices in TTS and STT services. + +- Added new `transcriptions.Language` enum. + +### Changed + +- `DailyTransport.on_joined` event now returns the full session data instead of + just the participant. + +- `CartesiaTTSService` is now a subclass of `TTSService`. + +- `DeepgramSTTService` is now a subclass of `STTService`. + +- `WhisperSTTService` is now a subclass of `SegmentedSTTService`. A + `SegmentedSTTService` is a `STTService` where the provided audio is given in a + big chunk (i.e. from when the user starts speaking until the user stops + speaking) instead of a continous stream. + ## [0.0.41] - 2024-08-22 ### Added From be923687fbcee22a126f93407ca4740d3399c141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 28 Aug 2024 15:13:27 -0700 Subject: [PATCH 16/17] processors(rtvi): user decices if bot interrupts on update config --- src/pipecat/processors/frameworks/rtvi.py | 34 ++++++++++++----------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 7ca1c3620..653a883d0 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -81,11 +81,6 @@ class RTVIAction(BaseModel): return super().model_post_init(__context) -# -# Client -> Pipecat messages. -# - - class RTVIServiceOptionConfig(BaseModel): name: str value: Any @@ -100,6 +95,16 @@ class RTVIConfig(BaseModel): config: List[RTVIServiceConfig] +# +# Client -> Pipecat messages. +# + + +class RTVIUpdateConfig(BaseModel): + config: List[RTVIServiceConfig] + interrupt: bool = False + + class RTVIActionRunArgument(BaseModel): name: str value: Any @@ -489,8 +494,8 @@ class RTVIProcessor(FrameProcessor): case "get-config": await self._handle_get_config(message.id) case "update-config": - config = RTVIConfig.model_validate(message.data) - await self._handle_update_config(message.id, config) + update_config = RTVIUpdateConfig.model_validate(message.data) + await self._handle_update_config(message.id, update_config) case "action": action = RTVIActionRun.model_validate(message.data) await self._handle_action(message.id, action) @@ -545,17 +550,14 @@ class RTVIProcessor(FrameProcessor): await handler(self, service.name, option) self._update_config_option(service.name, option) - async def _update_config(self, data: RTVIConfig): + async def _update_config(self, data: RTVIConfig, interrupt: bool): + if interrupt: + await self.interrupt_bot() for service_config in data.config: await self._update_service_config(service_config) - async def _handle_update_config(self, request_id: str, data: RTVIConfig): - # NOTE(aleix): The bot might be talking while we receive a new - # config. Let's interrupt it for now and update the config. Another - # solution is to wait until the bot stops speaking and then apply the - # config, but this definitely is more complicated to achieve. - await self.interrupt_bot() - await self._update_config(data) + async def _handle_update_config(self, request_id: str, data: RTVIUpdateConfig): + await self._update_config(RTVIConfig(config=data.config), data.interrupt) await self._handle_get_config(request_id) async def _handle_function_call_result(self, data): @@ -583,7 +585,7 @@ class RTVIProcessor(FrameProcessor): async def _maybe_send_bot_ready(self): if self._pipeline_started and self._client_ready: await self._send_bot_ready() - await self._update_config(self._config) + await self._update_config(self._config, False) async def _send_bot_ready(self): if not self._params.send_bot_ready: From c0ac5c6ae84139fb441a6737d4e7d80b261c9dc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 29 Aug 2024 11:11:24 -0700 Subject: [PATCH 17/17] services(lmnt): fix example and update README and CHANGELOG --- CHANGELOG.md | 3 +++ README.md | 2 +- examples/foundational/07k-interruptible-lmnt.py | 2 +- src/pipecat/services/ai_services.py | 2 +- src/pipecat/services/lmnt.py | 11 ++++------- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7acae5c82..7bf00326d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `LmntTTSService` text-to-speech service. + (see https://www.lmnt.com/) + - Added `TTSModelUpdateFrame`, `TTSLanguageUpdateFrame`, `STTModelUpdateFrame`, and `STTLanguageUpdateFrame` frames to allow you to switch models, language and voices in TTS and STT services. diff --git a/README.md b/README.md index 40f96636c..681fd3b91 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ pip install "pipecat-ai[option,...]" Your project may or may not need these, so they're made available as optional requirements. Here is a list: -- **AI services**: `anthropic`, `azure`, `deepgram`, `gladia`, `google`, `fal`, `moondream`, `openai`, `openpipe`, `playht`, `silero`, `whisper`, `xtts` +- **AI services**: `anthropic`, `azure`, `deepgram`, `gladia`, `google`, `fal`, `lmnt`, `moondream`, `openai`, `openpipe`, `playht`, `silero`, `whisper`, `xtts` - **Transports**: `local`, `websocket`, `daily` ## Code examples diff --git a/examples/foundational/07k-interruptible-lmnt.py b/examples/foundational/07k-interruptible-lmnt.py index 790c4794e..6e68564ea 100644 --- a/examples/foundational/07k-interruptible-lmnt.py +++ b/examples/foundational/07k-interruptible-lmnt.py @@ -50,7 +50,7 @@ async def main(): tts = LmntTTSService( api_key=os.getenv("LMNT_API_KEY"), - voice="morgan" + voice_id="morgan" ) llm = OpenAILLMService( diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index c10f3e46e..c5cb7cc0d 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -9,7 +9,7 @@ import io import wave from abc import abstractmethod -from typing import AsyncGenerator, Mapping, Optional +from typing import AsyncGenerator, Optional from pipecat.frames.frames import ( AudioRawFrame, diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index e6688dc78..f7afd6a41 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -4,23 +4,19 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import json -import uuid -import base64 import asyncio -import time from typing import AsyncGenerator from pipecat.processors.frame_processor import FrameDirection from pipecat.frames.frames import ( + AudioRawFrame, CancelFrame, + EndFrame, ErrorFrame, Frame, - AudioRawFrame, StartFrame, StartInterruptionFrame, - EndFrame, TTSStartedFrame, TTSStoppedFrame, ) @@ -95,7 +91,8 @@ class LmntTTSService(TTSService): async def _connect(self): try: self._speech = Speech() - self._connection = await self._speech.synthesize_streaming(self._voice_id, format="raw", sample_rate=self._output_format["sample_rate"]) + self._connection = await self._speech.synthesize_streaming( + self._voice_id, format="raw", sample_rate=self._output_format["sample_rate"]) self._receive_task = self.get_event_loop().create_task(self._receive_task_handler()) except Exception as e: logger.exception(f"{self} initialization error: {e}")