From d42d02d809e4a437bfc3ddf7c3fe0653205934bd Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 May 2025 11:21:06 +0200 Subject: [PATCH 1/3] Add Google streaming TTS as a base TTS service. Rename non-streaming service to GoogleHttpTTSService. --- src/pipecat/services/google/tts.py | 147 ++++++++++++++++++++++++++++- 1 file changed, 146 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index 9725aa1c3..4a596ede9 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -202,7 +202,7 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: return language_map.get(language) -class GoogleTTSService(TTSService): +class GoogleHttpTTSService(TTSService): class InputParams(BaseModel): pitch: Optional[str] = None rate: Optional[str] = None @@ -379,3 +379,148 @@ class GoogleTTSService(TTSService): logger.exception(f"{self} error generating TTS: {e}") error_message = f"TTS generation error: {str(e)}" yield ErrorFrame(error=error_message) + + +class GoogleTTSService(TTSService): + class InputParams(BaseModel): + pitch: Optional[str] = None + rate: Optional[str] = None + volume: Optional[str] = None + emphasis: Optional[Literal["strong", "moderate", "reduced", "none"]] = None + language: Optional[Language] = Language.EN + gender: Optional[Literal["male", "female", "neutral"]] = None + google_style: Optional[Literal["apologetic", "calm", "empathetic", "firm", "lively"]] = None + + def __init__( + self, + *, + credentials: Optional[str] = None, + credentials_path: Optional[str] = None, + voice_id: str = "en-US-Neural2-A", + sample_rate: Optional[int] = None, + params: InputParams = InputParams(), + **kwargs, + ): + super().__init__(sample_rate=sample_rate, **kwargs) + + self._settings = { + "pitch": params.pitch, + "rate": params.rate, + "volume": params.volume, + "emphasis": params.emphasis, + "language": self.language_to_service_language(params.language) + if params.language + else "en-US", + "gender": params.gender, + "google_style": params.google_style, + } + self.set_voice(voice_id) + self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( + credentials, credentials_path + ) + + def _create_client( + self, credentials: Optional[str], credentials_path: Optional[str] + ) -> texttospeech_v1.TextToSpeechAsyncClient: + creds: Optional[service_account.Credentials] = None + + # Create a Google Cloud service account for the Cloud Text-to-Speech API + # Using either the provided credentials JSON string or the path to a service account JSON + # file, create a Google Cloud service account and use it to authenticate with the API. + if credentials: + # Use provided credentials JSON string + json_account_info = json.loads(credentials) + creds = service_account.Credentials.from_service_account_info(json_account_info) + elif credentials_path: + # Use service account JSON file if provided + creds = service_account.Credentials.from_service_account_file(credentials_path) + else: + try: + creds, project_id = default( + scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + except GoogleAuthError: + pass + + if not creds: + raise ValueError("No valid credentials provided.") + + return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds) + + def can_generate_metrics(self) -> bool: + return True + + def language_to_service_language(self, language: Language) -> Optional[str]: + return language_to_google_tts_language(language) + + @traced_tts + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + # Chirp3 and Journey voices only. Does not support SSML. + logger.debug(f"{self}: Generating TTS [{text}]") + + try: + await self.start_ttfb_metrics() + + voice = texttospeech_v1.VoiceSelectionParams( + language_code=self._settings["language"], name=self._voice_id + ) + + streaming_config = texttospeech_v1.StreamingSynthesizeConfig( + voice=voice, + streaming_audio_config=texttospeech_v1.StreamingAudioConfig( + audio_encoding=texttospeech_v1.AudioEncoding.PCM, + sample_rate_hertz=self.sample_rate, + #speaking_rate=self._settings["rate"], + ), + ) + config_request = texttospeech_v1.StreamingSynthesizeRequest( + streaming_config=streaming_config + ) + async def request_generator(): + yield config_request + yield texttospeech_v1.StreamingSynthesizeRequest( + input=texttospeech_v1.StreamingSynthesisInput(text=text) + ) + + streaming_responses = await self._client.streaming_synthesize(request_generator()) + await self.start_tts_usage_metrics(text) + + yield TTSStartedFrame() + + audio_buffer = b"" + CHUNK_SIZE = 1024 + first_chunk_for_ttfb = False + + async for response in streaming_responses: + chunk = response.audio_content + if not chunk: + continue + + if not first_chunk_for_ttfb: + await self.stop_ttfb_metrics() + first_chunk_for_ttfb = True + + audio_buffer += chunk + while len(audio_buffer) >= CHUNK_SIZE: + piece = audio_buffer[:CHUNK_SIZE] + audio_buffer = audio_buffer[CHUNK_SIZE:] + yield TTSAudioRawFrame(piece, self.sample_rate, 1) + await asyncio.sleep(0) + + if audio_buffer: + yield TTSAudioRawFrame(audio_buffer, self.sample_rate, 1) + await asyncio.sleep(0) + + # Add 1 second pause between sentences + silence = b"\x00" * self.sample_rate + yield TTSAudioRawFrame( + audio=silence, sample_rate=self.sample_rate, num_channels=1 + ) + await asyncio.sleep(0) + + yield TTSStoppedFrame() + + except Exception as e: + logger.exception(f"{self} error generating TTS: {e}") + error_message = f"TTS generation error: {str(e)}" + yield ErrorFrame(error=error_message) From bfe9952c9a49ae915d434a02f971849688707834 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 23 May 2025 12:11:08 +0200 Subject: [PATCH 2/3] Remove sleep(0), add doc string etc. --- src/pipecat/services/google/tts.py | 84 ++++++++++++++++++------------ 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index 4a596ede9..ce56913cd 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -210,21 +210,23 @@ class GoogleHttpTTSService(TTSService): emphasis: Optional[Literal["strong", "moderate", "reduced", "none"]] = None language: Optional[Language] = Language.EN gender: Optional[Literal["male", "female", "neutral"]] = None - google_style: Optional[Literal["apologetic", "calm", "empathetic", "firm", "lively"]] = None + google_style: Optional[ + Literal["apologetic", "calm", "empathetic", "firm", "lively"] + ] = None def __init__( self, *, credentials: Optional[str] = None, credentials_path: Optional[str] = None, - voice_id: str = "en-US-Neural2-A", + voice_id: str = "en-US-Chirp3-HD-Charon", sample_rate: Optional[int] = None, params: Optional[InputParams] = None, **kwargs, ): super().__init__(sample_rate=sample_rate, **kwargs) - params = params or GoogleTTSService.InputParams() + params = params or GoogleHttpTTSService.InputParams() self._settings = { "pitch": params.pitch, @@ -253,10 +255,14 @@ class GoogleHttpTTSService(TTSService): if credentials: # Use provided credentials JSON string json_account_info = json.loads(credentials) - creds = service_account.Credentials.from_service_account_info(json_account_info) + creds = service_account.Credentials.from_service_account_info( + json_account_info + ) elif credentials_path: # Use service account JSON file if provided - creds = service_account.Credentials.from_service_account_file(credentials_path) + creds = service_account.Credentials.from_service_account_file( + credentials_path + ) else: try: creds, project_id = default( @@ -371,7 +377,6 @@ class GoogleHttpTTSService(TTSService): await self.stop_ttfb_metrics() frame = TTSAudioRawFrame(chunk, self.sample_rate, 1) yield frame - await asyncio.sleep(0) # Allow other tasks to run yield TTSStoppedFrame() @@ -382,14 +387,37 @@ class GoogleHttpTTSService(TTSService): class GoogleTTSService(TTSService): + """Text-to-Speech service using Google Cloud Text-to-Speech API. + + Converts text to speech using Google's TTS models with streaming synthesis + for low latency. Supports multiple languages and voices. + + Args: + credentials: JSON string containing Google Cloud service account credentials. + credentials_path: Path to Google Cloud service account JSON file. + voice_id: Google TTS voice identifier (e.g., "en-US-Chirp3-HD-Charon"). + sample_rate: Audio sample rate in Hz. + params: Language only. + + Notes: + Requires Google Cloud credentials via service account JSON, file path, or + default application credentials (GOOGLE_APPLICATION_CREDENTIALS env var). + Only Chirp 3 HD and Journey voices are supported. Use GoogleHttpTTSService for other voices. + + Example: + ```python + tts = GoogleTTSService( + credentials_path="/path/to/service-account.json", + voice_id="en-US-Chirp3-HD-Charon", + params=GoogleTTSService.InputParams( + language=Language.EN_US, + ) + ) + ``` + """ + class InputParams(BaseModel): - pitch: Optional[str] = None - rate: Optional[str] = None - volume: Optional[str] = None - emphasis: Optional[Literal["strong", "moderate", "reduced", "none"]] = None language: Optional[Language] = Language.EN - gender: Optional[Literal["male", "female", "neutral"]] = None - google_style: Optional[Literal["apologetic", "calm", "empathetic", "firm", "lively"]] = None def __init__( self, @@ -403,16 +431,12 @@ class GoogleTTSService(TTSService): ): super().__init__(sample_rate=sample_rate, **kwargs) + params = params or GoogleTTSService.InputParams() + self._settings = { - "pitch": params.pitch, - "rate": params.rate, - "volume": params.volume, - "emphasis": params.emphasis, "language": self.language_to_service_language(params.language) if params.language else "en-US", - "gender": params.gender, - "google_style": params.google_style, } self.set_voice(voice_id) self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client( @@ -430,10 +454,14 @@ class GoogleTTSService(TTSService): if credentials: # Use provided credentials JSON string json_account_info = json.loads(credentials) - creds = service_account.Credentials.from_service_account_info(json_account_info) + creds = service_account.Credentials.from_service_account_info( + json_account_info + ) elif credentials_path: # Use service account JSON file if provided - creds = service_account.Credentials.from_service_account_file(credentials_path) + creds = service_account.Credentials.from_service_account_file( + credentials_path + ) else: try: creds, project_id = default( @@ -455,7 +483,6 @@ class GoogleTTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - # Chirp3 and Journey voices only. Does not support SSML. logger.debug(f"{self}: Generating TTS [{text}]") try: @@ -470,19 +497,21 @@ class GoogleTTSService(TTSService): streaming_audio_config=texttospeech_v1.StreamingAudioConfig( audio_encoding=texttospeech_v1.AudioEncoding.PCM, sample_rate_hertz=self.sample_rate, - #speaking_rate=self._settings["rate"], ), ) config_request = texttospeech_v1.StreamingSynthesizeRequest( streaming_config=streaming_config ) + async def request_generator(): yield config_request yield texttospeech_v1.StreamingSynthesizeRequest( input=texttospeech_v1.StreamingSynthesisInput(text=text) ) - streaming_responses = await self._client.streaming_synthesize(request_generator()) + streaming_responses = await self._client.streaming_synthesize( + request_generator() + ) await self.start_tts_usage_metrics(text) yield TTSStartedFrame() @@ -505,18 +534,9 @@ class GoogleTTSService(TTSService): piece = audio_buffer[:CHUNK_SIZE] audio_buffer = audio_buffer[CHUNK_SIZE:] yield TTSAudioRawFrame(piece, self.sample_rate, 1) - await asyncio.sleep(0) if audio_buffer: yield TTSAudioRawFrame(audio_buffer, self.sample_rate, 1) - await asyncio.sleep(0) - - # Add 1 second pause between sentences - silence = b"\x00" * self.sample_rate - yield TTSAudioRawFrame( - audio=silence, sample_rate=self.sample_rate, num_channels=1 - ) - await asyncio.sleep(0) yield TTSStoppedFrame() From d3e2a9e5c06938e30bdf077d4b7f88b502a04d4b Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 24 May 2025 15:14:39 +0200 Subject: [PATCH 3/3] Change default voice and fix formatting --- CHANGELOG.md | 4 ++++ src/pipecat/services/google/tts.py | 26 +++++++------------------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dde0bae5..8208329a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `GoogleHttpTTSService` which uses Google's HTTP TTS API. + - Added `PipelineTask.add_observer()` and `PipelineTask.remove_observer()` to allow mangaging observers at runtime. This is useful for cases where the task is passed around to other code components that might want to observe the @@ -73,6 +75,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Updated `GoogleTTSService` to use Google's streaming TTS API. The default voice also updated to `en-US-Chirp3-HD-Charon`. + - `BaseTextFilter` methods `filter()`, `update_settings()`, `handle_interruption()` and `reset_interruption()` are now async. diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index ce56913cd..e28f9fadb 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -210,9 +210,7 @@ class GoogleHttpTTSService(TTSService): emphasis: Optional[Literal["strong", "moderate", "reduced", "none"]] = None language: Optional[Language] = Language.EN gender: Optional[Literal["male", "female", "neutral"]] = None - google_style: Optional[ - Literal["apologetic", "calm", "empathetic", "firm", "lively"] - ] = None + google_style: Optional[Literal["apologetic", "calm", "empathetic", "firm", "lively"]] = None def __init__( self, @@ -255,14 +253,10 @@ class GoogleHttpTTSService(TTSService): if credentials: # Use provided credentials JSON string json_account_info = json.loads(credentials) - creds = service_account.Credentials.from_service_account_info( - json_account_info - ) + creds = service_account.Credentials.from_service_account_info(json_account_info) elif credentials_path: # Use service account JSON file if provided - creds = service_account.Credentials.from_service_account_file( - credentials_path - ) + creds = service_account.Credentials.from_service_account_file(credentials_path) else: try: creds, project_id = default( @@ -424,7 +418,7 @@ class GoogleTTSService(TTSService): *, credentials: Optional[str] = None, credentials_path: Optional[str] = None, - voice_id: str = "en-US-Neural2-A", + voice_id: str = "en-US-Chirp3-HD-Charon", sample_rate: Optional[int] = None, params: InputParams = InputParams(), **kwargs, @@ -454,14 +448,10 @@ class GoogleTTSService(TTSService): if credentials: # Use provided credentials JSON string json_account_info = json.loads(credentials) - creds = service_account.Credentials.from_service_account_info( - json_account_info - ) + creds = service_account.Credentials.from_service_account_info(json_account_info) elif credentials_path: # Use service account JSON file if provided - creds = service_account.Credentials.from_service_account_file( - credentials_path - ) + creds = service_account.Credentials.from_service_account_file(credentials_path) else: try: creds, project_id = default( @@ -509,9 +499,7 @@ class GoogleTTSService(TTSService): input=texttospeech_v1.StreamingSynthesisInput(text=text) ) - streaming_responses = await self._client.streaming_synthesize( - request_generator() - ) + streaming_responses = await self._client.streaming_synthesize(request_generator()) await self.start_tts_usage_metrics(text) yield TTSStartedFrame()