services: restructure services into folders
This commit is contained in:
Aleix Conchillo Flaqué
364
src/pipecat/services/google/tts.py
Normal file
364
src/pipecat/services/google/tts.py
Normal file
@@ -0,0 +1,364 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
|
||||
# Suppress gRPC fork warnings
|
||||
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
|
||||
|
||||
from typing import AsyncGenerator, Literal, Optional
|
||||
|
||||
from loguru import logger
|
||||
from pydantic import BaseModel
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
ErrorFrame,
|
||||
Frame,
|
||||
TTSAudioRawFrame,
|
||||
TTSStartedFrame,
|
||||
TTSStoppedFrame,
|
||||
)
|
||||
from pipecat.services.ai_services import TTSService
|
||||
from pipecat.transcriptions.language import Language
|
||||
|
||||
try:
|
||||
from google.cloud import texttospeech_v1
|
||||
from google.oauth2 import service_account
|
||||
|
||||
except ModuleNotFoundError as e:
|
||||
logger.error(f"Exception: {e}")
|
||||
logger.error(
|
||||
"In order to use Google AI, you need to `pip install pipecat-ai[google]`. Also, set `GOOGLE_APPLICATION_CREDENTIALS` environment variable."
|
||||
)
|
||||
raise Exception(f"Missing module: {e}")
|
||||
|
||||
|
||||
def language_to_google_tts_language(language: Language) -> Optional[str]:
|
||||
language_map = {
|
||||
# Afrikaans
|
||||
Language.AF: "af-ZA",
|
||||
Language.AF_ZA: "af-ZA",
|
||||
# Arabic
|
||||
Language.AR: "ar-XA",
|
||||
# Bengali
|
||||
Language.BN: "bn-IN",
|
||||
Language.BN_IN: "bn-IN",
|
||||
# Bulgarian
|
||||
Language.BG: "bg-BG",
|
||||
Language.BG_BG: "bg-BG",
|
||||
# Catalan
|
||||
Language.CA: "ca-ES",
|
||||
Language.CA_ES: "ca-ES",
|
||||
# Chinese (Mandarin and Cantonese)
|
||||
Language.ZH: "cmn-CN",
|
||||
Language.ZH_CN: "cmn-CN",
|
||||
Language.ZH_TW: "cmn-TW",
|
||||
Language.ZH_HK: "yue-HK",
|
||||
# Czech
|
||||
Language.CS: "cs-CZ",
|
||||
Language.CS_CZ: "cs-CZ",
|
||||
# Danish
|
||||
Language.DA: "da-DK",
|
||||
Language.DA_DK: "da-DK",
|
||||
# Dutch
|
||||
Language.NL: "nl-NL",
|
||||
Language.NL_BE: "nl-BE",
|
||||
Language.NL_NL: "nl-NL",
|
||||
# English
|
||||
Language.EN: "en-US",
|
||||
Language.EN_US: "en-US",
|
||||
Language.EN_AU: "en-AU",
|
||||
Language.EN_GB: "en-GB",
|
||||
Language.EN_IN: "en-IN",
|
||||
# Estonian
|
||||
Language.ET: "et-EE",
|
||||
Language.ET_EE: "et-EE",
|
||||
# Filipino
|
||||
Language.FIL: "fil-PH",
|
||||
Language.FIL_PH: "fil-PH",
|
||||
# Finnish
|
||||
Language.FI: "fi-FI",
|
||||
Language.FI_FI: "fi-FI",
|
||||
# French
|
||||
Language.FR: "fr-FR",
|
||||
Language.FR_CA: "fr-CA",
|
||||
Language.FR_FR: "fr-FR",
|
||||
# Galician
|
||||
Language.GL: "gl-ES",
|
||||
Language.GL_ES: "gl-ES",
|
||||
# German
|
||||
Language.DE: "de-DE",
|
||||
Language.DE_DE: "de-DE",
|
||||
# Greek
|
||||
Language.EL: "el-GR",
|
||||
Language.EL_GR: "el-GR",
|
||||
# Gujarati
|
||||
Language.GU: "gu-IN",
|
||||
Language.GU_IN: "gu-IN",
|
||||
# Hebrew
|
||||
Language.HE: "he-IL",
|
||||
Language.HE_IL: "he-IL",
|
||||
# Hindi
|
||||
Language.HI: "hi-IN",
|
||||
Language.HI_IN: "hi-IN",
|
||||
# Hungarian
|
||||
Language.HU: "hu-HU",
|
||||
Language.HU_HU: "hu-HU",
|
||||
# Icelandic
|
||||
Language.IS: "is-IS",
|
||||
Language.IS_IS: "is-IS",
|
||||
# Indonesian
|
||||
Language.ID: "id-ID",
|
||||
Language.ID_ID: "id-ID",
|
||||
# Italian
|
||||
Language.IT: "it-IT",
|
||||
Language.IT_IT: "it-IT",
|
||||
# Japanese
|
||||
Language.JA: "ja-JP",
|
||||
Language.JA_JP: "ja-JP",
|
||||
# Kannada
|
||||
Language.KN: "kn-IN",
|
||||
Language.KN_IN: "kn-IN",
|
||||
# Korean
|
||||
Language.KO: "ko-KR",
|
||||
Language.KO_KR: "ko-KR",
|
||||
# Latvian
|
||||
Language.LV: "lv-LV",
|
||||
Language.LV_LV: "lv-LV",
|
||||
# Lithuanian
|
||||
Language.LT: "lt-LT",
|
||||
Language.LT_LT: "lt-LT",
|
||||
# Malay
|
||||
Language.MS: "ms-MY",
|
||||
Language.MS_MY: "ms-MY",
|
||||
# Malayalam
|
||||
Language.ML: "ml-IN",
|
||||
Language.ML_IN: "ml-IN",
|
||||
# Marathi
|
||||
Language.MR: "mr-IN",
|
||||
Language.MR_IN: "mr-IN",
|
||||
# Norwegian
|
||||
Language.NO: "nb-NO",
|
||||
Language.NB: "nb-NO",
|
||||
Language.NB_NO: "nb-NO",
|
||||
# Polish
|
||||
Language.PL: "pl-PL",
|
||||
Language.PL_PL: "pl-PL",
|
||||
# Portuguese
|
||||
Language.PT: "pt-PT",
|
||||
Language.PT_BR: "pt-BR",
|
||||
Language.PT_PT: "pt-PT",
|
||||
# Punjabi
|
||||
Language.PA: "pa-IN",
|
||||
Language.PA_IN: "pa-IN",
|
||||
# Romanian
|
||||
Language.RO: "ro-RO",
|
||||
Language.RO_RO: "ro-RO",
|
||||
# Russian
|
||||
Language.RU: "ru-RU",
|
||||
Language.RU_RU: "ru-RU",
|
||||
# Serbian
|
||||
Language.SR: "sr-RS",
|
||||
Language.SR_RS: "sr-RS",
|
||||
# Slovak
|
||||
Language.SK: "sk-SK",
|
||||
Language.SK_SK: "sk-SK",
|
||||
# Spanish
|
||||
Language.ES: "es-ES",
|
||||
Language.ES_ES: "es-ES",
|
||||
Language.ES_US: "es-US",
|
||||
# Swedish
|
||||
Language.SV: "sv-SE",
|
||||
Language.SV_SE: "sv-SE",
|
||||
# Tamil
|
||||
Language.TA: "ta-IN",
|
||||
Language.TA_IN: "ta-IN",
|
||||
# Telugu
|
||||
Language.TE: "te-IN",
|
||||
Language.TE_IN: "te-IN",
|
||||
# Thai
|
||||
Language.TH: "th-TH",
|
||||
Language.TH_TH: "th-TH",
|
||||
# Turkish
|
||||
Language.TR: "tr-TR",
|
||||
Language.TR_TR: "tr-TR",
|
||||
# Ukrainian
|
||||
Language.UK: "uk-UA",
|
||||
Language.UK_UA: "uk-UA",
|
||||
# Vietnamese
|
||||
Language.VI: "vi-VN",
|
||||
Language.VI_VN: "vi-VN",
|
||||
}
|
||||
|
||||
return language_map.get(language)
|
||||
|
||||
|
||||
class GoogleTTSService(TTSService):
|
||||
class InputParams(BaseModel):
|
||||
pitch: Optional[str] = None
|
||||
rate: Optional[str] = None
|
||||
volume: Optional[str] = None
|
||||
emphasis: Optional[Literal["strong", "moderate", "reduced", "none"]] = None
|
||||
language: Optional[Language] = Language.EN
|
||||
gender: Optional[Literal["male", "female", "neutral"]] = None
|
||||
google_style: Optional[Literal["apologetic", "calm", "empathetic", "firm", "lively"]] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
credentials: Optional[str] = None,
|
||||
credentials_path: Optional[str] = None,
|
||||
voice_id: str = "en-US-Neural2-A",
|
||||
sample_rate: Optional[int] = None,
|
||||
params: InputParams = InputParams(),
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(sample_rate=sample_rate, **kwargs)
|
||||
|
||||
self._settings = {
|
||||
"pitch": params.pitch,
|
||||
"rate": params.rate,
|
||||
"volume": params.volume,
|
||||
"emphasis": params.emphasis,
|
||||
"language": self.language_to_service_language(params.language)
|
||||
if params.language
|
||||
else "en-US",
|
||||
"gender": params.gender,
|
||||
"google_style": params.google_style,
|
||||
}
|
||||
self.set_voice(voice_id)
|
||||
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
|
||||
credentials, credentials_path
|
||||
)
|
||||
|
||||
def _create_client(
|
||||
self, credentials: Optional[str], credentials_path: Optional[str]
|
||||
) -> texttospeech_v1.TextToSpeechAsyncClient:
|
||||
creds: Optional[service_account.Credentials] = None
|
||||
|
||||
# Create a Google Cloud service account for the Cloud Text-to-Speech API
|
||||
# Using either the provided credentials JSON string or the path to a service account JSON
|
||||
# file, create a Google Cloud service account and use it to authenticate with the API.
|
||||
if credentials:
|
||||
# Use provided credentials JSON string
|
||||
json_account_info = json.loads(credentials)
|
||||
creds = service_account.Credentials.from_service_account_info(json_account_info)
|
||||
elif credentials_path:
|
||||
# Use service account JSON file if provided
|
||||
creds = service_account.Credentials.from_service_account_file(credentials_path)
|
||||
|
||||
return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds)
|
||||
|
||||
def can_generate_metrics(self) -> bool:
|
||||
return True
|
||||
|
||||
def language_to_service_language(self, language: Language) -> Optional[str]:
|
||||
return language_to_google_tts_language(language)
|
||||
|
||||
def _construct_ssml(self, text: str) -> str:
|
||||
ssml = "<speak>"
|
||||
|
||||
# Voice tag
|
||||
voice_attrs = [f"name='{self._voice_id}'"]
|
||||
|
||||
language = self._settings["language"]
|
||||
voice_attrs.append(f"language='{language}'")
|
||||
|
||||
if self._settings["gender"]:
|
||||
voice_attrs.append(f"gender='{self._settings['gender']}'")
|
||||
ssml += f"<voice {' '.join(voice_attrs)}>"
|
||||
|
||||
# Prosody tag
|
||||
prosody_attrs = []
|
||||
if self._settings["pitch"]:
|
||||
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
|
||||
if self._settings["rate"]:
|
||||
prosody_attrs.append(f"rate='{self._settings['rate']}'")
|
||||
if self._settings["volume"]:
|
||||
prosody_attrs.append(f"volume='{self._settings['volume']}'")
|
||||
|
||||
if prosody_attrs:
|
||||
ssml += f"<prosody {' '.join(prosody_attrs)}>"
|
||||
|
||||
# Emphasis tag
|
||||
if self._settings["emphasis"]:
|
||||
ssml += f"<emphasis level='{self._settings['emphasis']}'>"
|
||||
|
||||
# Google style tag
|
||||
if self._settings["google_style"]:
|
||||
ssml += f"<google:style name='{self._settings['google_style']}'>"
|
||||
|
||||
ssml += text
|
||||
|
||||
# Close tags
|
||||
if self._settings["google_style"]:
|
||||
ssml += "</google:style>"
|
||||
if self._settings["emphasis"]:
|
||||
ssml += "</emphasis>"
|
||||
if prosody_attrs:
|
||||
ssml += "</prosody>"
|
||||
ssml += "</voice></speak>"
|
||||
|
||||
return ssml
|
||||
|
||||
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
|
||||
logger.debug(f"{self}: Generating TTS [{text}]")
|
||||
|
||||
try:
|
||||
await self.start_ttfb_metrics()
|
||||
|
||||
# Check if the voice is a Chirp voice (including Chirp 3) or Journey voice
|
||||
is_chirp_voice = "chirp" in self._voice_id.lower()
|
||||
is_journey_voice = "journey" in self._voice_id.lower()
|
||||
|
||||
# Create synthesis input based on voice_id
|
||||
if is_chirp_voice or is_journey_voice:
|
||||
# Chirp and Journey voices don't support SSML, use plain text
|
||||
synthesis_input = texttospeech_v1.SynthesisInput(text=text)
|
||||
else:
|
||||
ssml = self._construct_ssml(text)
|
||||
synthesis_input = texttospeech_v1.SynthesisInput(ssml=ssml)
|
||||
|
||||
voice = texttospeech_v1.VoiceSelectionParams(
|
||||
language_code=self._settings["language"], name=self._voice_id
|
||||
)
|
||||
audio_config = texttospeech_v1.AudioConfig(
|
||||
audio_encoding=texttospeech_v1.AudioEncoding.LINEAR16,
|
||||
sample_rate_hertz=self.sample_rate,
|
||||
)
|
||||
|
||||
request = texttospeech_v1.SynthesizeSpeechRequest(
|
||||
input=synthesis_input, voice=voice, audio_config=audio_config
|
||||
)
|
||||
|
||||
response = await self._client.synthesize_speech(request=request)
|
||||
|
||||
await self.start_tts_usage_metrics(text)
|
||||
|
||||
yield TTSStartedFrame()
|
||||
|
||||
# Skip the first 44 bytes to remove the WAV header
|
||||
audio_content = response.audio_content[44:]
|
||||
|
||||
# Read and yield audio data in chunks
|
||||
chunk_size = 8192
|
||||
for i in range(0, len(audio_content), chunk_size):
|
||||
chunk = audio_content[i : i + chunk_size]
|
||||
if not chunk:
|
||||
break
|
||||
await self.stop_ttfb_metrics()
|
||||
frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
|
||||
yield frame
|
||||
await asyncio.sleep(0) # Allow other tasks to run
|
||||
|
||||
yield TTSStoppedFrame()
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"{self} error generating TTS: {e}")
|
||||
error_message = f"TTS generation error: {str(e)}"
|
||||
yield ErrorFrame(error=error_message)
|
||||
Reference in New Issue
Block a user