wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #527 from pipecat-ai/mb/google-tts-inputs

Browse Source 
Further consolidate service update settings into a single ServiceUpdateSettingsFrame class
This commit is contained in:
Mark Backman
2024-10-02 11:13:25 -04:00
committed by GitHub
parent 8aae4e9856 3d642df2b0
commit 096a15eef6
19 changed files with 867 additions and 725 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@@ -86,8 +86,8 @@ async def on_connected(processor):
### Changed
- Updated individual update settings frame classes into a single UpdateSettingsFrame
class for STT, LLM, and TTS.
- Updated individual update settings frame classes into a single
ServiceUpdateSettingsFrame class.
- We now distinguish between input and output audio and image frames. We
introduce `InputAudioRawFrame`, `OutputAudioRawFrame`, `InputImageRawFrame`

14
examples/foundational/07e-interruptible-playht.py
View File

@@ -4,11 +4,15 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import aiohttp
import asyncio
import os
import sys
import aiohttp
from dotenv import load_dotenv
from loguru import logger
from runner import configure
from pipecat.frames.frames import LLMMessagesFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
@@ -17,17 +21,11 @@ from pipecat.processors.aggregators.llm_response import (
LLMAssistantResponseAggregator,
LLMUserResponseAggregator,
)
from pipecat.services.playht import PlayHTTTSService
from pipecat.services.openai import OpenAILLMService
from pipecat.services.playht import PlayHTTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport
from pipecat.vad.silero import SileroVADAnalyzer
from runner import configure
from loguru import logger
from dotenv import load_dotenv
load_dotenv(override=True)
logger.remove(0)

15
examples/foundational/07g-interruptible-openai-tts.py
View File

@@ -4,11 +4,15 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import aiohttp
import asyncio
import os
import sys
import aiohttp
from dotenv import load_dotenv
from loguru import logger
from runner import configure
from pipecat.frames.frames import LLMMessagesFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
@@ -17,17 +21,10 @@ from pipecat.processors.aggregators.llm_response import (
LLMAssistantResponseAggregator,
LLMUserResponseAggregator,
)
from pipecat.services.openai import OpenAITTSService
from pipecat.services.openai import OpenAILLMService
from pipecat.services.openai import OpenAILLMService, OpenAITTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport
from pipecat.vad.silero import SileroVADAnalyzer
from runner import configure
from loguru import logger
from dotenv import load_dotenv
load_dotenv(override=True)
logger.remove(0)

12
examples/foundational/16-gpu-container-local-bot.py
View File

@@ -5,10 +5,14 @@
#
import asyncio
import aiohttp
import os
import sys
import aiohttp
from dotenv import load_dotenv
from loguru import logger
from runner import configure
from pipecat.frames.frames import LLMMessagesFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
@@ -26,12 +30,6 @@ from pipecat.transports.services.daily import (
)
from pipecat.vad.silero import SileroVADAnalyzer
from runner import configure
from loguru import logger
from dotenv import load_dotenv
load_dotenv(override=True)
logger.remove(0)

44
src/pipecat/frames/frames.py
View File

@@ -5,7 +5,7 @@
#
from dataclasses import dataclass, field
from typing import Any, List, Optional, Tuple, Union
from typing import Any, Dict, List, Optional, Tuple
from pipecat.clocks.base_clock import BaseClock
from pipecat.metrics.metrics import MetricsData
@@ -527,45 +527,25 @@ class UserImageRequestFrame(ControlFrame):
@dataclass
class LLMUpdateSettingsFrame(ControlFrame):
"""A control frame containing a request to update LLM settings."""
class ServiceUpdateSettingsFrame(ControlFrame):
"""A control frame containing a request to update service settings."""
model: Optional[str] = None
temperature: Optional[float] = None
top_k: Optional[int] = None
top_p: Optional[float] = None
frequency_penalty: Optional[float] = None
presence_penalty: Optional[float] = None
max_tokens: Optional[int] = None
seed: Optional[int] = None
extra: dict = field(default_factory=dict)
settings: Dict[str, Any]
@dataclass
class TTSUpdateSettingsFrame(ControlFrame):
"""A control frame containing a request to update TTS settings."""
model: Optional[str] = None
voice: Optional[str] = None
language: Optional[Language] = None
speed: Optional[Union[str, float]] = None
emotion: Optional[List[str]] = None
engine: Optional[str] = None
pitch: Optional[str] = None
rate: Optional[str] = None
volume: Optional[str] = None
emphasis: Optional[str] = None
style: Optional[str] = None
style_degree: Optional[str] = None
role: Optional[str] = None
class LLMUpdateSettingsFrame(ServiceUpdateSettingsFrame):
pass
@dataclass
class STTUpdateSettingsFrame(ControlFrame):
"""A control frame containing a request to update STT settings."""
class TTSUpdateSettingsFrame(ServiceUpdateSettingsFrame):
pass
model: Optional[str] = None
language: Optional[Language] = None
@dataclass
class STTUpdateSettingsFrame(ServiceUpdateSettingsFrame):
pass
@dataclass

131
src/pipecat/services/ai_services.py
View File

@@ -8,7 +8,7 @@ import asyncio
import io
import wave
from abc import abstractmethod
from typing import AsyncGenerator, List, Optional, Tuple, Union
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
from loguru import logger
@@ -45,6 +45,7 @@ class AIService(FrameProcessor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._model_name: str = ""
self._settings: Dict[str, Any] = {}
@property
def model_name(self) -> str:
@@ -63,6 +64,16 @@ class AIService(FrameProcessor):
async def cancel(self, frame: CancelFrame):
pass
async def _update_settings(self, settings: Dict[str, Any]):
for key, value in settings.items():
if key in self._settings:
logger.debug(f"Updating setting {key} to: [{value}] for {self.name}")
self._settings[key] = value
elif key == "model":
self.set_model_name(value)
else:
logger.warning(f"Unknown setting for {self.name} service: {key}")
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -169,6 +180,8 @@ class TTSService(AIService):
self._push_stop_frames: bool = push_stop_frames
self._stop_frame_timeout_s: float = stop_frame_timeout_s
self._sample_rate: int = sample_rate
self._voice_id: str = ""
self._settings: Dict[str, Any] = {}
self._stop_frame_task: Optional[asyncio.Task] = None
self._stop_frame_queue: asyncio.Queue = asyncio.Queue()
@@ -184,52 +197,8 @@ class TTSService(AIService):
self.set_model_name(model)
@abstractmethod
async def set_voice(self, voice: str):
pass
@abstractmethod
async def set_language(self, language: Language):
pass
@abstractmethod
async def set_speed(self, speed: Union[str, float]):
pass
@abstractmethod
async def set_emotion(self, emotion: List[str]):
pass
@abstractmethod
async def set_engine(self, engine: str):
pass
@abstractmethod
async def set_pitch(self, pitch: str):
pass
@abstractmethod
async def set_rate(self, rate: str):
pass
@abstractmethod
async def set_volume(self, volume: str):
pass
@abstractmethod
async def set_emphasis(self, emphasis: str):
pass
@abstractmethod
async def set_style(self, style: str):
pass
@abstractmethod
async def set_style_degree(self, style_degree: str):
pass
@abstractmethod
async def set_role(self, role: str):
pass
def set_voice(self, voice: str):
self._voice_id = voice
@abstractmethod
async def flush_audio(self):
@@ -259,6 +228,20 @@ class TTSService(AIService):
await self._stop_frame_task
self._stop_frame_task = None
async def _update_settings(self, settings: Dict[str, Any]):
for key, value in settings.items():
if key in self._settings:
logger.debug(f"Updating TTS setting {key} to: [{value}]")
self._settings[key] = value
if key == "language":
self._settings[key] = Language(value)
elif key == "model":
self.set_model_name(value)
elif key == "voice":
self.set_voice(value)
else:
logger.warning(f"Unknown setting for TTS service: {key}")
async def say(self, text: str):
aggregate_sentences = self._aggregate_sentences
self._aggregate_sentences = False
@@ -286,7 +269,7 @@ class TTSService(AIService):
await self._push_tts_frames(frame.text)
await self.flush_audio()
elif isinstance(frame, TTSUpdateSettingsFrame):
await self._update_tts_settings(frame)
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)
@@ -333,34 +316,6 @@ class TTSService(AIService):
# interrupted, the text is not added to the assistant context.
await self.push_frame(TextFrame(text))
async def _update_tts_settings(self, frame: TTSUpdateSettingsFrame):
if frame.model is not None:
await self.set_model(frame.model)
if frame.voice is not None:
await self.set_voice(frame.voice)
if frame.language is not None:
await self.set_language(frame.language)
if frame.speed is not None:
await self.set_speed(frame.speed)
if frame.emotion is not None:
await self.set_emotion(frame.emotion)
if frame.engine is not None:
await self.set_engine(frame.engine)
if frame.pitch is not None:
await self.set_pitch(frame.pitch)
if frame.rate is not None:
await self.set_rate(frame.rate)
if frame.volume is not None:
await self.set_volume(frame.volume)
if frame.emphasis is not None:
await self.set_emphasis(frame.emphasis)
if frame.style is not None:
await self.set_style(frame.style)
if frame.style_degree is not None:
await self.set_style_degree(frame.style_degree)
if frame.role is not None:
await self.set_role(frame.role)
async def _stop_frame_handler(self):
try:
has_started = False
@@ -446,25 +401,29 @@ class STTService(AIService):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._settings: Dict[str, Any] = {}
@abstractmethod
async def set_model(self, model: str):
self.set_model_name(model)
@abstractmethod
async def set_language(self, language: Language):
pass
@abstractmethod
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Returns transcript as a string"""
pass
async def _update_stt_settings(self, frame: STTUpdateSettingsFrame):
if frame.model is not None:
await self.set_model(frame.model)
if frame.language is not None:
await self.set_language(frame.language)
async def _update_settings(self, settings: Dict[str, Any]):
logger.debug(f"Updating STT settings: {self._settings}")
for key, value in settings.items():
if key in self._settings:
logger.debug(f"Updating STT setting {key} to: [{value}]")
self._settings[key] = value
if key == "language":
self._settings[key] = Language(value)
elif key == "model":
self.set_model_name(value)
else:
logger.warning(f"Unknown setting for STT service: {key}")
async def process_audio_frame(self, frame: AudioRawFrame):
await self.process_generator(self.run_stt(frame.audio))
@@ -478,7 +437,7 @@ class STTService(AIService):
# push a TextFrame. We don't really want to push audio frames down.
await self.process_audio_frame(frame)
elif isinstance(frame, STTUpdateSettingsFrame):
await self._update_stt_settings(frame)
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)

71
src/pipecat/services/anthropic.py
View File

@@ -96,12 +96,14 @@ class AnthropicLLMService(LLMService):
super().__init__(**kwargs)
self._client = AsyncAnthropic(api_key=api_key)
self.set_model_name(model)
self._max_tokens = params.max_tokens
self._enable_prompt_caching_beta: bool = params.enable_prompt_caching_beta or False
self._temperature = params.temperature
self._top_k = params.top_k
self._top_p = params.top_p
self._extra = params.extra if isinstance(params.extra, dict) else {}
self._settings = {
"max_tokens": params.max_tokens,
"enable_prompt_caching_beta": params.enable_prompt_caching_beta or False,
"temperature": params.temperature,
"top_k": params.top_k,
"top_p": params.top_p,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
def can_generate_metrics(self) -> bool:
return True
@@ -120,30 +122,6 @@ class AnthropicLLMService(LLMService):
)
return AnthropicContextAggregatorPair(_user=user, _assistant=assistant)
async def set_enable_prompt_caching_beta(self, enable_prompt_caching_beta: bool):
logger.debug(f"Switching LLM enable_prompt_caching_beta to: [{enable_prompt_caching_beta}]")
self._enable_prompt_caching_beta = enable_prompt_caching_beta
async def set_max_tokens(self, max_tokens: int):
logger.debug(f"Switching LLM max_tokens to: [{max_tokens}]")
self._max_tokens = max_tokens
async def set_temperature(self, temperature: float):
logger.debug(f"Switching LLM temperature to: [{temperature}]")
self._temperature = temperature
async def set_top_k(self, top_k: float):
logger.debug(f"Switching LLM top_k to: [{top_k}]")
self._top_k = top_k
async def set_top_p(self, top_p: float):
logger.debug(f"Switching LLM top_p to: [{top_p}]")
self._top_p = top_p
async def set_extra(self, extra: Dict[str, Any]):
logger.debug(f"Switching LLM extra to: [{extra}]")
self._extra = extra
async def _process_context(self, context: OpenAILLMContext):
# Usage tracking. We track the usage reported by Anthropic in prompt_tokens and
# completion_tokens. We also estimate the completion tokens from output text
@@ -165,11 +143,11 @@ class AnthropicLLMService(LLMService):
)
messages = context.messages
if self._enable_prompt_caching_beta:
if self._settings["enable_prompt_caching_beta"]:
messages = context.get_messages_with_cache_control_markers()
api_call = self._client.messages.create
if self._enable_prompt_caching_beta:
if self._settings["enable_prompt_caching_beta"]:
api_call = self._client.beta.prompt_caching.messages.create
await self.start_ttfb_metrics()
@@ -179,14 +157,14 @@ class AnthropicLLMService(LLMService):
"system": context.system,
"messages": messages,
"model": self.model_name,
"max_tokens": self._max_tokens,
"max_tokens": self._settings["max_tokens"],
"stream": True,
"temperature": self._temperature,
"top_k": self._top_k,
"top_p": self._top_p,
"temperature": self._settings["temperature"],
"top_k": self._settings["top_k"],
"top_p": self._settings["top_p"],
}
params.update(self._extra)
params.update(self._settings["extra"])
response = await api_call(**params)
@@ -284,21 +262,6 @@ class AnthropicLLMService(LLMService):
cache_read_input_tokens=cache_read_input_tokens,
)
async def _update_settings(self, frame: LLMUpdateSettingsFrame):
if frame.model is not None:
logger.debug(f"Switching LLM model to: [{frame.model}]")
self.set_model_name(frame.model)
if frame.max_tokens is not None:
await self.set_max_tokens(frame.max_tokens)
if frame.temperature is not None:
await self.set_temperature(frame.temperature)
if frame.top_k is not None:
await self.set_top_k(frame.top_k)
if frame.top_p is not None:
await self.set_top_p(frame.top_p)
if frame.extra:
await self.set_extra(frame.extra)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -314,10 +277,10 @@ class AnthropicLLMService(LLMService):
# to the context.
context = AnthropicLLMContext.from_image_frame(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame)
await self._update_settings(frame.settings)
elif isinstance(frame, LLMEnablePromptCachingFrame):
logger.debug(f"Setting enable prompt caching to: [{frame.enable}]")
self._enable_prompt_caching_beta = frame.enable
self._settings["enable_prompt_caching_beta"] = frame.enable
else:
await self.push_frame(frame, direction)

134
src/pipecat/services/aws.py
View File

@@ -6,6 +6,7 @@
from typing import AsyncGenerator, Optional
from loguru import logger
from pydantic import BaseModel
from pipecat.frames.frames import (
@@ -16,8 +17,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
from loguru import logger
from pipecat.transcriptions.language import Language
try:
import boto3
@@ -30,10 +30,71 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
def language_to_aws_language(language: Language) -> str | None:
match language:
case Language.CA:
return "ca-ES"
case Language.ZH:
return "cmn-CN"
case Language.DA:
return "da-DK"
case Language.NL:
return "nl-NL"
case Language.NL_BE:
return "nl-BE"
case Language.EN:
return "en-US"
case Language.EN_US:
return "en-US"
case Language.EN_AU:
return "en-AU"
case Language.EN_GB:
return "en-GB"
case Language.EN_NZ:
return "en-NZ"
case Language.EN_IN:
return "en-IN"
case Language.FI:
return "fi-FI"
case Language.FR:
return "fr-FR"
case Language.FR_CA:
return "fr-CA"
case Language.DE:
return "de-DE"
case Language.HI:
return "hi-IN"
case Language.IT:
return "it-IT"
case Language.JA:
return "ja-JP"
case Language.KO:
return "ko-KR"
case Language.NO:
return "nb-NO"
case Language.PL:
return "pl-PL"
case Language.PT:
return "pt-PT"
case Language.PT_BR:
return "pt-BR"
case Language.RO:
return "ro-RO"
case Language.RU:
return "ru-RU"
case Language.ES:
return "es-ES"
case Language.SV:
return "sv-SE"
case Language.TR:
return "tr-TR"
return None
class AWSTTSService(TTSService):
class InputParams(BaseModel):
engine: Optional[str] = None
language: Optional[str] = None
language: Optional[Language] = Language.EN
pitch: Optional[str] = None
rate: Optional[str] = None
volume: Optional[str] = None
@@ -57,9 +118,16 @@ class AWSTTSService(TTSService):
aws_secret_access_key=api_key,
region_name=region,
)
self._voice_id = voice_id
self._sample_rate = sample_rate
self._params = params
self._settings = {
"sample_rate": sample_rate,
"engine": params.engine,
"language": language_to_aws_language(params.language) if params.language else "en-US",
"pitch": params.pitch,
"rate": params.rate,
"volume": params.volume,
}
self.set_voice(voice_id)
def can_generate_metrics(self) -> bool:
return True
@@ -67,18 +135,18 @@ class AWSTTSService(TTSService):
def _construct_ssml(self, text: str) -> str:
ssml = "<speak>"
if self._params.language:
ssml += f"<lang xml:lang='{self._params.language}'>"
if self._settings["language"]:
ssml += f"<lang xml:lang='{self._settings["language"]}'>"
prosody_attrs = []
# Prosody tags are only supported for standard and neural engines
if self._params.engine != "generative":
if self._params.rate:
prosody_attrs.append(f"rate='{self._params.rate}'")
if self._params.pitch:
prosody_attrs.append(f"pitch='{self._params.pitch}'")
if self._params.volume:
prosody_attrs.append(f"volume='{self._params.volume}'")
if self._settings["engine"] != "generative":
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings["rate"]}'")
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings["pitch"]}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings["volume"]}'")
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
@@ -90,41 +158,13 @@ class AWSTTSService(TTSService):
if prosody_attrs:
ssml += "</prosody>"
if self._params.language:
if self._settings["language"]:
ssml += "</lang>"
ssml += "</speak>"
return ssml
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice_id = voice
async def set_engine(self, engine: str):
logger.debug(f"Switching TTS engine to: [{engine}]")
self._params.engine = engine
async def set_language(self, language: str):
logger.debug(f"Switching TTS language to: [{language}]")
self._params.language = language
async def set_pitch(self, pitch: str):
logger.debug(f"Switching TTS pitch to: [{pitch}]")
self._params.pitch = pitch
async def set_rate(self, rate: str):
logger.debug(f"Switching TTS rate to: [{rate}]")
self._params.rate = rate
async def set_volume(self, volume: str):
logger.debug(f"Switching TTS volume to: [{volume}]")
self._params.volume = volume
async def set_params(self, params: InputParams):
logger.debug(f"Switching TTS params to: [{params}]")
self._params = params
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
@@ -139,8 +179,8 @@ class AWSTTSService(TTSService):
"TextType": "ssml",
"OutputFormat": "pcm",
"VoiceId": self._voice_id,
"Engine": self._params.engine,
"SampleRate": str(self._sample_rate),
"Engine": self._settings["engine"],
"SampleRate": str(self._settings["sample_rate"]),
}
# Filter out None values
@@ -160,7 +200,7 @@ class AWSTTSService(TTSService):
chunk = audio_data[i : i + chunk_size]
if len(chunk) > 0:
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(chunk, self._sample_rate, 1)
frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
yield frame
await self.push_frame(TTSStoppedFrame())

216
src/pipecat/services/azure.py
View File

@@ -4,12 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import aiohttp
import asyncio
import io
from typing import AsyncGenerator, Optional
import aiohttp
from loguru import logger
from PIL import Image
from pydantic import BaseModel
from pipecat.frames.frames import (
@@ -26,12 +27,9 @@ from pipecat.frames.frames import (
)
from pipecat.services.ai_services import ImageGenService, STTService, TTSService
from pipecat.services.openai import BaseOpenAILLMService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from PIL import Image
from loguru import logger
# See .env.example for Azure configuration needed
try:
from azure.cognitiveservices.speech import (
@@ -73,10 +71,101 @@ class AzureLLMService(BaseOpenAILLMService):
)
def language_to_azure_language(language: Language) -> str | None:
match language:
case Language.BG:
return "bg-BG"
case Language.CA:
return "ca-ES"
case Language.ZH:
return "zh-CN"
case Language.ZH_TW:
return "zh-TW"
case Language.CS:
return "cs-CZ"
case Language.DA:
return "da-DK"
case Language.NL:
return "nl-NL"
case Language.EN:
return "en-US"
case Language.EN_US:
return "en-US"
case Language.EN_AU:
return "en-AU"
case Language.EN_GB:
return "en-GB"
case Language.EN_NZ:
return "en-NZ"
case Language.EN_IN:
return "en-IN"
case Language.ET:
return "et-EE"
case Language.FI:
return "fi-FI"
case Language.NL_BE:
return "nl-BE"
case Language.FR:
return "fr-FR"
case Language.FR_CA:
return "fr-CA"
case Language.DE:
return "de-DE"
case Language.DE_CH:
return "de-CH"
case Language.EL:
return "el-GR"
case Language.HI:
return "hi-IN"
case Language.HU:
return "hu-HU"
case Language.ID:
return "id-ID"
case Language.IT:
return "it-IT"
case Language.JA:
return "ja-JP"
case Language.KO:
return "ko-KR"
case Language.LV:
return "lv-LV"
case Language.LT:
return "lt-LT"
case Language.MS:
return "ms-MY"
case Language.NO:
return "nb-NO"
case Language.PL:
return "pl-PL"
case Language.PT:
return "pt-PT"
case Language.PT_BR:
return "pt-BR"
case Language.RO:
return "ro-RO"
case Language.RU:
return "ru-RU"
case Language.SK:
return "sk-SK"
case Language.ES:
return "es-ES"
case Language.SV:
return "sv-SE"
case Language.TH:
return "th-TH"
case Language.TR:
return "tr-TR"
case Language.UK:
return "uk-UA"
case Language.VI:
return "vi-VN"
return None
class AzureTTSService(TTSService):
class InputParams(BaseModel):
emphasis: Optional[str] = None
language: Optional[str] = "en-US"
language: Optional[Language] = Language.EN
pitch: Optional[str] = None
rate: Optional[str] = "1.05"
role: Optional[str] = None
@@ -99,114 +188,67 @@ class AzureTTSService(TTSService):
speech_config = SpeechConfig(subscription=api_key, region=region)
self._speech_synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=None)
self._voice = voice
self._sample_rate = sample_rate
self._params = params
self._settings = {
"sample_rate": sample_rate,
"emphasis": params.emphasis,
"language": language_to_azure_language(params.language) if params.language else "en-US",
"pitch": params.pitch,
"rate": params.rate,
"role": params.role,
"style": params.style,
"style_degree": params.style_degree,
"volume": params.volume,
}
self.set_voice(voice)
def can_generate_metrics(self) -> bool:
return True
def _construct_ssml(self, text: str) -> str:
ssml = (
f"<speak version='1.0' xml:lang='{self._params.language}' "
f"<speak version='1.0' xml:lang='{self._settings['language']}' "
"xmlns='http://www.w3.org/2001/10/synthesis' "
"xmlns:mstts='http://www.w3.org/2001/mstts'>"
f"<voice name='{self._voice}'>"
f"<voice name='{self._voice_id}'>"
"<mstts:silence type='Sentenceboundary' value='20ms' />"
)
if self._params.style:
ssml += f"<mstts:express-as style='{self._params.style}'"
if self._params.style_degree:
ssml += f" styledegree='{self._params.style_degree}'"
if self._params.role:
ssml += f" role='{self._params.role}'"
if self._settings["style"]:
ssml += f"<mstts:express-as style='{self._settings['style']}'"
if self._settings["style_degree"]:
ssml += f" styledegree='{self._settings['style_degree']}'"
if self._settings["role"]:
ssml += f" role='{self._settings['role']}'"
ssml += ">"
prosody_attrs = []
if self._params.rate:
prosody_attrs.append(f"rate='{self._params.rate}'")
if self._params.pitch:
prosody_attrs.append(f"pitch='{self._params.pitch}'")
if self._params.volume:
prosody_attrs.append(f"volume='{self._params.volume}'")
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
ssml += f"<prosody {' '.join(prosody_attrs)}>"
if self._params.emphasis:
ssml += f"<emphasis level='{self._params.emphasis}'>"
if self._settings["emphasis"]:
ssml += f"<emphasis level='{self._settings['emphasis']}'>"
ssml += text
if self._params.emphasis:
if self._settings["emphasis"]:
ssml += "</emphasis>"
ssml += "</prosody>"
if self._params.style:
if self._settings["style"]:
ssml += "</mstts:express-as>"
ssml += "</voice></speak>"
return ssml
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice = voice
async def set_emphasis(self, emphasis: str):
logger.debug(f"Setting TTS emphasis to: [{emphasis}]")
self._params.emphasis = emphasis
async def set_language(self, language: str):
logger.debug(f"Setting TTS language code to: [{language}]")
self._params.language = language
async def set_pitch(self, pitch: str):
logger.debug(f"Setting TTS pitch to: [{pitch}]")
self._params.pitch = pitch
async def set_rate(self, rate: str):
logger.debug(f"Setting TTS rate to: [{rate}]")
self._params.rate = rate
async def set_role(self, role: str):
logger.debug(f"Setting TTS role to: [{role}]")
self._params.role = role
async def set_style(self, style: str):
logger.debug(f"Setting TTS style to: [{style}]")
self._params.style = style
async def set_style_degree(self, style_degree: str):
logger.debug(f"Setting TTS style degree to: [{style_degree}]")
self._params.style_degree = style_degree
async def set_volume(self, volume: str):
logger.debug(f"Setting TTS volume to: [{volume}]")
self._params.volume = volume
async def set_params(self, **kwargs):
valid_params = {
"voice": self.set_voice,
"emphasis": self.set_emphasis,
"language_code": self.set_language,
"pitch": self.set_pitch,
"rate": self.set_rate,
"role": self.set_role,
"style": self.set_style,
"style_degree": self.set_style_degree,
"volume": self.set_volume,
}
for param, value in kwargs.items():
if param in valid_params:
await valid_params[param](value)
else:
logger.warning(f"Ignoring unknown parameter: {param}")
logger.debug(f"Updated TTS parameters: {', '.join(kwargs.keys())}")
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
@@ -222,7 +264,9 @@ class AzureTTSService(TTSService):
await self.push_frame(TTSStartedFrame())
# Azure always sends a 44-byte header. Strip it off.
yield TTSAudioRawFrame(
audio=result.audio_data[44:], sample_rate=self._sample_rate, num_channels=1
audio=result.audio_data[44:],
sample_rate=self._settings["sample_rate"],
num_channels=1,
)
await self.push_frame(TTSStoppedFrame())
elif result.reason == ResultReason.Canceled:

139
src/pipecat/services/cartesia.py
View File

@@ -4,36 +4,35 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import asyncio
import base64
import json
import uuid
import base64
import asyncio
from typing import AsyncGenerator, List, Optional, Union
from typing import AsyncGenerator, Optional, Union, List
from loguru import logger
from pydantic.main import BaseModel
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
ErrorFrame,
Frame,
StartInterruptionFrame,
LLMFullResponseEndFrame,
StartFrame,
EndFrame,
StartInterruptionFrame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
LLMFullResponseEndFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import TTSService, WordTTSService
from pipecat.transcriptions.language import Language
from pipecat.services.ai_services import WordTTSService, TTSService
from loguru import logger
# See .env.example for Cartesia configuration needed
try:
from cartesia import AsyncCartesia
import websockets
from cartesia import AsyncCartesia
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
@@ -66,7 +65,7 @@ class CartesiaTTSService(WordTTSService):
encoding: Optional[str] = "pcm_s16le"
sample_rate: Optional[int] = 16000
container: Optional[str] = "raw"
language: Optional[str] = "en"
language: Optional[Language] = Language.EN
speed: Optional[Union[str, float]] = ""
emotion: Optional[List[str]] = []
@@ -77,7 +76,7 @@ class CartesiaTTSService(WordTTSService):
voice_id: str,
cartesia_version: str = "2024-06-10",
url: str = "wss://api.cartesia.ai/tts/websocket",
model_id: str = "sonic-english",
model: str = "sonic-english",
params: InputParams = InputParams(),
**kwargs,
):
@@ -101,17 +100,18 @@ class CartesiaTTSService(WordTTSService):
self._api_key = api_key
self._cartesia_version = cartesia_version
self._url = url
self._voice_id = voice_id
self._model_id = model_id
self.set_model_name(model_id)
self._output_format = {
"container": params.container,
"encoding": params.encoding,
"sample_rate": params.sample_rate,
self._settings = {
"output_format": {
"container": params.container,
"encoding": params.encoding,
"sample_rate": params.sample_rate,
},
"language": language_to_cartesia_language(params.language) if params.language else "en",
"speed": params.speed,
"emotion": params.emotion,
}
self._language = params.language
self._speed = params.speed
self._emotion = params.emotion
self.set_model_name(model)
self.set_voice(voice_id)
self._websocket = None
self._context_id = None
@@ -125,42 +125,28 @@ class CartesiaTTSService(WordTTSService):
await super().set_model(model)
logger.debug(f"Switching TTS model to: [{model}]")
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice_id = voice
async def set_speed(self, speed: str):
logger.debug(f"Switching TTS speed to: [{speed}]")
self._speed = speed
async def set_emotion(self, emotion: list[str]):
logger.debug(f"Switching TTS emotion to: [{emotion}]")
self._emotion = emotion
async def set_language(self, language: Language):
logger.debug(f"Switching TTS language to: [{language}]")
self._language = language_to_cartesia_language(language)
def _build_msg(
self, text: str = "", continue_transcript: bool = True, add_timestamps: bool = True
):
voice_config = {"mode": "id", "id": self._voice_id}
voice_config = {}
voice_config["mode"] = "id"
voice_config["id"] = self._voice_id
if self._speed or self._emotion:
if self._settings["speed"] or self._settings["emotion"]:
voice_config["__experimental_controls"] = {}
if self._speed:
voice_config["__experimental_controls"]["speed"] = self._speed
if self._emotion:
voice_config["__experimental_controls"]["emotion"] = self._emotion
if self._settings["speed"]:
voice_config["__experimental_controls"]["speed"] = self._settings["speed"]
if self._settings["emotion"]:
voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"]
msg = {
"transcript": text,
"continue": continue_transcript,
"context_id": self._context_id,
"model_id": self._model_name,
"model_id": self.model_name,
"voice": voice_config,
"output_format": self._output_format,
"language": self._language,
"output_format": self._settings["output_format"],
"language": self._settings["language"],
"add_timestamps": add_timestamps,
}
return json.dumps(msg)
@@ -245,7 +231,7 @@ class CartesiaTTSService(WordTTSService):
self.start_word_timestamps()
frame = TTSAudioRawFrame(
audio=base64.b64decode(msg["data"]),
sample_rate=self._output_format["sample_rate"],
sample_rate=self._settings["output_format"]["sample_rate"],
num_channels=1,
)
await self.push_frame(frame)
@@ -294,7 +280,7 @@ class CartesiaHttpTTSService(TTSService):
encoding: Optional[str] = "pcm_s16le"
sample_rate: Optional[int] = 16000
container: Optional[str] = "raw"
language: Optional[str] = "en"
language: Optional[Language] = Language.EN
speed: Optional[Union[str, float]] = ""
emotion: Optional[List[str]] = []
@@ -303,7 +289,7 @@ class CartesiaHttpTTSService(TTSService):
*,
api_key: str,
voice_id: str,
model_id: str = "sonic-english",
model: str = "sonic-english",
base_url: str = "https://api.cartesia.ai",
params: InputParams = InputParams(),
**kwargs,
@@ -311,17 +297,18 @@ class CartesiaHttpTTSService(TTSService):
super().__init__(**kwargs)
self._api_key = api_key
self._voice_id = voice_id
self._model_id = model_id
self.set_model_name(model_id)
self._output_format = {
"container": params.container,
"encoding": params.encoding,
"sample_rate": params.sample_rate,
self._settings = {
"output_format": {
"container": params.container,
"encoding": params.encoding,
"sample_rate": params.sample_rate,
},
"language": language_to_cartesia_language(params.language) if params.language else None,
"speed": params.speed,
"emotion": params.emotion,
}
self._language = params.language
self._speed = params.speed
self._emotion = params.emotion
self.set_voice(voice_id)
self.set_model_name(model)
self._client = AsyncCartesia(api_key=api_key, base_url=base_url)
@@ -333,22 +320,6 @@ class CartesiaHttpTTSService(TTSService):
self._model_id = model
await super().set_model(model)
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice_id = voice
async def set_speed(self, speed: str):
logger.debug(f"Switching TTS speed to: [{speed}]")
self._speed = speed
async def set_emotion(self, emotion: list[str]):
logger.debug(f"Switching TTS emotion to: [{emotion}]")
self._emotion = emotion
async def set_language(self, language: Language):
logger.debug(f"Switching TTS language to: [{language}]")
self._language = language_to_cartesia_language(language)
async def stop(self, frame: EndFrame):
await super().stop(frame)
await self._client.close()
@@ -365,19 +336,19 @@ class CartesiaHttpTTSService(TTSService):
try:
voice_controls = None
if self._speed or self._emotion:
if self._settings["speed"] or self._settings["emotion"]:
voice_controls = {}
if self._speed:
voice_controls["speed"] = self._speed
if self._emotion:
voice_controls["emotion"] = self._emotion
if self._settings["speed"]:
voice_controls["speed"] = self._settings["speed"]
if self._settings["emotion"]:
voice_controls["emotion"] = self._settings["emotion"]
output = await self._client.tts.sse(
model_id=self._model_id,
transcript=text,
voice_id=self._voice_id,
output_format=self._output_format,
language=self._language,
output_format=self._settings["output_format"],
language=self._settings["language"],
stream=False,
_experimental_voice_controls=voice_controls,
)
@@ -386,7 +357,7 @@ class CartesiaHttpTTSService(TTSService):
frame = TTSAudioRawFrame(
audio=output["audio"],
sample_rate=self._output_format["sample_rate"],
sample_rate=self._settings["output_format"]["sample_rate"],
num_channels=1,
)
yield frame

43
src/pipecat/services/deepgram.py
View File

@@ -5,9 +5,10 @@
#
import asyncio
from typing import AsyncGenerator
from loguru import logger
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
@@ -24,8 +25,6 @@ from pipecat.services.ai_services import STTService, TTSService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from loguru import logger
# See .env.example for Deepgram configuration needed
try:
from deepgram import (
@@ -57,25 +56,23 @@ class DeepgramTTSService(TTSService):
):
super().__init__(**kwargs)
self._voice = voice
self._sample_rate = sample_rate
self._encoding = encoding
self._settings = {
"sample_rate": sample_rate,
"encoding": encoding,
}
self.set_voice(voice)
self._deepgram_client = DeepgramClient(api_key=api_key)
def can_generate_metrics(self) -> bool:
return True
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice = voice
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
options = SpeakOptions(
model=self._voice,
encoding=self._encoding,
sample_rate=self._sample_rate,
model=self._voice_id,
encoding=self._settings["encoding"],
sample_rate=self._settings["sample_rate"],
container="none",
)
@@ -103,7 +100,9 @@ class DeepgramTTSService(TTSService):
chunk = audio_buffer.read(chunk_size)
if not chunk:
break
frame = TTSAudioRawFrame(audio=chunk, sample_rate=self._sample_rate, num_channels=1)
frame = TTSAudioRawFrame(
audio=chunk, sample_rate=self._settings["sample_rate"], num_channels=1
)
yield frame
await self.push_frame(TTSStoppedFrame())
@@ -121,7 +120,7 @@ class DeepgramSTTService(STTService):
url: str = "",
live_options: LiveOptions = LiveOptions(
encoding="linear16",
language="en-US",
language=Language.EN,
model="nova-2-conversationalai",
sample_rate=16000,
channels=1,
@@ -135,7 +134,7 @@ class DeepgramSTTService(STTService):
):
super().__init__(**kwargs)
self._live_options = live_options
self._settings = vars(live_options)
self._client = DeepgramClient(
api_key, config=DeepgramClientOptions(url=url, options={"keepalive": "true"})
@@ -147,7 +146,7 @@ class DeepgramSTTService(STTService):
@property
def vad_enabled(self):
return self._live_options.vad_events
return self._settings["vad_events"]
def can_generate_metrics(self) -> bool:
return self.vad_enabled
@@ -155,13 +154,7 @@ class DeepgramSTTService(STTService):
async def set_model(self, model: str):
await super().set_model(model)
logger.debug(f"Switching STT model to: [{model}]")
self._live_options.model = model
await self._disconnect()
await self._connect()
async def set_language(self, language: Language):
logger.debug(f"Switching STT language to: [{language}]")
self._live_options.language = language
self._settings["model"] = model
await self._disconnect()
await self._connect()
@@ -182,7 +175,7 @@ class DeepgramSTTService(STTService):
yield None
async def _connect(self):
if await self._connection.start(self._live_options):
if await self._connection.start(self._settings):
logger.debug(f"{self}: Connected to Deepgram")
else:
logger.error(f"{self}: Unable to connect to Deepgram")

159
src/pipecat/services/elevenlabs.py
View File

@@ -24,6 +24,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import WordTTSService
from pipecat.transcriptions.language import Language
# See .env.example for ElevenLabs configuration needed
try:
@@ -49,6 +50,76 @@ def sample_rate_from_output_format(output_format: str) -> int:
return 16000
def language_to_elevenlabs_language(language: Language) -> str | None:
match language:
case Language.BG:
return "bg"
case Language.ZH:
return "zh"
case Language.CS:
return "cs"
case Language.DA:
return "da"
case Language.NL:
return "nl"
case (
Language.EN
| Language.EN_US
| Language.EN_AU
| Language.EN_GB
| Language.EN_NZ
| Language.EN_IN
):
return "en"
case Language.FI:
return "fi"
case Language.FR | Language.FR_CA:
return "fr"
case Language.DE | Language.DE_CH:
return "de"
case Language.EL:
return "el"
case Language.HI:
return "hi"
case Language.HU:
return "hu"
case Language.ID:
return "id"
case Language.IT:
return "it"
case Language.JA:
return "ja"
case Language.KO:
return "ko"
case Language.MS:
return "ms"
case Language.NO:
return "no"
case Language.PL:
return "pl"
case Language.PT:
return "pt-PT"
case Language.PT_BR:
return "pt-BR"
case Language.RO:
return "ro"
case Language.RU:
return "ru"
case Language.SK:
return "sk"
case Language.ES:
return "es"
case Language.SV:
return "sv"
case Language.TR:
return "tr"
case Language.UK:
return "uk"
case Language.VI:
return "vi"
return None
def calculate_word_times(
alignment_info: Mapping[str, Any], cumulative_time: float
) -> List[Tuple[str, float]]:
@@ -72,7 +143,7 @@ def calculate_word_times(
class ElevenLabsTTSService(WordTTSService):
class InputParams(BaseModel):
language: Optional[str] = None
language: Optional[Language] = Language.EN
output_format: Literal["pcm_16000", "pcm_22050", "pcm_24000", "pcm_44100"] = "pcm_16000"
optimize_streaming_latency: Optional[str] = None
stability: Optional[float] = None
@@ -124,10 +195,21 @@ class ElevenLabsTTSService(WordTTSService):
)
self._api_key = api_key
self._voice_id = voice_id
self.set_model_name(model)
self._url = url
self._params = params
self._settings = {
"sample_rate": sample_rate_from_output_format(params.output_format),
"language": language_to_elevenlabs_language(params.language)
if params.language
else "en",
"output_format": params.output_format,
"optimize_streaming_latency": params.optimize_streaming_latency,
"stability": params.stability,
"similarity_boost": params.similarity_boost,
"style": params.style,
"use_speaker_boost": params.use_speaker_boost,
}
self.set_model_name(model)
self.set_voice(voice_id)
self._voice_settings = self._set_voice_settings()
# Websocket connection to ElevenLabs.
@@ -142,19 +224,22 @@ class ElevenLabsTTSService(WordTTSService):
def _set_voice_settings(self):
voice_settings = {}
if self._params.stability is not None and self._params.similarity_boost is not None:
voice_settings["stability"] = self._params.stability
voice_settings["similarity_boost"] = self._params.similarity_boost
if self._params.style is not None:
voice_settings["style"] = self._params.style
if self._params.use_speaker_boost is not None:
voice_settings["use_speaker_boost"] = self._params.use_speaker_boost
if (
self._settings["stability"] is not None
and self._settings["similarity_boost"] is not None
):
voice_settings["stability"] = self._settings["stability"]
voice_settings["similarity_boost"] = self._settings["similarity_boost"]
if self._settings["style"] is not None:
voice_settings["style"] = self._settings["style"]
if self._settings["use_speaker_boost"] is not None:
voice_settings["use_speaker_boost"] = self._settings["use_speaker_boost"]
else:
if self._params.style is not None:
if self._settings["style"] is not None:
logger.warning(
"'style' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
)
if self._params.use_speaker_boost is not None:
if self._settings["use_speaker_boost"] is not None:
logger.warning(
"'use_speaker_boost' is set but will not be applied because 'stability' and 'similarity_boost' are not both set."
)
@@ -167,33 +252,13 @@ class ElevenLabsTTSService(WordTTSService):
await self._disconnect()
await self._connect()
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice_id = voice
await self._disconnect()
await self._connect()
async def set_voice_settings(
self,
stability: Optional[float] = None,
similarity_boost: Optional[float] = None,
style: Optional[float] = None,
use_speaker_boost: Optional[bool] = None,
):
self._params.stability = stability if stability is not None else self._params.stability
self._params.similarity_boost = (
similarity_boost if similarity_boost is not None else self._params.similarity_boost
)
self._params.style = style if style is not None else self._params.style
self._params.use_speaker_boost = (
use_speaker_boost if use_speaker_boost is not None else self._params.use_speaker_boost
)
self._set_voice_settings()
if self._websocket:
msg = {"voice_settings": self._voice_settings}
await self._websocket.send(json.dumps(msg))
async def _update_settings(self, settings: Dict[str, Any]):
prev_voice = self._voice_id
await super()._update_settings(settings)
if not prev_voice == self._voice_id:
await self._disconnect()
await self._connect()
logger.debug(f"Switching TTS voice to: [{self._voice_id}]")
async def start(self, frame: StartFrame):
await super().start(frame)
@@ -223,19 +288,19 @@ class ElevenLabsTTSService(WordTTSService):
try:
voice_id = self._voice_id
model = self.model_name
output_format = self._params.output_format
output_format = self._settings["output_format"]
url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}"
if self._params.optimize_streaming_latency:
url += f"&optimize_streaming_latency={self._params.optimize_streaming_latency}"
if self._settings["optimize_streaming_latency"]:
url += f"&optimize_streaming_latency={self._settings["optimize_streaming_latency"]}"
# language can only be used with the 'eleven_turbo_v2_5' model
if self._params.language:
if self._settings["language"]:
if model == "eleven_turbo_v2_5":
url += f"&language_code={self._params.language}"
url += f"&language_code={self._settings["language"]}"
else:
logger.debug(
f"Language code [{self._params.language}] not applied. Language codes can only be used with the 'eleven_turbo_v2_5' model."
f"Language code [{self._settings["language"]}] not applied. Language codes can only be used with the 'eleven_turbo_v2_5' model."
)
self._websocket = await websockets.connect(url)
@@ -286,7 +351,7 @@ class ElevenLabsTTSService(WordTTSService):
self.start_word_timestamps()
audio = base64.b64decode(msg["audio"])
frame = TTSAudioRawFrame(audio, self.sample_rate, 1)
frame = TTSAudioRawFrame(audio, self._settings["sample_rate"], 1)
await self.push_frame(frame)
if msg.get("alignment"):

100
src/pipecat/services/gladia.py
View File

@@ -6,8 +6,9 @@
import base64
import json
from typing import AsyncGenerator, Optional
from loguru import logger
from pydantic.main import BaseModel
from pipecat.frames.frames import (
@@ -19,10 +20,9 @@ from pipecat.frames.frames import (
TranscriptionFrame,
)
from pipecat.services.ai_services import STTService
from pipecat.transcriptions.language import Language
from pipecat.utils.time import time_now_iso8601
from loguru import logger
# See .env.example for Gladia configuration needed
try:
import websockets
@@ -34,10 +34,88 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
def language_to_gladia_language(language: Language) -> str | None:
match language:
case Language.BG:
return "bg"
case Language.CA:
return "ca"
case Language.ZH:
return "zh"
case Language.CS:
return "cs"
case Language.DA:
return "da"
case Language.NL:
return "nl"
case (
Language.EN
| Language.EN_US
| Language.EN_AU
| Language.EN_GB
| Language.EN_NZ
| Language.EN_IN
):
return "en"
case Language.ET:
return "et"
case Language.FI:
return "fi"
case Language.FR | Language.FR_CA:
return "fr"
case Language.DE | Language.DE_CH:
return "de"
case Language.EL:
return "el"
case Language.HI:
return "hi"
case Language.HU:
return "hu"
case Language.ID:
return "id"
case Language.IT:
return "it"
case Language.JA:
return "ja"
case Language.KO:
return "ko"
case Language.LV:
return "lv"
case Language.LT:
return "lt"
case Language.MS:
return "ms"
case Language.NO:
return "no"
case Language.PL:
return "pl"
case Language.PT | Language.PT_BR:
return "pt"
case Language.RO:
return "ro"
case Language.RU:
return "ru"
case Language.SK:
return "sk"
case Language.ES:
return "es"
case Language.SV:
return "sv"
case Language.TH:
return "th"
case Language.TR:
return "tr"
case Language.UK:
return "uk"
case Language.VI:
return "vi"
return None
class GladiaSTTService(STTService):
class InputParams(BaseModel):
sample_rate: Optional[int] = 16000
language: Optional[str] = "english"
language: Optional[Language] = Language.EN
transcription_hint: Optional[str] = None
endpointing: Optional[int] = 200
prosody: Optional[bool] = None
@@ -55,7 +133,13 @@ class GladiaSTTService(STTService):
self._api_key = api_key
self._url = url
self._params = params
self._settings = {
"sample_rate": params.sample_rate,
"language": language_to_gladia_language(params.language) if params.language else "en",
"transcription_hint": params.transcription_hint,
"endpointing": params.endpointing,
"prosody": params.prosody,
}
self._confidence = confidence
async def start(self, frame: StartFrame):
@@ -84,7 +168,11 @@ class GladiaSTTService(STTService):
"encoding": "WAV/PCM",
"model_type": "fast",
"language_behaviour": "manual",
**self._params.model_dump(exclude_none=True),
"sample_rate": self._settings["sample_rate"],
"language": self._settings["language"],
"transcription_hint": self._settings["transcription_hint"],
"endpointing": self._settings["endpointing"],
"prosody": self._settings["prosody"],
}
await self._websocket.send(json.dumps(configuration))

187
src/pipecat/services/google.py
View File

@@ -30,6 +30,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import LLMService, TTSService
from pipecat.transcriptions.language import Language
try:
import google.ai.generativelanguage as glm
@@ -137,9 +138,7 @@ class GoogleLLMService(LLMService):
elif isinstance(frame, VisionImageRawFrame):
context = OpenAILLMContext.from_image_frame(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
if frame.model is not None:
logger.debug(f"Switching LLM model to: [{frame.model}]")
self.set_model_name(frame.model)
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)
@@ -147,13 +146,100 @@ class GoogleLLMService(LLMService):
await self._process_context(context)
def language_to_google_language(language: Language) -> str | None:
match language:
case Language.BG:
return "bg-BG"
case Language.CA:
return "ca-ES"
case Language.ZH:
return "cmn-CN"
case Language.ZH_TW:
return "cmn-TW"
case Language.CS:
return "cs-CZ"
case Language.DA:
return "da-DK"
case Language.NL:
return "nl-NL"
case Language.EN:
return "en-US"
case Language.EN_US:
return "en-US"
case Language.EN_AU:
return "en-AU"
case Language.EN_GB:
return "en-GB"
case Language.EN_IN:
return "en-IN"
case Language.ET:
return "et-EE"
case Language.FI:
return "fi-FI"
case Language.NL_BE:
return "nl-BE"
case Language.FR:
return "fr-FR"
case Language.FR_CA:
return "fr-CA"
case Language.DE:
return "de-DE"
case Language.EL:
return "el-GR"
case Language.HI:
return "hi-IN"
case Language.HU:
return "hu-HU"
case Language.ID:
return "id-ID"
case Language.IT:
return "it-IT"
case Language.JA:
return "ja-JP"
case Language.KO:
return "ko-KR"
case Language.LV:
return "lv-LV"
case Language.LT:
return "lt-LT"
case Language.MS:
return "ms-MY"
case Language.NO:
return "nb-NO"
case Language.PL:
return "pl-PL"
case Language.PT:
return "pt-PT"
case Language.PT_BR:
return "pt-BR"
case Language.RO:
return "ro-RO"
case Language.RU:
return "ru-RU"
case Language.SK:
return "sk-SK"
case Language.ES:
return "es-ES"
case Language.SV:
return "sv-SE"
case Language.TH:
return "th-TH"
case Language.TR:
return "tr-TR"
case Language.UK:
return "uk-UA"
case Language.VI:
return "vi-VN"
return None
class GoogleTTSService(TTSService):
class InputParams(BaseModel):
pitch: Optional[str] = None
rate: Optional[str] = None
volume: Optional[str] = None
emphasis: Optional[Literal["strong", "moderate", "reduced", "none"]] = None
language: Optional[str] = "en-US"
language: Optional[Language] = Language.EN
gender: Optional[Literal["male", "female", "neutral"]] = None
google_style: Optional[Literal["apologetic", "calm", "empathetic", "firm", "lively"]] = None
@@ -169,8 +255,19 @@ class GoogleTTSService(TTSService):
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._voice_id: str = voice_id
self._params = params
self._settings = {
"sample_rate": sample_rate,
"pitch": params.pitch,
"rate": params.rate,
"volume": params.volume,
"emphasis": params.emphasis,
"language": language_to_google_language(params.language)
if params.language
else "en-US",
"gender": params.gender,
"google_style": params.google_style,
}
self.set_voice(voice_id)
self._client: texttospeech_v1.TextToSpeechAsyncClient = self._create_client(
credentials, credentials_path
)
@@ -203,38 +300,38 @@ class GoogleTTSService(TTSService):
# Voice tag
voice_attrs = [f"name='{self._voice_id}'"]
if self._params.language:
voice_attrs.append(f"language='{self._params.language}'")
if self._params.gender:
voice_attrs.append(f"gender='{self._params.gender}'")
if self._settings["language"]:
voice_attrs.append(f"language='{self._settings['language']}'")
if self._settings["gender"]:
voice_attrs.append(f"gender='{self._settings['gender']}'")
ssml += f"<voice {' '.join(voice_attrs)}>"
# Prosody tag
prosody_attrs = []
if self._params.pitch:
prosody_attrs.append(f"pitch='{self._params.pitch}'")
if self._params.rate:
prosody_attrs.append(f"rate='{self._params.rate}'")
if self._params.volume:
prosody_attrs.append(f"volume='{self._params.volume}'")
if self._settings["pitch"]:
prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
if self._settings["rate"]:
prosody_attrs.append(f"rate='{self._settings['rate']}'")
if self._settings["volume"]:
prosody_attrs.append(f"volume='{self._settings['volume']}'")
if prosody_attrs:
ssml += f"<prosody {' '.join(prosody_attrs)}>"
# Emphasis tag
if self._params.emphasis:
ssml += f"<emphasis level='{self._params.emphasis}'>"
if self._settings["emphasis"]:
ssml += f"<emphasis level='{self._settings['emphasis']}'>"
# Google style tag
if self._params.google_style:
ssml += f"<google:style name='{self._params.google_style}'>"
if self._settings["google_style"]:
ssml += f"<google:style name='{self._settings['google_style']}'>"
ssml += text
# Close tags
if self._params.google_style:
if self._settings["google_style"]:
ssml += "</google:style>"
if self._params.emphasis:
if self._settings["emphasis"]:
ssml += "</emphasis>"
if prosody_attrs:
ssml += "</prosody>"
@@ -242,46 +339,6 @@ class GoogleTTSService(TTSService):
return ssml
async def set_voice(self, voice: str) -> None:
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice_id = voice
async def set_language(self, language: str) -> None:
logger.debug(f"Switching TTS language to: [{language}]")
self._params.language = language
async def set_pitch(self, pitch: str) -> None:
logger.debug(f"Switching TTS pitch to: [{pitch}]")
self._params.pitch = pitch
async def set_rate(self, rate: str) -> None:
logger.debug(f"Switching TTS rate to: [{rate}]")
self._params.rate = rate
async def set_volume(self, volume: str) -> None:
logger.debug(f"Switching TTS volume to: [{volume}]")
self._params.volume = volume
async def set_emphasis(
self, emphasis: Literal["strong", "moderate", "reduced", "none"]
) -> None:
logger.debug(f"Switching TTS emphasis to: [{emphasis}]")
self._params.emphasis = emphasis
async def set_gender(self, gender: Literal["male", "female", "neutral"]) -> None:
logger.debug(f"Switch TTS gender to [{gender}]")
self._params.gender = gender
async def google_style(
self, google_style: Literal["apologetic", "calm", "empathetic", "firm", "lively"]
) -> None:
logger.debug(f"Switching TTS google style to: [{google_style}]")
self._params.google_style = google_style
async def set_params(self, params: InputParams) -> None:
logger.debug(f"Switching TTS params to: [{params}]")
self._params = params
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
@@ -291,11 +348,11 @@ class GoogleTTSService(TTSService):
ssml = self._construct_ssml(text)
synthesis_input = texttospeech_v1.SynthesisInput(ssml=ssml)
voice = texttospeech_v1.VoiceSelectionParams(
language_code=self._params.language, name=self._voice_id
language_code=self._settings["language"], name=self._voice_id
)
audio_config = texttospeech_v1.AudioConfig(
audio_encoding=texttospeech_v1.AudioEncoding.LINEAR16,
sample_rate_hertz=self.sample_rate,
sample_rate_hertz=self._settings["sample_rate"],
)
request = texttospeech_v1.SynthesizeSpeechRequest(
@@ -318,7 +375,7 @@ class GoogleTTSService(TTSService):
if not chunk:
break
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
yield frame
await asyncio.sleep(0) # Allow other tasks to run

61
src/pipecat/services/lmnt.py
View File

@@ -5,10 +5,10 @@
#
import asyncio
from typing import AsyncGenerator
from pipecat.processors.frame_processor import FrameDirection
from loguru import logger
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
@@ -20,9 +20,9 @@ from pipecat.frames.frames import (
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import TTSService
from loguru import logger
from pipecat.transcriptions.language import Language
# See .env.example for LMNT configuration needed
try:
@@ -35,6 +35,32 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
def language_to_lmnt_language(language: Language) -> str | None:
match language:
case Language.DE:
return "de"
case (
Language.EN
| Language.EN_US
| Language.EN_AU
| Language.EN_GB
| Language.EN_NZ
| Language.EN_IN
):
return "en"
case Language.ES:
return "es"
case Language.FR | Language.FR_CA:
return "fr"
case Language.PT | Language.PT_BR:
return "pt"
case Language.ZH | Language.ZH_TW:
return "zh"
case Language.KO:
return "ko"
return None
class LmntTTSService(TTSService):
def __init__(
self,
@@ -42,7 +68,7 @@ class LmntTTSService(TTSService):
api_key: str,
voice_id: str,
sample_rate: int = 24000,
language: str = "en",
language: Language = Language.EN,
**kwargs,
):
# Let TTSService produce TTSStoppedFrames after a short delay of
@@ -50,13 +76,16 @@ class LmntTTSService(TTSService):
super().__init__(push_stop_frames=True, sample_rate=sample_rate, **kwargs)
self._api_key = api_key
self._voice_id = voice_id
self._output_format = {
"container": "raw",
"encoding": "pcm_s16le",
"sample_rate": sample_rate,
self._settings = {
"output_format": {
"container": "raw",
"encoding": "pcm_s16le",
"sample_rate": sample_rate,
},
"language": language_to_lmnt_language(language) if language else "en",
}
self._language = language
self.set_voice(voice_id)
self._speech = None
self._connection = None
@@ -68,10 +97,6 @@ class LmntTTSService(TTSService):
def can_generate_metrics(self) -> bool:
return True
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice_id = voice
async def start(self, frame: StartFrame):
await super().start(frame)
await self._connect()
@@ -93,7 +118,9 @@ class LmntTTSService(TTSService):
try:
self._speech = Speech()
self._connection = await self._speech.synthesize_streaming(
self._voice_id, format="raw", sample_rate=self._output_format["sample_rate"]
self._voice_id,
format="raw",
sample_rate=self._settings["output_format"]["sample_rate"],
)
self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
except Exception as e:
@@ -130,7 +157,7 @@ class LmntTTSService(TTSService):
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(
audio=msg["audio"],
sample_rate=self._output_format["sample_rate"],
sample_rate=self._settings["output_format"]["sample_rate"],
num_channels=1,
)
await self.push_frame(frame)

85
src/pipecat/services/openai.py
View File

@@ -111,14 +111,16 @@ class BaseOpenAILLMService(LLMService):
**kwargs,
):
super().__init__(**kwargs)
self._settings = {
"frequency_penalty": params.frequency_penalty,
"presence_penalty": params.presence_penalty,
"seed": params.seed,
"temperature": params.temperature,
"top_p": params.top_p,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
self.set_model_name(model)
self._client = self.create_client(api_key=api_key, base_url=base_url, **kwargs)
self._frequency_penalty = params.frequency_penalty
self._presence_penalty = params.presence_penalty
self._seed = params.seed
self._temperature = params.temperature
self._top_p = params.top_p
self._extra = params.extra if isinstance(params.extra, dict) else {}
def create_client(self, api_key=None, base_url=None, **kwargs):
return AsyncOpenAI(
@@ -134,30 +136,6 @@ class BaseOpenAILLMService(LLMService):
def can_generate_metrics(self) -> bool:
return True
async def set_frequency_penalty(self, frequency_penalty: float):
logger.debug(f"Switching LLM frequency_penalty to: [{frequency_penalty}]")
self._frequency_penalty = frequency_penalty
async def set_presence_penalty(self, presence_penalty: float):
logger.debug(f"Switching LLM presence_penalty to: [{presence_penalty}]")
self._presence_penalty = presence_penalty
async def set_seed(self, seed: int):
logger.debug(f"Switching LLM seed to: [{seed}]")
self._seed = seed
async def set_temperature(self, temperature: float):
logger.debug(f"Switching LLM temperature to: [{temperature}]")
self._temperature = temperature
async def set_top_p(self, top_p: float):
logger.debug(f"Switching LLM top_p to: [{top_p}]")
self._top_p = top_p
async def set_extra(self, extra: Dict[str, Any]):
logger.debug(f"Switching LLM extra to: [{extra}]")
self._extra = extra
async def get_chat_completions(
self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam]
) -> AsyncStream[ChatCompletionChunk]:
@@ -168,14 +146,14 @@ class BaseOpenAILLMService(LLMService):
"tools": context.tools,
"tool_choice": context.tool_choice,
"stream_options": {"include_usage": True},
"frequency_penalty": self._frequency_penalty,
"presence_penalty": self._presence_penalty,
"seed": self._seed,
"temperature": self._temperature,
"top_p": self._top_p,
"frequency_penalty": self._settings["frequency_penalty"],
"presence_penalty": self._settings["presence_penalty"],
"seed": self._settings["seed"],
"temperature": self._settings["temperature"],
"top_p": self._settings["top_p"],
}
params.update(self._extra)
params.update(self._settings["extra"])
chunks = await self._client.chat.completions.create(**params)
return chunks
@@ -295,23 +273,6 @@ class BaseOpenAILLMService(LLMService):
f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function."
)
async def _update_settings(self, frame: LLMUpdateSettingsFrame):
if frame.model is not None:
logger.debug(f"Switching LLM model to: [{frame.model}]")
self.set_model_name(frame.model)
if frame.frequency_penalty is not None:
await self.set_frequency_penalty(frame.frequency_penalty)
if frame.presence_penalty is not None:
await self.set_presence_penalty(frame.presence_penalty)
if frame.seed is not None:
await self.set_seed(frame.seed)
if frame.temperature is not None:
await self.set_temperature(frame.temperature)
if frame.top_p is not None:
await self.set_top_p(frame.top_p)
if frame.extra:
await self.set_extra(frame.extra)
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -323,7 +284,7 @@ class BaseOpenAILLMService(LLMService):
elif isinstance(frame, VisionImageRawFrame):
context = OpenAILLMContext.from_image_frame(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame)
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)
@@ -427,22 +388,20 @@ class OpenAITTSService(TTSService):
):
super().__init__(sample_rate=sample_rate, **kwargs)
self._voice: ValidVoice = VALID_VOICES.get(voice, "alloy")
self._settings = {
"sample_rate": sample_rate,
}
self.set_model_name(model)
self._sample_rate = sample_rate
self.set_voice(voice)
self._client = AsyncOpenAI(api_key=api_key)
def can_generate_metrics(self) -> bool:
return True
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice = VALID_VOICES.get(voice, self._voice)
async def set_model(self, model: str):
logger.debug(f"Switching TTS model to: [{model}]")
self._model = model
self.set_model_name(model)
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
@@ -452,7 +411,7 @@ class OpenAITTSService(TTSService):
async with self._client.audio.speech.with_streaming_response.create(
input=text,
model=self.model_name,
voice=self._voice,
voice=VALID_VOICES[self._voice_id],
response_format="pcm",
) as r:
if r.status_code != 200:
@@ -471,7 +430,7 @@ class OpenAITTSService(TTSService):
async for chunk in r.iter_bytes(8192):
if len(chunk) > 0:
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
yield frame
await self.push_frame(TTSStoppedFrame())
except BadRequestError as e:

34
src/pipecat/services/playht.py
View File

@@ -6,17 +6,21 @@
import io
import struct
from typing import AsyncGenerator
from pipecat.frames.frames import Frame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame
from pipecat.services.ai_services import TTSService
from loguru import logger
from pipecat.frames.frames import (
Frame,
TTSAudioRawFrame,
TTSStartedFrame,
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
try:
from pyht.client import TTSOptions
from pyht.async_client import AsyncClient
from pyht.client import TTSOptions
from pyht.protos.api_pb2 import Format
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
@@ -39,17 +43,23 @@ class PlayHTTTSService(TTSService):
user_id=self._user_id,
api_key=self._speech_key,
)
self._settings = {
"sample_rate": sample_rate,
"quality": "higher",
"format": Format.FORMAT_WAV,
"voice_engine": "PlayHT2.0-turbo",
}
self.set_voice(voice_url)
self._options = TTSOptions(
voice=voice_url, sample_rate=sample_rate, quality="higher", format=Format.FORMAT_WAV
voice=self._voice_id,
sample_rate=self._settings["sample_rate"],
quality=self._settings["quality"],
format=self._settings["format"],
)
def can_generate_metrics(self) -> bool:
return True
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._options.voice = voice
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
@@ -60,7 +70,7 @@ class PlayHTTTSService(TTSService):
await self.start_ttfb_metrics()
playht_gen = self._client.tts(
text, voice_engine="PlayHT2.0-turbo", options=self._options
text, voice_engine=self._settings["voice_engine"], options=self._options
)
await self.start_tts_usage_metrics(text)
@@ -83,7 +93,7 @@ class PlayHTTTSService(TTSService):
else:
if len(chunk):
await self.stop_ttfb_metrics()
frame = TTSAudioRawFrame(chunk, 16000, 1)
frame = TTSAudioRawFrame(chunk, self._settings["sample_rate"], 1)
yield frame
await self.push_frame(TTSStoppedFrame())
except Exception as e:

68
src/pipecat/services/together.py
View File

@@ -5,16 +5,13 @@
#
from typing import Any, Dict, Optional
import httpx
from loguru import logger
from pydantic import BaseModel, Field
from pipecat.frames.frames import (
LLMUpdateSettingsFrame,
)
from pipecat.services.openai import OpenAILLMService
try:
# Together.ai is recommending OpenAI-compatible function calling, so we've switched over
# to using the OpenAI client library here rather than the Together Python client library.
@@ -53,13 +50,15 @@ class TogetherLLMService(OpenAILLMService):
):
super().__init__(api_key=api_key, base_url=base_url, model=model, params=params, **kwargs)
self.set_model_name(model)
self._max_tokens = params.max_tokens
self._frequency_penalty = params.frequency_penalty
self._presence_penalty = params.presence_penalty
self._temperature = params.temperature
self._top_k = params.top_k
self._top_p = params.top_p
self._extra = params.extra if isinstance(params.extra, dict) else {}
self._settings = {
"max_tokens": params.max_tokens,
"frequency_penalty": params.frequency_penalty,
"presence_penalty": params.presence_penalty,
"seed": params.seed,
"temperature": params.temperature,
"top_p": params.top_p,
"extra": params.extra if isinstance(params.extra, dict) else {},
}
def can_generate_metrics(self) -> bool:
return True
@@ -75,50 +74,3 @@ class TogetherLLMService(OpenAILLMService):
)
),
)
async def set_frequency_penalty(self, frequency_penalty: float):
logger.debug(f"Switching LLM frequency_penalty to: [{frequency_penalty}]")
self._frequency_penalty = frequency_penalty
async def set_max_tokens(self, max_tokens: int):
logger.debug(f"Switching LLM max_tokens to: [{max_tokens}]")
self._max_tokens = max_tokens
async def set_presence_penalty(self, presence_penalty: float):
logger.debug(f"Switching LLM presence_penalty to: [{presence_penalty}]")
self._presence_penalty = presence_penalty
async def set_temperature(self, temperature: float):
logger.debug(f"Switching LLM temperature to: [{temperature}]")
self._temperature = temperature
async def set_top_k(self, top_k: float):
logger.debug(f"Switching LLM top_k to: [{top_k}]")
self._top_k = top_k
async def set_top_p(self, top_p: float):
logger.debug(f"Switching LLM top_p to: [{top_p}]")
self._top_p = top_p
async def set_extra(self, extra: Dict[str, Any]):
logger.debug(f"Switching LLM extra to: [{extra}]")
self._extra = extra
async def _update_settings(self, frame: LLMUpdateSettingsFrame):
if frame.model is not None:
logger.debug(f"Switching LLM model to: [{frame.model}]")
self.set_model_name(frame.model)
if frame.frequency_penalty is not None:
await self.set_frequency_penalty(frame.frequency_penalty)
if frame.max_tokens is not None:
await self.set_max_tokens(frame.max_tokens)
if frame.presence_penalty is not None:
await self.set_presence_penalty(frame.presence_penalty)
if frame.temperature is not None:
await self.set_temperature(frame.temperature)
if frame.top_k is not None:
await self.set_top_k(frame.top_k)
if frame.top_p is not None:
await self.set_top_p(frame.top_p)
if frame.extra:
await self.set_extra(frame.extra)

75
src/pipecat/services/xtts.py
View File

@@ -4,10 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import aiohttp
from typing import Any, AsyncGenerator, Dict
import aiohttp
import numpy as np
from loguru import logger
from pipecat.frames.frames import (
ErrorFrame,
Frame,
@@ -17,10 +19,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
)
from pipecat.services.ai_services import TTSService
import numpy as np
from loguru import logger
from pipecat.transcriptions.language import Language
try:
import resampy
@@ -38,21 +37,67 @@ except ModuleNotFoundError as e:
# https://github.com/coqui-ai/xtts-streaming-server
def language_to_xtts_language(language: Language) -> str | None:
match language:
case Language.CS:
return "cs"
case Language.DE:
return "de"
case (
Language.EN
| Language.EN_US
| Language.EN_AU
| Language.EN_GB
| Language.EN_NZ
| Language.EN_IN
):
return "en"
case Language.ES:
return "es"
case Language.FR:
return "fr"
case Language.HI:
return "hi"
case Language.HU:
return "hu"
case Language.IT:
return "it"
case Language.JA:
return "ja"
case Language.KO:
return "ko"
case Language.NL:
return "nl"
case Language.PL:
return "pl"
case Language.PT | Language.PT_BR:
return "pt"
case Language.RU:
return "ru"
case Language.TR:
return "tr"
case Language.ZH:
return "zh-cn"
return None
class XTTSService(TTSService):
def __init__(
self,
*,
voice_id: str,
language: str,
language: Language,
base_url: str,
aiohttp_session: aiohttp.ClientSession,
**kwargs,
):
super().__init__(**kwargs)
self._voice_id = voice_id
self._language = language
self._base_url = base_url
self._settings = {
"language": language_to_xtts_language(language) if language else "en",
"base_url": base_url,
}
self.set_voice(voice_id)
self._studio_speakers: Dict[str, Any] | None = None
self._aiohttp_session = aiohttp_session
@@ -61,7 +106,7 @@ class XTTSService(TTSService):
async def start(self, frame: StartFrame):
await super().start(frame)
async with self._aiohttp_session.get(self._base_url + "/studio_speakers") as r:
async with self._aiohttp_session.get(self._settings["base_url"] + "/studio_speakers") as r:
if r.status != 200:
text = await r.text()
logger.error(
@@ -75,10 +120,6 @@ class XTTSService(TTSService):
return
self._studio_speakers = await r.json()
async def set_voice(self, voice: str):
logger.debug(f"Switching TTS voice to: [{voice}]")
self._voice_id = voice
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
logger.debug(f"Generating TTS: [{text}]")
@@ -88,11 +129,11 @@ class XTTSService(TTSService):
embeddings = self._studio_speakers[self._voice_id]
url = self._base_url + "/tts_stream"
url = self._settings["base_url"] + "/tts_stream"
payload = {
"text": text.replace(".", "").replace("*", ""),
"language": self._language,
"language": self._settings["language"],
"speaker_embedding": embeddings["speaker_embedding"],
"gpt_cond_latent": embeddings["gpt_cond_latent"],
"add_wav_header": False,