wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Consolidate service UpdateSettingsFrame into a single ServiceUpdateSettingsFrame

Browse Source
This commit is contained in:
Mark Backman
2024-10-01 10:35:59 -04:00
parent a397b859fe
commit 88cca7bf68
7 changed files with 106 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@@ -86,8 +86,8 @@ async def on_connected(processor):
### Changed
- Updated individual update settings frame classes into a single UpdateSettingsFrame
class for STT, LLM, and TTS.
- Updated individual update settings frame classes into a single
ServiceUpdateSettingsFrame class.
- We now distinguish between input and output audio and image frames. We
introduce `InputAudioRawFrame`, `OutputAudioRawFrame`, `InputImageRawFrame`

46
src/pipecat/frames/frames.py
View File

@@ -5,7 +5,7 @@
#
from dataclasses import dataclass, field
from typing import Any, List, Optional, Tuple, Union
from typing import Any, Dict, List, Optional, Tuple
from pipecat.clocks.base_clock import BaseClock
from pipecat.metrics.metrics import MetricsData
@@ -527,47 +527,11 @@ class UserImageRequestFrame(ControlFrame):
@dataclass
class LLMUpdateSettingsFrame(ControlFrame):
"""A control frame containing a request to update LLM settings."""
class ServiceUpdateSettingsFrame(ControlFrame):
"""A control frame containing a request to update service settings."""
model: Optional[str] = None
temperature: Optional[float] = None
top_k: Optional[int] = None
top_p: Optional[float] = None
frequency_penalty: Optional[float] = None
presence_penalty: Optional[float] = None
max_tokens: Optional[int] = None
seed: Optional[int] = None
extra: dict = field(default_factory=dict)
@dataclass
class TTSUpdateSettingsFrame(ControlFrame):
"""A control frame containing a request to update TTS settings."""
model: Optional[str] = None
voice: Optional[str] = None
language: Optional[Language] = None
speed: Optional[Union[str, float]] = None
emotion: Optional[List[str]] = None
engine: Optional[str] = None
pitch: Optional[str] = None
rate: Optional[str] = None
volume: Optional[str] = None
emphasis: Optional[str] = None
style: Optional[str] = None
style_degree: Optional[str] = None
role: Optional[str] = None
gender: Optional[str] = None
google_style: Optional[str] = None
@dataclass
class STTUpdateSettingsFrame(ControlFrame):
"""A control frame containing a request to update STT settings."""
model: Optional[str] = None
language: Optional[Language] = None
service_type: str
settings: Dict[str, Any]
@dataclass

87
src/pipecat/services/ai_services.py
View File

@@ -8,7 +8,7 @@ import asyncio
import io
import wave
from abc import abstractmethod
from typing import AsyncGenerator, List, Optional, Tuple, Union
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union
from loguru import logger
@@ -19,15 +19,14 @@ from pipecat.frames.frames import (
ErrorFrame,
Frame,
LLMFullResponseEndFrame,
ServiceUpdateSettingsFrame,
StartFrame,
StartInterruptionFrame,
STTUpdateSettingsFrame,
TextFrame,
TTSAudioRawFrame,
TTSSpeakFrame,
TTSStartedFrame,
TTSStoppedFrame,
TTSUpdateSettingsFrame,
UserImageRequestFrame,
VisionImageRawFrame,
)
@@ -169,6 +168,7 @@ class TTSService(AIService):
self._push_stop_frames: bool = push_stop_frames
self._stop_frame_timeout_s: float = stop_frame_timeout_s
self._sample_rate: int = sample_rate
self._settings: Dict[str, Any] = {}
self._stop_frame_task: Optional[asyncio.Task] = None
self._stop_frame_queue: asyncio.Queue = asyncio.Queue()
@@ -231,10 +231,6 @@ class TTSService(AIService):
async def set_role(self, role: str):
pass
@abstractmethod
async def flush_audio(self):
pass
@abstractmethod
async def set_gender(self, gender: str):
pass
@@ -243,6 +239,10 @@ class TTSService(AIService):
async def set_google_style(self, google_style: str):
pass
@abstractmethod
async def flush_audio(self):
pass
# Converts the text to audio.
@abstractmethod
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
@@ -267,6 +267,22 @@ class TTSService(AIService):
await self._stop_frame_task
self._stop_frame_task = None
async def _update_settings(self, settings: Dict[str, Any]):
for key, value in settings.items():
setter = getattr(self, f"set_{key}", None)
if setter and callable(setter):
try:
if key == "language":
await setter(Language(value))
else:
await setter(value)
except Exception as e:
logger.warning(f"Error setting {key}: {e}")
else:
logger.warning(f"Unknown setting for TTS service: {key}")
self._settings.update(settings)
async def say(self, text: str):
aggregate_sentences = self._aggregate_sentences
self._aggregate_sentences = False
@@ -293,8 +309,8 @@ class TTSService(AIService):
elif isinstance(frame, TTSSpeakFrame):
await self._push_tts_frames(frame.text)
await self.flush_audio()
elif isinstance(frame, TTSUpdateSettingsFrame):
await self._update_tts_settings(frame)
elif isinstance(frame, ServiceUpdateSettingsFrame) and frame.service_type == "tts":
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)
@@ -341,34 +357,6 @@ class TTSService(AIService):
# interrupted, the text is not added to the assistant context.
await self.push_frame(TextFrame(text))
async def _update_tts_settings(self, frame: TTSUpdateSettingsFrame):
if frame.model is not None:
await self.set_model(frame.model)
if frame.voice is not None:
await self.set_voice(frame.voice)
if frame.language is not None:
await self.set_language(frame.language)
if frame.speed is not None:
await self.set_speed(frame.speed)
if frame.emotion is not None:
await self.set_emotion(frame.emotion)
if frame.engine is not None:
await self.set_engine(frame.engine)
if frame.pitch is not None:
await self.set_pitch(frame.pitch)
if frame.rate is not None:
await self.set_rate(frame.rate)
if frame.volume is not None:
await self.set_volume(frame.volume)
if frame.emphasis is not None:
await self.set_emphasis(frame.emphasis)
if frame.style is not None:
await self.set_style(frame.style)
if frame.style_degree is not None:
await self.set_style_degree(frame.style_degree)
if frame.role is not None:
await self.set_role(frame.role)
async def _stop_frame_handler(self):
try:
has_started = False
@@ -454,6 +442,7 @@ class STTService(AIService):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._settings: Dict[str, Any] = {}
@abstractmethod
async def set_model(self, model: str):
@@ -468,11 +457,21 @@ class STTService(AIService):
"""Returns transcript as a string"""
pass
async def _update_stt_settings(self, frame: STTUpdateSettingsFrame):
if frame.model is not None:
await self.set_model(frame.model)
if frame.language is not None:
await self.set_language(frame.language)
async def _update_settings(self, settings: Dict[str, Any]):
for key, value in settings.items():
setter = getattr(self, f"set_{key}", None)
if setter and callable(setter):
try:
if key == "language":
await setter(Language(value))
else:
await setter(value)
except Exception as e:
logger.warning(f"Error setting {key}: {e}")
else:
logger.warning(f"Unknown setting for STT service: {key}")
self._settings.update(settings)
async def process_audio_frame(self, frame: AudioRawFrame):
await self.process_generator(self.run_stt(frame.audio))
@@ -485,8 +484,8 @@ class STTService(AIService):
# In this service we accumulate audio internally and at the end we
# push a TextFrame. We don't really want to push audio frames down.
await self.process_audio_frame(frame)
elif isinstance(frame, STTUpdateSettingsFrame):
await self._update_stt_settings(frame)
elif isinstance(frame, ServiceUpdateSettingsFrame) and frame.service_type == "stt":
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)

30
src/pipecat/services/anthropic.py
View File

@@ -25,7 +25,7 @@ from pipecat.frames.frames import (
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMUpdateSettingsFrame,
ServiceUpdateSettingsFrame,
StartInterruptionFrame,
TextFrame,
UserImageRawFrame,
@@ -284,20 +284,16 @@ class AnthropicLLMService(LLMService):
cache_read_input_tokens=cache_read_input_tokens,
)
async def _update_settings(self, frame: LLMUpdateSettingsFrame):
if frame.model is not None:
logger.debug(f"Switching LLM model to: [{frame.model}]")
self.set_model_name(frame.model)
if frame.max_tokens is not None:
await self.set_max_tokens(frame.max_tokens)
if frame.temperature is not None:
await self.set_temperature(frame.temperature)
if frame.top_k is not None:
await self.set_top_k(frame.top_k)
if frame.top_p is not None:
await self.set_top_p(frame.top_p)
if frame.extra:
await self.set_extra(frame.extra)
async def _update_settings(self, settings: Dict[str, Any]):
for key, value in settings.items():
setter = getattr(self, f"set_{key}", None)
if setter and callable(setter):
try:
await setter(value)
except Exception as e:
logger.warning(f"Error setting {key}: {e}")
else:
logger.warning(f"Unknown setting for Anthropic LLM service: {key}")
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -313,8 +309,8 @@ class AnthropicLLMService(LLMService):
# UserImageRawFrames coming through the pipeline and add them
# to the context.
context = AnthropicLLMContext.from_image_frame(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame)
elif isinstance(frame, ServiceUpdateSettingsFrame) and frame.service_type == "llm":
await self._update_settings(frame.settings)
elif isinstance(frame, LLMEnablePromptCachingFrame):
logger.debug(f"Setting enable prompt caching to: [{frame.enable}]")
self._enable_prompt_caching_beta = frame.enable

25
src/pipecat/services/google.py
View File

@@ -6,7 +6,7 @@
import asyncio
import json
from typing import AsyncGenerator, List, Literal, Optional
from typing import Any, AsyncGenerator, Dict, List, Literal, Optional
from loguru import logger
from pydantic import BaseModel
@@ -17,7 +17,7 @@ from pipecat.frames.frames import (
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMUpdateSettingsFrame,
ServiceUpdateSettingsFrame,
TextFrame,
TTSAudioRawFrame,
TTSStartedFrame,
@@ -64,6 +64,21 @@ class GoogleLLMService(LLMService):
self.set_model_name(model)
self._client = gai.GenerativeModel(model)
async def set_model(self, model: str):
logger.debug(f"Switching LLM model to: [{model}]")
self._create_client(model)
async def _update_settings(self, settings: Dict[str, Any]):
for key, value in settings.items():
setter = getattr(self, f"set_{key}", None)
if setter and callable(setter):
try:
await setter(value)
except Exception as e:
logger.warning(f"Error setting {key}: {e}")
else:
logger.warning(f"Unknown setting for Google LLM service: {key}")
def _get_messages_from_openai_context(self, context: OpenAILLMContext) -> List[glm.Content]:
openai_messages = context.get_messages()
google_messages = []
@@ -136,10 +151,8 @@ class GoogleLLMService(LLMService):
context = OpenAILLMContext.from_messages(frame.messages)
elif isinstance(frame, VisionImageRawFrame):
context = OpenAILLMContext.from_image_frame(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
if frame.model is not None:
logger.debug(f"Switching LLM model to: [{frame.model}]")
self.set_model_name(frame.model)
elif isinstance(frame, ServiceUpdateSettingsFrame) and frame.service_type == "llm":
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)

32
src/pipecat/services/openai.py
View File

@@ -24,7 +24,7 @@ from pipecat.frames.frames import (
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesFrame,
LLMUpdateSettingsFrame,
ServiceUpdateSettingsFrame,
StartInterruptionFrame,
TextFrame,
TTSAudioRawFrame,
@@ -295,22 +295,16 @@ class BaseOpenAILLMService(LLMService):
f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function."
)
async def _update_settings(self, frame: LLMUpdateSettingsFrame):
if frame.model is not None:
logger.debug(f"Switching LLM model to: [{frame.model}]")
self.set_model_name(frame.model)
if frame.frequency_penalty is not None:
await self.set_frequency_penalty(frame.frequency_penalty)
if frame.presence_penalty is not None:
await self.set_presence_penalty(frame.presence_penalty)
if frame.seed is not None:
await self.set_seed(frame.seed)
if frame.temperature is not None:
await self.set_temperature(frame.temperature)
if frame.top_p is not None:
await self.set_top_p(frame.top_p)
if frame.extra:
await self.set_extra(frame.extra)
async def _update_settings(self, settings: Dict[str, Any]):
for key, value in settings.items():
setter = getattr(self, f"set_{key}", None)
if setter and callable(setter):
try:
await setter(value)
except Exception as e:
logger.warning(f"Error setting {key}: {e}")
else:
logger.warning(f"Unknown setting for OpenAI LLM service: {key}")
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -322,8 +316,8 @@ class BaseOpenAILLMService(LLMService):
context = OpenAILLMContext.from_messages(frame.messages)
elif isinstance(frame, VisionImageRawFrame):
context = OpenAILLMContext.from_image_frame(frame)
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame)
elif isinstance(frame, ServiceUpdateSettingsFrame) and frame.service_type == "llm":
await self._update_settings(frame.settings)
else:
await self.push_frame(frame, direction)

33
src/pipecat/services/together.py
View File

@@ -5,16 +5,13 @@
#
from typing import Any, Dict, Optional
import httpx
from loguru import logger
from pydantic import BaseModel, Field
from pipecat.frames.frames import (
LLMUpdateSettingsFrame,
)
from pipecat.services.openai import OpenAILLMService
try:
# Together.ai is recommending OpenAI-compatible function calling, so we've switched over
# to using the OpenAI client library here rather than the Together Python client library.
@@ -104,21 +101,13 @@ class TogetherLLMService(OpenAILLMService):
logger.debug(f"Switching LLM extra to: [{extra}]")
self._extra = extra
async def _update_settings(self, frame: LLMUpdateSettingsFrame):
if frame.model is not None:
logger.debug(f"Switching LLM model to: [{frame.model}]")
self.set_model_name(frame.model)
if frame.frequency_penalty is not None:
await self.set_frequency_penalty(frame.frequency_penalty)
if frame.max_tokens is not None:
await self.set_max_tokens(frame.max_tokens)
if frame.presence_penalty is not None:
await self.set_presence_penalty(frame.presence_penalty)
if frame.temperature is not None:
await self.set_temperature(frame.temperature)
if frame.top_k is not None:
await self.set_top_k(frame.top_k)
if frame.top_p is not None:
await self.set_top_p(frame.top_p)
if frame.extra:
await self.set_extra(frame.extra)
async def _update_settings(self, settings: Dict[str, Any]):
for key, value in settings.items():
setter = getattr(self, f"set_{key}", None)
if setter and callable(setter):
try:
await setter(value)
except Exception as e:
logger.warning(f"Error setting {key}: {e}")
else:
logger.warning(f"Unknown setting for Together LLM service: {key}")