diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 90abfeda6..320d25e13 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -186,6 +186,13 @@ class LLMSetToolsFrame(DataFrame): tools: List[dict] +@dataclass +class LLMEnablePromptCachingFrame(DataFrame): + """A frame to enable/disable prompt caching in certain LLMs. + """ + enable: bool + + @dataclass class TTSSpeakFrame(DataFrame): """A frame that contains a text that should be spoken by the TTS in the diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 088db7861..bfe2e86f1 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -16,6 +16,7 @@ import re from pipecat.frames.frames import ( Frame, + LLMEnablePromptCachingFrame, LLMModelUpdateFrame, TextFrame, VisionImageRawFrame, @@ -62,10 +63,10 @@ class AnthropicContextAggregatorPair: _user: 'AnthropicUserContextAggregator' _assistant: 'AnthropicAssistantContextAggregator' - def user(self) -> str: + def user(self) -> 'AnthropicUserContextAggregator': return self._user - def assistant(self) -> str: + def assistant(self) -> 'AnthropicAssistantContextAggregator': return self._assistant @@ -227,6 +228,9 @@ class AnthropicLLMService(LLMService): elif isinstance(frame, LLMModelUpdateFrame): logger.debug(f"Switching LLM model to: [{frame.model}]") self._model = frame.model + elif isinstance(frame, LLMEnablePromptCachingFrame): + logger.debug(f"Setting enable prompt caching to: [{frame.enable}]") + self._enable_prompt_caching_beta = frame.enable else: await self.push_frame(frame, direction) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 12708647c..52da196d2 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -229,10 +229,10 @@ class OpenAIContextAggregatorPair: _user: 'OpenAIUserContextAggregator' _assistant: 'OpenAIAssistantContextAggregator' - def user(self) -> str: + def user(self) -> 'OpenAIUserContextAggregator': return self._user - def assistant(self) -> str: + def assistant(self) -> 'OpenAIAssistantContextAggregator': return self._assistant diff --git a/src/pipecat/services/together.py b/src/pipecat/services/together.py index bf34dd099..685609858 100644 --- a/src/pipecat/services/together.py +++ b/src/pipecat/services/together.py @@ -49,10 +49,10 @@ class TogetherContextAggregatorPair: _user: 'TogetherUserContextAggregator' _assistant: 'TogetherAssistantContextAggregator' - def user(self) -> str: + def user(self) -> 'TogetherUserContextAggregator': return self._user - def assistant(self) -> str: + def assistant(self) -> 'TogetherAssistantContextAggregator': return self._assistant @@ -75,7 +75,7 @@ class TogetherLLMService(LLMService): def can_generate_metrics(self) -> bool: return True - @ staticmethod + @staticmethod def create_context_aggregator(context: OpenAILLMContext) -> TogetherContextAggregatorPair: user = TogetherUserContextAggregator(context) assistant = TogetherAssistantContextAggregator(user) @@ -191,14 +191,14 @@ class TogetherLLMContext(OpenAILLMContext): ): super().__init__(messages=messages) - @ classmethod + @classmethod def from_openai_context(cls, openai_context: OpenAILLMContext): self = cls( messages=openai_context.messages, ) return self - @ classmethod + @classmethod def from_messages(cls, messages: List[dict]) -> "TogetherLLMContext": return cls(messages=messages)