From 9de2bd61a925d55c059954e542e707ef5802d42f Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 19 Aug 2025 13:07:56 -0400 Subject: [PATCH] Add `supports_universal_context` for `OpenAILLMService` subclasses so that we can gradually roll out support for universal `LLMContext` in a controlled manner. Also update `get_chat_completions()` implementations with the new argument type. --- src/pipecat/services/azure/llm.py | 9 +++++++ src/pipecat/services/cerebras/llm.py | 30 ++++++++++++++++------ src/pipecat/services/deepseek/llm.py | 30 ++++++++++++++++------ src/pipecat/services/fireworks/llm.py | 30 ++++++++++++++++------ src/pipecat/services/google/llm_openai.py | 11 +++++++- src/pipecat/services/google/llm_vertex.py | 9 +++++++ src/pipecat/services/grok/llm.py | 9 +++++++ src/pipecat/services/groq/llm.py | 9 +++++++ src/pipecat/services/nim/llm.py | 9 +++++++ src/pipecat/services/ollama/llm.py | 9 +++++++ src/pipecat/services/openai/base_llm.py | 30 +++++++++++++++++----- src/pipecat/services/openai/llm.py | 9 +++++++ src/pipecat/services/openpipe/llm.py | 23 +++++++++++------ src/pipecat/services/openrouter/llm.py | 9 +++++++ src/pipecat/services/perplexity/llm.py | 27 +++++++++++++++----- src/pipecat/services/qwen/llm.py | 9 +++++++ src/pipecat/services/sambanova/llm.py | 31 +++++++++++++++-------- src/pipecat/services/together/llm.py | 9 +++++++ 18 files changed, 245 insertions(+), 57 deletions(-) diff --git a/src/pipecat/services/azure/llm.py b/src/pipecat/services/azure/llm.py index a4b93f2a4..47a6ef280 100644 --- a/src/pipecat/services/azure/llm.py +++ b/src/pipecat/services/azure/llm.py @@ -60,3 +60,12 @@ class AzureLLMService(OpenAILLMService): azure_endpoint=self._endpoint, api_version=self._api_version, ) + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as Azure service does yet not support universal LLMContext. + """ + return False diff --git a/src/pipecat/services/cerebras/llm.py b/src/pipecat/services/cerebras/llm.py index c3577af82..9bdc5b963 100644 --- a/src/pipecat/services/cerebras/llm.py +++ b/src/pipecat/services/cerebras/llm.py @@ -9,9 +9,8 @@ from typing import List from loguru import logger -from openai.types.chat import ChatCompletionMessageParam -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams from pipecat.services.openai.llm import OpenAILLMService @@ -54,25 +53,40 @@ class CerebrasLLMService(OpenAILLMService): logger.debug(f"Creating Cerebras client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) - def build_chat_completion_params( - self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] - ) -> dict: + def build_chat_completion_params(self, params_from_context: OpenAILLMInvocationParams) -> dict: """Build parameters for Cerebras chat completion request. Cerebras supports a subset of OpenAI parameters, focusing on core completion settings without advanced features like frequency/presence penalties. + + Args: + params_from_context: Parameters, derived from the LLM context, to + use for the chat completion. Contains messages, tools, and tool + choice. + + Returns: + Dictionary of parameters for the chat completion request. """ params = { "model": self.model_name, "stream": True, - "messages": messages, - "tools": context.tools, - "tool_choice": context.tool_choice, "seed": self._settings["seed"], "temperature": self._settings["temperature"], "top_p": self._settings["top_p"], "max_completion_tokens": self._settings["max_completion_tokens"], } + # Messages, tools, tool_choice + params.update(params_from_context) + params.update(self._settings["extra"]) return params + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as Cerebras service does not yet support universal LLMContext. + """ + return False diff --git a/src/pipecat/services/deepseek/llm.py b/src/pipecat/services/deepseek/llm.py index 1e6bfcc5c..7b616a293 100644 --- a/src/pipecat/services/deepseek/llm.py +++ b/src/pipecat/services/deepseek/llm.py @@ -9,9 +9,8 @@ from typing import List from loguru import logger -from openai.types.chat import ChatCompletionMessageParam -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams from pipecat.services.openai.llm import OpenAILLMService @@ -54,19 +53,22 @@ class DeepSeekLLMService(OpenAILLMService): logger.debug(f"Creating DeepSeek client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) - def _build_chat_completion_params( - self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] - ) -> dict: + def _build_chat_completion_params(self, params_from_context: OpenAILLMInvocationParams) -> dict: """Build parameters for DeepSeek chat completion request. DeepSeek doesn't support some OpenAI parameters like seed and max_completion_tokens. + + Args: + params_from_context: Parameters, derived from the LLM context, to + use for the chat completion. Contains messages, tools, and tool + choice. + + Returns: + Dictionary of parameters for the chat completion request. """ params = { "model": self.model_name, "stream": True, - "messages": messages, - "tools": context.tools, - "tool_choice": context.tool_choice, "stream_options": {"include_usage": True}, "frequency_penalty": self._settings["frequency_penalty"], "presence_penalty": self._settings["presence_penalty"], @@ -75,5 +77,17 @@ class DeepSeekLLMService(OpenAILLMService): "max_tokens": self._settings["max_tokens"], } + # Messages, tools, tool_choice + params.update(params_from_context) + params.update(self._settings["extra"]) return params + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as DeepSeekLLMService does not yet support universal LLMContext. + """ + return False diff --git a/src/pipecat/services/fireworks/llm.py b/src/pipecat/services/fireworks/llm.py index e28ff9759..194adfc51 100644 --- a/src/pipecat/services/fireworks/llm.py +++ b/src/pipecat/services/fireworks/llm.py @@ -9,9 +9,8 @@ from typing import List from loguru import logger -from openai.types.chat import ChatCompletionMessageParam -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams from pipecat.services.openai.llm import OpenAILLMService @@ -54,20 +53,23 @@ class FireworksLLMService(OpenAILLMService): logger.debug(f"Creating Fireworks client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) - def build_chat_completion_params( - self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] - ) -> dict: + def build_chat_completion_params(self, params_from_context: OpenAILLMInvocationParams) -> dict: """Build parameters for Fireworks chat completion request. Fireworks doesn't support some OpenAI parameters like seed, max_completion_tokens, and stream_options. + + Args: + params_from_context: Parameters, derived from the LLM context, to + use for the chat completion. Contains messages, tools, and tool + choice. + + Returns: + Dictionary of parameters for the chat completion request. """ params = { "model": self.model_name, "stream": True, - "messages": messages, - "tools": context.tools, - "tool_choice": context.tool_choice, "frequency_penalty": self._settings["frequency_penalty"], "presence_penalty": self._settings["presence_penalty"], "temperature": self._settings["temperature"], @@ -75,5 +77,17 @@ class FireworksLLMService(OpenAILLMService): "max_tokens": self._settings["max_tokens"], } + # Messages, tools, tool_choice + params.update(params_from_context) + params.update(self._settings["extra"]) return params + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as FireworksLLMService does not yet support universal LLMContext. + """ + return False diff --git a/src/pipecat/services/google/llm_openai.py b/src/pipecat/services/google/llm_openai.py index bcd350380..59d1e0c1a 100644 --- a/src/pipecat/services/google/llm_openai.py +++ b/src/pipecat/services/google/llm_openai.py @@ -61,6 +61,15 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as GoogleLLMOpenAIBetaService does not yet support universal LLMContext. + """ + return False + async def _process_context(self, context: OpenAILLMContext): functions_list = [] arguments_list = [] @@ -72,7 +81,7 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): await self.start_ttfb_metrics() - chunk_stream: AsyncStream[ChatCompletionChunk] = await self._stream_chat_completions( + chunk_stream: AsyncStream[ChatCompletionChunk] = await self._stream_chat_completions_specific_context( context ) diff --git a/src/pipecat/services/google/llm_vertex.py b/src/pipecat/services/google/llm_vertex.py index 22b6258a5..bdbf2dda1 100644 --- a/src/pipecat/services/google/llm_vertex.py +++ b/src/pipecat/services/google/llm_vertex.py @@ -139,3 +139,12 @@ class GoogleVertexLLMService(OpenAILLMService): creds.refresh(Request()) # Ensure token is up-to-date, lifetime is 1 hour. return creds.token + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as GoogleVertexLLMService does not yet support universal LLMContext. + """ + return False diff --git a/src/pipecat/services/grok/llm.py b/src/pipecat/services/grok/llm.py index 2a9704008..49fe2e802 100644 --- a/src/pipecat/services/grok/llm.py +++ b/src/pipecat/services/grok/llm.py @@ -190,3 +190,12 @@ class GrokLLMService(OpenAILLMService): user = OpenAIUserContextAggregator(context, params=user_params) assistant = OpenAIAssistantContextAggregator(context, params=assistant_params) return GrokContextAggregatorPair(_user=user, _assistant=assistant) + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as GrokLLMService does not yet support universal LLMContext. + """ + return False diff --git a/src/pipecat/services/groq/llm.py b/src/pipecat/services/groq/llm.py index 57f2a533d..d3166ff8b 100644 --- a/src/pipecat/services/groq/llm.py +++ b/src/pipecat/services/groq/llm.py @@ -49,3 +49,12 @@ class GroqLLMService(OpenAILLMService): """ logger.debug(f"Creating Groq client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as GroqLLMService does not yet support universal LLMContext. + """ + return False diff --git a/src/pipecat/services/nim/llm.py b/src/pipecat/services/nim/llm.py index 052b94274..fdfb8bf6b 100644 --- a/src/pipecat/services/nim/llm.py +++ b/src/pipecat/services/nim/llm.py @@ -47,6 +47,15 @@ class NimLLMService(OpenAILLMService): self._has_reported_prompt_tokens = False self._is_processing = False + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as NimLLMService does not yet support universal LLMContext. + """ + return False + async def _process_context(self, context: OpenAILLMContext): """Process a context through the LLM and accumulate token usage metrics. diff --git a/src/pipecat/services/ollama/llm.py b/src/pipecat/services/ollama/llm.py index 2284a5070..aa6f58b59 100644 --- a/src/pipecat/services/ollama/llm.py +++ b/src/pipecat/services/ollama/llm.py @@ -43,3 +43,12 @@ class OLLamaLLMService(OpenAILLMService): """ logger.debug(f"Creating Ollama client with api {base_url}") return super().create_client(base_url=base_url, **kwargs) + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as OLLamaLLMService does not yet support universal LLMContext. + """ + return False diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 916236f12..3fdd6ee8f 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -190,12 +190,13 @@ class BaseOpenAILLMService(LLMService): Args: params_from_context: Parameters, derived from the LLM context, to - use for the chat completion. Contains messages, tools, and tool choice. + use for the chat completion. Contains messages, tools, and tool + choice. Returns: Async stream of chat completion chunks. """ - params = self.build_chat_completion_params(context, messages) + params = self.build_chat_completion_params(params_from_context) if self._retry_on_timeout: try: @@ -213,7 +214,7 @@ class BaseOpenAILLMService(LLMService): return chunks def build_chat_completion_params( - self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] + self, params_from_context: OpenAILLMInvocationParams ) -> dict: """Build parameters for chat completion request. @@ -245,7 +246,7 @@ class BaseOpenAILLMService(LLMService): params.update(self._settings["extra"]) return params - async def _stream_chat_completions( + async def _stream_chat_completions_specific_context( self, context: OpenAILLMContext ) -> AsyncStream[ChatCompletionChunk]: logger.debug( @@ -303,7 +304,7 @@ class BaseOpenAILLMService(LLMService): # Generate chat completions using either OpenAILLMContext or universal LLMContext chunk_stream = await ( - self._stream_chat_completions(context) + self._stream_chat_completions_specific_context(context) if isinstance(context, OpenAILLMContext) else self._stream_chat_completions_universal_context(context) ) @@ -389,6 +390,18 @@ class BaseOpenAILLMService(LLMService): await self.run_function_calls(function_calls) + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + Whether service supports universal LLMContext. + """ + # Return True in subclasses that support universal LLMContext + # This property lets us gradually roll out support for universal + # LLMContext to OpenAI-like services in a controlled manner. + return False + async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames for LLM completion requests. @@ -408,7 +421,12 @@ class BaseOpenAILLMService(LLMService): context = frame.context elif isinstance(frame, LLMContextFrame): # Handle universal (LLM-agnostic) LLM context frames - context = frame.context + if self.supports_universal_context: + context = frame.context + else: + raise NotImplementedError( + f"Universal LLMContext is not yet supported for {self.__class__.__name__}." + ) elif isinstance(frame, LLMMessagesFrame): # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal # LLMContext with it diff --git a/src/pipecat/services/openai/llm.py b/src/pipecat/services/openai/llm.py index 7919dd159..9f5d896b6 100644 --- a/src/pipecat/services/openai/llm.py +++ b/src/pipecat/services/openai/llm.py @@ -107,6 +107,15 @@ class OpenAILLMService(BaseOpenAILLMService): assistant = OpenAIAssistantContextAggregator(context, params=assistant_params) return OpenAIContextAggregatorPair(_user=user, _assistant=assistant) + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + True, as OpenAI service supports universal LLMContext. + """ + return True + class OpenAIUserContextAggregator(LLMUserContextAggregator): """OpenAI-specific user context aggregator. diff --git a/src/pipecat/services/openpipe/llm.py b/src/pipecat/services/openpipe/llm.py index 581bb045f..2e491ea26 100644 --- a/src/pipecat/services/openpipe/llm.py +++ b/src/pipecat/services/openpipe/llm.py @@ -13,9 +13,8 @@ enabling integration with OpenPipe's fine-tuning and monitoring capabilities. from typing import Dict, List, Optional from loguru import logger -from openai.types.chat import ChatCompletionMessageParam -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams from pipecat.services.openai.llm import OpenAILLMService try: @@ -86,22 +85,21 @@ class OpenPipeLLMService(OpenAILLMService): ) return client - def build_chat_completion_params( - self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] - ) -> dict: + def build_chat_completion_params(self, params_from_context: OpenAILLMInvocationParams) -> dict: """Build parameters for OpenPipe chat completion request. Adds OpenPipe-specific logging and tagging parameters. Args: - context: The LLM context containing tools and configuration. - messages: List of chat completion messages to send. + params_from_context: Parameters, derived from the LLM context, to + use for the chat completion. Contains messages, tools, and tool + choice. Returns: Dictionary of parameters for the chat completion request. """ # Start with base parameters - params = super().build_chat_completion_params(context, messages) + params = super().build_chat_completion_params(params_from_context) # Add OpenPipe-specific parameters params["openpipe"] = { @@ -110,3 +108,12 @@ class OpenPipeLLMService(OpenAILLMService): } return params + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as OpenPipeLLMService does not yet support universal LLMContext. + """ + return False diff --git a/src/pipecat/services/openrouter/llm.py b/src/pipecat/services/openrouter/llm.py index 97a9d336a..3ba1ae6f6 100644 --- a/src/pipecat/services/openrouter/llm.py +++ b/src/pipecat/services/openrouter/llm.py @@ -61,3 +61,12 @@ class OpenRouterLLMService(OpenAILLMService): """ logger.debug(f"Creating OpenRouter client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as OpenRouterLLMService does not yet support universal LLMContext. + """ + return False diff --git a/src/pipecat/services/perplexity/llm.py b/src/pipecat/services/perplexity/llm.py index 59cbe520b..3e39206c6 100644 --- a/src/pipecat/services/perplexity/llm.py +++ b/src/pipecat/services/perplexity/llm.py @@ -11,11 +11,9 @@ an OpenAI-compatible interface. It handles Perplexity's unique token usage reporting patterns while maintaining compatibility with the Pipecat framework. """ -from typing import List - from openai import NOT_GIVEN -from openai.types.chat import ChatCompletionMessageParam +from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai.llm import OpenAILLMService @@ -53,17 +51,23 @@ class PerplexityLLMService(OpenAILLMService): self._has_reported_prompt_tokens = False self._is_processing = False - def build_chat_completion_params( - self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] - ) -> dict: + def build_chat_completion_params(self, params_from_context: OpenAILLMInvocationParams) -> dict: """Build parameters for Perplexity chat completion request. Perplexity uses a subset of OpenAI parameters and doesn't support tools. + + Args: + params_from_context: Parameters, derived from the LLM context, to + use for the chat completion. Contains messages, tools, and tool + choice. + + Returns: + Dictionary of parameters for the chat completion request. """ params = { "model": self.model_name, "stream": True, - "messages": messages, + "messages": params_from_context["messages"], } # Add OpenAI-compatible parameters if they're set @@ -80,6 +84,15 @@ class PerplexityLLMService(OpenAILLMService): return params + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as PerplexityLLMService does not yet support universal LLMContext. + """ + return False + async def _process_context(self, context: OpenAILLMContext): """Process a context through the LLM and accumulate token usage metrics. diff --git a/src/pipecat/services/qwen/llm.py b/src/pipecat/services/qwen/llm.py index 648cbd9e8..1c842ded6 100644 --- a/src/pipecat/services/qwen/llm.py +++ b/src/pipecat/services/qwen/llm.py @@ -50,3 +50,12 @@ class QwenLLMService(OpenAILLMService): """ logger.debug(f"Creating Qwen client with base URL: {base_url}") return super().create_client(api_key, base_url, **kwargs) + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as QwenLLMService does not yet support universal LLMContext. + """ + return False diff --git a/src/pipecat/services/sambanova/llm.py b/src/pipecat/services/sambanova/llm.py index f2ee082ed..72ca5b8fb 100644 --- a/src/pipecat/services/sambanova/llm.py +++ b/src/pipecat/services/sambanova/llm.py @@ -7,12 +7,13 @@ """SambaNova LLM service implementation using OpenAI-compatible interface.""" import json -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional from loguru import logger from openai import AsyncStream -from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam +from openai.types.chat import ChatCompletionChunk +from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams from pipecat.frames.frames import ( LLMTextFrame, ) @@ -67,17 +68,16 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore logger.debug(f"Creating SambaNova client with API {base_url}") return super().create_client(api_key, base_url, **kwargs) - def build_chat_completion_params( - self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] - ) -> dict: + def build_chat_completion_params(self, params_from_context: OpenAILLMInvocationParams) -> dict: """Build parameters for SambaNova chat completion request. SambaNova doesn't support some OpenAI parameters like frequency_penalty, presence_penalty, and seed. Args: - context: The LLM context containing tools and configuration. - messages: List of chat completion messages to send. + params_from_context: Parameters, derived from the LLM context, to + use for the chat completion. Contains messages, tools, and tool + choice. Returns: Dictionary of parameters for the chat completion request. @@ -85,9 +85,6 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore params = { "model": self.model_name, "stream": True, - "messages": messages, - "tools": context.tools, - "tool_choice": context.tool_choice, "stream_options": {"include_usage": True}, "temperature": self._settings["temperature"], "top_p": self._settings["top_p"], @@ -95,6 +92,9 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore "max_completion_tokens": self._settings["max_completion_tokens"], } + # Messages, tools, tool_choice + params.update(params_from_context) + params.update(self._settings["extra"]) return params @@ -122,7 +122,7 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore await self.start_ttfb_metrics() - chunk_stream: AsyncStream[ChatCompletionChunk] = await self._stream_chat_completions( + chunk_stream: AsyncStream[ChatCompletionChunk] = await self._stream_chat_completions_specific_context( context ) @@ -210,3 +210,12 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore ) await self.run_function_calls(function_calls) + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as SambaNovaLLMService does not yet support universal LLMContext. + """ + return False diff --git a/src/pipecat/services/together/llm.py b/src/pipecat/services/together/llm.py index 7a22c885a..2a004f1c9 100644 --- a/src/pipecat/services/together/llm.py +++ b/src/pipecat/services/together/llm.py @@ -49,3 +49,12 @@ class TogetherLLMService(OpenAILLMService): """ logger.debug(f"Creating Together.ai client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) + + @property + def supports_universal_context(self) -> bool: + """Check if this service supports universal LLMContext. + + Returns: + False, as TogetherLLMService does not yet support universal LLMContext. + """ + return False