From cfb094b3c84840588765051012028024ea0fa409 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 13 Aug 2025 21:56:59 -0400 Subject: [PATCH] [WIP] Universal (LLM-agnostic) context machinery to support runtime LLM switching. - Make it so that tools in `LLMContext` are guaranteed to be either a `ToolsSchema` or `NOT_GIVEN` --- .../adapters/services/gemini_adapter.py | 1 + .../adapters/services/open_ai_adapter.py | 2 +- .../processors/aggregators/llm_context.py | 26 ++++++++++++------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 68c741d99..2b7c03e40 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -61,6 +61,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): return { "system_instruction": messages.system_instruction, "messages": messages.messages, + # NOTE; LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN) "tools": self.from_standard_tools(context.tools), } diff --git a/src/pipecat/adapters/services/open_ai_adapter.py b/src/pipecat/adapters/services/open_ai_adapter.py index 9a0494e55..f361b9edd 100644 --- a/src/pipecat/adapters/services/open_ai_adapter.py +++ b/src/pipecat/adapters/services/open_ai_adapter.py @@ -57,7 +57,7 @@ class OpenAILLMAdapter(BaseLLMAdapter): """ return { "messages": self._from_standard_messages(context.messages), - # TODO: doesn't seem quite right that we may or may not need to convert tools here; they should already be guaranteed to exist in a universal format in the universal LLMContext, right? + # NOTE; LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN) "tools": self.from_standard_tools(context.tools), "tool_choice": context.tool_choice, } diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index 3208d38f7..1a44d16c4 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -24,7 +24,6 @@ from openai._types import NotGiven as OpenAINotGiven from openai.types.chat import ( ChatCompletionMessageParam, ChatCompletionToolChoiceOptionParam, - ChatCompletionToolParam, ) from PIL import Image @@ -36,7 +35,6 @@ from pipecat.frames.frames import AudioRawFrame, Frame # diverge from OpenAI's, we should ditch this. In fact, audio frames already # diverge from OpenAI's standard format...we really ought to do this. LLMContextMessage = ChatCompletionMessageParam -LLMContextTool = ChatCompletionToolParam LLMContextToolChoice = ChatCompletionToolChoiceOptionParam NOT_GIVEN = OPEN_AI_NOT_GIVEN NotGiven = OpenAINotGiven @@ -53,7 +51,7 @@ class LLMContext: def __init__( self, messages: Optional[List[LLMContextMessage]] = None, - tools: List[LLMContextTool] | NotGiven | ToolsSchema = NOT_GIVEN, + tools: ToolsSchema | NotGiven = NOT_GIVEN, tool_choice: LLMContextToolChoice | NotGiven = NOT_GIVEN, ): """Initialize the LLM context. @@ -64,8 +62,9 @@ class LLMContext: tool_choice: Tool selection strategy for the LLM. """ self._messages: List[LLMContextMessage] = messages if messages else [] - self._tools: List[LLMContextTool] | NotGiven | ToolsSchema = tools + self._tools: ToolsSchema | NotGiven = tools self._tool_choice: LLMContextToolChoice | NotGiven = tool_choice + self._validate_tools() @property def messages(self) -> List[LLMContextMessage]: @@ -77,7 +76,7 @@ class LLMContext: return self._messages @property - def tools(self) -> List[LLMContextTool] | NotGiven | List[Any]: + def tools(self) -> ToolsSchema | NotGiven: """Get the tools list. Returns: @@ -118,17 +117,15 @@ class LLMContext: """ self._messages[:] = messages - def set_tools(self, tools: List[LLMContextTool] | NotGiven | ToolsSchema = NOT_GIVEN): + def set_tools(self, tools: ToolsSchema | NotGiven = NOT_GIVEN): """Set the available tools for the LLM. Args: - tools: List of tools available to the LLM, a ToolsSchema, or NOT_GIVEN to disable tools. + tools: A ToolsSchema or NOT_GIVEN to disable tools. """ # TODO: convert empty ToolsSchema to NOT_GIVEN if needed? - # TODO: maybe someday also convert provider-specific tools to ToolsSchema so it's always in a provider-neutral format here? See open_ai_adapter.py for related comment. Pipecat Flows is currently converting provider-specific tools to ToolsSchema... - if isinstance(tools, list) and len(tools) == 0: - tools = NOT_GIVEN self._tools = tools + self._validate_tools() def set_tool_choice(self, tool_choice: LLMContextToolChoice | NotGiven): """Set the tool choice configuration. @@ -195,3 +192,12 @@ class LLMContext: } ) self.add_message({"role": "user", "content": content}) + + def _validate_tools(self): + """Validate the tools schema. + + Raises: + TypeError: If tools are not a ToolsSchema or NotGiven. + """ + if self._tools is not NOT_GIVEN and not isinstance(self._tools, ToolsSchema): + raise TypeError("In LLMContext, tools must be a ToolsSchema object or NOT_GIVEN.")