From 15aa76efbaabbe7e760242522772eab70836161b Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 23 Oct 2025 12:49:48 -0400 Subject: [PATCH] Update `OpenAIRealtimeLLMService` to work with `LLMContext` and `LLMContextAggregatorPair` (cont'd). Maintain backward compatibility with functions specified in dict format. --- src/pipecat/adapters/schemas/tools_schema.py | 3 +++ .../services/aws_nova_sonic_adapter.py | 18 ++++++++++++++++-- .../services/open_ai_realtime_adapter.py | 18 ++++++++++++++++-- .../processors/aggregators/llm_context.py | 12 ++++++++++-- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/pipecat/adapters/schemas/tools_schema.py b/src/pipecat/adapters/schemas/tools_schema.py index 05710616d..d0d798569 100644 --- a/src/pipecat/adapters/schemas/tools_schema.py +++ b/src/pipecat/adapters/schemas/tools_schema.py @@ -22,9 +22,12 @@ class AdapterType(Enum): Parameters: GEMINI: Google Gemini adapter - currently the only service supporting custom tools. + SHIM: Backward compatibility shim for creating ToolsSchemas from lists of tools in + any format, used by LLMContext.from_openai_context. """ GEMINI = "gemini" # that is the only service where we are able to add custom tools for now + SHIM = "shim" # for use as backward compatibility shim for creating ToolsSchemas from list of tools in any format class ToolsSchema: diff --git a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py index 60f12798b..dcc42ba68 100644 --- a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py +++ b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py @@ -16,7 +16,7 @@ from loguru import logger from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage @@ -210,4 +210,18 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): List of dictionaries in AWS Nova Sonic function format. """ functions_schema = tools_schema.standard_tools - return [self._to_aws_nova_sonic_function_format(func) for func in functions_schema] + standard_tools = [ + self._to_aws_nova_sonic_function_format(func) for func in functions_schema + ] + + # For backward compatibility, AWS Nova Sonic can still be used with + # tools in dict format, even though it always uses `LLMContext` under + # the hood (via `LLMContext.from_openai_context()`). + # To support this behavior, we use "shimmed" custom tools here. + # (We maintain this backward compatibility because users aren't + # *knowingly* opting into the new `LLMContext`.) + shimmed_tools = [] + if tools_schema.custom_tools: + shimmed_tools = tools_schema.custom_tools.get(AdapterType.SHIM, []) + + return standard_tools + shimmed_tools diff --git a/src/pipecat/adapters/services/open_ai_realtime_adapter.py b/src/pipecat/adapters/services/open_ai_realtime_adapter.py index 3cdfefc86..3d3650633 100644 --- a/src/pipecat/adapters/services/open_ai_realtime_adapter.py +++ b/src/pipecat/adapters/services/open_ai_realtime_adapter.py @@ -15,7 +15,7 @@ from loguru import logger from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage from pipecat.services.openai.realtime import events @@ -225,4 +225,18 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): List of function definitions in OpenAI Realtime format. """ functions_schema = tools_schema.standard_tools - return [self._to_openai_realtime_function_format(func) for func in functions_schema] + standard_tools = [ + self._to_openai_realtime_function_format(func) for func in functions_schema + ] + + # For backward compatibility, OpenAI Realtime can still be used with + # tools in dict format, even though it always uses `LLMContext` under + # the hood (via `LLMContext.from_openai_context()`). + # To support this behavior, we use "shimmed" custom tools here. + # (We maintain this backward compatibility because users aren't + # *knowingly* opting into the new `LLMContext`.) + shimmed_tools = [] + if tools_schema.custom_tools: + shimmed_tools = tools_schema.custom_tools.get(AdapterType.SHIM, []) + + return standard_tools + shimmed_tools diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index 913566909..61241729e 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -29,7 +29,7 @@ from openai.types.chat import ( ) from PIL import Image -from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema from pipecat.frames.frames import AudioRawFrame if TYPE_CHECKING: @@ -83,9 +83,17 @@ class LLMContext: Returns: New LLMContext instance with converted messages and settings. """ + # Convert tools to ToolsSchema if needed. + # If the tools are already a ToolsSchema, this is a no-op. + # Otherwise, we wrap them in a shim ToolsSchema. + converted_tools = openai_context.tools + if isinstance(converted_tools, list): + converted_tools = ToolsSchema( + standard_tools=[], custom_tools={AdapterType.SHIM: converted_tools} + ) return LLMContext( messages=openai_context.get_messages(), - tools=openai_context.tools, + tools=converted_tools, tool_choice=openai_context.tool_choice, )