diff --git a/changelog/4248.added.md b/changelog/4248.added.md new file mode 100644 index 000000000..b48a6a432 --- /dev/null +++ b/changelog/4248.added.md @@ -0,0 +1 @@ +- `ToolsSchema` now accepts `custom_tools` for OpenAI LLM services (`OpenAILLMService`, `OpenAIResponsesLLMService`, `OpenAIResponsesHttpLLMService`, and `OpenAIRealtimeLLMService`), letting you pass provider-specific tools like `tool_search` alongside standard function tools. diff --git a/src/pipecat/adapters/schemas/tools_schema.py b/src/pipecat/adapters/schemas/tools_schema.py index e3940f6cb..1c1ba0dd3 100644 --- a/src/pipecat/adapters/schemas/tools_schema.py +++ b/src/pipecat/adapters/schemas/tools_schema.py @@ -21,10 +21,12 @@ class AdapterType(Enum): """Supported adapter types for custom tools. Parameters: - GEMINI: Google Gemini adapter - currently the only service supporting custom tools. + GEMINI: Google Gemini adapter. + OPENAI: OpenAI adapter (Chat Completions, Responses, and Realtime API). """ - GEMINI = "gemini" # that is the only service where we are able to add custom tools for now + GEMINI = "gemini" + OPENAI = "openai" class ToolsSchema: diff --git a/src/pipecat/adapters/services/open_ai_adapter.py b/src/pipecat/adapters/services/open_ai_adapter.py index 37adcfc9a..db9f3ee2f 100644 --- a/src/pipecat/adapters/services/open_ai_adapter.py +++ b/src/pipecat/adapters/services/open_ai_adapter.py @@ -17,7 +17,7 @@ from openai.types.chat import ( ) from pipecat.adapters.base_llm_adapter import BaseLLMAdapter -from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema from pipecat.processors.aggregators.llm_context import ( LLMContext, LLMContextMessage, @@ -107,10 +107,14 @@ class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]): with ChatCompletion API. """ functions_schema = tools_schema.standard_tools - return [ + formatted_standard_tools = [ ChatCompletionToolParam(type="function", function=func.to_default_dict()) for func in functions_schema ] + custom_openai_tools = [] + if tools_schema.custom_tools: + custom_openai_tools = tools_schema.custom_tools.get(AdapterType.OPENAI, []) + return formatted_standard_tools + custom_openai_tools def get_messages_for_logging(self, context: LLMContext) -> List[Dict[str, Any]]: """Get messages from a universal LLM context in a format ready for logging about OpenAI. diff --git a/src/pipecat/adapters/services/open_ai_realtime_adapter.py b/src/pipecat/adapters/services/open_ai_realtime_adapter.py index 64684e038..0481d1bd4 100644 --- a/src/pipecat/adapters/services/open_ai_realtime_adapter.py +++ b/src/pipecat/adapters/services/open_ai_realtime_adapter.py @@ -15,7 +15,7 @@ from loguru import logger from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema from pipecat.processors.aggregators.llm_context import LLMContext, LLMContextMessage from pipecat.services.openai.realtime import events @@ -236,4 +236,10 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): List of function definitions in OpenAI Realtime format. """ functions_schema = tools_schema.standard_tools - return [self._to_openai_realtime_function_format(func) for func in functions_schema] + formatted_standard_tools = [ + self._to_openai_realtime_function_format(func) for func in functions_schema + ] + custom_openai_tools = [] + if tools_schema.custom_tools: + custom_openai_tools = tools_schema.custom_tools.get(AdapterType.OPENAI, []) + return formatted_standard_tools + custom_openai_tools diff --git a/src/pipecat/adapters/services/open_ai_responses_adapter.py b/src/pipecat/adapters/services/open_ai_responses_adapter.py index 320bd11ef..a58b26c91 100644 --- a/src/pipecat/adapters/services/open_ai_responses_adapter.py +++ b/src/pipecat/adapters/services/open_ai_responses_adapter.py @@ -10,10 +10,10 @@ import copy from typing import Any, Dict, List, Optional, TypedDict from openai._types import NotGiven as OpenAINotGiven -from openai.types.responses import FunctionToolParam, ResponseInputItemParam +from openai.types.responses import FunctionToolParam, ResponseInputItemParam, ToolParam from pipecat.adapters.base_llm_adapter import BaseLLMAdapter -from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema from pipecat.processors.aggregators.llm_context import ( LLMContext, LLMContextMessage, @@ -25,7 +25,7 @@ class OpenAIResponsesLLMInvocationParams(TypedDict, total=False): """Context-based parameters for invoking OpenAI Responses API.""" input: List[ResponseInputItemParam] - tools: List[FunctionToolParam] | OpenAINotGiven + tools: List[ToolParam] | OpenAINotGiven instructions: str @@ -106,7 +106,7 @@ class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParam return params - def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[FunctionToolParam]: + def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[ToolParam]: """Convert function schemas to Responses API function tool format. Args: @@ -128,7 +128,10 @@ class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParam if "description" in d: tool["description"] = d["description"] result.append(tool) - return result + custom_openai_tools = [] + if tools_schema.custom_tools: + custom_openai_tools = tools_schema.custom_tools.get(AdapterType.OPENAI, []) + return result + custom_openai_tools def get_messages_for_logging(self, context: LLMContext) -> List[Dict[str, Any]]: """Get messages from context in a format ready for logging. diff --git a/tests/test_function_calling_adapters.py b/tests/test_function_calling_adapters.py index 621c8d139..4a1bfcb7e 100644 --- a/tests/test_function_calling_adapters.py +++ b/tests/test_function_calling_adapters.py @@ -7,6 +7,8 @@ import unittest from openai.types.chat import ChatCompletionToolParam +from openai.types.responses.function_tool_param import FunctionToolParam +from openai.types.responses.tool_search_tool_param import ToolSearchToolParam from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema @@ -16,6 +18,7 @@ from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter from pipecat.adapters.services.inworld_realtime_adapter import InworldRealtimeLLMAdapter from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter +from pipecat.adapters.services.open_ai_responses_adapter import OpenAIResponsesLLMAdapter class TestFunctionAdapters(unittest.TestCase): @@ -203,6 +206,133 @@ class TestFunctionAdapters(unittest.TestCase): tools_def.custom_tools = {AdapterType.GEMINI: [search_tool]} assert GeminiLLMAdapter().to_provider_tools_format(tools_def) == expected + def test_openai_adapter_with_custom_tools(self): + """Test OpenAI adapter appends custom tools.""" + tool_search = {"type": "tool_search"} + expected = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "get_weather", + "description": "Get the weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use.", + }, + }, + "required": ["location", "format"], + }, + }, + ), + tool_search, + ] + tools_def = self.tools_def + tools_def.custom_tools = {AdapterType.OPENAI: [tool_search]} + assert OpenAILLMAdapter().to_provider_tools_format(tools_def) == expected + + def test_openai_responses_adapter_with_custom_tools(self): + """Test OpenAI Responses adapter appends custom tools.""" + tool_search = {"type": "tool_search"} + expected = [ + FunctionToolParam( + type="function", + name="get_weather", + description="Get the weather in a given location", + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use.", + }, + }, + "required": ["location", "format"], + }, + strict=None, + ), + ToolSearchToolParam(type="tool_search"), + ] + tools_def = self.tools_def + tools_def.custom_tools = {AdapterType.OPENAI: [tool_search]} + assert OpenAIResponsesLLMAdapter().to_provider_tools_format(tools_def) == expected + + def test_openai_responses_adapter(self): + """Test OpenAI Responses adapter format transformation.""" + expected = [ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use.", + }, + }, + "required": ["location", "format"], + }, + "strict": None, + } + ] + assert OpenAIResponsesLLMAdapter().to_provider_tools_format(self.tools_def) == expected + + def test_openai_realtime_adapter_with_custom_tools(self): + """Test OpenAI Realtime adapter appends custom tools.""" + tool_search = {"type": "tool_search"} + expected = [ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use.", + }, + }, + "required": ["location", "format"], + }, + }, + tool_search, + ] + tools_def = self.tools_def + tools_def.custom_tools = {AdapterType.OPENAI: [tool_search]} + assert OpenAIRealtimeLLMAdapter().to_provider_tools_format(tools_def) == expected + + def test_openai_adapter_ignores_other_adapter_custom_tools(self): + """Test that OpenAI adapter ignores custom tools for other adapters.""" + expected = OpenAILLMAdapter().to_provider_tools_format(self.tools_def) + tools_def = self.tools_def + tools_def.custom_tools = {AdapterType.GEMINI: [{"google_search": {}}]} + assert OpenAILLMAdapter().to_provider_tools_format(tools_def) == expected + def test_bedrock_adapter(self): """Test AWS Bedrock adapter format transformation.""" expected = [