From 925a6cc2ef59d682cc6cfb4359a9380176478df7 Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Fri, 7 Nov 2025 09:50:26 -0500
Subject: [PATCH] Added support for passing in a `ToolsSchema` in lieu of a
 list of provider-specific dicts when initializing `OpenAIRealtimeLLMService`.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I chose to go the somewhat hacky route of adding the `ToolsSchema` support into the `events.SessionProperties` model itself—even though we should never serialize that type when creating events—because the alternative seemed to be to create a new type for `OpenAIRealtimeLLMService` initialization parameters and then we'd have to contend with backward compatibility, which seemed like a bigger headache.
---
 CHANGELOG.md                                   |  3 +++
 src/pipecat/services/openai/realtime/events.py | 10 +++++++++-
 src/pipecat/services/openai/realtime/llm.py    | 11 +++++++++++
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5c6ff6143..2ec09c493 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added support for passing in a `ToolsSchem` in lieu of a list of provider-
+  specific dicts when initializing `OpenAIRealtimeLLMService`.
+
 - Added `TransportParams.audio_out_silence_secs`, which specifies how many
   seconds of silence to output when an `EndFrame` reaches the output
   transport. This can help ensure that all audio data is fully delivered to
diff --git a/src/pipecat/services/openai/realtime/events.py b/src/pipecat/services/openai/realtime/events.py
index f52298ab3..42b5561d6 100644
--- a/src/pipecat/services/openai/realtime/events.py
+++ b/src/pipecat/services/openai/realtime/events.py
@@ -12,6 +12,8 @@ from typing import Any, Dict, List, Literal, Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Field
 
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+
 #
 # session properties
 #
@@ -186,6 +188,9 @@ class SessionProperties(BaseModel):
         include: Additional fields to include in server outputs.
     """
 
+    # Needed to support ToolSchema in tools field.
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     type: Optional[Literal["realtime"]] = "realtime"
     object: Optional[Literal["realtime.session"]] = None
     id: Optional[str] = None
@@ -193,7 +198,10 @@ class SessionProperties(BaseModel):
     output_modalities: Optional[List[Literal["text", "audio"]]] = None
     instructions: Optional[str] = None
     audio: Optional[AudioConfiguration] = None
-    tools: Optional[List[Dict]] = None
+    # Tools can only be ToolsSchema when provided by user in
+    # OpenAIRealtimeLLMService constructor. We'll never serialize/deserialize
+    # ToolsSchema.
+    tools: Optional[ToolsSchema | List[Dict]] = None
     tool_choice: Optional[Literal["auto", "none", "required"]] = None
     max_output_tokens: Optional[Union[int, Literal["inf"]]] = None
     tracing: Optional[Union[Literal["auto"], Dict]] = None
diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py
index 6d3604541..77e326eb8 100644
--- a/src/pipecat/services/openai/realtime/llm.py
+++ b/src/pipecat/services/openai/realtime/llm.py
@@ -14,6 +14,7 @@ from typing import Optional
 
 from loguru import logger
 
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
 from pipecat.adapters.services.open_ai_realtime_adapter import (
     OpenAIRealtimeLLMAdapter,
 )
@@ -155,6 +156,16 @@ class OpenAIRealtimeLLMService(LLMService):
         self._session_properties: events.SessionProperties = (
             session_properties or events.SessionProperties()
         )
+        # If needed, map session_properties.tools from ToolsSchema to list of
+        # dicts, which remote server expects
+        if self._session_properties.tools and isinstance(
+            self._session_properties.tools, ToolsSchema
+        ):
+            adapter = self.get_llm_adapter()
+            self._session_properties.tools = adapter.from_standard_tools(
+                self._session_properties.tools
+            )
+
         self._audio_input_paused = start_audio_paused
         self._websocket = None
         self._receive_task = None