From 925a6cc2ef59d682cc6cfb4359a9380176478df7 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 7 Nov 2025 09:50:26 -0500 Subject: [PATCH 1/3] Added support for passing in a `ToolsSchema` in lieu of a list of provider-specific dicts when initializing `OpenAIRealtimeLLMService`. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I chose to go the somewhat hacky route of adding the `ToolsSchema` support into the `events.SessionProperties` model itself—even though we should never serialize that type when creating events—because the alternative seemed to be to create a new type for `OpenAIRealtimeLLMService` initialization parameters and then we'd have to contend with backward compatibility, which seemed like a bigger headache. --- CHANGELOG.md | 3 +++ src/pipecat/services/openai/realtime/events.py | 10 +++++++++- src/pipecat/services/openai/realtime/llm.py | 11 +++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c6ff6143..2ec09c493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for passing in a `ToolsSchem` in lieu of a list of provider- + specific dicts when initializing `OpenAIRealtimeLLMService`. + - Added `TransportParams.audio_out_silence_secs`, which specifies how many seconds of silence to output when an `EndFrame` reaches the output transport. This can help ensure that all audio data is fully delivered to diff --git a/src/pipecat/services/openai/realtime/events.py b/src/pipecat/services/openai/realtime/events.py index f52298ab3..42b5561d6 100644 --- a/src/pipecat/services/openai/realtime/events.py +++ b/src/pipecat/services/openai/realtime/events.py @@ -12,6 +12,8 @@ from typing import Any, Dict, List, Literal, Optional, Union from pydantic import BaseModel, ConfigDict, Field +from pipecat.adapters.schemas.tools_schema import ToolsSchema + # # session properties # @@ -186,6 +188,9 @@ class SessionProperties(BaseModel): include: Additional fields to include in server outputs. """ + # Needed to support ToolSchema in tools field. + model_config = ConfigDict(arbitrary_types_allowed=True) + type: Optional[Literal["realtime"]] = "realtime" object: Optional[Literal["realtime.session"]] = None id: Optional[str] = None @@ -193,7 +198,10 @@ class SessionProperties(BaseModel): output_modalities: Optional[List[Literal["text", "audio"]]] = None instructions: Optional[str] = None audio: Optional[AudioConfiguration] = None - tools: Optional[List[Dict]] = None + # Tools can only be ToolsSchema when provided by user in + # OpenAIRealtimeLLMService constructor. We'll never serialize/deserialize + # ToolsSchema. + tools: Optional[ToolsSchema | List[Dict]] = None tool_choice: Optional[Literal["auto", "none", "required"]] = None max_output_tokens: Optional[Union[int, Literal["inf"]]] = None tracing: Optional[Union[Literal["auto"], Dict]] = None diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 6d3604541..77e326eb8 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -14,6 +14,7 @@ from typing import Optional from loguru import logger +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.services.open_ai_realtime_adapter import ( OpenAIRealtimeLLMAdapter, ) @@ -155,6 +156,16 @@ class OpenAIRealtimeLLMService(LLMService): self._session_properties: events.SessionProperties = ( session_properties or events.SessionProperties() ) + # If needed, map session_properties.tools from ToolsSchema to list of + # dicts, which remote server expects + if self._session_properties.tools and isinstance( + self._session_properties.tools, ToolsSchema + ): + adapter = self.get_llm_adapter() + self._session_properties.tools = adapter.from_standard_tools( + self._session_properties.tools + ) + self._audio_input_paused = start_audio_paused self._websocket = None self._receive_task = None From c3306bb4f2d6ed3e47132de5db673f4a2444980d Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 7 Nov 2025 10:17:53 -0500 Subject: [PATCH 2/3] Support for passing in a `ToolsSchema` in lieu of a list of provider-specific dicts when updating `OpenAIRealtimeLLMService` using `LLMUpdateSettingsFrame`. --- CHANGELOG.md | 3 ++- examples/foundational/19-openai-realtime.py | 14 +++++++++++++- src/pipecat/services/openai/realtime/events.py | 6 +++--- src/pipecat/services/openai/realtime/llm.py | 18 +++++++----------- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ec09c493..46be71bb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added support for passing in a `ToolsSchem` in lieu of a list of provider- - specific dicts when initializing `OpenAIRealtimeLLMService`. + specific dicts when initializing `OpenAIRealtimeLLMService` or when updating + it using `LLMUpdateSettingsFrame`. - Added `TransportParams.audio_out_silence_secs`, which specifies how many seconds of silence to output when an `EndFrame` reaches the output diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index b5edc0ff2..31af47bf9 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -14,8 +14,14 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMRunFrame, LLMSetToolsFrame, TranscriptionMessage +from pipecat.frames.frames import ( + LLMRunFrame, + LLMSetToolsFrame, + LLMUpdateSettingsFrame, + TranscriptionMessage, +) from pipecat.observers.loggers.transcription_log_observer import TranscriptionLogObserver from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -148,6 +154,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): noise_reduction=InputAudioNoiseReduction(type="near_field"), ) ), + # In this example we provide tools through the context, but you could + # alternatively provide them here. # tools=tools, instructions="""You are a helpful and friendly AI. @@ -223,6 +231,10 @@ Remember, your responses should be short. Just one or two sentences, usually. Re standard_tools=[weather_function, restaurant_function, get_news_function] ) await task.queue_frames([LLMSetToolsFrame(tools=new_tools)]) + # Alternative pattern, useful if you're changing other session properties too + # await task.queue_frames( + # [LLMUpdateSettingsFrame(settings=SessionProperties(tools=new_tools).model_dump())] + # ) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): diff --git a/src/pipecat/services/openai/realtime/events.py b/src/pipecat/services/openai/realtime/events.py index 42b5561d6..e1a69b43c 100644 --- a/src/pipecat/services/openai/realtime/events.py +++ b/src/pipecat/services/openai/realtime/events.py @@ -198,9 +198,9 @@ class SessionProperties(BaseModel): output_modalities: Optional[List[Literal["text", "audio"]]] = None instructions: Optional[str] = None audio: Optional[AudioConfiguration] = None - # Tools can only be ToolsSchema when provided by user in - # OpenAIRealtimeLLMService constructor. We'll never serialize/deserialize - # ToolsSchema. + # Tools can only be ToolsSchema when provided by the user, in either the + # OpenAIRealtimeLLMService constructor or through LLMUpdateSettingsFrame. + # We'll never serialize/deserialize ToolsSchema when talking to the server. tools: Optional[ToolsSchema | List[Dict]] = None tool_choice: Optional[Literal["auto", "none", "required"]] = None max_output_tokens: Optional[Union[int, Literal["inf"]]] = None diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 77e326eb8..0129a94f9 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -156,16 +156,6 @@ class OpenAIRealtimeLLMService(LLMService): self._session_properties: events.SessionProperties = ( session_properties or events.SessionProperties() ) - # If needed, map session_properties.tools from ToolsSchema to list of - # dicts, which remote server expects - if self._session_properties.tools and isinstance( - self._session_properties.tools, ToolsSchema - ): - adapter = self.get_llm_adapter() - self._session_properties.tools = adapter.from_standard_tools( - self._session_properties.tools - ) - self._audio_input_paused = start_audio_paused self._websocket = None self._receive_task = None @@ -492,9 +482,9 @@ class OpenAIRealtimeLLMService(LLMService): async def _update_settings(self): settings = self._session_properties + adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter() if self._context: - adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter() llm_invocation_params = adapter.get_llm_invocation_params(self._context) # tools given in the context override the tools in the session properties @@ -506,6 +496,12 @@ class OpenAIRealtimeLLMService(LLMService): if llm_invocation_params["system_instruction"]: settings.instructions = llm_invocation_params["system_instruction"] + # If needed, map settings.tools from ToolsSchema to list of dicts, + # which remote server expects. It would only be a ToolsSchema if that's + # how it was provided in the constructor or via LLMUpdateSettingsFrame. + if settings.tools and isinstance(settings.tools, ToolsSchema): + settings.tools = adapter.from_standard_tools(settings.tools) + await self.send_client_event(events.SessionUpdateEvent(session=settings)) # From 359d22016230295d0fba8b2f2e831c309dd8bbc3 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 7 Nov 2025 10:32:27 -0500 Subject: [PATCH 3/3] Document a `OpenAIRealtimeLLMService` gotcha in an example. --- examples/foundational/19-openai-realtime.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 31af47bf9..dca28b9fb 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -231,7 +231,12 @@ Remember, your responses should be short. Just one or two sentences, usually. Re standard_tools=[weather_function, restaurant_function, get_news_function] ) await task.queue_frames([LLMSetToolsFrame(tools=new_tools)]) - # Alternative pattern, useful if you're changing other session properties too + # Alternative pattern, useful if you're changing other session properties, too. + # (Though note that tools in your LLMContext take precedence over those + # in session properties, so if you have context-provided tools, prefer + # LLMSetToolsFrame instead, as it updates your context. Ditto for + # updating system instructions: send an LLMMessagesUpdateFrame with + # context messages updated with your new desired system message.) # await task.queue_frames( # [LLMUpdateSettingsFrame(settings=SessionProperties(tools=new_tools).model_dump())] # )