diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dae5ceab..aa07b2773 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for passing in a `ToolsSchem` in lieu of a list of provider- + specific dicts when initializing `OpenAIRealtimeLLMService` or when updating + it using `LLMUpdateSettingsFrame`. + - Added `TransportParams.audio_out_silence_secs`, which specifies how many seconds of silence to output when an `EndFrame` reaches the output transport. This can help ensure that all audio data is fully delivered to diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index b5edc0ff2..dca28b9fb 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -14,8 +14,14 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMRunFrame, LLMSetToolsFrame, TranscriptionMessage +from pipecat.frames.frames import ( + LLMRunFrame, + LLMSetToolsFrame, + LLMUpdateSettingsFrame, + TranscriptionMessage, +) from pipecat.observers.loggers.transcription_log_observer import TranscriptionLogObserver from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -148,6 +154,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): noise_reduction=InputAudioNoiseReduction(type="near_field"), ) ), + # In this example we provide tools through the context, but you could + # alternatively provide them here. # tools=tools, instructions="""You are a helpful and friendly AI. @@ -223,6 +231,15 @@ Remember, your responses should be short. Just one or two sentences, usually. Re standard_tools=[weather_function, restaurant_function, get_news_function] ) await task.queue_frames([LLMSetToolsFrame(tools=new_tools)]) + # Alternative pattern, useful if you're changing other session properties, too. + # (Though note that tools in your LLMContext take precedence over those + # in session properties, so if you have context-provided tools, prefer + # LLMSetToolsFrame instead, as it updates your context. Ditto for + # updating system instructions: send an LLMMessagesUpdateFrame with + # context messages updated with your new desired system message.) + # await task.queue_frames( + # [LLMUpdateSettingsFrame(settings=SessionProperties(tools=new_tools).model_dump())] + # ) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): diff --git a/src/pipecat/services/openai/realtime/events.py b/src/pipecat/services/openai/realtime/events.py index 3ca2f76c0..77886043f 100644 --- a/src/pipecat/services/openai/realtime/events.py +++ b/src/pipecat/services/openai/realtime/events.py @@ -12,6 +12,8 @@ from typing import Any, Dict, List, Literal, Optional, Union from pydantic import BaseModel, ConfigDict, Field +from pipecat.adapters.schemas.tools_schema import ToolsSchema + # # session properties # @@ -184,6 +186,9 @@ class SessionProperties(BaseModel): include: Additional fields to include in server outputs. """ + # Needed to support ToolSchema in tools field. + model_config = ConfigDict(arbitrary_types_allowed=True) + type: Optional[Literal["realtime"]] = "realtime" object: Optional[Literal["realtime.session"]] = None id: Optional[str] = None @@ -191,7 +196,10 @@ class SessionProperties(BaseModel): output_modalities: Optional[List[Literal["text", "audio"]]] = None instructions: Optional[str] = None audio: Optional[AudioConfiguration] = None - tools: Optional[List[Dict]] = None + # Tools can only be ToolsSchema when provided by the user, in either the + # OpenAIRealtimeLLMService constructor or through LLMUpdateSettingsFrame. + # We'll never serialize/deserialize ToolsSchema when talking to the server. + tools: Optional[ToolsSchema | List[Dict]] = None tool_choice: Optional[Literal["auto", "none", "required"]] = None max_output_tokens: Optional[Union[int, Literal["inf"]]] = None tracing: Optional[Union[Literal["auto"], Dict]] = None diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 6d3604541..0129a94f9 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -14,6 +14,7 @@ from typing import Optional from loguru import logger +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.services.open_ai_realtime_adapter import ( OpenAIRealtimeLLMAdapter, ) @@ -481,9 +482,9 @@ class OpenAIRealtimeLLMService(LLMService): async def _update_settings(self): settings = self._session_properties + adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter() if self._context: - adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter() llm_invocation_params = adapter.get_llm_invocation_params(self._context) # tools given in the context override the tools in the session properties @@ -495,6 +496,12 @@ class OpenAIRealtimeLLMService(LLMService): if llm_invocation_params["system_instruction"]: settings.instructions = llm_invocation_params["system_instruction"] + # If needed, map settings.tools from ToolsSchema to list of dicts, + # which remote server expects. It would only be a ToolsSchema if that's + # how it was provided in the constructor or via LLMUpdateSettingsFrame. + if settings.tools and isinstance(settings.tools, ToolsSchema): + settings.tools = adapter.from_standard_tools(settings.tools) + await self.send_client_event(events.SessionUpdateEvent(session=settings)) #