diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 0d59782f1..c85f6bb12 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -57,6 +57,10 @@ async def fetch_weather_from_api(params: FunctionCallParams): ) +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + async def get_news(params: FunctionCallParams): await params.result_callback( { @@ -69,10 +73,6 @@ async def get_news(params: FunctionCallParams): ) -async def fetch_restaurant_recommendation(params: FunctionCallParams): - await params.result_callback({"name": "The Golden Dragon"}) - - weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -90,13 +90,6 @@ weather_function = FunctionSchema( required=["location", "format"], ) -get_news_function = FunctionSchema( - name="get_news", - description="Get the current news.", - properties={}, - required=[], -) - restaurant_function = FunctionSchema( name="get_restaurant_recommendation", description="Get a restaurant recommendation", @@ -109,6 +102,13 @@ restaurant_function = FunctionSchema( required=["location"], ) +get_news_function = FunctionSchema( + name="get_news", + description="Get the current news.", + properties={}, + required=[], +) + # Create tools schema tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) @@ -215,8 +215,9 @@ Remember, your responses should be short. Just one or two sentences, usually. Re # Kick off the conversation. await task.queue_frames([LLMRunFrame()]) - # Add a new tool at runtime after a delay. + # Add a new tool (get_news) at runtime after a delay await asyncio.sleep(15) + logger.info(f"Adding new tool get_news at runtime...") new_tools = ToolsSchema( standard_tools=[weather_function, restaurant_function, get_news_function] ) diff --git a/examples/foundational/26b-gemini-live-function-calling.py b/examples/foundational/26b-gemini-live-function-calling.py index 7aea43036..a66925003 100644 --- a/examples/foundational/26b-gemini-live-function-calling.py +++ b/examples/foundational/26b-gemini-live-function-calling.py @@ -5,6 +5,7 @@ # +import asyncio import os from datetime import datetime @@ -15,7 +16,7 @@ from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame +from pipecat.frames.frames import LLMRunFrame, LLMSetToolsFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -51,6 +52,18 @@ async def fetch_restaurant_recommendation(params: FunctionCallParams): await params.result_callback({"name": "The Golden Dragon"}) +async def get_news(params: FunctionCallParams): + await params.result_callback( + { + "news": [ + "Massive UFO currently hovering above New York City", + "Stock markets reach all-time highs", + "Living dinosaur species discovered in the Amazon rainforest", + ], + } + ) + + system_instruction = """ You are a helpful assistant who can answer questions and use tools. @@ -109,6 +122,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, required=["location"], ) + get_news_function = FunctionSchema( + name="get_news", + description="Get the current news.", + properties={}, + required=[], + ) search_tool = {"google_search": {}} # KNOWN ISSUE: If using GeminiVertexLiveLLMService, it appears # you cannot use the "google_search" tool alongside other tools. @@ -126,6 +145,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm.register_function("get_current_weather", fetch_weather_from_api) llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) + llm.register_function("get_news", get_news) # You can provide the system instructions and tools in the context rather # than as arguments to GeminiLiveLLMService, but note that doing so will @@ -174,6 +194,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Kick off the conversation. await task.queue_frames([LLMRunFrame()]) + # Add a new tool (get_news) at runtime after a delay + await asyncio.sleep(15) + logger.info(f"Adding new tool get_news at runtime...") + new_tools = ToolsSchema( + standard_tools=[weather_function, restaurant_function, get_news_function] + ) + await task.queue_frames([LLMSetToolsFrame(tools=new_tools)]) + @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 871b1cbca..58e81d269 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -472,11 +472,13 @@ class LLMUserAggregator(LLMContextAggregator): await self._handle_llm_messages_update(frame) elif isinstance(frame, LLMSetToolsFrame): self.set_tools(frame.tools) - # Push the LLMSetToolsFrame as well, since speech-to-speech LLM - # services (like OpenAI Realtime) may need to know about tool - # changes; unlike text-based LLM services they won't just "pick up - # the change" on the next LLM run, as the LLM is continuously - # running. + # Push the LLMSetToolsFrame as well, since some realtime (aka + # speech-to-speech) LLM services (like OpenAI Realtime) may need to + # be directly be informed of tool changes; unlike text-based LLM + # services they can't necessarily rely on "picking up the change" + # from the context on the next LLM run, as the LLM is continuously + # running and they may need to apply the change sooner than the + # next context frame. await self.push_frame(frame, direction) elif isinstance(frame, LLMSetToolChoiceFrame): self.set_tool_choice(frame.tool_choice) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index ea60945c5..63dce5c1a 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -965,12 +965,10 @@ class GeminiLiveLLMService(LLMService): elif isinstance(frame, LLMUpdateSettingsFrame): await self._update_settings(frame.settings) elif isinstance(frame, LLMSetToolsFrame): - # TODO: you are here - setting tools doesn't work yet (requires reconnection) - # Do we have reference to previous tools to compare? - # New tools should already have been set on the context by the user aggregator. - # LLMSetToolsFrame without a user aggregator is not supported. - # If tools have changed, update context snapshot. - # await self._update_settings() + # We actually don't need to do anything here; the next time we get + # a context frame (like after a user transcription is appended), + # we'll detect that tools have changed and reconnect then to apply + # the new tools. pass else: await self.push_frame(frame, direction)