Add to the Gemini Live function calling example runtime tool addition with LLMSetToolsFrame

examples/foundational/19-openai-realtime.py

@@ -57,6 +57,10 @@ async def fetch_weather_from_api(params: FunctionCallParams):
     )
 
 
+async def fetch_restaurant_recommendation(params: FunctionCallParams):
+    await params.result_callback({"name": "The Golden Dragon"})
+
+
 async def get_news(params: FunctionCallParams):
     await params.result_callback(
         {
@@ -69,10 +73,6 @@ async def get_news(params: FunctionCallParams):
     )
 
 
-async def fetch_restaurant_recommendation(params: FunctionCallParams):
-    await params.result_callback({"name": "The Golden Dragon"})
-
-
 weather_function = FunctionSchema(
     name="get_current_weather",
     description="Get the current weather",
@@ -90,13 +90,6 @@ weather_function = FunctionSchema(
     required=["location", "format"],
 )
 
-get_news_function = FunctionSchema(
-    name="get_news",
-    description="Get the current news.",
-    properties={},
-    required=[],
-)
-
 restaurant_function = FunctionSchema(
     name="get_restaurant_recommendation",
     description="Get a restaurant recommendation",
@@ -109,6 +102,13 @@ restaurant_function = FunctionSchema(
     required=["location"],
 )
 
+get_news_function = FunctionSchema(
+    name="get_news",
+    description="Get the current news.",
+    properties={},
+    required=[],
+)
+
 # Create tools schema
 tools = ToolsSchema(standard_tools=[weather_function, restaurant_function])
 
@@ -215,8 +215,9 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
         # Kick off the conversation.
         await task.queue_frames([LLMRunFrame()])
 
-        # Add a new tool at runtime after a delay.
+        # Add a new tool (get_news) at runtime after a delay
         await asyncio.sleep(15)
+        logger.info(f"Adding new tool get_news at runtime...")
         new_tools = ToolsSchema(
             standard_tools=[weather_function, restaurant_function, get_news_function]
         )

examples/foundational/26b-gemini-live-function-calling.py

@@ -5,6 +5,7 @@
 #
 
 
+import asyncio
 import os
 from datetime import datetime
 
@@ -15,7 +16,7 @@ from pipecat.adapters.schemas.function_schema import FunctionSchema
 from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
 from pipecat.audio.vad.silero import SileroVADAnalyzer
 from pipecat.audio.vad.vad_analyzer import VADParams
-from pipecat.frames.frames import LLMRunFrame
+from pipecat.frames.frames import LLMRunFrame, LLMSetToolsFrame
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -51,6 +52,18 @@ async def fetch_restaurant_recommendation(params: FunctionCallParams):
     await params.result_callback({"name": "The Golden Dragon"})
 
 
+async def get_news(params: FunctionCallParams):
+    await params.result_callback(
+        {
+            "news": [
+                "Massive UFO currently hovering above New York City",
+                "Stock markets reach all-time highs",
+                "Living dinosaur species discovered in the Amazon rainforest",
+            ],
+        }
+    )
+
+
 system_instruction = """
 You are a helpful assistant who can answer questions and use tools.
 
@@ -109,6 +122,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
         },
         required=["location"],
     )
+    get_news_function = FunctionSchema(
+        name="get_news",
+        description="Get the current news.",
+        properties={},
+        required=[],
+    )
     search_tool = {"google_search": {}}
     # KNOWN ISSUE: If using GeminiVertexLiveLLMService, it appears
     # you cannot use the "google_search" tool alongside other tools.
@@ -126,6 +145,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
 
     llm.register_function("get_current_weather", fetch_weather_from_api)
     llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation)
+    llm.register_function("get_news", get_news)
 
     # You can provide the system instructions and tools in the context rather
     # than as arguments to GeminiLiveLLMService, but note that doing so will
@@ -174,6 +194,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
         # Kick off the conversation.
         await task.queue_frames([LLMRunFrame()])
 
+        # Add a new tool (get_news) at runtime after a delay
+        await asyncio.sleep(15)
+        logger.info(f"Adding new tool get_news at runtime...")
+        new_tools = ToolsSchema(
+            standard_tools=[weather_function, restaurant_function, get_news_function]
+        )
+        await task.queue_frames([LLMSetToolsFrame(tools=new_tools)])
+
     @transport.event_handler("on_client_disconnected")
     async def on_client_disconnected(transport, client):
         logger.info(f"Client disconnected")

src/pipecat/processors/aggregators/llm_response_universal.py

@@ -472,11 +472,13 @@ class LLMUserAggregator(LLMContextAggregator):
             await self._handle_llm_messages_update(frame)
         elif isinstance(frame, LLMSetToolsFrame):
             self.set_tools(frame.tools)
-            # Push the LLMSetToolsFrame as well, since speech-to-speech LLM
+            # Push the LLMSetToolsFrame as well, since some realtime (aka
-            # services (like OpenAI Realtime) may need to know about tool
+            # speech-to-speech) LLM services (like OpenAI Realtime) may need to
-            # changes; unlike text-based LLM services they won't just "pick up
+            # be directly be informed of tool changes; unlike text-based LLM
-            # the change" on the next LLM run, as the LLM is continuously
+            # services they can't necessarily rely on "picking up the change"
-            # running.
+            # from the context on the next LLM run, as the LLM is continuously
+            # running and they may need to apply the change sooner than the
+            # next context frame.
             await self.push_frame(frame, direction)
         elif isinstance(frame, LLMSetToolChoiceFrame):
             self.set_tool_choice(frame.tool_choice)

src/pipecat/services/google/gemini_live/llm.py

@@ -965,12 +965,10 @@ class GeminiLiveLLMService(LLMService):
         elif isinstance(frame, LLMUpdateSettingsFrame):
             await self._update_settings(frame.settings)
         elif isinstance(frame, LLMSetToolsFrame):
-            # TODO: you are here - setting tools doesn't work yet (requires reconnection)
+            # We actually don't need to do anything here; the next time we get
-            # Do we have reference to previous tools to compare?
+            # a context frame (like after a user transcription is appended),
-            # New tools should already have been set on the context by the user aggregator.
+            # we'll detect that tools have changed and reconnect then to apply
-            # LLMSetToolsFrame without a user aggregator is not supported.
+            # the new tools.
-            # If tools have changed, update context snapshot.
-            # await self._update_settings()
             pass
         else:
             await self.push_frame(frame, direction)