wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add to the Gemini Live function calling example runtime tool addition with LLMSetToolsFrame

Browse Source
This commit is contained in:
Paul Kompfner
2026-02-04 15:05:09 -05:00
parent ba3100be0d
commit 2aef572e38
4 changed files with 53 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
examples/foundational/19-openai-realtime.py
View File

@@ -57,6 +57,10 @@ async def fetch_weather_from_api(params: FunctionCallParams):
)
async def fetch_restaurant_recommendation(params: FunctionCallParams):
await params.result_callback({"name": "The Golden Dragon"})
async def get_news(params: FunctionCallParams):
await params.result_callback(
{
@@ -69,10 +73,6 @@ async def get_news(params: FunctionCallParams):
)
async def fetch_restaurant_recommendation(params: FunctionCallParams):
await params.result_callback({"name": "The Golden Dragon"})
weather_function = FunctionSchema(
name="get_current_weather",
description="Get the current weather",
@@ -90,13 +90,6 @@ weather_function = FunctionSchema(
required=["location", "format"],
)
get_news_function = FunctionSchema(
name="get_news",
description="Get the current news.",
properties={},
required=[],
)
restaurant_function = FunctionSchema(
name="get_restaurant_recommendation",
description="Get a restaurant recommendation",
@@ -109,6 +102,13 @@ restaurant_function = FunctionSchema(
required=["location"],
)
get_news_function = FunctionSchema(
name="get_news",
description="Get the current news.",
properties={},
required=[],
)
# Create tools schema
tools = ToolsSchema(standard_tools=[weather_function, restaurant_function])
@@ -215,8 +215,9 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
# Kick off the conversation.
await task.queue_frames([LLMRunFrame()])
# Add a new tool at runtime after a delay.
# Add a new tool (get_news) at runtime after a delay
await asyncio.sleep(15)
logger.info(f"Adding new tool get_news at runtime...")
new_tools = ToolsSchema(
standard_tools=[weather_function, restaurant_function, get_news_function]
)

30
examples/foundational/26b-gemini-live-function-calling.py
View File

@@ -5,6 +5,7 @@
#
import asyncio
import os
from datetime import datetime
@@ -15,7 +16,7 @@ from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.frames.frames import LLMRunFrame
from pipecat.frames.frames import LLMRunFrame, LLMSetToolsFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -51,6 +52,18 @@ async def fetch_restaurant_recommendation(params: FunctionCallParams):
await params.result_callback({"name": "The Golden Dragon"})
async def get_news(params: FunctionCallParams):
await params.result_callback(
{
"news": [
"Massive UFO currently hovering above New York City",
"Stock markets reach all-time highs",
"Living dinosaur species discovered in the Amazon rainforest",
],
}
)
system_instruction = """
You are a helpful assistant who can answer questions and use tools.
@@ -109,6 +122,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
},
required=["location"],
)
get_news_function = FunctionSchema(
name="get_news",
description="Get the current news.",
properties={},
required=[],
)
search_tool = {"google_search": {}}
# KNOWN ISSUE: If using GeminiVertexLiveLLMService, it appears
# you cannot use the "google_search" tool alongside other tools.
@@ -126,6 +145,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
llm.register_function("get_current_weather", fetch_weather_from_api)
llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation)
llm.register_function("get_news", get_news)
# You can provide the system instructions and tools in the context rather
# than as arguments to GeminiLiveLLMService, but note that doing so will
@@ -174,6 +194,14 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
# Kick off the conversation.
await task.queue_frames([LLMRunFrame()])
# Add a new tool (get_news) at runtime after a delay
await asyncio.sleep(15)
logger.info(f"Adding new tool get_news at runtime...")
new_tools = ToolsSchema(
standard_tools=[weather_function, restaurant_function, get_news_function]
)
await task.queue_frames([LLMSetToolsFrame(tools=new_tools)])
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")

12
src/pipecat/processors/aggregators/llm_response_universal.py
View File

@@ -472,11 +472,13 @@ class LLMUserAggregator(LLMContextAggregator):
await self._handle_llm_messages_update(frame)
elif isinstance(frame, LLMSetToolsFrame):
self.set_tools(frame.tools)
# Push the LLMSetToolsFrame as well, since speech-to-speech LLM
# services (like OpenAI Realtime) may need to know about tool
# changes; unlike text-based LLM services they won't just "pick up
# the change" on the next LLM run, as the LLM is continuously
# running.
# Push the LLMSetToolsFrame as well, since some realtime (aka
# speech-to-speech) LLM services (like OpenAI Realtime) may need to
# be directly be informed of tool changes; unlike text-based LLM
# services they can't necessarily rely on "picking up the change"
# from the context on the next LLM run, as the LLM is continuously
# running and they may need to apply the change sooner than the
# next context frame.
await self.push_frame(frame, direction)
elif isinstance(frame, LLMSetToolChoiceFrame):
self.set_tool_choice(frame.tool_choice)

10
src/pipecat/services/google/gemini_live/llm.py
View File

@@ -965,12 +965,10 @@ class GeminiLiveLLMService(LLMService):
elif isinstance(frame, LLMUpdateSettingsFrame):
await self._update_settings(frame.settings)
elif isinstance(frame, LLMSetToolsFrame):
# TODO: you are here - setting tools doesn't work yet (requires reconnection)
# Do we have reference to previous tools to compare?
# New tools should already have been set on the context by the user aggregator.
# LLMSetToolsFrame without a user aggregator is not supported.
# If tools have changed, update context snapshot.
# await self._update_settings()
# We actually don't need to do anything here; the next time we get
# a context frame (like after a user transcription is appended),
# we'll detect that tools have changed and reconnect then to apply
# the new tools.
pass
else:
await self.push_frame(frame, direction)