diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index 181cd576b..ccbe952fb 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -39,6 +39,10 @@ async def get_weather(params: FunctionCallParams): await params.result_callback(f"The weather in {location} is currently 72 degrees and sunny.") +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + async def get_image(params: FunctionCallParams): question = params.arguments["question"] logger.debug(f"Requesting image with user_id={client_id}, question={question}") @@ -92,6 +96,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") llm.register_function("get_weather", get_weather) llm.register_function("get_image", get_image) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) @llm.event_handler("on_function_calls_started") async def on_function_calls_started(service, function_calls): @@ -113,6 +118,17 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si }, required=["location", "format"], ) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) get_image_function = FunctionSchema( name="get_image", description="Get an image from the video stream.", @@ -124,14 +140,14 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si }, required=["question"], ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) + tools = ToolsSchema(standard_tools=[weather_function, get_image_function, restaurant_function]) system_prompt = """\ You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. Your response will be turned into speech so use only simple words and punctuation. -You have access to two tools: get_weather and get_image. +You have access to three tools: get_weather, get_restaurant_recommendation, and get_image. You can respond to questions about the weather using the get_weather tool. diff --git a/examples/foundational/14r-function-calling-aws.py b/examples/foundational/14r-function-calling-aws.py index f7796157a..4443bec27 100644 --- a/examples/foundational/14r-function-calling-aws.py +++ b/examples/foundational/14r-function-calling-aws.py @@ -32,6 +32,10 @@ async def fetch_weather_from_api(params: FunctionCallParams): await params.result_callback({"conditions": "nice", "temperature": "75"}) +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -74,6 +78,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) weather_function = FunctionSchema( name="get_current_weather", @@ -91,7 +96,18 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si }, required=["location", "format"], ) - tools = ToolsSchema(standard_tools=[weather_function]) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) messages = [ { diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index f1dd7269d..81e8dc6c3 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -45,6 +45,10 @@ async def fetch_weather_from_api(params: FunctionCallParams): ) +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -62,8 +66,20 @@ weather_function = FunctionSchema( required=["location", "format"], ) +restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], +) + # Create tools schema -tools = ToolsSchema(standard_tools=[weather_function]) +tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -100,7 +116,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si # turn_detection=False, input_audio_noise_reduction=InputAudioNoiseReduction(type="near_field"), # tools=tools, - instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. + instructions="""You are a helpful and friendly AI. Act like a human, but remember that you aren't a human and that you can't do human things in the real world. Your voice and personality should be warm and engaging, with a lively and @@ -113,6 +129,10 @@ even if you're asked about them. You are participating in a voice conversation. Keep your responses concise, short, and to the point unless specifically asked to elaborate on a topic. +You have access to the following tools: +- get_current_weather: Get the current weather for a given location. +- get_restaurant_recommendation: Get a restaurant recommendation for a given location. + Remember, your responses should be short. Just one or two sentences, usually.""", ) @@ -125,6 +145,7 @@ Remember, your responses should be short. Just one or two sentences, usually.""" # you can either register a single function for all function calls, or specific functions # llm.register_function(None, fetch_weather_from_api) llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) # Create a standard OpenAI LLM context object using the normal messages format. The # OpenAIRealtimeBetaLLMService will convert this internally to messages that the diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 76180e397..54f0302e5 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -43,6 +43,10 @@ async def fetch_weather_from_api(params: FunctionCallParams): ) +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + # Define weather function using standardized schema weather_function = FunctionSchema( name="get_current_weather", @@ -61,8 +65,20 @@ weather_function = FunctionSchema( required=["location", "format"], ) +restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], +) + # Create tools schema -tools = ToolsSchema(standard_tools=[weather_function]) +tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -98,7 +114,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si # Or set to False to disable openai turn detection and use transport VAD # turn_detection=False, # tools=tools, - instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. + instructions="""You are a helpful and friendly AI. Act like a human, but remember that you aren't a human and that you can't do human things in the real world. Your voice and personality should be warm and engaging, with a lively and @@ -111,6 +127,10 @@ even if you're asked about them. You are participating in a voice conversation. Keep your responses concise, short, and to the point unless specifically asked to elaborate on a topic. +You have access to the following tools: +- get_current_weather: Get the current weather for a given location. +- get_restaurant_recommendation: Get a restaurant recommendation for a given location. + Remember, your responses should be short. Just one or two sentences, usually.""", ) @@ -124,6 +144,7 @@ Remember, your responses should be short. Just one or two sentences, usually.""" # you can either register a single function for all function calls, or specific functions # llm.register_function(None, fetch_weather_from_api) llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) # Create a standard OpenAI LLM context object using the normal messages format. The # OpenAIRealtimeBetaLLMService will convert this internally to messages that the diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-multimodal-live-function-calling.py index 84c7d5045..15db2faa0 100644 --- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py +++ b/examples/foundational/26b-gemini-multimodal-live-function-calling.py @@ -40,11 +40,17 @@ async def fetch_weather_from_api(params: FunctionCallParams): ) +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + system_instruction = """ You are a helpful assistant who can answer questions and use tools. -You have a tool called "get_current_weather" that can be used to get the current weather. If the user asks -for the weather, call this function. +You have three tools available to you: +1. get_current_weather: Use this tool to get the current weather in a specific location. +2. get_restaurant_recommendation: Use this tool to get a restaurant recommendation in a specific location. +3. google_search: Use this tool to search the web for information. """ @@ -101,9 +107,21 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si }, required=["location", "format"], ) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) search_tool = {"google_search": {}} tools = ToolsSchema( - standard_tools=[weather_function], custom_tools={AdapterType.GEMINI: [search_tool]} + standard_tools=[weather_function, restaurant_function], + custom_tools={AdapterType.GEMINI: [search_tool]}, ) llm = GeminiMultimodalLiveLLMService( @@ -113,6 +131,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si ) llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) context = OpenAILLMContext( [{"role": "user", "content": "Say hello."}],