From af9462083914e90b9d8c385089214226a1055b57 Mon Sep 17 00:00:00 2001 From: Varun Singh <382354+vr000m@users.noreply.github.com> Date: Wed, 6 Aug 2025 23:14:47 -0700 Subject: [PATCH 1/2] Add OpenAI function calling example with Pipecat Introduces a new example script demonstrating how to use OpenAI's function calling capabilities within a Pipecat pipeline. The example integrates OpenAI STT, TTS, and LLM services, registers a weather function, and sets up a pipeline for real-time audio interaction over WebRTC. --- .../14v-function-calling-openai.py | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 examples/foundational/14v-function-calling-openai.py diff --git a/examples/foundational/14v-function-calling-openai.py b/examples/foundational/14v-function-calling-openai.py new file mode 100644 index 000000000..f9ad46d66 --- /dev/null +++ b/examples/foundational/14v-function-calling-openai.py @@ -0,0 +1,176 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.openai.stt import OpenAISTTService +from pipecat.services.openai.tts import OpenAITTSService +from pipecat.services.llm_service import FunctionCallParams +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams + +load_dotenv(override=True) + + +async def fetch_weather_from_api(params: FunctionCallParams): + await params.result_callback({"conditions": "nice", "temperature": "75"}) + + +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = OpenAISTTService( + api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4o-transcribe", + prompt="Expect words related weather, such as temperature and conditions. And restaurant names.", + ) + + # voice choices: ash, ballad, or any other voice available in the OpenAI TTS API + # see https://www.openai.fm/ + tts = OpenAITTSService( + api_key=os.getenv("OPENAI_API_KEY"), + voice="ballad", + instructions="Please speak clearly and at a moderate pace.", + ) + + # model choices: gpt-4o, gpt-4.1, etc. + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + # You can also register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) + + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() From 69541c883501e58b2c90281d914491a5ae04837e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 7 Aug 2025 13:19:01 -0400 Subject: [PATCH 2/2] Linting fix, plus update eval suite with 14v and others, tiny fix for 14m, too --- examples/foundational/14m-function-calling-openrouter.py | 4 ++-- examples/foundational/14v-function-calling-openai.py | 4 ++-- scripts/evals/run-release-evals.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index b5d62506d..719926275 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -63,8 +63,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = AzureTTSService( - api_key=os.getenv("AZURE_API_KEY"), - region="eastus", + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION"), voice="en-US-JennyNeural", params=AzureTTSService.InputParams(language="en-US", rate="1.1", style="cheerful"), ) diff --git a/examples/foundational/14v-function-calling-openai.py b/examples/foundational/14v-function-calling-openai.py index f9ad46d66..56a18353b 100644 --- a/examples/foundational/14v-function-calling-openai.py +++ b/examples/foundational/14v-function-calling-openai.py @@ -19,10 +19,10 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.openai.stt import OpenAISTTService -from pipecat.services.openai.tts import OpenAITTSService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService +from pipecat.services.openai.stt import OpenAISTTService +from pipecat.services.openai.tts import OpenAITTSService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams from pipecat.transports.services.daily import DailyParams diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 5353de9b2..ef47f2299 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -84,16 +84,16 @@ TESTS_14 = [ ("14h-function-calling-azure.py", PROMPT_WEATHER, EVAL_WEATHER), ("14i-function-calling-fireworks.py", PROMPT_WEATHER, EVAL_WEATHER), ("14j-function-calling-nim.py", PROMPT_WEATHER, EVAL_WEATHER), + ("14m-function-calling-openrouter.py", PROMPT_WEATHER, EVAL_WEATHER), ("14n-function-calling-perplexity.py", PROMPT_WEATHER, EVAL_WEATHER), ("14p-function-calling-gemini-vertex-ai.py", PROMPT_WEATHER, EVAL_WEATHER), ("14q-function-calling-qwen.py", PROMPT_WEATHER, EVAL_WEATHER), ("14r-function-calling-aws.py", PROMPT_WEATHER, EVAL_WEATHER), + ("14v-function-calling-openai.py.py", PROMPT_WEATHER, EVAL_WEATHER), # Currently not working. # ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER), - # ("14j-function-calling-nim.py", PROMPT_WEATHER, EVAL_WEATHER), # ("14k-function-calling-cerebras.py", PROMPT_WEATHER, EVAL_WEATHER), # ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER), - # ("14m-function-calling-openrouter.py", PROMPT_WEATHER, EVAL_WEATHER), # ("14o-function-calling-gemini-openai-format.py", PROMPT_WEATHER, EVAL_WEATHER), ]