diff --git a/CHANGELOG.md b/CHANGELOG.md index 052305702..812c47d71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Improved foundational examples 22b, 22c, and 22d to support function calling. + With these base examples, `FunctionCallInProgressFrame` and + `FunctionCallResultFrame` will no longer be blocked by the gates. + ### Fixed - Fixed a `SentryMetrics` issue that was preventing any metrics to be sent to diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 57639fe20..a3c373eac 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -12,6 +12,7 @@ import time import aiohttp from dotenv import load_dotenv from loguru import logger +from openai.types.chat import ChatCompletionToolParam from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -19,6 +20,8 @@ from pipecat.frames.frames import ( CancelFrame, EndFrame, Frame, + FunctionCallInProgressFrame, + FunctionCallResultFrame, LLMMessagesFrame, StartFrame, StartInterruptionFrame, @@ -26,6 +29,7 @@ from pipecat.frames.frames import ( SystemFrame, TextFrame, TranscriptionFrame, + TTSSpeakFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -156,6 +160,11 @@ class OutputGate(FrameProcessor): await self.push_frame(frame, direction) return + # Don't block function call frames + if isinstance(frame, (FunctionCallInProgressFrame, FunctionCallResultFrame)): + await self.push_frame(frame, direction) + return + # Ignore frames that are not following the direction of this gate. if direction != FrameDirection.DOWNSTREAM: await self.push_frame(frame, direction) @@ -186,6 +195,16 @@ class OutputGate(FrameProcessor): break +async def start_fetch_weather(function_name, llm, context): + """Push a frame to the LLM; this is handy when the LLM response might take a while.""" + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) + logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await result_callback({"conditions": "nice", "temperature": "75"}) + + async def main(): async with aiohttp.ClientSession() as session: (room_url, _) = await configure(session) @@ -216,6 +235,34 @@ async def main(): # This is the regular LLM. llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + # Register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + "required": ["location", "format"], + }, + }, + ) + ] messages = [ { @@ -224,7 +271,7 @@ async def main(): }, ] - context = OpenAILLMContext(messages) + context = OpenAILLMContext(messages, tools) context_aggregator = llm.create_context_aggregator(context) # We have instructed the LLM to return 'YES' if it thinks the user @@ -265,6 +312,8 @@ async def main(): or isinstance(frame, LLMMessagesFrame) or isinstance(frame, StartInterruptionFrame) or isinstance(frame, StopInterruptionFrame) + or isinstance(frame, FunctionCallInProgressFrame) + or isinstance(frame, FunctionCallResultFrame) ) pipeline = Pipeline( diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 1a24af21c..92f47ae92 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -12,6 +12,7 @@ import time import aiohttp from dotenv import load_dotenv from loguru import logger +from openai.types.chat import ChatCompletionToolParam from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -19,6 +20,8 @@ from pipecat.frames.frames import ( CancelFrame, EndFrame, Frame, + FunctionCallInProgressFrame, + FunctionCallResultFrame, LLMMessagesFrame, StartFrame, StartInterruptionFrame, @@ -26,6 +29,7 @@ from pipecat.frames.frames import ( SystemFrame, TextFrame, TranscriptionFrame, + TTSSpeakFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -360,6 +364,11 @@ class OutputGate(FrameProcessor): await self.push_frame(frame, direction) return + # Don't block function call frames + if isinstance(frame, (FunctionCallInProgressFrame, FunctionCallResultFrame)): + await self.push_frame(frame, direction) + return + # Ignore frames that are not following the direction of this gate. if direction != FrameDirection.DOWNSTREAM: await self.push_frame(frame, direction) @@ -390,6 +399,16 @@ class OutputGate(FrameProcessor): break +async def start_fetch_weather(function_name, llm, context): + """Push a frame to the LLM; this is handy when the LLM response might take a while.""" + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) + logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await result_callback({"conditions": "nice", "temperature": "75"}) + + async def main(): async with aiohttp.ClientSession() as session: (room_url, _) = await configure(session) @@ -426,6 +445,34 @@ async def main(): api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o", ) + # Register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + "required": ["location", "format"], + }, + }, + ) + ] messages = [ { @@ -434,7 +481,7 @@ async def main(): }, ] - context = OpenAILLMContext(messages) + context = OpenAILLMContext(messages, tools) context_aggregator = llm.create_context_aggregator(context) # We have instructed the LLM to return 'YES' if it thinks the user @@ -474,6 +521,8 @@ async def main(): or isinstance(frame, LLMMessagesFrame) or isinstance(frame, StartInterruptionFrame) or isinstance(frame, StopInterruptionFrame) + or isinstance(frame, FunctionCallInProgressFrame) + or isinstance(frame, FunctionCallResultFrame) ) pipeline = Pipeline( diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 6829f8370..860f1f574 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -20,6 +20,8 @@ from pipecat.frames.frames import ( CancelFrame, EndFrame, Frame, + FunctionCallInProgressFrame, + FunctionCallResultFrame, InputAudioRawFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, @@ -575,6 +577,11 @@ class OutputGate(FrameProcessor): await self.push_frame(frame, direction) return + # Don't block function call frames + if isinstance(frame, (FunctionCallInProgressFrame, FunctionCallResultFrame)): + await self.push_frame(frame, direction) + return + # Ignore frames that are not following the direction of this gate. if direction != FrameDirection.DOWNSTREAM: await self.push_frame(frame, direction) @@ -672,12 +679,6 @@ async def main(): context = OpenAILLMContext() context_aggregator = conversation_llm.create_context_aggregator(context) - # We have instructed the LLM to return 'True' if it thinks the user - # completed a sentence. So, if it's 'True' we will return true in this - # predicate which will wake up the notifier. - async def wake_check_filter(frame): - return frame.text == "True" - # This is a notifier that we use to synchronize the two LLMs. notifier = EventNotifier() @@ -694,14 +695,6 @@ async def main(): async def block_user_stopped_speaking(frame): return not isinstance(frame, UserStoppedSpeakingFrame) - async def pass_only_llm_trigger_frames(frame): - return ( - isinstance(frame, OpenAILLMContextFrame) - or isinstance(frame, LLMMessagesFrame) - or isinstance(frame, StartInterruptionFrame) - or isinstance(frame, StopInterruptionFrame) - ) - conversation_audio_context_assembler = ConversationAudioContextAssembler(context=context) user_aggregator_buffer = UserAggregatorBuffer()