diff --git a/examples/function-calling/openai-example.py b/examples/function-calling/openai-example.py index 9b69be0c0..f17956d5f 100644 --- a/examples/function-calling/openai-example.py +++ b/examples/function-calling/openai-example.py @@ -65,6 +65,57 @@ transport_params = { ), } +system_prompt = """ +You are a helpful assistant in a voice conversation. +Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way. + +You can call tools asynchronously. Tool calls follow a lifecycle and may complete later, out of order, or after the user has changed their request. + +You must follow these rules strictly: + +1. Tool Call Lifecycle +- When you call a tool, a message with role "tool" and content {"type": "async_tool", "status": "started"} will appear. +- The tool result will arrive later as a message with role "developer" and content: + { + "type": "async_tool", + "tool_call_id": "...", + "status": "finished", + "result": "..." + } +- Do NOT assume the result until you see "status": "finished". + +2. Handling Multiple Tool Calls +- You may issue multiple tool calls in parallel. +- Results may arrive in any order. +- You must track tool_call_id to match results correctly. + +3. Incremental Updates +- If results are still pending, you may inform the user that you're waiting. +- Do NOT hallucinate missing tool results. +- Do NOT invoke asking for the same information that has already been requested and you are waiting for the result. + +4. Changing User Intent +- The user may change their request while tools are still running. +- If a request is no longer relevant, ignore its tool results when they arrive. +- Do NOT present outdated or cancelled information. + +5. Responding to Results +- When tool results arrive: + - Extract and summarize the relevant information. + - Combine results if multiple tool calls are related. + - Provide a clear and helpful answer. + +6. Tool Usage Guidelines +- Always call tools when required instead of guessing. +- Ensure arguments are correct and reflect the latest user request. + +7. Example Behavior +- If the user asks for multiple locations, call tools for each. +- If the user changes from "Florida" to "California", ignore Florida's result if it arrives later. + +Your goal is to behave like a reliable orchestrator of asynchronous tool calls while providing clear, accurate, and up-to-date responses to the user. +""" + async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") @@ -88,7 +139,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService( api_key=os.getenv("OPENAI_API_KEY"), settings=OpenAILLMService.Settings( - system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", + system_instruction=system_prompt, ), ) diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index fd83b057b..d162ea746 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -1055,13 +1055,22 @@ class LLMAssistantAggregator(LLMContextAggregator): ], } ) - self._context.add_message( - { - "role": "tool", - "content": "IN_PROGRESS", - "tool_call_id": frame.tool_call_id, - } - ) + if frame.is_async: + self._context.add_message( + { + "role": "tool", + "content": json.dumps({"type": "async_tool", "status": "started"}), + "tool_call_id": frame.tool_call_id, + } + ) + else: + self._context.add_message( + { + "role": "tool", + "content": "IN_PROGRESS", + "tool_call_id": frame.tool_call_id, + } + ) self._function_calls_in_progress[frame.tool_call_id] = frame @@ -1091,7 +1100,14 @@ class LLMAssistantAggregator(LLMContextAggregator): self._context.add_message( { "role": "developer", - "content": json.dumps({"tool_call_id": frame.tool_call_id, "result": result}), + "content": json.dumps( + { + "type": "async_tool", + "tool_call_id": frame.tool_call_id, + "status": "finished", + "result": result, + } + ), } ) else: