Trying to structure async tool responses and improve the LLM prompt to teach it how to handle them.

examples/function-calling/openai-example.py

@@ -65,6 +65,57 @@ transport_params = {
     ),
 }
 
+system_prompt = """
+You are a helpful assistant in a voice conversation. 
+Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.
+
+You can call tools asynchronously. Tool calls follow a lifecycle and may complete later, out of order, or after the user has changed their request.
+
+You must follow these rules strictly:
+
+1. Tool Call Lifecycle
+- When you call a tool, a message with role "tool" and content {"type": "async_tool", "status": "started"} will appear.
+- The tool result will arrive later as a message with role "developer" and content:
+  {
+    "type": "async_tool",
+    "tool_call_id": "...",
+    "status": "finished",
+    "result": "..."
+  }
+- Do NOT assume the result until you see "status": "finished".
+
+2. Handling Multiple Tool Calls
+- You may issue multiple tool calls in parallel.
+- Results may arrive in any order.
+- You must track tool_call_id to match results correctly.
+
+3. Incremental Updates
+- If results are still pending, you may inform the user that you're waiting.
+- Do NOT hallucinate missing tool results.
+- Do NOT invoke asking for the same information that has already been requested and you are waiting for the result.
+
+4. Changing User Intent
+- The user may change their request while tools are still running.
+- If a request is no longer relevant, ignore its tool results when they arrive.
+- Do NOT present outdated or cancelled information.
+
+5. Responding to Results
+- When tool results arrive:
+  - Extract and summarize the relevant information.
+  - Combine results if multiple tool calls are related.
+  - Provide a clear and helpful answer.
+
+6. Tool Usage Guidelines
+- Always call tools when required instead of guessing.
+- Ensure arguments are correct and reflect the latest user request.
+
+7. Example Behavior
+- If the user asks for multiple locations, call tools for each.
+- If the user changes from "Florida" to "California", ignore Florida's result if it arrives later.
+
+Your goal is to behave like a reliable orchestrator of asynchronous tool calls while providing clear, accurate, and up-to-date responses to the user.
+"""
+
 
 async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
     logger.info(f"Starting bot")
@@ -88,7 +139,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
     llm = OpenAILLMService(
         api_key=os.getenv("OPENAI_API_KEY"),
         settings=OpenAILLMService.Settings(
-            system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.",
+            system_instruction=system_prompt,
         ),
     )
 

src/pipecat/processors/aggregators/llm_response_universal.py

@@ -1055,13 +1055,22 @@ class LLMAssistantAggregator(LLMContextAggregator):
                 ],
             }
         )
-        self._context.add_message(
+        if frame.is_async:
-            {
+            self._context.add_message(
-                "role": "tool",
+                {
-                "content": "IN_PROGRESS",
+                    "role": "tool",
-                "tool_call_id": frame.tool_call_id,
+                    "content": json.dumps({"type": "async_tool", "status": "started"}),
-            }
+                    "tool_call_id": frame.tool_call_id,
-        )
+                }
+            )
+        else:
+            self._context.add_message(
+                {
+                    "role": "tool",
+                    "content": "IN_PROGRESS",
+                    "tool_call_id": frame.tool_call_id,
+                }
+            )
 
         self._function_calls_in_progress[frame.tool_call_id] = frame
 
@@ -1091,7 +1100,14 @@ class LLMAssistantAggregator(LLMContextAggregator):
             self._context.add_message(
                 {
                     "role": "developer",
-                    "content": json.dumps({"tool_call_id": frame.tool_call_id, "result": result}),
+                    "content": json.dumps(
+                        {
+                            "type": "async_tool",
+                            "tool_call_id": frame.tool_call_id,
+                            "status": "finished",
+                            "result": result,
+                        }
+                    ),
                 }
             )
         else: