Merge pull request #701 from pipecat-ai/khk/anthropic-function-calling-fix

fixes for anthropic function calling

examples/foundational/14b-function-calling-anthropic-video.py

@@ -67,7 +67,8 @@ async def main():
 
         llm = AnthropicLLMService(
             api_key=os.getenv("ANTHROPIC_API_KEY"),
-            model="claude-3-5-sonnet-20240620",
+            # model="claude-3-5-sonnet-20240620",
+            model="claude-3-5-sonnet-latest",
             enable_prompt_caching_beta=True,
         )
         llm.register_function("get_weather", get_weather)

examples/foundational/20c-persistent-context-anthropic.py

@@ -98,12 +98,13 @@ async def load_conversation(function_name, tool_call_id, args, llm, context, res
 messages = [
     {
         "role": "system",
-        "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+        "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a succinct, creative and helpful way. Prefer responses that are one sentence long unless you are asked for a longer or more detailed response.",
     },
-    {"role": "user", "content": ""},
+    {"role": "user", "content": "Start the call by saying the word 'hello'. Say only that word."},
-    {"role": "assistant", "content": []},
+    # {"role": "user", "content": ""},
-    {"role": "user", "content": "Tell me"},
+    # {"role": "assistant", "content": []},
-    {"role": "user", "content": "a joke"},
+    # {"role": "user", "content": "Tell me"},
+    # {"role": "user", "content": "a joke"},
 ]
 tools = [
     {
@@ -183,7 +184,7 @@ async def main():
         )
 
         llm = AnthropicLLMService(
-            api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-20240620"
+            api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-latest"
         )
 
         # you can either register a single function for all function calls, or specific functions

src/pipecat/services/anthropic.py

@@ -671,6 +671,7 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
             ):
                 self._function_call_in_progress = None
                 self._function_call_result = frame
+                await self._push_aggregation()
             else:
                 logger.warning(
                     "FunctionCallResultFrame tool_call_id != InProgressFrame tool_call_id"
@@ -679,9 +680,12 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
                 self._function_call_result = None
         elif isinstance(frame, AnthropicImageMessageFrame):
             self._pending_image_frame_message = frame
+            await self._push_aggregation()
 
     async def _push_aggregation(self):
-        if not self._aggregation:
+        if not (
+            self._aggregation or self._function_call_result or self._pending_image_frame_message
+        ):
             return
 
         run_llm = False
@@ -694,20 +698,18 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
                 frame = self._function_call_result
                 self._function_call_result = None
                 if frame.result:
-                    self._context.add_message(
+                    assistant_message = {"role": "assistant", "content": []}
+                    if aggregation:
+                        assistant_message["content"].append({"type": "text", "text": aggregation})
+                    assistant_message["content"].append(
                         {
-                            "role": "assistant",
+                            "type": "tool_use",
-                            "content": [
+                            "id": frame.tool_call_id,
-                                {"type": "text", "text": aggregation},
+                            "name": frame.function_name,
-                                {
+                            "input": frame.arguments,
-                                    "type": "tool_use",
-                                    "id": frame.tool_call_id,
-                                    "name": frame.function_name,
-                                    "input": frame.arguments,
-                                },
-                            ],
                         }
                     )
+                    self._context.add_message(assistant_message)
                     self._context.add_message(
                         {
                             "role": "user",
@@ -721,7 +723,7 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
                         }
                     )
                     run_llm = True
-            else:
+            elif aggregation:
                 self._context.add_message({"role": "assistant", "content": aggregation})
 
             if self._pending_image_frame_message: