From a54ca08405181b33254b5ebea51af9cc3e84d261 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Fri, 8 Nov 2024 16:33:02 -0800 Subject: [PATCH 1/2] fixes for anthropic function calling --- .../14b-function-calling-anthropic-video.py | 3 +- src/pipecat/services/anthropic.py | 28 ++++++++++--------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index 8c3821ce4..cdc69556d 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -67,7 +67,8 @@ async def main(): llm = AnthropicLLMService( api_key=os.getenv("ANTHROPIC_API_KEY"), - model="claude-3-5-sonnet-20240620", + # model="claude-3-5-sonnet-20240620", + model="claude-3-5-sonnet-latest", enable_prompt_caching_beta=True, ) llm.register_function("get_weather", get_weather) diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index a87ed9423..d3445dbe0 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -671,6 +671,7 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): ): self._function_call_in_progress = None self._function_call_result = frame + await self._push_aggregation() else: logger.warning( "FunctionCallResultFrame tool_call_id != InProgressFrame tool_call_id" @@ -679,9 +680,12 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): self._function_call_result = None elif isinstance(frame, AnthropicImageMessageFrame): self._pending_image_frame_message = frame + await self._push_aggregation() async def _push_aggregation(self): - if not self._aggregation: + if not ( + self._aggregation or self._function_call_result or self._pending_image_frame_message + ): return run_llm = False @@ -694,20 +698,18 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): frame = self._function_call_result self._function_call_result = None if frame.result: - self._context.add_message( + assistant_message = {"role": "assistant", "content": []} + if aggregation: + assistant_message["content"].append({"type": "text", "text": aggregation}) + assistant_message["content"].append( { - "role": "assistant", - "content": [ - {"type": "text", "text": aggregation}, - { - "type": "tool_use", - "id": frame.tool_call_id, - "name": frame.function_name, - "input": frame.arguments, - }, - ], + "type": "tool_use", + "id": frame.tool_call_id, + "name": frame.function_name, + "input": frame.arguments, } ) + self._context.add_message(assistant_message) self._context.add_message( { "role": "user", @@ -721,7 +723,7 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): } ) run_llm = True - else: + elif aggregation: self._context.add_message({"role": "assistant", "content": aggregation}) if self._pending_image_frame_message: From 33fc5bf990004691b1754a0dda51cd60811baa50 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Fri, 8 Nov 2024 16:42:30 -0800 Subject: [PATCH 2/2] improved 20c-persistent-context-anthropic.py --- .../20c-persistent-context-anthropic.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 8e8e9c3f7..421b00603 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -98,12 +98,13 @@ async def load_conversation(function_name, tool_call_id, args, llm, context, res messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a succinct, creative and helpful way. Prefer responses that are one sentence long unless you are asked for a longer or more detailed response.", }, - {"role": "user", "content": ""}, - {"role": "assistant", "content": []}, - {"role": "user", "content": "Tell me"}, - {"role": "user", "content": "a joke"}, + {"role": "user", "content": "Start the call by saying the word 'hello'. Say only that word."}, + # {"role": "user", "content": ""}, + # {"role": "assistant", "content": []}, + # {"role": "user", "content": "Tell me"}, + # {"role": "user", "content": "a joke"}, ] tools = [ { @@ -183,7 +184,7 @@ async def main(): ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-20240620" + api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-latest" ) # you can either register a single function for all function calls, or specific functions