Merge pull request #1250 from pipecat-ai/aleix/context-aggregation-simulatenous-text-tools

AssistantContextAggregator: append aggregation and tools in the same turn

CHANGELOG.md

@@ -32,6 +32,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general"))
 
 ### Fixed
 
+- Fixed a context aggregator issue that would not append the LLM text response
+  to the context if a function call happened in the same LLM turn.
+
 - Fixed an issue that was causing HTTP TTS services to push `TTSStoppedFrame`
   more than once.
 

src/pipecat/processors/aggregators/llm_response.py

@@ -145,6 +145,9 @@ class LLMResponseAggregator(BaseLLMResponseAggregator):
             frame = LLMMessagesFrame(self._messages)
             await self.push_frame(frame)
 
+            # Reset our accumulator state.
+            self.reset()
+
 
 class LLMContextResponseAggregator(BaseLLMResponseAggregator):
     """This is a base LLM aggregator that uses an LLM context to store the

src/pipecat/services/anthropic.py

@@ -743,18 +743,19 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
         run_llm = False
         properties: Optional[FunctionCallResultProperties] = None
 
-        aggregation = self._aggregation
+        aggregation = self._aggregation.strip()
         self.reset()
 
         try:
+            if aggregation:
+                self._context.add_message({"role": "assistant", "content": aggregation})
+
             if self._function_call_result:
                 frame = self._function_call_result
                 properties = frame.properties
                 self._function_call_result = None
                 if frame.result:
                     assistant_message = {"role": "assistant", "content": []}
-                    if aggregation:
-                        assistant_message["content"].append({"type": "text", "text": aggregation})
                     assistant_message["content"].append(
                         {
                             "type": "tool_use",
@@ -782,8 +783,6 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
                     else:
                         # Default behavior
                         run_llm = True
-            elif aggregation:
-                self._context.add_message({"role": "assistant", "content": aggregation})
 
             if self._pending_image_frame_message:
                 frame = self._pending_image_frame_message

src/pipecat/services/google/google.py

@@ -565,10 +565,15 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator):
         run_llm = False
         properties: Optional[FunctionCallResultProperties] = None
 
-        aggregation = self._aggregation
+        aggregation = self._aggregation.strip()
         self.reset()
 
         try:
+            if aggregation:
+                self._context.add_message(
+                    glm.Content(role="model", parts=[glm.Part(text=aggregation)])
+                )
+
             if self._function_call_result:
                 frame = self._function_call_result
                 properties = frame.properties
@@ -608,11 +613,6 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator):
                     else:
                         # Default behavior is to run the LLM if there are no function calls in progress
                         run_llm = not bool(self._function_calls_in_progress)
-            else:
-                if aggregation.strip():
-                    self._context.add_message(
-                        glm.Content(role="model", parts=[glm.Part(text=aggregation)])
-                    )
 
             if self._pending_image_frame_message:
                 frame = self._pending_image_frame_message

src/pipecat/services/grok.py

@@ -37,10 +37,13 @@ class GrokAssistantContextAggregator(OpenAIAssistantContextAggregator):
         run_llm = False
         properties: Optional[FunctionCallResultProperties] = None
 
-        aggregation = self._aggregation
+        aggregation = self._aggregation.strip()
         self.reset()
 
         try:
+            if aggregation:
+                self._context.add_message({"role": "assistant", "content": aggregation})
+
             if self._function_call_result:
                 frame = self._function_call_result
                 properties = frame.properties
@@ -77,9 +80,6 @@ class GrokAssistantContextAggregator(OpenAIAssistantContextAggregator):
                         # Default behavior is to run the LLM if there are no function calls in progress
                         run_llm = not bool(self._function_calls_in_progress)
 
-            else:
-                self._context.add_message({"role": "assistant", "content": aggregation})
-
             if self._pending_image_frame_message:
                 frame = self._pending_image_frame_message
                 self._pending_image_frame_message = None

src/pipecat/services/openai.py

@@ -631,10 +631,13 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
         run_llm = False
         properties: Optional[FunctionCallResultProperties] = None
 
-        aggregation = self._aggregation
+        aggregation = self._aggregation.strip()
         self.reset()
 
         try:
+            if aggregation:
+                self._context.add_message({"role": "assistant", "content": aggregation})
+
             if self._function_call_result:
                 frame = self._function_call_result
                 properties = frame.properties
@@ -669,9 +672,6 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
                         # Default behavior is to run the LLM if there are no function calls in progress
                         run_llm = not bool(self._function_calls_in_progress)
 
-            else:
-                self._context.add_message({"role": "assistant", "content": aggregation})
-
             if self._pending_image_frame_message:
                 frame = self._pending_image_frame_message
                 self._pending_image_frame_message = None