From d04c4b36f3c37602e33cf5f402f22528412307f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Feb 2025 19:25:43 -0800 Subject: [PATCH] AssistantContextAggregator: append aggregation and tools in the same turn --- CHANGELOG.md | 3 +++ src/pipecat/processors/aggregators/llm_response.py | 3 +++ src/pipecat/services/anthropic.py | 9 ++++----- src/pipecat/services/google/google.py | 12 ++++++------ src/pipecat/services/grok.py | 8 ++++---- src/pipecat/services/openai.py | 8 ++++---- 6 files changed, 24 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc732c569..dd4ca7269 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed a context aggregator issue that would not append the LLM text response + to the context if a function call happened in the same LLM turn. + - Fixed an issue that was causing HTTP TTS services to push `TTSStoppedFrame` more than once. diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 2de3ebae4..6f1da5af4 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -145,6 +145,9 @@ class LLMResponseAggregator(BaseLLMResponseAggregator): frame = LLMMessagesFrame(self._messages) await self.push_frame(frame) + # Reset our accumulator state. + self.reset() + class LLMContextResponseAggregator(BaseLLMResponseAggregator): """This is a base LLM aggregator that uses an LLM context to store the diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 80235199b..a7c5fd5f0 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -743,18 +743,19 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): run_llm = False properties: Optional[FunctionCallResultProperties] = None - aggregation = self._aggregation + aggregation = self._aggregation.strip() self.reset() try: + if aggregation: + self._context.add_message({"role": "assistant", "content": aggregation}) + if self._function_call_result: frame = self._function_call_result properties = frame.properties self._function_call_result = None if frame.result: assistant_message = {"role": "assistant", "content": []} - if aggregation: - assistant_message["content"].append({"type": "text", "text": aggregation}) assistant_message["content"].append( { "type": "tool_use", @@ -782,8 +783,6 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): else: # Default behavior run_llm = True - elif aggregation: - self._context.add_message({"role": "assistant", "content": aggregation}) if self._pending_image_frame_message: frame = self._pending_image_frame_message diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 0a8a86580..67a2b3954 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -565,10 +565,15 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): run_llm = False properties: Optional[FunctionCallResultProperties] = None - aggregation = self._aggregation + aggregation = self._aggregation.strip() self.reset() try: + if aggregation: + self._context.add_message( + glm.Content(role="model", parts=[glm.Part(text=aggregation)]) + ) + if self._function_call_result: frame = self._function_call_result properties = frame.properties @@ -608,11 +613,6 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): else: # Default behavior is to run the LLM if there are no function calls in progress run_llm = not bool(self._function_calls_in_progress) - else: - if aggregation.strip(): - self._context.add_message( - glm.Content(role="model", parts=[glm.Part(text=aggregation)]) - ) if self._pending_image_frame_message: frame = self._pending_image_frame_message diff --git a/src/pipecat/services/grok.py b/src/pipecat/services/grok.py index 064dd8829..0011a1f4e 100644 --- a/src/pipecat/services/grok.py +++ b/src/pipecat/services/grok.py @@ -37,10 +37,13 @@ class GrokAssistantContextAggregator(OpenAIAssistantContextAggregator): run_llm = False properties: Optional[FunctionCallResultProperties] = None - aggregation = self._aggregation + aggregation = self._aggregation.strip() self.reset() try: + if aggregation: + self._context.add_message({"role": "assistant", "content": aggregation}) + if self._function_call_result: frame = self._function_call_result properties = frame.properties @@ -77,9 +80,6 @@ class GrokAssistantContextAggregator(OpenAIAssistantContextAggregator): # Default behavior is to run the LLM if there are no function calls in progress run_llm = not bool(self._function_calls_in_progress) - else: - self._context.add_message({"role": "assistant", "content": aggregation}) - if self._pending_image_frame_message: frame = self._pending_image_frame_message self._pending_image_frame_message = None diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 15c1a197a..4077f4354 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -631,10 +631,13 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): run_llm = False properties: Optional[FunctionCallResultProperties] = None - aggregation = self._aggregation + aggregation = self._aggregation.strip() self.reset() try: + if aggregation: + self._context.add_message({"role": "assistant", "content": aggregation}) + if self._function_call_result: frame = self._function_call_result properties = frame.properties @@ -669,9 +672,6 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): # Default behavior is to run the LLM if there are no function calls in progress run_llm = not bool(self._function_calls_in_progress) - else: - self._context.add_message({"role": "assistant", "content": aggregation}) - if self._pending_image_frame_message: frame = self._pending_image_frame_message self._pending_image_frame_message = None