From 2e1a18503b9da4c5cee86c188483ce39e6701811 Mon Sep 17 00:00:00 2001 From: balalo Date: Tue, 18 Mar 2025 10:41:43 +0100 Subject: [PATCH 1/6] Set tool choice from context aggregator --- src/pipecat/frames/frames.py | 6 ++++++ .../processors/aggregators/llm_response.py | 18 +++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 74dd2accb..aec433033 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -362,6 +362,12 @@ class LLMSetToolsFrame(DataFrame): tools: List[dict] +@dataclass +class LLMSetToolChoiceFrame(DataFrame): + """A frame containing a tool choice for an LLM to use for function calling.""" + + tool_choice: Literal["none", "auto", "required"] + @dataclass class LLMEnablePromptCachingFrame(DataFrame): diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index d8582e32f..620e34b2c 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -7,7 +7,7 @@ import asyncio import time from abc import abstractmethod -from typing import List +from typing import List, Literal from pipecat.frames.frames import ( CancelFrame, @@ -22,6 +22,7 @@ from pipecat.frames.frames import ( LLMMessagesFrame, LLMMessagesUpdateFrame, LLMSetToolsFrame, + LLMSetToolChoiceFrame, LLMTextFrame, StartFrame, StartInterruptionFrame, @@ -132,6 +133,11 @@ class BaseLLMResponseAggregator(FrameProcessor): """Set LLM tools to be used in the current conversation.""" pass + @abstractmethod + def set_tool_choice(self, tool_choice): + """Set the tool choice. This should modify the LLM context.""" + pass + @abstractmethod def reset(self): """Reset the internals of this aggregator. This should not modify the @@ -185,6 +191,9 @@ class LLMResponseAggregator(BaseLLMResponseAggregator): def set_tools(self, tools): pass + def set_tool_choice(self, tool_choice): + pass + def reset(self): self._aggregation = "" @@ -244,6 +253,9 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator): def set_tools(self, tools: List): self._context.set_tools(tools) + def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"]): + self._context.set_tool_choice(tool_choice) + def reset(self): self._aggregation = "" @@ -328,6 +340,8 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): self.set_messages(frame.messages) elif isinstance(frame, LLMSetToolsFrame): self.set_tools(frame.tools) + elif isinstance(frame, LLMSetToolChoiceFrame): + self.set_tool_choice(frame.tool_choice) else: await self.push_frame(frame, direction) @@ -448,6 +462,8 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): self.set_messages(frame.messages) elif isinstance(frame, LLMSetToolsFrame): self.set_tools(frame.tools) + elif isinstance(frame, LLMSetToolChoiceFrame): + self.set_tool_choice(frame.tool_choice) else: await self.push_frame(frame, direction) From 1c19777d5eda4cf237f83d8dcf3dc82c4f2a4540 Mon Sep 17 00:00:00 2001 From: balalo Date: Tue, 18 Mar 2025 11:09:40 +0100 Subject: [PATCH 2/6] Fix format --- src/pipecat/frames/frames.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index aec433033..7b07a331e 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -362,6 +362,7 @@ class LLMSetToolsFrame(DataFrame): tools: List[dict] + @dataclass class LLMSetToolChoiceFrame(DataFrame): """A frame containing a tool choice for an LLM to use for function calling.""" From dc5067407d6b2136e70b411e070cf774bdfea1e0 Mon Sep 17 00:00:00 2001 From: balalo Date: Tue, 18 Mar 2025 11:12:51 +0100 Subject: [PATCH 3/6] Fix ruff check --- src/pipecat/processors/aggregators/llm_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 620e34b2c..9c18e67ca 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -21,8 +21,8 @@ from pipecat.frames.frames import ( LLMMessagesAppendFrame, LLMMessagesFrame, LLMMessagesUpdateFrame, - LLMSetToolsFrame, LLMSetToolChoiceFrame, + LLMSetToolsFrame, LLMTextFrame, StartFrame, StartInterruptionFrame, From 48b6850df43d8bf6f74f72845f85188a0e2e153a Mon Sep 17 00:00:00 2001 From: balalo Date: Tue, 18 Mar 2025 20:45:31 +0100 Subject: [PATCH 4/6] allow other function names --- src/pipecat/frames/frames.py | 2 +- src/pipecat/processors/aggregators/llm_response.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 7b07a331e..734f0ea6a 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -367,7 +367,7 @@ class LLMSetToolsFrame(DataFrame): class LLMSetToolChoiceFrame(DataFrame): """A frame containing a tool choice for an LLM to use for function calling.""" - tool_choice: Literal["none", "auto", "required"] + tool_choice: Literal["none", "auto", "required"] | str @dataclass diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 9c18e67ca..44d41f535 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -253,7 +253,7 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator): def set_tools(self, tools: List): self._context.set_tools(tools) - def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"]): + def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | str): self._context.set_tool_choice(tool_choice) def reset(self): From ce9f75a8515e8741075abb87ac05611543f47c4b Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 26 Mar 2025 08:17:50 -0300 Subject: [PATCH 5/6] Fixing the tool choice extra type to be a dict instead of string. --- src/pipecat/frames/frames.py | 2 +- src/pipecat/processors/aggregators/llm_response.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 39e25fc11..11cd17801 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -367,7 +367,7 @@ class LLMSetToolsFrame(DataFrame): class LLMSetToolChoiceFrame(DataFrame): """A frame containing a tool choice for an LLM to use for function calling.""" - tool_choice: Literal["none", "auto", "required"] | str + tool_choice: Literal["none", "auto", "required"] | dict @dataclass diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index c0ef65f12..af8bf1a2e 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -210,7 +210,7 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator): def set_tools(self, tools: List): self._context.set_tools(tools) - def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | str): + def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | dict): self._context.set_tool_choice(tool_choice) def reset(self): From aeac40312e6f67e1c3c0dc420a448bde409f3ed0 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 26 Mar 2025 09:06:29 -0300 Subject: [PATCH 6/6] Added the feature to change dynamically the tool choice to the changelog. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47c2560e..e88155896 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a new frame, `LLMSetToolChoiceFrame`, which provides a mechanism + for modifying the `tool_choice` in the context. + - Added support in `DailyTransport` for updating remote participants' `canReceive` permission via the `update_remote_participants()` method, by bumping the daily-python dependency to >= 0.16.0.