feat: add configurable client tools and photo input

2026-07-30 19:06:03 +08:00
parent 510a277b5a
commit 913435785e
24 changed files with 1802 additions and 139 deletions
--- a/backend/tests/test_tool_policy.py
+++ b/backend/tests/test_tool_policy.py
@@ -0,0 +1,142 @@
+import unittest
+from types import SimpleNamespace
+
+from models import RuntimeTool
+from pipecat.frames.frames import (
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
+    FunctionCallResultFrame,
+    FunctionCallsStartedFrame,
+)
+from services.pipecat.processors import ToolInterruptionUserMuteStrategy
+from services.tool_policy import policy_for_tool
+
+
+def runtime_tool(tool_type: str, config: dict | None = None) -> RuntimeTool:
+    return RuntimeTool(
+        id=f"tool_{tool_type}",
+        name=tool_type,
+        function_name=f"run_{tool_type}",
+        type=tool_type,
+        definition={"type": tool_type, "config": config or {}},
+    )
+
+
+class ToolPolicyTests(unittest.TestCase):
+    def test_backwards_compatible_execution_defaults(self):
+        self.assertEqual(
+            policy_for_tool(runtime_tool("http")).execution_mode,
+            "immediate",
+        )
+        self.assertEqual(
+            policy_for_tool(runtime_tool("client")).execution_mode,
+            "async",
+        )
+
+    def test_explicit_policy_is_normalized(self):
+        policy = policy_for_tool(
+            runtime_tool(
+                "client",
+                {
+                    "allow_interruptions": False,
+                    "execution_mode": "immediate",
+                    "wait_for_response": False,
+                },
+            )
+        )
+        self.assertFalse(policy.allow_interruptions)
+        self.assertTrue(policy.cancel_on_interruption)
+        self.assertFalse(policy.wait_for_response)
+
+
+class ToolInterruptionStrategyTests(unittest.IsolatedAsyncioTestCase):
+    async def test_releases_only_after_tool_and_following_speech_finish(self):
+        strategy = ToolInterruptionUserMuteStrategy({"lookup_order"})
+        started = FunctionCallsStartedFrame(
+            function_calls=[
+                SimpleNamespace(
+                    function_name="lookup_order",
+                    tool_call_id="call_1",
+                )
+            ]
+        )
+
+        self.assertTrue(await strategy.process_frame(started))
+        self.assertTrue(
+            await strategy.process_frame(BotStartedSpeakingFrame())
+        )
+        self.assertTrue(
+            await strategy.process_frame(BotStoppedSpeakingFrame())
+        )
+        self.assertFalse(
+            await strategy.process_frame(
+                FunctionCallResultFrame(
+                    function_name="lookup_order",
+                    tool_call_id="call_1",
+                    arguments={},
+                    result={"status": "ok"},
+                )
+            )
+        )
+
+    async def test_waits_for_response_when_tool_finishes_first(self):
+        strategy = ToolInterruptionUserMuteStrategy({"lookup_order"})
+        await strategy.process_frame(
+            FunctionCallsStartedFrame(
+                function_calls=[
+                    SimpleNamespace(
+                        function_name="lookup_order",
+                        tool_call_id="call_2",
+                    )
+                ]
+            )
+        )
+        self.assertTrue(
+            await strategy.process_frame(
+                FunctionCallResultFrame(
+                    function_name="lookup_order",
+                    tool_call_id="call_2",
+                    arguments={},
+                    result={"status": "ok"},
+                )
+            )
+        )
+        await strategy.process_frame(BotStartedSpeakingFrame())
+        self.assertFalse(
+            await strategy.process_frame(BotStoppedSpeakingFrame())
+        )
+
+    async def test_immediate_tool_does_not_count_existing_preamble(self):
+        strategy = ToolInterruptionUserMuteStrategy(
+            {"lookup_order": "immediate"}
+        )
+        await strategy.process_frame(BotStartedSpeakingFrame())
+        await strategy.process_frame(
+            FunctionCallsStartedFrame(
+                function_calls=[
+                    SimpleNamespace(
+                        function_name="lookup_order",
+                        tool_call_id="call_3",
+                    )
+                ]
+            )
+        )
+        await strategy.process_frame(BotStoppedSpeakingFrame())
+        self.assertTrue(
+            await strategy.process_frame(
+                FunctionCallResultFrame(
+                    function_name="lookup_order",
+                    tool_call_id="call_3",
+                    arguments={},
+                    result={"status": "ok"},
+                )
+            )
+        )
+        await strategy.process_frame(BotStartedSpeakingFrame())
+        self.assertFalse(
+            await strategy.process_frame(BotStoppedSpeakingFrame())
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()