feat: add configurable client tools and photo input
This commit is contained in:
Xin Wang
@@ -34,9 +34,11 @@ from services.workflow.models import LLMRouteResult, RouteStatus
|
||||
class FakeLLM:
|
||||
def __init__(self):
|
||||
self.functions = {}
|
||||
self.function_options = {}
|
||||
|
||||
def register_function(self, name, handler):
|
||||
def register_function(self, name, handler, **options):
|
||||
self.functions[name] = handler
|
||||
self.function_options[name] = options
|
||||
|
||||
|
||||
class FakeCallEnd:
|
||||
|
||||
157
backend/tests/test_client_tools.py
Normal file
157
backend/tests/test_client_tools.py
Normal file
@@ -0,0 +1,157 @@
|
||||
import asyncio
|
||||
import unittest
|
||||
|
||||
from models import RuntimeTool
|
||||
from pipecat.frames.frames import (
|
||||
InputTransportMessageFrame,
|
||||
OutputTransportMessageUrgentFrame,
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameDirection
|
||||
from services.client_tools import ClientToolBroker, ClientToolError
|
||||
from services.runtime_variables import DynamicVariableStore
|
||||
from services.tool_executor import ToolExecutor
|
||||
from schemas import ClientToolConfig
|
||||
|
||||
|
||||
def client_tool() -> RuntimeTool:
|
||||
return RuntimeTool(
|
||||
id="tool_photo_button",
|
||||
name="显示拍照按钮",
|
||||
function_name="set_photo_button_visible",
|
||||
type="client",
|
||||
definition={
|
||||
"schema_version": 1,
|
||||
"type": "client",
|
||||
"config": {
|
||||
"parameters": [
|
||||
{
|
||||
"name": "visible",
|
||||
"type": "boolean",
|
||||
"required": True,
|
||||
}
|
||||
],
|
||||
"timeout_seconds": 3,
|
||||
"dynamic_variable_assignments": {
|
||||
"photo_button_visible": "visible"
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class FakeClientTools:
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
self.calls = []
|
||||
|
||||
async def call(
|
||||
self,
|
||||
function_name,
|
||||
arguments,
|
||||
*,
|
||||
timeout_seconds,
|
||||
wait_for_response=True,
|
||||
):
|
||||
self.calls.append(
|
||||
(function_name, arguments, timeout_seconds, wait_for_response)
|
||||
)
|
||||
return self.result
|
||||
|
||||
|
||||
class ClientToolExecutorTests(unittest.IsolatedAsyncioTestCase):
|
||||
def test_fire_and_forget_rejects_result_assignments(self):
|
||||
with self.assertRaisesRegex(ValueError, "不能配置结果变量赋值"):
|
||||
ClientToolConfig(
|
||||
wait_for_response=False,
|
||||
dynamic_variable_assignments={"visible": "visible"},
|
||||
)
|
||||
|
||||
async def test_success_updates_configured_variable(self):
|
||||
store = DynamicVariableStore(
|
||||
{"photo_button_visible": False},
|
||||
variable_types={"photo_button_visible": "boolean"},
|
||||
)
|
||||
port = FakeClientTools({"status": "ok", "data": {"visible": True}})
|
||||
executor = ToolExecutor(store, client_tools=port)
|
||||
|
||||
result = await executor.execute(client_tool(), {"visible": True})
|
||||
|
||||
self.assertEqual(result["updated_variables"], ["photo_button_visible"])
|
||||
self.assertIs(store.values["photo_button_visible"], True)
|
||||
self.assertEqual(
|
||||
port.calls,
|
||||
[("set_photo_button_visible", {"visible": True}, 3.0, True)],
|
||||
)
|
||||
|
||||
async def test_failure_does_not_update_variable(self):
|
||||
store = DynamicVariableStore({"photo_button_visible": False})
|
||||
executor = ToolExecutor(
|
||||
store,
|
||||
client_tools=FakeClientTools(
|
||||
{"status": "error", "message": "unsupported"}
|
||||
),
|
||||
)
|
||||
|
||||
result = await executor.execute(client_tool(), {"visible": True})
|
||||
|
||||
self.assertEqual(result["updated_variables"], [])
|
||||
self.assertIs(store.values["photo_button_visible"], False)
|
||||
|
||||
|
||||
class ClientToolBrokerTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_correlates_result_and_times_out(self):
|
||||
broker = ClientToolBroker()
|
||||
outbound = []
|
||||
|
||||
async def push_frame(frame, direction=FrameDirection.DOWNSTREAM):
|
||||
outbound.append((frame, direction))
|
||||
|
||||
broker.push_frame = push_frame
|
||||
call = asyncio.create_task(
|
||||
broker.call("set_photo_button_visible", {"visible": True}, timeout_seconds=1)
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
message = outbound[0][0].message
|
||||
self.assertIsInstance(outbound[0][0], OutputTransportMessageUrgentFrame)
|
||||
|
||||
await broker.process_frame(
|
||||
InputTransportMessageFrame(
|
||||
message={
|
||||
"type": "client-tool-result",
|
||||
"tool_call_id": message["tool_call_id"],
|
||||
"status": "ok",
|
||||
"data": {"visible": True},
|
||||
}
|
||||
),
|
||||
FrameDirection.DOWNSTREAM,
|
||||
)
|
||||
self.assertEqual(
|
||||
await call,
|
||||
{"status": "ok", "data": {"visible": True}},
|
||||
)
|
||||
|
||||
with self.assertRaisesRegex(ClientToolError, "超时"):
|
||||
await broker.call("never_returns", {}, timeout_seconds=0.001)
|
||||
|
||||
async def test_fire_and_forget_does_not_create_pending_call(self):
|
||||
broker = ClientToolBroker()
|
||||
outbound = []
|
||||
|
||||
async def push_frame(frame, direction=FrameDirection.DOWNSTREAM):
|
||||
outbound.append((frame, direction))
|
||||
|
||||
broker.push_frame = push_frame
|
||||
result = await broker.call(
|
||||
"open_panel",
|
||||
{"visible": True},
|
||||
timeout_seconds=1,
|
||||
wait_for_response=False,
|
||||
)
|
||||
|
||||
self.assertEqual(result, {"status": "ok", "data": {"dispatched": True}})
|
||||
self.assertEqual(broker._pending, {})
|
||||
self.assertFalse(outbound[0][0].message["wait_for_response"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -92,6 +92,44 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(brain.turns, ["我叫李白"])
|
||||
self.assertEqual(forwarded, [(frame, FrameDirection.DOWNSTREAM)])
|
||||
|
||||
async def test_routes_multimodal_user_message_by_its_text_part(self):
|
||||
class FakeBrain:
|
||||
def __init__(self):
|
||||
self.turns = []
|
||||
|
||||
async def on_user_turn_end(self, content):
|
||||
self.turns.append(content)
|
||||
return False
|
||||
|
||||
brain = FakeBrain()
|
||||
processor = UserTurnRoutingProcessor(brain)
|
||||
processor.push_frame = lambda *_args, **_kwargs: _async_none()
|
||||
context = LLMContext(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "看看这张照片"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/jpeg;base64,AA=="},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
await processor.process_frame(
|
||||
LLMContextFrame(context),
|
||||
FrameDirection.DOWNSTREAM,
|
||||
)
|
||||
|
||||
self.assertEqual(brain.turns, ["看看这张照片"])
|
||||
|
||||
|
||||
async def _async_none():
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
142
backend/tests/test_tool_policy.py
Normal file
142
backend/tests/test_tool_policy.py
Normal file
@@ -0,0 +1,142 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from models import RuntimeTool
|
||||
from pipecat.frames.frames import (
|
||||
BotStartedSpeakingFrame,
|
||||
BotStoppedSpeakingFrame,
|
||||
FunctionCallResultFrame,
|
||||
FunctionCallsStartedFrame,
|
||||
)
|
||||
from services.pipecat.processors import ToolInterruptionUserMuteStrategy
|
||||
from services.tool_policy import policy_for_tool
|
||||
|
||||
|
||||
def runtime_tool(tool_type: str, config: dict | None = None) -> RuntimeTool:
|
||||
return RuntimeTool(
|
||||
id=f"tool_{tool_type}",
|
||||
name=tool_type,
|
||||
function_name=f"run_{tool_type}",
|
||||
type=tool_type,
|
||||
definition={"type": tool_type, "config": config or {}},
|
||||
)
|
||||
|
||||
|
||||
class ToolPolicyTests(unittest.TestCase):
|
||||
def test_backwards_compatible_execution_defaults(self):
|
||||
self.assertEqual(
|
||||
policy_for_tool(runtime_tool("http")).execution_mode,
|
||||
"immediate",
|
||||
)
|
||||
self.assertEqual(
|
||||
policy_for_tool(runtime_tool("client")).execution_mode,
|
||||
"async",
|
||||
)
|
||||
|
||||
def test_explicit_policy_is_normalized(self):
|
||||
policy = policy_for_tool(
|
||||
runtime_tool(
|
||||
"client",
|
||||
{
|
||||
"allow_interruptions": False,
|
||||
"execution_mode": "immediate",
|
||||
"wait_for_response": False,
|
||||
},
|
||||
)
|
||||
)
|
||||
self.assertFalse(policy.allow_interruptions)
|
||||
self.assertTrue(policy.cancel_on_interruption)
|
||||
self.assertFalse(policy.wait_for_response)
|
||||
|
||||
|
||||
class ToolInterruptionStrategyTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_releases_only_after_tool_and_following_speech_finish(self):
|
||||
strategy = ToolInterruptionUserMuteStrategy({"lookup_order"})
|
||||
started = FunctionCallsStartedFrame(
|
||||
function_calls=[
|
||||
SimpleNamespace(
|
||||
function_name="lookup_order",
|
||||
tool_call_id="call_1",
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
self.assertTrue(await strategy.process_frame(started))
|
||||
self.assertTrue(
|
||||
await strategy.process_frame(BotStartedSpeakingFrame())
|
||||
)
|
||||
self.assertTrue(
|
||||
await strategy.process_frame(BotStoppedSpeakingFrame())
|
||||
)
|
||||
self.assertFalse(
|
||||
await strategy.process_frame(
|
||||
FunctionCallResultFrame(
|
||||
function_name="lookup_order",
|
||||
tool_call_id="call_1",
|
||||
arguments={},
|
||||
result={"status": "ok"},
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
async def test_waits_for_response_when_tool_finishes_first(self):
|
||||
strategy = ToolInterruptionUserMuteStrategy({"lookup_order"})
|
||||
await strategy.process_frame(
|
||||
FunctionCallsStartedFrame(
|
||||
function_calls=[
|
||||
SimpleNamespace(
|
||||
function_name="lookup_order",
|
||||
tool_call_id="call_2",
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
await strategy.process_frame(
|
||||
FunctionCallResultFrame(
|
||||
function_name="lookup_order",
|
||||
tool_call_id="call_2",
|
||||
arguments={},
|
||||
result={"status": "ok"},
|
||||
)
|
||||
)
|
||||
)
|
||||
await strategy.process_frame(BotStartedSpeakingFrame())
|
||||
self.assertFalse(
|
||||
await strategy.process_frame(BotStoppedSpeakingFrame())
|
||||
)
|
||||
|
||||
async def test_immediate_tool_does_not_count_existing_preamble(self):
|
||||
strategy = ToolInterruptionUserMuteStrategy(
|
||||
{"lookup_order": "immediate"}
|
||||
)
|
||||
await strategy.process_frame(BotStartedSpeakingFrame())
|
||||
await strategy.process_frame(
|
||||
FunctionCallsStartedFrame(
|
||||
function_calls=[
|
||||
SimpleNamespace(
|
||||
function_name="lookup_order",
|
||||
tool_call_id="call_3",
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
await strategy.process_frame(BotStoppedSpeakingFrame())
|
||||
self.assertTrue(
|
||||
await strategy.process_frame(
|
||||
FunctionCallResultFrame(
|
||||
function_name="lookup_order",
|
||||
tool_call_id="call_3",
|
||||
arguments={},
|
||||
result={"status": "ok"},
|
||||
)
|
||||
)
|
||||
)
|
||||
await strategy.process_frame(BotStartedSpeakingFrame())
|
||||
self.assertFalse(
|
||||
await strategy.process_frame(BotStoppedSpeakingFrame())
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
57
backend/tests/test_user_input.py
Normal file
57
backend/tests/test_user_input.py
Normal file
@@ -0,0 +1,57 @@
|
||||
import unittest
|
||||
|
||||
from services.pipecat.processors import UserInputError, parse_user_input
|
||||
|
||||
|
||||
class UserInputParserTests(unittest.TestCase):
|
||||
def test_parses_text_and_current_camera_frame(self):
|
||||
value = parse_user_input(
|
||||
{
|
||||
"type": "user-input",
|
||||
"schema_version": 1,
|
||||
"input_id": "input_1",
|
||||
"parts": [
|
||||
{"type": "input_text", "text": "帮我看看"},
|
||||
{
|
||||
"type": "input_image",
|
||||
"source": {
|
||||
"type": "camera_frame",
|
||||
"frame": "current",
|
||||
},
|
||||
},
|
||||
],
|
||||
"options": {
|
||||
"run_immediately": True,
|
||||
"interrupt": True,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertIsNotNone(value)
|
||||
self.assertEqual(value.text, "帮我看看")
|
||||
self.assertTrue(value.has_camera_frame)
|
||||
self.assertEqual(value.transcript_text, "帮我看看\n已发送一张图片")
|
||||
|
||||
def test_rejects_legacy_and_unsupported_image_sources(self):
|
||||
self.assertIsNone(parse_user_input({"type": "user-text", "text": "旧协议"}))
|
||||
with self.assertRaisesRegex(UserInputError, "当前摄像头"):
|
||||
parse_user_input(
|
||||
{
|
||||
"type": "user-input",
|
||||
"schema_version": 1,
|
||||
"input_id": "input_2",
|
||||
"parts": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"source": {
|
||||
"type": "uploaded_asset",
|
||||
"asset_id": "asset_1",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user