wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: route workflow image inputs natively

Browse Source
This commit is contained in:
Xin Wang
2026-08-03 10:55:57 +08:00
parent 2e84de0798
commit f3439b21d1
10 changed files with 412 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
backend/tests/test_brains.py
View File

@@ -6,6 +6,7 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
from models import AssistantConfig, RuntimeTool
from pipecat.flows import FlowManager
from pipecat.frames.frames import (
LLMContextFrame,
LLMFullResponseEndFrame,
@@ -28,7 +29,7 @@ from services.brains.dify_llm import (
last_user_text,
normalize_api_base,
)
from services.brains.workflow_brain import WorkflowBrain
from services.brains.workflow_brain import ConfiguredFlowManager, WorkflowBrain
from services.runtime_variables import prepare_dynamic_config
from services.action_runtime import ActionOutcome, ActionStatus
from services.workflow.models import (
@@ -789,6 +790,45 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
async def test_flow_manager_dispatches_native_vision_without_auxiliary_handler(self):
manager = object.__new__(ConfiguredFlowManager)
fallback_transition = AsyncMock()
native_handler = AsyncMock()
native_enabled = {"value": True}
async def flow_handler(_args, _manager):
return {"status": "ok"}
setattr(
flow_handler,
"_workflow_native_vision_handler",
native_handler,
)
setattr(
flow_handler,
"_workflow_native_vision_enabled",
lambda: native_enabled["value"],
)
with patch.object(
FlowManager,
"_create_transition_func",
new=AsyncMock(return_value=fallback_transition),
):
transition = await manager._create_transition_func(
"fetch_user_image",
flow_handler,
)
params = SimpleNamespace()
await transition(params)
native_handler.assert_awaited_once_with(params)
fallback_transition.assert_not_awaited()
native_enabled["value"] = False
await transition(params)
fallback_transition.assert_awaited_once_with(params)
def test_client_tool_session_wait_disables_flow_timeout(self):
brain = WorkflowBrain(
{
@@ -1922,9 +1962,11 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
class FakeRouter:
def __init__(self):
self.calls = 0
self.current_user_message = None
async def select_edge(self, **_kwargs):
async def select_edge(self, **kwargs):
self.calls += 1
self.current_user_message = kwargs.get("current_user_message")
return LLMRouteResult(
status=RouteStatus.MATCHED,
function_name="goto_eat",
@@ -1939,13 +1981,27 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(manager.current_node, "start")
self.assertEqual(router.calls, 0)
handled = await brain.on_user_turn_end("我想吃饭")
image_message = {
"role": "user",
"content": [
{"type": "text", "text": "我想吃饭"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,AA=="},
},
],
}
handled = await brain.on_user_turn_end(
"我想吃饭",
user_message=image_message,
)
self.assertTrue(handled)
self.assertEqual(router.calls, 1)
self.assertEqual(router.current_user_message, image_message)
self.assertEqual(manager.current_node, "eat")
self.assertIn(
{"role": "user", "content": "我想吃饭"},
image_message,
manager.config["task_messages"],
)
self.assertTrue(any(isinstance(frame, LLMRunFrame) for frame in queued))

103
backend/tests/test_pipeline_knowledge.py
View File

@@ -1,6 +1,6 @@
import unittest
from models import AssistantConfig
from models import AssistantConfig, RuntimeModelResource
from pipecat.frames.frames import LLMContextFrame
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameDirection
@@ -9,6 +9,7 @@ from services.pipecat.pipeline import (
KnowledgeRetrievalProcessor,
UserTurnRoutingProcessor,
_knowledge_tool_description,
_workflow_vision_uses_main_llm,
)
@@ -67,8 +68,8 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
def __init__(self):
self.turns = []
async def on_user_turn_end(self, content):
self.turns.append(content)
async def on_user_turn_end(self, content, user_message=None):
self.turns.append((content, user_message))
return True
brain = FakeBrain()
@@ -83,13 +84,19 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
frame = LLMContextFrame(context)
await processor.process_frame(frame, FrameDirection.DOWNSTREAM)
self.assertEqual(brain.turns, ["我叫李白"])
self.assertEqual(
brain.turns,
[("我叫李白", {"role": "user", "content": "我叫李白"})],
)
self.assertEqual(forwarded, [])
# A queued LLMRunFrame after the transition uses the same context. It
# must reach the target Agent without invoking routing a second time.
await processor.process_frame(frame, FrameDirection.DOWNSTREAM)
self.assertEqual(brain.turns, ["我叫李白"])
self.assertEqual(
brain.turns,
[("我叫李白", {"role": "user", "content": "我叫李白"})],
)
self.assertEqual(forwarded, [(frame, FrameDirection.DOWNSTREAM)])
async def test_routes_multimodal_user_message_by_its_text_part(self):
@@ -97,8 +104,8 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
def __init__(self):
self.turns = []
async def on_user_turn_end(self, content):
self.turns.append(content)
async def on_user_turn_end(self, content, user_message=None):
self.turns.append((content, user_message))
return False
brain = FakeBrain()
@@ -124,7 +131,87 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
FrameDirection.DOWNSTREAM,
)
self.assertEqual(brain.turns, ["看看这张照片"])
self.assertEqual(
brain.turns,
[
(
"看看这张照片",
{
"role": "user",
"content": [
{"type": "text", "text": "看看这张照片"},
{
"type": "image_url",
"image_url": {
"url": "data:image/jpeg;base64,AA=="
},
},
],
},
)
],
)
class WorkflowVisionModeTest(unittest.TestCase):
def test_uses_active_agent_llm_only_without_auxiliary_model(self):
cfg = AssistantConfig(
type="workflow",
workflow_model_resources={
"agent_llm": RuntimeModelResource(
id="agent_llm",
name="视觉 Agent",
capability="LLM",
interface_type="openai-llm",
support_image_input=True,
)
},
)
self.assertTrue(
_workflow_vision_uses_main_llm(
cfg,
{
"enabled": True,
"llm_resource_id": "agent_llm",
"vision_model_resource_id": None,
},
)
)
self.assertFalse(
_workflow_vision_uses_main_llm(
cfg,
{
"enabled": True,
"llm_resource_id": "agent_llm",
"vision_model_resource_id": "auxiliary_vision",
},
)
)
def test_rejects_a_non_visual_active_agent_llm(self):
cfg = AssistantConfig(
type="workflow",
workflow_model_resources={
"text_llm": RuntimeModelResource(
id="text_llm",
name="文本 Agent",
capability="LLM",
interface_type="openai-llm",
support_image_input=False,
)
},
)
with self.assertRaisesRegex(ValueError, "不支持图片输入"):
_workflow_vision_uses_main_llm(
cfg,
{
"enabled": True,
"llm_resource_id": "text_llm",
"vision_model_resource_id": None,
},
)
async def _async_none():

71
backend/tests/test_workflow_router.py
View File

@@ -72,6 +72,77 @@ class WorkflowLLMRouterTest(unittest.IsolatedAsyncioTestCase):
)
self.assertNotIn("developer", str(requests[0]["messages"]))
async def test_routes_with_the_current_multimodal_user_message(self):
requests = []
class FakeCompletions:
async def create(self, **kwargs):
requests.append(kwargs)
return SimpleNamespace(
choices=[
SimpleNamespace(
message=SimpleNamespace(
tool_calls=[
SimpleNamespace(
function=SimpleNamespace(
name="goto_confirm",
arguments="{}",
)
)
]
)
)
]
)
class FakeClient:
def __init__(self, **_kwargs):
self.chat = SimpleNamespace(completions=FakeCompletions())
async def close(self):
return None
router = WorkflowLLMRouter(
AssistantConfig(
type="workflow",
model="visual-model",
llm_api_key="secret",
llm_base_url="https://llm.test/v1",
)
)
image_message = {
"role": "user",
"content": [
{"type": "text", "text": "请检查车牌照片"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,AA=="},
},
],
}
with patch("services.workflow_router.AsyncOpenAI", FakeClient):
selected = await router.select_edge(
node_name="采集车牌",
node_prompt="确认车牌照片是否清晰",
edges=[{"id": "confirm", "data": {"condition": "车牌清晰"}}],
history=[
{"role": "user", "message": "之前的消息"},
{"role": "user", "message": "请检查车牌照片"},
],
variables={},
edge_name=lambda _edge: "goto_confirm",
edge_description=lambda _edge: "车牌清晰",
current_user_message=image_message,
)
self.assertEqual(selected.status, RouteStatus.MATCHED)
content = requests[0]["messages"][1]["content"]
self.assertIsInstance(content, list)
self.assertEqual(content[-1], image_message["content"][-1])
self.assertIn("之前的消息", content[0]["text"])
self.assertEqual(content[0]["text"].count("请检查车牌照片"), 0)
if __name__ == "__main__":
unittest.main()