feat: route workflow image inputs natively

This commit is contained in:
Xin Wang
2026-08-03 10:55:57 +08:00
parent 2e84de0798
commit f3439b21d1
10 changed files with 412 additions and 33 deletions

View File

@@ -6,6 +6,7 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
from models import AssistantConfig, RuntimeTool
from pipecat.flows import FlowManager
from pipecat.frames.frames import (
LLMContextFrame,
LLMFullResponseEndFrame,
@@ -28,7 +29,7 @@ from services.brains.dify_llm import (
last_user_text,
normalize_api_base,
)
from services.brains.workflow_brain import WorkflowBrain
from services.brains.workflow_brain import ConfiguredFlowManager, WorkflowBrain
from services.runtime_variables import prepare_dynamic_config
from services.action_runtime import ActionOutcome, ActionStatus
from services.workflow.models import (
@@ -789,6 +790,45 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
async def test_flow_manager_dispatches_native_vision_without_auxiliary_handler(self):
manager = object.__new__(ConfiguredFlowManager)
fallback_transition = AsyncMock()
native_handler = AsyncMock()
native_enabled = {"value": True}
async def flow_handler(_args, _manager):
return {"status": "ok"}
setattr(
flow_handler,
"_workflow_native_vision_handler",
native_handler,
)
setattr(
flow_handler,
"_workflow_native_vision_enabled",
lambda: native_enabled["value"],
)
with patch.object(
FlowManager,
"_create_transition_func",
new=AsyncMock(return_value=fallback_transition),
):
transition = await manager._create_transition_func(
"fetch_user_image",
flow_handler,
)
params = SimpleNamespace()
await transition(params)
native_handler.assert_awaited_once_with(params)
fallback_transition.assert_not_awaited()
native_enabled["value"] = False
await transition(params)
fallback_transition.assert_awaited_once_with(params)
def test_client_tool_session_wait_disables_flow_timeout(self):
brain = WorkflowBrain(
{
@@ -1922,9 +1962,11 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
class FakeRouter:
def __init__(self):
self.calls = 0
self.current_user_message = None
async def select_edge(self, **_kwargs):
async def select_edge(self, **kwargs):
self.calls += 1
self.current_user_message = kwargs.get("current_user_message")
return LLMRouteResult(
status=RouteStatus.MATCHED,
function_name="goto_eat",
@@ -1939,13 +1981,27 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(manager.current_node, "start")
self.assertEqual(router.calls, 0)
handled = await brain.on_user_turn_end("我想吃饭")
image_message = {
"role": "user",
"content": [
{"type": "text", "text": "我想吃饭"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,AA=="},
},
],
}
handled = await brain.on_user_turn_end(
"我想吃饭",
user_message=image_message,
)
self.assertTrue(handled)
self.assertEqual(router.calls, 1)
self.assertEqual(router.current_user_message, image_message)
self.assertEqual(manager.current_node, "eat")
self.assertIn(
{"role": "user", "content": "我想吃饭"},
image_message,
manager.config["task_messages"],
)
self.assertTrue(any(isinstance(frame, LLMRunFrame) for frame in queued))