feat: route workflow image inputs natively

This commit is contained in:
Xin Wang
2026-08-03 10:55:57 +08:00
parent 2e84de0798
commit f3439b21d1
10 changed files with 412 additions and 33 deletions

View File

@@ -166,11 +166,18 @@ class BaseBrain:
def record_user_message(self, content: str) -> None: def record_user_message(self, content: str) -> None:
"""Observe a committed user message for brain-owned routing state.""" """Observe a committed user message for brain-owned routing state."""
async def on_user_turn_end(self, content: str) -> bool: async def on_user_turn_end(
self,
content: str,
user_message: dict[str, Any] | None = None,
) -> bool:
"""Handle a complete user turn before the conversational LLM runs. """Handle a complete user turn before the conversational LLM runs.
Return True when the brain scheduled the next action itself and the Return True when the brain scheduled the next action itself and the
in-flight context frame must not reach the previous Agent's LLM. in-flight context frame must not reach the previous Agent's LLM.
``user_message`` preserves the current universal-context message for
brains that need multimodal routing. Text-only brains can ignore it.
""" """
self.record_user_message(content) self.record_user_message(content)
return False return False
@@ -222,7 +229,11 @@ class Brain(Protocol):
def record_user_message(self, content: str) -> None: ... def record_user_message(self, content: str) -> None: ...
async def on_user_turn_end(self, content: str) -> bool: ... async def on_user_turn_end(
self,
content: str,
user_message: dict[str, Any] | None = None,
) -> bool: ...
async def on_assistant_text_start(self, turn_id: str) -> None: ... async def on_assistant_text_start(self, turn_id: str) -> None: ...

View File

@@ -69,8 +69,9 @@ class _MessageContinuation:
token: int token: int
node_id: str node_id: str
context_messages: list[dict[str, str]] context_messages: list[dict[str, Any]]
triggering_user_text: str triggering_user_text: str
triggering_user_message: dict[str, Any] | None
task: asyncio.Task[None] | None = None task: asyncio.Task[None] | None = None
@@ -88,6 +89,26 @@ class ConfiguredFlowManager(FlowManager):
async def _create_transition_func(self, name, handler): async def _create_transition_func(self, name, handler):
transition = await super()._create_transition_func(name, handler) transition = await super()._create_transition_func(name, handler)
native_vision_handler = getattr(
handler,
"_workflow_native_vision_handler",
None,
)
native_vision_enabled = getattr(
handler,
"_workflow_native_vision_enabled",
None,
)
if callable(native_vision_handler) and callable(native_vision_enabled):
fallback_transition = transition
async def vision_transition(params: FunctionCallParams) -> None:
if native_vision_enabled():
await native_vision_handler(params)
return
await fallback_transition(params)
transition = vision_transition
if not getattr(handler, "_suppress_followup_llm", False): if not getattr(handler, "_suppress_followup_llm", False):
return transition return transition
@@ -290,14 +311,26 @@ class WorkflowBrain(BaseBrain):
if content and not self._ended: if content and not self._ended:
self._store.record("user", content) self._store.record("user", content)
async def on_user_turn_end(self, content: str) -> bool: async def on_user_turn_end(
self,
content: str,
user_message: dict[str, Any] | None = None,
) -> bool:
"""Route a complete user turn before the active stage may reply.""" """Route a complete user turn before the active stage may reply."""
if not content or self._ended: if not content or self._ended:
return True return True
async with self._turn_lock: async with self._turn_lock:
return await self._handle_user_turn_end(content) return await self._handle_user_turn_end(
content,
user_message=user_message,
)
async def _handle_user_turn_end(self, content: str) -> bool: async def _handle_user_turn_end(
self,
content: str,
*,
user_message: dict[str, Any] | None = None,
) -> bool:
"""Serialized implementation so one user turn cannot transition twice.""" """Serialized implementation so one user turn cannot transition twice."""
self.record_user_message(content) self.record_user_message(content)
self._state.begin_user_turn(content) self._state.begin_user_turn(content)
@@ -307,7 +340,10 @@ class WorkflowBrain(BaseBrain):
return True return True
self._state.status = WorkflowStatus.ROUTING self._state.status = WorkflowStatus.ROUTING
decision = await self._edge_evaluator.evaluate(current) decision = await self._edge_evaluator.evaluate(
current,
current_user_message=user_message,
)
if decision.status == RouteStatus.ERROR: if decision.status == RouteStatus.ERROR:
await self._require_output().emit_error( await self._require_output().emit_error(
decision.error or "工作流路由失败", decision.error or "工作流路由失败",
@@ -320,6 +356,7 @@ class WorkflowBrain(BaseBrain):
next_config = await self._follow_edge( next_config = await self._follow_edge(
decision.edge, decision.edge,
triggering_user_text=content, triggering_user_text=content,
triggering_user_message=user_message,
) )
await self._activate_node_config( await self._activate_node_config(
next_config, next_config,
@@ -387,7 +424,7 @@ class WorkflowBrain(BaseBrain):
def _agent_config( def _agent_config(
self, self,
node_id: str, node_id: str,
leading_messages: list[dict[str, str]] | None = None, leading_messages: list[dict[str, Any]] | None = None,
) -> NodeConfig: ) -> NodeConfig:
stage = self._engine.agent_stage_config(node_id) stage = self._engine.agent_stage_config(node_id)
functions: list[FlowsFunctionSchema] = [] functions: list[FlowsFunctionSchema] = []
@@ -468,7 +505,7 @@ class WorkflowBrain(BaseBrain):
def _passive_node_config( def _passive_node_config(
self, self,
node_id: str, node_id: str,
task_messages: list[dict[str, str]] | None = None, task_messages: list[dict[str, Any]] | None = None,
) -> NodeConfig: ) -> NodeConfig:
"""Keep a non-conversational terminal node active without ending the call.""" """Keep a non-conversational terminal node active without ending the call."""
return { return {
@@ -643,8 +680,9 @@ class WorkflowBrain(BaseBrain):
self, self,
edge: dict, edge: dict,
*, *,
leading_messages: list[dict[str, str]] | None = None, leading_messages: list[dict[str, Any]] | None = None,
triggering_user_text: str = "", triggering_user_text: str = "",
triggering_user_message: dict[str, Any] | None = None,
) -> NodeConfig: ) -> NodeConfig:
await self._begin_edge_transition(edge) await self._begin_edge_transition(edge)
context_messages = list(leading_messages or []) context_messages = list(leading_messages or [])
@@ -664,14 +702,16 @@ class WorkflowBrain(BaseBrain):
str(edge.get("target") or ""), str(edge.get("target") or ""),
leading_messages=context_messages, leading_messages=context_messages,
triggering_user_text=triggering_user_text, triggering_user_text=triggering_user_text,
triggering_user_message=triggering_user_message,
) )
async def _resolve_path( async def _resolve_path(
self, self,
node_id: str, node_id: str,
*, *,
leading_messages: list[dict[str, str]] | None = None, leading_messages: list[dict[str, Any]] | None = None,
triggering_user_text: str = "", triggering_user_text: str = "",
triggering_user_message: dict[str, Any] | None = None,
) -> NodeConfig: ) -> NodeConfig:
context_messages = list(leading_messages or []) context_messages = list(leading_messages or [])
for hop in range(MAX_AUTOMATIC_HOPS): for hop in range(MAX_AUTOMATIC_HOPS):
@@ -684,8 +724,13 @@ class WorkflowBrain(BaseBrain):
triggering_user_text triggering_user_text
and self._engine.data(node_id).get("contextPolicy") == "fresh" and self._engine.data(node_id).get("contextPolicy") == "fresh"
): ):
current_user_message = (
deepcopy(triggering_user_message)
if triggering_user_message
else {"role": "user", "content": triggering_user_text}
)
agent_messages = [ agent_messages = [
{"role": "user", "content": triggering_user_text}, current_user_message,
*context_messages, *context_messages,
] ]
return self._agent_config(node_id, agent_messages) return self._agent_config(node_id, agent_messages)
@@ -701,6 +746,7 @@ class WorkflowBrain(BaseBrain):
node_id, node_id,
context_messages=context_messages, context_messages=context_messages,
triggering_user_text=triggering_user_text, triggering_user_text=triggering_user_text,
triggering_user_message=triggering_user_message,
) )
return self._passive_node_config(node_id, context_messages) return self._passive_node_config(node_id, context_messages)
elif node_type == "handoff": elif node_type == "handoff":
@@ -736,8 +782,9 @@ class WorkflowBrain(BaseBrain):
self, self,
node_id: str, node_id: str,
*, *,
context_messages: list[dict[str, str]], context_messages: list[dict[str, Any]],
triggering_user_text: str, triggering_user_text: str,
triggering_user_message: dict[str, Any] | None,
) -> None: ) -> None:
"""Save the path state without waiting inside the pipeline call stack.""" """Save the path state without waiting inside the pipeline call stack."""
token = self._next_message_token token = self._next_message_token
@@ -747,6 +794,7 @@ class WorkflowBrain(BaseBrain):
node_id=node_id, node_id=node_id,
context_messages=[dict(message) for message in context_messages], context_messages=[dict(message) for message in context_messages],
triggering_user_text=triggering_user_text, triggering_user_text=triggering_user_text,
triggering_user_message=deepcopy(triggering_user_message),
) )
self._state.enter(node_id, WorkflowStatus.RUNNING_MESSAGE) self._state.enter(node_id, WorkflowStatus.RUNNING_MESSAGE)
@@ -847,6 +895,7 @@ class WorkflowBrain(BaseBrain):
edge, edge,
leading_messages=context_messages, leading_messages=context_messages,
triggering_user_text=continuation.triggering_user_text, triggering_user_text=continuation.triggering_user_text,
triggering_user_message=continuation.triggering_user_message,
) )
await self._activate_node_config( await self._activate_node_config(
next_config, next_config,

View File

@@ -141,6 +141,25 @@ def _vision_uses_main_llm(cfg: AssistantConfig) -> bool:
return not cfg.vision_model_resource_id and cfg.llm_support_image_input return not cfg.vision_model_resource_id and cfg.llm_support_image_input
def _workflow_vision_uses_main_llm(
cfg: AssistantConfig,
scope: dict[str, Any],
) -> bool:
"""Resolve native versus auxiliary vision for the active Workflow Agent."""
if not scope.get("enabled"):
raise ValueError("当前 Workflow Agent 节点未启用视觉能力")
if scope.get("vision_model_resource_id"):
return False
llm_resource_id = str(scope.get("llm_resource_id") or "")
resource = cfg.workflow_model_resources.get(llm_resource_id)
if not resource:
raise ValueError(f"当前 Workflow Agent 的 LLM 资源未加载:{llm_resource_id}")
if not resource.support_image_input:
raise ValueError("当前 Workflow Agent 的 LLM 不支持图片输入")
return True
def _image_data_uri(frame: UserImageRawFrame) -> str: def _image_data_uri(frame: UserImageRawFrame) -> str:
if not frame.format: if not frame.format:
raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型") raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型")
@@ -388,6 +407,11 @@ async def run_pipeline(
"vision_model_resource_id": None, "vision_model_resource_id": None,
"llm_resource_id": None, "llm_resource_id": None,
} }
def active_vision_uses_main_llm() -> bool:
if cfg.type == "workflow":
return _workflow_vision_uses_main_llm(cfg, workflow_vision_scope)
return vision_native_mode
vision_schema = FunctionSchema( vision_schema = FunctionSchema(
name=VISION_TOOL_NAME, name=VISION_TOOL_NAME,
description=( description=(
@@ -510,6 +534,27 @@ async def run_pipeline(
raise ValueError(f"视觉模型资源未加载:{vision_resource_id}") raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
if cfg.type == "workflow" and vision_enabled: if cfg.type == "workflow" and vision_enabled:
async def native_flow_fetch_user_image(params: FunctionCallParams) -> None:
question = str(params.arguments.get("question") or "请描述当前画面。")
user_id = vision_state.get("client_id")
if not user_id:
await params.result_callback(
{
"status": "no_video_client",
"message": "当前还没有可用的摄像头视频流。",
}
)
return
request = UserImageRequestFrame(
user_id=user_id,
text=question,
append_to_context=True,
function_name=params.function_name,
tool_call_id=params.tool_call_id,
result_callback=params.result_callback,
)
await params.llm.push_frame(request, FrameDirection.UPSTREAM)
async def flow_fetch_user_image(args, _flow_manager): async def flow_fetch_user_image(args, _flow_manager):
if not workflow_vision_scope.get("enabled"): if not workflow_vision_scope.get("enabled"):
return { return {
@@ -547,6 +592,20 @@ async def run_pipeline(
logger.warning(f"Workflow 视觉理解失败:{exc}") logger.warning(f"Workflow 视觉理解失败:{exc}")
return {"status": "error", "message": "视觉理解暂时不可用。"} return {"status": "error", "message": "视觉理解暂时不可用。"}
# ConfiguredFlowManager keeps the Flows handler for auxiliary models,
# but uses Pipecat's native function-call image path when the active
# Agent selected its own visual-capable LLM.
setattr(
flow_fetch_user_image,
"_workflow_native_vision_handler",
native_flow_fetch_user_image,
)
setattr(
flow_fetch_user_image,
"_workflow_native_vision_enabled",
active_vision_uses_main_llm,
)
workflow_vision_function = FlowsFunctionSchema( workflow_vision_function = FlowsFunctionSchema(
name=VISION_TOOL_NAME, name=VISION_TOOL_NAME,
description=vision_schema.description, description=vision_schema.description,
@@ -710,7 +769,8 @@ async def run_pipeline(
user_id = vision_state.get("client_id") user_id = vision_state.get("client_id")
if not user_id: if not user_id:
raise ValueError("当前没有可用的摄像头视频流") raise ValueError("当前没有可用的摄像头视频流")
analysis_cfg = None if vision_native_mode else active_vision_config() native_vision = active_vision_uses_main_llm()
analysis_cfg = None if native_vision else active_vision_config()
request = UserImageRequestFrame( request = UserImageRequestFrame(
user_id=user_id, user_id=user_id,
@@ -722,7 +782,7 @@ async def run_pipeline(
except asyncio.TimeoutError as exc: except asyncio.TimeoutError as exc:
raise ValueError("等待摄像头视频帧超时") from exc raise ValueError("等待摄像头视频帧超时") from exc
if vision_native_mode: if native_vision:
image_frame.text = value.prompt_text image_frame.text = value.prompt_text
image_frame.append_to_context = True image_frame.append_to_context = True
image_frame.request = None image_frame.request = None

View File

@@ -713,7 +713,10 @@ class UserTurnRoutingProcessor(FrameProcessor):
self._last_user_message = user_message self._last_user_message = user_message
content = message_text(user_message) content = message_text(user_message)
handled = await self._brain.on_user_turn_end(content) handled = await self._brain.on_user_turn_end(
content,
user_message=user_message,
)
if not handled: if not handled:
await self.push_frame(frame, direction) await self.push_frame(frame, direction)

View File

@@ -2,6 +2,8 @@
from __future__ import annotations from __future__ import annotations
from typing import Any
from models import AssistantConfig from models import AssistantConfig
from pipecat.flows import ContextStrategy, ContextStrategyConfig, NodeConfig from pipecat.flows import ContextStrategy, ContextStrategyConfig, NodeConfig
from pipecat.frames.frames import LLMUpdateSettingsFrame from pipecat.frames.frames import LLMUpdateSettingsFrame
@@ -106,7 +108,7 @@ class WorkflowAgentStage:
node_id: str, node_id: str,
*, *,
functions: list, functions: list,
leading_messages: list[dict[str, str]] | None = None, leading_messages: list[dict[str, Any]] | None = None,
) -> NodeConfig: ) -> NodeConfig:
data = self._engine.data(node_id) data = self._engine.data(node_id)
strategy = ( strategy = (

View File

@@ -3,6 +3,7 @@
from __future__ import annotations from __future__ import annotations
from collections.abc import Callable from collections.abc import Callable
from typing import Any
from services.runtime_variables import DynamicVariableStore from services.runtime_variables import DynamicVariableStore
from services.workflow.models import EdgeEvaluation, RouteStatus from services.workflow.models import EdgeEvaluation, RouteStatus
@@ -23,7 +24,12 @@ class WorkflowEdgeEvaluator:
self._store = store self._store = store
self._router_for_node = router_for_node self._router_for_node = router_for_node
async def evaluate(self, node_id: str) -> EdgeEvaluation: async def evaluate(
self,
node_id: str,
*,
current_user_message: dict[str, Any] | None = None,
) -> EdgeEvaluation:
"""Select the first matching conditional path, then the default path.""" """Select the first matching conditional path, then the default path."""
outgoing = self._engine.outgoing(node_id) outgoing = self._engine.outgoing(node_id)
expression_edge = self._engine.deterministic_edge( expression_edge = self._engine.deterministic_edge(
@@ -61,6 +67,7 @@ class WorkflowEdgeEvaluator:
node_prompt=self._engine.routing_prompt(node_id, self._store), node_prompt=self._engine.routing_prompt(node_id, self._store),
edges=llm_edges, edges=llm_edges,
history=self._store.history, history=self._store.history,
current_user_message=current_user_message,
variables={ variables={
key: value key: value
for key, value in self._store.values.items() for key, value in self._store.values.items()
@@ -94,4 +101,3 @@ class WorkflowEdgeEvaluator:
if edge is None: if edge is None:
return EdgeEvaluation(status=RouteStatus.NO_MATCH) return EdgeEvaluation(status=RouteStatus.NO_MATCH)
return EdgeEvaluation(status=RouteStatus.MATCHED, edge=edge) return EdgeEvaluation(status=RouteStatus.MATCHED, edge=edge)

View File

@@ -9,6 +9,7 @@ from __future__ import annotations
import json import json
from collections.abc import Callable from collections.abc import Callable
from copy import deepcopy
from typing import Any from typing import Any
from loguru import logger from loguru import logger
@@ -24,6 +25,32 @@ STAY_ON_CURRENT_AGENT = STAY_ON_CURRENT_NODE
MAX_ROUTING_HISTORY_ENTRIES = 20 MAX_ROUTING_HISTORY_ENTRIES = 20
def _routing_user_message(
routing_input: str,
current_user_message: dict[str, Any] | None,
) -> dict[str, Any]:
"""Combine routing metadata with the current text or multimodal turn."""
if not current_user_message:
return {"role": "user", "content": routing_input}
content = current_user_message.get("content")
if not isinstance(content, list):
current_text = str(content or "").strip()
suffix = f"\n\n[当前用户输入]\n{current_text}" if current_text else ""
return {"role": "user", "content": f"{routing_input}{suffix}"}
return {
"role": "user",
"content": [
{
"type": "text",
"text": f"{routing_input}\n\n[当前用户输入如下]",
},
*deepcopy(content),
],
}
class WorkflowLLMRouter: class WorkflowLLMRouter:
"""Select one LLM edge without allowing the router to speak.""" """Select one LLM edge without allowing the router to speak."""
@@ -40,6 +67,7 @@ class WorkflowLLMRouter:
variables: dict[str, Any], variables: dict[str, Any],
edge_name: Callable[[dict[str, Any]], str], edge_name: Callable[[dict[str, Any]], str],
edge_description: Callable[[dict[str, Any]], str], edge_description: Callable[[dict[str, Any]], str],
current_user_message: dict[str, Any] | None = None,
) -> LLMRouteResult: ) -> LLMRouteResult:
"""Return a typed match, no-match or technical error.""" """Return a typed match, no-match or technical error."""
if not edges: if not edges:
@@ -85,7 +113,13 @@ class WorkflowLLMRouter:
f"当前节点任务:{node_prompt or '未配置'}\n" f"当前节点任务:{node_prompt or '未配置'}\n"
f"转移条件:\n{ordered_conditions}" f"转移条件:\n{ordered_conditions}"
) )
recent_history = history[-MAX_ROUTING_HISTORY_ENTRIES:] # WorkflowBrain records the current turn before routing. When the full
# current message is supplied separately, keep only earlier history so
# the text is not duplicated and the image remains attached to its turn.
routing_history = (
history[:-1] if current_user_message and history else history
)
recent_history = routing_history[-MAX_ROUTING_HISTORY_ENTRIES:]
routing_input = json.dumps( routing_input = json.dumps(
{ {
"conversation": recent_history, "conversation": recent_history,
@@ -108,7 +142,7 @@ class WorkflowLLMRouter:
model=self._cfg.model, model=self._cfg.model,
messages=[ messages=[
{"role": "system", "content": router_prompt}, {"role": "system", "content": router_prompt},
{"role": "user", "content": routing_input}, _routing_user_message(routing_input, current_user_message),
], ],
tools=tools, tools=tools,
tool_choice="required", tool_choice="required",

View File

@@ -6,6 +6,7 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, patch from unittest.mock import AsyncMock, patch
from models import AssistantConfig, RuntimeTool from models import AssistantConfig, RuntimeTool
from pipecat.flows import FlowManager
from pipecat.frames.frames import ( from pipecat.frames.frames import (
LLMContextFrame, LLMContextFrame,
LLMFullResponseEndFrame, LLMFullResponseEndFrame,
@@ -28,7 +29,7 @@ from services.brains.dify_llm import (
last_user_text, last_user_text,
normalize_api_base, normalize_api_base,
) )
from services.brains.workflow_brain import WorkflowBrain from services.brains.workflow_brain import ConfiguredFlowManager, WorkflowBrain
from services.runtime_variables import prepare_dynamic_config from services.runtime_variables import prepare_dynamic_config
from services.action_runtime import ActionOutcome, ActionStatus from services.action_runtime import ActionOutcome, ActionStatus
from services.workflow.models import ( from services.workflow.models import (
@@ -789,6 +790,45 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
async def test_flow_manager_dispatches_native_vision_without_auxiliary_handler(self):
manager = object.__new__(ConfiguredFlowManager)
fallback_transition = AsyncMock()
native_handler = AsyncMock()
native_enabled = {"value": True}
async def flow_handler(_args, _manager):
return {"status": "ok"}
setattr(
flow_handler,
"_workflow_native_vision_handler",
native_handler,
)
setattr(
flow_handler,
"_workflow_native_vision_enabled",
lambda: native_enabled["value"],
)
with patch.object(
FlowManager,
"_create_transition_func",
new=AsyncMock(return_value=fallback_transition),
):
transition = await manager._create_transition_func(
"fetch_user_image",
flow_handler,
)
params = SimpleNamespace()
await transition(params)
native_handler.assert_awaited_once_with(params)
fallback_transition.assert_not_awaited()
native_enabled["value"] = False
await transition(params)
fallback_transition.assert_awaited_once_with(params)
def test_client_tool_session_wait_disables_flow_timeout(self): def test_client_tool_session_wait_disables_flow_timeout(self):
brain = WorkflowBrain( brain = WorkflowBrain(
{ {
@@ -1922,9 +1962,11 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
class FakeRouter: class FakeRouter:
def __init__(self): def __init__(self):
self.calls = 0 self.calls = 0
self.current_user_message = None
async def select_edge(self, **_kwargs): async def select_edge(self, **kwargs):
self.calls += 1 self.calls += 1
self.current_user_message = kwargs.get("current_user_message")
return LLMRouteResult( return LLMRouteResult(
status=RouteStatus.MATCHED, status=RouteStatus.MATCHED,
function_name="goto_eat", function_name="goto_eat",
@@ -1939,13 +1981,27 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(manager.current_node, "start") self.assertEqual(manager.current_node, "start")
self.assertEqual(router.calls, 0) self.assertEqual(router.calls, 0)
handled = await brain.on_user_turn_end("我想吃饭") image_message = {
"role": "user",
"content": [
{"type": "text", "text": "我想吃饭"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,AA=="},
},
],
}
handled = await brain.on_user_turn_end(
"我想吃饭",
user_message=image_message,
)
self.assertTrue(handled) self.assertTrue(handled)
self.assertEqual(router.calls, 1) self.assertEqual(router.calls, 1)
self.assertEqual(router.current_user_message, image_message)
self.assertEqual(manager.current_node, "eat") self.assertEqual(manager.current_node, "eat")
self.assertIn( self.assertIn(
{"role": "user", "content": "我想吃饭"}, image_message,
manager.config["task_messages"], manager.config["task_messages"],
) )
self.assertTrue(any(isinstance(frame, LLMRunFrame) for frame in queued)) self.assertTrue(any(isinstance(frame, LLMRunFrame) for frame in queued))

View File

@@ -1,6 +1,6 @@
import unittest import unittest
from models import AssistantConfig from models import AssistantConfig, RuntimeModelResource
from pipecat.frames.frames import LLMContextFrame from pipecat.frames.frames import LLMContextFrame
from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
@@ -9,6 +9,7 @@ from services.pipecat.pipeline import (
KnowledgeRetrievalProcessor, KnowledgeRetrievalProcessor,
UserTurnRoutingProcessor, UserTurnRoutingProcessor,
_knowledge_tool_description, _knowledge_tool_description,
_workflow_vision_uses_main_llm,
) )
@@ -67,8 +68,8 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
def __init__(self): def __init__(self):
self.turns = [] self.turns = []
async def on_user_turn_end(self, content): async def on_user_turn_end(self, content, user_message=None):
self.turns.append(content) self.turns.append((content, user_message))
return True return True
brain = FakeBrain() brain = FakeBrain()
@@ -83,13 +84,19 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
frame = LLMContextFrame(context) frame = LLMContextFrame(context)
await processor.process_frame(frame, FrameDirection.DOWNSTREAM) await processor.process_frame(frame, FrameDirection.DOWNSTREAM)
self.assertEqual(brain.turns, ["我叫李白"]) self.assertEqual(
brain.turns,
[("我叫李白", {"role": "user", "content": "我叫李白"})],
)
self.assertEqual(forwarded, []) self.assertEqual(forwarded, [])
# A queued LLMRunFrame after the transition uses the same context. It # A queued LLMRunFrame after the transition uses the same context. It
# must reach the target Agent without invoking routing a second time. # must reach the target Agent without invoking routing a second time.
await processor.process_frame(frame, FrameDirection.DOWNSTREAM) await processor.process_frame(frame, FrameDirection.DOWNSTREAM)
self.assertEqual(brain.turns, ["我叫李白"]) self.assertEqual(
brain.turns,
[("我叫李白", {"role": "user", "content": "我叫李白"})],
)
self.assertEqual(forwarded, [(frame, FrameDirection.DOWNSTREAM)]) self.assertEqual(forwarded, [(frame, FrameDirection.DOWNSTREAM)])
async def test_routes_multimodal_user_message_by_its_text_part(self): async def test_routes_multimodal_user_message_by_its_text_part(self):
@@ -97,8 +104,8 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
def __init__(self): def __init__(self):
self.turns = [] self.turns = []
async def on_user_turn_end(self, content): async def on_user_turn_end(self, content, user_message=None):
self.turns.append(content) self.turns.append((content, user_message))
return False return False
brain = FakeBrain() brain = FakeBrain()
@@ -124,7 +131,87 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
FrameDirection.DOWNSTREAM, FrameDirection.DOWNSTREAM,
) )
self.assertEqual(brain.turns, ["看看这张照片"]) self.assertEqual(
brain.turns,
[
(
"看看这张照片",
{
"role": "user",
"content": [
{"type": "text", "text": "看看这张照片"},
{
"type": "image_url",
"image_url": {
"url": "data:image/jpeg;base64,AA=="
},
},
],
},
)
],
)
class WorkflowVisionModeTest(unittest.TestCase):
def test_uses_active_agent_llm_only_without_auxiliary_model(self):
cfg = AssistantConfig(
type="workflow",
workflow_model_resources={
"agent_llm": RuntimeModelResource(
id="agent_llm",
name="视觉 Agent",
capability="LLM",
interface_type="openai-llm",
support_image_input=True,
)
},
)
self.assertTrue(
_workflow_vision_uses_main_llm(
cfg,
{
"enabled": True,
"llm_resource_id": "agent_llm",
"vision_model_resource_id": None,
},
)
)
self.assertFalse(
_workflow_vision_uses_main_llm(
cfg,
{
"enabled": True,
"llm_resource_id": "agent_llm",
"vision_model_resource_id": "auxiliary_vision",
},
)
)
def test_rejects_a_non_visual_active_agent_llm(self):
cfg = AssistantConfig(
type="workflow",
workflow_model_resources={
"text_llm": RuntimeModelResource(
id="text_llm",
name="文本 Agent",
capability="LLM",
interface_type="openai-llm",
support_image_input=False,
)
},
)
with self.assertRaisesRegex(ValueError, "不支持图片输入"):
_workflow_vision_uses_main_llm(
cfg,
{
"enabled": True,
"llm_resource_id": "text_llm",
"vision_model_resource_id": None,
},
)
async def _async_none(): async def _async_none():

View File

@@ -72,6 +72,77 @@ class WorkflowLLMRouterTest(unittest.IsolatedAsyncioTestCase):
) )
self.assertNotIn("developer", str(requests[0]["messages"])) self.assertNotIn("developer", str(requests[0]["messages"]))
async def test_routes_with_the_current_multimodal_user_message(self):
requests = []
class FakeCompletions:
async def create(self, **kwargs):
requests.append(kwargs)
return SimpleNamespace(
choices=[
SimpleNamespace(
message=SimpleNamespace(
tool_calls=[
SimpleNamespace(
function=SimpleNamespace(
name="goto_confirm",
arguments="{}",
)
)
]
)
)
]
)
class FakeClient:
def __init__(self, **_kwargs):
self.chat = SimpleNamespace(completions=FakeCompletions())
async def close(self):
return None
router = WorkflowLLMRouter(
AssistantConfig(
type="workflow",
model="visual-model",
llm_api_key="secret",
llm_base_url="https://llm.test/v1",
)
)
image_message = {
"role": "user",
"content": [
{"type": "text", "text": "请检查车牌照片"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,AA=="},
},
],
}
with patch("services.workflow_router.AsyncOpenAI", FakeClient):
selected = await router.select_edge(
node_name="采集车牌",
node_prompt="确认车牌照片是否清晰",
edges=[{"id": "confirm", "data": {"condition": "车牌清晰"}}],
history=[
{"role": "user", "message": "之前的消息"},
{"role": "user", "message": "请检查车牌照片"},
],
variables={},
edge_name=lambda _edge: "goto_confirm",
edge_description=lambda _edge: "车牌清晰",
current_user_message=image_message,
)
self.assertEqual(selected.status, RouteStatus.MATCHED)
content = requests[0]["messages"][1]["content"]
self.assertIsInstance(content, list)
self.assertEqual(content[-1], image_message["content"][-1])
self.assertIn("之前的消息", content[0]["text"])
self.assertEqual(content[0]["text"].count("请检查车牌照片"), 0)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()