wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance workflow routing and agent configuration management

Browse Source 
- Introduce WorkflowLLMRouter for pre-response LLM routing, allowing agents to determine the appropriate function to call based on user input.
- Implement UserTurnRoutingProcessor to manage user turns before reaching the LLM, ensuring proper routing and handling of user messages.
- Refactor WorkflowBrain to integrate new routing logic and enhance agent stage configuration, including entry modes and resource management.
- Update service factory to support dynamic LLM resource configuration based on workflow settings.
- Add tests for new routing functionality and ensure proper handling of user messages in various scenarios.
This commit is contained in:
Xin Wang
2026-07-14 09:36:28 +08:00
parent 32aef14ddb
commit 72856bf3a7
19 changed files with 2611 additions and 552 deletions
Download Patch File Download Diff File Expand all files Collapse all files

206
backend/tests/test_brains.py
View File

@@ -9,7 +9,10 @@ from pipecat.frames.frames import (
LLMContextFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMRunFrame,
LLMTextFrame,
OutputTransportMessageUrgentFrame,
TTSSpeakFrame,
)
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameDirection
@@ -388,10 +391,108 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
async def test_nodes_without_outgoing_edges_remain_active(self):
queued = []
async def queue_frame(frame):
queued.append(frame)
runtime = BrainRuntime(
context=LLMContext(messages=[]),
llm=FakeLLM(),
queue_frame=queue_frame,
set_system_prompt=lambda _prompt: None,
set_tools=lambda _tools: None,
call_end=FakeCallEnd(),
)
class FakeManager:
def __init__(self, current_node=None):
self.current_node = current_node
async def initialize(self, config):
self.current_node = config["name"]
start_brain = WorkflowBrain(
{
"specVersion": 3,
"settings": {},
"nodes": [{"id": "start", "type": "start", "data": {}}],
"edges": [],
}
)
start_brain._runtime = runtime
start_brain._manager = FakeManager()
await start_brain.on_connected()
self.assertEqual(start_brain._manager.current_node, "start")
agent_brain = WorkflowBrain(
{
"specVersion": 3,
"settings": {"globalPrompt": "全局规则"},
"nodes": [
{"id": "start", "type": "start", "data": {}},
{
"id": "agent",
"type": "agent",
"data": {"prompt": "持续回答"},
},
],
"edges": [
{
"id": "begin",
"source": "start",
"target": "agent",
"data": {"mode": "always", "priority": 0},
}
],
}
)
agent_brain._runtime = runtime
agent_brain._manager = FakeManager("agent")
queued.clear()
handled = await agent_brain.on_user_turn_end("请继续回答")
self.assertTrue(handled)
self.assertEqual(agent_brain._manager.current_node, "agent")
self.assertTrue(any(isinstance(frame, LLMRunFrame) for frame in queued))
handoff_brain = WorkflowBrain(
{
"specVersion": 3,
"settings": {},
"nodes": [
{"id": "start", "type": "start", "data": {}},
{
"id": "handoff",
"type": "handoff",
"data": {"targetType": "human"},
},
],
"edges": [],
}
)
handoff_brain._runtime = runtime
handoff_config = await handoff_brain._resolve_path("handoff")
self.assertEqual(handoff_config["name"], "handoff")
self.assertTrue(
any(
isinstance(frame, OutputTransportMessageUrgentFrame)
and frame.message.get("type") == "handoff-requested"
for frame in queued
)
)
async def test_transition_and_end_are_owned_by_workflow_brain(self):
graph = {
"specVersion": 3,
"settings": {"globalPrompt": "全局规则"},
"settings": {
"globalPrompt": "全局规则",
"defaultLlmResourceId": "llm_global",
"defaultAsrResourceId": "asr_global",
"defaultTtsResourceId": "tts_global",
"knowledgeBaseId": "kb_global",
"knowledgeMode": "automatic",
},
"nodes": [
{
"id": "start",
@@ -428,6 +529,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
"mode": "llm",
"priority": 10,
"condition": "需求已收集",
"transitionSpeech": "正在为你结束流程",
},
}
],
@@ -447,6 +549,8 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
llm = FakeLLM()
context = LLMContext(messages=[])
queued = []
service_switches = []
knowledge_scopes = []
call_end = FakeCallEnd()
class FakeWorker:
@@ -478,6 +582,9 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
async def queue_frame(frame):
queued.append(frame)
async def switch_services(llm_id, asr_id, tts_id):
service_switches.append((llm_id, asr_id, tts_id))
runtime = BrainRuntime(
context=context,
llm=llm,
@@ -487,29 +594,112 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
call_end=call_end,
worker=worker,
context_aggregator=pair,
switch_services=switch_services,
set_knowledge_scope=knowledge_scopes.append,
)
await brain.setup(cfg, runtime)
await brain.on_connected()
self.assertEqual(brain._manager.current_node, "agent")
self.assertEqual(
service_switches,
[("llm_global", "asr_global", "tts_global")],
)
self.assertEqual(knowledge_scopes[-1]["knowledge_base_id"], "kb_global")
brain._engine.data("agent").update(
{
"inheritGlobalConfig": False,
"llmResourceId": "llm_agent",
"asrResourceId": "asr_agent",
"ttsResourceId": "tts_agent",
"knowledgeBaseId": "kb_agent",
"knowledgeMode": "on_demand",
}
)
await brain._apply_agent_stage("agent")
self.assertEqual(
service_switches[-1],
("llm_agent", "asr_agent", "tts_agent"),
)
self.assertEqual(knowledge_scopes[-1]["knowledge_base_id"], "kb_agent")
agent_config = brain._agent_config("agent")
self.assertIn("王先生", agent_config["role_message"])
self.assertIn("完成当前阶段任务", agent_config["role_message"])
self.assertIn("工作流路由已在用户一轮输入结束时完成", agent_config["role_message"])
self.assertEqual(agent_config["task_messages"], [])
self.assertFalse(agent_config["respond_immediately"])
self.assertFalse(any(isinstance(frame, LLMRunFrame) for frame in worker.frames))
self.assertEqual(
agent_config["context_strategy"].strategy.value,
"reset",
)
edge_function = next(
function
for function in brain._agent_config("agent")["functions"]
if function.name == "goto_finish"
brain._engine.data("agent")["entryMode"] = "generate"
generate_config = brain._agent_config("agent")
self.assertTrue(generate_config["respond_immediately"])
worker.frames.clear()
await brain._manager.set_node_from_config(generate_config)
self.assertTrue(any(isinstance(frame, LLMRunFrame) for frame in worker.frames))
brain._engine.data("agent").update(
{"entryMode": "fixed_speech", "entrySpeech": "您好，{{user_name}}"}
)
_, terminal = await edge_function.handler({}, brain._manager)
self.assertEqual(terminal["name"], "end")
fixed_config = brain._agent_config("agent")
self.assertFalse(fixed_config["respond_immediately"])
self.assertEqual(
fixed_config["pre_actions"][0]["type"],
"workflow_fixed_speech",
)
self.assertEqual(fixed_config["pre_actions"][0]["text"], "您好，王先生")
self.assertEqual(
fixed_config["task_messages"],
[{"role": "assistant", "content": "您好，王先生"}],
)
worker.frames.clear()
queued.clear()
await brain._manager.set_node_from_config(fixed_config)
self.assertTrue(any(isinstance(frame, TTSSpeakFrame) for frame in queued))
self.assertFalse(any(isinstance(frame, LLMRunFrame) for frame in worker.frames))
self.assertFalse(
any(
function.name == "goto_finish"
for function in brain._agent_config("agent")["functions"]
)
)
await brain.on_assistant_text_end("old-turn", "需求已收集", False)
self.assertEqual(brain._manager.current_node, "agent")
class FakeRouter:
async def select_edge(self, **_kwargs):
return "goto_finish"
brain._router = FakeRouter()
handled = await brain.on_user_turn_end("我的需求已经说完了")
self.assertTrue(handled)
self.assertEqual(brain._manager.current_node, "end")
self.assertIn("我的需求已经说完了", brain._store.values["system__conversation_history"])
self.assertTrue(call_end.ending)
self.assertTrue(call_end.armed)
self.assertTrue(any(getattr(frame, "text", "") == "感谢来电" for frame in queued))
assistant_transcripts = [
frame.message.get("content")
for frame in queued
if isinstance(frame, OutputTransportMessageUrgentFrame)
and frame.message.get("type") == "transcript"
and frame.message.get("role") == "assistant"
]
self.assertEqual(
assistant_transcripts,
["您好，王先生", "正在为你结束流程", "感谢来电"],
)
self.assertIn(
"正在为你结束流程",
brain._store.values["system__conversation_history"],
)
self.assertIn(
"感谢来电",
brain._store.values["system__conversation_history"],
)
if __name__ == "__main__":

36
backend/tests/test_pipeline_knowledge.py
View File

@@ -1,9 +1,13 @@
import unittest
from models import AssistantConfig
from pipecat.frames.frames import LLMContextFrame
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameDirection
from services.pipecat.pipeline import (
KNOWLEDGE_CONTEXT_MARKER,
KnowledgeRetrievalProcessor,
UserTurnRoutingProcessor,
_knowledge_tool_description,
)
@@ -57,5 +61,37 @@ class KnowledgeToolDescriptionTest(unittest.TestCase):
self.assertFalse(any(message["role"] == "developer" for message in messages))
class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
async def test_routes_each_user_message_once_before_response_run(self):
class FakeBrain:
def __init__(self):
self.turns = []
async def on_user_turn_end(self, content):
self.turns.append(content)
return True
brain = FakeBrain()
processor = UserTurnRoutingProcessor(brain)
forwarded = []
async def push_frame(frame, direction):
forwarded.append((frame, direction))
processor.push_frame = push_frame
context = LLMContext(messages=[{"role": "user", "content": "我叫李白"}])
frame = LLMContextFrame(context)
await processor.process_frame(frame, FrameDirection.DOWNSTREAM)
self.assertEqual(brain.turns, ["我叫李白"])
self.assertEqual(forwarded, [])
# A queued LLMRunFrame after the transition uses the same context. It
# must reach the target Agent without invoking routing a second time.
await processor.process_frame(frame, FrameDirection.DOWNSTREAM)
self.assertEqual(brain.turns, ["我叫李白"])
self.assertEqual(forwarded, [(frame, FrameDirection.DOWNSTREAM)])
if __name__ == "__main__":
unittest.main()

75
backend/tests/test_workflow_router.py Normal file
View File

@@ -0,0 +1,75 @@
from __future__ import annotations
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from models import AssistantConfig
from services.workflow_router import WorkflowLLMRouter
class WorkflowLLMRouterTest(unittest.IsolatedAsyncioTestCase):
async def test_uses_required_tool_choice_without_developer_messages(self):
requests = []
class FakeCompletions:
async def create(self, **kwargs):
requests.append(kwargs)
return SimpleNamespace(
choices=[
SimpleNamespace(
message=SimpleNamespace(
tool_calls=[
SimpleNamespace(
function=SimpleNamespace(name="goto_age", arguments="{}")
)
]
)
)
]
)
class FakeClient:
def __init__(self, **_kwargs):
self.chat = SimpleNamespace(completions=FakeCompletions())
self.closed = False
async def close(self):
self.closed = True
cfg = AssistantConfig(
type="workflow",
model="deepseek-chat",
llm_api_key="secret",
llm_base_url="https://llm.test/v1",
)
router = WorkflowLLMRouter(cfg)
edges = [
{
"id": "age",
"data": {"condition": "用户已经回答姓名", "priority": 10},
}
]
with patch("services.workflow_router.AsyncOpenAI", FakeClient):
selected = await router.select_edge(
node_name="询问姓名",
node_prompt="询问用户姓名",
edges=edges,
history=[{"role": "user", "message": "我叫李白"}],
variables={"customer_type": "new"},
edge_name=lambda _edge: "goto_age",
edge_description=lambda _edge: "用户已经回答姓名",
)
self.assertEqual(selected, "goto_age")
self.assertEqual(requests[0]["tool_choice"], "required")
self.assertEqual(
[message["role"] for message in requests[0]["messages"]],
["system", "user"],
)
self.assertNotIn("developer", str(requests[0]["messages"]))
if __name__ == "__main__":
unittest.main()

212
backend/tests/test_workflow_v3.py
View File

@@ -4,7 +4,7 @@ import unittest
from models import AssistantConfig, RuntimeModelResource
from services.pipecat.service_factory import config_with_resource
from services.node_specs import normalize_graph, validate_graph
from services.node_specs import graph_references, normalize_graph, validate_graph
from services.runtime_variables import DynamicVariableStore, prepare_dynamic_config
from services.workflow_engine import WorkflowEngine
@@ -49,6 +49,22 @@ def valid_graph():
class WorkflowGraphTests(unittest.TestCase):
def test_agent_entry_mode_defaults_and_validation(self):
graph = valid_graph()
normalized = normalize_graph(graph)
agent = next(node for node in normalized["nodes"] if node["type"] == "agent")
self.assertEqual(agent["data"]["entryMode"], "wait_user")
self.assertEqual(agent["data"]["entrySpeech"], "")
self.assertTrue(agent["data"]["inheritGlobalConfig"])
self.assertEqual(agent["data"]["contextPolicy"], "fresh")
agent["data"]["entryMode"] = "fixed_speech"
self.assertTrue(
any("固定进入语不能为空" in error for error in validate_graph(normalized))
)
agent["data"]["entrySpeech"] = "您好，{{customer}}"
self.assertEqual(validate_graph(normalized), [])
def test_voice_resource_creates_isolated_runtime_config(self):
base = AssistantConfig(type="workflow", asr="default", voice="default")
asr = RuntimeModelResource(
@@ -63,6 +79,191 @@ class WorkflowGraphTests(unittest.TestCase):
self.assertEqual(resolved.stt_api_key, "secret")
self.assertEqual(base.asr, "default")
llm = RuntimeModelResource(
id="llm_1",
capability="LLM",
interface_type="openai-llm",
values={"modelId": "deepseek-chat", "apiUrl": "https://llm.test/v1"},
secrets={"apiKey": "llm-secret"},
)
llm_resolved = config_with_resource(base, llm)
self.assertEqual(llm_resolved.model, "deepseek-chat")
self.assertEqual(llm_resolved.llm_api_key, "llm-secret")
def test_global_and_custom_agent_references_are_preserved(self):
graph = valid_graph()
graph["settings"].update(
{
"defaultLlmResourceId": "llm_global",
"defaultAsrResourceId": "asr_global",
"defaultTtsResourceId": "tts_global",
"toolIds": ["tool_global"],
"knowledgeBaseId": "kb_global",
}
)
agent = next(node for node in graph["nodes"] if node["type"] == "agent")
agent["data"].update(
{
"inheritGlobalConfig": False,
"llmResourceId": "llm_agent",
"asrResourceId": "asr_agent",
"ttsResourceId": "tts_agent",
"toolIds": ["tool_agent"],
"knowledgeBaseId": "kb_agent",
}
)
refs = graph_references(graph)
self.assertEqual(
refs["model_resources"],
{
"llm_global",
"asr_global",
"tts_global",
"llm_agent",
"asr_agent",
"tts_agent",
},
)
self.assertEqual(refs["tools"], {"tool_global", "tool_agent"})
self.assertEqual(refs["knowledge_bases"], {"kb_global", "kb_agent"})
def test_existing_agent_override_disables_implicit_inheritance(self):
graph = valid_graph()
agent = next(node for node in graph["nodes"] if node["type"] == "agent")
agent["data"]["toolIds"] = ["legacy_tool"]
normalized = normalize_graph(graph)
normalized_agent = next(
node for node in normalized["nodes"] if node["type"] == "agent"
)
self.assertFalse(normalized_agent["data"]["inheritGlobalConfig"])
def test_inherited_agent_ignores_stale_custom_references(self):
graph = valid_graph()
agent = next(node for node in graph["nodes"] if node["type"] == "agent")
agent["data"].update(
{
"inheritGlobalConfig": True,
"llmResourceId": "stale_llm",
"asrResourceId": "stale_asr",
"ttsResourceId": "stale_tts",
"toolIds": ["stale_tool"],
"knowledgeBaseId": "stale_kb",
}
)
refs = graph_references(graph)
self.assertNotIn("stale_llm", refs["model_resources"])
self.assertNotIn("stale_tool", refs["tools"])
self.assertNotIn("stale_kb", refs["knowledge_bases"])
def test_agent_effective_config_inherits_then_switches_to_override(self):
graph = valid_graph()
graph["settings"].update(
{
"defaultLlmResourceId": "llm_global",
"defaultAsrResourceId": "asr_global",
"defaultTtsResourceId": "tts_global",
"toolIds": ["tool_global"],
"knowledgeBaseId": "kb_global",
"knowledgeMode": "on_demand",
"knowledgeTopN": 8,
"knowledgeScoreThreshold": 0.4,
}
)
engine = WorkflowEngine(graph)
inherited = engine.agent_stage_config("agent")
self.assertEqual(inherited.llm_resource_id, "llm_global")
self.assertEqual(inherited.tool_ids, ("tool_global",))
self.assertEqual(inherited.knowledge_mode, "on_demand")
engine.data("agent").update(
{
"inheritGlobalConfig": False,
"llmResourceId": "llm_agent",
"toolIds": ["tool_agent"],
"knowledgeBaseId": "",
}
)
custom = engine.agent_stage_config("agent")
self.assertEqual(custom.llm_resource_id, "llm_agent")
self.assertEqual(custom.tool_ids, ("tool_agent",))
self.assertEqual(custom.knowledge_mode, "disabled")
def test_start_agent_and_handoff_may_have_no_outgoing_edge(self):
terminal_graphs = [
{
"specVersion": 3,
"settings": {},
"nodes": [{"id": "start", "type": "start", "data": {}}],
"edges": [],
},
{
"specVersion": 3,
"settings": {},
"nodes": [
{"id": "start", "type": "start", "data": {}},
{
"id": "agent",
"type": "agent",
"data": {"prompt": "持续处理用户问题"},
},
],
"edges": [
{
"id": "begin",
"source": "start",
"target": "agent",
"data": {"mode": "always", "priority": 0},
}
],
},
{
"specVersion": 3,
"settings": {},
"nodes": [
{"id": "start", "type": "start", "data": {}},
{"id": "handoff", "type": "handoff", "data": {}},
],
"edges": [
{
"id": "begin",
"source": "start",
"target": "handoff",
"data": {"mode": "always", "priority": 0},
}
],
},
]
for graph in terminal_graphs:
with self.subTest(node=graph["nodes"][-1]["type"]):
self.assertEqual(validate_graph(graph), [])
action_without_exit = {
"specVersion": 3,
"settings": {},
"nodes": [
{"id": "start", "type": "start", "data": {}},
{"id": "action", "type": "action", "data": {}},
],
"edges": [
{
"id": "begin",
"source": "start",
"target": "action",
"data": {"mode": "always", "priority": 0},
}
],
}
self.assertTrue(
any(
"action 的出边不能少于 1" in error
for error in validate_graph(action_without_exit)
)
)
def test_v2_start_prompt_is_preserved_in_synthetic_agent(self):
graph = normalize_graph(
{
@@ -113,6 +314,15 @@ class WorkflowGraphTests(unittest.TestCase):
)
self.assertIn("王先生", engine.prompt_for("agent", store))
inherited_prompt = engine.prompt_for("agent", store)
self.assertIn("服务 王先生", inherited_prompt)
self.assertIn("处理订单", inherited_prompt)
engine.data("agent")["inheritGlobalConfig"] = False
custom_prompt = engine.prompt_for("agent", store)
self.assertNotIn("服务 王先生", custom_prompt)
self.assertIn("处理订单", custom_prompt)
if __name__ == "__main__":
unittest.main()