wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: route workflow image inputs natively

Browse Source
This commit is contained in:
Xin Wang
2026-08-03 10:55:57 +08:00
parent 2e84de0798
commit f3439b21d1
10 changed files with 412 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
backend/tests/test_pipeline_knowledge.py
View File

@@ -1,6 +1,6 @@
import unittest
from models import AssistantConfig
from models import AssistantConfig, RuntimeModelResource
from pipecat.frames.frames import LLMContextFrame
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameDirection
@@ -9,6 +9,7 @@ from services.pipecat.pipeline import (
KnowledgeRetrievalProcessor,
UserTurnRoutingProcessor,
_knowledge_tool_description,
_workflow_vision_uses_main_llm,
)
@@ -67,8 +68,8 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
def __init__(self):
self.turns = []
async def on_user_turn_end(self, content):
self.turns.append(content)
async def on_user_turn_end(self, content, user_message=None):
self.turns.append((content, user_message))
return True
brain = FakeBrain()
@@ -83,13 +84,19 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
frame = LLMContextFrame(context)
await processor.process_frame(frame, FrameDirection.DOWNSTREAM)
self.assertEqual(brain.turns, ["我叫李白"])
self.assertEqual(
brain.turns,
[("我叫李白", {"role": "user", "content": "我叫李白"})],
)
self.assertEqual(forwarded, [])
# A queued LLMRunFrame after the transition uses the same context. It
# must reach the target Agent without invoking routing a second time.
await processor.process_frame(frame, FrameDirection.DOWNSTREAM)
self.assertEqual(brain.turns, ["我叫李白"])
self.assertEqual(
brain.turns,
[("我叫李白", {"role": "user", "content": "我叫李白"})],
)
self.assertEqual(forwarded, [(frame, FrameDirection.DOWNSTREAM)])
async def test_routes_multimodal_user_message_by_its_text_part(self):
@@ -97,8 +104,8 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
def __init__(self):
self.turns = []
async def on_user_turn_end(self, content):
self.turns.append(content)
async def on_user_turn_end(self, content, user_message=None):
self.turns.append((content, user_message))
return False
brain = FakeBrain()
@@ -124,7 +131,87 @@ class UserTurnRoutingProcessorTest(unittest.IsolatedAsyncioTestCase):
FrameDirection.DOWNSTREAM,
)
self.assertEqual(brain.turns, ["看看这张照片"])
self.assertEqual(
brain.turns,
[
(
"看看这张照片",
{
"role": "user",
"content": [
{"type": "text", "text": "看看这张照片"},
{
"type": "image_url",
"image_url": {
"url": "data:image/jpeg;base64,AA=="
},
},
],
},
)
],
)
class WorkflowVisionModeTest(unittest.TestCase):
def test_uses_active_agent_llm_only_without_auxiliary_model(self):
cfg = AssistantConfig(
type="workflow",
workflow_model_resources={
"agent_llm": RuntimeModelResource(
id="agent_llm",
name="视觉 Agent",
capability="LLM",
interface_type="openai-llm",
support_image_input=True,
)
},
)
self.assertTrue(
_workflow_vision_uses_main_llm(
cfg,
{
"enabled": True,
"llm_resource_id": "agent_llm",
"vision_model_resource_id": None,
},
)
)
self.assertFalse(
_workflow_vision_uses_main_llm(
cfg,
{
"enabled": True,
"llm_resource_id": "agent_llm",
"vision_model_resource_id": "auxiliary_vision",
},
)
)
def test_rejects_a_non_visual_active_agent_llm(self):
cfg = AssistantConfig(
type="workflow",
workflow_model_resources={
"text_llm": RuntimeModelResource(
id="text_llm",
name="文本 Agent",
capability="LLM",
interface_type="openai-llm",
support_image_input=False,
)
},
)
with self.assertRaisesRegex(ValueError, "不支持图片输入"):
_workflow_vision_uses_main_llm(
cfg,
{
"enabled": True,
"llm_resource_id": "text_llm",
"vision_model_resource_id": None,
},
)
async def _async_none():