wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(workflow): add per-agent vision configuration

Browse Source
This commit is contained in:
Xin Wang
2026-07-18 00:00:28 +08:00
parent bdf3d3dd9c
commit 28c6380d8a
23 changed files with 553 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/services/brains/base.py
View File

@@ -81,6 +81,8 @@ class BrainRuntime:
Callable[[str | None, str | None, str | None], Awaitable[None]] | None
) = None
set_knowledge_scope: Callable[[dict[str, Any]], None] | None = None
set_vision_scope: Callable[[dict[str, Any]], None] | None = None
vision_function: Any = None
set_input_enabled: Callable[[bool], None] | None = None
apply_turn_config: (
Callable[[bool, dict[str, Any]], Awaitable[None]] | None

4
backend/services/brains/workflow_brain.py
View File

@@ -333,6 +333,8 @@ class WorkflowBrain(BaseBrain):
knowledge_function = self._knowledge_function(node_id)
if knowledge_function:
functions.append(knowledge_function)
if stage.vision_enabled and self._require_runtime().vision_function:
functions.append(self._require_runtime().vision_function)
return self._require_agent_stage().node_config(
node_id,
functions=functions,
@@ -703,6 +705,8 @@ class WorkflowBrain(BaseBrain):
runtime = self._require_runtime()
if runtime.set_knowledge_scope:
runtime.set_knowledge_scope({"mode": "disabled"})
if runtime.set_vision_scope:
runtime.set_vision_scope({"enabled": False})
if runtime.set_input_enabled:
runtime.set_input_enabled(False)
data = self._engine.data(node_id)

1
backend/services/config_resolver.py
View File

@@ -160,6 +160,7 @@ async def resolve_runtime_config(
interface_type=resource.interface_type,
values=resource.values or {},
secrets=resource.secrets or {},
support_image_input=bool(resource.support_image_input),
)
for resource in resources
if resource.enabled

16
backend/services/node_specs.py
View File

@@ -49,7 +49,7 @@ NODE_SPECS: list[dict[str, Any]] = [
"name": "agent",
"displayName": "Agent",
"category": "conversation_node",
"description": "阶段智能体：绑定上下文、工具、知识库及 ASR/TTS 资源。",
"description": "阶段智能体：绑定上下文、工具、知识库、视觉及语音资源。",
"icon": "Bot",
"accent": "sky",
"addable": True,
@@ -145,6 +145,8 @@ def _normalize_agent_data(data: dict[str, Any]) -> None:
data.get("llmResourceId"),
data.get("asrResourceId"),
data.get("ttsResourceId"),
data.get("visionEnabled"),
data.get("visionModelResourceId"),
data.get("knowledgeBaseId"),
data.get("toolIds"),
)
@@ -157,6 +159,8 @@ def _normalize_settings(settings: dict[str, Any], *, global_prompt: str = "") ->
settings.setdefault("defaultLlmResourceId", "")
settings.setdefault("defaultAsrResourceId", "")
settings.setdefault("defaultTtsResourceId", "")
settings.setdefault("visionEnabled", False)
settings.setdefault("visionModelResourceId", "")
settings.setdefault("toolIds", [])
settings.setdefault("knowledgeBaseId", "")
settings.setdefault("knowledgeMode", "automatic")
@@ -450,6 +454,11 @@ def graph_references(graph: dict[str, Any]) -> dict[str, set[str]]:
settings.get("defaultLlmResourceId"),
settings.get("defaultAsrResourceId"),
settings.get("defaultTtsResourceId"),
(
settings.get("visionModelResourceId")
if settings.get("visionEnabled")
else None
),
)
if value
}
@@ -469,6 +478,11 @@ def graph_references(graph: dict[str, Any]) -> dict[str, set[str]]:
data.get("llmResourceId"),
data.get("asrResourceId"),
data.get("ttsResourceId"),
(
data.get("visionModelResourceId")
if data.get("visionEnabled")
else None
),
):
if resource_id:
resources.add(str(resource_id))

77
backend/services/pipecat/pipeline.py
View File

@@ -29,6 +29,14 @@ from services.pipecat.service_factory import (
)
from db.session import SessionLocal
from services.knowledge import search as search_knowledge
from services.vision import (
VISION_ANALYSIS_SYSTEM_PROMPT,
VISION_SYSTEM_HINT,
VISION_TOOL_NAME,
config_with_main_llm_as_vision,
config_with_vision_resource,
)
from services.workflow_engine import WorkflowEngine
from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema
@@ -82,15 +90,6 @@ from services.pipecat.pipeline_events import (
from pipecat.workers.runner import WorkerRunner
VISION_TOOL_NAME = "fetch_user_image"
VISION_SYSTEM_HINT = (
"当前会话打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/"
"环境状态或需要你看一眼时,调用 fetch_user_image 获取当前视频帧,再基于画面回答。"
)
VISION_ANALYSIS_SYSTEM_PROMPT = (
"你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
"如果画面不足以判断,请明确说明不确定。"
)
KNOWLEDGE_TOOL_NAME = "search_knowledge_base"
AUTOMATIC_KNOWLEDGE_SYSTEM_HINT = (
"你已连接内部知识库。系统会在每轮用户问题前自动提供相关资料；"
@@ -205,6 +204,9 @@ async def run_pipeline(
只要有 .input() / .output() / event_handler 即可。
cfg: 助手配置(随请求内联传入)。
"""
if cfg.type == "workflow":
vision_enabled = WorkflowEngine(cfg.graph).uses_vision()
logger.info(
f"启动管线: assistant={cfg.name} type={cfg.type} "
f"mode={cfg.runtimeMode} vision={vision_enabled}"
@@ -298,7 +300,7 @@ async def run_pipeline(
def with_vision_hint(text: str) -> str:
hints = []
if vision_enabled:
if vision_enabled and cfg.type != "workflow":
hints.append(VISION_SYSTEM_HINT)
if cfg.knowledge_base_id:
hints.append(
@@ -354,8 +356,17 @@ async def run_pipeline(
top_n=knowledge_top_n,
score_threshold=knowledge_score_threshold,
)
vision_native_mode = vision_enabled and _vision_uses_main_llm(cfg)
vision_native_mode = (
vision_enabled
and cfg.type != "workflow"
and _vision_uses_main_llm(cfg)
)
vision_state: dict[str, str | None] = {"client_id": None}
workflow_vision_scope: dict[str, Any] = {
"enabled": False,
"vision_model_resource_id": None,
"llm_resource_id": None,
}
vision_schema = FunctionSchema(
name=VISION_TOOL_NAME,
description=(
@@ -458,8 +469,14 @@ async def run_pipeline(
)
flow_global_functions = []
workflow_vision_function = None
if cfg.type == "workflow" and vision_enabled:
async def flow_fetch_user_image(args, _flow_manager):
if not workflow_vision_scope.get("enabled"):
return {
"status": "disabled",
"message": "当前 Agent 节点未开启视觉理解。",
}
question = str((args or {}).get("question") or "请描述当前画面。")
user_id = vision_state.get("client_id")
if not user_id:
@@ -474,8 +491,26 @@ async def run_pipeline(
function_name=VISION_TOOL_NAME,
)
try:
vision_resource_id = str(
workflow_vision_scope.get("vision_model_resource_id") or ""
)
llm_resource_id = str(
workflow_vision_scope.get("llm_resource_id") or ""
)
resource_id = vision_resource_id or llm_resource_id
resource = cfg.workflow_model_resources.get(resource_id)
if resource:
vision_cfg = config_with_vision_resource(cfg, resource)
elif not vision_resource_id:
vision_cfg = config_with_main_llm_as_vision(cfg)
else:
raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
frame = await vision_capture.request_image(llm, request)
observation = await _analyze_image_with_vision_model(cfg, frame, question)
observation = await _analyze_image_with_vision_model(
vision_cfg,
frame,
question,
)
return {
"status": "ok",
"question": question,
@@ -487,15 +522,13 @@ async def run_pipeline(
logger.warning(f"Workflow 视觉理解失败:{exc}")
return {"status": "error", "message": "视觉理解暂时不可用。"}
flow_global_functions.append(
FlowsFunctionSchema(
name=VISION_TOOL_NAME,
description=vision_schema.description,
properties=vision_schema.properties,
required=vision_schema.required,
handler=flow_fetch_user_image,
cancel_on_interruption=True,
)
workflow_vision_function = FlowsFunctionSchema(
name=VISION_TOOL_NAME,
description=vision_schema.description,
properties=vision_schema.properties,
required=vision_schema.required,
handler=flow_fetch_user_image,
cancel_on_interruption=True,
)
if vision_enabled and cfg.type != "workflow":
@@ -612,6 +645,8 @@ async def run_pipeline(
transport=transport,
switch_services=service_controller.switch,
set_knowledge_scope=knowledge_retrieval.set_scope,
set_vision_scope=lambda scope: workflow_vision_scope.update(scope),
vision_function=workflow_vision_function,
set_input_enabled=lambda enabled: input_state.__setitem__("enabled", enabled),
apply_turn_config=apply_workflow_turn_config,
flow_global_functions=flow_global_functions,

58
backend/services/vision.py Normal file
View File

@@ -0,0 +1,58 @@
"""Shared vision capability metadata and Workflow model resolution."""
from __future__ import annotations
from models import AssistantConfig, RuntimeModelResource
VISION_TOOL_NAME = "fetch_user_image"
VISION_SYSTEM_HINT = (
"当前阶段打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/"
"环境状态或需要你看一眼时，调用 fetch_user_image 获取当前视频帧，"
"再基于画面回答。"
)
VISION_ANALYSIS_SYSTEM_PROMPT = (
"你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
"如果画面不足以判断，请明确说明不确定。"
)
def config_with_vision_resource(
cfg: AssistantConfig,
resource: RuntimeModelResource,
) -> AssistantConfig:
"""Create the config view used to analyze one Workflow camera frame."""
if resource.capability != "LLM":
raise ValueError(f"视觉模型资源能力无效:{resource.capability}")
if not resource.support_image_input:
raise ValueError(f"视觉模型不支持图片输入:{resource.id}")
result = cfg.model_copy(deep=True)
values = resource.values or {}
secrets = resource.secrets or {}
result.vision_model_resource_id = resource.id
result.vision_model = str(values.get("modelId") or "")
result.vision_llm_interface_type = resource.interface_type
result.vision_llm_values = values
result.vision_llm_secrets = secrets
result.vision_llm_support_image_input = resource.support_image_input
result.vision_llm_api_key = str(secrets.get("apiKey") or "")
result.vision_llm_base_url = str(values.get("apiUrl") or "")
return result
def config_with_main_llm_as_vision(cfg: AssistantConfig) -> AssistantConfig:
"""Legacy fallback when a Workflow LLM is not present in the resource map."""
if not cfg.llm_support_image_input:
raise ValueError("当前大语言模型不支持图片输入")
result = cfg.model_copy(deep=True)
result.vision_model_resource_id = None
result.vision_model = cfg.model
result.vision_llm_interface_type = cfg.llm_interface_type
result.vision_llm_values = dict(cfg.llm_values)
result.vision_llm_secrets = dict(cfg.llm_secrets)
result.vision_llm_support_image_input = cfg.llm_support_image_input
result.vision_llm_api_key = cfg.llm_api_key
result.vision_llm_base_url = cfg.llm_base_url
return result

12
backend/services/workflow/agent.py
View File

@@ -11,6 +11,7 @@ from pipecat.services.settings import LLMSettings
from services.brains.base import BrainRuntime
from services.runtime_variables import DynamicVariableStore
from services.vision import VISION_SYSTEM_HINT
from services.workflow_engine import WorkflowEngine
from services.workflow_router import WorkflowLLMRouter
@@ -39,6 +40,9 @@ class WorkflowAgentStage:
def role_message(self, node_id: str) -> str:
stage_prompt = self._engine.prompt_for(node_id, self._store)
stage = self._engine.agent_stage_config(node_id)
if stage.vision_enabled:
stage_prompt = f"{stage_prompt}\n\n[视觉能力]\n{VISION_SYSTEM_HINT}"
return (
f"{stage_prompt}\n\n[工作流执行规则]\n"
f"{AGENT_STAGE_INSTRUCTION}"
@@ -90,6 +94,14 @@ class WorkflowAgentStage:
"score_threshold": stage.knowledge_score_threshold,
}
)
if self._runtime.set_vision_scope:
self._runtime.set_vision_scope(
{
"enabled": stage.vision_enabled,
"vision_model_resource_id": stage.vision_model_resource_id,
"llm_resource_id": stage.llm_resource_id,
}
)
def node_config(
self,

14
backend/services/workflow_engine.py
View File

@@ -18,6 +18,8 @@ class AgentStageConfig:
llm_resource_id: str
asr_resource_id: str
tts_resource_id: str
vision_enabled: bool
vision_model_resource_id: str | None
tool_ids: tuple[str, ...]
knowledge_base_id: str | None
knowledge_mode: str
@@ -119,6 +121,10 @@ class WorkflowEngine:
llm_resource_id=str(source.get(llm_key) or ""),
asr_resource_id=str(source.get(asr_key) or ""),
tts_resource_id=str(source.get(tts_key) or ""),
vision_enabled=bool(source.get("visionEnabled", False)),
vision_model_resource_id=(
str(source.get("visionModelResourceId") or "") or None
),
tool_ids=tuple(str(tool_id) for tool_id in source.get("toolIds") or []),
knowledge_base_id=knowledge_base_id or None,
knowledge_mode=(
@@ -139,6 +145,14 @@ class WorkflowEngine:
turn_config=dict(turn_config),
)
def uses_vision(self) -> bool:
"""Return whether any effective Agent stage needs a camera stream."""
return any(
self.agent_stage_config(node_id).vision_enabled
for node_id, node in self.nodes.items()
if node.get("type") == "agent"
)
def prompt_for(self, node_id: str, store: DynamicVariableStore) -> str:
"""Build the Agent system prompt according to its inheritance setting."""
prompt = store.render(str(self.data(node_id).get("prompt") or "").strip())