wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add realtime workflow vision tools

Browse Source
This commit is contained in:
Xin Wang
2026-08-05 13:35:00 +08:00
parent 11374238f3
commit 10bea48d72
16 changed files with 362 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
backend/routes/assistants.py
View File

@@ -85,6 +85,7 @@ async def _validate_workflow_references(
graph = body.graph
engine = WorkflowEngine(graph)
settings = graph.get("settings") or {}
runtime_mode = str(settings.get("runtimeMode") or "pipeline")
resource_expectations: dict[str, str] = {}
vision_resource_ids: set[str] = set()
for key, capability in (
@@ -118,16 +119,22 @@ async def _validate_workflow_references(
stage = engine.agent_stage_config(node_id)
if not stage.vision_enabled:
continue
resource_id = (
stage.vision_model_resource_id or stage.llm_resource_id
)
if stage.vision_model_resource_id:
resource_id = stage.vision_model_resource_id
capability = "LLM"
elif runtime_mode == "realtime":
resource_id = str(settings.get("defaultRealtimeResourceId") or "")
capability = "Realtime"
else:
resource_id = stage.llm_resource_id
capability = "LLM"
if not resource_id:
raise HTTPException(
400,
f"Agent 节点 {node_id} 开启视觉理解时必须选择"
"支持图片输入的大语言模型或视觉模型",
"支持图片输入的当前模型或独立视觉模型",
)
resource_expectations[resource_id] = "LLM"
resource_expectations[resource_id] = capability
vision_resource_ids.add(resource_id)
for resource_id, capability in resource_expectations.items():
resource = await session.get(ModelResource, resource_id)

25
backend/routes/model_registry.py
View File

@@ -30,6 +30,19 @@ router = APIRouter(
tags=["model-registry"],
dependencies=[Depends(require_admin)],
)
NATIVE_IMAGE_UNSUPPORTED_REALTIME_INTERFACES = frozenset(
{"qwen-audio-realtime"}
)
def _supports_configurable_image_input(definition: InterfaceDefinition) -> bool:
if definition.capability == "LLM":
return True
return (
definition.capability == "Realtime"
and definition.interface_type
not in NATIVE_IMAGE_UNSUPPORTED_REALTIME_INTERFACES
)
def _definition_dict(row: InterfaceDefinition) -> dict:
@@ -156,9 +169,11 @@ async def create_model_resource(
interface_type=definition.interface_type,
values=body.values,
secrets=secrets,
support_image_input=body.support_image_input
if definition.capability == "LLM"
else False,
support_image_input=(
body.support_image_input
if _supports_configurable_image_input(definition)
else False
),
enabled=body.enabled,
is_default=body.is_default,
)
@@ -244,7 +259,9 @@ async def update_model_resource(
row.values = body.values
row.secrets = secrets
row.support_image_input = (
body.support_image_input if definition.capability == "LLM" else False
body.support_image_input
if _supports_configurable_image_input(definition)
else False
)
row.enabled = body.enabled
row.is_default = body.is_default

1
backend/services/brains/base.py
View File

@@ -111,6 +111,7 @@ class RealtimeBrainRuntime:
session_id: str = ""
client_tools: ClientToolPort | None = None
set_input_enabled: Callable[[bool], None] | None = None
capture_image: Callable[[str], Awaitable[Any | None]] | None = None
class BaseBrain:

6
backend/services/node_specs.py
View File

@@ -438,8 +438,6 @@ def validate_graph(graph: dict[str, Any]) -> list[str]:
"defaultRealtimeResourceId"
):
errors.append("Realtime 工作流必须选择 Realtime 模型")
if runtime_mode == "realtime" and settings.get("visionEnabled"):
errors.append("Realtime 工作流暂不支持视觉能力")
if (
runtime_mode == "realtime"
and settings.get("knowledgeBaseId")
@@ -488,10 +486,6 @@ def validate_graph(graph: dict[str, Any]) -> list[str]:
f"Realtime Agent 节点 {node_id} 不支持节点级模型、语音或交互策略覆盖"
)
source = settings if data.get("inheritGlobalConfig", True) else data
if source.get("visionEnabled"):
errors.append(
f"Realtime Agent 节点 {node_id} 暂不支持视觉能力"
)
if (
source.get("knowledgeBaseId")
and source.get("knowledgeMode") != "on_demand"

89
backend/services/pipecat/pipeline.py
View File

@@ -7,14 +7,10 @@
"""
import asyncio
import base64
from io import BytesIO
from typing import Any
from loguru import logger
from models import AssistantConfig
from openai import AsyncOpenAI
from PIL import Image
from services.brains import Brain, BrainRuntime, build_brain
from services.brains.base import RealtimeBrainRuntime
from services.conversation_history import ConversationRecorder
@@ -34,11 +30,12 @@ from services.knowledge import search as search_knowledge
from services.client_tools import ClientToolBroker
from services.tool_policy import policy_for_tool
from services.vision import (
VISION_ANALYSIS_SYSTEM_PROMPT,
VISION_SYSTEM_HINT,
VISION_TOOL_NAME,
analyze_image_with_vision_model,
config_with_main_llm_as_vision,
config_with_vision_resource,
image_data_uri,
)
from services.workflow_engine import WorkflowEngine
@@ -164,19 +161,6 @@ def _workflow_vision_uses_main_llm(
return True
def _image_data_uri(frame: UserImageRawFrame) -> str:
if not frame.format:
raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型")
buffer = BytesIO()
Image.frombytes(frame.format, frame.size, frame.image).save(
buffer,
format="JPEG",
quality=85,
)
encoded = base64.b64encode(buffer.getvalue()).decode("utf-8")
return f"data:image/jpeg;base64,{encoded}"
def _multimodal_user_input_frame(
image_frame: UserImageRawFrame,
prompt_text: str,
@@ -197,7 +181,7 @@ def _multimodal_user_input_frame(
{"type": "text", "text": prompt_text},
{
"type": "image_url",
"image_url": {"url": _image_data_uri(image_frame)},
"image_url": {"url": image_data_uri(image_frame)},
},
],
}
@@ -206,45 +190,6 @@ def _multimodal_user_input_frame(
)
async def _analyze_image_with_vision_model(
cfg: AssistantConfig,
frame: UserImageRawFrame,
question: str,
) -> str:
if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}:
raise ValueError(f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}")
data_uri = await asyncio.to_thread(_image_data_uri, frame)
extra_body = cfg.vision_llm_values.get("extraBody")
extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
client = AsyncOpenAI(
api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"),
base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"),
)
try:
response = await client.chat.completions.create(
model=_require(cfg.vision_model, "Vision LLM modelId"),
messages=[
{"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT},
{
"role": "user",
"content": [
{"type": "text", "text": question},
{"type": "image_url", "image_url": {"url": data_uri}},
],
},
],
**extra,
)
finally:
await client.close()
content = response.choices[0].message.content if response.choices else ""
if isinstance(content, str):
return content.strip()
return str(content or "").strip()
async def run_pipeline(
transport,
cfg: AssistantConfig,
@@ -278,12 +223,11 @@ async def run_pipeline(
raise ValueError(f"类型 {cfg.type} 不支持 realtime 运行模式")
if cfg.runtimeMode == "realtime":
if vision_enabled:
logger.warning("Realtime 模式暂未接入视频帧工具,本次仅启用语音通话")
await run_realtime_pipeline(
transport,
cfg,
brain=brain,
vision_enabled=vision_enabled,
assistant_id=assistant_id,
channel=channel,
)
@@ -524,7 +468,7 @@ async def run_pipeline(
)
try:
frame = await vision_capture.request_image(params.llm, request)
observation = await _analyze_image_with_vision_model(cfg, frame, question)
observation = await analyze_image_with_vision_model(cfg, frame, question)
except asyncio.TimeoutError:
await params.result_callback(
{
@@ -614,7 +558,7 @@ async def run_pipeline(
)
try:
frame = await vision_capture.request_image(llm, request)
observation = await _analyze_image_with_vision_model(
observation = await analyze_image_with_vision_model(
active_vision_config(),
frame,
question,
@@ -830,7 +774,7 @@ async def run_pipeline(
try:
assert analysis_cfg is not None
observation = await _analyze_image_with_vision_model(
observation = await analyze_image_with_vision_model(
analysis_cfg,
image_frame,
value.prompt_text,
@@ -883,6 +827,7 @@ async def run_realtime_pipeline(
cfg: AssistantConfig,
*,
brain: Brain,
vision_enabled: bool = False,
assistant_id: str | None = None,
channel: str = "webrtc",
) -> None:
@@ -894,6 +839,8 @@ async def run_realtime_pipeline(
input_sample_rate, output_sample_rate = realtime_audio_sample_rates(cfg)
worker_holder: dict[str, PipelineWorker] = {}
input_state = {"enabled": True}
vision_state: dict[str, str | None] = {"client_id": None}
vision_capture = VisionCaptureProcessor()
async def queue_call_end(reason: str) -> None:
worker = worker_holder.get("worker")
@@ -946,6 +893,18 @@ async def run_realtime_pipeline(
)
greeting = await brain.greeting(cfg)
async def capture_image(question: str):
user_id = vision_state["client_id"]
if not user_id:
return None
request = UserImageRequestFrame(
user_id=user_id,
text=question,
append_to_context=False,
function_name=VISION_TOOL_NAME,
)
return await vision_capture.request_image(realtime, request)
recorder = await ConversationRecorder.start(
assistant_id=assistant_id,
assistant_name=cfg.name,
@@ -961,6 +920,7 @@ async def run_realtime_pipeline(
pipeline = Pipeline(
[
transport.input(),
vision_capture,
client_tools,
session_update,
user_input,
@@ -995,6 +955,7 @@ async def run_realtime_pipeline(
session_id=cfg.conversation_id or "",
client_tools=client_tools,
set_input_enabled=set_input_enabled,
capture_image=capture_image if vision_enabled else None,
),
)
@@ -1005,6 +966,8 @@ async def run_realtime_pipeline(
brain=brain,
text_input=user_input,
greeting=greeting,
vision_enabled=vision_enabled,
vision_state=vision_state,
)
runner = WorkerRunner(handle_sigint=False)
run_status = "completed"

15
backend/services/pipecat/pipeline_events.py
View File

@@ -230,6 +230,8 @@ def bind_realtime_pipeline_events(
brain,
text_input,
greeting: str,
vision_enabled: bool,
vision_state: dict[str, str | None],
) -> None:
"""Connect text and lifecycle events for a realtime model pipeline."""
@@ -274,6 +276,19 @@ def bind_realtime_pipeline_events(
@transport.event_handler("on_client_connected")
async def on_client_connected(_transport, _client):
if vision_enabled:
try:
vision_state["client_id"] = get_transport_client_id(
_transport,
_client,
)
await maybe_capture_participant_camera(_transport, _client)
logger.info(
f"Realtime 视觉理解已接入视频客户端: "
f"{vision_state['client_id']}"
)
except Exception as exc: # noqa: BLE001 - video remains optional
logger.warning(f"Realtime 视觉理解摄像头捕获初始化失败: {exc}")
await brain.on_connected(greeting_pending=bool(greeting))
await brain.on_client_ready()
if greeting:

71
backend/services/vision.py
View File

@@ -1,8 +1,15 @@
"""Shared vision capability metadata and Workflow model resolution."""
"""Shared camera-frame analysis for Pipeline and Realtime runtimes."""
from __future__ import annotations
import asyncio
import base64
from io import BytesIO
from models import AssistantConfig, RuntimeModelResource
from openai import AsyncOpenAI
from PIL import Image
from pipecat.frames.frames import UserImageRawFrame
VISION_TOOL_NAME = "fetch_user_image"
@@ -17,6 +24,68 @@ VISION_ANALYSIS_SYSTEM_PROMPT = (
)
def _require(value: str, label: str) -> str:
if value:
return value
raise ValueError(f"缺少模型资源配置: {label}")
def image_data_uri(frame: UserImageRawFrame) -> str:
"""Encode one Pipecat camera frame for a vision chat-completion request."""
if not frame.format:
raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型")
buffer = BytesIO()
Image.frombytes(frame.format, frame.size, frame.image).save(
buffer,
format="JPEG",
quality=85,
)
encoded = base64.b64encode(buffer.getvalue()).decode("utf-8")
return f"data:image/jpeg;base64,{encoded}"
async def analyze_image_with_vision_model(
cfg: AssistantConfig,
frame: UserImageRawFrame,
question: str,
) -> str:
"""Analyze one frame with the configured independent vision LLM."""
if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}:
raise ValueError(
f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}"
)
data_uri = await asyncio.to_thread(image_data_uri, frame)
extra_body = cfg.vision_llm_values.get("extraBody")
extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
client = AsyncOpenAI(
api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"),
base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"),
)
try:
response = await client.chat.completions.create(
model=_require(cfg.vision_model, "Vision LLM modelId"),
messages=[
{"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT},
{
"role": "user",
"content": [
{"type": "text", "text": question},
{"type": "image_url", "image_url": {"url": data_uri}},
],
},
],
**extra,
)
finally:
await client.close()
content = response.choices[0].message.content if response.choices else ""
if isinstance(content, str):
return content.strip()
return str(content or "").strip()
def config_with_vision_resource(
cfg: AssistantConfig,
resource: RuntimeModelResource,

113
backend/services/workflow/realtime.py
View File

@@ -28,6 +28,12 @@ from services.runtime_variables import DynamicVariableError, DynamicVariableStor
from services.system_tools import state_update_properties, system_tool_kind
from services.tool_executor import ToolExecutionError, ToolExecutor
from services.tool_policy import policy_for_tool
from services.vision import (
VISION_SYSTEM_HINT,
VISION_TOOL_NAME,
analyze_image_with_vision_model,
config_with_vision_resource,
)
from services.workflow.agent import EDGE_TOOL_STAGE_INSTRUCTION
from services.workflow.models import WorkflowRuntimeState, WorkflowStatus
from services.workflow.output import WorkflowOutput
@@ -385,10 +391,13 @@ class WorkflowRealtimeController:
return RealtimeActivation(continue_response=generate)
def _agent_prompt(self, node_id: str) -> str:
return (
prompt = (
f"{self._engine.prompt_for(node_id, self._store)}\n\n"
f"[工作流执行规则]\n{EDGE_TOOL_STAGE_INSTRUCTION}"
)
if self._engine.agent_stage_config(node_id).vision_enabled:
prompt = f"{prompt}\n{VISION_SYSTEM_HINT}"
return prompt
def _build_agent_tools(self, node_id: str) -> list[RealtimeTool]:
stage = self._engine.agent_stage_config(node_id)
@@ -428,6 +437,9 @@ class WorkflowRealtimeController:
knowledge = self._knowledge_tool(node_id, transition_id)
if knowledge:
add(*knowledge)
vision = self._vision_tool(node_id, transition_id)
if vision:
add(*vision)
for edge in self._engine.edge_tool_edges(node_id):
add(*self._transition_tool(edge, node_id, transition_id))
self._handlers = handlers
@@ -519,6 +531,105 @@ class WorkflowRealtimeController:
handler,
)
def _vision_tool(
self,
node_id: str,
transition_id: int,
) -> tuple[RealtimeTool, ToolHandler] | None:
"""Build the on-demand camera tool for the active Realtime Agent."""
stage = self._engine.agent_stage_config(node_id)
if not stage.vision_enabled:
return None
vision_cfg: AssistantConfig | None = None
if stage.vision_model_resource_id:
resource = self._cfg.workflow_model_resources.get(
stage.vision_model_resource_id
)
if resource is not None:
vision_cfg = config_with_vision_resource(self._cfg, resource)
async def handler(arguments: dict[str, Any]) -> RealtimeToolResult:
if not self._is_current(node_id, transition_id):
return RealtimeToolResult(
{"status": "stale", "message": "当前 Agent 已经切换。"},
continue_response=False,
)
question = str(arguments.get("question") or "").strip()
if not question:
return RealtimeToolResult(
{"status": "error", "message": "视觉问题为空"}
)
if self._runtime.capture_image is None:
return RealtimeToolResult(
{"status": "no_video", "message": "当前会话未启用视频输入"}
)
try:
frame = await self._runtime.capture_image(question)
except TimeoutError:
return RealtimeToolResult(
{"status": "timeout", "message": "等待摄像头画面超时"}
)
except Exception as exc: # noqa: BLE001 - expose a stable tool error
logger.warning(f"Realtime Workflow 获取摄像头画面失败:{exc}")
return RealtimeToolResult(
{"status": "error", "message": "暂时无法获取摄像头画面"}
)
if frame is None:
return RealtimeToolResult(
{"status": "no_video", "message": "当前没有可用的视频客户端"}
)
if not self._is_current(node_id, transition_id):
return RealtimeToolResult(
{"status": "stale", "message": "获取画面时 Agent 已经切换。"},
continue_response=False,
)
try:
if vision_cfg is not None:
observation = await analyze_image_with_vision_model(
vision_cfg,
frame,
question,
)
else:
analyze = getattr(self._runtime.realtime, "analyze_image", None)
if not callable(analyze):
raise ValueError("当前 Realtime 适配器未实现原生视觉理解")
observation = await analyze(frame, question)
except Exception as exc: # noqa: BLE001 - provider errors become tool output
logger.warning(f"Realtime Workflow 视觉模型调用失败:{exc}")
return RealtimeToolResult(
{"status": "error", "message": "视觉模型暂时不可用"}
)
if not self._is_current(node_id, transition_id):
return RealtimeToolResult(
{"status": "stale", "message": "分析画面时 Agent 已经切换。"},
continue_response=False,
)
return RealtimeToolResult(
{
"status": "ok",
"question": question,
"observation": observation,
}
)
return (
RealtimeTool(
name=VISION_TOOL_NAME,
description="获取用户摄像头的当前画面，并回答一个视觉问题。",
properties={
"question": {
"type": "string",
"description": "需要根据当前画面判断的具体问题",
}
},
required=("question",),
),
handler,
)
def _system_tool(
self,
tool: RuntimeTool,

10
frontend/src/components/editor/vision-config-section.tsx
View File

@@ -19,6 +19,7 @@ export function VisionConfigSection({
enabled,
modelResourceId,
mainModelResourceId,
mainModelSupportsImageInput,
modelOptions,
onEnabledChange,
onModelResourceIdChange,
@@ -28,13 +29,14 @@ export function VisionConfigSection({
enabled: boolean;
modelResourceId: string;
mainModelResourceId: string;
mainModelSupportsImageInput?: boolean;
modelOptions: VisionModelOption[];
onEnabledChange: (enabled: boolean) => void;
onModelResourceIdChange: (modelResourceId: string) => void;
}) {
const mainModelSupportsVision = modelOptions.some(
(option) => option.value === mainModelResourceId,
);
const mainModelSupportsVision =
mainModelSupportsImageInput ??
modelOptions.some((option) => option.value === mainModelResourceId);
const independentModelOptions = modelOptions.filter(
(option) => option.value !== mainModelResourceId,
);
@@ -62,7 +64,7 @@ export function VisionConfigSection({
/>
{!modelResourceId && !mainModelSupportsVision && (
<p className="text-xs text-destructive">
当前大语言模型未标记支持图片输入，请选择独立视觉模型。
当前模型未标记支持图片输入，请选择独立视觉模型。
</p>
)}
</>

6
frontend/src/components/pages/AssistantPage.tsx
View File

@@ -411,7 +411,11 @@ export function AssistantPage(props: AssistantPageProps) {
const credOptions = (type: ModelResource["capability"]) =>
modelResources
.filter((c) => c.capability === type)
.map((c) => ({ value: c.id, label: c.name }));
.map((c) => ({
value: c.id,
label: c.name,
supportImageInput: c.supportImageInput,
}));
const agentOptions = (interfaceType: "dify" | "fastgpt" | "opencode") =>
modelResources
.filter(

45
frontend/src/components/pages/ComponentsModelsPage.tsx
View File

@@ -66,6 +66,20 @@ const capabilities: ModelType[] = [
];
const capabilityFilters = ["全部", ...capabilities] as const;
const nativeImageUnsupportedRealtimeInterfaces = new Set([
"qwen-audio-realtime",
]);
function canConfigureImageInput(
capability: ModelType,
interfaceType: string,
): boolean {
return (
capability === "LLM" ||
(capability === "Realtime" &&
!nativeImageUnsupportedRealtimeInterfaces.has(interfaceType))
);
}
type ResourceDraft = {
name: string;
@@ -294,6 +308,12 @@ export function ComponentsModelsPage() {
interfaceType,
values: definition ? defaults(definition) : {},
secrets: {},
supportImageInput: canConfigureImageInput(
previous.capability,
interfaceType,
)
? previous.supportImageInput
: false,
}));
setStoredSecretMasks({});
setExtraBodyDraft("");
@@ -310,8 +330,7 @@ export function ComponentsModelsPage() {
interfaceType: first?.interfaceType ?? "",
values: first ? defaults(first) : {},
secrets: {},
supportImageInput:
capability === "LLM" ? previous.supportImageInput : false,
supportImageInput: false,
}));
setStoredSecretMasks({});
setExtraBodyDraft("");
@@ -392,7 +411,9 @@ export function ComponentsModelsPage() {
? { ...storedSecretMasks, ...draft.secrets }
: draft.secrets,
supportImageInput:
draft.capability === "LLM" ? draft.supportImageInput : false,
canConfigureImageInput(draft.capability, draft.interfaceType)
? draft.supportImageInput
: false,
enabled: editingId
? resources.find((resource) => resource.id === editingId)?.enabled ?? true
: true,
@@ -747,11 +768,25 @@ export function ComponentsModelsPage() {
</Select>
</Field>
</div>
{draft.capability === "LLM" && (
{(draft.capability === "LLM" ||
draft.capability === "Realtime") && (
<div className="flex items-center justify-between rounded-xl border border-hairline p-4">
<span className="text-sm font-medium">支持图片输入</span>
<div className="space-y-1">
<p className="text-sm font-medium">原生支持图片输入</p>
{draft.capability === "Realtime" && (
<p className="text-xs text-muted-foreground">
Qwen Audio Realtime 请保持关闭，并为助手选择独立视觉模型。
</p>
)}
</div>
<Switch
checked={draft.supportImageInput}
disabled={
!canConfigureImageInput(
draft.capability,
draft.interfaceType,
)
}
onCheckedChange={(checked) =>
setDraft((previous) => ({
...previous,

1
frontend/src/components/workflow/WorkflowCanvas.tsx
View File

@@ -798,6 +798,7 @@ export function WorkflowCanvas({
llmOptions={modelOptions.llm}
asrOptions={modelOptions.asr}
ttsOptions={modelOptions.tts}
realtimeOptions={modelOptions.realtime}
visionOptions={modelOptions.vision}
dynamicVariableOptions={dynamicVariableOptions}
workflowSettings={settings}

37
frontend/src/components/workflow/panels/AgentNodePanel.tsx
View File

@@ -37,6 +37,7 @@ export function AgentNodePanel({
llmOptions,
asrOptions,
ttsOptions,
realtimeOptions,
visionOptions,
dynamicVariableOptions,
}: {
@@ -49,6 +50,7 @@ export function AgentNodePanel({
llmOptions: ModelOption[];
asrOptions: ModelOption[];
ttsOptions: ModelOption[];
realtimeOptions: ModelOption[];
visionOptions: ModelOption[];
dynamicVariableOptions: ModelOption[];
}) {
@@ -71,6 +73,11 @@ export function AgentNodePanel({
}
setPatch({
inheritGlobalConfig: false,
visionEnabled:
draft.visionEnabled ?? workflowSettings.visionEnabled,
visionModelResourceId:
(draft.visionModelResourceId as string) ||
workflowSettings.visionModelResourceId,
...(!isRealtime
? {
llmResourceId:
@@ -79,11 +86,6 @@ export function AgentNodePanel({
(draft.asrResourceId as string) || workflowSettings.asr || "",
ttsResourceId:
(draft.ttsResourceId as string) || workflowSettings.tts || "",
visionEnabled:
draft.visionEnabled ?? workflowSettings.visionEnabled,
visionModelResourceId:
(draft.visionModelResourceId as string) ||
workflowSettings.visionModelResourceId,
enableInterrupt:
draft.enableInterrupt ?? workflowSettings.allowInterrupt,
turnConfig: agentTurnConfig,
@@ -295,12 +297,29 @@ export function AgentNodePanel({
</PanelAnchor> : null}
<PanelAnchor id="capabilities">
{!isRealtime ? <VisionConfigSection
<VisionConfigSection
description="配置当前 Agent 是否可以按需理解用户摄像头画面"
hint="开启后，该 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，当前大语言模型必须支持图片输入。"
hint={
isRealtime
? "开启后，该 Agent 会获得读取当前视频画面的工具。Qwen Audio Realtime 需要选择独立视觉模型。"
: "开启后，该 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，当前大语言模型必须支持图片输入。"
}
enabled={Boolean(draft.visionEnabled)}
modelResourceId={draft.visionModelResourceId ?? ""}
mainModelResourceId={(draft.llmResourceId as string) || ""}
mainModelResourceId={
isRealtime
? workflowSettings.realtime ?? ""
: (draft.llmResourceId as string) || ""
}
mainModelSupportsImageInput={
isRealtime
? Boolean(
realtimeOptions.find(
(option) => option.value === workflowSettings.realtime,
)?.supportImageInput,
)
: undefined
}
modelOptions={visionOptions}
onEnabledChange={(visionEnabled) =>
setPatch({
@@ -311,7 +330,7 @@ export function AgentNodePanel({
onModelResourceIdChange={(visionModelResourceId) =>
set("visionModelResourceId", visionModelResourceId)
}
/> : null}
/>
<SectionCard
icon={<Database size={15} />}

25
frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
View File

@@ -175,12 +175,29 @@ export function GlobalSettingsPanel({
</PanelAnchor>
<PanelAnchor id="capabilities">
{settings.runtimeMode === "pipeline" ? <VisionConfigSection
<VisionConfigSection
description="配置继承全局设置的 Agent 是否可以按需理解用户摄像头画面"
hint="开启后，继承全局配置的 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，全局大语言模型必须支持图片输入。"
hint={
settings.runtimeMode === "realtime"
? "开启后，Agent 会获得读取当前视频画面的工具。Qwen Audio Realtime 需要选择独立视觉模型。"
: "开启后，继承全局配置的 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，全局大语言模型必须支持图片输入。"
}
enabled={settings.visionEnabled}
modelResourceId={settings.visionModelResourceId}
mainModelResourceId={settings.llm ?? ""}
mainModelResourceId={
settings.runtimeMode === "realtime"
? settings.realtime ?? ""
: settings.llm ?? ""
}
mainModelSupportsImageInput={
settings.runtimeMode === "realtime"
? Boolean(
modelOptions.realtime.find(
(option) => option.value === settings.realtime,
)?.supportImageInput,
)
: undefined
}
modelOptions={modelOptions.vision}
onEnabledChange={(visionEnabled) =>
onSettingsChange({
@@ -192,7 +209,7 @@ export function GlobalSettingsPanel({
onModelResourceIdChange={(visionModelResourceId) =>
onSettingsChange({ ...settings, visionModelResourceId })
}
/> : null}
/>
<SectionCard
icon={<Database size={15} />}

3
frontend/src/components/workflow/panels/NodeSettingsPanel.tsx
View File

@@ -31,6 +31,7 @@ export function NodeSettingsPanel({
llmOptions,
asrOptions,
ttsOptions,
realtimeOptions,
visionOptions,
dynamicVariableOptions,
workflowSettings,
@@ -44,6 +45,7 @@ export function NodeSettingsPanel({
llmOptions: ModelOption[];
asrOptions: ModelOption[];
ttsOptions: ModelOption[];
realtimeOptions: ModelOption[];
visionOptions: ModelOption[];
dynamicVariableOptions: ModelOption[];
workflowSettings: WorkflowSettings;
@@ -136,6 +138,7 @@ export function NodeSettingsPanel({
llmOptions={llmOptions}
asrOptions={asrOptions}
ttsOptions={ttsOptions}
realtimeOptions={realtimeOptions}
visionOptions={visionOptions}
dynamicVariableOptions={dynamicVariableOptions}
/>

1
frontend/src/components/workflow/types.ts
View File

@@ -30,6 +30,7 @@ export type ModelOption = {
value: string;
label: string;
disabled?: boolean;
supportImageInput?: boolean;
toolType?: Tool["type"];
systemKind?: SystemToolKind;
};