feat(workflow): add per-agent vision configuration

2026-07-18 00:00:28 +08:00
parent bdf3d3dd9c
commit 28c6380d8a
23 changed files with 553 additions and 34 deletions
--- a/frontend/src/components/workflow/panels/AgentNodePanel.tsx
+++ b/frontend/src/components/workflow/panels/AgentNodePanel.tsx
@@ -1,6 +1,7 @@
 "use client";

 import {
+  Camera,
  Bot,
  Brain,
  Database,
@@ -11,6 +12,10 @@ import {
  Wrench,
 } from "lucide-react";

+import {
+  ResourceSelectField,
+  ToggleRow,
+} from "@/components/assistant-editor/editor-controls";
 import { KnowledgeRetrievalConfigDialog } from "@/components/editor/knowledge-retrieval-config-dialog";
 import { SectionCard } from "@/components/editor/section-card";
 import { TurnConfigEditor } from "@/components/turn-config-editor";
@@ -34,6 +39,7 @@ export function AgentNodePanel({
  llmOptions,
  asrOptions,
  ttsOptions,
+  visionOptions,
 }: {
  draft: WorkflowNodeData;
  set: (key: string, val: unknown) => void;
@@ -44,6 +50,7 @@ export function AgentNodePanel({
  llmOptions: ModelOption[];
  asrOptions: ModelOption[];
  ttsOptions: ModelOption[];
+  visionOptions: ModelOption[];
 }) {
  const inheritsGlobal = draft.inheritGlobalConfig !== false;
  const knowledgeConfig: KnowledgeRetrievalConfig = {
@@ -69,6 +76,11 @@ export function AgentNodePanel({
        (draft.asrResourceId as string) || workflowSettings.asr || "",
      ttsResourceId:
        (draft.ttsResourceId as string) || workflowSettings.tts || "",
+      visionEnabled:
+        draft.visionEnabled ?? workflowSettings.visionEnabled,
+      visionModelResourceId:
+        (draft.visionModelResourceId as string) ||
+        workflowSettings.visionModelResourceId,
      toolIds: draft.toolIds?.length
        ? draft.toolIds
        : workflowSettings.toolIds,
@@ -112,7 +124,7 @@ export function AgentNodePanel({
      <SectionCard
        icon={<Settings2 size={15} />}
        title="配置范围"
-        description="默认复用工作流全局的模型、语音、知识库和工具"
+        description="默认复用工作流全局的模型、语音、视觉、知识库和工具"
      >
        <div className="flex items-center justify-between gap-4 rounded-xl border border-hairline bg-canvas-soft px-3.5 py-3">
          <div>
@@ -197,7 +209,14 @@ export function AgentNodePanel({
              label="大语言模型"
              value={(draft.llmResourceId as string) || ""}
              options={llmOptions}
-              onChange={(value) => set("llmResourceId", value || "")}
+              onChange={(llmResourceId) =>
+                setPatch({
+                  llmResourceId: llmResourceId || "",
+                  ...(draft.visionModelResourceId === llmResourceId
+                    ? { visionModelResourceId: "" }
+                    : {}),
+                })
+              }
              noneLabel="请选择模型"
            />
            <NodeSelect
@@ -216,6 +235,49 @@ export function AgentNodePanel({
            />
          </SectionCard>

+          <SectionCard
+            icon={<Camera size={15} />}
+            title="视觉理解"
+            description="配置当前 Agent 是否可以按需理解用户摄像头画面"
+          >
+            <ToggleRow
+              title="允许理解当前画面"
+              hint="开启后，该 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，当前大语言模型必须支持图片输入。"
+              checked={Boolean(draft.visionEnabled)}
+              onChange={(visionEnabled) =>
+                setPatch({
+                  visionEnabled,
+                  ...(!visionEnabled
+                    ? { visionModelResourceId: "" }
+                    : {}),
+                })
+              }
+            />
+            {draft.visionEnabled && (
+              <>
+                <ResourceSelectField
+                  label="视觉模型"
+                  value={draft.visionModelResourceId ?? ""}
+                  onChange={(visionModelResourceId) =>
+                    set("visionModelResourceId", visionModelResourceId)
+                  }
+                  options={visionOptions.filter(
+                    (option) => option.value !== draft.llmResourceId,
+                  )}
+                  noneLabel="模型自己"
+                />
+                {!draft.visionModelResourceId &&
+                  !visionOptions.some(
+                    (option) => option.value === draft.llmResourceId,
+                  ) && (
+                    <p className="text-xs text-destructive">
+                      当前大语言模型未标记支持图片输入，请选择独立视觉模型。
+                    </p>
+                  )}
+              </>
+            )}
+          </SectionCard>
+
          <SectionCard
            icon={<Database size={15} />}
            title="知识库配置"
@@ -279,5 +341,3 @@ export function AgentNodePanel({
    </div>
  );
 }
-
-
--- a/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
+++ b/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
@@ -9,6 +9,10 @@ import {
  Wrench,
 } from "lucide-react";

+import {
+  ResourceSelectField,
+  ToggleRow,
+} from "@/components/assistant-editor/editor-controls";
 import { KnowledgeRetrievalConfigDialog } from "@/components/editor/knowledge-retrieval-config-dialog";
 import { SectionCard } from "@/components/editor/section-card";
 import { TurnConfigEditor } from "@/components/turn-config-editor";
@@ -63,8 +67,51 @@ export function GlobalSettingsPanel({
          label="大语言模型"
          value={settings.llm}
          options={modelOptions.llm}
-          onChange={(v) => onSettingsChange({ ...settings, llm: v })}
+          onChange={(llm) =>
+            onSettingsChange({
+              ...settings,
+              llm,
+              ...(settings.visionModelResourceId === llm
+                ? { visionModelResourceId: "" }
+                : {}),
+            })
+          }
        />
+        <ToggleRow
+          title="视觉理解"
+          hint="开启后，继承全局配置的 Agent 可以按需理解当前视频画面。选择「模型自己」时，全局大语言模型必须支持图片输入。"
+          checked={settings.visionEnabled}
+          onChange={(visionEnabled) =>
+            onSettingsChange({
+              ...settings,
+              visionEnabled,
+              ...(!visionEnabled ? { visionModelResourceId: "" } : {}),
+            })
+          }
+        />
+        {settings.visionEnabled && (
+          <>
+            <ResourceSelectField
+              label="视觉模型"
+              value={settings.visionModelResourceId}
+              onChange={(visionModelResourceId) =>
+                onSettingsChange({ ...settings, visionModelResourceId })
+              }
+              options={modelOptions.vision.filter(
+                (option) => option.value !== settings.llm,
+              )}
+              noneLabel="模型自己"
+            />
+            {!settings.visionModelResourceId &&
+              !modelOptions.vision.some(
+                (option) => option.value === settings.llm,
+              ) && (
+                <p className="text-xs text-destructive">
+                  当前大语言模型未标记支持图片输入，请选择独立视觉模型。
+                </p>
+              )}
+          </>
+        )}
      </SectionCard>

      <SectionCard
@@ -146,5 +193,3 @@ export function GlobalSettingsPanel({
    </div>
  );
 }
-
-
--- a/frontend/src/components/workflow/panels/NodeSettingsPanel.tsx
+++ b/frontend/src/components/workflow/panels/NodeSettingsPanel.tsx
@@ -24,6 +24,7 @@ export function NodeSettingsPanel({
  llmOptions,
  asrOptions,
  ttsOptions,
+  visionOptions,
  workflowSettings,
  onChange,
 }: {
@@ -35,6 +36,7 @@ export function NodeSettingsPanel({
  llmOptions: ModelOption[];
  asrOptions: ModelOption[];
  ttsOptions: ModelOption[];
+  visionOptions: ModelOption[];
  workflowSettings: WorkflowSettings;
  onChange: (patch: WorkflowNodeData) => void;
 }) {
@@ -97,6 +99,7 @@ export function NodeSettingsPanel({
        llmOptions={llmOptions}
        asrOptions={asrOptions}
        ttsOptions={ttsOptions}
+        visionOptions={visionOptions}
      />
    );
  }
@@ -455,4 +458,3 @@ function WorkflowNodePanelForm({
 }

 /** Agent 右栏：与 AssistantPage 共用紧凑 SectionCard。 */
-