merge: add workflow vision configuration

2026-07-18 00:10:28 +08:00
parent e39bb48ba8 28c6380d8a
commit cd319d27b9
23 changed files with 553 additions and 34 deletions
--- a/frontend/src/components/assistant-editor/use-workflow-editor-state.ts
+++ b/frontend/src/components/assistant-editor/use-workflow-editor-state.ts
@@ -19,6 +19,8 @@ function settingsFromGraph(graph: WorkflowGraph): WorkflowSettings {
    llm: graph.settings.defaultLlmResourceId,
    asr: graph.settings.defaultAsrResourceId,
    tts: graph.settings.defaultTtsResourceId,
+    visionEnabled: graph.settings.visionEnabled,
+    visionModelResourceId: graph.settings.visionModelResourceId,
    toolIds: graph.settings.toolIds,
    knowledgeBaseId: graph.settings.knowledgeBaseId,
    knowledgeRetrievalConfig: {
@@ -49,6 +51,8 @@ export function useWorkflowEditorState() {
        defaultLlmResourceId: settings.llm ?? "",
        defaultAsrResourceId: settings.asr ?? "",
        defaultTtsResourceId: settings.tts ?? "",
+        visionEnabled: settings.visionEnabled,
+        visionModelResourceId: settings.visionModelResourceId,
        toolIds: settings.toolIds,
        knowledgeBaseId: settings.knowledgeBaseId,
        knowledgeMode: settings.knowledgeRetrievalConfig.mode,
--- a/frontend/src/components/assistant-editor/workflow-page.tsx
+++ b/frontend/src/components/assistant-editor/workflow-page.tsx
@@ -12,7 +12,10 @@ import {
 import { Button } from "@/components/ui/button";
 import { WorkflowEditor } from "@/components/workflow/WorkflowEditor";
 import type { WorkflowSettings } from "@/components/workflow/types";
-import type { WorkflowGraph } from "@/components/workflow/specs";
+import {
+  workflowUsesVision,
+  type WorkflowGraph,
+} from "@/components/workflow/specs";
 import type { DynamicVariableDefinition } from "@/lib/api";

 type ResourceOption = { value: string; label: string };
@@ -36,6 +39,7 @@ type WorkflowPageProps = {
    llm: ResourceOption[];
    asr: ResourceOption[];
    tts: ResourceOption[];
+    vision: ResourceOption[];
  };
  toolOptions: ResourceOption[];
  knowledgeOptions: ResourceOption[];
@@ -156,6 +160,7 @@ export function WorkflowPage({
              <DebugDrawer
                overlay
                assistantId={assistantId}
+                vision={workflowUsesVision(workflowGraph)}
                onClose={() => {
                  setWorkflowDebugOpen(false);
                  setActiveNodeId(null);
--- a/frontend/src/components/pages/AssistantPage.tsx
+++ b/frontend/src/components/pages/AssistantPage.tsx
@@ -56,6 +56,7 @@ import type { WorkflowSettings } from "@/components/workflow/types";
 import { SectionCard } from "@/components/editor/section-card";
 import {
  defaultGraph,
+  workflowUsesVision,
  type WorkflowGraph,
 } from "@/components/workflow/specs";
 import {
@@ -702,6 +703,12 @@ export function AssistantPage(props: AssistantPageProps) {
              assistant.modelResourceIds.LLM,
            asr: graph.settings?.defaultAsrResourceId || assistant.modelResourceIds.ASR,
            tts: graph.settings?.defaultTtsResourceId || assistant.modelResourceIds.TTS,
+            visionEnabled:
+              graph.settings?.visionEnabled ?? assistant.visionEnabled,
+            visionModelResourceId:
+              graph.settings?.visionModelResourceId ??
+              assistant.visionModelResourceId ??
+              "",
            toolIds: graph.settings?.toolIds ?? [],
            knowledgeBaseId:
              graph.settings?.knowledgeBaseId || assistant.knowledgeBaseId || "",
@@ -792,6 +799,8 @@ export function AssistantPage(props: AssistantPageProps) {
        type: "workflow",
        enableInterrupt: workflowSettings.allowInterrupt,
        turnConfig: workflowSettings.turnConfig,
+        visionEnabled: workflowUsesVision(workflowGraph),
+        visionModelResourceId: null,
        modelResourceIds: {
          ...(workflowSettings.llm ? { LLM: workflowSettings.llm } : {}),
          ...(workflowSettings.asr ? { ASR: workflowSettings.asr } : {}),
@@ -1314,6 +1323,7 @@ export function AssistantPage(props: AssistantPageProps) {
          llm: credOptions("LLM"),
          asr: credOptions("ASR"),
          tts: credOptions("TTS"),
+          vision: visionModelOptionsFor(""),
        }}
        toolOptions={tools
          .filter((tool) => tool.status === "active")
--- a/frontend/src/components/workflow/GenericNode.tsx
+++ b/frontend/src/components/workflow/GenericNode.tsx
@@ -47,6 +47,7 @@ export function GenericNode({ id, type, data, selected }: NodeProps) {
          nodeData.knowledgeBaseId ? "知识库" : null,
          nodeData.asrResourceId ? "独立 ASR" : null,
          nodeData.ttsResourceId ? "独立 TTS" : null,
+          nodeData.visionEnabled ? "视觉理解" : null,
        ].filter(Boolean)
    : type === "action" && nodeData.toolId
      ? ["确定性工具"]
--- a/frontend/src/components/workflow/WorkflowCanvas.tsx
+++ b/frontend/src/components/workflow/WorkflowCanvas.tsx
@@ -103,6 +103,8 @@ function fromFlow(nodes: Node[], edges: Edge[]): WorkflowGraph {
      defaultLlmResourceId: "",
      defaultAsrResourceId: "",
      defaultTtsResourceId: "",
+      visionEnabled: false,
+      visionModelResourceId: "",
      toolIds: [],
      knowledgeBaseId: "",
      knowledgeMode: "automatic",
@@ -170,6 +172,8 @@ export function WorkflowCanvas({
      defaultLlmResourceId: settings.llm ?? "",
      defaultAsrResourceId: settings.asr ?? "",
      defaultTtsResourceId: settings.tts ?? "",
+      visionEnabled: settings.visionEnabled,
+      visionModelResourceId: settings.visionModelResourceId,
      toolIds: settings.toolIds,
      knowledgeBaseId: settings.knowledgeBaseId,
      knowledgeMode: settings.knowledgeRetrievalConfig.mode,
@@ -187,6 +191,8 @@ export function WorkflowCanvas({
    settings.llm,
    settings.asr,
    settings.tts,
+    settings.visionEnabled,
+    settings.visionModelResourceId,
    settings.toolIds,
    settings.knowledgeBaseId,
    settings.knowledgeRetrievalConfig.mode,
@@ -645,6 +651,7 @@ export function WorkflowCanvas({
                          llmOptions={modelOptions.llm}
                          asrOptions={modelOptions.asr}
                          ttsOptions={modelOptions.tts}
+                          visionOptions={modelOptions.vision}
                          workflowSettings={settings}
                          onChange={(patch) =>
                            updateNodeData(editingNode.id, patch)
--- a/frontend/src/components/workflow/panels/AgentNodePanel.tsx
+++ b/frontend/src/components/workflow/panels/AgentNodePanel.tsx
@@ -1,6 +1,7 @@
 "use client";

 import {
+  Camera,
  Bot,
  Brain,
  Database,
@@ -11,6 +12,10 @@ import {
  Wrench,
 } from "lucide-react";

+import {
+  ResourceSelectField,
+  ToggleRow,
+} from "@/components/assistant-editor/editor-controls";
 import { KnowledgeRetrievalConfigDialog } from "@/components/editor/knowledge-retrieval-config-dialog";
 import { SectionCard } from "@/components/editor/section-card";
 import { TurnConfigEditor } from "@/components/turn-config-editor";
@@ -34,6 +39,7 @@ export function AgentNodePanel({
  llmOptions,
  asrOptions,
  ttsOptions,
+  visionOptions,
 }: {
  draft: WorkflowNodeData;
  set: (key: string, val: unknown) => void;
@@ -44,6 +50,7 @@ export function AgentNodePanel({
  llmOptions: ModelOption[];
  asrOptions: ModelOption[];
  ttsOptions: ModelOption[];
+  visionOptions: ModelOption[];
 }) {
  const inheritsGlobal = draft.inheritGlobalConfig !== false;
  const knowledgeConfig: KnowledgeRetrievalConfig = {
@@ -69,6 +76,11 @@ export function AgentNodePanel({
        (draft.asrResourceId as string) || workflowSettings.asr || "",
      ttsResourceId:
        (draft.ttsResourceId as string) || workflowSettings.tts || "",
+      visionEnabled:
+        draft.visionEnabled ?? workflowSettings.visionEnabled,
+      visionModelResourceId:
+        (draft.visionModelResourceId as string) ||
+        workflowSettings.visionModelResourceId,
      toolIds: draft.toolIds?.length
        ? draft.toolIds
        : workflowSettings.toolIds,
@@ -112,7 +124,7 @@ export function AgentNodePanel({
      <SectionCard
        icon={<Settings2 size={15} />}
        title="配置范围"
-        description="默认复用工作流全局的模型、语音、知识库和工具"
+        description="默认复用工作流全局的模型、语音、视觉、知识库和工具"
      >
        <div className="flex items-center justify-between gap-4 rounded-xl border border-hairline bg-canvas-soft px-3.5 py-3">
          <div>
@@ -197,7 +209,14 @@ export function AgentNodePanel({
              label="大语言模型"
              value={(draft.llmResourceId as string) || ""}
              options={llmOptions}
-              onChange={(value) => set("llmResourceId", value || "")}
+              onChange={(llmResourceId) =>
+                setPatch({
+                  llmResourceId: llmResourceId || "",
+                  ...(draft.visionModelResourceId === llmResourceId
+                    ? { visionModelResourceId: "" }
+                    : {}),
+                })
+              }
              noneLabel="请选择模型"
            />
            <NodeSelect
@@ -216,6 +235,49 @@ export function AgentNodePanel({
            />
          </SectionCard>

+          <SectionCard
+            icon={<Camera size={15} />}
+            title="视觉理解"
+            description="配置当前 Agent 是否可以按需理解用户摄像头画面"
+          >
+            <ToggleRow
+              title="允许理解当前画面"
+              hint="开启后，该 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，当前大语言模型必须支持图片输入。"
+              checked={Boolean(draft.visionEnabled)}
+              onChange={(visionEnabled) =>
+                setPatch({
+                  visionEnabled,
+                  ...(!visionEnabled
+                    ? { visionModelResourceId: "" }
+                    : {}),
+                })
+              }
+            />
+            {draft.visionEnabled && (
+              <>
+                <ResourceSelectField
+                  label="视觉模型"
+                  value={draft.visionModelResourceId ?? ""}
+                  onChange={(visionModelResourceId) =>
+                    set("visionModelResourceId", visionModelResourceId)
+                  }
+                  options={visionOptions.filter(
+                    (option) => option.value !== draft.llmResourceId,
+                  )}
+                  noneLabel="模型自己"
+                />
+                {!draft.visionModelResourceId &&
+                  !visionOptions.some(
+                    (option) => option.value === draft.llmResourceId,
+                  ) && (
+                    <p className="text-xs text-destructive">
+                      当前大语言模型未标记支持图片输入，请选择独立视觉模型。
+                    </p>
+                  )}
+              </>
+            )}
+          </SectionCard>
+
          <SectionCard
            icon={<Database size={15} />}
            title="知识库配置"
@@ -279,5 +341,3 @@ export function AgentNodePanel({
    </div>
  );
 }
-
-
--- a/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
+++ b/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
@@ -9,6 +9,10 @@ import {
  Wrench,
 } from "lucide-react";

+import {
+  ResourceSelectField,
+  ToggleRow,
+} from "@/components/assistant-editor/editor-controls";
 import { KnowledgeRetrievalConfigDialog } from "@/components/editor/knowledge-retrieval-config-dialog";
 import { SectionCard } from "@/components/editor/section-card";
 import { TurnConfigEditor } from "@/components/turn-config-editor";
@@ -63,8 +67,51 @@ export function GlobalSettingsPanel({
          label="大语言模型"
          value={settings.llm}
          options={modelOptions.llm}
-          onChange={(v) => onSettingsChange({ ...settings, llm: v })}
+          onChange={(llm) =>
+            onSettingsChange({
+              ...settings,
+              llm,
+              ...(settings.visionModelResourceId === llm
+                ? { visionModelResourceId: "" }
+                : {}),
+            })
+          }
        />
+        <ToggleRow
+          title="视觉理解"
+          hint="开启后，继承全局配置的 Agent 可以按需理解当前视频画面。选择「模型自己」时，全局大语言模型必须支持图片输入。"
+          checked={settings.visionEnabled}
+          onChange={(visionEnabled) =>
+            onSettingsChange({
+              ...settings,
+              visionEnabled,
+              ...(!visionEnabled ? { visionModelResourceId: "" } : {}),
+            })
+          }
+        />
+        {settings.visionEnabled && (
+          <>
+            <ResourceSelectField
+              label="视觉模型"
+              value={settings.visionModelResourceId}
+              onChange={(visionModelResourceId) =>
+                onSettingsChange({ ...settings, visionModelResourceId })
+              }
+              options={modelOptions.vision.filter(
+                (option) => option.value !== settings.llm,
+              )}
+              noneLabel="模型自己"
+            />
+            {!settings.visionModelResourceId &&
+              !modelOptions.vision.some(
+                (option) => option.value === settings.llm,
+              ) && (
+                <p className="text-xs text-destructive">
+                  当前大语言模型未标记支持图片输入，请选择独立视觉模型。
+                </p>
+              )}
+          </>
+        )}
      </SectionCard>

      <SectionCard
@@ -146,5 +193,3 @@ export function GlobalSettingsPanel({
    </div>
  );
 }
-
-
--- a/frontend/src/components/workflow/panels/NodeSettingsPanel.tsx
+++ b/frontend/src/components/workflow/panels/NodeSettingsPanel.tsx
@@ -24,6 +24,7 @@ export function NodeSettingsPanel({
  llmOptions,
  asrOptions,
  ttsOptions,
+  visionOptions,
  workflowSettings,
  onChange,
 }: {
@@ -35,6 +36,7 @@ export function NodeSettingsPanel({
  llmOptions: ModelOption[];
  asrOptions: ModelOption[];
  ttsOptions: ModelOption[];
+  visionOptions: ModelOption[];
  workflowSettings: WorkflowSettings;
  onChange: (patch: WorkflowNodeData) => void;
 }) {
@@ -97,6 +99,7 @@ export function NodeSettingsPanel({
        llmOptions={llmOptions}
        asrOptions={asrOptions}
        ttsOptions={ttsOptions}
+        visionOptions={visionOptions}
      />
    );
  }
@@ -455,4 +458,3 @@ function WorkflowNodePanelForm({
 }

 /** Agent 右栏：与 AssistantPage 共用紧凑 SectionCard。 */
-
--- a/frontend/src/components/workflow/specs.ts
+++ b/frontend/src/components/workflow/specs.ts
@@ -38,6 +38,8 @@ export type WorkflowNodeData = {
  llmResourceId?: string;
  asrResourceId?: string;
  ttsResourceId?: string;
+  visionEnabled?: boolean;
+  visionModelResourceId?: string;
  enableInterrupt?: boolean;
  turnConfig?: TurnConfig;
  toolId?: string;
@@ -140,6 +142,8 @@ export type WorkflowGraph = {
    defaultLlmResourceId: string;
    defaultAsrResourceId: string;
    defaultTtsResourceId: string;
+    visionEnabled: boolean;
+    visionModelResourceId: string;
    toolIds: string[];
    knowledgeBaseId: string;
    knowledgeMode: "automatic" | "on_demand";
@@ -172,6 +176,8 @@ export function defaultGraph(): WorkflowGraph {
      defaultLlmResourceId: "",
      defaultAsrResourceId: "",
      defaultTtsResourceId: "",
+      visionEnabled: false,
+      visionModelResourceId: "",
      toolIds: [],
      knowledgeBaseId: "",
      knowledgeMode: "automatic",
@@ -234,3 +240,14 @@ export function defaultGraph(): WorkflowGraph {
    ],
  };
 }
+
+/** The camera stays connected when at least one effective Agent can use it. */
+export function workflowUsesVision(graph: WorkflowGraph): boolean {
+  return graph.nodes.some((node) => {
+    if (node.type !== "agent") return false;
+    const inheritsGlobal = node.data.inheritGlobalConfig !== false;
+    return inheritsGlobal
+      ? graph.settings.visionEnabled
+      : Boolean(node.data.visionEnabled);
+  });
+}
--- a/frontend/src/components/workflow/types.ts
+++ b/frontend/src/components/workflow/types.ts
@@ -8,6 +8,8 @@ export type WorkflowSettings = {
  llm?: string;
  asr?: string;
  tts?: string;
+  visionEnabled: boolean;
+  visionModelResourceId: string;
  toolIds: string[];
  knowledgeBaseId: string;
  knowledgeRetrievalConfig: KnowledgeRetrievalConfig;
@@ -23,7 +25,12 @@ export type WorkflowEditorProps = {
  onChange?: (graph: WorkflowGraph) => void;
  settings: WorkflowSettings;
  onSettingsChange: (settings: WorkflowSettings) => void;
-  modelOptions: { llm: ModelOption[]; asr: ModelOption[]; tts: ModelOption[] };
+  modelOptions: {
+    llm: ModelOption[];
+    asr: ModelOption[];
+    tts: ModelOption[];
+    vision: ModelOption[];
+  };
  toolOptions?: ModelOption[];
  knowledgeOptions?: ModelOption[];
  onOpenDynamicVariables: () => void;