feat: add realtime workflow vision tools

2026-08-05 13:35:00 +08:00
parent 11374238f3
commit 10bea48d72
16 changed files with 362 additions and 103 deletions
--- a/frontend/src/components/editor/vision-config-section.tsx
+++ b/frontend/src/components/editor/vision-config-section.tsx
@@ -19,6 +19,7 @@ export function VisionConfigSection({
  enabled,
  modelResourceId,
  mainModelResourceId,
+  mainModelSupportsImageInput,
  modelOptions,
  onEnabledChange,
  onModelResourceIdChange,
@@ -28,13 +29,14 @@ export function VisionConfigSection({
  enabled: boolean;
  modelResourceId: string;
  mainModelResourceId: string;
+  mainModelSupportsImageInput?: boolean;
  modelOptions: VisionModelOption[];
  onEnabledChange: (enabled: boolean) => void;
  onModelResourceIdChange: (modelResourceId: string) => void;
 }) {
-  const mainModelSupportsVision = modelOptions.some(
-    (option) => option.value === mainModelResourceId,
-  );
+  const mainModelSupportsVision =
+    mainModelSupportsImageInput ??
+    modelOptions.some((option) => option.value === mainModelResourceId);
  const independentModelOptions = modelOptions.filter(
    (option) => option.value !== mainModelResourceId,
  );
@@ -62,7 +64,7 @@ export function VisionConfigSection({
          />
          {!modelResourceId && !mainModelSupportsVision && (
            <p className="text-xs text-destructive">
-              当前大语言模型未标记支持图片输入，请选择独立视觉模型。
+              当前模型未标记支持图片输入，请选择独立视觉模型。
            </p>
          )}
        </>
--- a/frontend/src/components/pages/AssistantPage.tsx
+++ b/frontend/src/components/pages/AssistantPage.tsx
@@ -411,7 +411,11 @@ export function AssistantPage(props: AssistantPageProps) {
  const credOptions = (type: ModelResource["capability"]) =>
    modelResources
      .filter((c) => c.capability === type)
-      .map((c) => ({ value: c.id, label: c.name }));
+      .map((c) => ({
+        value: c.id,
+        label: c.name,
+        supportImageInput: c.supportImageInput,
+      }));
  const agentOptions = (interfaceType: "dify" | "fastgpt" | "opencode") =>
    modelResources
      .filter(
--- a/frontend/src/components/pages/ComponentsModelsPage.tsx
+++ b/frontend/src/components/pages/ComponentsModelsPage.tsx
@@ -66,6 +66,20 @@ const capabilities: ModelType[] = [
 ];

 const capabilityFilters = ["全部", ...capabilities] as const;
+const nativeImageUnsupportedRealtimeInterfaces = new Set([
+  "qwen-audio-realtime",
+]);
+
+function canConfigureImageInput(
+  capability: ModelType,
+  interfaceType: string,
+): boolean {
+  return (
+    capability === "LLM" ||
+    (capability === "Realtime" &&
+      !nativeImageUnsupportedRealtimeInterfaces.has(interfaceType))
+  );
+}

 type ResourceDraft = {
  name: string;
@@ -294,6 +308,12 @@ export function ComponentsModelsPage() {
      interfaceType,
      values: definition ? defaults(definition) : {},
      secrets: {},
+      supportImageInput: canConfigureImageInput(
+        previous.capability,
+        interfaceType,
+      )
+        ? previous.supportImageInput
+        : false,
    }));
    setStoredSecretMasks({});
    setExtraBodyDraft("");
@@ -310,8 +330,7 @@ export function ComponentsModelsPage() {
      interfaceType: first?.interfaceType ?? "",
      values: first ? defaults(first) : {},
      secrets: {},
-      supportImageInput:
-        capability === "LLM" ? previous.supportImageInput : false,
+      supportImageInput: false,
    }));
    setStoredSecretMasks({});
    setExtraBodyDraft("");
@@ -392,7 +411,9 @@ export function ComponentsModelsPage() {
        ? { ...storedSecretMasks, ...draft.secrets }
        : draft.secrets,
      supportImageInput:
-        draft.capability === "LLM" ? draft.supportImageInput : false,
+        canConfigureImageInput(draft.capability, draft.interfaceType)
+          ? draft.supportImageInput
+          : false,
      enabled: editingId
        ? resources.find((resource) => resource.id === editingId)?.enabled ?? true
        : true,
@@ -747,11 +768,25 @@ export function ComponentsModelsPage() {
                    </Select>
                  </Field>
                </div>
-                {draft.capability === "LLM" && (
+                {(draft.capability === "LLM" ||
+                  draft.capability === "Realtime") && (
                  <div className="flex items-center justify-between rounded-xl border border-hairline p-4">
-                    <span className="text-sm font-medium">支持图片输入</span>
+                    <div className="space-y-1">
+                      <p className="text-sm font-medium">原生支持图片输入</p>
+                      {draft.capability === "Realtime" && (
+                        <p className="text-xs text-muted-foreground">
+                          Qwen Audio Realtime 请保持关闭，并为助手选择独立视觉模型。
+                        </p>
+                      )}
+                    </div>
                    <Switch
                      checked={draft.supportImageInput}
+                      disabled={
+                        !canConfigureImageInput(
+                          draft.capability,
+                          draft.interfaceType,
+                        )
+                      }
                      onCheckedChange={(checked) =>
                        setDraft((previous) => ({
                          ...previous,
--- a/frontend/src/components/workflow/WorkflowCanvas.tsx
+++ b/frontend/src/components/workflow/WorkflowCanvas.tsx
@@ -798,6 +798,7 @@ export function WorkflowCanvas({
                          llmOptions={modelOptions.llm}
                          asrOptions={modelOptions.asr}
                          ttsOptions={modelOptions.tts}
+                          realtimeOptions={modelOptions.realtime}
                          visionOptions={modelOptions.vision}
                          dynamicVariableOptions={dynamicVariableOptions}
                          workflowSettings={settings}
--- a/frontend/src/components/workflow/panels/AgentNodePanel.tsx
+++ b/frontend/src/components/workflow/panels/AgentNodePanel.tsx
@@ -37,6 +37,7 @@ export function AgentNodePanel({
  llmOptions,
  asrOptions,
  ttsOptions,
+  realtimeOptions,
  visionOptions,
  dynamicVariableOptions,
 }: {
@@ -49,6 +50,7 @@ export function AgentNodePanel({
  llmOptions: ModelOption[];
  asrOptions: ModelOption[];
  ttsOptions: ModelOption[];
+  realtimeOptions: ModelOption[];
  visionOptions: ModelOption[];
  dynamicVariableOptions: ModelOption[];
 }) {
@@ -71,6 +73,11 @@ export function AgentNodePanel({
    }
    setPatch({
      inheritGlobalConfig: false,
+      visionEnabled:
+        draft.visionEnabled ?? workflowSettings.visionEnabled,
+      visionModelResourceId:
+        (draft.visionModelResourceId as string) ||
+        workflowSettings.visionModelResourceId,
      ...(!isRealtime
        ? {
            llmResourceId:
@@ -79,11 +86,6 @@ export function AgentNodePanel({
              (draft.asrResourceId as string) || workflowSettings.asr || "",
            ttsResourceId:
              (draft.ttsResourceId as string) || workflowSettings.tts || "",
-            visionEnabled:
-              draft.visionEnabled ?? workflowSettings.visionEnabled,
-            visionModelResourceId:
-              (draft.visionModelResourceId as string) ||
-              workflowSettings.visionModelResourceId,
            enableInterrupt:
              draft.enableInterrupt ?? workflowSettings.allowInterrupt,
            turnConfig: agentTurnConfig,
@@ -295,12 +297,29 @@ export function AgentNodePanel({
          </PanelAnchor> : null}

          <PanelAnchor id="capabilities">
-            {!isRealtime ? <VisionConfigSection
+            <VisionConfigSection
              description="配置当前 Agent 是否可以按需理解用户摄像头画面"
-              hint="开启后，该 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，当前大语言模型必须支持图片输入。"
+              hint={
+                isRealtime
+                  ? "开启后，该 Agent 会获得读取当前视频画面的工具。Qwen Audio Realtime 需要选择独立视觉模型。"
+                  : "开启后，该 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，当前大语言模型必须支持图片输入。"
+              }
              enabled={Boolean(draft.visionEnabled)}
              modelResourceId={draft.visionModelResourceId ?? ""}
-              mainModelResourceId={(draft.llmResourceId as string) || ""}
+              mainModelResourceId={
+                isRealtime
+                  ? workflowSettings.realtime ?? ""
+                  : (draft.llmResourceId as string) || ""
+              }
+              mainModelSupportsImageInput={
+                isRealtime
+                  ? Boolean(
+                      realtimeOptions.find(
+                        (option) => option.value === workflowSettings.realtime,
+                      )?.supportImageInput,
+                    )
+                  : undefined
+              }
              modelOptions={visionOptions}
              onEnabledChange={(visionEnabled) =>
                setPatch({
@@ -311,7 +330,7 @@ export function AgentNodePanel({
              onModelResourceIdChange={(visionModelResourceId) =>
                set("visionModelResourceId", visionModelResourceId)
              }
-            /> : null}
+            />

          <SectionCard
            icon={<Database size={15} />}
--- a/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
+++ b/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
@@ -175,12 +175,29 @@ export function GlobalSettingsPanel({
      </PanelAnchor>

      <PanelAnchor id="capabilities">
-        {settings.runtimeMode === "pipeline" ? <VisionConfigSection
+        <VisionConfigSection
          description="配置继承全局设置的 Agent 是否可以按需理解用户摄像头画面"
-          hint="开启后，继承全局配置的 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，全局大语言模型必须支持图片输入。"
+          hint={
+            settings.runtimeMode === "realtime"
+              ? "开启后，Agent 会获得读取当前视频画面的工具。Qwen Audio Realtime 需要选择独立视觉模型。"
+              : "开启后，继承全局配置的 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，全局大语言模型必须支持图片输入。"
+          }
          enabled={settings.visionEnabled}
          modelResourceId={settings.visionModelResourceId}
-          mainModelResourceId={settings.llm ?? ""}
+          mainModelResourceId={
+            settings.runtimeMode === "realtime"
+              ? settings.realtime ?? ""
+              : settings.llm ?? ""
+          }
+          mainModelSupportsImageInput={
+            settings.runtimeMode === "realtime"
+              ? Boolean(
+                  modelOptions.realtime.find(
+                    (option) => option.value === settings.realtime,
+                  )?.supportImageInput,
+                )
+              : undefined
+          }
          modelOptions={modelOptions.vision}
          onEnabledChange={(visionEnabled) =>
            onSettingsChange({
@@ -192,7 +209,7 @@ export function GlobalSettingsPanel({
          onModelResourceIdChange={(visionModelResourceId) =>
            onSettingsChange({ ...settings, visionModelResourceId })
          }
-        /> : null}
+        />

        <SectionCard
          icon={<Database size={15} />}
--- a/frontend/src/components/workflow/panels/NodeSettingsPanel.tsx
+++ b/frontend/src/components/workflow/panels/NodeSettingsPanel.tsx
@@ -31,6 +31,7 @@ export function NodeSettingsPanel({
  llmOptions,
  asrOptions,
  ttsOptions,
+  realtimeOptions,
  visionOptions,
  dynamicVariableOptions,
  workflowSettings,
@@ -44,6 +45,7 @@ export function NodeSettingsPanel({
  llmOptions: ModelOption[];
  asrOptions: ModelOption[];
  ttsOptions: ModelOption[];
+  realtimeOptions: ModelOption[];
  visionOptions: ModelOption[];
  dynamicVariableOptions: ModelOption[];
  workflowSettings: WorkflowSettings;
@@ -136,6 +138,7 @@ export function NodeSettingsPanel({
        llmOptions={llmOptions}
        asrOptions={asrOptions}
        ttsOptions={ttsOptions}
+        realtimeOptions={realtimeOptions}
        visionOptions={visionOptions}
        dynamicVariableOptions={dynamicVariableOptions}
      />
--- a/frontend/src/components/workflow/types.ts
+++ b/frontend/src/components/workflow/types.ts
@@ -30,6 +30,7 @@ export type ModelOption = {
  value: string;
  label: string;
  disabled?: boolean;
+  supportImageInput?: boolean;
  toolType?: Tool["type"];
  systemKind?: SystemToolKind;
 };