feat(workflow): add edge-tool routing and realtime runtime

2026-08-04 22:29:04 +08:00
parent 1902ffc240
commit 59588ba88d
25 changed files with 1952 additions and 107 deletions
--- a/frontend/src/components/assistant-editor/workflow-page.tsx
+++ b/frontend/src/components/assistant-editor/workflow-page.tsx
@@ -36,6 +36,7 @@ type WorkflowPageProps = {
  dirty: boolean;
  saveError: string | null;
  modelOptions: {
+    realtime: ResourceOption[];
    llm: ResourceOption[];
    asr: ResourceOption[];
    tts: ResourceOption[];
--- a/frontend/src/components/pages/AssistantPage.tsx
+++ b/frontend/src/components/pages/AssistantPage.tsx
@@ -711,6 +711,13 @@ export function AssistantPage(props: AssistantPageProps) {
              ? (assistant.graph as WorkflowGraph)
              : defaultGraph();
          const wfSettings: WorkflowSettings = {
+            runtimeMode:
+              graph.settings?.runtimeMode ?? assistant.runtimeMode ?? "pipeline",
+            llmRoutingMode:
+              graph.settings?.llmRoutingMode ?? "llm_router",
+            realtime:
+              graph.settings?.defaultRealtimeResourceId ||
+              assistant.modelResourceIds.Realtime,
            llm:
              graph.settings?.defaultLlmResourceId ||
              assistant.modelResourceIds.LLM,
@@ -811,11 +818,15 @@ export function AssistantPage(props: AssistantPageProps) {
      baseUpsert({
        name: workflowName.trim(),
        type: "workflow",
+        runtimeMode: workflowSettings.runtimeMode,
        enableInterrupt: workflowSettings.allowInterrupt,
        turnConfig: workflowSettings.turnConfig,
        visionEnabled: workflowUsesVision(workflowGraph),
        visionModelResourceId: null,
        modelResourceIds: {
+          ...(workflowSettings.realtime
+            ? { Realtime: workflowSettings.realtime }
+            : {}),
          ...(workflowSettings.llm ? { LLM: workflowSettings.llm } : {}),
          ...(workflowSettings.asr ? { ASR: workflowSettings.asr } : {}),
          ...(workflowSettings.tts ? { TTS: workflowSettings.tts } : {}),
@@ -1294,6 +1305,7 @@ export function AssistantPage(props: AssistantPageProps) {
        dirty={dirty}
        saveError={saveError}
        modelOptions={{
+          realtime: credOptions("Realtime"),
          llm: credOptions("LLM"),
          asr: credOptions("ASR"),
          tts: credOptions("TTS"),
--- a/frontend/src/components/workflow/WorkflowCanvas.tsx
+++ b/frontend/src/components/workflow/WorkflowCanvas.tsx
@@ -119,7 +119,10 @@ function fromFlow(nodes: Node[], edges: Edge[]): WorkflowGraph {
  return {
    specVersion: 3,
    settings: {
+      runtimeMode: "pipeline",
+      llmRoutingMode: "edge_tool",
      globalPrompt: "",
+      defaultRealtimeResourceId: "",
      defaultLlmResourceId: "",
      defaultAsrResourceId: "",
      defaultTtsResourceId: "",
@@ -809,6 +812,7 @@ export function WorkflowCanvas({
                            editingEdge.source,
                            editingEdge.id,
                          )}
+                          llmRoutingMode={settings.llmRoutingMode}
                          onChange={(patch) =>
                            updateEdgeData(editingEdge.id, patch)
                          }
--- a/frontend/src/components/workflow/graph-settings.ts
+++ b/frontend/src/components/workflow/graph-settings.ts
@@ -6,6 +6,9 @@ export function settingsFromWorkflowGraph(
  graph: WorkflowGraph,
 ): WorkflowSettings {
  return {
+    runtimeMode: graph.settings.runtimeMode ?? "pipeline",
+    llmRoutingMode: graph.settings.llmRoutingMode ?? "llm_router",
+    realtime: graph.settings.defaultRealtimeResourceId,
    globalPrompt: graph.settings.globalPrompt,
    llm: graph.settings.defaultLlmResourceId,
    asr: graph.settings.defaultAsrResourceId,
@@ -32,7 +35,10 @@ export function workflowGraphWithSettings(
  return {
    ...graph,
    settings: {
+      runtimeMode: settings.runtimeMode,
+      llmRoutingMode: settings.llmRoutingMode,
      globalPrompt: settings.globalPrompt,
+      defaultRealtimeResourceId: settings.realtime ?? "",
      defaultLlmResourceId: settings.llm ?? "",
      defaultAsrResourceId: settings.asr ?? "",
      defaultTtsResourceId: settings.tts ?? "",
--- a/frontend/src/components/workflow/panels/ActionNodePanel.tsx
+++ b/frontend/src/components/workflow/panels/ActionNodePanel.tsx
@@ -23,6 +23,7 @@ type ActionNodePanelProps = {
  setArgumentsJson: (value: string) => void;
  setAssignmentsJson: (value: string) => void;
  commitActionJson: (argumentsValue: string, assignmentsValue: string) => void;
+  runtimeMode: "pipeline" | "realtime";
 };

 export function ActionNodePanel({
@@ -35,6 +36,7 @@ export function ActionNodePanel({
  setArgumentsJson,
  setAssignmentsJson,
  commitActionJson,
+  runtimeMode,
 }: ActionNodePanelProps) {
  const resultAssignmentMode = actionResultAssignmentMode(draft);
  const userInputPolicy = actionUserInputPolicy(draft);
@@ -113,14 +115,23 @@ export function ActionNodePanel({
        label="执行期间用户输入"
        value={userInputPolicy}
        options={[
-          { value: "queue", label: "允许输入并排队（默认）" },
+          ...(runtimeMode === "pipeline"
+            ? [{ value: "queue", label: "允许输入并排队（默认）" }]
+            : []),
          { value: "block", label: "暂时禁止输入" },
        ]}
-        onChange={(value) => set("userInputPolicy", value || "queue")}
+        onChange={(value) =>
+          set(
+            "userInputPolicy",
+            value || (runtimeMode === "realtime" ? "block" : "queue"),
+          )
+        }
        allowNone={false}
      />
      <p className="-mt-1 text-xs leading-5 text-muted-foreground">
-        {userInputPolicy === "queue"
+        {runtimeMode === "realtime"
+          ? "Realtime 模式必须阻止 Action 执行期间的输入，避免端到端模型提前生成回复。"
+          : userInputPolicy === "queue"
          ? "用户输入会保留，Action 完成并进入后续节点后再处理。"
          : "Action 执行期间忽略新的语音、文本和图片输入。"}
      </p>
--- a/frontend/src/components/workflow/panels/AgentNodePanel.tsx
+++ b/frontend/src/components/workflow/panels/AgentNodePanel.tsx
@@ -53,6 +53,7 @@ export function AgentNodePanel({
  dynamicVariableOptions: ModelOption[];
 }) {
  const inheritsGlobal = draft.inheritGlobalConfig !== false;
+  const isRealtime = workflowSettings.runtimeMode === "realtime";
  const knowledgeConfig: KnowledgeRetrievalConfig = {
    mode:
      draft.knowledgeMode === "on_demand" ? "on_demand" : "automatic",
@@ -70,17 +71,24 @@ export function AgentNodePanel({
    }
    setPatch({
      inheritGlobalConfig: false,
-      llmResourceId:
-        (draft.llmResourceId as string) || workflowSettings.llm || "",
-      asrResourceId:
-        (draft.asrResourceId as string) || workflowSettings.asr || "",
-      ttsResourceId:
-        (draft.ttsResourceId as string) || workflowSettings.tts || "",
-      visionEnabled:
-        draft.visionEnabled ?? workflowSettings.visionEnabled,
-      visionModelResourceId:
-        (draft.visionModelResourceId as string) ||
-        workflowSettings.visionModelResourceId,
+      ...(!isRealtime
+        ? {
+            llmResourceId:
+              (draft.llmResourceId as string) || workflowSettings.llm || "",
+            asrResourceId:
+              (draft.asrResourceId as string) || workflowSettings.asr || "",
+            ttsResourceId:
+              (draft.ttsResourceId as string) || workflowSettings.tts || "",
+            visionEnabled:
+              draft.visionEnabled ?? workflowSettings.visionEnabled,
+            visionModelResourceId:
+              (draft.visionModelResourceId as string) ||
+              workflowSettings.visionModelResourceId,
+            enableInterrupt:
+              draft.enableInterrupt ?? workflowSettings.allowInterrupt,
+            turnConfig: agentTurnConfig,
+          }
+        : {}),
      toolIds: draft.toolIds?.length
        ? draft.toolIds
        : workflowSettings.toolIds,
@@ -98,9 +106,6 @@ export function AgentNodePanel({
      knowledgeScoreThreshold:
        draft.knowledgeScoreThreshold ??
        workflowSettings.knowledgeRetrievalConfig.scoreThreshold,
-      enableInterrupt:
-        draft.enableInterrupt ?? workflowSettings.allowInterrupt,
-      turnConfig: agentTurnConfig,
    });
  };
  const selectedToolIds = inheritsGlobal
@@ -133,9 +138,13 @@ export function AgentNodePanel({
          : []),
        ...(!inheritsGlobal
          ? [
-              { id: "models", label: "模型与语音" },
+              ...(!isRealtime
+                ? [{ id: "models", label: "模型与语音" }]
+                : []),
              { id: "capabilities", label: "知识与工具" },
-              { id: "interaction", label: "交互策略" },
+              ...(!isRealtime
+                ? [{ id: "interaction", label: "交互策略" }]
+                : []),
            ]
          : []),
      ]}
@@ -248,7 +257,7 @@ export function AgentNodePanel({

      {!inheritsGlobal && (
        <>
-          <PanelAnchor id="models">
+          {!isRealtime ? <PanelAnchor id="models">
            <SectionCard
              icon={<Brain size={15} />}
              title="模型与语音"
@@ -283,10 +292,10 @@ export function AgentNodePanel({
                noneLabel="请选择语音合成"
              />
            </SectionCard>
-          </PanelAnchor>
+          </PanelAnchor> : null}

          <PanelAnchor id="capabilities">
-            <VisionConfigSection
+            {!isRealtime ? <VisionConfigSection
              description="配置当前 Agent 是否可以按需理解用户摄像头画面"
              hint="开启后，该 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，当前大语言模型必须支持图片输入。"
              enabled={Boolean(draft.visionEnabled)}
@@ -302,7 +311,7 @@ export function AgentNodePanel({
              onModelResourceIdChange={(visionModelResourceId) =>
                set("visionModelResourceId", visionModelResourceId)
              }
-            />
+            /> : null}

          <SectionCard
            icon={<Database size={15} />}
@@ -332,6 +341,13 @@ export function AgentNodePanel({
              onChange={(value) => set("knowledgeBaseId", value || "")}
              noneLabel="无"
            />
+            {isRealtime &&
+            draft.knowledgeBaseId &&
+            knowledgeConfig.mode !== "on_demand" ? (
+              <p className="text-xs leading-5 text-destructive">
+                Realtime 仅支持按需知识库，请将检索模式切换为按需调用。
+              </p>
+            ) : null}
          </SectionCard>

          <SectionCard
@@ -348,7 +364,7 @@ export function AgentNodePanel({

          </PanelAnchor>

-          <PanelAnchor id="interaction">
+          {!isRealtime ? <PanelAnchor id="interaction">
            <SectionCard
              icon={<Sparkles size={15} />}
              title="交互策略"
@@ -365,7 +381,7 @@ export function AgentNodePanel({
                onConfigChange={(turnConfig) => set("turnConfig", turnConfig)}
              />
            </SectionCard>
-          </PanelAnchor>
+          </PanelAnchor> : null}
        </>
      )}
    </PanelAnchorNavigation>
--- a/frontend/src/components/workflow/panels/EdgeSettingsPanel.tsx
+++ b/frontend/src/components/workflow/panels/EdgeSettingsPanel.tsx
@@ -33,12 +33,14 @@ export function EdgeSettingsPanel({
  sourceType,
  isOnlyOutgoing,
  hasOtherDefaultPath,
+  llmRoutingMode,
  onChange,
 }: {
  edge: Edge;
  sourceType?: string;
  isOnlyOutgoing: boolean;
  hasOtherDefaultPath: boolean;
+  llmRoutingMode: "llm_router" | "edge_tool";
  onChange: (patch: Partial<WorkflowEdgeData>) => void;
 }) {
  const data = (edge.data ?? { mode: "always", priority: 10 }) as WorkflowEdgeData;
@@ -110,11 +112,17 @@ export function EdgeSettingsPanel({
              value: "always",
              label: "默认路径",
              disabled:
+                (llmRoutingMode === "edge_tool" && sourceType === "agent") ||
                mode !== "always" &&
                (hasOtherDefaultPath ||
                  (sourceType === "agent" && isOnlyOutgoing)),
            },
-            { value: "llm", label: "大模型判断" },
+            {
+              value: "llm",
+              label: "大模型判断",
+              disabled:
+                llmRoutingMode === "edge_tool" && sourceType !== "agent",
+            },
            { value: "expression", label: "表达式" },
          ]}
          onChange={(value) => {
@@ -135,6 +143,13 @@ export function EdgeSettingsPanel({
            Agent 不能只有默认路径；请改为条件路径，或删除连接以持续对话。
          </span>
        )}
+        {llmRoutingMode === "edge_tool" &&
+        ((sourceType === "agent" && mode === "always") ||
+          (sourceType !== "agent" && mode === "llm")) ? (
+          <span className="-mt-1 block text-xs text-destructive">
+            边工具模式仅允许 Agent 发出大模型判断边；自动节点请使用表达式或默认路径。
+          </span>
+        ) : null}
        </SectionCard>
      </PanelAnchor>

--- a/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
+++ b/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
@@ -1,6 +1,7 @@
 "use client";

 import {
+  Activity,
  Brain,
  Database,
  MessageSquareText,
@@ -42,12 +43,65 @@ export function GlobalSettingsPanel({
    <PanelAnchorNavigation
      ariaLabel="工作流设置分区"
      sections={[
+        { id: "runtime", label: "运行与路由" },
        { id: "prompt", label: "提示词" },
        { id: "models", label: "模型与语音" },
        { id: "capabilities", label: "知识与工具" },
        { id: "interaction", label: "交互策略" },
      ]}
    >
+      <PanelAnchor id="runtime">
+        <SectionCard
+          icon={<Activity size={15} />}
+          title="运行与路由"
+          description="选择语音运行管线，以及大模型判断边的执行方式"
+        >
+          <NodeSelect
+            label="运行模式"
+            value={settings.runtimeMode}
+            options={[
+              { value: "pipeline", label: "Pipeline（ASR + LLM + TTS）" },
+              { value: "realtime", label: "Realtime（端到端语音模型）" },
+            ]}
+            onChange={(runtimeMode) =>
+              onSettingsChange({
+                ...settings,
+                runtimeMode:
+                  runtimeMode === "realtime" ? "realtime" : "pipeline",
+                ...(runtimeMode === "realtime"
+                  ? { llmRoutingMode: "edge_tool" }
+                  : {}),
+              })
+            }
+            allowNone={false}
+          />
+          <NodeSelect
+            label="大模型判断路由模式"
+            value={settings.llmRoutingMode}
+            options={
+              settings.runtimeMode === "realtime"
+                ? [{ value: "edge_tool", label: "边工具（Realtime 必需）" }]
+                : [
+                    { value: "edge_tool", label: "边工具（低延迟）" },
+                    { value: "llm_router", label: "独立 LLM Router（兼容）" },
+                  ]
+            }
+            onChange={(llmRoutingMode) =>
+              onSettingsChange({
+                ...settings,
+                llmRoutingMode:
+                  llmRoutingMode === "llm_router" ? "llm_router" : "edge_tool",
+              })
+            }
+            allowNone={false}
+          />
+          <p className="text-xs leading-5 text-muted-foreground">
+            此设置仅影响大模型判断边。边工具会将其注册为 function
+            tool；表达式始终由服务端直接判断。
+          </p>
+        </SectionCard>
+      </PanelAnchor>
+
      <PanelAnchor id="prompt">
        <SectionCard
          icon={<MessageSquareText size={15} />}
@@ -72,39 +126,56 @@ export function GlobalSettingsPanel({
        <SectionCard
          icon={<Brain size={15} />}
          title="模型与语音"
-          description="继承全局配置的 Agent 共用的推理、语音识别和语音合成资源"
+          description={
+            settings.runtimeMode === "realtime"
+              ? "整个工作流共用一个端到端语音模型，节点切换只更新提示词和工具"
+              : "继承全局配置的 Agent 共用的推理、语音识别和语音合成资源"
+          }
        >
-          <ModelSelect
-            label="大语言模型"
-            value={settings.llm}
-            options={modelOptions.llm}
-            onChange={(llm) =>
-              onSettingsChange({
-                ...settings,
-                llm,
-                ...(settings.visionModelResourceId === llm
-                  ? { visionModelResourceId: "" }
-                  : {}),
-              })
-            }
-          />
-          <ModelSelect
-            label="语音识别"
-            value={settings.asr}
-            options={modelOptions.asr}
-            onChange={(asr) => onSettingsChange({ ...settings, asr })}
-          />
-          <ModelSelect
-            label="语音合成"
-            value={settings.tts}
-            options={modelOptions.tts}
-            onChange={(tts) => onSettingsChange({ ...settings, tts })}
-          />
+          {settings.runtimeMode === "realtime" ? (
+            <ModelSelect
+              label="Realtime 模型"
+              value={settings.realtime}
+              options={modelOptions.realtime}
+              onChange={(realtime) =>
+                onSettingsChange({ ...settings, realtime })
+              }
+            />
+          ) : (
+            <>
+              <ModelSelect
+                label="大语言模型"
+                value={settings.llm}
+                options={modelOptions.llm}
+                onChange={(llm) =>
+                  onSettingsChange({
+                    ...settings,
+                    llm,
+                    ...(settings.visionModelResourceId === llm
+                      ? { visionModelResourceId: "" }
+                      : {}),
+                  })
+                }
+              />
+              <ModelSelect
+                label="语音识别"
+                value={settings.asr}
+                options={modelOptions.asr}
+                onChange={(asr) => onSettingsChange({ ...settings, asr })}
+              />
+              <ModelSelect
+                label="语音合成"
+                value={settings.tts}
+                options={modelOptions.tts}
+                onChange={(tts) => onSettingsChange({ ...settings, tts })}
+              />
+            </>
+          )}
        </SectionCard>
      </PanelAnchor>

      <PanelAnchor id="capabilities">
-        <VisionConfigSection
+        {settings.runtimeMode === "pipeline" ? <VisionConfigSection
          description="配置继承全局设置的 Agent 是否可以按需理解用户摄像头画面"
          hint="开启后，继承全局配置的 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，全局大语言模型必须支持图片输入。"
          enabled={settings.visionEnabled}
@@ -121,7 +192,7 @@ export function GlobalSettingsPanel({
          onModelResourceIdChange={(visionModelResourceId) =>
            onSettingsChange({ ...settings, visionModelResourceId })
          }
-        />
+        /> : null}

        <SectionCard
          icon={<Database size={15} />}
@@ -150,6 +221,13 @@ export function GlobalSettingsPanel({
            }
            noneLabel="无"
          />
+          {settings.runtimeMode === "realtime" &&
+          settings.knowledgeBaseId &&
+          settings.knowledgeRetrievalConfig.mode !== "on_demand" ? (
+            <p className="text-xs leading-5 text-destructive">
+              Realtime 仅支持按需知识库，请在上方检索设置中切换为按需调用。
+            </p>
+          ) : null}
        </SectionCard>

        <SectionCard
--- a/frontend/src/components/workflow/panels/NodeSettingsPanel.tsx
+++ b/frontend/src/components/workflow/panels/NodeSettingsPanel.tsx
@@ -120,6 +120,7 @@ export function NodeSettingsPanel({
            setAssignmentsJson={setAssignmentsJson}
            commitActionJson={commitActionJson}
            dynamicVariableOptions={dynamicVariableOptions}
+            runtimeMode={workflowSettings.runtimeMode}
          />
        </PanelAnchorNavigation>
      );
@@ -262,6 +263,7 @@ export function NodeSettingsPanel({
            setArgumentsJson={setArgumentsJson}
            setAssignmentsJson={setAssignmentsJson}
            commitActionJson={commitActionJson}
+            runtimeMode={workflowSettings.runtimeMode}
          />
        )}

@@ -322,6 +324,7 @@ function WorkflowNodePanelForm({
  setAssignmentsJson,
  commitActionJson,
  dynamicVariableOptions,
+  runtimeMode,
 }: {
  spec: RuntimeNodeSpec;
  draft: WorkflowNodeData;
@@ -335,6 +338,7 @@ function WorkflowNodePanelForm({
  setAssignmentsJson: (value: string) => void;
  commitActionJson: (argumentsValue: string, assignmentsValue: string) => void;
  dynamicVariableOptions: ModelOption[];
+  runtimeMode: "pipeline" | "realtime";
 }) {
  return (
    <div className="space-y-3">
@@ -350,6 +354,7 @@ function WorkflowNodePanelForm({
            setArgumentsJson={setArgumentsJson}
            setAssignmentsJson={setAssignmentsJson}
            commitActionJson={commitActionJson}
+            runtimeMode={runtimeMode}
          />
        </PanelAnchor>
      )}
--- a/frontend/src/components/workflow/specs.ts
+++ b/frontend/src/components/workflow/specs.ts
@@ -24,6 +24,8 @@ export type MessageCompletionPolicy =
 export type ActionResultAssignmentMode = "inherit" | "override" | "none";
 export type ActionUserInputPolicy = "queue" | "block";
 export type EdgeMode = "llm" | "expression" | "always";
+export type WorkflowRuntimeMode = "pipeline" | "realtime";
+export type WorkflowLlmRoutingMode = "llm_router" | "edge_tool";
 export type ExpressionOperator =
  | "eq"
  | "neq"
@@ -202,7 +204,10 @@ export type NodeSpecMap = Record<string, RuntimeNodeSpec>;
 export type WorkflowGraph = {
  specVersion: 3;
  settings: {
+    runtimeMode: WorkflowRuntimeMode;
+    llmRoutingMode: WorkflowLlmRoutingMode;
    globalPrompt: string;
+    defaultRealtimeResourceId: string;
    defaultLlmResourceId: string;
    defaultAsrResourceId: string;
    defaultTtsResourceId: string;
@@ -235,8 +240,11 @@ export function defaultGraph(): WorkflowGraph {
  return {
    specVersion: 3,
    settings: {
+      runtimeMode: "pipeline",
+      llmRoutingMode: "edge_tool",
      globalPrompt:
        "你是一个友好、专业的语音助手。请使用简短、自然、适合口语表达的句子。",
+      defaultRealtimeResourceId: "",
      defaultLlmResourceId: "",
      defaultAsrResourceId: "",
      defaultTtsResourceId: "",
--- a/frontend/src/components/workflow/types.ts
+++ b/frontend/src/components/workflow/types.ts
@@ -10,6 +10,9 @@ import type {
 import type { WorkflowGraph } from "./specs";

 export type WorkflowSettings = {
+  runtimeMode: "pipeline" | "realtime";
+  llmRoutingMode: "llm_router" | "edge_tool";
+  realtime?: string;
  llm?: string;
  asr?: string;
  tts?: string;
@@ -37,6 +40,7 @@ export type WorkflowEditorProps = {
  settings: WorkflowSettings;
  onSettingsChange: (settings: WorkflowSettings) => void;
  modelOptions: {
+    realtime: ModelOption[];
    llm: ModelOption[];
    asr: ModelOption[];
    tts: ModelOption[];