feat: add prompt startup actions and shared vision config

2026-08-01 23:31:14 +08:00
parent b747144ff1
commit 0331f8cd07
22 changed files with 1238 additions and 344 deletions
--- a/frontend/src/components/assistant-editor/prompt-editor.tsx
+++ b/frontend/src/components/assistant-editor/prompt-editor.tsx
@@ -29,8 +29,10 @@ import {
 } from "@/components/assistant-editor/editor-controls";
 import type { AssistantForm } from "@/components/assistant-editor/types";
 import { SectionCard } from "@/components/editor/section-card";
+import { VisionConfigSection } from "@/components/editor/vision-config-section";
 import { TurnConfigEditor } from "@/components/turn-config-editor";
 import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
 import type { DynamicVariableDefinition, Tool } from "@/lib/api";

 type ResourceOption = { value: string; label: string };
@@ -83,6 +85,81 @@ export function PromptEditor({
  handlePromptVisionEnabledChange,
  handlePromptModelChange,
 }: PromptEditorProps) {
+  const openingMessage = form.startup.actions.find(
+    (action) => action.id === "opening_message" && action.phase === "opening",
+  );
+  const openingArguments = openingMessage?.arguments ?? {};
+  const openingButtons = Array.isArray(openingArguments.actions)
+    ? openingArguments.actions
+    : [];
+  const openingButton =
+    openingButtons[0] && typeof openingButtons[0] === "object"
+      ? (openingButtons[0] as Record<string, unknown>)
+      : {};
+  const showMessageTools = tools
+    .filter(
+      (tool) =>
+        tool.status === "active" &&
+        tool.type === "client" &&
+        tool.functionName === "show_message" &&
+        tool.definition.type === "client" &&
+        tool.definition.config.waitForResponse &&
+        tool.definition.config.responseWaitMode === "session",
+    )
+    .map((tool) => ({ value: tool.id, label: tool.name }));
+
+  function setOpeningMessage(enabled: boolean) {
+    const otherActions = form.startup.actions.filter(
+      (action) => action.id !== "opening_message",
+    );
+    const defaultToolId = showMessageTools[0]?.value ?? "";
+    updateForm("startup", {
+      executionMode: "sequential",
+      actions: enabled
+        ? [
+            ...otherActions,
+            {
+              id: "opening_message",
+              phase: "opening",
+              toolId: defaultToolId,
+              required: true,
+              arguments: {
+                title: "重要提示",
+                message: "请确认已阅读以上信息。",
+                actions: [
+                  { id: "confirmed", label: "确认", style: "primary" },
+                ],
+                dismissible: false,
+              },
+            },
+          ]
+        : otherActions,
+    });
+    if (
+      enabled &&
+      defaultToolId &&
+      !form.toolIds.includes(defaultToolId)
+    ) {
+      updateForm("toolIds", [...form.toolIds, defaultToolId]);
+    }
+  }
+
+  function updateOpeningMessage(
+    patch: Partial<NonNullable<typeof openingMessage>>,
+  ) {
+    if (!openingMessage) return;
+    updateForm("startup", {
+      ...form.startup,
+      actions: form.startup.actions.map((action) =>
+        action.id === openingMessage.id ? { ...action, ...patch } : action,
+      ),
+    });
+  }
+
+  function updateOpeningArguments(patch: Record<string, unknown>) {
+    updateOpeningMessage({ arguments: { ...openingArguments, ...patch } });
+  }
+
  return (
    <div className="-mt-6 flex h-full flex-col gap-4">
      <div className="flex shrink-0 items-center justify-between gap-6 border-b border-hairline pb-3 pt-1">
@@ -161,11 +238,87 @@ export function PromptEditor({
              count={Object.keys(dynamicVariableDefinitions).length}
              onOpen={() => setDynamicVariablesOpen(true)}
            />
+            {form.runtimeMode === "pipeline" && (
+              <div className="mt-4 space-y-3 border-t border-hairline pt-4">
+                <ToggleRow
+                  title="开场确认弹窗"
+                  description="与开场白同时显示，确认且播报完成后才允许用户开始对话。"
+                  checked={Boolean(openingMessage)}
+                  onChange={setOpeningMessage}
+                />
+
+                {openingMessage && (
+                  <div className="space-y-3 rounded-xl border border-hairline bg-canvas-soft p-4">
+                    <ResourceSelectField
+                      label="消息工具"
+                      value={openingMessage.toolId}
+                      options={showMessageTools}
+                      noneLabel="请选择 show_message 工具"
+                      onChange={(toolId) => {
+                        updateOpeningMessage({ toolId });
+                        if (toolId && !form.toolIds.includes(toolId)) {
+                          updateForm("toolIds", [...form.toolIds, toolId]);
+                        }
+                      }}
+                    />
+                    {showMessageTools.length === 0 && (
+                      <p className="text-xs leading-5 text-muted-foreground">
+                        请先在组件管理中创建 functionName 为 show_message、会话内等待的 Client Tool。
+                      </p>
+                    )}
+                    <label className="block">
+                      <span className="mb-1.5 block text-sm font-medium text-foreground">
+                        弹窗标题
+                      </span>
+                      <Input
+                        value={String(openingArguments.title ?? "")}
+                        onChange={(event) =>
+                          updateOpeningArguments({ title: event.target.value })
+                        }
+                        placeholder="重要提示"
+                        className="border-hairline-strong bg-background"
+                      />
+                    </label>
+                    <TextAreaField
+                      label="重要信息"
+                      value={String(openingArguments.message ?? "")}
+                      onChange={(message) => updateOpeningArguments({ message })}
+                      placeholder="请输入需要用户确认的重要信息"
+                      rows={4}
+                    />
+                    <label className="block">
+                      <span className="mb-1.5 block text-sm font-medium text-foreground">
+                        确认按钮
+                      </span>
+                      <Input
+                        value={String(openingButton.label ?? "")}
+                        onChange={(event) =>
+                          updateOpeningArguments({
+                            actions: [
+                              {
+                                id: "confirmed",
+                                label: event.target.value,
+                                style: "primary",
+                              },
+                            ],
+                          })
+                        }
+                        placeholder="确认"
+                        className="border-hairline-strong bg-background"
+                      />
+                    </label>
+                    <p className="text-xs leading-5 text-muted-foreground">
+                      弹窗不可通过关闭按钮、Esc 或点击外部跳过；失败时将结束本次通话。
+                    </p>
+                  </div>
+                )}
+              </div>
+            )}
          </SectionCard>

          <SectionCard
            icon={<Brain size={15} />}
-            title="模型配置"
+            title="模型与语音"
            description={
              form.runtimeMode === "pipeline"
                ? "选择运行方式，以及大语言模型、语音识别与语音合成资源"
@@ -174,49 +327,40 @@ export function PromptEditor({
          >
            <RuntimeModeSelector
              value={form.runtimeMode}
-              onChange={(runtimeMode) => updateForm("runtimeMode", runtimeMode)}
+              onChange={(runtimeMode) => {
+                updateForm("runtimeMode", runtimeMode);
+                if (runtimeMode === "realtime" && form.startup.actions.length) {
+                  updateForm("startup", {
+                    executionMode: "sequential",
+                    actions: [],
+                  });
+                }
+              }}
            />

            {form.runtimeMode === "pipeline" ? (
              <>
-              <ToggleRow
-                title="视觉理解"
-                hint="开启后，开始对话时会允许助手按需理解当前视频画面。视觉模型选「模型自己」时，大语言模型本身必须支持图片输入。"
-                checked={form.visionEnabled}
-                onChange={handlePromptVisionEnabledChange}
-              />
-              {form.visionEnabled && (
                <ResourceSelectField
-                  label="视觉模型"
-                  value={form.visionModelResourceId}
-                  onChange={(value) =>
-                    updateForm("visionModelResourceId", value)
-                  }
-                  options={visionModelOptions}
-                  noneLabel="模型自己"
+                  label="大语言模型"
+                  value={form.model}
+                  onChange={handlePromptModelChange}
+                  options={llmOptions}
+                  noneLabel="无"
+                />
+                <ResourceSelectField
+                  label="语音识别"
+                  value={form.asr}
+                  onChange={(value) => updateForm("asr", value)}
+                  options={asrOptions}
+                  noneLabel="无"
+                />
+                <ResourceSelectField
+                  label="语音合成"
+                  value={form.voice}
+                  onChange={(value) => updateForm("voice", value)}
+                  options={ttsOptions}
+                  noneLabel="无"
                />
-              )}
-              <ResourceSelectField
-                label="大语言模型"
-                value={form.model}
-                onChange={handlePromptModelChange}
-                options={llmOptions}
-                noneLabel="无"
-              />
-              <ResourceSelectField
-                label="语音识别"
-                value={form.asr}
-                onChange={(value) => updateForm("asr", value)}
-                options={asrOptions}
-                noneLabel="无"
-              />
-              <ResourceSelectField
-                label="语音合成"
-                value={form.voice}
-                onChange={(value) => updateForm("voice", value)}
-                options={ttsOptions}
-                noneLabel="无"
-              />
              </>
            ) : (
              <ResourceSelectField
@@ -229,6 +373,21 @@ export function PromptEditor({
            )}
          </SectionCard>

+          {form.runtimeMode === "pipeline" && (
+            <VisionConfigSection
+              description="配置提示词助手是否可以按需理解用户摄像头画面"
+              hint="开启后，助手会获得读取当前视频画面的工具。选择「模型自己」时，大语言模型必须支持图片输入。"
+              enabled={form.visionEnabled}
+              modelResourceId={form.visionModelResourceId}
+              mainModelResourceId={form.model}
+              modelOptions={visionModelOptions}
+              onEnabledChange={handlePromptVisionEnabledChange}
+              onModelResourceIdChange={(value) =>
+                updateForm("visionModelResourceId", value)
+              }
+            />
+          )}
+
          {form.runtimeMode === "pipeline" && (
            <SectionCard
              icon={<Database size={15} />}
@@ -299,4 +458,3 @@ export function PromptEditor({
    </div>
  );
 }
-
--- a/frontend/src/components/assistant-editor/types.ts
+++ b/frontend/src/components/assistant-editor/types.ts
@@ -1,6 +1,7 @@
 import type {
  DynamicVariableDefinition,
  KnowledgeRetrievalConfig,
+  StartupConfig,
  TurnConfig,
 } from "@/lib/api";

@@ -20,6 +21,7 @@ export type AssistantForm = {
  knowledgeRetrievalConfig: KnowledgeRetrievalConfig;
  enableInterrupt: boolean;
  turnConfig: TurnConfig;
+  startup: StartupConfig;
  visionEnabled: boolean;
  visionModelResourceId: string;
  toolIds: string[];
--- a/frontend/src/components/editor/vision-config-section.tsx
+++ b/frontend/src/components/editor/vision-config-section.tsx
@@ -0,0 +1,72 @@
+"use client";
+
+import { Camera } from "lucide-react";
+
+import {
+  ResourceSelectField,
+  ToggleRow,
+} from "@/components/assistant-editor/editor-controls";
+import { SectionCard } from "@/components/editor/section-card";
+
+type VisionModelOption = {
+  value: string;
+  label: string;
+};
+
+export function VisionConfigSection({
+  description,
+  hint,
+  enabled,
+  modelResourceId,
+  mainModelResourceId,
+  modelOptions,
+  onEnabledChange,
+  onModelResourceIdChange,
+}: {
+  description: string;
+  hint: string;
+  enabled: boolean;
+  modelResourceId: string;
+  mainModelResourceId: string;
+  modelOptions: VisionModelOption[];
+  onEnabledChange: (enabled: boolean) => void;
+  onModelResourceIdChange: (modelResourceId: string) => void;
+}) {
+  const mainModelSupportsVision = modelOptions.some(
+    (option) => option.value === mainModelResourceId,
+  );
+  const independentModelOptions = modelOptions.filter(
+    (option) => option.value !== mainModelResourceId,
+  );
+
+  return (
+    <SectionCard
+      icon={<Camera size={15} />}
+      title="视觉理解"
+      description={description}
+    >
+      <ToggleRow
+        title="允许理解当前画面"
+        hint={hint}
+        checked={enabled}
+        onChange={onEnabledChange}
+      />
+      {enabled && (
+        <>
+          <ResourceSelectField
+            label="视觉模型"
+            value={modelResourceId}
+            onChange={onModelResourceIdChange}
+            options={independentModelOptions}
+            noneLabel="模型自己"
+          />
+          {!modelResourceId && !mainModelSupportsVision && (
+            <p className="text-xs text-destructive">
+              当前大语言模型未标记支持图片输入，请选择独立视觉模型。
+            </p>
+          )}
+        </>
+      )}
+    </SectionCard>
+  );
+}
--- a/frontend/src/components/pages/AssistantPage.tsx
+++ b/frontend/src/components/pages/AssistantPage.tsx
@@ -177,6 +177,7 @@ function blankPromptForm(name: string): AssistantForm {
    knowledgeRetrievalConfig: defaultKnowledgeRetrievalConfig(),
    enableInterrupt: true,
    turnConfig: defaultTurnConfig(),
+    startup: { executionMode: "sequential", actions: [] },
    visionEnabled: false,
    visionModelResourceId: "",
    toolIds: [],
@@ -409,7 +410,7 @@ export function AssistantPage(props: AssistantPageProps) {
          resource.interfaceType === interfaceType,
      )
      .map((resource) => ({ value: resource.id, label: resource.name }));
-  const visionModelOptionsFor = (currentModelId: string) => modelResources
+  const visionModelOptionsFor = (currentModelId?: string) => modelResources
    .filter(
      (c) =>
        c.capability === "LLM" &&
@@ -465,6 +466,7 @@ export function AssistantPage(props: AssistantPageProps) {
        a.knowledgeRetrievalConfig ?? defaultKnowledgeRetrievalConfig(),
      enableInterrupt: a.enableInterrupt,
      turnConfig: normalizeTurnConfig(a.turnConfig),
+      startup: a.startup ?? { executionMode: "sequential", actions: [] },
      visionEnabled: a.visionEnabled,
      visionModelResourceId: a.visionModelResourceId ?? "",
      toolIds: a.toolIds ?? [],
@@ -534,6 +536,7 @@ export function AssistantPage(props: AssistantPageProps) {
      greeting: "",
      enableInterrupt: true,
      turnConfig: defaultTurnConfig(),
+      startup: { executionMode: "sequential", actions: [] },
      visionEnabled: false,
      visionModelResourceId: null,
      modelResourceIds: {},
@@ -592,6 +595,7 @@ export function AssistantPage(props: AssistantPageProps) {
        greeting: form.greeting,
        enableInterrupt: form.enableInterrupt,
        turnConfig: form.turnConfig,
+        startup: form.startup,
        visionEnabled: form.visionEnabled,
        visionModelResourceId: form.visionModelResourceId || null,
        modelResourceIds: {
@@ -1558,7 +1562,7 @@ export function AssistantPage(props: AssistantPageProps) {
      asrOptions={credOptions("ASR")}
      ttsOptions={credOptions("TTS")}
      realtimeOptions={credOptions("Realtime")}
-      visionModelOptions={visionModelOptionsFor(form.model)}
+      visionModelOptions={visionModelOptionsFor()}
      knowledgeOptions={kbOptions}
      tools={tools}
      onBack={() => router.push("/assistants")}
--- a/frontend/src/components/workflow/panels/AgentNodePanel.tsx
+++ b/frontend/src/components/workflow/panels/AgentNodePanel.tsx
@@ -1,7 +1,6 @@
 "use client";

 import {
-  Camera,
  Bot,
  Brain,
  Database,
@@ -12,12 +11,9 @@ import {
  Wrench,
 } from "lucide-react";

-import {
-  ResourceSelectField,
-  ToggleRow,
-} from "@/components/assistant-editor/editor-controls";
 import { KnowledgeRetrievalConfigDialog } from "@/components/editor/knowledge-retrieval-config-dialog";
 import { SectionCard } from "@/components/editor/section-card";
+import { VisionConfigSection } from "@/components/editor/vision-config-section";
 import { TurnConfigEditor } from "@/components/turn-config-editor";
 import { Input } from "@/components/ui/input";
 import { Switch } from "@/components/ui/switch";
@@ -235,48 +231,23 @@ export function AgentNodePanel({
            />
          </SectionCard>

-          <SectionCard
-            icon={<Camera size={15} />}
-            title="视觉理解"
+          <VisionConfigSection
            description="配置当前 Agent 是否可以按需理解用户摄像头画面"
-          >
-            <ToggleRow
-              title="允许理解当前画面"
-              hint="开启后，该 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，当前大语言模型必须支持图片输入。"
-              checked={Boolean(draft.visionEnabled)}
-              onChange={(visionEnabled) =>
-                setPatch({
-                  visionEnabled,
-                  ...(!visionEnabled
-                    ? { visionModelResourceId: "" }
-                    : {}),
-                })
-              }
-            />
-            {draft.visionEnabled && (
-              <>
-                <ResourceSelectField
-                  label="视觉模型"
-                  value={draft.visionModelResourceId ?? ""}
-                  onChange={(visionModelResourceId) =>
-                    set("visionModelResourceId", visionModelResourceId)
-                  }
-                  options={visionOptions.filter(
-                    (option) => option.value !== draft.llmResourceId,
-                  )}
-                  noneLabel="模型自己"
-                />
-                {!draft.visionModelResourceId &&
-                  !visionOptions.some(
-                    (option) => option.value === draft.llmResourceId,
-                  ) && (
-                    <p className="text-xs text-destructive">
-                      当前大语言模型未标记支持图片输入，请选择独立视觉模型。
-                    </p>
-                  )}
-              </>
-            )}
-          </SectionCard>
+            hint="开启后，该 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，当前大语言模型必须支持图片输入。"
+            enabled={Boolean(draft.visionEnabled)}
+            modelResourceId={draft.visionModelResourceId ?? ""}
+            mainModelResourceId={(draft.llmResourceId as string) || ""}
+            modelOptions={visionOptions}
+            onEnabledChange={(visionEnabled) =>
+              setPatch({
+                visionEnabled,
+                ...(!visionEnabled ? { visionModelResourceId: "" } : {}),
+              })
+            }
+            onModelResourceIdChange={(visionModelResourceId) =>
+              set("visionModelResourceId", visionModelResourceId)
+            }
+          />

          <SectionCard
            icon={<Database size={15} />}
--- a/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
+++ b/frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
@@ -1,7 +1,6 @@
 "use client";

 import {
-  AudioLines,
  Brain,
  Database,
  MessageSquareText,
@@ -9,12 +8,9 @@ import {
  Wrench,
 } from "lucide-react";

-import {
-  ResourceSelectField,
-  ToggleRow,
-} from "@/components/assistant-editor/editor-controls";
 import { KnowledgeRetrievalConfigDialog } from "@/components/editor/knowledge-retrieval-config-dialog";
 import { SectionCard } from "@/components/editor/section-card";
+import { VisionConfigSection } from "@/components/editor/vision-config-section";
 import { TurnConfigEditor } from "@/components/turn-config-editor";
 import { Textarea } from "@/components/ui/textarea";

@@ -60,8 +56,8 @@ export function GlobalSettingsPanel({

      <SectionCard
        icon={<Brain size={15} />}
-        title="模型配置"
-        description="工作流中所有 Agent 共用的大语言模型"
+        title="模型与语音"
+        description="继承全局配置的 Agent 共用的推理、语音识别和语音合成资源"
      >
        <ModelSelect
          label="大语言模型"
@@ -77,62 +73,39 @@ export function GlobalSettingsPanel({
            })
          }
        />
-        <ToggleRow
-          title="视觉理解"
-          hint="开启后，继承全局配置的 Agent 可以按需理解当前视频画面。选择「模型自己」时，全局大语言模型必须支持图片输入。"
-          checked={settings.visionEnabled}
-          onChange={(visionEnabled) =>
-            onSettingsChange({
-              ...settings,
-              visionEnabled,
-              ...(!visionEnabled ? { visionModelResourceId: "" } : {}),
-            })
-          }
-        />
-        {settings.visionEnabled && (
-          <>
-            <ResourceSelectField
-              label="视觉模型"
-              value={settings.visionModelResourceId}
-              onChange={(visionModelResourceId) =>
-                onSettingsChange({ ...settings, visionModelResourceId })
-              }
-              options={modelOptions.vision.filter(
-                (option) => option.value !== settings.llm,
-              )}
-              noneLabel="模型自己"
-            />
-            {!settings.visionModelResourceId &&
-              !modelOptions.vision.some(
-                (option) => option.value === settings.llm,
-              ) && (
-                <p className="text-xs text-destructive">
-                  当前大语言模型未标记支持图片输入，请选择独立视觉模型。
-                </p>
-              )}
-          </>
-        )}
-      </SectionCard>
-
-      <SectionCard
-        icon={<AudioLines size={15} />}
-        title="语音配置"
-        description="Agent 节点未单独选择资源时继承这里的默认值"
-      >
        <ModelSelect
          label="语音识别"
          value={settings.asr}
          options={modelOptions.asr}
-          onChange={(v) => onSettingsChange({ ...settings, asr: v })}
+          onChange={(asr) => onSettingsChange({ ...settings, asr })}
        />
        <ModelSelect
          label="语音合成"
          value={settings.tts}
          options={modelOptions.tts}
-          onChange={(v) => onSettingsChange({ ...settings, tts: v })}
+          onChange={(tts) => onSettingsChange({ ...settings, tts })}
        />
      </SectionCard>

+      <VisionConfigSection
+        description="配置继承全局设置的 Agent 是否可以按需理解用户摄像头画面"
+        hint="开启后，继承全局配置的 Agent 会获得读取当前视频画面的工具。选择「模型自己」时，全局大语言模型必须支持图片输入。"
+        enabled={settings.visionEnabled}
+        modelResourceId={settings.visionModelResourceId}
+        mainModelResourceId={settings.llm ?? ""}
+        modelOptions={modelOptions.vision}
+        onEnabledChange={(visionEnabled) =>
+          onSettingsChange({
+            ...settings,
+            visionEnabled,
+            ...(!visionEnabled ? { visionModelResourceId: "" } : {}),
+          })
+        }
+        onModelResourceIdChange={(visionModelResourceId) =>
+          onSettingsChange({ ...settings, visionModelResourceId })
+        }
+      />
+
      <SectionCard
        icon={<Database size={15} />}
        title="知识库配置"
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -196,6 +196,19 @@ export type TurnConfig = {
  };
 };

+export type StartupAction = {
+  id: string;
+  phase: "preflight" | "opening";
+  toolId: string;
+  arguments: Record<string, unknown>;
+  required: boolean;
+};
+
+export type StartupConfig = {
+  executionMode: "sequential";
+  actions: StartupAction[];
+};
+
 /** 后端 AssistantOut(宽表 STI:瘦字段平铺,workflow 用 graph)。apiKey 读时打码 */
 export type Assistant = {
  id: string;
@@ -205,6 +218,7 @@ export type Assistant = {
  greeting: string;
  enableInterrupt: boolean;
  turnConfig: TurnConfig;
+  startup: StartupConfig;
  visionEnabled: boolean;
  visionModelResourceId: string | null;
  modelResourceIds: Partial<Record<ModelType, string>>;