feat: add configurable client tools and photo input

2026-07-30 19:06:03 +08:00
parent 510a277b5a
commit 913435785e
24 changed files with 1802 additions and 139 deletions
--- a/frontend/src/components/assistant-editor/debug-preview.tsx
+++ b/frontend/src/components/assistant-editor/debug-preview.tsx
@@ -12,6 +12,7 @@ import {
  Mic,
  Orbit,
  PhoneOff,
+  ScanLine,
  Send,
  Smartphone,
  Sparkles,
@@ -43,6 +44,7 @@ import { Textarea } from "@/components/ui/textarea";
 import { WaveVisualizer } from "@/components/ui/wave-visualizer";
 import { WaveformTimelinePanel } from "@/components/ui/waveform-timeline";
 import { useCameraPreview, type CameraPreview } from "@/hooks/use-camera-preview";
+import { usePhotoCaptureTool } from "@/hooks/use-photo-capture-tool";
 import {
  useVoicePreview,
  type ChatMessage,
@@ -626,6 +628,7 @@ function DebugVoicePanel({
  dynamicVariables: Record<string, string | number | boolean>;
  dynamicVariablesError: string;
 }) {
+  const photoCapture = usePhotoCaptureTool(preview);
  const {
    status,
    error,
@@ -689,6 +692,10 @@ function DebugVoicePanel({
          recording={recording}
          camera={camera}
          videoStream={preview.videoStream}
+          photoCaptureVisible={photoCapture.visible}
+          photoCapturing={photoCapture.capturing}
+          photoError={photoCapture.error}
+          onPhotoCapture={photoCapture.capture}
        />
      ) : view === "video" ? (
        showIdleHub ? (
@@ -966,6 +973,10 @@ function DebugVisionWorkspace({
  recording,
  camera,
  videoStream,
+  photoCaptureVisible,
+  photoCapturing,
+  photoError,
+  onPhotoCapture,
 }: {
  view: DebugView;
  onViewChange: (view: DebugView) => void;
@@ -973,6 +984,10 @@ function DebugVisionWorkspace({
  recording: boolean;
  camera: CameraPreview;
  videoStream: MediaStream | null;
+  photoCaptureVisible: boolean;
+  photoCapturing: boolean;
+  photoError: string | null;
+  onPhotoCapture: () => Promise<void>;
 }) {
  const containerRef = useRef<HTMLDivElement>(null);
  const [position, setPosition] = useState({ x: 16, y: 16 });
@@ -1038,6 +1053,14 @@ function DebugVisionWorkspace({
            </span>
          </span>
        </button>
+        {photoCaptureVisible && (
+          <DebugPhotoCaptureButton
+            capturing={photoCapturing}
+            enabled={Boolean(videoStream)}
+            error={photoError}
+            onCapture={onPhotoCapture}
+          />
+        )}
      </div>
    );
  }
@@ -1074,6 +1097,51 @@ function DebugVisionWorkspace({
      >
        <DebugVideoPanel camera={camera} streamOverride={videoStream} compact />
      </button>
+      {photoCaptureVisible && (
+        <DebugPhotoCaptureButton
+          capturing={photoCapturing}
+          enabled={Boolean(videoStream)}
+          error={photoError}
+          onCapture={onPhotoCapture}
+        />
+      )}
+    </div>
+  );
+}
+
+function DebugPhotoCaptureButton({
+  capturing,
+  enabled,
+  error,
+  onCapture,
+}: {
+  capturing: boolean;
+  enabled: boolean;
+  error: string | null;
+  onCapture: () => Promise<void>;
+}) {
+  return (
+    <div className="absolute bottom-4 left-1/2 z-20 flex -translate-x-1/2 flex-col items-center gap-2">
+      {error && (
+        <div className="max-w-xs rounded-full border border-destructive/20 bg-background/90 px-3 py-1 text-xs text-destructive shadow-sm backdrop-blur">
+          {error}
+        </div>
+      )}
+      <Button
+        type="button"
+        size="icon"
+        onClick={() => void onCapture()}
+        disabled={capturing || !enabled}
+        aria-label={capturing ? "正在提交照片" : "拍照并发送"}
+        title={capturing ? "正在提交照片" : "拍照并发送"}
+        className="size-12 rounded-full shadow-lg"
+      >
+        {capturing ? (
+          <Loader2 className="size-5 animate-spin" />
+        ) : (
+          <ScanLine className="size-5" />
+        )}
+      </Button>
    </div>
  );
 }
@@ -1250,4 +1318,3 @@ function DebugTranscriptPanel({
    </div>
  );
 }
-
--- a/frontend/src/components/assistant-editor/editor-controls.tsx
+++ b/frontend/src/components/assistant-editor/editor-controls.tsx
@@ -546,7 +546,9 @@ export function ToolPicker({
                            ? "End Call"
                            : tool.type === "mcp"
                              ? "MCP"
-                              : "HTTP"}
+                              : tool.type === "client"
+                                ? "Client"
+                                : "HTTP"}
                        </Badge>
                      </div>
                      <div className="mt-0.5 truncate font-mono text-xs text-muted-foreground">
--- a/frontend/src/components/pages/ComponentsToolsPage.tsx
+++ b/frontend/src/components/pages/ComponentsToolsPage.tsx
@@ -58,19 +58,26 @@ import {
  type McpServer,
  type Tool,
  type ToolParameter,
+  type ToolExecutionMode,
  type ToolStatus,
  type ToolUpsert,
 } from "@/lib/api";

-type ToolKind = "end_call" | "http";
+type ToolKind = "end_call" | "http" | "client";
 type HttpMethod = HttpToolDefinition["config"]["method"];
-type ToolFilter = "全部" | "End Call" | "HTTP" | "MCP";
+type ToolFilter = "全部" | "End Call" | "HTTP" | "Client" | "MCP";
 type SortOrder = "newest" | "oldest";
 type ToolResource =
  | { kind: "tool"; id: string; tool: Tool }
  | { kind: "mcp"; id: string; server: McpServer };

-const toolFilters: readonly ToolFilter[] = ["全部", "End Call", "HTTP", "MCP"];
+const toolFilters: readonly ToolFilter[] = [
+  "全部",
+  "End Call",
+  "HTTP",
+  "Client",
+  "MCP",
+];

 type ToolForm = {
  name: string;
@@ -84,10 +91,14 @@ type ToolForm = {
  method: HttpMethod;
  url: string;
  timeoutSeconds: string;
+  allowInterruptions: boolean;
+  executionMode: ToolExecutionMode;
+  waitForResponse: boolean;
  headers: string;
  secretHeaders: string;
  parameters: string;
  body: string;
+  updateDynamicVariables: boolean;
  dynamicVariableAssignments: string;
  secretDynamicVariables: string;
 };
@@ -108,10 +119,14 @@ function blankForm(): ToolForm {
    method: "GET",
    url: "",
    timeoutSeconds: "15",
+    allowInterruptions: true,
+    executionMode: "immediate",
+    waitForResponse: true,
    headers: EMPTY_OBJECT,
    secretHeaders: EMPTY_OBJECT,
    parameters: EMPTY_ARRAY,
    body: EMPTY_OBJECT,
+    updateDynamicVariables: false,
    dynamicVariableAssignments: EMPTY_OBJECT,
    secretDynamicVariables: EMPTY_OBJECT,
  };
@@ -136,7 +151,25 @@ function formFromTool(tool: Tool): ToolForm {
    return base;
  }
  if (tool.definition.type === "mcp") return base;
+  if (tool.definition.type === "client") {
+    base.allowInterruptions =
+      tool.definition.config.allowInterruptions ?? true;
+    base.executionMode = tool.definition.config.executionMode ?? "async";
+    base.waitForResponse = tool.definition.config.waitForResponse ?? true;
+    base.timeoutSeconds = String(tool.definition.config.timeoutSeconds);
+    base.parameters = pretty(tool.definition.config.parameters, EMPTY_ARRAY);
+    base.dynamicVariableAssignments = pretty(
+      tool.definition.config.dynamicVariableAssignments ?? {},
+      EMPTY_OBJECT,
+    );
+    base.updateDynamicVariables =
+      Object.keys(tool.definition.config.dynamicVariableAssignments ?? {}).length > 0;
+    return base;
+  }
  base.method = tool.definition.config.method;
+  base.allowInterruptions =
+    tool.definition.config.allowInterruptions ?? true;
+  base.executionMode = tool.definition.config.executionMode ?? "immediate";
  base.url = tool.definition.config.url;
  base.timeoutSeconds = String(tool.definition.config.timeoutSeconds);
  base.headers = pretty(tool.definition.config.headers, EMPTY_OBJECT);
@@ -215,6 +248,39 @@ function payloadFromForm(form: ToolForm): ToolUpsert {
      },
    };
  }
+  if (form.type === "client") {
+    const timeoutSeconds = Number(form.timeoutSeconds);
+    if (
+      !Number.isInteger(timeoutSeconds) ||
+      timeoutSeconds < 1 ||
+      timeoutSeconds > 30
+    ) {
+      throw new Error("Client Tool 超时时间必须是 1 到 30 秒之间的整数");
+    }
+    const dynamicVariableAssignments = form.updateDynamicVariables
+      ? parseObject(form.dynamicVariableAssignments, "变量赋值")
+      : {};
+    return {
+      name: form.name.trim(),
+      functionName: form.functionName,
+      description: form.description.trim(),
+      status: form.status,
+      definition: {
+        schemaVersion: 1,
+        type: "client",
+        config: {
+          allowInterruptions: form.allowInterruptions,
+          executionMode: form.executionMode,
+          waitForResponse: form.waitForResponse,
+          parameters: parseParameters(form.parameters),
+          timeoutSeconds,
+          dynamicVariableAssignments:
+            dynamicVariableAssignments as Record<string, string>,
+        },
+      },
+      secrets: {},
+    };
+  }
  const timeoutSeconds = Number(form.timeoutSeconds);
  if (!Number.isInteger(timeoutSeconds) || timeoutSeconds < 1 || timeoutSeconds > 120) {
    throw new Error("超时时间必须是 1 到 120 秒之间的整数");
@@ -240,6 +306,8 @@ function payloadFromForm(form: ToolForm): ToolUpsert {
      schemaVersion: 1,
      type: "http",
      config: {
+        allowInterruptions: form.allowInterruptions,
+        executionMode: form.executionMode,
        method: form.method,
        url: form.url.trim(),
        timeoutSeconds,
@@ -340,7 +408,8 @@ export function ComponentsToolsPage() {
      return (
        (filter === "全部" ||
          (filter === "End Call" && tool.type === "end_call") ||
-          (filter === "HTTP" && tool.type === "http")) &&
+          (filter === "HTTP" && tool.type === "http") ||
+          (filter === "Client" && tool.type === "client")) &&
        (!query ||
          [tool.name, tool.functionName, tool.description].some((value) =>
            value.toLowerCase().includes(query),
@@ -491,7 +560,9 @@ export function ComponentsToolsPage() {
            ? "MCP"
            : resource.tool.type === "end_call"
              ? "End Call"
-              : "HTTP"}
+              : resource.tool.type === "client"
+                ? "Client"
+                : "HTTP"}
        </Badge>
      ),
    },
@@ -684,13 +755,22 @@ export function ComponentsToolsPage() {
                  value={form.type}
                  disabled={Boolean(editing)}
                  onValueChange={(type: ToolKind) =>
-                    setForm((current) => ({ ...current, type }))
+                    setForm((current) => ({
+                      ...current,
+                      type,
+                      ...(type === "client"
+                        ? { timeoutSeconds: "3", executionMode: "async" as const }
+                        : type === "http"
+                          ? { timeoutSeconds: "15", executionMode: "immediate" as const }
+                          : {}),
+                    }))
                  }
                >
                  <SelectTrigger className="w-full"><SelectValue /></SelectTrigger>
                  <SelectContent>
                    <SelectItem value="end_call">End Call</SelectItem>
                    <SelectItem value="http">HTTP</SelectItem>
+                    <SelectItem value="client">Client Tool</SelectItem>
                  </SelectContent>
                </Select>
              </Field>
@@ -733,6 +813,8 @@ export function ComponentsToolsPage() {
            <FieldSection title="参数配置" scrollable tall>
              {form.type === "end_call" ? (
                <EndCallFields form={form} setForm={setForm} />
+              ) : form.type === "client" ? (
+                <ClientToolFields form={form} setForm={setForm} />
              ) : (
                <HttpFields form={form} setForm={setForm} />
              )}
@@ -813,6 +895,97 @@ function EndCallFields({
  );
 }

+function ClientToolFields({
+  form,
+  setForm,
+}: {
+  form: ToolForm;
+  setForm: React.Dispatch<React.SetStateAction<ToolForm>>;
+}) {
+  return (
+    <div className="space-y-4">
+      <ToolRuntimeFields form={form} setForm={setForm} />
+      <div className="flex items-center justify-between gap-4 rounded-lg border border-hairline-strong px-4 py-3">
+        <div>
+          <div className="font-medium text-foreground">等待客户端响应</div>
+          <div className="mt-0.5 text-xs text-muted-foreground">
+            关闭后只下发调用，不等待结果、错误或超时
+          </div>
+        </div>
+        <Switch
+          checked={form.waitForResponse}
+          onCheckedChange={(waitForResponse) =>
+            setForm((current) => ({
+              ...current,
+              waitForResponse,
+              updateDynamicVariables: waitForResponse
+                ? current.updateDynamicVariables
+                : false,
+            }))
+          }
+        />
+      </div>
+      {form.waitForResponse && (
+        <Field label="响应超时（秒）">
+          <Input
+            type="number"
+            min={1}
+            max={30}
+            value={form.timeoutSeconds}
+            onChange={(event) =>
+              setForm((current) => ({
+                ...current,
+                timeoutSeconds: event.target.value,
+              }))
+            }
+          />
+        </Field>
+      )}
+      <JsonField
+        label="参数定义"
+        value={form.parameters}
+        onChange={(parameters) =>
+          setForm((current) => ({ ...current, parameters }))
+        }
+        rows={8}
+      />
+      <div className="flex items-center justify-between gap-4 rounded-lg border border-hairline-strong px-4 py-3">
+        <div>
+          <div className="font-medium text-foreground">
+            成功后更新动态变量
+          </div>
+          <div className="mt-0.5 text-xs text-muted-foreground">
+            只在客户端返回 status=ok 时应用结果映射
+          </div>
+        </div>
+        <Switch
+          checked={form.updateDynamicVariables}
+          disabled={!form.waitForResponse}
+          onCheckedChange={(updateDynamicVariables) =>
+            setForm((current) => ({ ...current, updateDynamicVariables }))
+          }
+        />
+      </div>
+      {form.updateDynamicVariables && (
+        <JsonField
+          label="结果变量赋值"
+          value={form.dynamicVariableAssignments}
+          onChange={(dynamicVariableAssignments) =>
+            setForm((current) => ({
+              ...current,
+              dynamicVariableAssignments,
+            }))
+          }
+        />
+      )}
+      <p className="text-xs leading-5 text-muted-foreground">
+        函数名必须与客户端注册的 handler 名称一致，例如
+        set_photo_button_visible。
+      </p>
+    </div>
+  );
+}
+
 function HttpFields({
  form,
  setForm,
@@ -822,6 +995,7 @@ function HttpFields({
 }) {
  return (
    <div className="space-y-4">
+      <ToolRuntimeFields form={form} setForm={setForm} />
      <div className="grid gap-4 sm:grid-cols-[140px_1fr]">
        <Field label="方法">
          <Select
@@ -878,3 +1052,52 @@ function HttpFields({
    </div>
  );
 }
+
+function ToolRuntimeFields({
+  form,
+  setForm,
+}: {
+  form: ToolForm;
+  setForm: React.Dispatch<React.SetStateAction<ToolForm>>;
+}) {
+  return (
+    <div className="space-y-4 rounded-xl border border-hairline bg-canvas-soft p-4">
+      <div>
+        <div className="text-sm font-medium text-foreground">运行策略</div>
+        <div className="mt-1 text-xs leading-5 text-muted-foreground">
+          执行模式决定 Agent 是否等待结果；中断设置决定执行期间及随后播报是否接收用户插话。
+        </div>
+      </div>
+      <Field label="执行模式">
+        <Select
+          value={form.executionMode}
+          onValueChange={(executionMode: ToolExecutionMode) =>
+            setForm((current) => ({ ...current, executionMode }))
+          }
+        >
+          <SelectTrigger className="w-full border-hairline-strong bg-background">
+            <SelectValue />
+          </SelectTrigger>
+          <SelectContent>
+            <SelectItem value="immediate">立即执行 · 等待结果后继续</SelectItem>
+            <SelectItem value="async">异步执行 · 工具与对话并行</SelectItem>
+          </SelectContent>
+        </Select>
+      </Field>
+      <div className="flex items-center justify-between gap-4">
+        <div>
+          <div className="text-sm font-medium text-foreground">允许用户打断</div>
+          <div className="mt-0.5 text-xs text-muted-foreground">
+            关闭后，工具运行和紧随其后的 Agent 播报期间会忽略用户输入
+          </div>
+        </div>
+        <Switch
+          checked={form.allowInterruptions}
+          onCheckedChange={(allowInterruptions) =>
+            setForm((current) => ({ ...current, allowInterruptions }))
+          }
+        />
+      </div>
+    </div>
+  );
+}
--- a/frontend/src/components/pages/MobileCallPage.tsx
+++ b/frontend/src/components/pages/MobileCallPage.tsx
@@ -9,6 +9,7 @@ import {
  Mic,
  Phone,
  PhoneOff,
+  ScanLine,
  Video,
 } from "lucide-react";
 import { Button } from "@/components/ui/button";
@@ -30,6 +31,7 @@ import {
  SelectValue,
 } from "@/components/ui/select";
 import { useCameraPreview } from "@/hooks/use-camera-preview";
+import { usePhotoCaptureTool } from "@/hooks/use-photo-capture-tool";
 import {
  useVoicePreview,
  type ChatMessage,
@@ -420,6 +422,7 @@ function CallCameraDeviceSelect({

 export function MobileCallPage({ assistantId }: { assistantId: string }) {
  const preview = useVoicePreview(assistantId);
+  const photoCapture = usePhotoCaptureTool(preview);
  const camera = useCameraPreview();
  const {
    status,
@@ -475,7 +478,7 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
    // 首次打开页面时自动发起通话；hooks 会在卸载时各自释放媒体资源。
  }, [startCall]);

-  const error = previewError || cameraError;
+  const error = previewError || cameraError || photoCapture.error;

  return (
    <main className="flex h-dvh w-full items-center justify-center overflow-hidden bg-black lg:p-4">
@@ -562,6 +565,22 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
          >
            <PhoneOff className="size-6 min-[380px]:size-7" />
          </button>
+          {photoCapture.visible && (
+            <button
+              type="button"
+              onClick={() => void photoCapture.capture()}
+              disabled={photoCapture.capturing || !videoStream}
+              aria-label={photoCapture.capturing ? "正在提交照片" : "拍照并发送"}
+              title={photoCapture.capturing ? "正在提交照片" : "拍照并发送"}
+              className="flex size-12 items-center justify-center rounded-full border border-white/15 bg-white text-[#07101a] shadow-xl transition-transform hover:scale-105 hover:bg-white/90 active:scale-95 disabled:pointer-events-none disabled:opacity-50 min-[380px]:size-14"
+            >
+              {photoCapture.capturing ? (
+                <Loader2 className="size-5 animate-spin" />
+              ) : (
+                <ScanLine className="size-5" />
+              )}
+            </button>
+          )}
          <CallCameraDeviceSelect
            value={cameraDeviceId}
            devices={cameraDevices}
--- a/frontend/src/components/tools/McpServerDialog.tsx
+++ b/frontend/src/components/tools/McpServerDialog.tsx
@@ -20,6 +20,7 @@ import {
  DialogTitle,
 } from "@/components/ui/dialog";
 import { Input } from "@/components/ui/input";
+import { Switch } from "@/components/ui/switch";
 import {
  Select,
  SelectContent,
@@ -31,6 +32,8 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { Textarea } from "@/components/ui/textarea";
 import {
  mcpServersApi,
+  toolsApi,
+  type ToolExecutionMode,
  type McpServer,
  type McpServerUpsert,
  type McpToolDefinition,
@@ -364,6 +367,14 @@ export function McpServerDialog({
                      tool={tool}
                      expanded={expandedToolIds.has(tool.id)}
                      onToggle={() => toggleTool(tool.id)}
+                      onUpdated={(updated) => {
+                        setServerTools((current) =>
+                          current.map((item) =>
+                            item.id === updated.id ? updated : item,
+                          ),
+                        );
+                        void onChanged();
+                      }}
                    />
                  ))}
                </div>
@@ -390,12 +401,22 @@ function McpToolRow({
  tool,
  expanded,
  onToggle,
+  onUpdated,
 }: {
  tool: Tool;
  expanded: boolean;
  onToggle: () => void;
+  onUpdated: (tool: Tool) => void;
 }) {
  const definition = tool.definition as McpToolDefinition;
+  const [allowInterruptions, setAllowInterruptions] = useState(
+    definition.config.allowInterruptions ?? true,
+  );
+  const [executionMode, setExecutionMode] = useState<ToolExecutionMode>(
+    definition.config.executionMode ?? "immediate",
+  );
+  const [savingPolicy, setSavingPolicy] = useState(false);
+  const [policyError, setPolicyError] = useState<string | null>(null);
  const schema = definition.config.inputSchema ?? {};
  const properties = useMemo(
    () => Object.entries((schema.properties as Record<string, unknown> | undefined) ?? {}),
@@ -403,6 +424,37 @@ function McpToolRow({
  );
  const required = new Set(Array.isArray(schema.required) ? schema.required.map(String) : []);

+  async function savePolicy() {
+    if (savingPolicy) return;
+    setSavingPolicy(true);
+    setPolicyError(null);
+    try {
+      const updated = await toolsApi.update(tool.id, {
+        name: tool.name,
+        functionName: tool.functionName,
+        description: tool.description,
+        definition: {
+          ...definition,
+          config: {
+            ...definition.config,
+            allowInterruptions,
+            executionMode,
+          },
+        },
+        secrets: tool.secrets,
+        status: tool.status,
+        mcpServerId: tool.mcpServerId,
+      });
+      onUpdated(updated);
+    } catch (error) {
+      setPolicyError(
+        error instanceof Error ? error.message : "保存工具运行策略失败",
+      );
+    } finally {
+      setSavingPolicy(false);
+    }
+  }
+
  return (
    <div>
      <button
@@ -425,6 +477,47 @@ function McpToolRow({
          {tool.description && (
            <p className="mb-4 text-sm leading-6 text-muted-foreground">{tool.description}</p>
          )}
+          <div className="mb-4 grid gap-4 rounded-lg border border-hairline bg-card p-4 sm:grid-cols-2">
+            <Field label="执行模式">
+              <Select
+                value={executionMode}
+                onValueChange={(value: ToolExecutionMode) =>
+                  setExecutionMode(value)
+                }
+              >
+                <SelectTrigger className="w-full"><SelectValue /></SelectTrigger>
+                <SelectContent>
+                  <SelectItem value="immediate">立即执行 · 等待结果</SelectItem>
+                  <SelectItem value="async">异步执行 · 与对话并行</SelectItem>
+                </SelectContent>
+              </Select>
+            </Field>
+            <div className="flex items-center justify-between gap-4 rounded-lg border border-hairline px-3 py-2">
+              <div>
+                <div className="text-sm font-medium text-foreground">允许用户打断</div>
+                <div className="mt-0.5 text-xs text-muted-foreground">覆盖工具运行及随后播报</div>
+              </div>
+              <Switch
+                checked={allowInterruptions}
+                onCheckedChange={setAllowInterruptions}
+              />
+            </div>
+            <div className="flex items-center gap-3 sm:col-span-2">
+              <Button
+                type="button"
+                size="sm"
+                variant="outline"
+                disabled={savingPolicy}
+                onClick={() => void savePolicy()}
+              >
+                {savingPolicy && <Loader2 size={14} className="animate-spin" />}
+                保存运行策略
+              </Button>
+              {policyError && (
+                <span className="text-xs text-destructive">{policyError}</span>
+              )}
+            </div>
+          </div>
          {properties.length === 0 ? (
            <div className="text-sm text-muted-foreground">此工具不需要参数。</div>
          ) : (