feat: add configurable client tools and photo input
This commit is contained in:
Xin Wang
@@ -12,6 +12,7 @@ import {
|
||||
Mic,
|
||||
Orbit,
|
||||
PhoneOff,
|
||||
ScanLine,
|
||||
Send,
|
||||
Smartphone,
|
||||
Sparkles,
|
||||
@@ -43,6 +44,7 @@ import { Textarea } from "@/components/ui/textarea";
|
||||
import { WaveVisualizer } from "@/components/ui/wave-visualizer";
|
||||
import { WaveformTimelinePanel } from "@/components/ui/waveform-timeline";
|
||||
import { useCameraPreview, type CameraPreview } from "@/hooks/use-camera-preview";
|
||||
import { usePhotoCaptureTool } from "@/hooks/use-photo-capture-tool";
|
||||
import {
|
||||
useVoicePreview,
|
||||
type ChatMessage,
|
||||
@@ -626,6 +628,7 @@ function DebugVoicePanel({
|
||||
dynamicVariables: Record<string, string | number | boolean>;
|
||||
dynamicVariablesError: string;
|
||||
}) {
|
||||
const photoCapture = usePhotoCaptureTool(preview);
|
||||
const {
|
||||
status,
|
||||
error,
|
||||
@@ -689,6 +692,10 @@ function DebugVoicePanel({
|
||||
recording={recording}
|
||||
camera={camera}
|
||||
videoStream={preview.videoStream}
|
||||
photoCaptureVisible={photoCapture.visible}
|
||||
photoCapturing={photoCapture.capturing}
|
||||
photoError={photoCapture.error}
|
||||
onPhotoCapture={photoCapture.capture}
|
||||
/>
|
||||
) : view === "video" ? (
|
||||
showIdleHub ? (
|
||||
@@ -966,6 +973,10 @@ function DebugVisionWorkspace({
|
||||
recording,
|
||||
camera,
|
||||
videoStream,
|
||||
photoCaptureVisible,
|
||||
photoCapturing,
|
||||
photoError,
|
||||
onPhotoCapture,
|
||||
}: {
|
||||
view: DebugView;
|
||||
onViewChange: (view: DebugView) => void;
|
||||
@@ -973,6 +984,10 @@ function DebugVisionWorkspace({
|
||||
recording: boolean;
|
||||
camera: CameraPreview;
|
||||
videoStream: MediaStream | null;
|
||||
photoCaptureVisible: boolean;
|
||||
photoCapturing: boolean;
|
||||
photoError: string | null;
|
||||
onPhotoCapture: () => Promise<void>;
|
||||
}) {
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const [position, setPosition] = useState({ x: 16, y: 16 });
|
||||
@@ -1038,6 +1053,14 @@ function DebugVisionWorkspace({
|
||||
</span>
|
||||
</span>
|
||||
</button>
|
||||
{photoCaptureVisible && (
|
||||
<DebugPhotoCaptureButton
|
||||
capturing={photoCapturing}
|
||||
enabled={Boolean(videoStream)}
|
||||
error={photoError}
|
||||
onCapture={onPhotoCapture}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1074,6 +1097,51 @@ function DebugVisionWorkspace({
|
||||
>
|
||||
<DebugVideoPanel camera={camera} streamOverride={videoStream} compact />
|
||||
</button>
|
||||
{photoCaptureVisible && (
|
||||
<DebugPhotoCaptureButton
|
||||
capturing={photoCapturing}
|
||||
enabled={Boolean(videoStream)}
|
||||
error={photoError}
|
||||
onCapture={onPhotoCapture}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function DebugPhotoCaptureButton({
|
||||
capturing,
|
||||
enabled,
|
||||
error,
|
||||
onCapture,
|
||||
}: {
|
||||
capturing: boolean;
|
||||
enabled: boolean;
|
||||
error: string | null;
|
||||
onCapture: () => Promise<void>;
|
||||
}) {
|
||||
return (
|
||||
<div className="absolute bottom-4 left-1/2 z-20 flex -translate-x-1/2 flex-col items-center gap-2">
|
||||
{error && (
|
||||
<div className="max-w-xs rounded-full border border-destructive/20 bg-background/90 px-3 py-1 text-xs text-destructive shadow-sm backdrop-blur">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
<Button
|
||||
type="button"
|
||||
size="icon"
|
||||
onClick={() => void onCapture()}
|
||||
disabled={capturing || !enabled}
|
||||
aria-label={capturing ? "正在提交照片" : "拍照并发送"}
|
||||
title={capturing ? "正在提交照片" : "拍照并发送"}
|
||||
className="size-12 rounded-full shadow-lg"
|
||||
>
|
||||
{capturing ? (
|
||||
<Loader2 className="size-5 animate-spin" />
|
||||
) : (
|
||||
<ScanLine className="size-5" />
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1250,4 +1318,3 @@ function DebugTranscriptPanel({
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -546,7 +546,9 @@ export function ToolPicker({
|
||||
? "End Call"
|
||||
: tool.type === "mcp"
|
||||
? "MCP"
|
||||
: "HTTP"}
|
||||
: tool.type === "client"
|
||||
? "Client"
|
||||
: "HTTP"}
|
||||
</Badge>
|
||||
</div>
|
||||
<div className="mt-0.5 truncate font-mono text-xs text-muted-foreground">
|
||||
|
||||
@@ -58,19 +58,26 @@ import {
|
||||
type McpServer,
|
||||
type Tool,
|
||||
type ToolParameter,
|
||||
type ToolExecutionMode,
|
||||
type ToolStatus,
|
||||
type ToolUpsert,
|
||||
} from "@/lib/api";
|
||||
|
||||
type ToolKind = "end_call" | "http";
|
||||
type ToolKind = "end_call" | "http" | "client";
|
||||
type HttpMethod = HttpToolDefinition["config"]["method"];
|
||||
type ToolFilter = "全部" | "End Call" | "HTTP" | "MCP";
|
||||
type ToolFilter = "全部" | "End Call" | "HTTP" | "Client" | "MCP";
|
||||
type SortOrder = "newest" | "oldest";
|
||||
type ToolResource =
|
||||
| { kind: "tool"; id: string; tool: Tool }
|
||||
| { kind: "mcp"; id: string; server: McpServer };
|
||||
|
||||
const toolFilters: readonly ToolFilter[] = ["全部", "End Call", "HTTP", "MCP"];
|
||||
const toolFilters: readonly ToolFilter[] = [
|
||||
"全部",
|
||||
"End Call",
|
||||
"HTTP",
|
||||
"Client",
|
||||
"MCP",
|
||||
];
|
||||
|
||||
type ToolForm = {
|
||||
name: string;
|
||||
@@ -84,10 +91,14 @@ type ToolForm = {
|
||||
method: HttpMethod;
|
||||
url: string;
|
||||
timeoutSeconds: string;
|
||||
allowInterruptions: boolean;
|
||||
executionMode: ToolExecutionMode;
|
||||
waitForResponse: boolean;
|
||||
headers: string;
|
||||
secretHeaders: string;
|
||||
parameters: string;
|
||||
body: string;
|
||||
updateDynamicVariables: boolean;
|
||||
dynamicVariableAssignments: string;
|
||||
secretDynamicVariables: string;
|
||||
};
|
||||
@@ -108,10 +119,14 @@ function blankForm(): ToolForm {
|
||||
method: "GET",
|
||||
url: "",
|
||||
timeoutSeconds: "15",
|
||||
allowInterruptions: true,
|
||||
executionMode: "immediate",
|
||||
waitForResponse: true,
|
||||
headers: EMPTY_OBJECT,
|
||||
secretHeaders: EMPTY_OBJECT,
|
||||
parameters: EMPTY_ARRAY,
|
||||
body: EMPTY_OBJECT,
|
||||
updateDynamicVariables: false,
|
||||
dynamicVariableAssignments: EMPTY_OBJECT,
|
||||
secretDynamicVariables: EMPTY_OBJECT,
|
||||
};
|
||||
@@ -136,7 +151,25 @@ function formFromTool(tool: Tool): ToolForm {
|
||||
return base;
|
||||
}
|
||||
if (tool.definition.type === "mcp") return base;
|
||||
if (tool.definition.type === "client") {
|
||||
base.allowInterruptions =
|
||||
tool.definition.config.allowInterruptions ?? true;
|
||||
base.executionMode = tool.definition.config.executionMode ?? "async";
|
||||
base.waitForResponse = tool.definition.config.waitForResponse ?? true;
|
||||
base.timeoutSeconds = String(tool.definition.config.timeoutSeconds);
|
||||
base.parameters = pretty(tool.definition.config.parameters, EMPTY_ARRAY);
|
||||
base.dynamicVariableAssignments = pretty(
|
||||
tool.definition.config.dynamicVariableAssignments ?? {},
|
||||
EMPTY_OBJECT,
|
||||
);
|
||||
base.updateDynamicVariables =
|
||||
Object.keys(tool.definition.config.dynamicVariableAssignments ?? {}).length > 0;
|
||||
return base;
|
||||
}
|
||||
base.method = tool.definition.config.method;
|
||||
base.allowInterruptions =
|
||||
tool.definition.config.allowInterruptions ?? true;
|
||||
base.executionMode = tool.definition.config.executionMode ?? "immediate";
|
||||
base.url = tool.definition.config.url;
|
||||
base.timeoutSeconds = String(tool.definition.config.timeoutSeconds);
|
||||
base.headers = pretty(tool.definition.config.headers, EMPTY_OBJECT);
|
||||
@@ -215,6 +248,39 @@ function payloadFromForm(form: ToolForm): ToolUpsert {
|
||||
},
|
||||
};
|
||||
}
|
||||
if (form.type === "client") {
|
||||
const timeoutSeconds = Number(form.timeoutSeconds);
|
||||
if (
|
||||
!Number.isInteger(timeoutSeconds) ||
|
||||
timeoutSeconds < 1 ||
|
||||
timeoutSeconds > 30
|
||||
) {
|
||||
throw new Error("Client Tool 超时时间必须是 1 到 30 秒之间的整数");
|
||||
}
|
||||
const dynamicVariableAssignments = form.updateDynamicVariables
|
||||
? parseObject(form.dynamicVariableAssignments, "变量赋值")
|
||||
: {};
|
||||
return {
|
||||
name: form.name.trim(),
|
||||
functionName: form.functionName,
|
||||
description: form.description.trim(),
|
||||
status: form.status,
|
||||
definition: {
|
||||
schemaVersion: 1,
|
||||
type: "client",
|
||||
config: {
|
||||
allowInterruptions: form.allowInterruptions,
|
||||
executionMode: form.executionMode,
|
||||
waitForResponse: form.waitForResponse,
|
||||
parameters: parseParameters(form.parameters),
|
||||
timeoutSeconds,
|
||||
dynamicVariableAssignments:
|
||||
dynamicVariableAssignments as Record<string, string>,
|
||||
},
|
||||
},
|
||||
secrets: {},
|
||||
};
|
||||
}
|
||||
const timeoutSeconds = Number(form.timeoutSeconds);
|
||||
if (!Number.isInteger(timeoutSeconds) || timeoutSeconds < 1 || timeoutSeconds > 120) {
|
||||
throw new Error("超时时间必须是 1 到 120 秒之间的整数");
|
||||
@@ -240,6 +306,8 @@ function payloadFromForm(form: ToolForm): ToolUpsert {
|
||||
schemaVersion: 1,
|
||||
type: "http",
|
||||
config: {
|
||||
allowInterruptions: form.allowInterruptions,
|
||||
executionMode: form.executionMode,
|
||||
method: form.method,
|
||||
url: form.url.trim(),
|
||||
timeoutSeconds,
|
||||
@@ -340,7 +408,8 @@ export function ComponentsToolsPage() {
|
||||
return (
|
||||
(filter === "全部" ||
|
||||
(filter === "End Call" && tool.type === "end_call") ||
|
||||
(filter === "HTTP" && tool.type === "http")) &&
|
||||
(filter === "HTTP" && tool.type === "http") ||
|
||||
(filter === "Client" && tool.type === "client")) &&
|
||||
(!query ||
|
||||
[tool.name, tool.functionName, tool.description].some((value) =>
|
||||
value.toLowerCase().includes(query),
|
||||
@@ -491,7 +560,9 @@ export function ComponentsToolsPage() {
|
||||
? "MCP"
|
||||
: resource.tool.type === "end_call"
|
||||
? "End Call"
|
||||
: "HTTP"}
|
||||
: resource.tool.type === "client"
|
||||
? "Client"
|
||||
: "HTTP"}
|
||||
</Badge>
|
||||
),
|
||||
},
|
||||
@@ -684,13 +755,22 @@ export function ComponentsToolsPage() {
|
||||
value={form.type}
|
||||
disabled={Boolean(editing)}
|
||||
onValueChange={(type: ToolKind) =>
|
||||
setForm((current) => ({ ...current, type }))
|
||||
setForm((current) => ({
|
||||
...current,
|
||||
type,
|
||||
...(type === "client"
|
||||
? { timeoutSeconds: "3", executionMode: "async" as const }
|
||||
: type === "http"
|
||||
? { timeoutSeconds: "15", executionMode: "immediate" as const }
|
||||
: {}),
|
||||
}))
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="w-full"><SelectValue /></SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="end_call">End Call</SelectItem>
|
||||
<SelectItem value="http">HTTP</SelectItem>
|
||||
<SelectItem value="client">Client Tool</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</Field>
|
||||
@@ -733,6 +813,8 @@ export function ComponentsToolsPage() {
|
||||
<FieldSection title="参数配置" scrollable tall>
|
||||
{form.type === "end_call" ? (
|
||||
<EndCallFields form={form} setForm={setForm} />
|
||||
) : form.type === "client" ? (
|
||||
<ClientToolFields form={form} setForm={setForm} />
|
||||
) : (
|
||||
<HttpFields form={form} setForm={setForm} />
|
||||
)}
|
||||
@@ -813,6 +895,97 @@ function EndCallFields({
|
||||
);
|
||||
}
|
||||
|
||||
function ClientToolFields({
|
||||
form,
|
||||
setForm,
|
||||
}: {
|
||||
form: ToolForm;
|
||||
setForm: React.Dispatch<React.SetStateAction<ToolForm>>;
|
||||
}) {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<ToolRuntimeFields form={form} setForm={setForm} />
|
||||
<div className="flex items-center justify-between gap-4 rounded-lg border border-hairline-strong px-4 py-3">
|
||||
<div>
|
||||
<div className="font-medium text-foreground">等待客户端响应</div>
|
||||
<div className="mt-0.5 text-xs text-muted-foreground">
|
||||
关闭后只下发调用，不等待结果、错误或超时
|
||||
</div>
|
||||
</div>
|
||||
<Switch
|
||||
checked={form.waitForResponse}
|
||||
onCheckedChange={(waitForResponse) =>
|
||||
setForm((current) => ({
|
||||
...current,
|
||||
waitForResponse,
|
||||
updateDynamicVariables: waitForResponse
|
||||
? current.updateDynamicVariables
|
||||
: false,
|
||||
}))
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
{form.waitForResponse && (
|
||||
<Field label="响应超时（秒）">
|
||||
<Input
|
||||
type="number"
|
||||
min={1}
|
||||
max={30}
|
||||
value={form.timeoutSeconds}
|
||||
onChange={(event) =>
|
||||
setForm((current) => ({
|
||||
...current,
|
||||
timeoutSeconds: event.target.value,
|
||||
}))
|
||||
}
|
||||
/>
|
||||
</Field>
|
||||
)}
|
||||
<JsonField
|
||||
label="参数定义"
|
||||
value={form.parameters}
|
||||
onChange={(parameters) =>
|
||||
setForm((current) => ({ ...current, parameters }))
|
||||
}
|
||||
rows={8}
|
||||
/>
|
||||
<div className="flex items-center justify-between gap-4 rounded-lg border border-hairline-strong px-4 py-3">
|
||||
<div>
|
||||
<div className="font-medium text-foreground">
|
||||
成功后更新动态变量
|
||||
</div>
|
||||
<div className="mt-0.5 text-xs text-muted-foreground">
|
||||
只在客户端返回 status=ok 时应用结果映射
|
||||
</div>
|
||||
</div>
|
||||
<Switch
|
||||
checked={form.updateDynamicVariables}
|
||||
disabled={!form.waitForResponse}
|
||||
onCheckedChange={(updateDynamicVariables) =>
|
||||
setForm((current) => ({ ...current, updateDynamicVariables }))
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
{form.updateDynamicVariables && (
|
||||
<JsonField
|
||||
label="结果变量赋值"
|
||||
value={form.dynamicVariableAssignments}
|
||||
onChange={(dynamicVariableAssignments) =>
|
||||
setForm((current) => ({
|
||||
...current,
|
||||
dynamicVariableAssignments,
|
||||
}))
|
||||
}
|
||||
/>
|
||||
)}
|
||||
<p className="text-xs leading-5 text-muted-foreground">
|
||||
函数名必须与客户端注册的 handler 名称一致，例如
|
||||
set_photo_button_visible。
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function HttpFields({
|
||||
form,
|
||||
setForm,
|
||||
@@ -822,6 +995,7 @@ function HttpFields({
|
||||
}) {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<ToolRuntimeFields form={form} setForm={setForm} />
|
||||
<div className="grid gap-4 sm:grid-cols-[140px_1fr]">
|
||||
<Field label="方法">
|
||||
<Select
|
||||
@@ -878,3 +1052,52 @@ function HttpFields({
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ToolRuntimeFields({
|
||||
form,
|
||||
setForm,
|
||||
}: {
|
||||
form: ToolForm;
|
||||
setForm: React.Dispatch<React.SetStateAction<ToolForm>>;
|
||||
}) {
|
||||
return (
|
||||
<div className="space-y-4 rounded-xl border border-hairline bg-canvas-soft p-4">
|
||||
<div>
|
||||
<div className="text-sm font-medium text-foreground">运行策略</div>
|
||||
<div className="mt-1 text-xs leading-5 text-muted-foreground">
|
||||
执行模式决定 Agent 是否等待结果；中断设置决定执行期间及随后播报是否接收用户插话。
|
||||
</div>
|
||||
</div>
|
||||
<Field label="执行模式">
|
||||
<Select
|
||||
value={form.executionMode}
|
||||
onValueChange={(executionMode: ToolExecutionMode) =>
|
||||
setForm((current) => ({ ...current, executionMode }))
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="w-full border-hairline-strong bg-background">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="immediate">立即执行 · 等待结果后继续</SelectItem>
|
||||
<SelectItem value="async">异步执行 · 工具与对话并行</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</Field>
|
||||
<div className="flex items-center justify-between gap-4">
|
||||
<div>
|
||||
<div className="text-sm font-medium text-foreground">允许用户打断</div>
|
||||
<div className="mt-0.5 text-xs text-muted-foreground">
|
||||
关闭后，工具运行和紧随其后的 Agent 播报期间会忽略用户输入
|
||||
</div>
|
||||
</div>
|
||||
<Switch
|
||||
checked={form.allowInterruptions}
|
||||
onCheckedChange={(allowInterruptions) =>
|
||||
setForm((current) => ({ ...current, allowInterruptions }))
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
Mic,
|
||||
Phone,
|
||||
PhoneOff,
|
||||
ScanLine,
|
||||
Video,
|
||||
} from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
@@ -30,6 +31,7 @@ import {
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { useCameraPreview } from "@/hooks/use-camera-preview";
|
||||
import { usePhotoCaptureTool } from "@/hooks/use-photo-capture-tool";
|
||||
import {
|
||||
useVoicePreview,
|
||||
type ChatMessage,
|
||||
@@ -420,6 +422,7 @@ function CallCameraDeviceSelect({
|
||||
|
||||
export function MobileCallPage({ assistantId }: { assistantId: string }) {
|
||||
const preview = useVoicePreview(assistantId);
|
||||
const photoCapture = usePhotoCaptureTool(preview);
|
||||
const camera = useCameraPreview();
|
||||
const {
|
||||
status,
|
||||
@@ -475,7 +478,7 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
|
||||
// 首次打开页面时自动发起通话；hooks 会在卸载时各自释放媒体资源。
|
||||
}, [startCall]);
|
||||
|
||||
const error = previewError || cameraError;
|
||||
const error = previewError || cameraError || photoCapture.error;
|
||||
|
||||
return (
|
||||
<main className="flex h-dvh w-full items-center justify-center overflow-hidden bg-black lg:p-4">
|
||||
@@ -562,6 +565,22 @@ export function MobileCallPage({ assistantId }: { assistantId: string }) {
|
||||
>
|
||||
<PhoneOff className="size-6 min-[380px]:size-7" />
|
||||
</button>
|
||||
{photoCapture.visible && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void photoCapture.capture()}
|
||||
disabled={photoCapture.capturing || !videoStream}
|
||||
aria-label={photoCapture.capturing ? "正在提交照片" : "拍照并发送"}
|
||||
title={photoCapture.capturing ? "正在提交照片" : "拍照并发送"}
|
||||
className="flex size-12 items-center justify-center rounded-full border border-white/15 bg-white text-[#07101a] shadow-xl transition-transform hover:scale-105 hover:bg-white/90 active:scale-95 disabled:pointer-events-none disabled:opacity-50 min-[380px]:size-14"
|
||||
>
|
||||
{photoCapture.capturing ? (
|
||||
<Loader2 className="size-5 animate-spin" />
|
||||
) : (
|
||||
<ScanLine className="size-5" />
|
||||
)}
|
||||
</button>
|
||||
)}
|
||||
<CallCameraDeviceSelect
|
||||
value={cameraDeviceId}
|
||||
devices={cameraDevices}
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
DialogTitle,
|
||||
} from "@/components/ui/dialog";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
@@ -31,6 +32,8 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import {
|
||||
mcpServersApi,
|
||||
toolsApi,
|
||||
type ToolExecutionMode,
|
||||
type McpServer,
|
||||
type McpServerUpsert,
|
||||
type McpToolDefinition,
|
||||
@@ -364,6 +367,14 @@ export function McpServerDialog({
|
||||
tool={tool}
|
||||
expanded={expandedToolIds.has(tool.id)}
|
||||
onToggle={() => toggleTool(tool.id)}
|
||||
onUpdated={(updated) => {
|
||||
setServerTools((current) =>
|
||||
current.map((item) =>
|
||||
item.id === updated.id ? updated : item,
|
||||
),
|
||||
);
|
||||
void onChanged();
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
@@ -390,12 +401,22 @@ function McpToolRow({
|
||||
tool,
|
||||
expanded,
|
||||
onToggle,
|
||||
onUpdated,
|
||||
}: {
|
||||
tool: Tool;
|
||||
expanded: boolean;
|
||||
onToggle: () => void;
|
||||
onUpdated: (tool: Tool) => void;
|
||||
}) {
|
||||
const definition = tool.definition as McpToolDefinition;
|
||||
const [allowInterruptions, setAllowInterruptions] = useState(
|
||||
definition.config.allowInterruptions ?? true,
|
||||
);
|
||||
const [executionMode, setExecutionMode] = useState<ToolExecutionMode>(
|
||||
definition.config.executionMode ?? "immediate",
|
||||
);
|
||||
const [savingPolicy, setSavingPolicy] = useState(false);
|
||||
const [policyError, setPolicyError] = useState<string | null>(null);
|
||||
const schema = definition.config.inputSchema ?? {};
|
||||
const properties = useMemo(
|
||||
() => Object.entries((schema.properties as Record<string, unknown> | undefined) ?? {}),
|
||||
@@ -403,6 +424,37 @@ function McpToolRow({
|
||||
);
|
||||
const required = new Set(Array.isArray(schema.required) ? schema.required.map(String) : []);
|
||||
|
||||
async function savePolicy() {
|
||||
if (savingPolicy) return;
|
||||
setSavingPolicy(true);
|
||||
setPolicyError(null);
|
||||
try {
|
||||
const updated = await toolsApi.update(tool.id, {
|
||||
name: tool.name,
|
||||
functionName: tool.functionName,
|
||||
description: tool.description,
|
||||
definition: {
|
||||
...definition,
|
||||
config: {
|
||||
...definition.config,
|
||||
allowInterruptions,
|
||||
executionMode,
|
||||
},
|
||||
},
|
||||
secrets: tool.secrets,
|
||||
status: tool.status,
|
||||
mcpServerId: tool.mcpServerId,
|
||||
});
|
||||
onUpdated(updated);
|
||||
} catch (error) {
|
||||
setPolicyError(
|
||||
error instanceof Error ? error.message : "保存工具运行策略失败",
|
||||
);
|
||||
} finally {
|
||||
setSavingPolicy(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<button
|
||||
@@ -425,6 +477,47 @@ function McpToolRow({
|
||||
{tool.description && (
|
||||
<p className="mb-4 text-sm leading-6 text-muted-foreground">{tool.description}</p>
|
||||
)}
|
||||
<div className="mb-4 grid gap-4 rounded-lg border border-hairline bg-card p-4 sm:grid-cols-2">
|
||||
<Field label="执行模式">
|
||||
<Select
|
||||
value={executionMode}
|
||||
onValueChange={(value: ToolExecutionMode) =>
|
||||
setExecutionMode(value)
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="w-full"><SelectValue /></SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="immediate">立即执行 · 等待结果</SelectItem>
|
||||
<SelectItem value="async">异步执行 · 与对话并行</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</Field>
|
||||
<div className="flex items-center justify-between gap-4 rounded-lg border border-hairline px-3 py-2">
|
||||
<div>
|
||||
<div className="text-sm font-medium text-foreground">允许用户打断</div>
|
||||
<div className="mt-0.5 text-xs text-muted-foreground">覆盖工具运行及随后播报</div>
|
||||
</div>
|
||||
<Switch
|
||||
checked={allowInterruptions}
|
||||
onCheckedChange={setAllowInterruptions}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center gap-3 sm:col-span-2">
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="outline"
|
||||
disabled={savingPolicy}
|
||||
onClick={() => void savePolicy()}
|
||||
>
|
||||
{savingPolicy && <Loader2 size={14} className="animate-spin" />}
|
||||
保存运行策略
|
||||
</Button>
|
||||
{policyError && (
|
||||
<span className="text-xs text-destructive">{policyError}</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
{properties.length === 0 ? (
|
||||
<div className="text-sm text-muted-foreground">此工具不需要参数。</div>
|
||||
) : (
|
||||
|
||||
66
frontend/src/hooks/use-photo-capture-tool.ts
Normal file
66
frontend/src/hooks/use-photo-capture-tool.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
|
||||
import type { VoicePreview } from "@/hooks/use-voice-preview";
|
||||
|
||||
const PHOTO_BUTTON_TOOL = "set_photo_button_visible";
|
||||
|
||||
export function usePhotoCaptureTool(preview: VoicePreview) {
|
||||
const { registerClientTool, sendUserInput, status } = preview;
|
||||
const [visible, setVisible] = useState(false);
|
||||
const [capturing, setCapturing] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(
|
||||
() =>
|
||||
registerClientTool(PHOTO_BUTTON_TOOL, ({ visible: nextValue }) => {
|
||||
if (typeof nextValue !== "boolean") {
|
||||
throw new Error("visible 参数必须是布尔值");
|
||||
}
|
||||
setVisible(nextValue);
|
||||
setError(null);
|
||||
return { visible: nextValue };
|
||||
}),
|
||||
[registerClientTool],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (status === "connected") return;
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
setVisible(false);
|
||||
setCapturing(false);
|
||||
}, [status]);
|
||||
|
||||
const capture = useCallback(async () => {
|
||||
if (capturing || status !== "connected") return;
|
||||
setCapturing(true);
|
||||
setError(null);
|
||||
try {
|
||||
await sendUserInput(
|
||||
[
|
||||
{
|
||||
type: "input_image",
|
||||
source: { type: "camera_frame", frame: "current" },
|
||||
},
|
||||
],
|
||||
{ runImmediately: true, interrupt: true },
|
||||
);
|
||||
} catch (captureError) {
|
||||
setError(
|
||||
captureError instanceof Error ? captureError.message : "拍照提交失败",
|
||||
);
|
||||
} finally {
|
||||
setCapturing(false);
|
||||
}
|
||||
}, [capturing, sendUserInput, status]);
|
||||
|
||||
return {
|
||||
visible,
|
||||
capturing,
|
||||
error,
|
||||
capture,
|
||||
};
|
||||
}
|
||||
|
||||
export type PhotoCaptureTool = ReturnType<typeof usePhotoCaptureTool>;
|
||||
@@ -47,6 +47,25 @@ export type ChatMessage = {
|
||||
|
||||
type AppMessage = Record<string, unknown> & { type?: string };
|
||||
type DynamicVariableValue = string | number | boolean;
|
||||
export type UserInputPart =
|
||||
| { type: "input_text"; text: string }
|
||||
| {
|
||||
type: "input_image";
|
||||
source: { type: "camera_frame"; frame: "current" };
|
||||
};
|
||||
export type UserInputResult = {
|
||||
inputId: string;
|
||||
status: "accepted";
|
||||
};
|
||||
export type ClientToolHandler = (
|
||||
argumentsValue: Record<string, unknown>,
|
||||
) => unknown | Promise<unknown>;
|
||||
|
||||
type PendingUserInput = {
|
||||
resolve: (result: UserInputResult) => void;
|
||||
reject: (error: Error) => void;
|
||||
timeout: number;
|
||||
};
|
||||
|
||||
function publicVariableSnapshot(
|
||||
value: unknown,
|
||||
@@ -116,6 +135,13 @@ function encodeHeaderJson(value: unknown): string {
|
||||
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
||||
}
|
||||
|
||||
function newInputId(): string {
|
||||
if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
|
||||
return `input_${crypto.randomUUID()}`;
|
||||
}
|
||||
return `input_${Date.now()}_${Math.random().toString(16).slice(2)}`;
|
||||
}
|
||||
|
||||
function readNetworkMetrics(report: RTCStatsReport): NetworkMetrics | null {
|
||||
const outbound: Partial<Record<"audio" | "video", number>> = {};
|
||||
const lost: Partial<Record<"audio" | "video", number>> = {};
|
||||
@@ -219,6 +245,8 @@ export function useVoicePreview(
|
||||
const pendingAssistantTurnsRef = useRef(
|
||||
new Map<string, { timestamp: string }>(),
|
||||
);
|
||||
const pendingUserInputsRef = useRef(new Map<string, PendingUserInput>());
|
||||
const clientToolHandlersRef = useRef(new Map<string, ClientToolHandler>());
|
||||
const endedByServerRef = useRef(false);
|
||||
const selectedDeviceIdRef = useRef("");
|
||||
const selectedOutputDeviceIdRef = useRef("");
|
||||
@@ -273,6 +301,11 @@ export function useVoicePreview(
|
||||
if (audioRef.current) audioRef.current.srcObject = null;
|
||||
startingRef.current = false;
|
||||
pendingAssistantTurnsRef.current.clear();
|
||||
pendingUserInputsRef.current.forEach(({ reject, timeout }) => {
|
||||
window.clearTimeout(timeout);
|
||||
reject(new Error("连接已关闭，用户输入未完成"));
|
||||
});
|
||||
pendingUserInputsRef.current.clear();
|
||||
networkStatsRef.current = null;
|
||||
onNodeActiveRef.current?.(null);
|
||||
}, []);
|
||||
@@ -301,8 +334,78 @@ export function useVoicePreview(
|
||||
setStatus("failed");
|
||||
}, [releaseResources]);
|
||||
|
||||
const dispatchClientTool = useCallback(async (msg: AppMessage) => {
|
||||
const toolCallId =
|
||||
typeof msg.tool_call_id === "string" ? msg.tool_call_id : "";
|
||||
const functionName =
|
||||
typeof msg.function_name === "string" ? msg.function_name : "";
|
||||
const waitForResponse = msg.wait_for_response !== false;
|
||||
if (!toolCallId || !functionName) return;
|
||||
|
||||
const handler = clientToolHandlersRef.current.get(functionName);
|
||||
const transport = transportRef.current;
|
||||
if (!transport || transport.state !== "connected") return;
|
||||
if (!handler) {
|
||||
if (waitForResponse) {
|
||||
transport.sendAppMessage({
|
||||
type: "client-tool-result",
|
||||
tool_call_id: toolCallId,
|
||||
status: "error",
|
||||
message: `客户端未注册工具: ${functionName}`,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const argumentsValue =
|
||||
msg.arguments &&
|
||||
typeof msg.arguments === "object" &&
|
||||
!Array.isArray(msg.arguments)
|
||||
? (msg.arguments as Record<string, unknown>)
|
||||
: {};
|
||||
const data = await handler(argumentsValue);
|
||||
if (transportRef.current !== transport) return;
|
||||
if (!waitForResponse) return;
|
||||
transport.sendAppMessage({
|
||||
type: "client-tool-result",
|
||||
tool_call_id: toolCallId,
|
||||
status: "ok",
|
||||
data,
|
||||
});
|
||||
} catch (toolError) {
|
||||
if (transportRef.current !== transport) return;
|
||||
if (!waitForResponse) return;
|
||||
transport.sendAppMessage({
|
||||
type: "client-tool-result",
|
||||
tool_call_id: toolCallId,
|
||||
status: "error",
|
||||
message: errorMessage(toolError, "客户端工具执行失败"),
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handleAppMessage = useCallback((msg: AppMessage) => {
|
||||
if (
|
||||
if (msg.type === "client-tool-call") {
|
||||
void dispatchClientTool(msg);
|
||||
} else if (
|
||||
msg.type === "user-input-result" &&
|
||||
typeof msg.input_id === "string"
|
||||
) {
|
||||
const pending = pendingUserInputsRef.current.get(msg.input_id);
|
||||
if (!pending) return;
|
||||
pendingUserInputsRef.current.delete(msg.input_id);
|
||||
window.clearTimeout(pending.timeout);
|
||||
if (msg.status === "accepted") {
|
||||
pending.resolve({ inputId: msg.input_id, status: "accepted" });
|
||||
} else {
|
||||
pending.reject(
|
||||
new Error(
|
||||
typeof msg.message === "string" ? msg.message : "用户输入处理失败",
|
||||
),
|
||||
);
|
||||
}
|
||||
} else if (
|
||||
msg.type === "assistant-text-start" &&
|
||||
typeof msg.turn_id === "string"
|
||||
) {
|
||||
@@ -413,7 +516,7 @@ export function useVoicePreview(
|
||||
setCallEnded(true);
|
||||
disconnect();
|
||||
}
|
||||
}, [disconnect]);
|
||||
}, [disconnect, dispatchClientTool]);
|
||||
|
||||
const connect = useCallback(async (options: ConnectOptions = {}) => {
|
||||
if (startingRef.current || transportRef.current) return;
|
||||
@@ -601,10 +704,68 @@ export function useVoicePreview(
|
||||
const trimmed = text.trim();
|
||||
const transport = transportRef.current;
|
||||
if (!trimmed || !transport || transport.state !== "connected") return false;
|
||||
transport.sendAppMessage({ type: "user-text", text: trimmed });
|
||||
const inputId = newInputId();
|
||||
transport.sendAppMessage({
|
||||
type: "user-input",
|
||||
schema_version: 1,
|
||||
input_id: inputId,
|
||||
parts: [{ type: "input_text", text: trimmed }],
|
||||
options: { run_immediately: true, interrupt: true },
|
||||
});
|
||||
return true;
|
||||
}, []);
|
||||
|
||||
const sendUserInput = useCallback(
|
||||
(
|
||||
parts: UserInputPart[],
|
||||
options: { runImmediately?: boolean; interrupt?: boolean } = {},
|
||||
): Promise<UserInputResult> => {
|
||||
const transport = transportRef.current;
|
||||
if (!transport || transport.state !== "connected") {
|
||||
return Promise.reject(new Error("当前未连接语音服务"));
|
||||
}
|
||||
const inputId = newInputId();
|
||||
return new Promise<UserInputResult>((resolve, reject) => {
|
||||
const timeout = window.setTimeout(() => {
|
||||
pendingUserInputsRef.current.delete(inputId);
|
||||
reject(new Error("等待用户输入处理结果超时"));
|
||||
}, 30_000);
|
||||
pendingUserInputsRef.current.set(inputId, { resolve, reject, timeout });
|
||||
try {
|
||||
transport.sendAppMessage({
|
||||
type: "user-input",
|
||||
schema_version: 1,
|
||||
input_id: inputId,
|
||||
parts,
|
||||
options: {
|
||||
run_immediately: options.runImmediately ?? true,
|
||||
interrupt: options.interrupt ?? true,
|
||||
},
|
||||
});
|
||||
} catch (sendError) {
|
||||
window.clearTimeout(timeout);
|
||||
pendingUserInputsRef.current.delete(inputId);
|
||||
reject(
|
||||
new Error(errorMessage(sendError, "发送用户输入失败")),
|
||||
);
|
||||
}
|
||||
});
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const registerClientTool = useCallback(
|
||||
(functionName: string, handler: ClientToolHandler): (() => void) => {
|
||||
clientToolHandlersRef.current.set(functionName, handler);
|
||||
return () => {
|
||||
if (clientToolHandlersRef.current.get(functionName) === handler) {
|
||||
clientToolHandlersRef.current.delete(functionName);
|
||||
}
|
||||
};
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
useEffect(() => releaseResources, [releaseResources]);
|
||||
|
||||
return {
|
||||
@@ -627,6 +788,8 @@ export function useVoicePreview(
|
||||
selectOutputDevice,
|
||||
supportsOutputSelection,
|
||||
sendText,
|
||||
sendUserInput,
|
||||
registerClientTool,
|
||||
connect,
|
||||
replaceVideoStream,
|
||||
selectCamera,
|
||||
|
||||
@@ -315,6 +315,7 @@ export const conversationsApi = {
|
||||
|
||||
// ---------- 工具 ----------
|
||||
export type ToolStatus = "active" | "archived" | "draft";
|
||||
export type ToolExecutionMode = "immediate" | "async";
|
||||
export type ToolParameter = {
|
||||
name: string;
|
||||
type: "string" | "number" | "integer" | "boolean" | "object" | "array";
|
||||
@@ -337,6 +338,8 @@ export type HttpToolDefinition = {
|
||||
schemaVersion: number;
|
||||
type: "http";
|
||||
config: {
|
||||
allowInterruptions: boolean;
|
||||
executionMode: ToolExecutionMode;
|
||||
method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
|
||||
url: string;
|
||||
timeoutSeconds: number;
|
||||
@@ -347,10 +350,25 @@ export type HttpToolDefinition = {
|
||||
};
|
||||
};
|
||||
|
||||
export type ClientToolDefinition = {
|
||||
schemaVersion: number;
|
||||
type: "client";
|
||||
config: {
|
||||
allowInterruptions: boolean;
|
||||
executionMode: ToolExecutionMode;
|
||||
waitForResponse: boolean;
|
||||
parameters: ToolParameter[];
|
||||
timeoutSeconds: number;
|
||||
dynamicVariableAssignments: Record<string, string>;
|
||||
};
|
||||
};
|
||||
|
||||
export type McpToolDefinition = {
|
||||
schemaVersion: number;
|
||||
type: "mcp";
|
||||
config: {
|
||||
allowInterruptions: boolean;
|
||||
executionMode: ToolExecutionMode;
|
||||
remoteToolName: string;
|
||||
inputSchema: Record<string, unknown>;
|
||||
schemaHash: string;
|
||||
@@ -362,9 +380,13 @@ export type Tool = {
|
||||
id: string;
|
||||
name: string;
|
||||
functionName: string;
|
||||
type: "end_call" | "http" | "mcp";
|
||||
type: "end_call" | "http" | "mcp" | "client";
|
||||
description: string;
|
||||
definition: EndCallToolDefinition | HttpToolDefinition | McpToolDefinition;
|
||||
definition:
|
||||
| EndCallToolDefinition
|
||||
| HttpToolDefinition
|
||||
| ClientToolDefinition
|
||||
| McpToolDefinition;
|
||||
secrets: Record<string, unknown>;
|
||||
status: ToolStatus;
|
||||
mcpServerId?: string | null;
|
||||
|
||||
Reference in New Issue
Block a user