wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: show client tools in debug preview

Browse Source
This commit is contained in:
Xin Wang
2026-08-03 14:00:33 +08:00
parent ef434c80b2
commit dd40bfb90a
4 changed files with 242 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
frontend/src/components/assistant-editor/debug-preview.tsx
View File

@@ -18,6 +18,7 @@ import {
Sparkles,
Video,
Waves,
Wrench,
X,
} from "lucide-react";
@@ -49,6 +50,7 @@ import { usePhotoCaptureTool } from "@/hooks/use-photo-capture-tool";
import {
useVoicePreview,
type ChatMessage,
type ClientToolDefinition,
type VoicePreview,
type VoicePreviewStatus,
} from "@/hooks/use-voice-preview";
@@ -216,6 +218,7 @@ export function DebugDrawer({
/>
</div>
<div className="flex shrink-0 items-center gap-2">
<ClientToolsPopover tools={preview.clientTools} />
<CallPreviewLink assistantId={assistantId} />
{dynamicVariablesEnabled && (
<DynamicVariableValuesPopover
@@ -284,6 +287,131 @@ export function DebugDrawer({
);
}
function ClientToolsPopover({ tools }: { tools: ClientToolDefinition[] }) {
const [copiedToolName, setCopiedToolName] = useState<string | null>(null);
const copyToolName = useCallback(async (toolName: string) => {
await navigator.clipboard.writeText(toolName);
setCopiedToolName(toolName);
window.setTimeout(
() =>
setCopiedToolName((current) =>
current === toolName ? null : current,
),
1600,
);
}, []);
return (
<Popover>
<PopoverTrigger asChild>
<button
type="button"
aria-label="查看调试端支持的 Client Tools"
title="调试端支持的 Client Tools"
className="relative flex h-8 w-8 items-center justify-center rounded-full border border-hairline bg-canvas-soft text-muted-foreground transition-colors hover:bg-surface-strong hover:text-foreground"
>
<Wrench size={15} />
<span className="absolute -right-1 -top-1 flex h-4 min-w-4 items-center justify-center rounded-full border border-card bg-surface-strong px-1 text-[9px] tabular-nums text-foreground">
{tools.length}
</span>
</button>
</PopoverTrigger>
<PopoverContent
align="end"
side="bottom"
className="w-80 space-y-3 rounded-2xl p-4"
>
<div className="space-y-1">
<div className="flex items-center gap-2 text-sm font-medium text-foreground">
<Wrench size={15} />
Client Tools
</div>
<p className="text-xs leading-5 text-muted-foreground">
当前调试客户端已注册、能够响应的工具。
</p>
</div>
{tools.length === 0 ? (
<div className="rounded-xl border border-dashed border-hairline-strong bg-canvas-soft px-4 py-4 text-center text-xs text-muted-foreground">
当前没有可用的 Client Tool
</div>
) : (
<div className="max-h-72 space-y-2 overflow-y-auto pr-1">
{tools.map((tool) => (
<div
key={tool.name}
className="rounded-xl border border-hairline bg-canvas-soft p-3"
>
<div className="flex items-center justify-between gap-3">
<div className="text-xs font-medium text-foreground">
{tool.label}
</div>
<Badge
variant="secondary"
className="shrink-0 rounded-full px-2 py-0.5 text-[10px] font-medium"
>
可用
</Badge>
</div>
<button
type="button"
onClick={() => void copyToolName(tool.name)}
className="mt-1 flex w-full items-center justify-between gap-2 rounded-lg px-1 py-1 text-left text-foreground transition-colors hover:bg-surface-strong"
aria-label={`复制工具名 ${tool.name}`}
title="点击复制工具名"
>
<code className="min-w-0 break-all font-mono text-[11px]">
{tool.name}
</code>
{copiedToolName === tool.name ? (
<Check size={13} className="shrink-0 text-success" />
) : (
<Copy size={13} className="shrink-0 text-muted-soft" />
)}
</button>
<p className="mt-1 text-[11px] leading-5 text-muted-foreground">
{tool.description}
</p>
<div className="mt-2 border-t border-hairline pt-2">
<div className="mb-1.5 text-[10px] font-medium uppercase tracking-[0.08em] text-muted-soft">
参数
</div>
<div className="space-y-2">
{tool.parameters.map((parameter) => (
<div key={parameter.name} className="text-[11px] leading-4">
<div className="flex flex-wrap items-center gap-1.5">
<code className="font-mono font-medium text-foreground">
{parameter.name}
</code>
<span className="rounded-full bg-surface-strong px-1.5 py-0.5 font-mono text-[9px] text-muted-foreground">
{parameter.type}
</span>
<span
className={
parameter.required
? "text-destructive"
: "text-muted-soft"
}
>
{parameter.required ? "必填" : "可选"}
</span>
</div>
<p className="mt-0.5 text-muted-foreground">
{parameter.description}
</p>
</div>
))}
</div>
</div>
</div>
))}
</div>
)}
</PopoverContent>
</Popover>
);
}
function DynamicVariableValuesPopover({
entries,
values,
@@ -629,7 +757,7 @@ function DebugVoicePanel({
dynamicVariables: Record<string, string | number | boolean>;
dynamicVariablesError: string;
}) {
const photoCapture = usePhotoCaptureTool(preview);
const photoCapture = usePhotoCaptureTool(preview, vision);
const {
status,
error,

54
frontend/src/components/client-message-dialog.tsx
View File

@@ -15,6 +15,37 @@ import type { VoicePreview } from "@/hooks/use-voice-preview";
const SHOW_MESSAGE_TOOL = "show_message";
const MAX_ACTIONS = 5;
const SHOW_MESSAGE_DEFINITION = {
name: SHOW_MESSAGE_TOOL,
label: "显示消息",
description: "向用户显示消息弹窗，并等待用户选择操作。",
parameters: [
{
name: "message",
type: "string",
required: true,
description: "消息正文，最多 2000 个字符。",
},
{
name: "title",
type: "string",
required: false,
description: "弹窗标题，默认“提示”，最多 120 个字符。",
},
{
name: "actions",
type: "array",
required: false,
description: "操作列表，最多 5 项；每项包含 id、label 和可选 style。",
},
{
name: "dismissible",
type: "boolean",
required: false,
description: "是否允许关闭弹窗，默认为 true。",
},
],
} as const;
type MessageActionStyle = "primary" | "secondary" | "danger";
@@ -142,16 +173,19 @@ export function ClientMessageDialog({
}, []);
useEffect(() => {
const unregister = registerClientTool(SHOW_MESSAGE_TOOL, (argumentsValue) => {
if (pendingRef.current) {
throw new Error("已有消息弹窗正在等待用户操作");
}
const nextMessage = normalizeMessage(argumentsValue);
return new Promise<{ action: string }>((resolve, reject) => {
pendingRef.current = { resolve, reject };
setMessage(nextMessage);
});
});
const unregister = registerClientTool(
SHOW_MESSAGE_DEFINITION,
(argumentsValue) => {
if (pendingRef.current) {
throw new Error("已有消息弹窗正在等待用户操作");
}
const nextMessage = normalizeMessage(argumentsValue);
return new Promise<{ action: string }>((resolve, reject) => {
pendingRef.current = { resolve, reject };
setMessage(nextMessage);
});
},
);
return () => {
unregister();

33
frontend/src/hooks/use-photo-capture-tool.ts
View File

@@ -5,32 +5,47 @@ import { useCallback, useEffect, useState } from "react";
import type { VoicePreview } from "@/hooks/use-voice-preview";
const PHOTO_BUTTON_TOOL = "set_photo_button_visible";
const PHOTO_BUTTON_DEFINITION = {
name: PHOTO_BUTTON_TOOL,
label: "控制拍照按钮",
description: "显示或隐藏客户端的拍照按钮。",
parameters: [
{
name: "visible",
type: "boolean",
required: true,
description: "是否显示拍照按钮。",
},
],
} as const;
export function usePhotoCaptureTool(preview: VoicePreview) {
export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
const { registerClientTool, sendUserInput, status } = preview;
const [visible, setVisible] = useState(false);
const [capturing, setCapturing] = useState(false);
const [error, setError] = useState<string | null>(null);
useEffect(
() =>
registerClientTool(PHOTO_BUTTON_TOOL, ({ visible: nextValue }) => {
useEffect(() => {
if (!enabled) return;
return registerClientTool(
PHOTO_BUTTON_DEFINITION,
({ visible: nextValue }) => {
if (typeof nextValue !== "boolean") {
throw new Error("visible 参数必须是布尔值");
}
setVisible(nextValue);
setError(null);
return { visible: nextValue };
}),
[registerClientTool],
);
},
);
}, [enabled, registerClientTool]);
useEffect(() => {
if (status === "connected") return;
if (enabled && status === "connected") return;
// eslint-disable-next-line react-hooks/set-state-in-effect
setVisible(false);
setCapturing(false);
}, [status]);
}, [enabled, status]);
const capture = useCallback(async () => {
if (capturing || status !== "connected") return;

53
frontend/src/hooks/use-voice-preview.ts
View File

@@ -71,6 +71,23 @@ export type SessionUpdateResult = {
export type ClientToolHandler = (
argumentsValue: Record<string, unknown>,
) => unknown | Promise<unknown>;
export type ClientToolDefinition = {
name: string;
label: string;
description: string;
parameters: readonly ClientToolParameter[];
};
export type ClientToolParameter = {
name: string;
type: string;
required: boolean;
description: string;
};
type RegisteredClientTool = {
definition: ClientToolDefinition;
handler: ClientToolHandler;
};
type PendingUserInput = {
resolve: (result: UserInputResult) => void;
@@ -276,6 +293,7 @@ export function useVoicePreview(
const [audioOutputs, setAudioOutputs] = useState<MediaDeviceInfo[]>([]);
const [selectedDeviceId, setSelectedDeviceId] = useState("");
const [selectedOutputDeviceId, setSelectedOutputDeviceId] = useState("");
const [clientTools, setClientTools] = useState<ClientToolDefinition[]>([]);
const transportRef = useRef<AppSmallWebRTCTransport | null>(null);
const cleanupPromiseRef = useRef<Promise<void>>(Promise.resolve());
@@ -290,7 +308,9 @@ export function useVoicePreview(
const pendingSessionUpdatesRef = useRef(
new Map<string, PendingSessionUpdate>(),
);
const clientToolHandlersRef = useRef(new Map<string, ClientToolHandler>());
const clientToolHandlersRef = useRef(
new Map<string, RegisteredClientTool>(),
);
const endedByServerRef = useRef(false);
const selectedDeviceIdRef = useRef("");
const selectedOutputDeviceIdRef = useRef("");
@@ -396,10 +416,10 @@ export function useVoicePreview(
const waitForResponse = msg.wait_for_response !== false;
if (!toolCallId || !functionName) return;
const handler = clientToolHandlersRef.current.get(functionName);
const registeredTool = clientToolHandlersRef.current.get(functionName);
const transport = transportRef.current;
if (!transport || transport.state !== "connected") return;
if (!handler) {
if (!registeredTool) {
if (waitForResponse) {
transport.sendAppMessage({
type: "client-tool-result",
@@ -418,7 +438,7 @@ export function useVoicePreview(
!Array.isArray(msg.arguments)
? (msg.arguments as Record<string, unknown>)
: {};
const data = await handler(argumentsValue);
const data = await registeredTool.handler(argumentsValue);
if (transportRef.current !== transport) return;
if (!waitForResponse) return;
transport.sendAppMessage({
@@ -889,11 +909,27 @@ export function useVoicePreview(
);
const registerClientTool = useCallback(
(functionName: string, handler: ClientToolHandler): (() => void) => {
clientToolHandlersRef.current.set(functionName, handler);
(
definition: ClientToolDefinition,
handler: ClientToolHandler,
): (() => void) => {
const registeredTool = { definition, handler };
clientToolHandlersRef.current.set(definition.name, registeredTool);
setClientTools(
[...clientToolHandlersRef.current.values()].map(
(item) => item.definition,
),
);
return () => {
if (clientToolHandlersRef.current.get(functionName) === handler) {
clientToolHandlersRef.current.delete(functionName);
if (
clientToolHandlersRef.current.get(definition.name) === registeredTool
) {
clientToolHandlersRef.current.delete(definition.name);
setClientTools(
[...clientToolHandlersRef.current.values()].map(
(item) => item.definition,
),
);
}
};
},
@@ -911,6 +947,7 @@ export function useVoicePreview(
remoteStream,
messages,
sessionVariables,
clientTools,
callEnded,
networkQuality,
audioInputs,