diff --git a/frontend/src/components/assistant-editor/debug-preview.tsx b/frontend/src/components/assistant-editor/debug-preview.tsx index 7777b4b..5d368a1 100644 --- a/frontend/src/components/assistant-editor/debug-preview.tsx +++ b/frontend/src/components/assistant-editor/debug-preview.tsx @@ -18,6 +18,7 @@ import { Sparkles, Video, Waves, + Wrench, X, } from "lucide-react"; @@ -49,6 +50,7 @@ import { usePhotoCaptureTool } from "@/hooks/use-photo-capture-tool"; import { useVoicePreview, type ChatMessage, + type ClientToolDefinition, type VoicePreview, type VoicePreviewStatus, } from "@/hooks/use-voice-preview"; @@ -216,6 +218,7 @@ export function DebugDrawer({ />
+ {dynamicVariablesEnabled && ( (null); + + const copyToolName = useCallback(async (toolName: string) => { + await navigator.clipboard.writeText(toolName); + setCopiedToolName(toolName); + window.setTimeout( + () => + setCopiedToolName((current) => + current === toolName ? null : current, + ), + 1600, + ); + }, []); + + return ( + + + + + +
+
+ + Client Tools +
+

+ 当前调试客户端已注册、能够响应的工具。 +

+
+ {tools.length === 0 ? ( +
+ 当前没有可用的 Client Tool +
+ ) : ( +
+ {tools.map((tool) => ( +
+
+
+ {tool.label} +
+ + 可用 + +
+ +

+ {tool.description} +

+
+
+ 参数 +
+
+ {tool.parameters.map((parameter) => ( +
+
+ + {parameter.name} + + + {parameter.type} + + + {parameter.required ? "必填" : "可选"} + +
+

+ {parameter.description} +

+
+ ))} +
+
+
+ ))} +
+ )} + + + ); +} + function DynamicVariableValuesPopover({ entries, values, @@ -629,7 +757,7 @@ function DebugVoicePanel({ dynamicVariables: Record; dynamicVariablesError: string; }) { - const photoCapture = usePhotoCaptureTool(preview); + const photoCapture = usePhotoCaptureTool(preview, vision); const { status, error, diff --git a/frontend/src/components/client-message-dialog.tsx b/frontend/src/components/client-message-dialog.tsx index a2c0260..9b3228d 100644 --- a/frontend/src/components/client-message-dialog.tsx +++ b/frontend/src/components/client-message-dialog.tsx @@ -15,6 +15,37 @@ import type { VoicePreview } from "@/hooks/use-voice-preview"; const SHOW_MESSAGE_TOOL = "show_message"; const MAX_ACTIONS = 5; +const SHOW_MESSAGE_DEFINITION = { + name: SHOW_MESSAGE_TOOL, + label: "显示消息", + description: "向用户显示消息弹窗，并等待用户选择操作。", + parameters: [ + { + name: "message", + type: "string", + required: true, + description: "消息正文，最多 2000 个字符。", + }, + { + name: "title", + type: "string", + required: false, + description: "弹窗标题，默认“提示”，最多 120 个字符。", + }, + { + name: "actions", + type: "array", + required: false, + description: "操作列表，最多 5 项；每项包含 id、label 和可选 style。", + }, + { + name: "dismissible", + type: "boolean", + required: false, + description: "是否允许关闭弹窗，默认为 true。", + }, + ], +} as const; type MessageActionStyle = "primary" | "secondary" | "danger"; @@ -142,16 +173,19 @@ export function ClientMessageDialog({ }, []); useEffect(() => { - const unregister = registerClientTool(SHOW_MESSAGE_TOOL, (argumentsValue) => { - if (pendingRef.current) { - throw new Error("已有消息弹窗正在等待用户操作"); - } - const nextMessage = normalizeMessage(argumentsValue); - return new Promise<{ action: string }>((resolve, reject) => { - pendingRef.current = { resolve, reject }; - setMessage(nextMessage); - }); - }); + const unregister = registerClientTool( + SHOW_MESSAGE_DEFINITION, + (argumentsValue) => { + if (pendingRef.current) { + throw new Error("已有消息弹窗正在等待用户操作"); + } + const nextMessage = normalizeMessage(argumentsValue); + return new Promise<{ action: string }>((resolve, reject) => { + pendingRef.current = { resolve, reject }; + setMessage(nextMessage); + }); + }, + ); return () => { unregister(); diff --git a/frontend/src/hooks/use-photo-capture-tool.ts b/frontend/src/hooks/use-photo-capture-tool.ts index 1e0db9f..7343f20 100644 --- a/frontend/src/hooks/use-photo-capture-tool.ts +++ b/frontend/src/hooks/use-photo-capture-tool.ts @@ -5,32 +5,47 @@ import { useCallback, useEffect, useState } from "react"; import type { VoicePreview } from "@/hooks/use-voice-preview"; const PHOTO_BUTTON_TOOL = "set_photo_button_visible"; +const PHOTO_BUTTON_DEFINITION = { + name: PHOTO_BUTTON_TOOL, + label: "控制拍照按钮", + description: "显示或隐藏客户端的拍照按钮。", + parameters: [ + { + name: "visible", + type: "boolean", + required: true, + description: "是否显示拍照按钮。", + }, + ], +} as const; -export function usePhotoCaptureTool(preview: VoicePreview) { +export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) { const { registerClientTool, sendUserInput, status } = preview; const [visible, setVisible] = useState(false); const [capturing, setCapturing] = useState(false); const [error, setError] = useState(null); - useEffect( - () => - registerClientTool(PHOTO_BUTTON_TOOL, ({ visible: nextValue }) => { + useEffect(() => { + if (!enabled) return; + return registerClientTool( + PHOTO_BUTTON_DEFINITION, + ({ visible: nextValue }) => { if (typeof nextValue !== "boolean") { throw new Error("visible 参数必须是布尔值"); } setVisible(nextValue); setError(null); return { visible: nextValue }; - }), - [registerClientTool], - ); + }, + ); + }, [enabled, registerClientTool]); useEffect(() => { - if (status === "connected") return; + if (enabled && status === "connected") return; // eslint-disable-next-line react-hooks/set-state-in-effect setVisible(false); setCapturing(false); - }, [status]); + }, [enabled, status]); const capture = useCallback(async () => { if (capturing || status !== "connected") return; diff --git a/frontend/src/hooks/use-voice-preview.ts b/frontend/src/hooks/use-voice-preview.ts index e2f3702..45c7119 100644 --- a/frontend/src/hooks/use-voice-preview.ts +++ b/frontend/src/hooks/use-voice-preview.ts @@ -71,6 +71,23 @@ export type SessionUpdateResult = { export type ClientToolHandler = ( argumentsValue: Record, ) => unknown | Promise; +export type ClientToolDefinition = { + name: string; + label: string; + description: string; + parameters: readonly ClientToolParameter[]; +}; +export type ClientToolParameter = { + name: string; + type: string; + required: boolean; + description: string; +}; + +type RegisteredClientTool = { + definition: ClientToolDefinition; + handler: ClientToolHandler; +}; type PendingUserInput = { resolve: (result: UserInputResult) => void; @@ -276,6 +293,7 @@ export function useVoicePreview( const [audioOutputs, setAudioOutputs] = useState([]); const [selectedDeviceId, setSelectedDeviceId] = useState(""); const [selectedOutputDeviceId, setSelectedOutputDeviceId] = useState(""); + const [clientTools, setClientTools] = useState([]); const transportRef = useRef(null); const cleanupPromiseRef = useRef>(Promise.resolve()); @@ -290,7 +308,9 @@ export function useVoicePreview( const pendingSessionUpdatesRef = useRef( new Map(), ); - const clientToolHandlersRef = useRef(new Map()); + const clientToolHandlersRef = useRef( + new Map(), + ); const endedByServerRef = useRef(false); const selectedDeviceIdRef = useRef(""); const selectedOutputDeviceIdRef = useRef(""); @@ -396,10 +416,10 @@ export function useVoicePreview( const waitForResponse = msg.wait_for_response !== false; if (!toolCallId || !functionName) return; - const handler = clientToolHandlersRef.current.get(functionName); + const registeredTool = clientToolHandlersRef.current.get(functionName); const transport = transportRef.current; if (!transport || transport.state !== "connected") return; - if (!handler) { + if (!registeredTool) { if (waitForResponse) { transport.sendAppMessage({ type: "client-tool-result", @@ -418,7 +438,7 @@ export function useVoicePreview( !Array.isArray(msg.arguments) ? (msg.arguments as Record) : {}; - const data = await handler(argumentsValue); + const data = await registeredTool.handler(argumentsValue); if (transportRef.current !== transport) return; if (!waitForResponse) return; transport.sendAppMessage({ @@ -889,11 +909,27 @@ export function useVoicePreview( ); const registerClientTool = useCallback( - (functionName: string, handler: ClientToolHandler): (() => void) => { - clientToolHandlersRef.current.set(functionName, handler); + ( + definition: ClientToolDefinition, + handler: ClientToolHandler, + ): (() => void) => { + const registeredTool = { definition, handler }; + clientToolHandlersRef.current.set(definition.name, registeredTool); + setClientTools( + [...clientToolHandlersRef.current.values()].map( + (item) => item.definition, + ), + ); return () => { - if (clientToolHandlersRef.current.get(functionName) === handler) { - clientToolHandlersRef.current.delete(functionName); + if ( + clientToolHandlersRef.current.get(definition.name) === registeredTool + ) { + clientToolHandlersRef.current.delete(definition.name); + setClientTools( + [...clientToolHandlersRef.current.values()].map( + (item) => item.definition, + ), + ); } }; }, @@ -911,6 +947,7 @@ export function useVoicePreview( remoteStream, messages, sessionVariables, + clientTools, callEnded, networkQuality, audioInputs,