Add voice_message_prompt tool to API and UI components. Update DuplexPipeline, Assistants, and DebugDrawer to support new tool functionality, including parameter validation and speech synthesis integration. Ensure existing tools are preserved during seeding process in the database.
This commit is contained in:
Xin Wang
@@ -1682,6 +1682,13 @@ const TOOL_PARAMETER_HINTS: Record<string, any> = {
|
||||
},
|
||||
required: [],
|
||||
},
|
||||
voice_message_prompt: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
msg: { type: 'string', description: 'Message text to speak' },
|
||||
},
|
||||
required: ['msg'],
|
||||
},
|
||||
code_interpreter: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1694,6 +1701,15 @@ const TOOL_PARAMETER_HINTS: Record<string, any> = {
|
||||
const getDefaultToolParameters = (toolId: string) =>
|
||||
TOOL_PARAMETER_HINTS[toolId] || { type: 'object', properties: {} };
|
||||
|
||||
const DEBUG_CLIENT_TOOLS = [
|
||||
{ id: 'turn_on_camera', name: 'turn_on_camera', description: '打开摄像头' },
|
||||
{ id: 'turn_off_camera', name: 'turn_off_camera', description: '关闭摄像头' },
|
||||
{ id: 'increase_volume', name: 'increase_volume', description: '调高音量' },
|
||||
{ id: 'decrease_volume', name: 'decrease_volume', description: '调低音量' },
|
||||
{ id: 'voice_message_prompt', name: 'voice_message_prompt', description: '语音消息提示' },
|
||||
] as const;
|
||||
const DEBUG_CLIENT_TOOL_ID_SET = new Set<string>(DEBUG_CLIENT_TOOLS.map((item) => item.id));
|
||||
|
||||
type DynamicVariableEntry = {
|
||||
id: string;
|
||||
key: string;
|
||||
@@ -2034,6 +2050,13 @@ export const DebugDrawer: React.FC<{
|
||||
const [aecEnabled, setAecEnabled] = useState<boolean>(() => localStorage.getItem('debug_audio_aec') !== '0');
|
||||
const [nsEnabled, setNsEnabled] = useState<boolean>(() => localStorage.getItem('debug_audio_ns') !== '0');
|
||||
const [agcEnabled, setAgcEnabled] = useState<boolean>(() => localStorage.getItem('debug_audio_agc') !== '0');
|
||||
const [clientToolEnabledMap, setClientToolEnabledMap] = useState<Record<string, boolean>>(() => {
|
||||
const result: Record<string, boolean> = {};
|
||||
for (const tool of DEBUG_CLIENT_TOOLS) {
|
||||
result[tool.id] = localStorage.getItem(`debug_client_tool_enabled_${tool.id}`) !== '0';
|
||||
}
|
||||
return result;
|
||||
});
|
||||
|
||||
const micAudioCtxRef = useRef<AudioContext | null>(null);
|
||||
const micSourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
|
||||
@@ -2043,14 +2066,18 @@ export const DebugDrawer: React.FC<{
|
||||
const userDraftIndexRef = useRef<number | null>(null);
|
||||
const lastUserFinalRef = useRef<string>('');
|
||||
const debugVolumePercentRef = useRef<number>(50);
|
||||
const isClientToolEnabled = (toolId: string) => clientToolEnabledMap[toolId] !== false;
|
||||
const selectedToolSchemas = useMemo(() => {
|
||||
const ids = assistant.tools || [];
|
||||
if (!ids.length) return [];
|
||||
const ids = Array.from(new Set([...(assistant.tools || []), ...DEBUG_CLIENT_TOOLS.map((item) => item.id)]));
|
||||
const byId = new Map(tools.map((t) => [t.id, t]));
|
||||
return ids.map((id) => {
|
||||
const item = byId.get(id);
|
||||
const toolId = item?.id || id;
|
||||
const isClientTool = (item?.category || 'query') === 'system';
|
||||
const debugClientTool = DEBUG_CLIENT_TOOLS.find((tool) => tool.id === toolId);
|
||||
if (debugClientTool && !isClientToolEnabled(toolId)) {
|
||||
return null;
|
||||
}
|
||||
const isClientTool = debugClientTool ? true : (item?.category || 'query') === 'system';
|
||||
const parameterSchema = (item?.parameterSchema && typeof item.parameterSchema === 'object')
|
||||
? item.parameterSchema
|
||||
: getDefaultToolParameters(toolId);
|
||||
@@ -2063,12 +2090,12 @@ export const DebugDrawer: React.FC<{
|
||||
...(parameterDefaults && Object.keys(parameterDefaults).length > 0 ? { defaultArgs: parameterDefaults } : {}),
|
||||
function: {
|
||||
name: toolId,
|
||||
description: item?.description || item?.name || id,
|
||||
description: item?.description || item?.name || debugClientTool?.description || id,
|
||||
parameters: parameterSchema,
|
||||
},
|
||||
};
|
||||
});
|
||||
}, [assistant.tools, tools]);
|
||||
}).filter(Boolean) as Array<Record<string, any>>;
|
||||
}, [assistant.tools, tools, clientToolEnabledMap]);
|
||||
|
||||
const clearResponseTracking = () => {
|
||||
assistantDraftIndexRef.current = null;
|
||||
@@ -2129,6 +2156,12 @@ export const DebugDrawer: React.FC<{
|
||||
localStorage.setItem('debug_audio_agc', agcEnabled ? '1' : '0');
|
||||
}, [agcEnabled]);
|
||||
|
||||
useEffect(() => {
|
||||
for (const tool of DEBUG_CLIENT_TOOLS) {
|
||||
localStorage.setItem(`debug_client_tool_enabled_${tool.id}`, isClientToolEnabled(tool.id) ? '1' : '0');
|
||||
}
|
||||
}, [clientToolEnabledMap]);
|
||||
|
||||
// Auto-scroll logic
|
||||
useEffect(() => {
|
||||
if (scrollRef.current) {
|
||||
@@ -2940,6 +2973,10 @@ export const DebugDrawer: React.FC<{
|
||||
output: { message: `Unhandled client tool '${toolName}'` },
|
||||
status: { code: 501, message: 'not_implemented' },
|
||||
};
|
||||
if (DEBUG_CLIENT_TOOL_ID_SET.has(toolName) && !isClientToolEnabled(toolName)) {
|
||||
resultPayload.output = { message: `Client tool '${toolName}' is disabled in debug settings` };
|
||||
resultPayload.status = { code: 503, message: 'tool_disabled' };
|
||||
}
|
||||
const sendToolResult = () => {
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
@@ -2962,7 +2999,9 @@ export const DebugDrawer: React.FC<{
|
||||
]);
|
||||
};
|
||||
try {
|
||||
if (toolName === 'turn_on_camera') {
|
||||
if (resultPayload.status.code === 503) {
|
||||
// Keep disabled result as-is.
|
||||
} else if (toolName === 'turn_on_camera') {
|
||||
navigator.mediaDevices
|
||||
.getUserMedia({
|
||||
video: selectedCamera ? { deviceId: { exact: selectedCamera } } : true,
|
||||
@@ -3010,6 +3049,22 @@ export const DebugDrawer: React.FC<{
|
||||
level: debugVolumePercentRef.current,
|
||||
};
|
||||
resultPayload.status = { code: 200, message: 'ok' };
|
||||
} else if (toolName === 'voice_message_prompt') {
|
||||
const msg = String(parsedArgs?.msg || '').trim();
|
||||
if (!msg) {
|
||||
resultPayload.output = { message: "Missing required argument 'msg'" };
|
||||
resultPayload.status = { code: 422, message: 'invalid_arguments' };
|
||||
} else if (typeof window !== 'undefined' && 'speechSynthesis' in window) {
|
||||
const utterance = new SpeechSynthesisUtterance(msg);
|
||||
utterance.lang = 'zh-CN';
|
||||
window.speechSynthesis.cancel();
|
||||
window.speechSynthesis.speak(utterance);
|
||||
resultPayload.output = { message: 'voice_prompt_sent', msg };
|
||||
resultPayload.status = { code: 200, message: 'ok' };
|
||||
} else {
|
||||
resultPayload.output = { message: 'speech_synthesis_unavailable', msg };
|
||||
resultPayload.status = { code: 503, message: 'speech_output_unavailable' };
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
resultPayload.output = {
|
||||
@@ -3381,6 +3436,33 @@ export const DebugDrawer: React.FC<{
|
||||
Auto Gain Control (AGC)
|
||||
</label>
|
||||
</div>
|
||||
<div className="rounded-md border border-white/10 bg-black/20 p-2 space-y-2">
|
||||
<p className="text-[10px] uppercase tracking-widest text-muted-foreground">Client Tools</p>
|
||||
<p className="text-[11px] text-muted-foreground">滑块控制调试会话内客户端工具是否启用。</p>
|
||||
<div className="space-y-2">
|
||||
{DEBUG_CLIENT_TOOLS.map((tool) => {
|
||||
const enabled = isClientToolEnabled(tool.id);
|
||||
return (
|
||||
<div key={tool.id} className="flex items-center justify-between gap-3 rounded-md border border-white/10 bg-black/20 px-2 py-1.5">
|
||||
<div className="min-w-0">
|
||||
<div className="text-[11px] font-mono text-foreground truncate">{tool.name}</div>
|
||||
<div className="text-[10px] text-muted-foreground truncate">{tool.description}</div>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setClientToolEnabledMap((prev) => ({ ...prev, [tool.id]: !enabled }))}
|
||||
className={`relative h-6 w-11 rounded-full transition-colors ${enabled ? 'bg-emerald-500/80' : 'bg-white/20'}`}
|
||||
title={enabled ? '点击关闭' : '点击开启'}
|
||||
>
|
||||
<span
|
||||
className={`absolute top-0.5 h-5 w-5 rounded-full bg-white shadow transition-transform ${enabled ? 'translate-x-5' : 'translate-x-0.5'}`}
|
||||
/>
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
<div className="rounded-md border border-white/10 bg-black/20 p-2 space-y-2">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<p className="text-[10px] uppercase tracking-widest text-muted-foreground">Dynamic Variables</p>
|
||||
|
||||
Reference in New Issue
Block a user