Add text_msg_prompt tool to DuplexPipeline and Assistants. Update DebugDrawer to handle text message prompts, including parameter validation and state management for displaying messages. Ensure integration with existing tools and maintain functionality across components.
This commit is contained in:
Xin Wang
@@ -1689,6 +1689,13 @@ const TOOL_PARAMETER_HINTS: Record<string, any> = {
|
||||
},
|
||||
required: ['msg'],
|
||||
},
|
||||
text_msg_prompt: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
msg: { type: 'string', description: 'Message text to display in debug drawer modal' },
|
||||
},
|
||||
required: ['msg'],
|
||||
},
|
||||
code_interpreter: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -1707,6 +1714,7 @@ const DEBUG_CLIENT_TOOLS = [
|
||||
{ id: 'increase_volume', name: 'increase_volume', description: '调高音量' },
|
||||
{ id: 'decrease_volume', name: 'decrease_volume', description: '调低音量' },
|
||||
{ id: 'voice_message_prompt', name: 'voice_message_prompt', description: '语音消息提示' },
|
||||
{ id: 'text_msg_prompt', name: 'text_msg_prompt', description: '文本消息提示' },
|
||||
] as const;
|
||||
const DEBUG_CLIENT_TOOL_ID_SET = new Set<string>(DEBUG_CLIENT_TOOLS.map((item) => item.id));
|
||||
|
||||
@@ -1973,6 +1981,7 @@ export const DebugDrawer: React.FC<{
|
||||
const [inputText, setInputText] = useState('');
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const [callStatus, setCallStatus] = useState<'idle' | 'calling' | 'active'>('idle');
|
||||
const [textPromptDialog, setTextPromptDialog] = useState<{ open: boolean; message: string }>({ open: false, message: '' });
|
||||
const [textSessionStarted, setTextSessionStarted] = useState(false);
|
||||
const [wsStatus, setWsStatus] = useState<'disconnected' | 'connecting' | 'ready' | 'error'>('disconnected');
|
||||
const [wsError, setWsError] = useState('');
|
||||
@@ -2033,6 +2042,7 @@ export const DebugDrawer: React.FC<{
|
||||
const assistantDraftIndexRef = useRef<number | null>(null);
|
||||
const assistantResponseIndexByIdRef = useRef<Map<string, number>>(new Map());
|
||||
const pendingTtfbByResponseIdRef = useRef<Map<string, number>>(new Map());
|
||||
const interruptedResponseIdsRef = useRef<Set<string>>(new Set());
|
||||
const audioCtxRef = useRef<AudioContext | null>(null);
|
||||
const playbackTimeRef = useRef<number>(0);
|
||||
const activeAudioSourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set());
|
||||
@@ -2101,6 +2111,13 @@ export const DebugDrawer: React.FC<{
|
||||
assistantDraftIndexRef.current = null;
|
||||
assistantResponseIndexByIdRef.current.clear();
|
||||
pendingTtfbByResponseIdRef.current.clear();
|
||||
interruptedResponseIdsRef.current.clear();
|
||||
};
|
||||
|
||||
const extractResponseId = (payload: any): string | undefined => {
|
||||
const responseIdRaw = payload?.data?.response_id ?? payload?.response_id ?? payload?.responseId;
|
||||
const responseId = String(responseIdRaw || '').trim();
|
||||
return responseId || undefined;
|
||||
};
|
||||
|
||||
// Initialize
|
||||
@@ -2120,6 +2137,7 @@ export const DebugDrawer: React.FC<{
|
||||
stopVoiceCapture();
|
||||
stopMedia();
|
||||
closeWs();
|
||||
setTextPromptDialog({ open: false, message: '' });
|
||||
if (audioCtxRef.current) {
|
||||
void audioCtxRef.current.close();
|
||||
audioCtxRef.current = null;
|
||||
@@ -2480,6 +2498,7 @@ export const DebugDrawer: React.FC<{
|
||||
setCallStatus('idle');
|
||||
clearResponseTracking();
|
||||
setMessages([]);
|
||||
setTextPromptDialog({ open: false, message: '' });
|
||||
lastUserFinalRef.current = '';
|
||||
setIsLoading(false);
|
||||
};
|
||||
@@ -2903,6 +2922,14 @@ export const DebugDrawer: React.FC<{
|
||||
}
|
||||
|
||||
if (type === 'response.interrupted') {
|
||||
const interruptedResponseId = extractResponseId(payload);
|
||||
if (interruptedResponseId) {
|
||||
interruptedResponseIdsRef.current.add(interruptedResponseId);
|
||||
if (interruptedResponseIdsRef.current.size > 64) {
|
||||
const oldest = interruptedResponseIdsRef.current.values().next().value as string | undefined;
|
||||
if (oldest) interruptedResponseIdsRef.current.delete(oldest);
|
||||
}
|
||||
}
|
||||
assistantDraftIndexRef.current = null;
|
||||
setIsLoading(false);
|
||||
stopPlaybackImmediately();
|
||||
@@ -2913,8 +2940,8 @@ export const DebugDrawer: React.FC<{
|
||||
const maybeTtfb = Number(payload?.latencyMs ?? payload?.data?.latencyMs);
|
||||
if (!Number.isFinite(maybeTtfb) || maybeTtfb < 0) return;
|
||||
const ttfbMs = Math.round(maybeTtfb);
|
||||
const responseIdRaw = payload?.data?.response_id ?? payload?.response_id ?? payload?.responseId;
|
||||
const responseId = String(responseIdRaw || '').trim();
|
||||
const responseId = extractResponseId(payload);
|
||||
if (responseId && interruptedResponseIdsRef.current.has(responseId)) return;
|
||||
if (responseId) {
|
||||
const indexed = assistantResponseIndexByIdRef.current.get(responseId);
|
||||
if (typeof indexed === 'number') {
|
||||
@@ -3065,6 +3092,16 @@ export const DebugDrawer: React.FC<{
|
||||
resultPayload.output = { message: 'speech_synthesis_unavailable', msg };
|
||||
resultPayload.status = { code: 503, message: 'speech_output_unavailable' };
|
||||
}
|
||||
} else if (toolName === 'text_msg_prompt') {
|
||||
const msg = String(parsedArgs?.msg || '').trim();
|
||||
if (!msg) {
|
||||
resultPayload.output = { message: "Missing required argument 'msg'" };
|
||||
resultPayload.status = { code: 422, message: 'invalid_arguments' };
|
||||
} else {
|
||||
setTextPromptDialog({ open: true, message: msg });
|
||||
resultPayload.output = { message: 'text_prompt_shown', msg };
|
||||
resultPayload.status = { code: 200, message: 'ok' };
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
resultPayload.output = {
|
||||
@@ -3183,8 +3220,8 @@ export const DebugDrawer: React.FC<{
|
||||
if (type === 'assistant.response.delta') {
|
||||
const delta = String(payload.text || '');
|
||||
if (!delta) return;
|
||||
const responseIdRaw = payload?.data?.response_id ?? payload?.response_id ?? payload?.responseId;
|
||||
const responseId = String(responseIdRaw || '').trim() || undefined;
|
||||
const responseId = extractResponseId(payload);
|
||||
if (responseId && interruptedResponseIdsRef.current.has(responseId)) return;
|
||||
setMessages((prev) => {
|
||||
let idx = assistantDraftIndexRef.current;
|
||||
if (idx === null || !prev[idx] || prev[idx].role !== 'model') {
|
||||
@@ -3250,8 +3287,8 @@ export const DebugDrawer: React.FC<{
|
||||
|
||||
if (type === 'assistant.response.final') {
|
||||
const finalText = String(payload.text || '');
|
||||
const responseIdRaw = payload?.data?.response_id ?? payload?.response_id ?? payload?.responseId;
|
||||
const responseId = String(responseIdRaw || '').trim() || undefined;
|
||||
const responseId = extractResponseId(payload);
|
||||
if (responseId && interruptedResponseIdsRef.current.has(responseId)) return;
|
||||
setMessages((prev) => {
|
||||
let idx = assistantDraftIndexRef.current;
|
||||
assistantDraftIndexRef.current = null;
|
||||
@@ -3714,7 +3751,7 @@ export const DebugDrawer: React.FC<{
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className={mode === 'text' && textSessionStarted ? 'shrink-0 mt-3 px-1 mb-3' : mode === 'voice' ? 'shrink-0 space-y-3 mt-3 px-1 mb-3' : 'shrink-0 space-y-2 mt-2 px-1 mb-3'}>
|
||||
<div className={mode === 'text' && textSessionStarted ? 'shrink-0 mt-3 px-1 mb-3' : mode === 'voice' ? 'shrink-0 space-y-3 mt-3 px-1 mb-3' : 'shrink-0 space-y-2 mt-2 px-1 mb-3'}>
|
||||
{mode === 'voice' && (
|
||||
<div className="w-full flex items-center gap-2 pb-1">
|
||||
<span className="text-xs text-muted-foreground shrink-0">麦克风</span>
|
||||
@@ -3768,11 +3805,34 @@ export const DebugDrawer: React.FC<{
|
||||
>
|
||||
<Send className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{textPromptDialog.open && (
|
||||
<div className="absolute inset-0 z-40 flex items-center justify-center bg-black/55 backdrop-blur-[1px]">
|
||||
<div className="relative w-[92%] max-w-md rounded-xl border border-white/15 bg-card/95 p-4 shadow-2xl animate-in zoom-in-95 duration-200">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setTextPromptDialog({ open: false, message: '' })}
|
||||
className="absolute right-3 top-3 rounded-sm opacity-70 hover:opacity-100 text-muted-foreground hover:text-foreground transition-opacity"
|
||||
title="关闭"
|
||||
>
|
||||
<X className="h-4 w-4" />
|
||||
</button>
|
||||
<div className="mb-3 pr-6">
|
||||
<div className="text-[10px] font-black tracking-[0.14em] uppercase text-amber-300">文本消息提示</div>
|
||||
<p className="mt-2 text-sm leading-6 text-foreground whitespace-pre-wrap break-words">{textPromptDialog.message}</p>
|
||||
</div>
|
||||
<div className="flex justify-end">
|
||||
<Button size="sm" onClick={() => setTextPromptDialog({ open: false, message: '' })}>
|
||||
确认
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</Drawer>
|
||||
)}
|
||||
</div>
|
||||
</Drawer>
|
||||
{isOpen && (
|
||||
<div className="fixed inset-y-0 z-[51] right-[min(100vw,32rem)]">
|
||||
<button
|
||||
|
||||
Reference in New Issue
Block a user