Enhance DebugDrawer to support voice prompts in text prompt dialogs
- Added `promptType` and `voiceText` properties to `DebugTextPromptDialogState`. - Updated state management for text prompt dialogs to handle voice prompts. - Modified dialog activation logic to play voice prompts when applicable. - Adjusted UI to reflect the type of prompt being displayed (text or voice). - Ensured proper handling of prompt closure messages based on prompt type.
This commit is contained in:
@@ -2207,6 +2207,8 @@ type DebugTextPromptDialogState = {
|
|||||||
open: boolean;
|
open: boolean;
|
||||||
message: string;
|
message: string;
|
||||||
pendingResult?: DebugPromptPendingResult;
|
pendingResult?: DebugPromptPendingResult;
|
||||||
|
promptType?: 'text' | 'voice';
|
||||||
|
voiceText?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
type DebugChoicePromptDialogState = {
|
type DebugChoicePromptDialogState = {
|
||||||
@@ -2341,7 +2343,11 @@ export const DebugDrawer: React.FC<{
|
|||||||
const [inputText, setInputText] = useState('');
|
const [inputText, setInputText] = useState('');
|
||||||
const [isLoading, setIsLoading] = useState(false);
|
const [isLoading, setIsLoading] = useState(false);
|
||||||
const [callStatus, setCallStatus] = useState<'idle' | 'calling' | 'active'>('idle');
|
const [callStatus, setCallStatus] = useState<'idle' | 'calling' | 'active'>('idle');
|
||||||
const [textPromptDialog, setTextPromptDialog] = useState<DebugTextPromptDialogState>({ open: false, message: '' });
|
const [textPromptDialog, setTextPromptDialog] = useState<DebugTextPromptDialogState>({
|
||||||
|
open: false,
|
||||||
|
message: '',
|
||||||
|
promptType: 'text',
|
||||||
|
});
|
||||||
const [choicePromptDialog, setChoicePromptDialog] = useState<DebugChoicePromptDialogState>({ open: false, question: '', options: [] });
|
const [choicePromptDialog, setChoicePromptDialog] = useState<DebugChoicePromptDialogState>({ open: false, question: '', options: [] });
|
||||||
const textPromptDialogRef = useRef(textPromptDialog);
|
const textPromptDialogRef = useRef(textPromptDialog);
|
||||||
const choicePromptDialogRef = useRef(choicePromptDialog);
|
const choicePromptDialogRef = useRef(choicePromptDialog);
|
||||||
@@ -2538,7 +2544,7 @@ export const DebugDrawer: React.FC<{
|
|||||||
closeWs();
|
closeWs();
|
||||||
stopPromptVoicePlayback();
|
stopPromptVoicePlayback();
|
||||||
promptDialogQueueRef.current = [];
|
promptDialogQueueRef.current = [];
|
||||||
setTextPromptDialog({ open: false, message: '' });
|
setTextPromptDialog({ open: false, message: '', promptType: 'text' });
|
||||||
setChoicePromptDialog({ open: false, question: '', options: [] });
|
setChoicePromptDialog({ open: false, question: '', options: [] });
|
||||||
if (audioCtxRef.current) {
|
if (audioCtxRef.current) {
|
||||||
void audioCtxRef.current.close();
|
void audioCtxRef.current.close();
|
||||||
@@ -2861,11 +2867,17 @@ export const DebugDrawer: React.FC<{
|
|||||||
|
|
||||||
const activatePromptDialog = (item: DebugPromptQueueItem) => {
|
const activatePromptDialog = (item: DebugPromptQueueItem) => {
|
||||||
if (item.kind === 'text') {
|
if (item.kind === 'text') {
|
||||||
|
const nextVoiceText = String(item.payload.voiceText || '').trim();
|
||||||
setTextPromptDialog({
|
setTextPromptDialog({
|
||||||
open: true,
|
open: true,
|
||||||
message: item.payload.message,
|
message: item.payload.message,
|
||||||
pendingResult: item.payload.pendingResult,
|
pendingResult: item.payload.pendingResult,
|
||||||
|
promptType: item.payload.promptType || 'text',
|
||||||
|
voiceText: nextVoiceText || undefined,
|
||||||
});
|
});
|
||||||
|
if (nextVoiceText) {
|
||||||
|
void playPromptVoice(nextVoiceText);
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const nextVoiceText = String(item.payload.voiceText || '').trim();
|
const nextVoiceText = String(item.payload.voiceText || '').trim();
|
||||||
@@ -2902,16 +2914,19 @@ export const DebugDrawer: React.FC<{
|
|||||||
if (!snapshot.open && !opts?.force) return;
|
if (!snapshot.open && !opts?.force) return;
|
||||||
const pending = snapshot?.pendingResult;
|
const pending = snapshot?.pendingResult;
|
||||||
const message = snapshot?.message || '';
|
const message = snapshot?.message || '';
|
||||||
setTextPromptDialog({ open: false, message: '' });
|
const promptType = snapshot?.promptType === 'voice' ? 'voice' : 'text';
|
||||||
|
stopPromptVoicePlayback();
|
||||||
|
setTextPromptDialog({ open: false, message: '', promptType: 'text' });
|
||||||
if (pending?.waitForResponse) {
|
if (pending?.waitForResponse) {
|
||||||
emitClientToolResult(
|
emitClientToolResult(
|
||||||
{
|
{
|
||||||
tool_call_id: pending.toolCallId,
|
tool_call_id: pending.toolCallId,
|
||||||
name: pending.toolName,
|
name: pending.toolName,
|
||||||
output: {
|
output: {
|
||||||
message: 'text_prompt_closed',
|
message: promptType === 'voice' ? 'voice_prompt_closed' : 'text_prompt_closed',
|
||||||
action,
|
action,
|
||||||
msg: message,
|
msg: message,
|
||||||
|
prompt_type: promptType,
|
||||||
},
|
},
|
||||||
status: { code: 200, message: 'ok' },
|
status: { code: 200, message: 'ok' },
|
||||||
},
|
},
|
||||||
@@ -3108,7 +3123,7 @@ export const DebugDrawer: React.FC<{
|
|||||||
setCallStatus('idle');
|
setCallStatus('idle');
|
||||||
clearResponseTracking();
|
clearResponseTracking();
|
||||||
setMessages([]);
|
setMessages([]);
|
||||||
setTextPromptDialog({ open: false, message: '' });
|
setTextPromptDialog({ open: false, message: '', promptType: 'text' });
|
||||||
setChoicePromptDialog({ open: false, question: '', options: [] });
|
setChoicePromptDialog({ open: false, question: '', options: [] });
|
||||||
lastUserFinalRef.current = '';
|
lastUserFinalRef.current = '';
|
||||||
setIsLoading(false);
|
setIsLoading(false);
|
||||||
@@ -3467,7 +3482,7 @@ export const DebugDrawer: React.FC<{
|
|||||||
micFrameBufferRef.current = new Uint8Array(0);
|
micFrameBufferRef.current = new Uint8Array(0);
|
||||||
stopPromptVoicePlayback();
|
stopPromptVoicePlayback();
|
||||||
promptDialogQueueRef.current = [];
|
promptDialogQueueRef.current = [];
|
||||||
setTextPromptDialog({ open: false, message: '' });
|
setTextPromptDialog({ open: false, message: '', promptType: 'text' });
|
||||||
setChoicePromptDialog({ open: false, question: '', options: [] });
|
setChoicePromptDialog({ open: false, question: '', options: [] });
|
||||||
setTextSessionStarted(false);
|
setTextSessionStarted(false);
|
||||||
stopPlaybackImmediately();
|
stopPlaybackImmediately();
|
||||||
@@ -3729,13 +3744,26 @@ export const DebugDrawer: React.FC<{
|
|||||||
resultPayload.output = { message: "Missing required argument 'msg'" };
|
resultPayload.output = { message: "Missing required argument 'msg'" };
|
||||||
resultPayload.status = { code: 422, message: 'invalid_arguments' };
|
resultPayload.status = { code: 422, message: 'invalid_arguments' };
|
||||||
} else {
|
} else {
|
||||||
void playPromptVoice(msg);
|
enqueuePromptDialog({
|
||||||
if (waitForResponse) {
|
kind: 'text',
|
||||||
// Voice prompt playback is fire-and-forget; keep previous wait behavior stable.
|
payload: {
|
||||||
// Client ack is returned immediately after dispatch.
|
message: msg,
|
||||||
}
|
promptType: 'voice',
|
||||||
resultPayload.output = { message: 'voice_prompt_sent', msg };
|
voiceText: msg,
|
||||||
|
pendingResult: {
|
||||||
|
toolCallId: toolCallId,
|
||||||
|
toolName,
|
||||||
|
toolDisplayName,
|
||||||
|
waitForResponse,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (!waitForResponse) {
|
||||||
|
resultPayload.output = { message: 'voice_prompt_shown', msg };
|
||||||
resultPayload.status = { code: 200, message: 'ok' };
|
resultPayload.status = { code: 200, message: 'ok' };
|
||||||
|
} else {
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else if (toolName === 'text_msg_prompt') {
|
} else if (toolName === 'text_msg_prompt') {
|
||||||
const msg = String(parsedArgs?.msg || '').trim();
|
const msg = String(parsedArgs?.msg || '').trim();
|
||||||
@@ -4524,7 +4552,9 @@ export const DebugDrawer: React.FC<{
|
|||||||
<X className="h-4 w-4" />
|
<X className="h-4 w-4" />
|
||||||
</button>
|
</button>
|
||||||
<div className="mb-3 pr-6">
|
<div className="mb-3 pr-6">
|
||||||
<div className="text-[10px] font-black tracking-[0.14em] uppercase text-amber-300">文本消息提示</div>
|
<div className="text-[10px] font-black tracking-[0.14em] uppercase text-amber-300">
|
||||||
|
{textPromptDialog.promptType === 'voice' ? '语音消息提示' : '文本消息提示'}
|
||||||
|
</div>
|
||||||
<p className="mt-2 text-sm leading-6 text-foreground whitespace-pre-wrap break-words">{textPromptDialog.message}</p>
|
<p className="mt-2 text-sm leading-6 text-foreground whitespace-pre-wrap break-words">{textPromptDialog.message}</p>
|
||||||
</div>
|
</div>
|
||||||
<div className="flex justify-end">
|
<div className="flex justify-end">
|
||||||
|
|||||||
Reference in New Issue
Block a user