wx44wx/AI-VideoAssistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance DebugDrawer to support voice prompts in text prompt dialogs

Browse Source 
- Added `promptType` and `voiceText` properties to `DebugTextPromptDialogState`.
- Updated state management for text prompt dialogs to handle voice prompts.
- Modified dialog activation logic to play voice prompts when applicable.
- Adjusted UI to reflect the type of prompt being displayed (text or voice).
- Ensured proper handling of prompt closure messages based on prompt type.
This commit is contained in:
Xin Wang
2026-03-02 15:10:03 +08:00
parent 3aa9e0f432
commit 70b4043f9b
1 changed files with 43 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
web/pages/Assistants.tsx
View File

@@ -2207,6 +2207,8 @@ type DebugTextPromptDialogState = {
open: boolean;
message: string;
pendingResult?: DebugPromptPendingResult;
promptType?: 'text' | 'voice';
voiceText?: string;
};
type DebugChoicePromptDialogState = {
@@ -2341,7 +2343,11 @@ export const DebugDrawer: React.FC<{
const [inputText, setInputText] = useState('');
const [isLoading, setIsLoading] = useState(false);
const [callStatus, setCallStatus] = useState<'idle' | 'calling' | 'active'>('idle');
const [textPromptDialog, setTextPromptDialog] = useState<DebugTextPromptDialogState>({ open: false, message: '' });
const [textPromptDialog, setTextPromptDialog] = useState<DebugTextPromptDialogState>({
open: false,
message: '',
promptType: 'text',
});
const [choicePromptDialog, setChoicePromptDialog] = useState<DebugChoicePromptDialogState>({ open: false, question: '', options: [] });
const textPromptDialogRef = useRef(textPromptDialog);
const choicePromptDialogRef = useRef(choicePromptDialog);
@@ -2538,7 +2544,7 @@ export const DebugDrawer: React.FC<{
closeWs();
stopPromptVoicePlayback();
promptDialogQueueRef.current = [];
setTextPromptDialog({ open: false, message: '' });
setTextPromptDialog({ open: false, message: '', promptType: 'text' });
setChoicePromptDialog({ open: false, question: '', options: [] });
if (audioCtxRef.current) {
void audioCtxRef.current.close();
@@ -2861,11 +2867,17 @@ export const DebugDrawer: React.FC<{
const activatePromptDialog = (item: DebugPromptQueueItem) => {
if (item.kind === 'text') {
const nextVoiceText = String(item.payload.voiceText || '').trim();
setTextPromptDialog({
open: true,
message: item.payload.message,
pendingResult: item.payload.pendingResult,
promptType: item.payload.promptType || 'text',
voiceText: nextVoiceText || undefined,
});
if (nextVoiceText) {
void playPromptVoice(nextVoiceText);
}
return;
}
const nextVoiceText = String(item.payload.voiceText || '').trim();
@@ -2902,16 +2914,19 @@ export const DebugDrawer: React.FC<{
if (!snapshot.open && !opts?.force) return;
const pending = snapshot?.pendingResult;
const message = snapshot?.message || '';
setTextPromptDialog({ open: false, message: '' });
const promptType = snapshot?.promptType === 'voice' ? 'voice' : 'text';
stopPromptVoicePlayback();
setTextPromptDialog({ open: false, message: '', promptType: 'text' });
if (pending?.waitForResponse) {
emitClientToolResult(
{
tool_call_id: pending.toolCallId,
name: pending.toolName,
output: {
message: 'text_prompt_closed',
message: promptType === 'voice' ? 'voice_prompt_closed' : 'text_prompt_closed',
action,
msg: message,
prompt_type: promptType,
},
status: { code: 200, message: 'ok' },
},
@@ -3108,7 +3123,7 @@ export const DebugDrawer: React.FC<{
setCallStatus('idle');
clearResponseTracking();
setMessages([]);
setTextPromptDialog({ open: false, message: '' });
setTextPromptDialog({ open: false, message: '', promptType: 'text' });
setChoicePromptDialog({ open: false, question: '', options: [] });
lastUserFinalRef.current = '';
setIsLoading(false);
@@ -3467,7 +3482,7 @@ export const DebugDrawer: React.FC<{
micFrameBufferRef.current = new Uint8Array(0);
stopPromptVoicePlayback();
promptDialogQueueRef.current = [];
setTextPromptDialog({ open: false, message: '' });
setTextPromptDialog({ open: false, message: '', promptType: 'text' });
setChoicePromptDialog({ open: false, question: '', options: [] });
setTextSessionStarted(false);
stopPlaybackImmediately();
@@ -3729,13 +3744,26 @@ export const DebugDrawer: React.FC<{
resultPayload.output = { message: "Missing required argument 'msg'" };
resultPayload.status = { code: 422, message: 'invalid_arguments' };
} else {
void playPromptVoice(msg);
if (waitForResponse) {
// Voice prompt playback is fire-and-forget; keep previous wait behavior stable.
// Client ack is returned immediately after dispatch.
enqueuePromptDialog({
kind: 'text',
payload: {
message: msg,
promptType: 'voice',
voiceText: msg,
pendingResult: {
toolCallId: toolCallId,
toolName,
toolDisplayName,
waitForResponse,
},
},
});
if (!waitForResponse) {
resultPayload.output = { message: 'voice_prompt_shown', msg };
resultPayload.status = { code: 200, message: 'ok' };
} else {
return;
}
resultPayload.output = { message: 'voice_prompt_sent', msg };
resultPayload.status = { code: 200, message: 'ok' };
}
} else if (toolName === 'text_msg_prompt') {
const msg = String(parsedArgs?.msg || '').trim();
@@ -4524,7 +4552,9 @@ export const DebugDrawer: React.FC<{
<X className="h-4 w-4" />
</button>
<div className="mb-3 pr-6">
<div className="text-[10px] font-black tracking-[0.14em] uppercase text-amber-300">文本消息提示</div>
<div className="text-[10px] font-black tracking-[0.14em] uppercase text-amber-300">
{textPromptDialog.promptType === 'voice' ? '语音消息提示' : '文本消息提示'}
</div>
<p className="mt-2 text-sm leading-6 text-foreground whitespace-pre-wrap break-words">{textPromptDialog.message}</p>
</div>
<div className="flex justify-end">