From e8ef7c6da713a1971aa13e8e3210edc8fbb321e6 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Tue, 16 Dec 2025 17:54:37 +0800 Subject: [PATCH] bug fixed --- agents/my_basic_agent_1_2_9.py | 103 ++++++++++++++++--- src/components/playground/PhoneSimulator.tsx | 83 +++++++++------ src/components/playground/Playground.tsx | 22 +++- 3 files changed, 164 insertions(+), 44 deletions(-) diff --git a/agents/my_basic_agent_1_2_9.py b/agents/my_basic_agent_1_2_9.py index fd2b0eb..d18aa27 100644 --- a/agents/my_basic_agent_1_2_9.py +++ b/agents/my_basic_agent_1_2_9.py @@ -66,7 +66,7 @@ DEFAULT_INSTRUCTIONS = """# 角色 # 能力 - 你具有调用工具操作前端界面系统的能力 -- ask_image_capture工具被调用后会在系统播放拍摄的目标和需求，所以你每次在调用它之前不需要重复引导用户拍摄什么 +- ask_image_capture工具被调用后会在系统播放拍摄的目标和需求，所以你每次在调用它之前不需要重复引导用户拍摄什么，而是使用ask_image_capture来传递拍摄需求 # 任务 你的职责是全流程引导用户完成：事故信息采集 -> 现场证据拍照 -> 驾驶员信息核实。 @@ -75,6 +75,7 @@ DEFAULT_INSTRUCTIONS = """# 角色 - 在事故信息采集阶段：询问是否有人受伤，请求用户简单描述事故情况，询问事故发生时间并通过复述标准化时间（xx年xx月xx日xx时xx分）向用户确认，询问事故车辆数量，询问事故发生的原因（例如追尾、刮擦、碰撞等）。采集完成后进入现场证据拍照阶段 - 如果用户回答已包含需要问题的答案，改为与用户确认答案是否正确 - 采集完成之后进入现场证据拍照阶段 +- 这个阶段不使用ask_important_question和ask_image_capture工具 ## 现场证据拍照阶段 - 在现场证据拍照阶段：使用askImageCapture工具引导用户依次拍摄照片：1. 第一辆车的车牌；2. 第一辆车的碰撞位置；3. 第一辆车的驾驶员正脸； @@ -614,11 +615,20 @@ class MyAgent(Agent): mobile_suffix = ''.join([str(random.randint(0, 9)) for _ in range(9)]) random_mobile = f"{mobile_prefix}{mobile_suffix}" - return { + result = { "success": True, "plate": normalized_plate, "mobile": random_mobile, } + + await self._send_chat_message( + "┌─✅ Result: get_mobile_by_plate\n" + f"│ plate: \"{normalized_plate}\"\n" + f"│ mobile: \"{random_mobile}\"\n" + "└───────────────" + ) + + return result @function_tool() async def get_id_card_by_plate( @@ -652,11 +662,20 @@ class MyAgent(Agent): check_digit = 'X' if random.random() < 0.1 else str(random.randint(0, 9)) random_id_card = f"{area_code}{birth_date}{sequence}{check_digit}" - return { + result = { "success": True, "plate": normalized_plate, "id_card": random_id_card, } + + await self._send_chat_message( + "┌─✅ Result: get_id_card_by_plate\n" + f"│ plate: \"{normalized_plate}\"\n" + f"│ id_card: \"{random_id_card}\"\n" + "└───────────────" + ) + + return result @function_tool() async def validate_mobile_number( @@ -678,17 +697,33 @@ class MyAgent(Agent): ) is_valid = bool(re.fullmatch(r"1[3-9]\\d{9}", normalized)) if is_valid: - return { + result = { "success": True, "valid": True, "mobile": normalized, } - return { + await self._send_chat_message( + "┌─✅ Result: validate_mobile_number\n" + f"│ mobile: \"{normalized}\"\n" + f"│ valid: true\n" + "└───────────────" + ) + return result + + result = { "success": True, "valid": False, "mobile": normalized, "error": "手机号格式不正确，应为1[3-9]开头的11位数字", } + await self._send_chat_message( + "┌─✅ Result: validate_mobile_number\n" + f"│ mobile: \"{normalized}\"\n" + f"│ valid: false\n" + f"│ error: \"{result['error']}\"\n" + "└───────────────" + ) + return result @function_tool() async def validate_id_card_number( @@ -710,25 +745,44 @@ class MyAgent(Agent): ) is_valid = bool(re.fullmatch(r"(\\d{17}[\\dX]|\\d{15})", normalized)) if is_valid: - return { + result = { "success": True, "valid": True, "id_card": normalized, } - return { + await self._send_chat_message( + "┌─✅ Result: validate_id_card_number\n" + f"│ id_card: \"{normalized}\"\n" + f"│ valid: true\n" + "└───────────────" + ) + return result + + result = { "success": True, "valid": False, "id_card": normalized, "error": "身份证格式不正确，应为18位（末位可为X）或15位数字", } + await self._send_chat_message( + "┌─✅ Result: validate_id_card_number\n" + f"│ id_card: \"{normalized}\"\n" + f"│ valid: false\n" + f"│ error: \"{result['error']}\"\n" + "└───────────────" + ) + return result @function_tool() async def enter_hand_off_to_human_mode( self, context: RunContext, ): - """切换到“转人工”模式（前端电话界面进入人工处理）。返回成功/失败。""" - await self._send_chat_message("🔨 Call: enter_hand_off_to_human_mode") + """切换到"转人工"模式（前端电话界面进入人工处理）。返回成功/失败。""" + await self._send_chat_message( + "┌─🔨 Call: enter_hand_off_to_human_mode\n" + "└───────────────" + ) try: room = get_job_context().room participant_identity = next(iter(room.remote_participants)) @@ -739,10 +793,21 @@ class MyAgent(Agent): response_timeout=5.0, ) logger.info(f"Entered hand off to human mode: {response}") - await self._send_chat_message(f"✅ Result: enter_hand_off_to_human_mode\n • status: success") + await self._send_chat_message( + "┌─✅ Result: enter_hand_off_to_human_mode\n" + f"│ status: success\n" + f"│ response: {response}\n" + "└───────────────" + ) return response except Exception as e: logger.error(f"Failed to enter hand off to human mode: {e}") + await self._send_chat_message( + "┌─❌ Result: enter_hand_off_to_human_mode\n" + f"│ status: error\n" + f"│ error: \"{str(e)}\"\n" + "└───────────────" + ) raise ToolError(f"Unable to enter hand off to human mode: {str(e)}") @function_tool() @@ -751,7 +816,10 @@ class MyAgent(Agent): context: RunContext, ): """挂断当前通话（结束会话），返回成功/失败。""" - await self._send_chat_message("🔨 Call: hang_up_call") + await self._send_chat_message( + "┌─🔨 Call: hang_up_call\n" + "└───────────────" + ) try: room = get_job_context().room participant_identity = next(iter(room.remote_participants)) @@ -762,10 +830,21 @@ class MyAgent(Agent): response_timeout=5.0, ) logger.info(f"Hung up call: {response}") - await self._send_chat_message(f"✅ Result: hang_up_call\n • status: disconnected") + await self._send_chat_message( + "┌─✅ Result: hang_up_call\n" + f"│ status: disconnected\n" + f"│ response: {response}\n" + "└───────────────" + ) return response except Exception as e: logger.error(f"Failed to hang up call: {e}") + await self._send_chat_message( + "┌─❌ Result: hang_up_call\n" + f"│ status: error\n" + f"│ error: \"{str(e)}\"\n" + "└───────────────" + ) raise ToolError(f"Unable to hang up call: {str(e)}") @function_tool() diff --git a/src/components/playground/PhoneSimulator.tsx b/src/components/playground/PhoneSimulator.tsx index 797152c..a8ed4f0 100644 --- a/src/components/playground/PhoneSimulator.tsx +++ b/src/components/playground/PhoneSimulator.tsx @@ -164,35 +164,53 @@ export function PhoneSimulator({ const enteringMode = (mode: typeof phoneMode) => phoneMode === mode && lastPhoneMode.current !== mode; - // Entering important message / capture / hand_off: remember mic state and mute if needed - if (enteringMode("important_message") || enteringMode("capture") || enteringMode("hand_off")) { - wasMicEnabledRef.current = isMicEnabled; - if (isMicEnabled) { - localParticipant.setMicrophoneEnabled(false); - } - } - // Exiting important message mode or hand off mode or capture mode - else if ( - (phoneMode !== "important_message" && lastPhoneMode.current === "important_message") || - (phoneMode !== "hand_off" && lastPhoneMode.current === "hand_off") || - (phoneMode !== "capture" && lastPhoneMode.current === "capture") - ) { - // Restore mic to previous state - localParticipant.setMicrophoneEnabled(wasMicEnabledRef.current); + // Only proceed if connected and localParticipant is available + if (roomState !== ConnectionState.Connected || !localParticipant) return; - // If exiting capture mode, clear processing image - if (lastPhoneMode.current === "capture") { - setProcessingImage(null); - setProcessingSource(null); + const updateMicState = async () => { + // Entering important message / capture / hand_off: remember mic state and mute if needed + if (enteringMode("important_message") || enteringMode("capture") || enteringMode("hand_off")) { + wasMicEnabledRef.current = isMicEnabled; + if (isMicEnabled) { + try { + await localParticipant.setMicrophoneEnabled(false); + } catch (error) { + console.error("Failed to disable microphone:", error); + } + } } - } - // Enforce mic off in important message mode, hand off mode, or capture mode - else if ((phoneMode === "important_message" || phoneMode === "hand_off" || phoneMode === "capture") && isMicEnabled) { - localParticipant.setMicrophoneEnabled(false); - } + // Exiting important message mode or hand off mode or capture mode + else if ( + (phoneMode !== "important_message" && lastPhoneMode.current === "important_message") || + (phoneMode !== "hand_off" && lastPhoneMode.current === "hand_off") || + (phoneMode !== "capture" && lastPhoneMode.current === "capture") + ) { + // Restore mic to previous state + try { + await localParticipant.setMicrophoneEnabled(wasMicEnabledRef.current); + } catch (error) { + console.error("Failed to restore microphone:", error); + } + // If exiting capture mode, clear processing image + if (lastPhoneMode.current === "capture") { + setProcessingImage(null); + setProcessingSource(null); + } + } + // Enforce mic off in important message mode, hand off mode, or capture mode + else if ((phoneMode === "important_message" || phoneMode === "hand_off" || phoneMode === "capture") && isMicEnabled) { + try { + await localParticipant.setMicrophoneEnabled(false); + } catch (error) { + console.error("Failed to disable microphone:", error); + } + } + }; + + updateMicState(); lastPhoneMode.current = phoneMode; - }, [phoneMode, isMicEnabled, localParticipant]); + }, [phoneMode, isMicEnabled, localParticipant, roomState]); useEffect(() => { const updateTime = () => { @@ -218,10 +236,17 @@ export function PhoneSimulator({ ); const handleMicToggle = async () => { - if (isMicEnabled) { - await localParticipant.setMicrophoneEnabled(false); - } else { - await localParticipant.setMicrophoneEnabled(true); + if (roomState !== ConnectionState.Connected || !localParticipant) return; + + try { + if (isMicEnabled) { + await localParticipant.setMicrophoneEnabled(false); + } else { + await localParticipant.setMicrophoneEnabled(true); + } + } catch (error) { + console.error("Failed to toggle microphone:", error); + // Silently handle the error to avoid disrupting user experience } }; diff --git a/src/components/playground/Playground.tsx b/src/components/playground/Playground.tsx index 3af2719..4324204 100644 --- a/src/components/playground/Playground.tsx +++ b/src/components/playground/Playground.tsx @@ -118,9 +118,25 @@ export default function Playground({ }, [onConnect, cleanupRpcResolvers]); useEffect(() => { - if (roomState === ConnectionState.Connected) { - localParticipant.setCameraEnabled(config.settings.inputs.camera); - localParticipant.setMicrophoneEnabled(config.settings.inputs.mic); + if (roomState === ConnectionState.Connected && localParticipant) { + try { + localParticipant.setCameraEnabled(config.settings.inputs.camera); + localParticipant.setMicrophoneEnabled(config.settings.inputs.mic); + } catch (error) { + console.error("Failed to set camera/microphone:", error); + // Retry after a short delay if connection might not be fully ready + const retryTimeout = setTimeout(() => { + if (roomState === ConnectionState.Connected && localParticipant) { + try { + localParticipant.setCameraEnabled(config.settings.inputs.camera); + localParticipant.setMicrophoneEnabled(config.settings.inputs.mic); + } catch (retryError) { + console.error("Failed to set camera/microphone on retry:", retryError); + } + } + }, 500); + return () => clearTimeout(retryTimeout); + } } }, [config.settings.inputs.camera, config.settings.inputs.mic, localParticipant, roomState]);