Enhance voice interaction features and introduce voice preview functionality
- Update README to reflect the integration of the DebugVoicePanel with WebSocket support for voice interactions. - Refactor voice_webrtc.py to improve error handling during WebRTC signaling and include assistant_id in the offer payload. - Add useVoicePreview hook to manage microphone access and WebRTC connections for real-time voice previews. - Modify AssistantPage to incorporate new visualizer options and pass assistantId to DebugVoicePanel, enhancing user experience during audio interactions. - Update API model to include new fields for voice, speed, and language, supporting TTS and ASR configurations.
This commit is contained in:
@@ -100,5 +100,5 @@ docker compose --profile remote up -d
|
||||
- [ ] `pip install` 后跑通,核对 pipecat 版本的服务/transport 构造参数(代码内有注释)
|
||||
- [ ] 起本地 SenseVoice / CosyVoice 的 OpenAI 兼容服务
|
||||
- [ ] `realtime` 模式(目前只 `pipeline` 级联)
|
||||
- [ ] 前端 `DebugVoicePanel` 接 `/ws/voice`(抄 dograh `useWebSocketRTC.tsx`)
|
||||
- [x] 前端 `DebugVoicePanel` 接 `/ws/voice`(参考 dograh `useWebSocketRTC.tsx`)
|
||||
- [ ] 加 DB 后:助手配置入库(目前随请求内联)
|
||||
|
||||
@@ -2,9 +2,10 @@
|
||||
|
||||
参考 dograh 的 webrtc_signaling.py,砍掉鉴权/配额/DB/org/ICE 过滤策略/TURN。
|
||||
握手消息:
|
||||
client → {type:"offer", payload:{pc_id, sdp, type, config}}
|
||||
client → {type:"offer", payload:{pc_id, sdp, type, assistant_id}}
|
||||
server → {type:"answer", payload:{pc_id, sdp, type}}
|
||||
both → {type:"ice-candidate", payload:{pc_id, candidate:{...}}}
|
||||
server → {type:"error", payload:{message}}
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -36,10 +37,22 @@ async def voice_signaling(websocket: WebSocket):
|
||||
try:
|
||||
while True:
|
||||
message = await websocket.receive_json()
|
||||
if message.get("type") == "offer":
|
||||
await _handle_offer(websocket, message.get("payload", {}), peers)
|
||||
elif message.get("type") == "ice-candidate":
|
||||
await _handle_ice(message.get("payload", {}), peers)
|
||||
try:
|
||||
if message.get("type") == "offer":
|
||||
await _handle_offer(websocket, message.get("payload", {}), peers)
|
||||
elif message.get("type") == "ice-candidate":
|
||||
await _handle_ice(message.get("payload", {}), peers)
|
||||
except Exception as e:
|
||||
logger.exception(f"处理 WebRTC 信令消息失败: {e}")
|
||||
if websocket.application_state == WebSocketState.CONNECTED:
|
||||
await websocket.send_json(
|
||||
{
|
||||
"type": "error",
|
||||
"payload": {
|
||||
"message": f"语音会话启动失败: {type(e).__name__}"
|
||||
},
|
||||
}
|
||||
)
|
||||
except WebSocketDisconnect:
|
||||
logger.info("WebRTC 信令断开")
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user