ede5e2334b
- Add new HTTP endpoints for handling WebRTC offers and ICE candidates, enhancing the signaling process for voice interactions. - Introduce dynamic variable decoding from request headers to support flexible offer payloads. - Refactor existing WebSocket handling to accommodate new offer processing logic. - Update frontend dependencies to include Pipecat client libraries for improved WebRTC transport management. - Streamline voice preview functionality by integrating SmallWebRTCTransport for better media handling.
227 lines
8.4 KiB
Python
227 lines
8.4 KiB
Python
"""WebRTC 输出:SmallWebRTC 信令握手。
|
|
|
|
参考 dograh 的 webrtc_signaling.py,砍掉鉴权/配额/DB/org/ICE 过滤策略/TURN。
|
|
握手消息:
|
|
client → {type:"offer", payload:{pc_id, sdp, type, assistant_id}}
|
|
server → {type:"answer", payload:{pc_id, sdp, type}}
|
|
both → {type:"ice-candidate", payload:{pc_id, candidate:{...}}}
|
|
server → {type:"error", payload:{message}}
|
|
"""
|
|
|
|
import asyncio
|
|
import base64
|
|
import json
|
|
|
|
from db.session import SessionLocal
|
|
from fastapi import APIRouter, Body, Depends, Request, WebSocket
|
|
from loguru import logger
|
|
from models import AssistantConfig, SignalingOffer
|
|
from services.auth import require_admin, require_admin_websocket
|
|
from services.config_resolver import resolve_runtime_config
|
|
from services.runtime_variables import DynamicVariableError, prepare_dynamic_config
|
|
from starlette.websockets import WebSocketDisconnect, WebSocketState
|
|
|
|
from services.webrtc_ice import aiortc_ice_servers, client_ice_servers
|
|
|
|
# 注意:pipecat 是重依赖(语音才用),在 _handle_offer 等处惰性导入。
|
|
|
|
router = APIRouter(tags=["voice"])
|
|
_http_peers: dict[str, object] = {}
|
|
|
|
|
|
@router.get("/api/webrtc/ice-servers", dependencies=[Depends(require_admin)])
|
|
async def ice_servers():
|
|
"""Browser fetches STUN/TURN config (with ephemeral TURN creds when configured)."""
|
|
return {"iceServers": client_ice_servers()}
|
|
|
|
|
|
@router.post("/api/webrtc/offer", dependencies=[Depends(require_admin)])
|
|
async def http_offer(request: Request, payload: dict = Body(...)):
|
|
"""Official SmallWebRTC JS transport offer endpoint."""
|
|
request_data = payload.get("requestData") or {}
|
|
encoded_variables = request.headers.get("x-pipecat-dynamic-variables", "")
|
|
header_variables = {}
|
|
if encoded_variables:
|
|
padding = "=" * (-len(encoded_variables) % 4)
|
|
header_variables = json.loads(
|
|
base64.urlsafe_b64decode(encoded_variables + padding).decode("utf-8")
|
|
)
|
|
offer_payload = {
|
|
**payload,
|
|
"assistant_id": request.headers.get("x-pipecat-assistant-id"),
|
|
"dynamic_variables": header_variables,
|
|
**request_data,
|
|
}
|
|
answer = await _handle_offer_payload(offer_payload, _http_peers)
|
|
return answer
|
|
|
|
|
|
@router.patch("/api/webrtc/offer", dependencies=[Depends(require_admin)])
|
|
async def http_ice_candidates(payload: dict = Body(...)):
|
|
"""Accept the batched trickle ICE format used by the official JS client."""
|
|
pc_id = payload.get("pc_id")
|
|
pc = _http_peers.get(pc_id) if pc_id else None
|
|
if not pc:
|
|
return {"ok": False}
|
|
|
|
from aiortc.sdp import candidate_from_sdp
|
|
|
|
for item in payload.get("candidates") or []:
|
|
candidate = candidate_from_sdp(item.get("candidate", ""))
|
|
candidate.sdpMid = item.get("sdp_mid")
|
|
candidate.sdpMLineIndex = item.get("sdp_mline_index")
|
|
await pc.add_ice_candidate(candidate)
|
|
return {"ok": True}
|
|
|
|
|
|
@router.websocket("/ws/voice")
|
|
async def voice_signaling(websocket: WebSocket):
|
|
if not await require_admin_websocket(websocket):
|
|
return
|
|
await websocket.accept()
|
|
peers: dict = {}
|
|
try:
|
|
while True:
|
|
message = await websocket.receive_json()
|
|
try:
|
|
if message.get("type") == "offer":
|
|
await _handle_offer(websocket, message.get("payload", {}), peers)
|
|
elif message.get("type") == "ice-candidate":
|
|
await _handle_ice(message.get("payload", {}), peers)
|
|
except Exception as e:
|
|
logger.exception(f"处理 WebRTC 信令消息失败: {e}")
|
|
if websocket.application_state == WebSocketState.CONNECTED:
|
|
await websocket.send_json(
|
|
{
|
|
"type": "error",
|
|
"payload": {
|
|
"message": (
|
|
str(e)
|
|
if isinstance(e, DynamicVariableError)
|
|
else f"语音会话启动失败: {type(e).__name__}"
|
|
)
|
|
},
|
|
}
|
|
)
|
|
except WebSocketDisconnect:
|
|
logger.info("WebRTC 信令断开")
|
|
except Exception as e:
|
|
logger.error(f"WebRTC 信令出错: {e}")
|
|
finally:
|
|
# disconnect() triggers the registered closed callback, which removes
|
|
# the peer from this dict. Iterate over a snapshot to avoid mutation.
|
|
for pc in list(peers.values()):
|
|
await pc.disconnect()
|
|
|
|
|
|
async def _resolve_config(offer: SignalingOffer) -> AssistantConfig:
|
|
"""优先用 assistant_id 从 DB 解析(含真 key);否则用调试内联配置。"""
|
|
if offer.assistant_id:
|
|
async with SessionLocal() as session:
|
|
cfg = await resolve_runtime_config(session, offer.assistant_id)
|
|
return prepare_dynamic_config(
|
|
cfg,
|
|
offer.dynamic_variables,
|
|
assistant_id=offer.assistant_id,
|
|
)
|
|
if offer.inline_config:
|
|
return prepare_dynamic_config(
|
|
offer.inline_config,
|
|
offer.dynamic_variables,
|
|
assistant_id=None,
|
|
)
|
|
raise ValueError("offer 缺少 assistant_id 或 inline_config")
|
|
|
|
|
|
async def _handle_offer(websocket, payload, peers):
|
|
answer = await _handle_offer_payload(payload, peers)
|
|
if websocket.application_state == WebSocketState.CONNECTED:
|
|
await websocket.send_json(
|
|
{
|
|
"type": "answer",
|
|
"payload": answer,
|
|
}
|
|
)
|
|
|
|
|
|
async def _handle_offer_payload(payload, peers):
|
|
from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
|
|
from services.pipecat.pipeline import run_pipeline
|
|
from services.pipecat.transports import build_webrtc_transport
|
|
|
|
offer = SignalingOffer(**payload)
|
|
pc_id = offer.pc_id
|
|
|
|
restart_pc = bool(payload.get("restart_pc"))
|
|
if restart_pc and pc_id and pc_id in peers:
|
|
old_pc = peers.pop(pc_id)
|
|
await old_pc.disconnect()
|
|
|
|
if pc_id and pc_id in peers:
|
|
pc = peers[pc_id]
|
|
await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False)
|
|
else:
|
|
cfg = await _resolve_config(offer) # 解析放在建连前,配置错就别建连
|
|
# 服务端助手配置是视觉理解的唯一授权来源；客户端 offer 只负责携带媒体轨。
|
|
vision_enabled = cfg.vision_enabled
|
|
if vision_enabled:
|
|
has_native_vision = (
|
|
not cfg.vision_model_resource_id and cfg.llm_support_image_input
|
|
)
|
|
has_aux_vision_model = (
|
|
bool(cfg.vision_model_resource_id)
|
|
and cfg.vision_llm_support_image_input
|
|
)
|
|
if not (has_native_vision or has_aux_vision_model):
|
|
raise ValueError(
|
|
"当前模型不支持图片输入,请在模型资源中选择支持图片输入的视觉模型"
|
|
)
|
|
pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers())
|
|
if pc_id:
|
|
pc._pc_id = pc_id
|
|
await pc.initialize(sdp=offer.sdp, type=offer.type)
|
|
peers[pc.pc_id] = pc
|
|
|
|
@pc.event_handler("closed")
|
|
async def _on_closed(conn: SmallWebRTCConnection):
|
|
peers.pop(conn.pc_id, None)
|
|
|
|
# 后台跑管线:WebRTC transport + 解析出的运行时配置
|
|
transport = build_webrtc_transport(
|
|
pc,
|
|
video_in_enabled=vision_enabled,
|
|
)
|
|
asyncio.create_task(
|
|
run_pipeline(
|
|
transport,
|
|
cfg,
|
|
vision_enabled=vision_enabled,
|
|
assistant_id=offer.assistant_id,
|
|
channel="webrtc",
|
|
)
|
|
)
|
|
|
|
answer = pc.get_answer()
|
|
return {
|
|
"pc_id": answer["pc_id"],
|
|
"sdp": answer["sdp"],
|
|
"type": answer["type"],
|
|
}
|
|
|
|
|
|
async def _handle_ice(payload, peers):
|
|
from aiortc.sdp import candidate_from_sdp
|
|
|
|
pc_id = payload.get("pc_id")
|
|
candidate_data = payload.get("candidate")
|
|
pc = peers.get(pc_id) if pc_id else None
|
|
if not pc or not candidate_data:
|
|
return
|
|
try:
|
|
candidate = candidate_from_sdp(candidate_data.get("candidate", ""))
|
|
candidate.sdpMid = candidate_data.get("sdpMid")
|
|
candidate.sdpMLineIndex = candidate_data.get("sdpMLineIndex")
|
|
await pc.add_ice_candidate(candidate)
|
|
except Exception as e:
|
|
logger.error(f"添加 ICE candidate 失败: {e}")
|