Implement WebRTC offer handling and ICE candidate management in voice_webrtc.py
- Add new HTTP endpoints for handling WebRTC offers and ICE candidates, enhancing the signaling process for voice interactions. - Introduce dynamic variable decoding from request headers to support flexible offer payloads. - Refactor existing WebSocket handling to accommodate new offer processing logic. - Update frontend dependencies to include Pipecat client libraries for improved WebRTC transport management. - Streamline voice preview functionality by integrating SmallWebRTCTransport for better media handling.
This commit is contained in:
Xin Wang
@@ -9,9 +9,11 @@
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
|
||||
from db.session import SessionLocal
|
||||
from fastapi import APIRouter, Depends, WebSocket
|
||||
from fastapi import APIRouter, Body, Depends, Request, WebSocket
|
||||
from loguru import logger
|
||||
from models import AssistantConfig, SignalingOffer
|
||||
from services.auth import require_admin, require_admin_websocket
|
||||
@@ -24,6 +26,7 @@ from services.webrtc_ice import aiortc_ice_servers, client_ice_servers
|
||||
# 注意:pipecat 是重依赖(语音才用),在 _handle_offer 等处惰性导入。
|
||||
|
||||
router = APIRouter(tags=["voice"])
|
||||
_http_peers: dict[str, object] = {}
|
||||
|
||||
|
||||
@router.get("/api/webrtc/ice-servers", dependencies=[Depends(require_admin)])
|
||||
@@ -32,6 +35,45 @@ async def ice_servers():
|
||||
return {"iceServers": client_ice_servers()}
|
||||
|
||||
|
||||
@router.post("/api/webrtc/offer", dependencies=[Depends(require_admin)])
|
||||
async def http_offer(request: Request, payload: dict = Body(...)):
|
||||
"""Official SmallWebRTC JS transport offer endpoint."""
|
||||
request_data = payload.get("requestData") or {}
|
||||
encoded_variables = request.headers.get("x-pipecat-dynamic-variables", "")
|
||||
header_variables = {}
|
||||
if encoded_variables:
|
||||
padding = "=" * (-len(encoded_variables) % 4)
|
||||
header_variables = json.loads(
|
||||
base64.urlsafe_b64decode(encoded_variables + padding).decode("utf-8")
|
||||
)
|
||||
offer_payload = {
|
||||
**payload,
|
||||
"assistant_id": request.headers.get("x-pipecat-assistant-id"),
|
||||
"dynamic_variables": header_variables,
|
||||
**request_data,
|
||||
}
|
||||
answer = await _handle_offer_payload(offer_payload, _http_peers)
|
||||
return answer
|
||||
|
||||
|
||||
@router.patch("/api/webrtc/offer", dependencies=[Depends(require_admin)])
|
||||
async def http_ice_candidates(payload: dict = Body(...)):
|
||||
"""Accept the batched trickle ICE format used by the official JS client."""
|
||||
pc_id = payload.get("pc_id")
|
||||
pc = _http_peers.get(pc_id) if pc_id else None
|
||||
if not pc:
|
||||
return {"ok": False}
|
||||
|
||||
from aiortc.sdp import candidate_from_sdp
|
||||
|
||||
for item in payload.get("candidates") or []:
|
||||
candidate = candidate_from_sdp(item.get("candidate", ""))
|
||||
candidate.sdpMid = item.get("sdp_mid")
|
||||
candidate.sdpMLineIndex = item.get("sdp_mline_index")
|
||||
await pc.add_ice_candidate(candidate)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.websocket("/ws/voice")
|
||||
async def voice_signaling(websocket: WebSocket):
|
||||
if not await require_admin_websocket(websocket):
|
||||
@@ -92,6 +134,17 @@ async def _resolve_config(offer: SignalingOffer) -> AssistantConfig:
|
||||
|
||||
|
||||
async def _handle_offer(websocket, payload, peers):
|
||||
answer = await _handle_offer_payload(payload, peers)
|
||||
if websocket.application_state == WebSocketState.CONNECTED:
|
||||
await websocket.send_json(
|
||||
{
|
||||
"type": "answer",
|
||||
"payload": answer,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def _handle_offer_payload(payload, peers):
|
||||
from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
|
||||
from services.pipecat.pipeline import run_pipeline
|
||||
from services.pipecat.transports import build_webrtc_transport
|
||||
@@ -99,6 +152,11 @@ async def _handle_offer(websocket, payload, peers):
|
||||
offer = SignalingOffer(**payload)
|
||||
pc_id = offer.pc_id
|
||||
|
||||
restart_pc = bool(payload.get("restart_pc"))
|
||||
if restart_pc and pc_id and pc_id in peers:
|
||||
old_pc = peers.pop(pc_id)
|
||||
await old_pc.disconnect()
|
||||
|
||||
if pc_id and pc_id in peers:
|
||||
pc = peers[pc_id]
|
||||
await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False)
|
||||
@@ -144,17 +202,11 @@ async def _handle_offer(websocket, payload, peers):
|
||||
)
|
||||
|
||||
answer = pc.get_answer()
|
||||
if websocket.application_state == WebSocketState.CONNECTED:
|
||||
await websocket.send_json(
|
||||
{
|
||||
"type": "answer",
|
||||
"payload": {
|
||||
return {
|
||||
"pc_id": answer["pc_id"],
|
||||
"sdp": answer["sdp"],
|
||||
"type": answer["type"],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def _handle_ice(payload, peers):
|
||||
|
||||
254
frontend/package-lock.json
generated
254
frontend/package-lock.json
generated
@@ -8,6 +8,8 @@
|
||||
"name": "ai-video-admin-frontend",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@pipecat-ai/client-js": "^1.12.0",
|
||||
"@pipecat-ai/small-webrtc-transport": "^1.10.5",
|
||||
"@xyflow/react": "^12.11.0",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
@@ -404,6 +406,15 @@
|
||||
"@babel/core": "^7.0.0-0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/runtime": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz",
|
||||
"integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/template": {
|
||||
"version": "7.29.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz",
|
||||
@@ -449,6 +460,22 @@
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@daily-co/daily-js": {
|
||||
"version": "0.90.0",
|
||||
"resolved": "https://registry.npmjs.org/@daily-co/daily-js/-/daily-js-0.90.0.tgz",
|
||||
"integrity": "sha512-XnuuNIxLt8GOv+rFYoU+OPTSmVj+Mg9hRPK2EM6WYVY62F6rcw7vwzZBOSCisVuu1VnTIF/2J9V0VEBa83lDzw==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.12.5",
|
||||
"@sentry/browser": "^8.33.1",
|
||||
"bowser": "^2.8.1",
|
||||
"dequal": "^2.0.3",
|
||||
"events": "^3.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22.14.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@dotenvx/dotenvx": {
|
||||
"version": "1.71.0",
|
||||
"resolved": "https://registry.npmjs.org/@dotenvx/dotenvx/-/dotenvx-1.71.0.tgz",
|
||||
@@ -1888,6 +1915,34 @@
|
||||
"integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@pipecat-ai/client-js": {
|
||||
"version": "1.12.0",
|
||||
"resolved": "https://registry.npmjs.org/@pipecat-ai/client-js/-/client-js-1.12.0.tgz",
|
||||
"integrity": "sha512-TKHuxTfnO5Eq9VpmSqaPsV0y9lQ+jkb7aEG5MRsmjkhxyhhOW2ljMPwMDzOt+lwS89ZRtk12FUKEaRR/+yBGGg==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"@types/events": "^3.0.3",
|
||||
"bowser": "^2.11.0",
|
||||
"clone-deep": "^4.0.1",
|
||||
"events": "^3.3.0",
|
||||
"typed-emitter": "^2.1.0",
|
||||
"uuid": "^11.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@pipecat-ai/small-webrtc-transport": {
|
||||
"version": "1.10.5",
|
||||
"resolved": "https://registry.npmjs.org/@pipecat-ai/small-webrtc-transport/-/small-webrtc-transport-1.10.5.tgz",
|
||||
"integrity": "sha512-TF2TV/GQUVuC8Pr69fG/MSvUtuwEGcPcUg/H5d8O0RmSthVief0APgC5cKuyrjh/Rtl7+AjhxxWbrkuACdxulQ==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"@daily-co/daily-js": "^0.90.0",
|
||||
"dequal": "^2.0.3",
|
||||
"lodash": "^4.17.21"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@pipecat-ai/client-js": "~1.12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@radix-ui/number": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.1.tgz",
|
||||
@@ -3399,6 +3454,81 @@
|
||||
"integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@sentry-internal/browser-utils": {
|
||||
"version": "8.55.2",
|
||||
"resolved": "https://registry.npmjs.org/@sentry-internal/browser-utils/-/browser-utils-8.55.2.tgz",
|
||||
"integrity": "sha512-GnKod+gL/Y+1FUM/RGV8q6le1CoyiGbT40MitEK7eVwWe+bfTRq1gN7ioupyHFMUg1RlQkDQ4/sENmio/uow5A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@sentry/core": "8.55.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.18"
|
||||
}
|
||||
},
|
||||
"node_modules/@sentry-internal/feedback": {
|
||||
"version": "8.55.2",
|
||||
"resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-8.55.2.tgz",
|
||||
"integrity": "sha512-XQy//NWbL0mLLM5w8wNDWMNpXz39VUyW2397dUrH8++kR63WhUVAvTOtL0o0GMVadSAzl1b08oHP9zSUNFQwcg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@sentry/core": "8.55.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.18"
|
||||
}
|
||||
},
|
||||
"node_modules/@sentry-internal/replay": {
|
||||
"version": "8.55.2",
|
||||
"resolved": "https://registry.npmjs.org/@sentry-internal/replay/-/replay-8.55.2.tgz",
|
||||
"integrity": "sha512-+W43Z697EVe/OgpGW07B773sa8xO1UbpnW0Cr+E+3FMDb6ZbXlaBUoagPTUkkQPdwBe35SDh6r8y2M3EOPGbxg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@sentry-internal/browser-utils": "8.55.2",
|
||||
"@sentry/core": "8.55.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.18"
|
||||
}
|
||||
},
|
||||
"node_modules/@sentry-internal/replay-canvas": {
|
||||
"version": "8.55.2",
|
||||
"resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-8.55.2.tgz",
|
||||
"integrity": "sha512-P/jGiuR7dRLG9IzD/463fLgiibyYceauav/9prRG0ZxJm1AtuO02OKball2Fs3bbzdzwHCTlcsUuL2ivDF4b5A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@sentry-internal/replay": "8.55.2",
|
||||
"@sentry/core": "8.55.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.18"
|
||||
}
|
||||
},
|
||||
"node_modules/@sentry/browser": {
|
||||
"version": "8.55.2",
|
||||
"resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-8.55.2.tgz",
|
||||
"integrity": "sha512-xHuPIEKhx9zw5quWvv4YgZprnwoVMCfxIhmOIf6KJ9iizyUHeUDcKpLS59xERroqwX4RpvK+l/27AZu4zfZlzQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@sentry-internal/browser-utils": "8.55.2",
|
||||
"@sentry-internal/feedback": "8.55.2",
|
||||
"@sentry-internal/replay": "8.55.2",
|
||||
"@sentry-internal/replay-canvas": "8.55.2",
|
||||
"@sentry/core": "8.55.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.18"
|
||||
}
|
||||
},
|
||||
"node_modules/@sentry/core": {
|
||||
"version": "8.55.2",
|
||||
"resolved": "https://registry.npmjs.org/@sentry/core/-/core-8.55.2.tgz",
|
||||
"integrity": "sha512-YlEBwybUcOQ/KjMHDmof1vwweVnBtBxYlQp7DE3fOdtW4pqqdHWTnTntQs4VgYfxzjJYgtkd9LHlGtg8qy+JVQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.18"
|
||||
}
|
||||
},
|
||||
"node_modules/@sindresorhus/merge-streams": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz",
|
||||
@@ -3833,6 +3963,12 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/events": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.3.tgz",
|
||||
"integrity": "sha512-trOc4AAUThEz9hapPtSd7wf5tiQKvTtu5b371UxXdTuqzIh0ArcRspRP0i0Viu+LXstIQ1z96t1nsPxT9ol01g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/json-schema": {
|
||||
"version": "7.0.15",
|
||||
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
|
||||
@@ -4977,6 +5113,12 @@
|
||||
"url": "https://opencollective.com/express"
|
||||
}
|
||||
},
|
||||
"node_modules/bowser": {
|
||||
"version": "2.14.1",
|
||||
"resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz",
|
||||
"integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "1.1.15",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.15.tgz",
|
||||
@@ -5245,6 +5387,20 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/clone-deep": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz",
|
||||
"integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-plain-object": "^2.0.4",
|
||||
"kind-of": "^6.0.2",
|
||||
"shallow-clone": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/clsx": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
|
||||
@@ -5723,6 +5879,15 @@
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/dequal": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
|
||||
"integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/detect-libc": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
|
||||
@@ -6507,6 +6672,15 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/events": {
|
||||
"version": "3.3.0",
|
||||
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
|
||||
"integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.8.x"
|
||||
}
|
||||
},
|
||||
"node_modules/eventsource": {
|
||||
"version": "3.0.7",
|
||||
"resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz",
|
||||
@@ -7786,6 +7960,18 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/is-plain-object": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
|
||||
"integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"isobject": "^3.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/is-promise": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz",
|
||||
@@ -8001,6 +8187,15 @@
|
||||
"integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/isobject": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
|
||||
"integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/iterator.prototype": {
|
||||
"version": "1.1.5",
|
||||
"resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.5.tgz",
|
||||
@@ -8161,6 +8356,15 @@
|
||||
"json-buffer": "3.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/kind-of": {
|
||||
"version": "6.0.3",
|
||||
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
|
||||
"integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/kleur": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz",
|
||||
@@ -8487,6 +8691,12 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/lodash": {
|
||||
"version": "4.18.1",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz",
|
||||
"integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.merge": {
|
||||
"version": "4.6.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
|
||||
@@ -10028,6 +10238,16 @@
|
||||
"queue-microtask": "^1.2.2"
|
||||
}
|
||||
},
|
||||
"node_modules/rxjs": {
|
||||
"version": "7.8.2",
|
||||
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz",
|
||||
"integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/safe-array-concat": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.4.tgz",
|
||||
@@ -10306,6 +10526,18 @@
|
||||
"url": "https://github.com/sponsors/colinhacks"
|
||||
}
|
||||
},
|
||||
"node_modules/shallow-clone": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz",
|
||||
"integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"kind-of": "^6.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/sharp": {
|
||||
"version": "0.34.5",
|
||||
"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz",
|
||||
@@ -11165,6 +11397,15 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/typed-emitter": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz",
|
||||
"integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==",
|
||||
"license": "MIT",
|
||||
"optionalDependencies": {
|
||||
"rxjs": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
@@ -11403,6 +11644,19 @@
|
||||
"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/uuid": {
|
||||
"version": "11.1.1",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz",
|
||||
"integrity": "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==",
|
||||
"funding": [
|
||||
"https://github.com/sponsors/broofa",
|
||||
"https://github.com/sponsors/ctavan"
|
||||
],
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"uuid": "dist/esm/bin/uuid"
|
||||
}
|
||||
},
|
||||
"node_modules/validate-npm-package-name": {
|
||||
"version": "7.0.2",
|
||||
"resolved": "https://registry.npmjs.org/validate-npm-package-name/-/validate-npm-package-name-7.0.2.tgz",
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
"lint": "eslint"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pipecat-ai/client-js": "^1.12.0",
|
||||
"@pipecat-ai/small-webrtc-transport": "^1.10.5",
|
||||
"@xyflow/react": "^12.11.0",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
|
||||
@@ -1,52 +1,22 @@
|
||||
"use client";
|
||||
|
||||
/**
|
||||
* 语音预览:把麦克风接到后端 /ws/voice(WebRTC 信令),听到助手实时回应。
|
||||
* 语音预览。
|
||||
*
|
||||
* 走原生 RTCPeerConnection + 一条 ws 信令通道,与后端 voice_webrtc.py 的约定对齐:
|
||||
* client → {type:"offer", payload:{pc_id, sdp, type, assistant_id}}
|
||||
* server → {type:"answer", payload:{pc_id, sdp, type}}
|
||||
* client → {type:"ice-candidate", payload:{pc_id, candidate:{...}}}
|
||||
* 音频本身走 WebRTC 媒体流(Opus),不经 ws;后端 TTS 帧从 ontrack 拿到直接播放。
|
||||
*
|
||||
* 另开一条 data channel 与后端管线(pipeline.py)互通应用消息:
|
||||
* client → {type:"user-text", text} 文字输入(打断并触发新回复)
|
||||
* server → {type:"transcript", role, content, timestamp} 用户/助手最终转写(聊天记录)
|
||||
*
|
||||
* 纯本机(localhost)即可跑:localhost 是 secure context,麦克风可用,ws 用明文。
|
||||
* 局域网/别的设备要 https+wss,见 deploy/README.md。
|
||||
* WebRTC 生命周期交给 Pipecat 官方 SmallWebRTCTransport：它负责固定的
|
||||
* audio/video transceiver、trackStatus、keepalive、重协商和 ICE 恢复。
|
||||
* 本文件只把平台已有的业务消息映射到 React 状态。
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import type { PipecatClientOptions, RTVIMessage } from "@pipecat-ai/client-js";
|
||||
import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport";
|
||||
|
||||
import { API_BASE, webrtcApi } from "@/lib/api";
|
||||
|
||||
export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";
|
||||
export type NetworkQuality = "unknown" | "good" | "fair" | "poor";
|
||||
|
||||
const NETWORK_SAMPLE_INTERVAL_MS = 2_000;
|
||||
const VIDEO_BITRATE_BY_QUALITY: Record<
|
||||
Exclude<NetworkQuality, "unknown">,
|
||||
number
|
||||
> = {
|
||||
good: 600_000,
|
||||
fair: 500_000,
|
||||
poor: 300_000,
|
||||
};
|
||||
const INITIAL_VIDEO_BITRATE = VIDEO_BITRATE_BY_QUALITY.fair;
|
||||
|
||||
type NetworkStatsSample = {
|
||||
packetsSent: number;
|
||||
packetsLost: number;
|
||||
};
|
||||
|
||||
type VideoNetworkMetrics = {
|
||||
packetsSent: number | null;
|
||||
packetsLost: number | null;
|
||||
roundTripTime: number | null;
|
||||
availableOutgoingBitrate: number | null;
|
||||
};
|
||||
|
||||
type ConnectOptions = {
|
||||
visionEnabled?: boolean;
|
||||
videoStream?: MediaStream | null;
|
||||
@@ -57,29 +27,33 @@ export type ChatMessage = {
|
||||
id: string;
|
||||
role: "user" | "assistant";
|
||||
content: string;
|
||||
/** 后端给的 ISO 时间戳 */
|
||||
timestamp: string;
|
||||
sequence: number;
|
||||
turnId?: string;
|
||||
streaming?: boolean;
|
||||
};
|
||||
|
||||
// http→ws、https→wss,自动跟随 API 基址(同源反代时也对)
|
||||
function wsBaseUrl(): string {
|
||||
const url = new URL(API_BASE, window.location.origin);
|
||||
url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
|
||||
return url.toString().replace(/\/$/, "");
|
||||
type AppMessage = Record<string, unknown> & { type?: string };
|
||||
|
||||
class AppSmallWebRTCTransport extends SmallWebRTCTransport {
|
||||
onAppMessage?: (message: AppMessage) => void;
|
||||
|
||||
override handleMessage(raw: string): void {
|
||||
try {
|
||||
const message = JSON.parse(raw) as AppMessage;
|
||||
if (message.type === "signalling" || message.label === "rtvi-ai") {
|
||||
super.handleMessage(raw);
|
||||
} else {
|
||||
this.onAppMessage?.(message);
|
||||
}
|
||||
} catch {
|
||||
super.handleMessage(raw);
|
||||
}
|
||||
}
|
||||
|
||||
function generatePcId(): string {
|
||||
const bytes = new Uint8Array(16);
|
||||
crypto.getRandomValues(bytes);
|
||||
return (
|
||||
"PC-" +
|
||||
Array.from(bytes)
|
||||
.map((b) => b.toString(16).padStart(2, "0"))
|
||||
.join("")
|
||||
);
|
||||
sendAppMessage(message: AppMessage): void {
|
||||
super.sendMessage(message as unknown as RTVIMessage);
|
||||
}
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown, fallback: string): string {
|
||||
@@ -98,110 +72,13 @@ function sortMessages(messages: ChatMessage[]): ChatMessage[] {
|
||||
);
|
||||
}
|
||||
|
||||
function microphoneErrorMessage(error: unknown): string {
|
||||
if (error instanceof DOMException) {
|
||||
if (error.name === "NotAllowedError") {
|
||||
return "麦克风权限被拒绝，请在浏览器网站设置中允许 localhost 使用麦克风。";
|
||||
}
|
||||
if (error.name === "NotFoundError") {
|
||||
return "未检测到可用麦克风，请连接或启用麦克风后重试。";
|
||||
}
|
||||
if (error.name === "NotReadableError") {
|
||||
return "麦克风正被其他应用占用，或系统未允许浏览器访问麦克风。";
|
||||
}
|
||||
}
|
||||
return errorMessage(error, "无法访问麦克风。");
|
||||
}
|
||||
|
||||
async function setVideoBitrate(
|
||||
sender: RTCRtpSender,
|
||||
bitrate: number,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const parameters = sender.getParameters();
|
||||
if (!parameters.encodings.length) parameters.encodings = [{}];
|
||||
parameters.encodings[0].maxBitrate = bitrate;
|
||||
parameters.degradationPreference = "maintain-resolution";
|
||||
await sender.setParameters(parameters);
|
||||
return true;
|
||||
} catch {
|
||||
// 不支持发送参数的浏览器继续使用 WebRTC 自带的拥塞控制。
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readVideoNetworkMetrics(report: RTCStatsReport): VideoNetworkMetrics {
|
||||
const metrics: VideoNetworkMetrics = {
|
||||
packetsSent: null,
|
||||
packetsLost: null,
|
||||
roundTripTime: null,
|
||||
availableOutgoingBitrate: null,
|
||||
};
|
||||
|
||||
report.forEach((stat) => {
|
||||
const values = stat as RTCStats & Record<string, unknown>;
|
||||
const kind = values.kind ?? values.mediaType;
|
||||
if (values.type === "outbound-rtp" && kind === "video") {
|
||||
if (typeof values.packetsSent === "number") {
|
||||
metrics.packetsSent = values.packetsSent;
|
||||
}
|
||||
} else if (values.type === "remote-inbound-rtp" && kind === "video") {
|
||||
if (typeof values.packetsLost === "number") {
|
||||
metrics.packetsLost = values.packetsLost;
|
||||
}
|
||||
if (typeof values.roundTripTime === "number") {
|
||||
metrics.roundTripTime = values.roundTripTime;
|
||||
}
|
||||
} else if (
|
||||
values.type === "candidate-pair" &&
|
||||
values.state === "succeeded" &&
|
||||
(values.nominated === true || values.selected === true)
|
||||
) {
|
||||
if (typeof values.currentRoundTripTime === "number") {
|
||||
metrics.roundTripTime = values.currentRoundTripTime;
|
||||
}
|
||||
if (typeof values.availableOutgoingBitrate === "number") {
|
||||
metrics.availableOutgoingBitrate = values.availableOutgoingBitrate;
|
||||
}
|
||||
}
|
||||
function encodeHeaderJson(value: unknown): string {
|
||||
const bytes = new TextEncoder().encode(JSON.stringify(value));
|
||||
let binary = "";
|
||||
bytes.forEach((byte) => {
|
||||
binary += String.fromCharCode(byte);
|
||||
});
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
function classifyNetworkQuality(
|
||||
metrics: VideoNetworkMetrics,
|
||||
previous: NetworkStatsSample | null,
|
||||
): NetworkQuality {
|
||||
const hasPacketSample =
|
||||
previous !== null &&
|
||||
metrics.packetsSent !== null &&
|
||||
metrics.packetsLost !== null;
|
||||
const sentDelta = hasPacketSample
|
||||
? Math.max(0, metrics.packetsSent! - previous.packetsSent)
|
||||
: 0;
|
||||
const lostDelta = hasPacketSample
|
||||
? Math.max(0, metrics.packetsLost! - previous.packetsLost)
|
||||
: 0;
|
||||
const lossRate = sentDelta > 0 ? lostDelta / sentDelta : null;
|
||||
const { roundTripTime: rtt, availableOutgoingBitrate: bandwidth } = metrics;
|
||||
|
||||
if (lossRate === null && rtt === null && bandwidth === null) return "unknown";
|
||||
if (
|
||||
(lossRate !== null && lossRate >= 0.08) ||
|
||||
(rtt !== null && rtt >= 0.45) ||
|
||||
(bandwidth !== null && bandwidth < 350_000)
|
||||
) {
|
||||
return "poor";
|
||||
}
|
||||
if (
|
||||
(lossRate !== null && lossRate >= 0.03) ||
|
||||
(rtt !== null && rtt >= 0.25) ||
|
||||
(bandwidth !== null && bandwidth < 550_000)
|
||||
) {
|
||||
return "fair";
|
||||
}
|
||||
return "good";
|
||||
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
||||
}
|
||||
|
||||
export function useVoicePreview(
|
||||
@@ -212,36 +89,25 @@ export function useVoicePreview(
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [micWarning, setMicWarning] = useState<string | null>(null);
|
||||
const [localStream, setLocalStream] = useState<MediaStream | null>(null);
|
||||
// 远端(助手 TTS)媒体流:除挂到 <audio> 播放外,也暴露给波形可视化
|
||||
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||
// 后端结束节点/EndFrame 触发的正常结束。与 failed 区分，供独立通话页
|
||||
// 立即收起媒体画面并展示重新开始入口。
|
||||
const [callEnded, setCallEnded] = useState(false);
|
||||
const [networkQuality, setNetworkQuality] =
|
||||
useState<NetworkQuality>("unknown");
|
||||
// 可选麦克风/扬声器列表与当前选择(空串表示交给浏览器选默认设备)
|
||||
const [audioInputs, setAudioInputs] = useState<MediaDeviceInfo[]>([]);
|
||||
const [audioOutputs, setAudioOutputs] = useState<MediaDeviceInfo[]>([]);
|
||||
const [selectedDeviceId, setSelectedDeviceId] = useState<string>("");
|
||||
const [selectedOutputDeviceId, setSelectedOutputDeviceId] =
|
||||
useState<string>("");
|
||||
const [selectedDeviceId, setSelectedDeviceId] = useState("");
|
||||
const [selectedOutputDeviceId, setSelectedOutputDeviceId] = useState("");
|
||||
|
||||
const transportRef = useRef<AppSmallWebRTCTransport | null>(null);
|
||||
const startingRef = useRef(false);
|
||||
const messageSeqRef = useRef(0);
|
||||
const endedByServerRef = useRef(false);
|
||||
const selectedDeviceIdRef = useRef("");
|
||||
const selectedOutputDeviceIdRef = useRef("");
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const pcRef = useRef<RTCPeerConnection | null>(null);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const dataChannelRef = useRef<RTCDataChannel | null>(null);
|
||||
const localStreamRef = useRef<MediaStream | null>(null);
|
||||
const startingRef = useRef(false);
|
||||
const negotiationInProgressRef = useRef(false);
|
||||
const messageSeqRef = useRef(0);
|
||||
const networkStatsRef = useRef<NetworkStatsSample | null>(null);
|
||||
const videoBitrateRef = useRef(INITIAL_VIDEO_BITRATE);
|
||||
// 后端主动结束(工作流走到结束节点)标记:据此把随后的断开当作正常结束而非报错
|
||||
const endedByServerRef = useRef(false);
|
||||
// 工作流激活节点回调存进 ref,避免把它挂进 connect 依赖反复重建连接
|
||||
const onNodeActiveRef = useRef(onNodeActive);
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
onNodeActiveRef.current = onNodeActive;
|
||||
}, [onNodeActive]);
|
||||
@@ -256,97 +122,39 @@ export function useVoicePreview(
|
||||
try {
|
||||
await audio.setSinkId(deviceId);
|
||||
} catch {
|
||||
/* 忽略切换失败(设备不可用或不支持) */
|
||||
// 设备可能已拔出，交给浏览器回退默认输出。
|
||||
}
|
||||
}, [supportsOutputSelection]);
|
||||
|
||||
// 枚举可用麦克风与扬声器。未授权前 label 为空,授权(连接)后再刷新即可拿到名称。
|
||||
const refreshDevices = useCallback(async () => {
|
||||
if (!navigator.mediaDevices?.enumerateDevices) return;
|
||||
try {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||
// 未授权时设备可能没有 deviceId(空串),无法选择,直接过滤掉
|
||||
const inputs = devices.filter((d) => d.kind === "audioinput" && d.deviceId);
|
||||
const outputs = devices.filter(
|
||||
(d) => d.kind === "audiooutput" && d.deviceId,
|
||||
setAudioInputs(
|
||||
devices.filter((item) => item.kind === "audioinput" && item.deviceId),
|
||||
);
|
||||
setAudioOutputs(
|
||||
devices.filter((item) => item.kind === "audiooutput" && item.deviceId),
|
||||
);
|
||||
setAudioInputs(inputs);
|
||||
setAudioOutputs(outputs);
|
||||
// 选中的设备已被拔出时,回退到浏览器默认设备
|
||||
if (
|
||||
selectedDeviceIdRef.current &&
|
||||
!inputs.some((d) => d.deviceId === selectedDeviceIdRef.current)
|
||||
) {
|
||||
setSelectedDeviceId("");
|
||||
}
|
||||
if (
|
||||
selectedOutputDeviceIdRef.current &&
|
||||
!outputs.some((d) => d.deviceId === selectedOutputDeviceIdRef.current)
|
||||
) {
|
||||
setSelectedOutputDeviceId("");
|
||||
void applyOutputDevice("");
|
||||
}
|
||||
} catch {
|
||||
/* 忽略枚举失败 */
|
||||
// 设备枚举失败不影响仅接收模式。
|
||||
}
|
||||
}, [applyOutputDevice]);
|
||||
|
||||
// connect/refreshDevices 在回调里读最新选择,避免把它们挂进依赖反复重建
|
||||
useEffect(() => {
|
||||
selectedDeviceIdRef.current = selectedDeviceId;
|
||||
}, [selectedDeviceId]);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
selectedOutputDeviceIdRef.current = selectedOutputDeviceId;
|
||||
}, [selectedOutputDeviceId]);
|
||||
|
||||
useEffect(() => {
|
||||
void applyOutputDevice(selectedOutputDeviceId);
|
||||
}, [applyOutputDevice, selectedOutputDeviceId, remoteStream]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!navigator.mediaDevices?.enumerateDevices) return;
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
void refreshDevices();
|
||||
navigator.mediaDevices.addEventListener("devicechange", refreshDevices);
|
||||
return () => {
|
||||
navigator.mediaDevices.removeEventListener("devicechange", refreshDevices);
|
||||
};
|
||||
navigator.mediaDevices?.addEventListener("devicechange", refreshDevices);
|
||||
return () =>
|
||||
navigator.mediaDevices?.removeEventListener("devicechange", refreshDevices);
|
||||
}, [refreshDevices]);
|
||||
|
||||
const releaseResources = useCallback(() => {
|
||||
const ws = wsRef.current;
|
||||
wsRef.current = null;
|
||||
if (ws) {
|
||||
ws.onclose = null;
|
||||
ws.onerror = null;
|
||||
ws.onmessage = null;
|
||||
ws.close();
|
||||
}
|
||||
|
||||
const channel = dataChannelRef.current;
|
||||
dataChannelRef.current = null;
|
||||
if (channel) {
|
||||
channel.onopen = null;
|
||||
channel.onmessage = null;
|
||||
channel.close();
|
||||
}
|
||||
|
||||
const pc = pcRef.current;
|
||||
pcRef.current = null;
|
||||
if (pc) {
|
||||
pc.onconnectionstatechange = null;
|
||||
pc.onicecandidate = null;
|
||||
pc.oniceconnectionstatechange = null;
|
||||
pc.ontrack = null;
|
||||
pc.close();
|
||||
}
|
||||
|
||||
localStreamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
localStreamRef.current = null;
|
||||
const transport = transportRef.current;
|
||||
transportRef.current = null;
|
||||
transport?.disconnect().catch(() => {});
|
||||
if (audioRef.current) audioRef.current.srcObject = null;
|
||||
startingRef.current = false;
|
||||
negotiationInProgressRef.current = false;
|
||||
onNodeActiveRef.current?.(null);
|
||||
}, []);
|
||||
|
||||
@@ -359,217 +167,29 @@ export function useVoicePreview(
|
||||
setError(null);
|
||||
setMicWarning(null);
|
||||
setNetworkQuality("unknown");
|
||||
networkStatsRef.current = null;
|
||||
videoBitrateRef.current = INITIAL_VIDEO_BITRATE;
|
||||
setStatus("idle");
|
||||
}, [releaseResources]);
|
||||
|
||||
const fail = useCallback(
|
||||
(message: string) => {
|
||||
const fail = useCallback((message: string) => {
|
||||
releaseResources();
|
||||
setLocalStream(null);
|
||||
setRemoteStream(null);
|
||||
setError(message);
|
||||
setNetworkQuality("unknown");
|
||||
networkStatsRef.current = null;
|
||||
videoBitrateRef.current = INITIAL_VIDEO_BITRATE;
|
||||
setStatus("failed");
|
||||
},
|
||||
[releaseResources],
|
||||
);
|
||||
}, [releaseResources]);
|
||||
|
||||
// 连接断开时:若是后端主动收尾(call-ended),按正常结束处理(不报错);
|
||||
// 否则按异常失败处理。
|
||||
const closeOnRemoteEnd = useCallback(
|
||||
(message: string) => {
|
||||
if (endedByServerRef.current) {
|
||||
disconnect();
|
||||
} else {
|
||||
fail(message);
|
||||
}
|
||||
},
|
||||
[disconnect, fail],
|
||||
);
|
||||
|
||||
const connect = useCallback(async (options: ConnectOptions = {}) => {
|
||||
if (startingRef.current || pcRef.current || wsRef.current) return;
|
||||
if (!assistantId) {
|
||||
setError("请先保存助手，再开始语音预览。");
|
||||
setStatus("failed");
|
||||
return;
|
||||
}
|
||||
startingRef.current = true;
|
||||
setError(null);
|
||||
setMicWarning(null);
|
||||
setMessages([]); // 新会话清空上一轮聊天记录
|
||||
setCallEnded(false);
|
||||
endedByServerRef.current = false;
|
||||
setStatus("connecting");
|
||||
|
||||
// 麦克风是可选的:获取失败时继续建立仅接收后端音频的 WebRTC 会话。
|
||||
let stream: MediaStream | null = null;
|
||||
if (window.isSecureContext && navigator.mediaDevices?.getUserMedia) {
|
||||
try {
|
||||
const audioConstraints: MediaTrackConstraints = {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
};
|
||||
// 指定了具体设备就强制用它,否则交给浏览器选默认麦克风
|
||||
if (selectedDeviceIdRef.current) {
|
||||
audioConstraints.deviceId = { exact: selectedDeviceIdRef.current };
|
||||
}
|
||||
stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: audioConstraints,
|
||||
});
|
||||
// 授权后 label 才可见,刷新列表把真实设备名补上
|
||||
void refreshDevices();
|
||||
} catch (mediaError) {
|
||||
setMicWarning(microphoneErrorMessage(mediaError));
|
||||
}
|
||||
} else {
|
||||
setMicWarning("当前页面无法访问麦克风，已进入仅收听模式。");
|
||||
}
|
||||
if (stream) {
|
||||
localStreamRef.current = stream;
|
||||
setLocalStream(stream);
|
||||
}
|
||||
|
||||
const pcId = generatePcId();
|
||||
const ws = new WebSocket(`${wsBaseUrl()}/ws/voice`);
|
||||
wsRef.current = ws;
|
||||
|
||||
ws.onmessage = async (event) => {
|
||||
try {
|
||||
const msg = JSON.parse(event.data);
|
||||
if (msg.type === "answer") {
|
||||
const currentPc = pcRef.current;
|
||||
if (!currentPc) return;
|
||||
try {
|
||||
await currentPc.setRemoteDescription({
|
||||
type: "answer",
|
||||
sdp: msg.payload.sdp,
|
||||
});
|
||||
} finally {
|
||||
if (pcRef.current === currentPc) {
|
||||
negotiationInProgressRef.current = false;
|
||||
}
|
||||
}
|
||||
} else if (msg.type === "ice-candidate" && msg.payload?.candidate) {
|
||||
// 后端当前不主动 trickle,留兼容
|
||||
try {
|
||||
await pcRef.current?.addIceCandidate(msg.payload.candidate);
|
||||
} catch {
|
||||
/* 忽略迟到/重复 candidate */
|
||||
}
|
||||
} else if (msg.type === "error") {
|
||||
fail(msg.payload?.message || "后端无法启动语音会话。");
|
||||
}
|
||||
} catch {
|
||||
/* 非 JSON / 未知消息,忽略 */
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
// 1) 等 ws 连上
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
ws.onopen = () => resolve();
|
||||
ws.onerror = (e) => reject(e);
|
||||
ws.onclose = () => reject(new Error("语音信令连接已关闭。"));
|
||||
});
|
||||
// 连上后,信令异常或关闭都结束当前会话并保留失败状态。
|
||||
ws.onerror = () => {
|
||||
if (wsRef.current === ws) fail("语音信令连接失败。");
|
||||
};
|
||||
ws.onclose = () => {
|
||||
if (wsRef.current === ws) closeOnRemoteEnd("语音信令连接已断开。");
|
||||
};
|
||||
|
||||
// 2) 建 PeerConnection(STUN;公网跨网时后端会下发 TURN)
|
||||
const iceServers = await webrtcApi
|
||||
.iceServers()
|
||||
.then((r) => r.iceServers)
|
||||
.catch(() => [{ urls: "stun:stun.l.google.com:19302" }]);
|
||||
const pc = new RTCPeerConnection({ iceServers });
|
||||
pcRef.current = pc;
|
||||
|
||||
const sendOffer = async () => {
|
||||
const handleAppMessage = useCallback((msg: AppMessage) => {
|
||||
if (
|
||||
pcRef.current !== pc ||
|
||||
ws.readyState !== WebSocket.OPEN ||
|
||||
pc.signalingState !== "stable" ||
|
||||
negotiationInProgressRef.current
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
negotiationInProgressRef.current = true;
|
||||
try {
|
||||
const offer = await pc.createOffer();
|
||||
await pc.setLocalDescription(offer);
|
||||
const localDescription = pc.localDescription;
|
||||
if (!localDescription?.sdp) {
|
||||
throw new Error("浏览器无法创建 WebRTC offer。");
|
||||
}
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "offer",
|
||||
payload: {
|
||||
pc_id: pcId,
|
||||
sdp: localDescription.sdp,
|
||||
type: localDescription.type,
|
||||
assistant_id: assistantId,
|
||||
vision_enabled: Boolean(options.visionEnabled),
|
||||
dynamic_variables: options.dynamicVariables ?? {},
|
||||
},
|
||||
}),
|
||||
);
|
||||
} catch (offerError) {
|
||||
negotiationInProgressRef.current = false;
|
||||
throw offerError;
|
||||
}
|
||||
};
|
||||
|
||||
pc.onicecandidate = (e) => {
|
||||
if (ws.readyState !== WebSocket.OPEN) return;
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "ice-candidate",
|
||||
payload: {
|
||||
pc_id: pcId,
|
||||
candidate: e.candidate
|
||||
? {
|
||||
candidate: e.candidate.candidate,
|
||||
sdpMid: e.candidate.sdpMid,
|
||||
sdpMLineIndex: e.candidate.sdpMLineIndex,
|
||||
}
|
||||
: null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
// 应用消息通道:收后端转写(聊天记录),发文字输入。
|
||||
const channel = pc.createDataChannel("chat");
|
||||
dataChannelRef.current = channel;
|
||||
channel.onopen = () => {
|
||||
channel.send(JSON.stringify({ type: "client-ready" }));
|
||||
};
|
||||
channel.onmessage = async (event) => {
|
||||
try {
|
||||
const msg = JSON.parse(event.data);
|
||||
if (
|
||||
msg?.type === "signalling" &&
|
||||
msg?.message?.type === "renegotiate"
|
||||
) {
|
||||
await sendOffer();
|
||||
} else if (
|
||||
msg?.type === "assistant-text-start" &&
|
||||
msg.type === "assistant-text-start" &&
|
||||
typeof msg.turn_id === "string"
|
||||
) {
|
||||
messageSeqRef.current += 1;
|
||||
const next: ChatMessage = {
|
||||
id: `assistant-${msg.turn_id}`,
|
||||
setMessages((previous) =>
|
||||
sortMessages([
|
||||
...previous,
|
||||
{
|
||||
id: `assistant-${msg.turn_id as string}`,
|
||||
role: "assistant",
|
||||
content: "",
|
||||
timestamp:
|
||||
@@ -577,122 +197,166 @@ export function useVoicePreview(
|
||||
? msg.timestamp
|
||||
: new Date().toISOString(),
|
||||
sequence: messageSeqRef.current,
|
||||
turnId: msg.turn_id,
|
||||
turnId: msg.turn_id as string,
|
||||
streaming: true,
|
||||
};
|
||||
setMessages((prev) => sortMessages([...prev, next]));
|
||||
},
|
||||
]),
|
||||
);
|
||||
} else if (
|
||||
msg?.type === "assistant-text-delta" &&
|
||||
msg.type === "assistant-text-delta" &&
|
||||
typeof msg.turn_id === "string" &&
|
||||
typeof msg.delta === "string"
|
||||
) {
|
||||
setMessages((prev) =>
|
||||
prev.map((message) =>
|
||||
setMessages((previous) =>
|
||||
previous.map((message) =>
|
||||
message.turnId === msg.turn_id
|
||||
? { ...message, content: message.content + msg.delta }
|
||||
: message,
|
||||
),
|
||||
);
|
||||
} else if (
|
||||
msg?.type === "assistant-text-end" &&
|
||||
msg.type === "assistant-text-end" &&
|
||||
typeof msg.turn_id === "string"
|
||||
) {
|
||||
setMessages((prev) =>
|
||||
prev.map((message) =>
|
||||
setMessages((previous) =>
|
||||
previous.map((message) =>
|
||||
message.turnId === msg.turn_id
|
||||
? {
|
||||
...message,
|
||||
content:
|
||||
typeof msg.content === "string"
|
||||
? msg.content
|
||||
: message.content,
|
||||
typeof msg.content === "string" ? msg.content : message.content,
|
||||
streaming: false,
|
||||
}
|
||||
: message,
|
||||
),
|
||||
);
|
||||
} else if (
|
||||
msg?.type === "transcript" &&
|
||||
msg.type === "transcript" &&
|
||||
(msg.role === "user" || msg.role === "assistant") &&
|
||||
typeof msg.content === "string" &&
|
||||
msg.content.trim()
|
||||
) {
|
||||
messageSeqRef.current += 1;
|
||||
const next: ChatMessage = {
|
||||
setMessages((previous) =>
|
||||
sortMessages([
|
||||
...previous,
|
||||
{
|
||||
id: `msg-${messageSeqRef.current}`,
|
||||
role: msg.role,
|
||||
content: msg.content,
|
||||
role: msg.role as "user" | "assistant",
|
||||
content: msg.content as string,
|
||||
timestamp:
|
||||
typeof msg.timestamp === "string"
|
||||
? msg.timestamp
|
||||
: new Date().toISOString(),
|
||||
sequence: messageSeqRef.current,
|
||||
};
|
||||
setMessages((prev) => sortMessages([...prev, next]));
|
||||
} else if (
|
||||
msg?.type === "node-active" &&
|
||||
typeof msg.nodeId === "string"
|
||||
) {
|
||||
// 工作流:后端报告当前激活节点,交给画布高亮
|
||||
},
|
||||
]),
|
||||
);
|
||||
} else if (msg.type === "node-active" && typeof msg.nodeId === "string") {
|
||||
onNodeActiveRef.current?.(msg.nodeId);
|
||||
} else if (msg?.type === "call-ended") {
|
||||
// 后端已播完结束节点的收尾语，即将排入 EndFrame。此时主动释放
|
||||
// 浏览器侧资源，不依赖后续 WebRTC 状态事件是否及时到达。
|
||||
} else if (msg.type === "call-ended") {
|
||||
endedByServerRef.current = true;
|
||||
setCallEnded(true);
|
||||
onNodeActiveRef.current?.(null);
|
||||
disconnect();
|
||||
}
|
||||
} catch {
|
||||
/* 非 JSON / 未知消息,忽略 */
|
||||
}
|
||||
};
|
||||
}, [disconnect]);
|
||||
|
||||
pc.ontrack = (e) => {
|
||||
if (e.track.kind !== "audio") return;
|
||||
const remote = e.streams[0] ?? new MediaStream([e.track]);
|
||||
setRemoteStream(remote);
|
||||
const connect = useCallback(async (options: ConnectOptions = {}) => {
|
||||
if (startingRef.current || transportRef.current) return;
|
||||
if (!assistantId) {
|
||||
setError("请先保存助手，再开始语音预览。");
|
||||
setStatus("failed");
|
||||
return;
|
||||
}
|
||||
|
||||
startingRef.current = true;
|
||||
setStatus("connecting");
|
||||
setError(null);
|
||||
setMicWarning(null);
|
||||
setMessages([]);
|
||||
setCallEnded(false);
|
||||
endedByServerRef.current = false;
|
||||
|
||||
const iceServers = await webrtcApi
|
||||
.iceServers()
|
||||
.then((response) => response.iceServers)
|
||||
.catch(() => [{ urls: "stun:stun.l.google.com:19302" }]);
|
||||
const transport = new AppSmallWebRTCTransport({ iceServers });
|
||||
transportRef.current = transport;
|
||||
transport.onAppMessage = handleAppMessage;
|
||||
|
||||
const callbacks: NonNullable<PipecatClientOptions["callbacks"]> = {
|
||||
onConnected: () => setStatus("connected"),
|
||||
onDisconnected: () => {
|
||||
if (transportRef.current !== transport) return;
|
||||
if (endedByServerRef.current) disconnect();
|
||||
else fail("WebRTC 连接已断开。");
|
||||
},
|
||||
onTrackStarted: (track) => {
|
||||
if (track.kind !== "audio") return;
|
||||
const stream = new MediaStream([track]);
|
||||
setRemoteStream(stream);
|
||||
if (audioRef.current) {
|
||||
audioRef.current.srcObject = remote;
|
||||
audioRef.current.srcObject = stream;
|
||||
void applyOutputDevice(selectedOutputDeviceIdRef.current).then(() =>
|
||||
audioRef.current?.play().catch(() => {}),
|
||||
);
|
||||
}
|
||||
},
|
||||
onDeviceError: (deviceError) => {
|
||||
setMicWarning(deviceError.message || "无法访问麦克风。已尝试继续连接。");
|
||||
},
|
||||
onTransportStateChanged: (nextState) => {
|
||||
if (nextState === "connected" || nextState === "ready") {
|
||||
setStatus("connected");
|
||||
} else if (nextState === "error") {
|
||||
fail("WebRTC 连接失败。");
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
pc.onconnectionstatechange = () => {
|
||||
if (pcRef.current !== pc) return;
|
||||
if (pc.connectionState === "connected") setStatus("connected");
|
||||
else if (pc.connectionState === "failed")
|
||||
closeOnRemoteEnd("WebRTC 音频连接失败。");
|
||||
else if (pc.connectionState === "closed")
|
||||
closeOnRemoteEnd("WebRTC 音频连接已断开。");
|
||||
};
|
||||
transport.initialize(
|
||||
{
|
||||
transport,
|
||||
enableMic: true,
|
||||
enableCam: Boolean(options.visionEnabled),
|
||||
callbacks,
|
||||
},
|
||||
() => {},
|
||||
);
|
||||
|
||||
pc.oniceconnectionstatechange = () => {
|
||||
if (pcRef.current !== pc) return;
|
||||
const st = pc.iceConnectionState;
|
||||
if (st === "connected" || st === "completed") setStatus("connected");
|
||||
else if (st === "failed") closeOnRemoteEnd("WebRTC 音频连接失败。");
|
||||
else if (st === "disconnected")
|
||||
closeOnRemoteEnd("WebRTC 音频连接已断开。");
|
||||
};
|
||||
try {
|
||||
await transport.initDevices();
|
||||
if (selectedDeviceIdRef.current) {
|
||||
await transport.updateMic(selectedDeviceIdRef.current);
|
||||
}
|
||||
const localAudio = transport.tracks().local.audio;
|
||||
setLocalStream(localAudio ? new MediaStream([localAudio]) : null);
|
||||
void refreshDevices();
|
||||
|
||||
// 3) 有麦克风时双向音频；否则明确声明只接收后端音频。
|
||||
if (stream) {
|
||||
stream.getTracks().forEach((track) => pc.addTrack(track, stream));
|
||||
} else {
|
||||
pc.addTransceiver("audio", { direction: "recvonly" });
|
||||
}
|
||||
if (options.videoStream) {
|
||||
for (const track of options.videoStream.getVideoTracks()) {
|
||||
const sender = pc.addTrack(track, options.videoStream);
|
||||
await setVideoBitrate(sender, INITIAL_VIDEO_BITRATE);
|
||||
}
|
||||
}
|
||||
|
||||
// 4) 生成 offer 并发给后端；同一方法也响应 SmallWebRTC 的重新协商请求。
|
||||
await sendOffer();
|
||||
const request = new Request(`${API_BASE}/api/webrtc/offer`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"X-Pipecat-Assistant-ID": assistantId,
|
||||
"X-Pipecat-Dynamic-Variables": encodeHeaderJson(
|
||||
options.dynamicVariables ?? {},
|
||||
),
|
||||
},
|
||||
credentials: "include",
|
||||
});
|
||||
await transport.connect({
|
||||
webrtcRequestParams: {
|
||||
endpoint: request,
|
||||
requestData: {
|
||||
assistant_id: assistantId,
|
||||
vision_enabled: Boolean(options.visionEnabled),
|
||||
dynamic_variables: options.dynamicVariables ?? {},
|
||||
},
|
||||
},
|
||||
});
|
||||
transport.sendAppMessage({ type: "client-ready" });
|
||||
setStatus("connected");
|
||||
} catch (connectionError) {
|
||||
fail(errorMessage(connectionError, "无法连接语音服务。"));
|
||||
} finally {
|
||||
@@ -701,138 +365,47 @@ export function useVoicePreview(
|
||||
}, [
|
||||
assistantId,
|
||||
applyOutputDevice,
|
||||
fail,
|
||||
closeOnRemoteEnd,
|
||||
disconnect,
|
||||
fail,
|
||||
handleAppMessage,
|
||||
refreshDevices,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (status !== "connected") return;
|
||||
const pc = pcRef.current;
|
||||
if (!pc) return;
|
||||
const replaceVideoStream = useCallback(async (videoStream: MediaStream | null) => {
|
||||
const transport = transportRef.current;
|
||||
const deviceId = videoStream?.getVideoTracks()[0]?.getSettings().deviceId;
|
||||
if (transport && deviceId) await transport.updateCam(deviceId);
|
||||
}, []);
|
||||
|
||||
let cancelled = false;
|
||||
const sampleNetwork = async () => {
|
||||
try {
|
||||
const metrics = readVideoNetworkMetrics(await pc.getStats());
|
||||
if (cancelled || pcRef.current !== pc) return;
|
||||
const quality = classifyNetworkQuality(metrics, networkStatsRef.current);
|
||||
if (metrics.packetsSent !== null && metrics.packetsLost !== null) {
|
||||
networkStatsRef.current = {
|
||||
packetsSent: metrics.packetsSent,
|
||||
packetsLost: metrics.packetsLost,
|
||||
};
|
||||
}
|
||||
setNetworkQuality(quality);
|
||||
|
||||
if (quality === "unknown") return;
|
||||
const bitrate = VIDEO_BITRATE_BY_QUALITY[quality];
|
||||
if (bitrate === videoBitrateRef.current) return;
|
||||
const sender = pc
|
||||
.getSenders()
|
||||
.find((item) => item.track?.kind === "video");
|
||||
if (!sender) return;
|
||||
if (await setVideoBitrate(sender, bitrate)) {
|
||||
videoBitrateRef.current = bitrate;
|
||||
}
|
||||
} catch {
|
||||
// 个别浏览器不提供完整 WebRTC 统计信息时，保留底层拥塞控制。
|
||||
}
|
||||
};
|
||||
|
||||
void sampleNetwork();
|
||||
const interval = window.setInterval(
|
||||
() => void sampleNetwork(),
|
||||
NETWORK_SAMPLE_INTERVAL_MS,
|
||||
);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [status]);
|
||||
|
||||
const replaceVideoStream = useCallback(
|
||||
async (videoStream: MediaStream | null) => {
|
||||
const pc = pcRef.current;
|
||||
if (!pc) return;
|
||||
const sender = pc.getSenders().find((s) => s.track?.kind === "video");
|
||||
if (!sender) return;
|
||||
await sender.replaceTrack(videoStream?.getVideoTracks()[0] ?? null);
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
// 选择麦克风:更新选择;若会话正在发送麦克风音频,则用 WebRTC replaceTrack
|
||||
// 热切换轨道(无需重新协商),并把波形可视化重新接到新流。
|
||||
// 未连接时仅记下选择,留待下次 connect 生效。
|
||||
const selectDevice = useCallback(
|
||||
async (deviceId: string) => {
|
||||
const selectDevice = useCallback(async (deviceId: string) => {
|
||||
setSelectedDeviceId(deviceId);
|
||||
selectedDeviceIdRef.current = deviceId;
|
||||
|
||||
const pc = pcRef.current;
|
||||
if (!pc) return;
|
||||
// 只有本就在发送麦克风音频(存在 audio sender 轨道)时才热切换;
|
||||
// 仅收听模式下加麦克风需重新协商,这里不处理,留到下次连接。
|
||||
const sender = pc
|
||||
.getSenders()
|
||||
.find((s) => s.track?.kind === "audio");
|
||||
if (!sender) return;
|
||||
|
||||
const transport = transportRef.current;
|
||||
if (!transport) return;
|
||||
try {
|
||||
const audioConstraints: MediaTrackConstraints = {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
};
|
||||
if (deviceId) audioConstraints.deviceId = { exact: deviceId };
|
||||
const newStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: audioConstraints,
|
||||
});
|
||||
// 切换期间可能已断开,丢弃刚拿到的流
|
||||
if (pcRef.current !== pc) {
|
||||
newStream.getTracks().forEach((t) => t.stop());
|
||||
return;
|
||||
}
|
||||
const newTrack = newStream.getAudioTracks()[0];
|
||||
if (!newTrack) {
|
||||
newStream.getTracks().forEach((t) => t.stop());
|
||||
return;
|
||||
}
|
||||
await sender.replaceTrack(newTrack);
|
||||
// 旧轨道停掉,新流替换(波形/分析器随 localStream 变化自动重连)
|
||||
localStreamRef.current?.getTracks().forEach((t) => t.stop());
|
||||
localStreamRef.current = newStream;
|
||||
setLocalStream(newStream);
|
||||
await transport.updateMic(deviceId);
|
||||
const audio = transport.tracks().local.audio;
|
||||
setLocalStream(audio ? new MediaStream([audio]) : null);
|
||||
setMicWarning(null);
|
||||
} catch (mediaError) {
|
||||
setMicWarning(microphoneErrorMessage(mediaError));
|
||||
} catch (deviceError) {
|
||||
setMicWarning(errorMessage(deviceError, "无法切换麦克风。"));
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
}, []);
|
||||
|
||||
const selectOutputDevice = useCallback(
|
||||
(deviceId: string) => {
|
||||
const selectOutputDevice = useCallback((deviceId: string) => {
|
||||
setSelectedOutputDeviceId(deviceId);
|
||||
selectedOutputDeviceIdRef.current = deviceId;
|
||||
void applyOutputDevice(deviceId);
|
||||
},
|
||||
[applyOutputDevice],
|
||||
);
|
||||
}, [applyOutputDevice]);
|
||||
|
||||
// 发送文字消息:后端先打断当前播报,再按用户输入触发新回复。
|
||||
// 成功返回 true;通道未就绪(未开始对话/连接中)返回 false。
|
||||
const sendText = useCallback((text: string): boolean => {
|
||||
const trimmed = text.trim();
|
||||
const channel = dataChannelRef.current;
|
||||
if (!trimmed || !channel || channel.readyState !== "open") return false;
|
||||
channel.send(JSON.stringify({ type: "user-text", text: trimmed }));
|
||||
const transport = transportRef.current;
|
||||
if (!trimmed || !transport || transport.state !== "connected") return false;
|
||||
transport.sendAppMessage({ type: "user-text", text: trimmed });
|
||||
return true;
|
||||
}, []);
|
||||
|
||||
// 卸载时收尾
|
||||
useEffect(() => releaseResources, [releaseResources]);
|
||||
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user