diff --git a/backend/routes/voice_webrtc.py b/backend/routes/voice_webrtc.py index 798621e..0d89065 100644 --- a/backend/routes/voice_webrtc.py +++ b/backend/routes/voice_webrtc.py @@ -9,9 +9,11 @@ """ import asyncio +import base64 +import json from db.session import SessionLocal -from fastapi import APIRouter, Depends, WebSocket +from fastapi import APIRouter, Body, Depends, Request, WebSocket from loguru import logger from models import AssistantConfig, SignalingOffer from services.auth import require_admin, require_admin_websocket @@ -24,6 +26,7 @@ from services.webrtc_ice import aiortc_ice_servers, client_ice_servers # 注意:pipecat 是重依赖(语音才用),在 _handle_offer 等处惰性导入。 router = APIRouter(tags=["voice"]) +_http_peers: dict[str, object] = {} @router.get("/api/webrtc/ice-servers", dependencies=[Depends(require_admin)]) @@ -32,6 +35,45 @@ async def ice_servers(): return {"iceServers": client_ice_servers()} +@router.post("/api/webrtc/offer", dependencies=[Depends(require_admin)]) +async def http_offer(request: Request, payload: dict = Body(...)): + """Official SmallWebRTC JS transport offer endpoint.""" + request_data = payload.get("requestData") or {} + encoded_variables = request.headers.get("x-pipecat-dynamic-variables", "") + header_variables = {} + if encoded_variables: + padding = "=" * (-len(encoded_variables) % 4) + header_variables = json.loads( + base64.urlsafe_b64decode(encoded_variables + padding).decode("utf-8") + ) + offer_payload = { + **payload, + "assistant_id": request.headers.get("x-pipecat-assistant-id"), + "dynamic_variables": header_variables, + **request_data, + } + answer = await _handle_offer_payload(offer_payload, _http_peers) + return answer + + +@router.patch("/api/webrtc/offer", dependencies=[Depends(require_admin)]) +async def http_ice_candidates(payload: dict = Body(...)): + """Accept the batched trickle ICE format used by the official JS client.""" + pc_id = payload.get("pc_id") + pc = _http_peers.get(pc_id) if pc_id else None + if not pc: + return {"ok": False} + + from aiortc.sdp import candidate_from_sdp + + for item in payload.get("candidates") or []: + candidate = candidate_from_sdp(item.get("candidate", "")) + candidate.sdpMid = item.get("sdp_mid") + candidate.sdpMLineIndex = item.get("sdp_mline_index") + await pc.add_ice_candidate(candidate) + return {"ok": True} + + @router.websocket("/ws/voice") async def voice_signaling(websocket: WebSocket): if not await require_admin_websocket(websocket): @@ -92,6 +134,17 @@ async def _resolve_config(offer: SignalingOffer) -> AssistantConfig: async def _handle_offer(websocket, payload, peers): + answer = await _handle_offer_payload(payload, peers) + if websocket.application_state == WebSocketState.CONNECTED: + await websocket.send_json( + { + "type": "answer", + "payload": answer, + } + ) + + +async def _handle_offer_payload(payload, peers): from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection from services.pipecat.pipeline import run_pipeline from services.pipecat.transports import build_webrtc_transport @@ -99,6 +152,11 @@ async def _handle_offer(websocket, payload, peers): offer = SignalingOffer(**payload) pc_id = offer.pc_id + restart_pc = bool(payload.get("restart_pc")) + if restart_pc and pc_id and pc_id in peers: + old_pc = peers.pop(pc_id) + await old_pc.disconnect() + if pc_id and pc_id in peers: pc = peers[pc_id] await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False) @@ -144,17 +202,11 @@ async def _handle_offer(websocket, payload, peers): ) answer = pc.get_answer() - if websocket.application_state == WebSocketState.CONNECTED: - await websocket.send_json( - { - "type": "answer", - "payload": { - "pc_id": answer["pc_id"], - "sdp": answer["sdp"], - "type": answer["type"], - }, - } - ) + return { + "pc_id": answer["pc_id"], + "sdp": answer["sdp"], + "type": answer["type"], + } async def _handle_ice(payload, peers): diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0094f13..f14089a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,6 +8,8 @@ "name": "ai-video-admin-frontend", "version": "0.1.0", "dependencies": { + "@pipecat-ai/client-js": "^1.12.0", + "@pipecat-ai/small-webrtc-transport": "^1.10.5", "@xyflow/react": "^12.11.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -404,6 +406,15 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", @@ -449,6 +460,22 @@ "node": ">=6.9.0" } }, + "node_modules/@daily-co/daily-js": { + "version": "0.90.0", + "resolved": "https://registry.npmjs.org/@daily-co/daily-js/-/daily-js-0.90.0.tgz", + "integrity": "sha512-XnuuNIxLt8GOv+rFYoU+OPTSmVj+Mg9hRPK2EM6WYVY62F6rcw7vwzZBOSCisVuu1VnTIF/2J9V0VEBa83lDzw==", + "license": "BSD-2-Clause", + "dependencies": { + "@babel/runtime": "^7.12.5", + "@sentry/browser": "^8.33.1", + "bowser": "^2.8.1", + "dequal": "^2.0.3", + "events": "^3.1.0" + }, + "engines": { + "node": ">=22.14.0" + } + }, "node_modules/@dotenvx/dotenvx": { "version": "1.71.0", "resolved": "https://registry.npmjs.org/@dotenvx/dotenvx/-/dotenvx-1.71.0.tgz", @@ -1888,6 +1915,34 @@ "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", "license": "MIT" }, + "node_modules/@pipecat-ai/client-js": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/@pipecat-ai/client-js/-/client-js-1.12.0.tgz", + "integrity": "sha512-TKHuxTfnO5Eq9VpmSqaPsV0y9lQ+jkb7aEG5MRsmjkhxyhhOW2ljMPwMDzOt+lwS89ZRtk12FUKEaRR/+yBGGg==", + "license": "BSD-2-Clause", + "dependencies": { + "@types/events": "^3.0.3", + "bowser": "^2.11.0", + "clone-deep": "^4.0.1", + "events": "^3.3.0", + "typed-emitter": "^2.1.0", + "uuid": "^11.1.1" + } + }, + "node_modules/@pipecat-ai/small-webrtc-transport": { + "version": "1.10.5", + "resolved": "https://registry.npmjs.org/@pipecat-ai/small-webrtc-transport/-/small-webrtc-transport-1.10.5.tgz", + "integrity": "sha512-TF2TV/GQUVuC8Pr69fG/MSvUtuwEGcPcUg/H5d8O0RmSthVief0APgC5cKuyrjh/Rtl7+AjhxxWbrkuACdxulQ==", + "license": "BSD-2-Clause", + "dependencies": { + "@daily-co/daily-js": "^0.90.0", + "dequal": "^2.0.3", + "lodash": "^4.17.21" + }, + "peerDependencies": { + "@pipecat-ai/client-js": "~1.12.0" + } + }, "node_modules/@radix-ui/number": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.1.tgz", @@ -3399,6 +3454,81 @@ "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", "license": "MIT" }, + "node_modules/@sentry-internal/browser-utils": { + "version": "8.55.2", + "resolved": "https://registry.npmjs.org/@sentry-internal/browser-utils/-/browser-utils-8.55.2.tgz", + "integrity": "sha512-GnKod+gL/Y+1FUM/RGV8q6le1CoyiGbT40MitEK7eVwWe+bfTRq1gN7ioupyHFMUg1RlQkDQ4/sENmio/uow5A==", + "license": "MIT", + "dependencies": { + "@sentry/core": "8.55.2" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry-internal/feedback": { + "version": "8.55.2", + "resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-8.55.2.tgz", + "integrity": "sha512-XQy//NWbL0mLLM5w8wNDWMNpXz39VUyW2397dUrH8++kR63WhUVAvTOtL0o0GMVadSAzl1b08oHP9zSUNFQwcg==", + "license": "MIT", + "dependencies": { + "@sentry/core": "8.55.2" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry-internal/replay": { + "version": "8.55.2", + "resolved": "https://registry.npmjs.org/@sentry-internal/replay/-/replay-8.55.2.tgz", + "integrity": "sha512-+W43Z697EVe/OgpGW07B773sa8xO1UbpnW0Cr+E+3FMDb6ZbXlaBUoagPTUkkQPdwBe35SDh6r8y2M3EOPGbxg==", + "license": "MIT", + "dependencies": { + "@sentry-internal/browser-utils": "8.55.2", + "@sentry/core": "8.55.2" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry-internal/replay-canvas": { + "version": "8.55.2", + "resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-8.55.2.tgz", + "integrity": "sha512-P/jGiuR7dRLG9IzD/463fLgiibyYceauav/9prRG0ZxJm1AtuO02OKball2Fs3bbzdzwHCTlcsUuL2ivDF4b5A==", + "license": "MIT", + "dependencies": { + "@sentry-internal/replay": "8.55.2", + "@sentry/core": "8.55.2" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry/browser": { + "version": "8.55.2", + "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-8.55.2.tgz", + "integrity": "sha512-xHuPIEKhx9zw5quWvv4YgZprnwoVMCfxIhmOIf6KJ9iizyUHeUDcKpLS59xERroqwX4RpvK+l/27AZu4zfZlzQ==", + "license": "MIT", + "dependencies": { + "@sentry-internal/browser-utils": "8.55.2", + "@sentry-internal/feedback": "8.55.2", + "@sentry-internal/replay": "8.55.2", + "@sentry-internal/replay-canvas": "8.55.2", + "@sentry/core": "8.55.2" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry/core": { + "version": "8.55.2", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-8.55.2.tgz", + "integrity": "sha512-YlEBwybUcOQ/KjMHDmof1vwweVnBtBxYlQp7DE3fOdtW4pqqdHWTnTntQs4VgYfxzjJYgtkd9LHlGtg8qy+JVQ==", + "license": "MIT", + "engines": { + "node": ">=14.18" + } + }, "node_modules/@sindresorhus/merge-streams": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", @@ -3833,6 +3963,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/events": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.3.tgz", + "integrity": "sha512-trOc4AAUThEz9hapPtSd7wf5tiQKvTtu5b371UxXdTuqzIh0ArcRspRP0i0Viu+LXstIQ1z96t1nsPxT9ol01g==", + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -4977,6 +5113,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.15", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.15.tgz", @@ -5245,6 +5387,20 @@ "node": ">=8" } }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "license": "MIT", + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/clsx": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", @@ -5723,6 +5879,15 @@ "node": ">= 0.8" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -6507,6 +6672,15 @@ "node": ">= 0.6" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/eventsource": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", @@ -7786,6 +7960,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "license": "MIT", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-promise": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", @@ -8001,6 +8187,15 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "license": "ISC" }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/iterator.prototype": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.5.tgz", @@ -8161,6 +8356,15 @@ "json-buffer": "3.0.1" } }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/kleur": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", @@ -8487,6 +8691,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -10028,6 +10238,16 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/rxjs": { + "version": "7.8.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "tslib": "^2.1.0" + } + }, "node_modules/safe-array-concat": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.4.tgz", @@ -10306,6 +10526,18 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "license": "MIT", + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/sharp": { "version": "0.34.5", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", @@ -11165,6 +11397,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz", + "integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==", + "license": "MIT", + "optionalDependencies": { + "rxjs": "*" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -11403,6 +11644,19 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "license": "MIT" }, + "node_modules/uuid": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", + "integrity": "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, "node_modules/validate-npm-package-name": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/validate-npm-package-name/-/validate-npm-package-name-7.0.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index cadaaed..919b5fe 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -9,6 +9,8 @@ "lint": "eslint" }, "dependencies": { + "@pipecat-ai/client-js": "^1.12.0", + "@pipecat-ai/small-webrtc-transport": "^1.10.5", "@xyflow/react": "^12.11.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", diff --git a/frontend/src/hooks/use-voice-preview.ts b/frontend/src/hooks/use-voice-preview.ts index 199a8d9..fafecc0 100644 --- a/frontend/src/hooks/use-voice-preview.ts +++ b/frontend/src/hooks/use-voice-preview.ts @@ -1,52 +1,22 @@ "use client"; /** - * 语音预览:把麦克风接到后端 /ws/voice(WebRTC 信令),听到助手实时回应。 + * 语音预览。 * - * 走原生 RTCPeerConnection + 一条 ws 信令通道,与后端 voice_webrtc.py 的约定对齐: - * client → {type:"offer", payload:{pc_id, sdp, type, assistant_id}} - * server → {type:"answer", payload:{pc_id, sdp, type}} - * client → {type:"ice-candidate", payload:{pc_id, candidate:{...}}} - * 音频本身走 WebRTC 媒体流(Opus),不经 ws;后端 TTS 帧从 ontrack 拿到直接播放。 - * - * 另开一条 data channel 与后端管线(pipeline.py)互通应用消息: - * client → {type:"user-text", text} 文字输入(打断并触发新回复) - * server → {type:"transcript", role, content, timestamp} 用户/助手最终转写(聊天记录) - * - * 纯本机(localhost)即可跑:localhost 是 secure context,麦克风可用,ws 用明文。 - * 局域网/别的设备要 https+wss,见 deploy/README.md。 + * WebRTC 生命周期交给 Pipecat 官方 SmallWebRTCTransport:它负责固定的 + * audio/video transceiver、trackStatus、keepalive、重协商和 ICE 恢复。 + * 本文件只把平台已有的业务消息映射到 React 状态。 */ import { useCallback, useEffect, useRef, useState } from "react"; +import type { PipecatClientOptions, RTVIMessage } from "@pipecat-ai/client-js"; +import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport"; import { API_BASE, webrtcApi } from "@/lib/api"; export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed"; export type NetworkQuality = "unknown" | "good" | "fair" | "poor"; -const NETWORK_SAMPLE_INTERVAL_MS = 2_000; -const VIDEO_BITRATE_BY_QUALITY: Record< - Exclude, - number -> = { - good: 600_000, - fair: 500_000, - poor: 300_000, -}; -const INITIAL_VIDEO_BITRATE = VIDEO_BITRATE_BY_QUALITY.fair; - -type NetworkStatsSample = { - packetsSent: number; - packetsLost: number; -}; - -type VideoNetworkMetrics = { - packetsSent: number | null; - packetsLost: number | null; - roundTripTime: number | null; - availableOutgoingBitrate: number | null; -}; - type ConnectOptions = { visionEnabled?: boolean; videoStream?: MediaStream | null; @@ -57,29 +27,33 @@ export type ChatMessage = { id: string; role: "user" | "assistant"; content: string; - /** 后端给的 ISO 时间戳 */ timestamp: string; sequence: number; turnId?: string; streaming?: boolean; }; -// http→ws、https→wss,自动跟随 API 基址(同源反代时也对) -function wsBaseUrl(): string { - const url = new URL(API_BASE, window.location.origin); - url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; - return url.toString().replace(/\/$/, ""); -} +type AppMessage = Record & { type?: string }; -function generatePcId(): string { - const bytes = new Uint8Array(16); - crypto.getRandomValues(bytes); - return ( - "PC-" + - Array.from(bytes) - .map((b) => b.toString(16).padStart(2, "0")) - .join("") - ); +class AppSmallWebRTCTransport extends SmallWebRTCTransport { + onAppMessage?: (message: AppMessage) => void; + + override handleMessage(raw: string): void { + try { + const message = JSON.parse(raw) as AppMessage; + if (message.type === "signalling" || message.label === "rtvi-ai") { + super.handleMessage(raw); + } else { + this.onAppMessage?.(message); + } + } catch { + super.handleMessage(raw); + } + } + + sendAppMessage(message: AppMessage): void { + super.sendMessage(message as unknown as RTVIMessage); + } } function errorMessage(error: unknown, fallback: string): string { @@ -98,110 +72,13 @@ function sortMessages(messages: ChatMessage[]): ChatMessage[] { ); } -function microphoneErrorMessage(error: unknown): string { - if (error instanceof DOMException) { - if (error.name === "NotAllowedError") { - return "麦克风权限被拒绝,请在浏览器网站设置中允许 localhost 使用麦克风。"; - } - if (error.name === "NotFoundError") { - return "未检测到可用麦克风,请连接或启用麦克风后重试。"; - } - if (error.name === "NotReadableError") { - return "麦克风正被其他应用占用,或系统未允许浏览器访问麦克风。"; - } - } - return errorMessage(error, "无法访问麦克风。"); -} - -async function setVideoBitrate( - sender: RTCRtpSender, - bitrate: number, -): Promise { - try { - const parameters = sender.getParameters(); - if (!parameters.encodings.length) parameters.encodings = [{}]; - parameters.encodings[0].maxBitrate = bitrate; - parameters.degradationPreference = "maintain-resolution"; - await sender.setParameters(parameters); - return true; - } catch { - // 不支持发送参数的浏览器继续使用 WebRTC 自带的拥塞控制。 - return false; - } -} - -function readVideoNetworkMetrics(report: RTCStatsReport): VideoNetworkMetrics { - const metrics: VideoNetworkMetrics = { - packetsSent: null, - packetsLost: null, - roundTripTime: null, - availableOutgoingBitrate: null, - }; - - report.forEach((stat) => { - const values = stat as RTCStats & Record; - const kind = values.kind ?? values.mediaType; - if (values.type === "outbound-rtp" && kind === "video") { - if (typeof values.packetsSent === "number") { - metrics.packetsSent = values.packetsSent; - } - } else if (values.type === "remote-inbound-rtp" && kind === "video") { - if (typeof values.packetsLost === "number") { - metrics.packetsLost = values.packetsLost; - } - if (typeof values.roundTripTime === "number") { - metrics.roundTripTime = values.roundTripTime; - } - } else if ( - values.type === "candidate-pair" && - values.state === "succeeded" && - (values.nominated === true || values.selected === true) - ) { - if (typeof values.currentRoundTripTime === "number") { - metrics.roundTripTime = values.currentRoundTripTime; - } - if (typeof values.availableOutgoingBitrate === "number") { - metrics.availableOutgoingBitrate = values.availableOutgoingBitrate; - } - } +function encodeHeaderJson(value: unknown): string { + const bytes = new TextEncoder().encode(JSON.stringify(value)); + let binary = ""; + bytes.forEach((byte) => { + binary += String.fromCharCode(byte); }); - - return metrics; -} - -function classifyNetworkQuality( - metrics: VideoNetworkMetrics, - previous: NetworkStatsSample | null, -): NetworkQuality { - const hasPacketSample = - previous !== null && - metrics.packetsSent !== null && - metrics.packetsLost !== null; - const sentDelta = hasPacketSample - ? Math.max(0, metrics.packetsSent! - previous.packetsSent) - : 0; - const lostDelta = hasPacketSample - ? Math.max(0, metrics.packetsLost! - previous.packetsLost) - : 0; - const lossRate = sentDelta > 0 ? lostDelta / sentDelta : null; - const { roundTripTime: rtt, availableOutgoingBitrate: bandwidth } = metrics; - - if (lossRate === null && rtt === null && bandwidth === null) return "unknown"; - if ( - (lossRate !== null && lossRate >= 0.08) || - (rtt !== null && rtt >= 0.45) || - (bandwidth !== null && bandwidth < 350_000) - ) { - return "poor"; - } - if ( - (lossRate !== null && lossRate >= 0.03) || - (rtt !== null && rtt >= 0.25) || - (bandwidth !== null && bandwidth < 550_000) - ) { - return "fair"; - } - return "good"; + return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); } export function useVoicePreview( @@ -212,36 +89,25 @@ export function useVoicePreview( const [error, setError] = useState(null); const [micWarning, setMicWarning] = useState(null); const [localStream, setLocalStream] = useState(null); - // 远端(助手 TTS)媒体流:除挂到