Implement WebRTC offer handling and ICE candidate management in voice_webrtc.py

- Add new HTTP endpoints for handling WebRTC offers and ICE candidates, enhancing the signaling process for voice interactions.
- Introduce dynamic variable decoding from request headers to support flexible offer payloads.
- Refactor existing WebSocket handling to accommodate new offer processing logic.
- Update frontend dependencies to include Pipecat client libraries for improved WebRTC transport management.
- Streamline voice preview functionality by integrating SmallWebRTCTransport for better media handling.

backend/routes/voice_webrtc.py

@@ -9,9 +9,11 @@
 """
 
 import asyncio
+import base64
+import json
 
 from db.session import SessionLocal
-from fastapi import APIRouter, Depends, WebSocket
+from fastapi import APIRouter, Body, Depends, Request, WebSocket
 from loguru import logger
 from models import AssistantConfig, SignalingOffer
 from services.auth import require_admin, require_admin_websocket
@@ -24,6 +26,7 @@ from services.webrtc_ice import aiortc_ice_servers, client_ice_servers
 # 注意:pipecat 是重依赖(语音才用),在 _handle_offer 等处惰性导入。
 
 router = APIRouter(tags=["voice"])
+_http_peers: dict[str, object] = {}
 
 
 @router.get("/api/webrtc/ice-servers", dependencies=[Depends(require_admin)])
@@ -32,6 +35,45 @@ async def ice_servers():
     return {"iceServers": client_ice_servers()}
 
 
+@router.post("/api/webrtc/offer", dependencies=[Depends(require_admin)])
+async def http_offer(request: Request, payload: dict = Body(...)):
+    """Official SmallWebRTC JS transport offer endpoint."""
+    request_data = payload.get("requestData") or {}
+    encoded_variables = request.headers.get("x-pipecat-dynamic-variables", "")
+    header_variables = {}
+    if encoded_variables:
+        padding = "=" * (-len(encoded_variables) % 4)
+        header_variables = json.loads(
+            base64.urlsafe_b64decode(encoded_variables + padding).decode("utf-8")
+        )
+    offer_payload = {
+        **payload,
+        "assistant_id": request.headers.get("x-pipecat-assistant-id"),
+        "dynamic_variables": header_variables,
+        **request_data,
+    }
+    answer = await _handle_offer_payload(offer_payload, _http_peers)
+    return answer
+
+
+@router.patch("/api/webrtc/offer", dependencies=[Depends(require_admin)])
+async def http_ice_candidates(payload: dict = Body(...)):
+    """Accept the batched trickle ICE format used by the official JS client."""
+    pc_id = payload.get("pc_id")
+    pc = _http_peers.get(pc_id) if pc_id else None
+    if not pc:
+        return {"ok": False}
+
+    from aiortc.sdp import candidate_from_sdp
+
+    for item in payload.get("candidates") or []:
+        candidate = candidate_from_sdp(item.get("candidate", ""))
+        candidate.sdpMid = item.get("sdp_mid")
+        candidate.sdpMLineIndex = item.get("sdp_mline_index")
+        await pc.add_ice_candidate(candidate)
+    return {"ok": True}
+
+
 @router.websocket("/ws/voice")
 async def voice_signaling(websocket: WebSocket):
     if not await require_admin_websocket(websocket):
@@ -92,6 +134,17 @@ async def _resolve_config(offer: SignalingOffer) -> AssistantConfig:
 
 
 async def _handle_offer(websocket, payload, peers):
+    answer = await _handle_offer_payload(payload, peers)
+    if websocket.application_state == WebSocketState.CONNECTED:
+        await websocket.send_json(
+            {
+                "type": "answer",
+                "payload": answer,
+            }
+        )
+
+
+async def _handle_offer_payload(payload, peers):
     from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
     from services.pipecat.pipeline import run_pipeline
     from services.pipecat.transports import build_webrtc_transport
@@ -99,6 +152,11 @@ async def _handle_offer(websocket, payload, peers):
     offer = SignalingOffer(**payload)
     pc_id = offer.pc_id
 
+    restart_pc = bool(payload.get("restart_pc"))
+    if restart_pc and pc_id and pc_id in peers:
+        old_pc = peers.pop(pc_id)
+        await old_pc.disconnect()
+
     if pc_id and pc_id in peers:
         pc = peers[pc_id]
         await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False)
@@ -144,17 +202,11 @@ async def _handle_offer(websocket, payload, peers):
         )
 
     answer = pc.get_answer()
-    if websocket.application_state == WebSocketState.CONNECTED:
+    return {
-        await websocket.send_json(
-            {
-                "type": "answer",
-                "payload": {
         "pc_id": answer["pc_id"],
         "sdp": answer["sdp"],
         "type": answer["type"],
-                },
     }
-        )
 
 
 async def _handle_ice(payload, peers):

frontend/package-lock.json

@@ -8,6 +8,8 @@
       "name": "ai-video-admin-frontend",
       "version": "0.1.0",
       "dependencies": {
+        "@pipecat-ai/client-js": "^1.12.0",
+        "@pipecat-ai/small-webrtc-transport": "^1.10.5",
         "@xyflow/react": "^12.11.0",
         "class-variance-authority": "^0.7.1",
         "clsx": "^2.1.1",
@@ -404,6 +406,15 @@
         "@babel/core": "^7.0.0-0"
       }
     },
+    "node_modules/@babel/runtime": {
+      "version": "7.29.7",
+      "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz",
+      "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
     "node_modules/@babel/template": {
       "version": "7.29.7",
       "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz",
@@ -449,6 +460,22 @@
         "node": ">=6.9.0"
       }
     },
+    "node_modules/@daily-co/daily-js": {
+      "version": "0.90.0",
+      "resolved": "https://registry.npmjs.org/@daily-co/daily-js/-/daily-js-0.90.0.tgz",
+      "integrity": "sha512-XnuuNIxLt8GOv+rFYoU+OPTSmVj+Mg9hRPK2EM6WYVY62F6rcw7vwzZBOSCisVuu1VnTIF/2J9V0VEBa83lDzw==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "@babel/runtime": "^7.12.5",
+        "@sentry/browser": "^8.33.1",
+        "bowser": "^2.8.1",
+        "dequal": "^2.0.3",
+        "events": "^3.1.0"
+      },
+      "engines": {
+        "node": ">=22.14.0"
+      }
+    },
     "node_modules/@dotenvx/dotenvx": {
       "version": "1.71.0",
       "resolved": "https://registry.npmjs.org/@dotenvx/dotenvx/-/dotenvx-1.71.0.tgz",
@@ -1888,6 +1915,34 @@
       "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==",
       "license": "MIT"
     },
+    "node_modules/@pipecat-ai/client-js": {
+      "version": "1.12.0",
+      "resolved": "https://registry.npmjs.org/@pipecat-ai/client-js/-/client-js-1.12.0.tgz",
+      "integrity": "sha512-TKHuxTfnO5Eq9VpmSqaPsV0y9lQ+jkb7aEG5MRsmjkhxyhhOW2ljMPwMDzOt+lwS89ZRtk12FUKEaRR/+yBGGg==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "@types/events": "^3.0.3",
+        "bowser": "^2.11.0",
+        "clone-deep": "^4.0.1",
+        "events": "^3.3.0",
+        "typed-emitter": "^2.1.0",
+        "uuid": "^11.1.1"
+      }
+    },
+    "node_modules/@pipecat-ai/small-webrtc-transport": {
+      "version": "1.10.5",
+      "resolved": "https://registry.npmjs.org/@pipecat-ai/small-webrtc-transport/-/small-webrtc-transport-1.10.5.tgz",
+      "integrity": "sha512-TF2TV/GQUVuC8Pr69fG/MSvUtuwEGcPcUg/H5d8O0RmSthVief0APgC5cKuyrjh/Rtl7+AjhxxWbrkuACdxulQ==",
+      "license": "BSD-2-Clause",
+      "dependencies": {
+        "@daily-co/daily-js": "^0.90.0",
+        "dequal": "^2.0.3",
+        "lodash": "^4.17.21"
+      },
+      "peerDependencies": {
+        "@pipecat-ai/client-js": "~1.12.0"
+      }
+    },
     "node_modules/@radix-ui/number": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.1.tgz",
@@ -3399,6 +3454,81 @@
       "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==",
       "license": "MIT"
     },
+    "node_modules/@sentry-internal/browser-utils": {
+      "version": "8.55.2",
+      "resolved": "https://registry.npmjs.org/@sentry-internal/browser-utils/-/browser-utils-8.55.2.tgz",
+      "integrity": "sha512-GnKod+gL/Y+1FUM/RGV8q6le1CoyiGbT40MitEK7eVwWe+bfTRq1gN7ioupyHFMUg1RlQkDQ4/sENmio/uow5A==",
+      "license": "MIT",
+      "dependencies": {
+        "@sentry/core": "8.55.2"
+      },
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
+    "node_modules/@sentry-internal/feedback": {
+      "version": "8.55.2",
+      "resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-8.55.2.tgz",
+      "integrity": "sha512-XQy//NWbL0mLLM5w8wNDWMNpXz39VUyW2397dUrH8++kR63WhUVAvTOtL0o0GMVadSAzl1b08oHP9zSUNFQwcg==",
+      "license": "MIT",
+      "dependencies": {
+        "@sentry/core": "8.55.2"
+      },
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
+    "node_modules/@sentry-internal/replay": {
+      "version": "8.55.2",
+      "resolved": "https://registry.npmjs.org/@sentry-internal/replay/-/replay-8.55.2.tgz",
+      "integrity": "sha512-+W43Z697EVe/OgpGW07B773sa8xO1UbpnW0Cr+E+3FMDb6ZbXlaBUoagPTUkkQPdwBe35SDh6r8y2M3EOPGbxg==",
+      "license": "MIT",
+      "dependencies": {
+        "@sentry-internal/browser-utils": "8.55.2",
+        "@sentry/core": "8.55.2"
+      },
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
+    "node_modules/@sentry-internal/replay-canvas": {
+      "version": "8.55.2",
+      "resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-8.55.2.tgz",
+      "integrity": "sha512-P/jGiuR7dRLG9IzD/463fLgiibyYceauav/9prRG0ZxJm1AtuO02OKball2Fs3bbzdzwHCTlcsUuL2ivDF4b5A==",
+      "license": "MIT",
+      "dependencies": {
+        "@sentry-internal/replay": "8.55.2",
+        "@sentry/core": "8.55.2"
+      },
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
+    "node_modules/@sentry/browser": {
+      "version": "8.55.2",
+      "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-8.55.2.tgz",
+      "integrity": "sha512-xHuPIEKhx9zw5quWvv4YgZprnwoVMCfxIhmOIf6KJ9iizyUHeUDcKpLS59xERroqwX4RpvK+l/27AZu4zfZlzQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@sentry-internal/browser-utils": "8.55.2",
+        "@sentry-internal/feedback": "8.55.2",
+        "@sentry-internal/replay": "8.55.2",
+        "@sentry-internal/replay-canvas": "8.55.2",
+        "@sentry/core": "8.55.2"
+      },
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
+    "node_modules/@sentry/core": {
+      "version": "8.55.2",
+      "resolved": "https://registry.npmjs.org/@sentry/core/-/core-8.55.2.tgz",
+      "integrity": "sha512-YlEBwybUcOQ/KjMHDmof1vwweVnBtBxYlQp7DE3fOdtW4pqqdHWTnTntQs4VgYfxzjJYgtkd9LHlGtg8qy+JVQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
     "node_modules/@sindresorhus/merge-streams": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz",
@@ -3833,6 +3963,12 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/@types/events": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.3.tgz",
+      "integrity": "sha512-trOc4AAUThEz9hapPtSd7wf5tiQKvTtu5b371UxXdTuqzIh0ArcRspRP0i0Viu+LXstIQ1z96t1nsPxT9ol01g==",
+      "license": "MIT"
+    },
     "node_modules/@types/json-schema": {
       "version": "7.0.15",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
@@ -4977,6 +5113,12 @@
         "url": "https://opencollective.com/express"
       }
     },
+    "node_modules/bowser": {
+      "version": "2.14.1",
+      "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz",
+      "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==",
+      "license": "MIT"
+    },
     "node_modules/brace-expansion": {
       "version": "1.1.15",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.15.tgz",
@@ -5245,6 +5387,20 @@
         "node": ">=8"
       }
     },
+    "node_modules/clone-deep": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz",
+      "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==",
+      "license": "MIT",
+      "dependencies": {
+        "is-plain-object": "^2.0.4",
+        "kind-of": "^6.0.2",
+        "shallow-clone": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/clsx": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
@@ -5723,6 +5879,15 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/dequal": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
+      "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/detect-libc": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
@@ -6507,6 +6672,15 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/events": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
+      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.8.x"
+      }
+    },
     "node_modules/eventsource": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz",
@@ -7786,6 +7960,18 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
+      "license": "MIT",
+      "dependencies": {
+        "isobject": "^3.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/is-promise": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz",
@@ -8001,6 +8187,15 @@
       "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
       "license": "ISC"
     },
+    "node_modules/isobject": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
+      "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/iterator.prototype": {
       "version": "1.1.5",
       "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.5.tgz",
@@ -8161,6 +8356,15 @@
         "json-buffer": "3.0.1"
       }
     },
+    "node_modules/kind-of": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
+      "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/kleur": {
       "version": "4.1.5",
       "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz",
@@ -8487,6 +8691,12 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/lodash": {
+      "version": "4.18.1",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz",
+      "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==",
+      "license": "MIT"
+    },
     "node_modules/lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -10028,6 +10238,16 @@
         "queue-microtask": "^1.2.2"
       }
     },
+    "node_modules/rxjs": {
+      "version": "7.8.2",
+      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz",
+      "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
+      "license": "Apache-2.0",
+      "optional": true,
+      "dependencies": {
+        "tslib": "^2.1.0"
+      }
+    },
     "node_modules/safe-array-concat": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.4.tgz",
@@ -10306,6 +10526,18 @@
         "url": "https://github.com/sponsors/colinhacks"
       }
     },
+    "node_modules/shallow-clone": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz",
+      "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==",
+      "license": "MIT",
+      "dependencies": {
+        "kind-of": "^6.0.2"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/sharp": {
       "version": "0.34.5",
       "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz",
@@ -11165,6 +11397,15 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/typed-emitter": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz",
+      "integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==",
+      "license": "MIT",
+      "optionalDependencies": {
+        "rxjs": "*"
+      }
+    },
     "node_modules/typescript": {
       "version": "5.9.3",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
@@ -11403,6 +11644,19 @@
       "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
       "license": "MIT"
     },
+    "node_modules/uuid": {
+      "version": "11.1.1",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz",
+      "integrity": "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==",
+      "funding": [
+        "https://github.com/sponsors/broofa",
+        "https://github.com/sponsors/ctavan"
+      ],
+      "license": "MIT",
+      "bin": {
+        "uuid": "dist/esm/bin/uuid"
+      }
+    },
     "node_modules/validate-npm-package-name": {
       "version": "7.0.2",
       "resolved": "https://registry.npmjs.org/validate-npm-package-name/-/validate-npm-package-name-7.0.2.tgz",

frontend/package.json

@@ -9,6 +9,8 @@
     "lint": "eslint"
   },
   "dependencies": {
+    "@pipecat-ai/client-js": "^1.12.0",
+    "@pipecat-ai/small-webrtc-transport": "^1.10.5",
     "@xyflow/react": "^12.11.0",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",

frontend/src/hooks/use-voice-preview.ts

@@ -1,52 +1,22 @@
 "use client";
 
 /**
- * 语音预览:把麦克风接到后端 /ws/voice(WebRTC 信令),听到助手实时回应。
+ * 语音预览。
  *
- * 走原生 RTCPeerConnection + 一条 ws 信令通道,与后端 voice_webrtc.py 的约定对齐:
+ * WebRTC 生命周期交给 Pipecat 官方 SmallWebRTCTransport：它负责固定的
- *   client → {type:"offer",         payload:{pc_id, sdp, type, assistant_id}}
+ * audio/video transceiver、trackStatus、keepalive、重协商和 ICE 恢复。
- *   server → {type:"answer",        payload:{pc_id, sdp, type}}
+ * 本文件只把平台已有的业务消息映射到 React 状态。
- *   client → {type:"ice-candidate", payload:{pc_id, candidate:{...}}}
- * 音频本身走 WebRTC 媒体流(Opus),不经 ws;后端 TTS 帧从 ontrack 拿到直接播放。
- *
- * 另开一条 data channel 与后端管线(pipeline.py)互通应用消息:
- *   client → {type:"user-text",  text}                          文字输入(打断并触发新回复)
- *   server → {type:"transcript", role, content, timestamp}      用户/助手最终转写(聊天记录)
- *
- * 纯本机(localhost)即可跑:localhost 是 secure context,麦克风可用,ws 用明文。
- * 局域网/别的设备要 https+wss,见 deploy/README.md。
  */
 
 import { useCallback, useEffect, useRef, useState } from "react";
+import type { PipecatClientOptions, RTVIMessage } from "@pipecat-ai/client-js";
+import { SmallWebRTCTransport } from "@pipecat-ai/small-webrtc-transport";
 
 import { API_BASE, webrtcApi } from "@/lib/api";
 
 export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";
 export type NetworkQuality = "unknown" | "good" | "fair" | "poor";
 
-const NETWORK_SAMPLE_INTERVAL_MS = 2_000;
-const VIDEO_BITRATE_BY_QUALITY: Record<
-  Exclude<NetworkQuality, "unknown">,
-  number
-> = {
-  good: 600_000,
-  fair: 500_000,
-  poor: 300_000,
-};
-const INITIAL_VIDEO_BITRATE = VIDEO_BITRATE_BY_QUALITY.fair;
-
-type NetworkStatsSample = {
-  packetsSent: number;
-  packetsLost: number;
-};
-
-type VideoNetworkMetrics = {
-  packetsSent: number | null;
-  packetsLost: number | null;
-  roundTripTime: number | null;
-  availableOutgoingBitrate: number | null;
-};
-
 type ConnectOptions = {
   visionEnabled?: boolean;
   videoStream?: MediaStream | null;
@@ -57,29 +27,33 @@ export type ChatMessage = {
   id: string;
   role: "user" | "assistant";
   content: string;
-  /** 后端给的 ISO 时间戳 */
   timestamp: string;
   sequence: number;
   turnId?: string;
   streaming?: boolean;
 };
 
-// http→ws、https→wss,自动跟随 API 基址(同源反代时也对)
+type AppMessage = Record<string, unknown> & { type?: string };
-function wsBaseUrl(): string {
-  const url = new URL(API_BASE, window.location.origin);
-  url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
-  return url.toString().replace(/\/$/, "");
-}
 
-function generatePcId(): string {
+class AppSmallWebRTCTransport extends SmallWebRTCTransport {
-  const bytes = new Uint8Array(16);
+  onAppMessage?: (message: AppMessage) => void;
-  crypto.getRandomValues(bytes);
+
-  return (
+  override handleMessage(raw: string): void {
-    "PC-" +
+    try {
-    Array.from(bytes)
+      const message = JSON.parse(raw) as AppMessage;
-      .map((b) => b.toString(16).padStart(2, "0"))
+      if (message.type === "signalling" || message.label === "rtvi-ai") {
-      .join("")
+        super.handleMessage(raw);
-  );
+      } else {
+        this.onAppMessage?.(message);
+      }
+    } catch {
+      super.handleMessage(raw);
+    }
+  }
+
+  sendAppMessage(message: AppMessage): void {
+    super.sendMessage(message as unknown as RTVIMessage);
+  }
 }
 
 function errorMessage(error: unknown, fallback: string): string {
@@ -98,110 +72,13 @@ function sortMessages(messages: ChatMessage[]): ChatMessage[] {
   );
 }
 
-function microphoneErrorMessage(error: unknown): string {
+function encodeHeaderJson(value: unknown): string {
-  if (error instanceof DOMException) {
+  const bytes = new TextEncoder().encode(JSON.stringify(value));
-    if (error.name === "NotAllowedError") {
+  let binary = "";
-      return "麦克风权限被拒绝，请在浏览器网站设置中允许 localhost 使用麦克风。";
+  bytes.forEach((byte) => {
-    }
+    binary += String.fromCharCode(byte);
-    if (error.name === "NotFoundError") {
-      return "未检测到可用麦克风，请连接或启用麦克风后重试。";
-    }
-    if (error.name === "NotReadableError") {
-      return "麦克风正被其他应用占用，或系统未允许浏览器访问麦克风。";
-    }
-  }
-  return errorMessage(error, "无法访问麦克风。");
-}
-
-async function setVideoBitrate(
-  sender: RTCRtpSender,
-  bitrate: number,
-): Promise<boolean> {
-  try {
-    const parameters = sender.getParameters();
-    if (!parameters.encodings.length) parameters.encodings = [{}];
-    parameters.encodings[0].maxBitrate = bitrate;
-    parameters.degradationPreference = "maintain-resolution";
-    await sender.setParameters(parameters);
-    return true;
-  } catch {
-    // 不支持发送参数的浏览器继续使用 WebRTC 自带的拥塞控制。
-    return false;
-  }
-}
-
-function readVideoNetworkMetrics(report: RTCStatsReport): VideoNetworkMetrics {
-  const metrics: VideoNetworkMetrics = {
-    packetsSent: null,
-    packetsLost: null,
-    roundTripTime: null,
-    availableOutgoingBitrate: null,
-  };
-
-  report.forEach((stat) => {
-    const values = stat as RTCStats & Record<string, unknown>;
-    const kind = values.kind ?? values.mediaType;
-    if (values.type === "outbound-rtp" && kind === "video") {
-      if (typeof values.packetsSent === "number") {
-        metrics.packetsSent = values.packetsSent;
-      }
-    } else if (values.type === "remote-inbound-rtp" && kind === "video") {
-      if (typeof values.packetsLost === "number") {
-        metrics.packetsLost = values.packetsLost;
-      }
-      if (typeof values.roundTripTime === "number") {
-        metrics.roundTripTime = values.roundTripTime;
-      }
-    } else if (
-      values.type === "candidate-pair" &&
-      values.state === "succeeded" &&
-      (values.nominated === true || values.selected === true)
-    ) {
-      if (typeof values.currentRoundTripTime === "number") {
-        metrics.roundTripTime = values.currentRoundTripTime;
-      }
-      if (typeof values.availableOutgoingBitrate === "number") {
-        metrics.availableOutgoingBitrate = values.availableOutgoingBitrate;
-      }
-    }
   });
-
+  return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
-  return metrics;
-}
-
-function classifyNetworkQuality(
-  metrics: VideoNetworkMetrics,
-  previous: NetworkStatsSample | null,
-): NetworkQuality {
-  const hasPacketSample =
-    previous !== null &&
-    metrics.packetsSent !== null &&
-    metrics.packetsLost !== null;
-  const sentDelta = hasPacketSample
-    ? Math.max(0, metrics.packetsSent! - previous.packetsSent)
-    : 0;
-  const lostDelta = hasPacketSample
-    ? Math.max(0, metrics.packetsLost! - previous.packetsLost)
-    : 0;
-  const lossRate = sentDelta > 0 ? lostDelta / sentDelta : null;
-  const { roundTripTime: rtt, availableOutgoingBitrate: bandwidth } = metrics;
-
-  if (lossRate === null && rtt === null && bandwidth === null) return "unknown";
-  if (
-    (lossRate !== null && lossRate >= 0.08) ||
-    (rtt !== null && rtt >= 0.45) ||
-    (bandwidth !== null && bandwidth < 350_000)
-  ) {
-    return "poor";
-  }
-  if (
-    (lossRate !== null && lossRate >= 0.03) ||
-    (rtt !== null && rtt >= 0.25) ||
-    (bandwidth !== null && bandwidth < 550_000)
-  ) {
-    return "fair";
-  }
-  return "good";
 }
 
 export function useVoicePreview(
@@ -212,36 +89,25 @@ export function useVoicePreview(
   const [error, setError] = useState<string | null>(null);
   const [micWarning, setMicWarning] = useState<string | null>(null);
   const [localStream, setLocalStream] = useState<MediaStream | null>(null);
-  // 远端(助手 TTS)媒体流:除挂到 <audio> 播放外,也暴露给波形可视化
   const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
   const [messages, setMessages] = useState<ChatMessage[]>([]);
-  // 后端结束节点/EndFrame 触发的正常结束。与 failed 区分，供独立通话页
-  // 立即收起媒体画面并展示重新开始入口。
   const [callEnded, setCallEnded] = useState(false);
   const [networkQuality, setNetworkQuality] =
     useState<NetworkQuality>("unknown");
-  // 可选麦克风/扬声器列表与当前选择(空串表示交给浏览器选默认设备)
   const [audioInputs, setAudioInputs] = useState<MediaDeviceInfo[]>([]);
   const [audioOutputs, setAudioOutputs] = useState<MediaDeviceInfo[]>([]);
-  const [selectedDeviceId, setSelectedDeviceId] = useState<string>("");
+  const [selectedDeviceId, setSelectedDeviceId] = useState("");
-  const [selectedOutputDeviceId, setSelectedOutputDeviceId] =
+  const [selectedOutputDeviceId, setSelectedOutputDeviceId] = useState("");
-    useState<string>("");
+
+  const transportRef = useRef<AppSmallWebRTCTransport | null>(null);
+  const startingRef = useRef(false);
+  const messageSeqRef = useRef(0);
+  const endedByServerRef = useRef(false);
   const selectedDeviceIdRef = useRef("");
   const selectedOutputDeviceIdRef = useRef("");
-  const audioRef = useRef<HTMLAudioElement | null>(null);
-  const pcRef = useRef<RTCPeerConnection | null>(null);
-  const wsRef = useRef<WebSocket | null>(null);
-  const dataChannelRef = useRef<RTCDataChannel | null>(null);
-  const localStreamRef = useRef<MediaStream | null>(null);
-  const startingRef = useRef(false);
-  const negotiationInProgressRef = useRef(false);
-  const messageSeqRef = useRef(0);
-  const networkStatsRef = useRef<NetworkStatsSample | null>(null);
-  const videoBitrateRef = useRef(INITIAL_VIDEO_BITRATE);
-  // 后端主动结束(工作流走到结束节点)标记:据此把随后的断开当作正常结束而非报错
-  const endedByServerRef = useRef(false);
-  // 工作流激活节点回调存进 ref,避免把它挂进 connect 依赖反复重建连接
   const onNodeActiveRef = useRef(onNodeActive);
+  const audioRef = useRef<HTMLAudioElement | null>(null);
+
   useEffect(() => {
     onNodeActiveRef.current = onNodeActive;
   }, [onNodeActive]);
@@ -256,97 +122,39 @@ export function useVoicePreview(
     try {
       await audio.setSinkId(deviceId);
     } catch {
-      /* 忽略切换失败(设备不可用或不支持) */
+      // 设备可能已拔出，交给浏览器回退默认输出。
     }
   }, [supportsOutputSelection]);
 
-  // 枚举可用麦克风与扬声器。未授权前 label 为空,授权(连接)后再刷新即可拿到名称。
   const refreshDevices = useCallback(async () => {
     if (!navigator.mediaDevices?.enumerateDevices) return;
     try {
       const devices = await navigator.mediaDevices.enumerateDevices();
-      // 未授权时设备可能没有 deviceId(空串),无法选择,直接过滤掉
+      setAudioInputs(
-      const inputs = devices.filter((d) => d.kind === "audioinput" && d.deviceId);
+        devices.filter((item) => item.kind === "audioinput" && item.deviceId),
-      const outputs = devices.filter(
+      );
-        (d) => d.kind === "audiooutput" && d.deviceId,
+      setAudioOutputs(
+        devices.filter((item) => item.kind === "audiooutput" && item.deviceId),
       );
-      setAudioInputs(inputs);
-      setAudioOutputs(outputs);
-      // 选中的设备已被拔出时,回退到浏览器默认设备
-      if (
-        selectedDeviceIdRef.current &&
-        !inputs.some((d) => d.deviceId === selectedDeviceIdRef.current)
-      ) {
-        setSelectedDeviceId("");
-      }
-      if (
-        selectedOutputDeviceIdRef.current &&
-        !outputs.some((d) => d.deviceId === selectedOutputDeviceIdRef.current)
-      ) {
-        setSelectedOutputDeviceId("");
-        void applyOutputDevice("");
-      }
     } catch {
-      /* 忽略枚举失败 */
+      // 设备枚举失败不影响仅接收模式。
     }
-  }, [applyOutputDevice]);
+  }, []);
-
-  // connect/refreshDevices 在回调里读最新选择,避免把它们挂进依赖反复重建
-  useEffect(() => {
-    selectedDeviceIdRef.current = selectedDeviceId;
-  }, [selectedDeviceId]);
 
   useEffect(() => {
-    selectedOutputDeviceIdRef.current = selectedOutputDeviceId;
-  }, [selectedOutputDeviceId]);
-
-  useEffect(() => {
-    void applyOutputDevice(selectedOutputDeviceId);
-  }, [applyOutputDevice, selectedOutputDeviceId, remoteStream]);
-
-  useEffect(() => {
-    if (!navigator.mediaDevices?.enumerateDevices) return;
     // eslint-disable-next-line react-hooks/set-state-in-effect
     void refreshDevices();
-    navigator.mediaDevices.addEventListener("devicechange", refreshDevices);
+    navigator.mediaDevices?.addEventListener("devicechange", refreshDevices);
-    return () => {
+    return () =>
-      navigator.mediaDevices.removeEventListener("devicechange", refreshDevices);
+      navigator.mediaDevices?.removeEventListener("devicechange", refreshDevices);
-    };
   }, [refreshDevices]);
 
   const releaseResources = useCallback(() => {
-    const ws = wsRef.current;
+    const transport = transportRef.current;
-    wsRef.current = null;
+    transportRef.current = null;
-    if (ws) {
+    transport?.disconnect().catch(() => {});
-      ws.onclose = null;
-      ws.onerror = null;
-      ws.onmessage = null;
-      ws.close();
-    }
-
-    const channel = dataChannelRef.current;
-    dataChannelRef.current = null;
-    if (channel) {
-      channel.onopen = null;
-      channel.onmessage = null;
-      channel.close();
-    }
-
-    const pc = pcRef.current;
-    pcRef.current = null;
-    if (pc) {
-      pc.onconnectionstatechange = null;
-      pc.onicecandidate = null;
-      pc.oniceconnectionstatechange = null;
-      pc.ontrack = null;
-      pc.close();
-    }
-
-    localStreamRef.current?.getTracks().forEach((track) => track.stop());
-    localStreamRef.current = null;
     if (audioRef.current) audioRef.current.srcObject = null;
     startingRef.current = false;
-    negotiationInProgressRef.current = false;
     onNodeActiveRef.current?.(null);
   }, []);
 
@@ -359,217 +167,29 @@ export function useVoicePreview(
     setError(null);
     setMicWarning(null);
     setNetworkQuality("unknown");
-    networkStatsRef.current = null;
-    videoBitrateRef.current = INITIAL_VIDEO_BITRATE;
     setStatus("idle");
   }, [releaseResources]);
 
-  const fail = useCallback(
+  const fail = useCallback((message: string) => {
-    (message: string) => {
     releaseResources();
     setLocalStream(null);
     setRemoteStream(null);
     setError(message);
     setNetworkQuality("unknown");
-      networkStatsRef.current = null;
-      videoBitrateRef.current = INITIAL_VIDEO_BITRATE;
     setStatus("failed");
-    },
+  }, [releaseResources]);
-    [releaseResources],
-  );
 
-  // 连接断开时:若是后端主动收尾(call-ended),按正常结束处理(不报错);
+  const handleAppMessage = useCallback((msg: AppMessage) => {
-  // 否则按异常失败处理。
-  const closeOnRemoteEnd = useCallback(
-    (message: string) => {
-      if (endedByServerRef.current) {
-        disconnect();
-      } else {
-        fail(message);
-      }
-    },
-    [disconnect, fail],
-  );
-
-  const connect = useCallback(async (options: ConnectOptions = {}) => {
-    if (startingRef.current || pcRef.current || wsRef.current) return;
-    if (!assistantId) {
-      setError("请先保存助手，再开始语音预览。");
-      setStatus("failed");
-      return;
-    }
-    startingRef.current = true;
-    setError(null);
-    setMicWarning(null);
-    setMessages([]); // 新会话清空上一轮聊天记录
-    setCallEnded(false);
-    endedByServerRef.current = false;
-    setStatus("connecting");
-
-    // 麦克风是可选的:获取失败时继续建立仅接收后端音频的 WebRTC 会话。
-    let stream: MediaStream | null = null;
-    if (window.isSecureContext && navigator.mediaDevices?.getUserMedia) {
-      try {
-        const audioConstraints: MediaTrackConstraints = {
-          echoCancellation: true,
-          noiseSuppression: true,
-          autoGainControl: true,
-        };
-        // 指定了具体设备就强制用它,否则交给浏览器选默认麦克风
-        if (selectedDeviceIdRef.current) {
-          audioConstraints.deviceId = { exact: selectedDeviceIdRef.current };
-        }
-        stream = await navigator.mediaDevices.getUserMedia({
-          audio: audioConstraints,
-        });
-        // 授权后 label 才可见,刷新列表把真实设备名补上
-        void refreshDevices();
-      } catch (mediaError) {
-        setMicWarning(microphoneErrorMessage(mediaError));
-      }
-    } else {
-      setMicWarning("当前页面无法访问麦克风，已进入仅收听模式。");
-    }
-    if (stream) {
-      localStreamRef.current = stream;
-      setLocalStream(stream);
-    }
-
-    const pcId = generatePcId();
-    const ws = new WebSocket(`${wsBaseUrl()}/ws/voice`);
-    wsRef.current = ws;
-
-    ws.onmessage = async (event) => {
-      try {
-        const msg = JSON.parse(event.data);
-        if (msg.type === "answer") {
-          const currentPc = pcRef.current;
-          if (!currentPc) return;
-          try {
-            await currentPc.setRemoteDescription({
-              type: "answer",
-              sdp: msg.payload.sdp,
-            });
-          } finally {
-            if (pcRef.current === currentPc) {
-              negotiationInProgressRef.current = false;
-            }
-          }
-        } else if (msg.type === "ice-candidate" && msg.payload?.candidate) {
-          // 后端当前不主动 trickle,留兼容
-          try {
-            await pcRef.current?.addIceCandidate(msg.payload.candidate);
-          } catch {
-            /* 忽略迟到/重复 candidate */
-          }
-        } else if (msg.type === "error") {
-          fail(msg.payload?.message || "后端无法启动语音会话。");
-        }
-      } catch {
-        /* 非 JSON / 未知消息,忽略 */
-      }
-    };
-
-    try {
-      // 1) 等 ws 连上
-      await new Promise<void>((resolve, reject) => {
-        ws.onopen = () => resolve();
-        ws.onerror = (e) => reject(e);
-        ws.onclose = () => reject(new Error("语音信令连接已关闭。"));
-      });
-      // 连上后,信令异常或关闭都结束当前会话并保留失败状态。
-      ws.onerror = () => {
-        if (wsRef.current === ws) fail("语音信令连接失败。");
-      };
-      ws.onclose = () => {
-        if (wsRef.current === ws) closeOnRemoteEnd("语音信令连接已断开。");
-      };
-
-      // 2) 建 PeerConnection(STUN;公网跨网时后端会下发 TURN)
-      const iceServers = await webrtcApi
-        .iceServers()
-        .then((r) => r.iceServers)
-        .catch(() => [{ urls: "stun:stun.l.google.com:19302" }]);
-      const pc = new RTCPeerConnection({ iceServers });
-      pcRef.current = pc;
-
-      const sendOffer = async () => {
     if (
-          pcRef.current !== pc ||
+      msg.type === "assistant-text-start" &&
-          ws.readyState !== WebSocket.OPEN ||
-          pc.signalingState !== "stable" ||
-          negotiationInProgressRef.current
-        ) {
-          return;
-        }
-
-        negotiationInProgressRef.current = true;
-        try {
-          const offer = await pc.createOffer();
-          await pc.setLocalDescription(offer);
-          const localDescription = pc.localDescription;
-          if (!localDescription?.sdp) {
-            throw new Error("浏览器无法创建 WebRTC offer。");
-          }
-          ws.send(
-            JSON.stringify({
-              type: "offer",
-              payload: {
-                pc_id: pcId,
-                sdp: localDescription.sdp,
-                type: localDescription.type,
-                assistant_id: assistantId,
-                vision_enabled: Boolean(options.visionEnabled),
-                dynamic_variables: options.dynamicVariables ?? {},
-              },
-            }),
-          );
-        } catch (offerError) {
-          negotiationInProgressRef.current = false;
-          throw offerError;
-        }
-      };
-
-      pc.onicecandidate = (e) => {
-        if (ws.readyState !== WebSocket.OPEN) return;
-        ws.send(
-          JSON.stringify({
-            type: "ice-candidate",
-            payload: {
-              pc_id: pcId,
-              candidate: e.candidate
-                ? {
-                    candidate: e.candidate.candidate,
-                    sdpMid: e.candidate.sdpMid,
-                    sdpMLineIndex: e.candidate.sdpMLineIndex,
-                  }
-                : null,
-            },
-          }),
-        );
-      };
-
-      // 应用消息通道:收后端转写(聊天记录),发文字输入。
-      const channel = pc.createDataChannel("chat");
-      dataChannelRef.current = channel;
-      channel.onopen = () => {
-        channel.send(JSON.stringify({ type: "client-ready" }));
-      };
-      channel.onmessage = async (event) => {
-        try {
-          const msg = JSON.parse(event.data);
-          if (
-            msg?.type === "signalling" &&
-            msg?.message?.type === "renegotiate"
-          ) {
-            await sendOffer();
-          } else if (
-            msg?.type === "assistant-text-start" &&
       typeof msg.turn_id === "string"
     ) {
       messageSeqRef.current += 1;
-            const next: ChatMessage = {
+      setMessages((previous) =>
-              id: `assistant-${msg.turn_id}`,
+        sortMessages([
+          ...previous,
+          {
+            id: `assistant-${msg.turn_id as string}`,
             role: "assistant",
             content: "",
             timestamp:
@@ -577,122 +197,166 @@ export function useVoicePreview(
                 ? msg.timestamp
                 : new Date().toISOString(),
             sequence: messageSeqRef.current,
-              turnId: msg.turn_id,
+            turnId: msg.turn_id as string,
             streaming: true,
-            };
+          },
-            setMessages((prev) => sortMessages([...prev, next]));
+        ]),
+      );
     } else if (
-            msg?.type === "assistant-text-delta" &&
+      msg.type === "assistant-text-delta" &&
       typeof msg.turn_id === "string" &&
       typeof msg.delta === "string"
     ) {
-            setMessages((prev) =>
+      setMessages((previous) =>
-              prev.map((message) =>
+        previous.map((message) =>
           message.turnId === msg.turn_id
             ? { ...message, content: message.content + msg.delta }
             : message,
         ),
       );
     } else if (
-            msg?.type === "assistant-text-end" &&
+      msg.type === "assistant-text-end" &&
       typeof msg.turn_id === "string"
     ) {
-            setMessages((prev) =>
+      setMessages((previous) =>
-              prev.map((message) =>
+        previous.map((message) =>
           message.turnId === msg.turn_id
             ? {
                 ...message,
                 content:
-                        typeof msg.content === "string"
+                  typeof msg.content === "string" ? msg.content : message.content,
-                          ? msg.content
-                          : message.content,
                 streaming: false,
               }
             : message,
         ),
       );
     } else if (
-            msg?.type === "transcript" &&
+      msg.type === "transcript" &&
       (msg.role === "user" || msg.role === "assistant") &&
       typeof msg.content === "string" &&
       msg.content.trim()
     ) {
       messageSeqRef.current += 1;
-            const next: ChatMessage = {
+      setMessages((previous) =>
+        sortMessages([
+          ...previous,
+          {
             id: `msg-${messageSeqRef.current}`,
-              role: msg.role,
+            role: msg.role as "user" | "assistant",
-              content: msg.content,
+            content: msg.content as string,
             timestamp:
               typeof msg.timestamp === "string"
                 ? msg.timestamp
                 : new Date().toISOString(),
             sequence: messageSeqRef.current,
-            };
+          },
-            setMessages((prev) => sortMessages([...prev, next]));
+        ]),
-          } else if (
+      );
-            msg?.type === "node-active" &&
+    } else if (msg.type === "node-active" && typeof msg.nodeId === "string") {
-            typeof msg.nodeId === "string"
-          ) {
-            // 工作流:后端报告当前激活节点,交给画布高亮
       onNodeActiveRef.current?.(msg.nodeId);
-          } else if (msg?.type === "call-ended") {
+    } else if (msg.type === "call-ended") {
-            // 后端已播完结束节点的收尾语，即将排入 EndFrame。此时主动释放
-            // 浏览器侧资源，不依赖后续 WebRTC 状态事件是否及时到达。
       endedByServerRef.current = true;
       setCallEnded(true);
-            onNodeActiveRef.current?.(null);
       disconnect();
     }
-        } catch {
+  }, [disconnect]);
-          /* 非 JSON / 未知消息,忽略 */
-        }
-      };
 
-      pc.ontrack = (e) => {
+  const connect = useCallback(async (options: ConnectOptions = {}) => {
-        if (e.track.kind !== "audio") return;
+    if (startingRef.current || transportRef.current) return;
-        const remote = e.streams[0] ?? new MediaStream([e.track]);
+    if (!assistantId) {
-        setRemoteStream(remote);
+      setError("请先保存助手，再开始语音预览。");
+      setStatus("failed");
+      return;
+    }
+
+    startingRef.current = true;
+    setStatus("connecting");
+    setError(null);
+    setMicWarning(null);
+    setMessages([]);
+    setCallEnded(false);
+    endedByServerRef.current = false;
+
+    const iceServers = await webrtcApi
+      .iceServers()
+      .then((response) => response.iceServers)
+      .catch(() => [{ urls: "stun:stun.l.google.com:19302" }]);
+    const transport = new AppSmallWebRTCTransport({ iceServers });
+    transportRef.current = transport;
+    transport.onAppMessage = handleAppMessage;
+
+    const callbacks: NonNullable<PipecatClientOptions["callbacks"]> = {
+      onConnected: () => setStatus("connected"),
+      onDisconnected: () => {
+        if (transportRef.current !== transport) return;
+        if (endedByServerRef.current) disconnect();
+        else fail("WebRTC 连接已断开。");
+      },
+      onTrackStarted: (track) => {
+        if (track.kind !== "audio") return;
+        const stream = new MediaStream([track]);
+        setRemoteStream(stream);
         if (audioRef.current) {
-          audioRef.current.srcObject = remote;
+          audioRef.current.srcObject = stream;
           void applyOutputDevice(selectedOutputDeviceIdRef.current).then(() =>
             audioRef.current?.play().catch(() => {}),
           );
         }
+      },
+      onDeviceError: (deviceError) => {
+        setMicWarning(deviceError.message || "无法访问麦克风。已尝试继续连接。");
+      },
+      onTransportStateChanged: (nextState) => {
+        if (nextState === "connected" || nextState === "ready") {
+          setStatus("connected");
+        } else if (nextState === "error") {
+          fail("WebRTC 连接失败。");
+        }
+      },
     };
 
-      pc.onconnectionstatechange = () => {
+    transport.initialize(
-        if (pcRef.current !== pc) return;
+      {
-        if (pc.connectionState === "connected") setStatus("connected");
+        transport,
-        else if (pc.connectionState === "failed")
+        enableMic: true,
-          closeOnRemoteEnd("WebRTC 音频连接失败。");
+        enableCam: Boolean(options.visionEnabled),
-        else if (pc.connectionState === "closed")
+        callbacks,
-          closeOnRemoteEnd("WebRTC 音频连接已断开。");
+      },
-      };
+      () => {},
+    );
 
-      pc.oniceconnectionstatechange = () => {
+    try {
-        if (pcRef.current !== pc) return;
+      await transport.initDevices();
-        const st = pc.iceConnectionState;
+      if (selectedDeviceIdRef.current) {
-        if (st === "connected" || st === "completed") setStatus("connected");
+        await transport.updateMic(selectedDeviceIdRef.current);
-        else if (st === "failed") closeOnRemoteEnd("WebRTC 音频连接失败。");
+      }
-        else if (st === "disconnected")
+      const localAudio = transport.tracks().local.audio;
-          closeOnRemoteEnd("WebRTC 音频连接已断开。");
+      setLocalStream(localAudio ? new MediaStream([localAudio]) : null);
-      };
+      void refreshDevices();
 
-      // 3) 有麦克风时双向音频；否则明确声明只接收后端音频。
+      const request = new Request(`${API_BASE}/api/webrtc/offer`, {
-      if (stream) {
+        method: "POST",
-        stream.getTracks().forEach((track) => pc.addTrack(track, stream));
+        headers: {
-      } else {
+          "Content-Type": "application/json",
-        pc.addTransceiver("audio", { direction: "recvonly" });
+          "X-Pipecat-Assistant-ID": assistantId,
-      }
+          "X-Pipecat-Dynamic-Variables": encodeHeaderJson(
-      if (options.videoStream) {
+            options.dynamicVariables ?? {},
-        for (const track of options.videoStream.getVideoTracks()) {
+          ),
-          const sender = pc.addTrack(track, options.videoStream);
+        },
-          await setVideoBitrate(sender, INITIAL_VIDEO_BITRATE);
+        credentials: "include",
-        }
+      });
-      }
+      await transport.connect({
-
+        webrtcRequestParams: {
-      // 4) 生成 offer 并发给后端；同一方法也响应 SmallWebRTC 的重新协商请求。
+          endpoint: request,
-      await sendOffer();
+          requestData: {
+            assistant_id: assistantId,
+            vision_enabled: Boolean(options.visionEnabled),
+            dynamic_variables: options.dynamicVariables ?? {},
+          },
+        },
+      });
+      transport.sendAppMessage({ type: "client-ready" });
+      setStatus("connected");
     } catch (connectionError) {
       fail(errorMessage(connectionError, "无法连接语音服务。"));
     } finally {
@@ -701,138 +365,47 @@ export function useVoicePreview(
   }, [
     assistantId,
     applyOutputDevice,
-    fail,
-    closeOnRemoteEnd,
     disconnect,
+    fail,
+    handleAppMessage,
     refreshDevices,
   ]);
 
-  useEffect(() => {
+  const replaceVideoStream = useCallback(async (videoStream: MediaStream | null) => {
-    if (status !== "connected") return;
+    const transport = transportRef.current;
-    const pc = pcRef.current;
+    const deviceId = videoStream?.getVideoTracks()[0]?.getSettings().deviceId;
-    if (!pc) return;
+    if (transport && deviceId) await transport.updateCam(deviceId);
+  }, []);
 
-    let cancelled = false;
+  const selectDevice = useCallback(async (deviceId: string) => {
-    const sampleNetwork = async () => {
-      try {
-        const metrics = readVideoNetworkMetrics(await pc.getStats());
-        if (cancelled || pcRef.current !== pc) return;
-        const quality = classifyNetworkQuality(metrics, networkStatsRef.current);
-        if (metrics.packetsSent !== null && metrics.packetsLost !== null) {
-          networkStatsRef.current = {
-            packetsSent: metrics.packetsSent,
-            packetsLost: metrics.packetsLost,
-          };
-        }
-        setNetworkQuality(quality);
-
-        if (quality === "unknown") return;
-        const bitrate = VIDEO_BITRATE_BY_QUALITY[quality];
-        if (bitrate === videoBitrateRef.current) return;
-        const sender = pc
-          .getSenders()
-          .find((item) => item.track?.kind === "video");
-        if (!sender) return;
-        if (await setVideoBitrate(sender, bitrate)) {
-          videoBitrateRef.current = bitrate;
-        }
-      } catch {
-        // 个别浏览器不提供完整 WebRTC 统计信息时，保留底层拥塞控制。
-      }
-    };
-
-    void sampleNetwork();
-    const interval = window.setInterval(
-      () => void sampleNetwork(),
-      NETWORK_SAMPLE_INTERVAL_MS,
-    );
-    return () => {
-      cancelled = true;
-      window.clearInterval(interval);
-    };
-  }, [status]);
-
-  const replaceVideoStream = useCallback(
-    async (videoStream: MediaStream | null) => {
-      const pc = pcRef.current;
-      if (!pc) return;
-      const sender = pc.getSenders().find((s) => s.track?.kind === "video");
-      if (!sender) return;
-      await sender.replaceTrack(videoStream?.getVideoTracks()[0] ?? null);
-    },
-    [],
-  );
-
-  // 选择麦克风:更新选择;若会话正在发送麦克风音频,则用 WebRTC replaceTrack
-  // 热切换轨道(无需重新协商),并把波形可视化重新接到新流。
-  // 未连接时仅记下选择,留待下次 connect 生效。
-  const selectDevice = useCallback(
-    async (deviceId: string) => {
     setSelectedDeviceId(deviceId);
     selectedDeviceIdRef.current = deviceId;
-
+    const transport = transportRef.current;
-      const pc = pcRef.current;
+    if (!transport) return;
-      if (!pc) return;
-      // 只有本就在发送麦克风音频(存在 audio sender 轨道)时才热切换;
-      // 仅收听模式下加麦克风需重新协商,这里不处理,留到下次连接。
-      const sender = pc
-        .getSenders()
-        .find((s) => s.track?.kind === "audio");
-      if (!sender) return;
-
     try {
-        const audioConstraints: MediaTrackConstraints = {
+      await transport.updateMic(deviceId);
-          echoCancellation: true,
+      const audio = transport.tracks().local.audio;
-          noiseSuppression: true,
+      setLocalStream(audio ? new MediaStream([audio]) : null);
-          autoGainControl: true,
-        };
-        if (deviceId) audioConstraints.deviceId = { exact: deviceId };
-        const newStream = await navigator.mediaDevices.getUserMedia({
-          audio: audioConstraints,
-        });
-        // 切换期间可能已断开,丢弃刚拿到的流
-        if (pcRef.current !== pc) {
-          newStream.getTracks().forEach((t) => t.stop());
-          return;
-        }
-        const newTrack = newStream.getAudioTracks()[0];
-        if (!newTrack) {
-          newStream.getTracks().forEach((t) => t.stop());
-          return;
-        }
-        await sender.replaceTrack(newTrack);
-        // 旧轨道停掉,新流替换(波形/分析器随 localStream 变化自动重连)
-        localStreamRef.current?.getTracks().forEach((t) => t.stop());
-        localStreamRef.current = newStream;
-        setLocalStream(newStream);
       setMicWarning(null);
-      } catch (mediaError) {
+    } catch (deviceError) {
-        setMicWarning(microphoneErrorMessage(mediaError));
+      setMicWarning(errorMessage(deviceError, "无法切换麦克风。"));
     }
-    },
+  }, []);
-    [],
-  );
 
-  const selectOutputDevice = useCallback(
+  const selectOutputDevice = useCallback((deviceId: string) => {
-    (deviceId: string) => {
     setSelectedOutputDeviceId(deviceId);
     selectedOutputDeviceIdRef.current = deviceId;
     void applyOutputDevice(deviceId);
-    },
+  }, [applyOutputDevice]);
-    [applyOutputDevice],
-  );
 
-  // 发送文字消息:后端先打断当前播报,再按用户输入触发新回复。
-  // 成功返回 true;通道未就绪(未开始对话/连接中)返回 false。
   const sendText = useCallback((text: string): boolean => {
     const trimmed = text.trim();
-    const channel = dataChannelRef.current;
+    const transport = transportRef.current;
-    if (!trimmed || !channel || channel.readyState !== "open") return false;
+    if (!trimmed || !transport || transport.state !== "connected") return false;
-    channel.send(JSON.stringify({ type: "user-text", text: trimmed }));
+    transport.sendAppMessage({ type: "user-text", text: trimmed });
     return true;
   }, []);
 
-  // 卸载时收尾
   useEffect(() => releaseResources, [releaseResources]);
 
   return {