wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding support for renegotiation.

Browse Source
This commit is contained in:
Filipi Fuchter
2025-03-12 11:31:18 -03:00
parent da25e0c008
commit f24c5b0aa7
4 changed files with 71 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
examples/aiortc/video-transform/client/typescript/src/smallWebRTCTransport.ts
View File

@@ -27,6 +27,9 @@ export class SmallWebRTCTransport {
private pc: RTCPeerConnection | null = null;
private dc: RTCDataChannel | null = null;
private audioCodec: string | null = null;
private videoCodec: string | null = null;
private pc_id: string | null = null;
constructor(callbacks: SmallWebRTCTransportCallbacks) {
this._callbacks = callbacks
@@ -78,7 +81,7 @@ export class SmallWebRTCTransport {
return pc;
}
private async negotiate(audioCodec: string, videoCodec: string): Promise<void> {
private async negotiate(): Promise<void> {
if (!this.pc) {
return Promise.reject('Peer connection is not initialized');
}
@@ -107,22 +110,25 @@ export class SmallWebRTCTransport {
let codec: string;
// Filter audio codec
if (audioCodec !== 'default') {
if (this.audioCodec && this.audioCodec !== 'default') {
// @ts-ignore
offerSdp.sdp = this.sdpFilterCodec('audio', audioCodec, offerSdp.sdp);
offerSdp.sdp = this.sdpFilterCodec('audio', this.audioCodec, offerSdp.sdp);
}
// Filter video codec
if (videoCodec !== 'default') {
if (this.videoCodec && this.videoCodec !== 'default') {
// @ts-ignore
offerSdp.sdp = this.sdpFilterCodec('video', videoCodec, offerSdp.sdp);
offerSdp.sdp = this.sdpFilterCodec('video', this.videoCodec, offerSdp.sdp);
}
this.log(`Will create offer for peerId: ${this.pc_id}`)
// Send offer to server
const response = await fetch('/api/offer', {
body: JSON.stringify({
sdp: offerSdp.sdp,
type: offerSdp.type,
pc_id: this.pc_id
}),
headers: {
'Content-Type': 'application/json',
@@ -132,6 +138,8 @@ export class SmallWebRTCTransport {
const answer: RTCSessionDescriptionInit = await response.json();
// @ts-ignore
this.pc_id = answer.pc_id
// @ts-ignore
this.log(`Received answer for peer connection id ${answer.pc_id}`)
await this.pc!.setRemoteDescription(answer);
} catch (e) {
@@ -164,7 +172,9 @@ export class SmallWebRTCTransport {
this.addInitialTransceivers();
this.dc = this.createDataChannel('chat', { ordered: true });
await this.addUserMedias(audioDevice, videoDevice);
await this.negotiate(audioCodec, videoCodec);
this.audioCodec = audioCodec
this.videoCodec = videoCodec
await this.negotiate();
}
private async addUserMedias(audioDevice: string|undefined, videoDevice:string|undefined): Promise<void> {
@@ -195,7 +205,7 @@ export class SmallWebRTCTransport {
// Check if it's a signalling message
if (messageObj.type === SIGNALLING_TYPE) {
this.handleSignallingMessage(messageObj as SignallingMessageObject); // Delegate to handleSignallingMessage
void this.handleSignallingMessage(messageObj as SignallingMessageObject); // Delegate to handleSignallingMessage
} else {
// implement to handle the other messages in the future
}
@@ -205,15 +215,14 @@ export class SmallWebRTCTransport {
}
// Method to handle signalling messages specifically
handleSignallingMessage(messageObj: SignallingMessageObject): void {
async handleSignallingMessage(messageObj: SignallingMessageObject): Promise<void> {
// Cast the object to the correct type after verification
const signallingMessage = messageObj as SignallingMessageObject;
// Handle different signalling message types
switch (signallingMessage.message) {
case SignallingMessage.RENEGOTIATE:
this.log("Handling renegotiation...");
// TODO: implement it
await this.negotiate()
break;
default:

36
examples/aiortc/video-transform/server/server.py
View File

@@ -2,6 +2,7 @@ import argparse
import asyncio
import logging
from contextlib import asynccontextmanager
from typing import Dict
import uvicorn
from aiortc_bot import run_bot
@@ -17,33 +18,42 @@ logger = logging.getLogger("pc")
app = FastAPI()
pcs = set()
# Store connections by pc_id
pcs_map: Dict[str, SmallWebRTCConnection] = {}
@app.post("/api/offer")
async def offer(request: dict, background_tasks: BackgroundTasks):
pipecat_connection = SmallWebRTCConnection()
await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"])
pc_id = request.get("pc_id")
pcs.add(pipecat_connection)
if pc_id and pc_id in pcs_map:
pipecat_connection = pcs_map[pc_id]
logger.info(f"Reusing existing connection for pc_id: {pc_id}")
await pipecat_connection.renegotiate(sdp=request["sdp"], type=request["type"])
else:
pipecat_connection = SmallWebRTCConnection()
await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"])
@pipecat_connection.on("closed")
async def handle_disconnected():
logger.info("Discarding the peer connection.")
pcs.discard(pipecat_connection)
@pipecat_connection.on("closed")
async def handle_disconnected(webrtc_connection: SmallWebRTCConnection):
logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}")
pcs_map.pop(webrtc_connection.pc_id, None)
background_tasks.add_task(run_bot, pipecat_connection)
background_tasks.add_task(run_bot, pipecat_connection)
return pipecat_connection.get_answer()
answer = pipecat_connection.get_answer()
# Updating the peer connection inside the map
pcs_map[answer["pc_id"]] = pipecat_connection
return answer
@asynccontextmanager
async def lifespan(app: FastAPI):
yield # Run app
coros = [pc.close() for pc in pcs]
coros = [pc.close() for pc in pcs_map.values()]
await asyncio.gather(*coros)
pcs.clear()
pcs_map.clear()
if __name__ == "__main__":