wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1441 from pipecat-ai/aiortc_example_small_webrtc_transport

Browse Source 
P2P WebRTC transport - example improvements.
This commit is contained in:
Filipi da Silva Fuchter
2025-03-27 17:49:12 -03:00
committed by GitHub
parent 67a93d09c2 76f9626d35
commit 2355067f61
22 changed files with 349 additions and 583 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
examples/aiortc/video-transform/README.rst
View File

@@ -1,43 +0,0 @@
# Video transform
A Pipecat example demonstrating how to send and receive audio and video using SmallWebRTCTransport.
It also performs some image processing on the video frames using OpenCV.
## Quick Start
### First, start the bot server:
1. Navigate to the server directory:
```bash
cd server
```
2. Create and activate a virtual environment:
```bash
python3 -m venv venv
source venv/bin/activate # On Windows: venv\Scripts\activate
```
3. Install requirements:
```bash
pip install -r requirements.txt
```
4. Copy env.example to .env and configure:
- Add your API keys
5. Start the server:
```bash
python server.py
```
### Next, connect using the client app:
For client-side setup, refer to the [JavaScript Guide](client/typescript/README.md).
## Important Note
Ensure the bot server is running before using any client implementations.
## Requirements
- Python 3.10+
- Node.js 16+ (for JavaScript)
- Google API key
- Modern web browser with WebRTC support

473
examples/aiortc/video-transform/client/typescript/src/smallWebRTCTransport.ts
View File

@@ -1,473 +0,0 @@
// TODO: we should refactor everything that is inside this class,
// and create a Pipecat client transport here
// https://github.com/pipecat-ai/pipecat-client-web-transports
const SIGNALLING_TYPE = "signalling";
enum SignallingMessage {
RENEGOTIATE = "renegotiate",
}
// Interface for the structure of the signalling message
interface SignallingMessageObject {
type: string;
message: SignallingMessage;
}
export type SmallWebRTCTransportCallbacks = {
onLog(message: string): void;
onTrackStarted(track: MediaStreamTrack): void;
onConnected(): void;
onDisconnected(): void;
}
export class SmallWebRTCTransport {
private _callbacks: SmallWebRTCTransportCallbacks;
private pc: RTCPeerConnection | null = null;
private dc: RTCDataChannel | null = null;
private audioCodec: string | null = null;
private videoCodec: string | null = null;
private pc_id: string | null = null;
private reconnectionAttempts = 0;
private maxReconnectionAttempts = 3;
private isReconnecting = false;
private keepAliveInterval: number | null = null;
private audioDevice: string | undefined;
private videoDevice: string | undefined;
constructor(callbacks: SmallWebRTCTransportCallbacks) {
this._callbacks = callbacks
// for testing reconnections
// @ts-ignore
window.attemptReconnection = this.attemptReconnection.bind(this)
}
private log(message: string): void {
console.log(message);
this._callbacks.onLog(message)
}
private createPeerConnection(): RTCPeerConnection {
const config: RTCConfiguration = {
iceServers: [{ urls: ['stun:stun.l.google.com:19302'] }]
};
let pc = new RTCPeerConnection(config);
pc.addEventListener('icegatheringstatechange', () => {
this.log(`iceGatheringState: ${this.pc!.iceGatheringState}`)
});
this.log(`iceGatheringState: ${pc.iceGatheringState}`)
pc.addEventListener("iceconnectionstatechange", () => this.handleICEConnectionStateChange());
pc.addEventListener("connectionstatechange", () => this.handleConnectionStateChange());
this.log(`iceConnectionState: ${pc.iceConnectionState}`)
pc.addEventListener('signalingstatechange', () => {
this.log(`signalingState: ${this.pc!.signalingState}`)
if (this.pc!.signalingState == 'stable') {
this.handleReconnectionCompleted()
}
});
this.log(`signalingState: ${pc.signalingState}`)
pc.addEventListener('track', (evt: RTCTrackEvent) => {
this.log(`Received new track ${evt.track.kind}`)
this._callbacks.onTrackStarted(evt.track)
});
return pc;
}
private handleICEConnectionStateChange(): void {
if (!this.pc) return;
this.log(`ICE Connection State: ${this.pc.iceConnectionState}`);
if (this.pc.iceConnectionState === "failed") {
this.log("ICE connection failed, attempting restart.");
void this.attemptReconnection(true);
} else if (this.pc.iceConnectionState === "disconnected") {
// Waiting before trying to reconnect to see if it handles it automatically
setTimeout(() => {
if (this.pc?.iceConnectionState === "disconnected") {
this.log("Still disconnected, attempting reconnection.");
void this.attemptReconnection(true);
}
}, 5000);
}
}
private handleReconnectionCompleted() {
this.reconnectionAttempts = 0;
this.isReconnecting = false;
}
private handleConnectionStateChange(): void {
if (!this.pc) return;
this.log(`Connection State: ${this.pc.connectionState}`);
if (this.pc.connectionState === "connected") {
this.handleReconnectionCompleted()
this._callbacks.onConnected();
} else if (this.pc.connectionState === "failed") {
void this.attemptReconnection(true);
}
}
private async attemptReconnection(recreatePeerConnection: boolean = false): Promise<void> {
if (this.isReconnecting) {
this.log("Reconnection already in progress, skipping.");
return;
}
if (this.reconnectionAttempts >= this.maxReconnectionAttempts) {
this.log("Max reconnection attempts reached. Stopping transport.");
this.stop();
return;
}
this.isReconnecting = true;
this.reconnectionAttempts++;
this.log(`Reconnection attempt ${this.reconnectionAttempts}...`);
// aiortc it is not working fine when just trying to restart the ice
// so in this case we are creating a new peer connection on both sides
if (recreatePeerConnection) {
const oldPC = this.pc
await this.startNewPeerConnection(recreatePeerConnection)
if (oldPC) {
this.log("closing old peer connection")
this.closePeerConnection(oldPC)
}
} else {
await this.negotiate();
}
}
private async negotiate(recreatePeerConnection: boolean = false): Promise<void> {
if (!this.pc) {
return Promise.reject('Peer connection is not initialized');
}
try {
// Create offer
const offer = await this.pc.createOffer();
await this.pc.setLocalDescription(offer);
// Wait for ICE gathering to complete
/*await new Promise<void>((resolve) => {
if (this.pc!.iceGatheringState === 'complete') {
resolve();
} else {
const checkState = () => {
if (this.pc!.iceGatheringState === 'complete') {
this.pc!.removeEventListener('icegatheringstatechange', checkState);
resolve();
}
};
this.pc!.addEventListener('icegatheringstatechange', checkState);
}
});*/
let offerSdp = this.pc!.localDescription!;
let codec: string;
// Filter audio codec
if (this.audioCodec && this.audioCodec !== 'default') {
// @ts-ignore
offerSdp.sdp = this.sdpFilterCodec('audio', this.audioCodec, offerSdp.sdp);
}
// Filter video codec
if (this.videoCodec && this.videoCodec !== 'default') {
// @ts-ignore
offerSdp.sdp = this.sdpFilterCodec('video', this.videoCodec, offerSdp.sdp);
}
this.log(`Will create offer for peerId: ${this.pc_id}`)
// Send offer to server
const response = await fetch('/api/offer', {
body: JSON.stringify({
sdp: offerSdp.sdp,
type: offerSdp.type,
pc_id: this.pc_id,
restart_pc: recreatePeerConnection
}),
headers: {
'Content-Type': 'application/json',
},
method: 'POST',
});
const answer: RTCSessionDescriptionInit = await response.json();
// @ts-ignore
this.pc_id = answer.pc_id
// @ts-ignore
this.log(`Received answer for peer connection id ${answer.pc_id}`)
await this.pc!.setRemoteDescription(answer);
this.log(`Remote candidate supports trickle ice: ${this.pc.canTrickleIceCandidates}`)
} catch (e) {
this.log(`Reconnection attempt ${this.reconnectionAttempts} failed: ${e}`);
this.isReconnecting = false
setTimeout(() => this.attemptReconnection(true), 2000);
}
}
private addInitialTransceivers() {
// Transceivers always appear in creation-order for both peers
// For now we are only considering that we are going to have 02 transceivers,
// one for audio and one for video
this.pc!.addTransceiver('audio', { direction: 'sendrecv' });
this.pc!.addTransceiver('video', { direction: 'sendrecv' });
}
private getAudioTransceiver() {
// Transceivers always appear in creation-order for both peers
// Look at addInitialTransceivers
return this.pc!.getTransceivers()[0];
}
private getVideoTransceiver() {
// Transceivers always appear in creation-order for both peers
// Look at addInitialTransceivers
return this.pc!.getTransceivers()[1];
}
async start(audioDevice: string | undefined, audioCodec: string, videoCodec: string, videoDevice: string | undefined): Promise<void> {
this.audioDevice = audioDevice
this.videoDevice = videoDevice
this.audioCodec = audioCodec
this.videoCodec = videoCodec
await this.startNewPeerConnection()
}
private async startNewPeerConnection(recreatePeerConnection: boolean = false) {
this.pc = this.createPeerConnection();
this.addInitialTransceivers();
this.dc = this.createDataChannel('chat', { ordered: true });
await this.addUserMedias();
await this.negotiate(recreatePeerConnection);
}
private async addUserMedias(): Promise<void> {
this.log("Will send the audio and video");
const constraints = this.createMediaConstraints();
if (constraints.audio || constraints.video) {
try {
const stream = await navigator.mediaDevices.getUserMedia(constraints);
let videoTrack = stream.getVideoTracks()[0]
await this.getVideoTransceiver().sender.replaceTrack(videoTrack);
let audioTrack = stream.getAudioTracks()[0]
await this.getAudioTransceiver().sender.replaceTrack(audioTrack);
} catch (err) {
alert(`Could not acquire media: ${err}`);
}
}
}
// Method to handle a general message (this can be expanded for other types of messages)
handleMessage(message: string): void {
try {
this.log(message)
const messageObj = JSON.parse(message); // Type is `any` initially
// Check if it's a signalling message
if (messageObj.type === SIGNALLING_TYPE) {
void this.handleSignallingMessage(messageObj as SignallingMessageObject); // Delegate to handleSignallingMessage
} else {
// implement to handle the other messages in the future
}
} catch (error) {
console.error("Failed to parse JSON message:", error);
}
}
// Method to handle signalling messages specifically
async handleSignallingMessage(messageObj: SignallingMessageObject): Promise<void> {
// Cast the object to the correct type after verification
const signallingMessage = messageObj as SignallingMessageObject;
// Handle different signalling message types
switch (signallingMessage.message) {
case SignallingMessage.RENEGOTIATE:
void this.attemptReconnection(false)
break;
default:
console.warn("Unknown signalling message:", signallingMessage.message);
}
}
private createDataChannel(label: string, options: RTCDataChannelInit): RTCDataChannel {
const dc = this.pc!.createDataChannel(label, options);
let timeStart: number | null = null;
const getCurrentTimestamp = (): number => {
if (timeStart === null) {
timeStart = Date.now();
return 0;
}
return Date.now() - timeStart;
};
dc.addEventListener('close', () => {
this.log("datachannel closed")
if (this.keepAliveInterval) {
clearInterval(this.keepAliveInterval)
this.keepAliveInterval = null
}
});
dc.addEventListener('open', () => {
this.log("datachannel opened")
// Sending message that the client is ready, just for testing
dc.send(JSON.stringify({id: 'clientReady', label: 'rtvi-ai', type:'client-ready'}))
// @ts-ignore
this.keepAliveInterval = setInterval(() => {
const message = 'ping: ' + new Date().getTime()
dc.send(message);
}, 1000);
});
dc.addEventListener('message', (evt: MessageEvent) => {
let message = evt.data
this.handleMessage(message)
});
return dc;
}
private createMediaConstraints(): MediaStreamConstraints {
const constraints: MediaStreamConstraints = { audio: false, video: false };
const audioConstraints: MediaTrackConstraints = {};
if (this.audioDevice) audioConstraints.deviceId = { exact: this.audioDevice };
constraints.audio = Object.keys(audioConstraints).length ? audioConstraints : true;
const videoConstraints: MediaTrackConstraints = {};
if (this.videoDevice) videoConstraints.deviceId = { exact: this.videoDevice };
constraints.video = Object.keys(videoConstraints).length ? videoConstraints : true;
return constraints;
}
private closePeerConnection(pc:RTCPeerConnection) {
pc.getTransceivers().forEach((transceiver) => {
if (transceiver.stop) {
transceiver.stop();
}
});
pc.getSenders().forEach((sender) => {
sender.track?.stop();
});
pc.close();
}
stop(): void {
if (!this.pc) {
this.log("Peer connection is already closed or null.");
return;
}
if (this.dc) {
this.dc.close();
}
this.closePeerConnection(this.pc)
// For some reason after we close the peer connection, it is not triggering the listeners
this.pc_id = null
this.reconnectionAttempts = 0
this.isReconnecting = false;
this._callbacks.onDisconnected()
}
private async getAllDevices() {
return await navigator.mediaDevices.enumerateDevices();
}
async getAllCams() {
const devices = await this.getAllDevices();
return devices.filter((d) => d.kind === "videoinput");
}
async getAllMics() {
const devices = await this.getAllDevices();
return devices.filter((d) => d.kind === "audioinput");
}
private sdpFilterCodec(kind: string, codec: string, realSdp: string): string {
const allowed: number[] = [];
const rtxRegex = new RegExp('a=fmtp:(\\d+) apt=(\\d+)\\r$');
const codecRegex = new RegExp('a=rtpmap:([0-9]+) ' + this.escapeRegExp(codec));
const videoRegex = new RegExp('(m=' + kind + ' .*?)( ([0-9]+))*\\s*$');
const lines = realSdp.split('\n');
let isKind = false;
for (let i = 0; i < lines.length; i++) {
if (lines[i].startsWith('m=' + kind + ' ')) {
isKind = true;
} else if (lines[i].startsWith('m=')) {
isKind = false;
}
if (isKind) {
const match = lines[i].match(codecRegex);
if (match) {
allowed.push(parseInt(match[1]));
}
const matchRtx = lines[i].match(rtxRegex);
if (matchRtx && allowed.includes(parseInt(matchRtx[2]))) {
allowed.push(parseInt(matchRtx[1]));
}
}
}
const skipRegex = 'a=(fmtp|rtcp-fb|rtpmap):([0-9]+)';
let sdp = '';
isKind = false;
for (let i = 0; i < lines.length; i++) {
if (lines[i].startsWith('m=' + kind + ' ')) {
isKind = true;
} else if (lines[i].startsWith('m=')) {
isKind = false;
}
if (isKind) {
const skipMatch = lines[i].match(skipRegex);
if (skipMatch && !allowed.includes(parseInt(skipMatch[2]))) {
continue;
} else if (lines[i].match(videoRegex)) {
sdp += lines[i].replace(videoRegex, '$1 ' + allowed.join(' ')) + '\n';
} else {
sdp += lines[i] + '\n';
}
} else {
sdp += lines[i] + '\n';
}
}
return sdp;
}
private escapeRegExp(string: string): string {
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
}

34
examples/aiortc/voice-agent/README.rst
View File

@@ -1,34 +0,0 @@
# Video transform
A Pipecat example demonstrating the simplest way to create a voice agent with SmallWebRTCTransport.
## Quick Start
### First, start the bot server:
1. Create and activate a virtual environment:
```bash
python3 -m venv venv
source venv/bin/activate # On Windows: venv\Scripts\activate
```
2. Install requirements:
```bash
pip install -r requirements.txt
```
3. Copy env.example to .env and configure:
- Add your API keys
4. Start the server:
```bash
python server.py
```
### Next, connect using the client app:
Visit http://localhost:7860 in your browser.
## Requirements
- Python 3.10+
- Node.js 16+ (for JavaScript)
- Google API key
- Modern web browser with WebRTC support

59
examples/p2p-webrtc/video-transform/README.md Normal file
View File

@@ -0,0 +1,59 @@
# Video Transform
A Pipecat example demonstrating how to send and receive audio and video using `SmallWebRTCTransport`. This project also applies image processing to video frames using OpenCV.
## 🚀 Quick Start
### 1️⃣ Start the Bot Server
#### 📂 Navigate to the Server Directory
```bash
cd server
```
#### 🔧 Set Up the Environment
1. Create and activate a virtual environment:
```bash
python3 -m venv venv
source venv/bin/activate # On Windows: venv\Scripts\activate
```
2. Install dependencies:
```bash
pip install -r requirements.txt
```
3. Configure environment variables:
- Copy `env.example` to `.env`
```bash
cp env.example .env
```
- Add your API keys
#### ▶️ Run the Server
```bash
python server.py
```
### 2️⃣ Connect Using the Client App
For client-side setup, refer to the [JavaScript Guide](client/typescript/README.md).
## ⚠️ Important Note
Ensure the bot server is running before using any client implementations.
## 📌 Requirements
- Python **3.10+**
- Node.js **16+** (for JavaScript components)
- Google API Key
- Modern web browser with WebRTC support
---
### 💡 Notes
- Ensure all dependencies are installed before running the server.
- Check the `.env` file for missing configurations.
- WebRTC requires a secure environment (HTTPS) for full functionality in production.
Happy coding! 🎉

0
examples/aiortc/video-transform/client/typescript/README.md → examples/p2p-webrtc/video-transform/client/typescript/README.md
View File

0
examples/aiortc/video-transform/client/typescript/index.html → examples/p2p-webrtc/video-transform/client/typescript/index.html
View File

146
examples/aiortc/video-transform/client/typescript/package-lock.json → examples/p2p-webrtc/video-transform/client/typescript/package-lock.json generated
View File

@@ -9,7 +9,8 @@
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"@pipecat-ai/client-js": "^0.3.2"
"@pipecat-ai/client-js": "^0.3.2",
"@pipecat-ai/small-webrtc-transport": "^0.0.1"
},
"devDependencies": {
"@types/node": "^22.13.1",
@@ -18,6 +19,34 @@
"vite": "^6.0.2"
}
},
"node_modules/@babel/runtime": {
"version": "7.27.0",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.0.tgz",
"integrity": "sha512-VtPOkrdPHZsKc/clNqyi9WUA8TINkZ4cGk63UUE3u4pmB2k+ZMQRDuIOagv8UVd6j7k0T3+RRIb7beKTebNbcw==",
"license": "MIT",
"dependencies": {
"regenerator-runtime": "^0.14.0"
},
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@daily-co/daily-js": {
"version": "0.73.0",
"resolved": "https://registry.npmjs.org/@daily-co/daily-js/-/daily-js-0.73.0.tgz",
"integrity": "sha512-Wz8c60hgmkx8fcEeDAi4L4J0rbafiihWKyXFyhYoFYPsw2OdChHpA4RYwIB+1enRws5IK+/HdmzFDYLQsB4A6w==",
"license": "BSD-2-Clause",
"dependencies": {
"@babel/runtime": "^7.12.5",
"@sentry/browser": "^8.33.1",
"bowser": "^2.8.1",
"dequal": "^2.0.3",
"events": "^3.1.0"
},
"engines": {
"node": ">=10.0.0"
}
},
"node_modules/@esbuild/darwin-arm64": {
"version": "0.24.0",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.24.0.tgz",
@@ -36,9 +65,9 @@
}
},
"node_modules/@pipecat-ai/client-js": {
"version": "0.3.2",
"resolved": "https://registry.npmjs.org/@pipecat-ai/client-js/-/client-js-0.3.2.tgz",
"integrity": "sha512-psunOVrJjPka2SWlq53vxVWCA0Vt8pSXsXtn8pOLC0YTKFsUx+b7Z6quYUJcDZjCe1aAg9cKETek3Xal3Co8Tg==",
"version": "0.3.5",
"resolved": "https://registry.npmjs.org/@pipecat-ai/client-js/-/client-js-0.3.5.tgz",
"integrity": "sha512-qmhnDjwY2XUtLjww35ShsYf5TF9BCuAk0tIj0oHjpTe6v6QOlgKQt8JVCAdc32p5ycouzSZOeDFtBd2aNWuq1g==",
"license": "BSD-2-Clause",
"dependencies": {
"@types/events": "^3.0.3",
@@ -48,6 +77,19 @@
"uuid": "^10.0.0"
}
},
"node_modules/@pipecat-ai/small-webrtc-transport": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/@pipecat-ai/small-webrtc-transport/-/small-webrtc-transport-0.0.1.tgz",
"integrity": "sha512-WAOI7lT0V7cYOn0+qwUAryGxcOGe+wPVPEPzkR3qsM5GWIZ73spykZnuOndQGycq4UkcXVawCzERfNhpi+Uv7A==",
"license": "BSD-2-Clause",
"dependencies": {
"@daily-co/daily-js": "^0.73.0",
"dequal": "^2.0.3"
},
"peerDependencies": {
"@pipecat-ai/client-js": "~0.3.5"
}
},
"node_modules/@rollup/rollup-darwin-arm64": {
"version": "4.28.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.28.0.tgz",
@@ -62,6 +104,81 @@
"darwin"
]
},
"node_modules/@sentry-internal/browser-utils": {
"version": "8.55.0",
"resolved": "https://registry.npmjs.org/@sentry-internal/browser-utils/-/browser-utils-8.55.0.tgz",
"integrity": "sha512-ROgqtQfpH/82AQIpESPqPQe0UyWywKJsmVIqi3c5Fh+zkds5LUxnssTj3yNd1x+kxaPDVB023jAP+3ibNgeNDw==",
"license": "MIT",
"dependencies": {
"@sentry/core": "8.55.0"
},
"engines": {
"node": ">=14.18"
}
},
"node_modules/@sentry-internal/feedback": {
"version": "8.55.0",
"resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-8.55.0.tgz",
"integrity": "sha512-cP3BD/Q6pquVQ+YL+rwCnorKuTXiS9KXW8HNKu4nmmBAyf7urjs+F6Hr1k9MXP5yQ8W3yK7jRWd09Yu6DHWOiw==",
"license": "MIT",
"dependencies": {
"@sentry/core": "8.55.0"
},
"engines": {
"node": ">=14.18"
}
},
"node_modules/@sentry-internal/replay": {
"version": "8.55.0",
"resolved": "https://registry.npmjs.org/@sentry-internal/replay/-/replay-8.55.0.tgz",
"integrity": "sha512-roCDEGkORwolxBn8xAKedybY+Jlefq3xYmgN2fr3BTnsXjSYOPC7D1/mYqINBat99nDtvgFvNfRcZPiwwZ1hSw==",
"license": "MIT",
"dependencies": {
"@sentry-internal/browser-utils": "8.55.0",
"@sentry/core": "8.55.0"
},
"engines": {
"node": ">=14.18"
}
},
"node_modules/@sentry-internal/replay-canvas": {
"version": "8.55.0",
"resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-8.55.0.tgz",
"integrity": "sha512-nIkfgRWk1091zHdu4NbocQsxZF1rv1f7bbp3tTIlZYbrH62XVZosx5iHAuZG0Zc48AETLE7K4AX9VGjvQj8i9w==",
"license": "MIT",
"dependencies": {
"@sentry-internal/replay": "8.55.0",
"@sentry/core": "8.55.0"
},
"engines": {
"node": ">=14.18"
}
},
"node_modules/@sentry/browser": {
"version": "8.55.0",
"resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-8.55.0.tgz",
"integrity": "sha512-1A31mCEWCjaMxJt6qGUK+aDnLDcK6AwLAZnqpSchNysGni1pSn1RWSmk9TBF8qyTds5FH8B31H480uxMPUJ7Cw==",
"license": "MIT",
"dependencies": {
"@sentry-internal/browser-utils": "8.55.0",
"@sentry-internal/feedback": "8.55.0",
"@sentry-internal/replay": "8.55.0",
"@sentry-internal/replay-canvas": "8.55.0",
"@sentry/core": "8.55.0"
},
"engines": {
"node": ">=14.18"
}
},
"node_modules/@sentry/core": {
"version": "8.55.0",
"resolved": "https://registry.npmjs.org/@sentry/core/-/core-8.55.0.tgz",
"integrity": "sha512-6g7jpbefjHYs821Z+EBJ8r4Z7LT5h80YSWRJaylGS4nW5W5Z2KXzpdnyFarv37O7QjauzVC2E+PABmpkw5/JGA==",
"license": "MIT",
"engines": {
"node": ">=14.18"
}
},
"node_modules/@swc/core": {
"version": "1.10.14",
"resolved": "https://registry.npmjs.org/@swc/core/-/core-1.10.14.tgz",
@@ -171,6 +288,12 @@
"vite": "^4 || ^5 || ^6"
}
},
"node_modules/bowser": {
"version": "2.11.0",
"resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz",
"integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==",
"license": "MIT"
},
"node_modules/clone-deep": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz",
@@ -185,6 +308,15 @@
"node": ">=6"
}
},
"node_modules/dequal": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
"integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/esbuild": {
"version": "0.24.0",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.24.0.tgz",
@@ -334,6 +466,12 @@
"node": "^10 || ^12 || >=14"
}
},
"node_modules/regenerator-runtime": {
"version": "0.14.1",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz",
"integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==",
"license": "MIT"
},
"node_modules/rollup": {
"version": "4.28.0",
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.28.0.tgz",

3
examples/aiortc/video-transform/client/typescript/package.json → examples/p2p-webrtc/video-transform/client/typescript/package.json
View File

@@ -18,6 +18,7 @@
"vite": "^6.0.2"
},
"dependencies": {
"@pipecat-ai/client-js": "^0.3.2"
"@pipecat-ai/client-js": "^0.3.2",
"@pipecat-ai/small-webrtc-transport": "^0.0.1"
}
}

116
examples/aiortc/video-transform/client/typescript/src/app.ts → examples/p2p-webrtc/video-transform/client/typescript/src/app.ts
View File

@@ -1,4 +1,7 @@
import {SmallWebRTCTransport} from "./smallWebRTCTransport";
import {
SmallWebRTCTransport
} from "@pipecat-ai/small-webrtc-transport";
import {Participant, RTVIClient, RTVIClientOptions} from "@pipecat-ai/client-js";
class WebRTCApp {
@@ -16,29 +19,76 @@ class WebRTCApp {
private debugLog: HTMLElement | null = null;
private statusSpan: HTMLElement | null = null;
private smallWebRTCTransport: SmallWebRTCTransport;
private declare smallWebRTCTransport: SmallWebRTCTransport;
private declare rtviClient: RTVIClient;
constructor() {
this.setupDOMElements();
this.setupDOMEventListeners();
this.smallWebRTCTransport = new SmallWebRTCTransport({
onConnected: () => {
this.onConnectedHandler()
},
onDisconnected: () => {
this.onDisconnectedHandler()
},
onLog: (message: string) => {
this.log(message)
},
onTrackStarted: (track: MediaStreamTrack) => {
this.onTrackStarted(track)
}
});
this.initializeRTVIClient()
void this.populateDevices();
}
private initializeRTVIClient(): void {
const transport = new SmallWebRTCTransport();
const RTVIConfig: RTVIClientOptions = {
// need to understand why it is complaining
// @ts-ignore
transport,
params: {
baseUrl: "/api/offer"
},
enableMic: true,
enableCam: true,
callbacks: {
onTransportStateChanged: (state) => {
this.log(`Transport state: ${state}`)
},
onConnected: () => {
this.onConnectedHandler()
},
onBotReady: () => {
this.log("Bot is ready.")
},
onDisconnected: () => {
this.onDisconnectedHandler()
},
onUserStartedSpeaking: () => {
this.log("User started speaking.")
},
onUserStoppedSpeaking: () => {
this.log("User stopped speaking.")
},
onBotStartedSpeaking: () => {
this.log("Bot started speaking.")
},
onBotStoppedSpeaking: () => {
this.log("Bot stopped speaking.")
},
onUserTranscript: (transcript) => {
if (transcript.final) {
this.log(`User transcript: ${transcript.text}`)
}
},
onBotTranscript: (transcript) => {
this.log(`Bot transcript: ${transcript.text}`)
},
onTrackStarted: (track: MediaStreamTrack, participant?: Participant) => {
if (participant?.local) {
return
}
this.onBotTrackStarted(track)
},
onServerMessage: (msg) => {
this.log(`Server message: ${msg}`)
}
},
}
RTVIConfig.customConnectHandler = () => Promise.resolve();
this.rtviClient = new RTVIClient(RTVIConfig);
this.smallWebRTCTransport = transport
}
private setupDOMElements(): void {
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
@@ -58,6 +108,16 @@ class WebRTCApp {
private setupDOMEventListeners(): void {
this.connectBtn.addEventListener("click", () => this.start());
this.disconnectBtn.addEventListener("click", () => this.stop());
this.audioInput.addEventListener("change", (e) => {
// @ts-ignore
let audioDevice = e.target?.value
this.rtviClient.updateMic(audioDevice)
})
this.videoInput.addEventListener("change", (e) => {
// @ts-ignore
let videoDevice = e.target?.value
this.rtviClient.updateCam(videoDevice)
})
}
private log(message: string): void {
@@ -96,7 +156,7 @@ class WebRTCApp {
if (this.disconnectBtn) this.disconnectBtn.disabled = true;
}
private onTrackStarted(track: MediaStreamTrack) {
private onBotTrackStarted(track: MediaStreamTrack) {
if (track.kind === 'video') {
this.videoElement.srcObject = new MediaStream([track]);
} else {
@@ -117,9 +177,9 @@ class WebRTCApp {
};
try {
const audioDevices = await this.smallWebRTCTransport.getAllMics();
const audioDevices = await this.rtviClient.getAllMics();
populateSelect(this.audioInput, audioDevices);
const videoDevices = await this.smallWebRTCTransport.getAllCams();
const videoDevices = await this.rtviClient.getAllCams();
populateSelect(this.videoInput, videoDevices);
} catch (e) {
alert(e);
@@ -132,15 +192,19 @@ class WebRTCApp {
this.connectBtn.disabled = true;
this.updateStatus("Connecting")
const audioDevice = this.audioInput.value;
const audioCodec = this.audioCodec.value;
const videoDevice = this.videoInput.value;
const videoCodec = this.videoCodec.value;
await this.smallWebRTCTransport.start(audioDevice, audioCodec, videoCodec, videoDevice)
this.smallWebRTCTransport.setAudioCodec(this.audioCodec.value)
this.smallWebRTCTransport.setVideoCodec(this.videoCodec.value)
try {
await this.rtviClient.connect()
} catch (e) {
console.log(`Failed to connect ${e}`)
this.stop()
}
}
private stop(): void {
this.smallWebRTCTransport.stop()
void this.rtviClient.disconnect()
}
}

0
examples/aiortc/video-transform/client/typescript/src/style.css → examples/p2p-webrtc/video-transform/client/typescript/src/style.css
View File

0
examples/aiortc/video-transform/client/typescript/tsconfig.json → examples/p2p-webrtc/video-transform/client/typescript/tsconfig.json
View File

0
examples/aiortc/video-transform/client/typescript/vite.config.js → examples/p2p-webrtc/video-transform/client/typescript/vite.config.js
View File

0
examples/aiortc/video-transform/server/aiortc_bot.py → examples/p2p-webrtc/video-transform/server/bot.py
View File

0
examples/aiortc/video-transform/server/env.example → examples/p2p-webrtc/video-transform/server/env.example
View File

0
examples/aiortc/video-transform/server/requirements.txt → examples/p2p-webrtc/video-transform/server/requirements.txt
View File

2
examples/aiortc/video-transform/server/server.py → examples/p2p-webrtc/video-transform/server/server.py
View File

@@ -5,7 +5,7 @@ from contextlib import asynccontextmanager
from typing import Dict
import uvicorn
from aiortc_bot import run_bot
from bot import run_bot
from dotenv import load_dotenv
from fastapi import BackgroundTasks, FastAPI

54
examples/p2p-webrtc/voice-agent/README.md Normal file
View File

@@ -0,0 +1,54 @@
# Voice Agent
A Pipecat example demonstrating the simplest way to create a voice agent using `SmallWebRTCTransport`.
## 🚀 Quick Start
### 1️⃣ Start the Bot Server
#### 🔧 Set Up the Environment
1. Create and activate a virtual environment:
```bash
python3 -m venv venv
source venv/bin/activate # On Windows: venv\Scripts\activate
```
2. Install dependencies:
```bash
pip install -r requirements.txt
```
3. Configure environment variables:
- Copy `env.example` to `.env`
```bash
cp env.example .env
```
- Add your API keys
#### ▶️ Run the Server
```bash
python server.py
```
### 2️⃣ Connect Using the Client App
Open your browser and visit:
```
http://localhost:7860
```
## 📌 Requirements
- Python **3.10+**
- Node.js **16+** (for JavaScript components)
- Google API Key
- Modern web browser with WebRTC support
---
### 💡 Notes
- Ensure all dependencies are installed before running the server.
- Check the `.env` file for missing configurations.
- WebRTC requires a secure environment (HTTPS) for full functionality in production.
Happy coding! 🎉

0
examples/aiortc/voice-agent/aiortc_bot.py → examples/p2p-webrtc/voice-agent/bot.py
View File

0
examples/aiortc/voice-agent/env.example → examples/p2p-webrtc/voice-agent/env.example
View File

0
examples/aiortc/voice-agent/index.html → examples/p2p-webrtc/voice-agent/index.html
View File

0
examples/aiortc/voice-agent/requirements.txt → examples/p2p-webrtc/voice-agent/requirements.txt
View File

2
examples/aiortc/voice-agent/server.py → examples/p2p-webrtc/voice-agent/server.py
View File

@@ -5,7 +5,7 @@ from contextlib import asynccontextmanager
from typing import Dict
import uvicorn
from aiortc_bot import run_bot
from bot import run_bot
from dotenv import load_dotenv
from fastapi import BackgroundTasks, FastAPI
from fastapi.responses import FileResponse