From f884c93826dfb914a50befe386ec53ffc6cb184a Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 25 Mar 2025 17:32:25 -0300 Subject: [PATCH 1/5] Refactoring the video-transform example to use pipecat client. --- .../client/typescript/src/app.ts | 116 ++++- .../typescript/src/smallWebRTCTransport.ts | 473 ------------------ 2 files changed, 90 insertions(+), 499 deletions(-) delete mode 100644 examples/aiortc/video-transform/client/typescript/src/smallWebRTCTransport.ts diff --git a/examples/aiortc/video-transform/client/typescript/src/app.ts b/examples/aiortc/video-transform/client/typescript/src/app.ts index e2fabc1eb..6402d4c9a 100644 --- a/examples/aiortc/video-transform/client/typescript/src/app.ts +++ b/examples/aiortc/video-transform/client/typescript/src/app.ts @@ -1,4 +1,7 @@ -import {SmallWebRTCTransport} from "./smallWebRTCTransport"; +import { + SmallWebRTCTransport +} from "@pipecat-ai/small-webrtc-transport"; +import {Participant, RTVIClient, RTVIClientOptions} from "@pipecat-ai/client-js"; class WebRTCApp { @@ -16,29 +19,76 @@ class WebRTCApp { private debugLog: HTMLElement | null = null; private statusSpan: HTMLElement | null = null; - private smallWebRTCTransport: SmallWebRTCTransport; + private declare smallWebRTCTransport: SmallWebRTCTransport; + private declare rtviClient: RTVIClient; constructor() { this.setupDOMElements(); this.setupDOMEventListeners(); - - this.smallWebRTCTransport = new SmallWebRTCTransport({ - onConnected: () => { - this.onConnectedHandler() - }, - onDisconnected: () => { - this.onDisconnectedHandler() - }, - onLog: (message: string) => { - this.log(message) - }, - onTrackStarted: (track: MediaStreamTrack) => { - this.onTrackStarted(track) - } - }); + this.initializeRTVIClient() void this.populateDevices(); } + private initializeRTVIClient(): void { + const transport = new SmallWebRTCTransport(); + const RTVIConfig: RTVIClientOptions = { + // need to understand why it is complaining + // @ts-ignore + transport, + params: { + baseUrl: "/api/offer" + }, + enableMic: true, + enableCam: true, + callbacks: { + onTransportStateChanged: (state) => { + this.log(`Transport state: ${state}`) + }, + onConnected: () => { + this.onConnectedHandler() + }, + onBotReady: () => { + this.log("Bot is ready.") + }, + onDisconnected: () => { + this.onDisconnectedHandler() + }, + onUserStartedSpeaking: () => { + this.log("User started speaking.") + }, + onUserStoppedSpeaking: () => { + this.log("User stopped speaking.") + }, + onBotStartedSpeaking: () => { + this.log("Bot started speaking.") + }, + onBotStoppedSpeaking: () => { + this.log("Bot stopped speaking.") + }, + onUserTranscript: (transcript) => { + if (transcript.final) { + this.log(`User transcript: ${transcript.text}`) + } + }, + onBotTranscript: (transcript) => { + this.log(`Bot transcript: ${transcript.text}`) + }, + onTrackStarted: (track: MediaStreamTrack, participant?: Participant) => { + if (participant?.local) { + return + } + this.onBotTrackStarted(track) + }, + onServerMessage: (msg) => { + this.log(`Server message: ${msg}`) + } + }, + } + RTVIConfig.customConnectHandler = () => Promise.resolve(); + this.rtviClient = new RTVIClient(RTVIConfig); + this.smallWebRTCTransport = transport + } + private setupDOMElements(): void { this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement; this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement; @@ -58,6 +108,16 @@ class WebRTCApp { private setupDOMEventListeners(): void { this.connectBtn.addEventListener("click", () => this.start()); this.disconnectBtn.addEventListener("click", () => this.stop()); + this.audioInput.addEventListener("change", (e) => { + // @ts-ignore + let audioDevice = e.target?.value + this.rtviClient.updateMic(audioDevice) + }) + this.videoInput.addEventListener("change", (e) => { + // @ts-ignore + let videoDevice = e.target?.value + this.rtviClient.updateCam(videoDevice) + }) } private log(message: string): void { @@ -96,7 +156,7 @@ class WebRTCApp { if (this.disconnectBtn) this.disconnectBtn.disabled = true; } - private onTrackStarted(track: MediaStreamTrack) { + private onBotTrackStarted(track: MediaStreamTrack) { if (track.kind === 'video') { this.videoElement.srcObject = new MediaStream([track]); } else { @@ -117,9 +177,9 @@ class WebRTCApp { }; try { - const audioDevices = await this.smallWebRTCTransport.getAllMics(); + const audioDevices = await this.rtviClient.getAllMics(); populateSelect(this.audioInput, audioDevices); - const videoDevices = await this.smallWebRTCTransport.getAllCams(); + const videoDevices = await this.rtviClient.getAllCams(); populateSelect(this.videoInput, videoDevices); } catch (e) { alert(e); @@ -132,15 +192,19 @@ class WebRTCApp { this.connectBtn.disabled = true; this.updateStatus("Connecting") - const audioDevice = this.audioInput.value; - const audioCodec = this.audioCodec.value; - const videoDevice = this.videoInput.value; - const videoCodec = this.videoCodec.value; - await this.smallWebRTCTransport.start(audioDevice, audioCodec, videoCodec, videoDevice) + this.smallWebRTCTransport.setAudioCodec(this.audioCodec.value) + this.smallWebRTCTransport.setVideoCodec(this.videoCodec.value) + try { + await this.rtviClient.connect() + } catch (e) { + console.log(`Failed to connect ${e}`) + this.stop() + } + } private stop(): void { - this.smallWebRTCTransport.stop() + void this.rtviClient.disconnect() } } diff --git a/examples/aiortc/video-transform/client/typescript/src/smallWebRTCTransport.ts b/examples/aiortc/video-transform/client/typescript/src/smallWebRTCTransport.ts deleted file mode 100644 index 05de049c0..000000000 --- a/examples/aiortc/video-transform/client/typescript/src/smallWebRTCTransport.ts +++ /dev/null @@ -1,473 +0,0 @@ -// TODO: we should refactor everything that is inside this class, -// and create a Pipecat client transport here -// https://github.com/pipecat-ai/pipecat-client-web-transports - -const SIGNALLING_TYPE = "signalling"; - -enum SignallingMessage { - RENEGOTIATE = "renegotiate", -} - -// Interface for the structure of the signalling message -interface SignallingMessageObject { - type: string; - message: SignallingMessage; -} - -export type SmallWebRTCTransportCallbacks = { - onLog(message: string): void; - onTrackStarted(track: MediaStreamTrack): void; - onConnected(): void; - onDisconnected(): void; -} - -export class SmallWebRTCTransport { - - private _callbacks: SmallWebRTCTransportCallbacks; - - private pc: RTCPeerConnection | null = null; - private dc: RTCDataChannel | null = null; - private audioCodec: string | null = null; - private videoCodec: string | null = null; - private pc_id: string | null = null; - - private reconnectionAttempts = 0; - private maxReconnectionAttempts = 3; - private isReconnecting = false; - private keepAliveInterval: number | null = null; - private audioDevice: string | undefined; - private videoDevice: string | undefined; - - constructor(callbacks: SmallWebRTCTransportCallbacks) { - this._callbacks = callbacks - // for testing reconnections - // @ts-ignore - window.attemptReconnection = this.attemptReconnection.bind(this) - } - - private log(message: string): void { - console.log(message); - this._callbacks.onLog(message) - } - - private createPeerConnection(): RTCPeerConnection { - const config: RTCConfiguration = { - iceServers: [{ urls: ['stun:stun.l.google.com:19302'] }] - }; - - let pc = new RTCPeerConnection(config); - - pc.addEventListener('icegatheringstatechange', () => { - this.log(`iceGatheringState: ${this.pc!.iceGatheringState}`) - }); - this.log(`iceGatheringState: ${pc.iceGatheringState}`) - - pc.addEventListener("iceconnectionstatechange", () => this.handleICEConnectionStateChange()); - - pc.addEventListener("connectionstatechange", () => this.handleConnectionStateChange()); - - this.log(`iceConnectionState: ${pc.iceConnectionState}`) - - pc.addEventListener('signalingstatechange', () => { - this.log(`signalingState: ${this.pc!.signalingState}`) - if (this.pc!.signalingState == 'stable') { - this.handleReconnectionCompleted() - } - }); - this.log(`signalingState: ${pc.signalingState}`) - - pc.addEventListener('track', (evt: RTCTrackEvent) => { - this.log(`Received new track ${evt.track.kind}`) - this._callbacks.onTrackStarted(evt.track) - }); - - return pc; - } - - private handleICEConnectionStateChange(): void { - if (!this.pc) return; - this.log(`ICE Connection State: ${this.pc.iceConnectionState}`); - - if (this.pc.iceConnectionState === "failed") { - this.log("ICE connection failed, attempting restart."); - void this.attemptReconnection(true); - } else if (this.pc.iceConnectionState === "disconnected") { - // Waiting before trying to reconnect to see if it handles it automatically - setTimeout(() => { - if (this.pc?.iceConnectionState === "disconnected") { - this.log("Still disconnected, attempting reconnection."); - void this.attemptReconnection(true); - } - }, 5000); - } - } - - private handleReconnectionCompleted() { - this.reconnectionAttempts = 0; - this.isReconnecting = false; - } - - private handleConnectionStateChange(): void { - if (!this.pc) return; - this.log(`Connection State: ${this.pc.connectionState}`); - - if (this.pc.connectionState === "connected") { - this.handleReconnectionCompleted() - this._callbacks.onConnected(); - } else if (this.pc.connectionState === "failed") { - void this.attemptReconnection(true); - } - } - - private async attemptReconnection(recreatePeerConnection: boolean = false): Promise { - if (this.isReconnecting) { - this.log("Reconnection already in progress, skipping."); - return; - } - if (this.reconnectionAttempts >= this.maxReconnectionAttempts) { - this.log("Max reconnection attempts reached. Stopping transport."); - this.stop(); - return; - } - this.isReconnecting = true; - this.reconnectionAttempts++; - this.log(`Reconnection attempt ${this.reconnectionAttempts}...`); - // aiortc it is not working fine when just trying to restart the ice - // so in this case we are creating a new peer connection on both sides - if (recreatePeerConnection) { - const oldPC = this.pc - await this.startNewPeerConnection(recreatePeerConnection) - if (oldPC) { - this.log("closing old peer connection") - this.closePeerConnection(oldPC) - } - } else { - await this.negotiate(); - } - } - - private async negotiate(recreatePeerConnection: boolean = false): Promise { - if (!this.pc) { - return Promise.reject('Peer connection is not initialized'); - } - - try { - // Create offer - const offer = await this.pc.createOffer(); - await this.pc.setLocalDescription(offer); - - // Wait for ICE gathering to complete - /*await new Promise((resolve) => { - if (this.pc!.iceGatheringState === 'complete') { - resolve(); - } else { - const checkState = () => { - if (this.pc!.iceGatheringState === 'complete') { - this.pc!.removeEventListener('icegatheringstatechange', checkState); - resolve(); - } - }; - this.pc!.addEventListener('icegatheringstatechange', checkState); - } - });*/ - - let offerSdp = this.pc!.localDescription!; - let codec: string; - - // Filter audio codec - if (this.audioCodec && this.audioCodec !== 'default') { - // @ts-ignore - offerSdp.sdp = this.sdpFilterCodec('audio', this.audioCodec, offerSdp.sdp); - } - - // Filter video codec - if (this.videoCodec && this.videoCodec !== 'default') { - // @ts-ignore - offerSdp.sdp = this.sdpFilterCodec('video', this.videoCodec, offerSdp.sdp); - } - - this.log(`Will create offer for peerId: ${this.pc_id}`) - - // Send offer to server - const response = await fetch('/api/offer', { - body: JSON.stringify({ - sdp: offerSdp.sdp, - type: offerSdp.type, - pc_id: this.pc_id, - restart_pc: recreatePeerConnection - }), - headers: { - 'Content-Type': 'application/json', - }, - method: 'POST', - }); - - const answer: RTCSessionDescriptionInit = await response.json(); - // @ts-ignore - this.pc_id = answer.pc_id - // @ts-ignore - this.log(`Received answer for peer connection id ${answer.pc_id}`) - await this.pc!.setRemoteDescription(answer); - this.log(`Remote candidate supports trickle ice: ${this.pc.canTrickleIceCandidates}`) - } catch (e) { - this.log(`Reconnection attempt ${this.reconnectionAttempts} failed: ${e}`); - this.isReconnecting = false - setTimeout(() => this.attemptReconnection(true), 2000); - } - } - - private addInitialTransceivers() { - // Transceivers always appear in creation-order for both peers - // For now we are only considering that we are going to have 02 transceivers, - // one for audio and one for video - this.pc!.addTransceiver('audio', { direction: 'sendrecv' }); - this.pc!.addTransceiver('video', { direction: 'sendrecv' }); - } - - private getAudioTransceiver() { - // Transceivers always appear in creation-order for both peers - // Look at addInitialTransceivers - return this.pc!.getTransceivers()[0]; - } - - private getVideoTransceiver() { - // Transceivers always appear in creation-order for both peers - // Look at addInitialTransceivers - return this.pc!.getTransceivers()[1]; - } - - async start(audioDevice: string | undefined, audioCodec: string, videoCodec: string, videoDevice: string | undefined): Promise { - this.audioDevice = audioDevice - this.videoDevice = videoDevice - this.audioCodec = audioCodec - this.videoCodec = videoCodec - await this.startNewPeerConnection() - } - - private async startNewPeerConnection(recreatePeerConnection: boolean = false) { - this.pc = this.createPeerConnection(); - this.addInitialTransceivers(); - this.dc = this.createDataChannel('chat', { ordered: true }); - await this.addUserMedias(); - await this.negotiate(recreatePeerConnection); - } - - private async addUserMedias(): Promise { - this.log("Will send the audio and video"); - const constraints = this.createMediaConstraints(); - - if (constraints.audio || constraints.video) { - try { - const stream = await navigator.mediaDevices.getUserMedia(constraints); - - let videoTrack = stream.getVideoTracks()[0] - await this.getVideoTransceiver().sender.replaceTrack(videoTrack); - - let audioTrack = stream.getAudioTracks()[0] - await this.getAudioTransceiver().sender.replaceTrack(audioTrack); - } catch (err) { - alert(`Could not acquire media: ${err}`); - } - } - } - - // Method to handle a general message (this can be expanded for other types of messages) - handleMessage(message: string): void { - try { - this.log(message) - - const messageObj = JSON.parse(message); // Type is `any` initially - - // Check if it's a signalling message - if (messageObj.type === SIGNALLING_TYPE) { - void this.handleSignallingMessage(messageObj as SignallingMessageObject); // Delegate to handleSignallingMessage - } else { - // implement to handle the other messages in the future - } - } catch (error) { - console.error("Failed to parse JSON message:", error); - } - } - - // Method to handle signalling messages specifically - async handleSignallingMessage(messageObj: SignallingMessageObject): Promise { - // Cast the object to the correct type after verification - const signallingMessage = messageObj as SignallingMessageObject; - - // Handle different signalling message types - switch (signallingMessage.message) { - case SignallingMessage.RENEGOTIATE: - void this.attemptReconnection(false) - break; - - default: - console.warn("Unknown signalling message:", signallingMessage.message); - } - } - - private createDataChannel(label: string, options: RTCDataChannelInit): RTCDataChannel { - const dc = this.pc!.createDataChannel(label, options); - let timeStart: number | null = null; - - const getCurrentTimestamp = (): number => { - if (timeStart === null) { - timeStart = Date.now(); - return 0; - } - return Date.now() - timeStart; - }; - - dc.addEventListener('close', () => { - this.log("datachannel closed") - if (this.keepAliveInterval) { - clearInterval(this.keepAliveInterval) - this.keepAliveInterval = null - } - }); - - dc.addEventListener('open', () => { - this.log("datachannel opened") - // Sending message that the client is ready, just for testing - dc.send(JSON.stringify({id: 'clientReady', label: 'rtvi-ai', type:'client-ready'})) - // @ts-ignore - this.keepAliveInterval = setInterval(() => { - const message = 'ping: ' + new Date().getTime() - dc.send(message); - }, 1000); - - }); - - dc.addEventListener('message', (evt: MessageEvent) => { - let message = evt.data - this.handleMessage(message) - }); - - return dc; - } - - private createMediaConstraints(): MediaStreamConstraints { - const constraints: MediaStreamConstraints = { audio: false, video: false }; - - const audioConstraints: MediaTrackConstraints = {}; - if (this.audioDevice) audioConstraints.deviceId = { exact: this.audioDevice }; - - constraints.audio = Object.keys(audioConstraints).length ? audioConstraints : true; - - const videoConstraints: MediaTrackConstraints = {}; - if (this.videoDevice) videoConstraints.deviceId = { exact: this.videoDevice }; - - constraints.video = Object.keys(videoConstraints).length ? videoConstraints : true; - - return constraints; - } - - private closePeerConnection(pc:RTCPeerConnection) { - pc.getTransceivers().forEach((transceiver) => { - if (transceiver.stop) { - transceiver.stop(); - } - }); - - pc.getSenders().forEach((sender) => { - sender.track?.stop(); - }); - - pc.close(); - } - - stop(): void { - if (!this.pc) { - this.log("Peer connection is already closed or null."); - return; - } - - if (this.dc) { - this.dc.close(); - } - - this.closePeerConnection(this.pc) - - // For some reason after we close the peer connection, it is not triggering the listeners - this.pc_id = null - this.reconnectionAttempts = 0 - this.isReconnecting = false; - this._callbacks.onDisconnected() - } - - private async getAllDevices() { - return await navigator.mediaDevices.enumerateDevices(); - } - - async getAllCams() { - const devices = await this.getAllDevices(); - return devices.filter((d) => d.kind === "videoinput"); - } - - async getAllMics() { - const devices = await this.getAllDevices(); - return devices.filter((d) => d.kind === "audioinput"); - } - - private sdpFilterCodec(kind: string, codec: string, realSdp: string): string { - const allowed: number[] = []; - const rtxRegex = new RegExp('a=fmtp:(\\d+) apt=(\\d+)\\r$'); - const codecRegex = new RegExp('a=rtpmap:([0-9]+) ' + this.escapeRegExp(codec)); - const videoRegex = new RegExp('(m=' + kind + ' .*?)( ([0-9]+))*\\s*$'); - - const lines = realSdp.split('\n'); - - let isKind = false; - for (let i = 0; i < lines.length; i++) { - if (lines[i].startsWith('m=' + kind + ' ')) { - isKind = true; - } else if (lines[i].startsWith('m=')) { - isKind = false; - } - - if (isKind) { - const match = lines[i].match(codecRegex); - if (match) { - allowed.push(parseInt(match[1])); - } - - const matchRtx = lines[i].match(rtxRegex); - if (matchRtx && allowed.includes(parseInt(matchRtx[2]))) { - allowed.push(parseInt(matchRtx[1])); - } - } - } - - const skipRegex = 'a=(fmtp|rtcp-fb|rtpmap):([0-9]+)'; - let sdp = ''; - - isKind = false; - for (let i = 0; i < lines.length; i++) { - if (lines[i].startsWith('m=' + kind + ' ')) { - isKind = true; - } else if (lines[i].startsWith('m=')) { - isKind = false; - } - - if (isKind) { - const skipMatch = lines[i].match(skipRegex); - if (skipMatch && !allowed.includes(parseInt(skipMatch[2]))) { - continue; - } else if (lines[i].match(videoRegex)) { - sdp += lines[i].replace(videoRegex, '$1 ' + allowed.join(' ')) + '\n'; - } else { - sdp += lines[i] + '\n'; - } - } else { - sdp += lines[i] + '\n'; - } - } - - return sdp; - } - - private escapeRegExp(string: string): string { - return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - } - - -} \ No newline at end of file From 91a69b7029b0e19ca395b694397cb043d7a50a9b Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 27 Mar 2025 16:32:46 -0300 Subject: [PATCH 2/5] Improving the readmes for the webrtc examples. --- examples/aiortc/video-transform/README.rst | 66 ++++++++++++++-------- examples/aiortc/voice-agent/README.rst | 52 +++++++++++------ 2 files changed, 77 insertions(+), 41 deletions(-) diff --git a/examples/aiortc/video-transform/README.rst b/examples/aiortc/video-transform/README.rst index 531b0a8c9..7125e3e68 100644 --- a/examples/aiortc/video-transform/README.rst +++ b/examples/aiortc/video-transform/README.rst @@ -1,43 +1,59 @@ -# Video transform +# Video Transform -A Pipecat example demonstrating how to send and receive audio and video using SmallWebRTCTransport. -It also performs some image processing on the video frames using OpenCV. +A Pipecat example demonstrating how to send and receive audio and video using `SmallWebRTCTransport`. This project also applies image processing to video frames using OpenCV. -## Quick Start +## 🚀 Quick Start -### First, start the bot server: +### 1️⃣ Start the Bot Server -1. Navigate to the server directory: - ```bash - cd server - ``` -2. Create and activate a virtual environment: +#### 📂 Navigate to the Server Directory +```bash +cd server +``` + +#### 🔧 Set Up the Environment +1. Create and activate a virtual environment: ```bash python3 -m venv venv source venv/bin/activate # On Windows: venv\Scripts\activate ``` -3. Install requirements: + +2. Install dependencies: ```bash pip install -r requirements.txt ``` -4. Copy env.example to .env and configure: - - Add your API keys -5. Start the server: - ```bash - python server.py - ``` -### Next, connect using the client app: +3. Configure environment variables: + - Copy `env.example` to `.env` + ```bash + cp env.example .env + ``` + - Add your API keys + +#### ▶️ Run the Server +```bash +python server.py +``` + +### 2️⃣ Connect Using the Client App For client-side setup, refer to the [JavaScript Guide](client/typescript/README.md). -## Important Note - +## ⚠️ Important Note Ensure the bot server is running before using any client implementations. -## Requirements +## 📌 Requirements -- Python 3.10+ -- Node.js 16+ (for JavaScript) -- Google API key -- Modern web browser with WebRTC support \ No newline at end of file +- Python **3.10+** +- Node.js **16+** (for JavaScript components) +- Google API Key +- Modern web browser with WebRTC support + +--- + +### 💡 Notes +- Ensure all dependencies are installed before running the server. +- Check the `.env` file for missing configurations. +- WebRTC requires a secure environment (HTTPS) for full functionality in production. + +Happy coding! 🎉 \ No newline at end of file diff --git a/examples/aiortc/voice-agent/README.rst b/examples/aiortc/voice-agent/README.rst index 899adc861..17bf165af 100644 --- a/examples/aiortc/voice-agent/README.rst +++ b/examples/aiortc/voice-agent/README.rst @@ -1,34 +1,54 @@ -# Video transform +# Voice Agent -A Pipecat example demonstrating the simplest way to create a voice agent with SmallWebRTCTransport. +A Pipecat example demonstrating the simplest way to create a voice agent using `SmallWebRTCTransport`. -## Quick Start +## 🚀 Quick Start -### First, start the bot server: +### 1️⃣ Start the Bot Server +#### 🔧 Set Up the Environment 1. Create and activate a virtual environment: ```bash python3 -m venv venv source venv/bin/activate # On Windows: venv\Scripts\activate ``` -2. Install requirements: + +2. Install dependencies: ```bash pip install -r requirements.txt ``` -3. Copy env.example to .env and configure: - - Add your API keys -4. Start the server: + +3. Configure environment variables: + - Copy `env.example` to `.env` ```bash - python server.py + cp env.example .env ``` + - Add your API keys -### Next, connect using the client app: +#### ▶️ Run the Server +```bash +python server.py +``` -Visit http://localhost:7860 in your browser. +### 2️⃣ Connect Using the Client App -## Requirements +Open your browser and visit: +``` +http://localhost:7860 +``` -- Python 3.10+ -- Node.js 16+ (for JavaScript) -- Google API key -- Modern web browser with WebRTC support \ No newline at end of file +## 📌 Requirements + +- Python **3.10+** +- Node.js **16+** (for JavaScript components) +- Google API Key +- Modern web browser with WebRTC support + +--- + +### 💡 Notes +- Ensure all dependencies are installed before running the server. +- Check the `.env` file for missing configurations. +- WebRTC requires a secure environment (HTTPS) for full functionality in production. + +Happy coding! 🎉 \ No newline at end of file From 62cb0376f2eba25764e0249512b30d7618f18a60 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 27 Mar 2025 16:34:40 -0300 Subject: [PATCH 3/5] Changing the file types. --- examples/aiortc/video-transform/{README.rst => README.md} | 0 examples/aiortc/voice-agent/{README.rst => README.md} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename examples/aiortc/video-transform/{README.rst => README.md} (100%) rename examples/aiortc/voice-agent/{README.rst => README.md} (100%) diff --git a/examples/aiortc/video-transform/README.rst b/examples/aiortc/video-transform/README.md similarity index 100% rename from examples/aiortc/video-transform/README.rst rename to examples/aiortc/video-transform/README.md diff --git a/examples/aiortc/voice-agent/README.rst b/examples/aiortc/voice-agent/README.md similarity index 100% rename from examples/aiortc/voice-agent/README.rst rename to examples/aiortc/voice-agent/README.md From 311a5360add44b27f293b25d1689b855dc86e66e Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 27 Mar 2025 16:46:00 -0300 Subject: [PATCH 4/5] Renaming the example to p2p-webrtc --- examples/{aiortc => p2p-webrtc}/video-transform/README.md | 0 .../video-transform/client/typescript/README.md | 0 .../video-transform/client/typescript/index.html | 0 .../video-transform/client/typescript/package-lock.json | 0 .../video-transform/client/typescript/package.json | 0 .../video-transform/client/typescript/src/app.ts | 0 .../video-transform/client/typescript/src/style.css | 0 .../video-transform/client/typescript/tsconfig.json | 0 .../video-transform/client/typescript/vite.config.js | 0 .../aiortc_bot.py => p2p-webrtc/video-transform/server/bot.py} | 0 .../{aiortc => p2p-webrtc}/video-transform/server/env.example | 0 .../video-transform/server/requirements.txt | 0 .../{aiortc => p2p-webrtc}/video-transform/server/server.py | 2 +- examples/{aiortc => p2p-webrtc}/voice-agent/README.md | 0 .../voice-agent/aiortc_bot.py => p2p-webrtc/voice-agent/bot.py} | 0 examples/{aiortc => p2p-webrtc}/voice-agent/env.example | 0 examples/{aiortc => p2p-webrtc}/voice-agent/index.html | 0 examples/{aiortc => p2p-webrtc}/voice-agent/requirements.txt | 0 examples/{aiortc => p2p-webrtc}/voice-agent/server.py | 2 +- 19 files changed, 2 insertions(+), 2 deletions(-) rename examples/{aiortc => p2p-webrtc}/video-transform/README.md (100%) rename examples/{aiortc => p2p-webrtc}/video-transform/client/typescript/README.md (100%) rename examples/{aiortc => p2p-webrtc}/video-transform/client/typescript/index.html (100%) rename examples/{aiortc => p2p-webrtc}/video-transform/client/typescript/package-lock.json (100%) rename examples/{aiortc => p2p-webrtc}/video-transform/client/typescript/package.json (100%) rename examples/{aiortc => p2p-webrtc}/video-transform/client/typescript/src/app.ts (100%) rename examples/{aiortc => p2p-webrtc}/video-transform/client/typescript/src/style.css (100%) rename examples/{aiortc => p2p-webrtc}/video-transform/client/typescript/tsconfig.json (100%) rename examples/{aiortc => p2p-webrtc}/video-transform/client/typescript/vite.config.js (100%) rename examples/{aiortc/video-transform/server/aiortc_bot.py => p2p-webrtc/video-transform/server/bot.py} (100%) rename examples/{aiortc => p2p-webrtc}/video-transform/server/env.example (100%) rename examples/{aiortc => p2p-webrtc}/video-transform/server/requirements.txt (100%) rename examples/{aiortc => p2p-webrtc}/video-transform/server/server.py (98%) rename examples/{aiortc => p2p-webrtc}/voice-agent/README.md (100%) rename examples/{aiortc/voice-agent/aiortc_bot.py => p2p-webrtc/voice-agent/bot.py} (100%) rename examples/{aiortc => p2p-webrtc}/voice-agent/env.example (100%) rename examples/{aiortc => p2p-webrtc}/voice-agent/index.html (100%) rename examples/{aiortc => p2p-webrtc}/voice-agent/requirements.txt (100%) rename examples/{aiortc => p2p-webrtc}/voice-agent/server.py (98%) diff --git a/examples/aiortc/video-transform/README.md b/examples/p2p-webrtc/video-transform/README.md similarity index 100% rename from examples/aiortc/video-transform/README.md rename to examples/p2p-webrtc/video-transform/README.md diff --git a/examples/aiortc/video-transform/client/typescript/README.md b/examples/p2p-webrtc/video-transform/client/typescript/README.md similarity index 100% rename from examples/aiortc/video-transform/client/typescript/README.md rename to examples/p2p-webrtc/video-transform/client/typescript/README.md diff --git a/examples/aiortc/video-transform/client/typescript/index.html b/examples/p2p-webrtc/video-transform/client/typescript/index.html similarity index 100% rename from examples/aiortc/video-transform/client/typescript/index.html rename to examples/p2p-webrtc/video-transform/client/typescript/index.html diff --git a/examples/aiortc/video-transform/client/typescript/package-lock.json b/examples/p2p-webrtc/video-transform/client/typescript/package-lock.json similarity index 100% rename from examples/aiortc/video-transform/client/typescript/package-lock.json rename to examples/p2p-webrtc/video-transform/client/typescript/package-lock.json diff --git a/examples/aiortc/video-transform/client/typescript/package.json b/examples/p2p-webrtc/video-transform/client/typescript/package.json similarity index 100% rename from examples/aiortc/video-transform/client/typescript/package.json rename to examples/p2p-webrtc/video-transform/client/typescript/package.json diff --git a/examples/aiortc/video-transform/client/typescript/src/app.ts b/examples/p2p-webrtc/video-transform/client/typescript/src/app.ts similarity index 100% rename from examples/aiortc/video-transform/client/typescript/src/app.ts rename to examples/p2p-webrtc/video-transform/client/typescript/src/app.ts diff --git a/examples/aiortc/video-transform/client/typescript/src/style.css b/examples/p2p-webrtc/video-transform/client/typescript/src/style.css similarity index 100% rename from examples/aiortc/video-transform/client/typescript/src/style.css rename to examples/p2p-webrtc/video-transform/client/typescript/src/style.css diff --git a/examples/aiortc/video-transform/client/typescript/tsconfig.json b/examples/p2p-webrtc/video-transform/client/typescript/tsconfig.json similarity index 100% rename from examples/aiortc/video-transform/client/typescript/tsconfig.json rename to examples/p2p-webrtc/video-transform/client/typescript/tsconfig.json diff --git a/examples/aiortc/video-transform/client/typescript/vite.config.js b/examples/p2p-webrtc/video-transform/client/typescript/vite.config.js similarity index 100% rename from examples/aiortc/video-transform/client/typescript/vite.config.js rename to examples/p2p-webrtc/video-transform/client/typescript/vite.config.js diff --git a/examples/aiortc/video-transform/server/aiortc_bot.py b/examples/p2p-webrtc/video-transform/server/bot.py similarity index 100% rename from examples/aiortc/video-transform/server/aiortc_bot.py rename to examples/p2p-webrtc/video-transform/server/bot.py diff --git a/examples/aiortc/video-transform/server/env.example b/examples/p2p-webrtc/video-transform/server/env.example similarity index 100% rename from examples/aiortc/video-transform/server/env.example rename to examples/p2p-webrtc/video-transform/server/env.example diff --git a/examples/aiortc/video-transform/server/requirements.txt b/examples/p2p-webrtc/video-transform/server/requirements.txt similarity index 100% rename from examples/aiortc/video-transform/server/requirements.txt rename to examples/p2p-webrtc/video-transform/server/requirements.txt diff --git a/examples/aiortc/video-transform/server/server.py b/examples/p2p-webrtc/video-transform/server/server.py similarity index 98% rename from examples/aiortc/video-transform/server/server.py rename to examples/p2p-webrtc/video-transform/server/server.py index e99d21cf2..754c2ff69 100644 --- a/examples/aiortc/video-transform/server/server.py +++ b/examples/p2p-webrtc/video-transform/server/server.py @@ -5,7 +5,7 @@ from contextlib import asynccontextmanager from typing import Dict import uvicorn -from aiortc_bot import run_bot +from bot import run_bot from dotenv import load_dotenv from fastapi import BackgroundTasks, FastAPI diff --git a/examples/aiortc/voice-agent/README.md b/examples/p2p-webrtc/voice-agent/README.md similarity index 100% rename from examples/aiortc/voice-agent/README.md rename to examples/p2p-webrtc/voice-agent/README.md diff --git a/examples/aiortc/voice-agent/aiortc_bot.py b/examples/p2p-webrtc/voice-agent/bot.py similarity index 100% rename from examples/aiortc/voice-agent/aiortc_bot.py rename to examples/p2p-webrtc/voice-agent/bot.py diff --git a/examples/aiortc/voice-agent/env.example b/examples/p2p-webrtc/voice-agent/env.example similarity index 100% rename from examples/aiortc/voice-agent/env.example rename to examples/p2p-webrtc/voice-agent/env.example diff --git a/examples/aiortc/voice-agent/index.html b/examples/p2p-webrtc/voice-agent/index.html similarity index 100% rename from examples/aiortc/voice-agent/index.html rename to examples/p2p-webrtc/voice-agent/index.html diff --git a/examples/aiortc/voice-agent/requirements.txt b/examples/p2p-webrtc/voice-agent/requirements.txt similarity index 100% rename from examples/aiortc/voice-agent/requirements.txt rename to examples/p2p-webrtc/voice-agent/requirements.txt diff --git a/examples/aiortc/voice-agent/server.py b/examples/p2p-webrtc/voice-agent/server.py similarity index 98% rename from examples/aiortc/voice-agent/server.py rename to examples/p2p-webrtc/voice-agent/server.py index 9857e98d0..8d0617020 100644 --- a/examples/aiortc/voice-agent/server.py +++ b/examples/p2p-webrtc/voice-agent/server.py @@ -5,7 +5,7 @@ from contextlib import asynccontextmanager from typing import Dict import uvicorn -from aiortc_bot import run_bot +from bot import run_bot from dotenv import load_dotenv from fastapi import BackgroundTasks, FastAPI from fastapi.responses import FileResponse From 76f9626d35cbeb80333ffa001c09f5b5dd23fcd1 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 27 Mar 2025 17:48:32 -0300 Subject: [PATCH 5/5] Using the @pipecat-ai/small-webrtc-transport from npm. --- .../client/typescript/package-lock.json | 146 +++++++++++++++++- .../client/typescript/package.json | 3 +- 2 files changed, 144 insertions(+), 5 deletions(-) diff --git a/examples/p2p-webrtc/video-transform/client/typescript/package-lock.json b/examples/p2p-webrtc/video-transform/client/typescript/package-lock.json index 728bf5946..4fb1ecd50 100644 --- a/examples/p2p-webrtc/video-transform/client/typescript/package-lock.json +++ b/examples/p2p-webrtc/video-transform/client/typescript/package-lock.json @@ -9,7 +9,8 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "@pipecat-ai/client-js": "^0.3.2" + "@pipecat-ai/client-js": "^0.3.2", + "@pipecat-ai/small-webrtc-transport": "^0.0.1" }, "devDependencies": { "@types/node": "^22.13.1", @@ -18,6 +19,34 @@ "vite": "^6.0.2" } }, + "node_modules/@babel/runtime": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.0.tgz", + "integrity": "sha512-VtPOkrdPHZsKc/clNqyi9WUA8TINkZ4cGk63UUE3u4pmB2k+ZMQRDuIOagv8UVd6j7k0T3+RRIb7beKTebNbcw==", + "license": "MIT", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@daily-co/daily-js": { + "version": "0.73.0", + "resolved": "https://registry.npmjs.org/@daily-co/daily-js/-/daily-js-0.73.0.tgz", + "integrity": "sha512-Wz8c60hgmkx8fcEeDAi4L4J0rbafiihWKyXFyhYoFYPsw2OdChHpA4RYwIB+1enRws5IK+/HdmzFDYLQsB4A6w==", + "license": "BSD-2-Clause", + "dependencies": { + "@babel/runtime": "^7.12.5", + "@sentry/browser": "^8.33.1", + "bowser": "^2.8.1", + "dequal": "^2.0.3", + "events": "^3.1.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/@esbuild/darwin-arm64": { "version": "0.24.0", "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.24.0.tgz", @@ -36,9 +65,9 @@ } }, "node_modules/@pipecat-ai/client-js": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@pipecat-ai/client-js/-/client-js-0.3.2.tgz", - "integrity": "sha512-psunOVrJjPka2SWlq53vxVWCA0Vt8pSXsXtn8pOLC0YTKFsUx+b7Z6quYUJcDZjCe1aAg9cKETek3Xal3Co8Tg==", + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/@pipecat-ai/client-js/-/client-js-0.3.5.tgz", + "integrity": "sha512-qmhnDjwY2XUtLjww35ShsYf5TF9BCuAk0tIj0oHjpTe6v6QOlgKQt8JVCAdc32p5ycouzSZOeDFtBd2aNWuq1g==", "license": "BSD-2-Clause", "dependencies": { "@types/events": "^3.0.3", @@ -48,6 +77,19 @@ "uuid": "^10.0.0" } }, + "node_modules/@pipecat-ai/small-webrtc-transport": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/@pipecat-ai/small-webrtc-transport/-/small-webrtc-transport-0.0.1.tgz", + "integrity": "sha512-WAOI7lT0V7cYOn0+qwUAryGxcOGe+wPVPEPzkR3qsM5GWIZ73spykZnuOndQGycq4UkcXVawCzERfNhpi+Uv7A==", + "license": "BSD-2-Clause", + "dependencies": { + "@daily-co/daily-js": "^0.73.0", + "dequal": "^2.0.3" + }, + "peerDependencies": { + "@pipecat-ai/client-js": "~0.3.5" + } + }, "node_modules/@rollup/rollup-darwin-arm64": { "version": "4.28.0", "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.28.0.tgz", @@ -62,6 +104,81 @@ "darwin" ] }, + "node_modules/@sentry-internal/browser-utils": { + "version": "8.55.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/browser-utils/-/browser-utils-8.55.0.tgz", + "integrity": "sha512-ROgqtQfpH/82AQIpESPqPQe0UyWywKJsmVIqi3c5Fh+zkds5LUxnssTj3yNd1x+kxaPDVB023jAP+3ibNgeNDw==", + "license": "MIT", + "dependencies": { + "@sentry/core": "8.55.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry-internal/feedback": { + "version": "8.55.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-8.55.0.tgz", + "integrity": "sha512-cP3BD/Q6pquVQ+YL+rwCnorKuTXiS9KXW8HNKu4nmmBAyf7urjs+F6Hr1k9MXP5yQ8W3yK7jRWd09Yu6DHWOiw==", + "license": "MIT", + "dependencies": { + "@sentry/core": "8.55.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry-internal/replay": { + "version": "8.55.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/replay/-/replay-8.55.0.tgz", + "integrity": "sha512-roCDEGkORwolxBn8xAKedybY+Jlefq3xYmgN2fr3BTnsXjSYOPC7D1/mYqINBat99nDtvgFvNfRcZPiwwZ1hSw==", + "license": "MIT", + "dependencies": { + "@sentry-internal/browser-utils": "8.55.0", + "@sentry/core": "8.55.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry-internal/replay-canvas": { + "version": "8.55.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-8.55.0.tgz", + "integrity": "sha512-nIkfgRWk1091zHdu4NbocQsxZF1rv1f7bbp3tTIlZYbrH62XVZosx5iHAuZG0Zc48AETLE7K4AX9VGjvQj8i9w==", + "license": "MIT", + "dependencies": { + "@sentry-internal/replay": "8.55.0", + "@sentry/core": "8.55.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry/browser": { + "version": "8.55.0", + "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-8.55.0.tgz", + "integrity": "sha512-1A31mCEWCjaMxJt6qGUK+aDnLDcK6AwLAZnqpSchNysGni1pSn1RWSmk9TBF8qyTds5FH8B31H480uxMPUJ7Cw==", + "license": "MIT", + "dependencies": { + "@sentry-internal/browser-utils": "8.55.0", + "@sentry-internal/feedback": "8.55.0", + "@sentry-internal/replay": "8.55.0", + "@sentry-internal/replay-canvas": "8.55.0", + "@sentry/core": "8.55.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry/core": { + "version": "8.55.0", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-8.55.0.tgz", + "integrity": "sha512-6g7jpbefjHYs821Z+EBJ8r4Z7LT5h80YSWRJaylGS4nW5W5Z2KXzpdnyFarv37O7QjauzVC2E+PABmpkw5/JGA==", + "license": "MIT", + "engines": { + "node": ">=14.18" + } + }, "node_modules/@swc/core": { "version": "1.10.14", "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.10.14.tgz", @@ -171,6 +288,12 @@ "vite": "^4 || ^5 || ^6" } }, + "node_modules/bowser": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", + "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==", + "license": "MIT" + }, "node_modules/clone-deep": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", @@ -185,6 +308,15 @@ "node": ">=6" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/esbuild": { "version": "0.24.0", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.24.0.tgz", @@ -334,6 +466,12 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", + "license": "MIT" + }, "node_modules/rollup": { "version": "4.28.0", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.28.0.tgz", diff --git a/examples/p2p-webrtc/video-transform/client/typescript/package.json b/examples/p2p-webrtc/video-transform/client/typescript/package.json index a31f36b0f..e50084020 100644 --- a/examples/p2p-webrtc/video-transform/client/typescript/package.json +++ b/examples/p2p-webrtc/video-transform/client/typescript/package.json @@ -18,6 +18,7 @@ "vite": "^6.0.2" }, "dependencies": { - "@pipecat-ai/client-js": "^0.3.2" + "@pipecat-ai/client-js": "^0.3.2", + "@pipecat-ai/small-webrtc-transport": "^0.0.1" } }