wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated all examples with clients to use the new PipecatClient

Browse Source
This commit is contained in:
mattie ruth backman
2025-06-11 12:01:03 -04:00
committed by Mattie Ruth
parent 43049c865c
commit dc41ec7cb1
31 changed files with 571 additions and 621 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
examples/deployment/modal-example/client/javascript/src/app.js
View File

@@ -5,7 +5,7 @@
*/
/**
* RTVI Client Implementation
* Pipecat Client Implementation
*
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
* It handles audio/video streaming and manages the connection lifecycle.
@@ -16,7 +16,7 @@
* - Browser with WebRTC support
*/
import { RTVIClient, RTVIEvent } from '@pipecat-ai/client-js';
import { PipecatClient, RTVIEvent } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
/**
@@ -26,7 +26,7 @@ import { DailyTransport } from '@pipecat-ai/daily-transport';
class ChatbotClient {
constructor() {
// Initialize client state
this.rtviClient = null;
this.pcClient = null;
this.setupDOMElements();
this.initializeClientAndTransport();
this.setupEventListeners();
@@ -59,7 +59,7 @@ class ChatbotClient {
this.disconnectBtn.addEventListener('click', () => this.disconnect());
// Populate device selector
this.rtviClient.getAllMics().then((mics) => {
this.pcClient.getAllMics().then((mics) => {
console.log('Available mics:', mics);
mics.forEach((device) => {
const option = document.createElement('option');
@@ -71,16 +71,16 @@ class ChatbotClient {
this.deviceSelector.addEventListener('change', (event) => {
const selectedDeviceId = event.target.value;
console.log('Selected device ID:', selectedDeviceId);
this.rtviClient.updateMic(selectedDeviceId);
this.pcClient.updateMic(selectedDeviceId);
});
// Handle mic mute/unmute toggle
const micToggleBtn = document.getElementById('mic-toggle-btn');
micToggleBtn.addEventListener('click', () => {
let micEnabled = this.rtviClient.isMicEnabled;
let micEnabled = this.pcClient.isMicEnabled;
micToggleBtn.textContent = micEnabled ? 'Unmute Mic' : 'Mute Mic';
this.rtviClient.enableMic(!micEnabled);
this.pcClient.enableMic(!micEnabled);
// Add logic to mute/unmute the mic
if (micEnabled) {
console.log('Mic muted');
@@ -93,23 +93,12 @@ class ChatbotClient {
}
/**
* Set up the RTVI client and Daily transport
* Set up the Pipecat client and Daily transport
*/
async initializeClientAndTransport() {
// Initialize the RTVI client with a DailyTransport and our configuration
this.rtviClient = new RTVIClient({
// Initialize the Pipecat client with a DailyTransport and our configuration
this.pcClient = new PipecatClient({
transport: new DailyTransport(),
params: {
// REPLACE WITH YOUR MODAL URL ENDPOINT
baseUrl:
'https://<Modal workspace>--pipecat-modal-bot-launcher.modal.run',
endpoints: {
connect: '/connect',
},
requestData: {
bot_name: 'openai',
},
},
enableMic: true, // Enable microphone for user input
enableCam: false,
callbacks: {
@@ -176,8 +165,8 @@ class ChatbotClient {
// Set up listeners for media track events
this.setupTrackListeners();
await this.rtviClient.initDevices();
window.client = this.rtviClient;
await this.pcClient.initDevices();
window.client = this.pcClient;
}
/**
@@ -212,10 +201,10 @@ class ChatbotClient {
* This is called when the bot is ready or when the transport state changes to ready
*/
setupMediaTracks() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Get current tracks from the client
const tracks = this.rtviClient.tracks();
const tracks = this.pcClient.tracks();
// Set up any available bot tracks
if (tracks.bot?.audio) {
@@ -231,10 +220,10 @@ class ChatbotClient {
* This handles new tracks being added during the session
*/
setupTrackListeners() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks
if (!participant?.local) {
if (track.kind === 'audio') {
@@ -253,7 +242,7 @@ class ChatbotClient {
});
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
if (participant.local) {
this.log('Local mic muted');
return;
@@ -311,21 +300,27 @@ class ChatbotClient {
/**
* Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection
* This sets up the Pipecat client, initializes devices, and establishes the connection
*/
async connect() {
try {
const botSelector = document.getElementById('bot-selector');
const selectedBot = botSelector.value;
this.rtviClient.params.requestData.bot_name = selectedBot;
// Initialize audio/video devices
this.log('Initializing devices...');
await this.rtviClient.initDevices();
await this.pcClient.initDevices();
// Connect to the bot
this.log(`Connecting to bot: ${selectedBot}`);
await this.rtviClient.connect();
await this.pcClient.connect({
// REPLACE WITH YOUR MODAL URL ENDPOINT
endpoint:
'https://<your-workspace>--pipecat-modal-fastapi-app.modal.run/connect',
requestData: {
bot_name: selectedBot,
},
});
this.log('Connection complete');
} catch (error) {
@@ -336,9 +331,9 @@ class ChatbotClient {
this.updateStatus('Error');
// Clean up if there's an error
if (this.rtviClient) {
if (this.pcClient) {
try {
await this.rtviClient.disconnect();
await this.pcClient.disconnect();
} catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError.message}`);
}
@@ -350,10 +345,10 @@ class ChatbotClient {
* Disconnect from the bot and clean up media resources
*/
async disconnect() {
if (this.rtviClient) {
if (this.pcClient) {
try {
// Disconnect the RTVI client
await this.rtviClient.disconnect();
// Disconnect the Pipecat client
await this.pcClient.disconnect();
// Clean up audio
if (this.botAudio.srcObject) {

2
examples/fal-smart-turn/README.md
View File

@@ -44,7 +44,7 @@ Try the hosted version of the demo here: https://pcc-smart-turn.vercel.app/.
4. Run the server:
```bash
LOCAL=1 python server.py
LOCAL_RUN=1 python server.py
```
### Run the client

4
examples/fal-smart-turn/client/src/app/layout.tsx
View File

@@ -1,5 +1,5 @@
import './globals.css';
import { RTVIProvider } from '@/providers/RTVIProvider';
import { PipecatProvider } from '@/providers/PipecatProvider';
export const metadata = {
title: 'Pipecat React Client',
@@ -20,7 +20,7 @@ export default function RootLayout({
<link rel="icon" href="/favicon.svg" type="image/svg+xml" />
</head>
<body>
<RTVIProvider>{children}</RTVIProvider>
<PipecatProvider>{children}</PipecatProvider>
</body>
</html>
);

12
examples/fal-smart-turn/client/src/app/page.tsx
View File

@@ -1,22 +1,22 @@
'use client';
import {
RTVIClientAudio,
RTVIClientVideo,
useRTVIClientTransportState,
PipecatClientAudio,
PipecatClientVideo,
usePipecatClientTransportState,
} from '@pipecat-ai/client-react';
import { ConnectButton } from '../components/ConnectButton';
import { StatusDisplay } from '../components/StatusDisplay';
import { DebugDisplay } from '../components/DebugDisplay';
function BotVideo() {
const transportState = useRTVIClientTransportState();
const transportState = usePipecatClientTransportState();
const isConnected = transportState !== 'disconnected';
return (
<div className="bot-container">
<div className="video-container">
{isConnected && <RTVIClientVideo participant="bot" fit="cover" />}
{isConnected && <PipecatClientVideo participant="bot" fit="cover" />}
</div>
</div>
);
@@ -35,7 +35,7 @@ export default function Home() {
</div>
<DebugDisplay />
<RTVIClientAudio />
<PipecatClientAudio />
</div>
);
}

19
examples/fal-smart-turn/client/src/components/ConnectButton.tsx
View File

@@ -1,11 +1,17 @@
import {
useRTVIClient,
useRTVIClientTransportState,
usePipecatClient,
usePipecatClientTransportState,
} from '@pipecat-ai/client-react';
// Get the API base URL from environment variables
// Default to "/api" if not specified
// "/api" is the default for Next.js API routes and used
// for the Pipecat Cloud deployed agent
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
export function ConnectButton() {
const client = useRTVIClient();
const transportState = useRTVIClientTransportState();
const client = usePipecatClient();
const transportState = usePipecatClientTransportState();
const isConnected = ['connected', 'ready'].includes(transportState);
const handleClick = async () => {
@@ -18,7 +24,10 @@ export function ConnectButton() {
if (isConnected) {
await client.disconnect();
} else {
await client.connect();
await client.connect({
endpoint: `${API_BASE_URL}/connect`,
requestData: { foo: 'bar' },
});
}
} catch (error) {
console.error('Connection error:', error);

4
examples/fal-smart-turn/client/src/components/DebugDisplay.tsx
View File

@@ -6,7 +6,7 @@ import {
TranscriptData,
BotLLMTextData,
} from '@pipecat-ai/client-js';
import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
import { usePipecatClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
import './DebugDisplay.css';
interface SmartTurnResultData {
@@ -20,7 +20,7 @@ interface SmartTurnResultData {
export function DebugDisplay() {
const debugLogRef = useRef<HTMLDivElement>(null);
const client = useRTVIClient();
const client = usePipecatClient();
const log = useCallback((message: string) => {
if (!debugLogRef.current) return;

4
examples/fal-smart-turn/client/src/components/StatusDisplay.tsx
View File

@@ -1,7 +1,7 @@
import { useRTVIClientTransportState } from '@pipecat-ai/client-react';
import { usePipecatClientTransportState } from '@pipecat-ai/client-react';
export function StatusDisplay() {
const transportState = useRTVIClientTransportState();
const transportState = usePipecatClientTransportState();
return (
<div className="status">

28
examples/fal-smart-turn/client/src/providers/PipecatProvider.tsx Normal file
View File

@@ -0,0 +1,28 @@
'use client';
import { PipecatClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { PipecatClientProvider } from '@pipecat-ai/client-react';
import { PropsWithChildren, useEffect, useState } from 'react';
export function PipecatProvider({ children }: PropsWithChildren) {
const [client, setClient] = useState<PipecatClient | null>(null);
useEffect(() => {
const pcClient = new PipecatClient({
transport: new DailyTransport(),
enableMic: true,
enableCam: false,
});
setClient(pcClient);
}, []);
if (!client) {
return null;
}
return (
<PipecatClientProvider client={client}>{children}</PipecatClientProvider>
);
}

43
examples/fal-smart-turn/client/src/providers/RTVIProvider.tsx
View File

@@ -1,43 +0,0 @@
'use client';
import { RTVIClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { RTVIClientProvider } from '@pipecat-ai/client-react';
import { PropsWithChildren, useEffect, useState } from 'react';
// Get the API base URL from environment variables
// Default to "/api" if not specified
// "/api" is the default for Next.js API routes and used
// for the Pipecat Cloud deployed agent
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
console.log('Using API base URL:', API_BASE_URL);
export function RTVIProvider({ children }: PropsWithChildren) {
const [client, setClient] = useState<RTVIClient | null>(null);
useEffect(() => {
const transport = new DailyTransport();
const rtviClient = new RTVIClient({
transport,
params: {
baseUrl: API_BASE_URL,
endpoints: {
connect: '/connect',
},
requestData: { foo: 'bar' },
},
enableMic: true,
enableCam: false,
});
setClient(rtviClient);
}, []);
if (!client) {
return null;
}
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
}

2
examples/fal-smart-turn/server/bot.py
View File

@@ -45,7 +45,7 @@ from pipecat.transports.services.daily import DailyParams, DailyTransport
load_dotenv(override=True)
# Check if we're in local development mode
LOCAL = os.getenv("LOCAL")
LOCAL = os.getenv("LOCAL_RUN")
logger.remove()
logger.add(sys.stderr, level="DEBUG")

121
examples/freeze-test/client/src/app.ts
View File

@@ -20,11 +20,10 @@ import {
} from '@pipecat-ai/client-js';
import {
ProtobufFrameSerializer,
WebSocketTransport
} from "@pipecat-ai/websocket-transport";
WebSocketTransport,
} from '@pipecat-ai/websocket-transport';
class RecordingSerializer extends ProtobufFrameSerializer {
private lastTimestamp: number | null = null;
private recordingAudioToSend: boolean = false;
private _recordedAudio: { data: ArrayBuffer; delay: number }[] = [];
@@ -40,7 +39,11 @@ class RecordingSerializer extends ProtobufFrameSerializer {
}
// @ts-ignore
serializeAudio(data: ArrayBuffer, sampleRate: number, numChannels: number): Uint8Array | null {
serializeAudio(
data: ArrayBuffer,
sampleRate: number,
numChannels: number
): Uint8Array | null {
if (this.recordingAudioToSend) {
const now = Date.now();
// Compute delay since last packet
@@ -55,13 +58,13 @@ class RecordingSerializer extends ProtobufFrameSerializer {
}
public get recordedAudio() {
return this._recordedAudio
return this._recordedAudio;
}
}
class WebsocketClientApp {
private ENABLE_RECORDING_MODE = false
private RECORDING_TIME_MS = 10000
private ENABLE_RECORDING_MODE = false;
private RECORDING_TIME_MS = 10000;
private rtviClient: RTVIClient | null = null;
private connectBtn: HTMLButtonElement | null = null;
@@ -71,7 +74,7 @@ class WebsocketClientApp {
private botAudio: HTMLAudioElement;
private declare websocketTransport: WebSocketTransport;
private sendRecordedAudio: boolean = false
private sendRecordedAudio: boolean = false;
private declare recordingSerializer: RecordingSerializer;
private playBtn: HTMLButtonElement | null = null;
@@ -91,8 +94,12 @@ class WebsocketClientApp {
* Set up references to DOM elements and create necessary media elements
*/
private setupDOMElements(): void {
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
this.connectBtn = document.getElementById(
'connect-btn'
) as HTMLButtonElement;
this.disconnectBtn = document.getElementById(
'disconnect-btn'
) as HTMLButtonElement;
this.statusSpan = document.getElementById('connection-status');
this.debugLog = document.getElementById('debug-log');
this.playBtn = document.getElementById('play-btn') as HTMLButtonElement;
@@ -105,8 +112,12 @@ class WebsocketClientApp {
private setupEventListeners(): void {
this.connectBtn?.addEventListener('click', () => this.connect());
this.disconnectBtn?.addEventListener('click', () => this.disconnect());
this.playBtn?.addEventListener('click', () => this.startSendingRecordedAudio());
this.stopBtn?.addEventListener('click', () => this.stopSendingRecordedAudio());
this.playBtn?.addEventListener('click', () =>
this.startSendingRecordedAudio()
);
this.stopBtn?.addEventListener('click', () =>
this.stopSendingRecordedAudio()
);
}
/**
@@ -165,7 +176,9 @@ class WebsocketClientApp {
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`);
this.log(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
);
});
}
@@ -175,7 +188,10 @@ class WebsocketClientApp {
*/
private setupAudioTrack(track: MediaStreamTrack): void {
this.log('Setting up audio track');
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
if (oldTrack?.id === track.id) return;
}
@@ -190,27 +206,17 @@ class WebsocketClientApp {
try {
const startTime = Date.now();
this.recordingSerializer = new RecordingSerializer()
const transport = this.ENABLE_RECORDING_MODE ?
new WebSocketTransport({
serializer: this.recordingSerializer,
recorderSampleRate: 8000,
playerSampleRate:8000
}) :
new WebSocketTransport({
serializer: new ProtobufFrameSerializer(),
recorderSampleRate: 8000,
playerSampleRate:8000
});
this.websocketTransport = transport
this.recordingSerializer = new RecordingSerializer();
const ws_opts = {
serializer: this.ENABLE_RECORDING_MODE
? this.recordingSerializer
: new ProtobufFrameSerializer(),
recorderSampleRate: 8000,
playerSampleRate: 8000,
};
const RTVIConfig: RTVIClientOptions = {
transport,
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: { connect: '/connect' },
},
transport: new WebSocketTransport(ws_opts),
enableMic: true,
enableCam: false,
callbacks: {
@@ -238,27 +244,34 @@ class WebsocketClientApp {
onMessageError: (error) => console.error('Message error:', error),
onError: (error) => console.error('Error:', error),
},
}
};
this.rtviClient = new RTVIClient(RTVIConfig);
this.websocketTransport = this.rtviClient.transport;
this.setupTrackListeners();
this.log('Initializing devices...');
await this.rtviClient.initDevices();
this.log('Connecting to bot...');
await this.rtviClient.connect();
await this.rtviClient.connect({
endpoint: 'http://localhost:7860/connect',
});
const timeTaken = Date.now() - startTime;
this.log(`Connection complete, timeTaken: ${timeTaken}`);
if (this.ENABLE_RECORDING_MODE) {
this.log(`Starting to recording the next ${(this.RECORDING_TIME_MS/1000)}s of audio`);
this.recordingSerializer.startRecording()
await this.sleep(this.RECORDING_TIME_MS)
this.recordingSerializer.stopRecording()
this.log("Recording stopped");
this.rtviClient.enableMic(false)
this.startSendingRecordedAudio()
this.log(
`Starting to recording the next ${
this.RECORDING_TIME_MS / 1000
}s of audio`
);
this.recordingSerializer.startRecording();
await this.sleep(this.RECORDING_TIME_MS);
this.recordingSerializer.stopRecording();
this.log('Recording stopped');
this.rtviClient.enableMic(false);
this.startSendingRecordedAudio();
}
} catch (error) {
this.log(`Error connecting: ${(error as Error).message}`);
@@ -280,11 +293,16 @@ class WebsocketClientApp {
public async disconnect(): Promise<void> {
if (this.rtviClient) {
try {
this.stopSendingRecordedAudio()
this.stopSendingRecordedAudio();
await this.rtviClient.disconnect();
this.rtviClient = null;
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop());
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
this.botAudio.srcObject
.getAudioTracks()
.forEach((track) => track.stop());
this.botAudio.srcObject = null;
}
} catch (error) {
@@ -294,21 +312,21 @@ class WebsocketClientApp {
}
private startSendingRecordedAudio() {
this.sendRecordedAudio = true
this.sendRecordedAudio = true;
if (this.playBtn) this.playBtn.disabled = true;
if (this.stopBtn) this.stopBtn.disabled = false;
void this.replayAudio()
void this.replayAudio();
}
private stopSendingRecordedAudio() {
if (this.stopBtn) this.stopBtn.disabled = true;
if (this.playBtn) this.playBtn.disabled = false;
this.sendRecordedAudio = false
this.sendRecordedAudio = false;
}
private async replayAudio() {
if (this.sendRecordedAudio) {
this.log("Sending recorded audio")
this.log('Sending recorded audio');
for (const chunk of this.recordingSerializer.recordedAudio) {
await this.sleep(chunk.delay);
this.websocketTransport.handleUserAudioStream(chunk.data);
@@ -316,14 +334,13 @@ class WebsocketClientApp {
const randomDelay = 1000 + Math.random() * (10000 - 500);
await this.sleep(randomDelay);
void this.replayAudio()
void this.replayAudio();
}
}
private sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
return new Promise((resolve) => setTimeout(resolve, ms));
}
}
declare global {

85
examples/instant-voice/client/javascript/src/app.ts
View File

@@ -5,7 +5,7 @@
*/
/**
* RTVI Client Implementation
* Pipecat Client Implementation
*
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
* It handles audio/video streaming and manages the connection lifecycle.
@@ -18,20 +18,22 @@
import {
Participant,
RTVIClient,
RTVIClientOptions,
PipecatClient,
PipecatClientOptions,
RTVIEvent,
} from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import {
DailyEventCallbacks,
DailyTransport,
} from '@pipecat-ai/daily-transport';
import SoundUtils from './util/soundUtils';
import { InstantVoiceHelper } from './util/instantVoiceHelper';
/**
* InstantVoiceClient handles the connection and media management for a real-time
* voice and video interaction with an AI bot.
*/
class InstantVoiceClient {
private declare rtviClient: RTVIClient;
private declare pcClient: PipecatClient;
private connectBtn: HTMLButtonElement | null = null;
private disconnectBtn: HTMLButtonElement | null = null;
private statusSpan: HTMLElement | null = null;
@@ -46,7 +48,7 @@ class InstantVoiceClient {
document.body.appendChild(this.botAudio);
this.setupDOMElements();
this.setupEventListeners();
this.initializeRTVIClient();
this.initializePipecatClient();
}
/**
@@ -72,16 +74,11 @@ class InstantVoiceClient {
this.disconnectBtn?.addEventListener('click', () => this.disconnect());
}
private initializeRTVIClient(): void {
const RTVIConfig: RTVIClientOptions = {
private initializePipecatClient(): void {
const PipecatConfig: PipecatClientOptions = {
transport: new DailyTransport({
bufferLocalAudioUntilBotReady: true,
}),
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: { connect: '/connect' },
},
enableMic: true,
enableCam: false,
callbacks: {
@@ -113,30 +110,23 @@ class InstantVoiceClient {
onBotTranscript: (data) => this.log(`Bot: ${data.text}`),
onMessageError: (error) => console.error('Message error:', error),
onError: (error) => console.error('Error:', error),
},
onAudioBufferingStarted: () => {
SoundUtils.beep();
this.updateBufferingStatus('Yes');
this.log(
`onMicCaptureStarted, timeTaken: ${Date.now() - this.startTime}`
);
},
onAudioBufferingStopped: () => {
this.updateBufferingStatus('No');
this.log(
`onMicCaptureStopped, timeTaken: ${Date.now() - this.startTime}`
);
},
} as DailyEventCallbacks,
};
this.rtviClient = new RTVIClient(RTVIConfig);
this.rtviClient.registerHelper(
'transport',
new InstantVoiceHelper({
callbacks: {
onAudioBufferingStarted: () => {
SoundUtils.beep();
this.updateBufferingStatus('Yes');
this.log(
`onMicCaptureStarted, timeTaken: ${Date.now() - this.startTime}`
);
},
onAudioBufferingStopped: () => {
this.updateBufferingStatus('No');
this.log(
`onMicCaptureStopped, timeTaken: ${Date.now() - this.startTime}`
);
},
},
})
);
this.pcClient = new PipecatClient(PipecatConfig);
this.setupTrackListeners();
}
@@ -182,8 +172,8 @@ class InstantVoiceClient {
* This is called when the bot is ready or when the transport state changes to ready
*/
setupMediaTracks() {
if (!this.rtviClient) return;
const tracks = this.rtviClient.tracks();
if (!this.pcClient) return;
const tracks = this.pcClient.tracks();
if (tracks.bot?.audio) {
this.setupAudioTrack(tracks.bot.audio);
}
@@ -194,10 +184,10 @@ class InstantVoiceClient {
* This handles new tracks being added during the session
*/
setupTrackListeners() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks
if (!participant?.local && track.kind === 'audio') {
this.setupAudioTrack(track);
@@ -205,7 +195,7 @@ class InstantVoiceClient {
});
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
);
@@ -230,22 +220,25 @@ class InstantVoiceClient {
/**
* Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection
* This sets up the Pipecat client, initializes devices, and establishes the connection
*/
public async connect(): Promise<void> {
try {
this.startTime = Date.now();
this.log('Connecting to bot...');
await this.rtviClient.connect();
await this.pcClient.connect({
// The baseURL and endpoint of your bot server that the client will connect to
endpoint: 'http://localhost:7860/connect',
});
} catch (error) {
this.log(`Error connecting: ${(error as Error).message}`);
this.updateStatus('Error');
this.updateBufferingStatus('No');
// Clean up if there's an error
if (this.rtviClient) {
if (this.pcClient) {
try {
await this.rtviClient.disconnect();
await this.pcClient.disconnect();
} catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError}`);
}
@@ -258,7 +251,7 @@ class InstantVoiceClient {
*/
public async disconnect(): Promise<void> {
try {
await this.rtviClient.disconnect();
await this.pcClient.disconnect();
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject

39
examples/instant-voice/client/javascript/src/util/instantVoiceHelper.ts
View File

@@ -1,39 +0,0 @@
import {RTVIClientHelper, RTVIClientHelperOptions, RTVIMessage} from "@pipecat-ai/client-js";
import {DailyRTVIMessageType} from '@pipecat-ai/daily-transport';
export type InstantVoiceHelperCallbacks = Partial<{
onAudioBufferingStarted: () => void;
onAudioBufferingStopped: () => void;
}>;
// --- Interface and class
export interface InstantVoiceHelperOptions extends RTVIClientHelperOptions {
callbacks?: InstantVoiceHelperCallbacks;
}
export class InstantVoiceHelper extends RTVIClientHelper {
protected declare _options: InstantVoiceHelperOptions;
constructor(options: InstantVoiceHelperOptions) {
super(options);
}
handleMessage(rtviMessage: RTVIMessage): void {
switch (rtviMessage.type) {
case DailyRTVIMessageType.AUDIO_BUFFERING_STARTED:
if (this._options.callbacks?.onAudioBufferingStarted) {
this._options.callbacks?.onAudioBufferingStarted()
}
break;
case DailyRTVIMessageType.AUDIO_BUFFERING_STOPPED:
if (this._options.callbacks?.onAudioBufferingStopped) {
this._options.callbacks?.onAudioBufferingStopped()
}
break;
}
}
getMessageTypes(): string[] {
return [DailyRTVIMessageType.AUDIO_BUFFERING_STARTED, DailyRTVIMessageType.AUDIO_BUFFERING_STOPPED];
}
}

177
examples/news-chatbot/client/javascript/src/app.js
View File

@@ -5,7 +5,7 @@
*/
/**
* RTVI Client Implementation
* Pipecat Client Implementation
*
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
* It handles audio/video streaming and manages the connection lifecycle.
@@ -16,78 +16,9 @@
* - Browser with WebRTC support
*/
import {
LogLevel,
RTVIClient,
RTVIClientHelper,
RTVIEvent,
} from '@pipecat-ai/client-js';
import { LogLevel, PipecatClient, RTVIEvent } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
class SearchResponseHelper extends RTVIClientHelper {
constructor(contentPanel) {
super();
this.contentPanel = contentPanel;
}
handleMessage(rtviMessage) {
console.log('SearchResponseHelper, received message:', rtviMessage);
if (rtviMessage.data) {
// Clear existing content
this.contentPanel.innerHTML = '';
// Create a container for all content
const contentContainer = document.createElement('div');
contentContainer.className = 'content-container';
// Add the search_result
if (rtviMessage.data.search_result) {
const searchResultDiv = document.createElement('div');
searchResultDiv.className = 'search-result';
searchResultDiv.textContent = rtviMessage.data.search_result;
contentContainer.appendChild(searchResultDiv);
}
// Add the sources
if (rtviMessage.data.origins) {
const sourcesDiv = document.createElement('div');
sourcesDiv.className = 'sources';
const sourcesTitle = document.createElement('h3');
sourcesTitle.className = 'sources-title';
sourcesTitle.textContent = 'Sources:';
sourcesDiv.appendChild(sourcesTitle);
rtviMessage.data.origins.forEach((origin) => {
const sourceLink = document.createElement('a');
sourceLink.className = 'source-link';
sourceLink.href = origin.site_uri;
sourceLink.target = '_blank';
sourceLink.textContent = origin.site_title;
sourcesDiv.appendChild(sourceLink);
});
contentContainer.appendChild(sourcesDiv);
}
// Add the rendered_content in an iframe
if (rtviMessage.data.rendered_content) {
const iframe = document.createElement('iframe');
iframe.className = 'iframe-container';
iframe.srcdoc = rtviMessage.data.rendered_content;
contentContainer.appendChild(iframe);
}
// Append the content container to the content panel
this.contentPanel.appendChild(contentContainer);
}
}
getMessageTypes() {
return ['bot-llm-search-response'];
}
}
/**
* ChatbotClient handles the connection and media management for a real-time
* voice and video interaction with an AI bot.
@@ -95,7 +26,7 @@ class SearchResponseHelper extends RTVIClientHelper {
class ChatbotClient {
constructor() {
// Initialize client state
this.rtviClient = null;
this.pcClient = null;
this.setupDOMElements();
this.setupEventListeners();
}
@@ -160,10 +91,10 @@ class ChatbotClient {
* This is called when the bot is ready or when the transport state changes to ready
*/
setupMediaTracks() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Get current tracks from the client
const tracks = this.rtviClient.tracks();
const tracks = this.pcClient.tracks();
// Set up any available bot tracks
if (tracks.bot?.audio) {
@@ -176,10 +107,10 @@ class ChatbotClient {
* This handles new tracks being added during the session
*/
setupTrackListeners() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks
if (!participant?.local && track.kind === 'audio') {
this.setupAudioTrack(track);
@@ -187,7 +118,7 @@ class ChatbotClient {
});
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(
`Track stopped event: ${track.kind} from ${
participant?.name || 'unknown'
@@ -213,20 +144,13 @@ class ChatbotClient {
/**
* Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection
* This sets up the Pipecat client, initializes devices, and establishes the connection
*/
async connect() {
try {
// Initialize the RTVI client with a Daily WebRTC transport and our configuration
this.rtviClient = new RTVIClient({
// Initialize the Pipecat client with a Daily WebRTC transport and our configuration
this.pcClient = new PipecatClient({
transport: new DailyTransport(),
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: {
connect: '/connect',
},
},
enableMic: true, // Enable microphone for user input
enableCam: false,
callbacks: {
@@ -251,6 +175,8 @@ class ChatbotClient {
this.setupMediaTracks();
}
},
// Handle search response events
onBotLlmSearchResponse: this.handleSearchResponse.bind(this),
// Handle bot connection events
onBotConnected: (participant) => {
this.log(`Bot connected: ${JSON.stringify(participant)}`);
@@ -281,22 +207,22 @@ class ChatbotClient {
},
},
});
//this.rtviClient.setLogLevel(LogLevel.DEBUG)
this.rtviClient.registerHelper(
'llm',
new SearchResponseHelper(this.searchResultContainer)
);
//this.pcClient.setLogLevel(LogLevel.DEBUG)
// Set up listeners for media track events
this.setupTrackListeners();
// Initialize audio devices
this.log('Initializing devices...');
await this.rtviClient.initDevices();
await this.pcClient.initDevices();
// Connect to the bot
this.log('Connecting to bot...');
await this.rtviClient.connect();
await this.pcClient.connect({
// The baseURL and endpoint of your bot server that the client will connect to
endpoint: 'http://localhost:7860/connect',
});
this.log('Connection complete');
} catch (error) {
@@ -306,9 +232,9 @@ class ChatbotClient {
this.updateStatus('Error');
// Clean up if there's an error
if (this.rtviClient) {
if (this.pcClient) {
try {
await this.rtviClient.disconnect();
await this.pcClient.disconnect();
} catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError.message}`);
}
@@ -320,11 +246,11 @@ class ChatbotClient {
* Disconnect from the bot and clean up media resources
*/
async disconnect() {
if (this.rtviClient) {
if (this.pcClient) {
try {
// Disconnect the RTVI client
await this.rtviClient.disconnect();
this.rtviClient = null;
// Disconnect the Pipecat client
await this.pcClient.disconnect();
this.pcClient = null;
// Clean up audio
if (this.botAudio.srcObject) {
@@ -339,6 +265,57 @@ class ChatbotClient {
}
}
}
handleSearchResponse(response) {
console.log('SearchResponseHelper, received message:', response);
// Clear existing content
this.searchResultContainer.innerHTML = '';
// Create a container for all content
const contentContainer = document.createElement('div');
contentContainer.className = 'content-container';
// Add the search_result
if (response.search_result) {
const searchResultDiv = document.createElement('div');
searchResultDiv.className = 'search-result';
searchResultDiv.textContent = response.search_result;
contentContainer.appendChild(searchResultDiv);
}
// Add the sources
if (response.origins) {
const sourcesDiv = document.createElement('div');
sourcesDiv.className = 'sources';
const sourcesTitle = document.createElement('h3');
sourcesTitle.className = 'sources-title';
sourcesTitle.textContent = 'Sources:';
sourcesDiv.appendChild(sourcesTitle);
response.origins.forEach((origin) => {
const sourceLink = document.createElement('a');
sourceLink.className = 'source-link';
sourceLink.href = origin.site_uri;
sourceLink.target = '_blank';
sourceLink.textContent = origin.site_title;
sourcesDiv.appendChild(sourceLink);
});
contentContainer.appendChild(sourcesDiv);
}
// Add the rendered_content in an iframe
if (response.rendered_content) {
const iframe = document.createElement('iframe');
iframe.className = 'iframe-container';
iframe.srcdoc = response.rendered_content;
contentContainer.appendChild(iframe);
}
// Append the content container to the content panel
this.searchResultContainer.appendChild(contentContainer);
}
}
// Initialize the client when the page loads

53
examples/p2p-webrtc/video-transform/client/typescript/src/app.ts
View File

@@ -1,9 +1,11 @@
import { SmallWebRTCTransport } from '@pipecat-ai/small-webrtc-transport';
import {
BotLLMTextData,
Participant,
RTVIClient,
RTVIClientOptions,
Transport,
PipecatClient,
PipecatClientOptions,
TranscriptData,
TransportState,
} from '@pipecat-ai/client-js';
class WebRTCApp {
@@ -23,26 +25,22 @@ class WebRTCApp {
private statusSpan: HTMLElement | null = null;
private declare smallWebRTCTransport: SmallWebRTCTransport;
private declare rtviClient: RTVIClient;
private declare pcClient: PipecatClient;
constructor() {
this.setupDOMElements();
this.setupDOMEventListeners();
this.initializeRTVIClient();
this.initializePipecatClient();
void this.populateDevices();
}
private initializeRTVIClient(): void {
const transport = new SmallWebRTCTransport();
const RTVIConfig: RTVIClientOptions = {
params: {
baseUrl: '/api/offer',
},
transport: transport as Transport,
private initializePipecatClient(): void {
const opts: PipecatClientOptions = {
transport: new SmallWebRTCTransport({ connectionUrl: '/api/offer' }),
enableMic: true,
enableCam: true,
callbacks: {
onTransportStateChanged: (state) => {
onTransportStateChanged: (state: TransportState) => {
this.log(`Transport state: ${state}`);
},
onConnected: () => {
@@ -66,13 +64,13 @@ class WebRTCApp {
onBotStoppedSpeaking: () => {
this.log('Bot stopped speaking.');
},
onUserTranscript: (transcript) => {
onUserTranscript: (transcript: TranscriptData) => {
if (transcript.final) {
this.log(`User transcript: ${transcript.text}`);
}
},
onBotTranscript: (transcript) => {
this.log(`Bot transcript: ${transcript.text}`);
onBotTranscript: (data: BotLLMTextData) => {
this.log(`Bot transcript: ${data.text}`);
},
onTrackStarted: (
track: MediaStreamTrack,
@@ -83,14 +81,13 @@ class WebRTCApp {
}
this.onBotTrackStarted(track);
},
onServerMessage: (msg) => {
onServerMessage: (msg: unknown) => {
this.log(`Server message: ${msg}`);
},
},
};
RTVIConfig.customConnectHandler = () => Promise.resolve();
this.rtviClient = new RTVIClient(RTVIConfig);
this.smallWebRTCTransport = transport;
this.pcClient = new PipecatClient(opts);
this.smallWebRTCTransport = this.pcClient.transport as SmallWebRTCTransport;
}
private setupDOMElements(): void {
@@ -132,16 +129,16 @@ class WebRTCApp {
this.audioInput.addEventListener('change', (e) => {
// @ts-ignore
let audioDevice = e.target?.value;
this.rtviClient.updateMic(audioDevice);
this.pcClient.updateMic(audioDevice);
});
this.videoInput.addEventListener('change', (e) => {
// @ts-ignore
let videoDevice = e.target?.value;
this.rtviClient.updateCam(videoDevice);
this.pcClient.updateCam(videoDevice);
});
this.muteBtn.addEventListener('click', () => {
let isCamEnabled = this.rtviClient.isCamEnabled;
this.rtviClient.enableCam(!isCamEnabled);
let isCamEnabled = this.pcClient.isCamEnabled;
this.pcClient.enableCam(!isCamEnabled);
this.muteBtn.textContent = isCamEnabled ? '📵' : '📷';
});
}
@@ -206,9 +203,9 @@ class WebRTCApp {
};
try {
const audioDevices = await this.rtviClient.getAllMics();
const audioDevices = await this.pcClient.getAllMics();
populateSelect(this.audioInput, audioDevices);
const videoDevices = await this.rtviClient.getAllCams();
const videoDevices = await this.pcClient.getAllCams();
populateSelect(this.videoInput, videoDevices);
} catch (e) {
alert(e);
@@ -224,7 +221,7 @@ class WebRTCApp {
this.smallWebRTCTransport.setAudioCodec(this.audioCodec.value);
this.smallWebRTCTransport.setVideoCodec(this.videoCodec.value);
try {
await this.rtviClient.connect();
await this.pcClient.connect();
} catch (e) {
console.log(`Failed to connect ${e}`);
this.stop();
@@ -232,7 +229,7 @@ class WebRTCApp {
}
private stop(): void {
void this.rtviClient.disconnect();
void this.pcClient.disconnect();
}
}

66
examples/simple-chatbot/client/javascript/src/app.js
View File

@@ -5,7 +5,7 @@
*/
/**
* RTVI Client Implementation
* Pipecat Client Implementation
*
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
* It handles audio/video streaming and manages the connection lifecycle.
@@ -16,7 +16,7 @@
* - Browser with WebRTC support
*/
import { RTVIClient, RTVIEvent } from '@pipecat-ai/client-js';
import { PipecatClient, RTVIEvent } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
/**
@@ -26,7 +26,7 @@ import { DailyTransport } from '@pipecat-ai/daily-transport';
class ChatbotClient {
constructor() {
// Initialize client state
this.rtviClient = null;
this.pcClient = null;
this.setupDOMElements();
this.initializeClientAndTransport();
}
@@ -54,11 +54,14 @@ class ChatbotClient {
* Set up event listeners for connect/disconnect buttons
*/
setupEventListeners() {
this.connectBtn.addEventListener('click', () => this.connect());
this.connectBtn.addEventListener('click', () => {
console.log('click');
this.connect();
});
this.disconnectBtn.addEventListener('click', () => this.disconnect());
// Populate device selector
this.rtviClient.getAllMics().then((mics) => {
this.pcClient.getAllMics().then((mics) => {
console.log('Available mics:', mics);
mics.forEach((device) => {
const option = document.createElement('option');
@@ -70,33 +73,27 @@ class ChatbotClient {
this.deviceSelector.addEventListener('change', (event) => {
const selectedDeviceId = event.target.value;
console.log('Selected device ID:', selectedDeviceId);
this.rtviClient.updateMic(selectedDeviceId);
this.pcClient.updateMic(selectedDeviceId);
});
// Handle mic mute/unmute toggle
const micToggleBtn = document.getElementById('mic-toggle-btn');
micToggleBtn.addEventListener('click', () => {
let micEnabled = this.rtviClient.isMicEnabled;
let micEnabled = this.pcClient.isMicEnabled;
micToggleBtn.textContent = micEnabled ? 'Unmute Mic' : 'Mute Mic';
this.rtviClient.enableMic(!micEnabled);
this.pcClient.enableMic(!micEnabled);
});
}
/**
* Set up the RTVI client and Daily transport
* Set up the Pipecat client and Daily transport
*/
async initializeClientAndTransport() {
// Initialize the RTVI client with a DailyTransport and our configuration
this.rtviClient = new RTVIClient({
console.log('Initializing Pipecat client and transport...');
// Initialize the Pipecat client with a DailyTransport and our configuration
this.pcClient = new PipecatClient({
transport: new DailyTransport(),
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: {
connect: '/connect',
},
},
enableMic: true, // Enable microphone for user input
enableCam: false,
callbacks: {
@@ -160,7 +157,7 @@ class ChatbotClient {
// Set up listeners for media track events
this.setupTrackListeners();
await this.rtviClient.initDevices();
await this.pcClient.initDevices();
this.setupEventListeners();
}
@@ -196,10 +193,10 @@ class ChatbotClient {
* This is called when the bot is ready or when the transport state changes to ready
*/
setupMediaTracks() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Get current tracks from the client
const tracks = this.rtviClient.tracks();
const tracks = this.pcClient.tracks();
// Set up any available bot tracks
if (tracks.bot?.audio) {
@@ -215,10 +212,10 @@ class ChatbotClient {
* This handles new tracks being added during the session
*/
setupTrackListeners() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks
if (!participant?.local) {
if (track.kind === 'audio') {
@@ -230,7 +227,7 @@ class ChatbotClient {
});
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(
`Track stopped event: ${track.kind} from ${
participant?.name || 'unknown'
@@ -284,17 +281,16 @@ class ChatbotClient {
/**
* Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection
* This sets up the Pipecat client, initializes devices, and establishes the connection
*/
async connect() {
try {
// Initialize audio/video devices
this.log('Initializing devices...');
await this.rtviClient.initDevices();
// Connect to the bot
this.log('Connecting to bot...');
await this.rtviClient.connect();
await this.pcClient.connect({
endpoint: 'http://localhost:7860/connect',
timeout: 25000,
});
this.log('Connection complete');
} catch (error) {
@@ -304,9 +300,9 @@ class ChatbotClient {
this.updateStatus('Error');
// Clean up if there's an error
if (this.rtviClient) {
if (this.pcClient) {
try {
await this.rtviClient.disconnect();
await this.pcClient.disconnect();
} catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError.message}`);
}
@@ -318,10 +314,10 @@ class ChatbotClient {
* Disconnect from the bot and clean up media resources
*/
async disconnect() {
if (this.rtviClient) {
if (this.pcClient) {
try {
// Disconnect the RTVI client
await this.rtviClient.disconnect();
// Disconnect the Pipecat client
await this.pcClient.disconnect();
// Clean up audio
if (this.botAudio.srcObject) {

18
examples/simple-chatbot/client/react/src/App.tsx
View File

@@ -1,22 +1,22 @@
import {
RTVIClientAudio,
RTVIClientVideo,
useRTVIClientTransportState,
PipecatClientAudio,
PipecatClientVideo,
usePipecatClientTransportState,
} from '@pipecat-ai/client-react';
import { RTVIProvider } from './providers/RTVIProvider';
import { PipecatProvider } from './providers/PipecatProvider';
import { ConnectButton } from './components/ConnectButton';
import { StatusDisplay } from './components/StatusDisplay';
import { DebugDisplay } from './components/DebugDisplay';
import './App.css';
function BotVideo() {
const transportState = useRTVIClientTransportState();
const transportState = usePipecatClientTransportState();
const isConnected = transportState !== 'disconnected';
return (
<div className="bot-container">
<div className="video-container">
{isConnected && <RTVIClientVideo participant="bot" fit="cover" />}
{isConnected && <PipecatClientVideo participant="bot" fit="cover" />}
</div>
</div>
);
@@ -35,16 +35,16 @@ function AppContent() {
</div>
<DebugDisplay />
<RTVIClientAudio />
<PipecatClientAudio />
</div>
);
}
function App() {
return (
<RTVIProvider>
<PipecatProvider>
<AppContent />
</RTVIProvider>
</PipecatProvider>
);
}

12
examples/simple-chatbot/client/react/src/components/ConnectButton.tsx
View File

@@ -1,16 +1,16 @@
import {
useRTVIClient,
useRTVIClientTransportState,
usePipecatClient,
usePipecatClientTransportState,
} from '@pipecat-ai/client-react';
export function ConnectButton() {
const client = useRTVIClient();
const transportState = useRTVIClientTransportState();
const client = usePipecatClient();
const transportState = usePipecatClientTransportState();
const isConnected = ['connected', 'ready'].includes(transportState);
const handleClick = async () => {
if (!client) {
console.error('RTVI client is not initialized');
console.error('Pipecat client is not initialized');
return;
}
@@ -18,7 +18,7 @@ export function ConnectButton() {
if (isConnected) {
await client.disconnect();
} else {
await client.connect();
await client.connect({ endpoint: 'http://localhost:7860/connect' });
}
} catch (error) {
console.error('Connection error:', error);

4
examples/simple-chatbot/client/react/src/components/DebugDisplay.tsx
View File

@@ -6,12 +6,12 @@ import {
TranscriptData,
BotLLMTextData,
} from '@pipecat-ai/client-js';
import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
import { usePipecatClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
import './DebugDisplay.css';
export function DebugDisplay() {
const debugLogRef = useRef<HTMLDivElement>(null);
const client = useRTVIClient();
const client = usePipecatClient();
const log = useCallback((message: string) => {
if (!debugLogRef.current) return;

4
examples/simple-chatbot/client/react/src/components/StatusDisplay.tsx
View File

@@ -1,7 +1,7 @@
import { useRTVIClientTransportState } from '@pipecat-ai/client-react';
import { usePipecatClientTransportState } from '@pipecat-ai/client-react';
export function StatusDisplay() {
const transportState = useRTVIClientTransportState();
const transportState = usePipecatClientTransportState();
return (
<div className="status">

16
examples/simple-chatbot/client/react/src/providers/PipecatProvider.tsx Normal file
View File

@@ -0,0 +1,16 @@
import { type PropsWithChildren } from 'react';
import { PipecatClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { PipecatClientProvider } from '@pipecat-ai/client-react';
const client = new PipecatClient({
transport: new DailyTransport(),
enableMic: true,
enableCam: false,
});
export function PipecatProvider({ children }: PropsWithChildren) {
return (
<PipecatClientProvider client={client}>{children}</PipecatClientProvider>
);
}

22
examples/simple-chatbot/client/react/src/providers/RTVIProvider.tsx
View File

@@ -1,22 +0,0 @@
import { type PropsWithChildren } from 'react';
import { RTVIClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { RTVIClientProvider } from '@pipecat-ai/client-react';
const transport = new DailyTransport();
const client = new RTVIClient({
transport,
params: {
baseUrl: 'http://localhost:7860',
endpoints: {
connect: '/connect',
},
},
enableMic: true,
enableCam: false,
});
export function RTVIProvider({ children }: PropsWithChildren) {
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
}

78
examples/twilio-chatbot/client/typescript/src/app.ts
View File

@@ -12,12 +12,11 @@ import {
import {
WebSocketTransport,
TwilioSerializer,
} from "@pipecat-ai/websocket-transport";
} from '@pipecat-ai/websocket-transport';
class WebsocketClientApp {
private static STREAM_SID = "ws_mock_stream_sid"
private static CALL_SID = "ws_mock_call_sid"
private static STREAM_SID = 'ws_mock_stream_sid';
private static CALL_SID = 'ws_mock_call_sid';
private rtviClient: RTVIClient | null = null;
private connectBtn: HTMLButtonElement | null = null;
@@ -38,8 +37,12 @@ class WebsocketClientApp {
* Set up references to DOM elements and create necessary media elements
*/
private setupDOMElements(): void {
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
this.connectBtn = document.getElementById(
'connect-btn'
) as HTMLButtonElement;
this.disconnectBtn = document.getElementById(
'disconnect-btn'
) as HTMLButtonElement;
this.statusSpan = document.getElementById('connection-status');
this.debugLog = document.getElementById('debug-log');
}
@@ -80,13 +83,23 @@ class WebsocketClientApp {
}
private async emulateTwilioMessages() {
const connectedMessage={"event": "connected", "protocol": "Call", "version": "1.0.0"}
const connectedMessage = {
event: 'connected',
protocol: 'Call',
version: '1.0.0',
};
const websocketTransport = this.rtviClient?.transport as WebSocketTransport
void websocketTransport?.sendRawMessage(connectedMessage)
const websocketTransport = this.rtviClient?.transport as WebSocketTransport;
void websocketTransport?.sendRawMessage(connectedMessage);
const startMessage={"event": "start", "start": {"streamSid": WebsocketClientApp.STREAM_SID, "callSid": WebsocketClientApp.CALL_SID}}
void websocketTransport?.sendRawMessage(startMessage)
const startMessage = {
event: 'start',
start: {
streamSid: WebsocketClientApp.STREAM_SID,
callSid: WebsocketClientApp.CALL_SID,
},
};
void websocketTransport?.sendRawMessage(startMessage);
}
/**
@@ -118,7 +131,9 @@ class WebsocketClientApp {
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`);
this.log(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
);
});
}
@@ -128,7 +143,10 @@ class WebsocketClientApp {
*/
private setupAudioTrack(track: MediaStreamTrack): void {
this.log('Setting up audio track');
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
if (oldTrack?.id === track.id) return;
}
@@ -143,23 +161,19 @@ class WebsocketClientApp {
try {
const startTime = Date.now();
const transport = new WebSocketTransport({
const ws_opts = {
serializer: new TwilioSerializer(),
recorderSampleRate: 8000,
playerSampleRate: 8000
});
playerSampleRate: 8000,
ws_url: 'http://localhost:8765/ws',
};
const RTVIConfig: RTVIClientOptions = {
transport,
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:8765',
endpoints: { connect: '/' },
},
transport: new WebSocketTransport(ws_opts),
enableMic: true,
enableCam: false,
callbacks: {
onConnected: () => {
this.emulateTwilioMessages()
this.emulateTwilioMessages();
this.updateStatus('Connected');
if (this.connectBtn) this.connectBtn.disabled = true;
if (this.disconnectBtn) this.disconnectBtn.disabled = false;
@@ -183,13 +197,7 @@ class WebsocketClientApp {
onMessageError: (error) => console.error('Message error:', error),
onError: (error) => console.error('Error:', error),
},
}
// @ts-ignore
RTVIConfig.customConnectHandler = () => Promise.resolve(
{
ws_url: "/ws",
}
);
};
this.rtviClient = new RTVIClient(RTVIConfig);
this.setupTrackListeners();
@@ -223,8 +231,13 @@ class WebsocketClientApp {
try {
await this.rtviClient.disconnect();
this.rtviClient = null;
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop());
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
this.botAudio.srcObject
.getAudioTracks()
.forEach((track) => track.stop());
this.botAudio.srcObject = null;
}
} catch (error) {
@@ -232,7 +245,6 @@ class WebsocketClientApp {
}
}
}
}
declare global {

86
examples/websocket/client/src/app.ts
View File

@@ -5,7 +5,7 @@
*/
/**
* RTVI Client Implementation
* Pipecat Client Implementation
*
* This client connects to an RTVI-compatible bot server using WebSocket.
*
@@ -14,16 +14,14 @@
*/
import {
RTVIClient,
RTVIClientOptions,
PipecatClient,
PipecatClientOptions,
RTVIEvent,
} from '@pipecat-ai/client-js';
import {
WebSocketTransport
} from "@pipecat-ai/websocket-transport";
import { WebSocketTransport } from '@pipecat-ai/websocket-transport';
class WebsocketClientApp {
private rtviClient: RTVIClient | null = null;
private pcClient: PipecatClient | null = null;
private connectBtn: HTMLButtonElement | null = null;
private disconnectBtn: HTMLButtonElement | null = null;
private statusSpan: HTMLElement | null = null;
@@ -31,7 +29,7 @@ class WebsocketClientApp {
private botAudio: HTMLAudioElement;
constructor() {
console.log("WebsocketClientApp");
console.log('WebsocketClientApp');
this.botAudio = document.createElement('audio');
this.botAudio.autoplay = true;
//this.botAudio.playsInline = true;
@@ -45,8 +43,12 @@ class WebsocketClientApp {
* Set up references to DOM elements and create necessary media elements
*/
private setupDOMElements(): void {
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
this.connectBtn = document.getElementById(
'connect-btn'
) as HTMLButtonElement;
this.disconnectBtn = document.getElementById(
'disconnect-btn'
) as HTMLButtonElement;
this.statusSpan = document.getElementById('connection-status');
this.debugLog = document.getElementById('debug-log');
}
@@ -91,8 +93,8 @@ class WebsocketClientApp {
* This is called when the bot is ready or when the transport state changes to ready
*/
setupMediaTracks() {
if (!this.rtviClient) return;
const tracks = this.rtviClient.tracks();
if (!this.pcClient) return;
const tracks = this.pcClient.tracks();
if (tracks.bot?.audio) {
this.setupAudioTrack(tracks.bot.audio);
}
@@ -103,10 +105,10 @@ class WebsocketClientApp {
* This handles new tracks being added during the session
*/
setupTrackListeners() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks
if (!participant?.local && track.kind === 'audio') {
this.setupAudioTrack(track);
@@ -114,8 +116,10 @@ class WebsocketClientApp {
});
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`);
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
);
});
}
@@ -125,7 +129,10 @@ class WebsocketClientApp {
*/
private setupAudioTrack(track: MediaStreamTrack): void {
this.log('Setting up audio track');
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
if (oldTrack?.id === track.id) return;
}
@@ -134,21 +141,15 @@ class WebsocketClientApp {
/**
* Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection
* This sets up the Pipecat client, initializes devices, and establishes the connection
*/
public async connect(): Promise<void> {
try {
const startTime = Date.now();
//const transport = new DailyTransport();
const transport = new WebSocketTransport();
const RTVIConfig: RTVIClientOptions = {
transport,
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: { connect: '/connect' },
},
const PipecatConfig: PipecatClientOptions = {
transport: new WebSocketTransport(),
enableMic: true,
enableCam: false,
callbacks: {
@@ -176,15 +177,20 @@ class WebsocketClientApp {
onMessageError: (error) => console.error('Message error:', error),
onError: (error) => console.error('Error:', error),
},
}
this.rtviClient = new RTVIClient(RTVIConfig);
};
this.pcClient = new PipecatClient(PipecatConfig);
// @ts-ignore
window.pcClient = this.pcClient; // Expose for debugging
this.setupTrackListeners();
this.log('Initializing devices...');
await this.rtviClient.initDevices();
await this.pcClient.initDevices();
this.log('Connecting to bot...');
await this.rtviClient.connect();
await this.pcClient.connect({
// The baseURL and endpoint of your bot server that the client will connect to
endpoint: 'http://localhost:7860/connect',
});
const timeTaken = Date.now() - startTime;
this.log(`Connection complete, timeTaken: ${timeTaken}`);
@@ -192,9 +198,9 @@ class WebsocketClientApp {
this.log(`Error connecting: ${(error as Error).message}`);
this.updateStatus('Error');
// Clean up if there's an error
if (this.rtviClient) {
if (this.pcClient) {
try {
await this.rtviClient.disconnect();
await this.pcClient.disconnect();
} catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError}`);
}
@@ -206,12 +212,17 @@ class WebsocketClientApp {
* Disconnect from the bot and clean up media resources
*/
public async disconnect(): Promise<void> {
if (this.rtviClient) {
if (this.pcClient) {
try {
await this.rtviClient.disconnect();
this.rtviClient = null;
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop());
await this.pcClient.disconnect();
this.pcClient = null;
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
this.botAudio.srcObject
.getAudioTracks()
.forEach((track) => track.stop());
this.botAudio.srcObject = null;
}
} catch (error) {
@@ -219,7 +230,6 @@ class WebsocketClientApp {
}
}
}
}
declare global {

27
examples/word-wrangler-gemini-live/client/src/hooks/useConnectionState.ts
View File

@@ -1,16 +1,26 @@
import { useEffect, useCallback } from 'react';
import {
useRTVIClient,
useRTVIClientTransportState,
usePipecatClient,
usePipecatClientTransportState,
} from '@pipecat-ai/client-react';
import { CONNECTION_STATES } from '@/constants/gameConstants';
import { useConfigurationSettings } from '@/contexts/Configuration';
// Get the API base URL from environment variables
// Default to "/api" if not specified
// "/api" is the default for Next.js API routes and used
// for the Pipecat Cloud deployed agent
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
console.log('Using API base URL:', API_BASE_URL);
export function useConnectionState(
onConnected?: () => void,
onDisconnected?: () => void
) {
const client = useRTVIClient();
const transportState = useRTVIClientTransportState();
const client = usePipecatClient();
const transportState = usePipecatClientTransportState();
const config = useConfigurationSettings();
const isConnected = CONNECTION_STATES.ACTIVE.includes(transportState);
const isConnecting = CONNECTION_STATES.CONNECTING.includes(transportState);
@@ -35,12 +45,17 @@ export function useConnectionState(
if (isConnected) {
await client.disconnect();
} else {
await client.connect();
await client.connect({
endpoint: `${API_BASE_URL}/connect`,
requestData: {
personality: config.personality,
},
});
}
} catch (error) {
console.error('Connection error:', error);
}
}, [client, isConnected]);
}, [client, config, isConnected]);
return {
isConnected,

26
examples/word-wrangler-gemini-live/client/src/pages/_app.tsx
View File

@@ -1,15 +1,15 @@
import { ConfigurationProvider } from "@/contexts/Configuration";
import { RTVIProvider } from "@/providers/RTVIProvider";
import { RTVIClientAudio } from "@pipecat-ai/client-react";
import type { AppProps } from "next/app";
import { Nunito } from "next/font/google";
import Head from "next/head";
import "../styles/globals.css";
import { ConfigurationProvider } from '@/contexts/Configuration';
import { PipecatProvider } from '@/providers/PipecatProvider';
import { PipecatClientAudio } from '@pipecat-ai/client-react';
import type { AppProps } from 'next/app';
import { Nunito } from 'next/font/google';
import Head from 'next/head';
import '../styles/globals.css';
const nunito = Nunito({
subsets: ["latin"],
display: "swap",
variable: "--font-sans",
subsets: ['latin'],
display: 'swap',
variable: '--font-sans',
});
export default function App({ Component, pageProps }: AppProps) {
@@ -21,10 +21,10 @@ export default function App({ Component, pageProps }: AppProps) {
</Head>
<main className={`${nunito.variable}`}>
<ConfigurationProvider>
<RTVIProvider>
<RTVIClientAudio />
<PipecatProvider>
<PipecatClientAudio />
<Component {...pageProps} />
</RTVIProvider>
</PipecatProvider>
</ConfigurationProvider>
</main>
</>

20
examples/word-wrangler-gemini-live/client/src/pages/api/connect.ts
View File

@@ -1,11 +1,11 @@
import type { NextApiRequest, NextApiResponse } from "next";
import type { NextApiRequest, NextApiResponse } from 'next';
export default async function handler(
req: NextApiRequest,
res: NextApiResponse
) {
if (req.method !== "POST") {
return res.status(405).json({ error: "Method not allowed" });
if (req.method !== 'POST') {
return res.status(405).json({ error: 'Method not allowed' });
}
try {
@@ -15,16 +15,16 @@ export default async function handler(
if (!personality) {
return res
.status(400)
.json({ error: "Missing required configuration parameters" });
.json({ error: 'Missing required configuration parameters' });
}
const response = await fetch(
`https://api.pipecat.daily.co/v1/public/${process.env.AGENT_NAME}/start`,
{
method: "POST",
method: 'POST',
headers: {
Authorization: `Bearer ${process.env.PIPECAT_CLOUD_API_KEY}`,
"Content-Type": "application/json",
'Content-Type': 'application/json',
},
body: JSON.stringify({
createDailyRoom: true,
@@ -37,15 +37,15 @@ export default async function handler(
const data = await response.json();
console.log("Response from API:", JSON.stringify(data, null, 2));
console.log('Response from API:', JSON.stringify(data, null, 2));
// Transform the response to match what RTVI client expects
// Transform the response to match what Pipecat client expects
return res.status(200).json({
room_url: data.dailyRoom,
token: data.dailyToken,
});
} catch (error) {
console.error("Error starting agent:", error);
return res.status(500).json({ error: "Failed to start agent" });
console.error('Error starting agent:', error);
return res.status(500).json({ error: 'Failed to start agent' });
}
}

43
examples/word-wrangler-gemini-live/client/src/providers/PipecatProvider.tsx Normal file
View File

@@ -0,0 +1,43 @@
'use client';
import { PipecatClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { PipecatClientProvider } from '@pipecat-ai/client-react';
import { PropsWithChildren, useEffect, useState, useRef } from 'react';
export function PipecatProvider({ children }: PropsWithChildren) {
const [client, setClient] = useState<PipecatClient | null>(null);
const clientCreated = useRef(false);
useEffect(() => {
// Only create the client once
if (clientCreated.current) return;
const pcClient = new PipecatClient({
transport: new DailyTransport(),
enableMic: true,
enableCam: false,
});
setClient(pcClient);
clientCreated.current = true;
// Cleanup when component unmounts
return () => {
if (pcClient) {
pcClient.disconnect().catch((err) => {
console.error('Error disconnecting client:', err);
});
}
clientCreated.current = false;
};
}, []);
if (!client) {
return null;
}
return (
<PipecatClientProvider client={client}>{children}</PipecatClientProvider>
);
}

72
examples/word-wrangler-gemini-live/client/src/providers/RTVIProvider.tsx
View File

@@ -1,72 +0,0 @@
"use client";
import { RTVIClient } from "@pipecat-ai/client-js";
import { DailyTransport } from "@pipecat-ai/daily-transport";
import { RTVIClientProvider } from "@pipecat-ai/client-react";
import { PropsWithChildren, useEffect, useState, useRef } from "react";
import { useConfigurationSettings } from "@/contexts/Configuration";
// Get the API base URL from environment variables
// Default to "/api" if not specified
// "/api" is the default for Next.js API routes and used
// for the Pipecat Cloud deployed agent
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || "/api";
console.log("Using API base URL:", API_BASE_URL);
export function RTVIProvider({ children }: PropsWithChildren) {
const [client, setClient] = useState<RTVIClient | null>(null);
const config = useConfigurationSettings();
const clientCreated = useRef(false);
useEffect(() => {
// Only create the client once
if (clientCreated.current) return;
const transport = new DailyTransport();
const rtviClient = new RTVIClient({
transport,
params: {
baseUrl: API_BASE_URL,
endpoints: {
connect: "/connect",
},
requestData: {
personality: config.personality,
},
},
enableMic: true,
enableCam: false,
});
setClient(rtviClient);
clientCreated.current = true;
// Cleanup when component unmounts
return () => {
if (rtviClient) {
rtviClient.disconnect().catch((err) => {
console.error("Error disconnecting client:", err);
});
}
clientCreated.current = false;
};
}, []);
// Update the connectParams when config changes
useEffect(() => {
if (!client) return;
// Update the connect params without recreating the client
client.params.requestData = {
personality: config.personality,
};
}, [client, config.personality]);
if (!client) {
return null;
}
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
}

31
examples/word-wrangler-gemini-live/server/bot.py
View File

@@ -4,6 +4,7 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import argparse
import asyncio
import os
import sys
@@ -198,16 +199,15 @@ async def bot(args: DailySessionArguments):
# Local development
async def local_daily():
async def local_daily(args: DailySessionArguments):
"""Daily transport for local development."""
from runner import configure
# from runner import configure
try:
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
transport = DailyTransport(
room_url,
token,
room_url=args.room_url,
token=args.token,
bot_name="Bot",
params=DailyParams(
audio_in_enabled=True,
@@ -217,7 +217,7 @@ async def local_daily():
)
test_config = {
"personality": "witty",
"personality": args.personality,
}
await main(transport, test_config)
@@ -227,7 +227,24 @@ async def local_daily():
# Local development entry point
if LOCAL_RUN and __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Run the Word Wrangler bot in local development mode"
)
parser.add_argument(
"-u", "--room-url", type=str, default=os.getenv("DAILY_SAMPLE_ROOM_URL", "")
)
parser.add_argument(
"-t", "--token", type=str, default=os.getenv("DAILY_SAMPLE_ROOM_TOKEN", None)
)
parser.add_argument(
"-p",
"--personality",
default="witty",
choices=["friendly", "professional", "enthusiastic", "thoughtful", "witty"],
help="Personality preset for the bot (friendly, professional, enthusiastic, thoughtful, witty)",
)
args = parser.parse_args()
try:
asyncio.run(local_daily())
asyncio.run(local_daily(args))
except Exception as e:
logger.exception(f"Failed to run in local mode: {e}")

5
examples/word-wrangler-gemini-live/server/server.py
View File

@@ -160,14 +160,15 @@ async def rtvi_connect(request: Request) -> Dict[Any, Any]:
Raises:
HTTPException: If room creation, token generation, or bot startup fails
"""
print("Creating room for RTVI connection")
body = await request.json()
print("Creating room for RTVI connection", body)
room_url, token = await create_room_and_token()
print(f"Room URL: {room_url}")
# Start the bot process
try:
proc = subprocess.Popen(
[f"python3 -m bot -u {room_url} -t {token}"],
[f"python3 -m bot -u {room_url} -t {token} -p {body.get('personality', 'witty')}"],
shell=True,
bufsize=1,
cwd=os.path.dirname(os.path.abspath(__file__)),