Updated all examples with clients to use the new PipecatClient

This commit is contained in:
mattie ruth backman
2025-06-11 12:01:03 -04:00
committed by Mattie Ruth
parent 43049c865c
commit dc41ec7cb1
31 changed files with 571 additions and 621 deletions

View File

@@ -5,7 +5,7 @@
*/ */
/** /**
* RTVI Client Implementation * Pipecat Client Implementation
* *
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily). * This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
* It handles audio/video streaming and manages the connection lifecycle. * It handles audio/video streaming and manages the connection lifecycle.
@@ -16,7 +16,7 @@
* - Browser with WebRTC support * - Browser with WebRTC support
*/ */
import { RTVIClient, RTVIEvent } from '@pipecat-ai/client-js'; import { PipecatClient, RTVIEvent } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport'; import { DailyTransport } from '@pipecat-ai/daily-transport';
/** /**
@@ -26,7 +26,7 @@ import { DailyTransport } from '@pipecat-ai/daily-transport';
class ChatbotClient { class ChatbotClient {
constructor() { constructor() {
// Initialize client state // Initialize client state
this.rtviClient = null; this.pcClient = null;
this.setupDOMElements(); this.setupDOMElements();
this.initializeClientAndTransport(); this.initializeClientAndTransport();
this.setupEventListeners(); this.setupEventListeners();
@@ -59,7 +59,7 @@ class ChatbotClient {
this.disconnectBtn.addEventListener('click', () => this.disconnect()); this.disconnectBtn.addEventListener('click', () => this.disconnect());
// Populate device selector // Populate device selector
this.rtviClient.getAllMics().then((mics) => { this.pcClient.getAllMics().then((mics) => {
console.log('Available mics:', mics); console.log('Available mics:', mics);
mics.forEach((device) => { mics.forEach((device) => {
const option = document.createElement('option'); const option = document.createElement('option');
@@ -71,16 +71,16 @@ class ChatbotClient {
this.deviceSelector.addEventListener('change', (event) => { this.deviceSelector.addEventListener('change', (event) => {
const selectedDeviceId = event.target.value; const selectedDeviceId = event.target.value;
console.log('Selected device ID:', selectedDeviceId); console.log('Selected device ID:', selectedDeviceId);
this.rtviClient.updateMic(selectedDeviceId); this.pcClient.updateMic(selectedDeviceId);
}); });
// Handle mic mute/unmute toggle // Handle mic mute/unmute toggle
const micToggleBtn = document.getElementById('mic-toggle-btn'); const micToggleBtn = document.getElementById('mic-toggle-btn');
micToggleBtn.addEventListener('click', () => { micToggleBtn.addEventListener('click', () => {
let micEnabled = this.rtviClient.isMicEnabled; let micEnabled = this.pcClient.isMicEnabled;
micToggleBtn.textContent = micEnabled ? 'Unmute Mic' : 'Mute Mic'; micToggleBtn.textContent = micEnabled ? 'Unmute Mic' : 'Mute Mic';
this.rtviClient.enableMic(!micEnabled); this.pcClient.enableMic(!micEnabled);
// Add logic to mute/unmute the mic // Add logic to mute/unmute the mic
if (micEnabled) { if (micEnabled) {
console.log('Mic muted'); console.log('Mic muted');
@@ -93,23 +93,12 @@ class ChatbotClient {
} }
/** /**
* Set up the RTVI client and Daily transport * Set up the Pipecat client and Daily transport
*/ */
async initializeClientAndTransport() { async initializeClientAndTransport() {
// Initialize the RTVI client with a DailyTransport and our configuration // Initialize the Pipecat client with a DailyTransport and our configuration
this.rtviClient = new RTVIClient({ this.pcClient = new PipecatClient({
transport: new DailyTransport(), transport: new DailyTransport(),
params: {
// REPLACE WITH YOUR MODAL URL ENDPOINT
baseUrl:
'https://<Modal workspace>--pipecat-modal-bot-launcher.modal.run',
endpoints: {
connect: '/connect',
},
requestData: {
bot_name: 'openai',
},
},
enableMic: true, // Enable microphone for user input enableMic: true, // Enable microphone for user input
enableCam: false, enableCam: false,
callbacks: { callbacks: {
@@ -176,8 +165,8 @@ class ChatbotClient {
// Set up listeners for media track events // Set up listeners for media track events
this.setupTrackListeners(); this.setupTrackListeners();
await this.rtviClient.initDevices(); await this.pcClient.initDevices();
window.client = this.rtviClient; window.client = this.pcClient;
} }
/** /**
@@ -212,10 +201,10 @@ class ChatbotClient {
* This is called when the bot is ready or when the transport state changes to ready * This is called when the bot is ready or when the transport state changes to ready
*/ */
setupMediaTracks() { setupMediaTracks() {
if (!this.rtviClient) return; if (!this.pcClient) return;
// Get current tracks from the client // Get current tracks from the client
const tracks = this.rtviClient.tracks(); const tracks = this.pcClient.tracks();
// Set up any available bot tracks // Set up any available bot tracks
if (tracks.bot?.audio) { if (tracks.bot?.audio) {
@@ -231,10 +220,10 @@ class ChatbotClient {
* This handles new tracks being added during the session * This handles new tracks being added during the session
*/ */
setupTrackListeners() { setupTrackListeners() {
if (!this.rtviClient) return; if (!this.pcClient) return;
// Listen for new tracks starting // Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => { this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks // Only handle non-local (bot) tracks
if (!participant?.local) { if (!participant?.local) {
if (track.kind === 'audio') { if (track.kind === 'audio') {
@@ -253,7 +242,7 @@ class ChatbotClient {
}); });
// Listen for tracks stopping // Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => { this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
if (participant.local) { if (participant.local) {
this.log('Local mic muted'); this.log('Local mic muted');
return; return;
@@ -311,21 +300,27 @@ class ChatbotClient {
/** /**
* Initialize and connect to the bot * Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection * This sets up the Pipecat client, initializes devices, and establishes the connection
*/ */
async connect() { async connect() {
try { try {
const botSelector = document.getElementById('bot-selector'); const botSelector = document.getElementById('bot-selector');
const selectedBot = botSelector.value; const selectedBot = botSelector.value;
this.rtviClient.params.requestData.bot_name = selectedBot;
// Initialize audio/video devices // Initialize audio/video devices
this.log('Initializing devices...'); this.log('Initializing devices...');
await this.rtviClient.initDevices(); await this.pcClient.initDevices();
// Connect to the bot // Connect to the bot
this.log(`Connecting to bot: ${selectedBot}`); this.log(`Connecting to bot: ${selectedBot}`);
await this.rtviClient.connect(); await this.pcClient.connect({
// REPLACE WITH YOUR MODAL URL ENDPOINT
endpoint:
'https://<your-workspace>--pipecat-modal-fastapi-app.modal.run/connect',
requestData: {
bot_name: selectedBot,
},
});
this.log('Connection complete'); this.log('Connection complete');
} catch (error) { } catch (error) {
@@ -336,9 +331,9 @@ class ChatbotClient {
this.updateStatus('Error'); this.updateStatus('Error');
// Clean up if there's an error // Clean up if there's an error
if (this.rtviClient) { if (this.pcClient) {
try { try {
await this.rtviClient.disconnect(); await this.pcClient.disconnect();
} catch (disconnectError) { } catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError.message}`); this.log(`Error during disconnect: ${disconnectError.message}`);
} }
@@ -350,10 +345,10 @@ class ChatbotClient {
* Disconnect from the bot and clean up media resources * Disconnect from the bot and clean up media resources
*/ */
async disconnect() { async disconnect() {
if (this.rtviClient) { if (this.pcClient) {
try { try {
// Disconnect the RTVI client // Disconnect the Pipecat client
await this.rtviClient.disconnect(); await this.pcClient.disconnect();
// Clean up audio // Clean up audio
if (this.botAudio.srcObject) { if (this.botAudio.srcObject) {

View File

@@ -44,7 +44,7 @@ Try the hosted version of the demo here: https://pcc-smart-turn.vercel.app/.
4. Run the server: 4. Run the server:
```bash ```bash
LOCAL=1 python server.py LOCAL_RUN=1 python server.py
``` ```
### Run the client ### Run the client

View File

@@ -1,5 +1,5 @@
import './globals.css'; import './globals.css';
import { RTVIProvider } from '@/providers/RTVIProvider'; import { PipecatProvider } from '@/providers/PipecatProvider';
export const metadata = { export const metadata = {
title: 'Pipecat React Client', title: 'Pipecat React Client',
@@ -20,7 +20,7 @@ export default function RootLayout({
<link rel="icon" href="/favicon.svg" type="image/svg+xml" /> <link rel="icon" href="/favicon.svg" type="image/svg+xml" />
</head> </head>
<body> <body>
<RTVIProvider>{children}</RTVIProvider> <PipecatProvider>{children}</PipecatProvider>
</body> </body>
</html> </html>
); );

View File

@@ -1,22 +1,22 @@
'use client'; 'use client';
import { import {
RTVIClientAudio, PipecatClientAudio,
RTVIClientVideo, PipecatClientVideo,
useRTVIClientTransportState, usePipecatClientTransportState,
} from '@pipecat-ai/client-react'; } from '@pipecat-ai/client-react';
import { ConnectButton } from '../components/ConnectButton'; import { ConnectButton } from '../components/ConnectButton';
import { StatusDisplay } from '../components/StatusDisplay'; import { StatusDisplay } from '../components/StatusDisplay';
import { DebugDisplay } from '../components/DebugDisplay'; import { DebugDisplay } from '../components/DebugDisplay';
function BotVideo() { function BotVideo() {
const transportState = useRTVIClientTransportState(); const transportState = usePipecatClientTransportState();
const isConnected = transportState !== 'disconnected'; const isConnected = transportState !== 'disconnected';
return ( return (
<div className="bot-container"> <div className="bot-container">
<div className="video-container"> <div className="video-container">
{isConnected && <RTVIClientVideo participant="bot" fit="cover" />} {isConnected && <PipecatClientVideo participant="bot" fit="cover" />}
</div> </div>
</div> </div>
); );
@@ -35,7 +35,7 @@ export default function Home() {
</div> </div>
<DebugDisplay /> <DebugDisplay />
<RTVIClientAudio /> <PipecatClientAudio />
</div> </div>
); );
} }

View File

@@ -1,11 +1,17 @@
import { import {
useRTVIClient, usePipecatClient,
useRTVIClientTransportState, usePipecatClientTransportState,
} from '@pipecat-ai/client-react'; } from '@pipecat-ai/client-react';
// Get the API base URL from environment variables
// Default to "/api" if not specified
// "/api" is the default for Next.js API routes and used
// for the Pipecat Cloud deployed agent
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
export function ConnectButton() { export function ConnectButton() {
const client = useRTVIClient(); const client = usePipecatClient();
const transportState = useRTVIClientTransportState(); const transportState = usePipecatClientTransportState();
const isConnected = ['connected', 'ready'].includes(transportState); const isConnected = ['connected', 'ready'].includes(transportState);
const handleClick = async () => { const handleClick = async () => {
@@ -18,7 +24,10 @@ export function ConnectButton() {
if (isConnected) { if (isConnected) {
await client.disconnect(); await client.disconnect();
} else { } else {
await client.connect(); await client.connect({
endpoint: `${API_BASE_URL}/connect`,
requestData: { foo: 'bar' },
});
} }
} catch (error) { } catch (error) {
console.error('Connection error:', error); console.error('Connection error:', error);

View File

@@ -6,7 +6,7 @@ import {
TranscriptData, TranscriptData,
BotLLMTextData, BotLLMTextData,
} from '@pipecat-ai/client-js'; } from '@pipecat-ai/client-js';
import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react'; import { usePipecatClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
import './DebugDisplay.css'; import './DebugDisplay.css';
interface SmartTurnResultData { interface SmartTurnResultData {
@@ -20,7 +20,7 @@ interface SmartTurnResultData {
export function DebugDisplay() { export function DebugDisplay() {
const debugLogRef = useRef<HTMLDivElement>(null); const debugLogRef = useRef<HTMLDivElement>(null);
const client = useRTVIClient(); const client = usePipecatClient();
const log = useCallback((message: string) => { const log = useCallback((message: string) => {
if (!debugLogRef.current) return; if (!debugLogRef.current) return;

View File

@@ -1,7 +1,7 @@
import { useRTVIClientTransportState } from '@pipecat-ai/client-react'; import { usePipecatClientTransportState } from '@pipecat-ai/client-react';
export function StatusDisplay() { export function StatusDisplay() {
const transportState = useRTVIClientTransportState(); const transportState = usePipecatClientTransportState();
return ( return (
<div className="status"> <div className="status">

View File

@@ -0,0 +1,28 @@
'use client';
import { PipecatClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { PipecatClientProvider } from '@pipecat-ai/client-react';
import { PropsWithChildren, useEffect, useState } from 'react';
export function PipecatProvider({ children }: PropsWithChildren) {
const [client, setClient] = useState<PipecatClient | null>(null);
useEffect(() => {
const pcClient = new PipecatClient({
transport: new DailyTransport(),
enableMic: true,
enableCam: false,
});
setClient(pcClient);
}, []);
if (!client) {
return null;
}
return (
<PipecatClientProvider client={client}>{children}</PipecatClientProvider>
);
}

View File

@@ -1,43 +0,0 @@
'use client';
import { RTVIClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { RTVIClientProvider } from '@pipecat-ai/client-react';
import { PropsWithChildren, useEffect, useState } from 'react';
// Get the API base URL from environment variables
// Default to "/api" if not specified
// "/api" is the default for Next.js API routes and used
// for the Pipecat Cloud deployed agent
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
console.log('Using API base URL:', API_BASE_URL);
export function RTVIProvider({ children }: PropsWithChildren) {
const [client, setClient] = useState<RTVIClient | null>(null);
useEffect(() => {
const transport = new DailyTransport();
const rtviClient = new RTVIClient({
transport,
params: {
baseUrl: API_BASE_URL,
endpoints: {
connect: '/connect',
},
requestData: { foo: 'bar' },
},
enableMic: true,
enableCam: false,
});
setClient(rtviClient);
}, []);
if (!client) {
return null;
}
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
}

View File

@@ -45,7 +45,7 @@ from pipecat.transports.services.daily import DailyParams, DailyTransport
load_dotenv(override=True) load_dotenv(override=True)
# Check if we're in local development mode # Check if we're in local development mode
LOCAL = os.getenv("LOCAL") LOCAL = os.getenv("LOCAL_RUN")
logger.remove() logger.remove()
logger.add(sys.stderr, level="DEBUG") logger.add(sys.stderr, level="DEBUG")

View File

@@ -20,11 +20,10 @@ import {
} from '@pipecat-ai/client-js'; } from '@pipecat-ai/client-js';
import { import {
ProtobufFrameSerializer, ProtobufFrameSerializer,
WebSocketTransport WebSocketTransport,
} from "@pipecat-ai/websocket-transport"; } from '@pipecat-ai/websocket-transport';
class RecordingSerializer extends ProtobufFrameSerializer { class RecordingSerializer extends ProtobufFrameSerializer {
private lastTimestamp: number | null = null; private lastTimestamp: number | null = null;
private recordingAudioToSend: boolean = false; private recordingAudioToSend: boolean = false;
private _recordedAudio: { data: ArrayBuffer; delay: number }[] = []; private _recordedAudio: { data: ArrayBuffer; delay: number }[] = [];
@@ -40,7 +39,11 @@ class RecordingSerializer extends ProtobufFrameSerializer {
} }
// @ts-ignore // @ts-ignore
serializeAudio(data: ArrayBuffer, sampleRate: number, numChannels: number): Uint8Array | null { serializeAudio(
data: ArrayBuffer,
sampleRate: number,
numChannels: number
): Uint8Array | null {
if (this.recordingAudioToSend) { if (this.recordingAudioToSend) {
const now = Date.now(); const now = Date.now();
// Compute delay since last packet // Compute delay since last packet
@@ -55,13 +58,13 @@ class RecordingSerializer extends ProtobufFrameSerializer {
} }
public get recordedAudio() { public get recordedAudio() {
return this._recordedAudio return this._recordedAudio;
} }
} }
class WebsocketClientApp { class WebsocketClientApp {
private ENABLE_RECORDING_MODE = false private ENABLE_RECORDING_MODE = false;
private RECORDING_TIME_MS = 10000 private RECORDING_TIME_MS = 10000;
private rtviClient: RTVIClient | null = null; private rtviClient: RTVIClient | null = null;
private connectBtn: HTMLButtonElement | null = null; private connectBtn: HTMLButtonElement | null = null;
@@ -71,7 +74,7 @@ class WebsocketClientApp {
private botAudio: HTMLAudioElement; private botAudio: HTMLAudioElement;
private declare websocketTransport: WebSocketTransport; private declare websocketTransport: WebSocketTransport;
private sendRecordedAudio: boolean = false private sendRecordedAudio: boolean = false;
private declare recordingSerializer: RecordingSerializer; private declare recordingSerializer: RecordingSerializer;
private playBtn: HTMLButtonElement | null = null; private playBtn: HTMLButtonElement | null = null;
@@ -91,8 +94,12 @@ class WebsocketClientApp {
* Set up references to DOM elements and create necessary media elements * Set up references to DOM elements and create necessary media elements
*/ */
private setupDOMElements(): void { private setupDOMElements(): void {
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement; this.connectBtn = document.getElementById(
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement; 'connect-btn'
) as HTMLButtonElement;
this.disconnectBtn = document.getElementById(
'disconnect-btn'
) as HTMLButtonElement;
this.statusSpan = document.getElementById('connection-status'); this.statusSpan = document.getElementById('connection-status');
this.debugLog = document.getElementById('debug-log'); this.debugLog = document.getElementById('debug-log');
this.playBtn = document.getElementById('play-btn') as HTMLButtonElement; this.playBtn = document.getElementById('play-btn') as HTMLButtonElement;
@@ -105,8 +112,12 @@ class WebsocketClientApp {
private setupEventListeners(): void { private setupEventListeners(): void {
this.connectBtn?.addEventListener('click', () => this.connect()); this.connectBtn?.addEventListener('click', () => this.connect());
this.disconnectBtn?.addEventListener('click', () => this.disconnect()); this.disconnectBtn?.addEventListener('click', () => this.disconnect());
this.playBtn?.addEventListener('click', () => this.startSendingRecordedAudio()); this.playBtn?.addEventListener('click', () =>
this.stopBtn?.addEventListener('click', () => this.stopSendingRecordedAudio()); this.startSendingRecordedAudio()
);
this.stopBtn?.addEventListener('click', () =>
this.stopSendingRecordedAudio()
);
} }
/** /**
@@ -165,7 +176,9 @@ class WebsocketClientApp {
// Listen for tracks stopping // Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => { this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`); this.log(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
);
}); });
} }
@@ -175,7 +188,10 @@ class WebsocketClientApp {
*/ */
private setupAudioTrack(track: MediaStreamTrack): void { private setupAudioTrack(track: MediaStreamTrack): void {
this.log('Setting up audio track'); this.log('Setting up audio track');
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) { if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0]; const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
if (oldTrack?.id === track.id) return; if (oldTrack?.id === track.id) return;
} }
@@ -190,27 +206,17 @@ class WebsocketClientApp {
try { try {
const startTime = Date.now(); const startTime = Date.now();
this.recordingSerializer = new RecordingSerializer() this.recordingSerializer = new RecordingSerializer();
const transport = this.ENABLE_RECORDING_MODE ? const ws_opts = {
new WebSocketTransport({ serializer: this.ENABLE_RECORDING_MODE
serializer: this.recordingSerializer, ? this.recordingSerializer
recorderSampleRate: 8000, : new ProtobufFrameSerializer(),
playerSampleRate:8000 recorderSampleRate: 8000,
}) : playerSampleRate: 8000,
new WebSocketTransport({ };
serializer: new ProtobufFrameSerializer(),
recorderSampleRate: 8000,
playerSampleRate:8000
});
this.websocketTransport = transport
const RTVIConfig: RTVIClientOptions = { const RTVIConfig: RTVIClientOptions = {
transport, transport: new WebSocketTransport(ws_opts),
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: { connect: '/connect' },
},
enableMic: true, enableMic: true,
enableCam: false, enableCam: false,
callbacks: { callbacks: {
@@ -238,27 +244,34 @@ class WebsocketClientApp {
onMessageError: (error) => console.error('Message error:', error), onMessageError: (error) => console.error('Message error:', error),
onError: (error) => console.error('Error:', error), onError: (error) => console.error('Error:', error),
}, },
} };
this.rtviClient = new RTVIClient(RTVIConfig); this.rtviClient = new RTVIClient(RTVIConfig);
this.websocketTransport = this.rtviClient.transport;
this.setupTrackListeners(); this.setupTrackListeners();
this.log('Initializing devices...'); this.log('Initializing devices...');
await this.rtviClient.initDevices(); await this.rtviClient.initDevices();
this.log('Connecting to bot...'); this.log('Connecting to bot...');
await this.rtviClient.connect(); await this.rtviClient.connect({
endpoint: 'http://localhost:7860/connect',
});
const timeTaken = Date.now() - startTime; const timeTaken = Date.now() - startTime;
this.log(`Connection complete, timeTaken: ${timeTaken}`); this.log(`Connection complete, timeTaken: ${timeTaken}`);
if (this.ENABLE_RECORDING_MODE) { if (this.ENABLE_RECORDING_MODE) {
this.log(`Starting to recording the next ${(this.RECORDING_TIME_MS/1000)}s of audio`); this.log(
this.recordingSerializer.startRecording() `Starting to recording the next ${
await this.sleep(this.RECORDING_TIME_MS) this.RECORDING_TIME_MS / 1000
this.recordingSerializer.stopRecording() }s of audio`
this.log("Recording stopped"); );
this.rtviClient.enableMic(false) this.recordingSerializer.startRecording();
this.startSendingRecordedAudio() await this.sleep(this.RECORDING_TIME_MS);
this.recordingSerializer.stopRecording();
this.log('Recording stopped');
this.rtviClient.enableMic(false);
this.startSendingRecordedAudio();
} }
} catch (error) { } catch (error) {
this.log(`Error connecting: ${(error as Error).message}`); this.log(`Error connecting: ${(error as Error).message}`);
@@ -280,11 +293,16 @@ class WebsocketClientApp {
public async disconnect(): Promise<void> { public async disconnect(): Promise<void> {
if (this.rtviClient) { if (this.rtviClient) {
try { try {
this.stopSendingRecordedAudio() this.stopSendingRecordedAudio();
await this.rtviClient.disconnect(); await this.rtviClient.disconnect();
this.rtviClient = null; this.rtviClient = null;
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) { if (
this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop()); this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
this.botAudio.srcObject
.getAudioTracks()
.forEach((track) => track.stop());
this.botAudio.srcObject = null; this.botAudio.srcObject = null;
} }
} catch (error) { } catch (error) {
@@ -294,21 +312,21 @@ class WebsocketClientApp {
} }
private startSendingRecordedAudio() { private startSendingRecordedAudio() {
this.sendRecordedAudio = true this.sendRecordedAudio = true;
if (this.playBtn) this.playBtn.disabled = true; if (this.playBtn) this.playBtn.disabled = true;
if (this.stopBtn) this.stopBtn.disabled = false; if (this.stopBtn) this.stopBtn.disabled = false;
void this.replayAudio() void this.replayAudio();
} }
private stopSendingRecordedAudio() { private stopSendingRecordedAudio() {
if (this.stopBtn) this.stopBtn.disabled = true; if (this.stopBtn) this.stopBtn.disabled = true;
if (this.playBtn) this.playBtn.disabled = false; if (this.playBtn) this.playBtn.disabled = false;
this.sendRecordedAudio = false this.sendRecordedAudio = false;
} }
private async replayAudio() { private async replayAudio() {
if (this.sendRecordedAudio) { if (this.sendRecordedAudio) {
this.log("Sending recorded audio") this.log('Sending recorded audio');
for (const chunk of this.recordingSerializer.recordedAudio) { for (const chunk of this.recordingSerializer.recordedAudio) {
await this.sleep(chunk.delay); await this.sleep(chunk.delay);
this.websocketTransport.handleUserAudioStream(chunk.data); this.websocketTransport.handleUserAudioStream(chunk.data);
@@ -316,14 +334,13 @@ class WebsocketClientApp {
const randomDelay = 1000 + Math.random() * (10000 - 500); const randomDelay = 1000 + Math.random() * (10000 - 500);
await this.sleep(randomDelay); await this.sleep(randomDelay);
void this.replayAudio() void this.replayAudio();
} }
} }
private sleep(ms: number): Promise<void> { private sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
} }
declare global { declare global {

View File

@@ -5,7 +5,7 @@
*/ */
/** /**
* RTVI Client Implementation * Pipecat Client Implementation
* *
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily). * This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
* It handles audio/video streaming and manages the connection lifecycle. * It handles audio/video streaming and manages the connection lifecycle.
@@ -18,20 +18,22 @@
import { import {
Participant, Participant,
RTVIClient, PipecatClient,
RTVIClientOptions, PipecatClientOptions,
RTVIEvent, RTVIEvent,
} from '@pipecat-ai/client-js'; } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport'; import {
DailyEventCallbacks,
DailyTransport,
} from '@pipecat-ai/daily-transport';
import SoundUtils from './util/soundUtils'; import SoundUtils from './util/soundUtils';
import { InstantVoiceHelper } from './util/instantVoiceHelper';
/** /**
* InstantVoiceClient handles the connection and media management for a real-time * InstantVoiceClient handles the connection and media management for a real-time
* voice and video interaction with an AI bot. * voice and video interaction with an AI bot.
*/ */
class InstantVoiceClient { class InstantVoiceClient {
private declare rtviClient: RTVIClient; private declare pcClient: PipecatClient;
private connectBtn: HTMLButtonElement | null = null; private connectBtn: HTMLButtonElement | null = null;
private disconnectBtn: HTMLButtonElement | null = null; private disconnectBtn: HTMLButtonElement | null = null;
private statusSpan: HTMLElement | null = null; private statusSpan: HTMLElement | null = null;
@@ -46,7 +48,7 @@ class InstantVoiceClient {
document.body.appendChild(this.botAudio); document.body.appendChild(this.botAudio);
this.setupDOMElements(); this.setupDOMElements();
this.setupEventListeners(); this.setupEventListeners();
this.initializeRTVIClient(); this.initializePipecatClient();
} }
/** /**
@@ -72,16 +74,11 @@ class InstantVoiceClient {
this.disconnectBtn?.addEventListener('click', () => this.disconnect()); this.disconnectBtn?.addEventListener('click', () => this.disconnect());
} }
private initializeRTVIClient(): void { private initializePipecatClient(): void {
const RTVIConfig: RTVIClientOptions = { const PipecatConfig: PipecatClientOptions = {
transport: new DailyTransport({ transport: new DailyTransport({
bufferLocalAudioUntilBotReady: true, bufferLocalAudioUntilBotReady: true,
}), }),
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: { connect: '/connect' },
},
enableMic: true, enableMic: true,
enableCam: false, enableCam: false,
callbacks: { callbacks: {
@@ -113,30 +110,23 @@ class InstantVoiceClient {
onBotTranscript: (data) => this.log(`Bot: ${data.text}`), onBotTranscript: (data) => this.log(`Bot: ${data.text}`),
onMessageError: (error) => console.error('Message error:', error), onMessageError: (error) => console.error('Message error:', error),
onError: (error) => console.error('Error:', error), onError: (error) => console.error('Error:', error),
}, onAudioBufferingStarted: () => {
SoundUtils.beep();
this.updateBufferingStatus('Yes');
this.log(
`onMicCaptureStarted, timeTaken: ${Date.now() - this.startTime}`
);
},
onAudioBufferingStopped: () => {
this.updateBufferingStatus('No');
this.log(
`onMicCaptureStopped, timeTaken: ${Date.now() - this.startTime}`
);
},
} as DailyEventCallbacks,
}; };
this.rtviClient = new RTVIClient(RTVIConfig); this.pcClient = new PipecatClient(PipecatConfig);
this.rtviClient.registerHelper(
'transport',
new InstantVoiceHelper({
callbacks: {
onAudioBufferingStarted: () => {
SoundUtils.beep();
this.updateBufferingStatus('Yes');
this.log(
`onMicCaptureStarted, timeTaken: ${Date.now() - this.startTime}`
);
},
onAudioBufferingStopped: () => {
this.updateBufferingStatus('No');
this.log(
`onMicCaptureStopped, timeTaken: ${Date.now() - this.startTime}`
);
},
},
})
);
this.setupTrackListeners(); this.setupTrackListeners();
} }
@@ -182,8 +172,8 @@ class InstantVoiceClient {
* This is called when the bot is ready or when the transport state changes to ready * This is called when the bot is ready or when the transport state changes to ready
*/ */
setupMediaTracks() { setupMediaTracks() {
if (!this.rtviClient) return; if (!this.pcClient) return;
const tracks = this.rtviClient.tracks(); const tracks = this.pcClient.tracks();
if (tracks.bot?.audio) { if (tracks.bot?.audio) {
this.setupAudioTrack(tracks.bot.audio); this.setupAudioTrack(tracks.bot.audio);
} }
@@ -194,10 +184,10 @@ class InstantVoiceClient {
* This handles new tracks being added during the session * This handles new tracks being added during the session
*/ */
setupTrackListeners() { setupTrackListeners() {
if (!this.rtviClient) return; if (!this.pcClient) return;
// Listen for new tracks starting // Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => { this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks // Only handle non-local (bot) tracks
if (!participant?.local && track.kind === 'audio') { if (!participant?.local && track.kind === 'audio') {
this.setupAudioTrack(track); this.setupAudioTrack(track);
@@ -205,7 +195,7 @@ class InstantVoiceClient {
}); });
// Listen for tracks stopping // Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => { this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log( this.log(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}` `Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
); );
@@ -230,22 +220,25 @@ class InstantVoiceClient {
/** /**
* Initialize and connect to the bot * Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection * This sets up the Pipecat client, initializes devices, and establishes the connection
*/ */
public async connect(): Promise<void> { public async connect(): Promise<void> {
try { try {
this.startTime = Date.now(); this.startTime = Date.now();
this.log('Connecting to bot...'); this.log('Connecting to bot...');
await this.rtviClient.connect(); await this.pcClient.connect({
// The baseURL and endpoint of your bot server that the client will connect to
endpoint: 'http://localhost:7860/connect',
});
} catch (error) { } catch (error) {
this.log(`Error connecting: ${(error as Error).message}`); this.log(`Error connecting: ${(error as Error).message}`);
this.updateStatus('Error'); this.updateStatus('Error');
this.updateBufferingStatus('No'); this.updateBufferingStatus('No');
// Clean up if there's an error // Clean up if there's an error
if (this.rtviClient) { if (this.pcClient) {
try { try {
await this.rtviClient.disconnect(); await this.pcClient.disconnect();
} catch (disconnectError) { } catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError}`); this.log(`Error during disconnect: ${disconnectError}`);
} }
@@ -258,7 +251,7 @@ class InstantVoiceClient {
*/ */
public async disconnect(): Promise<void> { public async disconnect(): Promise<void> {
try { try {
await this.rtviClient.disconnect(); await this.pcClient.disconnect();
if ( if (
this.botAudio.srcObject && this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject 'getAudioTracks' in this.botAudio.srcObject

View File

@@ -1,39 +0,0 @@
import {RTVIClientHelper, RTVIClientHelperOptions, RTVIMessage} from "@pipecat-ai/client-js";
import {DailyRTVIMessageType} from '@pipecat-ai/daily-transport';
export type InstantVoiceHelperCallbacks = Partial<{
onAudioBufferingStarted: () => void;
onAudioBufferingStopped: () => void;
}>;
// --- Interface and class
export interface InstantVoiceHelperOptions extends RTVIClientHelperOptions {
callbacks?: InstantVoiceHelperCallbacks;
}
export class InstantVoiceHelper extends RTVIClientHelper {
protected declare _options: InstantVoiceHelperOptions;
constructor(options: InstantVoiceHelperOptions) {
super(options);
}
handleMessage(rtviMessage: RTVIMessage): void {
switch (rtviMessage.type) {
case DailyRTVIMessageType.AUDIO_BUFFERING_STARTED:
if (this._options.callbacks?.onAudioBufferingStarted) {
this._options.callbacks?.onAudioBufferingStarted()
}
break;
case DailyRTVIMessageType.AUDIO_BUFFERING_STOPPED:
if (this._options.callbacks?.onAudioBufferingStopped) {
this._options.callbacks?.onAudioBufferingStopped()
}
break;
}
}
getMessageTypes(): string[] {
return [DailyRTVIMessageType.AUDIO_BUFFERING_STARTED, DailyRTVIMessageType.AUDIO_BUFFERING_STOPPED];
}
}

View File

@@ -5,7 +5,7 @@
*/ */
/** /**
* RTVI Client Implementation * Pipecat Client Implementation
* *
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily). * This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
* It handles audio/video streaming and manages the connection lifecycle. * It handles audio/video streaming and manages the connection lifecycle.
@@ -16,78 +16,9 @@
* - Browser with WebRTC support * - Browser with WebRTC support
*/ */
import { import { LogLevel, PipecatClient, RTVIEvent } from '@pipecat-ai/client-js';
LogLevel,
RTVIClient,
RTVIClientHelper,
RTVIEvent,
} from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport'; import { DailyTransport } from '@pipecat-ai/daily-transport';
class SearchResponseHelper extends RTVIClientHelper {
constructor(contentPanel) {
super();
this.contentPanel = contentPanel;
}
handleMessage(rtviMessage) {
console.log('SearchResponseHelper, received message:', rtviMessage);
if (rtviMessage.data) {
// Clear existing content
this.contentPanel.innerHTML = '';
// Create a container for all content
const contentContainer = document.createElement('div');
contentContainer.className = 'content-container';
// Add the search_result
if (rtviMessage.data.search_result) {
const searchResultDiv = document.createElement('div');
searchResultDiv.className = 'search-result';
searchResultDiv.textContent = rtviMessage.data.search_result;
contentContainer.appendChild(searchResultDiv);
}
// Add the sources
if (rtviMessage.data.origins) {
const sourcesDiv = document.createElement('div');
sourcesDiv.className = 'sources';
const sourcesTitle = document.createElement('h3');
sourcesTitle.className = 'sources-title';
sourcesTitle.textContent = 'Sources:';
sourcesDiv.appendChild(sourcesTitle);
rtviMessage.data.origins.forEach((origin) => {
const sourceLink = document.createElement('a');
sourceLink.className = 'source-link';
sourceLink.href = origin.site_uri;
sourceLink.target = '_blank';
sourceLink.textContent = origin.site_title;
sourcesDiv.appendChild(sourceLink);
});
contentContainer.appendChild(sourcesDiv);
}
// Add the rendered_content in an iframe
if (rtviMessage.data.rendered_content) {
const iframe = document.createElement('iframe');
iframe.className = 'iframe-container';
iframe.srcdoc = rtviMessage.data.rendered_content;
contentContainer.appendChild(iframe);
}
// Append the content container to the content panel
this.contentPanel.appendChild(contentContainer);
}
}
getMessageTypes() {
return ['bot-llm-search-response'];
}
}
/** /**
* ChatbotClient handles the connection and media management for a real-time * ChatbotClient handles the connection and media management for a real-time
* voice and video interaction with an AI bot. * voice and video interaction with an AI bot.
@@ -95,7 +26,7 @@ class SearchResponseHelper extends RTVIClientHelper {
class ChatbotClient { class ChatbotClient {
constructor() { constructor() {
// Initialize client state // Initialize client state
this.rtviClient = null; this.pcClient = null;
this.setupDOMElements(); this.setupDOMElements();
this.setupEventListeners(); this.setupEventListeners();
} }
@@ -160,10 +91,10 @@ class ChatbotClient {
* This is called when the bot is ready or when the transport state changes to ready * This is called when the bot is ready or when the transport state changes to ready
*/ */
setupMediaTracks() { setupMediaTracks() {
if (!this.rtviClient) return; if (!this.pcClient) return;
// Get current tracks from the client // Get current tracks from the client
const tracks = this.rtviClient.tracks(); const tracks = this.pcClient.tracks();
// Set up any available bot tracks // Set up any available bot tracks
if (tracks.bot?.audio) { if (tracks.bot?.audio) {
@@ -176,10 +107,10 @@ class ChatbotClient {
* This handles new tracks being added during the session * This handles new tracks being added during the session
*/ */
setupTrackListeners() { setupTrackListeners() {
if (!this.rtviClient) return; if (!this.pcClient) return;
// Listen for new tracks starting // Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => { this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks // Only handle non-local (bot) tracks
if (!participant?.local && track.kind === 'audio') { if (!participant?.local && track.kind === 'audio') {
this.setupAudioTrack(track); this.setupAudioTrack(track);
@@ -187,7 +118,7 @@ class ChatbotClient {
}); });
// Listen for tracks stopping // Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => { this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log( this.log(
`Track stopped event: ${track.kind} from ${ `Track stopped event: ${track.kind} from ${
participant?.name || 'unknown' participant?.name || 'unknown'
@@ -213,20 +144,13 @@ class ChatbotClient {
/** /**
* Initialize and connect to the bot * Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection * This sets up the Pipecat client, initializes devices, and establishes the connection
*/ */
async connect() { async connect() {
try { try {
// Initialize the RTVI client with a Daily WebRTC transport and our configuration // Initialize the Pipecat client with a Daily WebRTC transport and our configuration
this.rtviClient = new RTVIClient({ this.pcClient = new PipecatClient({
transport: new DailyTransport(), transport: new DailyTransport(),
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: {
connect: '/connect',
},
},
enableMic: true, // Enable microphone for user input enableMic: true, // Enable microphone for user input
enableCam: false, enableCam: false,
callbacks: { callbacks: {
@@ -251,6 +175,8 @@ class ChatbotClient {
this.setupMediaTracks(); this.setupMediaTracks();
} }
}, },
// Handle search response events
onBotLlmSearchResponse: this.handleSearchResponse.bind(this),
// Handle bot connection events // Handle bot connection events
onBotConnected: (participant) => { onBotConnected: (participant) => {
this.log(`Bot connected: ${JSON.stringify(participant)}`); this.log(`Bot connected: ${JSON.stringify(participant)}`);
@@ -281,22 +207,22 @@ class ChatbotClient {
}, },
}, },
}); });
//this.rtviClient.setLogLevel(LogLevel.DEBUG)
this.rtviClient.registerHelper( //this.pcClient.setLogLevel(LogLevel.DEBUG)
'llm',
new SearchResponseHelper(this.searchResultContainer)
);
// Set up listeners for media track events // Set up listeners for media track events
this.setupTrackListeners(); this.setupTrackListeners();
// Initialize audio devices // Initialize audio devices
this.log('Initializing devices...'); this.log('Initializing devices...');
await this.rtviClient.initDevices(); await this.pcClient.initDevices();
// Connect to the bot // Connect to the bot
this.log('Connecting to bot...'); this.log('Connecting to bot...');
await this.rtviClient.connect(); await this.pcClient.connect({
// The baseURL and endpoint of your bot server that the client will connect to
endpoint: 'http://localhost:7860/connect',
});
this.log('Connection complete'); this.log('Connection complete');
} catch (error) { } catch (error) {
@@ -306,9 +232,9 @@ class ChatbotClient {
this.updateStatus('Error'); this.updateStatus('Error');
// Clean up if there's an error // Clean up if there's an error
if (this.rtviClient) { if (this.pcClient) {
try { try {
await this.rtviClient.disconnect(); await this.pcClient.disconnect();
} catch (disconnectError) { } catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError.message}`); this.log(`Error during disconnect: ${disconnectError.message}`);
} }
@@ -320,11 +246,11 @@ class ChatbotClient {
* Disconnect from the bot and clean up media resources * Disconnect from the bot and clean up media resources
*/ */
async disconnect() { async disconnect() {
if (this.rtviClient) { if (this.pcClient) {
try { try {
// Disconnect the RTVI client // Disconnect the Pipecat client
await this.rtviClient.disconnect(); await this.pcClient.disconnect();
this.rtviClient = null; this.pcClient = null;
// Clean up audio // Clean up audio
if (this.botAudio.srcObject) { if (this.botAudio.srcObject) {
@@ -339,6 +265,57 @@ class ChatbotClient {
} }
} }
} }
handleSearchResponse(response) {
console.log('SearchResponseHelper, received message:', response);
// Clear existing content
this.searchResultContainer.innerHTML = '';
// Create a container for all content
const contentContainer = document.createElement('div');
contentContainer.className = 'content-container';
// Add the search_result
if (response.search_result) {
const searchResultDiv = document.createElement('div');
searchResultDiv.className = 'search-result';
searchResultDiv.textContent = response.search_result;
contentContainer.appendChild(searchResultDiv);
}
// Add the sources
if (response.origins) {
const sourcesDiv = document.createElement('div');
sourcesDiv.className = 'sources';
const sourcesTitle = document.createElement('h3');
sourcesTitle.className = 'sources-title';
sourcesTitle.textContent = 'Sources:';
sourcesDiv.appendChild(sourcesTitle);
response.origins.forEach((origin) => {
const sourceLink = document.createElement('a');
sourceLink.className = 'source-link';
sourceLink.href = origin.site_uri;
sourceLink.target = '_blank';
sourceLink.textContent = origin.site_title;
sourcesDiv.appendChild(sourceLink);
});
contentContainer.appendChild(sourcesDiv);
}
// Add the rendered_content in an iframe
if (response.rendered_content) {
const iframe = document.createElement('iframe');
iframe.className = 'iframe-container';
iframe.srcdoc = response.rendered_content;
contentContainer.appendChild(iframe);
}
// Append the content container to the content panel
this.searchResultContainer.appendChild(contentContainer);
}
} }
// Initialize the client when the page loads // Initialize the client when the page loads

View File

@@ -1,9 +1,11 @@
import { SmallWebRTCTransport } from '@pipecat-ai/small-webrtc-transport'; import { SmallWebRTCTransport } from '@pipecat-ai/small-webrtc-transport';
import { import {
BotLLMTextData,
Participant, Participant,
RTVIClient, PipecatClient,
RTVIClientOptions, PipecatClientOptions,
Transport, TranscriptData,
TransportState,
} from '@pipecat-ai/client-js'; } from '@pipecat-ai/client-js';
class WebRTCApp { class WebRTCApp {
@@ -23,26 +25,22 @@ class WebRTCApp {
private statusSpan: HTMLElement | null = null; private statusSpan: HTMLElement | null = null;
private declare smallWebRTCTransport: SmallWebRTCTransport; private declare smallWebRTCTransport: SmallWebRTCTransport;
private declare rtviClient: RTVIClient; private declare pcClient: PipecatClient;
constructor() { constructor() {
this.setupDOMElements(); this.setupDOMElements();
this.setupDOMEventListeners(); this.setupDOMEventListeners();
this.initializeRTVIClient(); this.initializePipecatClient();
void this.populateDevices(); void this.populateDevices();
} }
private initializeRTVIClient(): void { private initializePipecatClient(): void {
const transport = new SmallWebRTCTransport(); const opts: PipecatClientOptions = {
const RTVIConfig: RTVIClientOptions = { transport: new SmallWebRTCTransport({ connectionUrl: '/api/offer' }),
params: {
baseUrl: '/api/offer',
},
transport: transport as Transport,
enableMic: true, enableMic: true,
enableCam: true, enableCam: true,
callbacks: { callbacks: {
onTransportStateChanged: (state) => { onTransportStateChanged: (state: TransportState) => {
this.log(`Transport state: ${state}`); this.log(`Transport state: ${state}`);
}, },
onConnected: () => { onConnected: () => {
@@ -66,13 +64,13 @@ class WebRTCApp {
onBotStoppedSpeaking: () => { onBotStoppedSpeaking: () => {
this.log('Bot stopped speaking.'); this.log('Bot stopped speaking.');
}, },
onUserTranscript: (transcript) => { onUserTranscript: (transcript: TranscriptData) => {
if (transcript.final) { if (transcript.final) {
this.log(`User transcript: ${transcript.text}`); this.log(`User transcript: ${transcript.text}`);
} }
}, },
onBotTranscript: (transcript) => { onBotTranscript: (data: BotLLMTextData) => {
this.log(`Bot transcript: ${transcript.text}`); this.log(`Bot transcript: ${data.text}`);
}, },
onTrackStarted: ( onTrackStarted: (
track: MediaStreamTrack, track: MediaStreamTrack,
@@ -83,14 +81,13 @@ class WebRTCApp {
} }
this.onBotTrackStarted(track); this.onBotTrackStarted(track);
}, },
onServerMessage: (msg) => { onServerMessage: (msg: unknown) => {
this.log(`Server message: ${msg}`); this.log(`Server message: ${msg}`);
}, },
}, },
}; };
RTVIConfig.customConnectHandler = () => Promise.resolve(); this.pcClient = new PipecatClient(opts);
this.rtviClient = new RTVIClient(RTVIConfig); this.smallWebRTCTransport = this.pcClient.transport as SmallWebRTCTransport;
this.smallWebRTCTransport = transport;
} }
private setupDOMElements(): void { private setupDOMElements(): void {
@@ -132,16 +129,16 @@ class WebRTCApp {
this.audioInput.addEventListener('change', (e) => { this.audioInput.addEventListener('change', (e) => {
// @ts-ignore // @ts-ignore
let audioDevice = e.target?.value; let audioDevice = e.target?.value;
this.rtviClient.updateMic(audioDevice); this.pcClient.updateMic(audioDevice);
}); });
this.videoInput.addEventListener('change', (e) => { this.videoInput.addEventListener('change', (e) => {
// @ts-ignore // @ts-ignore
let videoDevice = e.target?.value; let videoDevice = e.target?.value;
this.rtviClient.updateCam(videoDevice); this.pcClient.updateCam(videoDevice);
}); });
this.muteBtn.addEventListener('click', () => { this.muteBtn.addEventListener('click', () => {
let isCamEnabled = this.rtviClient.isCamEnabled; let isCamEnabled = this.pcClient.isCamEnabled;
this.rtviClient.enableCam(!isCamEnabled); this.pcClient.enableCam(!isCamEnabled);
this.muteBtn.textContent = isCamEnabled ? '📵' : '📷'; this.muteBtn.textContent = isCamEnabled ? '📵' : '📷';
}); });
} }
@@ -206,9 +203,9 @@ class WebRTCApp {
}; };
try { try {
const audioDevices = await this.rtviClient.getAllMics(); const audioDevices = await this.pcClient.getAllMics();
populateSelect(this.audioInput, audioDevices); populateSelect(this.audioInput, audioDevices);
const videoDevices = await this.rtviClient.getAllCams(); const videoDevices = await this.pcClient.getAllCams();
populateSelect(this.videoInput, videoDevices); populateSelect(this.videoInput, videoDevices);
} catch (e) { } catch (e) {
alert(e); alert(e);
@@ -224,7 +221,7 @@ class WebRTCApp {
this.smallWebRTCTransport.setAudioCodec(this.audioCodec.value); this.smallWebRTCTransport.setAudioCodec(this.audioCodec.value);
this.smallWebRTCTransport.setVideoCodec(this.videoCodec.value); this.smallWebRTCTransport.setVideoCodec(this.videoCodec.value);
try { try {
await this.rtviClient.connect(); await this.pcClient.connect();
} catch (e) { } catch (e) {
console.log(`Failed to connect ${e}`); console.log(`Failed to connect ${e}`);
this.stop(); this.stop();
@@ -232,7 +229,7 @@ class WebRTCApp {
} }
private stop(): void { private stop(): void {
void this.rtviClient.disconnect(); void this.pcClient.disconnect();
} }
} }

View File

@@ -5,7 +5,7 @@
*/ */
/** /**
* RTVI Client Implementation * Pipecat Client Implementation
* *
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily). * This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
* It handles audio/video streaming and manages the connection lifecycle. * It handles audio/video streaming and manages the connection lifecycle.
@@ -16,7 +16,7 @@
* - Browser with WebRTC support * - Browser with WebRTC support
*/ */
import { RTVIClient, RTVIEvent } from '@pipecat-ai/client-js'; import { PipecatClient, RTVIEvent } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport'; import { DailyTransport } from '@pipecat-ai/daily-transport';
/** /**
@@ -26,7 +26,7 @@ import { DailyTransport } from '@pipecat-ai/daily-transport';
class ChatbotClient { class ChatbotClient {
constructor() { constructor() {
// Initialize client state // Initialize client state
this.rtviClient = null; this.pcClient = null;
this.setupDOMElements(); this.setupDOMElements();
this.initializeClientAndTransport(); this.initializeClientAndTransport();
} }
@@ -54,11 +54,14 @@ class ChatbotClient {
* Set up event listeners for connect/disconnect buttons * Set up event listeners for connect/disconnect buttons
*/ */
setupEventListeners() { setupEventListeners() {
this.connectBtn.addEventListener('click', () => this.connect()); this.connectBtn.addEventListener('click', () => {
console.log('click');
this.connect();
});
this.disconnectBtn.addEventListener('click', () => this.disconnect()); this.disconnectBtn.addEventListener('click', () => this.disconnect());
// Populate device selector // Populate device selector
this.rtviClient.getAllMics().then((mics) => { this.pcClient.getAllMics().then((mics) => {
console.log('Available mics:', mics); console.log('Available mics:', mics);
mics.forEach((device) => { mics.forEach((device) => {
const option = document.createElement('option'); const option = document.createElement('option');
@@ -70,33 +73,27 @@ class ChatbotClient {
this.deviceSelector.addEventListener('change', (event) => { this.deviceSelector.addEventListener('change', (event) => {
const selectedDeviceId = event.target.value; const selectedDeviceId = event.target.value;
console.log('Selected device ID:', selectedDeviceId); console.log('Selected device ID:', selectedDeviceId);
this.rtviClient.updateMic(selectedDeviceId); this.pcClient.updateMic(selectedDeviceId);
}); });
// Handle mic mute/unmute toggle // Handle mic mute/unmute toggle
const micToggleBtn = document.getElementById('mic-toggle-btn'); const micToggleBtn = document.getElementById('mic-toggle-btn');
micToggleBtn.addEventListener('click', () => { micToggleBtn.addEventListener('click', () => {
let micEnabled = this.rtviClient.isMicEnabled; let micEnabled = this.pcClient.isMicEnabled;
micToggleBtn.textContent = micEnabled ? 'Unmute Mic' : 'Mute Mic'; micToggleBtn.textContent = micEnabled ? 'Unmute Mic' : 'Mute Mic';
this.rtviClient.enableMic(!micEnabled); this.pcClient.enableMic(!micEnabled);
}); });
} }
/** /**
* Set up the RTVI client and Daily transport * Set up the Pipecat client and Daily transport
*/ */
async initializeClientAndTransport() { async initializeClientAndTransport() {
// Initialize the RTVI client with a DailyTransport and our configuration console.log('Initializing Pipecat client and transport...');
this.rtviClient = new RTVIClient({ // Initialize the Pipecat client with a DailyTransport and our configuration
this.pcClient = new PipecatClient({
transport: new DailyTransport(), transport: new DailyTransport(),
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: {
connect: '/connect',
},
},
enableMic: true, // Enable microphone for user input enableMic: true, // Enable microphone for user input
enableCam: false, enableCam: false,
callbacks: { callbacks: {
@@ -160,7 +157,7 @@ class ChatbotClient {
// Set up listeners for media track events // Set up listeners for media track events
this.setupTrackListeners(); this.setupTrackListeners();
await this.rtviClient.initDevices(); await this.pcClient.initDevices();
this.setupEventListeners(); this.setupEventListeners();
} }
@@ -196,10 +193,10 @@ class ChatbotClient {
* This is called when the bot is ready or when the transport state changes to ready * This is called when the bot is ready or when the transport state changes to ready
*/ */
setupMediaTracks() { setupMediaTracks() {
if (!this.rtviClient) return; if (!this.pcClient) return;
// Get current tracks from the client // Get current tracks from the client
const tracks = this.rtviClient.tracks(); const tracks = this.pcClient.tracks();
// Set up any available bot tracks // Set up any available bot tracks
if (tracks.bot?.audio) { if (tracks.bot?.audio) {
@@ -215,10 +212,10 @@ class ChatbotClient {
* This handles new tracks being added during the session * This handles new tracks being added during the session
*/ */
setupTrackListeners() { setupTrackListeners() {
if (!this.rtviClient) return; if (!this.pcClient) return;
// Listen for new tracks starting // Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => { this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks // Only handle non-local (bot) tracks
if (!participant?.local) { if (!participant?.local) {
if (track.kind === 'audio') { if (track.kind === 'audio') {
@@ -230,7 +227,7 @@ class ChatbotClient {
}); });
// Listen for tracks stopping // Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => { this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log( this.log(
`Track stopped event: ${track.kind} from ${ `Track stopped event: ${track.kind} from ${
participant?.name || 'unknown' participant?.name || 'unknown'
@@ -284,17 +281,16 @@ class ChatbotClient {
/** /**
* Initialize and connect to the bot * Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection * This sets up the Pipecat client, initializes devices, and establishes the connection
*/ */
async connect() { async connect() {
try { try {
// Initialize audio/video devices
this.log('Initializing devices...');
await this.rtviClient.initDevices();
// Connect to the bot // Connect to the bot
this.log('Connecting to bot...'); this.log('Connecting to bot...');
await this.rtviClient.connect(); await this.pcClient.connect({
endpoint: 'http://localhost:7860/connect',
timeout: 25000,
});
this.log('Connection complete'); this.log('Connection complete');
} catch (error) { } catch (error) {
@@ -304,9 +300,9 @@ class ChatbotClient {
this.updateStatus('Error'); this.updateStatus('Error');
// Clean up if there's an error // Clean up if there's an error
if (this.rtviClient) { if (this.pcClient) {
try { try {
await this.rtviClient.disconnect(); await this.pcClient.disconnect();
} catch (disconnectError) { } catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError.message}`); this.log(`Error during disconnect: ${disconnectError.message}`);
} }
@@ -318,10 +314,10 @@ class ChatbotClient {
* Disconnect from the bot and clean up media resources * Disconnect from the bot and clean up media resources
*/ */
async disconnect() { async disconnect() {
if (this.rtviClient) { if (this.pcClient) {
try { try {
// Disconnect the RTVI client // Disconnect the Pipecat client
await this.rtviClient.disconnect(); await this.pcClient.disconnect();
// Clean up audio // Clean up audio
if (this.botAudio.srcObject) { if (this.botAudio.srcObject) {

View File

@@ -1,22 +1,22 @@
import { import {
RTVIClientAudio, PipecatClientAudio,
RTVIClientVideo, PipecatClientVideo,
useRTVIClientTransportState, usePipecatClientTransportState,
} from '@pipecat-ai/client-react'; } from '@pipecat-ai/client-react';
import { RTVIProvider } from './providers/RTVIProvider'; import { PipecatProvider } from './providers/PipecatProvider';
import { ConnectButton } from './components/ConnectButton'; import { ConnectButton } from './components/ConnectButton';
import { StatusDisplay } from './components/StatusDisplay'; import { StatusDisplay } from './components/StatusDisplay';
import { DebugDisplay } from './components/DebugDisplay'; import { DebugDisplay } from './components/DebugDisplay';
import './App.css'; import './App.css';
function BotVideo() { function BotVideo() {
const transportState = useRTVIClientTransportState(); const transportState = usePipecatClientTransportState();
const isConnected = transportState !== 'disconnected'; const isConnected = transportState !== 'disconnected';
return ( return (
<div className="bot-container"> <div className="bot-container">
<div className="video-container"> <div className="video-container">
{isConnected && <RTVIClientVideo participant="bot" fit="cover" />} {isConnected && <PipecatClientVideo participant="bot" fit="cover" />}
</div> </div>
</div> </div>
); );
@@ -35,16 +35,16 @@ function AppContent() {
</div> </div>
<DebugDisplay /> <DebugDisplay />
<RTVIClientAudio /> <PipecatClientAudio />
</div> </div>
); );
} }
function App() { function App() {
return ( return (
<RTVIProvider> <PipecatProvider>
<AppContent /> <AppContent />
</RTVIProvider> </PipecatProvider>
); );
} }

View File

@@ -1,16 +1,16 @@
import { import {
useRTVIClient, usePipecatClient,
useRTVIClientTransportState, usePipecatClientTransportState,
} from '@pipecat-ai/client-react'; } from '@pipecat-ai/client-react';
export function ConnectButton() { export function ConnectButton() {
const client = useRTVIClient(); const client = usePipecatClient();
const transportState = useRTVIClientTransportState(); const transportState = usePipecatClientTransportState();
const isConnected = ['connected', 'ready'].includes(transportState); const isConnected = ['connected', 'ready'].includes(transportState);
const handleClick = async () => { const handleClick = async () => {
if (!client) { if (!client) {
console.error('RTVI client is not initialized'); console.error('Pipecat client is not initialized');
return; return;
} }
@@ -18,7 +18,7 @@ export function ConnectButton() {
if (isConnected) { if (isConnected) {
await client.disconnect(); await client.disconnect();
} else { } else {
await client.connect(); await client.connect({ endpoint: 'http://localhost:7860/connect' });
} }
} catch (error) { } catch (error) {
console.error('Connection error:', error); console.error('Connection error:', error);

View File

@@ -6,12 +6,12 @@ import {
TranscriptData, TranscriptData,
BotLLMTextData, BotLLMTextData,
} from '@pipecat-ai/client-js'; } from '@pipecat-ai/client-js';
import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react'; import { usePipecatClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
import './DebugDisplay.css'; import './DebugDisplay.css';
export function DebugDisplay() { export function DebugDisplay() {
const debugLogRef = useRef<HTMLDivElement>(null); const debugLogRef = useRef<HTMLDivElement>(null);
const client = useRTVIClient(); const client = usePipecatClient();
const log = useCallback((message: string) => { const log = useCallback((message: string) => {
if (!debugLogRef.current) return; if (!debugLogRef.current) return;

View File

@@ -1,7 +1,7 @@
import { useRTVIClientTransportState } from '@pipecat-ai/client-react'; import { usePipecatClientTransportState } from '@pipecat-ai/client-react';
export function StatusDisplay() { export function StatusDisplay() {
const transportState = useRTVIClientTransportState(); const transportState = usePipecatClientTransportState();
return ( return (
<div className="status"> <div className="status">

View File

@@ -0,0 +1,16 @@
import { type PropsWithChildren } from 'react';
import { PipecatClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { PipecatClientProvider } from '@pipecat-ai/client-react';
const client = new PipecatClient({
transport: new DailyTransport(),
enableMic: true,
enableCam: false,
});
export function PipecatProvider({ children }: PropsWithChildren) {
return (
<PipecatClientProvider client={client}>{children}</PipecatClientProvider>
);
}

View File

@@ -1,22 +0,0 @@
import { type PropsWithChildren } from 'react';
import { RTVIClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { RTVIClientProvider } from '@pipecat-ai/client-react';
const transport = new DailyTransport();
const client = new RTVIClient({
transport,
params: {
baseUrl: 'http://localhost:7860',
endpoints: {
connect: '/connect',
},
},
enableMic: true,
enableCam: false,
});
export function RTVIProvider({ children }: PropsWithChildren) {
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
}

View File

@@ -12,12 +12,11 @@ import {
import { import {
WebSocketTransport, WebSocketTransport,
TwilioSerializer, TwilioSerializer,
} from "@pipecat-ai/websocket-transport"; } from '@pipecat-ai/websocket-transport';
class WebsocketClientApp { class WebsocketClientApp {
private static STREAM_SID = 'ws_mock_stream_sid';
private static STREAM_SID = "ws_mock_stream_sid" private static CALL_SID = 'ws_mock_call_sid';
private static CALL_SID = "ws_mock_call_sid"
private rtviClient: RTVIClient | null = null; private rtviClient: RTVIClient | null = null;
private connectBtn: HTMLButtonElement | null = null; private connectBtn: HTMLButtonElement | null = null;
@@ -38,8 +37,12 @@ class WebsocketClientApp {
* Set up references to DOM elements and create necessary media elements * Set up references to DOM elements and create necessary media elements
*/ */
private setupDOMElements(): void { private setupDOMElements(): void {
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement; this.connectBtn = document.getElementById(
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement; 'connect-btn'
) as HTMLButtonElement;
this.disconnectBtn = document.getElementById(
'disconnect-btn'
) as HTMLButtonElement;
this.statusSpan = document.getElementById('connection-status'); this.statusSpan = document.getElementById('connection-status');
this.debugLog = document.getElementById('debug-log'); this.debugLog = document.getElementById('debug-log');
} }
@@ -80,13 +83,23 @@ class WebsocketClientApp {
} }
private async emulateTwilioMessages() { private async emulateTwilioMessages() {
const connectedMessage={"event": "connected", "protocol": "Call", "version": "1.0.0"} const connectedMessage = {
event: 'connected',
protocol: 'Call',
version: '1.0.0',
};
const websocketTransport = this.rtviClient?.transport as WebSocketTransport const websocketTransport = this.rtviClient?.transport as WebSocketTransport;
void websocketTransport?.sendRawMessage(connectedMessage) void websocketTransport?.sendRawMessage(connectedMessage);
const startMessage={"event": "start", "start": {"streamSid": WebsocketClientApp.STREAM_SID, "callSid": WebsocketClientApp.CALL_SID}} const startMessage = {
void websocketTransport?.sendRawMessage(startMessage) event: 'start',
start: {
streamSid: WebsocketClientApp.STREAM_SID,
callSid: WebsocketClientApp.CALL_SID,
},
};
void websocketTransport?.sendRawMessage(startMessage);
} }
/** /**
@@ -118,7 +131,9 @@ class WebsocketClientApp {
// Listen for tracks stopping // Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => { this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`); this.log(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
);
}); });
} }
@@ -128,7 +143,10 @@ class WebsocketClientApp {
*/ */
private setupAudioTrack(track: MediaStreamTrack): void { private setupAudioTrack(track: MediaStreamTrack): void {
this.log('Setting up audio track'); this.log('Setting up audio track');
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) { if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0]; const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
if (oldTrack?.id === track.id) return; if (oldTrack?.id === track.id) return;
} }
@@ -143,23 +161,19 @@ class WebsocketClientApp {
try { try {
const startTime = Date.now(); const startTime = Date.now();
const transport = new WebSocketTransport({ const ws_opts = {
serializer: new TwilioSerializer(), serializer: new TwilioSerializer(),
recorderSampleRate: 8000, recorderSampleRate: 8000,
playerSampleRate: 8000 playerSampleRate: 8000,
}); ws_url: 'http://localhost:8765/ws',
};
const RTVIConfig: RTVIClientOptions = { const RTVIConfig: RTVIClientOptions = {
transport, transport: new WebSocketTransport(ws_opts),
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:8765',
endpoints: { connect: '/' },
},
enableMic: true, enableMic: true,
enableCam: false, enableCam: false,
callbacks: { callbacks: {
onConnected: () => { onConnected: () => {
this.emulateTwilioMessages() this.emulateTwilioMessages();
this.updateStatus('Connected'); this.updateStatus('Connected');
if (this.connectBtn) this.connectBtn.disabled = true; if (this.connectBtn) this.connectBtn.disabled = true;
if (this.disconnectBtn) this.disconnectBtn.disabled = false; if (this.disconnectBtn) this.disconnectBtn.disabled = false;
@@ -183,13 +197,7 @@ class WebsocketClientApp {
onMessageError: (error) => console.error('Message error:', error), onMessageError: (error) => console.error('Message error:', error),
onError: (error) => console.error('Error:', error), onError: (error) => console.error('Error:', error),
}, },
} };
// @ts-ignore
RTVIConfig.customConnectHandler = () => Promise.resolve(
{
ws_url: "/ws",
}
);
this.rtviClient = new RTVIClient(RTVIConfig); this.rtviClient = new RTVIClient(RTVIConfig);
this.setupTrackListeners(); this.setupTrackListeners();
@@ -223,8 +231,13 @@ class WebsocketClientApp {
try { try {
await this.rtviClient.disconnect(); await this.rtviClient.disconnect();
this.rtviClient = null; this.rtviClient = null;
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) { if (
this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop()); this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
this.botAudio.srcObject
.getAudioTracks()
.forEach((track) => track.stop());
this.botAudio.srcObject = null; this.botAudio.srcObject = null;
} }
} catch (error) { } catch (error) {
@@ -232,7 +245,6 @@ class WebsocketClientApp {
} }
} }
} }
} }
declare global { declare global {

View File

@@ -5,7 +5,7 @@
*/ */
/** /**
* RTVI Client Implementation * Pipecat Client Implementation
* *
* This client connects to an RTVI-compatible bot server using WebSocket. * This client connects to an RTVI-compatible bot server using WebSocket.
* *
@@ -14,16 +14,14 @@
*/ */
import { import {
RTVIClient, PipecatClient,
RTVIClientOptions, PipecatClientOptions,
RTVIEvent, RTVIEvent,
} from '@pipecat-ai/client-js'; } from '@pipecat-ai/client-js';
import { import { WebSocketTransport } from '@pipecat-ai/websocket-transport';
WebSocketTransport
} from "@pipecat-ai/websocket-transport";
class WebsocketClientApp { class WebsocketClientApp {
private rtviClient: RTVIClient | null = null; private pcClient: PipecatClient | null = null;
private connectBtn: HTMLButtonElement | null = null; private connectBtn: HTMLButtonElement | null = null;
private disconnectBtn: HTMLButtonElement | null = null; private disconnectBtn: HTMLButtonElement | null = null;
private statusSpan: HTMLElement | null = null; private statusSpan: HTMLElement | null = null;
@@ -31,7 +29,7 @@ class WebsocketClientApp {
private botAudio: HTMLAudioElement; private botAudio: HTMLAudioElement;
constructor() { constructor() {
console.log("WebsocketClientApp"); console.log('WebsocketClientApp');
this.botAudio = document.createElement('audio'); this.botAudio = document.createElement('audio');
this.botAudio.autoplay = true; this.botAudio.autoplay = true;
//this.botAudio.playsInline = true; //this.botAudio.playsInline = true;
@@ -45,8 +43,12 @@ class WebsocketClientApp {
* Set up references to DOM elements and create necessary media elements * Set up references to DOM elements and create necessary media elements
*/ */
private setupDOMElements(): void { private setupDOMElements(): void {
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement; this.connectBtn = document.getElementById(
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement; 'connect-btn'
) as HTMLButtonElement;
this.disconnectBtn = document.getElementById(
'disconnect-btn'
) as HTMLButtonElement;
this.statusSpan = document.getElementById('connection-status'); this.statusSpan = document.getElementById('connection-status');
this.debugLog = document.getElementById('debug-log'); this.debugLog = document.getElementById('debug-log');
} }
@@ -91,8 +93,8 @@ class WebsocketClientApp {
* This is called when the bot is ready or when the transport state changes to ready * This is called when the bot is ready or when the transport state changes to ready
*/ */
setupMediaTracks() { setupMediaTracks() {
if (!this.rtviClient) return; if (!this.pcClient) return;
const tracks = this.rtviClient.tracks(); const tracks = this.pcClient.tracks();
if (tracks.bot?.audio) { if (tracks.bot?.audio) {
this.setupAudioTrack(tracks.bot.audio); this.setupAudioTrack(tracks.bot.audio);
} }
@@ -103,10 +105,10 @@ class WebsocketClientApp {
* This handles new tracks being added during the session * This handles new tracks being added during the session
*/ */
setupTrackListeners() { setupTrackListeners() {
if (!this.rtviClient) return; if (!this.pcClient) return;
// Listen for new tracks starting // Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => { this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks // Only handle non-local (bot) tracks
if (!participant?.local && track.kind === 'audio') { if (!participant?.local && track.kind === 'audio') {
this.setupAudioTrack(track); this.setupAudioTrack(track);
@@ -114,8 +116,10 @@ class WebsocketClientApp {
}); });
// Listen for tracks stopping // Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => { this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`); this.log(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
);
}); });
} }
@@ -125,7 +129,10 @@ class WebsocketClientApp {
*/ */
private setupAudioTrack(track: MediaStreamTrack): void { private setupAudioTrack(track: MediaStreamTrack): void {
this.log('Setting up audio track'); this.log('Setting up audio track');
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) { if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0]; const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
if (oldTrack?.id === track.id) return; if (oldTrack?.id === track.id) return;
} }
@@ -134,21 +141,15 @@ class WebsocketClientApp {
/** /**
* Initialize and connect to the bot * Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection * This sets up the Pipecat client, initializes devices, and establishes the connection
*/ */
public async connect(): Promise<void> { public async connect(): Promise<void> {
try { try {
const startTime = Date.now(); const startTime = Date.now();
//const transport = new DailyTransport(); //const transport = new DailyTransport();
const transport = new WebSocketTransport(); const PipecatConfig: PipecatClientOptions = {
const RTVIConfig: RTVIClientOptions = { transport: new WebSocketTransport(),
transport,
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: { connect: '/connect' },
},
enableMic: true, enableMic: true,
enableCam: false, enableCam: false,
callbacks: { callbacks: {
@@ -176,15 +177,20 @@ class WebsocketClientApp {
onMessageError: (error) => console.error('Message error:', error), onMessageError: (error) => console.error('Message error:', error),
onError: (error) => console.error('Error:', error), onError: (error) => console.error('Error:', error),
}, },
} };
this.rtviClient = new RTVIClient(RTVIConfig); this.pcClient = new PipecatClient(PipecatConfig);
// @ts-ignore
window.pcClient = this.pcClient; // Expose for debugging
this.setupTrackListeners(); this.setupTrackListeners();
this.log('Initializing devices...'); this.log('Initializing devices...');
await this.rtviClient.initDevices(); await this.pcClient.initDevices();
this.log('Connecting to bot...'); this.log('Connecting to bot...');
await this.rtviClient.connect(); await this.pcClient.connect({
// The baseURL and endpoint of your bot server that the client will connect to
endpoint: 'http://localhost:7860/connect',
});
const timeTaken = Date.now() - startTime; const timeTaken = Date.now() - startTime;
this.log(`Connection complete, timeTaken: ${timeTaken}`); this.log(`Connection complete, timeTaken: ${timeTaken}`);
@@ -192,9 +198,9 @@ class WebsocketClientApp {
this.log(`Error connecting: ${(error as Error).message}`); this.log(`Error connecting: ${(error as Error).message}`);
this.updateStatus('Error'); this.updateStatus('Error');
// Clean up if there's an error // Clean up if there's an error
if (this.rtviClient) { if (this.pcClient) {
try { try {
await this.rtviClient.disconnect(); await this.pcClient.disconnect();
} catch (disconnectError) { } catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError}`); this.log(`Error during disconnect: ${disconnectError}`);
} }
@@ -206,12 +212,17 @@ class WebsocketClientApp {
* Disconnect from the bot and clean up media resources * Disconnect from the bot and clean up media resources
*/ */
public async disconnect(): Promise<void> { public async disconnect(): Promise<void> {
if (this.rtviClient) { if (this.pcClient) {
try { try {
await this.rtviClient.disconnect(); await this.pcClient.disconnect();
this.rtviClient = null; this.pcClient = null;
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) { if (
this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop()); this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
this.botAudio.srcObject
.getAudioTracks()
.forEach((track) => track.stop());
this.botAudio.srcObject = null; this.botAudio.srcObject = null;
} }
} catch (error) { } catch (error) {
@@ -219,7 +230,6 @@ class WebsocketClientApp {
} }
} }
} }
} }
declare global { declare global {

View File

@@ -1,16 +1,26 @@
import { useEffect, useCallback } from 'react'; import { useEffect, useCallback } from 'react';
import { import {
useRTVIClient, usePipecatClient,
useRTVIClientTransportState, usePipecatClientTransportState,
} from '@pipecat-ai/client-react'; } from '@pipecat-ai/client-react';
import { CONNECTION_STATES } from '@/constants/gameConstants'; import { CONNECTION_STATES } from '@/constants/gameConstants';
import { useConfigurationSettings } from '@/contexts/Configuration';
// Get the API base URL from environment variables
// Default to "/api" if not specified
// "/api" is the default for Next.js API routes and used
// for the Pipecat Cloud deployed agent
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
console.log('Using API base URL:', API_BASE_URL);
export function useConnectionState( export function useConnectionState(
onConnected?: () => void, onConnected?: () => void,
onDisconnected?: () => void onDisconnected?: () => void
) { ) {
const client = useRTVIClient(); const client = usePipecatClient();
const transportState = useRTVIClientTransportState(); const transportState = usePipecatClientTransportState();
const config = useConfigurationSettings();
const isConnected = CONNECTION_STATES.ACTIVE.includes(transportState); const isConnected = CONNECTION_STATES.ACTIVE.includes(transportState);
const isConnecting = CONNECTION_STATES.CONNECTING.includes(transportState); const isConnecting = CONNECTION_STATES.CONNECTING.includes(transportState);
@@ -35,12 +45,17 @@ export function useConnectionState(
if (isConnected) { if (isConnected) {
await client.disconnect(); await client.disconnect();
} else { } else {
await client.connect(); await client.connect({
endpoint: `${API_BASE_URL}/connect`,
requestData: {
personality: config.personality,
},
});
} }
} catch (error) { } catch (error) {
console.error('Connection error:', error); console.error('Connection error:', error);
} }
}, [client, isConnected]); }, [client, config, isConnected]);
return { return {
isConnected, isConnected,

View File

@@ -1,15 +1,15 @@
import { ConfigurationProvider } from "@/contexts/Configuration"; import { ConfigurationProvider } from '@/contexts/Configuration';
import { RTVIProvider } from "@/providers/RTVIProvider"; import { PipecatProvider } from '@/providers/PipecatProvider';
import { RTVIClientAudio } from "@pipecat-ai/client-react"; import { PipecatClientAudio } from '@pipecat-ai/client-react';
import type { AppProps } from "next/app"; import type { AppProps } from 'next/app';
import { Nunito } from "next/font/google"; import { Nunito } from 'next/font/google';
import Head from "next/head"; import Head from 'next/head';
import "../styles/globals.css"; import '../styles/globals.css';
const nunito = Nunito({ const nunito = Nunito({
subsets: ["latin"], subsets: ['latin'],
display: "swap", display: 'swap',
variable: "--font-sans", variable: '--font-sans',
}); });
export default function App({ Component, pageProps }: AppProps) { export default function App({ Component, pageProps }: AppProps) {
@@ -21,10 +21,10 @@ export default function App({ Component, pageProps }: AppProps) {
</Head> </Head>
<main className={`${nunito.variable}`}> <main className={`${nunito.variable}`}>
<ConfigurationProvider> <ConfigurationProvider>
<RTVIProvider> <PipecatProvider>
<RTVIClientAudio /> <PipecatClientAudio />
<Component {...pageProps} /> <Component {...pageProps} />
</RTVIProvider> </PipecatProvider>
</ConfigurationProvider> </ConfigurationProvider>
</main> </main>
</> </>

View File

@@ -1,11 +1,11 @@
import type { NextApiRequest, NextApiResponse } from "next"; import type { NextApiRequest, NextApiResponse } from 'next';
export default async function handler( export default async function handler(
req: NextApiRequest, req: NextApiRequest,
res: NextApiResponse res: NextApiResponse
) { ) {
if (req.method !== "POST") { if (req.method !== 'POST') {
return res.status(405).json({ error: "Method not allowed" }); return res.status(405).json({ error: 'Method not allowed' });
} }
try { try {
@@ -15,16 +15,16 @@ export default async function handler(
if (!personality) { if (!personality) {
return res return res
.status(400) .status(400)
.json({ error: "Missing required configuration parameters" }); .json({ error: 'Missing required configuration parameters' });
} }
const response = await fetch( const response = await fetch(
`https://api.pipecat.daily.co/v1/public/${process.env.AGENT_NAME}/start`, `https://api.pipecat.daily.co/v1/public/${process.env.AGENT_NAME}/start`,
{ {
method: "POST", method: 'POST',
headers: { headers: {
Authorization: `Bearer ${process.env.PIPECAT_CLOUD_API_KEY}`, Authorization: `Bearer ${process.env.PIPECAT_CLOUD_API_KEY}`,
"Content-Type": "application/json", 'Content-Type': 'application/json',
}, },
body: JSON.stringify({ body: JSON.stringify({
createDailyRoom: true, createDailyRoom: true,
@@ -37,15 +37,15 @@ export default async function handler(
const data = await response.json(); const data = await response.json();
console.log("Response from API:", JSON.stringify(data, null, 2)); console.log('Response from API:', JSON.stringify(data, null, 2));
// Transform the response to match what RTVI client expects // Transform the response to match what Pipecat client expects
return res.status(200).json({ return res.status(200).json({
room_url: data.dailyRoom, room_url: data.dailyRoom,
token: data.dailyToken, token: data.dailyToken,
}); });
} catch (error) { } catch (error) {
console.error("Error starting agent:", error); console.error('Error starting agent:', error);
return res.status(500).json({ error: "Failed to start agent" }); return res.status(500).json({ error: 'Failed to start agent' });
} }
} }

View File

@@ -0,0 +1,43 @@
'use client';
import { PipecatClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { PipecatClientProvider } from '@pipecat-ai/client-react';
import { PropsWithChildren, useEffect, useState, useRef } from 'react';
export function PipecatProvider({ children }: PropsWithChildren) {
const [client, setClient] = useState<PipecatClient | null>(null);
const clientCreated = useRef(false);
useEffect(() => {
// Only create the client once
if (clientCreated.current) return;
const pcClient = new PipecatClient({
transport: new DailyTransport(),
enableMic: true,
enableCam: false,
});
setClient(pcClient);
clientCreated.current = true;
// Cleanup when component unmounts
return () => {
if (pcClient) {
pcClient.disconnect().catch((err) => {
console.error('Error disconnecting client:', err);
});
}
clientCreated.current = false;
};
}, []);
if (!client) {
return null;
}
return (
<PipecatClientProvider client={client}>{children}</PipecatClientProvider>
);
}

View File

@@ -1,72 +0,0 @@
"use client";
import { RTVIClient } from "@pipecat-ai/client-js";
import { DailyTransport } from "@pipecat-ai/daily-transport";
import { RTVIClientProvider } from "@pipecat-ai/client-react";
import { PropsWithChildren, useEffect, useState, useRef } from "react";
import { useConfigurationSettings } from "@/contexts/Configuration";
// Get the API base URL from environment variables
// Default to "/api" if not specified
// "/api" is the default for Next.js API routes and used
// for the Pipecat Cloud deployed agent
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || "/api";
console.log("Using API base URL:", API_BASE_URL);
export function RTVIProvider({ children }: PropsWithChildren) {
const [client, setClient] = useState<RTVIClient | null>(null);
const config = useConfigurationSettings();
const clientCreated = useRef(false);
useEffect(() => {
// Only create the client once
if (clientCreated.current) return;
const transport = new DailyTransport();
const rtviClient = new RTVIClient({
transport,
params: {
baseUrl: API_BASE_URL,
endpoints: {
connect: "/connect",
},
requestData: {
personality: config.personality,
},
},
enableMic: true,
enableCam: false,
});
setClient(rtviClient);
clientCreated.current = true;
// Cleanup when component unmounts
return () => {
if (rtviClient) {
rtviClient.disconnect().catch((err) => {
console.error("Error disconnecting client:", err);
});
}
clientCreated.current = false;
};
}, []);
// Update the connectParams when config changes
useEffect(() => {
if (!client) return;
// Update the connect params without recreating the client
client.params.requestData = {
personality: config.personality,
};
}, [client, config.personality]);
if (!client) {
return null;
}
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
}

View File

@@ -4,6 +4,7 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import argparse
import asyncio import asyncio
import os import os
import sys import sys
@@ -198,16 +199,15 @@ async def bot(args: DailySessionArguments):
# Local development # Local development
async def local_daily(): async def local_daily(args: DailySessionArguments):
"""Daily transport for local development.""" """Daily transport for local development."""
from runner import configure # from runner import configure
try: try:
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
transport = DailyTransport( transport = DailyTransport(
room_url, room_url=args.room_url,
token, token=args.token,
bot_name="Bot", bot_name="Bot",
params=DailyParams( params=DailyParams(
audio_in_enabled=True, audio_in_enabled=True,
@@ -217,7 +217,7 @@ async def local_daily():
) )
test_config = { test_config = {
"personality": "witty", "personality": args.personality,
} }
await main(transport, test_config) await main(transport, test_config)
@@ -227,7 +227,24 @@ async def local_daily():
# Local development entry point # Local development entry point
if LOCAL_RUN and __name__ == "__main__": if LOCAL_RUN and __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Run the Word Wrangler bot in local development mode"
)
parser.add_argument(
"-u", "--room-url", type=str, default=os.getenv("DAILY_SAMPLE_ROOM_URL", "")
)
parser.add_argument(
"-t", "--token", type=str, default=os.getenv("DAILY_SAMPLE_ROOM_TOKEN", None)
)
parser.add_argument(
"-p",
"--personality",
default="witty",
choices=["friendly", "professional", "enthusiastic", "thoughtful", "witty"],
help="Personality preset for the bot (friendly, professional, enthusiastic, thoughtful, witty)",
)
args = parser.parse_args()
try: try:
asyncio.run(local_daily()) asyncio.run(local_daily(args))
except Exception as e: except Exception as e:
logger.exception(f"Failed to run in local mode: {e}") logger.exception(f"Failed to run in local mode: {e}")

View File

@@ -160,14 +160,15 @@ async def rtvi_connect(request: Request) -> Dict[Any, Any]:
Raises: Raises:
HTTPException: If room creation, token generation, or bot startup fails HTTPException: If room creation, token generation, or bot startup fails
""" """
print("Creating room for RTVI connection") body = await request.json()
print("Creating room for RTVI connection", body)
room_url, token = await create_room_and_token() room_url, token = await create_room_and_token()
print(f"Room URL: {room_url}") print(f"Room URL: {room_url}")
# Start the bot process # Start the bot process
try: try:
proc = subprocess.Popen( proc = subprocess.Popen(
[f"python3 -m bot -u {room_url} -t {token}"], [f"python3 -m bot -u {room_url} -t {token} -p {body.get('personality', 'witty')}"],
shell=True, shell=True,
bufsize=1, bufsize=1,
cwd=os.path.dirname(os.path.abspath(__file__)), cwd=os.path.dirname(os.path.abspath(__file__)),