Updated all examples with clients to use the new PipecatClient
This commit is contained in:
committed by
Mattie Ruth
parent
43049c865c
commit
dc41ec7cb1
@@ -5,7 +5,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RTVI Client Implementation
|
* Pipecat Client Implementation
|
||||||
*
|
*
|
||||||
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
|
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
|
||||||
* It handles audio/video streaming and manages the connection lifecycle.
|
* It handles audio/video streaming and manages the connection lifecycle.
|
||||||
@@ -16,7 +16,7 @@
|
|||||||
* - Browser with WebRTC support
|
* - Browser with WebRTC support
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { RTVIClient, RTVIEvent } from '@pipecat-ai/client-js';
|
import { PipecatClient, RTVIEvent } from '@pipecat-ai/client-js';
|
||||||
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -26,7 +26,7 @@ import { DailyTransport } from '@pipecat-ai/daily-transport';
|
|||||||
class ChatbotClient {
|
class ChatbotClient {
|
||||||
constructor() {
|
constructor() {
|
||||||
// Initialize client state
|
// Initialize client state
|
||||||
this.rtviClient = null;
|
this.pcClient = null;
|
||||||
this.setupDOMElements();
|
this.setupDOMElements();
|
||||||
this.initializeClientAndTransport();
|
this.initializeClientAndTransport();
|
||||||
this.setupEventListeners();
|
this.setupEventListeners();
|
||||||
@@ -59,7 +59,7 @@ class ChatbotClient {
|
|||||||
this.disconnectBtn.addEventListener('click', () => this.disconnect());
|
this.disconnectBtn.addEventListener('click', () => this.disconnect());
|
||||||
|
|
||||||
// Populate device selector
|
// Populate device selector
|
||||||
this.rtviClient.getAllMics().then((mics) => {
|
this.pcClient.getAllMics().then((mics) => {
|
||||||
console.log('Available mics:', mics);
|
console.log('Available mics:', mics);
|
||||||
mics.forEach((device) => {
|
mics.forEach((device) => {
|
||||||
const option = document.createElement('option');
|
const option = document.createElement('option');
|
||||||
@@ -71,16 +71,16 @@ class ChatbotClient {
|
|||||||
this.deviceSelector.addEventListener('change', (event) => {
|
this.deviceSelector.addEventListener('change', (event) => {
|
||||||
const selectedDeviceId = event.target.value;
|
const selectedDeviceId = event.target.value;
|
||||||
console.log('Selected device ID:', selectedDeviceId);
|
console.log('Selected device ID:', selectedDeviceId);
|
||||||
this.rtviClient.updateMic(selectedDeviceId);
|
this.pcClient.updateMic(selectedDeviceId);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Handle mic mute/unmute toggle
|
// Handle mic mute/unmute toggle
|
||||||
const micToggleBtn = document.getElementById('mic-toggle-btn');
|
const micToggleBtn = document.getElementById('mic-toggle-btn');
|
||||||
|
|
||||||
micToggleBtn.addEventListener('click', () => {
|
micToggleBtn.addEventListener('click', () => {
|
||||||
let micEnabled = this.rtviClient.isMicEnabled;
|
let micEnabled = this.pcClient.isMicEnabled;
|
||||||
micToggleBtn.textContent = micEnabled ? 'Unmute Mic' : 'Mute Mic';
|
micToggleBtn.textContent = micEnabled ? 'Unmute Mic' : 'Mute Mic';
|
||||||
this.rtviClient.enableMic(!micEnabled);
|
this.pcClient.enableMic(!micEnabled);
|
||||||
// Add logic to mute/unmute the mic
|
// Add logic to mute/unmute the mic
|
||||||
if (micEnabled) {
|
if (micEnabled) {
|
||||||
console.log('Mic muted');
|
console.log('Mic muted');
|
||||||
@@ -93,23 +93,12 @@ class ChatbotClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set up the RTVI client and Daily transport
|
* Set up the Pipecat client and Daily transport
|
||||||
*/
|
*/
|
||||||
async initializeClientAndTransport() {
|
async initializeClientAndTransport() {
|
||||||
// Initialize the RTVI client with a DailyTransport and our configuration
|
// Initialize the Pipecat client with a DailyTransport and our configuration
|
||||||
this.rtviClient = new RTVIClient({
|
this.pcClient = new PipecatClient({
|
||||||
transport: new DailyTransport(),
|
transport: new DailyTransport(),
|
||||||
params: {
|
|
||||||
// REPLACE WITH YOUR MODAL URL ENDPOINT
|
|
||||||
baseUrl:
|
|
||||||
'https://<Modal workspace>--pipecat-modal-bot-launcher.modal.run',
|
|
||||||
endpoints: {
|
|
||||||
connect: '/connect',
|
|
||||||
},
|
|
||||||
requestData: {
|
|
||||||
bot_name: 'openai',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
enableMic: true, // Enable microphone for user input
|
enableMic: true, // Enable microphone for user input
|
||||||
enableCam: false,
|
enableCam: false,
|
||||||
callbacks: {
|
callbacks: {
|
||||||
@@ -176,8 +165,8 @@ class ChatbotClient {
|
|||||||
// Set up listeners for media track events
|
// Set up listeners for media track events
|
||||||
this.setupTrackListeners();
|
this.setupTrackListeners();
|
||||||
|
|
||||||
await this.rtviClient.initDevices();
|
await this.pcClient.initDevices();
|
||||||
window.client = this.rtviClient;
|
window.client = this.pcClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -212,10 +201,10 @@ class ChatbotClient {
|
|||||||
* This is called when the bot is ready or when the transport state changes to ready
|
* This is called when the bot is ready or when the transport state changes to ready
|
||||||
*/
|
*/
|
||||||
setupMediaTracks() {
|
setupMediaTracks() {
|
||||||
if (!this.rtviClient) return;
|
if (!this.pcClient) return;
|
||||||
|
|
||||||
// Get current tracks from the client
|
// Get current tracks from the client
|
||||||
const tracks = this.rtviClient.tracks();
|
const tracks = this.pcClient.tracks();
|
||||||
|
|
||||||
// Set up any available bot tracks
|
// Set up any available bot tracks
|
||||||
if (tracks.bot?.audio) {
|
if (tracks.bot?.audio) {
|
||||||
@@ -231,10 +220,10 @@ class ChatbotClient {
|
|||||||
* This handles new tracks being added during the session
|
* This handles new tracks being added during the session
|
||||||
*/
|
*/
|
||||||
setupTrackListeners() {
|
setupTrackListeners() {
|
||||||
if (!this.rtviClient) return;
|
if (!this.pcClient) return;
|
||||||
|
|
||||||
// Listen for new tracks starting
|
// Listen for new tracks starting
|
||||||
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
|
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
|
||||||
// Only handle non-local (bot) tracks
|
// Only handle non-local (bot) tracks
|
||||||
if (!participant?.local) {
|
if (!participant?.local) {
|
||||||
if (track.kind === 'audio') {
|
if (track.kind === 'audio') {
|
||||||
@@ -253,7 +242,7 @@ class ChatbotClient {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Listen for tracks stopping
|
// Listen for tracks stopping
|
||||||
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
||||||
if (participant.local) {
|
if (participant.local) {
|
||||||
this.log('Local mic muted');
|
this.log('Local mic muted');
|
||||||
return;
|
return;
|
||||||
@@ -311,21 +300,27 @@ class ChatbotClient {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize and connect to the bot
|
* Initialize and connect to the bot
|
||||||
* This sets up the RTVI client, initializes devices, and establishes the connection
|
* This sets up the Pipecat client, initializes devices, and establishes the connection
|
||||||
*/
|
*/
|
||||||
async connect() {
|
async connect() {
|
||||||
try {
|
try {
|
||||||
const botSelector = document.getElementById('bot-selector');
|
const botSelector = document.getElementById('bot-selector');
|
||||||
const selectedBot = botSelector.value;
|
const selectedBot = botSelector.value;
|
||||||
this.rtviClient.params.requestData.bot_name = selectedBot;
|
|
||||||
|
|
||||||
// Initialize audio/video devices
|
// Initialize audio/video devices
|
||||||
this.log('Initializing devices...');
|
this.log('Initializing devices...');
|
||||||
await this.rtviClient.initDevices();
|
await this.pcClient.initDevices();
|
||||||
|
|
||||||
// Connect to the bot
|
// Connect to the bot
|
||||||
this.log(`Connecting to bot: ${selectedBot}`);
|
this.log(`Connecting to bot: ${selectedBot}`);
|
||||||
await this.rtviClient.connect();
|
await this.pcClient.connect({
|
||||||
|
// REPLACE WITH YOUR MODAL URL ENDPOINT
|
||||||
|
endpoint:
|
||||||
|
'https://<your-workspace>--pipecat-modal-fastapi-app.modal.run/connect',
|
||||||
|
requestData: {
|
||||||
|
bot_name: selectedBot,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
this.log('Connection complete');
|
this.log('Connection complete');
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -336,9 +331,9 @@ class ChatbotClient {
|
|||||||
this.updateStatus('Error');
|
this.updateStatus('Error');
|
||||||
|
|
||||||
// Clean up if there's an error
|
// Clean up if there's an error
|
||||||
if (this.rtviClient) {
|
if (this.pcClient) {
|
||||||
try {
|
try {
|
||||||
await this.rtviClient.disconnect();
|
await this.pcClient.disconnect();
|
||||||
} catch (disconnectError) {
|
} catch (disconnectError) {
|
||||||
this.log(`Error during disconnect: ${disconnectError.message}`);
|
this.log(`Error during disconnect: ${disconnectError.message}`);
|
||||||
}
|
}
|
||||||
@@ -350,10 +345,10 @@ class ChatbotClient {
|
|||||||
* Disconnect from the bot and clean up media resources
|
* Disconnect from the bot and clean up media resources
|
||||||
*/
|
*/
|
||||||
async disconnect() {
|
async disconnect() {
|
||||||
if (this.rtviClient) {
|
if (this.pcClient) {
|
||||||
try {
|
try {
|
||||||
// Disconnect the RTVI client
|
// Disconnect the Pipecat client
|
||||||
await this.rtviClient.disconnect();
|
await this.pcClient.disconnect();
|
||||||
|
|
||||||
// Clean up audio
|
// Clean up audio
|
||||||
if (this.botAudio.srcObject) {
|
if (this.botAudio.srcObject) {
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ Try the hosted version of the demo here: https://pcc-smart-turn.vercel.app/.
|
|||||||
4. Run the server:
|
4. Run the server:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
LOCAL=1 python server.py
|
LOCAL_RUN=1 python server.py
|
||||||
```
|
```
|
||||||
|
|
||||||
### Run the client
|
### Run the client
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import './globals.css';
|
import './globals.css';
|
||||||
import { RTVIProvider } from '@/providers/RTVIProvider';
|
import { PipecatProvider } from '@/providers/PipecatProvider';
|
||||||
|
|
||||||
export const metadata = {
|
export const metadata = {
|
||||||
title: 'Pipecat React Client',
|
title: 'Pipecat React Client',
|
||||||
@@ -20,7 +20,7 @@ export default function RootLayout({
|
|||||||
<link rel="icon" href="/favicon.svg" type="image/svg+xml" />
|
<link rel="icon" href="/favicon.svg" type="image/svg+xml" />
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<RTVIProvider>{children}</RTVIProvider>
|
<PipecatProvider>{children}</PipecatProvider>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,22 +1,22 @@
|
|||||||
'use client';
|
'use client';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
RTVIClientAudio,
|
PipecatClientAudio,
|
||||||
RTVIClientVideo,
|
PipecatClientVideo,
|
||||||
useRTVIClientTransportState,
|
usePipecatClientTransportState,
|
||||||
} from '@pipecat-ai/client-react';
|
} from '@pipecat-ai/client-react';
|
||||||
import { ConnectButton } from '../components/ConnectButton';
|
import { ConnectButton } from '../components/ConnectButton';
|
||||||
import { StatusDisplay } from '../components/StatusDisplay';
|
import { StatusDisplay } from '../components/StatusDisplay';
|
||||||
import { DebugDisplay } from '../components/DebugDisplay';
|
import { DebugDisplay } from '../components/DebugDisplay';
|
||||||
|
|
||||||
function BotVideo() {
|
function BotVideo() {
|
||||||
const transportState = useRTVIClientTransportState();
|
const transportState = usePipecatClientTransportState();
|
||||||
const isConnected = transportState !== 'disconnected';
|
const isConnected = transportState !== 'disconnected';
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="bot-container">
|
<div className="bot-container">
|
||||||
<div className="video-container">
|
<div className="video-container">
|
||||||
{isConnected && <RTVIClientVideo participant="bot" fit="cover" />}
|
{isConnected && <PipecatClientVideo participant="bot" fit="cover" />}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
@@ -35,7 +35,7 @@ export default function Home() {
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<DebugDisplay />
|
<DebugDisplay />
|
||||||
<RTVIClientAudio />
|
<PipecatClientAudio />
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,17 @@
|
|||||||
import {
|
import {
|
||||||
useRTVIClient,
|
usePipecatClient,
|
||||||
useRTVIClientTransportState,
|
usePipecatClientTransportState,
|
||||||
} from '@pipecat-ai/client-react';
|
} from '@pipecat-ai/client-react';
|
||||||
|
|
||||||
|
// Get the API base URL from environment variables
|
||||||
|
// Default to "/api" if not specified
|
||||||
|
// "/api" is the default for Next.js API routes and used
|
||||||
|
// for the Pipecat Cloud deployed agent
|
||||||
|
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
|
||||||
|
|
||||||
export function ConnectButton() {
|
export function ConnectButton() {
|
||||||
const client = useRTVIClient();
|
const client = usePipecatClient();
|
||||||
const transportState = useRTVIClientTransportState();
|
const transportState = usePipecatClientTransportState();
|
||||||
const isConnected = ['connected', 'ready'].includes(transportState);
|
const isConnected = ['connected', 'ready'].includes(transportState);
|
||||||
|
|
||||||
const handleClick = async () => {
|
const handleClick = async () => {
|
||||||
@@ -18,7 +24,10 @@ export function ConnectButton() {
|
|||||||
if (isConnected) {
|
if (isConnected) {
|
||||||
await client.disconnect();
|
await client.disconnect();
|
||||||
} else {
|
} else {
|
||||||
await client.connect();
|
await client.connect({
|
||||||
|
endpoint: `${API_BASE_URL}/connect`,
|
||||||
|
requestData: { foo: 'bar' },
|
||||||
|
});
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Connection error:', error);
|
console.error('Connection error:', error);
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import {
|
|||||||
TranscriptData,
|
TranscriptData,
|
||||||
BotLLMTextData,
|
BotLLMTextData,
|
||||||
} from '@pipecat-ai/client-js';
|
} from '@pipecat-ai/client-js';
|
||||||
import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
|
import { usePipecatClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
|
||||||
import './DebugDisplay.css';
|
import './DebugDisplay.css';
|
||||||
|
|
||||||
interface SmartTurnResultData {
|
interface SmartTurnResultData {
|
||||||
@@ -20,7 +20,7 @@ interface SmartTurnResultData {
|
|||||||
|
|
||||||
export function DebugDisplay() {
|
export function DebugDisplay() {
|
||||||
const debugLogRef = useRef<HTMLDivElement>(null);
|
const debugLogRef = useRef<HTMLDivElement>(null);
|
||||||
const client = useRTVIClient();
|
const client = usePipecatClient();
|
||||||
|
|
||||||
const log = useCallback((message: string) => {
|
const log = useCallback((message: string) => {
|
||||||
if (!debugLogRef.current) return;
|
if (!debugLogRef.current) return;
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { useRTVIClientTransportState } from '@pipecat-ai/client-react';
|
import { usePipecatClientTransportState } from '@pipecat-ai/client-react';
|
||||||
|
|
||||||
export function StatusDisplay() {
|
export function StatusDisplay() {
|
||||||
const transportState = useRTVIClientTransportState();
|
const transportState = usePipecatClientTransportState();
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="status">
|
<div className="status">
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
'use client';
|
||||||
|
|
||||||
|
import { PipecatClient } from '@pipecat-ai/client-js';
|
||||||
|
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
||||||
|
import { PipecatClientProvider } from '@pipecat-ai/client-react';
|
||||||
|
import { PropsWithChildren, useEffect, useState } from 'react';
|
||||||
|
|
||||||
|
export function PipecatProvider({ children }: PropsWithChildren) {
|
||||||
|
const [client, setClient] = useState<PipecatClient | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const pcClient = new PipecatClient({
|
||||||
|
transport: new DailyTransport(),
|
||||||
|
enableMic: true,
|
||||||
|
enableCam: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
setClient(pcClient);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
if (!client) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<PipecatClientProvider client={client}>{children}</PipecatClientProvider>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
'use client';
|
|
||||||
|
|
||||||
import { RTVIClient } from '@pipecat-ai/client-js';
|
|
||||||
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
|
||||||
import { RTVIClientProvider } from '@pipecat-ai/client-react';
|
|
||||||
import { PropsWithChildren, useEffect, useState } from 'react';
|
|
||||||
|
|
||||||
// Get the API base URL from environment variables
|
|
||||||
// Default to "/api" if not specified
|
|
||||||
// "/api" is the default for Next.js API routes and used
|
|
||||||
// for the Pipecat Cloud deployed agent
|
|
||||||
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
|
|
||||||
|
|
||||||
console.log('Using API base URL:', API_BASE_URL);
|
|
||||||
|
|
||||||
export function RTVIProvider({ children }: PropsWithChildren) {
|
|
||||||
const [client, setClient] = useState<RTVIClient | null>(null);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
const transport = new DailyTransport();
|
|
||||||
|
|
||||||
const rtviClient = new RTVIClient({
|
|
||||||
transport,
|
|
||||||
params: {
|
|
||||||
baseUrl: API_BASE_URL,
|
|
||||||
endpoints: {
|
|
||||||
connect: '/connect',
|
|
||||||
},
|
|
||||||
requestData: { foo: 'bar' },
|
|
||||||
},
|
|
||||||
enableMic: true,
|
|
||||||
enableCam: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
setClient(rtviClient);
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
if (!client) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
|
|
||||||
}
|
|
||||||
@@ -45,7 +45,7 @@ from pipecat.transports.services.daily import DailyParams, DailyTransport
|
|||||||
load_dotenv(override=True)
|
load_dotenv(override=True)
|
||||||
|
|
||||||
# Check if we're in local development mode
|
# Check if we're in local development mode
|
||||||
LOCAL = os.getenv("LOCAL")
|
LOCAL = os.getenv("LOCAL_RUN")
|
||||||
|
|
||||||
logger.remove()
|
logger.remove()
|
||||||
logger.add(sys.stderr, level="DEBUG")
|
logger.add(sys.stderr, level="DEBUG")
|
||||||
|
|||||||
@@ -20,11 +20,10 @@ import {
|
|||||||
} from '@pipecat-ai/client-js';
|
} from '@pipecat-ai/client-js';
|
||||||
import {
|
import {
|
||||||
ProtobufFrameSerializer,
|
ProtobufFrameSerializer,
|
||||||
WebSocketTransport
|
WebSocketTransport,
|
||||||
} from "@pipecat-ai/websocket-transport";
|
} from '@pipecat-ai/websocket-transport';
|
||||||
|
|
||||||
class RecordingSerializer extends ProtobufFrameSerializer {
|
class RecordingSerializer extends ProtobufFrameSerializer {
|
||||||
|
|
||||||
private lastTimestamp: number | null = null;
|
private lastTimestamp: number | null = null;
|
||||||
private recordingAudioToSend: boolean = false;
|
private recordingAudioToSend: boolean = false;
|
||||||
private _recordedAudio: { data: ArrayBuffer; delay: number }[] = [];
|
private _recordedAudio: { data: ArrayBuffer; delay: number }[] = [];
|
||||||
@@ -40,7 +39,11 @@ class RecordingSerializer extends ProtobufFrameSerializer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
serializeAudio(data: ArrayBuffer, sampleRate: number, numChannels: number): Uint8Array | null {
|
serializeAudio(
|
||||||
|
data: ArrayBuffer,
|
||||||
|
sampleRate: number,
|
||||||
|
numChannels: number
|
||||||
|
): Uint8Array | null {
|
||||||
if (this.recordingAudioToSend) {
|
if (this.recordingAudioToSend) {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
// Compute delay since last packet
|
// Compute delay since last packet
|
||||||
@@ -55,13 +58,13 @@ class RecordingSerializer extends ProtobufFrameSerializer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public get recordedAudio() {
|
public get recordedAudio() {
|
||||||
return this._recordedAudio
|
return this._recordedAudio;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class WebsocketClientApp {
|
class WebsocketClientApp {
|
||||||
private ENABLE_RECORDING_MODE = false
|
private ENABLE_RECORDING_MODE = false;
|
||||||
private RECORDING_TIME_MS = 10000
|
private RECORDING_TIME_MS = 10000;
|
||||||
|
|
||||||
private rtviClient: RTVIClient | null = null;
|
private rtviClient: RTVIClient | null = null;
|
||||||
private connectBtn: HTMLButtonElement | null = null;
|
private connectBtn: HTMLButtonElement | null = null;
|
||||||
@@ -71,7 +74,7 @@ class WebsocketClientApp {
|
|||||||
private botAudio: HTMLAudioElement;
|
private botAudio: HTMLAudioElement;
|
||||||
|
|
||||||
private declare websocketTransport: WebSocketTransport;
|
private declare websocketTransport: WebSocketTransport;
|
||||||
private sendRecordedAudio: boolean = false
|
private sendRecordedAudio: boolean = false;
|
||||||
private declare recordingSerializer: RecordingSerializer;
|
private declare recordingSerializer: RecordingSerializer;
|
||||||
|
|
||||||
private playBtn: HTMLButtonElement | null = null;
|
private playBtn: HTMLButtonElement | null = null;
|
||||||
@@ -91,8 +94,12 @@ class WebsocketClientApp {
|
|||||||
* Set up references to DOM elements and create necessary media elements
|
* Set up references to DOM elements and create necessary media elements
|
||||||
*/
|
*/
|
||||||
private setupDOMElements(): void {
|
private setupDOMElements(): void {
|
||||||
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
|
this.connectBtn = document.getElementById(
|
||||||
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
|
'connect-btn'
|
||||||
|
) as HTMLButtonElement;
|
||||||
|
this.disconnectBtn = document.getElementById(
|
||||||
|
'disconnect-btn'
|
||||||
|
) as HTMLButtonElement;
|
||||||
this.statusSpan = document.getElementById('connection-status');
|
this.statusSpan = document.getElementById('connection-status');
|
||||||
this.debugLog = document.getElementById('debug-log');
|
this.debugLog = document.getElementById('debug-log');
|
||||||
this.playBtn = document.getElementById('play-btn') as HTMLButtonElement;
|
this.playBtn = document.getElementById('play-btn') as HTMLButtonElement;
|
||||||
@@ -105,8 +112,12 @@ class WebsocketClientApp {
|
|||||||
private setupEventListeners(): void {
|
private setupEventListeners(): void {
|
||||||
this.connectBtn?.addEventListener('click', () => this.connect());
|
this.connectBtn?.addEventListener('click', () => this.connect());
|
||||||
this.disconnectBtn?.addEventListener('click', () => this.disconnect());
|
this.disconnectBtn?.addEventListener('click', () => this.disconnect());
|
||||||
this.playBtn?.addEventListener('click', () => this.startSendingRecordedAudio());
|
this.playBtn?.addEventListener('click', () =>
|
||||||
this.stopBtn?.addEventListener('click', () => this.stopSendingRecordedAudio());
|
this.startSendingRecordedAudio()
|
||||||
|
);
|
||||||
|
this.stopBtn?.addEventListener('click', () =>
|
||||||
|
this.stopSendingRecordedAudio()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -165,7 +176,9 @@ class WebsocketClientApp {
|
|||||||
|
|
||||||
// Listen for tracks stopping
|
// Listen for tracks stopping
|
||||||
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
||||||
this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`);
|
this.log(
|
||||||
|
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
|
||||||
|
);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -175,7 +188,10 @@ class WebsocketClientApp {
|
|||||||
*/
|
*/
|
||||||
private setupAudioTrack(track: MediaStreamTrack): void {
|
private setupAudioTrack(track: MediaStreamTrack): void {
|
||||||
this.log('Setting up audio track');
|
this.log('Setting up audio track');
|
||||||
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
|
if (
|
||||||
|
this.botAudio.srcObject &&
|
||||||
|
'getAudioTracks' in this.botAudio.srcObject
|
||||||
|
) {
|
||||||
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
|
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
|
||||||
if (oldTrack?.id === track.id) return;
|
if (oldTrack?.id === track.id) return;
|
||||||
}
|
}
|
||||||
@@ -190,27 +206,17 @@ class WebsocketClientApp {
|
|||||||
try {
|
try {
|
||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
|
|
||||||
this.recordingSerializer = new RecordingSerializer()
|
this.recordingSerializer = new RecordingSerializer();
|
||||||
const transport = this.ENABLE_RECORDING_MODE ?
|
const ws_opts = {
|
||||||
new WebSocketTransport({
|
serializer: this.ENABLE_RECORDING_MODE
|
||||||
serializer: this.recordingSerializer,
|
? this.recordingSerializer
|
||||||
recorderSampleRate: 8000,
|
: new ProtobufFrameSerializer(),
|
||||||
playerSampleRate:8000
|
recorderSampleRate: 8000,
|
||||||
}) :
|
playerSampleRate: 8000,
|
||||||
new WebSocketTransport({
|
};
|
||||||
serializer: new ProtobufFrameSerializer(),
|
|
||||||
recorderSampleRate: 8000,
|
|
||||||
playerSampleRate:8000
|
|
||||||
});
|
|
||||||
this.websocketTransport = transport
|
|
||||||
|
|
||||||
const RTVIConfig: RTVIClientOptions = {
|
const RTVIConfig: RTVIClientOptions = {
|
||||||
transport,
|
transport: new WebSocketTransport(ws_opts),
|
||||||
params: {
|
|
||||||
// The baseURL and endpoint of your bot server that the client will connect to
|
|
||||||
baseUrl: 'http://localhost:7860',
|
|
||||||
endpoints: { connect: '/connect' },
|
|
||||||
},
|
|
||||||
enableMic: true,
|
enableMic: true,
|
||||||
enableCam: false,
|
enableCam: false,
|
||||||
callbacks: {
|
callbacks: {
|
||||||
@@ -238,27 +244,34 @@ class WebsocketClientApp {
|
|||||||
onMessageError: (error) => console.error('Message error:', error),
|
onMessageError: (error) => console.error('Message error:', error),
|
||||||
onError: (error) => console.error('Error:', error),
|
onError: (error) => console.error('Error:', error),
|
||||||
},
|
},
|
||||||
}
|
};
|
||||||
this.rtviClient = new RTVIClient(RTVIConfig);
|
this.rtviClient = new RTVIClient(RTVIConfig);
|
||||||
|
this.websocketTransport = this.rtviClient.transport;
|
||||||
this.setupTrackListeners();
|
this.setupTrackListeners();
|
||||||
|
|
||||||
this.log('Initializing devices...');
|
this.log('Initializing devices...');
|
||||||
await this.rtviClient.initDevices();
|
await this.rtviClient.initDevices();
|
||||||
|
|
||||||
this.log('Connecting to bot...');
|
this.log('Connecting to bot...');
|
||||||
await this.rtviClient.connect();
|
await this.rtviClient.connect({
|
||||||
|
endpoint: 'http://localhost:7860/connect',
|
||||||
|
});
|
||||||
|
|
||||||
const timeTaken = Date.now() - startTime;
|
const timeTaken = Date.now() - startTime;
|
||||||
this.log(`Connection complete, timeTaken: ${timeTaken}`);
|
this.log(`Connection complete, timeTaken: ${timeTaken}`);
|
||||||
|
|
||||||
if (this.ENABLE_RECORDING_MODE) {
|
if (this.ENABLE_RECORDING_MODE) {
|
||||||
this.log(`Starting to recording the next ${(this.RECORDING_TIME_MS/1000)}s of audio`);
|
this.log(
|
||||||
this.recordingSerializer.startRecording()
|
`Starting to recording the next ${
|
||||||
await this.sleep(this.RECORDING_TIME_MS)
|
this.RECORDING_TIME_MS / 1000
|
||||||
this.recordingSerializer.stopRecording()
|
}s of audio`
|
||||||
this.log("Recording stopped");
|
);
|
||||||
this.rtviClient.enableMic(false)
|
this.recordingSerializer.startRecording();
|
||||||
this.startSendingRecordedAudio()
|
await this.sleep(this.RECORDING_TIME_MS);
|
||||||
|
this.recordingSerializer.stopRecording();
|
||||||
|
this.log('Recording stopped');
|
||||||
|
this.rtviClient.enableMic(false);
|
||||||
|
this.startSendingRecordedAudio();
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.log(`Error connecting: ${(error as Error).message}`);
|
this.log(`Error connecting: ${(error as Error).message}`);
|
||||||
@@ -280,11 +293,16 @@ class WebsocketClientApp {
|
|||||||
public async disconnect(): Promise<void> {
|
public async disconnect(): Promise<void> {
|
||||||
if (this.rtviClient) {
|
if (this.rtviClient) {
|
||||||
try {
|
try {
|
||||||
this.stopSendingRecordedAudio()
|
this.stopSendingRecordedAudio();
|
||||||
await this.rtviClient.disconnect();
|
await this.rtviClient.disconnect();
|
||||||
this.rtviClient = null;
|
this.rtviClient = null;
|
||||||
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
|
if (
|
||||||
this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop());
|
this.botAudio.srcObject &&
|
||||||
|
'getAudioTracks' in this.botAudio.srcObject
|
||||||
|
) {
|
||||||
|
this.botAudio.srcObject
|
||||||
|
.getAudioTracks()
|
||||||
|
.forEach((track) => track.stop());
|
||||||
this.botAudio.srcObject = null;
|
this.botAudio.srcObject = null;
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -294,21 +312,21 @@ class WebsocketClientApp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private startSendingRecordedAudio() {
|
private startSendingRecordedAudio() {
|
||||||
this.sendRecordedAudio = true
|
this.sendRecordedAudio = true;
|
||||||
if (this.playBtn) this.playBtn.disabled = true;
|
if (this.playBtn) this.playBtn.disabled = true;
|
||||||
if (this.stopBtn) this.stopBtn.disabled = false;
|
if (this.stopBtn) this.stopBtn.disabled = false;
|
||||||
void this.replayAudio()
|
void this.replayAudio();
|
||||||
}
|
}
|
||||||
|
|
||||||
private stopSendingRecordedAudio() {
|
private stopSendingRecordedAudio() {
|
||||||
if (this.stopBtn) this.stopBtn.disabled = true;
|
if (this.stopBtn) this.stopBtn.disabled = true;
|
||||||
if (this.playBtn) this.playBtn.disabled = false;
|
if (this.playBtn) this.playBtn.disabled = false;
|
||||||
this.sendRecordedAudio = false
|
this.sendRecordedAudio = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async replayAudio() {
|
private async replayAudio() {
|
||||||
if (this.sendRecordedAudio) {
|
if (this.sendRecordedAudio) {
|
||||||
this.log("Sending recorded audio")
|
this.log('Sending recorded audio');
|
||||||
for (const chunk of this.recordingSerializer.recordedAudio) {
|
for (const chunk of this.recordingSerializer.recordedAudio) {
|
||||||
await this.sleep(chunk.delay);
|
await this.sleep(chunk.delay);
|
||||||
this.websocketTransport.handleUserAudioStream(chunk.data);
|
this.websocketTransport.handleUserAudioStream(chunk.data);
|
||||||
@@ -316,14 +334,13 @@ class WebsocketClientApp {
|
|||||||
const randomDelay = 1000 + Math.random() * (10000 - 500);
|
const randomDelay = 1000 + Math.random() * (10000 - 500);
|
||||||
await this.sleep(randomDelay);
|
await this.sleep(randomDelay);
|
||||||
|
|
||||||
void this.replayAudio()
|
void this.replayAudio();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private sleep(ms: number): Promise<void> {
|
private sleep(ms: number): Promise<void> {
|
||||||
return new Promise(resolve => setTimeout(resolve, ms));
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
declare global {
|
declare global {
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RTVI Client Implementation
|
* Pipecat Client Implementation
|
||||||
*
|
*
|
||||||
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
|
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
|
||||||
* It handles audio/video streaming and manages the connection lifecycle.
|
* It handles audio/video streaming and manages the connection lifecycle.
|
||||||
@@ -18,20 +18,22 @@
|
|||||||
|
|
||||||
import {
|
import {
|
||||||
Participant,
|
Participant,
|
||||||
RTVIClient,
|
PipecatClient,
|
||||||
RTVIClientOptions,
|
PipecatClientOptions,
|
||||||
RTVIEvent,
|
RTVIEvent,
|
||||||
} from '@pipecat-ai/client-js';
|
} from '@pipecat-ai/client-js';
|
||||||
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
import {
|
||||||
|
DailyEventCallbacks,
|
||||||
|
DailyTransport,
|
||||||
|
} from '@pipecat-ai/daily-transport';
|
||||||
import SoundUtils from './util/soundUtils';
|
import SoundUtils from './util/soundUtils';
|
||||||
import { InstantVoiceHelper } from './util/instantVoiceHelper';
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* InstantVoiceClient handles the connection and media management for a real-time
|
* InstantVoiceClient handles the connection and media management for a real-time
|
||||||
* voice and video interaction with an AI bot.
|
* voice and video interaction with an AI bot.
|
||||||
*/
|
*/
|
||||||
class InstantVoiceClient {
|
class InstantVoiceClient {
|
||||||
private declare rtviClient: RTVIClient;
|
private declare pcClient: PipecatClient;
|
||||||
private connectBtn: HTMLButtonElement | null = null;
|
private connectBtn: HTMLButtonElement | null = null;
|
||||||
private disconnectBtn: HTMLButtonElement | null = null;
|
private disconnectBtn: HTMLButtonElement | null = null;
|
||||||
private statusSpan: HTMLElement | null = null;
|
private statusSpan: HTMLElement | null = null;
|
||||||
@@ -46,7 +48,7 @@ class InstantVoiceClient {
|
|||||||
document.body.appendChild(this.botAudio);
|
document.body.appendChild(this.botAudio);
|
||||||
this.setupDOMElements();
|
this.setupDOMElements();
|
||||||
this.setupEventListeners();
|
this.setupEventListeners();
|
||||||
this.initializeRTVIClient();
|
this.initializePipecatClient();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -72,16 +74,11 @@ class InstantVoiceClient {
|
|||||||
this.disconnectBtn?.addEventListener('click', () => this.disconnect());
|
this.disconnectBtn?.addEventListener('click', () => this.disconnect());
|
||||||
}
|
}
|
||||||
|
|
||||||
private initializeRTVIClient(): void {
|
private initializePipecatClient(): void {
|
||||||
const RTVIConfig: RTVIClientOptions = {
|
const PipecatConfig: PipecatClientOptions = {
|
||||||
transport: new DailyTransport({
|
transport: new DailyTransport({
|
||||||
bufferLocalAudioUntilBotReady: true,
|
bufferLocalAudioUntilBotReady: true,
|
||||||
}),
|
}),
|
||||||
params: {
|
|
||||||
// The baseURL and endpoint of your bot server that the client will connect to
|
|
||||||
baseUrl: 'http://localhost:7860',
|
|
||||||
endpoints: { connect: '/connect' },
|
|
||||||
},
|
|
||||||
enableMic: true,
|
enableMic: true,
|
||||||
enableCam: false,
|
enableCam: false,
|
||||||
callbacks: {
|
callbacks: {
|
||||||
@@ -113,30 +110,23 @@ class InstantVoiceClient {
|
|||||||
onBotTranscript: (data) => this.log(`Bot: ${data.text}`),
|
onBotTranscript: (data) => this.log(`Bot: ${data.text}`),
|
||||||
onMessageError: (error) => console.error('Message error:', error),
|
onMessageError: (error) => console.error('Message error:', error),
|
||||||
onError: (error) => console.error('Error:', error),
|
onError: (error) => console.error('Error:', error),
|
||||||
},
|
onAudioBufferingStarted: () => {
|
||||||
|
SoundUtils.beep();
|
||||||
|
this.updateBufferingStatus('Yes');
|
||||||
|
this.log(
|
||||||
|
`onMicCaptureStarted, timeTaken: ${Date.now() - this.startTime}`
|
||||||
|
);
|
||||||
|
},
|
||||||
|
onAudioBufferingStopped: () => {
|
||||||
|
this.updateBufferingStatus('No');
|
||||||
|
this.log(
|
||||||
|
`onMicCaptureStopped, timeTaken: ${Date.now() - this.startTime}`
|
||||||
|
);
|
||||||
|
},
|
||||||
|
} as DailyEventCallbacks,
|
||||||
};
|
};
|
||||||
|
|
||||||
this.rtviClient = new RTVIClient(RTVIConfig);
|
this.pcClient = new PipecatClient(PipecatConfig);
|
||||||
this.rtviClient.registerHelper(
|
|
||||||
'transport',
|
|
||||||
new InstantVoiceHelper({
|
|
||||||
callbacks: {
|
|
||||||
onAudioBufferingStarted: () => {
|
|
||||||
SoundUtils.beep();
|
|
||||||
this.updateBufferingStatus('Yes');
|
|
||||||
this.log(
|
|
||||||
`onMicCaptureStarted, timeTaken: ${Date.now() - this.startTime}`
|
|
||||||
);
|
|
||||||
},
|
|
||||||
onAudioBufferingStopped: () => {
|
|
||||||
this.updateBufferingStatus('No');
|
|
||||||
this.log(
|
|
||||||
`onMicCaptureStopped, timeTaken: ${Date.now() - this.startTime}`
|
|
||||||
);
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
);
|
|
||||||
this.setupTrackListeners();
|
this.setupTrackListeners();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -182,8 +172,8 @@ class InstantVoiceClient {
|
|||||||
* This is called when the bot is ready or when the transport state changes to ready
|
* This is called when the bot is ready or when the transport state changes to ready
|
||||||
*/
|
*/
|
||||||
setupMediaTracks() {
|
setupMediaTracks() {
|
||||||
if (!this.rtviClient) return;
|
if (!this.pcClient) return;
|
||||||
const tracks = this.rtviClient.tracks();
|
const tracks = this.pcClient.tracks();
|
||||||
if (tracks.bot?.audio) {
|
if (tracks.bot?.audio) {
|
||||||
this.setupAudioTrack(tracks.bot.audio);
|
this.setupAudioTrack(tracks.bot.audio);
|
||||||
}
|
}
|
||||||
@@ -194,10 +184,10 @@ class InstantVoiceClient {
|
|||||||
* This handles new tracks being added during the session
|
* This handles new tracks being added during the session
|
||||||
*/
|
*/
|
||||||
setupTrackListeners() {
|
setupTrackListeners() {
|
||||||
if (!this.rtviClient) return;
|
if (!this.pcClient) return;
|
||||||
|
|
||||||
// Listen for new tracks starting
|
// Listen for new tracks starting
|
||||||
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
|
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
|
||||||
// Only handle non-local (bot) tracks
|
// Only handle non-local (bot) tracks
|
||||||
if (!participant?.local && track.kind === 'audio') {
|
if (!participant?.local && track.kind === 'audio') {
|
||||||
this.setupAudioTrack(track);
|
this.setupAudioTrack(track);
|
||||||
@@ -205,7 +195,7 @@ class InstantVoiceClient {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Listen for tracks stopping
|
// Listen for tracks stopping
|
||||||
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
||||||
this.log(
|
this.log(
|
||||||
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
|
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
|
||||||
);
|
);
|
||||||
@@ -230,22 +220,25 @@ class InstantVoiceClient {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize and connect to the bot
|
* Initialize and connect to the bot
|
||||||
* This sets up the RTVI client, initializes devices, and establishes the connection
|
* This sets up the Pipecat client, initializes devices, and establishes the connection
|
||||||
*/
|
*/
|
||||||
public async connect(): Promise<void> {
|
public async connect(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
this.startTime = Date.now();
|
this.startTime = Date.now();
|
||||||
this.log('Connecting to bot...');
|
this.log('Connecting to bot...');
|
||||||
await this.rtviClient.connect();
|
await this.pcClient.connect({
|
||||||
|
// The baseURL and endpoint of your bot server that the client will connect to
|
||||||
|
endpoint: 'http://localhost:7860/connect',
|
||||||
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.log(`Error connecting: ${(error as Error).message}`);
|
this.log(`Error connecting: ${(error as Error).message}`);
|
||||||
this.updateStatus('Error');
|
this.updateStatus('Error');
|
||||||
this.updateBufferingStatus('No');
|
this.updateBufferingStatus('No');
|
||||||
|
|
||||||
// Clean up if there's an error
|
// Clean up if there's an error
|
||||||
if (this.rtviClient) {
|
if (this.pcClient) {
|
||||||
try {
|
try {
|
||||||
await this.rtviClient.disconnect();
|
await this.pcClient.disconnect();
|
||||||
} catch (disconnectError) {
|
} catch (disconnectError) {
|
||||||
this.log(`Error during disconnect: ${disconnectError}`);
|
this.log(`Error during disconnect: ${disconnectError}`);
|
||||||
}
|
}
|
||||||
@@ -258,7 +251,7 @@ class InstantVoiceClient {
|
|||||||
*/
|
*/
|
||||||
public async disconnect(): Promise<void> {
|
public async disconnect(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
await this.rtviClient.disconnect();
|
await this.pcClient.disconnect();
|
||||||
if (
|
if (
|
||||||
this.botAudio.srcObject &&
|
this.botAudio.srcObject &&
|
||||||
'getAudioTracks' in this.botAudio.srcObject
|
'getAudioTracks' in this.botAudio.srcObject
|
||||||
|
|||||||
@@ -1,39 +0,0 @@
|
|||||||
import {RTVIClientHelper, RTVIClientHelperOptions, RTVIMessage} from "@pipecat-ai/client-js";
|
|
||||||
import {DailyRTVIMessageType} from '@pipecat-ai/daily-transport';
|
|
||||||
|
|
||||||
export type InstantVoiceHelperCallbacks = Partial<{
|
|
||||||
onAudioBufferingStarted: () => void;
|
|
||||||
onAudioBufferingStopped: () => void;
|
|
||||||
}>;
|
|
||||||
|
|
||||||
// --- Interface and class
|
|
||||||
export interface InstantVoiceHelperOptions extends RTVIClientHelperOptions {
|
|
||||||
callbacks?: InstantVoiceHelperCallbacks;
|
|
||||||
}
|
|
||||||
export class InstantVoiceHelper extends RTVIClientHelper {
|
|
||||||
|
|
||||||
protected declare _options: InstantVoiceHelperOptions;
|
|
||||||
|
|
||||||
constructor(options: InstantVoiceHelperOptions) {
|
|
||||||
super(options);
|
|
||||||
}
|
|
||||||
|
|
||||||
handleMessage(rtviMessage: RTVIMessage): void {
|
|
||||||
switch (rtviMessage.type) {
|
|
||||||
case DailyRTVIMessageType.AUDIO_BUFFERING_STARTED:
|
|
||||||
if (this._options.callbacks?.onAudioBufferingStarted) {
|
|
||||||
this._options.callbacks?.onAudioBufferingStarted()
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
case DailyRTVIMessageType.AUDIO_BUFFERING_STOPPED:
|
|
||||||
if (this._options.callbacks?.onAudioBufferingStopped) {
|
|
||||||
this._options.callbacks?.onAudioBufferingStopped()
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
getMessageTypes(): string[] {
|
|
||||||
return [DailyRTVIMessageType.AUDIO_BUFFERING_STARTED, DailyRTVIMessageType.AUDIO_BUFFERING_STOPPED];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -5,7 +5,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RTVI Client Implementation
|
* Pipecat Client Implementation
|
||||||
*
|
*
|
||||||
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
|
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
|
||||||
* It handles audio/video streaming and manages the connection lifecycle.
|
* It handles audio/video streaming and manages the connection lifecycle.
|
||||||
@@ -16,78 +16,9 @@
|
|||||||
* - Browser with WebRTC support
|
* - Browser with WebRTC support
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import {
|
import { LogLevel, PipecatClient, RTVIEvent } from '@pipecat-ai/client-js';
|
||||||
LogLevel,
|
|
||||||
RTVIClient,
|
|
||||||
RTVIClientHelper,
|
|
||||||
RTVIEvent,
|
|
||||||
} from '@pipecat-ai/client-js';
|
|
||||||
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
||||||
|
|
||||||
class SearchResponseHelper extends RTVIClientHelper {
|
|
||||||
constructor(contentPanel) {
|
|
||||||
super();
|
|
||||||
this.contentPanel = contentPanel;
|
|
||||||
}
|
|
||||||
|
|
||||||
handleMessage(rtviMessage) {
|
|
||||||
console.log('SearchResponseHelper, received message:', rtviMessage);
|
|
||||||
if (rtviMessage.data) {
|
|
||||||
// Clear existing content
|
|
||||||
this.contentPanel.innerHTML = '';
|
|
||||||
|
|
||||||
// Create a container for all content
|
|
||||||
const contentContainer = document.createElement('div');
|
|
||||||
contentContainer.className = 'content-container';
|
|
||||||
|
|
||||||
// Add the search_result
|
|
||||||
if (rtviMessage.data.search_result) {
|
|
||||||
const searchResultDiv = document.createElement('div');
|
|
||||||
searchResultDiv.className = 'search-result';
|
|
||||||
searchResultDiv.textContent = rtviMessage.data.search_result;
|
|
||||||
contentContainer.appendChild(searchResultDiv);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add the sources
|
|
||||||
if (rtviMessage.data.origins) {
|
|
||||||
const sourcesDiv = document.createElement('div');
|
|
||||||
sourcesDiv.className = 'sources';
|
|
||||||
|
|
||||||
const sourcesTitle = document.createElement('h3');
|
|
||||||
sourcesTitle.className = 'sources-title';
|
|
||||||
sourcesTitle.textContent = 'Sources:';
|
|
||||||
sourcesDiv.appendChild(sourcesTitle);
|
|
||||||
|
|
||||||
rtviMessage.data.origins.forEach((origin) => {
|
|
||||||
const sourceLink = document.createElement('a');
|
|
||||||
sourceLink.className = 'source-link';
|
|
||||||
sourceLink.href = origin.site_uri;
|
|
||||||
sourceLink.target = '_blank';
|
|
||||||
sourceLink.textContent = origin.site_title;
|
|
||||||
sourcesDiv.appendChild(sourceLink);
|
|
||||||
});
|
|
||||||
|
|
||||||
contentContainer.appendChild(sourcesDiv);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add the rendered_content in an iframe
|
|
||||||
if (rtviMessage.data.rendered_content) {
|
|
||||||
const iframe = document.createElement('iframe');
|
|
||||||
iframe.className = 'iframe-container';
|
|
||||||
iframe.srcdoc = rtviMessage.data.rendered_content;
|
|
||||||
contentContainer.appendChild(iframe);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Append the content container to the content panel
|
|
||||||
this.contentPanel.appendChild(contentContainer);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
getMessageTypes() {
|
|
||||||
return ['bot-llm-search-response'];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ChatbotClient handles the connection and media management for a real-time
|
* ChatbotClient handles the connection and media management for a real-time
|
||||||
* voice and video interaction with an AI bot.
|
* voice and video interaction with an AI bot.
|
||||||
@@ -95,7 +26,7 @@ class SearchResponseHelper extends RTVIClientHelper {
|
|||||||
class ChatbotClient {
|
class ChatbotClient {
|
||||||
constructor() {
|
constructor() {
|
||||||
// Initialize client state
|
// Initialize client state
|
||||||
this.rtviClient = null;
|
this.pcClient = null;
|
||||||
this.setupDOMElements();
|
this.setupDOMElements();
|
||||||
this.setupEventListeners();
|
this.setupEventListeners();
|
||||||
}
|
}
|
||||||
@@ -160,10 +91,10 @@ class ChatbotClient {
|
|||||||
* This is called when the bot is ready or when the transport state changes to ready
|
* This is called when the bot is ready or when the transport state changes to ready
|
||||||
*/
|
*/
|
||||||
setupMediaTracks() {
|
setupMediaTracks() {
|
||||||
if (!this.rtviClient) return;
|
if (!this.pcClient) return;
|
||||||
|
|
||||||
// Get current tracks from the client
|
// Get current tracks from the client
|
||||||
const tracks = this.rtviClient.tracks();
|
const tracks = this.pcClient.tracks();
|
||||||
|
|
||||||
// Set up any available bot tracks
|
// Set up any available bot tracks
|
||||||
if (tracks.bot?.audio) {
|
if (tracks.bot?.audio) {
|
||||||
@@ -176,10 +107,10 @@ class ChatbotClient {
|
|||||||
* This handles new tracks being added during the session
|
* This handles new tracks being added during the session
|
||||||
*/
|
*/
|
||||||
setupTrackListeners() {
|
setupTrackListeners() {
|
||||||
if (!this.rtviClient) return;
|
if (!this.pcClient) return;
|
||||||
|
|
||||||
// Listen for new tracks starting
|
// Listen for new tracks starting
|
||||||
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
|
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
|
||||||
// Only handle non-local (bot) tracks
|
// Only handle non-local (bot) tracks
|
||||||
if (!participant?.local && track.kind === 'audio') {
|
if (!participant?.local && track.kind === 'audio') {
|
||||||
this.setupAudioTrack(track);
|
this.setupAudioTrack(track);
|
||||||
@@ -187,7 +118,7 @@ class ChatbotClient {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Listen for tracks stopping
|
// Listen for tracks stopping
|
||||||
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
||||||
this.log(
|
this.log(
|
||||||
`Track stopped event: ${track.kind} from ${
|
`Track stopped event: ${track.kind} from ${
|
||||||
participant?.name || 'unknown'
|
participant?.name || 'unknown'
|
||||||
@@ -213,20 +144,13 @@ class ChatbotClient {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize and connect to the bot
|
* Initialize and connect to the bot
|
||||||
* This sets up the RTVI client, initializes devices, and establishes the connection
|
* This sets up the Pipecat client, initializes devices, and establishes the connection
|
||||||
*/
|
*/
|
||||||
async connect() {
|
async connect() {
|
||||||
try {
|
try {
|
||||||
// Initialize the RTVI client with a Daily WebRTC transport and our configuration
|
// Initialize the Pipecat client with a Daily WebRTC transport and our configuration
|
||||||
this.rtviClient = new RTVIClient({
|
this.pcClient = new PipecatClient({
|
||||||
transport: new DailyTransport(),
|
transport: new DailyTransport(),
|
||||||
params: {
|
|
||||||
// The baseURL and endpoint of your bot server that the client will connect to
|
|
||||||
baseUrl: 'http://localhost:7860',
|
|
||||||
endpoints: {
|
|
||||||
connect: '/connect',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
enableMic: true, // Enable microphone for user input
|
enableMic: true, // Enable microphone for user input
|
||||||
enableCam: false,
|
enableCam: false,
|
||||||
callbacks: {
|
callbacks: {
|
||||||
@@ -251,6 +175,8 @@ class ChatbotClient {
|
|||||||
this.setupMediaTracks();
|
this.setupMediaTracks();
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
// Handle search response events
|
||||||
|
onBotLlmSearchResponse: this.handleSearchResponse.bind(this),
|
||||||
// Handle bot connection events
|
// Handle bot connection events
|
||||||
onBotConnected: (participant) => {
|
onBotConnected: (participant) => {
|
||||||
this.log(`Bot connected: ${JSON.stringify(participant)}`);
|
this.log(`Bot connected: ${JSON.stringify(participant)}`);
|
||||||
@@ -281,22 +207,22 @@ class ChatbotClient {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
//this.rtviClient.setLogLevel(LogLevel.DEBUG)
|
|
||||||
this.rtviClient.registerHelper(
|
//this.pcClient.setLogLevel(LogLevel.DEBUG)
|
||||||
'llm',
|
|
||||||
new SearchResponseHelper(this.searchResultContainer)
|
|
||||||
);
|
|
||||||
|
|
||||||
// Set up listeners for media track events
|
// Set up listeners for media track events
|
||||||
this.setupTrackListeners();
|
this.setupTrackListeners();
|
||||||
|
|
||||||
// Initialize audio devices
|
// Initialize audio devices
|
||||||
this.log('Initializing devices...');
|
this.log('Initializing devices...');
|
||||||
await this.rtviClient.initDevices();
|
await this.pcClient.initDevices();
|
||||||
|
|
||||||
// Connect to the bot
|
// Connect to the bot
|
||||||
this.log('Connecting to bot...');
|
this.log('Connecting to bot...');
|
||||||
await this.rtviClient.connect();
|
await this.pcClient.connect({
|
||||||
|
// The baseURL and endpoint of your bot server that the client will connect to
|
||||||
|
endpoint: 'http://localhost:7860/connect',
|
||||||
|
});
|
||||||
|
|
||||||
this.log('Connection complete');
|
this.log('Connection complete');
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -306,9 +232,9 @@ class ChatbotClient {
|
|||||||
this.updateStatus('Error');
|
this.updateStatus('Error');
|
||||||
|
|
||||||
// Clean up if there's an error
|
// Clean up if there's an error
|
||||||
if (this.rtviClient) {
|
if (this.pcClient) {
|
||||||
try {
|
try {
|
||||||
await this.rtviClient.disconnect();
|
await this.pcClient.disconnect();
|
||||||
} catch (disconnectError) {
|
} catch (disconnectError) {
|
||||||
this.log(`Error during disconnect: ${disconnectError.message}`);
|
this.log(`Error during disconnect: ${disconnectError.message}`);
|
||||||
}
|
}
|
||||||
@@ -320,11 +246,11 @@ class ChatbotClient {
|
|||||||
* Disconnect from the bot and clean up media resources
|
* Disconnect from the bot and clean up media resources
|
||||||
*/
|
*/
|
||||||
async disconnect() {
|
async disconnect() {
|
||||||
if (this.rtviClient) {
|
if (this.pcClient) {
|
||||||
try {
|
try {
|
||||||
// Disconnect the RTVI client
|
// Disconnect the Pipecat client
|
||||||
await this.rtviClient.disconnect();
|
await this.pcClient.disconnect();
|
||||||
this.rtviClient = null;
|
this.pcClient = null;
|
||||||
|
|
||||||
// Clean up audio
|
// Clean up audio
|
||||||
if (this.botAudio.srcObject) {
|
if (this.botAudio.srcObject) {
|
||||||
@@ -339,6 +265,57 @@ class ChatbotClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
handleSearchResponse(response) {
|
||||||
|
console.log('SearchResponseHelper, received message:', response);
|
||||||
|
// Clear existing content
|
||||||
|
this.searchResultContainer.innerHTML = '';
|
||||||
|
|
||||||
|
// Create a container for all content
|
||||||
|
const contentContainer = document.createElement('div');
|
||||||
|
contentContainer.className = 'content-container';
|
||||||
|
|
||||||
|
// Add the search_result
|
||||||
|
if (response.search_result) {
|
||||||
|
const searchResultDiv = document.createElement('div');
|
||||||
|
searchResultDiv.className = 'search-result';
|
||||||
|
searchResultDiv.textContent = response.search_result;
|
||||||
|
contentContainer.appendChild(searchResultDiv);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add the sources
|
||||||
|
if (response.origins) {
|
||||||
|
const sourcesDiv = document.createElement('div');
|
||||||
|
sourcesDiv.className = 'sources';
|
||||||
|
|
||||||
|
const sourcesTitle = document.createElement('h3');
|
||||||
|
sourcesTitle.className = 'sources-title';
|
||||||
|
sourcesTitle.textContent = 'Sources:';
|
||||||
|
sourcesDiv.appendChild(sourcesTitle);
|
||||||
|
|
||||||
|
response.origins.forEach((origin) => {
|
||||||
|
const sourceLink = document.createElement('a');
|
||||||
|
sourceLink.className = 'source-link';
|
||||||
|
sourceLink.href = origin.site_uri;
|
||||||
|
sourceLink.target = '_blank';
|
||||||
|
sourceLink.textContent = origin.site_title;
|
||||||
|
sourcesDiv.appendChild(sourceLink);
|
||||||
|
});
|
||||||
|
|
||||||
|
contentContainer.appendChild(sourcesDiv);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add the rendered_content in an iframe
|
||||||
|
if (response.rendered_content) {
|
||||||
|
const iframe = document.createElement('iframe');
|
||||||
|
iframe.className = 'iframe-container';
|
||||||
|
iframe.srcdoc = response.rendered_content;
|
||||||
|
contentContainer.appendChild(iframe);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Append the content container to the content panel
|
||||||
|
this.searchResultContainer.appendChild(contentContainer);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize the client when the page loads
|
// Initialize the client when the page loads
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
import { SmallWebRTCTransport } from '@pipecat-ai/small-webrtc-transport';
|
import { SmallWebRTCTransport } from '@pipecat-ai/small-webrtc-transport';
|
||||||
import {
|
import {
|
||||||
|
BotLLMTextData,
|
||||||
Participant,
|
Participant,
|
||||||
RTVIClient,
|
PipecatClient,
|
||||||
RTVIClientOptions,
|
PipecatClientOptions,
|
||||||
Transport,
|
TranscriptData,
|
||||||
|
TransportState,
|
||||||
} from '@pipecat-ai/client-js';
|
} from '@pipecat-ai/client-js';
|
||||||
|
|
||||||
class WebRTCApp {
|
class WebRTCApp {
|
||||||
@@ -23,26 +25,22 @@ class WebRTCApp {
|
|||||||
private statusSpan: HTMLElement | null = null;
|
private statusSpan: HTMLElement | null = null;
|
||||||
|
|
||||||
private declare smallWebRTCTransport: SmallWebRTCTransport;
|
private declare smallWebRTCTransport: SmallWebRTCTransport;
|
||||||
private declare rtviClient: RTVIClient;
|
private declare pcClient: PipecatClient;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.setupDOMElements();
|
this.setupDOMElements();
|
||||||
this.setupDOMEventListeners();
|
this.setupDOMEventListeners();
|
||||||
this.initializeRTVIClient();
|
this.initializePipecatClient();
|
||||||
void this.populateDevices();
|
void this.populateDevices();
|
||||||
}
|
}
|
||||||
|
|
||||||
private initializeRTVIClient(): void {
|
private initializePipecatClient(): void {
|
||||||
const transport = new SmallWebRTCTransport();
|
const opts: PipecatClientOptions = {
|
||||||
const RTVIConfig: RTVIClientOptions = {
|
transport: new SmallWebRTCTransport({ connectionUrl: '/api/offer' }),
|
||||||
params: {
|
|
||||||
baseUrl: '/api/offer',
|
|
||||||
},
|
|
||||||
transport: transport as Transport,
|
|
||||||
enableMic: true,
|
enableMic: true,
|
||||||
enableCam: true,
|
enableCam: true,
|
||||||
callbacks: {
|
callbacks: {
|
||||||
onTransportStateChanged: (state) => {
|
onTransportStateChanged: (state: TransportState) => {
|
||||||
this.log(`Transport state: ${state}`);
|
this.log(`Transport state: ${state}`);
|
||||||
},
|
},
|
||||||
onConnected: () => {
|
onConnected: () => {
|
||||||
@@ -66,13 +64,13 @@ class WebRTCApp {
|
|||||||
onBotStoppedSpeaking: () => {
|
onBotStoppedSpeaking: () => {
|
||||||
this.log('Bot stopped speaking.');
|
this.log('Bot stopped speaking.');
|
||||||
},
|
},
|
||||||
onUserTranscript: (transcript) => {
|
onUserTranscript: (transcript: TranscriptData) => {
|
||||||
if (transcript.final) {
|
if (transcript.final) {
|
||||||
this.log(`User transcript: ${transcript.text}`);
|
this.log(`User transcript: ${transcript.text}`);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
onBotTranscript: (transcript) => {
|
onBotTranscript: (data: BotLLMTextData) => {
|
||||||
this.log(`Bot transcript: ${transcript.text}`);
|
this.log(`Bot transcript: ${data.text}`);
|
||||||
},
|
},
|
||||||
onTrackStarted: (
|
onTrackStarted: (
|
||||||
track: MediaStreamTrack,
|
track: MediaStreamTrack,
|
||||||
@@ -83,14 +81,13 @@ class WebRTCApp {
|
|||||||
}
|
}
|
||||||
this.onBotTrackStarted(track);
|
this.onBotTrackStarted(track);
|
||||||
},
|
},
|
||||||
onServerMessage: (msg) => {
|
onServerMessage: (msg: unknown) => {
|
||||||
this.log(`Server message: ${msg}`);
|
this.log(`Server message: ${msg}`);
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
RTVIConfig.customConnectHandler = () => Promise.resolve();
|
this.pcClient = new PipecatClient(opts);
|
||||||
this.rtviClient = new RTVIClient(RTVIConfig);
|
this.smallWebRTCTransport = this.pcClient.transport as SmallWebRTCTransport;
|
||||||
this.smallWebRTCTransport = transport;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private setupDOMElements(): void {
|
private setupDOMElements(): void {
|
||||||
@@ -132,16 +129,16 @@ class WebRTCApp {
|
|||||||
this.audioInput.addEventListener('change', (e) => {
|
this.audioInput.addEventListener('change', (e) => {
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
let audioDevice = e.target?.value;
|
let audioDevice = e.target?.value;
|
||||||
this.rtviClient.updateMic(audioDevice);
|
this.pcClient.updateMic(audioDevice);
|
||||||
});
|
});
|
||||||
this.videoInput.addEventListener('change', (e) => {
|
this.videoInput.addEventListener('change', (e) => {
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
let videoDevice = e.target?.value;
|
let videoDevice = e.target?.value;
|
||||||
this.rtviClient.updateCam(videoDevice);
|
this.pcClient.updateCam(videoDevice);
|
||||||
});
|
});
|
||||||
this.muteBtn.addEventListener('click', () => {
|
this.muteBtn.addEventListener('click', () => {
|
||||||
let isCamEnabled = this.rtviClient.isCamEnabled;
|
let isCamEnabled = this.pcClient.isCamEnabled;
|
||||||
this.rtviClient.enableCam(!isCamEnabled);
|
this.pcClient.enableCam(!isCamEnabled);
|
||||||
this.muteBtn.textContent = isCamEnabled ? '📵' : '📷';
|
this.muteBtn.textContent = isCamEnabled ? '📵' : '📷';
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -206,9 +203,9 @@ class WebRTCApp {
|
|||||||
};
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const audioDevices = await this.rtviClient.getAllMics();
|
const audioDevices = await this.pcClient.getAllMics();
|
||||||
populateSelect(this.audioInput, audioDevices);
|
populateSelect(this.audioInput, audioDevices);
|
||||||
const videoDevices = await this.rtviClient.getAllCams();
|
const videoDevices = await this.pcClient.getAllCams();
|
||||||
populateSelect(this.videoInput, videoDevices);
|
populateSelect(this.videoInput, videoDevices);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
alert(e);
|
alert(e);
|
||||||
@@ -224,7 +221,7 @@ class WebRTCApp {
|
|||||||
this.smallWebRTCTransport.setAudioCodec(this.audioCodec.value);
|
this.smallWebRTCTransport.setAudioCodec(this.audioCodec.value);
|
||||||
this.smallWebRTCTransport.setVideoCodec(this.videoCodec.value);
|
this.smallWebRTCTransport.setVideoCodec(this.videoCodec.value);
|
||||||
try {
|
try {
|
||||||
await this.rtviClient.connect();
|
await this.pcClient.connect();
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.log(`Failed to connect ${e}`);
|
console.log(`Failed to connect ${e}`);
|
||||||
this.stop();
|
this.stop();
|
||||||
@@ -232,7 +229,7 @@ class WebRTCApp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private stop(): void {
|
private stop(): void {
|
||||||
void this.rtviClient.disconnect();
|
void this.pcClient.disconnect();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RTVI Client Implementation
|
* Pipecat Client Implementation
|
||||||
*
|
*
|
||||||
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
|
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
|
||||||
* It handles audio/video streaming and manages the connection lifecycle.
|
* It handles audio/video streaming and manages the connection lifecycle.
|
||||||
@@ -16,7 +16,7 @@
|
|||||||
* - Browser with WebRTC support
|
* - Browser with WebRTC support
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { RTVIClient, RTVIEvent } from '@pipecat-ai/client-js';
|
import { PipecatClient, RTVIEvent } from '@pipecat-ai/client-js';
|
||||||
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -26,7 +26,7 @@ import { DailyTransport } from '@pipecat-ai/daily-transport';
|
|||||||
class ChatbotClient {
|
class ChatbotClient {
|
||||||
constructor() {
|
constructor() {
|
||||||
// Initialize client state
|
// Initialize client state
|
||||||
this.rtviClient = null;
|
this.pcClient = null;
|
||||||
this.setupDOMElements();
|
this.setupDOMElements();
|
||||||
this.initializeClientAndTransport();
|
this.initializeClientAndTransport();
|
||||||
}
|
}
|
||||||
@@ -54,11 +54,14 @@ class ChatbotClient {
|
|||||||
* Set up event listeners for connect/disconnect buttons
|
* Set up event listeners for connect/disconnect buttons
|
||||||
*/
|
*/
|
||||||
setupEventListeners() {
|
setupEventListeners() {
|
||||||
this.connectBtn.addEventListener('click', () => this.connect());
|
this.connectBtn.addEventListener('click', () => {
|
||||||
|
console.log('click');
|
||||||
|
this.connect();
|
||||||
|
});
|
||||||
this.disconnectBtn.addEventListener('click', () => this.disconnect());
|
this.disconnectBtn.addEventListener('click', () => this.disconnect());
|
||||||
|
|
||||||
// Populate device selector
|
// Populate device selector
|
||||||
this.rtviClient.getAllMics().then((mics) => {
|
this.pcClient.getAllMics().then((mics) => {
|
||||||
console.log('Available mics:', mics);
|
console.log('Available mics:', mics);
|
||||||
mics.forEach((device) => {
|
mics.forEach((device) => {
|
||||||
const option = document.createElement('option');
|
const option = document.createElement('option');
|
||||||
@@ -70,33 +73,27 @@ class ChatbotClient {
|
|||||||
this.deviceSelector.addEventListener('change', (event) => {
|
this.deviceSelector.addEventListener('change', (event) => {
|
||||||
const selectedDeviceId = event.target.value;
|
const selectedDeviceId = event.target.value;
|
||||||
console.log('Selected device ID:', selectedDeviceId);
|
console.log('Selected device ID:', selectedDeviceId);
|
||||||
this.rtviClient.updateMic(selectedDeviceId);
|
this.pcClient.updateMic(selectedDeviceId);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Handle mic mute/unmute toggle
|
// Handle mic mute/unmute toggle
|
||||||
const micToggleBtn = document.getElementById('mic-toggle-btn');
|
const micToggleBtn = document.getElementById('mic-toggle-btn');
|
||||||
|
|
||||||
micToggleBtn.addEventListener('click', () => {
|
micToggleBtn.addEventListener('click', () => {
|
||||||
let micEnabled = this.rtviClient.isMicEnabled;
|
let micEnabled = this.pcClient.isMicEnabled;
|
||||||
micToggleBtn.textContent = micEnabled ? 'Unmute Mic' : 'Mute Mic';
|
micToggleBtn.textContent = micEnabled ? 'Unmute Mic' : 'Mute Mic';
|
||||||
this.rtviClient.enableMic(!micEnabled);
|
this.pcClient.enableMic(!micEnabled);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set up the RTVI client and Daily transport
|
* Set up the Pipecat client and Daily transport
|
||||||
*/
|
*/
|
||||||
async initializeClientAndTransport() {
|
async initializeClientAndTransport() {
|
||||||
// Initialize the RTVI client with a DailyTransport and our configuration
|
console.log('Initializing Pipecat client and transport...');
|
||||||
this.rtviClient = new RTVIClient({
|
// Initialize the Pipecat client with a DailyTransport and our configuration
|
||||||
|
this.pcClient = new PipecatClient({
|
||||||
transport: new DailyTransport(),
|
transport: new DailyTransport(),
|
||||||
params: {
|
|
||||||
// The baseURL and endpoint of your bot server that the client will connect to
|
|
||||||
baseUrl: 'http://localhost:7860',
|
|
||||||
endpoints: {
|
|
||||||
connect: '/connect',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
enableMic: true, // Enable microphone for user input
|
enableMic: true, // Enable microphone for user input
|
||||||
enableCam: false,
|
enableCam: false,
|
||||||
callbacks: {
|
callbacks: {
|
||||||
@@ -160,7 +157,7 @@ class ChatbotClient {
|
|||||||
// Set up listeners for media track events
|
// Set up listeners for media track events
|
||||||
this.setupTrackListeners();
|
this.setupTrackListeners();
|
||||||
|
|
||||||
await this.rtviClient.initDevices();
|
await this.pcClient.initDevices();
|
||||||
this.setupEventListeners();
|
this.setupEventListeners();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -196,10 +193,10 @@ class ChatbotClient {
|
|||||||
* This is called when the bot is ready or when the transport state changes to ready
|
* This is called when the bot is ready or when the transport state changes to ready
|
||||||
*/
|
*/
|
||||||
setupMediaTracks() {
|
setupMediaTracks() {
|
||||||
if (!this.rtviClient) return;
|
if (!this.pcClient) return;
|
||||||
|
|
||||||
// Get current tracks from the client
|
// Get current tracks from the client
|
||||||
const tracks = this.rtviClient.tracks();
|
const tracks = this.pcClient.tracks();
|
||||||
|
|
||||||
// Set up any available bot tracks
|
// Set up any available bot tracks
|
||||||
if (tracks.bot?.audio) {
|
if (tracks.bot?.audio) {
|
||||||
@@ -215,10 +212,10 @@ class ChatbotClient {
|
|||||||
* This handles new tracks being added during the session
|
* This handles new tracks being added during the session
|
||||||
*/
|
*/
|
||||||
setupTrackListeners() {
|
setupTrackListeners() {
|
||||||
if (!this.rtviClient) return;
|
if (!this.pcClient) return;
|
||||||
|
|
||||||
// Listen for new tracks starting
|
// Listen for new tracks starting
|
||||||
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
|
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
|
||||||
// Only handle non-local (bot) tracks
|
// Only handle non-local (bot) tracks
|
||||||
if (!participant?.local) {
|
if (!participant?.local) {
|
||||||
if (track.kind === 'audio') {
|
if (track.kind === 'audio') {
|
||||||
@@ -230,7 +227,7 @@ class ChatbotClient {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Listen for tracks stopping
|
// Listen for tracks stopping
|
||||||
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
||||||
this.log(
|
this.log(
|
||||||
`Track stopped event: ${track.kind} from ${
|
`Track stopped event: ${track.kind} from ${
|
||||||
participant?.name || 'unknown'
|
participant?.name || 'unknown'
|
||||||
@@ -284,17 +281,16 @@ class ChatbotClient {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize and connect to the bot
|
* Initialize and connect to the bot
|
||||||
* This sets up the RTVI client, initializes devices, and establishes the connection
|
* This sets up the Pipecat client, initializes devices, and establishes the connection
|
||||||
*/
|
*/
|
||||||
async connect() {
|
async connect() {
|
||||||
try {
|
try {
|
||||||
// Initialize audio/video devices
|
|
||||||
this.log('Initializing devices...');
|
|
||||||
await this.rtviClient.initDevices();
|
|
||||||
|
|
||||||
// Connect to the bot
|
// Connect to the bot
|
||||||
this.log('Connecting to bot...');
|
this.log('Connecting to bot...');
|
||||||
await this.rtviClient.connect();
|
await this.pcClient.connect({
|
||||||
|
endpoint: 'http://localhost:7860/connect',
|
||||||
|
timeout: 25000,
|
||||||
|
});
|
||||||
|
|
||||||
this.log('Connection complete');
|
this.log('Connection complete');
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -304,9 +300,9 @@ class ChatbotClient {
|
|||||||
this.updateStatus('Error');
|
this.updateStatus('Error');
|
||||||
|
|
||||||
// Clean up if there's an error
|
// Clean up if there's an error
|
||||||
if (this.rtviClient) {
|
if (this.pcClient) {
|
||||||
try {
|
try {
|
||||||
await this.rtviClient.disconnect();
|
await this.pcClient.disconnect();
|
||||||
} catch (disconnectError) {
|
} catch (disconnectError) {
|
||||||
this.log(`Error during disconnect: ${disconnectError.message}`);
|
this.log(`Error during disconnect: ${disconnectError.message}`);
|
||||||
}
|
}
|
||||||
@@ -318,10 +314,10 @@ class ChatbotClient {
|
|||||||
* Disconnect from the bot and clean up media resources
|
* Disconnect from the bot and clean up media resources
|
||||||
*/
|
*/
|
||||||
async disconnect() {
|
async disconnect() {
|
||||||
if (this.rtviClient) {
|
if (this.pcClient) {
|
||||||
try {
|
try {
|
||||||
// Disconnect the RTVI client
|
// Disconnect the Pipecat client
|
||||||
await this.rtviClient.disconnect();
|
await this.pcClient.disconnect();
|
||||||
|
|
||||||
// Clean up audio
|
// Clean up audio
|
||||||
if (this.botAudio.srcObject) {
|
if (this.botAudio.srcObject) {
|
||||||
|
|||||||
@@ -1,22 +1,22 @@
|
|||||||
import {
|
import {
|
||||||
RTVIClientAudio,
|
PipecatClientAudio,
|
||||||
RTVIClientVideo,
|
PipecatClientVideo,
|
||||||
useRTVIClientTransportState,
|
usePipecatClientTransportState,
|
||||||
} from '@pipecat-ai/client-react';
|
} from '@pipecat-ai/client-react';
|
||||||
import { RTVIProvider } from './providers/RTVIProvider';
|
import { PipecatProvider } from './providers/PipecatProvider';
|
||||||
import { ConnectButton } from './components/ConnectButton';
|
import { ConnectButton } from './components/ConnectButton';
|
||||||
import { StatusDisplay } from './components/StatusDisplay';
|
import { StatusDisplay } from './components/StatusDisplay';
|
||||||
import { DebugDisplay } from './components/DebugDisplay';
|
import { DebugDisplay } from './components/DebugDisplay';
|
||||||
import './App.css';
|
import './App.css';
|
||||||
|
|
||||||
function BotVideo() {
|
function BotVideo() {
|
||||||
const transportState = useRTVIClientTransportState();
|
const transportState = usePipecatClientTransportState();
|
||||||
const isConnected = transportState !== 'disconnected';
|
const isConnected = transportState !== 'disconnected';
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="bot-container">
|
<div className="bot-container">
|
||||||
<div className="video-container">
|
<div className="video-container">
|
||||||
{isConnected && <RTVIClientVideo participant="bot" fit="cover" />}
|
{isConnected && <PipecatClientVideo participant="bot" fit="cover" />}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
@@ -35,16 +35,16 @@ function AppContent() {
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<DebugDisplay />
|
<DebugDisplay />
|
||||||
<RTVIClientAudio />
|
<PipecatClientAudio />
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function App() {
|
function App() {
|
||||||
return (
|
return (
|
||||||
<RTVIProvider>
|
<PipecatProvider>
|
||||||
<AppContent />
|
<AppContent />
|
||||||
</RTVIProvider>
|
</PipecatProvider>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,16 +1,16 @@
|
|||||||
import {
|
import {
|
||||||
useRTVIClient,
|
usePipecatClient,
|
||||||
useRTVIClientTransportState,
|
usePipecatClientTransportState,
|
||||||
} from '@pipecat-ai/client-react';
|
} from '@pipecat-ai/client-react';
|
||||||
|
|
||||||
export function ConnectButton() {
|
export function ConnectButton() {
|
||||||
const client = useRTVIClient();
|
const client = usePipecatClient();
|
||||||
const transportState = useRTVIClientTransportState();
|
const transportState = usePipecatClientTransportState();
|
||||||
const isConnected = ['connected', 'ready'].includes(transportState);
|
const isConnected = ['connected', 'ready'].includes(transportState);
|
||||||
|
|
||||||
const handleClick = async () => {
|
const handleClick = async () => {
|
||||||
if (!client) {
|
if (!client) {
|
||||||
console.error('RTVI client is not initialized');
|
console.error('Pipecat client is not initialized');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -18,7 +18,7 @@ export function ConnectButton() {
|
|||||||
if (isConnected) {
|
if (isConnected) {
|
||||||
await client.disconnect();
|
await client.disconnect();
|
||||||
} else {
|
} else {
|
||||||
await client.connect();
|
await client.connect({ endpoint: 'http://localhost:7860/connect' });
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Connection error:', error);
|
console.error('Connection error:', error);
|
||||||
|
|||||||
@@ -6,12 +6,12 @@ import {
|
|||||||
TranscriptData,
|
TranscriptData,
|
||||||
BotLLMTextData,
|
BotLLMTextData,
|
||||||
} from '@pipecat-ai/client-js';
|
} from '@pipecat-ai/client-js';
|
||||||
import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
|
import { usePipecatClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
|
||||||
import './DebugDisplay.css';
|
import './DebugDisplay.css';
|
||||||
|
|
||||||
export function DebugDisplay() {
|
export function DebugDisplay() {
|
||||||
const debugLogRef = useRef<HTMLDivElement>(null);
|
const debugLogRef = useRef<HTMLDivElement>(null);
|
||||||
const client = useRTVIClient();
|
const client = usePipecatClient();
|
||||||
|
|
||||||
const log = useCallback((message: string) => {
|
const log = useCallback((message: string) => {
|
||||||
if (!debugLogRef.current) return;
|
if (!debugLogRef.current) return;
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { useRTVIClientTransportState } from '@pipecat-ai/client-react';
|
import { usePipecatClientTransportState } from '@pipecat-ai/client-react';
|
||||||
|
|
||||||
export function StatusDisplay() {
|
export function StatusDisplay() {
|
||||||
const transportState = useRTVIClientTransportState();
|
const transportState = usePipecatClientTransportState();
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="status">
|
<div className="status">
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { type PropsWithChildren } from 'react';
|
||||||
|
import { PipecatClient } from '@pipecat-ai/client-js';
|
||||||
|
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
||||||
|
import { PipecatClientProvider } from '@pipecat-ai/client-react';
|
||||||
|
|
||||||
|
const client = new PipecatClient({
|
||||||
|
transport: new DailyTransport(),
|
||||||
|
enableMic: true,
|
||||||
|
enableCam: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
export function PipecatProvider({ children }: PropsWithChildren) {
|
||||||
|
return (
|
||||||
|
<PipecatClientProvider client={client}>{children}</PipecatClientProvider>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
import { type PropsWithChildren } from 'react';
|
|
||||||
import { RTVIClient } from '@pipecat-ai/client-js';
|
|
||||||
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
|
||||||
import { RTVIClientProvider } from '@pipecat-ai/client-react';
|
|
||||||
|
|
||||||
const transport = new DailyTransport();
|
|
||||||
|
|
||||||
const client = new RTVIClient({
|
|
||||||
transport,
|
|
||||||
params: {
|
|
||||||
baseUrl: 'http://localhost:7860',
|
|
||||||
endpoints: {
|
|
||||||
connect: '/connect',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
enableMic: true,
|
|
||||||
enableCam: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
export function RTVIProvider({ children }: PropsWithChildren) {
|
|
||||||
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
|
|
||||||
}
|
|
||||||
@@ -12,12 +12,11 @@ import {
|
|||||||
import {
|
import {
|
||||||
WebSocketTransport,
|
WebSocketTransport,
|
||||||
TwilioSerializer,
|
TwilioSerializer,
|
||||||
} from "@pipecat-ai/websocket-transport";
|
} from '@pipecat-ai/websocket-transport';
|
||||||
|
|
||||||
class WebsocketClientApp {
|
class WebsocketClientApp {
|
||||||
|
private static STREAM_SID = 'ws_mock_stream_sid';
|
||||||
private static STREAM_SID = "ws_mock_stream_sid"
|
private static CALL_SID = 'ws_mock_call_sid';
|
||||||
private static CALL_SID = "ws_mock_call_sid"
|
|
||||||
|
|
||||||
private rtviClient: RTVIClient | null = null;
|
private rtviClient: RTVIClient | null = null;
|
||||||
private connectBtn: HTMLButtonElement | null = null;
|
private connectBtn: HTMLButtonElement | null = null;
|
||||||
@@ -38,8 +37,12 @@ class WebsocketClientApp {
|
|||||||
* Set up references to DOM elements and create necessary media elements
|
* Set up references to DOM elements and create necessary media elements
|
||||||
*/
|
*/
|
||||||
private setupDOMElements(): void {
|
private setupDOMElements(): void {
|
||||||
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
|
this.connectBtn = document.getElementById(
|
||||||
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
|
'connect-btn'
|
||||||
|
) as HTMLButtonElement;
|
||||||
|
this.disconnectBtn = document.getElementById(
|
||||||
|
'disconnect-btn'
|
||||||
|
) as HTMLButtonElement;
|
||||||
this.statusSpan = document.getElementById('connection-status');
|
this.statusSpan = document.getElementById('connection-status');
|
||||||
this.debugLog = document.getElementById('debug-log');
|
this.debugLog = document.getElementById('debug-log');
|
||||||
}
|
}
|
||||||
@@ -80,13 +83,23 @@ class WebsocketClientApp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async emulateTwilioMessages() {
|
private async emulateTwilioMessages() {
|
||||||
const connectedMessage={"event": "connected", "protocol": "Call", "version": "1.0.0"}
|
const connectedMessage = {
|
||||||
|
event: 'connected',
|
||||||
|
protocol: 'Call',
|
||||||
|
version: '1.0.0',
|
||||||
|
};
|
||||||
|
|
||||||
const websocketTransport = this.rtviClient?.transport as WebSocketTransport
|
const websocketTransport = this.rtviClient?.transport as WebSocketTransport;
|
||||||
void websocketTransport?.sendRawMessage(connectedMessage)
|
void websocketTransport?.sendRawMessage(connectedMessage);
|
||||||
|
|
||||||
const startMessage={"event": "start", "start": {"streamSid": WebsocketClientApp.STREAM_SID, "callSid": WebsocketClientApp.CALL_SID}}
|
const startMessage = {
|
||||||
void websocketTransport?.sendRawMessage(startMessage)
|
event: 'start',
|
||||||
|
start: {
|
||||||
|
streamSid: WebsocketClientApp.STREAM_SID,
|
||||||
|
callSid: WebsocketClientApp.CALL_SID,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
void websocketTransport?.sendRawMessage(startMessage);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -118,7 +131,9 @@ class WebsocketClientApp {
|
|||||||
|
|
||||||
// Listen for tracks stopping
|
// Listen for tracks stopping
|
||||||
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
||||||
this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`);
|
this.log(
|
||||||
|
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
|
||||||
|
);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,7 +143,10 @@ class WebsocketClientApp {
|
|||||||
*/
|
*/
|
||||||
private setupAudioTrack(track: MediaStreamTrack): void {
|
private setupAudioTrack(track: MediaStreamTrack): void {
|
||||||
this.log('Setting up audio track');
|
this.log('Setting up audio track');
|
||||||
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
|
if (
|
||||||
|
this.botAudio.srcObject &&
|
||||||
|
'getAudioTracks' in this.botAudio.srcObject
|
||||||
|
) {
|
||||||
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
|
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
|
||||||
if (oldTrack?.id === track.id) return;
|
if (oldTrack?.id === track.id) return;
|
||||||
}
|
}
|
||||||
@@ -143,23 +161,19 @@ class WebsocketClientApp {
|
|||||||
try {
|
try {
|
||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
|
|
||||||
const transport = new WebSocketTransport({
|
const ws_opts = {
|
||||||
serializer: new TwilioSerializer(),
|
serializer: new TwilioSerializer(),
|
||||||
recorderSampleRate: 8000,
|
recorderSampleRate: 8000,
|
||||||
playerSampleRate: 8000
|
playerSampleRate: 8000,
|
||||||
});
|
ws_url: 'http://localhost:8765/ws',
|
||||||
|
};
|
||||||
const RTVIConfig: RTVIClientOptions = {
|
const RTVIConfig: RTVIClientOptions = {
|
||||||
transport,
|
transport: new WebSocketTransport(ws_opts),
|
||||||
params: {
|
|
||||||
// The baseURL and endpoint of your bot server that the client will connect to
|
|
||||||
baseUrl: 'http://localhost:8765',
|
|
||||||
endpoints: { connect: '/' },
|
|
||||||
},
|
|
||||||
enableMic: true,
|
enableMic: true,
|
||||||
enableCam: false,
|
enableCam: false,
|
||||||
callbacks: {
|
callbacks: {
|
||||||
onConnected: () => {
|
onConnected: () => {
|
||||||
this.emulateTwilioMessages()
|
this.emulateTwilioMessages();
|
||||||
this.updateStatus('Connected');
|
this.updateStatus('Connected');
|
||||||
if (this.connectBtn) this.connectBtn.disabled = true;
|
if (this.connectBtn) this.connectBtn.disabled = true;
|
||||||
if (this.disconnectBtn) this.disconnectBtn.disabled = false;
|
if (this.disconnectBtn) this.disconnectBtn.disabled = false;
|
||||||
@@ -183,13 +197,7 @@ class WebsocketClientApp {
|
|||||||
onMessageError: (error) => console.error('Message error:', error),
|
onMessageError: (error) => console.error('Message error:', error),
|
||||||
onError: (error) => console.error('Error:', error),
|
onError: (error) => console.error('Error:', error),
|
||||||
},
|
},
|
||||||
}
|
};
|
||||||
// @ts-ignore
|
|
||||||
RTVIConfig.customConnectHandler = () => Promise.resolve(
|
|
||||||
{
|
|
||||||
ws_url: "/ws",
|
|
||||||
}
|
|
||||||
);
|
|
||||||
this.rtviClient = new RTVIClient(RTVIConfig);
|
this.rtviClient = new RTVIClient(RTVIConfig);
|
||||||
this.setupTrackListeners();
|
this.setupTrackListeners();
|
||||||
|
|
||||||
@@ -223,8 +231,13 @@ class WebsocketClientApp {
|
|||||||
try {
|
try {
|
||||||
await this.rtviClient.disconnect();
|
await this.rtviClient.disconnect();
|
||||||
this.rtviClient = null;
|
this.rtviClient = null;
|
||||||
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
|
if (
|
||||||
this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop());
|
this.botAudio.srcObject &&
|
||||||
|
'getAudioTracks' in this.botAudio.srcObject
|
||||||
|
) {
|
||||||
|
this.botAudio.srcObject
|
||||||
|
.getAudioTracks()
|
||||||
|
.forEach((track) => track.stop());
|
||||||
this.botAudio.srcObject = null;
|
this.botAudio.srcObject = null;
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -232,7 +245,6 @@ class WebsocketClientApp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
declare global {
|
declare global {
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RTVI Client Implementation
|
* Pipecat Client Implementation
|
||||||
*
|
*
|
||||||
* This client connects to an RTVI-compatible bot server using WebSocket.
|
* This client connects to an RTVI-compatible bot server using WebSocket.
|
||||||
*
|
*
|
||||||
@@ -14,16 +14,14 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import {
|
import {
|
||||||
RTVIClient,
|
PipecatClient,
|
||||||
RTVIClientOptions,
|
PipecatClientOptions,
|
||||||
RTVIEvent,
|
RTVIEvent,
|
||||||
} from '@pipecat-ai/client-js';
|
} from '@pipecat-ai/client-js';
|
||||||
import {
|
import { WebSocketTransport } from '@pipecat-ai/websocket-transport';
|
||||||
WebSocketTransport
|
|
||||||
} from "@pipecat-ai/websocket-transport";
|
|
||||||
|
|
||||||
class WebsocketClientApp {
|
class WebsocketClientApp {
|
||||||
private rtviClient: RTVIClient | null = null;
|
private pcClient: PipecatClient | null = null;
|
||||||
private connectBtn: HTMLButtonElement | null = null;
|
private connectBtn: HTMLButtonElement | null = null;
|
||||||
private disconnectBtn: HTMLButtonElement | null = null;
|
private disconnectBtn: HTMLButtonElement | null = null;
|
||||||
private statusSpan: HTMLElement | null = null;
|
private statusSpan: HTMLElement | null = null;
|
||||||
@@ -31,7 +29,7 @@ class WebsocketClientApp {
|
|||||||
private botAudio: HTMLAudioElement;
|
private botAudio: HTMLAudioElement;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
console.log("WebsocketClientApp");
|
console.log('WebsocketClientApp');
|
||||||
this.botAudio = document.createElement('audio');
|
this.botAudio = document.createElement('audio');
|
||||||
this.botAudio.autoplay = true;
|
this.botAudio.autoplay = true;
|
||||||
//this.botAudio.playsInline = true;
|
//this.botAudio.playsInline = true;
|
||||||
@@ -45,8 +43,12 @@ class WebsocketClientApp {
|
|||||||
* Set up references to DOM elements and create necessary media elements
|
* Set up references to DOM elements and create necessary media elements
|
||||||
*/
|
*/
|
||||||
private setupDOMElements(): void {
|
private setupDOMElements(): void {
|
||||||
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
|
this.connectBtn = document.getElementById(
|
||||||
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
|
'connect-btn'
|
||||||
|
) as HTMLButtonElement;
|
||||||
|
this.disconnectBtn = document.getElementById(
|
||||||
|
'disconnect-btn'
|
||||||
|
) as HTMLButtonElement;
|
||||||
this.statusSpan = document.getElementById('connection-status');
|
this.statusSpan = document.getElementById('connection-status');
|
||||||
this.debugLog = document.getElementById('debug-log');
|
this.debugLog = document.getElementById('debug-log');
|
||||||
}
|
}
|
||||||
@@ -91,8 +93,8 @@ class WebsocketClientApp {
|
|||||||
* This is called when the bot is ready or when the transport state changes to ready
|
* This is called when the bot is ready or when the transport state changes to ready
|
||||||
*/
|
*/
|
||||||
setupMediaTracks() {
|
setupMediaTracks() {
|
||||||
if (!this.rtviClient) return;
|
if (!this.pcClient) return;
|
||||||
const tracks = this.rtviClient.tracks();
|
const tracks = this.pcClient.tracks();
|
||||||
if (tracks.bot?.audio) {
|
if (tracks.bot?.audio) {
|
||||||
this.setupAudioTrack(tracks.bot.audio);
|
this.setupAudioTrack(tracks.bot.audio);
|
||||||
}
|
}
|
||||||
@@ -103,10 +105,10 @@ class WebsocketClientApp {
|
|||||||
* This handles new tracks being added during the session
|
* This handles new tracks being added during the session
|
||||||
*/
|
*/
|
||||||
setupTrackListeners() {
|
setupTrackListeners() {
|
||||||
if (!this.rtviClient) return;
|
if (!this.pcClient) return;
|
||||||
|
|
||||||
// Listen for new tracks starting
|
// Listen for new tracks starting
|
||||||
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
|
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
|
||||||
// Only handle non-local (bot) tracks
|
// Only handle non-local (bot) tracks
|
||||||
if (!participant?.local && track.kind === 'audio') {
|
if (!participant?.local && track.kind === 'audio') {
|
||||||
this.setupAudioTrack(track);
|
this.setupAudioTrack(track);
|
||||||
@@ -114,8 +116,10 @@ class WebsocketClientApp {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Listen for tracks stopping
|
// Listen for tracks stopping
|
||||||
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
|
||||||
this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`);
|
this.log(
|
||||||
|
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
|
||||||
|
);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -125,7 +129,10 @@ class WebsocketClientApp {
|
|||||||
*/
|
*/
|
||||||
private setupAudioTrack(track: MediaStreamTrack): void {
|
private setupAudioTrack(track: MediaStreamTrack): void {
|
||||||
this.log('Setting up audio track');
|
this.log('Setting up audio track');
|
||||||
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
|
if (
|
||||||
|
this.botAudio.srcObject &&
|
||||||
|
'getAudioTracks' in this.botAudio.srcObject
|
||||||
|
) {
|
||||||
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
|
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
|
||||||
if (oldTrack?.id === track.id) return;
|
if (oldTrack?.id === track.id) return;
|
||||||
}
|
}
|
||||||
@@ -134,21 +141,15 @@ class WebsocketClientApp {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize and connect to the bot
|
* Initialize and connect to the bot
|
||||||
* This sets up the RTVI client, initializes devices, and establishes the connection
|
* This sets up the Pipecat client, initializes devices, and establishes the connection
|
||||||
*/
|
*/
|
||||||
public async connect(): Promise<void> {
|
public async connect(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
|
|
||||||
//const transport = new DailyTransport();
|
//const transport = new DailyTransport();
|
||||||
const transport = new WebSocketTransport();
|
const PipecatConfig: PipecatClientOptions = {
|
||||||
const RTVIConfig: RTVIClientOptions = {
|
transport: new WebSocketTransport(),
|
||||||
transport,
|
|
||||||
params: {
|
|
||||||
// The baseURL and endpoint of your bot server that the client will connect to
|
|
||||||
baseUrl: 'http://localhost:7860',
|
|
||||||
endpoints: { connect: '/connect' },
|
|
||||||
},
|
|
||||||
enableMic: true,
|
enableMic: true,
|
||||||
enableCam: false,
|
enableCam: false,
|
||||||
callbacks: {
|
callbacks: {
|
||||||
@@ -176,15 +177,20 @@ class WebsocketClientApp {
|
|||||||
onMessageError: (error) => console.error('Message error:', error),
|
onMessageError: (error) => console.error('Message error:', error),
|
||||||
onError: (error) => console.error('Error:', error),
|
onError: (error) => console.error('Error:', error),
|
||||||
},
|
},
|
||||||
}
|
};
|
||||||
this.rtviClient = new RTVIClient(RTVIConfig);
|
this.pcClient = new PipecatClient(PipecatConfig);
|
||||||
|
// @ts-ignore
|
||||||
|
window.pcClient = this.pcClient; // Expose for debugging
|
||||||
this.setupTrackListeners();
|
this.setupTrackListeners();
|
||||||
|
|
||||||
this.log('Initializing devices...');
|
this.log('Initializing devices...');
|
||||||
await this.rtviClient.initDevices();
|
await this.pcClient.initDevices();
|
||||||
|
|
||||||
this.log('Connecting to bot...');
|
this.log('Connecting to bot...');
|
||||||
await this.rtviClient.connect();
|
await this.pcClient.connect({
|
||||||
|
// The baseURL and endpoint of your bot server that the client will connect to
|
||||||
|
endpoint: 'http://localhost:7860/connect',
|
||||||
|
});
|
||||||
|
|
||||||
const timeTaken = Date.now() - startTime;
|
const timeTaken = Date.now() - startTime;
|
||||||
this.log(`Connection complete, timeTaken: ${timeTaken}`);
|
this.log(`Connection complete, timeTaken: ${timeTaken}`);
|
||||||
@@ -192,9 +198,9 @@ class WebsocketClientApp {
|
|||||||
this.log(`Error connecting: ${(error as Error).message}`);
|
this.log(`Error connecting: ${(error as Error).message}`);
|
||||||
this.updateStatus('Error');
|
this.updateStatus('Error');
|
||||||
// Clean up if there's an error
|
// Clean up if there's an error
|
||||||
if (this.rtviClient) {
|
if (this.pcClient) {
|
||||||
try {
|
try {
|
||||||
await this.rtviClient.disconnect();
|
await this.pcClient.disconnect();
|
||||||
} catch (disconnectError) {
|
} catch (disconnectError) {
|
||||||
this.log(`Error during disconnect: ${disconnectError}`);
|
this.log(`Error during disconnect: ${disconnectError}`);
|
||||||
}
|
}
|
||||||
@@ -206,12 +212,17 @@ class WebsocketClientApp {
|
|||||||
* Disconnect from the bot and clean up media resources
|
* Disconnect from the bot and clean up media resources
|
||||||
*/
|
*/
|
||||||
public async disconnect(): Promise<void> {
|
public async disconnect(): Promise<void> {
|
||||||
if (this.rtviClient) {
|
if (this.pcClient) {
|
||||||
try {
|
try {
|
||||||
await this.rtviClient.disconnect();
|
await this.pcClient.disconnect();
|
||||||
this.rtviClient = null;
|
this.pcClient = null;
|
||||||
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
|
if (
|
||||||
this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop());
|
this.botAudio.srcObject &&
|
||||||
|
'getAudioTracks' in this.botAudio.srcObject
|
||||||
|
) {
|
||||||
|
this.botAudio.srcObject
|
||||||
|
.getAudioTracks()
|
||||||
|
.forEach((track) => track.stop());
|
||||||
this.botAudio.srcObject = null;
|
this.botAudio.srcObject = null;
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -219,7 +230,6 @@ class WebsocketClientApp {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
declare global {
|
declare global {
|
||||||
|
|||||||
@@ -1,16 +1,26 @@
|
|||||||
import { useEffect, useCallback } from 'react';
|
import { useEffect, useCallback } from 'react';
|
||||||
import {
|
import {
|
||||||
useRTVIClient,
|
usePipecatClient,
|
||||||
useRTVIClientTransportState,
|
usePipecatClientTransportState,
|
||||||
} from '@pipecat-ai/client-react';
|
} from '@pipecat-ai/client-react';
|
||||||
import { CONNECTION_STATES } from '@/constants/gameConstants';
|
import { CONNECTION_STATES } from '@/constants/gameConstants';
|
||||||
|
import { useConfigurationSettings } from '@/contexts/Configuration';
|
||||||
|
|
||||||
|
// Get the API base URL from environment variables
|
||||||
|
// Default to "/api" if not specified
|
||||||
|
// "/api" is the default for Next.js API routes and used
|
||||||
|
// for the Pipecat Cloud deployed agent
|
||||||
|
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
|
||||||
|
|
||||||
|
console.log('Using API base URL:', API_BASE_URL);
|
||||||
|
|
||||||
export function useConnectionState(
|
export function useConnectionState(
|
||||||
onConnected?: () => void,
|
onConnected?: () => void,
|
||||||
onDisconnected?: () => void
|
onDisconnected?: () => void
|
||||||
) {
|
) {
|
||||||
const client = useRTVIClient();
|
const client = usePipecatClient();
|
||||||
const transportState = useRTVIClientTransportState();
|
const transportState = usePipecatClientTransportState();
|
||||||
|
const config = useConfigurationSettings();
|
||||||
|
|
||||||
const isConnected = CONNECTION_STATES.ACTIVE.includes(transportState);
|
const isConnected = CONNECTION_STATES.ACTIVE.includes(transportState);
|
||||||
const isConnecting = CONNECTION_STATES.CONNECTING.includes(transportState);
|
const isConnecting = CONNECTION_STATES.CONNECTING.includes(transportState);
|
||||||
@@ -35,12 +45,17 @@ export function useConnectionState(
|
|||||||
if (isConnected) {
|
if (isConnected) {
|
||||||
await client.disconnect();
|
await client.disconnect();
|
||||||
} else {
|
} else {
|
||||||
await client.connect();
|
await client.connect({
|
||||||
|
endpoint: `${API_BASE_URL}/connect`,
|
||||||
|
requestData: {
|
||||||
|
personality: config.personality,
|
||||||
|
},
|
||||||
|
});
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Connection error:', error);
|
console.error('Connection error:', error);
|
||||||
}
|
}
|
||||||
}, [client, isConnected]);
|
}, [client, config, isConnected]);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
isConnected,
|
isConnected,
|
||||||
|
|||||||
@@ -1,15 +1,15 @@
|
|||||||
import { ConfigurationProvider } from "@/contexts/Configuration";
|
import { ConfigurationProvider } from '@/contexts/Configuration';
|
||||||
import { RTVIProvider } from "@/providers/RTVIProvider";
|
import { PipecatProvider } from '@/providers/PipecatProvider';
|
||||||
import { RTVIClientAudio } from "@pipecat-ai/client-react";
|
import { PipecatClientAudio } from '@pipecat-ai/client-react';
|
||||||
import type { AppProps } from "next/app";
|
import type { AppProps } from 'next/app';
|
||||||
import { Nunito } from "next/font/google";
|
import { Nunito } from 'next/font/google';
|
||||||
import Head from "next/head";
|
import Head from 'next/head';
|
||||||
import "../styles/globals.css";
|
import '../styles/globals.css';
|
||||||
|
|
||||||
const nunito = Nunito({
|
const nunito = Nunito({
|
||||||
subsets: ["latin"],
|
subsets: ['latin'],
|
||||||
display: "swap",
|
display: 'swap',
|
||||||
variable: "--font-sans",
|
variable: '--font-sans',
|
||||||
});
|
});
|
||||||
|
|
||||||
export default function App({ Component, pageProps }: AppProps) {
|
export default function App({ Component, pageProps }: AppProps) {
|
||||||
@@ -21,10 +21,10 @@ export default function App({ Component, pageProps }: AppProps) {
|
|||||||
</Head>
|
</Head>
|
||||||
<main className={`${nunito.variable}`}>
|
<main className={`${nunito.variable}`}>
|
||||||
<ConfigurationProvider>
|
<ConfigurationProvider>
|
||||||
<RTVIProvider>
|
<PipecatProvider>
|
||||||
<RTVIClientAudio />
|
<PipecatClientAudio />
|
||||||
<Component {...pageProps} />
|
<Component {...pageProps} />
|
||||||
</RTVIProvider>
|
</PipecatProvider>
|
||||||
</ConfigurationProvider>
|
</ConfigurationProvider>
|
||||||
</main>
|
</main>
|
||||||
</>
|
</>
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
import type { NextApiRequest, NextApiResponse } from "next";
|
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||||
|
|
||||||
export default async function handler(
|
export default async function handler(
|
||||||
req: NextApiRequest,
|
req: NextApiRequest,
|
||||||
res: NextApiResponse
|
res: NextApiResponse
|
||||||
) {
|
) {
|
||||||
if (req.method !== "POST") {
|
if (req.method !== 'POST') {
|
||||||
return res.status(405).json({ error: "Method not allowed" });
|
return res.status(405).json({ error: 'Method not allowed' });
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -15,16 +15,16 @@ export default async function handler(
|
|||||||
if (!personality) {
|
if (!personality) {
|
||||||
return res
|
return res
|
||||||
.status(400)
|
.status(400)
|
||||||
.json({ error: "Missing required configuration parameters" });
|
.json({ error: 'Missing required configuration parameters' });
|
||||||
}
|
}
|
||||||
|
|
||||||
const response = await fetch(
|
const response = await fetch(
|
||||||
`https://api.pipecat.daily.co/v1/public/${process.env.AGENT_NAME}/start`,
|
`https://api.pipecat.daily.co/v1/public/${process.env.AGENT_NAME}/start`,
|
||||||
{
|
{
|
||||||
method: "POST",
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
Authorization: `Bearer ${process.env.PIPECAT_CLOUD_API_KEY}`,
|
Authorization: `Bearer ${process.env.PIPECAT_CLOUD_API_KEY}`,
|
||||||
"Content-Type": "application/json",
|
'Content-Type': 'application/json',
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
createDailyRoom: true,
|
createDailyRoom: true,
|
||||||
@@ -37,15 +37,15 @@ export default async function handler(
|
|||||||
|
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
|
|
||||||
console.log("Response from API:", JSON.stringify(data, null, 2));
|
console.log('Response from API:', JSON.stringify(data, null, 2));
|
||||||
|
|
||||||
// Transform the response to match what RTVI client expects
|
// Transform the response to match what Pipecat client expects
|
||||||
return res.status(200).json({
|
return res.status(200).json({
|
||||||
room_url: data.dailyRoom,
|
room_url: data.dailyRoom,
|
||||||
token: data.dailyToken,
|
token: data.dailyToken,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error starting agent:", error);
|
console.error('Error starting agent:', error);
|
||||||
return res.status(500).json({ error: "Failed to start agent" });
|
return res.status(500).json({ error: 'Failed to start agent' });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
'use client';
|
||||||
|
|
||||||
|
import { PipecatClient } from '@pipecat-ai/client-js';
|
||||||
|
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
||||||
|
import { PipecatClientProvider } from '@pipecat-ai/client-react';
|
||||||
|
import { PropsWithChildren, useEffect, useState, useRef } from 'react';
|
||||||
|
|
||||||
|
export function PipecatProvider({ children }: PropsWithChildren) {
|
||||||
|
const [client, setClient] = useState<PipecatClient | null>(null);
|
||||||
|
const clientCreated = useRef(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
// Only create the client once
|
||||||
|
if (clientCreated.current) return;
|
||||||
|
|
||||||
|
const pcClient = new PipecatClient({
|
||||||
|
transport: new DailyTransport(),
|
||||||
|
enableMic: true,
|
||||||
|
enableCam: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
setClient(pcClient);
|
||||||
|
clientCreated.current = true;
|
||||||
|
|
||||||
|
// Cleanup when component unmounts
|
||||||
|
return () => {
|
||||||
|
if (pcClient) {
|
||||||
|
pcClient.disconnect().catch((err) => {
|
||||||
|
console.error('Error disconnecting client:', err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
clientCreated.current = false;
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
if (!client) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<PipecatClientProvider client={client}>{children}</PipecatClientProvider>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
"use client";
|
|
||||||
|
|
||||||
import { RTVIClient } from "@pipecat-ai/client-js";
|
|
||||||
import { DailyTransport } from "@pipecat-ai/daily-transport";
|
|
||||||
import { RTVIClientProvider } from "@pipecat-ai/client-react";
|
|
||||||
import { PropsWithChildren, useEffect, useState, useRef } from "react";
|
|
||||||
import { useConfigurationSettings } from "@/contexts/Configuration";
|
|
||||||
|
|
||||||
// Get the API base URL from environment variables
|
|
||||||
// Default to "/api" if not specified
|
|
||||||
// "/api" is the default for Next.js API routes and used
|
|
||||||
// for the Pipecat Cloud deployed agent
|
|
||||||
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || "/api";
|
|
||||||
|
|
||||||
console.log("Using API base URL:", API_BASE_URL);
|
|
||||||
|
|
||||||
export function RTVIProvider({ children }: PropsWithChildren) {
|
|
||||||
const [client, setClient] = useState<RTVIClient | null>(null);
|
|
||||||
const config = useConfigurationSettings();
|
|
||||||
const clientCreated = useRef(false);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
// Only create the client once
|
|
||||||
if (clientCreated.current) return;
|
|
||||||
|
|
||||||
const transport = new DailyTransport();
|
|
||||||
|
|
||||||
const rtviClient = new RTVIClient({
|
|
||||||
transport,
|
|
||||||
params: {
|
|
||||||
baseUrl: API_BASE_URL,
|
|
||||||
endpoints: {
|
|
||||||
connect: "/connect",
|
|
||||||
},
|
|
||||||
requestData: {
|
|
||||||
personality: config.personality,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
enableMic: true,
|
|
||||||
enableCam: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
setClient(rtviClient);
|
|
||||||
clientCreated.current = true;
|
|
||||||
|
|
||||||
// Cleanup when component unmounts
|
|
||||||
return () => {
|
|
||||||
if (rtviClient) {
|
|
||||||
rtviClient.disconnect().catch((err) => {
|
|
||||||
console.error("Error disconnecting client:", err);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
clientCreated.current = false;
|
|
||||||
};
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
// Update the connectParams when config changes
|
|
||||||
useEffect(() => {
|
|
||||||
if (!client) return;
|
|
||||||
|
|
||||||
// Update the connect params without recreating the client
|
|
||||||
client.params.requestData = {
|
|
||||||
personality: config.personality,
|
|
||||||
};
|
|
||||||
}, [client, config.personality]);
|
|
||||||
|
|
||||||
if (!client) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
|
|
||||||
}
|
|
||||||
@@ -4,6 +4,7 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
@@ -198,16 +199,15 @@ async def bot(args: DailySessionArguments):
|
|||||||
|
|
||||||
|
|
||||||
# Local development
|
# Local development
|
||||||
async def local_daily():
|
async def local_daily(args: DailySessionArguments):
|
||||||
"""Daily transport for local development."""
|
"""Daily transport for local development."""
|
||||||
from runner import configure
|
# from runner import configure
|
||||||
|
|
||||||
try:
|
try:
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
(room_url, token) = await configure(session)
|
|
||||||
transport = DailyTransport(
|
transport = DailyTransport(
|
||||||
room_url,
|
room_url=args.room_url,
|
||||||
token,
|
token=args.token,
|
||||||
bot_name="Bot",
|
bot_name="Bot",
|
||||||
params=DailyParams(
|
params=DailyParams(
|
||||||
audio_in_enabled=True,
|
audio_in_enabled=True,
|
||||||
@@ -217,7 +217,7 @@ async def local_daily():
|
|||||||
)
|
)
|
||||||
|
|
||||||
test_config = {
|
test_config = {
|
||||||
"personality": "witty",
|
"personality": args.personality,
|
||||||
}
|
}
|
||||||
|
|
||||||
await main(transport, test_config)
|
await main(transport, test_config)
|
||||||
@@ -227,7 +227,24 @@ async def local_daily():
|
|||||||
|
|
||||||
# Local development entry point
|
# Local development entry point
|
||||||
if LOCAL_RUN and __name__ == "__main__":
|
if LOCAL_RUN and __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Run the Word Wrangler bot in local development mode"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-u", "--room-url", type=str, default=os.getenv("DAILY_SAMPLE_ROOM_URL", "")
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-t", "--token", type=str, default=os.getenv("DAILY_SAMPLE_ROOM_TOKEN", None)
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-p",
|
||||||
|
"--personality",
|
||||||
|
default="witty",
|
||||||
|
choices=["friendly", "professional", "enthusiastic", "thoughtful", "witty"],
|
||||||
|
help="Personality preset for the bot (friendly, professional, enthusiastic, thoughtful, witty)",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
try:
|
try:
|
||||||
asyncio.run(local_daily())
|
asyncio.run(local_daily(args))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception(f"Failed to run in local mode: {e}")
|
logger.exception(f"Failed to run in local mode: {e}")
|
||||||
|
|||||||
@@ -160,14 +160,15 @@ async def rtvi_connect(request: Request) -> Dict[Any, Any]:
|
|||||||
Raises:
|
Raises:
|
||||||
HTTPException: If room creation, token generation, or bot startup fails
|
HTTPException: If room creation, token generation, or bot startup fails
|
||||||
"""
|
"""
|
||||||
print("Creating room for RTVI connection")
|
body = await request.json()
|
||||||
|
print("Creating room for RTVI connection", body)
|
||||||
room_url, token = await create_room_and_token()
|
room_url, token = await create_room_and_token()
|
||||||
print(f"Room URL: {room_url}")
|
print(f"Room URL: {room_url}")
|
||||||
|
|
||||||
# Start the bot process
|
# Start the bot process
|
||||||
try:
|
try:
|
||||||
proc = subprocess.Popen(
|
proc = subprocess.Popen(
|
||||||
[f"python3 -m bot -u {room_url} -t {token}"],
|
[f"python3 -m bot -u {room_url} -t {token} -p {body.get('personality', 'witty')}"],
|
||||||
shell=True,
|
shell=True,
|
||||||
bufsize=1,
|
bufsize=1,
|
||||||
cwd=os.path.dirname(os.path.abspath(__file__)),
|
cwd=os.path.dirname(os.path.abspath(__file__)),
|
||||||
|
|||||||
Reference in New Issue
Block a user