/** * Copyright (c) 2024–2025, Daily * * SPDX-License-Identifier: BSD 2-Clause License */ /** * RTVI Client Implementation * * This client connects to an RTVI-compatible bot server using WebSocket. * * Requirements: * - A running RTVI bot server (defaults to http://localhost:7860) */ import { RTVIClient, RTVIClientOptions, RTVIEvent, } from '@pipecat-ai/client-js'; import { ProtobufFrameSerializer, WebSocketTransport } from "@pipecat-ai/websocket-transport"; class RecordingSerializer extends ProtobufFrameSerializer { private lastTimestamp: number | null = null; private recordingAudioToSend: boolean = false; private _recordedAudio: { data: ArrayBuffer; delay: number }[] = []; public startRecording() { this.recordingAudioToSend = true; this._recordedAudio = []; this.lastTimestamp = null; } public stopRecording() { this.recordingAudioToSend = false; } // @ts-ignore serializeAudio(data: ArrayBuffer, sampleRate: number, numChannels: number): Uint8Array | null { if (this.recordingAudioToSend) { const now = Date.now(); // Compute delay since last packet const delay = this.lastTimestamp ? now - this.lastTimestamp : 0; this.lastTimestamp = now; // Save audio chunk and delay this._recordedAudio.push({ data, delay }); return null; } else { return super.serializeAudio(data, sampleRate, numChannels); } } public get recordedAudio() { return this._recordedAudio } } class WebsocketClientApp { private ENABLE_RECORDING_MODE = false private RECORDING_TIME_MS = 10000 private rtviClient: RTVIClient | null = null; private connectBtn: HTMLButtonElement | null = null; private disconnectBtn: HTMLButtonElement | null = null; private statusSpan: HTMLElement | null = null; private debugLog: HTMLElement | null = null; private botAudio: HTMLAudioElement; private declare websocketTransport: WebSocketTransport; private sendRecordedAudio: boolean = false private declare recordingSerializer: RecordingSerializer; private playBtn: HTMLButtonElement | null = null; private stopBtn: HTMLButtonElement | null = null; constructor() { this.botAudio = document.createElement('audio'); this.botAudio.autoplay = true; //this.botAudio.playsInline = true; document.body.appendChild(this.botAudio); this.setupDOMElements(); this.setupEventListeners(); } /** * Set up references to DOM elements and create necessary media elements */ private setupDOMElements(): void { this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement; this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement; this.statusSpan = document.getElementById('connection-status'); this.debugLog = document.getElementById('debug-log'); this.playBtn = document.getElementById('play-btn') as HTMLButtonElement; this.stopBtn = document.getElementById('stop-btn') as HTMLButtonElement; } /** * Set up event listeners for connect/disconnect buttons */ private setupEventListeners(): void { this.connectBtn?.addEventListener('click', () => this.connect()); this.disconnectBtn?.addEventListener('click', () => this.disconnect()); this.playBtn?.addEventListener('click', () => this.startSendingRecordedAudio()); this.stopBtn?.addEventListener('click', () => this.stopSendingRecordedAudio()); } /** * Add a timestamped message to the debug log */ private log(message: string): void { if (!this.debugLog) return; const entry = document.createElement('div'); entry.textContent = `${new Date().toISOString()} - ${message}`; if (message.startsWith('User: ')) { entry.style.color = '#2196F3'; } else if (message.startsWith('Bot: ')) { entry.style.color = '#4CAF50'; } this.debugLog.appendChild(entry); this.debugLog.scrollTop = this.debugLog.scrollHeight; console.log(message); } /** * Update the connection status display */ private updateStatus(status: string): void { if (this.statusSpan) { this.statusSpan.textContent = status; } this.log(`Status: ${status}`); } /** * Check for available media tracks and set them up if present * This is called when the bot is ready or when the transport state changes to ready */ setupMediaTracks() { if (!this.rtviClient) return; const tracks = this.rtviClient.tracks(); if (tracks.bot?.audio) { this.setupAudioTrack(tracks.bot.audio); } } /** * Set up listeners for track events (start/stop) * This handles new tracks being added during the session */ setupTrackListeners() { if (!this.rtviClient) return; // Listen for new tracks starting this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => { // Only handle non-local (bot) tracks if (!participant?.local && track.kind === 'audio') { this.setupAudioTrack(track); } }); // Listen for tracks stopping this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => { this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`); }); } /** * Set up an audio track for playback * Handles both initial setup and track updates */ private setupAudioTrack(track: MediaStreamTrack): void { this.log('Setting up audio track'); if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) { const oldTrack = this.botAudio.srcObject.getAudioTracks()[0]; if (oldTrack?.id === track.id) return; } this.botAudio.srcObject = new MediaStream([track]); } /** * Initialize and connect to the bot * This sets up the RTVI client, initializes devices, and establishes the connection */ public async connect(): Promise { try { const startTime = Date.now(); this.recordingSerializer = new RecordingSerializer() const transport = this.ENABLE_RECORDING_MODE ? new WebSocketTransport({ serializer: this.recordingSerializer, recorderSampleRate: 8000, playerSampleRate:8000 }) : new WebSocketTransport({ serializer: new ProtobufFrameSerializer(), recorderSampleRate: 8000, playerSampleRate:8000 }); this.websocketTransport = transport const RTVIConfig: RTVIClientOptions = { transport, params: { // The baseURL and endpoint of your bot server that the client will connect to baseUrl: 'http://localhost:7860', endpoints: { connect: '/connect' }, }, enableMic: true, enableCam: false, callbacks: { onConnected: () => { this.updateStatus('Connected'); if (this.connectBtn) this.connectBtn.disabled = true; if (this.disconnectBtn) this.disconnectBtn.disabled = false; }, onDisconnected: () => { this.updateStatus('Disconnected'); if (this.connectBtn) this.connectBtn.disabled = false; if (this.disconnectBtn) this.disconnectBtn.disabled = true; this.log('Client disconnected'); }, onBotReady: (data) => { this.log(`Bot ready: ${JSON.stringify(data)}`); this.setupMediaTracks(); }, onUserTranscript: (data) => { if (data.final) { this.log(`User: ${data.text}`); } }, onBotTranscript: (data) => this.log(`Bot: ${data.text}`), onMessageError: (error) => console.error('Message error:', error), onError: (error) => console.error('Error:', error), }, } this.rtviClient = new RTVIClient(RTVIConfig); this.setupTrackListeners(); this.log('Initializing devices...'); await this.rtviClient.initDevices(); this.log('Connecting to bot...'); await this.rtviClient.connect(); const timeTaken = Date.now() - startTime; this.log(`Connection complete, timeTaken: ${timeTaken}`); if (this.ENABLE_RECORDING_MODE) { this.log(`Starting to recording the next ${(this.RECORDING_TIME_MS/1000)}s of audio`); this.recordingSerializer.startRecording() await this.sleep(this.RECORDING_TIME_MS) this.recordingSerializer.stopRecording() this.log("Recording stopped"); this.rtviClient.enableMic(false) this.startSendingRecordedAudio() } } catch (error) { this.log(`Error connecting: ${(error as Error).message}`); this.updateStatus('Error'); // Clean up if there's an error if (this.rtviClient) { try { await this.rtviClient.disconnect(); } catch (disconnectError) { this.log(`Error during disconnect: ${disconnectError}`); } } } } /** * Disconnect from the bot and clean up media resources */ public async disconnect(): Promise { if (this.rtviClient) { try { this.stopSendingRecordedAudio() await this.rtviClient.disconnect(); this.rtviClient = null; if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) { this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop()); this.botAudio.srcObject = null; } } catch (error) { this.log(`Error disconnecting: ${(error as Error).message}`); } } } private startSendingRecordedAudio() { this.sendRecordedAudio = true if (this.playBtn) this.playBtn.disabled = true; if (this.stopBtn) this.stopBtn.disabled = false; void this.replayAudio() } private stopSendingRecordedAudio() { if (this.stopBtn) this.stopBtn.disabled = true; if (this.playBtn) this.playBtn.disabled = false; this.sendRecordedAudio = false } private async replayAudio() { if (this.sendRecordedAudio) { this.log("Sending recorded audio") for (const chunk of this.recordingSerializer.recordedAudio) { await this.sleep(chunk.delay); this.websocketTransport.handleUserAudioStream(chunk.data); } const randomDelay = 1000 + Math.random() * (10000 - 500); await this.sleep(randomDelay); void this.replayAudio() } } private sleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } } declare global { interface Window { WebsocketClientApp: typeof WebsocketClientApp; } } window.addEventListener('DOMContentLoaded', () => { window.WebsocketClientApp = WebsocketClientApp; new WebsocketClientApp(); });