Merge pull request #784 from pipecat-ai/mb/simple-bot-client

Update the simple-chatbot demo to have JS and React clients

CHANGELOG.md

@@ -58,6 +58,9 @@ async def on_audio_data(processor, audio, sample_rate, num_channels):
 - Updated the `FireworksLLMService` to use the `OpenAILLMService`. Updated the
   default model to `accounts/fireworks/models/firefunction-v2`.
 
+- Updated the `simple-chatbot` example to include a Javascript and React client
+  example, using RTVI JS and React.
+
 ### Removed
 
 - Removed `AppFrame`. This was used as a special user custom frame, but there's

examples/simple-chatbot/.gitignore

@@ -1,161 +1,51 @@
-# Byte-compiled / optimized / DLL files
+# Python
 __pycache__/
 *.py[cod]
 *$py.class
-
-# C extensions
 *.so
-
-# Distribution / packaging
 .Python
 build/
-develop-eggs/
 dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
-MANIFEST
+.pytest_cache/
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
 .coverage
 .coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
 .env
 .venv
 env/
 venv/
 ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
 .mypy_cache/
 .dmypy.json
 dmypy.json
 
-# Pyre type checker
+# JavaScript/Node.js
-.pyre/
+node_modules/
+dist/
+dist-ssr/
+*.local
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
 
-# pytype static type analyzer
+# Logs
-.pytype/
+logs/
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
 
-# Cython debug symbols
+# Editor/IDE
-cython_debug/
+.vscode/*
+!.vscode/extensions.json
+.idea/
+*.swp
+*.swo
+.DS_Store
 
-# PyCharm
+# Project specific
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+runpod.toml
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-runpod.toml

examples/simple-chatbot/README.md

@@ -2,36 +2,79 @@
 
 <img src="image.png" width="420px">
 
-This app connects you to a chatbot powered by GPT-4, complete with animations generated by Stable Video Diffusion.
+This repository demonstrates a simple AI chatbot with real-time audio/video interaction, implemented in three different ways. The bot server remains the same, but you can connect to it using three different client approaches.
 
-See a video of it in action: https://x.com/kwindla/status/1778628911817183509
+## Three Ways to Connect
 
-And a quick video walkthrough of the code: https://www.loom.com/share/13df1967161f4d24ade054e7f8753416
+1. **Daily Prebuilt** (Simplest)
 
-ℹ️ The first time, things might take extra time to get started since VAD (Voice Activity Detection) model needs to be downloaded.
+   - Direct connection through a Daily Prebuilt room
+   - For demo purposes only; handy for quick testing
 
-## Get started
+2. **JavaScript**
 
-```python
+   - Basic implementation using RTVI JavaScript SDK
-python3 -m venv venv
+   - No framework dependencies
-source venv/bin/activate
+   - Good for learning the fundamentals
-pip install -r requirements.txt
 
-cp env.example .env # and add your credentials
+3. **React**
+   - Basic impelmentation using RTVI React SDK
+   - Demonstrates the basic client principles with RTVI React
+
+## Quick Start
+
+### First, start the bot server:
+
+1. Navigate to the server directory:
+   ```bash
+   cd server
+   ```
+2. Create and activate a virtual environment:
+   ```bash
+   python3 -m venv venv
+   source venv/bin/activate  # On Windows: venv\Scripts\activate
+   ```
+3. Install requirements:
+   ```bash
+   pip install -r requirements.txt
+   ```
+4. Copy env.example to .env and add your credentials
+
+5. Start the server:
+   ```bash
+   python server.py
+   ```
+
+### Next, connect using your preferred client app:
+
+- [Daily Prebuilt](examples/prebuilt/README.md)
+- [Vanilla JavaScript Guide](examples/javascript/README.md)
+- [React Guide](examples/react/README.md)
+
+## Important Note
+
+The bot server must be running for any of the client implementations to work. Start the server first before trying any of the client apps.
+
+## Requirements
+
+- Python 3.10+
+- Node.js 16+ (for JavaScript and React implementations)
+- Daily API key
+- OpenAI API key
+- Cartesia API key
+- Modern web browser with WebRTC support
+
+## Project Structure
 
 ```
-
+simple-chatbot-full-stack/
-## Run the server
+├── server/             # Bot server implementation
-
+│   ├── bot.py          # Bot logic and media handling
-```bash
+│   ├── runner.py       # Server runner utilities
-python server.py
+│   ├── server.py       # FastAPI server
-```
+│   └── requirements.txt
-
+└── examples/           # Client implementations
-Then, visit `http://localhost:7860/` in your browser to start a chatbot session.
+    ├── prebuilt/       # Daily Prebuilt connection
-
+    ├── javascript/     # JavaScript RTVI client
-## Build and test the Docker image
+    └── react/          # React RTVI client
-
-```
-docker build -t chatbot .
-docker run --env-file .env -p 7860:7860 chatbot
 ```

examples/simple-chatbot/examples/javascript/README.md

@@ -0,0 +1,27 @@
+# JavaScript Implementation
+
+Basic implementation using the RTVI JavaScript SDK.
+
+## Setup
+
+1. Run the bot server; see [README](../../README).
+
+2. Navigate to the `examples/javascript` directory:
+
+```bash
+cd examples/javascript
+```
+
+3. Install dependencies:
+
+```bash
+npm install
+```
+
+4. Run the client app:
+
+```
+npm run dev
+```
+
+5. Visit http://localhost:5173 in your browser.

examples/simple-chatbot/examples/javascript/index.html

@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>AI Chatbot</title>
+</head>
+
+<body>
+  <div class="container">
+    <div class="status-bar">
+      <div class="status">
+        Status: <span id="connection-status">Disconnected</span>
+      </div>
+      <div class="controls">
+        <button id="connect-btn">Connect</button>
+        <button id="disconnect-btn" disabled>Disconnect</button>
+      </div>
+    </div>
+
+    <div class="main-content">
+      <div class="bot-container">
+        <div id="bot-video-container">
+        </div>
+        <audio id="bot-audio" autoplay></audio>
+      </div>
+    </div>
+
+    <div class="debug-panel">
+      <h3>Debug Info</h3>
+      <div id="debug-log"></div>
+    </div>
+  </div>
+
+  <script type="module" src="/src/app.js"></script>
+  <link rel="stylesheet" href="/src/style.css">
+</body>
+
+</html>

examples/simple-chatbot/examples/javascript/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "client",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "dependencies": {
+    "@daily-co/realtime-ai-daily": "^0.2.1",
+    "realtime-ai": "^0.2.1"
+  },
+  "devDependencies": {
+    "vite": "^6.0.2"
+  }
+}

examples/simple-chatbot/examples/javascript/src/app.js

@@ -0,0 +1,289 @@
+/**
+ * Copyright (c) 2024, Daily
+ *
+ * SPDX-License-Identifier: BSD 2-Clause License
+ */
+
+/**
+ * RTVI Client Implementation
+ *
+ * This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
+ * It handles audio/video streaming and manages the connection lifecycle.
+ *
+ * Requirements:
+ * - A running RTVI bot server (defaults to http://localhost:7860)
+ * - The server must implement the /connect endpoint that returns Daily.co room credentials
+ * - Browser with WebRTC support
+ */
+
+import { RTVIClient, RTVIEvent } from 'realtime-ai';
+import { DailyTransport } from '@daily-co/realtime-ai-daily';
+
+/**
+ * ChatbotClient handles the connection and media management for a real-time
+ * voice and video interaction with an AI bot.
+ */
+class ChatbotClient {
+  constructor() {
+    // Initialize client state
+    this.rtviClient = null;
+    this.setupDOMElements();
+    this.setupEventListeners();
+  }
+
+  /**
+   * Set up references to DOM elements and create necessary media elements
+   */
+  setupDOMElements() {
+    // Get references to UI control elements
+    this.connectBtn = document.getElementById('connect-btn');
+    this.disconnectBtn = document.getElementById('disconnect-btn');
+    this.statusSpan = document.getElementById('connection-status');
+    this.debugLog = document.getElementById('debug-log');
+    this.botVideoContainer = document.getElementById('bot-video-container');
+
+    // Create an audio element for bot's voice output
+    this.botAudio = document.createElement('audio');
+    this.botAudio.autoplay = true;
+    this.botAudio.playsInline = true;
+    document.body.appendChild(this.botAudio);
+  }
+
+  /**
+   * Set up event listeners for connect/disconnect buttons
+   */
+  setupEventListeners() {
+    this.connectBtn.addEventListener('click', () => this.connect());
+    this.disconnectBtn.addEventListener('click', () => this.disconnect());
+  }
+
+  /**
+   * Add a timestamped message to the debug log
+   */
+  log(message) {
+    const entry = document.createElement('div');
+    entry.textContent = `${new Date().toISOString()} - ${message}`;
+    this.debugLog.appendChild(entry);
+    this.debugLog.scrollTop = this.debugLog.scrollHeight;
+    console.log(message);
+  }
+
+  /**
+   * Update the connection status display
+   */
+  updateStatus(status) {
+    this.statusSpan.textContent = status;
+    this.log(`Status: ${status}`);
+  }
+
+  /**
+   * Check for available media tracks and set them up if present
+   * This is called when the bot is ready or when the transport state changes to ready
+   */
+  setupMediaTracks() {
+    if (!this.rtviClient) return;
+
+    // Get current tracks from the client
+    const tracks = this.rtviClient.tracks();
+
+    // Set up any available bot tracks
+    if (tracks.bot?.audio) {
+      this.setupAudioTrack(tracks.bot.audio);
+    }
+    if (tracks.bot?.video) {
+      this.setupVideoTrack(tracks.bot.video);
+    }
+  }
+
+  /**
+   * Set up listeners for track events (start/stop)
+   * This handles new tracks being added during the session
+   */
+  setupTrackListeners() {
+    if (!this.rtviClient) return;
+
+    // Listen for new tracks starting
+    this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
+      // Only handle non-local (bot) tracks
+      if (!participant?.local) {
+        if (track.kind === 'audio') {
+          this.setupAudioTrack(track);
+        } else if (track.kind === 'video') {
+          this.setupVideoTrack(track);
+        }
+      }
+    });
+
+    // Listen for tracks stopping
+    this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
+      this.log(
+        `Track stopped event: ${track.kind} from ${
+          participant?.name || 'unknown'
+        }`
+      );
+    });
+  }
+
+  /**
+   * Set up an audio track for playback
+   * Handles both initial setup and track updates
+   */
+  setupAudioTrack(track) {
+    this.log('Setting up audio track');
+    // Check if we're already playing this track
+    if (this.botAudio.srcObject) {
+      const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
+      if (oldTrack?.id === track.id) return;
+    }
+    // Create a new MediaStream with the track and set it as the audio source
+    this.botAudio.srcObject = new MediaStream([track]);
+  }
+
+  /**
+   * Set up a video track for display
+   * Handles both initial setup and track updates
+   */
+  setupVideoTrack(track) {
+    this.log('Setting up video track');
+    const videoEl = document.createElement('video');
+    videoEl.autoplay = true;
+    videoEl.playsInline = true;
+    videoEl.muted = true;
+    videoEl.style.width = '100%';
+    videoEl.style.height = '100%';
+    videoEl.style.objectFit = 'cover';
+
+    // Check if we're already displaying this track
+    if (this.botVideoContainer.querySelector('video')?.srcObject) {
+      const oldTrack = this.botVideoContainer
+        .querySelector('video')
+        .srcObject.getVideoTracks()[0];
+      if (oldTrack?.id === track.id) return;
+    }
+
+    // Create a new MediaStream with the track and set it as the video source
+    videoEl.srcObject = new MediaStream([track]);
+    this.botVideoContainer.innerHTML = '';
+    this.botVideoContainer.appendChild(videoEl);
+  }
+
+  /**
+   * Initialize and connect to the bot
+   * This sets up the RTVI client, initializes devices, and establishes the connection
+   */
+  async connect() {
+    try {
+      // Create a new Daily transport for WebRTC communication
+      const transport = new DailyTransport();
+
+      // Initialize the RTVI client with our configuration
+      this.rtviClient = new RTVIClient({
+        transport,
+        params: {
+          // The baseURL and endpoint of your bot server that the client will connect to
+          baseUrl: 'http://localhost:7860',
+          endpoints: {
+            connect: '/connect',
+          },
+        },
+        enableMic: true, // Enable microphone for user input
+        enableCam: false,
+        callbacks: {
+          // Handle connection state changes
+          onConnected: () => {
+            this.updateStatus('Connected');
+            this.connectBtn.disabled = true;
+            this.disconnectBtn.disabled = false;
+            this.log('Client connected');
+          },
+          onDisconnected: () => {
+            this.updateStatus('Disconnected');
+            this.connectBtn.disabled = false;
+            this.disconnectBtn.disabled = true;
+            this.log('Client disconnected');
+          },
+          // Handle transport state changes
+          onTransportStateChanged: (state) => {
+            this.updateStatus(`Transport: ${state}`);
+            this.log(`Transport state changed: ${state}`);
+            if (state === 'ready') {
+              this.setupMediaTracks();
+            }
+          },
+          // Handle bot connection events
+          onBotConnected: (participant) => {
+            this.log(`Bot connected: ${JSON.stringify(participant)}`);
+          },
+          onBotDisconnected: (participant) => {
+            this.log(`Bot disconnected: ${JSON.stringify(participant)}`);
+          },
+          onBotReady: (data) => {
+            this.log(`Bot ready: ${JSON.stringify(data)}`);
+            this.setupMediaTracks();
+          },
+        },
+      });
+
+      // Set up listeners for media track events
+      this.setupTrackListeners();
+
+      // Initialize audio/video devices
+      this.log('Initializing devices...');
+      await this.rtviClient.initDevices();
+
+      // Connect to the bot
+      this.log('Connecting to bot...');
+      await this.rtviClient.connect();
+
+      this.log('Connection complete');
+    } catch (error) {
+      // Handle any errors during connection
+      this.log(`Error connecting: ${error.message}`);
+      this.log(`Error stack: ${error.stack}`);
+      this.updateStatus('Error');
+
+      // Clean up if there's an error
+      if (this.rtviClient) {
+        try {
+          await this.rtviClient.disconnect();
+        } catch (disconnectError) {
+          this.log(`Error during disconnect: ${disconnectError.message}`);
+        }
+      }
+    }
+  }
+
+  /**
+   * Disconnect from the bot and clean up media resources
+   */
+  async disconnect() {
+    if (this.rtviClient) {
+      try {
+        // Disconnect the RTVI client
+        await this.rtviClient.disconnect();
+        this.rtviClient = null;
+
+        // Clean up audio
+        if (this.botAudio.srcObject) {
+          this.botAudio.srcObject.getTracks().forEach((track) => track.stop());
+          this.botAudio.srcObject = null;
+        }
+
+        // Clean up video
+        if (this.botVideoContainer.querySelector('video')?.srcObject) {
+          const video = this.botVideoContainer.querySelector('video');
+          video.srcObject.getTracks().forEach((track) => track.stop());
+          video.srcObject = null;
+        }
+        this.botVideoContainer.innerHTML = '';
+      } catch (error) {
+        this.log(`Error disconnecting: ${error.message}`);
+      }
+    }
+  }
+}
+
+// Initialize the client when the page loads
+window.addEventListener('DOMContentLoaded', () => {
+  new ChatbotClient();
+});

examples/simple-chatbot/examples/javascript/src/style.css

@@ -0,0 +1,90 @@
+body {
+  margin: 0;
+  padding: 20px;
+  font-family: Arial, sans-serif;
+  background-color: #f0f0f0;
+}
+
+.container {
+  max-width: 1200px;
+  margin: 0 auto;
+}
+
+.status-bar {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  padding: 10px;
+  background-color: #fff;
+  border-radius: 8px;
+  margin-bottom: 20px;
+}
+
+.controls button {
+  padding: 8px 16px;
+  margin-left: 10px;
+  border: none;
+  border-radius: 4px;
+  cursor: pointer;
+}
+
+#connect-btn {
+  background-color: #4caf50;
+  color: white;
+}
+
+#disconnect-btn {
+  background-color: #f44336;
+  color: white;
+}
+
+button:disabled {
+  opacity: 0.5;
+  cursor: not-allowed;
+}
+
+.main-content {
+  background-color: #fff;
+  border-radius: 8px;
+  padding: 20px;
+  margin-bottom: 20px;
+}
+
+.bot-container {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+#bot-video-container {
+  width: 640px;
+  height: 360px;
+  background-color: #e0e0e0;
+  border-radius: 8px;
+  margin: 20px auto;
+  overflow: hidden;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+#bot-video-container video {
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+}
+
+.debug-panel {
+  background-color: #fff;
+  border-radius: 8px;
+  padding: 20px;
+}
+
+#debug-log {
+  height: 200px;
+  overflow-y: auto;
+  background-color: #f8f8f8;
+  padding: 10px;
+  border-radius: 4px;
+  font-family: monospace;
+}

examples/simple-chatbot/examples/prebuilt/README.md

@@ -0,0 +1,15 @@
+# Daily Prebuilt Connection
+
+The simplest way to connect to the chatbot using Daily's Prebuilt UI.
+
+1. Start the bot server
+
+```bash
+python server/server.py
+```
+
+2. Visit http://localhost:7860
+
+3. Allow microphone access when prompted
+
+4. Start talking with the bot

examples/simple-chatbot/examples/react/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

examples/simple-chatbot/examples/react/README.md

@@ -0,0 +1,27 @@
+# React Implementation
+
+Basic implementation using the RTVI React SDK.
+
+## Setup
+
+1. Run the bot server; see [README](../../README).
+
+2. Navigate to the `examples/react` directory:
+
+```bash
+cd examples/react
+```
+
+3. Install dependencies:
+
+```bash
+npm install
+```
+
+4. Run the client app:
+
+```
+npm run dev
+```
+
+5. Visit http://localhost:5173 in your browser.

examples/simple-chatbot/examples/react/eslint.config.js

@@ -0,0 +1,28 @@
+import js from '@eslint/js'
+import globals from 'globals'
+import reactHooks from 'eslint-plugin-react-hooks'
+import reactRefresh from 'eslint-plugin-react-refresh'
+import tseslint from 'typescript-eslint'
+
+export default tseslint.config(
+  { ignores: ['dist'] },
+  {
+    extends: [js.configs.recommended, ...tseslint.configs.recommended],
+    files: ['**/*.{ts,tsx}'],
+    languageOptions: {
+      ecmaVersion: 2020,
+      globals: globals.browser,
+    },
+    plugins: {
+      'react-hooks': reactHooks,
+      'react-refresh': reactRefresh,
+    },
+    rules: {
+      ...reactHooks.configs.recommended.rules,
+      'react-refresh/only-export-components': [
+        'warn',
+        { allowConstantExport: true },
+      ],
+    },
+  },
+)

examples/simple-chatbot/examples/react/index.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>RTVI React Client</title>
+</head>
+
+<body>
+  <div id="root"></div>
+  <script type="module" src="/src/main.tsx"></script>
+</body>
+
+</html>

examples/simple-chatbot/examples/react/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "react",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && vite build",
+    "lint": "eslint .",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@daily-co/realtime-ai-daily": "^0.2.1",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "realtime-ai": "^0.2.1",
+    "realtime-ai-react": "^0.2.1"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.15.0",
+    "@types/react": "^18.3.12",
+    "@types/react-dom": "^18.3.1",
+    "@vitejs/plugin-react": "^4.3.4",
+    "eslint": "^9.15.0",
+    "eslint-plugin-react-hooks": "^5.0.0",
+    "eslint-plugin-react-refresh": "^0.4.14",
+    "globals": "^15.12.0",
+    "typescript": "~5.6.2",
+    "typescript-eslint": "^8.15.0",
+    "vite": "^6.0.1"
+  }
+}

examples/simple-chatbot/examples/react/src/App.css

@@ -0,0 +1,82 @@
+body {
+  margin: 0;
+  padding: 20px;
+  font-family: Arial, sans-serif;
+  background-color: #f0f0f0;
+}
+
+.app {
+  max-width: 1200px;
+  margin: 0 auto;
+}
+
+.status-bar {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  padding: 10px;
+  background-color: #fff;
+  border-radius: 8px;
+  margin-bottom: 20px;
+}
+
+.controls button {
+  padding: 8px 16px;
+  margin-left: 10px;
+  border: none;
+  border-radius: 4px;
+  cursor: pointer;
+}
+
+button:disabled {
+  opacity: 0.5;
+  cursor: not-allowed;
+}
+
+.connect-btn {
+  background-color: #4caf50;
+  color: white;
+}
+
+.disconnect-btn {
+  background-color: #f44336;
+  color: white;
+}
+
+.main-content {
+  background-color: #fff;
+  border-radius: 8px;
+  padding: 20px;
+  margin-bottom: 20px;
+}
+
+.bot-container {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+.video-container {
+  width: 640px;
+  height: 360px;
+  background-color: #ddd;
+  margin-bottom: 20px;
+  border-radius: 8px;
+  overflow: hidden;
+}
+
+.video-container video {
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+}
+
+.mic-enabled {
+  background-color: #4caf50;
+  color: white;
+}
+
+.mic-disabled {
+  background-color: #f44336;
+  color: white;
+}

examples/simple-chatbot/examples/react/src/App.tsx

@@ -0,0 +1,51 @@
+import {
+  RTVIClientAudio,
+  RTVIClientVideo,
+  useRTVIClientTransportState,
+} from 'realtime-ai-react';
+import { RTVIProvider } from './providers/RTVIProvider';
+import { ConnectButton } from './components/ConnectButton';
+import { StatusDisplay } from './components/StatusDisplay';
+import { DebugDisplay } from './components/DebugDisplay';
+import './App.css';
+
+function BotVideo() {
+  const transportState = useRTVIClientTransportState();
+  const isConnected = transportState !== 'disconnected';
+
+  return (
+    <div className="bot-container">
+      <div className="video-container">
+        {isConnected && <RTVIClientVideo participant="bot" fit="cover" />}
+      </div>
+    </div>
+  );
+}
+
+function AppContent() {
+  return (
+    <div className="app">
+      <div className="status-bar">
+        <StatusDisplay />
+        <ConnectButton />
+      </div>
+
+      <div className="main-content">
+        <BotVideo />
+      </div>
+
+      <DebugDisplay />
+      <RTVIClientAudio />
+    </div>
+  );
+}
+
+function App() {
+  return (
+    <RTVIProvider>
+      <AppContent />
+    </RTVIProvider>
+  );
+}
+
+export default App;

examples/simple-chatbot/examples/react/src/components/ConnectButton.tsx

@@ -0,0 +1,37 @@
+import { useRTVIClient, useRTVIClientTransportState } from 'realtime-ai-react';
+
+export function ConnectButton() {
+  const client = useRTVIClient();
+  const transportState = useRTVIClientTransportState();
+  const isConnected = ['connected', 'ready'].includes(transportState);
+
+  const handleClick = async () => {
+    if (!client) {
+      console.error('RTVI client is not initialized');
+      return;
+    }
+
+    try {
+      if (isConnected) {
+        await client.disconnect();
+      } else {
+        await client.connect();
+      }
+    } catch (error) {
+      console.error('Connection error:', error);
+    }
+  };
+
+  return (
+    <div className="controls">
+      <button
+        className={isConnected ? 'disconnect-btn' : 'connect-btn'}
+        onClick={handleClick}
+        disabled={
+          !client || ['connecting', 'disconnecting'].includes(transportState)
+        }>
+        {isConnected ? 'Disconnect' : 'Connect'}
+      </button>
+    </div>
+  );
+}

examples/simple-chatbot/examples/react/src/components/DebugDisplay.css

@@ -0,0 +1,26 @@
+.debug-panel {
+  background-color: #fff;
+  border-radius: 8px;
+  padding: 20px;
+}
+
+.debug-panel h3 {
+  margin: 0 0 10px 0;
+  font-size: 16px;
+  font-weight: bold;
+}
+
+.debug-log {
+  height: 200px;
+  overflow-y: auto;
+  background-color: #f8f8f8;
+  padding: 10px;
+  border-radius: 4px;
+  font-family: monospace;
+  font-size: 12px;
+  line-height: 1.4;
+}
+
+.debug-log div {
+  margin-bottom: 4px;
+}

examples/simple-chatbot/examples/react/src/components/DebugDisplay.tsx

@@ -0,0 +1,106 @@
+import { useRef, useCallback } from 'react';
+import { Participant, RTVIEvent, TransportState } from 'realtime-ai';
+import { useRTVIClient, useRTVIClientEvent } from 'realtime-ai-react';
+import './DebugDisplay.css';
+
+export function DebugDisplay() {
+  const debugLogRef = useRef<HTMLDivElement>(null);
+  const client = useRTVIClient();
+
+  const log = useCallback((message: string) => {
+    if (!debugLogRef.current) return;
+
+    const entry = document.createElement('div');
+    entry.textContent = `${new Date().toISOString()} - ${message}`;
+    debugLogRef.current.appendChild(entry);
+    debugLogRef.current.scrollTop = debugLogRef.current.scrollHeight;
+  }, []);
+
+  // Log transport state changes
+  useRTVIClientEvent(
+    RTVIEvent.TransportStateChanged,
+    useCallback(
+      (state: TransportState) => {
+        log(`Transport state changed: ${state}`);
+      },
+      [log]
+    )
+  );
+
+  // Log bot connection events
+  useRTVIClientEvent(
+    RTVIEvent.BotConnected,
+    useCallback(
+      (participant?: Participant) => {
+        log(`Bot connected: ${JSON.stringify(participant)}`);
+      },
+      [log]
+    )
+  );
+
+  useRTVIClientEvent(
+    RTVIEvent.BotDisconnected,
+    useCallback(
+      (participant?: Participant) => {
+        log(`Bot disconnected: ${JSON.stringify(participant)}`);
+      },
+      [log]
+    )
+  );
+
+  // Log track events
+  useRTVIClientEvent(
+    RTVIEvent.TrackStarted,
+    useCallback(
+      (track: MediaStreamTrack, participant?: Participant) => {
+        log(
+          `Track started: ${track.kind} from ${participant?.name || 'unknown'}`
+        );
+      },
+      [log]
+    )
+  );
+
+  useRTVIClientEvent(
+    RTVIEvent.TrackedStopped,
+    useCallback(
+      (track: MediaStreamTrack, participant?: Participant) => {
+        log(
+          `Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
+        );
+      },
+      [log]
+    )
+  );
+
+  // Log bot ready state and check tracks
+  useRTVIClientEvent(
+    RTVIEvent.BotReady,
+    useCallback(() => {
+      log(`Bot ready`);
+
+      if (!client) return;
+
+      const tracks = client.tracks();
+      log(
+        `Available tracks: ${JSON.stringify({
+          local: {
+            audio: !!tracks.local.audio,
+            video: !!tracks.local.video,
+          },
+          bot: {
+            audio: !!tracks.bot?.audio,
+            video: !!tracks.bot?.video,
+          },
+        })}`
+      );
+    }, [client, log])
+  );
+
+  return (
+    <div className="debug-panel">
+      <h3>Debug Info</h3>
+      <div ref={debugLogRef} className="debug-log" />
+    </div>
+  );
+}

examples/simple-chatbot/examples/react/src/components/StatusDisplay.tsx

@@ -0,0 +1,11 @@
+import { useRTVIClientTransportState } from 'realtime-ai-react';
+
+export function StatusDisplay() {
+  const transportState = useRTVIClientTransportState();
+
+  return (
+    <div className="status">
+      Status: <span>{transportState}</span>
+    </div>
+  );
+}

examples/simple-chatbot/examples/react/src/main.tsx

@@ -0,0 +1,9 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App';
+
+ReactDOM.createRoot(document.getElementById('root')!).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+);

examples/simple-chatbot/examples/react/src/providers/RTVIProvider.tsx

@@ -0,0 +1,22 @@
+import { type PropsWithChildren } from 'react';
+import { RTVIClient } from 'realtime-ai';
+import { DailyTransport } from '@daily-co/realtime-ai-daily';
+import { RTVIClientProvider } from 'realtime-ai-react';
+
+const transport = new DailyTransport();
+
+const client = new RTVIClient({
+  transport,
+  params: {
+    baseUrl: 'http://localhost:7860',
+    endpoints: {
+      connect: '/connect',
+    },
+  },
+  enableMic: true,
+  enableCam: false,
+});
+
+export function RTVIProvider({ children }: PropsWithChildren) {
+  return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
+}

examples/simple-chatbot/examples/react/tsconfig.json

@@ -0,0 +1,25 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "useDefineForClassFields": true,
+    "lib": ["ES2020", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+
+    /* Bundler mode */
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx",
+
+    /* Linting */
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true
+  },
+  "include": ["src"],
+  "references": [{ "path": "./tsconfig.node.json" }]
+}

examples/simple-chatbot/examples/react/tsconfig.node.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "skipLibCheck": true,
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "allowSyntheticDefaultImports": true
+  },
+  "include": ["vite.config.ts"]
+}

examples/simple-chatbot/examples/react/vite.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from 'vite'
+import react from '@vitejs/plugin-react'
+
+// https://vite.dev/config/
+export default defineConfig({
+  plugins: [react()],
+})

examples/simple-chatbot/server.py

@@ -1,141 +0,0 @@
-#
-# Copyright (c) 2024, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-import aiohttp
-import os
-import argparse
-import subprocess
-
-from contextlib import asynccontextmanager
-
-from fastapi import FastAPI, Request, HTTPException
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse, RedirectResponse
-
-from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper, DailyRoomParams
-
-from dotenv import load_dotenv
-
-load_dotenv(override=True)
-
-MAX_BOTS_PER_ROOM = 1
-
-# Bot sub-process dict for status reporting and concurrency control
-bot_procs = {}
-
-daily_helpers = {}
-
-
-def cleanup():
-    # Clean up function, just to be extra safe
-    for entry in bot_procs.values():
-        proc = entry[0]
-        proc.terminate()
-        proc.wait()
-
-
-@asynccontextmanager
-async def lifespan(app: FastAPI):
-    aiohttp_session = aiohttp.ClientSession()
-    daily_helpers["rest"] = DailyRESTHelper(
-        daily_api_key=os.getenv("DAILY_API_KEY", ""),
-        daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
-        aiohttp_session=aiohttp_session,
-    )
-    yield
-    await aiohttp_session.close()
-    cleanup()
-
-
-app = FastAPI(lifespan=lifespan)
-
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-
-@app.get("/")
-async def start_agent(request: Request):
-    print(f"!!! Creating room")
-    room = await daily_helpers["rest"].create_room(DailyRoomParams())
-    print(f"!!! Room URL: {room.url}")
-    # Ensure the room property is present
-    if not room.url:
-        raise HTTPException(
-            status_code=500,
-            detail="Missing 'room' property in request data. Cannot start agent without a target room!",
-        )
-
-    # Check if there is already an existing process running in this room
-    num_bots_in_room = sum(
-        1 for proc in bot_procs.values() if proc[1] == room.url and proc[0].poll() is None
-    )
-    if num_bots_in_room >= MAX_BOTS_PER_ROOM:
-        raise HTTPException(status_code=500, detail=f"Max bot limited reach for room: {room.url}")
-
-    # Get the token for the room
-    token = await daily_helpers["rest"].get_token(room.url)
-
-    if not token:
-        raise HTTPException(status_code=500, detail=f"Failed to get token for room: {room.url}")
-
-    # Spawn a new agent, and join the user session
-    # Note: this is mostly for demonstration purposes (refer to 'deployment' in README)
-    try:
-        proc = subprocess.Popen(
-            [f"python3 -m bot -u {room.url} -t {token}"],
-            shell=True,
-            bufsize=1,
-            cwd=os.path.dirname(os.path.abspath(__file__)),
-        )
-        bot_procs[proc.pid] = (proc, room.url)
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}")
-
-    return RedirectResponse(room.url)
-
-
-@app.get("/status/{pid}")
-def get_status(pid: int):
-    # Look up the subprocess
-    proc = bot_procs.get(pid)
-
-    # If the subprocess doesn't exist, return an error
-    if not proc:
-        raise HTTPException(status_code=404, detail=f"Bot with process id: {pid} not found")
-
-    # Check the status of the subprocess
-    if proc[0].poll() is None:
-        status = "running"
-    else:
-        status = "finished"
-
-    return JSONResponse({"bot_id": pid, "status": status})
-
-
-if __name__ == "__main__":
-    import uvicorn
-
-    default_host = os.getenv("HOST", "0.0.0.0")
-    default_port = int(os.getenv("FAST_API_PORT", "7860"))
-
-    parser = argparse.ArgumentParser(description="Daily Storyteller FastAPI server")
-    parser.add_argument("--host", type=str, default=default_host, help="Host address")
-    parser.add_argument("--port", type=int, default=default_port, help="Port number")
-    parser.add_argument("--reload", action="store_true", help="Reload code on change")
-
-    config = parser.parse_args()
-
-    uvicorn.run(
-        "server:app",
-        host=config.host,
-        port=config.port,
-        reload=config.reload,
-    )

examples/simple-chatbot/server/README.md

@@ -0,0 +1,43 @@
+# Simple Chatbot Server
+
+A FastAPI server that manages bot instances and provides endpoints for both Daily Prebuilt and RTVI client connections.
+
+## Endpoints
+
+- `GET /` - Direct browser access, redirects to a Daily Prebuilt room
+- `POST /connect` - RTVI client connection endpoint
+- `GET /status/{pid}` - Get status of a specific bot process
+
+## Environment Variables
+
+Copy `env.example` to `.env` and configure:
+
+```ini
+DAILY_API_KEY=           # Your Daily API key
+DAILY_API_URL=           # Optional: Daily API URL (defaults to https://api.daily.co/v1)
+OPENAI_API_KEY=          # Your OpenAI API key
+CARTESIA_API_KEY=        # Your Cartesia API key
+HOST=                    # Optional: Host address (defaults to 0.0.0.0)
+FAST_API_PORT=           # Optional: Port number (defaults to 7860)
+```
+
+## Running the Server
+
+Set up and activate your virtual environment:
+
+```bash
+python3 -m venv venv
+source venv/bin/activate  # On Windows: venv\Scripts\activate
+```
+
+Install dependencies:
+
+```bash
+pip install -r requirements.txt
+```
+
+Run the server:
+
+```bash
+python server.py
+```

examples/simple-chatbot/server/bot.py

@@ -18,6 +18,7 @@ from pipecat.audio.vad.silero import SileroVADAnalyzer
 from pipecat.frames.frames import (
     BotStartedSpeakingFrame,
     BotStoppedSpeakingFrame,
+    EndFrame,
     Frame,
     LLMMessagesFrame,
     OutputImageRawFrame,
@@ -162,6 +163,11 @@ async def main():
             await transport.capture_participant_transcription(participant["id"])
             await task.queue_frames([LLMMessagesFrame(messages)])
 
+        @transport.event_handler("on_participant_left")
+        async def on_participant_left(transport, participant, reason):
+            print(f"Participant left: {participant}")
+            await task.queue_frame(EndFrame())
+
         runner = PipelineRunner()
 
         await runner.run(task)

examples/simple-chatbot/server/server.py

@@ -0,0 +1,231 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""
+RTVI Bot Server Implementation
+
+This FastAPI server manages RTVI bot instances and provides endpoints for both
+direct browser access and RTVI client connections. It handles:
+- Creating Daily rooms
+- Managing bot processes
+- Providing connection credentials
+- Monitoring bot status
+
+Requirements:
+- Daily API key (set in .env file)
+- Python 3.10+
+- FastAPI
+- Running bot implementation
+"""
+
+import argparse
+import os
+import subprocess
+from contextlib import asynccontextmanager
+from typing import Any, Dict
+
+import aiohttp
+from dotenv import load_dotenv
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse, RedirectResponse
+
+from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper, DailyRoomParams
+
+# Load environment variables from .env file
+load_dotenv(override=True)
+
+# Maximum number of bot instances allowed per room
+MAX_BOTS_PER_ROOM = 1
+
+# Dictionary to track bot processes: {pid: (process, room_url)}
+bot_procs = {}
+
+# Store Daily API helpers
+daily_helpers = {}
+
+
+def cleanup():
+    """
+    Cleanup function to terminate all bot processes.
+    Called during server shutdown.
+    """
+    for entry in bot_procs.values():
+        proc = entry[0]
+        proc.terminate()
+        proc.wait()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """
+    FastAPI lifespan manager that handles startup and shutdown tasks:
+    - Creates aiohttp session
+    - Initializes Daily API helper
+    - Cleans up resources on shutdown
+    """
+    aiohttp_session = aiohttp.ClientSession()
+    daily_helpers["rest"] = DailyRESTHelper(
+        daily_api_key=os.getenv("DAILY_API_KEY", ""),
+        daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
+        aiohttp_session=aiohttp_session,
+    )
+    yield
+    await aiohttp_session.close()
+    cleanup()
+
+
+# Initialize FastAPI app with lifespan manager
+app = FastAPI(lifespan=lifespan)
+
+# Configure CORS to allow requests from any origin
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+async def create_room_and_token() -> tuple[str, str]:
+    """
+    Helper function to create a Daily room and generate an access token.
+
+    Returns:
+        tuple[str, str]: A tuple containing (room_url, token)
+
+    Raises:
+        HTTPException: If room creation or token generation fails
+    """
+    room = await daily_helpers["rest"].create_room(DailyRoomParams())
+    if not room.url:
+        raise HTTPException(status_code=500, detail="Failed to create room")
+
+    token = await daily_helpers["rest"].get_token(room.url)
+    if not token:
+        raise HTTPException(status_code=500, detail=f"Failed to get token for room: {room.url}")
+
+    return room.url, token
+
+
+@app.get("/")
+async def start_agent(request: Request):
+    """
+    Endpoint for direct browser access to the bot.
+    Creates a room, starts a bot instance, and redirects to the Daily room URL.
+
+    Returns:
+        RedirectResponse: Redirects to the Daily room URL
+
+    Raises:
+        HTTPException: If room creation, token generation, or bot startup fails
+    """
+    print("Creating room")
+    room_url, token = await create_room_and_token()
+    print(f"Room URL: {room_url}")
+
+    # Check if there is already an existing process running in this room
+    num_bots_in_room = sum(
+        1 for proc in bot_procs.values() if proc[1] == room_url and proc[0].poll() is None
+    )
+    if num_bots_in_room >= MAX_BOTS_PER_ROOM:
+        raise HTTPException(status_code=500, detail=f"Max bot limit reached for room: {room_url}")
+
+    # Spawn a new bot process
+    try:
+        proc = subprocess.Popen(
+            [f"python3 -m bot -u {room_url} -t {token}"],
+            shell=True,
+            bufsize=1,
+            cwd=os.path.dirname(os.path.abspath(__file__)),
+        )
+        bot_procs[proc.pid] = (proc, room_url)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}")
+
+    return RedirectResponse(room_url)
+
+
+@app.post("/connect")
+async def rtvi_connect(request: Request) -> Dict[Any, Any]:
+    """
+    RTVI connect endpoint that creates a room and returns connection credentials.
+    This endpoint is called by RTVI clients to establish a connection.
+
+    Returns:
+        Dict[Any, Any]: Authentication bundle containing room_url and token
+
+    Raises:
+        HTTPException: If room creation, token generation, or bot startup fails
+    """
+    print("Creating room for RTVI connection")
+    room_url, token = await create_room_and_token()
+    print(f"Room URL: {room_url}")
+
+    # Start the bot process
+    try:
+        proc = subprocess.Popen(
+            [f"python3 -m bot -u {room_url} -t {token}"],
+            shell=True,
+            bufsize=1,
+            cwd=os.path.dirname(os.path.abspath(__file__)),
+        )
+        bot_procs[proc.pid] = (proc, room_url)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}")
+
+    # Return the authentication bundle in format expected by DailyTransport
+    return {"room_url": room_url, "token": token}
+
+
+@app.get("/status/{pid}")
+def get_status(pid: int):
+    """
+    Get the status of a specific bot process.
+
+    Args:
+        pid (int): Process ID of the bot
+
+    Returns:
+        JSONResponse: Status information for the bot
+
+    Raises:
+        HTTPException: If the specified bot process is not found
+    """
+    # Look up the subprocess
+    proc = bot_procs.get(pid)
+
+    # If the subprocess doesn't exist, return an error
+    if not proc:
+        raise HTTPException(status_code=404, detail=f"Bot with process id: {pid} not found")
+
+    # Check the status of the subprocess
+    status = "running" if proc[0].poll() is None else "finished"
+    return JSONResponse({"bot_id": pid, "status": status})
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    # Parse command line arguments for server configuration
+    default_host = os.getenv("HOST", "0.0.0.0")
+    default_port = int(os.getenv("FAST_API_PORT", "7860"))
+
+    parser = argparse.ArgumentParser(description="Daily Storyteller FastAPI server")
+    parser.add_argument("--host", type=str, default=default_host, help="Host address")
+    parser.add_argument("--port", type=int, default=default_port, help="Port number")
+    parser.add_argument("--reload", action="store_true", help="Reload code on change")
+
+    config = parser.parse_args()
+
+    # Start the FastAPI server
+    uvicorn.run(
+        "server:app",
+        host=config.host,
+        port=config.port,
+        reload=config.reload,
+    )