Merge branch 'main' into groundingMetadata

This commit is contained in:
Pete
2025-07-20 19:54:30 -04:00
committed by GitHub
105 changed files with 8529 additions and 4312 deletions

View File

@@ -0,0 +1,60 @@
# AWS Strands Examples
This folder contains two Python examples demonstrating how to use Pipecat with the AWS Strands agent.
## Overview
These examples show how to delegate complex, multi-step tasks to a Strands agent, which can reason step-by-step and call tools to accomplish user requests.
These examples are intentionally simplified for demonstration, using mock API calls. They work best if you ask it:
> What's the weather where the Golden Gate Bridge is?
## Example Scripts
### `black-box.py`
A minimal example that demonstrates how to use the Strands agent with Pipecat. The agent can handle multi-step queries by calling tools, but does not explain its reasoning out loud.
### `explain-thinking.py`
An enhanced example where the Strands agent explains each step of its reasoning in clear, simple language as it works through a multi-step task.
## Quick Start
1. **Clone the repository and navigate to this example:**
```bash
git clone https://github.com/pipecat-ai/pipecat.git
cd pipecat/examples/aws-strands
```
2. **Set up a virtual environment:**
```bash
python -m venv venv
source venv/bin/activate # On Windows: venv\Scripts\activate
```
3. **Install dependencies:**
```bash
pip install -r requirements.txt
```
4. **Configure environment variables:**
Copy the provided `env.example` file to `.env` and fill in the necessary credentials:
```bash
cp env.example .env
# Then edit .env with your preferred editor
```
5. **Run an example:**
```bash
python black-box.py
# or
python explain-thinking.py
```

View File

@@ -0,0 +1,206 @@
#
# Copyright (c) 2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import argparse
import asyncio
import os
from dotenv import load_dotenv
from loguru import logger
from strands import Agent, tool
from strands.models import BedrockModel
from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import TTSSpeakFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.llm_service import FunctionCallParams
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams
from pipecat.transports.services.daily import DailyParams
load_dotenv(override=True)
"""This example demonstrates how to use the Strands agent with Pipecat.
You can delegate complex, multi-step tasks to the Strands agent, which can cycle through LLM-based reasoning and tool calls to accomplish the task.
Try asking: "What's the weather where the Golden Gate Bridge is?"
"""
# Strands agent tools
@tool
def get_location_name_from_landmark(landmark: str) -> str:
"""
Get the location name from a landmark.
Args:
landmark (str): The name of the landmark, e.g. "Golden Gate Bridge".
"""
# Simulate fetching location
return "San Francisco, CA"
@tool
def get_lat_long_from_location_name(location: str) -> dict:
"""
Get the latitude and longitude for a location name.
Args:
location (str): The city and state, e.g. "San Francisco, CA".
"""
# Simulate fetching lat/long from a geocoding service
return {"lat": 37.7749, "long": -122.4194}
@tool
def get_current_weather_from_lat_long(lat: float, long: float) -> dict:
"""
Get the current weather for a specific latitude and longitude.
Args:
lat (float): The latitude of the location.
long (float): The longitude of the location.
"""
# Simulate fetching weather data from a weather service
return {"conditions": "nice", "temperature": "75"}
# We store functions so objects (e.g. SileroVADAnalyzer) don't get
# instantiated. The function will be called when the desired transport gets
# selected.
transport_params = {
"daily": lambda: DailyParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
"twilio": lambda: FastAPIWebsocketParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
"webrtc": lambda: TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
}
async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool):
logger.info(f"Starting bot")
strands_agent = Agent(
model=BedrockModel(
model_id="us.anthropic.claude-3-7-sonnet-20250219-v1:0", max_tokens=64000
),
tools=[
get_location_name_from_landmark,
get_lat_long_from_location_name,
get_current_weather_from_lat_long,
],
system_prompt="""
You are a helpful personal assistant who can look up information about places and weather.
Your key capabilities:
1. Look up where landmarks are located.
2. Find latitude and longitude for a location.
3. Look up the current weather for a specific latitude and longitude.
Explain each step of your reasoning in clear, simple, and concise language. Your responses will be converted to audio, so avoid special characters and numbered lists.
""",
)
async def handle_location_or_weather_related_queries(params: FunctionCallParams, query: str):
"""
Handle location or weather related queries.
Args:
query (str): The user's query, e.g. "What's the weather where the Golden Gate Bridge is?".
"""
# Run in a background thread
# (Otherwise the agent blocks the event loop; one effect of that is that we don't hear
# "let me check on that" until the agent finishes)
loop = asyncio.get_running_loop()
result = await loop.run_in_executor(None, strands_agent, query)
await params.result_callback(result.message)
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
)
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
llm.register_direct_function(handle_location_or_weather_related_queries)
@llm.event_handler("on_function_calls_started")
async def on_function_calls_started(service, function_calls):
await tts.queue_frame(TTSSpeakFrame("Let me check on that."))
tools = ToolsSchema(standard_tools=[handle_location_or_weather_related_queries])
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. Start by suggesting that the user ask about the weather where the Golden Gate Bridge is.",
},
]
context = OpenAILLMContext(messages, tools)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(),
stt,
context_aggregator.user(),
llm,
tts,
transport.output(),
context_aggregator.assistant(),
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
enable_metrics=True,
enable_usage_metrics=True,
),
)
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
await task.cancel()
runner = PipelineRunner(handle_sigint=handle_sigint)
await runner.run(task)
if __name__ == "__main__":
from pipecat.examples.run import main
main(run_example, transport_params=transport_params)

View File

@@ -0,0 +1,8 @@
OPENAI_API_KEY=
CARTESIA_API_KEY=
DEEPGRAM_API_KEY=
DAILY_API_KEY=
DAILY_SAMPLE_ROOM_URL=
AWS_SECRET_ACCESS_KEY=
AWS_ACCESS_KEY_ID=
AWS_REGION=

View File

@@ -0,0 +1,249 @@
#
# Copyright (c) 2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import argparse
import asyncio
import os
import threading
import time
from dotenv import load_dotenv
from loguru import logger
from strands import Agent, tool
from strands.models import BedrockModel
from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import TTSSpeakFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.llm_service import FunctionCallParams
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams
from pipecat.transports.services.daily import DailyParams
load_dotenv(override=True)
"""This example demonstrates how to use the Strands agent with Pipecat in a way where the agent explains its reasoning step-by-step.
You can delegate complex, multi-step tasks to the Strands agent, which can cycle through LLM-based reasoning and tool calls to accomplish the task.
Try asking: "What's the weather where the Golden Gate Bridge is?"
"""
# Strands agent tools
@tool
def get_location_name_from_landmark(landmark: str) -> str:
"""
Get the location name from a landmark.
Args:
landmark (str): The name of the landmark, e.g. "Golden Gate Bridge".
"""
# Simulate fetching location (slowly)
time.sleep(3)
return "San Francisco, CA"
@tool
def get_lat_long_from_location_name(location: str) -> dict:
"""
Get the latitude and longitude for a location name.
Args:
location (str): The city and state, e.g. "San Francisco, CA".
"""
# Simulate fetching lat/long from a geocoding service (slowly)
time.sleep(3)
return {"lat": 37.7749, "long": -122.4194}
@tool
def get_current_weather_from_lat_long(lat: float, long: float) -> dict:
"""
Get the current weather for a specific latitude and longitude.
Args:
lat (float): The latitude of the location.
long (float): The longitude of the location.
"""
# Simulate fetching weather data from a weather service (slowly)
time.sleep(3)
return {"conditions": "nice", "temperature": "75"}
# We store functions so objects (e.g. SileroVADAnalyzer) don't get
# instantiated. The function will be called when the desired transport gets
# selected.
transport_params = {
"daily": lambda: DailyParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
"twilio": lambda: FastAPIWebsocketParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
"webrtc": lambda: TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
}
async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool):
logger.info(f"Starting bot")
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
)
next_strands_message_is_last = False
strands_messages_queue = asyncio.Queue()
def strands_callback_handler(**kwargs):
"""
Handle events from the Strands agent.
"""
nonlocal next_strands_message_is_last
if "event" in kwargs:
event_obj = kwargs["event"]
if event_obj and "messageStop" in event_obj:
message_stop = event_obj["messageStop"]
if message_stop and "stopReason" in message_stop:
stop_reason = message_stop["stopReason"]
if stop_reason == "end_turn":
next_strands_message_is_last = True
elif "message" in kwargs:
message_obj = kwargs["message"]
if message_obj and "content" in message_obj and "role" in message_obj:
role = message_obj["role"]
content = message_obj["content"]
if role == "assistant" and isinstance(content, list):
for content_obj in content:
if isinstance(content_obj, dict) and "text" in content_obj:
message = content_obj["text"]
if not next_strands_message_is_last:
strands_messages_queue.put_nowait(message)
async def process_strands_messages():
while True:
message = await strands_messages_queue.get()
await tts.queue_frame(TTSSpeakFrame(message))
strands_messages_queue.task_done()
asyncio.create_task(process_strands_messages())
strands_agent = Agent(
model=BedrockModel(
model_id="us.anthropic.claude-3-7-sonnet-20250219-v1:0", max_tokens=64000
),
tools=[
get_location_name_from_landmark,
get_lat_long_from_location_name,
get_current_weather_from_lat_long,
],
system_prompt="""
You are a helpful personal assistant who can look up information about places and weather.
Your key capabilities:
1. Look up where landmarks are located.
2. Find latitude and longitude for a location.
3. Look up the current weather for a specific latitude and longitude.
Explain each step of your reasoning in clear, simple, and concise language. Your responses will be converted to audio, so avoid special characters and numbered lists.
""",
callback_handler=strands_callback_handler,
)
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
async def handle_location_or_weather_related_queries(params: FunctionCallParams, query: str):
"""
Handle location or weather related queries.
Args:
query (str): The user's query, e.g. "What's the weather where the Golden Gate Bridge is?".
"""
# Run in a background thread
# (Otherwise the agent blocks the event loop; one effect of that is that we don't hear
# the agent's "thinking" messages until the agent finishes)
loop = asyncio.get_running_loop()
result = await loop.run_in_executor(None, strands_agent, query)
await params.result_callback(result.message)
llm.register_direct_function(handle_location_or_weather_related_queries)
@llm.event_handler("on_function_calls_started")
async def on_function_calls_started(service, function_calls):
await tts.queue_frame(TTSSpeakFrame("Let me check on that."))
tools = ToolsSchema(standard_tools=[handle_location_or_weather_related_queries])
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. Start by suggesting that the user ask about the weather where the Golden Gate Bridge is.",
},
]
context = OpenAILLMContext(messages, tools)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(),
stt,
context_aggregator.user(),
llm,
tts,
transport.output(),
context_aggregator.assistant(),
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
enable_metrics=True,
enable_usage_metrics=True,
),
)
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
await task.cancel()
runner = PipelineRunner(handle_sigint=handle_sigint)
await runner.run(task)
if __name__ == "__main__":
from pipecat.examples.run import main
main(run_example, transport_params=transport_params)

View File

@@ -0,0 +1,6 @@
fastapi
uvicorn
python-dotenv
pipecat-ai[webrtc,daily,deepgram,cartesia]
pipecat-ai-small-webrtc-prebuilt
strands-agents

File diff suppressed because it is too large Load Diff

View File

@@ -15,7 +15,7 @@
"vite": "^6.3.5"
},
"dependencies": {
"@pipecat-ai/client-js": "^0.3.5",
"@pipecat-ai/daily-transport": "^0.3.10"
"@pipecat-ai/client-js": "^1.0.0",
"@pipecat-ai/daily-transport": "^1.0.0"
}
}

View File

@@ -5,7 +5,7 @@
*/
/**
* RTVI Client Implementation
* Pipecat Client Implementation
*
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
* It handles audio/video streaming and manages the connection lifecycle.
@@ -16,7 +16,7 @@
* - Browser with WebRTC support
*/
import { RTVIClient, RTVIEvent } from '@pipecat-ai/client-js';
import { PipecatClient, RTVIEvent } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
/**
@@ -26,7 +26,7 @@ import { DailyTransport } from '@pipecat-ai/daily-transport';
class ChatbotClient {
constructor() {
// Initialize client state
this.rtviClient = null;
this.pcClient = null;
this.setupDOMElements();
this.initializeClientAndTransport();
this.setupEventListeners();
@@ -59,7 +59,7 @@ class ChatbotClient {
this.disconnectBtn.addEventListener('click', () => this.disconnect());
// Populate device selector
this.rtviClient.getAllMics().then((mics) => {
this.pcClient.getAllMics().then((mics) => {
console.log('Available mics:', mics);
mics.forEach((device) => {
const option = document.createElement('option');
@@ -71,16 +71,16 @@ class ChatbotClient {
this.deviceSelector.addEventListener('change', (event) => {
const selectedDeviceId = event.target.value;
console.log('Selected device ID:', selectedDeviceId);
this.rtviClient.updateMic(selectedDeviceId);
this.pcClient.updateMic(selectedDeviceId);
});
// Handle mic mute/unmute toggle
const micToggleBtn = document.getElementById('mic-toggle-btn');
micToggleBtn.addEventListener('click', () => {
let micEnabled = this.rtviClient.isMicEnabled;
let micEnabled = this.pcClient.isMicEnabled;
micToggleBtn.textContent = micEnabled ? 'Unmute Mic' : 'Mute Mic';
this.rtviClient.enableMic(!micEnabled);
this.pcClient.enableMic(!micEnabled);
// Add logic to mute/unmute the mic
if (micEnabled) {
console.log('Mic muted');
@@ -93,23 +93,12 @@ class ChatbotClient {
}
/**
* Set up the RTVI client and Daily transport
* Set up the Pipecat client and Daily transport
*/
async initializeClientAndTransport() {
// Initialize the RTVI client with a DailyTransport and our configuration
this.rtviClient = new RTVIClient({
// Initialize the Pipecat client with a DailyTransport and our configuration
this.pcClient = new PipecatClient({
transport: new DailyTransport(),
params: {
// REPLACE WITH YOUR MODAL URL ENDPOINT
baseUrl:
'https://<Modal workspace>--pipecat-modal-bot-launcher.modal.run',
endpoints: {
connect: '/connect',
},
requestData: {
bot_name: 'openai',
},
},
enableMic: true, // Enable microphone for user input
enableCam: false,
callbacks: {
@@ -176,8 +165,8 @@ class ChatbotClient {
// Set up listeners for media track events
this.setupTrackListeners();
await this.rtviClient.initDevices();
window.client = this.rtviClient;
await this.pcClient.initDevices();
window.client = this.pcClient;
}
/**
@@ -212,10 +201,10 @@ class ChatbotClient {
* This is called when the bot is ready or when the transport state changes to ready
*/
setupMediaTracks() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Get current tracks from the client
const tracks = this.rtviClient.tracks();
const tracks = this.pcClient.tracks();
// Set up any available bot tracks
if (tracks.bot?.audio) {
@@ -231,10 +220,10 @@ class ChatbotClient {
* This handles new tracks being added during the session
*/
setupTrackListeners() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks
if (!participant?.local) {
if (track.kind === 'audio') {
@@ -253,7 +242,7 @@ class ChatbotClient {
});
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
if (participant.local) {
this.log('Local mic muted');
return;
@@ -311,21 +300,27 @@ class ChatbotClient {
/**
* Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection
* This sets up the Pipecat client, initializes devices, and establishes the connection
*/
async connect() {
try {
const botSelector = document.getElementById('bot-selector');
const selectedBot = botSelector.value;
this.rtviClient.params.requestData.bot_name = selectedBot;
// Initialize audio/video devices
this.log('Initializing devices...');
await this.rtviClient.initDevices();
await this.pcClient.initDevices();
// Connect to the bot
this.log(`Connecting to bot: ${selectedBot}`);
await this.rtviClient.connect();
await this.pcClient.connect({
// REPLACE WITH YOUR MODAL URL ENDPOINT
endpoint:
'https://<your-workspace>--pipecat-modal-fastapi-app.modal.run/connect',
requestData: {
bot_name: selectedBot,
},
});
this.log('Connection complete');
} catch (error) {
@@ -336,9 +331,9 @@ class ChatbotClient {
this.updateStatus('Error');
// Clean up if there's an error
if (this.rtviClient) {
if (this.pcClient) {
try {
await this.rtviClient.disconnect();
await this.pcClient.disconnect();
} catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError.message}`);
}
@@ -350,10 +345,10 @@ class ChatbotClient {
* Disconnect from the bot and clean up media resources
*/
async disconnect() {
if (this.rtviClient) {
if (this.pcClient) {
try {
// Disconnect the RTVI client
await this.rtviClient.disconnect();
// Disconnect the Pipecat client
await this.pcClient.disconnect();
// Clean up audio
if (this.botAudio.srcObject) {

View File

@@ -301,7 +301,7 @@ def fastapi_app():
allow_headers=["*"],
)
# Include the endpoints from endpoints.py
# Include the endpoints from this file
web_app.include_router(router)
return web_app

View File

@@ -1,2 +1,3 @@
python-dotenv==1.0.1
modal==0.71.3
modal==1.0.5
fastapi[all]

View File

@@ -103,7 +103,7 @@ export default async function handler(req, res) {
const sip_config = {
display_name: From,
sip_mode: 'dial-in',
num_endpoints: call_transfer !== null ? 2 : 1,
num_endpoints: (call_transfer !== undefined && call_transfer !== null) ? 2 : 1,
codecs: {"audio": ["OPUS"]},
};
daily_room_properties.sip = sip_config;

View File

@@ -44,7 +44,7 @@ Try the hosted version of the demo here: https://pcc-smart-turn.vercel.app/.
4. Run the server:
```bash
LOCAL=1 python server.py
LOCAL_RUN=1 python server.py
```
### Run the client

File diff suppressed because it is too large Load Diff

View File

@@ -9,9 +9,9 @@
"lint": "next lint"
},
"dependencies": {
"@pipecat-ai/client-js": "^0.3.5",
"@pipecat-ai/client-react": "^0.3.5",
"@pipecat-ai/daily-transport": "^0.3.10",
"@pipecat-ai/client-js": "^1.0.0",
"@pipecat-ai/client-react": "^1.0.0",
"@pipecat-ai/daily-transport": "^1.0.0",
"next": "15.3.1",
"react": "^19.0.0",
"react-dom": "^19.0.0"

View File

@@ -1,5 +1,5 @@
import './globals.css';
import { RTVIProvider } from '@/providers/RTVIProvider';
import { PipecatProvider } from '@/providers/PipecatProvider';
export const metadata = {
title: 'Pipecat React Client',
@@ -20,7 +20,7 @@ export default function RootLayout({
<link rel="icon" href="/favicon.svg" type="image/svg+xml" />
</head>
<body>
<RTVIProvider>{children}</RTVIProvider>
<PipecatProvider>{children}</PipecatProvider>
</body>
</html>
);

View File

@@ -1,22 +1,22 @@
'use client';
import {
RTVIClientAudio,
RTVIClientVideo,
useRTVIClientTransportState,
PipecatClientAudio,
PipecatClientVideo,
usePipecatClientTransportState,
} from '@pipecat-ai/client-react';
import { ConnectButton } from '../components/ConnectButton';
import { StatusDisplay } from '../components/StatusDisplay';
import { DebugDisplay } from '../components/DebugDisplay';
function BotVideo() {
const transportState = useRTVIClientTransportState();
const transportState = usePipecatClientTransportState();
const isConnected = transportState !== 'disconnected';
return (
<div className="bot-container">
<div className="video-container">
{isConnected && <RTVIClientVideo participant="bot" fit="cover" />}
{isConnected && <PipecatClientVideo participant="bot" fit="cover" />}
</div>
</div>
);
@@ -35,7 +35,7 @@ export default function Home() {
</div>
<DebugDisplay />
<RTVIClientAudio />
<PipecatClientAudio />
</div>
);
}

View File

@@ -1,11 +1,17 @@
import {
useRTVIClient,
useRTVIClientTransportState,
usePipecatClient,
usePipecatClientTransportState,
} from '@pipecat-ai/client-react';
// Get the API base URL from environment variables
// Default to "/api" if not specified
// "/api" is the default for Next.js API routes and used
// for the Pipecat Cloud deployed agent
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
export function ConnectButton() {
const client = useRTVIClient();
const transportState = useRTVIClientTransportState();
const client = usePipecatClient();
const transportState = usePipecatClientTransportState();
const isConnected = ['connected', 'ready'].includes(transportState);
const handleClick = async () => {
@@ -18,7 +24,10 @@ export function ConnectButton() {
if (isConnected) {
await client.disconnect();
} else {
await client.connect();
await client.connect({
endpoint: `${API_BASE_URL}/connect`,
requestData: { foo: 'bar' },
});
}
} catch (error) {
console.error('Connection error:', error);

View File

@@ -6,7 +6,7 @@ import {
TranscriptData,
BotLLMTextData,
} from '@pipecat-ai/client-js';
import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
import { usePipecatClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
import './DebugDisplay.css';
interface SmartTurnResultData {
@@ -20,7 +20,7 @@ interface SmartTurnResultData {
export function DebugDisplay() {
const debugLogRef = useRef<HTMLDivElement>(null);
const client = useRTVIClient();
const client = usePipecatClient();
const log = useCallback((message: string) => {
if (!debugLogRef.current) return;

View File

@@ -1,7 +1,7 @@
import { useRTVIClientTransportState } from '@pipecat-ai/client-react';
import { usePipecatClientTransportState } from '@pipecat-ai/client-react';
export function StatusDisplay() {
const transportState = useRTVIClientTransportState();
const transportState = usePipecatClientTransportState();
return (
<div className="status">

View File

@@ -0,0 +1,28 @@
'use client';
import { PipecatClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { PipecatClientProvider } from '@pipecat-ai/client-react';
import { PropsWithChildren, useEffect, useState } from 'react';
export function PipecatProvider({ children }: PropsWithChildren) {
const [client, setClient] = useState<PipecatClient | null>(null);
useEffect(() => {
const pcClient = new PipecatClient({
transport: new DailyTransport(),
enableMic: true,
enableCam: false,
});
setClient(pcClient);
}, []);
if (!client) {
return null;
}
return (
<PipecatClientProvider client={client}>{children}</PipecatClientProvider>
);
}

View File

@@ -1,43 +0,0 @@
'use client';
import { RTVIClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { RTVIClientProvider } from '@pipecat-ai/client-react';
import { PropsWithChildren, useEffect, useState } from 'react';
// Get the API base URL from environment variables
// Default to "/api" if not specified
// "/api" is the default for Next.js API routes and used
// for the Pipecat Cloud deployed agent
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
console.log('Using API base URL:', API_BASE_URL);
export function RTVIProvider({ children }: PropsWithChildren) {
const [client, setClient] = useState<RTVIClient | null>(null);
useEffect(() => {
const transport = new DailyTransport();
const rtviClient = new RTVIClient({
transport,
params: {
baseUrl: API_BASE_URL,
endpoints: {
connect: '/connect',
},
requestData: { foo: 'bar' },
},
enableMic: true,
enableCam: false,
});
setClient(rtviClient);
}, []);
if (!client) {
return null;
}
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
}

View File

@@ -45,7 +45,7 @@ from pipecat.transports.services.daily import DailyParams, DailyTransport
load_dotenv(override=True)
# Check if we're in local development mode
LOCAL = os.getenv("LOCAL")
LOCAL = os.getenv("LOCAL_RUN")
logger.remove()
logger.add(sys.stderr, level="DEBUG")

View File

@@ -0,0 +1,109 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import argparse
import os
from dotenv import load_dotenv
from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.services.soniox.stt import SonioxSTTService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams
from pipecat.transports.services.daily import DailyParams
load_dotenv(override=True)
transport_params = {
"daily": lambda: DailyParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
"twilio": lambda: FastAPIWebsocketParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
"webrtc": lambda: TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
}
async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool):
logger.info(f"Starting bot")
stt = SonioxSTTService(
api_key=os.getenv("SONIOX_API_KEY"),
)
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
)
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
},
]
context = OpenAILLMContext(messages)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(), # Transport user input
stt,
context_aggregator.user(), # User responses
llm, # LLM
tts, # TTS
transport.output(), # Transport bot output
context_aggregator.assistant(), # Assistant spoken responses
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
enable_metrics=True,
enable_usage_metrics=True,
),
)
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation.
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
await task.cancel()
runner = PipelineRunner(handle_sigint=handle_sigint)
await runner.run(task)
if __name__ == "__main__":
from pipecat.examples.run import main
main(run_example, transport_params=transport_params)

View File

@@ -0,0 +1,81 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import argparse
import os
from dotenv import load_dotenv
from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import Frame, TranscriptionFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.soniox.stt import SonioxSTTService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams
from pipecat.transports.services.daily import DailyParams
load_dotenv(override=True)
class TranscriptionLogger(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame):
print(f"Transcription: {frame.text}")
transport_params = {
"daily": lambda: DailyParams(
audio_in_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
"twilio": lambda: FastAPIWebsocketParams(
audio_in_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
"webrtc": lambda: TransportParams(
audio_in_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
}
async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool):
logger.info(f"Starting bot")
stt = SonioxSTTService(
api_key=os.getenv("SONIOX_API_KEY"),
)
tl = TranscriptionLogger()
pipeline = Pipeline([transport.input(), stt, tl])
task = PipelineTask(pipeline)
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
runner = PipelineRunner(handle_sigint=False)
await runner.run(task)
if __name__ == "__main__":
from pipecat.examples.run import main
main(run_example, transport_params=transport_params)

View File

@@ -11,7 +11,7 @@ from dotenv import load_dotenv
from loguru import logger
from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams
from pipecat.audio.turn.smart_turn.local_smart_turn import LocalSmartTurnAnalyzer
from pipecat.audio.turn.smart_turn.local_smart_turn_v2 import LocalSmartTurnAnalyzerV2
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.pipeline.pipeline import Pipeline
@@ -37,7 +37,7 @@ load_dotenv(override=True)
# # Hugging Face uses LFS to store large model files, including .mlpackage
# git lfs install
# # Clone the repo with the smart_turn_classifier.mlpackage
# git clone https://huggingface.co/pipecat-ai/smart-turn
# git clone https://huggingface.co/pipecat-ai/smart-turn-v2
#
# Then set the env variable:
# export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn
@@ -52,7 +52,7 @@ transport_params = {
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
turn_analyzer=LocalSmartTurnAnalyzer(
turn_analyzer=LocalSmartTurnAnalyzerV2(
smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams()
),
),
@@ -60,7 +60,7 @@ transport_params = {
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
turn_analyzer=LocalSmartTurnAnalyzer(
turn_analyzer=LocalSmartTurnAnalyzerV2(
smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams()
),
),
@@ -68,7 +68,7 @@ transport_params = {
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
turn_analyzer=LocalSmartTurnAnalyzer(
turn_analyzer=LocalSmartTurnAnalyzerV2(
smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams()
),
),

View File

@@ -20,11 +20,10 @@ import {
} from '@pipecat-ai/client-js';
import {
ProtobufFrameSerializer,
WebSocketTransport
} from "@pipecat-ai/websocket-transport";
WebSocketTransport,
} from '@pipecat-ai/websocket-transport';
class RecordingSerializer extends ProtobufFrameSerializer {
private lastTimestamp: number | null = null;
private recordingAudioToSend: boolean = false;
private _recordedAudio: { data: ArrayBuffer; delay: number }[] = [];
@@ -40,7 +39,11 @@ class RecordingSerializer extends ProtobufFrameSerializer {
}
// @ts-ignore
serializeAudio(data: ArrayBuffer, sampleRate: number, numChannels: number): Uint8Array | null {
serializeAudio(
data: ArrayBuffer,
sampleRate: number,
numChannels: number
): Uint8Array | null {
if (this.recordingAudioToSend) {
const now = Date.now();
// Compute delay since last packet
@@ -55,13 +58,13 @@ class RecordingSerializer extends ProtobufFrameSerializer {
}
public get recordedAudio() {
return this._recordedAudio
return this._recordedAudio;
}
}
class WebsocketClientApp {
private ENABLE_RECORDING_MODE = false
private RECORDING_TIME_MS = 10000
private ENABLE_RECORDING_MODE = false;
private RECORDING_TIME_MS = 10000;
private rtviClient: RTVIClient | null = null;
private connectBtn: HTMLButtonElement | null = null;
@@ -71,7 +74,7 @@ class WebsocketClientApp {
private botAudio: HTMLAudioElement;
private declare websocketTransport: WebSocketTransport;
private sendRecordedAudio: boolean = false
private sendRecordedAudio: boolean = false;
private declare recordingSerializer: RecordingSerializer;
private playBtn: HTMLButtonElement | null = null;
@@ -91,8 +94,12 @@ class WebsocketClientApp {
* Set up references to DOM elements and create necessary media elements
*/
private setupDOMElements(): void {
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
this.connectBtn = document.getElementById(
'connect-btn'
) as HTMLButtonElement;
this.disconnectBtn = document.getElementById(
'disconnect-btn'
) as HTMLButtonElement;
this.statusSpan = document.getElementById('connection-status');
this.debugLog = document.getElementById('debug-log');
this.playBtn = document.getElementById('play-btn') as HTMLButtonElement;
@@ -105,8 +112,12 @@ class WebsocketClientApp {
private setupEventListeners(): void {
this.connectBtn?.addEventListener('click', () => this.connect());
this.disconnectBtn?.addEventListener('click', () => this.disconnect());
this.playBtn?.addEventListener('click', () => this.startSendingRecordedAudio());
this.stopBtn?.addEventListener('click', () => this.stopSendingRecordedAudio());
this.playBtn?.addEventListener('click', () =>
this.startSendingRecordedAudio()
);
this.stopBtn?.addEventListener('click', () =>
this.stopSendingRecordedAudio()
);
}
/**
@@ -165,7 +176,9 @@ class WebsocketClientApp {
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`);
this.log(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
);
});
}
@@ -175,7 +188,10 @@ class WebsocketClientApp {
*/
private setupAudioTrack(track: MediaStreamTrack): void {
this.log('Setting up audio track');
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
if (oldTrack?.id === track.id) return;
}
@@ -190,27 +206,17 @@ class WebsocketClientApp {
try {
const startTime = Date.now();
this.recordingSerializer = new RecordingSerializer()
const transport = this.ENABLE_RECORDING_MODE ?
new WebSocketTransport({
serializer: this.recordingSerializer,
recorderSampleRate: 8000,
playerSampleRate:8000
}) :
new WebSocketTransport({
serializer: new ProtobufFrameSerializer(),
recorderSampleRate: 8000,
playerSampleRate:8000
});
this.websocketTransport = transport
this.recordingSerializer = new RecordingSerializer();
const ws_opts = {
serializer: this.ENABLE_RECORDING_MODE
? this.recordingSerializer
: new ProtobufFrameSerializer(),
recorderSampleRate: 8000,
playerSampleRate: 8000,
};
const RTVIConfig: RTVIClientOptions = {
transport,
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: { connect: '/connect' },
},
transport: new WebSocketTransport(ws_opts),
enableMic: true,
enableCam: false,
callbacks: {
@@ -238,27 +244,34 @@ class WebsocketClientApp {
onMessageError: (error) => console.error('Message error:', error),
onError: (error) => console.error('Error:', error),
},
}
};
this.rtviClient = new RTVIClient(RTVIConfig);
this.websocketTransport = this.rtviClient.transport;
this.setupTrackListeners();
this.log('Initializing devices...');
await this.rtviClient.initDevices();
this.log('Connecting to bot...');
await this.rtviClient.connect();
await this.rtviClient.connect({
endpoint: 'http://localhost:7860/connect',
});
const timeTaken = Date.now() - startTime;
this.log(`Connection complete, timeTaken: ${timeTaken}`);
if (this.ENABLE_RECORDING_MODE) {
this.log(`Starting to recording the next ${(this.RECORDING_TIME_MS/1000)}s of audio`);
this.recordingSerializer.startRecording()
await this.sleep(this.RECORDING_TIME_MS)
this.recordingSerializer.stopRecording()
this.log("Recording stopped");
this.rtviClient.enableMic(false)
this.startSendingRecordedAudio()
this.log(
`Starting to recording the next ${
this.RECORDING_TIME_MS / 1000
}s of audio`
);
this.recordingSerializer.startRecording();
await this.sleep(this.RECORDING_TIME_MS);
this.recordingSerializer.stopRecording();
this.log('Recording stopped');
this.rtviClient.enableMic(false);
this.startSendingRecordedAudio();
}
} catch (error) {
this.log(`Error connecting: ${(error as Error).message}`);
@@ -280,11 +293,16 @@ class WebsocketClientApp {
public async disconnect(): Promise<void> {
if (this.rtviClient) {
try {
this.stopSendingRecordedAudio()
this.stopSendingRecordedAudio();
await this.rtviClient.disconnect();
this.rtviClient = null;
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop());
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
this.botAudio.srcObject
.getAudioTracks()
.forEach((track) => track.stop());
this.botAudio.srcObject = null;
}
} catch (error) {
@@ -294,21 +312,21 @@ class WebsocketClientApp {
}
private startSendingRecordedAudio() {
this.sendRecordedAudio = true
this.sendRecordedAudio = true;
if (this.playBtn) this.playBtn.disabled = true;
if (this.stopBtn) this.stopBtn.disabled = false;
void this.replayAudio()
void this.replayAudio();
}
private stopSendingRecordedAudio() {
if (this.stopBtn) this.stopBtn.disabled = true;
if (this.playBtn) this.playBtn.disabled = false;
this.sendRecordedAudio = false
this.sendRecordedAudio = false;
}
private async replayAudio() {
if (this.sendRecordedAudio) {
this.log("Sending recorded audio")
this.log('Sending recorded audio');
for (const chunk of this.recordingSerializer.recordedAudio) {
await this.sleep(chunk.delay);
this.websocketTransport.handleUserAudioStream(chunk.data);
@@ -316,14 +334,13 @@ class WebsocketClientApp {
const randomDelay = 1000 + Math.random() * (10000 - 500);
await this.sleep(randomDelay);
void this.replayAudio()
void this.replayAudio();
}
}
private sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
return new Promise((resolve) => setTimeout(resolve, ms));
}
}
declare global {

File diff suppressed because it is too large Load Diff

View File

@@ -18,7 +18,7 @@
"vite": "^6.3.5"
},
"dependencies": {
"@pipecat-ai/client-js": "^0.3.5",
"@pipecat-ai/daily-transport": "^0.3.8"
"@pipecat-ai/client-js": "^1.0.0",
"@pipecat-ai/daily-transport": "^1.0.0"
}
}

View File

@@ -5,7 +5,7 @@
*/
/**
* RTVI Client Implementation
* Pipecat Client Implementation
*
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
* It handles audio/video streaming and manages the connection lifecycle.
@@ -18,20 +18,22 @@
import {
Participant,
RTVIClient,
RTVIClientOptions,
PipecatClient,
PipecatClientOptions,
RTVIEvent,
} from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import {
DailyEventCallbacks,
DailyTransport,
} from '@pipecat-ai/daily-transport';
import SoundUtils from './util/soundUtils';
import { InstantVoiceHelper } from './util/instantVoiceHelper';
/**
* InstantVoiceClient handles the connection and media management for a real-time
* voice and video interaction with an AI bot.
*/
class InstantVoiceClient {
private declare rtviClient: RTVIClient;
private declare pcClient: PipecatClient;
private connectBtn: HTMLButtonElement | null = null;
private disconnectBtn: HTMLButtonElement | null = null;
private statusSpan: HTMLElement | null = null;
@@ -46,7 +48,7 @@ class InstantVoiceClient {
document.body.appendChild(this.botAudio);
this.setupDOMElements();
this.setupEventListeners();
this.initializeRTVIClient();
this.initializePipecatClient();
}
/**
@@ -72,16 +74,11 @@ class InstantVoiceClient {
this.disconnectBtn?.addEventListener('click', () => this.disconnect());
}
private initializeRTVIClient(): void {
const RTVIConfig: RTVIClientOptions = {
private initializePipecatClient(): void {
const PipecatConfig: PipecatClientOptions = {
transport: new DailyTransport({
bufferLocalAudioUntilBotReady: true,
}),
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: { connect: '/connect' },
},
enableMic: true,
enableCam: false,
callbacks: {
@@ -113,30 +110,23 @@ class InstantVoiceClient {
onBotTranscript: (data) => this.log(`Bot: ${data.text}`),
onMessageError: (error) => console.error('Message error:', error),
onError: (error) => console.error('Error:', error),
},
onAudioBufferingStarted: () => {
SoundUtils.beep();
this.updateBufferingStatus('Yes');
this.log(
`onMicCaptureStarted, timeTaken: ${Date.now() - this.startTime}`
);
},
onAudioBufferingStopped: () => {
this.updateBufferingStatus('No');
this.log(
`onMicCaptureStopped, timeTaken: ${Date.now() - this.startTime}`
);
},
} as DailyEventCallbacks,
};
this.rtviClient = new RTVIClient(RTVIConfig);
this.rtviClient.registerHelper(
'transport',
new InstantVoiceHelper({
callbacks: {
onAudioBufferingStarted: () => {
SoundUtils.beep();
this.updateBufferingStatus('Yes');
this.log(
`onMicCaptureStarted, timeTaken: ${Date.now() - this.startTime}`
);
},
onAudioBufferingStopped: () => {
this.updateBufferingStatus('No');
this.log(
`onMicCaptureStopped, timeTaken: ${Date.now() - this.startTime}`
);
},
},
})
);
this.pcClient = new PipecatClient(PipecatConfig);
this.setupTrackListeners();
}
@@ -182,8 +172,8 @@ class InstantVoiceClient {
* This is called when the bot is ready or when the transport state changes to ready
*/
setupMediaTracks() {
if (!this.rtviClient) return;
const tracks = this.rtviClient.tracks();
if (!this.pcClient) return;
const tracks = this.pcClient.tracks();
if (tracks.bot?.audio) {
this.setupAudioTrack(tracks.bot.audio);
}
@@ -194,10 +184,10 @@ class InstantVoiceClient {
* This handles new tracks being added during the session
*/
setupTrackListeners() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks
if (!participant?.local && track.kind === 'audio') {
this.setupAudioTrack(track);
@@ -205,7 +195,7 @@ class InstantVoiceClient {
});
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
);
@@ -230,22 +220,25 @@ class InstantVoiceClient {
/**
* Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection
* This sets up the Pipecat client, initializes devices, and establishes the connection
*/
public async connect(): Promise<void> {
try {
this.startTime = Date.now();
this.log('Connecting to bot...');
await this.rtviClient.connect();
await this.pcClient.connect({
// The baseURL and endpoint of your bot server that the client will connect to
endpoint: 'http://localhost:7860/connect',
});
} catch (error) {
this.log(`Error connecting: ${(error as Error).message}`);
this.updateStatus('Error');
this.updateBufferingStatus('No');
// Clean up if there's an error
if (this.rtviClient) {
if (this.pcClient) {
try {
await this.rtviClient.disconnect();
await this.pcClient.disconnect();
} catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError}`);
}
@@ -258,7 +251,7 @@ class InstantVoiceClient {
*/
public async disconnect(): Promise<void> {
try {
await this.rtviClient.disconnect();
await this.pcClient.disconnect();
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject

View File

@@ -1,39 +0,0 @@
import {RTVIClientHelper, RTVIClientHelperOptions, RTVIMessage} from "@pipecat-ai/client-js";
import {DailyRTVIMessageType} from '@pipecat-ai/daily-transport';
export type InstantVoiceHelperCallbacks = Partial<{
onAudioBufferingStarted: () => void;
onAudioBufferingStopped: () => void;
}>;
// --- Interface and class
export interface InstantVoiceHelperOptions extends RTVIClientHelperOptions {
callbacks?: InstantVoiceHelperCallbacks;
}
export class InstantVoiceHelper extends RTVIClientHelper {
protected declare _options: InstantVoiceHelperOptions;
constructor(options: InstantVoiceHelperOptions) {
super(options);
}
handleMessage(rtviMessage: RTVIMessage): void {
switch (rtviMessage.type) {
case DailyRTVIMessageType.AUDIO_BUFFERING_STARTED:
if (this._options.callbacks?.onAudioBufferingStarted) {
this._options.callbacks?.onAudioBufferingStarted()
}
break;
case DailyRTVIMessageType.AUDIO_BUFFERING_STOPPED:
if (this._options.callbacks?.onAudioBufferingStopped) {
this._options.callbacks?.onAudioBufferingStopped()
}
break;
}
}
getMessageTypes(): string[] {
return [DailyRTVIMessageType.AUDIO_BUFFERING_STARTED, DailyRTVIMessageType.AUDIO_BUFFERING_STOPPED];
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -15,7 +15,7 @@
"vite": "^6.3.5"
},
"dependencies": {
"@pipecat-ai/client-js": "^0.3.5",
"@pipecat-ai/daily-transport": "^0.3.8"
"@pipecat-ai/client-js": "^1.0.0",
"@pipecat-ai/daily-transport": "^1.0.0"
}
}

View File

@@ -5,7 +5,7 @@
*/
/**
* RTVI Client Implementation
* Pipecat Client Implementation
*
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
* It handles audio/video streaming and manages the connection lifecycle.
@@ -16,78 +16,9 @@
* - Browser with WebRTC support
*/
import {
LogLevel,
RTVIClient,
RTVIClientHelper,
RTVIEvent,
} from '@pipecat-ai/client-js';
import { LogLevel, PipecatClient, RTVIEvent } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
class SearchResponseHelper extends RTVIClientHelper {
constructor(contentPanel) {
super();
this.contentPanel = contentPanel;
}
handleMessage(rtviMessage) {
console.log('SearchResponseHelper, received message:', rtviMessage);
if (rtviMessage.data) {
// Clear existing content
this.contentPanel.innerHTML = '';
// Create a container for all content
const contentContainer = document.createElement('div');
contentContainer.className = 'content-container';
// Add the search_result
if (rtviMessage.data.search_result) {
const searchResultDiv = document.createElement('div');
searchResultDiv.className = 'search-result';
searchResultDiv.textContent = rtviMessage.data.search_result;
contentContainer.appendChild(searchResultDiv);
}
// Add the sources
if (rtviMessage.data.origins) {
const sourcesDiv = document.createElement('div');
sourcesDiv.className = 'sources';
const sourcesTitle = document.createElement('h3');
sourcesTitle.className = 'sources-title';
sourcesTitle.textContent = 'Sources:';
sourcesDiv.appendChild(sourcesTitle);
rtviMessage.data.origins.forEach((origin) => {
const sourceLink = document.createElement('a');
sourceLink.className = 'source-link';
sourceLink.href = origin.site_uri;
sourceLink.target = '_blank';
sourceLink.textContent = origin.site_title;
sourcesDiv.appendChild(sourceLink);
});
contentContainer.appendChild(sourcesDiv);
}
// Add the rendered_content in an iframe
if (rtviMessage.data.rendered_content) {
const iframe = document.createElement('iframe');
iframe.className = 'iframe-container';
iframe.srcdoc = rtviMessage.data.rendered_content;
contentContainer.appendChild(iframe);
}
// Append the content container to the content panel
this.contentPanel.appendChild(contentContainer);
}
}
getMessageTypes() {
return ['bot-llm-search-response'];
}
}
/**
* ChatbotClient handles the connection and media management for a real-time
* voice and video interaction with an AI bot.
@@ -95,7 +26,7 @@ class SearchResponseHelper extends RTVIClientHelper {
class ChatbotClient {
constructor() {
// Initialize client state
this.rtviClient = null;
this.pcClient = null;
this.setupDOMElements();
this.setupEventListeners();
}
@@ -160,10 +91,10 @@ class ChatbotClient {
* This is called when the bot is ready or when the transport state changes to ready
*/
setupMediaTracks() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Get current tracks from the client
const tracks = this.rtviClient.tracks();
const tracks = this.pcClient.tracks();
// Set up any available bot tracks
if (tracks.bot?.audio) {
@@ -176,10 +107,10 @@ class ChatbotClient {
* This handles new tracks being added during the session
*/
setupTrackListeners() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks
if (!participant?.local && track.kind === 'audio') {
this.setupAudioTrack(track);
@@ -187,7 +118,7 @@ class ChatbotClient {
});
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(
`Track stopped event: ${track.kind} from ${
participant?.name || 'unknown'
@@ -213,20 +144,13 @@ class ChatbotClient {
/**
* Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection
* This sets up the Pipecat client, initializes devices, and establishes the connection
*/
async connect() {
try {
// Initialize the RTVI client with a Daily WebRTC transport and our configuration
this.rtviClient = new RTVIClient({
// Initialize the Pipecat client with a Daily WebRTC transport and our configuration
this.pcClient = new PipecatClient({
transport: new DailyTransport(),
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: {
connect: '/connect',
},
},
enableMic: true, // Enable microphone for user input
enableCam: false,
callbacks: {
@@ -251,6 +175,8 @@ class ChatbotClient {
this.setupMediaTracks();
}
},
// Handle search response events
onBotLlmSearchResponse: this.handleSearchResponse.bind(this),
// Handle bot connection events
onBotConnected: (participant) => {
this.log(`Bot connected: ${JSON.stringify(participant)}`);
@@ -281,22 +207,22 @@ class ChatbotClient {
},
},
});
//this.rtviClient.setLogLevel(LogLevel.DEBUG)
this.rtviClient.registerHelper(
'llm',
new SearchResponseHelper(this.searchResultContainer)
);
//this.pcClient.setLogLevel(LogLevel.DEBUG)
// Set up listeners for media track events
this.setupTrackListeners();
// Initialize audio devices
this.log('Initializing devices...');
await this.rtviClient.initDevices();
await this.pcClient.initDevices();
// Connect to the bot
this.log('Connecting to bot...');
await this.rtviClient.connect();
await this.pcClient.connect({
// The baseURL and endpoint of your bot server that the client will connect to
endpoint: 'http://localhost:7860/connect',
});
this.log('Connection complete');
} catch (error) {
@@ -306,9 +232,9 @@ class ChatbotClient {
this.updateStatus('Error');
// Clean up if there's an error
if (this.rtviClient) {
if (this.pcClient) {
try {
await this.rtviClient.disconnect();
await this.pcClient.disconnect();
} catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError.message}`);
}
@@ -320,11 +246,11 @@ class ChatbotClient {
* Disconnect from the bot and clean up media resources
*/
async disconnect() {
if (this.rtviClient) {
if (this.pcClient) {
try {
// Disconnect the RTVI client
await this.rtviClient.disconnect();
this.rtviClient = null;
// Disconnect the Pipecat client
await this.pcClient.disconnect();
this.pcClient = null;
// Clean up audio
if (this.botAudio.srcObject) {
@@ -339,6 +265,57 @@ class ChatbotClient {
}
}
}
handleSearchResponse(response) {
console.log('SearchResponseHelper, received message:', response);
// Clear existing content
this.searchResultContainer.innerHTML = '';
// Create a container for all content
const contentContainer = document.createElement('div');
contentContainer.className = 'content-container';
// Add the search_result
if (response.search_result) {
const searchResultDiv = document.createElement('div');
searchResultDiv.className = 'search-result';
searchResultDiv.textContent = response.search_result;
contentContainer.appendChild(searchResultDiv);
}
// Add the sources
if (response.origins) {
const sourcesDiv = document.createElement('div');
sourcesDiv.className = 'sources';
const sourcesTitle = document.createElement('h3');
sourcesTitle.className = 'sources-title';
sourcesTitle.textContent = 'Sources:';
sourcesDiv.appendChild(sourcesTitle);
response.origins.forEach((origin) => {
const sourceLink = document.createElement('a');
sourceLink.className = 'source-link';
sourceLink.href = origin.site_uri;
sourceLink.target = '_blank';
sourceLink.textContent = origin.site_title;
sourcesDiv.appendChild(sourceLink);
});
contentContainer.appendChild(sourcesDiv);
}
// Add the rendered_content in an iframe
if (response.rendered_content) {
const iframe = document.createElement('iframe');
iframe.className = 'iframe-container';
iframe.srcdoc = response.rendered_content;
contentContainer.appendChild(iframe);
}
// Append the content container to the content panel
this.searchResultContainer.appendChild(contentContainer);
}
}
// Initialize the client when the page loads

File diff suppressed because it is too large Load Diff

View File

@@ -18,7 +18,7 @@
"vite": "^6.3.5"
},
"dependencies": {
"@pipecat-ai/client-js": "^0.3.2",
"@pipecat-ai/small-webrtc-transport": "^0.0.2"
"@pipecat-ai/client-js": "^1.0.0",
"@pipecat-ai/small-webrtc-transport": "^1.0.0"
}
}

View File

@@ -1,217 +1,236 @@
import { SmallWebRTCTransport } from '@pipecat-ai/small-webrtc-transport';
import {
SmallWebRTCTransport
} from "@pipecat-ai/small-webrtc-transport";
import {Participant, RTVIClient, RTVIClientOptions, Transport} from "@pipecat-ai/client-js";
BotLLMTextData,
Participant,
PipecatClient,
PipecatClientOptions,
TranscriptData,
TransportState,
} from '@pipecat-ai/client-js';
class WebRTCApp {
private declare connectBtn: HTMLButtonElement;
private declare disconnectBtn: HTMLButtonElement;
private declare muteBtn: HTMLButtonElement;
private declare connectBtn: HTMLButtonElement;
private declare disconnectBtn: HTMLButtonElement;
private declare muteBtn: HTMLButtonElement;
private declare audioInput: HTMLSelectElement;
private declare videoInput: HTMLSelectElement;
private declare audioCodec: HTMLSelectElement;
private declare videoCodec: HTMLSelectElement;
private declare audioInput: HTMLSelectElement;
private declare videoInput: HTMLSelectElement;
private declare audioCodec: HTMLSelectElement;
private declare videoCodec: HTMLSelectElement;
private declare videoElement: HTMLVideoElement;
private declare audioElement: HTMLAudioElement;
private declare videoElement: HTMLVideoElement;
private declare audioElement: HTMLAudioElement;
private debugLog: HTMLElement | null = null;
private statusSpan: HTMLElement | null = null;
private debugLog: HTMLElement | null = null;
private statusSpan: HTMLElement | null = null;
private declare smallWebRTCTransport: SmallWebRTCTransport;
private declare pcClient: PipecatClient;
private declare smallWebRTCTransport: SmallWebRTCTransport;
private declare rtviClient: RTVIClient;
constructor() {
this.setupDOMElements();
this.setupDOMEventListeners();
this.initializePipecatClient();
void this.populateDevices();
}
constructor() {
this.setupDOMElements();
this.setupDOMEventListeners();
this.initializeRTVIClient()
void this.populateDevices();
private initializePipecatClient(): void {
const opts: PipecatClientOptions = {
transport: new SmallWebRTCTransport({ connectionUrl: '/api/offer' }),
enableMic: true,
enableCam: true,
callbacks: {
onTransportStateChanged: (state: TransportState) => {
this.log(`Transport state: ${state}`);
},
onConnected: () => {
this.onConnectedHandler();
},
onBotReady: () => {
this.log('Bot is ready.');
},
onDisconnected: () => {
this.onDisconnectedHandler();
},
onUserStartedSpeaking: () => {
this.log('User started speaking.');
},
onUserStoppedSpeaking: () => {
this.log('User stopped speaking.');
},
onBotStartedSpeaking: () => {
this.log('Bot started speaking.');
},
onBotStoppedSpeaking: () => {
this.log('Bot stopped speaking.');
},
onUserTranscript: (transcript: TranscriptData) => {
if (transcript.final) {
this.log(`User transcript: ${transcript.text}`);
}
},
onBotTranscript: (data: BotLLMTextData) => {
this.log(`Bot transcript: ${data.text}`);
},
onTrackStarted: (
track: MediaStreamTrack,
participant?: Participant
) => {
if (participant?.local) {
return;
}
this.onBotTrackStarted(track);
},
onServerMessage: (msg: unknown) => {
this.log(`Server message: ${msg}`);
},
},
};
this.pcClient = new PipecatClient(opts);
this.smallWebRTCTransport = this.pcClient.transport as SmallWebRTCTransport;
}
private setupDOMElements(): void {
this.connectBtn = document.getElementById(
'connect-btn'
) as HTMLButtonElement;
this.disconnectBtn = document.getElementById(
'disconnect-btn'
) as HTMLButtonElement;
this.muteBtn = document.getElementById('mute-btn') as HTMLButtonElement;
this.audioInput = document.getElementById(
'audio-input'
) as HTMLSelectElement;
this.videoInput = document.getElementById(
'video-input'
) as HTMLSelectElement;
this.audioCodec = document.getElementById(
'audio-codec'
) as HTMLSelectElement;
this.videoCodec = document.getElementById(
'video-codec'
) as HTMLSelectElement;
this.videoElement = document.getElementById(
'bot-video'
) as HTMLVideoElement;
this.audioElement = document.getElementById(
'bot-audio'
) as HTMLAudioElement;
this.debugLog = document.getElementById('debug-log');
this.statusSpan = document.getElementById('connection-status');
}
private setupDOMEventListeners(): void {
this.connectBtn.addEventListener('click', () => this.start());
this.disconnectBtn.addEventListener('click', () => this.stop());
this.audioInput.addEventListener('change', (e) => {
// @ts-ignore
let audioDevice = e.target?.value;
this.pcClient.updateMic(audioDevice);
});
this.videoInput.addEventListener('change', (e) => {
// @ts-ignore
let videoDevice = e.target?.value;
this.pcClient.updateCam(videoDevice);
});
this.muteBtn.addEventListener('click', () => {
let isCamEnabled = this.pcClient.isCamEnabled;
this.pcClient.enableCam(!isCamEnabled);
this.muteBtn.textContent = isCamEnabled ? '📵' : '📷';
});
}
private log(message: string): void {
if (!this.debugLog) return;
const entry = document.createElement('div');
entry.textContent = `${new Date().toISOString()} - ${message}`;
if (message.startsWith('User: ')) {
entry.style.color = '#2196F3';
} else if (message.startsWith('Bot: ')) {
entry.style.color = '#4CAF50';
}
this.debugLog.appendChild(entry);
this.debugLog.scrollTop = this.debugLog.scrollHeight;
}
private initializeRTVIClient(): void {
const transport = new SmallWebRTCTransport();
const RTVIConfig: RTVIClientOptions = {
params: {
baseUrl: "/api/offer"
},
transport: transport as Transport,
enableMic: true,
enableCam: true,
callbacks: {
onTransportStateChanged: (state) => {
this.log(`Transport state: ${state}`)
},
onConnected: () => {
this.onConnectedHandler()
},
onBotReady: () => {
this.log("Bot is ready.")
},
onDisconnected: () => {
this.onDisconnectedHandler()
},
onUserStartedSpeaking: () => {
this.log("User started speaking.")
},
onUserStoppedSpeaking: () => {
this.log("User stopped speaking.")
},
onBotStartedSpeaking: () => {
this.log("Bot started speaking.")
},
onBotStoppedSpeaking: () => {
this.log("Bot stopped speaking.")
},
onUserTranscript: (transcript) => {
if (transcript.final) {
this.log(`User transcript: ${transcript.text}`)
}
},
onBotTranscript: (transcript) => {
this.log(`Bot transcript: ${transcript.text}`)
},
onTrackStarted: (track: MediaStreamTrack, participant?: Participant) => {
if (participant?.local) {
return
}
this.onBotTrackStarted(track)
},
onServerMessage: (msg) => {
this.log(`Server message: ${msg}`)
}
},
}
RTVIConfig.customConnectHandler = () => Promise.resolve();
this.rtviClient = new RTVIClient(RTVIConfig);
this.smallWebRTCTransport = transport
private clearAllLogs() {
this.debugLog!.innerText = '';
}
private updateStatus(status: string): void {
if (this.statusSpan) {
this.statusSpan.textContent = status;
}
this.log(`Status: ${status}`);
}
private setupDOMElements(): void {
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
this.muteBtn = document.getElementById('mute-btn') as HTMLButtonElement;
private onConnectedHandler() {
this.updateStatus('Connected');
if (this.connectBtn) this.connectBtn.disabled = true;
if (this.disconnectBtn) this.disconnectBtn.disabled = false;
}
this.audioInput = document.getElementById('audio-input') as HTMLSelectElement;
this.videoInput = document.getElementById('video-input') as HTMLSelectElement;
this.audioCodec = document.getElementById('audio-codec') as HTMLSelectElement;
this.videoCodec = document.getElementById('video-codec') as HTMLSelectElement;
private onDisconnectedHandler() {
this.updateStatus('Disconnected');
if (this.connectBtn) this.connectBtn.disabled = false;
if (this.disconnectBtn) this.disconnectBtn.disabled = true;
}
this.videoElement = document.getElementById('bot-video') as HTMLVideoElement;
this.audioElement = document.getElementById('bot-audio') as HTMLAudioElement;
this.debugLog = document.getElementById('debug-log');
this.statusSpan = document.getElementById('connection-status');
private onBotTrackStarted(track: MediaStreamTrack) {
if (track.kind === 'video') {
this.videoElement.srcObject = new MediaStream([track]);
} else {
this.audioElement.srcObject = new MediaStream([track]);
}
}
private setupDOMEventListeners(): void {
this.connectBtn.addEventListener("click", () => this.start());
this.disconnectBtn.addEventListener("click", () => this.stop());
this.audioInput.addEventListener("change", (e) => {
// @ts-ignore
let audioDevice = e.target?.value
this.rtviClient.updateMic(audioDevice)
})
this.videoInput.addEventListener("change", (e) => {
// @ts-ignore
let videoDevice = e.target?.value
this.rtviClient.updateCam(videoDevice)
})
this.muteBtn.addEventListener('click', () => {
let isCamEnabled = this.rtviClient.isCamEnabled
this.rtviClient.enableCam(!isCamEnabled)
this.muteBtn.textContent = isCamEnabled ? '📵' : '📷';
});
private async populateDevices(): Promise<void> {
const populateSelect = (
select: HTMLSelectElement,
devices: MediaDeviceInfo[]
): void => {
let counter = 1;
devices.forEach((device) => {
const option = document.createElement('option');
option.value = device.deviceId;
option.text = device.label || 'Device #' + counter;
select.appendChild(option);
counter += 1;
});
};
try {
const audioDevices = await this.pcClient.getAllMics();
populateSelect(this.audioInput, audioDevices);
const videoDevices = await this.pcClient.getAllCams();
populateSelect(this.videoInput, videoDevices);
} catch (e) {
alert(e);
}
}
private log(message: string): void {
if (!this.debugLog) return;
const entry = document.createElement('div');
entry.textContent = `${new Date().toISOString()} - ${message}`;
if (message.startsWith('User: ')) {
entry.style.color = '#2196F3';
} else if (message.startsWith('Bot: ')) {
entry.style.color = '#4CAF50';
}
this.debugLog.appendChild(entry);
this.debugLog.scrollTop = this.debugLog.scrollHeight;
private async start(): Promise<void> {
this.clearAllLogs();
this.connectBtn.disabled = true;
this.updateStatus('Connecting');
this.smallWebRTCTransport.setAudioCodec(this.audioCodec.value);
this.smallWebRTCTransport.setVideoCodec(this.videoCodec.value);
try {
await this.pcClient.connect();
} catch (e) {
console.log(`Failed to connect ${e}`);
this.stop();
}
}
private clearAllLogs() {
this.debugLog!.innerText = ''
}
private updateStatus(status: string): void {
if (this.statusSpan) {
this.statusSpan.textContent = status;
}
this.log(`Status: ${status}`);
}
private onConnectedHandler() {
this.updateStatus('Connected');
if (this.connectBtn) this.connectBtn.disabled = true;
if (this.disconnectBtn) this.disconnectBtn.disabled = false;
}
private onDisconnectedHandler() {
this.updateStatus('Disconnected');
if (this.connectBtn) this.connectBtn.disabled = false;
if (this.disconnectBtn) this.disconnectBtn.disabled = true;
}
private onBotTrackStarted(track: MediaStreamTrack) {
if (track.kind === 'video') {
this.videoElement.srcObject = new MediaStream([track]);
} else {
this.audioElement.srcObject = new MediaStream([track]);
}
}
private async populateDevices(): Promise<void> {
const populateSelect = (select: HTMLSelectElement, devices: MediaDeviceInfo[]): void => {
let counter = 1;
devices.forEach((device) => {
const option = document.createElement('option');
option.value = device.deviceId;
option.text = device.label || ('Device #' + counter);
select.appendChild(option);
counter += 1;
});
};
try {
const audioDevices = await this.rtviClient.getAllMics();
populateSelect(this.audioInput, audioDevices);
const videoDevices = await this.rtviClient.getAllCams();
populateSelect(this.videoInput, videoDevices);
} catch (e) {
alert(e);
}
}
private async start(): Promise<void> {
this.clearAllLogs()
this.connectBtn.disabled = true;
this.updateStatus("Connecting")
this.smallWebRTCTransport.setAudioCodec(this.audioCodec.value)
this.smallWebRTCTransport.setVideoCodec(this.videoCodec.value)
try {
await this.rtviClient.connect()
} catch (e) {
console.log(`Failed to connect ${e}`)
this.stop()
}
}
private stop(): void {
void this.rtviClient.disconnect()
}
private stop(): void {
void this.pcClient.disconnect();
}
}
// Create the WebRTCConnection instance

View File

@@ -1,40 +1,51 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>AI Chatbot</title>
</head>
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>AI Chatbot</title>
</head>
<body>
<div class="container">
<div class="status-bar">
<div class="status">
Status: <span id="connection-status">Disconnected</span>
</div>
<div class="controls">
<button id="connect-btn">Connect</button>
<button id="disconnect-btn" disabled>Disconnect</button>
</div>
</div>
<div class="main-content">
<div class="bot-container">
<div id="bot-video-container">
<body>
<div class="container">
<div class="status-bar">
<div class="status">
Status: <span id="connection-status">Disconnected</span>
</div>
<audio id="bot-audio" autoplay></audio>
<div class="controls">
<button id="connect-btn">Connect</button>
<button id="disconnect-btn" disabled>Disconnect</button>
</div>
</div>
<div class="main-content">
<div class="bot-container">
<div id="bot-video-container"></div>
<audio id="bot-audio" autoplay></audio>
</div>
</div>
<div class="device-bar">
<div class="device-controls">
<select id="device-selector"></select>
<button id="mic-toggle-btn">Unmute Mic</button>
</div>
<div class="text-input-container">
<input
type="text"
id="text-input"
placeholder="Type your message..." />
<button id="send-text-btn" disabled>Send</button>
</div>
</div>
<div class="debug-panel">
<h3>Debug Info</h3>
<div id="debug-log"></div>
</div>
</div>
<div class="debug-panel">
<h3>Debug Info</h3>
<div id="debug-log"></div>
</div>
</div>
<script type="module" src="/src/app.js"></script>
<link rel="stylesheet" href="/src/style.css">
</body>
</html>
<script type="module" src="/src/app.js"></script>
<link rel="stylesheet" href="/src/style.css" />
</body>
</html>

File diff suppressed because it is too large Load Diff

View File

@@ -15,7 +15,7 @@
"vite": "^6.3.5"
},
"dependencies": {
"@pipecat-ai/client-js": "^0.3.5",
"@pipecat-ai/daily-transport": "^0.3.8"
"@pipecat-ai/client-js": "^1.0.0",
"@pipecat-ai/daily-transport": "^1.0.0"
}
}

View File

@@ -5,7 +5,7 @@
*/
/**
* RTVI Client Implementation
* Pipecat Client Implementation
*
* This client connects to an RTVI-compatible bot server using WebRTC (via Daily).
* It handles audio/video streaming and manages the connection lifecycle.
@@ -16,7 +16,7 @@
* - Browser with WebRTC support
*/
import { RTVIClient, RTVIEvent } from '@pipecat-ai/client-js';
import { PipecatClient, RTVIEvent } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
/**
@@ -26,9 +26,8 @@ import { DailyTransport } from '@pipecat-ai/daily-transport';
class ChatbotClient {
constructor() {
// Initialize client state
this.rtviClient = null;
this.pcClient = null;
this.setupDOMElements();
this.setupEventListeners();
this.initializeClientAndTransport();
}
@@ -42,6 +41,9 @@ class ChatbotClient {
this.statusSpan = document.getElementById('connection-status');
this.debugLog = document.getElementById('debug-log');
this.botVideoContainer = document.getElementById('bot-video-container');
this.deviceSelector = document.getElementById('device-selector');
this.micToggleBtn = document.getElementById('mic-toggle-btn');
this.sendTextBtn = document.getElementById('send-text-btn');
// Create an audio element for bot's voice output
this.botAudio = document.createElement('audio');
@@ -54,25 +56,78 @@ class ChatbotClient {
* Set up event listeners for connect/disconnect buttons
*/
setupEventListeners() {
this.connectBtn.addEventListener('click', () => this.connect());
this.connectBtn.addEventListener('click', () => {
console.log('click');
this.connect();
});
this.disconnectBtn.addEventListener('click', () => this.disconnect());
// Populate device selector
this.pcClient.getAllMics().then((mics) => {
console.log('Available mics:', mics);
mics.forEach((device) => {
const option = document.createElement('option');
option.value = device.deviceId;
option.textContent = device.label || `Microphone ${device.deviceId}`;
this.deviceSelector.appendChild(option);
});
});
this.deviceSelector.addEventListener('change', (event) => {
const selectedDeviceId = event.target.value;
console.log('Selected device ID:', selectedDeviceId);
this.pcClient.updateMic(selectedDeviceId);
});
// Handle mic mute/unmute toggle
const micToggleBtn = document.getElementById('mic-toggle-btn');
micToggleBtn.addEventListener('click', async () => {
if (this.pcClient.state === 'disconnected') {
await this.pcClient.initDevices();
} else {
this.pcClient.enableMic(!this.pcClient.isMicEnabled);
}
});
const textInput = document.getElementById('text-input');
const sendTextToLLM = () => {
this.sendTextBtn.disabled = true; // Disable button to prevent multiple clicks
const text = textInput.value.trim();
if (text) {
void this.pcClient.appendToContext({
role: 'user',
content: text,
run_immediately: true,
});
}
textInput.value = ''; // Clear the input
this.sendTextBtn.disabled = false; // Re-enable button after sending
};
this.sendTextBtn.addEventListener('click', sendTextToLLM);
// Also handle Enter key in the input
textInput.addEventListener('keypress', (e) => {
if (e.key === 'Enter') {
sendTextToLLM();
}
});
}
updateMicToggleButton(micEnabled) {
console.log('Mic enabled:', micEnabled, this.pcClient?.isMicEnabled);
this.micToggleBtn.textContent = micEnabled ? 'Mute Mic' : 'Unmute Mic';
}
/**
* Set up the RTVI client and Daily transport
* Set up the Pipecat client and Daily transport
*/
initializeClientAndTransport() {
// Initialize the RTVI client with a DailyTransport and our configuration
this.rtviClient = new RTVIClient({
async initializeClientAndTransport() {
console.log('Initializing Pipecat client and transport...');
// Initialize the Pipecat client with a DailyTransport and our configuration
this.pcClient = new PipecatClient({
transport: new DailyTransport(),
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: {
connect: '/connect',
},
},
enableMic: true, // Enable microphone for user input
enableMic: true,
enableCam: false,
callbacks: {
// Handle connection state changes
@@ -86,7 +141,9 @@ class ChatbotClient {
this.updateStatus('Disconnected');
this.connectBtn.disabled = false;
this.disconnectBtn.disabled = true;
this.sendTextBtn.disabled = true;
this.log('Client disconnected');
this.updateMicToggleButton(false);
},
// Handle transport state changes
onTransportStateChanged: (state) => {
@@ -106,6 +163,7 @@ class ChatbotClient {
onBotReady: (data) => {
this.log(`Bot ready: ${JSON.stringify(data)}`);
this.setupMediaTracks();
this.sendTextBtn.disabled = false;
},
// Transcript events
onUserTranscript: (data) => {
@@ -121,14 +179,20 @@ class ChatbotClient {
onMessageError: (error) => {
console.log('Message error:', error);
},
onMicUpdated: (data) => {
console.log('Mic updated:', data);
this.deviceSelector.value = data.deviceId;
},
onError: (error) => {
console.log('Error:', JSON.stringify(error));
},
},
});
window.client = this; // Expose client globally for debugging
// Set up listeners for media track events
this.setupTrackListeners();
this.setupEventListeners();
}
/**
@@ -163,10 +227,10 @@ class ChatbotClient {
* This is called when the bot is ready or when the transport state changes to ready
*/
setupMediaTracks() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Get current tracks from the client
const tracks = this.rtviClient.tracks();
const tracks = this.pcClient.tracks();
// Set up any available bot tracks
if (tracks.bot?.audio) {
@@ -182,27 +246,34 @@ class ChatbotClient {
* This handles new tracks being added during the session
*/
setupTrackListeners() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
if (!participant?.local) {
if (track.kind === 'audio') {
this.setupAudioTrack(track);
} else if (track.kind === 'video') {
this.setupVideoTrack(track);
}
} else if (track.kind === 'audio') {
console.log(`Local audio track started: `, this.pcClient.tracks());
// If local audio track starts, update mic
this.updateMicToggleButton(true);
}
});
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(
`Track stopped event: ${track.kind} from ${
participant?.name || 'unknown'
participant ? (participant.local ? 'local' : 'bot') : 'unknown'
}`
);
if (participant?.local && track.kind === 'audio') {
// If local audio track stops, update mic toggle button
this.updateMicToggleButton(false);
}
});
}
@@ -251,17 +322,16 @@ class ChatbotClient {
/**
* Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection
* This sets up the Pipecat client, initializes devices, and establishes the connection
*/
async connect() {
try {
// Initialize audio/video devices
this.log('Initializing devices...');
await this.rtviClient.initDevices();
// Connect to the bot
this.log('Connecting to bot...');
await this.rtviClient.connect();
await this.pcClient.connect({
endpoint: 'http://localhost:7860/connect',
timeout: 25000,
});
this.log('Connection complete');
} catch (error) {
@@ -271,9 +341,9 @@ class ChatbotClient {
this.updateStatus('Error');
// Clean up if there's an error
if (this.rtviClient) {
if (this.pcClient) {
try {
await this.rtviClient.disconnect();
await this.pcClient.disconnect();
} catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError.message}`);
}
@@ -285,10 +355,10 @@ class ChatbotClient {
* Disconnect from the bot and clean up media resources
*/
async disconnect() {
if (this.rtviClient) {
if (this.pcClient) {
try {
// Disconnect the RTVI client
await this.rtviClient.disconnect();
// Disconnect the Pipecat client
await this.pcClient.disconnect();
// Clean up audio
if (this.botAudio.srcObject) {

View File

@@ -10,7 +10,8 @@ body {
margin: 0 auto;
}
.status-bar {
.status-bar,
.device-bar {
display: flex;
justify-content: space-between;
align-items: center;
@@ -20,7 +21,24 @@ body {
margin-bottom: 20px;
}
.controls button {
.device-bar {
flex-direction: column;
gap: 10px;
}
.controls,
.device-controls {
display: flex;
align-items: center;
gap: 10px; /* Adds spacing between elements */
}
.device-controls {
margin-left: auto;
}
.controls button,
.device-controls button {
padding: 8px 16px;
margin-left: 10px;
border: none;
@@ -28,6 +46,56 @@ body {
cursor: pointer;
}
#bot-selector,
#device-selector {
padding: 8px 16px;
padding-right: 40px;
border: none;
border-radius: 4px;
background-color: #6c757d; /* Gray background */
color: white; /* White text */
cursor: pointer;
appearance: none; /* Removes default browser styling for dropdowns */
background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='white'%3E%3Cpath d='M7 10l5 5 5-5z'/%3E%3C/svg%3E"); /* Custom arrow */
background-repeat: no-repeat;
background-position: right 8px center; /* Position the arrow */
}
#bot-selector:focus,
#device-selector:focus {
outline: none;
box-shadow: 0 0 4px rgba(0, 0, 0, 0.3); /* Add a subtle focus effect */
}
.text-input-container {
display: flex;
gap: 8px;
margin-left: 10px;
width: 100%;
flex: 1;
}
#text-input {
flex: 1;
padding: 8px 16px;
border: 1px solid #e0e0e0;
border-radius: 4px;
min-width: 200px;
}
#send-text-btn {
padding: 8px 16px;
border: none;
border-radius: 4px;
background-color: #007bff;
color: white;
flex-shrink: 0;
}
#send-text-btn:hover {
background-color: #0056b3;
}
#connect-btn {
background-color: #4caf50;
color: white;
@@ -38,6 +106,9 @@ body {
color: white;
}
#mic-toggle-btn {
}
button:disabled {
opacity: 0.5;
cursor: not-allowed;

File diff suppressed because it is too large Load Diff

View File

@@ -10,9 +10,9 @@
"preview": "vite preview"
},
"dependencies": {
"@pipecat-ai/client-js": "^0.3.5",
"@pipecat-ai/client-react": "^0.3.5",
"@pipecat-ai/daily-transport": "^0.3.8",
"@pipecat-ai/client-js": "^1.0.0",
"@pipecat-ai/client-react": "^1.0.0",
"@pipecat-ai/daily-transport": "^1.0.0",
"react": "^18.3.1",
"react-dom": "^18.3.1"
},

View File

@@ -1,22 +1,22 @@
import {
RTVIClientAudio,
RTVIClientVideo,
useRTVIClientTransportState,
PipecatClientAudio,
PipecatClientVideo,
usePipecatClientTransportState,
} from '@pipecat-ai/client-react';
import { RTVIProvider } from './providers/RTVIProvider';
import { PipecatProvider } from './providers/PipecatProvider';
import { ConnectButton } from './components/ConnectButton';
import { StatusDisplay } from './components/StatusDisplay';
import { DebugDisplay } from './components/DebugDisplay';
import './App.css';
function BotVideo() {
const transportState = useRTVIClientTransportState();
const transportState = usePipecatClientTransportState();
const isConnected = transportState !== 'disconnected';
return (
<div className="bot-container">
<div className="video-container">
{isConnected && <RTVIClientVideo participant="bot" fit="cover" />}
{isConnected && <PipecatClientVideo participant="bot" fit="cover" />}
</div>
</div>
);
@@ -35,16 +35,16 @@ function AppContent() {
</div>
<DebugDisplay />
<RTVIClientAudio />
<PipecatClientAudio />
</div>
);
}
function App() {
return (
<RTVIProvider>
<PipecatProvider>
<AppContent />
</RTVIProvider>
</PipecatProvider>
);
}

View File

@@ -1,16 +1,16 @@
import {
useRTVIClient,
useRTVIClientTransportState,
usePipecatClient,
usePipecatClientTransportState,
} from '@pipecat-ai/client-react';
export function ConnectButton() {
const client = useRTVIClient();
const transportState = useRTVIClientTransportState();
const client = usePipecatClient();
const transportState = usePipecatClientTransportState();
const isConnected = ['connected', 'ready'].includes(transportState);
const handleClick = async () => {
if (!client) {
console.error('RTVI client is not initialized');
console.error('Pipecat client is not initialized');
return;
}
@@ -18,7 +18,7 @@ export function ConnectButton() {
if (isConnected) {
await client.disconnect();
} else {
await client.connect();
await client.connect({ endpoint: 'http://localhost:7860/connect' });
}
} catch (error) {
console.error('Connection error:', error);

View File

@@ -6,12 +6,12 @@ import {
TranscriptData,
BotLLMTextData,
} from '@pipecat-ai/client-js';
import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
import { usePipecatClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
import './DebugDisplay.css';
export function DebugDisplay() {
const debugLogRef = useRef<HTMLDivElement>(null);
const client = useRTVIClient();
const client = usePipecatClient();
const log = useCallback((message: string) => {
if (!debugLogRef.current) return;

View File

@@ -1,7 +1,7 @@
import { useRTVIClientTransportState } from '@pipecat-ai/client-react';
import { usePipecatClientTransportState } from '@pipecat-ai/client-react';
export function StatusDisplay() {
const transportState = useRTVIClientTransportState();
const transportState = usePipecatClientTransportState();
return (
<div className="status">

View File

@@ -0,0 +1,16 @@
import { type PropsWithChildren } from 'react';
import { PipecatClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { PipecatClientProvider } from '@pipecat-ai/client-react';
const client = new PipecatClient({
transport: new DailyTransport(),
enableMic: true,
enableCam: false,
});
export function PipecatProvider({ children }: PropsWithChildren) {
return (
<PipecatClientProvider client={client}>{children}</PipecatClientProvider>
);
}

View File

@@ -1,22 +0,0 @@
import { type PropsWithChildren } from 'react';
import { RTVIClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { RTVIClientProvider } from '@pipecat-ai/client-react';
const transport = new DailyTransport();
const client = new RTVIClient({
transport,
params: {
baseUrl: 'http://localhost:7860',
endpoints: {
connect: '/connect',
},
},
enableMic: true,
enableCam: false,
});
export function RTVIProvider({ children }: PropsWithChildren) {
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
}

File diff suppressed because it is too large Load Diff

View File

@@ -19,7 +19,7 @@
"vite": "^6.0.2"
},
"dependencies": {
"@pipecat-ai/client-js": "^0.4.0",
"@pipecat-ai/websocket-transport": "^0.4.2"
"@pipecat-ai/client-js": "^1.0.0",
"@pipecat-ai/websocket-transport": "^1.0.0"
}
}

View File

@@ -5,21 +5,22 @@
*/
import {
RTVIClient,
RTVIClientOptions,
RTVIEvent,
BotLLMTextData,
Participant,
PipecatClient,
PipecatClientOptions,
RTVIEvent, RTVIMessage, TranscriptData,
} from '@pipecat-ai/client-js';
import {
WebSocketTransport,
TwilioSerializer,
} from "@pipecat-ai/websocket-transport";
} from '@pipecat-ai/websocket-transport';
class WebsocketClientApp {
private static STREAM_SID = 'ws_mock_stream_sid';
private static CALL_SID = 'ws_mock_call_sid';
private static STREAM_SID = "ws_mock_stream_sid"
private static CALL_SID = "ws_mock_call_sid"
private rtviClient: RTVIClient | null = null;
private rtviClient: PipecatClient | null = null;
private connectBtn: HTMLButtonElement | null = null;
private disconnectBtn: HTMLButtonElement | null = null;
private statusSpan: HTMLElement | null = null;
@@ -38,8 +39,12 @@ class WebsocketClientApp {
* Set up references to DOM elements and create necessary media elements
*/
private setupDOMElements(): void {
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
this.connectBtn = document.getElementById(
'connect-btn'
) as HTMLButtonElement;
this.disconnectBtn = document.getElementById(
'disconnect-btn'
) as HTMLButtonElement;
this.statusSpan = document.getElementById('connection-status');
this.debugLog = document.getElementById('debug-log');
}
@@ -80,13 +85,23 @@ class WebsocketClientApp {
}
private async emulateTwilioMessages() {
const connectedMessage={"event": "connected", "protocol": "Call", "version": "1.0.0"}
const connectedMessage = {
event: 'connected',
protocol: 'Call',
version: '1.0.0',
};
const websocketTransport = this.rtviClient?.transport as WebSocketTransport
void websocketTransport?.sendRawMessage(connectedMessage)
const websocketTransport = this.rtviClient?.transport as WebSocketTransport;
void websocketTransport?.sendRawMessage(connectedMessage);
const startMessage={"event": "start", "start": {"streamSid": WebsocketClientApp.STREAM_SID, "callSid": WebsocketClientApp.CALL_SID}}
void websocketTransport?.sendRawMessage(startMessage)
const startMessage = {
event: 'start',
start: {
streamSid: WebsocketClientApp.STREAM_SID,
callSid: WebsocketClientApp.CALL_SID,
},
};
void websocketTransport?.sendRawMessage(startMessage);
}
/**
@@ -109,7 +124,7 @@ class WebsocketClientApp {
if (!this.rtviClient) return;
// Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
this.rtviClient.on(RTVIEvent.TrackStarted, (track: MediaStreamTrack, participant?: Participant) => {
// Only handle non-local (bot) tracks
if (!participant?.local && track.kind === 'audio') {
this.setupAudioTrack(track);
@@ -117,8 +132,10 @@ class WebsocketClientApp {
});
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`);
this.rtviClient.on(RTVIEvent.TrackStopped, (track: MediaStreamTrack, participant?: Participant) => {
this.log(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
);
});
}
@@ -128,7 +145,10 @@ class WebsocketClientApp {
*/
private setupAudioTrack(track: MediaStreamTrack): void {
this.log('Setting up audio track');
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
if (oldTrack?.id === track.id) return;
}
@@ -143,23 +163,19 @@ class WebsocketClientApp {
try {
const startTime = Date.now();
const transport = new WebSocketTransport({
const ws_opts = {
serializer: new TwilioSerializer(),
recorderSampleRate: 8000,
playerSampleRate: 8000
});
const RTVIConfig: RTVIClientOptions = {
transport,
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:8765',
endpoints: { connect: '/' },
},
playerSampleRate: 8000,
ws_url: 'http://localhost:8765/ws',
};
const RTVIConfig: PipecatClientOptions = {
transport: new WebSocketTransport(ws_opts),
enableMic: true,
enableCam: false,
callbacks: {
onConnected: () => {
this.emulateTwilioMessages()
this.emulateTwilioMessages();
this.updateStatus('Connected');
if (this.connectBtn) this.connectBtn.disabled = true;
if (this.disconnectBtn) this.disconnectBtn.disabled = false;
@@ -170,27 +186,21 @@ class WebsocketClientApp {
if (this.disconnectBtn) this.disconnectBtn.disabled = true;
this.log('Client disconnected');
},
onBotReady: (data) => {
onBotReady: (data: any) => {
this.log(`Bot ready: ${JSON.stringify(data)}`);
this.setupMediaTracks();
},
onUserTranscript: (data) => {
onUserTranscript: (data: TranscriptData) => {
if (data.final) {
this.log(`User: ${data.text}`);
}
},
onBotTranscript: (data) => this.log(`Bot: ${data.text}`),
onMessageError: (error) => console.error('Message error:', error),
onError: (error) => console.error('Error:', error),
onBotTranscript: (data: BotLLMTextData) => this.log(`Bot: ${data.text}`),
onMessageError: (error: RTVIMessage) => console.error('Message error:', error),
onError: (error: RTVIMessage) => console.error('Error:', error),
},
}
// @ts-ignore
RTVIConfig.customConnectHandler = () => Promise.resolve(
{
ws_url: "/ws",
}
);
this.rtviClient = new RTVIClient(RTVIConfig);
};
this.rtviClient = new PipecatClient(RTVIConfig);
this.setupTrackListeners();
this.log('Initializing devices...');
@@ -223,8 +233,13 @@ class WebsocketClientApp {
try {
await this.rtviClient.disconnect();
this.rtviClient = null;
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop());
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
this.botAudio.srcObject
.getAudioTracks()
.forEach((track) => track.stop());
this.botAudio.srcObject = null;
}
} catch (error) {
@@ -232,7 +247,6 @@ class WebsocketClientApp {
}
}
}
}
declare global {

File diff suppressed because it is too large Load Diff

View File

@@ -19,8 +19,8 @@
"vite": "^6.3.5"
},
"dependencies": {
"@pipecat-ai/client-js": "^0.4.0",
"@pipecat-ai/websocket-transport": "^0.4.2",
"@pipecat-ai/client-js": "^1.0.0",
"@pipecat-ai/websocket-transport": "^1.0.0",
"protobufjs": "^7.4.0"
}
}

View File

@@ -5,7 +5,7 @@
*/
/**
* RTVI Client Implementation
* Pipecat Client Implementation
*
* This client connects to an RTVI-compatible bot server using WebSocket.
*
@@ -14,16 +14,14 @@
*/
import {
RTVIClient,
RTVIClientOptions,
PipecatClient,
PipecatClientOptions,
RTVIEvent,
} from '@pipecat-ai/client-js';
import {
WebSocketTransport
} from "@pipecat-ai/websocket-transport";
import { WebSocketTransport } from '@pipecat-ai/websocket-transport';
class WebsocketClientApp {
private rtviClient: RTVIClient | null = null;
private pcClient: PipecatClient | null = null;
private connectBtn: HTMLButtonElement | null = null;
private disconnectBtn: HTMLButtonElement | null = null;
private statusSpan: HTMLElement | null = null;
@@ -31,7 +29,7 @@ class WebsocketClientApp {
private botAudio: HTMLAudioElement;
constructor() {
console.log("WebsocketClientApp");
console.log('WebsocketClientApp');
this.botAudio = document.createElement('audio');
this.botAudio.autoplay = true;
//this.botAudio.playsInline = true;
@@ -45,8 +43,12 @@ class WebsocketClientApp {
* Set up references to DOM elements and create necessary media elements
*/
private setupDOMElements(): void {
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
this.connectBtn = document.getElementById(
'connect-btn'
) as HTMLButtonElement;
this.disconnectBtn = document.getElementById(
'disconnect-btn'
) as HTMLButtonElement;
this.statusSpan = document.getElementById('connection-status');
this.debugLog = document.getElementById('debug-log');
}
@@ -91,8 +93,8 @@ class WebsocketClientApp {
* This is called when the bot is ready or when the transport state changes to ready
*/
setupMediaTracks() {
if (!this.rtviClient) return;
const tracks = this.rtviClient.tracks();
if (!this.pcClient) return;
const tracks = this.pcClient.tracks();
if (tracks.bot?.audio) {
this.setupAudioTrack(tracks.bot.audio);
}
@@ -103,10 +105,10 @@ class WebsocketClientApp {
* This handles new tracks being added during the session
*/
setupTrackListeners() {
if (!this.rtviClient) return;
if (!this.pcClient) return;
// Listen for new tracks starting
this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
this.pcClient.on(RTVIEvent.TrackStarted, (track, participant) => {
// Only handle non-local (bot) tracks
if (!participant?.local && track.kind === 'audio') {
this.setupAudioTrack(track);
@@ -114,8 +116,10 @@ class WebsocketClientApp {
});
// Listen for tracks stopping
this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`);
this.pcClient.on(RTVIEvent.TrackStopped, (track, participant) => {
this.log(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
);
});
}
@@ -125,7 +129,10 @@ class WebsocketClientApp {
*/
private setupAudioTrack(track: MediaStreamTrack): void {
this.log('Setting up audio track');
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
if (oldTrack?.id === track.id) return;
}
@@ -134,21 +141,15 @@ class WebsocketClientApp {
/**
* Initialize and connect to the bot
* This sets up the RTVI client, initializes devices, and establishes the connection
* This sets up the Pipecat client, initializes devices, and establishes the connection
*/
public async connect(): Promise<void> {
try {
const startTime = Date.now();
//const transport = new DailyTransport();
const transport = new WebSocketTransport();
const RTVIConfig: RTVIClientOptions = {
transport,
params: {
// The baseURL and endpoint of your bot server that the client will connect to
baseUrl: 'http://localhost:7860',
endpoints: { connect: '/connect' },
},
const PipecatConfig: PipecatClientOptions = {
transport: new WebSocketTransport(),
enableMic: true,
enableCam: false,
callbacks: {
@@ -176,15 +177,20 @@ class WebsocketClientApp {
onMessageError: (error) => console.error('Message error:', error),
onError: (error) => console.error('Error:', error),
},
}
this.rtviClient = new RTVIClient(RTVIConfig);
};
this.pcClient = new PipecatClient(PipecatConfig);
// @ts-ignore
window.pcClient = this.pcClient; // Expose for debugging
this.setupTrackListeners();
this.log('Initializing devices...');
await this.rtviClient.initDevices();
await this.pcClient.initDevices();
this.log('Connecting to bot...');
await this.rtviClient.connect();
await this.pcClient.connect({
// The baseURL and endpoint of your bot server that the client will connect to
endpoint: 'http://localhost:7860/connect',
});
const timeTaken = Date.now() - startTime;
this.log(`Connection complete, timeTaken: ${timeTaken}`);
@@ -192,9 +198,9 @@ class WebsocketClientApp {
this.log(`Error connecting: ${(error as Error).message}`);
this.updateStatus('Error');
// Clean up if there's an error
if (this.rtviClient) {
if (this.pcClient) {
try {
await this.rtviClient.disconnect();
await this.pcClient.disconnect();
} catch (disconnectError) {
this.log(`Error during disconnect: ${disconnectError}`);
}
@@ -206,12 +212,17 @@ class WebsocketClientApp {
* Disconnect from the bot and clean up media resources
*/
public async disconnect(): Promise<void> {
if (this.rtviClient) {
if (this.pcClient) {
try {
await this.rtviClient.disconnect();
this.rtviClient = null;
if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop());
await this.pcClient.disconnect();
this.pcClient = null;
if (
this.botAudio.srcObject &&
'getAudioTracks' in this.botAudio.srcObject
) {
this.botAudio.srcObject
.getAudioTracks()
.forEach((track) => track.stop());
this.botAudio.srcObject = null;
}
} catch (error) {
@@ -219,7 +230,6 @@ class WebsocketClientApp {
}
}
}
}
declare global {

File diff suppressed because it is too large Load Diff

View File

@@ -9,11 +9,12 @@
"lint": "next lint"
},
"dependencies": {
"@pipecat-ai/client-js": "^0.3.5",
"@pipecat-ai/client-react": "^0.3.5",
"@pipecat-ai/daily-transport": "^0.3.10",
"@pipecat-ai/client-js": "^1.0.0",
"@pipecat-ai/client-react": "^1.0.0",
"@pipecat-ai/daily-transport": "^1.0.0",
"@tabler/icons-react": "^3.31.0",
"@tailwindcss/postcss": "^4.1.3",
"jotai": "^2.12.5",
"js-confetti": "^0.12.0",
"next": "15.2.4",
"react": "^19.0.0",

View File

@@ -1,16 +1,26 @@
import { useEffect, useCallback } from 'react';
import {
useRTVIClient,
useRTVIClientTransportState,
usePipecatClient,
usePipecatClientTransportState,
} from '@pipecat-ai/client-react';
import { CONNECTION_STATES } from '@/constants/gameConstants';
import { useConfigurationSettings } from '@/contexts/Configuration';
// Get the API base URL from environment variables
// Default to "/api" if not specified
// "/api" is the default for Next.js API routes and used
// for the Pipecat Cloud deployed agent
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
console.log('Using API base URL:', API_BASE_URL);
export function useConnectionState(
onConnected?: () => void,
onDisconnected?: () => void
) {
const client = useRTVIClient();
const transportState = useRTVIClientTransportState();
const client = usePipecatClient();
const transportState = usePipecatClientTransportState();
const config = useConfigurationSettings();
const isConnected = CONNECTION_STATES.ACTIVE.includes(transportState);
const isConnecting = CONNECTION_STATES.CONNECTING.includes(transportState);
@@ -35,12 +45,17 @@ export function useConnectionState(
if (isConnected) {
await client.disconnect();
} else {
await client.connect();
await client.connect({
endpoint: `${API_BASE_URL}/connect`,
requestData: {
personality: config.personality,
},
});
}
} catch (error) {
console.error('Connection error:', error);
}
}, [client, isConnected]);
}, [client, config, isConnected]);
return {
isConnected,

View File

@@ -1,15 +1,15 @@
import { ConfigurationProvider } from "@/contexts/Configuration";
import { RTVIProvider } from "@/providers/RTVIProvider";
import { RTVIClientAudio } from "@pipecat-ai/client-react";
import type { AppProps } from "next/app";
import { Nunito } from "next/font/google";
import Head from "next/head";
import "../styles/globals.css";
import { ConfigurationProvider } from '@/contexts/Configuration';
import { PipecatProvider } from '@/providers/PipecatProvider';
import { PipecatClientAudio } from '@pipecat-ai/client-react';
import type { AppProps } from 'next/app';
import { Nunito } from 'next/font/google';
import Head from 'next/head';
import '../styles/globals.css';
const nunito = Nunito({
subsets: ["latin"],
display: "swap",
variable: "--font-sans",
subsets: ['latin'],
display: 'swap',
variable: '--font-sans',
});
export default function App({ Component, pageProps }: AppProps) {
@@ -21,10 +21,10 @@ export default function App({ Component, pageProps }: AppProps) {
</Head>
<main className={`${nunito.variable}`}>
<ConfigurationProvider>
<RTVIProvider>
<RTVIClientAudio />
<PipecatProvider>
<PipecatClientAudio />
<Component {...pageProps} />
</RTVIProvider>
</PipecatProvider>
</ConfigurationProvider>
</main>
</>

View File

@@ -1,11 +1,11 @@
import type { NextApiRequest, NextApiResponse } from "next";
import type { NextApiRequest, NextApiResponse } from 'next';
export default async function handler(
req: NextApiRequest,
res: NextApiResponse
) {
if (req.method !== "POST") {
return res.status(405).json({ error: "Method not allowed" });
if (req.method !== 'POST') {
return res.status(405).json({ error: 'Method not allowed' });
}
try {
@@ -15,16 +15,16 @@ export default async function handler(
if (!personality) {
return res
.status(400)
.json({ error: "Missing required configuration parameters" });
.json({ error: 'Missing required configuration parameters' });
}
const response = await fetch(
`https://api.pipecat.daily.co/v1/public/${process.env.AGENT_NAME}/start`,
{
method: "POST",
method: 'POST',
headers: {
Authorization: `Bearer ${process.env.PIPECAT_CLOUD_API_KEY}`,
"Content-Type": "application/json",
'Content-Type': 'application/json',
},
body: JSON.stringify({
createDailyRoom: true,
@@ -37,15 +37,15 @@ export default async function handler(
const data = await response.json();
console.log("Response from API:", JSON.stringify(data, null, 2));
console.log('Response from API:', JSON.stringify(data, null, 2));
// Transform the response to match what RTVI client expects
// Transform the response to match what Pipecat client expects
return res.status(200).json({
room_url: data.dailyRoom,
token: data.dailyToken,
});
} catch (error) {
console.error("Error starting agent:", error);
return res.status(500).json({ error: "Failed to start agent" });
console.error('Error starting agent:', error);
return res.status(500).json({ error: 'Failed to start agent' });
}
}

View File

@@ -0,0 +1,43 @@
'use client';
import { PipecatClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { PipecatClientProvider } from '@pipecat-ai/client-react';
import { PropsWithChildren, useEffect, useState, useRef } from 'react';
export function PipecatProvider({ children }: PropsWithChildren) {
const [client, setClient] = useState<PipecatClient | null>(null);
const clientCreated = useRef(false);
useEffect(() => {
// Only create the client once
if (clientCreated.current) return;
const pcClient = new PipecatClient({
transport: new DailyTransport(),
enableMic: true,
enableCam: false,
});
setClient(pcClient);
clientCreated.current = true;
// Cleanup when component unmounts
return () => {
if (pcClient) {
pcClient.disconnect().catch((err) => {
console.error('Error disconnecting client:', err);
});
}
clientCreated.current = false;
};
}, []);
if (!client) {
return null;
}
return (
<PipecatClientProvider client={client}>{children}</PipecatClientProvider>
);
}

View File

@@ -1,72 +0,0 @@
"use client";
import { RTVIClient } from "@pipecat-ai/client-js";
import { DailyTransport } from "@pipecat-ai/daily-transport";
import { RTVIClientProvider } from "@pipecat-ai/client-react";
import { PropsWithChildren, useEffect, useState, useRef } from "react";
import { useConfigurationSettings } from "@/contexts/Configuration";
// Get the API base URL from environment variables
// Default to "/api" if not specified
// "/api" is the default for Next.js API routes and used
// for the Pipecat Cloud deployed agent
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || "/api";
console.log("Using API base URL:", API_BASE_URL);
export function RTVIProvider({ children }: PropsWithChildren) {
const [client, setClient] = useState<RTVIClient | null>(null);
const config = useConfigurationSettings();
const clientCreated = useRef(false);
useEffect(() => {
// Only create the client once
if (clientCreated.current) return;
const transport = new DailyTransport();
const rtviClient = new RTVIClient({
transport,
params: {
baseUrl: API_BASE_URL,
endpoints: {
connect: "/connect",
},
requestData: {
personality: config.personality,
},
},
enableMic: true,
enableCam: false,
});
setClient(rtviClient);
clientCreated.current = true;
// Cleanup when component unmounts
return () => {
if (rtviClient) {
rtviClient.disconnect().catch((err) => {
console.error("Error disconnecting client:", err);
});
}
clientCreated.current = false;
};
}, []);
// Update the connectParams when config changes
useEffect(() => {
if (!client) return;
// Update the connect params without recreating the client
client.params.requestData = {
personality: config.personality,
};
}, [client, config.personality]);
if (!client) {
return null;
}
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
}

View File

@@ -4,6 +4,7 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
import argparse
import asyncio
import os
import sys
@@ -198,16 +199,15 @@ async def bot(args: DailySessionArguments):
# Local development
async def local_daily():
async def local_daily(args: DailySessionArguments):
"""Daily transport for local development."""
from runner import configure
# from runner import configure
try:
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
transport = DailyTransport(
room_url,
token,
room_url=args.room_url,
token=args.token,
bot_name="Bot",
params=DailyParams(
audio_in_enabled=True,
@@ -217,7 +217,7 @@ async def local_daily():
)
test_config = {
"personality": "witty",
"personality": args.personality,
}
await main(transport, test_config)
@@ -227,7 +227,24 @@ async def local_daily():
# Local development entry point
if LOCAL_RUN and __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Run the Word Wrangler bot in local development mode"
)
parser.add_argument(
"-u", "--room-url", type=str, default=os.getenv("DAILY_SAMPLE_ROOM_URL", "")
)
parser.add_argument(
"-t", "--token", type=str, default=os.getenv("DAILY_SAMPLE_ROOM_TOKEN", None)
)
parser.add_argument(
"-p",
"--personality",
default="witty",
choices=["friendly", "professional", "enthusiastic", "thoughtful", "witty"],
help="Personality preset for the bot (friendly, professional, enthusiastic, thoughtful, witty)",
)
args = parser.parse_args()
try:
asyncio.run(local_daily())
asyncio.run(local_daily(args))
except Exception as e:
logger.exception(f"Failed to run in local mode: {e}")

View File

@@ -160,14 +160,15 @@ async def rtvi_connect(request: Request) -> Dict[Any, Any]:
Raises:
HTTPException: If room creation, token generation, or bot startup fails
"""
print("Creating room for RTVI connection")
body = await request.json()
print("Creating room for RTVI connection", body)
room_url, token = await create_room_and_token()
print(f"Room URL: {room_url}")
# Start the bot process
try:
proc = subprocess.Popen(
[f"python3 -m bot -u {room_url} -t {token}"],
[f"python3 -m bot -u {room_url} -t {token} -p {body.get('personality', 'witty')}"],
shell=True,
bufsize=1,
cwd=os.path.dirname(os.path.abspath(__file__)),