Compare commits
7 Commits
hush/usage
...
cb/pcc-tra
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
974e989cb5 | ||
|
|
8f33f00c3f | ||
|
|
0fede6bb48 | ||
|
|
943c75c622 | ||
|
|
46b2e925f2 | ||
|
|
e6e8a03c5f | ||
|
|
e5ee3e3720 |
@@ -15,12 +15,16 @@ from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||
from pipecat.services.gemini_multimodal_live import GeminiMultimodalLiveLLMService
|
||||
from pipecat.transports.base_transport import TransportParams
|
||||
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||
from pipecat.transports.services.pipecat_cloud import (
|
||||
PipecatCloudParams,
|
||||
PipecatCloudTransport,
|
||||
SessionArguments,
|
||||
)
|
||||
|
||||
load_dotenv(override=True)
|
||||
|
||||
logger.remove(0)
|
||||
# logger.remove(0)
|
||||
logger.add(sys.stderr, level="DEBUG")
|
||||
|
||||
|
||||
@@ -36,9 +40,11 @@ Respond to what the user said in a creative and helpful way. Keep your responses
|
||||
|
||||
|
||||
async def run_bot(webrtc_connection):
|
||||
pipecat_transport = SmallWebRTCTransport(
|
||||
webrtc_connection=webrtc_connection,
|
||||
params=TransportParams(
|
||||
pipecat_transport = PipecatCloudTransport(
|
||||
session_args=SessionArguments(
|
||||
webrtc_connection=webrtc_connection,
|
||||
),
|
||||
params=PipecatCloudParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_enabled=True,
|
||||
|
||||
157
examples/pcc-transport-chatbot/README.md
Normal file
@@ -0,0 +1,157 @@
|
||||
# Simple Chatbot for Pipecat Cloud
|
||||
|
||||
This project demonstrates how to build a complete Pipecat AI agent application with both client and server components. It includes a Next.js client for interacting with a Pipecat AI bot server through Daily.co's WebRTC transport.
|
||||
|
||||
<img src="image.png" width="420px">
|
||||
|
||||
## Project Overview
|
||||
|
||||
- **Server**: Python-based Pipecat bot with video/audio processing capabilities
|
||||
- **Client**: Next.js TypeScript web application using the Pipecat React & JS SDKs
|
||||
- **Infrastructure**: Deployable to Pipecat Cloud (server) and Vercel (client)
|
||||
|
||||
> See the [simple-chatbot example](https://github.com/pipecat-ai/pipecat/tree/main/examples/simple-chatbot) with different client and server implementations.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Server Setup
|
||||
|
||||
Navigate to the server directory:
|
||||
|
||||
```bash
|
||||
cd server
|
||||
```
|
||||
|
||||
Create and activate a virtual environment:
|
||||
|
||||
```bash
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate # On Windows: venv\Scripts\activate
|
||||
```
|
||||
|
||||
Install requirements:
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
Copy env.example to .env and add your API keys:
|
||||
|
||||
```bash
|
||||
cp env.example .env
|
||||
# Edit .env to add OPENAI_API_KEY and CARTESIA_API_KEY
|
||||
```
|
||||
|
||||
Run the server locally to test before deploying:
|
||||
|
||||
```bash
|
||||
LOCAL_RUN=1 python bot.py
|
||||
```
|
||||
|
||||
This will open a browser window with a Daily.co room where you can test your bot directly.
|
||||
|
||||
### 2. Client Setup
|
||||
|
||||
In a separate terminal, navigate to the client directory:
|
||||
|
||||
```bash
|
||||
cd client-react
|
||||
```
|
||||
|
||||
Install dependencies:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
Create `.env.local` file with your Pipecat Cloud API key:
|
||||
|
||||
```bash
|
||||
cp env.local.example .env.local
|
||||
```
|
||||
|
||||
> Create a Pipecat Cloud API key using the dashboard
|
||||
|
||||
Start the development server:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Open [http://localhost:3000](http://localhost:3000) to interact with your agent through the Next.js client.
|
||||
|
||||
## Deployment
|
||||
|
||||
> See the [Pipecat Cloud Quickstart](https://docs.pipecat.daily.co/quickstart) for a complete walkthrough.
|
||||
|
||||
### Deploy Server to Pipecat Cloud
|
||||
|
||||
1. Install the Pipecat Cloud CLI:
|
||||
|
||||
```bash
|
||||
pip install pipecatcloud
|
||||
```
|
||||
|
||||
2. Authenticate:
|
||||
|
||||
```bash
|
||||
pcc auth login
|
||||
```
|
||||
|
||||
3. Build and push your Docker image:
|
||||
|
||||
```bash
|
||||
cd server
|
||||
chmod +x build.sh
|
||||
./build.sh
|
||||
```
|
||||
|
||||
> IMPORTANT: Before running this build script, you need to add your DOCKER_USERNAME
|
||||
|
||||
4. Create a secret set for your API keys:
|
||||
|
||||
```bash
|
||||
pcc secrets set simple-chatbot-secrets --file .env
|
||||
```
|
||||
|
||||
5. Deploy to Pipecat Cloud:
|
||||
|
||||
```bash
|
||||
pcc deploy
|
||||
```
|
||||
|
||||
> IMPORTANT: Before deploying, you need to add your Docker Hub username
|
||||
|
||||
### Deploy Client to Vercel
|
||||
|
||||
1. Push your Next.js client to GitHub
|
||||
|
||||
2. Connect your GitHub repository to Vercel
|
||||
|
||||
3. Add your `PIPECAT_CLOUD_API_KEY` environment variable in Vercel
|
||||
|
||||
4. Deploy with the Vercel dashboard or CLI
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
simple-chatbot/
|
||||
├── client-next/ # Next.js client application
|
||||
│ ├── src/
|
||||
│ │ ├── app/ # Next.js app routes
|
||||
│ │ │ └── api/
|
||||
│ │ │ └── connect/ # API endpoint for Daily.co connection
|
||||
│ │ ├── components/ # React components
|
||||
│ │ └── providers/ # React providers including RTVIProvider
|
||||
│ ├── package.json
|
||||
│ └── README.md # Client-specific documentation
|
||||
│
|
||||
└── server/ # Pipecat bot server
|
||||
├── assets/ # Robot animation frames
|
||||
├── bot.py # The Pipecat pipeline implementation
|
||||
├── Dockerfile # For building the container image
|
||||
├── build.sh # Script for building and pushing Docker image
|
||||
├── requirements.txt # Python dependencies
|
||||
├── pcc-deploy.toml # Pipecat Cloud deployment config
|
||||
└── README.md # Server-specific documentation
|
||||
```
|
||||
41
examples/pcc-transport-chatbot/client-react/.gitignore
vendored
Normal file
@@ -0,0 +1,41 @@
|
||||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
/.pnp
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/versions
|
||||
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# next.js
|
||||
/.next/
|
||||
/out/
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
*.pem
|
||||
|
||||
# debug
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env*
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
# typescript
|
||||
*.tsbuildinfo
|
||||
next-env.d.ts
|
||||
33
examples/pcc-transport-chatbot/client-react/README.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Simple Chatbot Client
|
||||
|
||||
A Next.js application using TypeScript and the Pipecat React SDK to connect to a Pipecat AI agent.
|
||||
|
||||
## Features
|
||||
|
||||
- Next.js App Router architecture
|
||||
- TypeScript for type safety
|
||||
- RTVI client integration for real-time voice and video
|
||||
- Daily.co WebRTC transport
|
||||
- Custom API endpoint for Daily room creation
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. Install dependencies:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
2. Create `.env.local` file with your Pipecat Cloud API key:
|
||||
|
||||
```
|
||||
PIPECAT_CLOUD_API_KEY=your_pipecat_cloud_key
|
||||
```
|
||||
|
||||
3. Start the development server:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
4. Open [http://localhost:3000](http://localhost:3000) in your browser
|
||||
@@ -0,0 +1 @@
|
||||
PIPECAT_CLOUD_API_KEY=your_api_key_here
|
||||
@@ -0,0 +1,16 @@
|
||||
import { dirname } from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import { FlatCompat } from "@eslint/eslintrc";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
const compat = new FlatCompat({
|
||||
baseDirectory: __dirname,
|
||||
});
|
||||
|
||||
const eslintConfig = [
|
||||
...compat.extends("next/core-web-vitals", "next/typescript"),
|
||||
];
|
||||
|
||||
export default eslintConfig;
|
||||
@@ -0,0 +1,7 @@
|
||||
import type { NextConfig } from "next";
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
/* config options here */
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
5419
examples/pcc-transport-chatbot/client-react/package-lock.json
generated
Normal file
28
examples/pcc-transport-chatbot/client-react/package.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "my-nextjs-app",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"lint": "next lint"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pipecat-ai/client-js": "^0.3.5",
|
||||
"@pipecat-ai/client-react": "^0.3.5",
|
||||
"@pipecat-ai/daily-transport": "^0.3.7",
|
||||
"next": "15.2.3",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/eslintrc": "^3",
|
||||
"@types/node": "^20",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"eslint": "^9",
|
||||
"eslint-config-next": "15.2.3",
|
||||
"typescript": "^5"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { NextResponse, NextRequest } from 'next/server';
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const { MY_CUSTOM_DATA } = await request.json();
|
||||
|
||||
const response = await fetch(
|
||||
'https://api.pipecat.daily.co/v1/public/simple-chatbot/start',
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${process.env.PIPECAT_CLOUD_API_KEY}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
// Create Daily room
|
||||
createDailyRoom: true,
|
||||
// Optionally set Daily room properties
|
||||
dailyRoomProperties: { start_video_off: true },
|
||||
// Optionally pass custom data to the bot
|
||||
body: { MY_CUSTOM_DATA },
|
||||
}),
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`API responded with status: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
// Transform the response to match what RTVI client expects
|
||||
return NextResponse.json({
|
||||
room_url: data.dailyRoom,
|
||||
token: data.dailyToken,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('API error:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to start agent' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 20px;
|
||||
font-family: Arial, sans-serif;
|
||||
background-color: #f0f0f0;
|
||||
}
|
||||
|
||||
.app {
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.status-bar {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 10px;
|
||||
background-color: #fff;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.controls button {
|
||||
padding: 8px 16px;
|
||||
margin-left: 10px;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
button:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.connect-btn {
|
||||
background-color: #4caf50;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.disconnect-btn {
|
||||
background-color: #f44336;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.main-content {
|
||||
background-color: #fff;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.bot-container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.video-container {
|
||||
width: 640px;
|
||||
height: 360px;
|
||||
background-color: #ddd;
|
||||
margin-bottom: 20px;
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.video-container video {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.mic-enabled {
|
||||
background-color: #4caf50;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.mic-disabled {
|
||||
background-color: #f44336;
|
||||
color: white;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import './globals.css';
|
||||
import { RTVIProvider } from '@/providers/RTVIProvider';
|
||||
|
||||
export const metadata = {
|
||||
title: 'Pipecat React Client',
|
||||
description: 'Pipecat RTVI Client using Next.js',
|
||||
};
|
||||
|
||||
export default function RootLayout({
|
||||
children,
|
||||
}: {
|
||||
children: React.ReactNode;
|
||||
}) {
|
||||
return (
|
||||
<html lang="en">
|
||||
<body>
|
||||
<RTVIProvider>{children}</RTVIProvider>
|
||||
</body>
|
||||
</html>
|
||||
);
|
||||
}
|
||||
41
examples/pcc-transport-chatbot/client-react/src/app/page.tsx
Normal file
@@ -0,0 +1,41 @@
|
||||
'use client';
|
||||
|
||||
import {
|
||||
RTVIClientAudio,
|
||||
RTVIClientVideo,
|
||||
useRTVIClientTransportState,
|
||||
} from '@pipecat-ai/client-react';
|
||||
import { ConnectButton } from '../components/ConnectButton';
|
||||
import { StatusDisplay } from '../components/StatusDisplay';
|
||||
import { DebugDisplay } from '../components/DebugDisplay';
|
||||
|
||||
function BotVideo() {
|
||||
const transportState = useRTVIClientTransportState();
|
||||
const isConnected = transportState !== 'disconnected';
|
||||
|
||||
return (
|
||||
<div className="bot-container">
|
||||
<div className="video-container">
|
||||
{isConnected && <RTVIClientVideo participant="bot" fit="cover" />}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Home() {
|
||||
return (
|
||||
<div className="app">
|
||||
<div className="status-bar">
|
||||
<StatusDisplay />
|
||||
<ConnectButton />
|
||||
</div>
|
||||
|
||||
<div className="main-content">
|
||||
<BotVideo />
|
||||
</div>
|
||||
|
||||
<DebugDisplay />
|
||||
<RTVIClientAudio />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import {
|
||||
useRTVIClient,
|
||||
useRTVIClientTransportState,
|
||||
} from '@pipecat-ai/client-react';
|
||||
|
||||
export function ConnectButton() {
|
||||
const client = useRTVIClient();
|
||||
const transportState = useRTVIClientTransportState();
|
||||
const isConnected = ['connected', 'ready'].includes(transportState);
|
||||
|
||||
const handleClick = async () => {
|
||||
if (!client) {
|
||||
console.error('RTVI client is not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (isConnected) {
|
||||
await client.disconnect();
|
||||
} else {
|
||||
await client.connect();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Connection error:', error);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="controls">
|
||||
<button
|
||||
className={isConnected ? 'disconnect-btn' : 'connect-btn'}
|
||||
onClick={handleClick}
|
||||
disabled={
|
||||
!client || ['connecting', 'disconnecting'].includes(transportState)
|
||||
}>
|
||||
{isConnected ? 'Disconnect' : 'Connect'}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
.debug-panel {
|
||||
background-color: #fff;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.debug-panel h3 {
|
||||
margin: 0 0 10px 0;
|
||||
font-size: 16px;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.debug-log {
|
||||
height: 200px;
|
||||
overflow-y: auto;
|
||||
background-color: #f8f8f8;
|
||||
padding: 10px;
|
||||
border-radius: 4px;
|
||||
font-family: monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.debug-log div {
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
import { useRef, useCallback } from 'react';
|
||||
import {
|
||||
Participant,
|
||||
RTVIEvent,
|
||||
TransportState,
|
||||
TranscriptData,
|
||||
BotLLMTextData,
|
||||
} from '@pipecat-ai/client-js';
|
||||
import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
|
||||
import './DebugDisplay.css';
|
||||
|
||||
export function DebugDisplay() {
|
||||
const debugLogRef = useRef<HTMLDivElement>(null);
|
||||
const client = useRTVIClient();
|
||||
|
||||
const log = useCallback((message: string) => {
|
||||
if (!debugLogRef.current) return;
|
||||
|
||||
const entry = document.createElement('div');
|
||||
entry.textContent = `${new Date().toISOString()} - ${message}`;
|
||||
|
||||
// Add styling based on message type
|
||||
if (message.startsWith('User: ')) {
|
||||
entry.style.color = '#2196F3'; // blue for user
|
||||
} else if (message.startsWith('Bot: ')) {
|
||||
entry.style.color = '#4CAF50'; // green for bot
|
||||
}
|
||||
|
||||
debugLogRef.current.appendChild(entry);
|
||||
debugLogRef.current.scrollTop = debugLogRef.current.scrollHeight;
|
||||
}, []);
|
||||
|
||||
// Log transport state changes
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.TransportStateChanged,
|
||||
useCallback(
|
||||
(state: TransportState) => {
|
||||
log(`Transport state changed: ${state}`);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
// Log bot connection events
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.BotConnected,
|
||||
useCallback(
|
||||
(participant?: Participant) => {
|
||||
log(`Bot connected: ${JSON.stringify(participant)}`);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.BotDisconnected,
|
||||
useCallback(
|
||||
(participant?: Participant) => {
|
||||
log(`Bot disconnected: ${JSON.stringify(participant)}`);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
// Log track events
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.TrackStarted,
|
||||
useCallback(
|
||||
(track: MediaStreamTrack, participant?: Participant) => {
|
||||
log(
|
||||
`Track started: ${track.kind} from ${participant?.name || 'unknown'}`
|
||||
);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.TrackStopped,
|
||||
useCallback(
|
||||
(track: MediaStreamTrack, participant?: Participant) => {
|
||||
log(
|
||||
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
|
||||
);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
// Log bot ready state and check tracks
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.BotReady,
|
||||
useCallback(() => {
|
||||
log(`Bot ready`);
|
||||
|
||||
if (!client) return;
|
||||
|
||||
const tracks = client.tracks();
|
||||
log(
|
||||
`Available tracks: ${JSON.stringify({
|
||||
local: {
|
||||
audio: !!tracks.local.audio,
|
||||
video: !!tracks.local.video,
|
||||
},
|
||||
bot: {
|
||||
audio: !!tracks.bot?.audio,
|
||||
video: !!tracks.bot?.video,
|
||||
},
|
||||
})}`
|
||||
);
|
||||
}, [client, log])
|
||||
);
|
||||
|
||||
// Log transcripts
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.UserTranscript,
|
||||
useCallback(
|
||||
(data: TranscriptData) => {
|
||||
// Only log final transcripts
|
||||
if (data.final) {
|
||||
log(`User: ${data.text}`);
|
||||
}
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.BotTranscript,
|
||||
useCallback(
|
||||
(data: BotLLMTextData) => {
|
||||
log(`Bot: ${data.text}`);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="debug-panel">
|
||||
<h3>Debug Info</h3>
|
||||
<div ref={debugLogRef} className="debug-log" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { useRTVIClientTransportState } from '@pipecat-ai/client-react';
|
||||
|
||||
export function StatusDisplay() {
|
||||
const transportState = useRTVIClientTransportState();
|
||||
|
||||
return (
|
||||
<div className="status">
|
||||
Status: <span>{transportState}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
'use client';
|
||||
|
||||
import { RTVIClient } from '@pipecat-ai/client-js';
|
||||
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
||||
import { RTVIClientProvider } from '@pipecat-ai/client-react';
|
||||
import { PropsWithChildren, useEffect, useState } from 'react';
|
||||
|
||||
const MY_CUSTOM_DATA = { foo: 'bar' };
|
||||
|
||||
export function RTVIProvider({ children }: PropsWithChildren) {
|
||||
const [client, setClient] = useState<RTVIClient | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
console.log('Setting up Transport and Client');
|
||||
const transport = new DailyTransport();
|
||||
|
||||
const rtviClient = new RTVIClient({
|
||||
transport,
|
||||
params: {
|
||||
baseUrl: '/api',
|
||||
endpoints: {
|
||||
connect: '/connect',
|
||||
},
|
||||
requestData: { MY_CUSTOM_DATA },
|
||||
},
|
||||
enableMic: true,
|
||||
enableCam: false,
|
||||
});
|
||||
|
||||
setClient(rtviClient);
|
||||
}, []);
|
||||
|
||||
if (!client) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
|
||||
}
|
||||
28
examples/pcc-transport-chatbot/client-react/tsconfig.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2017",
|
||||
"lib": ["dom", "dom.iterable", "esnext"],
|
||||
"allowJs": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"module": "esnext",
|
||||
"moduleResolution": "bundler",
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"jsx": "preserve",
|
||||
"incremental": true,
|
||||
"plugins": [
|
||||
{
|
||||
"name": "next"
|
||||
}
|
||||
],
|
||||
"paths": {
|
||||
"@/components/*": ["./src/components/*"],
|
||||
"@/providers/*": ["./src/providers/*"]
|
||||
}
|
||||
},
|
||||
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
BIN
examples/pcc-transport-chatbot/image.png
Normal file
|
After Width: | Height: | Size: 733 KiB |
51
examples/pcc-transport-chatbot/server/.gitignore
vendored
Normal file
@@ -0,0 +1,51 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# JavaScript/Node.js
|
||||
node_modules/
|
||||
dist/
|
||||
dist-ssr/
|
||||
*.local
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
|
||||
# Editor/IDE
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
.DS_Store
|
||||
|
||||
# Project specific
|
||||
runpod.toml
|
||||
12
examples/pcc-transport-chatbot/server/Dockerfile
Normal file
@@ -0,0 +1,12 @@
|
||||
FROM dailyco/pipecat-base:latest
|
||||
RUN apt-get update && apt-get install ffmpeg -y
|
||||
|
||||
COPY ./pipecat pipecat
|
||||
|
||||
COPY ./requirements.txt requirements.txt
|
||||
|
||||
COPY ./assets assets
|
||||
|
||||
RUN pip install --no-cache-dir --upgrade -r requirements.txt
|
||||
|
||||
COPY ./bot.py bot.py
|
||||
33
examples/pcc-transport-chatbot/server/README.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Simple Chatbot Server
|
||||
|
||||
A Pipecat bot.py file that is built to be deployed to Pipecat Cloud.
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Copy `env.example` to `.env` and configure:
|
||||
|
||||
```ini
|
||||
OPENAI_API_KEY= # Your OpenAI API key (required for OpenAI bot)
|
||||
CARTESIA_API_KEY= # Your Cartesia API key
|
||||
```
|
||||
|
||||
## Running the server locally
|
||||
|
||||
Set up and activate your virtual environment:
|
||||
|
||||
```bash
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate # On Windows: venv\Scripts\activate
|
||||
```
|
||||
|
||||
Install dependencies:
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
Run the server:
|
||||
|
||||
```bash
|
||||
LOCAL_RUN=1 python bot.py
|
||||
```
|
||||
BIN
examples/pcc-transport-chatbot/server/assets/robot01.png
Normal file
|
After Width: | Height: | Size: 759 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot010.png
Normal file
|
After Width: | Height: | Size: 884 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot011.png
Normal file
|
After Width: | Height: | Size: 876 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot012.png
Normal file
|
After Width: | Height: | Size: 881 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot013.png
Normal file
|
After Width: | Height: | Size: 866 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot014.png
Normal file
|
After Width: | Height: | Size: 874 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot015.png
Normal file
|
After Width: | Height: | Size: 882 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot016.png
Normal file
|
After Width: | Height: | Size: 885 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot017.png
Normal file
|
After Width: | Height: | Size: 888 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot018.png
Normal file
|
After Width: | Height: | Size: 890 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot019.png
Normal file
|
After Width: | Height: | Size: 898 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot02.png
Normal file
|
After Width: | Height: | Size: 836 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot020.png
Normal file
|
After Width: | Height: | Size: 903 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot021.png
Normal file
|
After Width: | Height: | Size: 908 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot022.png
Normal file
|
After Width: | Height: | Size: 908 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot023.png
Normal file
|
After Width: | Height: | Size: 905 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot024.png
Normal file
|
After Width: | Height: | Size: 903 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot025.png
Normal file
|
After Width: | Height: | Size: 866 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot03.png
Normal file
|
After Width: | Height: | Size: 849 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot04.png
Normal file
|
After Width: | Height: | Size: 866 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot05.png
Normal file
|
After Width: | Height: | Size: 866 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot06.png
Normal file
|
After Width: | Height: | Size: 864 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot07.png
Normal file
|
After Width: | Height: | Size: 858 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot08.png
Normal file
|
After Width: | Height: | Size: 875 KiB |
BIN
examples/pcc-transport-chatbot/server/assets/robot09.png
Normal file
|
After Width: | Height: | Size: 881 KiB |
330
examples/pcc-transport-chatbot/server/bot.py
Normal file
@@ -0,0 +1,330 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
"""OpenAI Bot Implementation.
|
||||
|
||||
This module implements a chatbot using OpenAI's GPT-4 model for natural language
|
||||
processing. It includes:
|
||||
- Real-time audio/video interaction through Daily
|
||||
- Animated robot avatar
|
||||
- Text-to-speech using ElevenLabs
|
||||
- Support for both English and Spanish
|
||||
|
||||
The bot runs as part of a pipeline that processes audio/video frames and manages
|
||||
the conversation flow.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import aiohttp
|
||||
from dotenv import load_dotenv
|
||||
from loguru import logger
|
||||
from PIL import Image
|
||||
from pipecatcloud.agent import (
|
||||
DailySessionArguments,
|
||||
)
|
||||
|
||||
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
||||
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||
from pipecat.frames.frames import (
|
||||
BotStartedSpeakingFrame,
|
||||
BotStoppedSpeakingFrame,
|
||||
Frame,
|
||||
OutputImageRawFrame,
|
||||
SpriteFrame,
|
||||
TTSSpeakFrame,
|
||||
)
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor
|
||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||
from pipecat.services.gladia.stt import GladiaSTTService
|
||||
from pipecat.services.openai.llm import OpenAILLMService
|
||||
from pipecat.transports.services.daily import DailyTransport
|
||||
from pipecat.transports.services.pipecat_cloud import (
|
||||
PipecatCloudParams,
|
||||
PipecatCloudSessionArguments,
|
||||
PipecatCloudTransport,
|
||||
)
|
||||
|
||||
load_dotenv(override=True)
|
||||
logger.add(sys.stderr, level="DEBUG")
|
||||
|
||||
print(f"DailyTransport: {DailyTransport}")
|
||||
|
||||
# Check if we're in local development mode
|
||||
LOCAL_RUN = os.getenv("LOCAL_RUN")
|
||||
if LOCAL_RUN:
|
||||
import asyncio
|
||||
import webbrowser
|
||||
|
||||
try:
|
||||
from local_runner import configure
|
||||
except ImportError:
|
||||
logger.error("Could not import local_runner module. Local development mode may not work.")
|
||||
|
||||
# Logger for local dev
|
||||
logger.add(sys.stderr, level="DEBUG")
|
||||
|
||||
sprites = []
|
||||
script_dir = os.path.dirname(__file__)
|
||||
|
||||
# Load sequential animation frames
|
||||
for i in range(1, 26):
|
||||
# Build the full path to the image file
|
||||
full_path = os.path.join(script_dir, f"assets/robot0{i}.png")
|
||||
# Get the filename without the extension to use as the dictionary key
|
||||
# Open the image and convert it to bytes
|
||||
with Image.open(full_path) as img:
|
||||
sprites.append(OutputImageRawFrame(image=img.tobytes(), size=img.size, format=img.format))
|
||||
|
||||
# Create a smooth animation by adding reversed frames
|
||||
flipped = sprites[::-1]
|
||||
sprites.extend(flipped)
|
||||
|
||||
# Define static and animated states
|
||||
quiet_frame = sprites[0] # Static frame for when bot is listening
|
||||
talking_frame = SpriteFrame(images=sprites) # Animation sequence for when bot is talking
|
||||
|
||||
|
||||
class TalkingAnimation(FrameProcessor):
|
||||
"""Manages the bot's visual animation states.
|
||||
|
||||
Switches between static (listening) and animated (talking) states based on
|
||||
the bot's current speaking status.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._is_talking = False
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||
"""Process incoming frames and update animation state.
|
||||
|
||||
Args:
|
||||
frame: The incoming frame to process
|
||||
direction: The direction of frame flow in the pipeline
|
||||
"""
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
# Switch to talking animation when bot starts speaking
|
||||
if isinstance(frame, BotStartedSpeakingFrame):
|
||||
if not self._is_talking:
|
||||
await self.push_frame(talking_frame)
|
||||
self._is_talking = True
|
||||
# Return to static frame when bot stops speaking
|
||||
elif isinstance(frame, BotStoppedSpeakingFrame):
|
||||
await self.push_frame(quiet_frame)
|
||||
self._is_talking = False
|
||||
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
|
||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||
"""Fetch weather data dummy function.
|
||||
|
||||
This function simulates fetching weather data from an external API.
|
||||
It demonstrates how to call an external service from the language model.
|
||||
"""
|
||||
await llm.push_frame(TTSSpeakFrame("Let me check on that."))
|
||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||
|
||||
|
||||
async def main(session_args: PipecatCloudSessionArguments):
|
||||
"""Main bot execution function.
|
||||
|
||||
Sets up and runs the bot pipeline including:
|
||||
- Daily video transport
|
||||
- Speech-to-text and text-to-speech services
|
||||
- Language model integration
|
||||
- Animation processing
|
||||
- RTVI event handling
|
||||
"""
|
||||
logger.info(f"session args: {session_args}")
|
||||
|
||||
# Set up Daily transport with video/audio parameters
|
||||
transport = PipecatCloudTransport(
|
||||
session_args=session_args,
|
||||
params=PipecatCloudParams(
|
||||
audio_out_enabled=True, # Enable output audio for the bot
|
||||
camera_out_enabled=True, # Enable the camera output for the bot
|
||||
camera_out_width=1024, # Set the camera output width
|
||||
camera_out_height=576, # Set the camera output height
|
||||
transcription_enabled=True, # Enable transcription for the user
|
||||
vad_enabled=True, # Enable VAD to handle user speech
|
||||
vad_analyzer=SileroVADAnalyzer(), # Use the Silero VAD analyzer
|
||||
vad_audio_passthrough=True, # Pass audio through VAD for user speech to the rest of the pipeline
|
||||
),
|
||||
)
|
||||
|
||||
# Initialize text-to-speech service
|
||||
tts = CartesiaTTSService(
|
||||
api_key=os.getenv("CARTESIA_API_KEY"),
|
||||
voice_id="c45bc5ec-dc68-4feb-8829-6e6b2748095d", # Movieman
|
||||
)
|
||||
|
||||
stt = GladiaSTTService(api_key=os.getenv("GLADIA_API_KEY"))
|
||||
|
||||
# Initialize LLM service
|
||||
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
||||
|
||||
# Register your function call providing the function name and callback
|
||||
llm.register_function("get_current_weather", fetch_weather_from_api)
|
||||
|
||||
# Define your function call using the FunctionSchema
|
||||
# Learn more about function calling in Pipecat:
|
||||
# https://docs.pipecat.ai/guides/features/function-calling
|
||||
weather_function = FunctionSchema(
|
||||
name="get_current_weather",
|
||||
description="Get the current weather",
|
||||
properties={
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA",
|
||||
},
|
||||
"format": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
"description": "The temperature unit to use. Infer this from the user's location.",
|
||||
},
|
||||
},
|
||||
required=["location", "format"],
|
||||
)
|
||||
|
||||
# Set up the tools schema with your weather function call
|
||||
tools = ToolsSchema(standard_tools=[weather_function])
|
||||
|
||||
# Set up initial messages for the bot
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are Chatbot, a friendly, helpful robot. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but keep your responses brief. Start by introducing yourself.",
|
||||
},
|
||||
]
|
||||
|
||||
# Set up conversation context and management
|
||||
# The context_aggregator will automatically collect conversation context
|
||||
# Pass your initial messages and tools to the context to initialize the context
|
||||
context = OpenAILLMContext(messages, tools)
|
||||
context_aggregator = llm.create_context_aggregator(context)
|
||||
|
||||
ta = TalkingAnimation()
|
||||
|
||||
# RTVI events for Pipecat client UI
|
||||
rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
|
||||
|
||||
# Add your processors to the pipeline
|
||||
pipeline = Pipeline(
|
||||
[
|
||||
transport.input(),
|
||||
stt,
|
||||
rtvi,
|
||||
context_aggregator.user(),
|
||||
llm,
|
||||
tts,
|
||||
ta,
|
||||
transport.output(),
|
||||
context_aggregator.assistant(),
|
||||
]
|
||||
)
|
||||
|
||||
# Create a PipelineTask to manage the pipeline
|
||||
task = PipelineTask(
|
||||
pipeline,
|
||||
params=PipelineParams(
|
||||
allow_interruptions=True,
|
||||
enable_metrics=True,
|
||||
enable_usage_metrics=True,
|
||||
),
|
||||
observers=[RTVIObserver(rtvi)],
|
||||
)
|
||||
|
||||
@rtvi.event_handler("on_client_ready")
|
||||
async def on_client_ready(rtvi):
|
||||
# Notify the client that the bot is ready
|
||||
await rtvi.set_bot_ready()
|
||||
|
||||
@transport.event_handler("on_client_connected")
|
||||
async def on_client_connected(transport, participant):
|
||||
# Push a static frame to show the bot is listening
|
||||
await task.queue_frame(quiet_frame)
|
||||
# Capture the first participant's transcription
|
||||
# await transport.capture_participant_transcription(participant["id"])
|
||||
# Kick off the conversation by pushing a context frame to the pipeline
|
||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||
|
||||
@transport.event_handler("on_client_disconnected")
|
||||
async def on_client_disconnected(transport, participant):
|
||||
logger.debug(f"Participant left: {participant}")
|
||||
# Cancel the PipelineTask to stop processing
|
||||
await task.cancel()
|
||||
|
||||
runner = PipelineRunner()
|
||||
|
||||
await runner.run(task)
|
||||
|
||||
|
||||
async def bot(args: DailySessionArguments):
|
||||
"""Main bot entry point compatible with Pipecat Cloud.
|
||||
|
||||
Args:
|
||||
room_url: The Daily room URL
|
||||
token: The Daily room token
|
||||
body: The configuration object from the request body
|
||||
session_id: The session ID for logging
|
||||
"""
|
||||
pcc_args = PipecatCloudSessionArguments(
|
||||
room_url=args.room.url,
|
||||
token=args.token,
|
||||
body=args.body,
|
||||
session_id=args.session_id,
|
||||
)
|
||||
logger.info(f"Bot process initialized {pcc_args.room_url} {pcc_args.token}")
|
||||
|
||||
try:
|
||||
await main(pcc_args)
|
||||
logger.info("Bot process completed")
|
||||
except Exception as e:
|
||||
logger.exception(f"Error in bot process: {str(e)}")
|
||||
raise
|
||||
|
||||
|
||||
# Local development
|
||||
async def local_daily():
|
||||
# TODO-CB: This becomes SmallWebRTCTransport
|
||||
"""Function for local development testing."""
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
(room_url, token) = await configure(session)
|
||||
args = PipecatCloudSessionArguments(
|
||||
room_url=room_url, token=token, body={}, session_id=None
|
||||
)
|
||||
logger.warning("_")
|
||||
logger.warning("_")
|
||||
logger.warning(f"Talk to your voice agent here: {room_url}")
|
||||
logger.warning("_")
|
||||
logger.warning("_")
|
||||
webbrowser.open(room_url)
|
||||
await main(args)
|
||||
except Exception as e:
|
||||
logger.exception(f"Error in local development mode: {e}")
|
||||
|
||||
|
||||
async def local_webrtc(webrtc_connection):
|
||||
await main(PipecatCloudSessionArguments(webrtc_connection=webrtc_connection))
|
||||
|
||||
|
||||
# Local development entry point
|
||||
if LOCAL_RUN and __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(local_daily())
|
||||
except Exception as e:
|
||||
logger.exception(f"Failed to run in local mode: {e}")
|
||||
19
examples/pcc-transport-chatbot/server/build.sh
Executable file
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
VERSION="0.2"
|
||||
DOCKER_USERNAME="chadbailey59"
|
||||
AGENT_NAME="pcc-transport-chatbot"
|
||||
|
||||
# Build the Docker image with the correct context
|
||||
echo "Building Docker image..."
|
||||
docker build --platform=linux/arm64 -t "$DOCKER_USERNAME/$AGENT_NAME:$VERSION" -t "$DOCKER_USERNAME/$AGENT_NAME:latest" .
|
||||
|
||||
# Push the Docker images
|
||||
echo "Pushing Docker image $DOCKER_USERNAME/$AGENT_NAME:$VERSION..."
|
||||
docker push "$DOCKER_USERNAME/$AGENT_NAME:$VERSION"
|
||||
|
||||
echo "Pushing Docker image $DOCKER_USERNAME/$AGENT_NAME:latest..."
|
||||
docker push "$DOCKER_USERNAME/$AGENT_NAME:latest"
|
||||
|
||||
echo "Successfully built and pushed $DOCKER_USERNAME/$AGENT_NAME:$VERSION and $DOCKER_USERNAME/$AGENT_NAME:latest"
|
||||
2
examples/pcc-transport-chatbot/server/env.example
Normal file
@@ -0,0 +1,2 @@
|
||||
OPENAI_API_KEY=sk-PL...
|
||||
CARTESIA_API_KEY=aeb...
|
||||
100
examples/pcc-transport-chatbot/server/index.html
Normal file
@@ -0,0 +1,100 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>WebRTC Voice Agent</title>
|
||||
<style>
|
||||
body { font-family: Arial, sans-serif; text-align: center; margin-top: 50px; }
|
||||
#status { font-size: 20px; margin: 20px; }
|
||||
button { padding: 10px 20px; font-size: 16px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>WebRTC Voice Agent</h1>
|
||||
<p id="status">Disconnected</p>
|
||||
<button id="connect-btn">Connect</button>
|
||||
<audio id="audio-el" autoplay></audio>
|
||||
|
||||
<script>
|
||||
const statusEl = document.getElementById("status")
|
||||
const buttonEl = document.getElementById("connect-btn")
|
||||
const audioEl = document.getElementById("audio-el")
|
||||
|
||||
let connected = false
|
||||
let peerConnection = null
|
||||
|
||||
/*const waitForIceGatheringComplete = async (pc) => {
|
||||
if (pc.iceGatheringState === 'complete') return;
|
||||
return new Promise((resolve) => {
|
||||
const checkState = () => {
|
||||
if (pc.iceGatheringState === 'complete') {
|
||||
pc.removeEventListener('icegatheringstatechange', checkState);
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
pc.addEventListener('icegatheringstatechange', checkState);
|
||||
});
|
||||
}*/
|
||||
|
||||
const createSmallWebRTCConnection = async (audioTrack) => {
|
||||
const pc = new RTCPeerConnection()
|
||||
pc.ontrack = e => audioEl.srcObject = e.streams[0]
|
||||
pc.addTransceiver(audioTrack, { direction: 'sendrecv' })
|
||||
await pc.setLocalDescription(await pc.createOffer())
|
||||
//await waitForIceGatheringComplete(pc)
|
||||
const offer = pc.localDescription
|
||||
const response = await fetch('/api/offer', {
|
||||
body: JSON.stringify({ sdp: offer.sdp, type: offer.type}),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
method: 'POST',
|
||||
});
|
||||
const answer = await response.json()
|
||||
await pc.setRemoteDescription(answer)
|
||||
return pc
|
||||
}
|
||||
|
||||
const connect = async () => {
|
||||
const audioStream = await navigator.mediaDevices.getUserMedia({audio: true})
|
||||
peerConnection= await createSmallWebRTCConnection(audioStream.getAudioTracks()[0])
|
||||
peerConnection.onconnectionstatechange = () => {
|
||||
let connectionState = peerConnection?.connectionState
|
||||
if (connectionState === 'connected') {
|
||||
_onConnected()
|
||||
} else if (connectionState === 'disconnected') {
|
||||
_onDisconnected()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const _onConnected = () => {
|
||||
statusEl.textContent = "Connected"
|
||||
buttonEl.textContent = "Disconnect"
|
||||
connected = true
|
||||
}
|
||||
|
||||
const _onDisconnected = () => {
|
||||
statusEl.textContent = "Disconnected"
|
||||
buttonEl.textContent = "Connect"
|
||||
connected = false
|
||||
}
|
||||
|
||||
const disconnect = () => {
|
||||
if (!peerConnection) {
|
||||
return
|
||||
}
|
||||
peerConnection.close()
|
||||
peerConnection = null
|
||||
_onDisconnected()
|
||||
}
|
||||
|
||||
buttonEl.addEventListener("click", async () => {
|
||||
if (!connected) {
|
||||
await connect()
|
||||
} else {
|
||||
disconnect()
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
46
examples/pcc-transport-chatbot/server/local_runner.py
Normal file
@@ -0,0 +1,46 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
import os
|
||||
|
||||
import aiohttp
|
||||
|
||||
from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper, DailyRoomParams
|
||||
|
||||
|
||||
async def configure(aiohttp_session: aiohttp.ClientSession):
|
||||
(url, token) = await configure_with_args(aiohttp_session)
|
||||
return (url, token)
|
||||
|
||||
|
||||
async def configure_with_args(aiohttp_session: aiohttp.ClientSession = None):
|
||||
key = os.getenv("DAILY_API_KEY")
|
||||
if not key:
|
||||
raise Exception(
|
||||
"No Daily API key specified. set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers."
|
||||
)
|
||||
|
||||
daily_rest_helper = DailyRESTHelper(
|
||||
daily_api_key=key,
|
||||
daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
|
||||
aiohttp_session=aiohttp_session,
|
||||
)
|
||||
|
||||
room = await daily_rest_helper.create_room(
|
||||
DailyRoomParams(properties={"enable_prejoin_ui": False})
|
||||
)
|
||||
if not room.url:
|
||||
raise HTTPException(status_code=500, detail="Failed to create room")
|
||||
|
||||
url = room.url
|
||||
|
||||
# Create a meeting token for the given room with an expiration 1 hour in
|
||||
# the future.
|
||||
expiry_time: float = 60 * 60
|
||||
|
||||
token = await daily_rest_helper.get_token(url, expiry_time)
|
||||
|
||||
return (url, token)
|
||||
7
examples/pcc-transport-chatbot/server/pcc-deploy.toml
Normal file
@@ -0,0 +1,7 @@
|
||||
agent_name = "pcc-transport-chatbot"
|
||||
image = "chadbailey59/pcc-transport-chatbot:0.2"
|
||||
secret_set = "pcc-transport-chatbot-secrets"
|
||||
|
||||
[scaling]
|
||||
min_instances = 0
|
||||
max_instances = 2
|
||||
5
examples/pcc-transport-chatbot/server/requirements.txt
Normal file
@@ -0,0 +1,5 @@
|
||||
python-dotenv
|
||||
fastapi[all]
|
||||
uvicorn
|
||||
-e ./pipecat[daily,cartesia,openai,silero,gladia,webrtc]
|
||||
pipecatcloud
|
||||
81
examples/pcc-transport-chatbot/server/server.py
Normal file
@@ -0,0 +1,81 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Dict
|
||||
|
||||
import uvicorn
|
||||
from bot import local_webrtc
|
||||
from dotenv import load_dotenv
|
||||
from fastapi import BackgroundTasks, FastAPI
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||
|
||||
# Load environment variables
|
||||
load_dotenv(override=True)
|
||||
|
||||
logger = logging.getLogger("pc")
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
# Store connections by pc_id
|
||||
pcs_map: Dict[str, SmallWebRTCConnection] = {}
|
||||
|
||||
|
||||
@app.post("/api/offer")
|
||||
async def offer(request: dict, background_tasks: BackgroundTasks):
|
||||
pc_id = request.get("pc_id")
|
||||
|
||||
if pc_id and pc_id in pcs_map:
|
||||
pipecat_connection = pcs_map[pc_id]
|
||||
logger.info(f"Reusing existing connection for pc_id: {pc_id}")
|
||||
await pipecat_connection.renegotiate(sdp=request["sdp"], type=request["type"])
|
||||
else:
|
||||
pipecat_connection = SmallWebRTCConnection()
|
||||
await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"])
|
||||
|
||||
@pipecat_connection.event_handler("closed")
|
||||
async def handle_disconnected(webrtc_connection: SmallWebRTCConnection):
|
||||
logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}")
|
||||
pcs_map.pop(webrtc_connection.pc_id, None)
|
||||
|
||||
background_tasks.add_task(local_webrtc, pipecat_connection)
|
||||
|
||||
answer = pipecat_connection.get_answer()
|
||||
# Updating the peer connection inside the map
|
||||
pcs_map[answer["pc_id"]] = pipecat_connection
|
||||
|
||||
return answer
|
||||
|
||||
|
||||
@app.get("/")
|
||||
async def serve_index():
|
||||
return FileResponse("index.html")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
yield # Run app
|
||||
coros = [pc.close() for pc in pcs_map.values()]
|
||||
await asyncio.gather(*coros)
|
||||
pcs_map.clear()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="WebRTC demo")
|
||||
parser.add_argument(
|
||||
"--host", default="localhost", help="Host for HTTP server (default: localhost)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=7860, help="Port for HTTP server (default: 7860)"
|
||||
)
|
||||
parser.add_argument("--verbose", "-v", action="count")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
else:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
uvicorn.run(app, host=args.host, port=args.port)
|
||||
@@ -74,6 +74,7 @@ openai = [ "websockets~=13.1" ]
|
||||
openpipe = [ "openpipe~=4.48.0" ]
|
||||
openrouter = []
|
||||
perplexity = []
|
||||
pipecatcloud = ["pipecatcloud" ]
|
||||
playht = [ "pyht~=0.1.12", "websockets~=13.1" ]
|
||||
qwen = []
|
||||
rime = [ "websockets~=13.1" ]
|
||||
|
||||
313
src/pipecat/transports/services/pipecat_cloud.py
Normal file
@@ -0,0 +1,313 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from functools import wraps
|
||||
from typing import Any, Callable, Optional, Union
|
||||
|
||||
from fastapi import WebSocket
|
||||
from pipecatcloud.agent import DailySessionArguments, SessionArguments, WebSocketSessionArguments
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
||||
from pipecat.transports.network.fastapi_websocket import (
|
||||
FastAPIWebsocketParams,
|
||||
FastAPIWebsocketTransport,
|
||||
FrameSerializer,
|
||||
)
|
||||
from pipecat.transports.network.small_webrtc import (
|
||||
SmallWebRTCConnection,
|
||||
SmallWebRTCTransport,
|
||||
)
|
||||
from pipecat.transports.services.daily import (
|
||||
DailyDialinSettings,
|
||||
DailyParams,
|
||||
DailyTranscriptionSettings,
|
||||
DailyTransport,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WebRTCSessionArguments(SessionArguments):
|
||||
"""WebRTC based agent session arguments. The arguments are received by the
|
||||
bot() entry point.
|
||||
"""
|
||||
|
||||
webrtc_connection: SmallWebRTCConnection
|
||||
|
||||
|
||||
class PipecatCloudParams(TransportParams):
|
||||
"""Parameters for PipecatCloudTransport.
|
||||
|
||||
This class combines parameters from all transport types:
|
||||
- TransportParams (inherited)
|
||||
- FastAPIWebsocketParams
|
||||
- DailyParams
|
||||
"""
|
||||
|
||||
# FastAPIWebsocketParams fields
|
||||
add_wav_header: bool = False
|
||||
serializer: Optional[FrameSerializer] = None
|
||||
session_timeout: Optional[int] = None
|
||||
|
||||
# DailyParams fields
|
||||
api_url: str = "https://api.daily.co/v1"
|
||||
api_key: str = ""
|
||||
dialin_settings: Optional[DailyDialinSettings] = None
|
||||
transcription_enabled: bool = False
|
||||
transcription_settings: DailyTranscriptionSettings = DailyTranscriptionSettings()
|
||||
|
||||
def to_transport_params(self) -> TransportParams:
|
||||
"""Convert to TransportParams."""
|
||||
return self
|
||||
|
||||
def to_websocket_params(self) -> FastAPIWebsocketParams:
|
||||
"""Convert to FastAPIWebsocketParams."""
|
||||
base_params = self.model_dump()
|
||||
# Remove WebSocket-specific fields since we'll add them explicitly
|
||||
websocket_fields = ["add_wav_header", "serializer", "session_timeout"]
|
||||
for field in websocket_fields:
|
||||
base_params.pop(field, None)
|
||||
|
||||
return FastAPIWebsocketParams(
|
||||
**base_params,
|
||||
add_wav_header=self.add_wav_header,
|
||||
serializer=self.serializer,
|
||||
session_timeout=self.session_timeout,
|
||||
)
|
||||
|
||||
def to_daily_params(self) -> DailyParams:
|
||||
"""Convert to DailyParams."""
|
||||
base_params = self.model_dump()
|
||||
# Remove Daily-specific fields since we'll add them explicitly
|
||||
daily_fields = [
|
||||
"api_url",
|
||||
"api_key",
|
||||
"dialin_settings",
|
||||
"transcription_enabled",
|
||||
"transcription_settings",
|
||||
]
|
||||
for field in daily_fields:
|
||||
base_params.pop(field, None)
|
||||
|
||||
return DailyParams(
|
||||
**base_params,
|
||||
api_url=self.api_url,
|
||||
api_key=self.api_key,
|
||||
dialin_settings=self.dialin_settings,
|
||||
transcription_enabled=self.transcription_enabled,
|
||||
transcription_settings=self.transcription_settings,
|
||||
)
|
||||
|
||||
|
||||
class PipecatCloudSessionArguments:
|
||||
"""Arguments for creating a PipecatCloudTransport session.
|
||||
|
||||
This class can be initialized with arguments for any of the supported transport types:
|
||||
- WebSocket: Pass websocket=WebSocket
|
||||
- Daily: Pass room_url=str, token=str, bot_name=str
|
||||
- WebRTC: Pass webrtc_connection=SmallWebRTCConnection
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
websocket: Optional[WebSocket] = None,
|
||||
room_url: Optional[str] = None,
|
||||
token: Optional[str] = None,
|
||||
webrtc_connection: Optional[SmallWebRTCConnection] = None,
|
||||
session_id: Optional[str] = None,
|
||||
body: Optional[dict] = None,
|
||||
):
|
||||
"""Initialize session arguments for any supported transport type."""
|
||||
if websocket is not None:
|
||||
self._args = WebSocketSessionArguments(websocket=websocket, session_id=session_id)
|
||||
elif room_url is not None:
|
||||
self._args = DailySessionArguments(
|
||||
# TODO-CB: bot_name is missing from DailySessionArguments?
|
||||
room_url=room_url,
|
||||
token=token,
|
||||
session_id=session_id,
|
||||
body=body,
|
||||
)
|
||||
elif webrtc_connection is not None:
|
||||
self._args = WebRTCSessionArguments(
|
||||
webrtc_connection=webrtc_connection, session_id=session_id
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Must provide either websocket, (room_url, token, bot_name), or webrtc_connection"
|
||||
)
|
||||
|
||||
@property
|
||||
def args(self):
|
||||
"""Get the underlying session arguments."""
|
||||
return self._args
|
||||
|
||||
|
||||
class PipecatCloudTransport(BaseTransport):
|
||||
"""A transport that wraps FastAPIWebsocketTransport, SmallWebRTCTransport, and DailyTransport.
|
||||
|
||||
This transport will instantiate one of the three underlying transports based on the
|
||||
session arguments provided to the constructor.
|
||||
|
||||
Event handlers:
|
||||
@event_handler("on_client_connected"): Called when a client connects. Maps to:
|
||||
- FastAPIWebsocketTransport.event_handler("on_client_connected")
|
||||
- SmallWebRTCTransport.event_handler("on_client_connected")
|
||||
- DailyTransport.event_handler("on_first_participant_joined")
|
||||
|
||||
@event_handler("on_client_disconnected"): Called when a client disconnects. Maps to:
|
||||
- FastAPIWebsocketTransport.event_handler("on_client_disconnected")
|
||||
- SmallWebRTCTransport.event_handler("on_client_disconnected")
|
||||
- DailyTransport.event_handler("on_participant_left")
|
||||
|
||||
Other event handlers are passed through directly to the underlying transport.
|
||||
|
||||
Args:
|
||||
session_args: Arguments for creating the session. The type of arguments determines
|
||||
which transport will be used.
|
||||
params: Configuration parameters for the transport. Parameters will be extracted
|
||||
based on the session arguments type.
|
||||
input_name: Optional name for the input transport.
|
||||
output_name: Optional name for the output transport.
|
||||
"""
|
||||
|
||||
# Event name mappings for each transport type
|
||||
# Only include events that need to be mapped differently
|
||||
_EVENT_MAPPINGS = {
|
||||
DailyTransport: {
|
||||
"on_client_connected": "on_first_participant_joined",
|
||||
"on_client_disconnected": "on_participant_left",
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_args: PipecatCloudSessionArguments,
|
||||
params: Optional[Union[PipecatCloudParams, TransportParams]] = None,
|
||||
*,
|
||||
input_name: Optional[str] = None,
|
||||
output_name: Optional[str] = None,
|
||||
):
|
||||
super().__init__(input_name=input_name, output_name=output_name)
|
||||
|
||||
# Convert TransportParams to PipecatCloudParams if needed
|
||||
if isinstance(params, TransportParams):
|
||||
cloud_params = PipecatCloudParams()
|
||||
for field_name, field_value in params.model_dump().items():
|
||||
setattr(cloud_params, field_name, field_value)
|
||||
params = cloud_params
|
||||
else:
|
||||
params = params or PipecatCloudParams()
|
||||
|
||||
self._pending_handlers = {}
|
||||
|
||||
# Create the appropriate transport based on session arguments type
|
||||
args = session_args.args
|
||||
if isinstance(args, WebSocketSessionArguments):
|
||||
logger.info("Using FastAPIWebsocketTransport")
|
||||
websocket_params = params.to_websocket_params()
|
||||
self._transport = FastAPIWebsocketTransport(
|
||||
args.websocket,
|
||||
websocket_params,
|
||||
input_name=input_name,
|
||||
output_name=output_name,
|
||||
)
|
||||
elif isinstance(args, DailySessionArguments):
|
||||
logger.info("Using DailyTransport")
|
||||
daily_params = params.to_daily_params()
|
||||
self._transport = DailyTransport(
|
||||
args.room_url,
|
||||
args.token,
|
||||
# TODO-CB: Bot name is missing from DailySessionArguments
|
||||
"Bot",
|
||||
params=daily_params,
|
||||
input_name=input_name,
|
||||
output_name=output_name,
|
||||
)
|
||||
elif isinstance(args, WebRTCSessionArguments):
|
||||
logger.info("Using SmallWebRTCTransport")
|
||||
transport_params = params.to_transport_params()
|
||||
self._transport = SmallWebRTCTransport(
|
||||
args.webrtc_connection,
|
||||
transport_params,
|
||||
input_name=input_name,
|
||||
output_name=output_name,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported session arguments type: {type(args)}")
|
||||
|
||||
# Register any handlers that were added before transport creation
|
||||
for event_name, handlers in self._pending_handlers.items():
|
||||
for handler in handlers:
|
||||
self._register_handler(event_name, handler)
|
||||
|
||||
def _register_handler(self, event_name: str, handler: Callable[..., Any]) -> None:
|
||||
"""Register a handler with the appropriate transport method."""
|
||||
transport_type = type(self._transport)
|
||||
|
||||
# If the transport type has mappings and the event needs to be mapped
|
||||
if (
|
||||
transport_type in self._EVENT_MAPPINGS
|
||||
and event_name in self._EVENT_MAPPINGS[transport_type]
|
||||
):
|
||||
mapped_event = self._EVENT_MAPPINGS[transport_type][event_name]
|
||||
else:
|
||||
# Pass through the event name directly if no mapping exists
|
||||
mapped_event = event_name
|
||||
|
||||
self._transport.event_handler(mapped_event)(handler)
|
||||
|
||||
def event_handler(self, event_name: str) -> Callable[..., Any]:
|
||||
"""Register an event handler.
|
||||
|
||||
Args:
|
||||
event_name: The name of the event to handle. Common events:
|
||||
- "on_client_connected": Called when a client connects
|
||||
- "on_client_disconnected": Called when a client disconnects
|
||||
Other event names are passed through to the underlying transport.
|
||||
|
||||
Returns:
|
||||
A decorator that registers the handler function.
|
||||
"""
|
||||
|
||||
def decorator(handler: Callable[..., Any]) -> Callable[..., Any]:
|
||||
if not hasattr(self, "_transport"):
|
||||
# Store the handler to be registered when the transport is created
|
||||
if event_name not in self._pending_handlers:
|
||||
self._pending_handlers[event_name] = []
|
||||
self._pending_handlers[event_name].append(handler)
|
||||
else:
|
||||
self._register_handler(event_name, handler)
|
||||
return handler
|
||||
|
||||
return decorator
|
||||
|
||||
async def start(self, frame):
|
||||
"""Start the transport."""
|
||||
await self._transport.start(frame)
|
||||
|
||||
async def stop(self, frame):
|
||||
"""Stop the transport."""
|
||||
await self._transport.stop(frame)
|
||||
|
||||
async def cancel(self, frame):
|
||||
"""Cancel the transport."""
|
||||
await self._transport.cancel(frame)
|
||||
|
||||
@property
|
||||
def input(self):
|
||||
"""Get the input transport."""
|
||||
return self._transport.input
|
||||
|
||||
@property
|
||||
def output(self):
|
||||
"""Get the output transport."""
|
||||
return self._transport.output
|
||||