wip - pcc-transport example

examples/pcc-transport/README.md

@@ -0,0 +1,157 @@
+# Simple Chatbot for Pipecat Cloud
+
+This project demonstrates how to build a complete Pipecat AI agent application with both client and server components. It includes a Next.js client for interacting with a Pipecat AI bot server through Daily.co's WebRTC transport.
+
+<img src="image.png" width="420px">
+
+## Project Overview
+
+- **Server**: Python-based Pipecat bot with video/audio processing capabilities
+- **Client**: Next.js TypeScript web application using the Pipecat React & JS SDKs
+- **Infrastructure**: Deployable to Pipecat Cloud (server) and Vercel (client)
+
+> See the [simple-chatbot example](https://github.com/pipecat-ai/pipecat/tree/main/examples/simple-chatbot) with different client and server implementations.
+
+## Quick Start
+
+### 1. Server Setup
+
+Navigate to the server directory:
+
+```bash
+cd server
+```
+
+Create and activate a virtual environment:
+
+```bash
+python3 -m venv venv
+source venv/bin/activate  # On Windows: venv\Scripts\activate
+```
+
+Install requirements:
+
+```bash
+pip install -r requirements.txt
+```
+
+Copy env.example to .env and add your API keys:
+
+```bash
+cp env.example .env
+# Edit .env to add OPENAI_API_KEY and CARTESIA_API_KEY
+```
+
+Run the server locally to test before deploying:
+
+```bash
+LOCAL_RUN=1 python bot.py
+```
+
+This will open a browser window with a Daily.co room where you can test your bot directly.
+
+### 2. Client Setup
+
+In a separate terminal, navigate to the client directory:
+
+```bash
+cd client-react
+```
+
+Install dependencies:
+
+```bash
+npm install
+```
+
+Create `.env.local` file with your Pipecat Cloud API key:
+
+```bash
+cp env.local.example .env.local
+```
+
+> Create a Pipecat Cloud API key using the dashboard
+
+Start the development server:
+
+```bash
+npm run dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) to interact with your agent through the Next.js client.
+
+## Deployment
+
+> See the [Pipecat Cloud Quickstart](https://docs.pipecat.daily.co/quickstart) for a complete walkthrough.
+
+### Deploy Server to Pipecat Cloud
+
+1. Install the Pipecat Cloud CLI:
+
+```bash
+pip install pipecatcloud
+```
+
+2. Authenticate:
+
+```bash
+pcc auth login
+```
+
+3. Build and push your Docker image:
+
+```bash
+cd server
+chmod +x build.sh
+./build.sh
+```
+
+> IMPORTANT: Before running this build script, you need to add your DOCKER_USERNAME
+
+4. Create a secret set for your API keys:
+
+```bash
+pcc secrets set simple-chatbot-secrets --file .env
+```
+
+5. Deploy to Pipecat Cloud:
+
+```bash
+pcc deploy
+```
+
+> IMPORTANT: Before deploying, you need to add your Docker Hub username
+
+### Deploy Client to Vercel
+
+1. Push your Next.js client to GitHub
+
+2. Connect your GitHub repository to Vercel
+
+3. Add your `PIPECAT_CLOUD_API_KEY` environment variable in Vercel
+
+4. Deploy with the Vercel dashboard or CLI
+
+## Project Structure
+
+```
+simple-chatbot/
+├── client-next/            # Next.js client application
+│   ├── src/
+│   │   ├── app/            # Next.js app routes
+│   │   │   └── api/
+│   │   │       └── connect/ # API endpoint for Daily.co connection
+│   │   ├── components/     # React components
+│   │   └── providers/      # React providers including RTVIProvider
+│   ├── package.json
+│   └── README.md          # Client-specific documentation
+│
+└── server/                # Pipecat bot server
+    ├── assets/            # Robot animation frames
+    ├── bot.py             # The Pipecat pipeline implementation
+    ├── Dockerfile         # For building the container image
+    ├── build.sh           # Script for building and pushing Docker image
+    ├── requirements.txt   # Python dependencies
+    ├── pcc-deploy.toml    # Pipecat Cloud deployment config
+    └── README.md          # Server-specific documentation
+```

examples/pcc-transport/client-react/.gitignore

@@ -0,0 +1,41 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# env files (can opt-in for committing if needed)
+.env*
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

examples/pcc-transport/client-react/README.md

@@ -0,0 +1,33 @@
+# Simple Chatbot Client
+
+A Next.js application using TypeScript and the Pipecat React SDK to connect to a Pipecat AI agent.
+
+## Features
+
+- Next.js App Router architecture
+- TypeScript for type safety
+- RTVI client integration for real-time voice and video
+- Daily.co WebRTC transport
+- Custom API endpoint for Daily room creation
+
+## Getting Started
+
+1. Install dependencies:
+
+```bash
+npm install
+```
+
+2. Create `.env.local` file with your Pipecat Cloud API key:
+
+```
+PIPECAT_CLOUD_API_KEY=your_pipecat_cloud_key
+```
+
+3. Start the development server:
+
+```bash
+npm run dev
+```
+
+4. Open [http://localhost:3000](http://localhost:3000) in your browser

examples/pcc-transport/client-react/env.local.example

@@ -0,0 +1 @@
+PIPECAT_CLOUD_API_KEY=your_api_key_here

examples/pcc-transport/client-react/eslint.config.mjs

@@ -0,0 +1,16 @@
+import { dirname } from "path";
+import { fileURLToPath } from "url";
+import { FlatCompat } from "@eslint/eslintrc";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const compat = new FlatCompat({
+  baseDirectory: __dirname,
+});
+
+const eslintConfig = [
+  ...compat.extends("next/core-web-vitals", "next/typescript"),
+];
+
+export default eslintConfig;

examples/pcc-transport/client-react/next.config.ts

@@ -0,0 +1,7 @@
+import type { NextConfig } from "next";
+
+const nextConfig: NextConfig = {
+  /* config options here */
+};
+
+export default nextConfig;

examples/pcc-transport/client-react/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "my-nextjs-app",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "@pipecat-ai/client-js": "^0.3.5",
+    "@pipecat-ai/client-react": "^0.3.5",
+    "@pipecat-ai/daily-transport": "^0.3.7",
+    "next": "15.2.3",
+    "react": "^19.0.0",
+    "react-dom": "^19.0.0"
+  },
+  "devDependencies": {
+    "@eslint/eslintrc": "^3",
+    "@types/node": "^20",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "eslint": "^9",
+    "eslint-config-next": "15.2.3",
+    "typescript": "^5"
+  }
+}

examples/pcc-transport/client-react/src/app/api/connect/route.ts

@@ -0,0 +1,44 @@
+import { NextResponse, NextRequest } from 'next/server';
+
+export async function POST(request: NextRequest) {
+  try {
+    const { MY_CUSTOM_DATA } = await request.json();
+
+    const response = await fetch(
+      'https://api.pipecat.daily.co/v1/public/simple-chatbot/start',
+      {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${process.env.PIPECAT_CLOUD_API_KEY}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          // Create Daily room
+          createDailyRoom: true,
+          // Optionally set Daily room properties
+          dailyRoomProperties: { start_video_off: true },
+          // Optionally pass custom data to the bot
+          body: { MY_CUSTOM_DATA },
+        }),
+      }
+    );
+
+    if (!response.ok) {
+      throw new Error(`API responded with status: ${response.status}`);
+    }
+
+    const data = await response.json();
+
+    // Transform the response to match what RTVI client expects
+    return NextResponse.json({
+      room_url: data.dailyRoom,
+      token: data.dailyToken,
+    });
+  } catch (error) {
+    console.error('API error:', error);
+    return NextResponse.json(
+      { error: 'Failed to start agent' },
+      { status: 500 }
+    );
+  }
+}

examples/pcc-transport/client-react/src/app/globals.css

@@ -0,0 +1,82 @@
+body {
+  margin: 0;
+  padding: 20px;
+  font-family: Arial, sans-serif;
+  background-color: #f0f0f0;
+}
+
+.app {
+  max-width: 1200px;
+  margin: 0 auto;
+}
+
+.status-bar {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  padding: 10px;
+  background-color: #fff;
+  border-radius: 8px;
+  margin-bottom: 20px;
+}
+
+.controls button {
+  padding: 8px 16px;
+  margin-left: 10px;
+  border: none;
+  border-radius: 4px;
+  cursor: pointer;
+}
+
+button:disabled {
+  opacity: 0.5;
+  cursor: not-allowed;
+}
+
+.connect-btn {
+  background-color: #4caf50;
+  color: white;
+}
+
+.disconnect-btn {
+  background-color: #f44336;
+  color: white;
+}
+
+.main-content {
+  background-color: #fff;
+  border-radius: 8px;
+  padding: 20px;
+  margin-bottom: 20px;
+}
+
+.bot-container {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+.video-container {
+  width: 640px;
+  height: 360px;
+  background-color: #ddd;
+  margin-bottom: 20px;
+  border-radius: 8px;
+  overflow: hidden;
+}
+
+.video-container video {
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+}
+
+.mic-enabled {
+  background-color: #4caf50;
+  color: white;
+}
+
+.mic-disabled {
+  background-color: #f44336;
+  color: white;
+}

examples/pcc-transport/client-react/src/app/layout.tsx

@@ -0,0 +1,21 @@
+import './globals.css';
+import { RTVIProvider } from '@/providers/RTVIProvider';
+
+export const metadata = {
+  title: 'Pipecat React Client',
+  description: 'Pipecat RTVI Client using Next.js',
+};
+
+export default function RootLayout({
+  children,
+}: {
+  children: React.ReactNode;
+}) {
+  return (
+    <html lang="en">
+      <body>
+        <RTVIProvider>{children}</RTVIProvider>
+      </body>
+    </html>
+  );
+}

examples/pcc-transport/client-react/src/app/page.tsx

@@ -0,0 +1,41 @@
+'use client';
+
+import {
+  RTVIClientAudio,
+  RTVIClientVideo,
+  useRTVIClientTransportState,
+} from '@pipecat-ai/client-react';
+import { ConnectButton } from '../components/ConnectButton';
+import { StatusDisplay } from '../components/StatusDisplay';
+import { DebugDisplay } from '../components/DebugDisplay';
+
+function BotVideo() {
+  const transportState = useRTVIClientTransportState();
+  const isConnected = transportState !== 'disconnected';
+
+  return (
+    <div className="bot-container">
+      <div className="video-container">
+        {isConnected && <RTVIClientVideo participant="bot" fit="cover" />}
+      </div>
+    </div>
+  );
+}
+
+export default function Home() {
+  return (
+    <div className="app">
+      <div className="status-bar">
+        <StatusDisplay />
+        <ConnectButton />
+      </div>
+
+      <div className="main-content">
+        <BotVideo />
+      </div>
+
+      <DebugDisplay />
+      <RTVIClientAudio />
+    </div>
+  );
+}

examples/pcc-transport/client-react/src/components/ConnectButton.tsx

@@ -0,0 +1,40 @@
+import {
+  useRTVIClient,
+  useRTVIClientTransportState,
+} from '@pipecat-ai/client-react';
+
+export function ConnectButton() {
+  const client = useRTVIClient();
+  const transportState = useRTVIClientTransportState();
+  const isConnected = ['connected', 'ready'].includes(transportState);
+
+  const handleClick = async () => {
+    if (!client) {
+      console.error('RTVI client is not initialized');
+      return;
+    }
+
+    try {
+      if (isConnected) {
+        await client.disconnect();
+      } else {
+        await client.connect();
+      }
+    } catch (error) {
+      console.error('Connection error:', error);
+    }
+  };
+
+  return (
+    <div className="controls">
+      <button
+        className={isConnected ? 'disconnect-btn' : 'connect-btn'}
+        onClick={handleClick}
+        disabled={
+          !client || ['connecting', 'disconnecting'].includes(transportState)
+        }>
+        {isConnected ? 'Disconnect' : 'Connect'}
+      </button>
+    </div>
+  );
+}

examples/pcc-transport/client-react/src/components/DebugDisplay.css

@@ -0,0 +1,26 @@
+.debug-panel {
+  background-color: #fff;
+  border-radius: 8px;
+  padding: 20px;
+}
+
+.debug-panel h3 {
+  margin: 0 0 10px 0;
+  font-size: 16px;
+  font-weight: bold;
+}
+
+.debug-log {
+  height: 200px;
+  overflow-y: auto;
+  background-color: #f8f8f8;
+  padding: 10px;
+  border-radius: 4px;
+  font-family: monospace;
+  font-size: 12px;
+  line-height: 1.4;
+}
+
+.debug-log div {
+  margin-bottom: 4px;
+}

examples/pcc-transport/client-react/src/components/DebugDisplay.tsx

@@ -0,0 +1,144 @@
+import { useRef, useCallback } from 'react';
+import {
+  Participant,
+  RTVIEvent,
+  TransportState,
+  TranscriptData,
+  BotLLMTextData,
+} from '@pipecat-ai/client-js';
+import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
+import './DebugDisplay.css';
+
+export function DebugDisplay() {
+  const debugLogRef = useRef<HTMLDivElement>(null);
+  const client = useRTVIClient();
+
+  const log = useCallback((message: string) => {
+    if (!debugLogRef.current) return;
+
+    const entry = document.createElement('div');
+    entry.textContent = `${new Date().toISOString()} - ${message}`;
+
+    // Add styling based on message type
+    if (message.startsWith('User: ')) {
+      entry.style.color = '#2196F3'; // blue for user
+    } else if (message.startsWith('Bot: ')) {
+      entry.style.color = '#4CAF50'; // green for bot
+    }
+
+    debugLogRef.current.appendChild(entry);
+    debugLogRef.current.scrollTop = debugLogRef.current.scrollHeight;
+  }, []);
+
+  // Log transport state changes
+  useRTVIClientEvent(
+    RTVIEvent.TransportStateChanged,
+    useCallback(
+      (state: TransportState) => {
+        log(`Transport state changed: ${state}`);
+      },
+      [log]
+    )
+  );
+
+  // Log bot connection events
+  useRTVIClientEvent(
+    RTVIEvent.BotConnected,
+    useCallback(
+      (participant?: Participant) => {
+        log(`Bot connected: ${JSON.stringify(participant)}`);
+      },
+      [log]
+    )
+  );
+
+  useRTVIClientEvent(
+    RTVIEvent.BotDisconnected,
+    useCallback(
+      (participant?: Participant) => {
+        log(`Bot disconnected: ${JSON.stringify(participant)}`);
+      },
+      [log]
+    )
+  );
+
+  // Log track events
+  useRTVIClientEvent(
+    RTVIEvent.TrackStarted,
+    useCallback(
+      (track: MediaStreamTrack, participant?: Participant) => {
+        log(
+          `Track started: ${track.kind} from ${participant?.name || 'unknown'}`
+        );
+      },
+      [log]
+    )
+  );
+
+  useRTVIClientEvent(
+    RTVIEvent.TrackStopped,
+    useCallback(
+      (track: MediaStreamTrack, participant?: Participant) => {
+        log(
+          `Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
+        );
+      },
+      [log]
+    )
+  );
+
+  // Log bot ready state and check tracks
+  useRTVIClientEvent(
+    RTVIEvent.BotReady,
+    useCallback(() => {
+      log(`Bot ready`);
+
+      if (!client) return;
+
+      const tracks = client.tracks();
+      log(
+        `Available tracks: ${JSON.stringify({
+          local: {
+            audio: !!tracks.local.audio,
+            video: !!tracks.local.video,
+          },
+          bot: {
+            audio: !!tracks.bot?.audio,
+            video: !!tracks.bot?.video,
+          },
+        })}`
+      );
+    }, [client, log])
+  );
+
+  // Log transcripts
+  useRTVIClientEvent(
+    RTVIEvent.UserTranscript,
+    useCallback(
+      (data: TranscriptData) => {
+        // Only log final transcripts
+        if (data.final) {
+          log(`User: ${data.text}`);
+        }
+      },
+      [log]
+    )
+  );
+
+  useRTVIClientEvent(
+    RTVIEvent.BotTranscript,
+    useCallback(
+      (data: BotLLMTextData) => {
+        log(`Bot: ${data.text}`);
+      },
+      [log]
+    )
+  );
+
+  return (
+    <div className="debug-panel">
+      <h3>Debug Info</h3>
+      <div ref={debugLogRef} className="debug-log" />
+    </div>
+  );
+}

examples/pcc-transport/client-react/src/components/StatusDisplay.tsx

@@ -0,0 +1,11 @@
+import { useRTVIClientTransportState } from '@pipecat-ai/client-react';
+
+export function StatusDisplay() {
+  const transportState = useRTVIClientTransportState();
+
+  return (
+    <div className="status">
+      Status: <span>{transportState}</span>
+    </div>
+  );
+}

examples/pcc-transport/client-react/src/providers/RTVIProvider.tsx

@@ -0,0 +1,38 @@
+'use client';
+
+import { RTVIClient } from '@pipecat-ai/client-js';
+import { DailyTransport } from '@pipecat-ai/daily-transport';
+import { RTVIClientProvider } from '@pipecat-ai/client-react';
+import { PropsWithChildren, useEffect, useState } from 'react';
+
+const MY_CUSTOM_DATA = { foo: 'bar' };
+
+export function RTVIProvider({ children }: PropsWithChildren) {
+  const [client, setClient] = useState<RTVIClient | null>(null);
+
+  useEffect(() => {
+    console.log('Setting up Transport and Client');
+    const transport = new DailyTransport();
+
+    const rtviClient = new RTVIClient({
+      transport,
+      params: {
+        baseUrl: '/api',
+        endpoints: {
+          connect: '/connect',
+        },
+        requestData: { MY_CUSTOM_DATA },
+      },
+      enableMic: true,
+      enableCam: false,
+    });
+
+    setClient(rtviClient);
+  }, []);
+
+  if (!client) {
+    return null;
+  }
+
+  return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
+}

examples/pcc-transport/client-react/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "target": "ES2017",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/components/*": ["./src/components/*"],
+      "@/providers/*": ["./src/providers/*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}