Fal Smart Turn example
This commit is contained in:
Mark Backman
44
examples/fal-smart-turn/client/src/app/api/connect/route.ts
Normal file
44
examples/fal-smart-turn/client/src/app/api/connect/route.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
import { NextResponse, NextRequest } from 'next/server';
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const { MY_CUSTOM_DATA } = await request.json();
|
||||
|
||||
try {
|
||||
const response = await fetch(
|
||||
`https://api.pipecat.daily.co/v1/public/${process.env.AGENT_NAME}/start`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${process.env.PIPECAT_CLOUD_API_KEY}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
// Create Daily room
|
||||
createDailyRoom: true,
|
||||
// Optionally set Daily room properties
|
||||
dailyRoomProperties: { start_video_off: true },
|
||||
// Optionally pass custom data to the bot
|
||||
body: { MY_CUSTOM_DATA },
|
||||
}),
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`API responded with status: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
// Transform the response to match what RTVI client expects
|
||||
return NextResponse.json({
|
||||
room_url: data.dailyRoom,
|
||||
token: data.dailyToken,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('API error:', error);
|
||||
return NextResponse.json(
|
||||
{ error: 'Failed to start agent' },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
82
examples/fal-smart-turn/client/src/app/globals.css
Normal file
82
examples/fal-smart-turn/client/src/app/globals.css
Normal file
@@ -0,0 +1,82 @@
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 20px;
|
||||
font-family: Arial, sans-serif;
|
||||
background-color: #f0f0f0;
|
||||
}
|
||||
|
||||
.app {
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.status-bar {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 10px;
|
||||
background-color: #fff;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.controls button {
|
||||
padding: 8px 16px;
|
||||
margin-left: 10px;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
button:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.connect-btn {
|
||||
background-color: #4caf50;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.disconnect-btn {
|
||||
background-color: #f44336;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.main-content {
|
||||
background-color: #fff;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.bot-container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.video-container {
|
||||
width: 640px;
|
||||
height: 360px;
|
||||
background-color: #ddd;
|
||||
margin-bottom: 20px;
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.video-container video {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.mic-enabled {
|
||||
background-color: #4caf50;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.mic-disabled {
|
||||
background-color: #f44336;
|
||||
color: white;
|
||||
}
|
||||
27
examples/fal-smart-turn/client/src/app/layout.tsx
Normal file
27
examples/fal-smart-turn/client/src/app/layout.tsx
Normal file
@@ -0,0 +1,27 @@
|
||||
import './globals.css';
|
||||
import { RTVIProvider } from '@/providers/RTVIProvider';
|
||||
|
||||
export const metadata = {
|
||||
title: 'Pipecat React Client',
|
||||
description: 'Pipecat RTVI Client using Next.js',
|
||||
icons: {
|
||||
icon: [{ url: '/favicon.svg', type: 'image/svg+xml' }],
|
||||
},
|
||||
};
|
||||
|
||||
export default function RootLayout({
|
||||
children,
|
||||
}: {
|
||||
children: React.ReactNode;
|
||||
}) {
|
||||
return (
|
||||
<html lang="en">
|
||||
<head>
|
||||
<link rel="icon" href="/favicon.svg" type="image/svg+xml" />
|
||||
</head>
|
||||
<body>
|
||||
<RTVIProvider>{children}</RTVIProvider>
|
||||
</body>
|
||||
</html>
|
||||
);
|
||||
}
|
||||
41
examples/fal-smart-turn/client/src/app/page.tsx
Normal file
41
examples/fal-smart-turn/client/src/app/page.tsx
Normal file
@@ -0,0 +1,41 @@
|
||||
'use client';
|
||||
|
||||
import {
|
||||
RTVIClientAudio,
|
||||
RTVIClientVideo,
|
||||
useRTVIClientTransportState,
|
||||
} from '@pipecat-ai/client-react';
|
||||
import { ConnectButton } from '../components/ConnectButton';
|
||||
import { StatusDisplay } from '../components/StatusDisplay';
|
||||
import { DebugDisplay } from '../components/DebugDisplay';
|
||||
|
||||
function BotVideo() {
|
||||
const transportState = useRTVIClientTransportState();
|
||||
const isConnected = transportState !== 'disconnected';
|
||||
|
||||
return (
|
||||
<div className="bot-container">
|
||||
<div className="video-container">
|
||||
{isConnected && <RTVIClientVideo participant="bot" fit="cover" />}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Home() {
|
||||
return (
|
||||
<div className="app">
|
||||
<div className="status-bar">
|
||||
<StatusDisplay />
|
||||
<ConnectButton />
|
||||
</div>
|
||||
|
||||
<div className="main-content">
|
||||
<BotVideo />
|
||||
</div>
|
||||
|
||||
<DebugDisplay />
|
||||
<RTVIClientAudio />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import {
|
||||
useRTVIClient,
|
||||
useRTVIClientTransportState,
|
||||
} from '@pipecat-ai/client-react';
|
||||
|
||||
export function ConnectButton() {
|
||||
const client = useRTVIClient();
|
||||
const transportState = useRTVIClientTransportState();
|
||||
const isConnected = ['connected', 'ready'].includes(transportState);
|
||||
|
||||
const handleClick = async () => {
|
||||
if (!client) {
|
||||
console.error('RTVI client is not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (isConnected) {
|
||||
await client.disconnect();
|
||||
} else {
|
||||
await client.connect();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Connection error:', error);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="controls">
|
||||
<button
|
||||
className={isConnected ? 'disconnect-btn' : 'connect-btn'}
|
||||
onClick={handleClick}
|
||||
disabled={
|
||||
!client || ['connecting', 'disconnecting'].includes(transportState)
|
||||
}>
|
||||
{isConnected ? 'Disconnect' : 'Connect'}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
.debug-panel {
|
||||
background-color: #fff;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.debug-panel h3 {
|
||||
margin: 0 0 10px 0;
|
||||
font-size: 16px;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.debug-log {
|
||||
height: 200px;
|
||||
overflow-y: auto;
|
||||
background-color: #f8f8f8;
|
||||
padding: 10px;
|
||||
border-radius: 4px;
|
||||
font-family: monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.debug-log div {
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
171
examples/fal-smart-turn/client/src/components/DebugDisplay.tsx
Normal file
171
examples/fal-smart-turn/client/src/components/DebugDisplay.tsx
Normal file
@@ -0,0 +1,171 @@
|
||||
import { useRef, useCallback } from 'react';
|
||||
import {
|
||||
Participant,
|
||||
RTVIEvent,
|
||||
TransportState,
|
||||
TranscriptData,
|
||||
BotLLMTextData,
|
||||
} from '@pipecat-ai/client-js';
|
||||
import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
|
||||
import './DebugDisplay.css';
|
||||
|
||||
interface SmartTurnResultData {
|
||||
type: 'smart_turn_result';
|
||||
is_complete: boolean;
|
||||
probability: number;
|
||||
inference_time_ms: number; // Pure model inference time
|
||||
server_total_time_ms: number; // Server processing time
|
||||
e2e_processing_time_ms: number; // Complete end-to-end time
|
||||
}
|
||||
|
||||
export function DebugDisplay() {
|
||||
const debugLogRef = useRef<HTMLDivElement>(null);
|
||||
const client = useRTVIClient();
|
||||
|
||||
const log = useCallback((message: string) => {
|
||||
if (!debugLogRef.current) return;
|
||||
|
||||
const entry = document.createElement('div');
|
||||
entry.textContent = `${new Date().toISOString()} - ${message}`;
|
||||
|
||||
// Add styling based on message type
|
||||
if (message.startsWith('User: ')) {
|
||||
entry.style.color = '#2196F3'; // blue for user
|
||||
} else if (message.startsWith('Bot: ')) {
|
||||
entry.style.color = '#4CAF50'; // green for bot
|
||||
} else if (message.includes('Smart Turn:')) {
|
||||
entry.style.color = '#9C27B0'; // purple for smart turn
|
||||
}
|
||||
|
||||
debugLogRef.current.appendChild(entry);
|
||||
debugLogRef.current.scrollTop = debugLogRef.current.scrollHeight;
|
||||
}, []);
|
||||
|
||||
// Log transport state changes
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.TransportStateChanged,
|
||||
useCallback(
|
||||
(state: TransportState) => {
|
||||
log(`Transport state changed: ${state}`);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
// Log bot connection events
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.BotConnected,
|
||||
useCallback(
|
||||
(participant?: Participant) => {
|
||||
log(`Bot connected: ${JSON.stringify(participant)}`);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.BotDisconnected,
|
||||
useCallback(
|
||||
(participant?: Participant) => {
|
||||
log(`Bot disconnected: ${JSON.stringify(participant)}`);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
// Log track events
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.TrackStarted,
|
||||
useCallback(
|
||||
(track: MediaStreamTrack, participant?: Participant) => {
|
||||
log(
|
||||
`Track started: ${track.kind} from ${participant?.name || 'unknown'}`
|
||||
);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.TrackStopped,
|
||||
useCallback(
|
||||
(track: MediaStreamTrack, participant?: Participant) => {
|
||||
log(
|
||||
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`
|
||||
);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
// Log bot ready state and check tracks
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.BotReady,
|
||||
useCallback(() => {
|
||||
log(`Bot ready`);
|
||||
|
||||
if (!client) return;
|
||||
|
||||
const tracks = client.tracks();
|
||||
log(
|
||||
`Available tracks: ${JSON.stringify({
|
||||
local: {
|
||||
audio: !!tracks.local.audio,
|
||||
video: !!tracks.local.video,
|
||||
},
|
||||
bot: {
|
||||
audio: !!tracks.bot?.audio,
|
||||
video: !!tracks.bot?.video,
|
||||
},
|
||||
})}`
|
||||
);
|
||||
}, [client, log])
|
||||
);
|
||||
|
||||
// Log transcripts
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.UserTranscript,
|
||||
useCallback(
|
||||
(data: TranscriptData) => {
|
||||
// Only log final transcripts
|
||||
if (data.final) {
|
||||
log(`User: ${data.text}`);
|
||||
}
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.BotTranscript,
|
||||
useCallback(
|
||||
(data: BotLLMTextData) => {
|
||||
log(`Bot: ${data.text}`);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.ServerMessage,
|
||||
useCallback(
|
||||
(data: SmartTurnResultData) => {
|
||||
log(
|
||||
`Smart Turn:
|
||||
${data.is_complete ? 'COMPLETE' : 'INCOMPLETE'},
|
||||
Probability: ${(data.probability * 100).toFixed(1)}%,
|
||||
Model inference: ${data.inference_time_ms?.toFixed(2) || 'N/A'}ms,
|
||||
Server processing: ${data.server_total_time_ms?.toFixed(2) || 'N/A'}ms,
|
||||
End-to-end: ${data.e2e_processing_time_ms?.toFixed(2) || 'N/A'}ms`
|
||||
);
|
||||
},
|
||||
[log]
|
||||
)
|
||||
);
|
||||
return (
|
||||
<div className="debug-panel">
|
||||
<h3>Debug Info</h3>
|
||||
<div ref={debugLogRef} className="debug-log" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { useRTVIClientTransportState } from '@pipecat-ai/client-react';
|
||||
|
||||
export function StatusDisplay() {
|
||||
const transportState = useRTVIClientTransportState();
|
||||
|
||||
return (
|
||||
<div className="status">
|
||||
Status: <span>{transportState}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
'use client';
|
||||
|
||||
import { RTVIClient } from '@pipecat-ai/client-js';
|
||||
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
||||
import { RTVIClientProvider } from '@pipecat-ai/client-react';
|
||||
import { PropsWithChildren, useEffect, useState } from 'react';
|
||||
|
||||
// Get the API base URL from environment variables
|
||||
// Default to "/api" if not specified
|
||||
// "/api" is the default for Next.js API routes and used
|
||||
// for the Pipecat Cloud deployed agent
|
||||
const API_BASE_URL = process.env.NEXT_PUBLIC_API_BASE_URL || '/api';
|
||||
|
||||
console.log('Using API base URL:', API_BASE_URL);
|
||||
|
||||
export function RTVIProvider({ children }: PropsWithChildren) {
|
||||
const [client, setClient] = useState<RTVIClient | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const transport = new DailyTransport();
|
||||
|
||||
const rtviClient = new RTVIClient({
|
||||
transport,
|
||||
params: {
|
||||
baseUrl: API_BASE_URL,
|
||||
endpoints: {
|
||||
connect: '/connect',
|
||||
},
|
||||
requestData: { foo: 'bar' },
|
||||
},
|
||||
enableMic: true,
|
||||
enableCam: false,
|
||||
});
|
||||
|
||||
setClient(rtviClient);
|
||||
}, []);
|
||||
|
||||
if (!client) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
|
||||
}
|
||||
Reference in New Issue
Block a user