Add mem0 as a service integration
This commit is contained in:
51
examples/personalized-voice-agent/.gitignore
vendored
Normal file
51
examples/personalized-voice-agent/.gitignore
vendored
Normal file
@@ -0,0 +1,51 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# JavaScript/Node.js
|
||||
node_modules/
|
||||
dist/
|
||||
dist-ssr/
|
||||
*.local
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
|
||||
# Editor/IDE
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
.DS_Store
|
||||
|
||||
# Project specific
|
||||
runpod.toml
|
||||
80
examples/personalized-voice-agent/README.md
Normal file
80
examples/personalized-voice-agent/README.md
Normal file
@@ -0,0 +1,80 @@
|
||||
# Personalized Voice Agent
|
||||
|
||||
This repository demonstrates a personalized voice agent with real-time audio/video interaction, implemented using different client and server options. The bot server supports multiple AI backends, and you can connect to it using five different client approaches.
|
||||
|
||||
Here is a demo video:
|
||||
|
||||
[](https://www.youtube.com/watch?v=FR0yCDw29SI)
|
||||
|
||||
## Bot Options
|
||||
|
||||
- **OpenAI Bot** (Default)
|
||||
- Uses gpt-4o for conversation
|
||||
- Requires OpenAI API key
|
||||
|
||||
## Client Option
|
||||
|
||||
- **React**
|
||||
- Basic implementation using [Pipecat React SDK](https://docs.pipecat.ai/client/react/introduction)
|
||||
- Demonstrates the basic client principles with Pipecat React
|
||||
|
||||
## Quick Start
|
||||
|
||||
### First, start the bot server:
|
||||
|
||||
1. Navigate to the server directory:
|
||||
```bash
|
||||
cd server
|
||||
```
|
||||
|
||||
2. Create and activate a virtual environment:
|
||||
```bash
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate # On Windows: venv\Scripts\activate
|
||||
```
|
||||
|
||||
3. Install requirements:
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
4. Copy env.example to .env and configure:
|
||||
- Add your API keys
|
||||
|
||||
5. Start the server:
|
||||
```bash
|
||||
python server.py
|
||||
```
|
||||
|
||||
6. Next, connect using [react client](client/react/README.md)
|
||||
|
||||
## Important Note
|
||||
|
||||
The bot server must be running for any of the client implementations to work. Start the server first before trying any of the client apps.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.10+
|
||||
- Node.js 16+ (for JavaScript and React implementations)
|
||||
- Daily API key
|
||||
- OpenAI API key (for OpenAI bot)
|
||||
- ElevenLabs API key
|
||||
- Mem0 API Key
|
||||
- Modern web browser with WebRTC support
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
personalized-voice-agent/
|
||||
├── server/ # Bot server implementation
|
||||
│ ├── bot-mem0.py # Mem0 bot implementation
|
||||
│ ├── runner.py # Server runner utilities
|
||||
│ ├── server.py # FastAPI server
|
||||
│ └── requirements.txt
|
||||
└── client/ # Client implementations
|
||||
├── android/ # Daily Android connection
|
||||
├── ios/ # Daily iOS connection
|
||||
├── javascript/ # Daily JavaScript connection
|
||||
├── prebuilt/ # Pipecat Prebuilt client
|
||||
└── react/ # Pipecat React client
|
||||
```
|
||||
26
examples/personalized-voice-agent/client/react/.gitignore
vendored
Normal file
26
examples/personalized-voice-agent/client/react/.gitignore
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
lerna-debug.log*
|
||||
|
||||
!**/lib
|
||||
|
||||
node_modules
|
||||
dist
|
||||
dist-ssr
|
||||
*.local
|
||||
|
||||
# Editor directories and files
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
.idea
|
||||
.DS_Store
|
||||
*.suo
|
||||
*.ntvs*
|
||||
*.njsproj
|
||||
*.sln
|
||||
*.sw?
|
||||
27
examples/personalized-voice-agent/client/react/README.md
Normal file
27
examples/personalized-voice-agent/client/react/README.md
Normal file
@@ -0,0 +1,27 @@
|
||||
# React Implementation
|
||||
|
||||
Basic implementation using the [Pipecat React SDK](https://docs.pipecat.ai/client/react/introduction).
|
||||
|
||||
## Setup
|
||||
|
||||
1. Run the bot server; see [README](../../README).
|
||||
|
||||
2. Navigate to the `client/react` directory:
|
||||
|
||||
```bash
|
||||
cd client/react
|
||||
```
|
||||
|
||||
3. Install dependencies:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
4. Run the client app:
|
||||
|
||||
```
|
||||
npm run dev
|
||||
```
|
||||
|
||||
5. Visit http://localhost:5173 in your browser.
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"$schema": "https://ui.shadcn.com/schema.json",
|
||||
"style": "new-york",
|
||||
"rsc": false,
|
||||
"tsx": true,
|
||||
"tailwind": {
|
||||
"config": "",
|
||||
"css": "src/App.css",
|
||||
"baseColor": "zinc",
|
||||
"cssVariables": true,
|
||||
"prefix": ""
|
||||
},
|
||||
"aliases": {
|
||||
"components": "@/components",
|
||||
"utils": "@/lib/utils",
|
||||
"ui": "@/components/ui",
|
||||
"lib": "@/lib",
|
||||
"hooks": "@/hooks"
|
||||
},
|
||||
"iconLibrary": "lucide"
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import js from '@eslint/js'
|
||||
import globals from 'globals'
|
||||
import reactHooks from 'eslint-plugin-react-hooks'
|
||||
import reactRefresh from 'eslint-plugin-react-refresh'
|
||||
import tseslint from 'typescript-eslint'
|
||||
|
||||
export default tseslint.config(
|
||||
{ ignores: ['dist'] },
|
||||
{
|
||||
extends: [js.configs.recommended, ...tseslint.configs.recommended],
|
||||
files: ['**/*.{ts,tsx}'],
|
||||
languageOptions: {
|
||||
ecmaVersion: 2020,
|
||||
globals: globals.browser,
|
||||
},
|
||||
plugins: {
|
||||
'react-hooks': reactHooks,
|
||||
'react-refresh': reactRefresh,
|
||||
},
|
||||
rules: {
|
||||
...reactHooks.configs.recommended.rules,
|
||||
'react-refresh/only-export-components': [
|
||||
'warn',
|
||||
{ allowConstantExport: true },
|
||||
],
|
||||
},
|
||||
},
|
||||
)
|
||||
15
examples/personalized-voice-agent/client/react/index.html
Normal file
15
examples/personalized-voice-agent/client/react/index.html
Normal file
@@ -0,0 +1,15 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Mem0 - Pipecat React Client</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
4387
examples/personalized-voice-agent/client/react/package-lock.json
generated
Normal file
4387
examples/personalized-voice-agent/client/react/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
48
examples/personalized-voice-agent/client/react/package.json
Normal file
48
examples/personalized-voice-agent/client/react/package.json
Normal file
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"name": "react",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc && vite build",
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"@phosphor-icons/react": "^2.1.7",
|
||||
"@pipecat-ai/client-js": "^0.3.2",
|
||||
"@pipecat-ai/client-react": "^0.3.2",
|
||||
"@pipecat-ai/daily-transport": "^0.3.4",
|
||||
"@radix-ui/react-scroll-area": "^1.2.3",
|
||||
"@radix-ui/react-slot": "^1.1.2",
|
||||
"@tailwindcss/vite": "^4.0.12",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"date-fns": "^4.1.0",
|
||||
"framer-motion": "^12.4.10",
|
||||
"lucide-react": "^0.479.0",
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1",
|
||||
"tailwind-merge": "^3.0.2",
|
||||
"tailwindcss-animate": "^1.0.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.15.0",
|
||||
"@types/node": "^22.13.9",
|
||||
"@types/react": "^18.3.12",
|
||||
"@types/react-dom": "^18.3.1",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
"autoprefixer": "^10.4.20",
|
||||
"eslint": "^9.15.0",
|
||||
"eslint-plugin-react-hooks": "^5.0.0",
|
||||
"eslint-plugin-react-refresh": "^0.4.14",
|
||||
"globals": "^15.12.0",
|
||||
"postcss": "^8.5.3",
|
||||
"tailwindcss": "^4.0.12",
|
||||
"typescript": "~5.6.2",
|
||||
"typescript-eslint": "^8.15.0",
|
||||
"vite": "^6.0.9"
|
||||
},
|
||||
"packageManager": "pnpm@10.5.2+sha512.da9dc28cd3ff40d0592188235ab25d3202add8a207afbedc682220e4a0029ffbff4562102b9e6e46b4e3f9e8bd53e6d05de48544b0c57d4b0179e22c76d1199b"
|
||||
}
|
||||
267
examples/personalized-voice-agent/client/react/src/App.css
Normal file
267
examples/personalized-voice-agent/client/react/src/App.css
Normal file
@@ -0,0 +1,267 @@
|
||||
@import "tailwindcss";
|
||||
|
||||
@plugin "tailwindcss-animate";
|
||||
|
||||
@custom-variant dark (&:is(.dark *));
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 20px;
|
||||
font-family: Arial, sans-serif;
|
||||
background-color: #f0f0f0;
|
||||
}
|
||||
|
||||
@keyframes message-pop {
|
||||
0% {
|
||||
opacity: 0;
|
||||
transform: scale(0.8);
|
||||
}
|
||||
50% {
|
||||
transform: scale(1.05);
|
||||
}
|
||||
100% {
|
||||
opacity: 1;
|
||||
transform: scale(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@keyframes pulse-shadow {
|
||||
0% {
|
||||
box-shadow: 0 0 0 0 rgba(161, 161, 170, 0.7);
|
||||
}
|
||||
70% {
|
||||
box-shadow: 0 0 0 20px rgba(161, 161, 170, 0);
|
||||
}
|
||||
100% {
|
||||
box-shadow: 0 0 0 0 rgba(161, 161, 170, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes wave {
|
||||
0% { transform: scaleY(0.2); }
|
||||
50% { transform: scaleY(1); }
|
||||
100% { transform: scaleY(0.2); }
|
||||
}
|
||||
|
||||
@keyframes wave-listening {
|
||||
0% { transform: scaleY(0.3); }
|
||||
50% { transform: scaleY(0.5); }
|
||||
100% { transform: scaleY(0.3); }
|
||||
}
|
||||
|
||||
@keyframes wave-user {
|
||||
0% { transform: scaleY(0.4); }
|
||||
50% { transform: scaleY(1.2); }
|
||||
100% { transform: scaleY(0.4); }
|
||||
}
|
||||
|
||||
@keyframes wave-assistant {
|
||||
0% { transform: scaleY(0.3); }
|
||||
50% { transform: scaleY(0.9); }
|
||||
100% { transform: scaleY(0.3); }
|
||||
}
|
||||
|
||||
@keyframes memory-indicator-pop {
|
||||
0% {
|
||||
opacity: 0;
|
||||
transform: translateY(-10px);
|
||||
height: 0;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
30% {
|
||||
height: 24px;
|
||||
margin-bottom: 0.25rem;
|
||||
}
|
||||
60% {
|
||||
opacity: 0.7;
|
||||
}
|
||||
100% {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
height: 24px;
|
||||
margin-bottom: 0.25rem;
|
||||
}
|
||||
}
|
||||
|
||||
.animate-message-pop {
|
||||
animation: message-pop 0.3s ease-out forwards;
|
||||
}
|
||||
|
||||
.animate-pulse-shadow {
|
||||
animation: pulse-shadow 2s infinite;
|
||||
}
|
||||
|
||||
.waveform-bar {
|
||||
animation: wave-user 0.8s ease-in-out infinite;
|
||||
transform-origin: bottom;
|
||||
}
|
||||
|
||||
.waveform-bar-listening {
|
||||
animation: wave-listening 1.5s ease-in-out infinite;
|
||||
transform-origin: bottom;
|
||||
}
|
||||
|
||||
.waveform-bar.bg-zinc-600 {
|
||||
animation: wave-assistant 1s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.waveform-bar.bg-zinc-400 {
|
||||
animation: wave-listening 1.5s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.waveform-bar:nth-child(2) { animation-delay: 0.1s; }
|
||||
.waveform-bar:nth-child(3) { animation-delay: 0.2s; }
|
||||
.waveform-bar:nth-child(4) { animation-delay: 0.3s; }
|
||||
.waveform-bar:nth-child(5) { animation-delay: 0.4s; }
|
||||
|
||||
.slide-up-enter {
|
||||
opacity: 0;
|
||||
transform: translateY(20px);
|
||||
}
|
||||
|
||||
.slide-up-enter-active {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
transition: opacity 300ms, transform 300ms;
|
||||
}
|
||||
|
||||
.slide-up-exit {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
}
|
||||
|
||||
.slide-up-exit-active {
|
||||
opacity: 0;
|
||||
transform: translateY(-20px);
|
||||
transition: opacity 300ms, transform 300ms;
|
||||
}
|
||||
|
||||
/* Smooth scrolling for the entire page */
|
||||
html {
|
||||
scroll-behavior: smooth;
|
||||
}
|
||||
|
||||
:root {
|
||||
--background: oklch(1 0 0);
|
||||
--foreground: oklch(0.141 0.005 285.823);
|
||||
--card: oklch(1 0 0);
|
||||
--card-foreground: oklch(0.141 0.005 285.823);
|
||||
--popover: oklch(1 0 0);
|
||||
--popover-foreground: oklch(0.141 0.005 285.823);
|
||||
--primary: oklch(0.21 0.006 285.885);
|
||||
--primary-foreground: oklch(0.985 0 0);
|
||||
--secondary: oklch(0.967 0.001 286.375);
|
||||
--secondary-foreground: oklch(0.21 0.006 285.885);
|
||||
--muted: oklch(0.967 0.001 286.375);
|
||||
--muted-foreground: oklch(0.552 0.016 285.938);
|
||||
--accent: oklch(0.967 0.001 286.375);
|
||||
--accent-foreground: oklch(0.21 0.006 285.885);
|
||||
--destructive: oklch(0.577 0.245 27.325);
|
||||
--destructive-foreground: oklch(0.577 0.245 27.325);
|
||||
--border: oklch(0.92 0.004 286.32);
|
||||
--input: oklch(0.92 0.004 286.32);
|
||||
--ring: oklch(0.705 0.015 286.067);
|
||||
--chart-1: oklch(0.646 0.222 41.116);
|
||||
--chart-2: oklch(0.6 0.118 184.704);
|
||||
--chart-3: oklch(0.398 0.07 227.392);
|
||||
--chart-4: oklch(0.828 0.189 84.429);
|
||||
--chart-5: oklch(0.769 0.188 70.08);
|
||||
--radius: 0.625rem;
|
||||
--sidebar: oklch(0.985 0 0);
|
||||
--sidebar-foreground: oklch(0.141 0.005 285.823);
|
||||
--sidebar-primary: oklch(0.21 0.006 285.885);
|
||||
--sidebar-primary-foreground: oklch(0.985 0 0);
|
||||
--sidebar-accent: oklch(0.967 0.001 286.375);
|
||||
--sidebar-accent-foreground: oklch(0.21 0.006 285.885);
|
||||
--sidebar-border: oklch(0.92 0.004 286.32);
|
||||
--sidebar-ring: oklch(0.705 0.015 286.067);
|
||||
}
|
||||
|
||||
.dark {
|
||||
--background: oklch(0.141 0.005 285.823);
|
||||
--foreground: oklch(0.985 0 0);
|
||||
--card: oklch(0.141 0.005 285.823);
|
||||
--card-foreground: oklch(0.985 0 0);
|
||||
--popover: oklch(0.141 0.005 285.823);
|
||||
--popover-foreground: oklch(0.985 0 0);
|
||||
--primary: oklch(0.985 0 0);
|
||||
--primary-foreground: oklch(0.21 0.006 285.885);
|
||||
--secondary: oklch(0.274 0.006 286.033);
|
||||
--secondary-foreground: oklch(0.985 0 0);
|
||||
--muted: oklch(0.274 0.006 286.033);
|
||||
--muted-foreground: oklch(0.705 0.015 286.067);
|
||||
--accent: oklch(0.274 0.006 286.033);
|
||||
--accent-foreground: oklch(0.985 0 0);
|
||||
--destructive: oklch(0.396 0.141 25.723);
|
||||
--destructive-foreground: oklch(0.637 0.237 25.331);
|
||||
--border: oklch(0.274 0.006 286.033);
|
||||
--input: oklch(0.274 0.006 286.033);
|
||||
--ring: oklch(0.442 0.017 285.786);
|
||||
--chart-1: oklch(0.488 0.243 264.376);
|
||||
--chart-2: oklch(0.696 0.17 162.48);
|
||||
--chart-3: oklch(0.769 0.188 70.08);
|
||||
--chart-4: oklch(0.627 0.265 303.9);
|
||||
--chart-5: oklch(0.645 0.246 16.439);
|
||||
--sidebar: oklch(0.21 0.006 285.885);
|
||||
--sidebar-foreground: oklch(0.985 0 0);
|
||||
--sidebar-primary: oklch(0.488 0.243 264.376);
|
||||
--sidebar-primary-foreground: oklch(0.985 0 0);
|
||||
--sidebar-accent: oklch(0.274 0.006 286.033);
|
||||
--sidebar-accent-foreground: oklch(0.985 0 0);
|
||||
--sidebar-border: oklch(0.274 0.006 286.033);
|
||||
--sidebar-ring: oklch(0.442 0.017 285.786);
|
||||
}
|
||||
|
||||
@theme inline {
|
||||
--color-background: var(--background);
|
||||
--color-foreground: var(--foreground);
|
||||
--color-card: var(--card);
|
||||
--color-card-foreground: var(--card-foreground);
|
||||
--color-popover: var(--popover);
|
||||
--color-popover-foreground: var(--popover-foreground);
|
||||
--color-primary: var(--primary);
|
||||
--color-primary-foreground: var(--primary-foreground);
|
||||
--color-secondary: var(--secondary);
|
||||
--color-secondary-foreground: var(--secondary-foreground);
|
||||
--color-muted: var(--muted);
|
||||
--color-muted-foreground: var(--muted-foreground);
|
||||
--color-accent: var(--accent);
|
||||
--color-accent-foreground: var(--accent-foreground);
|
||||
--color-destructive: var(--destructive);
|
||||
--color-destructive-foreground: var(--destructive-foreground);
|
||||
--color-border: var(--border);
|
||||
--color-input: var(--input);
|
||||
--color-ring: var(--ring);
|
||||
--color-chart-1: var(--chart-1);
|
||||
--color-chart-2: var(--chart-2);
|
||||
--color-chart-3: var(--chart-3);
|
||||
--color-chart-4: var(--chart-4);
|
||||
--color-chart-5: var(--chart-5);
|
||||
--radius-sm: calc(var(--radius) - 4px);
|
||||
--radius-md: calc(var(--radius) - 2px);
|
||||
--radius-lg: var(--radius);
|
||||
--radius-xl: calc(var(--radius) + 4px);
|
||||
--color-sidebar: var(--sidebar);
|
||||
--color-sidebar-foreground: var(--sidebar-foreground);
|
||||
--color-sidebar-primary: var(--sidebar-primary);
|
||||
--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
|
||||
--color-sidebar-accent: var(--sidebar-accent);
|
||||
--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
|
||||
--color-sidebar-border: var(--sidebar-border);
|
||||
--color-sidebar-ring: var(--sidebar-ring);
|
||||
}
|
||||
|
||||
@layer base {
|
||||
* {
|
||||
@apply border-border outline-ring/50;
|
||||
}
|
||||
body {
|
||||
@apply bg-background text-foreground;
|
||||
}
|
||||
}
|
||||
|
||||
.animate-memory-indicator {
|
||||
animation: memory-indicator-pop 0.6s cubic-bezier(0.4, 0, 0.2, 1) forwards;
|
||||
transform-origin: top;
|
||||
}
|
||||
168
examples/personalized-voice-agent/client/react/src/App.tsx
Normal file
168
examples/personalized-voice-agent/client/react/src/App.tsx
Normal file
@@ -0,0 +1,168 @@
|
||||
import {
|
||||
RTVIClientAudio,
|
||||
useRTVIClient,
|
||||
useRTVIClientTransportState,
|
||||
} from '@pipecat-ai/client-react';
|
||||
import { RTVIProvider } from './providers/RTVIProvider';
|
||||
import { DebugDisplay } from './components/DebugDisplay';
|
||||
import { Waveform } from './components/Waveform';
|
||||
import StaticMemoryPanel from './components/StaticMemoryPanel';
|
||||
import './App.css';
|
||||
import Navbar from './components/Navbar';
|
||||
import { ScrollArea } from './components/ui/scroll-area';
|
||||
import { InteractiveHoverButton } from "@/components/magicui/interactive-hover-button";
|
||||
import { useState, useEffect } from 'react';
|
||||
import { CaretRight, Memory } from "@phosphor-icons/react";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { Button } from "./components/ui/button";
|
||||
|
||||
function AppContent() {
|
||||
const client = useRTVIClient();
|
||||
const transportState = useRTVIClientTransportState();
|
||||
const isConnected = ['connected', 'ready'].includes(transportState);
|
||||
const isConnecting = ['connecting', 'disconnecting'].includes(transportState);
|
||||
const [speakingState, setSpeakingState] = useState<'user' | 'assistant' | 'system' | 'idle'>('idle');
|
||||
const [leftPanelCollapsed, setLeftPanelCollapsed] = useState(true);
|
||||
const [rightPanelCollapsed, setRightPanelCollapsed] = useState(false);
|
||||
const [isFirstConnectionConnected, setIsFirstConnectionConnected] = useState(false);
|
||||
|
||||
const handleReset = () => {
|
||||
setIsFirstConnectionConnected(false);
|
||||
}
|
||||
|
||||
// Simulated memory counts - replace with actual counts from your components
|
||||
const leftMemoryCount = 3;
|
||||
const rightMemoryCount = 3;
|
||||
|
||||
const handleConnect = async () => {
|
||||
if (!client) {
|
||||
console.error('RTVI client is not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (isConnected) {
|
||||
await client.disconnect();
|
||||
} else {
|
||||
await client.connect();
|
||||
}
|
||||
setIsFirstConnectionConnected(true);
|
||||
} catch (error) {
|
||||
console.error('Connection error:', error);
|
||||
}
|
||||
};
|
||||
|
||||
// Listen for message events from DebugDisplay
|
||||
useEffect(() => {
|
||||
const handleMessage = (event: CustomEvent<{ type: 'user' | 'assistant' | 'system' }>) => {
|
||||
setSpeakingState(event.detail.type);
|
||||
// Reset to idle after animation
|
||||
setTimeout(() => setSpeakingState('idle'), 2000);
|
||||
};
|
||||
|
||||
window.addEventListener('newMessage', handleMessage as EventListener);
|
||||
return () => window.removeEventListener('newMessage', handleMessage as EventListener);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<div className="mb-24">
|
||||
<Navbar onReset={handleReset} />
|
||||
</div>
|
||||
|
||||
{!isConnected && !isFirstConnectionConnected ? (
|
||||
<div className="h-[calc(100vh-12rem)] flex items-center justify-center">
|
||||
<InteractiveHoverButton
|
||||
onClick={handleConnect}
|
||||
disabled={isConnecting}
|
||||
className="px-8 py-4 text-xl animate-pulse-shadow"
|
||||
>
|
||||
{isConnecting ? 'Connecting...' : 'Connect'}
|
||||
</InteractiveHoverButton>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex px-4 relative">
|
||||
{/* Main Content */}
|
||||
<div className={cn(
|
||||
"transition-all duration-300 ease-in-out flex-1 relative",
|
||||
leftPanelCollapsed && rightPanelCollapsed ? "mx-4" : "",
|
||||
!leftPanelCollapsed && !rightPanelCollapsed ? "mx-4" : "",
|
||||
!leftPanelCollapsed && rightPanelCollapsed ? "ml-4 mr-4" : "",
|
||||
leftPanelCollapsed && !rightPanelCollapsed ? "ml-4 mr-4" : ""
|
||||
)}>
|
||||
<ScrollArea className="w-full h-[calc(100vh-10rem)] rounded-xl bg-zinc-50 dark:bg-neutral-950">
|
||||
<DebugDisplay onNewMessage={(type) => setSpeakingState(type)} />
|
||||
</ScrollArea>
|
||||
|
||||
<div className="absolute bottom-6 left-1/2 -translate-x-1/2">
|
||||
<div className="p-4 rounded-full shadow-lg border border-neutral-200 dark:border-neutral-800 bg-white dark:bg-neutral-900 transition-all duration-300 hover:shadow-xl">
|
||||
<Waveform speakingState={speakingState} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Right Panel */}
|
||||
<div className={cn(
|
||||
"transition-all duration-300 ease-in-out group",
|
||||
rightPanelCollapsed ? "w-12" : "w-[25rem]"
|
||||
)}>
|
||||
{/* Collapsed State with Memory Icons */}
|
||||
<div className={cn(
|
||||
"absolute top-0 right-0 h-full w-12 flex flex-col items-center pt-4 gap-2",
|
||||
rightPanelCollapsed ? "opacity-100" : "opacity-0 pointer-events-none"
|
||||
)}>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={() => setRightPanelCollapsed(false)}
|
||||
className="h-10 w-10 rounded-xl bg-violet-100 dark:bg-violet-900/30 hover:bg-violet-200 dark:hover:bg-violet-900/50"
|
||||
>
|
||||
<CaretRight
|
||||
className="h-5 w-5 text-violet-600 dark:text-violet-400 rotate-180"
|
||||
/>
|
||||
</Button>
|
||||
<div className="flex flex-col items-center gap-1 mt-2">
|
||||
<Memory className="h-5 w-5 text-violet-500" weight="duotone" />
|
||||
<span className="text-xs font-medium text-violet-600 dark:text-violet-400">{rightMemoryCount}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Expanded State */}
|
||||
<div className={cn(
|
||||
"transition-all duration-300 w-[25rem]",
|
||||
rightPanelCollapsed ? "opacity-0 pointer-events-none" : "opacity-100"
|
||||
)}>
|
||||
<div className="relative">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={() => setRightPanelCollapsed(true)}
|
||||
className="absolute left-2 top-2 z-10 h-10 w-10 rounded-xl bg-violet-100 dark:bg-violet-900/30 hover:bg-violet-200 dark:hover:bg-violet-900/50"
|
||||
>
|
||||
<CaretRight
|
||||
className="h-5 w-5 text-violet-600 dark:text-violet-400"
|
||||
/>
|
||||
</Button>
|
||||
<ScrollArea>
|
||||
<StaticMemoryPanel />
|
||||
</ScrollArea>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<RTVIClientAudio />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function App() {
|
||||
return (
|
||||
<RTVIProvider>
|
||||
<AppContent />
|
||||
</RTVIProvider>
|
||||
);
|
||||
}
|
||||
|
||||
export default App;
|
||||
@@ -0,0 +1,39 @@
|
||||
import {
|
||||
useRTVIClient,
|
||||
useRTVIClientTransportState,
|
||||
} from '@pipecat-ai/client-react';
|
||||
import { InteractiveHoverButton } from "@/components/magicui/interactive-hover-button";
|
||||
|
||||
|
||||
export function ConnectButton() {
|
||||
const client = useRTVIClient();
|
||||
const transportState = useRTVIClientTransportState();
|
||||
const isConnected = ['connected', 'ready'].includes(transportState);
|
||||
|
||||
const handleClick = async () => {
|
||||
if (!client) {
|
||||
console.error('RTVI client is not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (isConnected) {
|
||||
await client.disconnect();
|
||||
} else {
|
||||
await client.connect();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Connection error:', error);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<InteractiveHoverButton
|
||||
onClick={handleClick}
|
||||
disabled={
|
||||
!client || ['connecting', 'disconnecting'].includes(transportState)
|
||||
}>
|
||||
{!client || ['connecting', 'disconnecting'].includes(transportState) ? 'Connecting...' : isConnected ? 'Disconnect' : 'Connect'}
|
||||
</InteractiveHoverButton>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
import { useRef, useCallback, useState, useEffect } from 'react';
|
||||
import {
|
||||
Participant,
|
||||
RTVIEvent,
|
||||
TransportState,
|
||||
TranscriptData,
|
||||
BotLLMTextData,
|
||||
} from '@pipecat-ai/client-js';
|
||||
import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
|
||||
|
||||
interface Message {
|
||||
id: string;
|
||||
text: string;
|
||||
type: 'user' | 'assistant' | 'system';
|
||||
timestamp: Date;
|
||||
showMemoryRefresh?: boolean;
|
||||
memoryRefreshTime?: number;
|
||||
showMemoryIndicator?: boolean;
|
||||
}
|
||||
|
||||
interface DebugDisplayProps {
|
||||
onNewMessage: (type: 'user' | 'assistant' | 'system') => void;
|
||||
}
|
||||
|
||||
// Create a shared context for memory refresh
|
||||
export const memoryRefreshTrigger = {
|
||||
value: 0,
|
||||
subscribers: new Set<(value: number) => void>(),
|
||||
increment() {
|
||||
this.value++;
|
||||
this.subscribers.forEach(callback => callback(this.value));
|
||||
}
|
||||
};
|
||||
|
||||
export function DebugDisplay({ onNewMessage }: DebugDisplayProps) {
|
||||
const [messages, setMessages] = useState<Message[]>([]);
|
||||
const messagesEndRef = useRef<HTMLDivElement>(null);
|
||||
const client = useRTVIClient();
|
||||
|
||||
// Add effect to handle delayed appearance of memory refresh indicators
|
||||
useEffect(() => {
|
||||
const timeouts: NodeJS.Timeout[] = [];
|
||||
|
||||
messages.forEach(message => {
|
||||
if (message.showMemoryRefresh && !message.showMemoryIndicator) {
|
||||
const timeout = setTimeout(() => {
|
||||
setMessages(prev => prev.map(msg =>
|
||||
msg.id === message.id
|
||||
? { ...msg, showMemoryIndicator: true }
|
||||
: msg
|
||||
));
|
||||
}, 800);
|
||||
timeouts.push(timeout);
|
||||
}
|
||||
});
|
||||
|
||||
return () => {
|
||||
timeouts.forEach(timeout => clearTimeout(timeout));
|
||||
};
|
||||
}, [messages]);
|
||||
|
||||
const generateRandomRefreshTime = () => {
|
||||
return Number((Math.random() * (100 - 50) + 50).toFixed(2));
|
||||
};
|
||||
|
||||
const scrollToBottom = useCallback(() => {
|
||||
messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
|
||||
}, []);
|
||||
|
||||
const addMessage = useCallback((text: string, type: 'user' | 'assistant' | 'system') => {
|
||||
const newMessage: Message = {
|
||||
id: `${Date.now()}-${Math.random()}`,
|
||||
text,
|
||||
type,
|
||||
timestamp: new Date(),
|
||||
showMemoryRefresh: type === 'user',
|
||||
memoryRefreshTime: type === 'user' ? generateRandomRefreshTime() : undefined
|
||||
};
|
||||
|
||||
setMessages(prev => [...prev, newMessage]);
|
||||
onNewMessage(type);
|
||||
if (type === 'assistant') {
|
||||
memoryRefreshTrigger.increment();
|
||||
}
|
||||
setTimeout(scrollToBottom, 100);
|
||||
}, [scrollToBottom, onNewMessage]);
|
||||
|
||||
// Log transport state changes
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.TransportStateChanged,
|
||||
useCallback(
|
||||
(state: TransportState) => {
|
||||
addMessage(`Transport state changed: ${state}`, 'system');
|
||||
},
|
||||
[addMessage]
|
||||
)
|
||||
);
|
||||
|
||||
// Log bot connection events
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.BotConnected,
|
||||
useCallback(
|
||||
(participant?: Participant) => {
|
||||
addMessage(`Bot connected: ${JSON.stringify(participant)}`, 'system');
|
||||
},
|
||||
[addMessage]
|
||||
)
|
||||
);
|
||||
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.BotDisconnected,
|
||||
useCallback(
|
||||
(participant?: Participant) => {
|
||||
addMessage(`Bot disconnected: ${JSON.stringify(participant)}`, 'system');
|
||||
},
|
||||
[addMessage]
|
||||
)
|
||||
);
|
||||
|
||||
// Log track events
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.TrackStarted,
|
||||
useCallback(
|
||||
(track: MediaStreamTrack, participant?: Participant) => {
|
||||
addMessage(
|
||||
`Track started: ${track.kind} from ${participant?.name || 'unknown'}`,
|
||||
'system'
|
||||
);
|
||||
},
|
||||
[addMessage]
|
||||
)
|
||||
);
|
||||
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.TrackStopped,
|
||||
useCallback(
|
||||
(track: MediaStreamTrack, participant?: Participant) => {
|
||||
addMessage(
|
||||
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`,
|
||||
'system'
|
||||
);
|
||||
},
|
||||
[addMessage]
|
||||
)
|
||||
);
|
||||
|
||||
// Log bot ready state and check tracks
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.BotReady,
|
||||
useCallback(() => {
|
||||
addMessage('Bot ready', 'system');
|
||||
|
||||
if (!client) return;
|
||||
|
||||
const tracks = client.tracks();
|
||||
addMessage(
|
||||
`Available tracks: ${JSON.stringify({
|
||||
local: {
|
||||
audio: !!tracks.local.audio,
|
||||
video: !!tracks.local.video,
|
||||
},
|
||||
bot: {
|
||||
audio: !!tracks.bot?.audio,
|
||||
video: !!tracks.bot?.video,
|
||||
},
|
||||
})}`,
|
||||
'system'
|
||||
);
|
||||
}, [client, addMessage])
|
||||
);
|
||||
|
||||
// Log transcripts
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.UserTranscript,
|
||||
useCallback(
|
||||
(data: TranscriptData) => {
|
||||
// Only log final transcripts
|
||||
if (data.final) {
|
||||
addMessage(data.text, 'user');
|
||||
}
|
||||
},
|
||||
[addMessage]
|
||||
)
|
||||
);
|
||||
|
||||
useRTVIClientEvent(
|
||||
RTVIEvent.BotTranscript,
|
||||
useCallback(
|
||||
(data: BotLLMTextData) => {
|
||||
addMessage(data.text, 'assistant');
|
||||
},
|
||||
[addMessage]
|
||||
)
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="w-full h-full mx-auto p-6 bg-zinc-50">
|
||||
<div className="flex flex-col gap-4 pb-16">
|
||||
{messages.map((message) => (
|
||||
<div
|
||||
key={message.id}
|
||||
className={`
|
||||
flex animate-message-pop
|
||||
${message.type === 'system' ? 'justify-center mx-auto w-full' : 'max-w-[80%]'}
|
||||
${message.type === 'user' ? 'justify-end ml-auto' : ''}
|
||||
${message.type === 'assistant' ? 'justify-start mr-auto' : ''}
|
||||
`}
|
||||
>
|
||||
{message.type === 'system' ? (
|
||||
<div className="text-sm text-zinc-500 font-medium text-center">
|
||||
{message.text}
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex flex-col">
|
||||
{!message.showMemoryRefresh && message.showMemoryIndicator && (
|
||||
<div className="text-xs bg-blue-100 text-blue-400 px-2 py-1 rounded-md self-end animate-memory-indicator overflow-hidden">
|
||||
Memory refresh: {message.memoryRefreshTime}ms
|
||||
</div>
|
||||
)}
|
||||
<div
|
||||
className={`
|
||||
px-4 py-3 rounded-2xl shadow-sm transition-all duration-300 hover:shadow-md
|
||||
${message.type === 'user' ? 'bg-zinc-800 text-white' : ''}
|
||||
${message.type === 'assistant' ? 'bg-white text-zinc-800' : ''}
|
||||
`}
|
||||
>
|
||||
<div className="break-words">{message.text}</div>
|
||||
<div className="text-xs mt-1 opacity-70">
|
||||
{message.timestamp.toLocaleTimeString([], {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
second: '2-digit'
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
<div ref={messagesEndRef} />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import type React from "react"
|
||||
import { Card, CardContent } from "@/components/ui/card"
|
||||
import { Badge } from "@/components/ui/badge"
|
||||
import { Tag, Clock } from "lucide-react"
|
||||
import { Memory } from "@phosphor-icons/react"
|
||||
import { formatDistanceToNow } from "date-fns"
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
interface MemoryCardProps {
|
||||
memory: {
|
||||
id: string
|
||||
content: string
|
||||
createdAt: string
|
||||
tags?: string[]
|
||||
categories?: string[]
|
||||
}
|
||||
}
|
||||
|
||||
export const MemoryCard: React.FC<MemoryCardProps> = ({ memory }) => {
|
||||
return (
|
||||
<Card className="w-full overflow-hidden border border-neutral-200 dark:border-neutral-800 bg-white dark:bg-neutral-900">
|
||||
<CardContent className="p-4">
|
||||
<div className="flex items-center gap-3 mb-4">
|
||||
<div
|
||||
className={cn(
|
||||
"h-8 w-8 rounded-full flex items-center justify-center",
|
||||
"bg-violet-100 dark:bg-violet-900/30",
|
||||
)}
|
||||
>
|
||||
<Memory className="h-4 w-4 text-violet-500" weight="duotone" />
|
||||
</div>
|
||||
<div className="flex items-center gap-2 text-xs text-neutral-500 dark:text-neutral-400">
|
||||
<Clock className="h-3 w-3" />
|
||||
<span>{formatDistanceToNow(new Date(memory.createdAt), { addSuffix: true })}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="relative pl-4">
|
||||
<div className="absolute -left-0 top-0 bottom-0 w-[2px] rounded-full bg-violet-100 dark:bg-violet-900" />
|
||||
|
||||
<div className="prose prose-sm dark:prose-invert max-w-none">
|
||||
<p className="text-sm text-neutral-800 dark:text-neutral-200 leading-relaxed">{memory.content}</p>
|
||||
</div>
|
||||
|
||||
{memory.tags && memory.tags.length > 0 && (
|
||||
<div className="flex flex-wrap gap-1.5 mt-3">
|
||||
{memory.tags.map((tag, tagIndex) => (
|
||||
<Badge
|
||||
key={tagIndex}
|
||||
variant="secondary"
|
||||
className="px-1.5 py-0 h-4 text-[10px] bg-violet-50 dark:bg-violet-900/20 text-violet-600 dark:text-violet-400 hover:bg-violet-100 dark:hover:bg-violet-900/40 transition-colors"
|
||||
>
|
||||
<Tag className="h-2.5 w-2.5 mr-1" />
|
||||
{tag}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{memory.categories && memory.categories.length > 0 && (
|
||||
<div className="flex flex-wrap gap-1.5 mt-3">
|
||||
{memory.categories.map((category, categoryIndex) => (
|
||||
<Badge
|
||||
key={categoryIndex}
|
||||
variant="secondary"
|
||||
className="px-1.5 py-0 h-4 text-[10px] bg-violet-50 dark:bg-violet-900/20 text-violet-600 dark:text-violet-400 hover:bg-violet-100 dark:hover:bg-violet-900/40 transition-colors"
|
||||
>
|
||||
<Tag className="h-2.5 w-2.5 mr-1" />
|
||||
{category}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { ConnectButton } from './ConnectButton'
|
||||
import ThemeAwareLogo from './ThemeAwareLogo'
|
||||
|
||||
const Navbar = ({ onReset }: { onReset: () => void }) => {
|
||||
return (
|
||||
<nav className="fixed top-4 left-1/2 -translate-x-1/2 w-full max-w-[95%] bg-white rounded-3xl shadow-lg px-6 py-2 flex items-center justify-between z-50">
|
||||
<div onClick={onReset} className="cursor-pointer">
|
||||
<ThemeAwareLogo />
|
||||
</div>
|
||||
<button className="p-2 hover:bg-zinc-100 rounded-full transition-colors">
|
||||
<ConnectButton />
|
||||
</button>
|
||||
</nav>
|
||||
)
|
||||
}
|
||||
|
||||
export default Navbar
|
||||
@@ -0,0 +1,134 @@
|
||||
import React, { useState, useEffect, useCallback } from "react"
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
|
||||
import { ScrollArea } from "./ui/scroll-area"
|
||||
import { Memory, CaretRight } from "@phosphor-icons/react"
|
||||
import { cn } from "@/lib/utils"
|
||||
import { Button } from "./ui/button"
|
||||
import { MemoryCard } from "./MemoryCard"
|
||||
import { memoryRefreshTrigger } from "./DebugDisplay"
|
||||
|
||||
interface MemoryItem {
|
||||
id: string // Kept for React key prop
|
||||
content: string
|
||||
createdAt: string
|
||||
categories: string[]
|
||||
}
|
||||
|
||||
const fetchMemories = async () => {
|
||||
const response = await fetch("http://localhost:7860/memories")
|
||||
const data = await response.json()
|
||||
return data
|
||||
}
|
||||
|
||||
const MemorySkeleton = () => (
|
||||
<div className="animate-pulse space-y-4">
|
||||
<div className="flex items-start space-x-4">
|
||||
<div className="h-10 w-10 rounded-full bg-neutral-200 dark:bg-neutral-800" />
|
||||
<div className="space-y-3 flex-1">
|
||||
<div className="h-4 w-1/4 bg-neutral-200 dark:bg-neutral-800 rounded" />
|
||||
<div className="space-y-2">
|
||||
<div className="h-4 w-full bg-neutral-200 dark:bg-neutral-800 rounded" />
|
||||
<div className="h-4 w-5/6 bg-neutral-200 dark:bg-neutral-800 rounded" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
|
||||
export default function StaticMemoryPanel() {
|
||||
const [isCollapsed, setIsCollapsed] = useState(false)
|
||||
const [memories, setMemories] = useState<MemoryItem[]>([])
|
||||
const [isInitialLoading, setIsInitialLoading] = useState(true)
|
||||
const [localRefreshTrigger, setLocalRefreshTrigger] = useState(0)
|
||||
|
||||
const fetchEffectMemories = useCallback(async () => {
|
||||
// Only set loading if this is the initial fetch (no memories)
|
||||
if (memories.length === 0) {
|
||||
setIsInitialLoading(true)
|
||||
}
|
||||
|
||||
try {
|
||||
const memories = await fetchMemories()
|
||||
if (memories) {
|
||||
setMemories(memories as unknown as MemoryItem[])
|
||||
} else {
|
||||
setMemories([])
|
||||
console.log("No memories found")
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error fetching memories:", error)
|
||||
setMemories([])
|
||||
} finally {
|
||||
setIsInitialLoading(false)
|
||||
}
|
||||
}, [memories.length])
|
||||
|
||||
useEffect(() => {
|
||||
fetchEffectMemories()
|
||||
}, [fetchEffectMemories, localRefreshTrigger])
|
||||
|
||||
useEffect(() => {
|
||||
const handleRefresh = (value: number) => {
|
||||
setLocalRefreshTrigger(value);
|
||||
};
|
||||
|
||||
memoryRefreshTrigger.subscribers.add(handleRefresh);
|
||||
return () => {
|
||||
memoryRefreshTrigger.subscribers.delete(handleRefresh);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<Card className="w-full overflow-hidden border border-neutral-200 dark:border-neutral-800 bg-zinc-50 dark:bg-neutral-900">
|
||||
<CardHeader className="p-4 pb-2 flex flex-row items-center justify-between space-y-0 border-b border-neutral-200 dark:border-neutral-800">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className={cn("h-8 w-8 rounded-full flex items-center justify-center bg-violet-100 dark:bg-violet-900/30")}>
|
||||
<Memory className="h-4 w-4 text-violet-500" weight="duotone" />
|
||||
</div>
|
||||
<div className="flex flex-col">
|
||||
<CardTitle className="text-base font-medium text-neutral-900 dark:text-neutral-100">
|
||||
Memories ({memories.length})
|
||||
</CardTitle>
|
||||
</div>
|
||||
</div>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={() => setIsCollapsed(!isCollapsed)}
|
||||
className="h-8 w-8"
|
||||
>
|
||||
<CaretRight
|
||||
className={cn(
|
||||
"h-4 w-4 text-neutral-500 transition-transform",
|
||||
isCollapsed ? "" : "rotate-90"
|
||||
)}
|
||||
/>
|
||||
</Button>
|
||||
</CardHeader>
|
||||
<div className={cn(
|
||||
"transition-all duration-300 ease-in-out",
|
||||
isCollapsed ? "h-0" : "h-[calc(100vh-218px)]"
|
||||
)}>
|
||||
{!isCollapsed && (
|
||||
<ScrollArea className="h-full">
|
||||
<CardContent className="py-4">
|
||||
<div className="space-y-6">
|
||||
{isInitialLoading && memories.length === 0 ? (
|
||||
<>
|
||||
<MemorySkeleton />
|
||||
<MemorySkeleton />
|
||||
<MemorySkeleton />
|
||||
</>
|
||||
) : (
|
||||
memories.map((memory) => (
|
||||
<MemoryCard key={memory.id} memory={memory} />
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
</CardContent>
|
||||
</ScrollArea>
|
||||
)}
|
||||
</div>
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { useRTVIClientTransportState } from '@pipecat-ai/client-react';
|
||||
|
||||
export function StatusDisplay() {
|
||||
const transportState = useRTVIClientTransportState();
|
||||
|
||||
return (
|
||||
<div className="status">
|
||||
Status: <span>{transportState}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
|
||||
import mem0Logo from "./light.svg";
|
||||
|
||||
export default function ThemeAwareLogo({
|
||||
width = 100,
|
||||
height = 35,
|
||||
}: {
|
||||
width?: number;
|
||||
height?: number;
|
||||
}) {
|
||||
return <img src={mem0Logo} alt="Mem0.ai" width={width} height={height} />;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
interface WaveformProps {
|
||||
speakingState: 'user' | 'assistant' | 'system' | 'idle';
|
||||
}
|
||||
|
||||
export function Waveform({ speakingState }: WaveformProps) {
|
||||
const getAnimationClass = () => {
|
||||
switch (speakingState) {
|
||||
case 'user':
|
||||
return 'waveform-bar bg-zinc-800';
|
||||
case 'assistant':
|
||||
return 'waveform-bar bg-zinc-600';
|
||||
case 'system':
|
||||
return 'waveform-bar-listening bg-zinc-400';
|
||||
default:
|
||||
return 'waveform-bar-listening bg-zinc-300';
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-end gap-1 h-8">
|
||||
{[...Array(5)].map((_, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className={`w-1 rounded-full ${getAnimationClass()}`}
|
||||
style={{
|
||||
height: speakingState === 'idle' ? '40%' : `${Math.random() * 60 + 40}%`,
|
||||
animationDelay: `${i * 0.1}s`
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 13 KiB |
@@ -0,0 +1,35 @@
|
||||
import React from "react";
|
||||
import { ArrowRight } from "lucide-react";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
interface InteractiveHoverButtonProps
|
||||
extends React.ButtonHTMLAttributes<HTMLButtonElement> {}
|
||||
|
||||
export const InteractiveHoverButton = React.forwardRef<
|
||||
HTMLButtonElement,
|
||||
InteractiveHoverButtonProps
|
||||
>(({ children, className, ...props }, ref) => {
|
||||
return (
|
||||
<button
|
||||
ref={ref}
|
||||
className={cn(
|
||||
"group relative w-auto cursor-pointer overflow-hidden rounded-full border bg-background p-2 px-6 text-center font-semibold hover:text-white",
|
||||
className,
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="h-2 w-2 rounded-full bg-black transition-all duration-300 group-hover:scale-[100.8]"></div>
|
||||
<span className="inline-block transition-all duration-300 group-hover:translate-x-12 group-hover:opacity-0">
|
||||
{children}
|
||||
</span>
|
||||
</div>
|
||||
<div className="absolute top-0 z-10 flex h-full w-full translate-x-12 items-center justify-center gap-2 text-primary-foreground opacity-0 transition-all duration-300 group-hover:-translate-x-5 group-hover:opacity-100">
|
||||
<span>{children}</span>
|
||||
<ArrowRight />
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
});
|
||||
|
||||
InteractiveHoverButton.displayName = "InteractiveHoverButton";
|
||||
@@ -0,0 +1,56 @@
|
||||
import { useEffect, useRef } from "react"
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
interface BorderTrailProps extends React.HTMLAttributes<HTMLDivElement> {
|
||||
size?: number
|
||||
duration?: number
|
||||
}
|
||||
|
||||
export const BorderTrail: React.FC<BorderTrailProps> = ({ className, size = 100, duration = 5, ...props }) => {
|
||||
const borderRef = useRef<HTMLDivElement>(null)
|
||||
|
||||
useEffect(() => {
|
||||
const borderElement = borderRef.current
|
||||
if (!borderElement) return
|
||||
|
||||
const animateBorder = () => {
|
||||
const startTime = performance.now()
|
||||
let animationFrameId: number
|
||||
|
||||
const animate = (currentTime: number) => {
|
||||
const elapsed = (currentTime - startTime) / 1000
|
||||
const progress = (elapsed % duration) / duration
|
||||
const rotation = progress * 360
|
||||
|
||||
if (borderElement) {
|
||||
borderElement.style.transform = `rotate(${rotation}deg)`
|
||||
}
|
||||
|
||||
animationFrameId = requestAnimationFrame(animate)
|
||||
}
|
||||
|
||||
animationFrameId = requestAnimationFrame(animate)
|
||||
|
||||
return () => {
|
||||
cancelAnimationFrame(animationFrameId)
|
||||
}
|
||||
}
|
||||
|
||||
const cleanup = animateBorder()
|
||||
return cleanup
|
||||
}, [duration])
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={borderRef}
|
||||
className={cn("absolute inset-0 rounded-full opacity-70", className)}
|
||||
style={{
|
||||
width: `${size}%`,
|
||||
height: `${size}%`,
|
||||
left: `${(100 - size) / 2}%`,
|
||||
top: `${(100 - size) / 2}%`,
|
||||
}}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { useEffect, useRef } from "react"
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
interface TextShimmerProps extends React.HTMLAttributes<HTMLDivElement> {
|
||||
duration?: number
|
||||
children: React.ReactNode
|
||||
}
|
||||
|
||||
export const TextShimmer: React.FC<TextShimmerProps> = ({ className, duration = 3, children, ...props }) => {
|
||||
const shimmerRef = useRef<HTMLDivElement>(null)
|
||||
|
||||
useEffect(() => {
|
||||
const shimmerElement = shimmerRef.current
|
||||
if (!shimmerElement) return
|
||||
|
||||
const animateShimmer = () => {
|
||||
const startTime = performance.now()
|
||||
let animationFrameId: number
|
||||
|
||||
const animate = (currentTime: number) => {
|
||||
const elapsed = (currentTime - startTime) / 1000
|
||||
const progress = (elapsed % duration) / duration
|
||||
const position = progress * 200 - 100 // -100% to 100%
|
||||
|
||||
if (shimmerElement) {
|
||||
shimmerElement.style.backgroundPosition = `${position}% 0`
|
||||
}
|
||||
|
||||
animationFrameId = requestAnimationFrame(animate)
|
||||
}
|
||||
|
||||
animationFrameId = requestAnimationFrame(animate)
|
||||
|
||||
return () => {
|
||||
cancelAnimationFrame(animationFrameId)
|
||||
}
|
||||
}
|
||||
|
||||
const cleanup = animateShimmer()
|
||||
return cleanup
|
||||
}, [duration])
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={shimmerRef}
|
||||
className={cn(
|
||||
"inline-block bg-gradient-to-r from-transparent via-violet-400/20 to-transparent bg-[length:200%_100%]",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
{children}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import * as React from "react"
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
export interface BadgeProps extends React.HTMLAttributes<HTMLDivElement> {
|
||||
variant?: "default" | "secondary" | "destructive" | "outline"
|
||||
}
|
||||
|
||||
export function Badge({
|
||||
className,
|
||||
variant = "default",
|
||||
...props
|
||||
}: BadgeProps) {
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"inline-flex items-center rounded-full border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2",
|
||||
{
|
||||
"border-transparent bg-primary text-primary-foreground hover:bg-primary/80":
|
||||
variant === "default",
|
||||
"border-transparent bg-secondary text-secondary-foreground hover:bg-secondary/80":
|
||||
variant === "secondary",
|
||||
"border-transparent bg-destructive text-destructive-foreground hover:bg-destructive/80":
|
||||
variant === "destructive",
|
||||
"text-foreground": variant === "outline",
|
||||
},
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import * as React from "react"
|
||||
import { Slot } from "@radix-ui/react-slot"
|
||||
import { cva, type VariantProps } from "class-variance-authority"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const buttonVariants = cva(
|
||||
"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-[color,box-shadow] disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive",
|
||||
{
|
||||
variants: {
|
||||
variant: {
|
||||
default:
|
||||
"bg-primary text-primary-foreground shadow-xs hover:bg-primary/90",
|
||||
destructive:
|
||||
"bg-destructive text-white shadow-xs hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40",
|
||||
outline:
|
||||
"border border-input bg-background shadow-xs hover:bg-accent hover:text-accent-foreground",
|
||||
secondary:
|
||||
"bg-secondary text-secondary-foreground shadow-xs hover:bg-secondary/80",
|
||||
ghost: "hover:bg-accent hover:text-accent-foreground",
|
||||
link: "text-primary underline-offset-4 hover:underline",
|
||||
},
|
||||
size: {
|
||||
default: "h-9 px-4 py-2 has-[>svg]:px-3",
|
||||
sm: "h-8 rounded-md gap-1.5 px-3 has-[>svg]:px-2.5",
|
||||
lg: "h-10 rounded-md px-6 has-[>svg]:px-4",
|
||||
icon: "size-9",
|
||||
},
|
||||
},
|
||||
defaultVariants: {
|
||||
variant: "default",
|
||||
size: "default",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
function Button({
|
||||
className,
|
||||
variant,
|
||||
size,
|
||||
asChild = false,
|
||||
...props
|
||||
}: React.ComponentProps<"button"> &
|
||||
VariantProps<typeof buttonVariants> & {
|
||||
asChild?: boolean
|
||||
}) {
|
||||
const Comp = asChild ? Slot : "button"
|
||||
|
||||
return (
|
||||
<Comp
|
||||
data-slot="button"
|
||||
className={cn(buttonVariants({ variant, size, className }))}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
export { Button, buttonVariants }
|
||||
@@ -0,0 +1,54 @@
|
||||
import * as React from "react"
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const Card = React.forwardRef<
|
||||
HTMLDivElement,
|
||||
React.HTMLAttributes<HTMLDivElement>
|
||||
>(({ className, ...props }, ref) => (
|
||||
<div
|
||||
ref={ref}
|
||||
className={cn(
|
||||
"rounded-lg border border-neutral-200 bg-white text-neutral-950 shadow-sm dark:border-neutral-800 dark:bg-neutral-950 dark:text-neutral-50",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
))
|
||||
Card.displayName = "Card"
|
||||
|
||||
const CardHeader = React.forwardRef<
|
||||
HTMLDivElement,
|
||||
React.HTMLAttributes<HTMLDivElement>
|
||||
>(({ className, ...props }, ref) => (
|
||||
<div
|
||||
ref={ref}
|
||||
className={cn("flex flex-col space-y-1.5 p-6", className)}
|
||||
{...props}
|
||||
/>
|
||||
))
|
||||
CardHeader.displayName = "CardHeader"
|
||||
|
||||
const CardTitle = React.forwardRef<
|
||||
HTMLParagraphElement,
|
||||
React.HTMLAttributes<HTMLHeadingElement>
|
||||
>(({ className, ...props }, ref) => (
|
||||
<h3
|
||||
ref={ref}
|
||||
className={cn(
|
||||
"text-2xl font-semibold leading-none tracking-tight",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
))
|
||||
CardTitle.displayName = "CardTitle"
|
||||
|
||||
const CardContent = React.forwardRef<
|
||||
HTMLDivElement,
|
||||
React.HTMLAttributes<HTMLDivElement>
|
||||
>(({ className, ...props }, ref) => (
|
||||
<div ref={ref} className={cn("p-6 pt-0", className)} {...props} />
|
||||
))
|
||||
CardContent.displayName = "CardContent"
|
||||
|
||||
export { Card, CardHeader, CardTitle, CardContent }
|
||||
@@ -0,0 +1,56 @@
|
||||
import * as React from "react"
|
||||
import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
function ScrollArea({
|
||||
className,
|
||||
children,
|
||||
...props
|
||||
}: React.ComponentProps<typeof ScrollAreaPrimitive.Root>) {
|
||||
return (
|
||||
<ScrollAreaPrimitive.Root
|
||||
data-slot="scroll-area"
|
||||
className={cn("relative", className)}
|
||||
{...props}
|
||||
>
|
||||
<ScrollAreaPrimitive.Viewport
|
||||
data-slot="scroll-area-viewport"
|
||||
className="ring-ring/10 dark:ring-ring/20 dark:outline-ring/40 outline-ring/50 size-full rounded-[inherit] transition-[color,box-shadow] focus-visible:ring-4 focus-visible:outline-1"
|
||||
>
|
||||
{children}
|
||||
</ScrollAreaPrimitive.Viewport>
|
||||
<ScrollBar />
|
||||
<ScrollAreaPrimitive.Corner />
|
||||
</ScrollAreaPrimitive.Root>
|
||||
)
|
||||
}
|
||||
|
||||
function ScrollBar({
|
||||
className,
|
||||
orientation = "vertical",
|
||||
...props
|
||||
}: React.ComponentProps<typeof ScrollAreaPrimitive.ScrollAreaScrollbar>) {
|
||||
return (
|
||||
<ScrollAreaPrimitive.ScrollAreaScrollbar
|
||||
data-slot="scroll-area-scrollbar"
|
||||
orientation={orientation}
|
||||
className={cn(
|
||||
"flex touch-none p-px transition-colors select-none",
|
||||
orientation === "vertical" &&
|
||||
"h-full w-2.5 border-l border-l-transparent",
|
||||
orientation === "horizontal" &&
|
||||
"h-2.5 flex-col border-t border-t-transparent",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
<ScrollAreaPrimitive.ScrollAreaThumb
|
||||
data-slot="scroll-area-thumb"
|
||||
className="bg-border relative flex-1 rounded-full"
|
||||
/>
|
||||
</ScrollAreaPrimitive.ScrollAreaScrollbar>
|
||||
)
|
||||
}
|
||||
|
||||
export { ScrollArea, ScrollBar }
|
||||
@@ -0,0 +1,6 @@
|
||||
import { type ClassValue, clsx } from 'clsx';
|
||||
import { twMerge } from 'tailwind-merge';
|
||||
|
||||
export function cn(...inputs: ClassValue[]) {
|
||||
return twMerge(clsx(inputs));
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import React from 'react';
|
||||
import ReactDOM from 'react-dom/client';
|
||||
import App from './App';
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
</React.StrictMode>
|
||||
);
|
||||
@@ -0,0 +1,22 @@
|
||||
import { type PropsWithChildren } from 'react';
|
||||
import { RTVIClient } from '@pipecat-ai/client-js';
|
||||
import { DailyTransport } from '@pipecat-ai/daily-transport';
|
||||
import { RTVIClientProvider } from '@pipecat-ai/client-react';
|
||||
|
||||
const transport = new DailyTransport();
|
||||
|
||||
const client = new RTVIClient({
|
||||
transport,
|
||||
params: {
|
||||
baseUrl: 'http://localhost:7860',
|
||||
endpoints: {
|
||||
connect: '/connect',
|
||||
},
|
||||
},
|
||||
enableMic: true,
|
||||
enableCam: false,
|
||||
});
|
||||
|
||||
export function RTVIProvider({ children }: PropsWithChildren) {
|
||||
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/** @type {import('tailwindcss').Config} */
|
||||
export default {
|
||||
content: [
|
||||
"./index.html",
|
||||
"./src/**/*.{js,ts,jsx,tsx}",
|
||||
],
|
||||
theme: {
|
||||
extend: {
|
||||
keyframes: {
|
||||
'message-pop': {
|
||||
'0%': {
|
||||
opacity: '0',
|
||||
transform: 'scale(0.95) translateY(10px)'
|
||||
},
|
||||
'100%': {
|
||||
opacity: '1',
|
||||
transform: 'scale(1) translateY(0)'
|
||||
}
|
||||
},
|
||||
'memory-pop': {
|
||||
'0%': {
|
||||
opacity: '0',
|
||||
transform: 'translateY(10px)'
|
||||
},
|
||||
'100%': {
|
||||
opacity: '1',
|
||||
transform: 'translateY(0)'
|
||||
}
|
||||
},
|
||||
'loading-bar': {
|
||||
'0%': {
|
||||
transform: 'translateX(-100%)'
|
||||
},
|
||||
'100%': {
|
||||
transform: 'translateX(100%)'
|
||||
}
|
||||
},
|
||||
'shimmer': {
|
||||
'100%': {
|
||||
transform: 'translateX(100%)'
|
||||
}
|
||||
}
|
||||
},
|
||||
animation: {
|
||||
'message-pop': 'message-pop 0.3s ease-out',
|
||||
'memory-pop': 'memory-pop 0.3s ease-out forwards',
|
||||
'loading-bar': 'loading-bar 2s infinite',
|
||||
'shimmer': 'shimmer 1.5s infinite'
|
||||
}
|
||||
},
|
||||
},
|
||||
plugins: [],
|
||||
}
|
||||
30
examples/personalized-voice-agent/client/react/tsconfig.json
Normal file
30
examples/personalized-voice-agent/client/react/tsconfig.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"useDefineForClassFields": true,
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"skipLibCheck": true,
|
||||
|
||||
/* Bundler mode */
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"noEmit": true,
|
||||
"jsx": "react-jsx",
|
||||
|
||||
/* Linting */
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["src"],
|
||||
"references": [{ "path": "./tsconfig.node.json" }]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"skipLibCheck": true,
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["vite.config.ts"]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import path from 'path'
|
||||
import { defineConfig } from 'vite'
|
||||
import react from '@vitejs/plugin-react'
|
||||
import tailwindcss from '@tailwindcss/vite'
|
||||
|
||||
// https://vite.dev/config/
|
||||
export default defineConfig({
|
||||
plugins: [react(), tailwindcss()],
|
||||
resolve: {
|
||||
alias: {
|
||||
"@": path.resolve(__dirname, "./src"),
|
||||
},
|
||||
},
|
||||
})
|
||||
BIN
examples/personalized-voice-agent/image.png
Normal file
BIN
examples/personalized-voice-agent/image.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 360 KiB |
17
examples/personalized-voice-agent/server/Dockerfile
Normal file
17
examples/personalized-voice-agent/server/Dockerfile
Normal file
@@ -0,0 +1,17 @@
|
||||
FROM python:3.10-bullseye
|
||||
|
||||
RUN mkdir /app
|
||||
RUN mkdir /app/assets
|
||||
RUN mkdir /app/utils
|
||||
COPY requirements.txt /app/
|
||||
|
||||
WORKDIR /app
|
||||
RUN pip3 install -r requirements.txt
|
||||
|
||||
COPY *.py /app/
|
||||
COPY assets/* /app/assets/
|
||||
COPY .env /app/.env
|
||||
|
||||
EXPOSE 7860
|
||||
|
||||
CMD ["python3", "server.py"]
|
||||
57
examples/personalized-voice-agent/server/README.md
Normal file
57
examples/personalized-voice-agent/server/README.md
Normal file
@@ -0,0 +1,57 @@
|
||||
# Personalized Voice Agent Server
|
||||
|
||||
A FastAPI server that manages bot instances and provides endpoints for both Daily Prebuilt and Pipecat client connections.
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Copy `env.example` to `.env` and configure:
|
||||
|
||||
```ini
|
||||
# Required API Keys
|
||||
DAILY_API_KEY= # Your Daily API key
|
||||
MEM0_API_KEY= # Your Mem0 API key
|
||||
OPENAI_API_KEY= # Your OpenAI API key (required for OpenAI bot)
|
||||
ELEVENLABS_API_KEY= # Your ElevenLabs API key
|
||||
|
||||
# Optional Configuration
|
||||
DAILY_API_URL= # Optional: Daily API URL (defaults to https://api.daily.co/v1)
|
||||
DAILY_SAMPLE_ROOM_URL= # Optional: Fixed room URL for development
|
||||
HOST= # Optional: Host address (defaults to 0.0.0.0)
|
||||
FAST_API_PORT= # Optional: Port number (defaults to 7860)
|
||||
```
|
||||
|
||||
## Available Bots
|
||||
|
||||
The server supports two bot implementations:
|
||||
|
||||
1. **OpenAI Bot** (Default)
|
||||
|
||||
- Uses GPT-4 for conversation
|
||||
- Requires OPENAI_API_KEY
|
||||
|
||||
## Running the Server
|
||||
|
||||
Set up and activate your virtual environment:
|
||||
|
||||
```bash
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate # On Windows: venv\Scripts\activate
|
||||
```
|
||||
|
||||
Install dependencies:
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
If you want to use the local version of `pipecat` in this repo rather than the last published version, also run:
|
||||
|
||||
```bash
|
||||
pip install --editable "../../../[daily,elevenlabs,openai,silero,mem0ai]"
|
||||
```
|
||||
|
||||
Run the server:
|
||||
|
||||
```bash
|
||||
python server.py
|
||||
```
|
||||
243
examples/personalized-voice-agent/server/bot-openai.py
Normal file
243
examples/personalized-voice-agent/server/bot-openai.py
Normal file
@@ -0,0 +1,243 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
"""OpenAI Bot Implementation.
|
||||
|
||||
This module implements a chatbot using OpenAI's GPT-4 model for natural language
|
||||
processing. It includes:
|
||||
- Real-time audio/video interaction through Daily
|
||||
- Animated robot avatar
|
||||
- Text-to-speech using ElevenLabs
|
||||
- Support for both English and Spanish
|
||||
|
||||
The bot runs as part of a pipeline that processes audio/video frames and manages
|
||||
the conversation flow.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
|
||||
import aiohttp
|
||||
from dotenv import load_dotenv
|
||||
from loguru import logger
|
||||
from runner import configure
|
||||
|
||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||
from pipecat.services.mem0 import Mem0MemoryService
|
||||
from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor
|
||||
from pipecat.services.elevenlabs import ElevenLabsTTSService
|
||||
from pipecat.services.openai import OpenAILLMService
|
||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
||||
load_dotenv(override=True)
|
||||
logger.remove(0)
|
||||
logger.add(sys.stderr, level="DEBUG")
|
||||
|
||||
from pipecat.processors.aggregators.openai_llm_context import (
|
||||
OpenAILLMContext,
|
||||
)
|
||||
|
||||
try:
|
||||
from mem0 import MemoryClient
|
||||
except ModuleNotFoundError as e:
|
||||
logger.error(f"Exception: {e}")
|
||||
logger.error(
|
||||
"In order to use Mem0, you need to `pip install mem0ai`. Also, set the environment variable MEM0_API_KEY."
|
||||
)
|
||||
raise Exception(f"Missing module: {e}")
|
||||
|
||||
|
||||
async def get_initial_greeting(memory_client: MemoryClient, user_id: str, agent_id: str, run_id: str) -> str:
|
||||
"""Fetch all memories for the user and create a personalized greeting.
|
||||
|
||||
Returns:
|
||||
A personalized greeting based on user memories
|
||||
"""
|
||||
try:
|
||||
# Create filters based on available IDs
|
||||
id_pairs = [("user_id", user_id), ("agent_id", agent_id), ("run_id", run_id)]
|
||||
clauses = [{name: value} for name, value in id_pairs if value is not None]
|
||||
filters = {"AND": clauses} if clauses else {}
|
||||
|
||||
# Get all memories for this user
|
||||
memories = memory_client.get_all(filters=filters, version="v2")
|
||||
|
||||
if not memories or len(memories) == 0:
|
||||
return "Hello! It's nice to meet you. How can I help you today?"
|
||||
|
||||
# Create a personalized greeting based on memories
|
||||
greeting = "Hello! It's great to see you again. "
|
||||
|
||||
# Add some personalization based on memories (limit to 3 memories for brevity)
|
||||
if len(memories) > 0:
|
||||
greeting += "Based on our previous conversations, I remember: "
|
||||
for i, memory in enumerate(memories[:3], 1):
|
||||
memory_content = memory.get('memory', '')
|
||||
# Keep memory references brief
|
||||
if len(memory_content) > 100:
|
||||
memory_content = memory_content[:97] + "..."
|
||||
greeting += f"{memory_content} "
|
||||
|
||||
greeting += "How can I help you today?"
|
||||
|
||||
logger.debug(f"Created personalized greeting from {len(memories)} memories")
|
||||
return greeting
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving initial memories from Mem0: {e}")
|
||||
return "Hello! How can I help you today?"
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main bot execution function.
|
||||
|
||||
Sets up and runs the bot pipeline including:
|
||||
- Daily video transport
|
||||
- Speech-to-text and text-to-speech services
|
||||
- Language model integration
|
||||
- Mem0 memory service
|
||||
- RTVI event handling
|
||||
"""
|
||||
# Note: You can pass the user_id as a parameter in API call
|
||||
USER_ID = "deshraj"
|
||||
async with aiohttp.ClientSession() as session:
|
||||
(room_url, token) = await configure(session)
|
||||
|
||||
# Set up Daily transport with video/audio parameters
|
||||
transport = DailyTransport(
|
||||
room_url,
|
||||
token,
|
||||
"Chatbot",
|
||||
DailyParams(
|
||||
audio_out_enabled=True,
|
||||
camera_out_enabled=True,
|
||||
camera_out_width=1024,
|
||||
camera_out_height=576,
|
||||
vad_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(),
|
||||
transcription_enabled=True,
|
||||
#
|
||||
# Spanish
|
||||
#
|
||||
# transcription_settings=DailyTranscriptionSettings(
|
||||
# language="es",
|
||||
# tier="nova",
|
||||
# model="2-general"
|
||||
# )
|
||||
),
|
||||
)
|
||||
|
||||
# Initialize text-to-speech service
|
||||
tts = ElevenLabsTTSService(
|
||||
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
||||
#
|
||||
# English
|
||||
#
|
||||
voice_id="pNInz6obpgDQGcFmaJgB",
|
||||
#
|
||||
# Spanish
|
||||
#
|
||||
# model="eleven_multilingual_v2",
|
||||
# voice_id="gD1IexrzCvsXPHUuT0s3",
|
||||
)
|
||||
|
||||
# Initialize Mem0 memory service
|
||||
memory = Mem0MemoryService(
|
||||
api_key=os.getenv("MEM0_API_KEY"),
|
||||
user_id=USER_ID, # Unique identifier for the user
|
||||
# agent_id="life_coach_bot", # Optional identifier for the agent
|
||||
# run_id="session_1", # Optional identifier for the run
|
||||
params=Mem0MemoryService.InputParams(
|
||||
search_limit=10,
|
||||
search_threshold=0.3,
|
||||
api_version="v2",
|
||||
system_prompt="Based on previous conversations, I recall: \n\n",
|
||||
add_as_system_message=True,
|
||||
position=1
|
||||
)
|
||||
)
|
||||
|
||||
# Initialize LLM service
|
||||
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": """You are a personal assistant. You can remember things about the person you are talking to.
|
||||
Some Guidelines:
|
||||
- Make sure your responses are friendly yet short and concise.
|
||||
- If the user asks you to remember something, make sure to remember it.
|
||||
- Greet the user by their name if you know about it.
|
||||
"""
|
||||
},
|
||||
]
|
||||
|
||||
# Set up conversation context and management
|
||||
# The context_aggregator will automatically collect conversation context
|
||||
context = OpenAILLMContext(messages)
|
||||
context_aggregator = llm.create_context_aggregator(context)
|
||||
|
||||
#
|
||||
# RTVI events for Pipecat client UI
|
||||
#
|
||||
rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
|
||||
|
||||
pipeline = Pipeline(
|
||||
[
|
||||
transport.input(),
|
||||
rtvi,
|
||||
context_aggregator.user(),
|
||||
memory,
|
||||
llm,
|
||||
tts,
|
||||
transport.output(),
|
||||
context_aggregator.assistant(),
|
||||
]
|
||||
)
|
||||
|
||||
task = PipelineTask(
|
||||
pipeline,
|
||||
params=PipelineParams(
|
||||
allow_interruptions=True,
|
||||
enable_metrics=True,
|
||||
enable_usage_metrics=True,
|
||||
),
|
||||
observers=[RTVIObserver(rtvi)],
|
||||
)
|
||||
|
||||
@rtvi.event_handler("on_client_ready")
|
||||
async def on_client_ready(rtvi):
|
||||
await rtvi.set_bot_ready()
|
||||
|
||||
@transport.event_handler("on_first_participant_joined")
|
||||
async def on_first_participant_joined(transport, participant):
|
||||
await transport.capture_participant_transcription(participant["id"])
|
||||
|
||||
# Get personalized greeting based on user memories. Can pass agent_id and run_id as per requirement of the application to manage short term memory or agent specific memory.
|
||||
greeting = await get_initial_greeting(memory_client=memory.memory_client, user_id=USER_ID, agent_id=None, run_id=None)
|
||||
|
||||
# Add the greeting as an assistant message to start the conversation
|
||||
context.add_message({"role": "assistant", "content": greeting})
|
||||
|
||||
# Queue the context frame to start the conversation
|
||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||
|
||||
@transport.event_handler("on_participant_left")
|
||||
async def on_participant_left(transport, participant, reason):
|
||||
print(f"Participant left: {participant}")
|
||||
await task.cancel()
|
||||
|
||||
runner = PipelineRunner()
|
||||
|
||||
await runner.run(task)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
6
examples/personalized-voice-agent/server/env.example
Normal file
6
examples/personalized-voice-agent/server/env.example
Normal file
@@ -0,0 +1,6 @@
|
||||
DAILY_SAMPLE_ROOM_URL=https://yourdomain.daily.co/yourroom # (for joining the bot to the same room repeatedly for local dev)
|
||||
DAILY_API_KEY=7df...
|
||||
OPENAI_API_KEY=sk-PL...
|
||||
GEMINI_API_KEY=AIza...
|
||||
ELEVENLABS_API_KEY=aeb...
|
||||
MEM0_API_KEY=m0-...
|
||||
@@ -0,0 +1,5 @@
|
||||
python-dotenv
|
||||
fastapi[all]
|
||||
uvicorn
|
||||
pipecat-ai[daily,elevenlabs,openai,silero,google]
|
||||
mem0ai>=0.1.69
|
||||
56
examples/personalized-voice-agent/server/runner.py
Normal file
56
examples/personalized-voice-agent/server/runner.py
Normal file
@@ -0,0 +1,56 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import aiohttp
|
||||
|
||||
from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper
|
||||
|
||||
|
||||
async def configure(aiohttp_session: aiohttp.ClientSession):
|
||||
"""Configure the Daily room and Daily REST helper."""
|
||||
parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
|
||||
parser.add_argument(
|
||||
"-u", "--url", type=str, required=False, help="URL of the Daily room to join"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-k",
|
||||
"--apikey",
|
||||
type=str,
|
||||
required=False,
|
||||
help="Daily API Key (needed to create an owner token for the room)",
|
||||
)
|
||||
|
||||
args, unknown = parser.parse_known_args()
|
||||
|
||||
url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL")
|
||||
key = args.apikey or os.getenv("DAILY_API_KEY")
|
||||
|
||||
if not url:
|
||||
raise Exception(
|
||||
"No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL."
|
||||
)
|
||||
|
||||
if not key:
|
||||
raise Exception(
|
||||
"No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers."
|
||||
)
|
||||
|
||||
daily_rest_helper = DailyRESTHelper(
|
||||
daily_api_key=key,
|
||||
daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
|
||||
aiohttp_session=aiohttp_session,
|
||||
)
|
||||
|
||||
# Create a meeting token for the given room with an expiration 1 hour in
|
||||
# the future.
|
||||
expiry_time: float = 60 * 60
|
||||
|
||||
token = await daily_rest_helper.get_token(url, expiry_time)
|
||||
|
||||
return (url, token)
|
||||
270
examples/personalized-voice-agent/server/server.py
Normal file
270
examples/personalized-voice-agent/server/server.py
Normal file
@@ -0,0 +1,270 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
"""RTVI Bot Server Implementation.
|
||||
|
||||
This FastAPI server manages RTVI bot instances and provides endpoints for both
|
||||
direct browser access and RTVI client connections. It handles:
|
||||
- Creating Daily rooms
|
||||
- Managing bot processes
|
||||
- Providing connection credentials
|
||||
- Monitoring bot status
|
||||
|
||||
Requirements:
|
||||
- Daily API key (set in .env file)
|
||||
- Python 3.10+
|
||||
- FastAPI
|
||||
- Running bot implementation
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any, Dict
|
||||
|
||||
import aiohttp
|
||||
from dotenv import load_dotenv
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse, RedirectResponse
|
||||
from mem0 import MemoryClient
|
||||
from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper, DailyRoomParams
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv(override=True)
|
||||
|
||||
# Maximum number of bot instances allowed per room
|
||||
MAX_BOTS_PER_ROOM = 1
|
||||
|
||||
# Dictionary to track bot processes: {pid: (process, room_url)}
|
||||
bot_procs = {}
|
||||
|
||||
# Store Daily API helpers
|
||||
daily_helpers = {}
|
||||
|
||||
|
||||
def cleanup():
|
||||
"""Cleanup function to terminate all bot processes.
|
||||
|
||||
Called during server shutdown.
|
||||
"""
|
||||
for entry in bot_procs.values():
|
||||
proc = entry[0]
|
||||
proc.terminate()
|
||||
proc.wait()
|
||||
|
||||
|
||||
def get_bot_file():
|
||||
bot_implementation = os.getenv("BOT_IMPLEMENTATION", "openai").lower().strip()
|
||||
# If blank or None, default to openai
|
||||
if not bot_implementation:
|
||||
bot_implementation = "openai"
|
||||
if bot_implementation not in ["openai", "gemini"]:
|
||||
raise ValueError(
|
||||
f"Invalid BOT_IMPLEMENTATION: {bot_implementation}. Must be 'openai' or 'gemini'"
|
||||
)
|
||||
return f"bot-{bot_implementation}"
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""FastAPI lifespan manager that handles startup and shutdown tasks.
|
||||
|
||||
- Creates aiohttp session
|
||||
- Initializes Daily API helper
|
||||
- Cleans up resources on shutdown
|
||||
"""
|
||||
aiohttp_session = aiohttp.ClientSession()
|
||||
daily_helpers["rest"] = DailyRESTHelper(
|
||||
daily_api_key=os.getenv("DAILY_API_KEY", ""),
|
||||
daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
|
||||
aiohttp_session=aiohttp_session,
|
||||
)
|
||||
yield
|
||||
await aiohttp_session.close()
|
||||
cleanup()
|
||||
|
||||
|
||||
# Initialize FastAPI app with lifespan manager
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
|
||||
# Configure CORS to allow requests from any origin
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
async def create_room_and_token() -> tuple[str, str]:
|
||||
"""Helper function to create a Daily room and generate an access token.
|
||||
|
||||
Returns:
|
||||
tuple[str, str]: A tuple containing (room_url, token)
|
||||
|
||||
Raises:
|
||||
HTTPException: If room creation or token generation fails
|
||||
"""
|
||||
room = await daily_helpers["rest"].create_room(DailyRoomParams())
|
||||
if not room.url:
|
||||
raise HTTPException(status_code=500, detail="Failed to create room")
|
||||
|
||||
token = await daily_helpers["rest"].get_token(room.url)
|
||||
if not token:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to get token for room: {room.url}")
|
||||
|
||||
return room.url, token
|
||||
|
||||
|
||||
@app.get("/")
|
||||
async def start_agent(request: Request):
|
||||
"""Endpoint for direct browser access to the bot.
|
||||
|
||||
Creates a room, starts a bot instance, and redirects to the Daily room URL.
|
||||
|
||||
Returns:
|
||||
RedirectResponse: Redirects to the Daily room URL
|
||||
|
||||
Raises:
|
||||
HTTPException: If room creation, token generation, or bot startup fails
|
||||
"""
|
||||
print("Creating room")
|
||||
room_url, token = await create_room_and_token()
|
||||
print(f"Room URL: {room_url}")
|
||||
|
||||
# Check if there is already an existing process running in this room
|
||||
num_bots_in_room = sum(
|
||||
1 for proc in bot_procs.values() if proc[1] == room_url and proc[0].poll() is None
|
||||
)
|
||||
if num_bots_in_room >= MAX_BOTS_PER_ROOM:
|
||||
raise HTTPException(status_code=500, detail=f"Max bot limit reached for room: {room_url}")
|
||||
|
||||
# Spawn a new bot process
|
||||
try:
|
||||
bot_file = get_bot_file()
|
||||
proc = subprocess.Popen(
|
||||
[f"python3 -m {bot_file} -u {room_url} -t {token}"],
|
||||
shell=True,
|
||||
bufsize=1,
|
||||
cwd=os.path.dirname(os.path.abspath(__file__)),
|
||||
)
|
||||
bot_procs[proc.pid] = (proc, room_url)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}")
|
||||
|
||||
return RedirectResponse(room_url)
|
||||
|
||||
|
||||
@app.get("/memories")
|
||||
async def get_memories(request: Request):
|
||||
"""Endpoint for getting memories from the bot.
|
||||
|
||||
Returns:
|
||||
Dict[Any, Any]: Memories from the bot
|
||||
"""
|
||||
memory_client = MemoryClient(api_key=os.getenv("MEM0_API_KEY"))
|
||||
filters = {
|
||||
"AND": [
|
||||
{"user_id": "deshraj"},
|
||||
]
|
||||
}
|
||||
memories = memory_client.get_all(filters=filters, version="v2")
|
||||
|
||||
# Format memories for emission
|
||||
formatted_memories = [
|
||||
{
|
||||
"id": str(memory.get("id", "")),
|
||||
"content": memory.get("memory", ""),
|
||||
"createdAt": memory.get("created_at", ""),
|
||||
"categories": memory.get("categories", [])
|
||||
} for memory in memories
|
||||
]
|
||||
|
||||
return formatted_memories
|
||||
|
||||
|
||||
@app.post("/connect")
|
||||
async def rtvi_connect(request: Request) -> Dict[Any, Any]:
|
||||
"""RTVI connect endpoint that creates a room and returns connection credentials.
|
||||
|
||||
This endpoint is called by RTVI clients to establish a connection.
|
||||
|
||||
Returns:
|
||||
Dict[Any, Any]: Authentication bundle containing room_url and token
|
||||
|
||||
Raises:
|
||||
HTTPException: If room creation, token generation, or bot startup fails
|
||||
"""
|
||||
print("Creating room for RTVI connection")
|
||||
room_url, token = await create_room_and_token()
|
||||
print(f"Room URL: {room_url}")
|
||||
|
||||
# Start the bot process
|
||||
try:
|
||||
bot_file = get_bot_file()
|
||||
proc = subprocess.Popen(
|
||||
[f"python3 -m {bot_file} -u {room_url} -t {token}"],
|
||||
shell=True,
|
||||
bufsize=1,
|
||||
cwd=os.path.dirname(os.path.abspath(__file__)),
|
||||
)
|
||||
bot_procs[proc.pid] = (proc, room_url)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}")
|
||||
|
||||
# Return the authentication bundle in format expected by DailyTransport
|
||||
return {"room_url": room_url, "token": token}
|
||||
|
||||
|
||||
@app.get("/status/{pid}")
|
||||
def get_status(pid: int):
|
||||
"""Get the status of a specific bot process.
|
||||
|
||||
Args:
|
||||
pid (int): Process ID of the bot
|
||||
|
||||
Returns:
|
||||
JSONResponse: Status information for the bot
|
||||
|
||||
Raises:
|
||||
HTTPException: If the specified bot process is not found
|
||||
"""
|
||||
# Look up the subprocess
|
||||
proc = bot_procs.get(pid)
|
||||
|
||||
# If the subprocess doesn't exist, return an error
|
||||
if not proc:
|
||||
raise HTTPException(status_code=404, detail=f"Bot with process id: {pid} not found")
|
||||
|
||||
# Check the status of the subprocess
|
||||
status = "running" if proc[0].poll() is None else "finished"
|
||||
return JSONResponse({"bot_id": pid, "status": status})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
|
||||
# Parse command line arguments for server configuration
|
||||
default_host = os.getenv("HOST", "0.0.0.0")
|
||||
default_port = int(os.getenv("FAST_API_PORT", "7860"))
|
||||
|
||||
parser = argparse.ArgumentParser(description="Daily Storyteller FastAPI server")
|
||||
parser.add_argument("--host", type=str, default=default_host, help="Host address")
|
||||
parser.add_argument("--port", type=int, default=default_port, help="Port number")
|
||||
parser.add_argument("--reload", action="store_true", help="Reload code on change")
|
||||
|
||||
config = parser.parse_args()
|
||||
|
||||
# Start the FastAPI server
|
||||
uvicorn.run(
|
||||
"server:app",
|
||||
host=config.host,
|
||||
port=config.port,
|
||||
reload=config.reload,
|
||||
)
|
||||
@@ -84,6 +84,7 @@ together = []
|
||||
ultravox = [ "transformers~=4.48.0", "vllm~=0.7.3" ]
|
||||
websocket = [ "websockets~=13.1", "fastapi~=0.115.6" ]
|
||||
whisper = [ "faster-whisper~=1.1.1" ]
|
||||
mem0 = [ "mem0ai~=0.1.76" ]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
# All the following settings are optional:
|
||||
|
||||
199
src/pipecat/services/mem0.py
Normal file
199
src/pipecat/services/mem0.py
Normal file
@@ -0,0 +1,199 @@
|
||||
from typing import Any, Dict, List
|
||||
from loguru import logger
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
ErrorFrame,
|
||||
Frame,
|
||||
LLMMessagesFrame,
|
||||
)
|
||||
from pipecat.processors.aggregators.openai_llm_context import (
|
||||
OpenAILLMContext,
|
||||
OpenAILLMContextFrame,
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
|
||||
try:
|
||||
from mem0 import MemoryClient
|
||||
except ModuleNotFoundError as e:
|
||||
logger.error(f"Exception: {e}")
|
||||
logger.error(
|
||||
"In order to use Mem0, you need to `pip install mem0ai`. Also, set the environment variable MEM0_API_KEY."
|
||||
)
|
||||
raise Exception(f"Missing module: {e}")
|
||||
|
||||
|
||||
class Mem0MemoryService(FrameProcessor):
|
||||
"""A standalone memory service that integrates with Mem0.
|
||||
|
||||
This service intercepts message frames in the pipeline, stores them in Mem0,
|
||||
and enhances context with relevant memories before passing them downstream.
|
||||
|
||||
Args:
|
||||
api_key (str): The API key for accessing Mem0's API
|
||||
user_id (str): The user ID to associate with memories in Mem0
|
||||
params (InputParams, optional): Configuration parameters for memory retrieval
|
||||
"""
|
||||
|
||||
class InputParams(BaseModel):
|
||||
search_limit: int = Field(default=10, ge=1)
|
||||
search_threshold: float = Field(default=0.1, ge=0.0, le=1.0)
|
||||
api_version: str = Field(default="v2")
|
||||
system_prompt: str = Field(
|
||||
default="Based on previous conversations, I recall: \n\n"
|
||||
)
|
||||
add_as_system_message: bool = Field(default=True)
|
||||
position: int = Field(default=1)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_key: str,
|
||||
user_id: str = None,
|
||||
agent_id: str = None,
|
||||
run_id: str = None,
|
||||
params: InputParams = InputParams(),
|
||||
):
|
||||
# Important: Call the parent class __init__ first
|
||||
super().__init__()
|
||||
|
||||
self.memory_client = MemoryClient(api_key=api_key)
|
||||
# At least one of user_id, agent_id, or run_id must be provided
|
||||
if not any([user_id, agent_id, run_id]):
|
||||
raise ValueError("At least one of user_id, agent_id, or run_id must be provided")
|
||||
|
||||
self.user_id = user_id
|
||||
self.agent_id = agent_id
|
||||
self.run_id = run_id
|
||||
self.search_limit = params.search_limit
|
||||
self.search_threshold = params.search_threshold
|
||||
self.api_version = params.api_version
|
||||
self.system_prompt = params.system_prompt
|
||||
self.add_as_system_message = params.add_as_system_message
|
||||
self.position = params.position
|
||||
self.last_query = None
|
||||
logger.info(f"Initialized Mem0MemoryService with {user_id=}, {agent_id=}, {run_id=}")
|
||||
|
||||
def _store_messages(self, messages: List[Dict[str, Any]]):
|
||||
"""Store messages in Mem0.
|
||||
|
||||
Args:
|
||||
messages: List of message dictionaries to store
|
||||
"""
|
||||
try:
|
||||
logger.debug(f"Storing {len(messages)} messages in Mem0")
|
||||
params = {"messages": messages, "metadata": {"platform": "pipecat"}}
|
||||
for id in ["user_id", "agent_id", "run_id"]:
|
||||
if getattr(self, id):
|
||||
params[id] = getattr(self, id)
|
||||
# Note: You can run this in background to avoid blocking the conversation
|
||||
self.memory_client.add(**params)
|
||||
except Exception as e:
|
||||
logger.error(f"Error storing messages in Mem0: {e}")
|
||||
|
||||
def _retrieve_memories(self, query: str) -> List[Dict[str, Any]]:
|
||||
"""Retrieve relevant memories from Mem0.
|
||||
|
||||
Args:
|
||||
query: The query to search for relevant memories
|
||||
|
||||
Returns:
|
||||
List of relevant memory dictionaries
|
||||
"""
|
||||
try:
|
||||
logger.debug(f"Retrieving memories for query: {query}")
|
||||
id_pairs = [("user_id", self.user_id), ("agent_id", self.agent_id), ("run_id", self.run_id)]
|
||||
clauses = [{name: value} for name, value in id_pairs if value is not None]
|
||||
filters = {"AND": clauses} if clauses else {}
|
||||
results = self.memory_client.search(
|
||||
query=query,
|
||||
filters=filters,
|
||||
version=self.api_version,
|
||||
top_k=self.search_limit,
|
||||
threshold=self.search_threshold,
|
||||
)
|
||||
|
||||
logger.debug(f"Retrieved {len(results)} memories from Mem0")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrieving memories from Mem0: {e}")
|
||||
return []
|
||||
|
||||
def _enhance_context_with_memories(self, context: OpenAILLMContext, query: str):
|
||||
"""Enhance the LLM context with relevant memories.
|
||||
|
||||
Args:
|
||||
context: The OpenAILLMContext to enhance
|
||||
query: The query to search for relevant memories
|
||||
"""
|
||||
# Skip if this is the same query we just processed
|
||||
if self.last_query == query:
|
||||
return
|
||||
|
||||
self.last_query = query
|
||||
|
||||
memories = self._retrieve_memories(query)
|
||||
if not memories:
|
||||
return
|
||||
|
||||
# Format memories as a message
|
||||
memory_text = self.system_prompt
|
||||
for i, memory in enumerate(memories, 1):
|
||||
memory_text += f"{i}. {memory.get('memory', '')}\n\n"
|
||||
|
||||
# Add memories as a system message or user message based on configuration
|
||||
if self.add_as_system_message:
|
||||
context.add_message({"role": "system", "content": memory_text})
|
||||
else:
|
||||
# Add as a user message that provides context
|
||||
context.add_message({"role": "user", "content": memory_text})
|
||||
logger.debug(f"Enhanced context with {len(memories)} memories")
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||
"""Process incoming frames, intercept context frames for memory integration.
|
||||
|
||||
Args:
|
||||
frame: The incoming frame to process
|
||||
direction: The direction of frame flow in the pipeline
|
||||
"""
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
context = None
|
||||
messages = None
|
||||
|
||||
if isinstance(frame, OpenAILLMContextFrame):
|
||||
context = frame.context
|
||||
elif isinstance(frame, LLMMessagesFrame):
|
||||
messages = frame.messages
|
||||
context = OpenAILLMContext.from_messages(messages)
|
||||
|
||||
if context:
|
||||
try:
|
||||
# Get the latest user message to use as a query for memory retrieval
|
||||
context_messages = context.get_messages()
|
||||
latest_user_message = None
|
||||
|
||||
for message in reversed(context_messages):
|
||||
if message.get("role") == "user" and isinstance(message.get("content"), str):
|
||||
latest_user_message = message.get("content")
|
||||
break
|
||||
|
||||
if latest_user_message:
|
||||
# Enhance context with memories before passing it downstream
|
||||
self._enhance_context_with_memories(context, latest_user_message)
|
||||
# Store the conversation in Mem0. Only call this when user message is detected
|
||||
self._store_messages(context_messages)
|
||||
|
||||
# If we received an LLMMessagesFrame, create a new one with the enhanced messages
|
||||
if messages is not None:
|
||||
await self.push_frame(LLMMessagesFrame(context.get_messages()))
|
||||
else:
|
||||
# Otherwise, pass the enhanced context frame downstream
|
||||
await self.push_frame(frame)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing with Mem0: {str(e)}")
|
||||
await self.push_frame(ErrorFrame(f"Error processing with Mem0: {str(e)}"))
|
||||
await self.push_frame(frame) # Still pass the original frame through
|
||||
else:
|
||||
# For non-context frames, just pass them through
|
||||
await self.push_frame(frame, direction)
|
||||
Reference in New Issue
Block a user