Add mem0 as a service integration

This commit is contained in:
Deshraj Yadav
2025-03-18 01:02:49 -07:00
parent f3b50bc3c4
commit 7ad36eeaf4
44 changed files with 7041 additions and 0 deletions

View File

@@ -0,0 +1,51 @@
# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
build/
dist/
*.egg-info/
.installed.cfg
*.egg
.pytest_cache/
.coverage
.coverage.*
.env
.venv
env/
venv/
ENV/
.mypy_cache/
.dmypy.json
dmypy.json
# JavaScript/Node.js
node_modules/
dist/
dist-ssr/
*.local
.env.local
.env.development.local
.env.test.local
.env.production.local
# Logs
logs/
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
# Editor/IDE
.vscode/*
!.vscode/extensions.json
.idea/
*.swp
*.swo
.DS_Store
# Project specific
runpod.toml

View File

@@ -0,0 +1,80 @@
# Personalized Voice Agent
This repository demonstrates a personalized voice agent with real-time audio/video interaction, implemented using different client and server options. The bot server supports multiple AI backends, and you can connect to it using five different client approaches.
Here is a demo video:
[![Watch the video](https://img.youtube.com/vi/FR0yCDw29SI/0.jpg)](https://www.youtube.com/watch?v=FR0yCDw29SI)
## Bot Options
- **OpenAI Bot** (Default)
- Uses gpt-4o for conversation
- Requires OpenAI API key
## Client Option
- **React**
- Basic implementation using [Pipecat React SDK](https://docs.pipecat.ai/client/react/introduction)
- Demonstrates the basic client principles with Pipecat React
## Quick Start
### First, start the bot server:
1. Navigate to the server directory:
```bash
cd server
```
2. Create and activate a virtual environment:
```bash
python3 -m venv venv
source venv/bin/activate # On Windows: venv\Scripts\activate
```
3. Install requirements:
```bash
pip install -r requirements.txt
```
4. Copy env.example to .env and configure:
- Add your API keys
5. Start the server:
```bash
python server.py
```
6. Next, connect using [react client](client/react/README.md)
## Important Note
The bot server must be running for any of the client implementations to work. Start the server first before trying any of the client apps.
## Requirements
- Python 3.10+
- Node.js 16+ (for JavaScript and React implementations)
- Daily API key
- OpenAI API key (for OpenAI bot)
- ElevenLabs API key
- Mem0 API Key
- Modern web browser with WebRTC support
## Project Structure
```
personalized-voice-agent/
├── server/ # Bot server implementation
│ ├── bot-mem0.py # Mem0 bot implementation
│ ├── runner.py # Server runner utilities
│ ├── server.py # FastAPI server
│ └── requirements.txt
└── client/ # Client implementations
├── android/ # Daily Android connection
├── ios/ # Daily iOS connection
├── javascript/ # Daily JavaScript connection
├── prebuilt/ # Pipecat Prebuilt client
└── react/ # Pipecat React client
```

View File

@@ -0,0 +1,26 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
!**/lib
node_modules
dist
dist-ssr
*.local
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?

View File

@@ -0,0 +1,27 @@
# React Implementation
Basic implementation using the [Pipecat React SDK](https://docs.pipecat.ai/client/react/introduction).
## Setup
1. Run the bot server; see [README](../../README).
2. Navigate to the `client/react` directory:
```bash
cd client/react
```
3. Install dependencies:
```bash
npm install
```
4. Run the client app:
```
npm run dev
```
5. Visit http://localhost:5173 in your browser.

View File

@@ -0,0 +1,21 @@
{
"$schema": "https://ui.shadcn.com/schema.json",
"style": "new-york",
"rsc": false,
"tsx": true,
"tailwind": {
"config": "",
"css": "src/App.css",
"baseColor": "zinc",
"cssVariables": true,
"prefix": ""
},
"aliases": {
"components": "@/components",
"utils": "@/lib/utils",
"ui": "@/components/ui",
"lib": "@/lib",
"hooks": "@/hooks"
},
"iconLibrary": "lucide"
}

View File

@@ -0,0 +1,28 @@
import js from '@eslint/js'
import globals from 'globals'
import reactHooks from 'eslint-plugin-react-hooks'
import reactRefresh from 'eslint-plugin-react-refresh'
import tseslint from 'typescript-eslint'
export default tseslint.config(
{ ignores: ['dist'] },
{
extends: [js.configs.recommended, ...tseslint.configs.recommended],
files: ['**/*.{ts,tsx}'],
languageOptions: {
ecmaVersion: 2020,
globals: globals.browser,
},
plugins: {
'react-hooks': reactHooks,
'react-refresh': reactRefresh,
},
rules: {
...reactHooks.configs.recommended.rules,
'react-refresh/only-export-components': [
'warn',
{ allowConstantExport: true },
],
},
},
)

View File

@@ -0,0 +1,15 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Mem0 - Pipecat React Client</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,48 @@
{
"name": "react",
"private": true,
"version": "0.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "tsc && vite build",
"lint": "eslint .",
"preview": "vite preview"
},
"dependencies": {
"@phosphor-icons/react": "^2.1.7",
"@pipecat-ai/client-js": "^0.3.2",
"@pipecat-ai/client-react": "^0.3.2",
"@pipecat-ai/daily-transport": "^0.3.4",
"@radix-ui/react-scroll-area": "^1.2.3",
"@radix-ui/react-slot": "^1.1.2",
"@tailwindcss/vite": "^4.0.12",
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"date-fns": "^4.1.0",
"framer-motion": "^12.4.10",
"lucide-react": "^0.479.0",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"tailwind-merge": "^3.0.2",
"tailwindcss-animate": "^1.0.7"
},
"devDependencies": {
"@eslint/js": "^9.15.0",
"@types/node": "^22.13.9",
"@types/react": "^18.3.12",
"@types/react-dom": "^18.3.1",
"@vitejs/plugin-react": "^4.3.4",
"autoprefixer": "^10.4.20",
"eslint": "^9.15.0",
"eslint-plugin-react-hooks": "^5.0.0",
"eslint-plugin-react-refresh": "^0.4.14",
"globals": "^15.12.0",
"postcss": "^8.5.3",
"tailwindcss": "^4.0.12",
"typescript": "~5.6.2",
"typescript-eslint": "^8.15.0",
"vite": "^6.0.9"
},
"packageManager": "pnpm@10.5.2+sha512.da9dc28cd3ff40d0592188235ab25d3202add8a207afbedc682220e4a0029ffbff4562102b9e6e46b4e3f9e8bd53e6d05de48544b0c57d4b0179e22c76d1199b"
}

View File

@@ -0,0 +1,267 @@
@import "tailwindcss";
@plugin "tailwindcss-animate";
@custom-variant dark (&:is(.dark *));
body {
margin: 0;
padding: 20px;
font-family: Arial, sans-serif;
background-color: #f0f0f0;
}
@keyframes message-pop {
0% {
opacity: 0;
transform: scale(0.8);
}
50% {
transform: scale(1.05);
}
100% {
opacity: 1;
transform: scale(1);
}
}
@keyframes pulse-shadow {
0% {
box-shadow: 0 0 0 0 rgba(161, 161, 170, 0.7);
}
70% {
box-shadow: 0 0 0 20px rgba(161, 161, 170, 0);
}
100% {
box-shadow: 0 0 0 0 rgba(161, 161, 170, 0);
}
}
@keyframes wave {
0% { transform: scaleY(0.2); }
50% { transform: scaleY(1); }
100% { transform: scaleY(0.2); }
}
@keyframes wave-listening {
0% { transform: scaleY(0.3); }
50% { transform: scaleY(0.5); }
100% { transform: scaleY(0.3); }
}
@keyframes wave-user {
0% { transform: scaleY(0.4); }
50% { transform: scaleY(1.2); }
100% { transform: scaleY(0.4); }
}
@keyframes wave-assistant {
0% { transform: scaleY(0.3); }
50% { transform: scaleY(0.9); }
100% { transform: scaleY(0.3); }
}
@keyframes memory-indicator-pop {
0% {
opacity: 0;
transform: translateY(-10px);
height: 0;
margin-bottom: 0;
}
30% {
height: 24px;
margin-bottom: 0.25rem;
}
60% {
opacity: 0.7;
}
100% {
opacity: 1;
transform: translateY(0);
height: 24px;
margin-bottom: 0.25rem;
}
}
.animate-message-pop {
animation: message-pop 0.3s ease-out forwards;
}
.animate-pulse-shadow {
animation: pulse-shadow 2s infinite;
}
.waveform-bar {
animation: wave-user 0.8s ease-in-out infinite;
transform-origin: bottom;
}
.waveform-bar-listening {
animation: wave-listening 1.5s ease-in-out infinite;
transform-origin: bottom;
}
.waveform-bar.bg-zinc-600 {
animation: wave-assistant 1s ease-in-out infinite;
}
.waveform-bar.bg-zinc-400 {
animation: wave-listening 1.5s ease-in-out infinite;
}
.waveform-bar:nth-child(2) { animation-delay: 0.1s; }
.waveform-bar:nth-child(3) { animation-delay: 0.2s; }
.waveform-bar:nth-child(4) { animation-delay: 0.3s; }
.waveform-bar:nth-child(5) { animation-delay: 0.4s; }
.slide-up-enter {
opacity: 0;
transform: translateY(20px);
}
.slide-up-enter-active {
opacity: 1;
transform: translateY(0);
transition: opacity 300ms, transform 300ms;
}
.slide-up-exit {
opacity: 1;
transform: translateY(0);
}
.slide-up-exit-active {
opacity: 0;
transform: translateY(-20px);
transition: opacity 300ms, transform 300ms;
}
/* Smooth scrolling for the entire page */
html {
scroll-behavior: smooth;
}
:root {
--background: oklch(1 0 0);
--foreground: oklch(0.141 0.005 285.823);
--card: oklch(1 0 0);
--card-foreground: oklch(0.141 0.005 285.823);
--popover: oklch(1 0 0);
--popover-foreground: oklch(0.141 0.005 285.823);
--primary: oklch(0.21 0.006 285.885);
--primary-foreground: oklch(0.985 0 0);
--secondary: oklch(0.967 0.001 286.375);
--secondary-foreground: oklch(0.21 0.006 285.885);
--muted: oklch(0.967 0.001 286.375);
--muted-foreground: oklch(0.552 0.016 285.938);
--accent: oklch(0.967 0.001 286.375);
--accent-foreground: oklch(0.21 0.006 285.885);
--destructive: oklch(0.577 0.245 27.325);
--destructive-foreground: oklch(0.577 0.245 27.325);
--border: oklch(0.92 0.004 286.32);
--input: oklch(0.92 0.004 286.32);
--ring: oklch(0.705 0.015 286.067);
--chart-1: oklch(0.646 0.222 41.116);
--chart-2: oklch(0.6 0.118 184.704);
--chart-3: oklch(0.398 0.07 227.392);
--chart-4: oklch(0.828 0.189 84.429);
--chart-5: oklch(0.769 0.188 70.08);
--radius: 0.625rem;
--sidebar: oklch(0.985 0 0);
--sidebar-foreground: oklch(0.141 0.005 285.823);
--sidebar-primary: oklch(0.21 0.006 285.885);
--sidebar-primary-foreground: oklch(0.985 0 0);
--sidebar-accent: oklch(0.967 0.001 286.375);
--sidebar-accent-foreground: oklch(0.21 0.006 285.885);
--sidebar-border: oklch(0.92 0.004 286.32);
--sidebar-ring: oklch(0.705 0.015 286.067);
}
.dark {
--background: oklch(0.141 0.005 285.823);
--foreground: oklch(0.985 0 0);
--card: oklch(0.141 0.005 285.823);
--card-foreground: oklch(0.985 0 0);
--popover: oklch(0.141 0.005 285.823);
--popover-foreground: oklch(0.985 0 0);
--primary: oklch(0.985 0 0);
--primary-foreground: oklch(0.21 0.006 285.885);
--secondary: oklch(0.274 0.006 286.033);
--secondary-foreground: oklch(0.985 0 0);
--muted: oklch(0.274 0.006 286.033);
--muted-foreground: oklch(0.705 0.015 286.067);
--accent: oklch(0.274 0.006 286.033);
--accent-foreground: oklch(0.985 0 0);
--destructive: oklch(0.396 0.141 25.723);
--destructive-foreground: oklch(0.637 0.237 25.331);
--border: oklch(0.274 0.006 286.033);
--input: oklch(0.274 0.006 286.033);
--ring: oklch(0.442 0.017 285.786);
--chart-1: oklch(0.488 0.243 264.376);
--chart-2: oklch(0.696 0.17 162.48);
--chart-3: oklch(0.769 0.188 70.08);
--chart-4: oklch(0.627 0.265 303.9);
--chart-5: oklch(0.645 0.246 16.439);
--sidebar: oklch(0.21 0.006 285.885);
--sidebar-foreground: oklch(0.985 0 0);
--sidebar-primary: oklch(0.488 0.243 264.376);
--sidebar-primary-foreground: oklch(0.985 0 0);
--sidebar-accent: oklch(0.274 0.006 286.033);
--sidebar-accent-foreground: oklch(0.985 0 0);
--sidebar-border: oklch(0.274 0.006 286.033);
--sidebar-ring: oklch(0.442 0.017 285.786);
}
@theme inline {
--color-background: var(--background);
--color-foreground: var(--foreground);
--color-card: var(--card);
--color-card-foreground: var(--card-foreground);
--color-popover: var(--popover);
--color-popover-foreground: var(--popover-foreground);
--color-primary: var(--primary);
--color-primary-foreground: var(--primary-foreground);
--color-secondary: var(--secondary);
--color-secondary-foreground: var(--secondary-foreground);
--color-muted: var(--muted);
--color-muted-foreground: var(--muted-foreground);
--color-accent: var(--accent);
--color-accent-foreground: var(--accent-foreground);
--color-destructive: var(--destructive);
--color-destructive-foreground: var(--destructive-foreground);
--color-border: var(--border);
--color-input: var(--input);
--color-ring: var(--ring);
--color-chart-1: var(--chart-1);
--color-chart-2: var(--chart-2);
--color-chart-3: var(--chart-3);
--color-chart-4: var(--chart-4);
--color-chart-5: var(--chart-5);
--radius-sm: calc(var(--radius) - 4px);
--radius-md: calc(var(--radius) - 2px);
--radius-lg: var(--radius);
--radius-xl: calc(var(--radius) + 4px);
--color-sidebar: var(--sidebar);
--color-sidebar-foreground: var(--sidebar-foreground);
--color-sidebar-primary: var(--sidebar-primary);
--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
--color-sidebar-accent: var(--sidebar-accent);
--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
--color-sidebar-border: var(--sidebar-border);
--color-sidebar-ring: var(--sidebar-ring);
}
@layer base {
* {
@apply border-border outline-ring/50;
}
body {
@apply bg-background text-foreground;
}
}
.animate-memory-indicator {
animation: memory-indicator-pop 0.6s cubic-bezier(0.4, 0, 0.2, 1) forwards;
transform-origin: top;
}

View File

@@ -0,0 +1,168 @@
import {
RTVIClientAudio,
useRTVIClient,
useRTVIClientTransportState,
} from '@pipecat-ai/client-react';
import { RTVIProvider } from './providers/RTVIProvider';
import { DebugDisplay } from './components/DebugDisplay';
import { Waveform } from './components/Waveform';
import StaticMemoryPanel from './components/StaticMemoryPanel';
import './App.css';
import Navbar from './components/Navbar';
import { ScrollArea } from './components/ui/scroll-area';
import { InteractiveHoverButton } from "@/components/magicui/interactive-hover-button";
import { useState, useEffect } from 'react';
import { CaretRight, Memory } from "@phosphor-icons/react";
import { cn } from "@/lib/utils";
import { Button } from "./components/ui/button";
function AppContent() {
const client = useRTVIClient();
const transportState = useRTVIClientTransportState();
const isConnected = ['connected', 'ready'].includes(transportState);
const isConnecting = ['connecting', 'disconnecting'].includes(transportState);
const [speakingState, setSpeakingState] = useState<'user' | 'assistant' | 'system' | 'idle'>('idle');
const [leftPanelCollapsed, setLeftPanelCollapsed] = useState(true);
const [rightPanelCollapsed, setRightPanelCollapsed] = useState(false);
const [isFirstConnectionConnected, setIsFirstConnectionConnected] = useState(false);
const handleReset = () => {
setIsFirstConnectionConnected(false);
}
// Simulated memory counts - replace with actual counts from your components
const leftMemoryCount = 3;
const rightMemoryCount = 3;
const handleConnect = async () => {
if (!client) {
console.error('RTVI client is not initialized');
return;
}
try {
if (isConnected) {
await client.disconnect();
} else {
await client.connect();
}
setIsFirstConnectionConnected(true);
} catch (error) {
console.error('Connection error:', error);
}
};
// Listen for message events from DebugDisplay
useEffect(() => {
const handleMessage = (event: CustomEvent<{ type: 'user' | 'assistant' | 'system' }>) => {
setSpeakingState(event.detail.type);
// Reset to idle after animation
setTimeout(() => setSpeakingState('idle'), 2000);
};
window.addEventListener('newMessage', handleMessage as EventListener);
return () => window.removeEventListener('newMessage', handleMessage as EventListener);
}, []);
return (
<div className="app">
<div className="mb-24">
<Navbar onReset={handleReset} />
</div>
{!isConnected && !isFirstConnectionConnected ? (
<div className="h-[calc(100vh-12rem)] flex items-center justify-center">
<InteractiveHoverButton
onClick={handleConnect}
disabled={isConnecting}
className="px-8 py-4 text-xl animate-pulse-shadow"
>
{isConnecting ? 'Connecting...' : 'Connect'}
</InteractiveHoverButton>
</div>
) : (
<div className="flex px-4 relative">
{/* Main Content */}
<div className={cn(
"transition-all duration-300 ease-in-out flex-1 relative",
leftPanelCollapsed && rightPanelCollapsed ? "mx-4" : "",
!leftPanelCollapsed && !rightPanelCollapsed ? "mx-4" : "",
!leftPanelCollapsed && rightPanelCollapsed ? "ml-4 mr-4" : "",
leftPanelCollapsed && !rightPanelCollapsed ? "ml-4 mr-4" : ""
)}>
<ScrollArea className="w-full h-[calc(100vh-10rem)] rounded-xl bg-zinc-50 dark:bg-neutral-950">
<DebugDisplay onNewMessage={(type) => setSpeakingState(type)} />
</ScrollArea>
<div className="absolute bottom-6 left-1/2 -translate-x-1/2">
<div className="p-4 rounded-full shadow-lg border border-neutral-200 dark:border-neutral-800 bg-white dark:bg-neutral-900 transition-all duration-300 hover:shadow-xl">
<Waveform speakingState={speakingState} />
</div>
</div>
</div>
{/* Right Panel */}
<div className={cn(
"transition-all duration-300 ease-in-out group",
rightPanelCollapsed ? "w-12" : "w-[25rem]"
)}>
{/* Collapsed State with Memory Icons */}
<div className={cn(
"absolute top-0 right-0 h-full w-12 flex flex-col items-center pt-4 gap-2",
rightPanelCollapsed ? "opacity-100" : "opacity-0 pointer-events-none"
)}>
<Button
variant="ghost"
size="icon"
onClick={() => setRightPanelCollapsed(false)}
className="h-10 w-10 rounded-xl bg-violet-100 dark:bg-violet-900/30 hover:bg-violet-200 dark:hover:bg-violet-900/50"
>
<CaretRight
className="h-5 w-5 text-violet-600 dark:text-violet-400 rotate-180"
/>
</Button>
<div className="flex flex-col items-center gap-1 mt-2">
<Memory className="h-5 w-5 text-violet-500" weight="duotone" />
<span className="text-xs font-medium text-violet-600 dark:text-violet-400">{rightMemoryCount}</span>
</div>
</div>
{/* Expanded State */}
<div className={cn(
"transition-all duration-300 w-[25rem]",
rightPanelCollapsed ? "opacity-0 pointer-events-none" : "opacity-100"
)}>
<div className="relative">
<Button
variant="ghost"
size="icon"
onClick={() => setRightPanelCollapsed(true)}
className="absolute left-2 top-2 z-10 h-10 w-10 rounded-xl bg-violet-100 dark:bg-violet-900/30 hover:bg-violet-200 dark:hover:bg-violet-900/50"
>
<CaretRight
className="h-5 w-5 text-violet-600 dark:text-violet-400"
/>
</Button>
<ScrollArea>
<StaticMemoryPanel />
</ScrollArea>
</div>
</div>
</div>
</div>
)}
<RTVIClientAudio />
</div>
);
}
function App() {
return (
<RTVIProvider>
<AppContent />
</RTVIProvider>
);
}
export default App;

View File

@@ -0,0 +1,39 @@
import {
useRTVIClient,
useRTVIClientTransportState,
} from '@pipecat-ai/client-react';
import { InteractiveHoverButton } from "@/components/magicui/interactive-hover-button";
export function ConnectButton() {
const client = useRTVIClient();
const transportState = useRTVIClientTransportState();
const isConnected = ['connected', 'ready'].includes(transportState);
const handleClick = async () => {
if (!client) {
console.error('RTVI client is not initialized');
return;
}
try {
if (isConnected) {
await client.disconnect();
} else {
await client.connect();
}
} catch (error) {
console.error('Connection error:', error);
}
};
return (
<InteractiveHoverButton
onClick={handleClick}
disabled={
!client || ['connecting', 'disconnecting'].includes(transportState)
}>
{!client || ['connecting', 'disconnecting'].includes(transportState) ? 'Connecting...' : isConnected ? 'Disconnect' : 'Connect'}
</InteractiveHoverButton>
);
}

View File

@@ -0,0 +1,244 @@
import { useRef, useCallback, useState, useEffect } from 'react';
import {
Participant,
RTVIEvent,
TransportState,
TranscriptData,
BotLLMTextData,
} from '@pipecat-ai/client-js';
import { useRTVIClient, useRTVIClientEvent } from '@pipecat-ai/client-react';
interface Message {
id: string;
text: string;
type: 'user' | 'assistant' | 'system';
timestamp: Date;
showMemoryRefresh?: boolean;
memoryRefreshTime?: number;
showMemoryIndicator?: boolean;
}
interface DebugDisplayProps {
onNewMessage: (type: 'user' | 'assistant' | 'system') => void;
}
// Create a shared context for memory refresh
export const memoryRefreshTrigger = {
value: 0,
subscribers: new Set<(value: number) => void>(),
increment() {
this.value++;
this.subscribers.forEach(callback => callback(this.value));
}
};
export function DebugDisplay({ onNewMessage }: DebugDisplayProps) {
const [messages, setMessages] = useState<Message[]>([]);
const messagesEndRef = useRef<HTMLDivElement>(null);
const client = useRTVIClient();
// Add effect to handle delayed appearance of memory refresh indicators
useEffect(() => {
const timeouts: NodeJS.Timeout[] = [];
messages.forEach(message => {
if (message.showMemoryRefresh && !message.showMemoryIndicator) {
const timeout = setTimeout(() => {
setMessages(prev => prev.map(msg =>
msg.id === message.id
? { ...msg, showMemoryIndicator: true }
: msg
));
}, 800);
timeouts.push(timeout);
}
});
return () => {
timeouts.forEach(timeout => clearTimeout(timeout));
};
}, [messages]);
const generateRandomRefreshTime = () => {
return Number((Math.random() * (100 - 50) + 50).toFixed(2));
};
const scrollToBottom = useCallback(() => {
messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
}, []);
const addMessage = useCallback((text: string, type: 'user' | 'assistant' | 'system') => {
const newMessage: Message = {
id: `${Date.now()}-${Math.random()}`,
text,
type,
timestamp: new Date(),
showMemoryRefresh: type === 'user',
memoryRefreshTime: type === 'user' ? generateRandomRefreshTime() : undefined
};
setMessages(prev => [...prev, newMessage]);
onNewMessage(type);
if (type === 'assistant') {
memoryRefreshTrigger.increment();
}
setTimeout(scrollToBottom, 100);
}, [scrollToBottom, onNewMessage]);
// Log transport state changes
useRTVIClientEvent(
RTVIEvent.TransportStateChanged,
useCallback(
(state: TransportState) => {
addMessage(`Transport state changed: ${state}`, 'system');
},
[addMessage]
)
);
// Log bot connection events
useRTVIClientEvent(
RTVIEvent.BotConnected,
useCallback(
(participant?: Participant) => {
addMessage(`Bot connected: ${JSON.stringify(participant)}`, 'system');
},
[addMessage]
)
);
useRTVIClientEvent(
RTVIEvent.BotDisconnected,
useCallback(
(participant?: Participant) => {
addMessage(`Bot disconnected: ${JSON.stringify(participant)}`, 'system');
},
[addMessage]
)
);
// Log track events
useRTVIClientEvent(
RTVIEvent.TrackStarted,
useCallback(
(track: MediaStreamTrack, participant?: Participant) => {
addMessage(
`Track started: ${track.kind} from ${participant?.name || 'unknown'}`,
'system'
);
},
[addMessage]
)
);
useRTVIClientEvent(
RTVIEvent.TrackStopped,
useCallback(
(track: MediaStreamTrack, participant?: Participant) => {
addMessage(
`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`,
'system'
);
},
[addMessage]
)
);
// Log bot ready state and check tracks
useRTVIClientEvent(
RTVIEvent.BotReady,
useCallback(() => {
addMessage('Bot ready', 'system');
if (!client) return;
const tracks = client.tracks();
addMessage(
`Available tracks: ${JSON.stringify({
local: {
audio: !!tracks.local.audio,
video: !!tracks.local.video,
},
bot: {
audio: !!tracks.bot?.audio,
video: !!tracks.bot?.video,
},
})}`,
'system'
);
}, [client, addMessage])
);
// Log transcripts
useRTVIClientEvent(
RTVIEvent.UserTranscript,
useCallback(
(data: TranscriptData) => {
// Only log final transcripts
if (data.final) {
addMessage(data.text, 'user');
}
},
[addMessage]
)
);
useRTVIClientEvent(
RTVIEvent.BotTranscript,
useCallback(
(data: BotLLMTextData) => {
addMessage(data.text, 'assistant');
},
[addMessage]
)
);
return (
<div className="w-full h-full mx-auto p-6 bg-zinc-50">
<div className="flex flex-col gap-4 pb-16">
{messages.map((message) => (
<div
key={message.id}
className={`
flex animate-message-pop
${message.type === 'system' ? 'justify-center mx-auto w-full' : 'max-w-[80%]'}
${message.type === 'user' ? 'justify-end ml-auto' : ''}
${message.type === 'assistant' ? 'justify-start mr-auto' : ''}
`}
>
{message.type === 'system' ? (
<div className="text-sm text-zinc-500 font-medium text-center">
{message.text}
</div>
) : (
<div className="flex flex-col">
{!message.showMemoryRefresh && message.showMemoryIndicator && (
<div className="text-xs bg-blue-100 text-blue-400 px-2 py-1 rounded-md self-end animate-memory-indicator overflow-hidden">
Memory refresh: {message.memoryRefreshTime}ms
</div>
)}
<div
className={`
px-4 py-3 rounded-2xl shadow-sm transition-all duration-300 hover:shadow-md
${message.type === 'user' ? 'bg-zinc-800 text-white' : ''}
${message.type === 'assistant' ? 'bg-white text-zinc-800' : ''}
`}
>
<div className="break-words">{message.text}</div>
<div className="text-xs mt-1 opacity-70">
{message.timestamp.toLocaleTimeString([], {
hour: '2-digit',
minute: '2-digit',
second: '2-digit'
})}
</div>
</div>
</div>
)}
</div>
))}
<div ref={messagesEndRef} />
</div>
</div>
);
}

View File

@@ -0,0 +1,78 @@
import type React from "react"
import { Card, CardContent } from "@/components/ui/card"
import { Badge } from "@/components/ui/badge"
import { Tag, Clock } from "lucide-react"
import { Memory } from "@phosphor-icons/react"
import { formatDistanceToNow } from "date-fns"
import { cn } from "@/lib/utils"
interface MemoryCardProps {
memory: {
id: string
content: string
createdAt: string
tags?: string[]
categories?: string[]
}
}
export const MemoryCard: React.FC<MemoryCardProps> = ({ memory }) => {
return (
<Card className="w-full overflow-hidden border border-neutral-200 dark:border-neutral-800 bg-white dark:bg-neutral-900">
<CardContent className="p-4">
<div className="flex items-center gap-3 mb-4">
<div
className={cn(
"h-8 w-8 rounded-full flex items-center justify-center",
"bg-violet-100 dark:bg-violet-900/30",
)}
>
<Memory className="h-4 w-4 text-violet-500" weight="duotone" />
</div>
<div className="flex items-center gap-2 text-xs text-neutral-500 dark:text-neutral-400">
<Clock className="h-3 w-3" />
<span>{formatDistanceToNow(new Date(memory.createdAt), { addSuffix: true })}</span>
</div>
</div>
<div className="relative pl-4">
<div className="absolute -left-0 top-0 bottom-0 w-[2px] rounded-full bg-violet-100 dark:bg-violet-900" />
<div className="prose prose-sm dark:prose-invert max-w-none">
<p className="text-sm text-neutral-800 dark:text-neutral-200 leading-relaxed">{memory.content}</p>
</div>
{memory.tags && memory.tags.length > 0 && (
<div className="flex flex-wrap gap-1.5 mt-3">
{memory.tags.map((tag, tagIndex) => (
<Badge
key={tagIndex}
variant="secondary"
className="px-1.5 py-0 h-4 text-[10px] bg-violet-50 dark:bg-violet-900/20 text-violet-600 dark:text-violet-400 hover:bg-violet-100 dark:hover:bg-violet-900/40 transition-colors"
>
<Tag className="h-2.5 w-2.5 mr-1" />
{tag}
</Badge>
))}
</div>
)}
{memory.categories && memory.categories.length > 0 && (
<div className="flex flex-wrap gap-1.5 mt-3">
{memory.categories.map((category, categoryIndex) => (
<Badge
key={categoryIndex}
variant="secondary"
className="px-1.5 py-0 h-4 text-[10px] bg-violet-50 dark:bg-violet-900/20 text-violet-600 dark:text-violet-400 hover:bg-violet-100 dark:hover:bg-violet-900/40 transition-colors"
>
<Tag className="h-2.5 w-2.5 mr-1" />
{category}
</Badge>
))}
</div>
)}
</div>
</CardContent>
</Card>
)
}

View File

@@ -0,0 +1,17 @@
import { ConnectButton } from './ConnectButton'
import ThemeAwareLogo from './ThemeAwareLogo'
const Navbar = ({ onReset }: { onReset: () => void }) => {
return (
<nav className="fixed top-4 left-1/2 -translate-x-1/2 w-full max-w-[95%] bg-white rounded-3xl shadow-lg px-6 py-2 flex items-center justify-between z-50">
<div onClick={onReset} className="cursor-pointer">
<ThemeAwareLogo />
</div>
<button className="p-2 hover:bg-zinc-100 rounded-full transition-colors">
<ConnectButton />
</button>
</nav>
)
}
export default Navbar

View File

@@ -0,0 +1,134 @@
import React, { useState, useEffect, useCallback } from "react"
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
import { ScrollArea } from "./ui/scroll-area"
import { Memory, CaretRight } from "@phosphor-icons/react"
import { cn } from "@/lib/utils"
import { Button } from "./ui/button"
import { MemoryCard } from "./MemoryCard"
import { memoryRefreshTrigger } from "./DebugDisplay"
interface MemoryItem {
id: string // Kept for React key prop
content: string
createdAt: string
categories: string[]
}
const fetchMemories = async () => {
const response = await fetch("http://localhost:7860/memories")
const data = await response.json()
return data
}
const MemorySkeleton = () => (
<div className="animate-pulse space-y-4">
<div className="flex items-start space-x-4">
<div className="h-10 w-10 rounded-full bg-neutral-200 dark:bg-neutral-800" />
<div className="space-y-3 flex-1">
<div className="h-4 w-1/4 bg-neutral-200 dark:bg-neutral-800 rounded" />
<div className="space-y-2">
<div className="h-4 w-full bg-neutral-200 dark:bg-neutral-800 rounded" />
<div className="h-4 w-5/6 bg-neutral-200 dark:bg-neutral-800 rounded" />
</div>
</div>
</div>
</div>
)
export default function StaticMemoryPanel() {
const [isCollapsed, setIsCollapsed] = useState(false)
const [memories, setMemories] = useState<MemoryItem[]>([])
const [isInitialLoading, setIsInitialLoading] = useState(true)
const [localRefreshTrigger, setLocalRefreshTrigger] = useState(0)
const fetchEffectMemories = useCallback(async () => {
// Only set loading if this is the initial fetch (no memories)
if (memories.length === 0) {
setIsInitialLoading(true)
}
try {
const memories = await fetchMemories()
if (memories) {
setMemories(memories as unknown as MemoryItem[])
} else {
setMemories([])
console.log("No memories found")
}
} catch (error) {
console.error("Error fetching memories:", error)
setMemories([])
} finally {
setIsInitialLoading(false)
}
}, [memories.length])
useEffect(() => {
fetchEffectMemories()
}, [fetchEffectMemories, localRefreshTrigger])
useEffect(() => {
const handleRefresh = (value: number) => {
setLocalRefreshTrigger(value);
};
memoryRefreshTrigger.subscribers.add(handleRefresh);
return () => {
memoryRefreshTrigger.subscribers.delete(handleRefresh);
};
}, []);
return (
<Card className="w-full overflow-hidden border border-neutral-200 dark:border-neutral-800 bg-zinc-50 dark:bg-neutral-900">
<CardHeader className="p-4 pb-2 flex flex-row items-center justify-between space-y-0 border-b border-neutral-200 dark:border-neutral-800">
<div className="flex items-center gap-3">
<div className={cn("h-8 w-8 rounded-full flex items-center justify-center bg-violet-100 dark:bg-violet-900/30")}>
<Memory className="h-4 w-4 text-violet-500" weight="duotone" />
</div>
<div className="flex flex-col">
<CardTitle className="text-base font-medium text-neutral-900 dark:text-neutral-100">
Memories ({memories.length})
</CardTitle>
</div>
</div>
<Button
variant="ghost"
size="icon"
onClick={() => setIsCollapsed(!isCollapsed)}
className="h-8 w-8"
>
<CaretRight
className={cn(
"h-4 w-4 text-neutral-500 transition-transform",
isCollapsed ? "" : "rotate-90"
)}
/>
</Button>
</CardHeader>
<div className={cn(
"transition-all duration-300 ease-in-out",
isCollapsed ? "h-0" : "h-[calc(100vh-218px)]"
)}>
{!isCollapsed && (
<ScrollArea className="h-full">
<CardContent className="py-4">
<div className="space-y-6">
{isInitialLoading && memories.length === 0 ? (
<>
<MemorySkeleton />
<MemorySkeleton />
<MemorySkeleton />
</>
) : (
memories.map((memory) => (
<MemoryCard key={memory.id} memory={memory} />
))
)}
</div>
</CardContent>
</ScrollArea>
)}
</div>
</Card>
)
}

View File

@@ -0,0 +1,11 @@
import { useRTVIClientTransportState } from '@pipecat-ai/client-react';
export function StatusDisplay() {
const transportState = useRTVIClientTransportState();
return (
<div className="status">
Status: <span>{transportState}</span>
</div>
);
}

View File

@@ -0,0 +1,12 @@
import mem0Logo from "./light.svg";
export default function ThemeAwareLogo({
width = 100,
height = 35,
}: {
width?: number;
height?: number;
}) {
return <img src={mem0Logo} alt="Mem0.ai" width={width} height={height} />;
}

View File

@@ -0,0 +1,33 @@
interface WaveformProps {
speakingState: 'user' | 'assistant' | 'system' | 'idle';
}
export function Waveform({ speakingState }: WaveformProps) {
const getAnimationClass = () => {
switch (speakingState) {
case 'user':
return 'waveform-bar bg-zinc-800';
case 'assistant':
return 'waveform-bar bg-zinc-600';
case 'system':
return 'waveform-bar-listening bg-zinc-400';
default:
return 'waveform-bar-listening bg-zinc-300';
}
};
return (
<div className="flex items-end gap-1 h-8">
{[...Array(5)].map((_, i) => (
<div
key={i}
className={`w-1 rounded-full ${getAnimationClass()}`}
style={{
height: speakingState === 'idle' ? '40%' : `${Math.random() * 60 + 40}%`,
animationDelay: `${i * 0.1}s`
}}
/>
))}
</div>
);
}

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 13 KiB

View File

@@ -0,0 +1,35 @@
import React from "react";
import { ArrowRight } from "lucide-react";
import { cn } from "@/lib/utils";
interface InteractiveHoverButtonProps
extends React.ButtonHTMLAttributes<HTMLButtonElement> {}
export const InteractiveHoverButton = React.forwardRef<
HTMLButtonElement,
InteractiveHoverButtonProps
>(({ children, className, ...props }, ref) => {
return (
<button
ref={ref}
className={cn(
"group relative w-auto cursor-pointer overflow-hidden rounded-full border bg-background p-2 px-6 text-center font-semibold hover:text-white",
className,
)}
{...props}
>
<div className="flex items-center gap-2">
<div className="h-2 w-2 rounded-full bg-black transition-all duration-300 group-hover:scale-[100.8]"></div>
<span className="inline-block transition-all duration-300 group-hover:translate-x-12 group-hover:opacity-0">
{children}
</span>
</div>
<div className="absolute top-0 z-10 flex h-full w-full translate-x-12 items-center justify-center gap-2 text-primary-foreground opacity-0 transition-all duration-300 group-hover:-translate-x-5 group-hover:opacity-100">
<span>{children}</span>
<ArrowRight />
</div>
</button>
);
});
InteractiveHoverButton.displayName = "InteractiveHoverButton";

View File

@@ -0,0 +1,56 @@
import { useEffect, useRef } from "react"
import { cn } from "@/lib/utils"
interface BorderTrailProps extends React.HTMLAttributes<HTMLDivElement> {
size?: number
duration?: number
}
export const BorderTrail: React.FC<BorderTrailProps> = ({ className, size = 100, duration = 5, ...props }) => {
const borderRef = useRef<HTMLDivElement>(null)
useEffect(() => {
const borderElement = borderRef.current
if (!borderElement) return
const animateBorder = () => {
const startTime = performance.now()
let animationFrameId: number
const animate = (currentTime: number) => {
const elapsed = (currentTime - startTime) / 1000
const progress = (elapsed % duration) / duration
const rotation = progress * 360
if (borderElement) {
borderElement.style.transform = `rotate(${rotation}deg)`
}
animationFrameId = requestAnimationFrame(animate)
}
animationFrameId = requestAnimationFrame(animate)
return () => {
cancelAnimationFrame(animationFrameId)
}
}
const cleanup = animateBorder()
return cleanup
}, [duration])
return (
<div
ref={borderRef}
className={cn("absolute inset-0 rounded-full opacity-70", className)}
style={{
width: `${size}%`,
height: `${size}%`,
left: `${(100 - size) / 2}%`,
top: `${(100 - size) / 2}%`,
}}
{...props}
/>
)
}

View File

@@ -0,0 +1,55 @@
import { useEffect, useRef } from "react"
import { cn } from "@/lib/utils"
interface TextShimmerProps extends React.HTMLAttributes<HTMLDivElement> {
duration?: number
children: React.ReactNode
}
export const TextShimmer: React.FC<TextShimmerProps> = ({ className, duration = 3, children, ...props }) => {
const shimmerRef = useRef<HTMLDivElement>(null)
useEffect(() => {
const shimmerElement = shimmerRef.current
if (!shimmerElement) return
const animateShimmer = () => {
const startTime = performance.now()
let animationFrameId: number
const animate = (currentTime: number) => {
const elapsed = (currentTime - startTime) / 1000
const progress = (elapsed % duration) / duration
const position = progress * 200 - 100 // -100% to 100%
if (shimmerElement) {
shimmerElement.style.backgroundPosition = `${position}% 0`
}
animationFrameId = requestAnimationFrame(animate)
}
animationFrameId = requestAnimationFrame(animate)
return () => {
cancelAnimationFrame(animationFrameId)
}
}
const cleanup = animateShimmer()
return cleanup
}, [duration])
return (
<div
ref={shimmerRef}
className={cn(
"inline-block bg-gradient-to-r from-transparent via-violet-400/20 to-transparent bg-[length:200%_100%]",
className
)}
{...props}
>
{children}
</div>
)
}

View File

@@ -0,0 +1,31 @@
import * as React from "react"
import { cn } from "@/lib/utils"
export interface BadgeProps extends React.HTMLAttributes<HTMLDivElement> {
variant?: "default" | "secondary" | "destructive" | "outline"
}
export function Badge({
className,
variant = "default",
...props
}: BadgeProps) {
return (
<div
className={cn(
"inline-flex items-center rounded-full border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2",
{
"border-transparent bg-primary text-primary-foreground hover:bg-primary/80":
variant === "default",
"border-transparent bg-secondary text-secondary-foreground hover:bg-secondary/80":
variant === "secondary",
"border-transparent bg-destructive text-destructive-foreground hover:bg-destructive/80":
variant === "destructive",
"text-foreground": variant === "outline",
},
className
)}
{...props}
/>
)
}

View File

@@ -0,0 +1,58 @@
import * as React from "react"
import { Slot } from "@radix-ui/react-slot"
import { cva, type VariantProps } from "class-variance-authority"
import { cn } from "@/lib/utils"
const buttonVariants = cva(
"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-[color,box-shadow] disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive",
{
variants: {
variant: {
default:
"bg-primary text-primary-foreground shadow-xs hover:bg-primary/90",
destructive:
"bg-destructive text-white shadow-xs hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40",
outline:
"border border-input bg-background shadow-xs hover:bg-accent hover:text-accent-foreground",
secondary:
"bg-secondary text-secondary-foreground shadow-xs hover:bg-secondary/80",
ghost: "hover:bg-accent hover:text-accent-foreground",
link: "text-primary underline-offset-4 hover:underline",
},
size: {
default: "h-9 px-4 py-2 has-[>svg]:px-3",
sm: "h-8 rounded-md gap-1.5 px-3 has-[>svg]:px-2.5",
lg: "h-10 rounded-md px-6 has-[>svg]:px-4",
icon: "size-9",
},
},
defaultVariants: {
variant: "default",
size: "default",
},
}
)
function Button({
className,
variant,
size,
asChild = false,
...props
}: React.ComponentProps<"button"> &
VariantProps<typeof buttonVariants> & {
asChild?: boolean
}) {
const Comp = asChild ? Slot : "button"
return (
<Comp
data-slot="button"
className={cn(buttonVariants({ variant, size, className }))}
{...props}
/>
)
}
export { Button, buttonVariants }

View File

@@ -0,0 +1,54 @@
import * as React from "react"
import { cn } from "@/lib/utils"
const Card = React.forwardRef<
HTMLDivElement,
React.HTMLAttributes<HTMLDivElement>
>(({ className, ...props }, ref) => (
<div
ref={ref}
className={cn(
"rounded-lg border border-neutral-200 bg-white text-neutral-950 shadow-sm dark:border-neutral-800 dark:bg-neutral-950 dark:text-neutral-50",
className
)}
{...props}
/>
))
Card.displayName = "Card"
const CardHeader = React.forwardRef<
HTMLDivElement,
React.HTMLAttributes<HTMLDivElement>
>(({ className, ...props }, ref) => (
<div
ref={ref}
className={cn("flex flex-col space-y-1.5 p-6", className)}
{...props}
/>
))
CardHeader.displayName = "CardHeader"
const CardTitle = React.forwardRef<
HTMLParagraphElement,
React.HTMLAttributes<HTMLHeadingElement>
>(({ className, ...props }, ref) => (
<h3
ref={ref}
className={cn(
"text-2xl font-semibold leading-none tracking-tight",
className
)}
{...props}
/>
))
CardTitle.displayName = "CardTitle"
const CardContent = React.forwardRef<
HTMLDivElement,
React.HTMLAttributes<HTMLDivElement>
>(({ className, ...props }, ref) => (
<div ref={ref} className={cn("p-6 pt-0", className)} {...props} />
))
CardContent.displayName = "CardContent"
export { Card, CardHeader, CardTitle, CardContent }

View File

@@ -0,0 +1,56 @@
import * as React from "react"
import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area"
import { cn } from "@/lib/utils"
function ScrollArea({
className,
children,
...props
}: React.ComponentProps<typeof ScrollAreaPrimitive.Root>) {
return (
<ScrollAreaPrimitive.Root
data-slot="scroll-area"
className={cn("relative", className)}
{...props}
>
<ScrollAreaPrimitive.Viewport
data-slot="scroll-area-viewport"
className="ring-ring/10 dark:ring-ring/20 dark:outline-ring/40 outline-ring/50 size-full rounded-[inherit] transition-[color,box-shadow] focus-visible:ring-4 focus-visible:outline-1"
>
{children}
</ScrollAreaPrimitive.Viewport>
<ScrollBar />
<ScrollAreaPrimitive.Corner />
</ScrollAreaPrimitive.Root>
)
}
function ScrollBar({
className,
orientation = "vertical",
...props
}: React.ComponentProps<typeof ScrollAreaPrimitive.ScrollAreaScrollbar>) {
return (
<ScrollAreaPrimitive.ScrollAreaScrollbar
data-slot="scroll-area-scrollbar"
orientation={orientation}
className={cn(
"flex touch-none p-px transition-colors select-none",
orientation === "vertical" &&
"h-full w-2.5 border-l border-l-transparent",
orientation === "horizontal" &&
"h-2.5 flex-col border-t border-t-transparent",
className
)}
{...props}
>
<ScrollAreaPrimitive.ScrollAreaThumb
data-slot="scroll-area-thumb"
className="bg-border relative flex-1 rounded-full"
/>
</ScrollAreaPrimitive.ScrollAreaScrollbar>
)
}
export { ScrollArea, ScrollBar }

View File

@@ -0,0 +1,6 @@
import { type ClassValue, clsx } from 'clsx';
import { twMerge } from 'tailwind-merge';
export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs));
}

View File

@@ -0,0 +1,9 @@
import React from 'react';
import ReactDOM from 'react-dom/client';
import App from './App';
ReactDOM.createRoot(document.getElementById('root')!).render(
<React.StrictMode>
<App />
</React.StrictMode>
);

View File

@@ -0,0 +1,22 @@
import { type PropsWithChildren } from 'react';
import { RTVIClient } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
import { RTVIClientProvider } from '@pipecat-ai/client-react';
const transport = new DailyTransport();
const client = new RTVIClient({
transport,
params: {
baseUrl: 'http://localhost:7860',
endpoints: {
connect: '/connect',
},
},
enableMic: true,
enableCam: false,
});
export function RTVIProvider({ children }: PropsWithChildren) {
return <RTVIClientProvider client={client}>{children}</RTVIClientProvider>;
}

View File

@@ -0,0 +1,53 @@
/** @type {import('tailwindcss').Config} */
export default {
content: [
"./index.html",
"./src/**/*.{js,ts,jsx,tsx}",
],
theme: {
extend: {
keyframes: {
'message-pop': {
'0%': {
opacity: '0',
transform: 'scale(0.95) translateY(10px)'
},
'100%': {
opacity: '1',
transform: 'scale(1) translateY(0)'
}
},
'memory-pop': {
'0%': {
opacity: '0',
transform: 'translateY(10px)'
},
'100%': {
opacity: '1',
transform: 'translateY(0)'
}
},
'loading-bar': {
'0%': {
transform: 'translateX(-100%)'
},
'100%': {
transform: 'translateX(100%)'
}
},
'shimmer': {
'100%': {
transform: 'translateX(100%)'
}
}
},
animation: {
'message-pop': 'message-pop 0.3s ease-out',
'memory-pop': 'memory-pop 0.3s ease-out forwards',
'loading-bar': 'loading-bar 2s infinite',
'shimmer': 'shimmer 1.5s infinite'
}
},
},
plugins: [],
}

View File

@@ -0,0 +1,30 @@
{
"compilerOptions": {
"target": "ES2020",
"useDefineForClassFields": true,
"lib": ["ES2020", "DOM", "DOM.Iterable"],
"module": "ESNext",
"skipLibCheck": true,
/* Bundler mode */
"moduleResolution": "bundler",
"allowImportingTsExtensions": true,
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"jsx": "react-jsx",
/* Linting */
"strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noFallthroughCasesInSwitch": true,
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"]
}
},
"include": ["src"],
"references": [{ "path": "./tsconfig.node.json" }]
}

View File

@@ -0,0 +1,14 @@
{
"compilerOptions": {
"composite": true,
"skipLibCheck": true,
"module": "ESNext",
"moduleResolution": "bundler",
"allowSyntheticDefaultImports": true,
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"]
}
},
"include": ["vite.config.ts"]
}

View File

@@ -0,0 +1,14 @@
import path from 'path'
import { defineConfig } from 'vite'
import react from '@vitejs/plugin-react'
import tailwindcss from '@tailwindcss/vite'
// https://vite.dev/config/
export default defineConfig({
plugins: [react(), tailwindcss()],
resolve: {
alias: {
"@": path.resolve(__dirname, "./src"),
},
},
})

Binary file not shown.

After

Width:  |  Height:  |  Size: 360 KiB

View File

@@ -0,0 +1,17 @@
FROM python:3.10-bullseye
RUN mkdir /app
RUN mkdir /app/assets
RUN mkdir /app/utils
COPY requirements.txt /app/
WORKDIR /app
RUN pip3 install -r requirements.txt
COPY *.py /app/
COPY assets/* /app/assets/
COPY .env /app/.env
EXPOSE 7860
CMD ["python3", "server.py"]

View File

@@ -0,0 +1,57 @@
# Personalized Voice Agent Server
A FastAPI server that manages bot instances and provides endpoints for both Daily Prebuilt and Pipecat client connections.
## Environment Variables
Copy `env.example` to `.env` and configure:
```ini
# Required API Keys
DAILY_API_KEY= # Your Daily API key
MEM0_API_KEY= # Your Mem0 API key
OPENAI_API_KEY= # Your OpenAI API key (required for OpenAI bot)
ELEVENLABS_API_KEY= # Your ElevenLabs API key
# Optional Configuration
DAILY_API_URL= # Optional: Daily API URL (defaults to https://api.daily.co/v1)
DAILY_SAMPLE_ROOM_URL= # Optional: Fixed room URL for development
HOST= # Optional: Host address (defaults to 0.0.0.0)
FAST_API_PORT= # Optional: Port number (defaults to 7860)
```
## Available Bots
The server supports two bot implementations:
1. **OpenAI Bot** (Default)
- Uses GPT-4 for conversation
- Requires OPENAI_API_KEY
## Running the Server
Set up and activate your virtual environment:
```bash
python3 -m venv venv
source venv/bin/activate # On Windows: venv\Scripts\activate
```
Install dependencies:
```bash
pip install -r requirements.txt
```
If you want to use the local version of `pipecat` in this repo rather than the last published version, also run:
```bash
pip install --editable "../../../[daily,elevenlabs,openai,silero,mem0ai]"
```
Run the server:
```bash
python server.py
```

View File

@@ -0,0 +1,243 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""OpenAI Bot Implementation.
This module implements a chatbot using OpenAI's GPT-4 model for natural language
processing. It includes:
- Real-time audio/video interaction through Daily
- Animated robot avatar
- Text-to-speech using ElevenLabs
- Support for both English and Spanish
The bot runs as part of a pipeline that processes audio/video frames and manages
the conversation flow.
"""
import asyncio
import os
import sys
import aiohttp
from dotenv import load_dotenv
from loguru import logger
from runner import configure
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.mem0 import Mem0MemoryService
from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor
from pipecat.services.elevenlabs import ElevenLabsTTSService
from pipecat.services.openai import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport
load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
)
try:
from mem0 import MemoryClient
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Mem0, you need to `pip install mem0ai`. Also, set the environment variable MEM0_API_KEY."
)
raise Exception(f"Missing module: {e}")
async def get_initial_greeting(memory_client: MemoryClient, user_id: str, agent_id: str, run_id: str) -> str:
"""Fetch all memories for the user and create a personalized greeting.
Returns:
A personalized greeting based on user memories
"""
try:
# Create filters based on available IDs
id_pairs = [("user_id", user_id), ("agent_id", agent_id), ("run_id", run_id)]
clauses = [{name: value} for name, value in id_pairs if value is not None]
filters = {"AND": clauses} if clauses else {}
# Get all memories for this user
memories = memory_client.get_all(filters=filters, version="v2")
if not memories or len(memories) == 0:
return "Hello! It's nice to meet you. How can I help you today?"
# Create a personalized greeting based on memories
greeting = "Hello! It's great to see you again. "
# Add some personalization based on memories (limit to 3 memories for brevity)
if len(memories) > 0:
greeting += "Based on our previous conversations, I remember: "
for i, memory in enumerate(memories[:3], 1):
memory_content = memory.get('memory', '')
# Keep memory references brief
if len(memory_content) > 100:
memory_content = memory_content[:97] + "..."
greeting += f"{memory_content} "
greeting += "How can I help you today?"
logger.debug(f"Created personalized greeting from {len(memories)} memories")
return greeting
except Exception as e:
logger.error(f"Error retrieving initial memories from Mem0: {e}")
return "Hello! How can I help you today?"
async def main():
"""Main bot execution function.
Sets up and runs the bot pipeline including:
- Daily video transport
- Speech-to-text and text-to-speech services
- Language model integration
- Mem0 memory service
- RTVI event handling
"""
# Note: You can pass the user_id as a parameter in API call
USER_ID = "deshraj"
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
# Set up Daily transport with video/audio parameters
transport = DailyTransport(
room_url,
token,
"Chatbot",
DailyParams(
audio_out_enabled=True,
camera_out_enabled=True,
camera_out_width=1024,
camera_out_height=576,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
transcription_enabled=True,
#
# Spanish
#
# transcription_settings=DailyTranscriptionSettings(
# language="es",
# tier="nova",
# model="2-general"
# )
),
)
# Initialize text-to-speech service
tts = ElevenLabsTTSService(
api_key=os.getenv("ELEVENLABS_API_KEY"),
#
# English
#
voice_id="pNInz6obpgDQGcFmaJgB",
#
# Spanish
#
# model="eleven_multilingual_v2",
# voice_id="gD1IexrzCvsXPHUuT0s3",
)
# Initialize Mem0 memory service
memory = Mem0MemoryService(
api_key=os.getenv("MEM0_API_KEY"),
user_id=USER_ID, # Unique identifier for the user
# agent_id="life_coach_bot", # Optional identifier for the agent
# run_id="session_1", # Optional identifier for the run
params=Mem0MemoryService.InputParams(
search_limit=10,
search_threshold=0.3,
api_version="v2",
system_prompt="Based on previous conversations, I recall: \n\n",
add_as_system_message=True,
position=1
)
)
# Initialize LLM service
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
messages = [
{
"role": "system",
"content": """You are a personal assistant. You can remember things about the person you are talking to.
Some Guidelines:
- Make sure your responses are friendly yet short and concise.
- If the user asks you to remember something, make sure to remember it.
- Greet the user by their name if you know about it.
"""
},
]
# Set up conversation context and management
# The context_aggregator will automatically collect conversation context
context = OpenAILLMContext(messages)
context_aggregator = llm.create_context_aggregator(context)
#
# RTVI events for Pipecat client UI
#
rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
pipeline = Pipeline(
[
transport.input(),
rtvi,
context_aggregator.user(),
memory,
llm,
tts,
transport.output(),
context_aggregator.assistant(),
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
allow_interruptions=True,
enable_metrics=True,
enable_usage_metrics=True,
),
observers=[RTVIObserver(rtvi)],
)
@rtvi.event_handler("on_client_ready")
async def on_client_ready(rtvi):
await rtvi.set_bot_ready()
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
await transport.capture_participant_transcription(participant["id"])
# Get personalized greeting based on user memories. Can pass agent_id and run_id as per requirement of the application to manage short term memory or agent specific memory.
greeting = await get_initial_greeting(memory_client=memory.memory_client, user_id=USER_ID, agent_id=None, run_id=None)
# Add the greeting as an assistant message to start the conversation
context.add_message({"role": "assistant", "content": greeting})
# Queue the context frame to start the conversation
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_participant_left")
async def on_participant_left(transport, participant, reason):
print(f"Participant left: {participant}")
await task.cancel()
runner = PipelineRunner()
await runner.run(task)
if __name__ == "__main__":
asyncio.run(main())

View File

@@ -0,0 +1,6 @@
DAILY_SAMPLE_ROOM_URL=https://yourdomain.daily.co/yourroom # (for joining the bot to the same room repeatedly for local dev)
DAILY_API_KEY=7df...
OPENAI_API_KEY=sk-PL...
GEMINI_API_KEY=AIza...
ELEVENLABS_API_KEY=aeb...
MEM0_API_KEY=m0-...

View File

@@ -0,0 +1,5 @@
python-dotenv
fastapi[all]
uvicorn
pipecat-ai[daily,elevenlabs,openai,silero,google]
mem0ai>=0.1.69

View File

@@ -0,0 +1,56 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import argparse
import os
import aiohttp
from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper
async def configure(aiohttp_session: aiohttp.ClientSession):
"""Configure the Daily room and Daily REST helper."""
parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=False, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=False,
help="Daily API Key (needed to create an owner token for the room)",
)
args, unknown = parser.parse_known_args()
url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL")
key = args.apikey or os.getenv("DAILY_API_KEY")
if not url:
raise Exception(
"No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL."
)
if not key:
raise Exception(
"No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers."
)
daily_rest_helper = DailyRESTHelper(
daily_api_key=key,
daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
aiohttp_session=aiohttp_session,
)
# Create a meeting token for the given room with an expiration 1 hour in
# the future.
expiry_time: float = 60 * 60
token = await daily_rest_helper.get_token(url, expiry_time)
return (url, token)

View File

@@ -0,0 +1,270 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""RTVI Bot Server Implementation.
This FastAPI server manages RTVI bot instances and provides endpoints for both
direct browser access and RTVI client connections. It handles:
- Creating Daily rooms
- Managing bot processes
- Providing connection credentials
- Monitoring bot status
Requirements:
- Daily API key (set in .env file)
- Python 3.10+
- FastAPI
- Running bot implementation
"""
import argparse
import os
import subprocess
from contextlib import asynccontextmanager
from typing import Any, Dict
import aiohttp
from dotenv import load_dotenv
from fastapi import FastAPI, HTTPException, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse, RedirectResponse
from mem0 import MemoryClient
from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper, DailyRoomParams
# Load environment variables from .env file
load_dotenv(override=True)
# Maximum number of bot instances allowed per room
MAX_BOTS_PER_ROOM = 1
# Dictionary to track bot processes: {pid: (process, room_url)}
bot_procs = {}
# Store Daily API helpers
daily_helpers = {}
def cleanup():
"""Cleanup function to terminate all bot processes.
Called during server shutdown.
"""
for entry in bot_procs.values():
proc = entry[0]
proc.terminate()
proc.wait()
def get_bot_file():
bot_implementation = os.getenv("BOT_IMPLEMENTATION", "openai").lower().strip()
# If blank or None, default to openai
if not bot_implementation:
bot_implementation = "openai"
if bot_implementation not in ["openai", "gemini"]:
raise ValueError(
f"Invalid BOT_IMPLEMENTATION: {bot_implementation}. Must be 'openai' or 'gemini'"
)
return f"bot-{bot_implementation}"
@asynccontextmanager
async def lifespan(app: FastAPI):
"""FastAPI lifespan manager that handles startup and shutdown tasks.
- Creates aiohttp session
- Initializes Daily API helper
- Cleans up resources on shutdown
"""
aiohttp_session = aiohttp.ClientSession()
daily_helpers["rest"] = DailyRESTHelper(
daily_api_key=os.getenv("DAILY_API_KEY", ""),
daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
aiohttp_session=aiohttp_session,
)
yield
await aiohttp_session.close()
cleanup()
# Initialize FastAPI app with lifespan manager
app = FastAPI(lifespan=lifespan)
# Configure CORS to allow requests from any origin
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
async def create_room_and_token() -> tuple[str, str]:
"""Helper function to create a Daily room and generate an access token.
Returns:
tuple[str, str]: A tuple containing (room_url, token)
Raises:
HTTPException: If room creation or token generation fails
"""
room = await daily_helpers["rest"].create_room(DailyRoomParams())
if not room.url:
raise HTTPException(status_code=500, detail="Failed to create room")
token = await daily_helpers["rest"].get_token(room.url)
if not token:
raise HTTPException(status_code=500, detail=f"Failed to get token for room: {room.url}")
return room.url, token
@app.get("/")
async def start_agent(request: Request):
"""Endpoint for direct browser access to the bot.
Creates a room, starts a bot instance, and redirects to the Daily room URL.
Returns:
RedirectResponse: Redirects to the Daily room URL
Raises:
HTTPException: If room creation, token generation, or bot startup fails
"""
print("Creating room")
room_url, token = await create_room_and_token()
print(f"Room URL: {room_url}")
# Check if there is already an existing process running in this room
num_bots_in_room = sum(
1 for proc in bot_procs.values() if proc[1] == room_url and proc[0].poll() is None
)
if num_bots_in_room >= MAX_BOTS_PER_ROOM:
raise HTTPException(status_code=500, detail=f"Max bot limit reached for room: {room_url}")
# Spawn a new bot process
try:
bot_file = get_bot_file()
proc = subprocess.Popen(
[f"python3 -m {bot_file} -u {room_url} -t {token}"],
shell=True,
bufsize=1,
cwd=os.path.dirname(os.path.abspath(__file__)),
)
bot_procs[proc.pid] = (proc, room_url)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}")
return RedirectResponse(room_url)
@app.get("/memories")
async def get_memories(request: Request):
"""Endpoint for getting memories from the bot.
Returns:
Dict[Any, Any]: Memories from the bot
"""
memory_client = MemoryClient(api_key=os.getenv("MEM0_API_KEY"))
filters = {
"AND": [
{"user_id": "deshraj"},
]
}
memories = memory_client.get_all(filters=filters, version="v2")
# Format memories for emission
formatted_memories = [
{
"id": str(memory.get("id", "")),
"content": memory.get("memory", ""),
"createdAt": memory.get("created_at", ""),
"categories": memory.get("categories", [])
} for memory in memories
]
return formatted_memories
@app.post("/connect")
async def rtvi_connect(request: Request) -> Dict[Any, Any]:
"""RTVI connect endpoint that creates a room and returns connection credentials.
This endpoint is called by RTVI clients to establish a connection.
Returns:
Dict[Any, Any]: Authentication bundle containing room_url and token
Raises:
HTTPException: If room creation, token generation, or bot startup fails
"""
print("Creating room for RTVI connection")
room_url, token = await create_room_and_token()
print(f"Room URL: {room_url}")
# Start the bot process
try:
bot_file = get_bot_file()
proc = subprocess.Popen(
[f"python3 -m {bot_file} -u {room_url} -t {token}"],
shell=True,
bufsize=1,
cwd=os.path.dirname(os.path.abspath(__file__)),
)
bot_procs[proc.pid] = (proc, room_url)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}")
# Return the authentication bundle in format expected by DailyTransport
return {"room_url": room_url, "token": token}
@app.get("/status/{pid}")
def get_status(pid: int):
"""Get the status of a specific bot process.
Args:
pid (int): Process ID of the bot
Returns:
JSONResponse: Status information for the bot
Raises:
HTTPException: If the specified bot process is not found
"""
# Look up the subprocess
proc = bot_procs.get(pid)
# If the subprocess doesn't exist, return an error
if not proc:
raise HTTPException(status_code=404, detail=f"Bot with process id: {pid} not found")
# Check the status of the subprocess
status = "running" if proc[0].poll() is None else "finished"
return JSONResponse({"bot_id": pid, "status": status})
if __name__ == "__main__":
import uvicorn
# Parse command line arguments for server configuration
default_host = os.getenv("HOST", "0.0.0.0")
default_port = int(os.getenv("FAST_API_PORT", "7860"))
parser = argparse.ArgumentParser(description="Daily Storyteller FastAPI server")
parser.add_argument("--host", type=str, default=default_host, help="Host address")
parser.add_argument("--port", type=int, default=default_port, help="Port number")
parser.add_argument("--reload", action="store_true", help="Reload code on change")
config = parser.parse_args()
# Start the FastAPI server
uvicorn.run(
"server:app",
host=config.host,
port=config.port,
reload=config.reload,
)

View File

@@ -84,6 +84,7 @@ together = []
ultravox = [ "transformers~=4.48.0", "vllm~=0.7.3" ]
websocket = [ "websockets~=13.1", "fastapi~=0.115.6" ]
whisper = [ "faster-whisper~=1.1.1" ]
mem0 = [ "mem0ai~=0.1.76" ]
[tool.setuptools.packages.find]
# All the following settings are optional:

View File

@@ -0,0 +1,199 @@
from typing import Any, Dict, List
from loguru import logger
from pydantic import BaseModel, Field
from pipecat.frames.frames import (
ErrorFrame,
Frame,
LLMMessagesFrame,
)
from pipecat.processors.aggregators.openai_llm_context import (
OpenAILLMContext,
OpenAILLMContextFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
try:
from mem0 import MemoryClient
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use Mem0, you need to `pip install mem0ai`. Also, set the environment variable MEM0_API_KEY."
)
raise Exception(f"Missing module: {e}")
class Mem0MemoryService(FrameProcessor):
"""A standalone memory service that integrates with Mem0.
This service intercepts message frames in the pipeline, stores them in Mem0,
and enhances context with relevant memories before passing them downstream.
Args:
api_key (str): The API key for accessing Mem0's API
user_id (str): The user ID to associate with memories in Mem0
params (InputParams, optional): Configuration parameters for memory retrieval
"""
class InputParams(BaseModel):
search_limit: int = Field(default=10, ge=1)
search_threshold: float = Field(default=0.1, ge=0.0, le=1.0)
api_version: str = Field(default="v2")
system_prompt: str = Field(
default="Based on previous conversations, I recall: \n\n"
)
add_as_system_message: bool = Field(default=True)
position: int = Field(default=1)
def __init__(
self,
*,
api_key: str,
user_id: str = None,
agent_id: str = None,
run_id: str = None,
params: InputParams = InputParams(),
):
# Important: Call the parent class __init__ first
super().__init__()
self.memory_client = MemoryClient(api_key=api_key)
# At least one of user_id, agent_id, or run_id must be provided
if not any([user_id, agent_id, run_id]):
raise ValueError("At least one of user_id, agent_id, or run_id must be provided")
self.user_id = user_id
self.agent_id = agent_id
self.run_id = run_id
self.search_limit = params.search_limit
self.search_threshold = params.search_threshold
self.api_version = params.api_version
self.system_prompt = params.system_prompt
self.add_as_system_message = params.add_as_system_message
self.position = params.position
self.last_query = None
logger.info(f"Initialized Mem0MemoryService with {user_id=}, {agent_id=}, {run_id=}")
def _store_messages(self, messages: List[Dict[str, Any]]):
"""Store messages in Mem0.
Args:
messages: List of message dictionaries to store
"""
try:
logger.debug(f"Storing {len(messages)} messages in Mem0")
params = {"messages": messages, "metadata": {"platform": "pipecat"}}
for id in ["user_id", "agent_id", "run_id"]:
if getattr(self, id):
params[id] = getattr(self, id)
# Note: You can run this in background to avoid blocking the conversation
self.memory_client.add(**params)
except Exception as e:
logger.error(f"Error storing messages in Mem0: {e}")
def _retrieve_memories(self, query: str) -> List[Dict[str, Any]]:
"""Retrieve relevant memories from Mem0.
Args:
query: The query to search for relevant memories
Returns:
List of relevant memory dictionaries
"""
try:
logger.debug(f"Retrieving memories for query: {query}")
id_pairs = [("user_id", self.user_id), ("agent_id", self.agent_id), ("run_id", self.run_id)]
clauses = [{name: value} for name, value in id_pairs if value is not None]
filters = {"AND": clauses} if clauses else {}
results = self.memory_client.search(
query=query,
filters=filters,
version=self.api_version,
top_k=self.search_limit,
threshold=self.search_threshold,
)
logger.debug(f"Retrieved {len(results)} memories from Mem0")
return results
except Exception as e:
logger.error(f"Error retrieving memories from Mem0: {e}")
return []
def _enhance_context_with_memories(self, context: OpenAILLMContext, query: str):
"""Enhance the LLM context with relevant memories.
Args:
context: The OpenAILLMContext to enhance
query: The query to search for relevant memories
"""
# Skip if this is the same query we just processed
if self.last_query == query:
return
self.last_query = query
memories = self._retrieve_memories(query)
if not memories:
return
# Format memories as a message
memory_text = self.system_prompt
for i, memory in enumerate(memories, 1):
memory_text += f"{i}. {memory.get('memory', '')}\n\n"
# Add memories as a system message or user message based on configuration
if self.add_as_system_message:
context.add_message({"role": "system", "content": memory_text})
else:
# Add as a user message that provides context
context.add_message({"role": "user", "content": memory_text})
logger.debug(f"Enhanced context with {len(memories)} memories")
async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames, intercept context frames for memory integration.
Args:
frame: The incoming frame to process
direction: The direction of frame flow in the pipeline
"""
await super().process_frame(frame, direction)
context = None
messages = None
if isinstance(frame, OpenAILLMContextFrame):
context = frame.context
elif isinstance(frame, LLMMessagesFrame):
messages = frame.messages
context = OpenAILLMContext.from_messages(messages)
if context:
try:
# Get the latest user message to use as a query for memory retrieval
context_messages = context.get_messages()
latest_user_message = None
for message in reversed(context_messages):
if message.get("role") == "user" and isinstance(message.get("content"), str):
latest_user_message = message.get("content")
break
if latest_user_message:
# Enhance context with memories before passing it downstream
self._enhance_context_with_memories(context, latest_user_message)
# Store the conversation in Mem0. Only call this when user message is detected
self._store_messages(context_messages)
# If we received an LLMMessagesFrame, create a new one with the enhanced messages
if messages is not None:
await self.push_frame(LLMMessagesFrame(context.get_messages()))
else:
# Otherwise, pass the enhanced context frame downstream
await self.push_frame(frame)
except Exception as e:
logger.error(f"Error processing with Mem0: {str(e)}")
await self.push_frame(ErrorFrame(f"Error processing with Mem0: {str(e)}"))
await self.push_frame(frame) # Still pass the original frame through
else:
# For non-context frames, just pass them through
await self.push_frame(frame, direction)