wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'pipecat-ai:main' into mcp-http-gemini-support

Browse Source
This commit is contained in:
Yousif
2025-07-01 23:54:42 -07:00
committed by GitHub
parent 58675f4d5a 05753fb207
commit 09aa168840
236 changed files with 13728 additions and 2205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
examples/foundational/07n-interruptible-google.py
View File

@@ -61,7 +61,12 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si
credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"),
)
llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"))
llm = GoogleLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
model="gemini-2.5-flash",
# turn on thinking if you want it
# params=GoogleLLMService.InputParams(extra={"thinking_config": {"thinking_budget": 4096}}),)
)
messages = [
{

7
examples/foundational/07s-interruptible-google-audio-in.py
View File

@@ -214,7 +214,12 @@ transport_params = {
async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool):
logger.info(f"Starting bot")
llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001")
llm = GoogleLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
model="gemini-2.5-flash",
# turn on thinking if you want it
# params=GoogleLLMService.InputParams(extra={"thinking_config": {"thinking_budget": 4096}}),
)
tts = GoogleTTSService(
voice_id="en-US-Chirp3-HD-Charon",

146
examples/foundational/14t-function-calling-direct.py Normal file
View File

@@ -0,0 +1,146 @@
#
# Copyright (c) 2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import argparse
import os
from dotenv import load_dotenv
from loguru import logger
from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import TTSSpeakFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
from pipecat.services.llm_service import FunctionCallParams
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams
from pipecat.transports.services.daily import DailyParams
load_dotenv(override=True)
async def get_current_weather(params: FunctionCallParams, location: str, format: str):
"""
Get the current weather.
Args:
location (str): The city and state, e.g. "San Francisco, CA".
format (str): The temperature unit to use. Must be either "celsius" or "fahrenheit". Infer this from the user's location.
"""
await params.result_callback({"conditions": "nice", "temperature": "75"})
async def get_restaurant_recommendation(params: FunctionCallParams, location: str):
"""
Get a restaurant recommendation.
Args:
location (str): The city and state, e.g. "San Francisco, CA".
"""
await params.result_callback({"name": "The Golden Dragon"})
# We store functions so objects (e.g. SileroVADAnalyzer) don't get
# instantiated. The function will be called when the desired transport gets
# selected.
transport_params = {
"daily": lambda: DailyParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
"twilio": lambda: FastAPIWebsocketParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
"webrtc": lambda: TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
}
async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool):
logger.info(f"Starting bot")
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
)
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
# You can also register a function_name of None to get all functions
# sent to the same callback with an additional function_name parameter.
llm.register_direct_function(get_current_weather)
llm.register_direct_function(get_restaurant_recommendation)
@llm.event_handler("on_function_calls_started")
async def on_function_calls_started(service, function_calls):
await tts.queue_frame(TTSSpeakFrame("Let me check on that."))
tools = ToolsSchema(standard_tools=[get_current_weather, get_restaurant_recommendation])
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
},
]
context = OpenAILLMContext(messages, tools)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(),
stt,
context_aggregator.user(),
llm,
tts,
transport.output(),
context_aggregator.assistant(),
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
enable_metrics=True,
enable_usage_metrics=True,
),
)
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()])
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
await task.cancel()
runner = PipelineRunner(handle_sigint=handle_sigint)
await runner.run(task)
if __name__ == "__main__":
from pipecat.examples.run import main
main(run_example, transport_params=transport_params)

242
examples/foundational/26f-gemini-multimodal-live-files-api.py Normal file
View File

@@ -0,0 +1,242 @@
#
# Copyright (c) 2024–2025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import argparse
import os
import tempfile
from dotenv import load_dotenv
from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.gemini_multimodal_live.gemini import (
GeminiMultimodalLiveContext,
GeminiMultimodalLiveLLMService,
)
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.services.daily import DailyParams
load_dotenv(override=True)
# We store functions so objects (e.g. SileroVADAnalyzer) don't get
# instantiated. The function will be called when the desired transport gets
# selected.
transport_params = {
"daily": lambda: DailyParams(
audio_in_enabled=True,
audio_out_enabled=True,
video_in_enabled=False,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
),
"twilio": lambda: FastAPIWebsocketParams(
audio_in_enabled=True,
audio_out_enabled=True,
video_in_enabled=False,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
),
"webrtc": lambda: TransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
video_in_enabled=False,
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)),
),
}
sample_file_path = ""
async def create_sample_file():
if sample_file_path:
return sample_file_path
else:
"""Create a sample text file for testing the File API."""
content = """# Sample Document for Gemini File API Test
This is a test document to demonstrate the Gemini File API functionality.
## Key Information:
- This document was created for testing purposes
- It contains information about AI assistants
- The document should be analyzed by Gemini
- The secret phrase for the test is "Pineapple Pizza"
## AI Assistant Capabilities:
1. Natural language processing
2. File analysis and understanding
3. Context-aware conversations
4. Multi-modal interactions
## Conclusion:
This document serves as a test case for the Gemini File API integration with Pipecat.
The AI should be able to reference and discuss the contents of this file.
"""
# Create a temporary file
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
f.write(content)
return f.name
async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool):
logger.info(f"Starting File API bot")
# Create a sample file to upload
sample_file_path = await create_sample_file()
logger.info(f"Created sample file: {sample_file_path}")
system_instruction = """
You are a helpful AI assistant with access to a document that has been uploaded for analysis.
The document contains test information.
You should be able to:
- Reference and discuss the contents of the uploaded document
- Answer questions about what's in the document
- Use the information from the document in our conversation
Your output will be converted to audio so don't include special characters in your answers.
Be friendly and demonstrate your ability to work with the uploaded file.
"""
# Initialize Gemini service with File API support
llm = GeminiMultimodalLiveLLMService(
api_key=os.getenv("GOOGLE_API_KEY"),
system_instruction=system_instruction,
voice_id="Charon", # Aoede, Charon, Fenrir, Kore, Puck
transcribe_user_audio=True,
)
# Upload the sample file to Gemini File API
logger.info("Uploading file to Gemini File API...")
file_info = None
try:
file_info = await llm.file_api.upload_file(
sample_file_path, display_name="Sample Test Document"
)
logger.info(f"File uploaded successfully: {file_info['file']['name']}")
# Get file URI and mime type
file_uri = file_info["file"]["uri"]
mime_type = "text/plain"
# Create context with file reference
context = OpenAILLMContext(
[
{
"role": "user",
"content": [
{
"type": "text",
"text": "Greet the user and let them know you have access to a document they can ask you about. Mention that you can discuss its contents.",
},
{
"type": "file_data",
"file_data": {"mime_type": mime_type, "file_uri": file_uri},
},
],
}
]
)
logger.info("File reference added to conversation context")
except Exception as e:
logger.error(f"Error uploading file: {e}")
# Continue with a basic context if file upload fails
context = OpenAILLMContext(
[
{
"role": "user",
"content": "Greet the user and explain that there was an issue with file upload, but you're ready to help with other tasks.",
}
]
)
# Create context aggregator
context_aggregator = llm.create_context_aggregator(context)
# Build the pipeline
pipeline = Pipeline(
[
transport.input(),
context_aggregator.user(),
llm,
transport.output(),
context_aggregator.assistant(),
]
)
# Configure the pipeline task
task = PipelineTask(
pipeline,
params=PipelineParams(
allow_interruptions=True,
enable_metrics=True,
enable_usage_metrics=True,
),
)
# Handle client connection event
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected")
# Kick off the conversation using standard context frame
await task.queue_frames([context_aggregator.user().get_context_frame()])
# Handle client disconnection events
@transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client):
logger.info(f"Client disconnected")
@transport.event_handler("on_client_closed")
async def on_client_closed(transport, client):
logger.info(f"Client closed connection")
await task.cancel()
# Run the pipeline
runner = PipelineRunner(handle_sigint=False)
await runner.run(task)
# Clean up: delete the uploaded file and temporary file
if file_info:
try:
await llm.file_api.delete_file(file_info["file"]["name"])
logger.info("Cleaned up uploaded file from Gemini")
except Exception as e:
logger.error(f"Error cleaning up file: {e}")
# Remove temporary file
try:
os.unlink(sample_file_path)
logger.info("Cleaned up temporary file")
except Exception as e:
logger.error(f"Error removing temporary file: {e}")
if __name__ == "__main__":
from pipecat.examples.run import main
upload_example_file = input("""
Please pass in a TEXT filepath to test upload.
NOTE: Files are stored on Google's servers for 48 hours.
Press Enter to use a default test file.
text filepath : """)
if upload_example_file:
print(f"Uploading file: {upload_example_file}")
sample_file_path = upload_example_file.strip()
else:
print(f"Using default file")
main(run_example, transport_params=transport_params)

1
examples/foundational/40-aws-nova-sonic.py
View File

@@ -102,6 +102,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si
secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
region=os.getenv("AWS_REGION"), # as of 2025-05-06, us-east-1 is the only supported region
session_token=os.getenv("AWS_SESSION_TOKEN"),
voice_id="tiffany", # matthew, tiffany, amy
# you could choose to pass instruction here rather than via context
# system_instruction=system_instruction

2
examples/foundational/42-interruption-config.py
View File

@@ -10,8 +10,8 @@ import os
from dotenv import load_dotenv
from loguru import logger
from pipecat.audio.interruptions.min_words_interruption_strategy import MinWordsInterruptionStrategy
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import MinWordsInterruptionStrategy
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask

12
examples/freeze-test/client/src/app.ts
View File

@@ -191,7 +191,17 @@ class WebsocketClientApp {
const startTime = Date.now();
this.recordingSerializer = new RecordingSerializer()
const transport = this.ENABLE_RECORDING_MODE ? new WebSocketTransport({serializer: this.recordingSerializer}) : new WebSocketTransport();
const transport = this.ENABLE_RECORDING_MODE ?
new WebSocketTransport({
serializer: this.recordingSerializer,
recorderSampleRate: 8000,
playerSampleRate:8000
}) :
new WebSocketTransport({
serializer: new ProtobufFrameSerializer(),
recorderSampleRate: 8000,
playerSampleRate:8000
});
this.websocketTransport = transport
const RTVIConfig: RTVIClientOptions = {

4
examples/freeze-test/env.example Normal file
View File

@@ -0,0 +1,4 @@
SENTRY_DSN=
DEEPGRAM_API_KEY=
CARTESIA_API_KEY=
OPENAI_API_KEY=

45
examples/freeze-test/freeze_test_bot.py
View File

@@ -18,7 +18,6 @@ from fastapi import FastAPI, Request, WebSocket
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import RedirectResponse
from loguru import logger
from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import (
@@ -27,11 +26,13 @@ from pipecat.frames.frames import (
Frame,
InterimTranscriptionFrame,
LLMFullResponseEndFrame,
LLMMessagesFrame,
StartFrame,
StartInterruptionFrame,
StopFrame,
StopInterruptionFrame,
TranscriptionFrame,
TTSSpeakFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
@@ -47,6 +48,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIProcessor
from pipecat.processors.metrics.sentry import SentryMetrics
from pipecat.processors.user_idle_processor import UserIdleProcessor
from pipecat.serializers.protobuf import ProtobufFrameSerializer
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.services.deepgram.stt import DeepgramSTTService
@@ -78,9 +80,6 @@ app.add_middleware(
allow_headers=["*"],
)
# Mount the frontend at /
app.mount("/client", SmallWebRTCPrebuiltUI)
class SimulateFreezeInput(FrameProcessor):
def __init__(
@@ -188,6 +187,37 @@ async def run_example(websocket_client):
stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
async def handle_user_idle(user_idle: UserIdleProcessor, retry_count: int) -> bool:
if retry_count == 1:
# First attempt: Add a gentle prompt to the conversation
messages.append(
{
"role": "system",
"content": "The user has been quiet. Politely and briefly ask if they're still there.",
}
)
await user_idle.push_frame(LLMMessagesFrame(messages))
return True
elif retry_count == 2:
# Second attempt: More direct prompt
messages.append(
{
"role": "system",
"content": "The user is still inactive. Ask if they'd like to continue our conversation.",
}
)
await user_idle.push_frame(LLMMessagesFrame(messages))
return True
else:
# Third attempt: End the conversation
await user_idle.push_frame(
TTSSpeakFrame("It seems like you're busy right now. Have a nice day!")
)
await task.queue_frame(EndFrame())
return False
user_idle = UserIdleProcessor(callback=handle_user_idle, timeout=10.0)
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
@@ -222,6 +252,7 @@ async def run_example(websocket_client):
stt,
],
),
user_idle,
rtvi,
context_aggregator.user(), # User responses
llm, # LLM
@@ -238,6 +269,8 @@ async def run_example(websocket_client):
enable_metrics=True,
enable_usage_metrics=True,
report_only_initial_ttfb=True,
audio_in_sample_rate=8000,
audio_out_sample_rate=8000,
),
idle_timeout_secs=120,
observers=[
@@ -249,6 +282,10 @@ async def run_example(websocket_client):
# LLMTextFrame: None,
OpenAILLMContextFrame: None,
LLMFullResponseEndFrame: None,
UserStartedSpeakingFrame: None,
UserStoppedSpeakingFrame: None,
StartInterruptionFrame: None,
StopInterruptionFrame: None,
},
exclude_fields={
"result",

4
examples/freeze-test/requirements.txt Normal file
View File

@@ -0,0 +1,4 @@
python-dotenv
fastapi[all]
uvicorn
pipecat-ai[silero,websocket,openai, deepgram, cartesia, sentry]