wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into mcp-http-gemini-support

Browse Source
This commit is contained in:
Vanessa Pyne
2025-07-02 11:47:18 -05:00
committed by GitHub
parent 18817fd81b 2902362886
commit d80e228c6f
16 changed files with 63 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/daily-multi-translation/requirements.txt
View File

@@ -2,4 +2,4 @@ aiofiles
python-dotenv
fastapi[all]
uvicorn
pipecat-ai[daily,deepgram,openai,silero,cartesia]
pipecat-ai[daily,deepgram,openai,silero,cartesia,soundfile]

2
examples/open-telemetry/jaeger/requirements.txt
View File

@@ -1,6 +1,6 @@
fastapi
uvicorn
python-dotenv
pipecat-ai[webrtc,silero,cartesia,deepgram,openai,tracing]
pipecat-ai[daily,webrtc,silero,cartesia,deepgram,openai,tracing]
pipecat-ai-small-webrtc-prebuilt
opentelemetry-exporter-otlp-proto-grpc

2
examples/open-telemetry/langfuse/requirements.txt
View File

@@ -1,6 +1,6 @@
fastapi
uvicorn
python-dotenv
pipecat-ai[webrtc,silero,cartesia,deepgram,openai,tracing]
pipecat-ai[daily,webrtc,silero,cartesia,deepgram,openai,tracing]
pipecat-ai-small-webrtc-prebuilt
opentelemetry-exporter-otlp-proto-http

2
examples/phone-chatbot/daily-twilio-sip-dial-in/requirements.txt
View File

@@ -1,4 +1,4 @@
pipecat-ai[daily,elevenlabs,openai,silero]
pipecat-ai[daily,cartesia,openai,silero]
fastapi==0.115.6
uvicorn
python-dotenv

23
examples/simple-chatbot/client/ios/SimpleChatbot.xcodeproj/project.pbxproj
View File

@@ -15,7 +15,6 @@
90031FC22C616EE900408370 /* SimpleChatbotUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90031FC12C616EE900408370 /* SimpleChatbotUITests.swift */; };
90031FC42C616EE900408370 /* SimpleChatbotUITestsLaunchTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90031FC32C616EE900408370 /* SimpleChatbotUITestsLaunchTests.swift */; };
90031FDC2C6D5DD700408370 /* ToastModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90031FDB2C6D5DD700408370 /* ToastModifier.swift */; };
907C98842D37E6AF0079441F /* PipecatClientIOSDaily in Frameworks */ = {isa = PBXBuildFile; productRef = 907C98832D37E6AF0079441F /* PipecatClientIOSDaily */; };
90ABB98E2C735ED6000D9CC7 /* MeetingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB98D2C735ED6000D9CC7 /* MeetingView.swift */; };
90ABB9902C736A8B000D9CC7 /* WaveformView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB98F2C736A8B000D9CC7 /* WaveformView.swift */; };
90ABB9932C73820D000D9CC7 /* MicrophoneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9922C73820D000D9CC7 /* MicrophoneView.swift */; };
@@ -25,6 +24,8 @@
90ABB9A32C74E1CE000D9CC7 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9A22C74E1CE000D9CC7 /* SettingsView.swift */; };
90ABB9A62C74EA8A000D9CC7 /* SettingsPreference.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9A52C74EA8A000D9CC7 /* SettingsPreference.swift */; };
90ABB9A82C74EAB1000D9CC7 /* SettingsManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9A72C74EAB1000D9CC7 /* SettingsManager.swift */; };
90CC98B02E158093003C2706 /* PipecatClientIOSDaily in Frameworks */ = {isa = PBXBuildFile; productRef = 90CC98AF2E158093003C2706 /* PipecatClientIOSDaily */; };
90CC98B62E15820B003C2706 /* PipecatClientIOSDaily in Frameworks */ = {isa = PBXBuildFile; productRef = 90CC98B52E15820B003C2706 /* PipecatClientIOSDaily */; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
@@ -73,7 +74,8 @@
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
907C98842D37E6AF0079441F /* PipecatClientIOSDaily in Frameworks */,
90CC98B62E15820B003C2706 /* PipecatClientIOSDaily in Frameworks */,
90CC98B02E158093003C2706 /* PipecatClientIOSDaily in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -218,7 +220,8 @@
);
name = SimpleChatbot;
packageProductDependencies = (
907C98832D37E6AF0079441F /* PipecatClientIOSDaily */,
90CC98AF2E158093003C2706 /* PipecatClientIOSDaily */,
90CC98B52E15820B003C2706 /* PipecatClientIOSDaily */,
);
productName = SimpleChatbot;
productReference = 90031FA32C616EE700408370 /* SimpleChatbot.app */;
@@ -293,7 +296,7 @@
);
mainGroup = 90031F9A2C616EE700408370;
packageReferences = (
907C98822D37E6AF0079441F /* XCRemoteSwiftPackageReference "pipecat-client-ios-daily" */,
90CC98B42E15820B003C2706 /* XCRemoteSwiftPackageReference "pipecat-client-ios-daily" */,
);
productRefGroup = 90031FA42C616EE700408370 /* Products */;
projectDirPath = "";
@@ -682,20 +685,24 @@
/* End XCConfigurationList section */
/* Begin XCRemoteSwiftPackageReference section */
907C98822D37E6AF0079441F /* XCRemoteSwiftPackageReference "pipecat-client-ios-daily" */ = {
90CC98B42E15820B003C2706 /* XCRemoteSwiftPackageReference "pipecat-client-ios-daily" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/pipecat-ai/pipecat-client-ios-daily/";
requirement = {
kind = upToNextMajorVersion;
minimumVersion = 0.3.2;
minimumVersion = 0.3.6;
};
};
/* End XCRemoteSwiftPackageReference section */
/* Begin XCSwiftPackageProductDependency section */
907C98832D37E6AF0079441F /* PipecatClientIOSDaily */ = {
90CC98AF2E158093003C2706 /* PipecatClientIOSDaily */ = {
isa = XCSwiftPackageProductDependency;
package = 907C98822D37E6AF0079441F /* XCRemoteSwiftPackageReference "pipecat-client-ios-daily" */;
productName = PipecatClientIOSDaily;
};
90CC98B52E15820B003C2706 /* PipecatClientIOSDaily */ = {
isa = XCSwiftPackageProductDependency;
package = 90CC98B42E15820B003C2706 /* XCRemoteSwiftPackageReference "pipecat-client-ios-daily" */;
productName = PipecatClientIOSDaily;
};
/* End XCSwiftPackageProductDependency section */

15
examples/simple-chatbot/client/ios/SimpleChatbot.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
View File

@@ -1,12 +1,13 @@
{
"originHash" : "cc17f08b06def9570d775e9c6f7a8dc10d1588b98127e977c47d052abac659b7",
"pins" : [
{
"identity" : "daily-client-ios",
"kind" : "remoteSourceControl",
"location" : "https://github.com/daily-co/daily-client-ios.git",
"state" : {
"revision" : "15804ce495780da3ec2d05ab99736315f7bfbd24",
"version" : "0.28.0"
"revision" : "431938db25e5807120e89e2dc5bab1c076729f59",
"version" : "0.31.0"
}
},
{
@@ -14,8 +15,8 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/pipecat-ai/pipecat-client-ios.git",
"state" : {
"revision" : "c679512e367002a1a67da85d503fec72d9b17191",
"version" : "0.3.2"
"revision" : "f92b5e68e56a8311f7d8ead68a7a5674843cbc40",
"version" : "0.3.6"
}
},
{
@@ -23,10 +24,10 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/pipecat-ai/pipecat-client-ios-daily/",
"state" : {
"revision" : "a337fe6642c52376d2f90eafcb965f5be772ce72",
"version" : "0.3.2"
"revision" : "8f494da903192c22c367ecf9e51248c9b651fbc6",
"version" : "0.3.6"
}
}
],
"version" : 2
"version" : 3
}

7
examples/simple-chatbot/client/ios/SimpleChatbot/model/CallContainerModel.swift
View File

@@ -78,10 +78,11 @@ class CallContainerModel: ObservableObject {
self.saveCredentials(backendURL: baseUrl)
}
@MainActor
func disconnect() {
self.rtviClientIOS?.disconnect(completion: nil)
self.rtviClientIOS?.release()
Task { @MainActor in
try await self.rtviClientIOS?.disconnect()
self.rtviClientIOS?.release()
}
}
func showError(message: String) {

4
src/pipecat/audio/vad/silero.py
View File

@@ -46,10 +46,6 @@ class SileroOnnxModel:
path: Path to the ONNX model file.
force_onnx_cpu: Whether to force CPU execution provider.
"""
import numpy as np
global np
opts = onnxruntime.SessionOptions()
opts.inter_op_num_threads = 1
opts.intra_op_num_threads = 1

5
src/pipecat/observers/loggers/debug_log_observer.py
View File

@@ -47,7 +47,7 @@ class DebugLogObserver(BaseObserver):
Log specific frame types from any source/destination::
from pipecat.frames.frames import TranscriptionFrame, InterimTranscriptionFrame
from pipecat.frames.frames import LLMTextFrame, TranscriptionFrame
observers=[
DebugLogObserver(frame_types=(LLMTextFrame,TranscriptionFrame,)),
]
@@ -55,7 +55,8 @@ class DebugLogObserver(BaseObserver):
Log frames with specific source/destination filters::
from pipecat.frames.frames import StartInterruptionFrame, UserStartedSpeakingFrame, LLMTextFrame
from pipecat.transports.base_output_transport import BaseOutputTransport
from pipecat.observers.loggers.debug_log_observer import DebugLogObserver, FrameEndpoint
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.services.stt_service import STTService
observers=[

23
src/pipecat/processors/aggregators/gated.py
View File

@@ -26,29 +26,6 @@ class GatedAggregator(FrameProcessor):
until and not including the gate-closed frame. The aggregator maintains an
internal gate state that controls whether frames are passed through immediately
or accumulated for later release.
Doctest: FIXME to work with asyncio
>>> from pipecat.frames.frames import ImageRawFrame
>>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame):
... if isinstance(frame, TextFrame):
... print(frame.text)
... else:
... print(frame.__class__.__name__)
>>> aggregator = GatedAggregator(
... gate_close_fn=lambda x: isinstance(x, LLMResponseStartFrame),
... gate_open_fn=lambda x: isinstance(x, ImageRawFrame),
... start_open=False)
>>> asyncio.run(print_frames(aggregator, TextFrame("Hello")))
>>> asyncio.run(print_frames(aggregator, TextFrame("Hello again.")))
>>> asyncio.run(print_frames(aggregator, ImageRawFrame(image=bytes([]), size=(0, 0))))
ImageRawFrame
Hello
Hello again.
>>> asyncio.run(print_frames(aggregator, TextFrame("Goodbye.")))
Goodbye.
"""
def __init__(

14
src/pipecat/processors/aggregators/sentence.py
View File

@@ -23,20 +23,10 @@ class SentenceAggregator(FrameProcessor):
Useful for ensuring downstream processors receive coherent, complete sentences
rather than fragmented text.
Frame input/output:
Frame input/output::
TextFrame("Hello,") -> None
TextFrame(" world.") -> TextFrame("Hello, world.")
Doctest: FIXME to work with asyncio
>>> import asyncio
>>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame):
... print(frame.text)
>>> aggregator = SentenceAggregator()
>>> asyncio.run(print_frames(aggregator, TextFrame("Hello,")))
>>> asyncio.run(print_frames(aggregator, TextFrame(" world.")))
Hello, world.
"""
def __init__(self):

11
src/pipecat/processors/aggregators/vision_image_frame.py
View File

@@ -20,17 +20,6 @@ class VisionImageFrameAggregator(FrameProcessor):
This aggregator waits for a consecutive TextFrame and an InputImageRawFrame.
After the InputImageRawFrame arrives it will output a VisionImageRawFrame
combining both the text and image data for multimodal processing.
>>> from pipecat.frames.frames import ImageFrame
>>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame):
... print(frame)
>>> aggregator = VisionImageFrameAggregator()
>>> asyncio.run(print_frames(aggregator, TextFrame("What do you see?")))
>>> asyncio.run(print_frames(aggregator, ImageFrame(image=bytes([]), size=(0, 0))))
VisionImageFrame, text: What do you see?, image size: 0x0, buffer size: 0 B
"""
def __init__(self):

8
src/pipecat/processors/text_transformer.py
View File

@@ -18,14 +18,6 @@ class StatelessTextTransformer(FrameProcessor):
This processor intercepts TextFrame objects and applies a user-provided
transformation function to the text content. The function can be either
synchronous or asynchronous (coroutine).
>>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame):
... print(frame.text)
>>> aggregator = StatelessTextTransformer(lambda x: x.upper())
>>> asyncio.run(print_frames(aggregator, TextFrame("Hello")))
HELLO
"""
def __init__(

6
src/pipecat/services/gemini_multimodal_live/file_api.py
View File

@@ -6,9 +6,9 @@
"""Gemini File API client for uploading and managing files.
This module provides the GeminiFileAPI class for interacting with Google's Gemini File API,
including uploading, fetching, listing, and deleting files that can be used with Gemini
generative models.
This module provides a client for Google's Gemini File API, enabling file
uploads, metadata retrieval, listing, and deletion. Files uploaded through
this API can be referenced in Gemini generative model calls.
"""
import mimetypes

26
src/pipecat/services/gemini_multimodal_live/gemini.py
View File

@@ -382,7 +382,14 @@ class GeminiMultimodalModalities(Enum):
class GeminiMediaResolution(str, Enum):
"""Media resolution options for Gemini Multimodal Live."""
"""Media resolution options for Gemini Multimodal Live.
Parameters:
UNSPECIFIED: Use default resolution setting.
LOW: Low resolution with 64 tokens.
MEDIUM: Medium resolution with 256 tokens.
HIGH: High resolution with zoomed reframing and 256 tokens.
"""
UNSPECIFIED = "MEDIA_RESOLUTION_UNSPECIFIED" # Use default
LOW = "MEDIA_RESOLUTION_LOW" # 64 tokens
@@ -499,7 +506,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
params: Configuration parameters for the model. Defaults to InputParams().
inference_on_context_initialization: Whether to generate a response when context
is first set. Defaults to True.
file_api_base_url: Base URL for the file API. Defaults to the official Gemini Live endpoint.
file_api_base_url: Base URL for the Gemini File API. Defaults to the official endpoint.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(base_url=base_url, **kwargs)
@@ -772,6 +779,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self._ws_send(event.model_dump(exclude_none=True))
async def _connect(self):
"""Establish WebSocket connection to Gemini Live API."""
if self._websocket:
# Here we assume that if we have a websocket, we are connected. We
# handle disconnections in the send/recv code paths.
@@ -876,6 +884,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._websocket = None
async def _disconnect(self):
"""Disconnect from Gemini Live API and clean up resources."""
logger.info("Disconnecting from Gemini service")
try:
self._disconnecting = True
@@ -892,6 +901,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
logger.error(f"{self} error disconnecting: {e}")
async def _ws_send(self, message):
"""Send a message to the WebSocket connection."""
# logger.debug(f"Sending message to websocket: {message}")
try:
if self._websocket:
@@ -912,6 +922,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
#
async def _receive_task_handler(self):
"""Handle incoming messages from the WebSocket connection."""
async for message in WatchdogAsyncIterator(self._websocket, manager=self.task_manager):
evt = events.parse_server_event(message)
# logger.debug(f"Received event: {message[:500]}")
@@ -940,6 +951,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
#
async def _send_user_audio(self, frame):
"""Send user audio frame to Gemini Live API."""
if self._audio_input_paused:
return
# Send all audio to Gemini
@@ -956,6 +968,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._user_audio_buffer = self._user_audio_buffer[-length:]
async def _send_user_video(self, frame):
"""Send user video frame to Gemini Live API."""
if self._video_input_paused:
return
@@ -969,6 +982,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self.send_client_event(evt)
async def _create_initial_response(self):
"""Create initial response based on context history."""
if not self._api_session_ready:
self._run_llm_when_api_session_ready = True
return
@@ -994,6 +1008,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
self._needs_turn_complete_message = True
async def _create_single_response(self, messages_list):
"""Create a single response from a list of messages."""
# Refactor to combine this logic with same logic in GeminiMultimodalLiveContext
messages = []
for item in messages_list:
@@ -1046,6 +1061,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
@traced_gemini_live(operation="llm_tool_result")
async def _tool_result(self, tool_result_message):
"""Send tool result back to the API."""
# For now we're shoving the name into the tool_call_id field, so this
# will work until we revisit that.
id = tool_result_message.get("tool_call_id")
@@ -1071,6 +1087,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
@traced_gemini_live(operation="llm_setup")
async def _handle_evt_setup_complete(self, evt):
"""Handle the setup complete event."""
# If this is our first context frame, run the LLM
self._api_session_ready = True
# Now that we've configured the session, we can run the LLM if we need to.
@@ -1079,6 +1096,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self._create_initial_response()
async def _handle_evt_model_turn(self, evt):
"""Handle the model turn event."""
part = evt.serverContent.modelTurn.parts[0]
if not part:
return
@@ -1120,6 +1138,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
@traced_gemini_live(operation="llm_tool_call")
async def _handle_evt_tool_call(self, evt):
"""Handle tool call events."""
function_calls = evt.toolCall.functionCalls
if not function_calls:
return
@@ -1140,6 +1159,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
@traced_gemini_live(operation="llm_response")
async def _handle_evt_turn_complete(self, evt):
"""Handle the turn complete event."""
self._bot_is_speaking = False
text = self._bot_text_buffer
@@ -1223,6 +1243,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
)
async def _handle_evt_output_transcription(self, evt):
"""Handle the output transcription event."""
if not evt.serverContent.outputTranscription:
return
@@ -1241,6 +1262,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
await self.push_frame(TTSTextFrame(text=text))
async def _handle_evt_usage_metadata(self, evt):
"""Handle the usage metadata event."""
if not evt.usageMetadata:
return

19
src/pipecat/utils/utils.py
View File

@@ -25,15 +25,6 @@ def obj_id() -> int:
Returns:
A unique integer identifier that increments globally across all objects.
Examples::
>>> obj_id()
0
>>> obj_id()
1
>>> obj_id()
2
"""
with _ID_LOCK:
return next(_ID)
@@ -47,16 +38,6 @@ def obj_count(obj) -> int:
Returns:
A unique integer count that increments per class type.
Examples::
>>> obj_count(object())
0
>>> obj_count(object())
1
>>> new_type = type('NewType', (object,), {})
>>> obj_count(new_type())
0
"""
with _COUNTS_LOCK:
return next(_COUNTS[obj.__class__.__name__])