Merge branch 'main' into mcp-http-gemini-support

examples/daily-multi-translation/requirements.txt

@@ -2,4 +2,4 @@ aiofiles
 python-dotenv
 fastapi[all]
 uvicorn
-pipecat-ai[daily,deepgram,openai,silero,cartesia]
+pipecat-ai[daily,deepgram,openai,silero,cartesia,soundfile]

examples/open-telemetry/jaeger/requirements.txt

@@ -1,6 +1,6 @@
 fastapi
 uvicorn
 python-dotenv
-pipecat-ai[webrtc,silero,cartesia,deepgram,openai,tracing]
+pipecat-ai[daily,webrtc,silero,cartesia,deepgram,openai,tracing]
 pipecat-ai-small-webrtc-prebuilt
 opentelemetry-exporter-otlp-proto-grpc

examples/open-telemetry/langfuse/requirements.txt

@@ -1,6 +1,6 @@
 fastapi
 uvicorn
 python-dotenv
-pipecat-ai[webrtc,silero,cartesia,deepgram,openai,tracing]
+pipecat-ai[daily,webrtc,silero,cartesia,deepgram,openai,tracing]
 pipecat-ai-small-webrtc-prebuilt
 opentelemetry-exporter-otlp-proto-http

examples/phone-chatbot/daily-twilio-sip-dial-in/requirements.txt

@@ -1,4 +1,4 @@
-pipecat-ai[daily,elevenlabs,openai,silero]
+pipecat-ai[daily,cartesia,openai,silero]
 fastapi==0.115.6
 uvicorn
 python-dotenv

examples/simple-chatbot/client/ios/SimpleChatbot.xcodeproj/project.pbxproj

@@ -15,7 +15,6 @@
 		90031FC22C616EE900408370 /* SimpleChatbotUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90031FC12C616EE900408370 /* SimpleChatbotUITests.swift */; };
 		90031FC42C616EE900408370 /* SimpleChatbotUITestsLaunchTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90031FC32C616EE900408370 /* SimpleChatbotUITestsLaunchTests.swift */; };
 		90031FDC2C6D5DD700408370 /* ToastModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90031FDB2C6D5DD700408370 /* ToastModifier.swift */; };
-		907C98842D37E6AF0079441F /* PipecatClientIOSDaily in Frameworks */ = {isa = PBXBuildFile; productRef = 907C98832D37E6AF0079441F /* PipecatClientIOSDaily */; };
 		90ABB98E2C735ED6000D9CC7 /* MeetingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB98D2C735ED6000D9CC7 /* MeetingView.swift */; };
 		90ABB9902C736A8B000D9CC7 /* WaveformView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB98F2C736A8B000D9CC7 /* WaveformView.swift */; };
 		90ABB9932C73820D000D9CC7 /* MicrophoneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9922C73820D000D9CC7 /* MicrophoneView.swift */; };
@@ -25,6 +24,8 @@
 		90ABB9A32C74E1CE000D9CC7 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9A22C74E1CE000D9CC7 /* SettingsView.swift */; };
 		90ABB9A62C74EA8A000D9CC7 /* SettingsPreference.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9A52C74EA8A000D9CC7 /* SettingsPreference.swift */; };
 		90ABB9A82C74EAB1000D9CC7 /* SettingsManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9A72C74EAB1000D9CC7 /* SettingsManager.swift */; };
+		90CC98B02E158093003C2706 /* PipecatClientIOSDaily in Frameworks */ = {isa = PBXBuildFile; productRef = 90CC98AF2E158093003C2706 /* PipecatClientIOSDaily */; };
+		90CC98B62E15820B003C2706 /* PipecatClientIOSDaily in Frameworks */ = {isa = PBXBuildFile; productRef = 90CC98B52E15820B003C2706 /* PipecatClientIOSDaily */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -73,7 +74,8 @@
 			isa = PBXFrameworksBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
-				907C98842D37E6AF0079441F /* PipecatClientIOSDaily in Frameworks */,
+				90CC98B62E15820B003C2706 /* PipecatClientIOSDaily in Frameworks */,
+				90CC98B02E158093003C2706 /* PipecatClientIOSDaily in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -218,7 +220,8 @@
 			);
 			name = SimpleChatbot;
 			packageProductDependencies = (
-				907C98832D37E6AF0079441F /* PipecatClientIOSDaily */,
+				90CC98AF2E158093003C2706 /* PipecatClientIOSDaily */,
+				90CC98B52E15820B003C2706 /* PipecatClientIOSDaily */,
 			);
 			productName = SimpleChatbot;
 			productReference = 90031FA32C616EE700408370 /* SimpleChatbot.app */;
@@ -293,7 +296,7 @@
 			);
 			mainGroup = 90031F9A2C616EE700408370;
 			packageReferences = (
-				907C98822D37E6AF0079441F /* XCRemoteSwiftPackageReference "pipecat-client-ios-daily" */,
+				90CC98B42E15820B003C2706 /* XCRemoteSwiftPackageReference "pipecat-client-ios-daily" */,
 			);
 			productRefGroup = 90031FA42C616EE700408370 /* Products */;
 			projectDirPath = "";
@@ -682,20 +685,24 @@
 /* End XCConfigurationList section */
 
 /* Begin XCRemoteSwiftPackageReference section */
-		907C98822D37E6AF0079441F /* XCRemoteSwiftPackageReference "pipecat-client-ios-daily" */ = {
+		90CC98B42E15820B003C2706 /* XCRemoteSwiftPackageReference "pipecat-client-ios-daily" */ = {
 			isa = XCRemoteSwiftPackageReference;
 			repositoryURL = "https://github.com/pipecat-ai/pipecat-client-ios-daily/";
 			requirement = {
 				kind = upToNextMajorVersion;
-				minimumVersion = 0.3.2;
+				minimumVersion = 0.3.6;
 			};
 		};
 /* End XCRemoteSwiftPackageReference section */
 
 /* Begin XCSwiftPackageProductDependency section */
-		907C98832D37E6AF0079441F /* PipecatClientIOSDaily */ = {
+		90CC98AF2E158093003C2706 /* PipecatClientIOSDaily */ = {
 			isa = XCSwiftPackageProductDependency;
-			package = 907C98822D37E6AF0079441F /* XCRemoteSwiftPackageReference "pipecat-client-ios-daily" */;
+			productName = PipecatClientIOSDaily;
+		};
+		90CC98B52E15820B003C2706 /* PipecatClientIOSDaily */ = {
+			isa = XCSwiftPackageProductDependency;
+			package = 90CC98B42E15820B003C2706 /* XCRemoteSwiftPackageReference "pipecat-client-ios-daily" */;
 			productName = PipecatClientIOSDaily;
 		};
 /* End XCSwiftPackageProductDependency section */

examples/simple-chatbot/client/ios/SimpleChatbot.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -1,12 +1,13 @@
 {
+  "originHash" : "cc17f08b06def9570d775e9c6f7a8dc10d1588b98127e977c47d052abac659b7",
   "pins" : [
     {
       "identity" : "daily-client-ios",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/daily-co/daily-client-ios.git",
       "state" : {
-        "revision" : "15804ce495780da3ec2d05ab99736315f7bfbd24",
+        "revision" : "431938db25e5807120e89e2dc5bab1c076729f59",
-        "version" : "0.28.0"
+        "version" : "0.31.0"
       }
     },
     {
@@ -14,8 +15,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/pipecat-ai/pipecat-client-ios.git",
       "state" : {
-        "revision" : "c679512e367002a1a67da85d503fec72d9b17191",
+        "revision" : "f92b5e68e56a8311f7d8ead68a7a5674843cbc40",
-        "version" : "0.3.2"
+        "version" : "0.3.6"
       }
     },
     {
@@ -23,10 +24,10 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/pipecat-ai/pipecat-client-ios-daily/",
       "state" : {
-        "revision" : "a337fe6642c52376d2f90eafcb965f5be772ce72",
+        "revision" : "8f494da903192c22c367ecf9e51248c9b651fbc6",
-        "version" : "0.3.2"
+        "version" : "0.3.6"
       }
     }
   ],
-  "version" : 2
+  "version" : 3
 }

examples/simple-chatbot/client/ios/SimpleChatbot/model/CallContainerModel.swift

@@ -78,10 +78,11 @@ class CallContainerModel: ObservableObject {
         self.saveCredentials(backendURL: baseUrl)
     }
     
-    @MainActor
     func disconnect() {
-        self.rtviClientIOS?.disconnect(completion: nil)
+        Task { @MainActor in
-        self.rtviClientIOS?.release()
+            try await self.rtviClientIOS?.disconnect()
+            self.rtviClientIOS?.release()
+        }
     }
     
     func showError(message: String) {

src/pipecat/audio/vad/silero.py

@@ -46,10 +46,6 @@ class SileroOnnxModel:
             path: Path to the ONNX model file.
             force_onnx_cpu: Whether to force CPU execution provider.
         """
-        import numpy as np
-
-        global np
-
         opts = onnxruntime.SessionOptions()
         opts.inter_op_num_threads = 1
         opts.intra_op_num_threads = 1

src/pipecat/observers/loggers/debug_log_observer.py

@@ -47,7 +47,7 @@ class DebugLogObserver(BaseObserver):
 
         Log specific frame types from any source/destination::
 
-            from pipecat.frames.frames import TranscriptionFrame, InterimTranscriptionFrame
+            from pipecat.frames.frames import LLMTextFrame, TranscriptionFrame
             observers=[
                 DebugLogObserver(frame_types=(LLMTextFrame,TranscriptionFrame,)),
             ]
@@ -55,7 +55,8 @@ class DebugLogObserver(BaseObserver):
         Log frames with specific source/destination filters::
 
             from pipecat.frames.frames import StartInterruptionFrame, UserStartedSpeakingFrame, LLMTextFrame
-            from pipecat.transports.base_output_transport import BaseOutputTransport
+            from pipecat.observers.loggers.debug_log_observer import DebugLogObserver, FrameEndpoint
+            from pipecat.transports.base_output import BaseOutputTransport
             from pipecat.services.stt_service import STTService
 
             observers=[

src/pipecat/processors/aggregators/gated.py

@@ -26,29 +26,6 @@ class GatedAggregator(FrameProcessor):
     until and not including the gate-closed frame. The aggregator maintains an
     internal gate state that controls whether frames are passed through immediately
     or accumulated for later release.
-
-    Doctest: FIXME to work with asyncio
-    >>> from pipecat.frames.frames import ImageRawFrame
-
-    >>> async def print_frames(aggregator, frame):
-    ...     async for frame in aggregator.process_frame(frame):
-    ...         if isinstance(frame, TextFrame):
-    ...             print(frame.text)
-    ...         else:
-    ...             print(frame.__class__.__name__)
-
-    >>> aggregator = GatedAggregator(
-    ...     gate_close_fn=lambda x: isinstance(x, LLMResponseStartFrame),
-    ...     gate_open_fn=lambda x: isinstance(x, ImageRawFrame),
-    ...     start_open=False)
-    >>> asyncio.run(print_frames(aggregator, TextFrame("Hello")))
-    >>> asyncio.run(print_frames(aggregator, TextFrame("Hello again.")))
-    >>> asyncio.run(print_frames(aggregator, ImageRawFrame(image=bytes([]), size=(0, 0))))
-    ImageRawFrame
-    Hello
-    Hello again.
-    >>> asyncio.run(print_frames(aggregator, TextFrame("Goodbye.")))
-    Goodbye.
     """
 
     def __init__(

src/pipecat/processors/aggregators/sentence.py

@@ -23,20 +23,10 @@ class SentenceAggregator(FrameProcessor):
     Useful for ensuring downstream processors receive coherent, complete sentences
     rather than fragmented text.
 
-    Frame input/output:
+    Frame input/output::
+
         TextFrame("Hello,") -> None
         TextFrame(" world.") -> TextFrame("Hello, world.")
-
-    Doctest: FIXME to work with asyncio
-    >>> import asyncio
-    >>> async def print_frames(aggregator, frame):
-    ...     async for frame in aggregator.process_frame(frame):
-    ...         print(frame.text)
-
-    >>> aggregator = SentenceAggregator()
-    >>> asyncio.run(print_frames(aggregator, TextFrame("Hello,")))
-    >>> asyncio.run(print_frames(aggregator, TextFrame(" world.")))
-    Hello, world.
     """
 
     def __init__(self):

src/pipecat/processors/aggregators/vision_image_frame.py

@@ -20,17 +20,6 @@ class VisionImageFrameAggregator(FrameProcessor):
     This aggregator waits for a consecutive TextFrame and an InputImageRawFrame.
     After the InputImageRawFrame arrives it will output a VisionImageRawFrame
     combining both the text and image data for multimodal processing.
-
-    >>> from pipecat.frames.frames import ImageFrame
-
-    >>> async def print_frames(aggregator, frame):
-    ...     async for frame in aggregator.process_frame(frame):
-    ...         print(frame)
-
-    >>> aggregator = VisionImageFrameAggregator()
-    >>> asyncio.run(print_frames(aggregator, TextFrame("What do you see?")))
-    >>> asyncio.run(print_frames(aggregator, ImageFrame(image=bytes([]), size=(0, 0))))
-    VisionImageFrame, text: What do you see?, image size: 0x0, buffer size: 0 B
     """
 
     def __init__(self):

src/pipecat/processors/text_transformer.py

@@ -18,14 +18,6 @@ class StatelessTextTransformer(FrameProcessor):
     This processor intercepts TextFrame objects and applies a user-provided
     transformation function to the text content. The function can be either
     synchronous or asynchronous (coroutine).
-
-    >>> async def print_frames(aggregator, frame):
-    ...     async for frame in aggregator.process_frame(frame):
-    ...         print(frame.text)
-
-    >>> aggregator = StatelessTextTransformer(lambda x: x.upper())
-    >>> asyncio.run(print_frames(aggregator, TextFrame("Hello")))
-    HELLO
     """
 
     def __init__(

src/pipecat/services/gemini_multimodal_live/file_api.py

@@ -6,9 +6,9 @@
 
 """Gemini File API client for uploading and managing files.
 
-This module provides the GeminiFileAPI class for interacting with Google's Gemini File API,
+This module provides a client for Google's Gemini File API, enabling file
-including uploading, fetching, listing, and deleting files that can be used with Gemini
+uploads, metadata retrieval, listing, and deletion. Files uploaded through
-generative models.
+this API can be referenced in Gemini generative model calls.
 """
 
 import mimetypes

src/pipecat/services/gemini_multimodal_live/gemini.py

@@ -382,7 +382,14 @@ class GeminiMultimodalModalities(Enum):
 
 
 class GeminiMediaResolution(str, Enum):
-    """Media resolution options for Gemini Multimodal Live."""
+    """Media resolution options for Gemini Multimodal Live.
+
+    Parameters:
+        UNSPECIFIED: Use default resolution setting.
+        LOW: Low resolution with 64 tokens.
+        MEDIUM: Medium resolution with 256 tokens.
+        HIGH: High resolution with zoomed reframing and 256 tokens.
+    """
 
     UNSPECIFIED = "MEDIA_RESOLUTION_UNSPECIFIED"  # Use default
     LOW = "MEDIA_RESOLUTION_LOW"  # 64 tokens
@@ -499,7 +506,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
             params: Configuration parameters for the model. Defaults to InputParams().
             inference_on_context_initialization: Whether to generate a response when context
                 is first set. Defaults to True.
-            file_api_base_url: Base URL for the file API. Defaults to the official Gemini Live endpoint.
+            file_api_base_url: Base URL for the Gemini File API. Defaults to the official endpoint.
             **kwargs: Additional arguments passed to parent LLMService.
         """
         super().__init__(base_url=base_url, **kwargs)
@@ -772,6 +779,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
         await self._ws_send(event.model_dump(exclude_none=True))
 
     async def _connect(self):
+        """Establish WebSocket connection to Gemini Live API."""
         if self._websocket:
             # Here we assume that if we have a websocket, we are connected. We
             # handle disconnections in the send/recv code paths.
@@ -876,6 +884,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
             self._websocket = None
 
     async def _disconnect(self):
+        """Disconnect from Gemini Live API and clean up resources."""
         logger.info("Disconnecting from Gemini service")
         try:
             self._disconnecting = True
@@ -892,6 +901,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
             logger.error(f"{self} error disconnecting: {e}")
 
     async def _ws_send(self, message):
+        """Send a message to the WebSocket connection."""
         # logger.debug(f"Sending message to websocket: {message}")
         try:
             if self._websocket:
@@ -912,6 +922,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
     #
 
     async def _receive_task_handler(self):
+        """Handle incoming messages from the WebSocket connection."""
         async for message in WatchdogAsyncIterator(self._websocket, manager=self.task_manager):
             evt = events.parse_server_event(message)
             # logger.debug(f"Received event: {message[:500]}")
@@ -940,6 +951,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
     #
 
     async def _send_user_audio(self, frame):
+        """Send user audio frame to Gemini Live API."""
         if self._audio_input_paused:
             return
         # Send all audio to Gemini
@@ -956,6 +968,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
             self._user_audio_buffer = self._user_audio_buffer[-length:]
 
     async def _send_user_video(self, frame):
+        """Send user video frame to Gemini Live API."""
         if self._video_input_paused:
             return
 
@@ -969,6 +982,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
         await self.send_client_event(evt)
 
     async def _create_initial_response(self):
+        """Create initial response based on context history."""
         if not self._api_session_ready:
             self._run_llm_when_api_session_ready = True
             return
@@ -994,6 +1008,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
             self._needs_turn_complete_message = True
 
     async def _create_single_response(self, messages_list):
+        """Create a single response from a list of messages."""
         # Refactor to combine this logic with same logic in GeminiMultimodalLiveContext
         messages = []
         for item in messages_list:
@@ -1046,6 +1061,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
 
     @traced_gemini_live(operation="llm_tool_result")
     async def _tool_result(self, tool_result_message):
+        """Send tool result back to the API."""
         # For now we're shoving the name into the tool_call_id field, so this
         # will work until we revisit that.
         id = tool_result_message.get("tool_call_id")
@@ -1071,6 +1087,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
 
     @traced_gemini_live(operation="llm_setup")
     async def _handle_evt_setup_complete(self, evt):
+        """Handle the setup complete event."""
         # If this is our first context frame, run the LLM
         self._api_session_ready = True
         # Now that we've configured the session, we can run the LLM if we need to.
@@ -1079,6 +1096,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
             await self._create_initial_response()
 
     async def _handle_evt_model_turn(self, evt):
+        """Handle the model turn event."""
         part = evt.serverContent.modelTurn.parts[0]
         if not part:
             return
@@ -1120,6 +1138,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
 
     @traced_gemini_live(operation="llm_tool_call")
     async def _handle_evt_tool_call(self, evt):
+        """Handle tool call events."""
         function_calls = evt.toolCall.functionCalls
         if not function_calls:
             return
@@ -1140,6 +1159,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
 
     @traced_gemini_live(operation="llm_response")
     async def _handle_evt_turn_complete(self, evt):
+        """Handle the turn complete event."""
         self._bot_is_speaking = False
         text = self._bot_text_buffer
 
@@ -1223,6 +1243,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
             )
 
     async def _handle_evt_output_transcription(self, evt):
+        """Handle the output transcription event."""
         if not evt.serverContent.outputTranscription:
             return
 
@@ -1241,6 +1262,7 @@ class GeminiMultimodalLiveLLMService(LLMService):
         await self.push_frame(TTSTextFrame(text=text))
 
     async def _handle_evt_usage_metadata(self, evt):
+        """Handle the usage metadata event."""
         if not evt.usageMetadata:
             return
 

src/pipecat/utils/utils.py

@@ -25,15 +25,6 @@ def obj_id() -> int:
 
     Returns:
         A unique integer identifier that increments globally across all objects.
-
-    Examples::
-
-        >>> obj_id()
-        0
-        >>> obj_id()
-        1
-        >>> obj_id()
-        2
     """
     with _ID_LOCK:
         return next(_ID)
@@ -47,16 +38,6 @@ def obj_count(obj) -> int:
 
     Returns:
         A unique integer count that increments per class type.
-
-    Examples::
-
-        >>> obj_count(object())
-        0
-        >>> obj_count(object())
-        1
-        >>> new_type = type('NewType', (object,), {})
-        >>> obj_count(new_type())
-        0
     """
     with _COUNTS_LOCK:
         return next(_COUNTS[obj.__class__.__name__])