Merge remote-tracking branch 'upstream/main'

src/pipecat/clocks/base_clock.py

@@ -8,7 +8,6 @@ from abc import ABC, abstractmethod
 
 
 class BaseClock(ABC):
-
     @abstractmethod
     def get_time(self) -> int:
         pass

src/pipecat/clocks/system_clock.py

@@ -10,7 +10,6 @@ from pipecat.clocks.base_clock import BaseClock
 
 
 class SystemClock(BaseClock):
-
     def __init__(self):
         self._time = 0
 

src/pipecat/frames/frames.py

@@ -43,6 +43,7 @@ class DataFrame(Frame):
 @dataclass
 class AudioRawFrame(DataFrame):
     """A chunk of audio."""
+
     audio: bytes
     sample_rate: int
     num_channels: int
@@ -58,9 +59,8 @@ class AudioRawFrame(DataFrame):
 
 @dataclass
 class InputAudioRawFrame(AudioRawFrame):
-    """A chunk of audio usually coming from an input transport.
+    """A chunk of audio usually coming from an input transport."""
 
-    """
     pass
 
 
@@ -70,14 +70,14 @@ class OutputAudioRawFrame(AudioRawFrame):
     transport's microphone has been enabled.
 
     """
+
     pass
 
 
 @dataclass
 class TTSAudioRawFrame(OutputAudioRawFrame):
-    """A chunk of output audio generated by a TTS service.
+    """A chunk of output audio generated by a TTS service."""
 
-    """
     pass
 
 
@@ -87,6 +87,7 @@ class ImageRawFrame(DataFrame):
     enabled.
 
     """
+
     image: bytes
     size: Tuple[int, int]
     format: str | None
@@ -112,6 +113,7 @@ class UserImageRawFrame(InputImageRawFrame):
     transport's camera is enabled.
 
     """
+
     user_id: str
 
     def __str__(self):
@@ -125,11 +127,14 @@ class VisionImageRawFrame(InputImageRawFrame):
     shown by the transport if the transport's camera is enabled.
 
     """
+
     text: str | None
 
     def __str__(self):
         pts = format_pts(self.pts)
-        return f"{self.name}(pts: {pts}, text: {self.text}, size: {self.size}, format: {self.format})"
+        return (
+            f"{self.name}(pts: {pts}, text: {self.text}, size: {self.size}, format: {self.format})"
+        )
 
 
 @dataclass
@@ -138,6 +143,7 @@ class URLImageRawFrame(OutputImageRawFrame):
     transport's camera is enabled.
 
     """
+
     url: str | None
 
     def __str__(self):
@@ -152,6 +158,7 @@ class SpriteFrame(Frame):
     `camera_out_framerate` constructor parameter.
 
     """
+
     images: List[ImageRawFrame]
 
     def __str__(self):
@@ -165,6 +172,7 @@ class TextFrame(DataFrame):
     be used to send text through pipelines.
 
     """
+
     text: str
 
     def __str__(self):
@@ -178,6 +186,7 @@ class TranscriptionFrame(TextFrame):
     transport's receive queue when a participant speaks.
 
     """
+
     user_id: str
     timestamp: str
     language: Language | None = None
@@ -190,6 +199,7 @@ class TranscriptionFrame(TextFrame):
 class InterimTranscriptionFrame(TextFrame):
     """A text frame with interim transcription-specific data. Will be placed in
     the transport's receive queue when a participant speaks."""
+
     user_id: str
     timestamp: str
     language: Language | None = None
@@ -207,6 +217,7 @@ class LLMMessagesFrame(DataFrame):
     processors.
 
     """
+
     messages: List[dict]
 
 
@@ -216,6 +227,7 @@ class LLMMessagesAppendFrame(DataFrame):
     current context.
 
     """
+
     messages: List[dict]
 
 
@@ -226,6 +238,7 @@ class LLMMessagesUpdateFrame(DataFrame):
     LLMMessagesFrame.
 
     """
+
     messages: List[dict]
 
 
@@ -235,13 +248,14 @@ class LLMSetToolsFrame(DataFrame):
     The specific format depends on the LLM being used, but it should typically
     contain JSON Schema objects.
     """
+
     tools: List[dict]
 
 
 @dataclass
 class LLMEnablePromptCachingFrame(DataFrame):
-    """A frame to enable/disable prompt caching in certain LLMs.
-    """
+    """A frame to enable/disable prompt caching in certain LLMs."""
+
     enable: bool
 
 
@@ -251,6 +265,7 @@ class TTSSpeakFrame(DataFrame):
     pipeline (if any).
 
     """
+
     text: str
 
 
@@ -262,6 +277,7 @@ class TransportMessageFrame(DataFrame):
     def __str__(self):
         return f"{self.name}(message: {self.message})"
 
+
 #
 # App frames. Application user-defined frames.
 #
@@ -271,6 +287,7 @@ class TransportMessageFrame(DataFrame):
 class AppFrame(Frame):
     pass
 
+
 #
 # System frames
 #
@@ -284,6 +301,7 @@ class SystemFrame(Frame):
 @dataclass
 class StartFrame(SystemFrame):
     """This is the first frame that should be pushed down a pipeline."""
+
     clock: BaseClock
     allow_interruptions: bool = False
     enable_metrics: bool = False
@@ -294,6 +312,7 @@ class StartFrame(SystemFrame):
 @dataclass
 class CancelFrame(SystemFrame):
     """Indicates that a pipeline needs to stop right away."""
+
     pass
 
 
@@ -304,6 +323,7 @@ class ErrorFrame(SystemFrame):
     bot should exit.
 
     """
+
     error: str
     fatal: bool = False
 
@@ -317,6 +337,7 @@ class FatalErrorFrame(ErrorFrame):
     that the bot should exit.
 
     """
+
     fatal: bool = field(default=True, init=False)
 
 
@@ -327,6 +348,7 @@ class StopTaskFrame(SystemFrame):
     the pipeline task.
 
     """
+
     pass
 
 
@@ -338,6 +360,7 @@ class StartInterruptionFrame(SystemFrame):
     guaranteed).
 
     """
+
     pass
 
 
@@ -349,6 +372,7 @@ class StopInterruptionFrame(SystemFrame):
     guaranteed).
 
     """
+
     pass
 
 
@@ -359,13 +383,14 @@ class BotInterruptionFrame(SystemFrame):
     UserStartedSpeakingFrame and UserStoppedSpeakingFrame won't be generated.
 
     """
+
     pass
 
 
 @dataclass
 class MetricsFrame(SystemFrame):
-    """Emitted by processor that can compute metrics like latencies.
-    """
+    """Emitted by processor that can compute metrics like latencies."""
+
     data: List[MetricsData]
 
 
@@ -388,6 +413,7 @@ class EndFrame(ControlFrame):
     was sent (unline system frames).
 
     """
+
     pass
 
 
@@ -395,12 +421,14 @@ class EndFrame(ControlFrame):
 class LLMFullResponseStartFrame(ControlFrame):
     """Used to indicate the beginning of an LLM response. Following by one or
     more TextFrame and a final LLMFullResponseEndFrame."""
+
     pass
 
 
 @dataclass
 class LLMFullResponseEndFrame(ControlFrame):
     """Indicates the end of an LLM response."""
+
     pass
 
 
@@ -412,28 +440,28 @@ class UserStartedSpeakingFrame(ControlFrame):
     with a TranscriptionFrame)
 
     """
+
     pass
 
 
 @dataclass
 class UserStoppedSpeakingFrame(ControlFrame):
     """Emitted by the VAD to indicate that a user stopped speaking."""
+
     pass
 
 
 @dataclass
 class BotStartedSpeakingFrame(ControlFrame):
-    """Emitted upstream by transport outputs to indicate the bot started speaking.
+    """Emitted upstream by transport outputs to indicate the bot started speaking."""
 
-    """
     pass
 
 
 @dataclass
 class BotStoppedSpeakingFrame(ControlFrame):
-    """Emitted upstream by transport outputs to indicate the bot stopped speaking.
+    """Emitted upstream by transport outputs to indicate the bot stopped speaking."""
 
-    """
     pass
 
 
@@ -445,6 +473,7 @@ class BotSpeakingFrame(ControlFrame):
     since the user might be listening.
 
     """
+
     pass
 
 
@@ -457,18 +486,21 @@ class TTSStartedFrame(ControlFrame):
     needing to control this in the TTS service.
 
     """
+
     pass
 
 
 @dataclass
 class TTSStoppedFrame(ControlFrame):
     """Indicates the end of a TTS response."""
+
     pass
 
 
 @dataclass
 class UserImageRequestFrame(ControlFrame):
     """A frame user to request an image from the given user."""
+
     user_id: str
     context: Optional[Any] = None
 
@@ -478,29 +510,29 @@ class UserImageRequestFrame(ControlFrame):
 
 @dataclass
 class LLMModelUpdateFrame(ControlFrame):
-    """A control frame containing a request to update to a new LLM model.
-    """
+    """A control frame containing a request to update to a new LLM model."""
+
     model: str
 
 
 @dataclass
 class LLMTemperatureUpdateFrame(ControlFrame):
-    """A control frame containing a request to update to a new LLM temperature.
-    """
+    """A control frame containing a request to update to a new LLM temperature."""
+
     temperature: float
 
 
 @dataclass
 class LLMTopKUpdateFrame(ControlFrame):
-    """A control frame containing a request to update to a new LLM top_k.
-    """
+    """A control frame containing a request to update to a new LLM top_k."""
+
     top_k: int
 
 
 @dataclass
 class LLMTopPUpdateFrame(ControlFrame):
-    """A control frame containing a request to update to a new LLM top_p.
-    """
+    """A control frame containing a request to update to a new LLM top_p."""
+
     top_p: float
 
 
@@ -510,6 +542,7 @@ class LLMFrequencyPenaltyUpdateFrame(ControlFrame):
     penalty.
 
     """
+
     frequency_penalty: float
 
 
@@ -519,41 +552,42 @@ class LLMPresencePenaltyUpdateFrame(ControlFrame):
     penalty.
 
     """
+
     presence_penalty: float
 
 
 @dataclass
 class LLMMaxTokensUpdateFrame(ControlFrame):
-    """A control frame containing a request to update to a new LLM max tokens.
-    """
+    """A control frame containing a request to update to a new LLM max tokens."""
+
     max_tokens: int
 
 
 @dataclass
 class LLMSeedUpdateFrame(ControlFrame):
-    """A control frame containing a request to update to a new LLM seed.
-    """
+    """A control frame containing a request to update to a new LLM seed."""
+
     seed: int
 
 
 @dataclass
 class LLMExtraUpdateFrame(ControlFrame):
-    """A control frame containing a request to update to a new LLM extra params.
-    """
+    """A control frame containing a request to update to a new LLM extra params."""
+
     extra: dict
 
 
 @dataclass
 class TTSModelUpdateFrame(ControlFrame):
-    """A control frame containing a request to update the TTS model.
-    """
+    """A control frame containing a request to update the TTS model."""
+
     model: str
 
 
 @dataclass
 class TTSVoiceUpdateFrame(ControlFrame):
-    """A control frame containing a request to update to a new TTS voice.
-    """
+    """A control frame containing a request to update to a new TTS voice."""
+
     voice: str
 
 
@@ -563,6 +597,7 @@ class TTSLanguageUpdateFrame(ControlFrame):
     optional voice.
 
     """
+
     language: Language
 
 
@@ -572,20 +607,21 @@ class STTModelUpdateFrame(ControlFrame):
     language.
 
     """
+
     model: str
 
 
 @dataclass
 class STTLanguageUpdateFrame(ControlFrame):
-    """A control frame containing a request to update to STT language.
-    """
+    """A control frame containing a request to update to STT language."""
+
     language: Language
 
 
 @dataclass
 class FunctionCallInProgressFrame(SystemFrame):
-    """A frame signaling that a function call is in progress.
-    """
+    """A frame signaling that a function call is in progress."""
+
     function_name: str
     tool_call_id: str
     arguments: str
@@ -593,8 +629,8 @@ class FunctionCallInProgressFrame(SystemFrame):
 
 @dataclass
 class FunctionCallResultFrame(DataFrame):
-    """A frame containing the result of an LLM function (tool) call.
-    """
+    """A frame containing the result of an LLM function (tool) call."""
+
     function_name: str
     tool_call_id: str
     arguments: str
@@ -606,4 +642,5 @@ class VADParamsUpdateFrame(ControlFrame):
     """A control frame containing a request to update VAD params. Intended
     to be pushed upstream from RTVI processor.
     """
+
     params: VADParams

src/pipecat/pipeline/base_pipeline.py

@@ -12,7 +12,6 @@ from pipecat.processors.frame_processor import FrameProcessor
 
 
 class BasePipeline(FrameProcessor):
-
     def __init__(self):
         super().__init__()
 

src/pipecat/pipeline/parallel_pipeline.py

@@ -18,7 +18,6 @@ from loguru import logger
 
 
 class Source(FrameProcessor):
-
     def __init__(self, upstream_queue: asyncio.Queue):
         super().__init__()
         self._up_queue = upstream_queue
@@ -34,7 +33,6 @@ class Source(FrameProcessor):
 
 
 class Sink(FrameProcessor):
-
     def __init__(self, downstream_queue: asyncio.Queue):
         super().__init__()
         self._down_queue = downstream_queue

src/pipecat/pipeline/pipeline.py

@@ -12,7 +12,6 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
 
 class PipelineSource(FrameProcessor):
-
     def __init__(self, upstream_push_frame: Callable[[Frame, FrameDirection], Coroutine]):
         super().__init__()
         self._upstream_push_frame = upstream_push_frame
@@ -28,7 +27,6 @@ class PipelineSource(FrameProcessor):
 
 
 class PipelineSink(FrameProcessor):
-
     def __init__(self, downstream_push_frame: Callable[[Frame, FrameDirection], Coroutine]):
         super().__init__()
         self._downstream_push_frame = downstream_push_frame
@@ -44,7 +42,6 @@ class PipelineSink(FrameProcessor):
 
 
 class Pipeline(BasePipeline):
-
     def __init__(self, processors: List[FrameProcessor]):
         super().__init__()
 

src/pipecat/pipeline/runner.py

@@ -14,7 +14,6 @@ from loguru import logger
 
 
 class PipelineRunner:
-
     def __init__(self, *, name: str | None = None, handle_sigint: bool = True):
         self.id: int = obj_id()
         self.name: str = name or f"{self.__class__.__name__}#{obj_count(self)}"
@@ -42,12 +41,10 @@ class PipelineRunner:
     def _setup_sigint(self):
         loop = asyncio.get_running_loop()
         loop.add_signal_handler(
-            signal.SIGINT,
-            lambda *args: asyncio.create_task(self._sig_handler())
+            signal.SIGINT, lambda *args: asyncio.create_task(self._sig_handler())
         )
         loop.add_signal_handler(
-            signal.SIGTERM,
-            lambda *args: asyncio.create_task(self._sig_handler())
+            signal.SIGTERM, lambda *args: asyncio.create_task(self._sig_handler())
         )
 
     async def _sig_handler(self):

src/pipecat/pipeline/sync_parallel_pipeline.py

@@ -18,7 +18,6 @@ from loguru import logger
 
 
 class Source(FrameProcessor):
-
     def __init__(self, upstream_queue: asyncio.Queue):
         super().__init__()
         self._up_queue = upstream_queue
@@ -34,7 +33,6 @@ class Source(FrameProcessor):
 
 
 class Sink(FrameProcessor):
-
     def __init__(self, downstream_queue: asyncio.Queue):
         super().__init__()
         self._down_queue = downstream_queue

src/pipecat/pipeline/task.py

@@ -19,7 +19,8 @@ from pipecat.frames.frames import (
     Frame,
     MetricsFrame,
     StartFrame,
-    StopTaskFrame)
+    StopTaskFrame,
+)
 from pipecat.metrics.metrics import TTFBMetricsData, ProcessingMetricsData
 from pipecat.pipeline.base_pipeline import BasePipeline
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -37,7 +38,6 @@ class PipelineParams(BaseModel):
 
 
 class Source(FrameProcessor):
-
     def __init__(self, up_queue: asyncio.Queue):
         super().__init__()
         self._up_queue = up_queue
@@ -62,12 +62,12 @@ class Source(FrameProcessor):
 
 
 class PipelineTask:
-
     def __init__(
-            self,
-            pipeline: BasePipeline,
-            params: PipelineParams = PipelineParams(),
-            clock: BaseClock = SystemClock()):
+        self,
+        pipeline: BasePipeline,
+        params: PipelineParams = PipelineParams(),
+        clock: BaseClock = SystemClock(),
+    ):
         self.id: int = obj_id()
         self.name: str = f"{self.__class__.__name__}#{obj_count(self)}"
 
@@ -133,12 +133,14 @@ class PipelineTask:
             enable_metrics=self._params.enable_metrics,
             enable_usage_metrics=self._params.enable_metrics,
             report_only_initial_ttfb=self._params.report_only_initial_ttfb,
-            clock=self._clock
+            clock=self._clock,
         )
         await self._source.process_frame(start_frame, FrameDirection.DOWNSTREAM)
 
         if self._params.enable_metrics and self._params.send_initial_empty_metrics:
-            await self._source.process_frame(self._initial_metrics_frame(), FrameDirection.DOWNSTREAM)
+            await self._source.process_frame(
+                self._initial_metrics_frame(), FrameDirection.DOWNSTREAM
+            )
 
         running = True
         should_cleanup = True

src/pipecat/pipeline/to_be_updated/merge_pipeline.py

@@ -15,9 +15,7 @@ class SequentialMergePipeline(Pipeline):
         for idx, pipeline in enumerate(self.pipelines):
             while True:
                 frame = await pipeline.sink.get()
-                if isinstance(
-                        frame, EndFrame) or isinstance(
-                        frame, EndPipeFrame):
+                if isinstance(frame, EndFrame) or isinstance(frame, EndPipeFrame):
                     break
                 await self.sink.put(frame)
 

src/pipecat/processors/aggregators/gated.py

@@ -41,8 +41,13 @@ class GatedAggregator(FrameProcessor):
     Goodbye.
     """
 
-    def __init__(self, gate_open_fn, gate_close_fn, start_open,
-                 direction: FrameDirection = FrameDirection.DOWNSTREAM):
+    def __init__(
+        self,
+        gate_open_fn,
+        gate_close_fn,
+        start_open,
+        direction: FrameDirection = FrameDirection.DOWNSTREAM,
+    ):
         super().__init__()
         self._gate_open_fn = gate_open_fn
         self._gate_close_fn = gate_close_fn
@@ -75,7 +80,7 @@ class GatedAggregator(FrameProcessor):
 
         if self._gate_open:
             await self.push_frame(frame, direction)
-            for (f, d) in self._accumulator:
+            for f, d in self._accumulator:
                 await self.push_frame(f, d)
             self._accumulator = []
         else:

src/pipecat/processors/aggregators/llm_response.py

@@ -6,7 +6,10 @@
 
 from typing import List, Type
 
-from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame, OpenAILLMContext
+from pipecat.processors.aggregators.openai_llm_context import (
+    OpenAILLMContextFrame,
+    OpenAILLMContext,
+)
 
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.frames.frames import (
@@ -22,11 +25,11 @@ from pipecat.frames.frames import (
     TranscriptionFrame,
     TextFrame,
     UserStartedSpeakingFrame,
-    UserStoppedSpeakingFrame)
+    UserStoppedSpeakingFrame,
+)
 
 
 class LLMResponseAggregator(FrameProcessor):
-
     def __init__(
         self,
         *,
@@ -36,7 +39,7 @@ class LLMResponseAggregator(FrameProcessor):
         end_frame,
         accumulator_frame: Type[TextFrame],
         interim_accumulator_frame: Type[TextFrame] | None = None,
-        handle_interruptions: bool = False
+        handle_interruptions: bool = False,
     ):
         super().__init__()
 
@@ -175,7 +178,7 @@ class LLMAssistantResponseAggregator(LLMResponseAggregator):
             start_frame=LLMFullResponseStartFrame,
             end_frame=LLMFullResponseEndFrame,
             accumulator_frame=TextFrame,
-            handle_interruptions=True
+            handle_interruptions=True,
         )
 
 
@@ -187,7 +190,7 @@ class LLMUserResponseAggregator(LLMResponseAggregator):
             start_frame=UserStartedSpeakingFrame,
             end_frame=UserStoppedSpeakingFrame,
             accumulator_frame=TranscriptionFrame,
-            interim_accumulator_frame=InterimTranscriptionFrame
+            interim_accumulator_frame=InterimTranscriptionFrame,
         )
 
 
@@ -295,7 +298,7 @@ class LLMAssistantContextAggregator(LLMContextAggregator):
             start_frame=LLMFullResponseStartFrame,
             end_frame=LLMFullResponseEndFrame,
             accumulator_frame=TextFrame,
-            handle_interruptions=True
+            handle_interruptions=True,
         )
 
 
@@ -308,5 +311,5 @@ class LLMUserContextAggregator(LLMContextAggregator):
             start_frame=UserStartedSpeakingFrame,
             end_frame=UserStoppedSpeakingFrame,
             accumulator_frame=TranscriptionFrame,
-            interim_accumulator_frame=InterimTranscriptionFrame
+            interim_accumulator_frame=InterimTranscriptionFrame,
         )

src/pipecat/processors/aggregators/openai_llm_context.py

@@ -17,7 +17,8 @@ from pipecat.frames.frames import (
     Frame,
     VisionImageRawFrame,
     FunctionCallInProgressFrame,
-    FunctionCallResultFrame)
+    FunctionCallResultFrame,
+)
 from pipecat.processors.frame_processor import FrameProcessor
 
 from loguru import logger
@@ -28,12 +29,13 @@ try:
     from openai.types.chat import (
         ChatCompletionToolParam,
         ChatCompletionToolChoiceOptionParam,
-        ChatCompletionMessageParam
+        ChatCompletionMessageParam,
     )
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable.")
+        "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable."
+    )
     raise Exception(f"Missing module: {e}")
 
 # JSON custom encoder to handle bytes arrays so that we can log contexts
@@ -44,20 +46,18 @@ class CustomEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, io.BytesIO):
             # Convert the first 8 bytes to an ASCII hex string
-            return (f"{obj.getbuffer()[0:8].hex()}...")
+            return f"{obj.getbuffer()[0:8].hex()}..."
         return super().default(obj)
 
 
 class OpenAILLMContext:
-
     def __init__(
         self,
         messages: List[ChatCompletionMessageParam] | None = None,
         tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN,
-        tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN
+        tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN,
     ):
-        self._messages: List[ChatCompletionMessageParam] = messages if messages else [
-        ]
+        self._messages: List[ChatCompletionMessageParam] = messages if messages else []
         self._tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice
         self._tools: List[ChatCompletionToolParam] | NotGiven = tools
 
@@ -81,19 +81,10 @@ class OpenAILLMContext:
         """
         context = OpenAILLMContext()
         buffer = io.BytesIO()
-        Image.frombytes(
-            frame.format,
-            frame.size,
-            frame.image
-        ).save(
-            buffer,
-            format="JPEG")
-        context.add_message({
-            "content": frame.text,
-            "role": "user",
-            "data": buffer,
-            "mime_type": "image/jpeg"
-        })
+        Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG")
+        context.add_message(
+            {"content": frame.text, "role": "user", "data": buffer, "mime_type": "image/jpeg"}
+        )
         return context
 
     @property
@@ -123,9 +114,7 @@ class OpenAILLMContext:
     def get_messages_json(self) -> str:
         return json.dumps(self._messages, cls=CustomEncoder)
 
-    def set_tool_choice(
-        self, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven
-    ):
+    def set_tool_choice(self, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven):
         self._tool_choice = tool_choice
 
     def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN):
@@ -133,37 +122,40 @@ class OpenAILLMContext:
             tools = NOT_GIVEN
         self._tools = tools
 
-    async def call_function(self,
-                            f: Callable[[str,
-                                         str,
-                                         Any,
-                                         FrameProcessor,
-                                         'OpenAILLMContext',
-                                         Callable[[Any],
-                                                  Awaitable[None]]],
-                                        Awaitable[None]],
-                            *,
-                            function_name: str,
-                            tool_call_id: str,
-                            arguments: str,
-                            llm: FrameProcessor) -> None:
-
+    async def call_function(
+        self,
+        f: Callable[
+            [str, str, Any, FrameProcessor, "OpenAILLMContext", Callable[[Any], Awaitable[None]]],
+            Awaitable[None],
+        ],
+        *,
+        function_name: str,
+        tool_call_id: str,
+        arguments: str,
+        llm: FrameProcessor,
+    ) -> None:
         # Push a SystemFrame downstream. This frame will let our assistant context aggregator
         # know that we are in the middle of a function call. Some contexts/aggregators may
         # not need this. But some definitely do (Anthropic, for example).
-        await llm.push_frame(FunctionCallInProgressFrame(
-            function_name=function_name,
-            tool_call_id=tool_call_id,
-            arguments=arguments,
-        ))
-
-        # Define a callback function that pushes a FunctionCallResultFrame downstream.
-        async def function_call_result_callback(result):
-            await llm.push_frame(FunctionCallResultFrame(
+        await llm.push_frame(
+            FunctionCallInProgressFrame(
                 function_name=function_name,
                 tool_call_id=tool_call_id,
                 arguments=arguments,
-                result=result))
+            )
+        )
+
+        # Define a callback function that pushes a FunctionCallResultFrame downstream.
+        async def function_call_result_callback(result):
+            await llm.push_frame(
+                FunctionCallResultFrame(
+                    function_name=function_name,
+                    tool_call_id=tool_call_id,
+                    arguments=arguments,
+                    result=result,
+                )
+            )
+
         await f(function_name, tool_call_id, arguments, llm, self, function_call_result_callback)
 
 
@@ -174,4 +166,5 @@ class OpenAILLMContextFrame(Frame):
     OpenAIContextAggregator frame processor.
 
     """
+
     context: OpenAILLMContext

src/pipecat/processors/aggregators/user_response.py

@@ -12,7 +12,8 @@ from pipecat.frames.frames import (
     TextFrame,
     TranscriptionFrame,
     UserStartedSpeakingFrame,
-    UserStoppedSpeakingFrame)
+    UserStoppedSpeakingFrame,
+)
 
 
 class ResponseAggregator(FrameProcessor):
@@ -49,7 +50,7 @@ class ResponseAggregator(FrameProcessor):
         start_frame,
         end_frame,
         accumulator_frame: TextFrame,
-        interim_accumulator_frame: TextFrame | None = None
+        interim_accumulator_frame: TextFrame | None = None,
     ):
         super().__init__()
 

src/pipecat/processors/aggregators/vision_image_frame.py

@@ -4,12 +4,7 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-from pipecat.frames.frames import (
-    Frame,
-    InputImageRawFrame,
-    TextFrame,
-    VisionImageRawFrame
-)
+from pipecat.frames.frames import Frame, InputImageRawFrame, TextFrame, VisionImageRawFrame
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
 
@@ -46,7 +41,8 @@ class VisionImageFrameAggregator(FrameProcessor):
                     text=self._describe_text,
                     image=frame.image,
                     size=frame.size,
-                    format=frame.format)
+                    format=frame.format,
+                )
                 await self.push_frame(frame)
                 self._describe_text = None
         else:

src/pipecat/processors/filters/frame_filter.py

@@ -11,7 +11,6 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
 
 class FrameFilter(FrameProcessor):
-
     def __init__(self, types: List[type]):
         super().__init__()
         self._types = types
@@ -25,9 +24,11 @@ class FrameFilter(FrameProcessor):
             if isinstance(frame, t):
                 return True
 
-        return (isinstance(frame, AppFrame)
-                or isinstance(frame, ControlFrame)
-                or isinstance(frame, SystemFrame))
+        return (
+            isinstance(frame, AppFrame)
+            or isinstance(frame, ControlFrame)
+            or isinstance(frame, SystemFrame)
+        )
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)

src/pipecat/processors/filters/function_filter.py

@@ -11,7 +11,6 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
 
 class FunctionFilter(FrameProcessor):
-
     def __init__(self, filter: Callable[[Frame], Awaitable[bool]]):
         super().__init__()
         self._filter = filter

src/pipecat/processors/filters/wake_check_filter.py

@@ -21,6 +21,7 @@ class WakeCheckFilter(FrameProcessor):
     after a wake phrase has been detected. It also has a keepalive timeout to allow for a brief
     period of continued conversation after a wake phrase has been detected.
     """
+
     class WakeState(Enum):
         IDLE = 1
         AWAKE = 2
@@ -38,8 +39,9 @@ class WakeCheckFilter(FrameProcessor):
         self._keepalive_timeout = keepalive_timeout
         self._wake_patterns = []
         for name in wake_phrases:
-            pattern = re.compile(r'\b' + r'\s*'.join(re.escape(word)
-                                 for word in name.split()) + r'\b', re.IGNORECASE)
+            pattern = re.compile(
+                r"\b" + r"\s*".join(re.escape(word) for word in name.split()) + r"\b", re.IGNORECASE
+            )
             self._wake_patterns.append(pattern)
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -57,7 +59,8 @@ class WakeCheckFilter(FrameProcessor):
                 if p.state == WakeCheckFilter.WakeState.AWAKE:
                     if time.time() - p.wake_timer < self._keepalive_timeout:
                         logger.debug(
-                            f"Wake phrase keepalive timeout has not expired. Pushing {frame}")
+                            f"Wake phrase keepalive timeout has not expired. Pushing {frame}"
+                        )
                         p.wake_timer = time.time()
                         await self.push_frame(frame)
                         return
@@ -73,7 +76,7 @@ class WakeCheckFilter(FrameProcessor):
                         # and modify the frame in place.
                         p.state = WakeCheckFilter.WakeState.AWAKE
                         p.wake_timer = time.time()
-                        frame.text = p.accumulator[match.start():]
+                        frame.text = p.accumulator[match.start() :]
                         p.accumulator = ""
                         await self.push_frame(frame)
                     else:

src/pipecat/processors/frame_processor.py

@@ -14,18 +14,13 @@ from pipecat.frames.frames import (
     EndFrame,
     ErrorFrame,
     Frame,
-    MetricsFrame,
     StartFrame,
     StartInterruptionFrame,
     StopInterruptionFrame,
-    SystemFrame)
-from pipecat.metrics.metrics import (
-    LLMTokenUsage,
-    LLMUsageMetricsData,
-    MetricsData,
-    ProcessingMetricsData,
-    TTFBMetricsData,
-    TTSUsageMetricsData)
+    SystemFrame,
+)
+from pipecat.metrics.metrics import LLMTokenUsage, MetricsData
+from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics
 from pipecat.utils.utils import obj_count, obj_id
 
 from loguru import logger
@@ -36,81 +31,16 @@ class FrameDirection(Enum):
     UPSTREAM = 2
 
 
-class FrameProcessorMetrics:
-    def __init__(self, name: str):
-        self._core_metrics_data = MetricsData(processor=name)
-        self._start_ttfb_time = 0
-        self._start_processing_time = 0
-        self._should_report_ttfb = True
-
-    def _processor_name(self):
-        return self._core_metrics_data.processor
-
-    def _model_name(self):
-        return self._core_metrics_data.model
-
-    def set_core_metrics_data(self, data: MetricsData):
-        self._core_metrics_data = data
-
-    async def start_ttfb_metrics(self, report_only_initial_ttfb):
-        if self._should_report_ttfb:
-            self._start_ttfb_time = time.time()
-            self._should_report_ttfb = not report_only_initial_ttfb
-
-    async def stop_ttfb_metrics(self):
-        if self._start_ttfb_time == 0:
-            return None
-
-        value = time.time() - self._start_ttfb_time
-        logger.debug(f"{self._processor_name()} TTFB: {value}")
-        ttfb = TTFBMetricsData(
-            processor=self._processor_name(),
-            value=value,
-            model=self._model_name())
-        self._start_ttfb_time = 0
-        return MetricsFrame(data=[ttfb])
-
-    async def start_processing_metrics(self):
-        self._start_processing_time = time.time()
-
-    async def stop_processing_metrics(self):
-        if self._start_processing_time == 0:
-            return None
-
-        value = time.time() - self._start_processing_time
-        logger.debug(f"{self._processor_name()} processing time: {value}")
-        processing = ProcessingMetricsData(
-            processor=self._processor_name(), value=value, model=self._model_name())
-        self._start_processing_time = 0
-        return MetricsFrame(data=[processing])
-
-    async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
-        logger.debug(
-            f"{self._processor_name()} prompt tokens: {tokens.prompt_tokens}, completion tokens: {tokens.completion_tokens}")
-        value = LLMUsageMetricsData(
-            processor=self._processor_name(),
-            model=self._model_name(),
-            value=tokens)
-        return MetricsFrame(data=[value])
-
-    async def start_tts_usage_metrics(self, text: str):
-        characters = TTSUsageMetricsData(
-            processor=self._processor_name(),
-            model=self._model_name(),
-            value=len(text))
-        logger.debug(f"{self._processor_name()} usage characters: {characters.value}")
-        return MetricsFrame(data=[characters])
-
-
 class FrameProcessor:
-
     def __init__(
-            self,
-            *,
-            name: str | None = None,
-            sync: bool = True,
-            loop: asyncio.AbstractEventLoop | None = None,
-            **kwargs):
+        self,
+        *,
+        name: str | None = None,
+        metrics: FrameProcessorMetrics | None = None,
+        sync: bool = True,
+        loop: asyncio.AbstractEventLoop | None = None,
+        **kwargs,
+    ):
         self.id: int = obj_id()
         self.name = name or f"{self.__class__.__name__}#{obj_count(self)}"
         self._parent: "FrameProcessor" | None = None
@@ -129,7 +59,8 @@ class FrameProcessor:
         self._report_only_initial_ttfb = False
 
         # Metrics
-        self._metrics = FrameProcessorMetrics(name=self.name)
+        self._metrics = metrics or FrameProcessorMetrics()
+        self._metrics.set_processor_name(self.name)
 
         # Every processor in Pipecat should only output frames from a single
         # task. This avoid problems like audio overlapping. System frames are

src/pipecat/processors/frameworks/langchain.py

@@ -11,7 +11,8 @@ from pipecat.frames.frames import (
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
     LLMMessagesFrame,
-    TextFrame)
+    TextFrame,
+)
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
 from loguru import logger
@@ -20,9 +21,7 @@ try:
     from langchain_core.messages import AIMessageChunk
     from langchain_core.runnables import Runnable
 except ModuleNotFoundError as e:
-    logger.exception(
-        "In order to use Langchain, you need to `pip install pipecat-ai[langchain]`. "
-    )
+    logger.exception("In order to use Langchain, you need to `pip install pipecat-ai[langchain]`. ")
     raise Exception(f"Missing module: {e}")
 
 

src/pipecat/processors/frameworks/rtvi.py

@@ -8,12 +8,14 @@ import asyncio
 
 from typing import Any, Awaitable, Callable, Dict, List, Literal, Optional, Union
 from pydantic import BaseModel, Field, PrivateAttr, ValidationError
+from dataclasses import dataclass
 
 from pipecat.frames.frames import (
     BotInterruptionFrame,
     BotStartedSpeakingFrame,
     BotStoppedSpeakingFrame,
     CancelFrame,
+    DataFrame,
     EndFrame,
     ErrorFrame,
     Frame,
@@ -24,7 +26,8 @@ from pipecat.frames.frames import (
     TransportMessageFrame,
     UserStartedSpeakingFrame,
     FunctionCallResultFrame,
-    UserStoppedSpeakingFrame)
+    UserStoppedSpeakingFrame,
+)
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
@@ -39,8 +42,9 @@ ActionResult = Union[bool, int, float, str, list, dict]
 class RTVIServiceOption(BaseModel):
     name: str
     type: Literal["bool", "number", "string", "array", "object"]
-    handler: Callable[["RTVIProcessor", str, "RTVIServiceOptionConfig"],
-                      Awaitable[None]] = Field(exclude=True)
+    handler: Callable[["RTVIProcessor", str, "RTVIServiceOptionConfig"], Awaitable[None]] = Field(
+        exclude=True
+    )
 
 
 class RTVIService(BaseModel):
@@ -70,8 +74,9 @@ class RTVIAction(BaseModel):
     action: str
     arguments: List[RTVIActionArgument] = []
     result: Literal["bool", "number", "string", "array", "object"]
-    handler: Callable[["RTVIProcessor", str, Dict[str, Any]],
-                      Awaitable[ActionResult]] = Field(exclude=True)
+    handler: Callable[["RTVIProcessor", str, Dict[str, Any]], Awaitable[ActionResult]] = Field(
+        exclude=True
+    )
     _arguments_dict: Dict[str, RTVIActionArgument] = PrivateAttr(default={})
 
     def model_post_init(self, __context: Any) -> None:
@@ -116,12 +121,19 @@ class RTVIActionRun(BaseModel):
     arguments: Optional[List[RTVIActionRunArgument]] = None
 
 
+@dataclass
+class RTVIActionFrame(DataFrame):
+    rtvi_action_run: RTVIActionRun
+    message_id: Optional[str] = None
+
+
 class RTVIMessage(BaseModel):
     label: Literal["rtvi-ai"] = "rtvi-ai"
     type: str
     id: str
     data: Optional[Dict[str, Any]] = None
 
+
 #
 # Pipecat -> Client responses and messages.
 #
@@ -268,12 +280,13 @@ class RTVIProcessorParams(BaseModel):
 
 
 class RTVIProcessor(FrameProcessor):
-
-    def __init__(self,
-                 *,
-                 config: RTVIConfig = RTVIConfig(config=[]),
-                 params: RTVIProcessorParams = RTVIProcessorParams(),
-                 **kwargs):
+    def __init__(
+        self,
+        *,
+        config: RTVIConfig = RTVIConfig(config=[]),
+        params: RTVIProcessorParams = RTVIProcessorParams(),
+        **kwargs,
+    ):
         super().__init__(sync=False, **kwargs)
         self._config = config
         self._params = params
@@ -310,25 +323,23 @@ class RTVIProcessor(FrameProcessor):
             await self._maybe_send_bot_ready()
 
     async def handle_function_call(
-            self,
-            function_name: str,
-            tool_call_id: str,
-            arguments: dict,
-            llm: FrameProcessor,
-            context: OpenAILLMContext,
-            result_callback):
+        self,
+        function_name: str,
+        tool_call_id: str,
+        arguments: dict,
+        llm: FrameProcessor,
+        context: OpenAILLMContext,
+        result_callback,
+    ):
         fn = RTVILLMFunctionCallMessageData(
-            function_name=function_name,
-            tool_call_id=tool_call_id,
-            args=arguments)
+            function_name=function_name, tool_call_id=tool_call_id, args=arguments
+        )
         message = RTVILLMFunctionCallMessage(data=fn)
         await self._push_transport_message(message, exclude_none=False)
 
     async def handle_function_call_start(
-            self,
-            function_name: str,
-            llm: FrameProcessor,
-            context: OpenAILLMContext):
+        self, function_name: str, llm: FrameProcessor, context: OpenAILLMContext
+    ):
         fn = RTVILLMFunctionCallStartMessageData(function_name=function_name)
         message = RTVILLMFunctionCallStartMessage(data=fn)
         await self._push_transport_message(message, exclude_none=False)
@@ -357,10 +368,14 @@ class RTVIProcessor(FrameProcessor):
             # finish and the task finishes when EndFrame is processed.
             await self.push_frame(frame, direction)
             await self._stop(frame)
-        elif isinstance(frame, UserStartedSpeakingFrame) or isinstance(frame, UserStoppedSpeakingFrame):
+        elif isinstance(frame, UserStartedSpeakingFrame) or isinstance(
+            frame, UserStoppedSpeakingFrame
+        ):
             await self._handle_interruptions(frame)
             await self.push_frame(frame, direction)
-        elif isinstance(frame, BotStartedSpeakingFrame) or isinstance(frame, BotStoppedSpeakingFrame):
+        elif isinstance(frame, BotStartedSpeakingFrame) or isinstance(
+            frame, BotStoppedSpeakingFrame
+        ):
             await self._handle_bot_speaking(frame)
             await self.push_frame(frame, direction)
         # Data frames
@@ -369,6 +384,8 @@ class RTVIProcessor(FrameProcessor):
             await self.push_frame(frame, direction)
         elif isinstance(frame, TransportMessageFrame):
             await self._message_queue.put(frame)
+        elif isinstance(frame, RTVIActionFrame):
+            await self._handle_action(frame.message_id, frame.rtvi_action_run)
         # Other frames
         else:
             await self.push_frame(frame, direction)
@@ -393,8 +410,8 @@ class RTVIProcessor(FrameProcessor):
 
     async def _push_transport_message(self, model: BaseModel, exclude_none: bool = True):
         frame = TransportMessageFrame(
-            message=model.model_dump(exclude_none=exclude_none),
-            urgent=True)
+            message=model.model_dump(exclude_none=exclude_none), urgent=True
+        )
         await self.push_frame(frame)
 
     async def _handle_transcriptions(self, frame: Frame):
@@ -405,17 +422,15 @@ class RTVIProcessor(FrameProcessor):
         if isinstance(frame, TranscriptionFrame):
             message = RTVITranscriptionMessage(
                 data=RTVITranscriptionMessageData(
-                    text=frame.text,
-                    user_id=frame.user_id,
-                    timestamp=frame.timestamp,
-                    final=True))
+                    text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp, final=True
+                )
+            )
         elif isinstance(frame, InterimTranscriptionFrame):
             message = RTVITranscriptionMessage(
                 data=RTVITranscriptionMessageData(
-                    text=frame.text,
-                    user_id=frame.user_id,
-                    timestamp=frame.timestamp,
-                    final=False))
+                    text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp, final=False
+                )
+            )
 
         if message:
             await self._push_transport_message(message)
@@ -539,10 +554,11 @@ class RTVIProcessor(FrameProcessor):
             function_name=data.function_name,
             tool_call_id=data.tool_call_id,
             arguments=data.arguments,
-            result=data.result)
+            result=data.result,
+        )
         await self.push_frame(frame)
 
-    async def _handle_action(self, request_id: str, data: RTVIActionRun):
+    async def _handle_action(self, request_id: str | None, data: RTVIActionRun):
         action_id = self._action_id(data.service, data.action)
         if action_id not in self._registered_actions:
             await self._send_error_response(request_id, f"Action {action_id} not registered")
@@ -553,8 +569,11 @@ class RTVIProcessor(FrameProcessor):
             for arg in data.arguments:
                 arguments[arg.name] = arg.value
         result = await action.handler(self, action.service, arguments)
-        message = RTVIActionResponse(id=request_id, data=RTVIActionResponseData(result=result))
-        await self._push_transport_message(message)
+        # Only send a response if request_id is present. Things that don't care about
+        # action responses (such as webhooks) don't set a request_id
+        if request_id:
+            message = RTVIActionResponse(id=request_id, data=RTVIActionResponseData(result=result))
+            await self._push_transport_message(message)
 
     async def _maybe_send_bot_ready(self):
         if self._pipeline_started and self._client_ready:
@@ -567,9 +586,8 @@ class RTVIProcessor(FrameProcessor):
 
         message = RTVIBotReady(
             id=self._client_ready_id,
-            data=RTVIBotReadyData(
-                version=RTVI_PROTOCOL_VERSION,
-                config=self._config.config))
+            data=RTVIBotReadyData(version=RTVI_PROTOCOL_VERSION, config=self._config.config),
+        )
         await self._push_transport_message(message)
 
     async def _send_error_frame(self, frame: ErrorFrame):

src/pipecat/processors/gstreamer/pipeline_source.py

@@ -15,20 +15,23 @@ from pipecat.frames.frames import (
     OutputAudioRawFrame,
     OutputImageRawFrame,
     StartFrame,
-    SystemFrame)
+    SystemFrame,
+)
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
 from loguru import logger
 
 try:
     import gi
-    gi.require_version('Gst', '1.0')
-    gi.require_version('GstApp', '1.0')
+
+    gi.require_version("Gst", "1.0")
+    gi.require_version("GstApp", "1.0")
     from gi.repository import Gst, GstApp
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use GStreamer, you need to `pip install pipecat-ai[gstreamer]`. Also, you need to install GStreamer in your system.")
+        "In order to use GStreamer, you need to `pip install pipecat-ai[gstreamer]`. Also, you need to install GStreamer in your system."
+    )
     raise Exception(f"Missing module: {e}")
 
 
@@ -120,7 +123,8 @@ class GStreamerPipelineSource(FrameProcessor):
         audioresample = Gst.ElementFactory.make("audioresample", None)
         audiocapsfilter = Gst.ElementFactory.make("capsfilter", None)
         audiocaps = Gst.Caps.from_string(
-            f"audio/x-raw,format=S16LE,rate={self._out_params.audio_sample_rate},channels={self._out_params.audio_channels},layout=interleaved")
+            f"audio/x-raw,format=S16LE,rate={self._out_params.audio_sample_rate},channels={self._out_params.audio_channels},layout=interleaved"
+        )
         audiocapsfilter.set_property("caps", audiocaps)
         appsink_audio = Gst.ElementFactory.make("appsink", None)
         appsink_audio.set_property("emit-signals", True)
@@ -152,7 +156,8 @@ class GStreamerPipelineSource(FrameProcessor):
         videoscale = Gst.ElementFactory.make("videoscale", None)
         videocapsfilter = Gst.ElementFactory.make("capsfilter", None)
         videocaps = Gst.Caps.from_string(
-            f"video/x-raw,format=RGB,width={self._out_params.video_width},height={self._out_params.video_height}")
+            f"video/x-raw,format=RGB,width={self._out_params.video_width},height={self._out_params.video_height}"
+        )
         videocapsfilter.set_property("caps", videocaps)
 
         appsink_video = Gst.ElementFactory.make("appsink", None)
@@ -182,9 +187,11 @@ class GStreamerPipelineSource(FrameProcessor):
     def _appsink_audio_new_sample(self, appsink: GstApp.AppSink):
         buffer = appsink.pull_sample().get_buffer()
         (_, info) = buffer.map(Gst.MapFlags.READ)
-        frame = OutputAudioRawFrame(audio=info.data,
-                                    sample_rate=self._out_params.audio_sample_rate,
-                                    num_channels=self._out_params.audio_channels)
+        frame = OutputAudioRawFrame(
+            audio=info.data,
+            sample_rate=self._out_params.audio_sample_rate,
+            num_channels=self._out_params.audio_channels,
+        )
         asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())
         buffer.unmap(info)
         return Gst.FlowReturn.OK
@@ -195,7 +202,8 @@ class GStreamerPipelineSource(FrameProcessor):
         frame = OutputImageRawFrame(
             image=info.data,
             size=(self._out_params.video_width, self._out_params.video_height),
-            format="RGB")
+            format="RGB",
+        )
         asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())
         buffer.unmap(info)
         return Gst.FlowReturn.OK

src/pipecat/processors/idle_frame_processor.py

@@ -19,12 +19,13 @@ class IdleFrameProcessor(FrameProcessor):
     """
 
     def __init__(
-            self,
-            *,
-            callback: Callable[["IdleFrameProcessor"], Awaitable[None]],
-            timeout: float,
-            types: List[type] = [],
-            **kwargs):
+        self,
+        *,
+        callback: Callable[["IdleFrameProcessor"], Awaitable[None]],
+        timeout: float,
+        types: List[type] = [],
+        **kwargs,
+    ):
         super().__init__(sync=False, **kwargs)
 
         self._callback = callback

src/pipecat/processors/logger.py

@@ -8,6 +8,7 @@ from pipecat.frames.frames import BotSpeakingFrame, Frame, AudioRawFrame, Transp
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from loguru import logger
 from typing import Optional
+
 logger = logger.opt(ansi=True)
 
 
@@ -19,7 +20,9 @@ class FrameLogger(FrameProcessor):
         ignored_frame_types: Optional[list] = [
             BotSpeakingFrame,
             AudioRawFrame,
-            TransportMessageFrame]):
+            TransportMessageFrame,
+        ],
+    ):
         super().__init__()
         self._prefix = prefix
         self._color = color

src/pipecat/processors/metrics/frame_processor_metrics.py

@@ -0,0 +1,80 @@
+import time
+
+from pipecat.frames.frames import MetricsFrame
+from pipecat.metrics.metrics import (
+    LLMTokenUsage,
+    LLMUsageMetricsData,
+    MetricsData,
+    ProcessingMetricsData,
+    TTFBMetricsData,
+    TTSUsageMetricsData,
+)
+
+from loguru import logger
+
+
+class FrameProcessorMetrics:
+    def __init__(self):
+        self._start_ttfb_time = 0
+        self._start_processing_time = 0
+        self._should_report_ttfb = True
+
+    def _processor_name(self):
+        return self._core_metrics_data.processor
+
+    def _model_name(self):
+        return self._core_metrics_data.model
+
+    def set_core_metrics_data(self, data: MetricsData):
+        self._core_metrics_data = data
+
+    def set_processor_name(self, name: str):
+        self._core_metrics_data = MetricsData(processor=name)
+
+    async def start_ttfb_metrics(self, report_only_initial_ttfb):
+        if self._should_report_ttfb:
+            self._start_ttfb_time = time.time()
+            self._should_report_ttfb = not report_only_initial_ttfb
+
+    async def stop_ttfb_metrics(self):
+        if self._start_ttfb_time == 0:
+            return None
+
+        value = time.time() - self._start_ttfb_time
+        logger.debug(f"{self._processor_name()} TTFB: {value}")
+        ttfb = TTFBMetricsData(
+            processor=self._processor_name(), value=value, model=self._model_name()
+        )
+        self._start_ttfb_time = 0
+        return MetricsFrame(data=[ttfb])
+
+    async def start_processing_metrics(self):
+        self._start_processing_time = time.time()
+
+    async def stop_processing_metrics(self):
+        if self._start_processing_time == 0:
+            return None
+
+        value = time.time() - self._start_processing_time
+        logger.debug(f"{self._processor_name()} processing time: {value}")
+        processing = ProcessingMetricsData(
+            processor=self._processor_name(), value=value, model=self._model_name()
+        )
+        self._start_processing_time = 0
+        return MetricsFrame(data=[processing])
+
+    async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
+        logger.debug(
+            f"{self._processor_name()} prompt tokens: {tokens.prompt_tokens}, completion tokens: {tokens.completion_tokens}"
+        )
+        value = LLMUsageMetricsData(
+            processor=self._processor_name(), model=self._model_name(), value=tokens
+        )
+        return MetricsFrame(data=[value])
+
+    async def start_tts_usage_metrics(self, text: str):
+        characters = TTSUsageMetricsData(
+            processor=self._processor_name(), model=self._model_name(), value=len(text)
+        )
+        logger.debug(f"{self._processor_name()} usage characters: {characters.value}")
+        return MetricsFrame(data=[characters])

src/pipecat/processors/metrics/sentry.py

@@ -0,0 +1,55 @@
+import time
+from loguru import logger
+
+try:
+    import sentry_sdk
+
+    sentry_available = sentry_sdk.is_initialized()
+    if not sentry_available:
+        logger.warning("Sentry SDK not initialized. Sentry features will be disabled.")
+except ImportError:
+    sentry_available = False
+    logger.warning("Sentry SDK not installed. Sentry features will be disabled.")
+
+from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics
+
+
+class SentryMetrics(FrameProcessorMetrics):
+    def __init__(self):
+        super().__init__()
+        self._ttfb_metrics_span = None
+        self._processing_metrics_span = None
+
+    async def start_ttfb_metrics(self, report_only_initial_ttfb):
+        if self._should_report_ttfb:
+            self._start_ttfb_time = time.time()
+            if sentry_available:
+                self._ttfb_metrics_span = sentry_sdk.start_span(
+                    op="ttfb",
+                    description=f"TTFB for {self._processor_name()}",
+                    start_timestamp=self._start_ttfb_time,
+                )
+                logger.debug(f"Sentry Span ID: {self._ttfb_metrics_span.span_id} Description: {
+                             self._ttfb_metrics_span.description} started.")
+            self._should_report_ttfb = not report_only_initial_ttfb
+
+    async def stop_ttfb_metrics(self):
+        stop_time = time.time()
+        if sentry_available:
+            self._ttfb_metrics_span.finish(end_timestamp=stop_time)
+
+    async def start_processing_metrics(self):
+        self._start_processing_time = time.time()
+        if sentry_available:
+            self._processing_metrics_span = sentry_sdk.start_span(
+                op="processing",
+                description=f"Processing for {self._processor_name()}",
+                start_timestamp=self._start_processing_time,
+            )
+            logger.debug(f"Sentry Span ID: {self._processing_metrics_span.span_id} Description: {
+                         self._processing_metrics_span.description} started.")
+
+    async def stop_processing_metrics(self):
+        stop_time = time.time()
+        if sentry_available:
+            self._processing_metrics_span.finish(end_timestamp=stop_time)

src/pipecat/processors/user_idle_processor.py

@@ -12,7 +12,8 @@ from pipecat.frames.frames import (
     BotSpeakingFrame,
     Frame,
     UserStartedSpeakingFrame,
-    UserStoppedSpeakingFrame)
+    UserStoppedSpeakingFrame,
+)
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
 
@@ -24,11 +25,12 @@ class UserIdleProcessor(FrameProcessor):
     """
 
     def __init__(
-            self,
-            *,
-            callback: Callable[["UserIdleProcessor"], Awaitable[None]],
-            timeout: float,
-            **kwargs):
+        self,
+        *,
+        callback: Callable[["UserIdleProcessor"], Awaitable[None]],
+        timeout: float,
+        **kwargs,
+    ):
         super().__init__(sync=False, **kwargs)
 
         self._callback = callback

src/pipecat/serializers/base_serializer.py

@@ -10,7 +10,6 @@ from pipecat.frames.frames import Frame
 
 
 class FrameSerializer(ABC):
-
     @abstractmethod
     def serialize(self, frame: Frame) -> str | bytes | None:
         pass

src/pipecat/serializers/livekit.py

@@ -7,10 +7,7 @@
 import ctypes
 import pickle
 
-from pipecat.frames.frames import (
-    Frame,
-    InputAudioRawFrame,
-    OutputAudioRawFrame)
+from pipecat.frames.frames import Frame, InputAudioRawFrame, OutputAudioRawFrame
 from pipecat.serializers.base_serializer import FrameSerializer
 
 from loguru import logger
@@ -19,8 +16,7 @@ try:
     from livekit.rtc import AudioFrame
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
-    logger.error(
-        "In order to use LiveKit, you need to `pip install pipecat-ai[livekit]`.")
+    logger.error("In order to use LiveKit, you need to `pip install pipecat-ai[livekit]`.")
     raise Exception(f"Missing module: {e}")
 
 
@@ -37,7 +33,7 @@ class LivekitFrameSerializer(FrameSerializer):
         return pickle.dumps(audio_frame)
 
     def deserialize(self, data: str | bytes) -> Frame | None:
-        audio_frame: AudioFrame = pickle.loads(data)['frame']
+        audio_frame: AudioFrame = pickle.loads(data)["frame"]
         return InputAudioRawFrame(
             audio=bytes(audio_frame.data),
             sample_rate=audio_frame.sample_rate,

src/pipecat/serializers/protobuf.py

@@ -8,11 +8,7 @@ import dataclasses
 
 import pipecat.frames.protobufs.frames_pb2 as frame_protos
 
-from pipecat.frames.frames import (
-    AudioRawFrame,
-    Frame,
-    TextFrame,
-    TranscriptionFrame)
+from pipecat.frames.frames import AudioRawFrame, Frame, TextFrame, TranscriptionFrame
 from pipecat.serializers.base_serializer import FrameSerializer
 
 from loguru import logger
@@ -22,7 +18,7 @@ class ProtobufFrameSerializer(FrameSerializer):
     SERIALIZABLE_TYPES = {
         TextFrame: "text",
         AudioRawFrame: "audio",
-        TranscriptionFrame: "transcription"
+        TranscriptionFrame: "transcription",
     }
 
     SERIALIZABLE_FIELDS = {v: k for k, v in SERIALIZABLE_TYPES.items()}

src/pipecat/serializers/twilio.py

@@ -9,10 +9,7 @@ import json
 
 from pydantic import BaseModel
 
-from pipecat.frames.frames import (
-    AudioRawFrame,
-    Frame,
-    StartInterruptionFrame)
+from pipecat.frames.frames import AudioRawFrame, Frame, StartInterruptionFrame
 from pipecat.serializers.base_serializer import FrameSerializer
 from pipecat.utils.audio import ulaw_to_pcm, pcm_to_ulaw
 
@@ -30,15 +27,12 @@ class TwilioFrameSerializer(FrameSerializer):
         if isinstance(frame, AudioRawFrame):
             data = frame.audio
 
-            serialized_data = pcm_to_ulaw(
-                data, frame.sample_rate, self._params.twilio_sample_rate)
+            serialized_data = pcm_to_ulaw(data, frame.sample_rate, self._params.twilio_sample_rate)
             payload = base64.b64encode(serialized_data).decode("utf-8")
             answer = {
                 "event": "media",
                 "streamSid": self._stream_sid,
-                "media": {
-                    "payload": payload
-                }
+                "media": {"payload": payload},
             }
 
             return json.dumps(answer)
@@ -57,11 +51,9 @@ class TwilioFrameSerializer(FrameSerializer):
             payload = base64.b64decode(payload_base64)
 
             deserialized_data = ulaw_to_pcm(
-                payload,
-                self._params.twilio_sample_rate,
-                self._params.sample_rate)
+                payload, self._params.twilio_sample_rate, self._params.sample_rate
+            )
             audio_frame = AudioRawFrame(
-                audio=deserialized_data,
-                num_channels=1,
-                sample_rate=self._params.sample_rate)
+                audio=deserialized_data, num_channels=1, sample_rate=self._params.sample_rate
+            )
             return audio_frame

src/pipecat/services/ai_services.py

@@ -31,7 +31,7 @@ from pipecat.frames.frames import (
     TTSVoiceUpdateFrame,
     TextFrame,
     UserImageRequestFrame,
-    VisionImageRawFrame
+    VisionImageRawFrame,
 )
 from pipecat.metrics.metrics import MetricsData
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -114,12 +114,8 @@ class LLMService(AIService):
         return function_name in self._callbacks.keys()
 
     async def call_function(
-            self,
-            *,
-            context: OpenAILLMContext,
-            tool_call_id: str,
-            function_name: str,
-            arguments: str) -> None:
+        self, *, context: OpenAILLMContext, tool_call_id: str, function_name: str, arguments: str
+    ) -> None:
         f = None
         if function_name in self._callbacks.keys():
             f = self._callbacks[function_name]
@@ -128,11 +124,8 @@ class LLMService(AIService):
         else:
             return None
         await context.call_function(
-            f,
-            function_name=function_name,
-            tool_call_id=tool_call_id,
-            arguments=arguments,
-            llm=self)
+            f, function_name=function_name, tool_call_id=tool_call_id, arguments=arguments, llm=self
+        )
 
     # QUESTION FOR CB: maybe this isn't needed anymore?
     async def call_start_function(self, context: OpenAILLMContext, function_name: str):
@@ -142,21 +135,23 @@ class LLMService(AIService):
             return await self._start_callbacks[None](function_name, self, context)
 
     async def request_image_frame(self, user_id: str, *, text_content: str | None = None):
-        await self.push_frame(UserImageRequestFrame(user_id=user_id, context=text_content),
-                              FrameDirection.UPSTREAM)
+        await self.push_frame(
+            UserImageRequestFrame(user_id=user_id, context=text_content), FrameDirection.UPSTREAM
+        )
 
 
 class TTSService(AIService):
     def __init__(
-            self,
-            *,
-            aggregate_sentences: bool = True,
-            # if True, TTSService will push TextFrames and LLMFullResponseEndFrames,
-            # otherwise subclass must do it
-            push_text_frames: bool = True,
-            # TTS output sample rate
-            sample_rate: int = 16000,
-            **kwargs):
+        self,
+        *,
+        aggregate_sentences: bool = True,
+        # if True, TTSService will push TextFrames and LLMFullResponseEndFrames,
+        # otherwise subclass must do it
+        push_text_frames: bool = True,
+        # TTS output sample rate
+        sample_rate: int = 16000,
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self._aggregate_sentences: bool = aggregate_sentences
         self._push_text_frames: bool = push_text_frames
@@ -247,12 +242,13 @@ class TTSService(AIService):
 
 class AsyncTTSService(TTSService):
     def __init__(
-            self,
-            # if True, TTSService will push TTSStoppedFrames, otherwise subclass must do it
-            push_stop_frames: bool = False,
-            # if push_stop_frames is True, wait for this idle period before pushing TTSStoppedFrame
-            stop_frame_timeout_s: float = 1.0,
-            **kwargs):
+        self,
+        # if True, TTSService will push TTSStoppedFrames, otherwise subclass must do it
+        push_stop_frames: bool = False,
+        # if push_stop_frames is True, wait for this idle period before pushing TTSStoppedFrame
+        stop_frame_timeout_s: float = 1.0,
+        **kwargs,
+    ):
         super().__init__(sync=False, **kwargs)
         self._push_stop_frames: bool = push_stop_frames
         self._stop_frame_timeout_s: float = stop_frame_timeout_s
@@ -286,10 +282,11 @@ class AsyncTTSService(TTSService):
         await super().push_frame(frame, direction)
 
         if self._push_stop_frames and (
-                isinstance(frame, StartInterruptionFrame) or
-                isinstance(frame, TTSStartedFrame) or
-                isinstance(frame, TTSAudioRawFrame) or
-                isinstance(frame, TTSStoppedFrame)):
+            isinstance(frame, StartInterruptionFrame)
+            or isinstance(frame, TTSStartedFrame)
+            or isinstance(frame, TTSAudioRawFrame)
+            or isinstance(frame, TTSStoppedFrame)
+        ):
             await self._stop_frame_queue.put(frame)
 
     async def _stop_frame_handler(self):
@@ -297,8 +294,9 @@ class AsyncTTSService(TTSService):
             has_started = False
             while True:
                 try:
-                    frame = await asyncio.wait_for(self._stop_frame_queue.get(),
-                                                   self._stop_frame_timeout_s)
+                    frame = await asyncio.wait_for(
+                        self._stop_frame_queue.get(), self._stop_frame_timeout_s
+                    )
                     if isinstance(frame, TTSStartedFrame):
                         has_started = True
                     elif isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
@@ -327,7 +325,7 @@ class AsyncWordTTSService(AsyncTTSService):
         self._word_timestamps = []
 
     async def add_word_timestamps(self, word_times: List[Tuple[str, float]]):
-        for (word, timestamp) in word_times:
+        for word, timestamp in word_times:
             await self._words_queue.put((word, seconds_to_nanoseconds(timestamp)))
 
     async def stop(self, frame: EndFrame):
@@ -414,14 +412,16 @@ class SegmentedSTTService(STTService):
 
     """
 
-    def __init__(self,
-                 *,
-                 min_volume: float = 0.6,
-                 max_silence_secs: float = 0.3,
-                 max_buffer_secs: float = 1.5,
-                 sample_rate: int = 16000,
-                 num_channels: int = 1,
-                 **kwargs):
+    def __init__(
+        self,
+        *,
+        min_volume: float = 0.6,
+        max_silence_secs: float = 0.3,
+        max_buffer_secs: float = 1.5,
+        sample_rate: int = 16000,
+        num_channels: int = 1,
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self._min_volume = min_volume
         self._max_silence_secs = max_silence_secs
@@ -450,7 +450,8 @@ class SegmentedSTTService(STTService):
         silence_secs = self._silence_num_frames / self._sample_rate
         buffer_secs = self._wave.getnframes() / self._sample_rate
         if self._content.tell() > 0 and (
-                buffer_secs > self._max_buffer_secs or silence_secs > self._max_silence_secs):
+            buffer_secs > self._max_buffer_secs or silence_secs > self._max_silence_secs
+        ):
             self._silence_num_frames = 0
             self._wave.close()
             self._content.seek(0)
@@ -477,7 +478,6 @@ class SegmentedSTTService(STTService):
 
 
 class ImageGenService(AIService):
-
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 

src/pipecat/services/anthropic.py

@@ -28,18 +28,18 @@ from pipecat.frames.frames import (
     LLMFullResponseEndFrame,
     FunctionCallResultFrame,
     FunctionCallInProgressFrame,
-    StartInterruptionFrame
+    StartInterruptionFrame,
 )
 from pipecat.metrics.metrics import LLMTokenUsage
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.ai_services import LLMService
 from pipecat.processors.aggregators.openai_llm_context import (
     OpenAILLMContext,
-    OpenAILLMContextFrame
+    OpenAILLMContextFrame,
 )
 from pipecat.processors.aggregators.llm_response import (
     LLMUserContextAggregator,
-    LLMAssistantContextAggregator
+    LLMAssistantContextAggregator,
 )
 
 from loguru import logger
@@ -49,8 +49,9 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use Anthropic, you need to `pip install pipecat-ai[anthropic]`. " +
-        "Also, set `ANTHROPIC_API_KEY` environment variable.")
+        "In order to use Anthropic, you need to `pip install pipecat-ai[anthropic]`. "
+        + "Also, set `ANTHROPIC_API_KEY` environment variable."
+    )
     raise Exception(f"Missing module: {e}")
 
 
@@ -62,19 +63,19 @@ class AnthropicImageMessageFrame(Frame):
 
 @dataclass
 class AnthropicContextAggregatorPair:
-    _user: 'AnthropicUserContextAggregator'
-    _assistant: 'AnthropicAssistantContextAggregator'
+    _user: "AnthropicUserContextAggregator"
+    _assistant: "AnthropicAssistantContextAggregator"
 
-    def user(self) -> 'AnthropicUserContextAggregator':
+    def user(self) -> "AnthropicUserContextAggregator":
         return self._user
 
-    def assistant(self) -> 'AnthropicAssistantContextAggregator':
+    def assistant(self) -> "AnthropicAssistantContextAggregator":
         return self._assistant
 
 
 class AnthropicLLMService(LLMService):
-    """This class implements inference with Anthropic's AI models
-    """
+    """This class implements inference with Anthropic's AI models"""
+
     class InputParams(BaseModel):
         enable_prompt_caching_beta: Optional[bool] = False
         max_tokens: Optional[int] = Field(default_factory=lambda: 4096, ge=1)
@@ -84,12 +85,13 @@ class AnthropicLLMService(LLMService):
         extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
 
     def __init__(
-            self,
-            *,
-            api_key: str,
-            model: str = "claude-3-5-sonnet-20240620",
-            params: InputParams = InputParams(),
-            **kwargs):
+        self,
+        *,
+        api_key: str,
+        model: str = "claude-3-5-sonnet-20240620",
+        params: InputParams = InputParams(),
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self._client = AsyncAnthropic(api_key=api_key)
         self.set_model_name(model)
@@ -111,10 +113,7 @@ class AnthropicLLMService(LLMService):
     def create_context_aggregator(context: OpenAILLMContext) -> AnthropicContextAggregatorPair:
         user = AnthropicUserContextAggregator(context)
         assistant = AnthropicAssistantContextAggregator(user)
-        return AnthropicContextAggregatorPair(
-            _user=user,
-            _assistant=assistant
-        )
+        return AnthropicContextAggregatorPair(_user=user, _assistant=assistant)
 
     async def set_enable_prompt_caching_beta(self, enable_prompt_caching_beta: bool):
         logger.debug(f"Switching LLM enable_prompt_caching_beta to: [{enable_prompt_caching_beta}]")
@@ -157,7 +156,8 @@ class AnthropicLLMService(LLMService):
             await self.start_processing_metrics()
 
             logger.debug(
-                f"Generating chat: {context.system} | {context.get_messages_for_logging()}")
+                f"Generating chat: {context.system} | {context.get_messages_for_logging()}"
+            )
 
             messages = context.messages
             if self._enable_prompt_caching_beta:
@@ -178,7 +178,7 @@ class AnthropicLLMService(LLMService):
                 "stream": True,
                 "temperature": self._temperature,
                 "top_k": self._top_k,
-                "top_p": self._top_p
+                "top_p": self._top_p,
             }
 
             params.update(self._extra)
@@ -189,54 +189,70 @@ class AnthropicLLMService(LLMService):
 
             # Function calling
             tool_use_block = None
-            json_accumulator = ''
+            json_accumulator = ""
 
             async for event in response:
                 # logger.debug(f"Anthropic LLM event: {event}")
 
                 # Aggregate streaming content, create frames, trigger events
 
-                if (event.type == "content_block_delta"):
-                    if hasattr(event.delta, 'text'):
+                if event.type == "content_block_delta":
+                    if hasattr(event.delta, "text"):
                         await self.push_frame(TextFrame(event.delta.text))
                         completion_tokens_estimate += self._estimate_tokens(event.delta.text)
-                    elif hasattr(event.delta, 'partial_json') and tool_use_block:
+                    elif hasattr(event.delta, "partial_json") and tool_use_block:
                         json_accumulator += event.delta.partial_json
                         completion_tokens_estimate += self._estimate_tokens(
-                            event.delta.partial_json)
-                elif (event.type == "content_block_start"):
+                            event.delta.partial_json
+                        )
+                elif event.type == "content_block_start":
                     if event.content_block.type == "tool_use":
                         tool_use_block = event.content_block
-                        json_accumulator = ''
-                elif ((event.type == "message_delta" and
-                      hasattr(event.delta, 'stop_reason')
-                      and event.delta.stop_reason == 'tool_use')):
+                        json_accumulator = ""
+                elif (
+                    event.type == "message_delta"
+                    and hasattr(event.delta, "stop_reason")
+                    and event.delta.stop_reason == "tool_use"
+                ):
                     if tool_use_block:
-                        await self.call_function(context=context,
-                                                 tool_call_id=tool_use_block.id,
-                                                 function_name=tool_use_block.name,
-                                                 arguments=json.loads(json_accumulator) if json_accumulator else dict()
-                                                 )
+                        await self.call_function(
+                            context=context,
+                            tool_call_id=tool_use_block.id,
+                            function_name=tool_use_block.name,
+                            arguments=json.loads(json_accumulator) if json_accumulator else dict(),
+                        )
 
                 # Calculate usage. Do this here in its own if statement, because there may be usage
                 # data embedded in messages that we do other processing for, above.
                 if hasattr(event, "usage"):
-                    prompt_tokens += event.usage.input_tokens if hasattr(
-                        event.usage, "input_tokens") else 0
-                    completion_tokens += event.usage.output_tokens if hasattr(
-                        event.usage, "output_tokens") else 0
+                    prompt_tokens += (
+                        event.usage.input_tokens if hasattr(event.usage, "input_tokens") else 0
+                    )
+                    completion_tokens += (
+                        event.usage.output_tokens if hasattr(event.usage, "output_tokens") else 0
+                    )
                 elif hasattr(event, "message") and hasattr(event.message, "usage"):
-                    prompt_tokens += event.message.usage.input_tokens if hasattr(
-                        event.message.usage, "input_tokens") else 0
-                    completion_tokens += event.message.usage.output_tokens if hasattr(
-                        event.message.usage, "output_tokens") else 0
+                    prompt_tokens += (
+                        event.message.usage.input_tokens
+                        if hasattr(event.message.usage, "input_tokens")
+                        else 0
+                    )
+                    completion_tokens += (
+                        event.message.usage.output_tokens
+                        if hasattr(event.message.usage, "output_tokens")
+                        else 0
+                    )
                     if hasattr(event.message.usage, "cache_creation_input_tokens"):
-                        cache_creation_input_tokens += event.message.usage.cache_creation_input_tokens
+                        cache_creation_input_tokens += (
+                            event.message.usage.cache_creation_input_tokens
+                        )
                         logger.debug(f"Cache creation input tokens: {cache_creation_input_tokens}")
                     if hasattr(event.message.usage, "cache_read_input_tokens"):
                         cache_read_input_tokens += event.message.usage.cache_read_input_tokens
                         logger.debug(f"Cache read input tokens: {cache_read_input_tokens}")
-                    total_input_tokens = prompt_tokens + cache_creation_input_tokens + cache_read_input_tokens
+                    total_input_tokens = (
+                        prompt_tokens + cache_creation_input_tokens + cache_read_input_tokens
+                    )
                     if total_input_tokens >= 1024:
                         context.turns_above_cache_threshold += 1
 
@@ -251,12 +267,16 @@ class AnthropicLLMService(LLMService):
         finally:
             await self.stop_processing_metrics()
             await self.push_frame(LLMFullResponseEndFrame())
-            comp_tokens = completion_tokens if not use_completion_tokens_estimate else completion_tokens_estimate
+            comp_tokens = (
+                completion_tokens
+                if not use_completion_tokens_estimate
+                else completion_tokens_estimate
+            )
             await self._report_usage_metrics(
                 prompt_tokens=prompt_tokens,
                 completion_tokens=comp_tokens,
                 cache_creation_input_tokens=cache_creation_input_tokens,
-                cache_read_input_tokens=cache_read_input_tokens
+                cache_read_input_tokens=cache_read_input_tokens,
             )
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -286,21 +306,27 @@ class AnthropicLLMService(LLMService):
             await self._process_context(context)
 
     def _estimate_tokens(self, text: str) -> int:
-        return int(len(re.split(r'[^\w]+', text)) * 1.3)
+        return int(len(re.split(r"[^\w]+", text)) * 1.3)
 
     async def _report_usage_metrics(
-            self,
-            prompt_tokens: int,
-            completion_tokens: int,
-            cache_creation_input_tokens: int,
-            cache_read_input_tokens: int):
-        if prompt_tokens or completion_tokens or cache_creation_input_tokens or cache_read_input_tokens:
+        self,
+        prompt_tokens: int,
+        completion_tokens: int,
+        cache_creation_input_tokens: int,
+        cache_read_input_tokens: int,
+    ):
+        if (
+            prompt_tokens
+            or completion_tokens
+            or cache_creation_input_tokens
+            or cache_read_input_tokens
+        ):
             tokens = LLMTokenUsage(
                 prompt_tokens=prompt_tokens,
                 completion_tokens=completion_tokens,
                 cache_creation_input_tokens=cache_creation_input_tokens,
                 cache_read_input_tokens=cache_read_input_tokens,
-                total_tokens=prompt_tokens + completion_tokens
+                total_tokens=prompt_tokens + completion_tokens,
             )
             await self.start_llm_usage_metrics(tokens)
 
@@ -312,7 +338,7 @@ class AnthropicLLMContext(OpenAILLMContext):
         tools: list[dict] | None = None,
         tool_choice: dict | None = None,
         *,
-        system: str | NotGiven = NOT_GIVEN
+        system: str | NotGiven = NOT_GIVEN,
     ):
         super().__init__(messages=messages, tools=tools, tool_choice=tool_choice)
         self._user_image_request_context = {}
@@ -345,10 +371,8 @@ class AnthropicLLMContext(OpenAILLMContext):
     def from_image_frame(cls, frame: VisionImageRawFrame) -> "AnthropicLLMContext":
         context = cls()
         context.add_image_frame_message(
-            format=frame.format,
-            size=frame.size,
-            image=frame.image,
-            text=frame.text)
+            format=frame.format, size=frame.size, image=frame.image, text=frame.text
+        )
         return context
 
     def set_messages(self, messages: List):
@@ -357,18 +381,23 @@ class AnthropicLLMContext(OpenAILLMContext):
         self._restructure_from_openai_messages()
 
     def add_image_frame_message(
-            self, *, format: str, size: tuple[int, int], image: bytes, text: str = None):
+        self, *, format: str, size: tuple[int, int], image: bytes, text: str = None
+    ):
         buffer = io.BytesIO()
         Image.frombytes(format, size, image).save(buffer, format="JPEG")
         encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
 
         # Anthropic docs say that the image should be the first content block in the message.
-        content = [{"type": "image",
-                    "source": {
-                        "type": "base64",
-                        "media_type": "image/jpeg",
-                        "data": encoded_image,
-                    }}]
+        content = [
+            {
+                "type": "image",
+                "source": {
+                    "type": "base64",
+                    "media_type": "image/jpeg",
+                    "data": encoded_image,
+                },
+            }
+        ]
         if text:
             content.append({"type": "text", "text": text})
         self.add_message({"role": "user", "content": content})
@@ -382,8 +411,9 @@ class AnthropicLLMContext(OpenAILLMContext):
                     # if the last message has just a content string, convert it to a list
                     # in the proper format
                     if isinstance(self.messages[-1]["content"], str):
-                        self.messages[-1]["content"] = [{"type": "text",
-                                                         "text": self.messages[-1]["content"]}]
+                        self.messages[-1]["content"] = [
+                            {"type": "text", "text": self.messages[-1]["content"]}
+                        ]
                     # if this message has just a content string, convert it to a list
                     # in the proper format
                     if isinstance(message["content"], str):
@@ -404,8 +434,11 @@ class AnthropicLLMContext(OpenAILLMContext):
                 if isinstance(messages[-1]["content"], str):
                     messages[-1]["content"] = [{"type": "text", "text": messages[-1]["content"]}]
                 messages[-1]["content"][-1]["cache_control"] = {"type": "ephemeral"}
-            if (self.turns_above_cache_threshold >= 2 and
-                    len(messages) > 2 and messages[-3]["role"] == "user"):
+            if (
+                self.turns_above_cache_threshold >= 2
+                and len(messages) > 2
+                and messages[-3]["role"] == "user"
+            ):
                 if isinstance(messages[-3]["content"], str):
                     messages[-3]["content"] = [{"type": "text", "text": messages[-3]["content"]}]
                 messages[-3]["content"][-1]["cache_control"] = {"type": "ephemeral"}
@@ -459,12 +492,13 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator):
                 # The LLM sends a UserImageRequestFrame upstream. Cache any context provided with
                 # that frame so we can use it when we assemble the image message in the assistant
                 # context aggregator.
-                if (frame.context):
+                if frame.context:
                     if isinstance(frame.context, str):
                         self._context._user_image_request_context[frame.user_id] = frame.context
                     else:
                         logger.error(
-                            f"Unexpected UserImageRequestFrame context type: {type(frame.context)}")
+                            f"Unexpected UserImageRequestFrame context type: {type(frame.context)}"
+                        )
                         del self._context._user_image_request_context[frame.user_id]
                 else:
                     if frame.user_id in self._context._user_image_request_context:
@@ -481,6 +515,7 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator):
         except Exception as e:
             logger.error(f"Error processing frame: {e}")
 
+
 #
 # Claude returns a text content block along with a tool use content block. This works quite nicely
 # with streaming. We get the text first, so we can start streaming it right away. Then we get the
@@ -508,13 +543,16 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
         elif isinstance(frame, FunctionCallInProgressFrame):
             self._function_call_in_progress = frame
         elif isinstance(frame, FunctionCallResultFrame):
-            if (self._function_call_in_progress and self._function_call_in_progress.tool_call_id ==
-                    frame.tool_call_id):
+            if (
+                self._function_call_in_progress
+                and self._function_call_in_progress.tool_call_id == frame.tool_call_id
+            ):
                 self._function_call_in_progress = None
                 self._function_call_result = frame
             else:
                 logger.warning(
-                    "FunctionCallResultFrame tool_call_id != InProgressFrame tool_call_id")
+                    "FunctionCallResultFrame tool_call_id != InProgressFrame tool_call_id"
+                )
                 self._function_call_in_progress = None
                 self._function_call_result = None
         elif isinstance(frame, AnthropicImageMessageFrame):
@@ -534,31 +572,32 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
                 frame = self._function_call_result
                 self._function_call_result = None
                 if frame.result:
-                    self._context.add_message({
-                        "role": "assistant",
-                        "content": [
-                            {
-                                "type": "text",
-                                "text": aggregation
-                            },
-                            {
-                                "type": "tool_use",
-                                "id": frame.tool_call_id,
-                                "name": frame.function_name,
-                                "input": frame.arguments
-                            }
-                        ]
-                    })
-                    self._context.add_message({
-                        "role": "user",
-                        "content": [
-                            {
-                                "type": "tool_result",
-                                "tool_use_id": frame.tool_call_id,
-                                "content": json.dumps(frame.result)
-                            }
-                        ]
-                    })
+                    self._context.add_message(
+                        {
+                            "role": "assistant",
+                            "content": [
+                                {"type": "text", "text": aggregation},
+                                {
+                                    "type": "tool_use",
+                                    "id": frame.tool_call_id,
+                                    "name": frame.function_name,
+                                    "input": frame.arguments,
+                                },
+                            ],
+                        }
+                    )
+                    self._context.add_message(
+                        {
+                            "role": "user",
+                            "content": [
+                                {
+                                    "type": "tool_result",
+                                    "tool_use_id": frame.tool_call_id,
+                                    "content": json.dumps(frame.result),
+                                }
+                            ],
+                        }
+                    )
                     run_llm = True
             else:
                 self._context.add_message({"role": "assistant", "content": aggregation})
@@ -570,7 +609,8 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
                     format=frame.user_image_raw_frame.format,
                     size=frame.user_image_raw_frame.size,
                     image=frame.user_image_raw_frame.image,
-                    text=frame.text)
+                    text=frame.text,
+                )
                 run_llm = True
 
             if run_llm:

src/pipecat/services/azure.py

@@ -21,7 +21,8 @@ from pipecat.frames.frames import (
     TTSStartedFrame,
     TTSStoppedFrame,
     TranscriptionFrame,
-    URLImageRawFrame)
+    URLImageRawFrame,
+)
 from pipecat.metrics.metrics import TTSUsageMetricsData
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.ai_services import STTService, TTSService, ImageGenService
@@ -45,18 +46,15 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use Azure, you need to `pip install pipecat-ai[azure]`. Also, set `AZURE_SPEECH_API_KEY` and `AZURE_SPEECH_REGION` environment variables.")
+        "In order to use Azure, you need to `pip install pipecat-ai[azure]`. Also, set `AZURE_SPEECH_API_KEY` and `AZURE_SPEECH_REGION` environment variables."
+    )
     raise Exception(f"Missing module: {e}")
 
 
 class AzureLLMService(BaseOpenAILLMService):
     def __init__(
-            self,
-            *,
-            api_key: str,
-            endpoint: str,
-            model: str,
-            api_version: str = "2023-12-01-preview"):
+        self, *, api_key: str, endpoint: str, model: str, api_version: str = "2023-12-01-preview"
+    ):
         # Initialize variables before calling parent __init__() because that
         # will call create_client() and we need those values there.
         self._endpoint = endpoint
@@ -73,13 +71,14 @@ class AzureLLMService(BaseOpenAILLMService):
 
 class AzureTTSService(TTSService):
     def __init__(
-            self,
-            *,
-            api_key: str,
-            region: str,
-            voice="en-US-SaraNeural",
-            sample_rate: int = 16000,
-            **kwargs):
+        self,
+        *,
+        api_key: str,
+        region: str,
+        voice="en-US-SaraNeural",
+        sample_rate: int = 16000,
+        **kwargs,
+    ):
         super().__init__(sample_rate=sample_rate, **kwargs)
 
         speech_config = SpeechConfig(subscription=api_key, region=region)
@@ -108,7 +107,8 @@ class AzureTTSService(TTSService):
             "<mstts:express-as style='lyrical' styledegree='2' role='SeniorFemale'>"
             "<prosody rate='1.05'>"
             f"{text}"
-            "</prosody></mstts:express-as></voice></speak> ")
+            "</prosody></mstts:express-as></voice></speak> "
+        )
 
         result = await asyncio.to_thread(self._speech_synthesizer.speak_ssml, (ssml))
 
@@ -117,7 +117,9 @@ class AzureTTSService(TTSService):
             await self.stop_ttfb_metrics()
             await self.push_frame(TTSStartedFrame())
             # Azure always sends a 44-byte header. Strip it off.
-            yield TTSAudioRawFrame(audio=result.audio_data[44:], sample_rate=self._sample_rate, num_channels=1)
+            yield TTSAudioRawFrame(
+                audio=result.audio_data[44:], sample_rate=self._sample_rate, num_channels=1
+            )
             await self.push_frame(TTSStoppedFrame())
         elif result.reason == ResultReason.Canceled:
             cancellation_details = result.cancellation_details
@@ -128,14 +130,15 @@ class AzureTTSService(TTSService):
 
 class AzureSTTService(STTService):
     def __init__(
-            self,
-            *,
-            api_key: str,
-            region: str,
-            language="en-US",
-            sample_rate=16000,
-            channels=1,
-            **kwargs):
+        self,
+        *,
+        api_key: str,
+        region: str,
+        language="en-US",
+        sample_rate=16000,
+        channels=1,
+        **kwargs,
+    ):
         super().__init__(**kwargs)
 
         speech_config = SpeechConfig(subscription=api_key, region=region)
@@ -146,7 +149,8 @@ class AzureSTTService(STTService):
 
         audio_config = AudioConfig(stream=self._audio_stream)
         self._speech_recognizer = SpeechRecognizer(
-            speech_config=speech_config, audio_config=audio_config)
+            speech_config=speech_config, audio_config=audio_config
+        )
         self._speech_recognizer.recognized.connect(self._on_handle_recognized)
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
@@ -176,7 +180,6 @@ class AzureSTTService(STTService):
 
 
 class AzureImageGenServiceREST(ImageGenService):
-
     def __init__(
         self,
         *,
@@ -199,9 +202,7 @@ class AzureImageGenServiceREST(ImageGenService):
     async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
         url = f"{self._azure_endpoint}openai/images/generations:submit?api-version={self._api_version}"
 
-        headers = {
-            "api-key": self._api_key,
-            "Content-Type": "application/json"}
+        headers = {"api-key": self._api_key, "Content-Type": "application/json"}
 
         body = {
             # Enter your prompt text here
@@ -243,8 +244,6 @@ class AzureImageGenServiceREST(ImageGenService):
                 image_stream = io.BytesIO(await response.content.read())
                 image = Image.open(image_stream)
                 frame = URLImageRawFrame(
-                    url=image_url,
-                    image=image.tobytes(),
-                    size=image.size,
-                    format=image.format)
+                    url=image_url, image=image.tobytes(), size=image.size, format=image.format
+                )
                 yield frame

src/pipecat/services/cartesia.py

@@ -22,7 +22,7 @@ from pipecat.frames.frames import (
     TTSAudioRawFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
-    LLMFullResponseEndFrame
+    LLMFullResponseEndFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.transcriptions.language import Language
@@ -37,7 +37,8 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use Cartesia, you need to `pip install pipecat-ai[cartesia]`. Also, set `CARTESIA_API_KEY` environment variable.")
+        "In order to use Cartesia, you need to `pip install pipecat-ai[cartesia]`. Also, set `CARTESIA_API_KEY` environment variable."
+    )
     raise Exception(f"Missing module: {e}")
 
 
@@ -89,8 +90,9 @@ class CartesiaTTSService(AsyncWordTTSService):
         # if we're interrupted. Cartesia gives us word-by-word timestamps. We
         # can use those to generate text frames ourselves aligned with the
         # playout timing of the audio!
-        super().__init__(aggregate_sentences=True,
-                         push_text_frames=False, sample_rate=sample_rate, **kwargs)
+        super().__init__(
+            aggregate_sentences=True, push_text_frames=False, sample_rate=sample_rate, **kwargs
+        )
 
         self._api_key = api_key
         self._cartesia_version = cartesia_version
@@ -193,10 +195,7 @@ class CartesiaTTSService(AsyncWordTTSService):
             "continue": False,
             "context_id": self._context_id,
             "model_id": self.model_name,
-            "voice": {
-                "mode": "id",
-                "id": self._voice_id
-            },
+            "voice": {"mode": "id", "id": self._voice_id},
             "output_format": self._output_format,
             "language": self._language,
             "add_timestamps": True,
@@ -228,7 +227,7 @@ class CartesiaTTSService(AsyncWordTTSService):
                     frame = TTSAudioRawFrame(
                         audio=base64.b64decode(msg["data"]),
                         sample_rate=self._output_format["sample_rate"],
-                        num_channels=1
+                        num_channels=1,
                     )
                     await self.push_frame(frame)
                 elif msg["type"] == "error":
@@ -293,18 +292,18 @@ class CartesiaTTSService(AsyncWordTTSService):
 
 
 class CartesiaHttpTTSService(TTSService):
-
     def __init__(
-            self,
-            *,
-            api_key: str,
-            voice_id: str,
-            model_id: str = "sonic-english",
-            base_url: str = "https://api.cartesia.ai",
-            encoding: str = "pcm_s16le",
-            sample_rate: int = 16000,
-            language: str = "en",
-            **kwargs):
+        self,
+        *,
+        api_key: str,
+        voice_id: str,
+        model_id: str = "sonic-english",
+        base_url: str = "https://api.cartesia.ai",
+        encoding: str = "pcm_s16le",
+        sample_rate: int = 16000,
+        language: str = "en",
+        **kwargs,
+    ):
         super().__init__(**kwargs)
 
         self._api_key = api_key
@@ -355,7 +354,7 @@ class CartesiaHttpTTSService(TTSService):
                 voice_id=self._voice_id,
                 output_format=self._output_format,
                 language=self._language,
-                stream=False
+                stream=False,
             )
 
             await self.stop_ttfb_metrics()
@@ -363,7 +362,7 @@ class CartesiaHttpTTSService(TTSService):
             frame = TTSAudioRawFrame(
                 audio=output["audio"],
                 sample_rate=self._output_format["sample_rate"],
-                num_channels=1
+                num_channels=1,
             )
             yield frame
         except Exception as e:

src/pipecat/services/deepgram.py

@@ -18,7 +18,8 @@ from pipecat.frames.frames import (
     TTSAudioRawFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
-    TranscriptionFrame)
+    TranscriptionFrame,
+)
 from pipecat.services.ai_services import STTService, TTSService
 from pipecat.transcriptions.language import Language
 from pipecat.utils.time import time_now_iso8601
@@ -34,27 +35,28 @@ try:
         DeepgramClientOptions,
         LiveTranscriptionEvents,
         LiveOptions,
-        LiveResultResponse
+        LiveResultResponse,
     )
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use Deepgram, you need to `pip install pipecat-ai[deepgram]`. Also, set `DEEPGRAM_API_KEY` environment variable.")
+        "In order to use Deepgram, you need to `pip install pipecat-ai[deepgram]`. Also, set `DEEPGRAM_API_KEY` environment variable."
+    )
     raise Exception(f"Missing module: {e}")
 
 
 class DeepgramTTSService(TTSService):
-
     def __init__(
-            self,
-            *,
-            api_key: str,
-            aiohttp_session: aiohttp.ClientSession,
-            voice: str = "aura-helios-en",
-            base_url: str = "https://api.deepgram.com/v1/speak",
-            sample_rate: int = 16000,
-            encoding: str = "linear16",
-            **kwargs):
+        self,
+        *,
+        api_key: str,
+        aiohttp_session: aiohttp.ClientSession,
+        voice: str = "aura-helios-en",
+        base_url: str = "https://api.deepgram.com/v1/speak",
+        sample_rate: int = 16000,
+        encoding: str = "linear16",
+        **kwargs,
+    ):
         super().__init__(**kwargs)
 
         self._voice = voice
@@ -75,7 +77,8 @@ class DeepgramTTSService(TTSService):
         logger.debug(f"Generating TTS: [{text}]")
 
         base_url = self._base_url
-        request_url = f"{base_url}?model={self._voice}&encoding={self._encoding}&container=none&sample_rate={self._sample_rate}"
+        request_url = f"{base_url}?model={self._voice}&encoding={
+            self._encoding}&container=none&sample_rate={self._sample_rate}"
         headers = {"authorization": f"token {self._api_key}"}
         body = {"text": text}
 
@@ -92,8 +95,11 @@ class DeepgramTTSService(TTSService):
                         return
 
                     logger.error(
-                        f"{self} error getting audio (status: {r.status}, error: {response_text})")
-                    yield ErrorFrame(f"Error getting audio (status: {r.status}, error: {response_text})")
+                        f"{self} error getting audio (status: {r.status}, error: {response_text})"
+                    )
+                    yield ErrorFrame(
+                        f"Error getting audio (status: {r.status}, error: {response_text})"
+                    )
                     return
 
                 await self.start_tts_usage_metrics(text)
@@ -102,7 +108,8 @@ class DeepgramTTSService(TTSService):
                 async for data in r.content:
                     await self.stop_ttfb_metrics()
                     frame = TTSAudioRawFrame(
-                        audio=data, sample_rate=self._sample_rate, num_channels=1)
+                        audio=data, sample_rate=self._sample_rate, num_channels=1
+                    )
                     yield frame
                 await self.push_frame(TTSStoppedFrame())
         except Exception as e:
@@ -110,30 +117,43 @@ class DeepgramTTSService(TTSService):
 
 
 class DeepgramSTTService(STTService):
-    def __init__(self,
-                 *,
-                 api_key: str,
-                 url: str = "",
-                 live_options: LiveOptions = LiveOptions(
-                     encoding="linear16",
-                     language="en-US",
-                     model="nova-2-conversationalai",
-                     sample_rate=16000,
-                     channels=1,
-                     interim_results=True,
-                     smart_format=True,
-                     punctuate=True,
-                     profanity_filter=True,
-                 ),
-                 **kwargs):
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        url: str = "",
+        live_options: LiveOptions = LiveOptions(
+            encoding="linear16",
+            language="en-US",
+            model="nova-2-conversationalai",
+            sample_rate=16000,
+            channels=1,
+            interim_results=True,
+            smart_format=True,
+            punctuate=True,
+            profanity_filter=True,
+            vad_events=False,
+        ),
+        **kwargs,
+    ):
         super().__init__(**kwargs)
 
         self._live_options = live_options
 
         self._client = DeepgramClient(
-            api_key, config=DeepgramClientOptions(url=url, options={"keepalive": "true"}))
+            api_key, config=DeepgramClientOptions(url=url, options={"keepalive": "true"})
+        )
         self._connection: AsyncListenWebSocketClient = self._client.listen.asyncwebsocket.v("1")
         self._connection.on(LiveTranscriptionEvents.Transcript, self._on_message)
+        if self.vad_enabled:
+            self._connection.on(LiveTranscriptionEvents.SpeechStarted, self._on_speech_started)
+
+    @property
+    def vad_enabled(self):
+        return self._live_options.vad_events
+
+    def can_generate_metrics(self) -> bool:
+        return self.vad_enabled
 
     async def set_model(self, model: str):
         await super().set_model(model)
@@ -161,9 +181,7 @@ class DeepgramSTTService(STTService):
         await self._disconnect()
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
-        await self.start_processing_metrics()
         await self._connection.send(audio)
-        await self.stop_processing_metrics()
         yield None
 
     async def _connect(self):
@@ -177,6 +195,10 @@ class DeepgramSTTService(STTService):
             await self._connection.finish()
             logger.debug(f"{self}: Disconnected from Deepgram")
 
+    async def _on_speech_started(self, *args, **kwargs):
+        await self.start_ttfb_metrics()
+        await self.start_processing_metrics()
+
     async def _on_message(self, *args, **kwargs):
         result: LiveResultResponse = kwargs["result"]
         if len(result.channel.alternatives) == 0:
@@ -188,7 +210,13 @@ class DeepgramSTTService(STTService):
             language = result.channel.alternatives[0].languages[0]
             language = Language(language)
         if len(transcript) > 0:
+            await self.stop_ttfb_metrics()
             if is_final:
-                await self.push_frame(TranscriptionFrame(transcript, "", time_now_iso8601(), language))
+                await self.push_frame(
+                    TranscriptionFrame(transcript, "", time_now_iso8601(), language)
+                )
+                await self.stop_processing_metrics()
             else:
-                await self.push_frame(InterimTranscriptionFrame(transcript, "", time_now_iso8601(), language))
+                await self.push_frame(
+                    InterimTranscriptionFrame(transcript, "", time_now_iso8601(), language)
+                )

src/pipecat/services/fal.py

@@ -22,7 +22,8 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use Fal, you need to `pip install pipecat-ai[fal]`. Also, set `FAL_KEY` environment variable.")
+        "In order to use Fal, you need to `pip install pipecat-ai[fal]`. Also, set `FAL_KEY` environment variable."
+    )
     raise Exception(f"Missing module: {e}")
 
 
@@ -43,7 +44,7 @@ class FalImageGenService(ImageGenService):
         aiohttp_session: aiohttp.ClientSession,
         model: str = "fal-ai/fast-sdxl",
         key: str | None = None,
-        **kwargs
+        **kwargs,
     ):
         super().__init__(**kwargs)
         self.set_model_name(model)
@@ -57,7 +58,7 @@ class FalImageGenService(ImageGenService):
 
         response = await fal_client.run_async(
             self.model_name,
-            arguments={"prompt": prompt, **self._params.model_dump(exclude_none=True)}
+            arguments={"prompt": prompt, **self._params.model_dump(exclude_none=True)},
         )
 
         image_url = response["images"][0]["url"] if response else None
@@ -77,8 +78,6 @@ class FalImageGenService(ImageGenService):
             image = Image.open(image_stream)
 
             frame = URLImageRawFrame(
-                url=image_url,
-                image=image.tobytes(),
-                size=image.size,
-                format=image.format)
+                url=image_url, image=image.tobytes(), size=image.size, format=image.format
+            )
             yield frame

src/pipecat/services/fireworks.py

@@ -13,13 +13,16 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use Fireworks, you need to `pip install pipecat-ai[fireworks]`. Also, set the `FIREWORKS_API_KEY` environment variable.")
+        "In order to use Fireworks, you need to `pip install pipecat-ai[fireworks]`. Also, set the `FIREWORKS_API_KEY` environment variable."
+    )
     raise Exception(f"Missing module: {e}")
 
 
 class FireworksLLMService(BaseOpenAILLMService):
-    def __init__(self,
-                 *,
-                 model: str = "accounts/fireworks/models/firefunction-v1",
-                 base_url: str = "https://api.fireworks.ai/inference/v1"):
+    def __init__(
+        self,
+        *,
+        model: str = "accounts/fireworks/models/firefunction-v1",
+        base_url: str = "https://api.fireworks.ai/inference/v1",
+    ):
         super().__init__(model=model, base_url=base_url)

src/pipecat/services/gladia.py

@@ -16,7 +16,8 @@ from pipecat.frames.frames import (
     Frame,
     InterimTranscriptionFrame,
     StartFrame,
-    TranscriptionFrame)
+    TranscriptionFrame,
+)
 from pipecat.services.ai_services import STTService
 from pipecat.utils.time import time_now_iso8601
 
@@ -28,7 +29,8 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use Gladia, you need to `pip install pipecat-ai[gladia]`. Also, set `GLADIA_API_KEY` environment variable.")
+        "In order to use Gladia, you need to `pip install pipecat-ai[gladia]`. Also, set `GLADIA_API_KEY` environment variable."
+    )
     raise Exception(f"Missing module: {e}")
 
 
@@ -40,13 +42,15 @@ class GladiaSTTService(STTService):
         endpointing: Optional[int] = 200
         prosody: Optional[bool] = None
 
-    def __init__(self,
-                 *,
-                 api_key: str,
-                 url: str = "wss://api.gladia.io/audio/text/audio-transcription",
-                 confidence: float = 0.5,
-                 params: InputParams = InputParams(),
-                 **kwargs):
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        url: str = "wss://api.gladia.io/audio/text/audio-transcription",
+        confidence: float = 0.5,
+        params: InputParams = InputParams(),
+        **kwargs,
+    ):
         super().__init__(sync=False, **kwargs)
 
         self._api_key = api_key
@@ -80,15 +84,13 @@ class GladiaSTTService(STTService):
             "encoding": "WAV/PCM",
             "model_type": "fast",
             "language_behaviour": "manual",
-            **self._params.model_dump(exclude_none=True)
+            **self._params.model_dump(exclude_none=True),
         }
 
         await self._websocket.send(json.dumps(configuration))
 
     async def _send_audio(self, audio: bytes):
-        message = {
-            'frames': base64.b64encode(audio).decode("utf-8")
-        }
+        message = {"frames": base64.b64encode(audio).decode("utf-8")}
         await self._websocket.send(json.dumps(message))
 
     async def _receive_task_handler(self):
@@ -106,6 +108,10 @@ class GladiaSTTService(STTService):
                 transcript = utterance["transcription"]
                 if confidence >= self._confidence:
                     if type == "final":
-                        await self.push_frame(TranscriptionFrame(transcript, "", time_now_iso8601()))
+                        await self.push_frame(
+                            TranscriptionFrame(transcript, "", time_now_iso8601())
+                        )
                     else:
-                        await self.push_frame(InterimTranscriptionFrame(transcript, "", time_now_iso8601()))
+                        await self.push_frame(
+                            InterimTranscriptionFrame(transcript, "", time_now_iso8601())
+                        )

src/pipecat/services/google.py

@@ -15,11 +15,14 @@ from pipecat.frames.frames import (
     VisionImageRawFrame,
     LLMMessagesFrame,
     LLMFullResponseStartFrame,
-    LLMFullResponseEndFrame
+    LLMFullResponseEndFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.ai_services import LLMService
-from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext, OpenAILLMContextFrame
+from pipecat.processors.aggregators.openai_llm_context import (
+    OpenAILLMContext,
+    OpenAILLMContextFrame,
+)
 
 from loguru import logger
 
@@ -29,7 +32,8 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use Google AI, you need to `pip install pipecat-ai[google]`. Also, set `GOOGLE_API_KEY` environment variable.")
+        "In order to use Google AI, you need to `pip install pipecat-ai[google]`. Also, set `GOOGLE_API_KEY` environment variable."
+    )
     raise Exception(f"Missing module: {e}")
 
 
@@ -53,8 +57,7 @@ class GoogleLLMService(LLMService):
         self.set_model_name(model)
         self._client = gai.GenerativeModel(model)
 
-    def _get_messages_from_openai_context(
-            self, context: OpenAILLMContext) -> List[glm.Content]:
+    def _get_messages_from_openai_context(self, context: OpenAILLMContext) -> List[glm.Content]:
         openai_messages = context.get_messages()
         google_messages = []
 
@@ -69,10 +72,12 @@ class GoogleLLMService(LLMService):
             parts = [glm.Part(text=content)]
             if "mime_type" in message:
                 parts.append(
-                    glm.Part(inline_data=glm.Blob(
-                        mime_type=message["mime_type"],
-                        data=message["data"].getvalue()
-                    )))
+                    glm.Part(
+                        inline_data=glm.Blob(
+                            mime_type=message["mime_type"], data=message["data"].getvalue()
+                        )
+                    )
+                )
             google_messages.append({"role": role, "parts": parts})
 
         return google_messages
@@ -103,7 +108,8 @@ class GoogleLLMService(LLMService):
                     # Google LLMs seem to flag safety issues a lot!
                     if chunk.candidates[0].finish_reason == 3:
                         logger.debug(
-                            f"LLM refused to generate content for safety reasons - {messages}.")
+                            f"LLM refused to generate content for safety reasons - {messages}."
+                        )
                     else:
                         logger.exception(f"{self} error: {e}")
 

src/pipecat/services/lmnt.py

@@ -30,20 +30,21 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use LMNT, you need to `pip install pipecat-ai[lmnt]`. Also, set `LMNT_API_KEY` environment variable.")
+        "In order to use LMNT, you need to `pip install pipecat-ai[lmnt]`. Also, set `LMNT_API_KEY` environment variable."
+    )
     raise Exception(f"Missing module: {e}")
 
 
 class LmntTTSService(AsyncTTSService):
-
     def __init__(
-            self,
-            *,
-            api_key: str,
-            voice_id: str,
-            sample_rate: int = 24000,
-            language: str = "en",
-            **kwargs):
+        self,
+        *,
+        api_key: str,
+        voice_id: str,
+        sample_rate: int = 24000,
+        language: str = "en",
+        **kwargs,
+    ):
         # Let TTSService produce TTSStoppedFrames after a short delay of
         # no activity.
         super().__init__(sync=False, push_stop_frames=True, sample_rate=sample_rate, **kwargs)
@@ -92,7 +93,8 @@ class LmntTTSService(AsyncTTSService):
         try:
             self._speech = Speech()
             self._connection = await self._speech.synthesize_streaming(
-                self._voice_id, format="raw", sample_rate=self._output_format["sample_rate"])
+                self._voice_id, format="raw", sample_rate=self._output_format["sample_rate"]
+            )
             self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
         except Exception as e:
             logger.exception(f"{self} initialization error: {e}")
@@ -129,7 +131,7 @@ class LmntTTSService(AsyncTTSService):
                     frame = TTSAudioRawFrame(
                         audio=msg["audio"],
                         sample_rate=self._output_format["sample_rate"],
-                        num_channels=1
+                        num_channels=1,
                     )
                     await self.push_frame(frame)
                 else:

src/pipecat/services/moondream.py

@@ -31,6 +31,7 @@ def detect_device():
     """
     try:
         import intel_extension_for_pytorch
+
         if torch.xpu.is_available():
             return torch.device("xpu"), torch.float32
     except ImportError:
@@ -45,12 +46,7 @@ def detect_device():
 
 class MoondreamService(VisionService):
     def __init__(
-        self,
-        *,
-        model="vikhyatk/moondream2",
-        revision="2024-08-26",
-        use_cpu=False,
-        **kwargs
+        self, *, model="vikhyatk/moondream2", revision="2024-08-26", use_cpu=False, **kwargs
     ):
         super().__init__(**kwargs)
 
@@ -85,9 +81,8 @@ class MoondreamService(VisionService):
             image = Image.frombytes(frame.format, frame.size, frame.image)
             image_embeds = self._model.encode_image(image)
             description = self._model.answer_question(
-                image_embeds=image_embeds,
-                question=frame.text,
-                tokenizer=self._tokenizer)
+                image_embeds=image_embeds, question=frame.text, tokenizer=self._tokenizer
+            )
             return description
 
         description = await asyncio.to_thread(get_image_description, frame)

src/pipecat/services/ollama.py

@@ -8,6 +8,5 @@ from pipecat.services.openai import BaseOpenAILLMService
 
 
 class OLLamaLLMService(BaseOpenAILLMService):
-
     def __init__(self, *, model: str = "llama2", base_url: str = "http://localhost:11434/v1"):
         super().__init__(model=model, base_url=base_url, api_key="ollama")

src/pipecat/services/openai.py

@@ -32,21 +32,20 @@ from pipecat.frames.frames import (
     VisionImageRawFrame,
     FunctionCallResultFrame,
     FunctionCallInProgressFrame,
-    StartInterruptionFrame
+    StartInterruptionFrame,
 )
 from pipecat.metrics.metrics import LLMTokenUsage
-from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator, LLMAssistantContextAggregator
+from pipecat.processors.aggregators.llm_response import (
+    LLMUserContextAggregator,
+    LLMAssistantContextAggregator,
+)
 
 from pipecat.processors.aggregators.openai_llm_context import (
     OpenAILLMContext,
-    OpenAILLMContextFrame
+    OpenAILLMContextFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.ai_services import (
-    ImageGenService,
-    LLMService,
-    TTSService
-)
+from pipecat.services.ai_services import ImageGenService, LLMService, TTSService
 
 try:
     from openai import AsyncOpenAI, AsyncStream, DefaultAsyncHttpxClient, BadRequestError, NOT_GIVEN
@@ -54,7 +53,8 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable.")
+        "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable."
+    )
     raise Exception(f"Missing module: {e}")
 
 ValidVoice = Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"]
@@ -82,24 +82,28 @@ class BaseOpenAILLMService(LLMService):
     as well as tool choices and the tool, which is used if requesting function
     calls from the LLM.
     """
+
     class InputParams(BaseModel):
         frequency_penalty: Optional[float] = Field(
-            default_factory=lambda: NOT_GIVEN, ge=-2.0, le=2.0)
+            default_factory=lambda: NOT_GIVEN, ge=-2.0, le=2.0
+        )
         presence_penalty: Optional[float] = Field(
-            default_factory=lambda: NOT_GIVEN, ge=-2.0, le=2.0)
+            default_factory=lambda: NOT_GIVEN, ge=-2.0, le=2.0
+        )
         seed: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=0)
         temperature: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=2.0)
         top_p: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0)
         extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
 
     def __init__(
-            self,
-            *,
-            model: str,
-            api_key=None,
-            base_url=None,
-            params: InputParams = InputParams(),
-            **kwargs):
+        self,
+        *,
+        model: str,
+        api_key=None,
+        base_url=None,
+        params: InputParams = InputParams(),
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self.set_model_name(model)
         self._client = self.create_client(api_key=api_key, base_url=base_url, **kwargs)
@@ -116,9 +120,10 @@ class BaseOpenAILLMService(LLMService):
             base_url=base_url,
             http_client=DefaultAsyncHttpxClient(
                 limits=httpx.Limits(
-                    max_keepalive_connections=100,
-                    max_connections=1000,
-                    keepalive_expiry=None)))
+                    max_keepalive_connections=100, max_connections=1000, keepalive_expiry=None
+                )
+            ),
+        )
 
     def can_generate_metrics(self) -> bool:
         return True
@@ -148,10 +153,8 @@ class BaseOpenAILLMService(LLMService):
         self._extra = extra
 
     async def get_chat_completions(
-            self,
-            context: OpenAILLMContext,
-            messages: List[ChatCompletionMessageParam]) -> AsyncStream[ChatCompletionChunk]:
-
+        self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam]
+    ) -> AsyncStream[ChatCompletionChunk]:
         params = {
             "model": self.model_name,
             "stream": True,
@@ -172,7 +175,8 @@ class BaseOpenAILLMService(LLMService):
         return chunks
 
     async def _stream_chat_completions(
-            self, context: OpenAILLMContext) -> AsyncStream[ChatCompletionChunk]:
+        self, context: OpenAILLMContext
+    ) -> AsyncStream[ChatCompletionChunk]:
         logger.debug(f"Generating chat: {context.get_messages_json()}")
 
         messages: List[ChatCompletionMessageParam] = context.get_messages()
@@ -184,7 +188,10 @@ class BaseOpenAILLMService(LLMService):
                 text = message["content"]
                 message["content"] = [
                     {"type": "text", "text": text},
-                    {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}}
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"},
+                    },
                 ]
                 del message["data"]
                 del message["mime_type"]
@@ -200,8 +207,8 @@ class BaseOpenAILLMService(LLMService):
 
         await self.start_ttfb_metrics()
 
-        chunk_stream: AsyncStream[ChatCompletionChunk] = (
-            await self._stream_chat_completions(context)
+        chunk_stream: AsyncStream[ChatCompletionChunk] = await self._stream_chat_completions(
+            context
         )
 
         async for chunk in chunk_stream:
@@ -209,7 +216,7 @@ class BaseOpenAILLMService(LLMService):
                 tokens = LLMTokenUsage(
                     prompt_tokens=chunk.usage.prompt_tokens,
                     completion_tokens=chunk.usage.completion_tokens,
-                    total_tokens=chunk.usage.total_tokens
+                    total_tokens=chunk.usage.total_tokens,
                 )
                 await self.start_llm_usage_metrics(tokens)
 
@@ -250,21 +257,16 @@ class BaseOpenAILLMService(LLMService):
                 await self._handle_function_call(context, tool_call_id, function_name, arguments)
             else:
                 raise OpenAIUnhandledFunctionException(
-                    f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function.")
+                    f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function."
+                )
 
-    async def _handle_function_call(
-            self,
-            context,
-            tool_call_id,
-            function_name,
-            arguments
-    ):
+    async def _handle_function_call(self, context, tool_call_id, function_name, arguments):
         arguments = json.loads(arguments)
         await self.call_function(
             context=context,
             tool_call_id=tool_call_id,
             function_name=function_name,
-            arguments=arguments
+            arguments=arguments,
         )
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -293,38 +295,34 @@ class BaseOpenAILLMService(LLMService):
 
 @dataclass
 class OpenAIContextAggregatorPair:
-    _user: 'OpenAIUserContextAggregator'
-    _assistant: 'OpenAIAssistantContextAggregator'
+    _user: "OpenAIUserContextAggregator"
+    _assistant: "OpenAIAssistantContextAggregator"
 
-    def user(self) -> 'OpenAIUserContextAggregator':
+    def user(self) -> "OpenAIUserContextAggregator":
         return self._user
 
-    def assistant(self) -> 'OpenAIAssistantContextAggregator':
+    def assistant(self) -> "OpenAIAssistantContextAggregator":
         return self._assistant
 
 
 class OpenAILLMService(BaseOpenAILLMService):
-
     def __init__(
-            self,
-            *,
-            model: str = "gpt-4o",
-            params: BaseOpenAILLMService.InputParams = BaseOpenAILLMService.InputParams(),
-            **kwargs):
+        self,
+        *,
+        model: str = "gpt-4o",
+        params: BaseOpenAILLMService.InputParams = BaseOpenAILLMService.InputParams(),
+        **kwargs,
+    ):
         super().__init__(model=model, params=params, **kwargs)
 
     @staticmethod
     def create_context_aggregator(context: OpenAILLMContext) -> OpenAIContextAggregatorPair:
         user = OpenAIUserContextAggregator(context)
         assistant = OpenAIAssistantContextAggregator(user)
-        return OpenAIContextAggregatorPair(
-            _user=user,
-            _assistant=assistant
-        )
+        return OpenAIContextAggregatorPair(_user=user, _assistant=assistant)
 
 
 class OpenAIImageGenService(ImageGenService):
-
     def __init__(
         self,
         *,
@@ -343,10 +341,7 @@ class OpenAIImageGenService(ImageGenService):
         logger.debug(f"Generating image from prompt: {prompt}")
 
         image = await self._client.images.generate(
-            prompt=prompt,
-            model=self.model_name,
-            n=1,
-            size=self._image_size
+            prompt=prompt, model=self.model_name, n=1, size=self._image_size
         )
 
         image_url = image.data[0].url
@@ -376,13 +371,14 @@ class OpenAITTSService(TTSService):
     """
 
     def __init__(
-            self,
-            *,
-            api_key: str | None = None,
-            voice: str = "alloy",
-            model: Literal["tts-1", "tts-1-hd"] = "tts-1",
-            sample_rate: int = 24000,
-            **kwargs):
+        self,
+        *,
+        api_key: str | None = None,
+        voice: str = "alloy",
+        model: Literal["tts-1", "tts-1-hd"] = "tts-1",
+        sample_rate: int = 24000,
+        **kwargs,
+    ):
         super().__init__(sample_rate=sample_rate, **kwargs)
 
         self._voice: ValidVoice = VALID_VOICES.get(voice, "alloy")
@@ -408,16 +404,19 @@ class OpenAITTSService(TTSService):
             await self.start_ttfb_metrics()
 
             async with self._client.audio.speech.with_streaming_response.create(
-                    input=text,
-                    model=self.model_name,
-                    voice=self._voice,
-                    response_format="pcm",
+                input=text,
+                model=self.model_name,
+                voice=self._voice,
+                response_format="pcm",
             ) as r:
                 if r.status_code != 200:
                     error = await r.text()
                     logger.error(
-                        f"{self} error getting audio (status: {r.status_code}, error: {error})")
-                    yield ErrorFrame(f"Error getting audio (status: {r.status_code}, error: {error})")
+                        f"{self} error getting audio (status: {r.status_code}, error: {error})"
+                    )
+                    yield ErrorFrame(
+                        f"Error getting audio (status: {r.status_code}, error: {error})"
+                    )
                     return
 
                 await self.start_tts_usage_metrics(text)
@@ -454,14 +453,18 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
         elif isinstance(frame, FunctionCallInProgressFrame):
             self._function_call_in_progress = frame
         elif isinstance(frame, FunctionCallResultFrame):
-            if self._function_call_in_progress and self._function_call_in_progress.tool_call_id == frame.tool_call_id:
+            if (
+                self._function_call_in_progress
+                and self._function_call_in_progress.tool_call_id == frame.tool_call_id
+            ):
                 self._function_call_in_progress = None
                 self._function_call_result = frame
                 # TODO-CB: Kwin wants us to refactor this out of here but I REFUSE
                 await self._push_aggregation()
             else:
                 logger.warning(
-                    f"FunctionCallResultFrame tool_call_id does not match FunctionCallInProgressFrame tool_call_id")
+                    f"FunctionCallResultFrame tool_call_id does not match FunctionCallInProgressFrame tool_call_id"
+                )
                 self._function_call_in_progress = None
                 self._function_call_result = None
 
@@ -479,24 +482,28 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
                 frame = self._function_call_result
                 self._function_call_result = None
                 if frame.result:
-                    self._context.add_message({
-                        "role": "assistant",
-                        "tool_calls": [
-                            {
-                                "id": frame.tool_call_id,
-                                "function": {
-                                    "name": frame.function_name,
-                                    "arguments": json.dumps(frame.arguments)
-                                },
-                                "type": "function"
-                            }
-                        ]
-                    })
-                    self._context.add_message({
-                        "role": "tool",
-                        "content": json.dumps(frame.result),
-                        "tool_call_id": frame.tool_call_id
-                    })
+                    self._context.add_message(
+                        {
+                            "role": "assistant",
+                            "tool_calls": [
+                                {
+                                    "id": frame.tool_call_id,
+                                    "function": {
+                                        "name": frame.function_name,
+                                        "arguments": json.dumps(frame.arguments),
+                                    },
+                                    "type": "function",
+                                }
+                            ],
+                        }
+                    )
+                    self._context.add_message(
+                        {
+                            "role": "tool",
+                            "content": json.dumps(frame.result),
+                            "tool_call_id": frame.tool_call_id,
+                        }
+                    )
                     run_llm = True
             else:
                 self._context.add_message({"role": "assistant", "content": aggregation})

src/pipecat/services/openpipe.py

@@ -13,33 +13,35 @@ from loguru import logger
 
 try:
     from openpipe import AsyncOpenAI as OpenPipeAI, AsyncStream
-    from openai.types.chat import (ChatCompletionMessageParam, ChatCompletionChunk)
+    from openai.types.chat import ChatCompletionMessageParam, ChatCompletionChunk
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use OpenPipe, you need to `pip install pipecat-ai[openpipe]`. Also, set `OPENPIPE_API_KEY` and `OPENAI_API_KEY` environment variables.")
+        "In order to use OpenPipe, you need to `pip install pipecat-ai[openpipe]`. Also, set `OPENPIPE_API_KEY` and `OPENAI_API_KEY` environment variables."
+    )
     raise Exception(f"Missing module: {e}")
 
 
 class OpenPipeLLMService(BaseOpenAILLMService):
-
     def __init__(
-            self,
-            *,
-            model: str = "gpt-4o",
-            api_key: str | None = None,
-            base_url: str | None = None,
-            openpipe_api_key: str | None = None,
-            openpipe_base_url: str = "https://app.openpipe.ai/api/v1",
-            tags: Dict[str, str] | None = None,
-            **kwargs):
+        self,
+        *,
+        model: str = "gpt-4o",
+        api_key: str | None = None,
+        base_url: str | None = None,
+        openpipe_api_key: str | None = None,
+        openpipe_base_url: str = "https://app.openpipe.ai/api/v1",
+        tags: Dict[str, str] | None = None,
+        **kwargs,
+    ):
         super().__init__(
             model=model,
             api_key=api_key,
             base_url=base_url,
             openpipe_api_key=openpipe_api_key,
             openpipe_base_url=openpipe_base_url,
-            **kwargs)
+            **kwargs,
+        )
         self._tags = tags
 
     def create_client(self, api_key=None, base_url=None, **kwargs):
@@ -48,24 +50,17 @@ class OpenPipeLLMService(BaseOpenAILLMService):
         client = OpenPipeAI(
             api_key=api_key,
             base_url=base_url,
-            openpipe={
-                "api_key": openpipe_api_key,
-                "base_url": openpipe_base_url
-            }
+            openpipe={"api_key": openpipe_api_key, "base_url": openpipe_base_url},
         )
         return client
 
     async def get_chat_completions(
-            self,
-            context: OpenAILLMContext,
-            messages: List[ChatCompletionMessageParam]) -> AsyncStream[ChatCompletionChunk]:
+        self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam]
+    ) -> AsyncStream[ChatCompletionChunk]:
         chunks = await self._client.chat.completions.create(
             model=self.model_name,
             stream=True,
             messages=messages,
-            openpipe={
-                "tags": self._tags,
-                "log_request": True
-            }
+            openpipe={"tags": self._tags, "log_request": True},
         )
         return chunks

src/pipecat/services/playht.py

@@ -9,11 +9,7 @@ import struct
 
 from typing import AsyncGenerator
 
-from pipecat.frames.frames import (
-    Frame,
-    TTSAudioRawFrame,
-    TTSStartedFrame,
-    TTSStoppedFrame)
+from pipecat.frames.frames import Frame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame
 from pipecat.services.ai_services import TTSService
 
 from loguru import logger
@@ -25,20 +21,15 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use PlayHT, you need to `pip install pipecat-ai[playht]`. Also, set `PLAY_HT_USER_ID` and `PLAY_HT_API_KEY` environment variables.")
+        "In order to use PlayHT, you need to `pip install pipecat-ai[playht]`. Also, set `PLAY_HT_USER_ID` and `PLAY_HT_API_KEY` environment variables."
+    )
     raise Exception(f"Missing module: {e}")
 
 
 class PlayHTTTSService(TTSService):
-
     def __init__(
-            self,
-            *,
-            api_key: str,
-            user_id: str,
-            voice_url: str,
-            sample_rate: int = 16000,
-            **kwargs):
+        self, *, api_key: str, user_id: str, voice_url: str, sample_rate: int = 16000, **kwargs
+    ):
         super().__init__(sample_rate=sample_rate, **kwargs)
 
         self._user_id = user_id
@@ -49,10 +40,8 @@ class PlayHTTTSService(TTSService):
             api_key=self._speech_key,
         )
         self._options = TTSOptions(
-            voice=voice_url,
-            sample_rate=sample_rate,
-            quality="higher",
-            format=Format.FORMAT_WAV)
+            voice=voice_url, sample_rate=sample_rate, quality="higher", format=Format.FORMAT_WAV
+        )
 
     def can_generate_metrics(self) -> bool:
         return True
@@ -71,9 +60,8 @@ class PlayHTTTSService(TTSService):
             await self.start_ttfb_metrics()
 
             playht_gen = self._client.tts(
-                text,
-                voice_engine="PlayHT2.0-turbo",
-                options=self._options)
+                text, voice_engine="PlayHT2.0-turbo", options=self._options
+            )
 
             await self.start_tts_usage_metrics(text)
 
@@ -87,10 +75,10 @@ class PlayHTTTSService(TTSService):
                     else:
                         fh = io.BytesIO(b)
                         fh.seek(36)
-                        (data, size) = struct.unpack('<4sI', fh.read(8))
-                        while data != b'data':
+                        (data, size) = struct.unpack("<4sI", fh.read(8))
+                        while data != b"data":
                             fh.read(size)
-                            (data, size) = struct.unpack('<4sI', fh.read(8))
+                            (data, size) = struct.unpack("<4sI", fh.read(8))
                         in_header = False
                 else:
                     if len(chunk):

src/pipecat/services/to_be_updated/cloudflare_ai_service.py

@@ -12,15 +12,14 @@ class CloudflareAIService(AIService):
         self.cloudflare_account_id = os.getenv("CLOUDFLARE_ACCOUNT_ID")
         self.cloudflare_api_token = os.getenv("CLOUDFLARE_API_TOKEN")
 
-        self.api_base_url = f'https://api.cloudflare.com/client/v4/accounts/{self.cloudflare_account_id}/ai/run/'
-        self.headers = {"Authorization": f'Bearer {self.cloudflare_api_token}'}
+        self.api_base_url = (
+            f"https://api.cloudflare.com/client/v4/accounts/{self.cloudflare_account_id}/ai/run/"
+        )
+        self.headers = {"Authorization": f"Bearer {self.cloudflare_api_token}"}
 
     # base endpoint, used by the others
     def run(self, model, input):
-        response = requests.post(
-            f"{self.api_base_url}{model}",
-            headers=self.headers,
-            json=input)
+        response = requests.post(f"{self.api_base_url}{model}", headers=self.headers, json=input)
         return response.json()
 
     # https://developers.cloudflare.com/workers-ai/models/llm/
@@ -28,7 +27,7 @@ class CloudflareAIService(AIService):
         input = {
             "messages": [
                 {"role": "system", "content": "You are a friendly assistant"},
-                {"role": "user", "content": sentence}
+                {"role": "user", "content": sentence},
             ]
         }
 
@@ -36,16 +35,14 @@ class CloudflareAIService(AIService):
 
     # https://developers.cloudflare.com/workers-ai/models/translation/
     def run_text_translation(self, sentence, source_language, target_language):
-        return self.run('@cf/meta/m2m100-1.2b', {
-            "text": sentence,
-            "source_lang": source_language,
-            "target_lang": target_language
-        })
+        return self.run(
+            "@cf/meta/m2m100-1.2b",
+            {"text": sentence, "source_lang": source_language, "target_lang": target_language},
+        )
 
     # https://developers.cloudflare.com/workers-ai/models/sentiment-analysis/
     def run_text_sentiment(self, sentence):
-        return self.run("@cf/huggingface/distilbert-sst-2-int8",
-                        {"text": sentence})
+        return self.run("@cf/huggingface/distilbert-sst-2-int8", {"text": sentence})
 
     # https://developers.cloudflare.com/workers-ai/models/image-classification/
     def run_image_classification(self, image_url):
@@ -65,7 +62,7 @@ class CloudflareAIService(AIService):
         models = {
             "small": "@cf/baai/bge-small-en-v1.5",  # 384 output dimensions
             "medium": "@cf/baai/bge-base-en-v1.5",  # 768 output dimensions
-            "large": "@cf/baai/bge-large-en-v1.5"  # 1024 output dimensions
+            "large": "@cf/baai/bge-large-en-v1.5",  # 1024 output dimensions
         }
 
         return self.run(models[size], {"text": texts})

src/pipecat/services/to_be_updated/google_ai_service.py

@@ -18,14 +18,12 @@ class GoogleAIService(AIService):
         )
 
         self.audio_config = texttospeech.AudioConfig(
-            audio_encoding=texttospeech.AudioEncoding.LINEAR16,
-            sample_rate_hertz=16000
+            audio_encoding=texttospeech.AudioEncoding.LINEAR16, sample_rate_hertz=16000
         )
 
     def run_tts(self, sentence):
         synthesis_input = texttospeech.SynthesisInput(text=sentence.strip())
         result = self.client.synthesize_speech(
-            input=synthesis_input,
-            voice=self.voice,
-            audio_config=self.audio_config)
+            input=synthesis_input, voice=self.voice, audio_config=self.audio_config
+        )
         return result

src/pipecat/services/to_be_updated/huggingface_ai_service.py

@@ -19,8 +19,8 @@ class HuggingFaceAIService(AIService):
     # models use 2-character language codes**)
     def run_text_translation(self, sentence, source_language, target_language):
         translator = pipeline(
-            f"translation",
-            model=f"Helsinki-NLP/opus-mt-{source_language}-{target_language}")
+            f"translation", model=f"Helsinki-NLP/opus-mt-{source_language}-{target_language}"
+        )
 
         return translator(sentence)[0]["translation_text"]
 

src/pipecat/services/together.py

@@ -23,13 +23,19 @@ from pipecat.frames.frames import (
     LLMFullResponseEndFrame,
     FunctionCallResultFrame,
     FunctionCallInProgressFrame,
-    StartInterruptionFrame
+    StartInterruptionFrame,
 )
 from pipecat.metrics.metrics import LLMTokenUsage
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.ai_services import LLMService
-from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext, OpenAILLMContextFrame
-from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator, LLMAssistantContextAggregator
+from pipecat.processors.aggregators.openai_llm_context import (
+    OpenAILLMContext,
+    OpenAILLMContextFrame,
+)
+from pipecat.processors.aggregators.llm_response import (
+    LLMUserContextAggregator,
+    LLMAssistantContextAggregator,
+)
 
 from loguru import logger
 
@@ -38,25 +44,26 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use Together.ai, you need to `pip install pipecat-ai[together]`. Also, set `TOGETHER_API_KEY` environment variable.")
+        "In order to use Together.ai, you need to `pip install pipecat-ai[together]`. Also, set `TOGETHER_API_KEY` environment variable."
+    )
     raise Exception(f"Missing module: {e}")
 
 
 @dataclass
 class TogetherContextAggregatorPair:
-    _user: 'TogetherUserContextAggregator'
-    _assistant: 'TogetherAssistantContextAggregator'
+    _user: "TogetherUserContextAggregator"
+    _assistant: "TogetherAssistantContextAggregator"
 
-    def user(self) -> 'TogetherUserContextAggregator':
+    def user(self) -> "TogetherUserContextAggregator":
         return self._user
 
-    def assistant(self) -> 'TogetherAssistantContextAggregator':
+    def assistant(self) -> "TogetherAssistantContextAggregator":
         return self._assistant
 
 
 class TogetherLLMService(LLMService):
-    """This class implements inference with Together's Llama 3.1 models
-    """
+    """This class implements inference with Together's Llama 3.1 models"""
+
     class InputParams(BaseModel):
         frequency_penalty: Optional[float] = Field(default=None, ge=-2.0, le=2.0)
         max_tokens: Optional[int] = Field(default=4096, ge=1)
@@ -67,12 +74,13 @@ class TogetherLLMService(LLMService):
         extra: Optional[Dict[str, Any]] = Field(default_factory=dict)
 
     def __init__(
-            self,
-            *,
-            api_key: str,
-            model: str = "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
-            params: InputParams = InputParams(),
-            **kwargs):
+        self,
+        *,
+        api_key: str,
+        model: str = "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
+        params: InputParams = InputParams(),
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self._client = AsyncTogether(api_key=api_key)
         self.set_model_name(model)
@@ -91,10 +99,7 @@ class TogetherLLMService(LLMService):
     def create_context_aggregator(context: OpenAILLMContext) -> TogetherContextAggregatorPair:
         user = TogetherUserContextAggregator(context)
         assistant = TogetherAssistantContextAggregator(user)
-        return TogetherContextAggregatorPair(
-            _user=user,
-            _assistant=assistant
-        )
+        return TogetherContextAggregatorPair(_user=user, _assistant=assistant)
 
     async def set_frequency_penalty(self, frequency_penalty: float):
         logger.debug(f"Switching LLM frequency_penalty to: [{frequency_penalty}]")
@@ -142,7 +147,7 @@ class TogetherLLMService(LLMService):
                 "presence_penalty": self._presence_penalty,
                 "temperature": self._temperature,
                 "top_k": self._top_k,
-                "top_p": self._top_p
+                "top_p": self._top_p,
             }
 
             params.update(self._extra)
@@ -160,7 +165,7 @@ class TogetherLLMService(LLMService):
                     tokens = LLMTokenUsage(
                         prompt_tokens=chunk.usage.prompt_tokens,
                         completion_tokens=chunk.usage.completion_tokens,
-                        total_tokens=chunk.usage.total_tokens
+                        total_tokens=chunk.usage.total_tokens,
                     )
                     await self.start_llm_usage_metrics(tokens)
 
@@ -180,7 +185,7 @@ class TogetherLLMService(LLMService):
                     else:
                         await self.push_frame(TextFrame(chunk.choices[0].delta.content))
 
-                if chunk.choices[0].finish_reason == 'eos' and accumulating_function_call:
+                if chunk.choices[0].finish_reason == "eos" and accumulating_function_call:
                     await self._extract_function_call(context, function_call_accumulator)
 
         except CancelledError as e:
@@ -219,10 +224,12 @@ class TogetherLLMService(LLMService):
             function_name, args_string = match.groups()
             try:
                 arguments = json.loads(args_string)
-                await self.call_function(context=context,
-                                         tool_call_id=str(uuid.uuid4()),
-                                         function_name=function_name,
-                                         arguments=arguments)
+                await self.call_function(
+                    context=context,
+                    tool_call_id=str(uuid.uuid4()),
+                    function_name=function_name,
+                    arguments=arguments,
+                )
                 return
             except json.JSONDecodeError as error:
                 # We get here if the LLM returns a function call with invalid JSON arguments. This could happen
@@ -281,12 +288,13 @@ class TogetherUserContextAggregator(LLMUserContextAggregator):
                 # The LLM sends a UserImageRequestFrame upstream. Cache any context provided with
                 # that frame so we can use it when we assemble the image message in the assistant
                 # context aggregator.
-                if (frame.context):
+                if frame.context:
                     if isinstance(frame.context, str):
                         self._context._user_image_request_context[frame.user_id] = frame.context
                     else:
                         logger.error(
-                            f"Unexpected UserImageRequestFrame context type: {type(frame.context)}")
+                            f"Unexpected UserImageRequestFrame context type: {type(frame.context)}"
+                        )
                         del self._context._user_image_request_context[frame.user_id]
                 else:
                     if frame.user_id in self._context._user_image_request_context:
@@ -294,6 +302,7 @@ class TogetherUserContextAggregator(LLMUserContextAggregator):
         except Exception as e:
             logger.error(f"Error processing frame: {e}")
 
+
 #
 # Claude returns a text content block along with a tool use content block. This works quite nicely
 # with streaming. We get the text first, so we can start streaming it right away. Then we get the
@@ -320,13 +329,17 @@ class TogetherAssistantContextAggregator(LLMAssistantContextAggregator):
         elif isinstance(frame, FunctionCallInProgressFrame):
             self._function_call_in_progress = frame
         elif isinstance(frame, FunctionCallResultFrame):
-            if self._function_call_in_progress and self._function_call_in_progress.tool_call_id == frame.tool_call_id:
+            if (
+                self._function_call_in_progress
+                and self._function_call_in_progress.tool_call_id == frame.tool_call_id
+            ):
                 self._function_call_in_progress = None
                 self._function_call_result = frame
                 await self._push_aggregation()
             else:
                 logger.warning(
-                    f"FunctionCallResultFrame tool_call_id does not match FunctionCallInProgressFrame tool_call_id")
+                    f"FunctionCallResultFrame tool_call_id does not match FunctionCallInProgressFrame tool_call_id"
+                )
                 self._function_call_in_progress = None
                 self._function_call_result = None
 
@@ -346,11 +359,13 @@ class TogetherAssistantContextAggregator(LLMAssistantContextAggregator):
             if self._function_call_result:
                 frame = self._function_call_result
                 self._function_call_result = None
-                self._context.add_message({
-                    "role": "tool",
-                    # Together expects the content here to be a string, so stringify it
-                    "content": str(frame.result)
-                })
+                self._context.add_message(
+                    {
+                        "role": "tool",
+                        # Together expects the content here to be a string, so stringify it
+                        "content": str(frame.result),
+                    }
+                )
                 run_llm = True
             else:
                 self._context.add_message({"role": "assistant", "content": aggregation})

src/pipecat/services/whisper.py

@@ -23,13 +23,13 @@ try:
     from faster_whisper import WhisperModel
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
-    logger.error(
-        "In order to use Whisper, you need to `pip install pipecat-ai[whisper]`.")
+    logger.error("In order to use Whisper, you need to `pip install pipecat-ai[whisper]`.")
     raise Exception(f"Missing module: {e}")
 
 
 class Model(Enum):
     """Class of basic Whisper model selection options"""
+
     TINY = "tiny"
     BASE = "base"
     MEDIUM = "medium"
@@ -41,14 +41,15 @@ class Model(Enum):
 class WhisperSTTService(SegmentedSTTService):
     """Class to transcribe audio with a locally-downloaded Whisper model"""
 
-    def __init__(self,
-                 *,
-                 model: str | Model = Model.DISTIL_MEDIUM_EN,
-                 device: str = "auto",
-                 compute_type: str = "default",
-                 no_speech_prob: float = 0.4,
-                 **kwargs):
-
+    def __init__(
+        self,
+        *,
+        model: str | Model = Model.DISTIL_MEDIUM_EN,
+        device: str = "auto",
+        compute_type: str = "default",
+        no_speech_prob: float = 0.4,
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self._device: str = device
         self._compute_type = compute_type
@@ -65,9 +66,8 @@ class WhisperSTTService(SegmentedSTTService):
         this model is being run, it will take time to download."""
         logger.debug("Loading Whisper model...")
         self._model = WhisperModel(
-            self.model_name,
-            device=self._device,
-            compute_type=self._compute_type)
+            self.model_name, device=self._device, compute_type=self._compute_type
+        )
         logger.debug("Loaded Whisper model")
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:

src/pipecat/services/xtts.py

@@ -14,7 +14,8 @@ from pipecat.frames.frames import (
     StartFrame,
     TTSAudioRawFrame,
     TTSStartedFrame,
-    TTSStoppedFrame)
+    TTSStoppedFrame,
+)
 from pipecat.services.ai_services import TTSService
 
 from loguru import logger
@@ -38,15 +39,15 @@ except ModuleNotFoundError as e:
 
 
 class XTTSService(TTSService):
-
     def __init__(
-            self,
-            *,
-            voice_id: str,
-            language: str,
-            base_url: str,
-            aiohttp_session: aiohttp.ClientSession,
-            **kwargs):
+        self,
+        *,
+        voice_id: str,
+        language: str,
+        base_url: str,
+        aiohttp_session: aiohttp.ClientSession,
+        **kwargs,
+    ):
         super().__init__(**kwargs)
 
         self._voice_id = voice_id
@@ -64,9 +65,13 @@ class XTTSService(TTSService):
             if r.status != 200:
                 text = await r.text()
                 logger.error(
-                    f"{self} error getting studio speakers (status: {r.status}, error: {text})")
+                    f"{self} error getting studio speakers (status: {r.status}, error: {text})"
+                )
                 await self.push_error(
-                    ErrorFrame(f"Error error getting studio speakers (status: {r.status}, error: {text})"))
+                    ErrorFrame(
+                        f"Error error getting studio speakers (status: {r.status}, error: {text})"
+                    )
+                )
                 return
             self._studio_speakers = await r.json()
 
@@ -86,7 +91,7 @@ class XTTSService(TTSService):
         url = self._base_url + "/tts_stream"
 
         payload = {
-            "text": text.replace('.', '').replace('*', ''),
+            "text": text.replace(".", "").replace("*", ""),
             "language": self._language,
             "speaker_embedding": embeddings["speaker_embedding"],
             "gpt_cond_latent": embeddings["gpt_cond_latent"],
@@ -115,7 +120,9 @@ class XTTSService(TTSService):
                     buffer.extend(chunk)
 
                     # Check if buffer has enough data for processing
-                    while len(buffer) >= 48000:  # Assuming at least 0.5 seconds of audio data at 24000 Hz
+                    while (
+                        len(buffer) >= 48000
+                    ):  # Assuming at least 0.5 seconds of audio data at 24000 Hz
                         # Process the buffer up to a safe size for resampling
                         process_data = buffer[:48000]
                         # Remove processed data from buffer

src/pipecat/transcriptions/language.py

@@ -9,6 +9,7 @@ import sys
 from enum import Enum
 
 if sys.version_info < (3, 11):
+
     class StrEnum(str, Enum):
         def __new__(cls, value):
             obj = str.__new__(cls, value)
@@ -19,46 +20,46 @@ else:
 
 
 class Language(StrEnum):
-    BG = "bg"            # Bulgarian
-    CA = "ca"            # Catalan
-    ZH = "zh"            # Chinese simplified
-    ZH_TW = "zh-TW"      # Chinese traditional
-    CS = "cs"            # Czech
-    DA = "da"            # Danish
-    NL = "nl"            # Dutch
-    EN = "en"            # English
-    EN_US = "en-US"      # English (USA)
-    EN_AU = "en-AU"      # English (Australia)
-    EN_GB = "en-GB"      # English (Great Britain)
-    EN_NZ = "en-NZ"      # English (New Zealand)
-    EN_IN = "en-IN"      # English (India)
-    ET = "et"            # Estonian
-    FI = "fi"            # Finnish
-    NL_BE = "nl-BE"      # Flemmish
-    FR = "fr"            # French
-    FR_CA = "fr-CA"      # French (Canada)
-    DE = "de"            # German
-    DE_CH = "de-CH"      # German (Switzerland)
-    EL = "el"            # Greek
-    HI = "hi"            # Hindi
-    HU = "hu"            # Hungarian
-    ID = "id"            # Indonesian
-    IT = "it"            # Italian
-    JA = "ja"            # Japanese
-    KO = "ko"            # Korean
-    LV = "lv"            # Latvian
-    LT = "lt"            # Lithuanian
-    MS = "ms"            # Malay
-    NO = "no"            # Norwegian
-    PL = "pl"            # Polish
-    PT = "pt"            # Portuguese
-    PT_BR = "pt-BR"      # Portuguese (Brazil)
-    RO = "ro"            # Romanian
-    RU = "ru"            # Russian
-    SK = "sk"            # Slovak
-    ES = "es"            # Spanish
-    SV = "sv"            # Swedish
-    TH = "th"            # Thai
-    TR = "tr"            # Turkish
-    UK = "uk"            # Ukrainian
-    VI = "vi"            # Vietnamese
+    BG = "bg"  # Bulgarian
+    CA = "ca"  # Catalan
+    ZH = "zh"  # Chinese simplified
+    ZH_TW = "zh-TW"  # Chinese traditional
+    CS = "cs"  # Czech
+    DA = "da"  # Danish
+    NL = "nl"  # Dutch
+    EN = "en"  # English
+    EN_US = "en-US"  # English (USA)
+    EN_AU = "en-AU"  # English (Australia)
+    EN_GB = "en-GB"  # English (Great Britain)
+    EN_NZ = "en-NZ"  # English (New Zealand)
+    EN_IN = "en-IN"  # English (India)
+    ET = "et"  # Estonian
+    FI = "fi"  # Finnish
+    NL_BE = "nl-BE"  # Flemmish
+    FR = "fr"  # French
+    FR_CA = "fr-CA"  # French (Canada)
+    DE = "de"  # German
+    DE_CH = "de-CH"  # German (Switzerland)
+    EL = "el"  # Greek
+    HI = "hi"  # Hindi
+    HU = "hu"  # Hungarian
+    ID = "id"  # Indonesian
+    IT = "it"  # Italian
+    JA = "ja"  # Japanese
+    KO = "ko"  # Korean
+    LV = "lv"  # Latvian
+    LT = "lt"  # Lithuanian
+    MS = "ms"  # Malay
+    NO = "no"  # Norwegian
+    PL = "pl"  # Polish
+    PT = "pt"  # Portuguese
+    PT_BR = "pt-BR"  # Portuguese (Brazil)
+    RO = "ro"  # Romanian
+    RU = "ru"  # Russian
+    SK = "sk"  # Slovak
+    ES = "es"  # Spanish
+    SV = "sv"  # Swedish
+    TH = "th"  # Thai
+    TR = "tr"  # Turkish
+    UK = "uk"  # Ukrainian
+    VI = "vi"  # Vietnamese

src/pipecat/transports/base_input.py

@@ -21,7 +21,8 @@ from pipecat.frames.frames import (
     SystemFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
-    VADParamsUpdateFrame)
+    VADParamsUpdateFrame,
+)
 from pipecat.transports.base_transport import TransportParams
 from pipecat.vad.vad_analyzer import VADAnalyzer, VADState
 
@@ -29,7 +30,6 @@ from loguru import logger
 
 
 class BaseInputTransport(FrameProcessor):
-
     def __init__(self, params: TransportParams, **kwargs):
         super().__init__(sync=False, **kwargs)
 
@@ -129,12 +129,17 @@ class BaseInputTransport(FrameProcessor):
         vad_analyzer = self.vad_analyzer()
         if vad_analyzer:
             state = await self.get_event_loop().run_in_executor(
-                self._executor, vad_analyzer.analyze_audio, audio_frames)
+                self._executor, vad_analyzer.analyze_audio, audio_frames
+            )
         return state
 
     async def _handle_vad(self, audio_frames: bytes, vad_state: VADState):
         new_vad_state = await self._vad_analyze(audio_frames)
-        if new_vad_state != vad_state and new_vad_state != VADState.STARTING and new_vad_state != VADState.STOPPING:
+        if (
+            new_vad_state != vad_state
+            and new_vad_state != VADState.STARTING
+            and new_vad_state != VADState.STOPPING
+        ):
             frame = None
             if new_vad_state == VADState.SPEAKING:
                 frame = UserStartedSpeakingFrame()

src/pipecat/transports/base_output.py

@@ -32,7 +32,8 @@ from pipecat.frames.frames import (
     TTSStartedFrame,
     TTSStoppedFrame,
     TextFrame,
-    TransportMessageFrame)
+    TransportMessageFrame,
+)
 from pipecat.transports.base_transport import TransportParams
 
 from loguru import logger
@@ -41,7 +42,6 @@ from pipecat.utils.time import nanoseconds_to_seconds
 
 
 class BaseOutputTransport(FrameProcessor):
-
     def __init__(self, params: TransportParams, **kwargs):
         super().__init__(sync=False, **kwargs)
 
@@ -53,8 +53,9 @@ class BaseOutputTransport(FrameProcessor):
 
         # We will write 20ms audio at a time. If we receive long audio frames we
         # will chunk them. This will help with interruption handling.
-        audio_bytes_10ms = int(self._params.audio_out_sample_rate / 100) * \
-            self._params.audio_out_channels * 2
+        audio_bytes_10ms = (
+            int(self._params.audio_out_sample_rate / 100) * self._params.audio_out_channels * 2
+        )
         self._audio_chunk_size = audio_bytes_10ms * 2
         self._audio_buffer = bytearray()
 
@@ -74,7 +75,9 @@ class BaseOutputTransport(FrameProcessor):
         # Create camera output queue and task if needed.
         if self._params.camera_out_enabled:
             self._camera_out_queue = asyncio.Queue()
-            self._camera_out_task = self.get_event_loop().create_task(self._camera_out_task_handler())
+            self._camera_out_task = self.get_event_loop().create_task(
+                self._camera_out_task_handler()
+            )
         # Create audio output queue and task if needed.
         if self._params.audio_out_enabled and self._params.audio_out_is_live:
             self._audio_out_queue = asyncio.Queue()
@@ -201,11 +204,12 @@ class BaseOutputTransport(FrameProcessor):
             self._audio_buffer.extend(frame.audio)
             while len(self._audio_buffer) >= self._audio_chunk_size:
                 chunk = OutputAudioRawFrame(
-                    bytes(self._audio_buffer[:self._audio_chunk_size]),
-                    sample_rate=frame.sample_rate, num_channels=frame.num_channels
+                    bytes(self._audio_buffer[: self._audio_chunk_size]),
+                    sample_rate=frame.sample_rate,
+                    num_channels=frame.num_channels,
                 )
                 await self._sink_queue.put(chunk)
-                self._audio_buffer = self._audio_buffer[self._audio_chunk_size:]
+                self._audio_buffer = self._audio_buffer[self._audio_chunk_size :]
 
     async def _handle_image(self, frame: OutputImageRawFrame | SpriteFrame):
         if not self._params.camera_out_enabled:
@@ -316,12 +320,10 @@ class BaseOutputTransport(FrameProcessor):
         if frame.size != desired_size:
             image = Image.frombytes(frame.format, frame.size, frame.image)
             resized_image = image.resize(desired_size)
-            logger.warning(
-                f"{frame} does not have the expected size {desired_size}, resizing")
+            logger.warning(f"{frame} does not have the expected size {desired_size}, resizing")
             frame = OutputImageRawFrame(
-                resized_image.tobytes(),
-                resized_image.size,
-                resized_image.format)
+                resized_image.tobytes(), resized_image.size, resized_image.format
+            )
 
         await self.write_frame_to_camera(frame)
 

src/pipecat/transports/base_transport.py

@@ -42,11 +42,12 @@ class TransportParams(BaseModel):
 
 
 class BaseTransport(ABC):
-
-    def __init__(self,
-                 input_name: str | None = None,
-                 output_name: str | None = None,
-                 loop: asyncio.AbstractEventLoop | None = None):
+    def __init__(
+        self,
+        input_name: str | None = None,
+        output_name: str | None = None,
+        loop: asyncio.AbstractEventLoop | None = None,
+    ):
         self._input_name = input_name
         self._output_name = output_name
         self._loop = loop or asyncio.get_running_loop()
@@ -64,6 +65,7 @@ class BaseTransport(ABC):
         def decorator(handler):
             self.add_event_handler(event_name, handler)
             return handler
+
         return decorator
 
     def add_event_handler(self, event_name: str, handler):

src/pipecat/transports/local/audio.py

@@ -21,12 +21,12 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use local audio, you need to `pip install pipecat-ai[local]`. On MacOS, you also need to `brew install portaudio`.")
+        "In order to use local audio, you need to `pip install pipecat-ai[local]`. On MacOS, you also need to `brew install portaudio`."
+    )
     raise Exception(f"Missing module: {e}")
 
 
 class LocalAudioInputTransport(BaseInputTransport):
-
     def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
         super().__init__(params)
 
@@ -39,7 +39,8 @@ class LocalAudioInputTransport(BaseInputTransport):
             rate=params.audio_in_sample_rate,
             frames_per_buffer=num_frames,
             stream_callback=self._audio_in_callback,
-            input=True)
+            input=True,
+        )
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
@@ -54,9 +55,11 @@ class LocalAudioInputTransport(BaseInputTransport):
         self._in_stream.close()
 
     def _audio_in_callback(self, in_data, frame_count, time_info, status):
-        frame = InputAudioRawFrame(audio=in_data,
-                                   sample_rate=self._params.audio_in_sample_rate,
-                                   num_channels=self._params.audio_in_channels)
+        frame = InputAudioRawFrame(
+            audio=in_data,
+            sample_rate=self._params.audio_in_sample_rate,
+            num_channels=self._params.audio_in_channels,
+        )
 
         asyncio.run_coroutine_threadsafe(self.push_audio_frame(frame), self.get_event_loop())
 
@@ -64,7 +67,6 @@ class LocalAudioInputTransport(BaseInputTransport):
 
 
 class LocalAudioOutputTransport(BaseOutputTransport):
-
     def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
         super().__init__(params)
 
@@ -74,7 +76,8 @@ class LocalAudioOutputTransport(BaseOutputTransport):
             format=py_audio.get_format_from_width(2),
             channels=params.audio_out_channels,
             rate=params.audio_out_sample_rate,
-            output=True)
+            output=True,
+        )
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
@@ -93,7 +96,6 @@ class LocalAudioOutputTransport(BaseOutputTransport):
 
 
 class LocalAudioTransport(BaseTransport):
-
     def __init__(self, params: TransportParams):
         self._params = params
         self._pyaudio = pyaudio.PyAudio()

src/pipecat/transports/local/tk.py

@@ -23,7 +23,8 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use local audio, you need to `pip install pipecat-ai[local]`. On MacOS, you also need to `brew install portaudio`.")
+        "In order to use local audio, you need to `pip install pipecat-ai[local]`. On MacOS, you also need to `brew install portaudio`."
+    )
     raise Exception(f"Missing module: {e}")
 
 try:
@@ -35,7 +36,6 @@ except ModuleNotFoundError as e:
 
 
 class TkInputTransport(BaseInputTransport):
-
     def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams):
         super().__init__(params)
 
@@ -48,7 +48,8 @@ class TkInputTransport(BaseInputTransport):
             rate=params.audio_in_sample_rate,
             frames_per_buffer=num_frames,
             stream_callback=self._audio_in_callback,
-            input=True)
+            input=True,
+        )
 
     async def start(self, frame: StartFrame):
         await super().start(frame)
@@ -63,9 +64,11 @@ class TkInputTransport(BaseInputTransport):
         self._in_stream.close()
 
     def _audio_in_callback(self, in_data, frame_count, time_info, status):
-        frame = InputAudioRawFrame(audio=in_data,
-                                   sample_rate=self._params.audio_in_sample_rate,
-                                   num_channels=self._params.audio_in_channels)
+        frame = InputAudioRawFrame(
+            audio=in_data,
+            sample_rate=self._params.audio_in_sample_rate,
+            num_channels=self._params.audio_in_channels,
+        )
 
         asyncio.run_coroutine_threadsafe(self.push_audio_frame(frame), self.get_event_loop())
 
@@ -73,7 +76,6 @@ class TkInputTransport(BaseInputTransport):
 
 
 class TkOutputTransport(BaseOutputTransport):
-
     def __init__(self, tk_root: tk.Tk, py_audio: pyaudio.PyAudio, params: TransportParams):
         super().__init__(params)
 
@@ -83,7 +85,8 @@ class TkOutputTransport(BaseOutputTransport):
             format=py_audio.get_format_from_width(2),
             channels=params.audio_out_channels,
             rate=params.audio_out_sample_rate,
-            output=True)
+            output=True,
+        )
 
         # Start with a neutral gray background.
         array = np.ones((1024, 1024, 3)) * 128
@@ -114,11 +117,7 @@ class TkOutputTransport(BaseOutputTransport):
         width = frame.size[0]
         height = frame.size[1]
         data = f"P6 {width} {height} 255 ".encode() + frame.image
-        photo = tk.PhotoImage(
-            width=width,
-            height=height,
-            data=data,
-            format="PPM")
+        photo = tk.PhotoImage(width=width, height=height, data=data, format="PPM")
         self._image_label.config(image=photo)
 
         # This holds a reference to the photo, preventing it from being garbage
@@ -127,7 +126,6 @@ class TkOutputTransport(BaseOutputTransport):
 
 
 class TkLocalTransport(BaseTransport):
-
     def __init__(self, tk_root: tk.Tk, params: TransportParams):
         self._tk_root = tk_root
         self._params = params

src/pipecat/transports/network/fastapi_websocket.py

@@ -19,7 +19,7 @@ from pipecat.frames.frames import (
     Frame,
     InputAudioRawFrame,
     StartFrame,
-    StartInterruptionFrame
+    StartInterruptionFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.serializers.base_serializer import FrameSerializer
@@ -35,7 +35,8 @@ try:
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use FastAPI websockets, you need to `pip install pipecat-ai[websocket]`.")
+        "In order to use FastAPI websockets, you need to `pip install pipecat-ai[websocket]`."
+    )
     raise Exception(f"Missing module: {e}")
 
 
@@ -51,13 +52,13 @@ class FastAPIWebsocketCallbacks(BaseModel):
 
 
 class FastAPIWebsocketInputTransport(BaseInputTransport):
-
     def __init__(
-            self,
-            websocket: WebSocket,
-            params: FastAPIWebsocketParams,
-            callbacks: FastAPIWebsocketCallbacks,
-            **kwargs):
+        self,
+        websocket: WebSocket,
+        params: FastAPIWebsocketParams,
+        callbacks: FastAPIWebsocketCallbacks,
+        **kwargs,
+    ):
         super().__init__(params, **kwargs)
 
         self._websocket = websocket
@@ -87,17 +88,18 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
                 continue
 
             if isinstance(frame, AudioRawFrame):
-                await self.push_audio_frame(InputAudioRawFrame(
-                    audio=frame.audio,
-                    sample_rate=frame.sample_rate,
-                    num_channels=frame.num_channels)
+                await self.push_audio_frame(
+                    InputAudioRawFrame(
+                        audio=frame.audio,
+                        sample_rate=frame.sample_rate,
+                        num_channels=frame.num_channels,
+                    )
                 )
 
         await self._callbacks.on_client_disconnected(self._websocket)
 
 
 class FastAPIWebsocketOutputTransport(BaseOutputTransport):
-
     def __init__(self, websocket: WebSocket, params: FastAPIWebsocketParams, **kwargs):
         super().__init__(params, **kwargs)
 
@@ -115,10 +117,9 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
         self._websocket_audio_buffer += frames
         while len(self._websocket_audio_buffer):
             frame = AudioRawFrame(
-                audio=self._websocket_audio_buffer[:
-                                                   self._params.audio_frame_size],
+                audio=self._websocket_audio_buffer[: self._params.audio_frame_size],
                 sample_rate=self._params.audio_out_sample_rate,
-                num_channels=self._params.audio_out_channels
+                num_channels=self._params.audio_out_channels,
             )
 
             if self._params.add_wav_header:
@@ -131,9 +132,8 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
                 ww.close()
                 content.seek(0)
                 wav_frame = AudioRawFrame(
-                    content.read(),
-                    sample_rate=frame.sample_rate,
-                    num_channels=frame.num_channels)
+                    content.read(), sample_rate=frame.sample_rate, num_channels=frame.num_channels
+                )
                 frame = wav_frame
 
             payload = self._params.serializer.serialize(frame)
@@ -141,7 +141,8 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
                 await self._websocket.send_text(payload)
 
             self._websocket_audio_buffer = self._websocket_audio_buffer[
-                self._params.audio_frame_size:]
+                self._params.audio_frame_size :
+            ]
 
     async def _write_frame(self, frame: Frame):
         payload = self._params.serializer.serialize(frame)
@@ -150,26 +151,28 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
 
 
 class FastAPIWebsocketTransport(BaseTransport):
-
     def __init__(
-            self,
-            websocket: WebSocket,
-            params: FastAPIWebsocketParams,
-            input_name: str | None = None,
-            output_name: str | None = None,
-            loop: asyncio.AbstractEventLoop | None = None):
+        self,
+        websocket: WebSocket,
+        params: FastAPIWebsocketParams,
+        input_name: str | None = None,
+        output_name: str | None = None,
+        loop: asyncio.AbstractEventLoop | None = None,
+    ):
         super().__init__(input_name=input_name, output_name=output_name, loop=loop)
         self._params = params
 
         self._callbacks = FastAPIWebsocketCallbacks(
             on_client_connected=self._on_client_connected,
-            on_client_disconnected=self._on_client_disconnected
+            on_client_disconnected=self._on_client_disconnected,
         )
 
         self._input = FastAPIWebsocketInputTransport(
-            websocket, self._params, self._callbacks, name=self._input_name)
+            websocket, self._params, self._callbacks, name=self._input_name
+        )
         self._output = FastAPIWebsocketOutputTransport(
-            websocket, self._params, name=self._output_name)
+            websocket, self._params, name=self._output_name
+        )
 
         # Register supported handlers. The user will only be able to register
         # these handlers.

src/pipecat/transports/network/websocket_server.py

@@ -11,7 +11,13 @@ import wave
 from typing import Awaitable, Callable
 from pydantic.main import BaseModel
 
-from pipecat.frames.frames import AudioRawFrame, CancelFrame, EndFrame, InputAudioRawFrame, StartFrame
+from pipecat.frames.frames import (
+    AudioRawFrame,
+    CancelFrame,
+    EndFrame,
+    InputAudioRawFrame,
+    StartFrame,
+)
 from pipecat.serializers.base_serializer import FrameSerializer
 from pipecat.serializers.protobuf import ProtobufFrameSerializer
 from pipecat.transports.base_input import BaseInputTransport
@@ -40,14 +46,14 @@ class WebsocketServerCallbacks(BaseModel):
 
 
 class WebsocketServerInputTransport(BaseInputTransport):
-
     def __init__(
-            self,
-            host: str,
-            port: int,
-            params: WebsocketServerParams,
-            callbacks: WebsocketServerCallbacks,
-            **kwargs):
+        self,
+        host: str,
+        port: int,
+        params: WebsocketServerParams,
+        callbacks: WebsocketServerCallbacks,
+        **kwargs,
+    ):
         super().__init__(params, **kwargs)
 
         self._host = host
@@ -97,10 +103,12 @@ class WebsocketServerInputTransport(BaseInputTransport):
                 continue
 
             if isinstance(frame, AudioRawFrame):
-                await self.push_audio_frame(InputAudioRawFrame(
-                    audio=frame.audio,
-                    sample_rate=frame.sample_rate,
-                    num_channels=frame.num_channels)
+                await self.push_audio_frame(
+                    InputAudioRawFrame(
+                        audio=frame.audio,
+                        sample_rate=frame.sample_rate,
+                        num_channels=frame.num_channels,
+                    )
                 )
             else:
                 await self.push_frame(frame)
@@ -115,7 +123,6 @@ class WebsocketServerInputTransport(BaseInputTransport):
 
 
 class WebsocketServerOutputTransport(BaseOutputTransport):
-
     def __init__(self, params: WebsocketServerParams, **kwargs):
         super().__init__(params, **kwargs)
 
@@ -138,9 +145,9 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
         self._websocket_audio_buffer += frames
         while len(self._websocket_audio_buffer) >= self._params.audio_frame_size:
             frame = AudioRawFrame(
-                audio=self._websocket_audio_buffer[:self._params.audio_frame_size],
+                audio=self._websocket_audio_buffer[: self._params.audio_frame_size],
                 sample_rate=self._params.audio_out_sample_rate,
-                num_channels=self._params.audio_out_channels
+                num_channels=self._params.audio_out_channels,
             )
 
             if self._params.add_wav_header:
@@ -153,28 +160,29 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
                 ww.close()
                 content.seek(0)
                 wav_frame = AudioRawFrame(
-                    content.read(),
-                    sample_rate=frame.sample_rate,
-                    num_channels=frame.num_channels)
+                    content.read(), sample_rate=frame.sample_rate, num_channels=frame.num_channels
+                )
                 frame = wav_frame
 
             proto = self._params.serializer.serialize(frame)
             if proto:
                 await self._websocket.send(proto)
 
-            self._websocket_audio_buffer = self._websocket_audio_buffer[self._params.audio_frame_size:]
+            self._websocket_audio_buffer = self._websocket_audio_buffer[
+                self._params.audio_frame_size :
+            ]
 
 
 class WebsocketServerTransport(BaseTransport):
-
     def __init__(
-            self,
-            host: str = "localhost",
-            port: int = 8765,
-            params: WebsocketServerParams = WebsocketServerParams(),
-            input_name: str | None = None,
-            output_name: str | None = None,
-            loop: asyncio.AbstractEventLoop | None = None):
+        self,
+        host: str = "localhost",
+        port: int = 8765,
+        params: WebsocketServerParams = WebsocketServerParams(),
+        input_name: str | None = None,
+        output_name: str | None = None,
+        loop: asyncio.AbstractEventLoop | None = None,
+    ):
         super().__init__(input_name=input_name, output_name=output_name, loop=loop)
         self._host = host
         self._port = port
@@ -182,7 +190,7 @@ class WebsocketServerTransport(BaseTransport):
 
         self._callbacks = WebsocketServerCallbacks(
             on_client_connected=self._on_client_connected,
-            on_client_disconnected=self._on_client_disconnected
+            on_client_disconnected=self._on_client_disconnected,
         )
         self._input: WebsocketServerInputTransport | None = None
         self._output: WebsocketServerOutputTransport | None = None
@@ -196,7 +204,8 @@ class WebsocketServerTransport(BaseTransport):
     def input(self) -> WebsocketServerInputTransport:
         if not self._input:
             self._input = WebsocketServerInputTransport(
-                self._host, self._port, self._params, self._callbacks, name=self._input_name)
+                self._host, self._port, self._params, self._callbacks, name=self._input_name
+            )
         return self._input
 
     def output(self) -> WebsocketServerOutputTransport:

src/pipecat/transports/services/daily.py

@@ -18,7 +18,8 @@ from daily import (
     EventHandler,
     VirtualCameraDevice,
     VirtualMicrophoneDevice,
-    VirtualSpeakerDevice)
+    VirtualSpeakerDevice,
+)
 from pydantic.main import BaseModel
 
 from pipecat.frames.frames import (
@@ -35,8 +36,14 @@ from pipecat.frames.frames import (
     TranscriptionFrame,
     TransportMessageFrame,
     UserImageRawFrame,
-    UserImageRequestFrame)
-from pipecat.metrics.metrics import LLMUsageMetricsData, ProcessingMetricsData, TTFBMetricsData, TTSUsageMetricsData
+    UserImageRequestFrame,
+)
+from pipecat.metrics.metrics import (
+    LLMUsageMetricsData,
+    ProcessingMetricsData,
+    TTFBMetricsData,
+    TTSUsageMetricsData,
+)
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.transcriptions.language import Language
 from pipecat.transports.base_input import BaseInputTransport
@@ -47,11 +54,12 @@ from pipecat.vad.vad_analyzer import VADAnalyzer, VADParams
 from loguru import logger
 
 try:
-    from daily import (EventHandler, CallClient, Daily)
+    from daily import EventHandler, CallClient, Daily
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(
-        "In order to use the Daily transport, you need to `pip install pipecat-ai[daily]`.")
+        "In order to use the Daily transport, you need to `pip install pipecat-ai[daily]`."
+    )
     raise Exception(f"Missing module: {e}")
 
 VAD_RESET_PERIOD_MS = 2000
@@ -63,14 +71,11 @@ class DailyTransportMessageFrame(TransportMessageFrame):
 
 
 class WebRTCVADAnalyzer(VADAnalyzer):
-
     def __init__(self, *, sample_rate=16000, num_channels=1, params: VADParams = VADParams()):
         super().__init__(sample_rate=sample_rate, num_channels=num_channels, params=params)
 
         self._webrtc_vad = Daily.create_native_vad(
-            reset_period_ms=VAD_RESET_PERIOD_MS,
-            sample_rate=sample_rate,
-            channels=num_channels
+            reset_period_ms=VAD_RESET_PERIOD_MS, sample_rate=sample_rate, channels=num_channels
         )
         logger.debug("Loaded native WebRTC VAD")
 
@@ -98,9 +103,7 @@ class DailyTranscriptionSettings(BaseModel):
     endpointing: bool = True
     punctuate: bool = True
     includeRawResponse: bool = True
-    extra: Mapping[str, Any] = {
-        "interim_results": True
-    }
+    extra: Mapping[str, Any] = {"interim_results": True}
 
 
 class DailyParams(TransportParams):
@@ -139,12 +142,13 @@ def completion_callback(future):
                     future.set_result(*args)
             except asyncio.InvalidStateError:
                 pass
+
         future.get_loop().call_soon_threadsafe(set_result, future, *args)
+
     return _callback
 
 
 class DailyTransportClient(EventHandler):
-
     _daily_initialized: bool = False
 
     # This is necessary to override EventHandler's __new__ method.
@@ -152,13 +156,14 @@ class DailyTransportClient(EventHandler):
         return super().__new__(cls)
 
     def __init__(
-            self,
-            room_url: str,
-            token: str | None,
-            bot_name: str,
-            params: DailyParams,
-            callbacks: DailyCallbacks,
-            loop: asyncio.AbstractEventLoop):
+        self,
+        room_url: str,
+        token: str | None,
+        bot_name: str,
+        params: DailyParams,
+        callbacks: DailyCallbacks,
+        loop: asyncio.AbstractEventLoop,
+    ):
         super().__init__()
 
         if not DailyTransportClient._daily_initialized:
@@ -191,7 +196,8 @@ class DailyTransportClient(EventHandler):
                 self._camera_name(),
                 width=self._params.camera_out_width,
                 height=self._params.camera_out_height,
-                color_format=self._params.camera_out_color_format)
+                color_format=self._params.camera_out_color_format,
+            )
 
         self._mic: VirtualMicrophoneDevice | None = None
         if self._params.audio_out_enabled:
@@ -199,7 +205,8 @@ class DailyTransportClient(EventHandler):
                 self._mic_name(),
                 sample_rate=self._params.audio_out_sample_rate,
                 channels=self._params.audio_out_channels,
-                non_blocking=True)
+                non_blocking=True,
+            )
 
         self._speaker: VirtualSpeakerDevice | None = None
         if self._params.audio_in_enabled or self._params.vad_enabled:
@@ -207,7 +214,8 @@ class DailyTransportClient(EventHandler):
                 self._speaker_name(),
                 sample_rate=self._params.audio_in_sample_rate,
                 channels=self._params.audio_in_channels,
-                non_blocking=True)
+                non_blocking=True,
+            )
             Daily.select_speaker_device(self._speaker_name())
 
     def _camera_name(self):
@@ -236,9 +244,8 @@ class DailyTransportClient(EventHandler):
 
         future = self._loop.create_future()
         self._client.send_app_message(
-            frame.message,
-            participant_id,
-            completion=completion_callback(future))
+            frame.message, participant_id, completion=completion_callback(future)
+        )
         await future
 
     async def read_next_audio_frame(self) -> InputAudioRawFrame | None:
@@ -255,9 +262,8 @@ class DailyTransportClient(EventHandler):
 
         if len(audio) > 0:
             return InputAudioRawFrame(
-                audio=audio,
-                sample_rate=sample_rate,
-                num_channels=num_channels)
+                audio=audio, sample_rate=sample_rate, num_channels=num_channels
+            )
         else:
             # If we don't read any audio it could be there's no participant
             # connected. daily-python will return immediately if that's the
@@ -290,12 +296,9 @@ class DailyTransportClient(EventHandler):
 
         # For performance reasons, never subscribe to video streams (unless a
         # video renderer is registered).
-        self._client.update_subscription_profiles({
-            "base": {
-                "camera": "unsubscribed",
-                "screenVideo": "unsubscribed"
-            }
-        })
+        self._client.update_subscription_profiles(
+            {"base": {"camera": "unsubscribed", "screenVideo": "unsubscribed"}}
+        )
 
         self._client.set_user_name(self._bot_name)
 
@@ -327,7 +330,7 @@ class DailyTransportClient(EventHandler):
         future = self._loop.create_future()
         self._client.start_transcription(
             settings=self._params.transcription_settings.model_dump(exclude_none=True),
-            completion=completion_callback(future)
+            completion=completion_callback(future),
         )
         error = await future
         if error:
@@ -374,12 +377,15 @@ class DailyTransportClient(EventHandler):
                     },
                     "microphone": {
                         "sendSettings": {
-                            "channelConfig": "stereo" if self._params.audio_out_channels == 2 else "mono",
+                            "channelConfig": "stereo"
+                            if self._params.audio_out_channels == 2
+                            else "mono",
                             "bitrate": self._params.audio_out_bitrate,
                         }
-                    }
+                    },
                 },
-            })
+            },
+        )
 
         return await asyncio.wait_for(future, timeout=10)
 
@@ -456,18 +462,17 @@ class DailyTransportClient(EventHandler):
         self._transcription_renderers[participant_id] = callback
 
     def capture_participant_video(
-            self,
-            participant_id: str,
-            callback: Callable,
-            framerate: int = 30,
-            video_source: str = "camera",
-            color_format: str = "RGB"):
+        self,
+        participant_id: str,
+        callback: Callable,
+        framerate: int = 30,
+        video_source: str = "camera",
+        color_format: str = "RGB",
+    ):
         # Only enable camera subscription on this participant
-        self._client.update_subscriptions(participant_settings={
-            participant_id: {
-                "media": "subscribed"
-            }
-        })
+        self._client.update_subscriptions(
+            participant_settings={participant_id: {"media": "subscribed"}}
+        )
 
         self._video_renderers[participant_id] = callback
 
@@ -475,7 +480,8 @@ class DailyTransportClient(EventHandler):
             participant_id,
             self._video_frame_received,
             video_source=video_source,
-            color_format=color_format)
+            color_format=color_format,
+        )
 
     #
     #
@@ -553,9 +559,9 @@ class DailyTransportClient(EventHandler):
             callback,
             participant_id,
             video_frame.buffer,
-            (video_frame.width,
-             video_frame.height),
-            video_frame.color_format)
+            (video_frame.width, video_frame.height),
+            video_frame.color_format,
+        )
 
     def _call_async_callback(self, callback, *args):
         future = asyncio.run_coroutine_threadsafe(callback(*args), self._loop)
@@ -563,7 +569,6 @@ class DailyTransportClient(EventHandler):
 
 
 class DailyInputTransport(BaseInputTransport):
-
     def __init__(self, client: DailyTransportClient, params: DailyParams, **kwargs):
         super().__init__(params, **kwargs)
 
@@ -576,7 +581,8 @@ class DailyInputTransport(BaseInputTransport):
         if params.vad_enabled and not params.vad_analyzer:
             self._vad_analyzer = WebRTCVADAnalyzer(
                 sample_rate=self._params.audio_in_sample_rate,
-                num_channels=self._params.audio_in_channels)
+                num_channels=self._params.audio_in_channels,
+            )
 
     async def start(self, frame: StartFrame):
         # Parent start.
@@ -654,11 +660,12 @@ class DailyInputTransport(BaseInputTransport):
     #
 
     def capture_participant_video(
-            self,
-            participant_id: str,
-            framerate: int = 30,
-            video_source: str = "camera",
-            color_format: str = "RGB"):
+        self,
+        participant_id: str,
+        framerate: int = 30,
+        video_source: str = "camera",
+        color_format: str = "RGB",
+    ):
         self._video_renderers[participant_id] = {
             "framerate": framerate,
             "timestamp": 0,
@@ -666,11 +673,7 @@ class DailyInputTransport(BaseInputTransport):
         }
 
         self._client.capture_participant_video(
-            participant_id,
-            self._on_participant_video_frame,
-            framerate,
-            video_source,
-            color_format
+            participant_id, self._on_participant_video_frame, framerate, video_source, color_format
         )
 
     def request_participant_image(self, participant_id: str):
@@ -693,17 +696,14 @@ class DailyInputTransport(BaseInputTransport):
 
         if render_frame:
             frame = UserImageRawFrame(
-                user_id=participant_id,
-                image=buffer,
-                size=size,
-                format=format)
+                user_id=participant_id, image=buffer, size=size, format=format
+            )
             await self.push_frame(frame)
 
         self._video_renderers[participant_id]["timestamp"] = curr_time
 
 
 class DailyOutputTransport(BaseOutputTransport):
-
     def __init__(self, client: DailyTransportClient, params: DailyParams, **kwargs):
         super().__init__(params, **kwargs)
 
@@ -754,10 +754,9 @@ class DailyOutputTransport(BaseOutputTransport):
                     metrics["characters"] = []
                 metrics["characters"].append(d.model_dump(exclude_none=True))
 
-        message = DailyTransportMessageFrame(message={
-            "type": "pipecat-metrics",
-            "metrics": metrics
-        })
+        message = DailyTransportMessageFrame(
+            message={"type": "pipecat-metrics", "metrics": metrics}
+        )
         await self._client.send_message(message)
 
     async def write_raw_audio_frames(self, frames: bytes):
@@ -768,16 +767,16 @@ class DailyOutputTransport(BaseOutputTransport):
 
 
 class DailyTransport(BaseTransport):
-
     def __init__(
-            self,
-            room_url: str,
-            token: str | None,
-            bot_name: str,
-            params: DailyParams = DailyParams(),
-            input_name: str | None = None,
-            output_name: str | None = None,
-            loop: asyncio.AbstractEventLoop | None = None):
+        self,
+        room_url: str,
+        token: str | None,
+        bot_name: str,
+        params: DailyParams = DailyParams(),
+        input_name: str | None = None,
+        output_name: str | None = None,
+        loop: asyncio.AbstractEventLoop | None = None,
+    ):
         super().__init__(input_name=input_name, output_name=output_name, loop=loop)
 
         callbacks = DailyCallbacks(
@@ -800,7 +799,8 @@ class DailyTransport(BaseTransport):
         self._params = params
 
         self._client = DailyTransportClient(
-            room_url, token, bot_name, params, callbacks, self._loop)
+            room_url, token, bot_name, params, callbacks, self._loop
+        )
         self._input: DailyInputTransport | None = None
         self._output: DailyOutputTransport | None = None
 
@@ -871,19 +871,20 @@ class DailyTransport(BaseTransport):
 
     def capture_participant_transcription(self, participant_id: str):
         self._client.capture_participant_transcription(
-            participant_id,
-            self._on_transcription_message
+            participant_id, self._on_transcription_message
         )
 
     def capture_participant_video(
-            self,
-            participant_id: str,
-            framerate: int = 30,
-            video_source: str = "camera",
-            color_format: str = "RGB"):
+        self,
+        participant_id: str,
+        framerate: int = 30,
+        video_source: str = "camera",
+        color_format: str = "RGB",
+    ):
         if self._input:
             self._input.capture_participant_video(
-                participant_id, framerate, video_source, color_format)
+                participant_id, framerate, video_source, color_format
+            )
 
     async def _on_joined(self, data):
         await self._call_event_handler("on_joined", data)
@@ -911,12 +912,12 @@ class DailyTransport(BaseTransport):
         async with aiohttp.ClientSession() as session:
             headers = {
                 "Authorization": f"Bearer {self._params.api_key}",
-                "Content-Type": "application/json"
+                "Content-Type": "application/json",
             }
             data = {
                 "callId": self._params.dialin_settings.call_id,
                 "callDomain": self._params.dialin_settings.call_domain,
-                "sipUri": sip_endpoint
+                "sipUri": sip_endpoint,
             }
 
             url = f"{self._params.api_url}/dialin/pinlessCallUpdate"
@@ -926,7 +927,8 @@ class DailyTransport(BaseTransport):
                     if r.status != 200:
                         text = await r.text()
                         logger.error(
-                            f"Unable to handle dialin-ready event (status: {r.status}, error: {text})")
+                            f"Unable to handle dialin-ready event (status: {r.status}, error: {text})"
+                        )
                         return
 
                     logger.debug("Event dialin-ready was handled successfully")

src/pipecat/transports/services/helpers/daily_rest.py

@@ -41,12 +41,12 @@ class DailyRoomProperties(BaseModel, extra="allow"):
         if not self.sip_uri:
             return ""
         else:
-            return "sip:%s" % self.sip_uri['endpoint']
+            return "sip:%s" % self.sip_uri["endpoint"]
 
 
 class DailyRoomParams(BaseModel):
     name: Optional[str] = None
-    privacy: Literal['private', 'public'] = "public"
+    privacy: Literal["private", "public"] = "public"
     properties: DailyRoomProperties = Field(default_factory=DailyRoomProperties)
 
 
@@ -61,11 +61,13 @@ class DailyRoomObject(BaseModel):
 
 
 class DailyRESTHelper:
-    def __init__(self,
-                 *,
-                 daily_api_key: str,
-                 daily_api_url: str = "https://api.daily.co/v1",
-                 aiohttp_session: aiohttp.ClientSession):
+    def __init__(
+        self,
+        *,
+        daily_api_key: str,
+        daily_api_url: str = "https://api.daily.co/v1",
+        aiohttp_session: aiohttp.ClientSession,
+    ):
         self.daily_api_key = daily_api_key
         self.daily_api_url = daily_api_url
         self.aiohttp_session = aiohttp_session
@@ -80,7 +82,9 @@ class DailyRESTHelper:
     async def create_room(self, params: DailyRoomParams) -> DailyRoomObject:
         headers = {"Authorization": f"Bearer {self.daily_api_key}"}
         json = {**params.model_dump(exclude_none=True)}
-        async with self.aiohttp_session.post(f"{self.daily_api_url}/rooms", headers=headers, json=json) as r:
+        async with self.aiohttp_session.post(
+            f"{self.daily_api_url}/rooms", headers=headers, json=json
+        ) as r:
             if r.status != 200:
                 text = await r.text()
                 raise Exception(f"Unable to create room (status: {r.status}): {text}")
@@ -95,27 +99,22 @@ class DailyRESTHelper:
         return room
 
     async def get_token(
-            self,
-            room_url: str,
-            expiry_time: float = 60 * 60,
-            owner: bool = True) -> str:
+        self, room_url: str, expiry_time: float = 60 * 60, owner: bool = True
+    ) -> str:
         if not room_url:
             raise Exception(
-                "No Daily room specified. You must specify a Daily room in order a token to be generated.")
+                "No Daily room specified. You must specify a Daily room in order a token to be generated."
+            )
 
         expiration: float = time.time() + expiry_time
 
         room_name = self.get_name_from_url(room_url)
 
         headers = {"Authorization": f"Bearer {self.daily_api_key}"}
-        json = {
-            "properties": {
-                "room_name": room_name,
-                "is_owner": owner,
-                "exp": expiration
-            }
-        }
-        async with self.aiohttp_session.post(f"{self.daily_api_url}/meeting-tokens", headers=headers, json=json) as r:
+        json = {"properties": {"room_name": room_name, "is_owner": owner, "exp": expiration}}
+        async with self.aiohttp_session.post(
+            f"{self.daily_api_url}/meeting-tokens", headers=headers, json=json
+        ) as r:
             if r.status != 200:
                 text = await r.text()
                 raise Exception(f"Failed to create meeting token (status: {r.status}): {text}")
@@ -130,7 +129,9 @@ class DailyRESTHelper:
 
     async def delete_room_by_name(self, room_name: str) -> bool:
         headers = {"Authorization": f"Bearer {self.daily_api_key}"}
-        async with self.aiohttp_session.delete(f"{self.daily_api_url}/rooms/{room_name}", headers=headers) as r:
+        async with self.aiohttp_session.delete(
+            f"{self.daily_api_url}/rooms/{room_name}", headers=headers
+        ) as r:
             if r.status != 200 and r.status != 404:
                 text = await r.text()
                 raise Exception(f"Failed to delete room [{room_name}] (status: {r.status}): {text}")
@@ -139,7 +140,9 @@ class DailyRESTHelper:
 
     async def _get_room_from_name(self, room_name: str) -> DailyRoomObject:
         headers = {"Authorization": f"Bearer {self.daily_api_key}"}
-        async with self.aiohttp_session.get(f"{self.daily_api_url}/rooms/{room_name}", headers=headers) as r:
+        async with self.aiohttp_session.get(
+            f"{self.daily_api_url}/rooms/{room_name}", headers=headers
+        ) as r:
             if r.status != 200:
                 raise Exception(f"Room not found: {room_name}")
 

src/pipecat/utils/test_frame_processor.py

@@ -15,7 +15,9 @@ class TestFrameProcessor(FrameProcessor):
     async def process_frame(self, frame, direction):
         await super().process_frame(frame, direction)
 
-        if not self.test_frames[0]:  # then we've run out of required frames but the generator is still going?
+        if not self.test_frames[
+            0
+        ]:  # then we've run out of required frames but the generator is still going?
             raise TestException(f"Oops, got an extra frame, {frame}")
         if isinstance(self.test_frames[0], List):
             # We need to consume frames until we see the next frame type after this

src/pipecat/vad/silero.py

@@ -8,32 +8,111 @@ import time
 
 import numpy as np
 
-from pipecat.frames.frames import AudioRawFrame, Frame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame
+from pipecat.frames.frames import (
+    AudioRawFrame,
+    Frame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
+)
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.vad.vad_analyzer import VADAnalyzer, VADParams, VADState
 
 from loguru import logger
 
+# How often should we reset internal model state
+_MODEL_RESET_STATES_TIME = 5.0
+
 try:
-    from silero_vad import load_silero_vad
-    import torch
+    import onnxruntime
 
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error("In order to use Silero VAD, you need to `pip install pipecat-ai[silero]`.")
     raise Exception(f"Missing module(s): {e}")
 
-# How often should we reset internal model state
-_MODEL_RESET_STATES_TIME = 5.0
+
+class SileroOnnxModel:
+    def __init__(self, path, force_onnx_cpu=True):
+        import numpy as np
+
+        global np
+
+        opts = onnxruntime.SessionOptions()
+        opts.inter_op_num_threads = 1
+        opts.intra_op_num_threads = 1
+
+        if force_onnx_cpu and "CPUExecutionProvider" in onnxruntime.get_available_providers():
+            self.session = onnxruntime.InferenceSession(
+                path, providers=["CPUExecutionProvider"], sess_options=opts
+            )
+        else:
+            self.session = onnxruntime.InferenceSession(path, sess_options=opts)
+
+        self.reset_states()
+        self.sample_rates = [8000, 16000]
+
+    def _validate_input(self, x, sr: int):
+        if np.ndim(x) == 1:
+            x = np.expand_dims(x, 0)
+        if np.ndim(x) > 2:
+            raise ValueError(f"Too many dimensions for input audio chunk {x.dim()}")
+
+        if sr not in self.sample_rates:
+            raise ValueError(
+                f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)"
+            )
+        if sr / np.shape(x)[1] > 31.25:
+            raise ValueError("Input audio chunk is too short")
+
+        return x, sr
+
+    def reset_states(self, batch_size=1):
+        self._state = np.zeros((2, batch_size, 128), dtype="float32")
+        self._context = np.zeros((batch_size, 0), dtype="float32")
+        self._last_sr = 0
+        self._last_batch_size = 0
+
+    def __call__(self, x, sr: int):
+        x, sr = self._validate_input(x, sr)
+        num_samples = 512 if sr == 16000 else 256
+
+        if np.shape(x)[-1] != num_samples:
+            raise ValueError(
+                f"Provided number of samples is {np.shape(x)[-1]} (Supported values: 256 for 8000 sample rate, 512 for 16000)"
+            )
+
+        batch_size = np.shape(x)[0]
+        context_size = 64 if sr == 16000 else 32
+
+        if not self._last_batch_size:
+            self.reset_states(batch_size)
+        if (self._last_sr) and (self._last_sr != sr):
+            self.reset_states(batch_size)
+        if (self._last_batch_size) and (self._last_batch_size != batch_size):
+            self.reset_states(batch_size)
+
+        if not np.shape(self._context)[1]:
+            self._context = np.zeros((batch_size, context_size), dtype="float32")
+
+        x = np.concatenate((self._context, x), axis=1)
+
+        if sr in [8000, 16000]:
+            ort_inputs = {"input": x, "state": self._state, "sr": np.array(sr, dtype="int64")}
+            ort_outs = self.session.run(None, ort_inputs)
+            out, state = ort_outs
+            self._state = state
+        else:
+            raise ValueError()
+
+        self._context = x[..., -context_size:]
+        self._last_sr = sr
+        self._last_batch_size = batch_size
+
+        return out
 
 
 class SileroVADAnalyzer(VADAnalyzer):
-
-    def __init__(
-            self,
-            *,
-            sample_rate: int = 16000,
-            params: VADParams = VADParams()):
+    def __init__(self, *, sample_rate: int = 16000, params: VADParams = VADParams()):
         super().__init__(sample_rate=sample_rate, num_channels=1, params=params)
 
         if sample_rate != 16000 and sample_rate != 8000:
@@ -41,7 +120,23 @@ class SileroVADAnalyzer(VADAnalyzer):
 
         logger.debug("Loading Silero VAD model...")
 
-        self._model = load_silero_vad()
+        model_name = "silero_vad.onnx"
+        package_path = "pipecat.vad.data"
+
+        try:
+            import importlib_resources as impresources
+
+            model_file_path = str(impresources.files(package_path).joinpath(model_name))
+        except BaseException:
+            from importlib import resources as impresources
+
+            try:
+                with impresources.path(package_path, model_name) as f:
+                    model_file_path = f
+            except BaseException:
+                model_file_path = str(impresources.files(package_path).joinpath(model_name))
+
+        self._model = SileroOnnxModel(model_file_path, force_onnx_cpu=True)
 
         self._last_reset_time = 0
 
@@ -59,7 +154,7 @@ class SileroVADAnalyzer(VADAnalyzer):
             audio_int16 = np.frombuffer(buffer, np.int16)
             # Divide by 32768 because we have signed 16-bit data.
             audio_float32 = np.frombuffer(audio_int16, dtype=np.int16).astype(np.float32) / 32768.0
-            new_confidence = self._model(torch.from_numpy(audio_float32), self.sample_rate).item()
+            new_confidence = self._model(audio_float32, self.sample_rate)[0]
 
             # We need to reset the model from time to time because it doesn't
             # really need all the data and memory will keep growing otherwise.
@@ -77,18 +172,16 @@ class SileroVADAnalyzer(VADAnalyzer):
 
 
 class SileroVAD(FrameProcessor):
-
     def __init__(
-            self,
-            *,
-            sample_rate: int = 16000,
-            vad_params: VADParams = VADParams(),
-            audio_passthrough: bool = False):
+        self,
+        *,
+        sample_rate: int = 16000,
+        vad_params: VADParams = VADParams(),
+        audio_passthrough: bool = False,
+    ):
         super().__init__()
 
-        self._vad_analyzer = SileroVADAnalyzer(
-            sample_rate=sample_rate,
-            params=vad_params)
+        self._vad_analyzer = SileroVADAnalyzer(sample_rate=sample_rate, params=vad_params)
         self._audio_passthrough = audio_passthrough
 
         self._processor_vad_state: VADState = VADState.QUIET
@@ -111,7 +204,11 @@ class SileroVAD(FrameProcessor):
         # Check VAD and push event if necessary. We just care about changes
         # from QUIET to SPEAKING and vice versa.
         new_vad_state = self._vad_analyzer.analyze_audio(frame.audio)
-        if new_vad_state != self._processor_vad_state and new_vad_state != VADState.STARTING and new_vad_state != VADState.STOPPING:
+        if (
+            new_vad_state != self._processor_vad_state
+            and new_vad_state != VADState.STARTING
+            and new_vad_state != VADState.STOPPING
+        ):
             new_frame = None
 
             if new_vad_state == VADState.SPEAKING:

src/pipecat/vad/vad_analyzer.py

@@ -29,7 +29,6 @@ class VADParams(BaseModel):
 
 
 class VADAnalyzer:
-
     def __init__(self, *, sample_rate: int, num_channels: int, params: VADParams):
         self._sample_rate = sample_rate
         self._num_channels = num_channels