From d1a3f404a55751dd041bc4f4db305d48ce42939b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 11:29:21 -0800 Subject: [PATCH 01/28] improve task creation and cancellation If a FrameProcessor needs to create a task it should use FrameProcessor.create_task() and FrameProcessor.cancel_task(). This gives Pipecat more control over all the tasks that are created in Pipecat. Both functions internally use the utils module: utils.create_task() and utils.cancel_task() which should also be used outside of FrameProcessors. That is, unless strictly necessary, we should avoid using asyncio.create_task(). --- .../22b-natural-conversation-proposal.py | 3 +- .../22c-natural-conversation-mixed-llms.py | 39 ++-- .../22d-natural-conversation-gemini-audio.py | 9 +- src/pipecat/audio/vad/silero.py | 2 +- src/pipecat/pipeline/parallel_pipeline.py | 50 +++--- src/pipecat/pipeline/task.py | 166 +++++++++--------- src/pipecat/pipeline/task_observer.py | 26 +-- .../aggregators/gated_openai_llm_context.py | 18 +- src/pipecat/processors/frame_processor.py | 71 ++++---- src/pipecat/processors/frameworks/rtvi.py | 28 ++- .../processors/idle_frame_processor.py | 7 +- src/pipecat/processors/user_idle_processor.py | 10 +- src/pipecat/services/ai_services.py | 83 ++++----- src/pipecat/services/cartesia.py | 7 +- src/pipecat/services/elevenlabs.py | 21 +-- src/pipecat/services/fish.py | 7 +- .../services/gemini_multimodal_live/gemini.py | 13 +- src/pipecat/services/gladia.py | 2 +- src/pipecat/services/lmnt.py | 7 +- .../services/openai_realtime_beta/openai.py | 68 +++---- src/pipecat/services/playht.py | 7 +- src/pipecat/services/riva.py | 19 +- src/pipecat/services/simli.py | 64 +++---- src/pipecat/services/websocket_service.py | 4 - src/pipecat/transports/base_input.py | 41 ++--- src/pipecat/transports/base_output.py | 79 ++++----- .../transports/network/fastapi_websocket.py | 13 +- .../transports/network/websocket_server.py | 3 +- src/pipecat/transports/services/daily.py | 32 ++-- src/pipecat/transports/services/livekit.py | 81 ++++++--- src/pipecat/utils/utils.py | 39 ++++ 31 files changed, 467 insertions(+), 552 deletions(-) diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 328d691c4..c188ae7d4 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -169,8 +169,7 @@ class OutputGate(FrameProcessor): self._gate_task = self.get_event_loop().create_task(self._gate_task_handler()) async def _stop(self): - self._gate_task.cancel() - await self._gate_task + await self.cancel_task(self._gate_task) async def _gate_task_handler(self): while True: diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 6281175b7..ff53cd838 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -101,12 +101,12 @@ HIGH PRIORITY SIGNALS: Examples: # Complete Wh-question -[{"role": "assistant", "content": "I can help you learn."}, +[{"role": "assistant", "content": "I can help you learn."}, {"role": "user", "content": "What's the fastest way to learn Spanish"}] Output: YES # Complete Yes/No question despite STT error -[{"role": "assistant", "content": "I know about planets."}, +[{"role": "assistant", "content": "I know about planets."}, {"role": "user", "content": "Is is Jupiter the biggest planet"}] Output: YES @@ -118,12 +118,12 @@ Output: YES Examples: # Direct instruction -[{"role": "assistant", "content": "I can explain many topics."}, +[{"role": "assistant", "content": "I can explain many topics."}, {"role": "user", "content": "Tell me about black holes"}] Output: YES # Action demand -[{"role": "assistant", "content": "I can help with math."}, +[{"role": "assistant", "content": "I can help with math."}, {"role": "user", "content": "Solve this equation x plus 5 equals 12"}] Output: YES @@ -134,12 +134,12 @@ Output: YES Examples: # Specific answer -[{"role": "assistant", "content": "What's your favorite color?"}, +[{"role": "assistant", "content": "What's your favorite color?"}, {"role": "user", "content": "I really like blue"}] Output: YES # Option selection -[{"role": "assistant", "content": "Would you prefer morning or evening?"}, +[{"role": "assistant", "content": "Would you prefer morning or evening?"}, {"role": "user", "content": "Morning"}] Output: YES @@ -153,17 +153,17 @@ MEDIUM PRIORITY SIGNALS: Examples: # Self-correction reaching completion -[{"role": "assistant", "content": "What would you like to know?"}, +[{"role": "assistant", "content": "What would you like to know?"}, {"role": "user", "content": "Tell me about... no wait, explain how rainbows form"}] Output: YES # Topic change with complete thought -[{"role": "assistant", "content": "The weather is nice today."}, +[{"role": "assistant", "content": "The weather is nice today."}, {"role": "user", "content": "Actually can you tell me who invented the telephone"}] Output: YES # Mid-sentence completion -[{"role": "assistant", "content": "Hello I'm ready."}, +[{"role": "assistant", "content": "Hello I'm ready."}, {"role": "user", "content": "What's the capital of? France"}] Output: YES @@ -175,12 +175,12 @@ Output: YES Examples: # Acknowledgment -[{"role": "assistant", "content": "Should we talk about history?"}, +[{"role": "assistant", "content": "Should we talk about history?"}, {"role": "user", "content": "Sure"}] Output: YES # Disagreement with completion -[{"role": "assistant", "content": "Is that what you meant?"}, +[{"role": "assistant", "content": "Is that what you meant?"}, {"role": "user", "content": "No not really"}] Output: YES @@ -194,12 +194,12 @@ LOW PRIORITY SIGNALS: Examples: # Word repetition but complete -[{"role": "assistant", "content": "I can help with that."}, +[{"role": "assistant", "content": "I can help with that."}, {"role": "user", "content": "What what is the time right now"}] Output: YES # Missing punctuation but complete -[{"role": "assistant", "content": "I can explain that."}, +[{"role": "assistant", "content": "I can explain that."}, {"role": "user", "content": "Please tell me how computers work"}] Output: YES @@ -211,12 +211,12 @@ Output: YES Examples: # Filler words but complete -[{"role": "assistant", "content": "What would you like to know?"}, +[{"role": "assistant", "content": "What would you like to know?"}, {"role": "user", "content": "Um uh how do airplanes fly"}] Output: YES # Thinking pause but incomplete -[{"role": "assistant", "content": "I can explain anything."}, +[{"role": "assistant", "content": "I can explain anything."}, {"role": "user", "content": "Well um I want to know about the"}] Output: NO @@ -241,17 +241,17 @@ DECISION RULES: Examples: # Incomplete despite corrections -[{"role": "assistant", "content": "What would you like to know about?"}, +[{"role": "assistant", "content": "What would you like to know about?"}, {"role": "user", "content": "Can you tell me about"}] Output: NO # Complete despite multiple artifacts -[{"role": "assistant", "content": "I can help you learn."}, +[{"role": "assistant", "content": "I can help you learn."}, {"role": "user", "content": "How do you I mean what's the best way to learn programming"}] Output: YES # Trailing off incomplete -[{"role": "assistant", "content": "I can explain anything."}, +[{"role": "assistant", "content": "I can explain anything."}, {"role": "user", "content": "I was wondering if you could tell me why"}] Output: NO """ @@ -374,8 +374,7 @@ class OutputGate(FrameProcessor): self._gate_task = self.get_event_loop().create_task(self._gate_task_handler()) async def _stop(self): - self._gate_task.cancel() - await self._gate_task + await cancel_task(self._gate_task) async def _gate_task_handler(self): while True: diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 921aa0f3b..83ccef2ee 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -44,9 +44,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.processors.user_idle_processor import UserIdleProcessor from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.deepgram import DeepgramSTTService from pipecat.services.google import GoogleLLMContext, GoogleLLMService from pipecat.sync.base_notifier import BaseNotifier from pipecat.sync.event_notifier import EventNotifier @@ -440,11 +438,11 @@ class CompletenessCheck(FrameProcessor): if isinstance(frame, UserStartedSpeakingFrame): if self._idle_task: - self._idle_task.cancel() + await self.cancel_task(self._idle_task) elif isinstance(frame, TextFrame) and frame.text.startswith("YES"): logger.debug("Completeness check YES") if self._idle_task: - self._idle_task.cancel() + await self.cancel_task(self._idle_task) await self.push_frame(UserStoppedSpeakingFrame()) await self._audio_accumulator.reset() await self._notifier.notify() @@ -602,8 +600,7 @@ class OutputGate(FrameProcessor): self._gate_task = self.get_event_loop().create_task(self._gate_task_handler()) async def _stop(self): - self._gate_task.cancel() - await self._gate_task + await self.cancel_task(self._gate_task) async def _gate_task_handler(self): while True: diff --git a/src/pipecat/audio/vad/silero.py b/src/pipecat/audio/vad/silero.py index fab01ae70..00a1358a5 100644 --- a/src/pipecat/audio/vad/silero.py +++ b/src/pipecat/audio/vad/silero.py @@ -159,5 +159,5 @@ class SileroVADAnalyzer(VADAnalyzer): return new_confidence except Exception as e: # This comes from an empty audio array - logger.exception(f"Error analyzing audio with Silero VAD: {e}") + logger.error(f"Error analyzing audio with Silero VAD: {e}") return 0 diff --git a/src/pipecat/pipeline/parallel_pipeline.py b/src/pipecat/pipeline/parallel_pipeline.py index 874a8c3a5..2d5bd7761 100644 --- a/src/pipecat/pipeline/parallel_pipeline.py +++ b/src/pipecat/pipeline/parallel_pipeline.py @@ -150,22 +150,18 @@ class ParallelPipeline(BasePipeline): async def _stop(self): # The up task doesn't receive an EndFrame, so we just cancel it. - self._up_task.cancel() - await self._up_task - # The down tasks waits for the last EndFrame send by the internal + await self.cancel_task(self._up_task) + # The down tasks waits for the last EndFrame sent by the internal # pipelines. await self._down_task async def _cancel(self): - self._up_task.cancel() - await self._up_task - self._down_task.cancel() - await self._down_task + await self.cancel_task(self._up_task) + await self.cancel_task(self._down_task) async def _create_tasks(self): - loop = self.get_event_loop() - self._up_task = loop.create_task(self._process_up_queue()) - self._down_task = loop.create_task(self._process_down_queue()) + self._up_task = self.create_task(self._process_up_queue()) + self._down_task = self.create_task(self._process_down_queue()) async def _drain_queues(self): while not self._up_queue.empty: @@ -185,32 +181,26 @@ class ParallelPipeline(BasePipeline): async def _process_up_queue(self): while True: - try: - frame = await self._up_queue.get() - await self._parallel_push_frame(frame, FrameDirection.UPSTREAM) - self._up_queue.task_done() - except asyncio.CancelledError: - break + frame = await self._up_queue.get() + await self._parallel_push_frame(frame, FrameDirection.UPSTREAM) + self._up_queue.task_done() async def _process_down_queue(self): running = True while running: - try: - frame = await self._down_queue.get() + frame = await self._down_queue.get() - endframe_counter = self._endframe_counter.get(frame.id, 0) + endframe_counter = self._endframe_counter.get(frame.id, 0) - # If we have a counter, decrement it. - if endframe_counter > 0: - self._endframe_counter[frame.id] -= 1 - endframe_counter = self._endframe_counter[frame.id] + # If we have a counter, decrement it. + if endframe_counter > 0: + self._endframe_counter[frame.id] -= 1 + endframe_counter = self._endframe_counter[frame.id] - # If we don't have a counter or we reached 0, push the frame. - if endframe_counter == 0: - await self._parallel_push_frame(frame, FrameDirection.DOWNSTREAM) + # If we don't have a counter or we reached 0, push the frame. + if endframe_counter == 0: + await self._parallel_push_frame(frame, FrameDirection.DOWNSTREAM) - running = not (endframe_counter == 0 and isinstance(frame, EndFrame)) + running = not (endframe_counter == 0 and isinstance(frame, EndFrame)) - self._down_queue.task_done() - except asyncio.CancelledError: - break + self._down_queue.task_done() diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 82918c239..34a7bb5eb 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -30,7 +30,7 @@ from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.pipeline.base_task import BaseTask from pipecat.pipeline.task_observer import TaskObserver from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.utils.utils import obj_count, obj_id +from pipecat.utils.utils import cancel_task, create_task, obj_count, obj_id HEARTBEAT_SECONDS = 1.0 HEARTBEAT_MONITOR_SECONDS = HEARTBEAT_SECONDS * 5 @@ -49,7 +49,7 @@ class PipelineParams(BaseModel): heartbeats_period_secs: float = HEARTBEAT_SECONDS -class Source(FrameProcessor): +class PipelineTaskSource(FrameProcessor): """This is the source processor that is linked at the beginning of the pipeline given to the pipeline task. It allows us to easily push frames downstream to the pipeline and also receive upstream frames coming from the @@ -57,8 +57,8 @@ class Source(FrameProcessor): """ - def __init__(self, up_queue: asyncio.Queue): - super().__init__() + def __init__(self, up_queue: asyncio.Queue, **kwargs): + super().__init__(**kwargs) self._up_queue = up_queue async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -71,15 +71,15 @@ class Source(FrameProcessor): await self.push_frame(frame, direction) -class Sink(FrameProcessor): +class PipelineTaskSink(FrameProcessor): """This is the sink processor that is linked at the end of the pipeline given to the pipeline task. It allows us to receive downstream frames and act on them, for example, waiting to receive an EndFrame. """ - def __init__(self, down_queue: asyncio.Queue): - super().__init__() + def __init__(self, down_queue: asyncio.Queue, **kwargs): + super().__init__(**kwargs) self._down_queue = down_queue async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -115,10 +115,10 @@ class PipelineTask(BaseTask): # down queue. self._endframe_event = asyncio.Event() - self._source = Source(self._up_queue) + self._source = PipelineTaskSource(self._up_queue) self._source.link(pipeline) - self._sink = Sink(self._down_queue) + self._sink = PipelineTaskSink(self._down_queue) pipeline.link(self._sink) self._observer = TaskObserver(params.observers) @@ -148,13 +148,22 @@ class PipelineTask(BaseTask): # we want to cancel right away. await self._source.push_frame(CancelFrame()) await self._cancel_tasks(True) + await self._cleanup() async def run(self): """ Starts running the given pipeline. """ - tasks = self._create_tasks() - await asyncio.gather(*tasks) + try: + push_task = self._create_tasks() + await asyncio.gather(push_task) + except asyncio.CancelledError: + # We are awaiting on the push task and it might be cancelled + # (e.g. Ctrl-C). This means we will get a CancelledError here as + # well, because you get a CancelledError in every place you are + # awaiting a task. + pass + await self._cancel_tasks(False) self._finished = True async def queue_frame(self, frame: Frame): @@ -175,41 +184,44 @@ class PipelineTask(BaseTask): await self.queue_frame(frame) def _create_tasks(self): - tasks = [] - self._process_up_task = asyncio.create_task(self._process_up_queue()) - self._process_down_task = asyncio.create_task(self._process_down_queue()) - self._process_push_task = asyncio.create_task(self._process_push_queue()) + loop = asyncio.get_running_loop() + self._process_up_task = create_task( + loop, self._process_up_queue(), f"{self}::_process_up_queue" + ) + self._process_down_task = create_task( + loop, self._process_down_queue(), f"{self}::_process_down_queue" + ) + self._process_push_task = create_task( + loop, self._process_push_queue(), f"{self}::_process_push_queue" + ) - tasks = [self._process_up_task, self._process_down_task, self._process_push_task] - - return tasks + return self._process_push_task def _maybe_start_heartbeat_tasks(self): if self._params.enable_heartbeats: - self._heartbeat_push_task = asyncio.create_task(self._heartbeat_push_handler()) - self._heartbeat_monitor_task = asyncio.create_task(self._heartbeat_monitor_handler()) + loop = asyncio.get_running_loop() + self._heartbeat_push_task = create_task( + loop, self._heartbeat_push_handler(), f"{self}::_heartbeat_push_handler" + ) + self._heartbeat_monitor_task = create_task( + loop, self._heartbeat_monitor_handler(), f"{self}::_heartbeat_monitor_handler" + ) async def _cancel_tasks(self, cancel_push: bool): await self._maybe_cancel_heartbeat_tasks() if cancel_push: - self._process_push_task.cancel() - await self._process_push_task + await cancel_task(self._process_push_task) - self._process_up_task.cancel() - await self._process_up_task - - self._process_down_task.cancel() - await self._process_down_task + await cancel_task(self._process_up_task) + await cancel_task(self._process_down_task) await self._observer.stop() async def _maybe_cancel_heartbeat_tasks(self): if self._params.enable_heartbeats: - self._heartbeat_push_task.cancel() - await self._heartbeat_push_task - self._heartbeat_monitor_task.cancel() - await self._heartbeat_monitor_task + await cancel_task(self._heartbeat_push_task) + await cancel_task(self._heartbeat_monitor_task) def _initial_metrics_frame(self) -> MetricsFrame: processors = self._pipeline.processors_with_metrics() @@ -223,6 +235,11 @@ class PipelineTask(BaseTask): await self._endframe_event.wait() self._endframe_event.clear() + async def _cleanup(self): + await self._source.cleanup() + await self._pipeline.cleanup() + await self._sink.cleanup() + async def _process_push_queue(self): """This is the task that runs the pipeline for the first time by sending a StartFrame and by pushing any other frames queued by the user. It runs @@ -249,24 +266,16 @@ class PipelineTask(BaseTask): running = True should_cleanup = True while running: - try: - frame = await self._push_queue.get() - await self._source.queue_frame(frame, FrameDirection.DOWNSTREAM) - if isinstance(frame, EndFrame): - await self._wait_for_endframe() - running = not isinstance(frame, (StopTaskFrame, EndFrame)) - should_cleanup = not isinstance(frame, StopTaskFrame) - self._push_queue.task_done() - except asyncio.CancelledError: - break + frame = await self._push_queue.get() + await self._source.queue_frame(frame, FrameDirection.DOWNSTREAM) + if isinstance(frame, EndFrame): + await self._wait_for_endframe() + running = not isinstance(frame, (StopTaskFrame, EndFrame)) + should_cleanup = not isinstance(frame, StopTaskFrame) + self._push_queue.task_done() # Cleanup only if we need to. if should_cleanup: - await self._source.cleanup() - await self._pipeline.cleanup() - await self._sink.cleanup() - # Finally, cancel internal tasks. We don't cancel the push tasks because - # that's us. - await self._cancel_tasks(False) + await self._cleanup() async def _process_up_queue(self): """This is the task that processes frames coming upstream from the @@ -276,26 +285,23 @@ class PipelineTask(BaseTask): """ while True: - try: - frame = await self._up_queue.get() - if isinstance(frame, EndTaskFrame): - # Tell the task we should end nicely. - await self.queue_frame(EndFrame()) - elif isinstance(frame, CancelTaskFrame): - # Tell the task we should end right away. + frame = await self._up_queue.get() + if isinstance(frame, EndTaskFrame): + # Tell the task we should end nicely. + await self.queue_frame(EndFrame()) + elif isinstance(frame, CancelTaskFrame): + # Tell the task we should end right away. + await self.queue_frame(CancelFrame()) + elif isinstance(frame, StopTaskFrame): + await self.queue_frame(StopTaskFrame()) + elif isinstance(frame, ErrorFrame): + logger.error(f"Error running app: {frame}") + if frame.fatal: + # Cancel all tasks downstream. await self.queue_frame(CancelFrame()) - elif isinstance(frame, StopTaskFrame): + # Tell the task we should stop. await self.queue_frame(StopTaskFrame()) - elif isinstance(frame, ErrorFrame): - logger.error(f"Error running app: {frame}") - if frame.fatal: - # Cancel all tasks downstream. - await self.queue_frame(CancelFrame()) - # Tell the task we should stop. - await self.queue_frame(StopTaskFrame()) - self._up_queue.task_done() - except asyncio.CancelledError: - break + self._up_queue.task_done() async def _process_down_queue(self): """This tasks process frames coming downstream from the pipeline. For @@ -305,29 +311,23 @@ class PipelineTask(BaseTask): """ while True: - try: - frame = await self._down_queue.get() - if isinstance(frame, EndFrame): - self._endframe_event.set() - elif isinstance(frame, HeartbeatFrame): - await self._heartbeat_queue.put(frame) - self._down_queue.task_done() - except asyncio.CancelledError: - break + frame = await self._down_queue.get() + if isinstance(frame, EndFrame): + self._endframe_event.set() + elif isinstance(frame, HeartbeatFrame): + await self._heartbeat_queue.put(frame) + self._down_queue.task_done() async def _heartbeat_push_handler(self): """ This tasks pushes a heartbeat frame every heartbeat period. """ while True: - try: - # Don't use `queue_frame()` because if an EndFrame is queued the - # task will just stop waiting for the pipeline to finish not - # allowing more frames to be pushed. - await self._source.queue_frame(HeartbeatFrame(timestamp=self._clock.get_time())) - await asyncio.sleep(self._params.heartbeats_period_secs) - except asyncio.CancelledError: - break + # Don't use `queue_frame()` because if an EndFrame is queued the + # task will just stop waiting for the pipeline to finish not + # allowing more frames to be pushed. + await self._source.queue_frame(HeartbeatFrame(timestamp=self._clock.get_time())) + await asyncio.sleep(self._params.heartbeats_period_secs) async def _heartbeat_monitor_handler(self): """This tasks monitors heartbeat frames. If a heartbeat frame has not @@ -347,8 +347,6 @@ class PipelineTask(BaseTask): logger.warning( f"{self}: heartbeat frame not received for more than {wait_time} seconds" ) - except asyncio.CancelledError: - break def __str__(self): return self.name diff --git a/src/pipecat/pipeline/task_observer.py b/src/pipecat/pipeline/task_observer.py index 2fd13f517..1bf4ff0f9 100644 --- a/src/pipecat/pipeline/task_observer.py +++ b/src/pipecat/pipeline/task_observer.py @@ -12,6 +12,7 @@ from attr import dataclass from pipecat.frames.frames import Frame from pipecat.observers.base_observer import BaseObserver from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.utils.utils import cancel_task, create_task, obj_count @dataclass @@ -54,13 +55,13 @@ class TaskObserver(BaseObserver): """ def __init__(self, observers: List[BaseObserver] = []): + self.name: str = f"{self.__class__.__name__}#{obj_count(self)}" self._proxies: List[Proxy] = self._create_proxies(observers) async def stop(self): """Stops all proxy observer tasks.""" for proxy in self._proxies: - proxy.task.cancel() - await proxy.task + await cancel_task(proxy.task) async def on_push_frame( self, @@ -79,19 +80,24 @@ class TaskObserver(BaseObserver): def _create_proxies(self, observers) -> List[Proxy]: proxies = [] + loop = asyncio.get_running_loop() for observer in observers: queue = asyncio.Queue() - task = asyncio.create_task(self._proxy_task_handler(queue, observer)) + task = create_task( + loop, + self._proxy_task_handler(queue, observer), + f"{self}::{observer.__class__.__name__}", + ) proxy = Proxy(queue=queue, task=task, observer=observer) proxies.append(proxy) return proxies async def _proxy_task_handler(self, queue: asyncio.Queue, observer: BaseObserver): while True: - try: - data = await queue.get() - await observer.on_push_frame( - data.src, data.dst, data.frame, data.direction, data.timestamp - ) - except asyncio.CancelledError: - break + data = await queue.get() + await observer.on_push_frame( + data.src, data.dst, data.frame, data.direction, data.timestamp + ) + + def __str__(self): + return self.name diff --git a/src/pipecat/processors/aggregators/gated_openai_llm_context.py b/src/pipecat/processors/aggregators/gated_openai_llm_context.py index a185528f2..8c3f496eb 100644 --- a/src/pipecat/processors/aggregators/gated_openai_llm_context.py +++ b/src/pipecat/processors/aggregators/gated_openai_llm_context.py @@ -4,8 +4,6 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio - from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -38,18 +36,14 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): await self.push_frame(frame, direction) async def _start(self): - self._gate_task = self.get_event_loop().create_task(self._gate_task_handler()) + self._gate_task = self.create_task(self._gate_task_handler()) async def _stop(self): - self._gate_task.cancel() - await self._gate_task + await self.cancel_task(self._gate_task) async def _gate_task_handler(self): while True: - try: - await self._notifier.wait() - if self._last_context_frame: - await self.push_frame(self._last_context_frame) - self._last_context_frame = None - except asyncio.CancelledError: - break + await self._notifier.wait() + if self._last_context_frame: + await self.push_frame(self._last_context_frame) + self._last_context_frame = None diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index cc483465a..1895147dd 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -6,15 +6,15 @@ import asyncio import inspect +import sys from enum import Enum -from typing import Awaitable, Callable, Optional +from typing import Awaitable, Callable, Coroutine, Optional from loguru import logger from pipecat.clocks.base_clock import BaseClock from pipecat.frames.frames import ( CancelFrame, - EndFrame, ErrorFrame, Frame, StartFrame, @@ -24,7 +24,7 @@ from pipecat.frames.frames import ( ) from pipecat.metrics.metrics import LLMTokenUsage, MetricsData from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics -from pipecat.utils.utils import obj_count, obj_id +from pipecat.utils.utils import cancel_task, create_task, obj_count, obj_id class FrameDirection(Enum): @@ -141,6 +141,13 @@ class FrameProcessor: await self.stop_ttfb_metrics() await self.stop_processing_metrics() + def create_task(self, coroutine: Coroutine) -> asyncio.Task: + name = f"{self}::{coroutine.cr_code.co_name}" + return create_task(self.get_event_loop(), coroutine, name) + + async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None): + await cancel_task(task, timeout) + async def cleanup(self): await self.__cancel_input_task() await self.__cancel_push_task() @@ -188,7 +195,6 @@ class FrameProcessor: async def resume_processing_frames(self): logger.trace(f"{self}: resuming frame processing") self.__input_event.set() - self.__should_block_frames = False async def process_frame(self, frame: Frame, direction: FrameDirection): if isinstance(frame, StartFrame): @@ -283,61 +289,44 @@ class FrameProcessor: def __create_input_task(self): self.__should_block_frames = False self.__input_queue = asyncio.Queue() - self.__input_frame_task = self.get_event_loop().create_task( - self.__input_frame_task_handler() - ) self.__input_event = asyncio.Event() + self.__input_frame_task = self.create_task(self.__input_frame_task_handler()) async def __cancel_input_task(self): - self.__input_frame_task.cancel() - await self.__input_frame_task + await self.cancel_task(self.__input_frame_task) async def __input_frame_task_handler(self): while True: - try: - if self.__should_block_frames: - logger.trace(f"{self}: frame processing paused") - await self.__input_event.wait() - self.__input_event.clear() - logger.trace(f"{self}: frame processing resumed") + if self.__should_block_frames: + logger.trace(f"{self}: frame processing paused") + await self.__input_event.wait() + self.__input_event.clear() + self.__should_block_frames = False + logger.trace(f"{self}: frame processing resumed") - (frame, direction, callback) = await self.__input_queue.get() + (frame, direction, callback) = await self.__input_queue.get() - # Process the frame. - await self.process_frame(frame, direction) + # Process the frame. + await self.process_frame(frame, direction) - # If this frame has an associated callback, call it now. - if callback: - await callback(self, frame, direction) + # If this frame has an associated callback, call it now. + if callback: + await callback(self, frame, direction) - self.__input_queue.task_done() - except asyncio.CancelledError: - logger.trace(f"{self}: cancelled input task") - break - except Exception as e: - logger.exception(f"{self}: Uncaught exception {e}") - await self.push_error(ErrorFrame(str(e))) + self.__input_queue.task_done() def __create_push_task(self): self.__push_queue = asyncio.Queue() - self.__push_frame_task = self.get_event_loop().create_task(self.__push_frame_task_handler()) + self.__push_frame_task = self.create_task(self.__push_frame_task_handler()) async def __cancel_push_task(self): - self.__push_frame_task.cancel() - await self.__push_frame_task + await self.cancel_task(self.__push_frame_task) async def __push_frame_task_handler(self): while True: - try: - (frame, direction) = await self.__push_queue.get() - await self.__internal_push_frame(frame, direction) - self.__push_queue.task_done() - except asyncio.CancelledError: - logger.trace(f"{self}: cancelled push task") - break - except Exception as e: - logger.exception(f"{self}: Uncaught exception {e}") - await self.push_error(ErrorFrame(str(e))) + (frame, direction) = await self.__push_queue.get() + await self.__internal_push_frame(frame, direction) + self.__push_queue.task_done() async def _call_event_handler(self, event_name: str, *args, **kwargs): try: diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index fcfb73237..f59b6d48f 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -764,11 +764,11 @@ class RTVIProcessor(FrameProcessor): # A task to process incoming action frames. self._action_queue = asyncio.Queue() - self._action_task = self.get_event_loop().create_task(self._action_task_handler()) + self._action_task = self.create_task(self._action_task_handler()) # A task to process incoming transport messages. self._message_queue = asyncio.Queue() - self._message_task = self.get_event_loop().create_task(self._message_task_handler()) + self._message_task = self.create_task(self._message_task_handler()) self._register_event_handler("on_bot_started") self._register_event_handler("on_client_ready") @@ -873,13 +873,11 @@ class RTVIProcessor(FrameProcessor): async def _cancel_tasks(self): if self._action_task: - self._action_task.cancel() - await self._action_task + await self.cancel_task(self._action_task) self._action_task = None if self._message_task: - self._message_task.cancel() - await self._message_task + await self.cancel_task(self._message_task) self._message_task = None async def _push_transport_message(self, model: BaseModel, exclude_none: bool = True): @@ -888,21 +886,15 @@ class RTVIProcessor(FrameProcessor): async def _action_task_handler(self): while True: - try: - frame = await self._action_queue.get() - await self._handle_action(frame.message_id, frame.rtvi_action_run) - self._action_queue.task_done() - except asyncio.CancelledError: - break + frame = await self._action_queue.get() + await self._handle_action(frame.message_id, frame.rtvi_action_run) + self._action_queue.task_done() async def _message_task_handler(self): while True: - try: - message = await self._message_queue.get() - await self._handle_message(message) - self._message_queue.task_done() - except asyncio.CancelledError: - break + message = await self._message_queue.get() + await self._handle_message(message) + self._message_queue.task_done() async def _handle_transport_message(self, frame: TransportMessageUrgentFrame): try: diff --git a/src/pipecat/processors/idle_frame_processor.py b/src/pipecat/processors/idle_frame_processor.py index 3f5f51e45..85ea215cf 100644 --- a/src/pipecat/processors/idle_frame_processor.py +++ b/src/pipecat/processors/idle_frame_processor.py @@ -49,12 +49,11 @@ class IdleFrameProcessor(FrameProcessor): self._idle_event.set() async def cleanup(self): - self._idle_task.cancel() - await self._idle_task + await self.cancel_task(self._idle_task) def _create_idle_task(self): self._idle_event = asyncio.Event() - self._idle_task = self.get_event_loop().create_task(self._idle_task_handler()) + self._idle_task = self.create_task(self._idle_task_handler()) async def _idle_task_handler(self): while True: @@ -62,7 +61,5 @@ class IdleFrameProcessor(FrameProcessor): await asyncio.wait_for(self._idle_event.wait(), timeout=self._timeout) except asyncio.TimeoutError: await self._callback(self) - except asyncio.CancelledError: - break finally: self._idle_event.clear() diff --git a/src/pipecat/processors/user_idle_processor.py b/src/pipecat/processors/user_idle_processor.py index 7c2f1c763..3aadf0793 100644 --- a/src/pipecat/processors/user_idle_processor.py +++ b/src/pipecat/processors/user_idle_processor.py @@ -103,7 +103,7 @@ class UserIdleProcessor(FrameProcessor): def _create_idle_task(self) -> None: """Creates the idle task if it hasn't been created yet.""" if self._idle_task is None: - self._idle_task = self.get_event_loop().create_task(self._idle_task_handler()) + self._idle_task = self.create_task(self._idle_task_handler()) @property def retry_count(self) -> int: @@ -113,11 +113,7 @@ class UserIdleProcessor(FrameProcessor): async def _stop(self) -> None: """Stops and cleans up the idle monitoring task.""" if self._idle_task is not None: - self._idle_task.cancel() - try: - await self._idle_task - except asyncio.CancelledError: - pass # Expected when task is cancelled + await self.cancel_task(self._idle_task) self._idle_task = None async def process_frame(self, frame: Frame, direction: FrameDirection) -> None: @@ -178,7 +174,5 @@ class UserIdleProcessor(FrameProcessor): if not should_continue: await self._stop() break - except asyncio.CancelledError: - break finally: self._idle_event.clear() diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index bfd155d38..2b0485fdd 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -253,20 +253,18 @@ class TTSService(AIService): async def start(self, frame: StartFrame): await super().start(frame) if self._push_stop_frames: - self._stop_frame_task = self.get_event_loop().create_task(self._stop_frame_handler()) + self._stop_frame_task = self.create_task(self._stop_frame_handler()) async def stop(self, frame: EndFrame): await super().stop(frame) if self._stop_frame_task: - self._stop_frame_task.cancel() - await self._stop_frame_task + await self.cancel_task(self._stop_frame_task) self._stop_frame_task = None async def cancel(self, frame: CancelFrame): await super().cancel(frame) if self._stop_frame_task: - self._stop_frame_task.cancel() - await self._stop_frame_task + await self.cancel_task(self._stop_frame_task) self._stop_frame_task = None async def _update_settings(self, settings: Dict[str, Any]): @@ -364,23 +362,20 @@ class TTSService(AIService): await self.push_frame(TTSTextFrame(text)) async def _stop_frame_handler(self): - try: - has_started = False - while True: - try: - frame = await asyncio.wait_for( - self._stop_frame_queue.get(), self._stop_frame_timeout_s - ) - if isinstance(frame, TTSStartedFrame): - has_started = True - elif isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): - has_started = False - except asyncio.TimeoutError: - if has_started: - await self.push_frame(TTSStoppedFrame()) - has_started = False - except asyncio.CancelledError: - pass + has_started = False + while True: + try: + frame = await asyncio.wait_for( + self._stop_frame_queue.get(), self._stop_frame_timeout_s + ) + if isinstance(frame, TTSStartedFrame): + has_started = True + elif isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): + has_started = False + except asyncio.TimeoutError: + if has_started: + await self.push_frame(TTSStoppedFrame()) + has_started = False class WordTTSService(TTSService): @@ -388,7 +383,7 @@ class WordTTSService(TTSService): super().__init__(**kwargs) self._initial_word_timestamp = -1 self._words_queue = asyncio.Queue() - self._words_task = self.get_event_loop().create_task(self._words_task_handler()) + self._words_task = self.create_task(self._words_task_handler()) def start_word_timestamps(self): if self._initial_word_timestamp == -1: @@ -421,35 +416,29 @@ class WordTTSService(TTSService): async def _stop_words_task(self): if self._words_task: - self._words_task.cancel() - await self._words_task + await self.cancel_task(self._words_task) self._words_task = None async def _words_task_handler(self): last_pts = 0 while True: - try: - (word, timestamp) = await self._words_queue.get() - if word == "Reset" and timestamp == 0: - self.reset_word_timestamps() - frame = None - elif word == "LLMFullResponseEndFrame" and timestamp == 0: - frame = LLMFullResponseEndFrame() - frame.pts = last_pts - elif word == "TTSStoppedFrame" and timestamp == 0: - frame = TTSStoppedFrame() - frame.pts = last_pts - else: - frame = TTSTextFrame(word) - frame.pts = self._initial_word_timestamp + timestamp - if frame: - last_pts = frame.pts - await self.push_frame(frame) - self._words_queue.task_done() - except asyncio.CancelledError: - break - except Exception as e: - logger.exception(f"{self} exception: {e}") + (word, timestamp) = await self._words_queue.get() + if word == "Reset" and timestamp == 0: + self.reset_word_timestamps() + frame = None + elif word == "LLMFullResponseEndFrame" and timestamp == 0: + frame = LLMFullResponseEndFrame() + frame.pts = last_pts + elif word == "TTSStoppedFrame" and timestamp == 0: + frame = TTSStoppedFrame() + frame.pts = last_pts + else: + frame = TTSTextFrame(word) + frame.pts = self._initial_word_timestamp + timestamp + if frame: + last_pts = frame.pts + await self.push_frame(frame) + self._words_queue.task_done() class STTService(AIService): diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 35856ea52..75da116d5 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -187,16 +187,13 @@ class CartesiaTTSService(WordTTSService, WebsocketService): async def _connect(self): await self._connect_websocket() - self._receive_task = self.get_event_loop().create_task( - self._receive_task_handler(self.push_error) - ) + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): await self._disconnect_websocket() if self._receive_task: - self._receive_task.cancel() - await self._receive_task + await self.cancel_task(self._receive_task) self._receive_task = None async def _connect_websocket(self): diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 6a5898347..0854ae61a 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -299,20 +299,16 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService): async def _connect(self): await self._connect_websocket() - self._receive_task = self.get_event_loop().create_task( - self._receive_task_handler(self.push_error) - ) - self._keepalive_task = self.get_event_loop().create_task(self._keepalive_task_handler()) + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._keepalive_task = self.create_task(self._keepalive_task_handler()) async def _disconnect(self): if self._receive_task: - self._receive_task.cancel() - await self._receive_task + await self.cancel_task(self._receive_task) self._receive_task = None if self._keepalive_task: - self._keepalive_task.cancel() - await self._keepalive_task + await self.cancel_task(self._keepalive_task) self._keepalive_task = None await self._disconnect_websocket() @@ -383,13 +379,8 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService): async def _keepalive_task_handler(self): while True: - try: - await asyncio.sleep(10) - await self._send_text("") - except asyncio.CancelledError: - break - except Exception as e: - logger.error(f"{self} exception: {e}") + await asyncio.sleep(10) + await self._send_text("") async def _send_text(self, text: str): if self._websocket: diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index 710ab04ec..6fadab683 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -104,15 +104,12 @@ class FishAudioTTSService(TTSService, WebsocketService): async def _connect(self): await self._connect_websocket() - self._receive_task = self.get_event_loop().create_task( - self._receive_task_handler(self.push_error) - ) + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): await self._disconnect_websocket() if self._receive_task: - self._receive_task.cancel() - await self._receive_task + await self.cancel_task(self._receive_task) self._receive_task = None async def _connect_websocket(self): diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 0241b3b64..ef509e4ae 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -275,7 +275,7 @@ class GeminiMultimodalLiveLLMService(LLMService): ) await self.send_client_event(evt) if self._transcribe_user_audio and self._context: - asyncio.create_task(self._handle_transcribe_user_audio(audio, self._context)) + self.create_task(self._handle_transcribe_user_audio(audio, self._context)) async def _handle_transcribe_user_audio(self, audio, context): text = await self._transcribe_audio(audio, context) @@ -391,7 +391,7 @@ class GeminiMultimodalLiveLLMService(LLMService): uri = f"wss://{self.base_url}/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key={self.api_key}" logger.info(f"Connecting to {uri}") self._websocket = await websockets.connect(uri=uri) - self._receive_task = self.get_event_loop().create_task(self._receive_task_handler()) + self._receive_task = self.create_task(self._receive_task_handler()) config = events.Config.model_validate( { "setup": { @@ -441,11 +441,7 @@ class GeminiMultimodalLiveLLMService(LLMService): await self._websocket.close() self._websocket = None if self._receive_task: - self._receive_task.cancel() - try: - await asyncio.wait_for(self._receive_task, timeout=1.0) - except asyncio.TimeoutError: - logger.warning("Timed out waiting for receive task to finish") + await self.cancel_task(self._receive_task, timeout=1.0) self._receive_task = None self._disconnecting = False except Exception as e: @@ -497,6 +493,7 @@ class GeminiMultimodalLiveLLMService(LLMService): pass except asyncio.CancelledError: logger.debug("websocket receive task cancelled") + raise except Exception as e: logger.error(f"{self} exception: {e}") @@ -679,7 +676,7 @@ class GeminiMultimodalLiveLLMService(LLMService): self._bot_text_buffer = "" if audio and self._transcribe_model_audio and self._context: - asyncio.create_task(self._handle_transcribe_model_audio(audio, self._context)) + self.create_task(self._handle_transcribe_model_audio(audio, self._context)) elif text: await self.push_frame(LLMFullResponseEndFrame()) diff --git a/src/pipecat/services/gladia.py b/src/pipecat/services/gladia.py index 3487be5ae..e240252a2 100644 --- a/src/pipecat/services/gladia.py +++ b/src/pipecat/services/gladia.py @@ -180,7 +180,7 @@ class GladiaSTTService(STTService): await super().start(frame) response = await self._setup_gladia() self._websocket = await websockets.connect(response["url"]) - self._receive_task = self.get_event_loop().create_task(self._receive_task_handler()) + self._receive_task = self.create_task(self._receive_task_handler()) async def stop(self, frame: EndFrame): await super().stop(frame) diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index 6f654293a..327901423 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -113,16 +113,13 @@ class LmntTTSService(TTSService, WebsocketService): async def _connect(self): await self._connect_websocket() - self._receive_task = self.get_event_loop().create_task( - self._receive_task_handler(self.push_error) - ) + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): await self._disconnect_websocket() if self._receive_task: - self._receive_task.cancel() - await self._receive_task + await self.cancel_task(self._receive_task) self._receive_task = None async def _connect_websocket(self): diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index d5acf5afd..a42ed7b9a 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -277,7 +277,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): "OpenAI-Beta": "realtime=v1", }, ) - self._receive_task = self.get_event_loop().create_task(self._receive_task_handler()) + self._receive_task = self.create_task(self._receive_task_handler()) except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None @@ -291,11 +291,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self._websocket.close() self._websocket = None if self._receive_task: - self._receive_task.cancel() - try: - await asyncio.wait_for(self._receive_task, timeout=1.0) - except asyncio.TimeoutError: - logger.warning("Timed out waiting for receive task to finish") + await self.cancel_task(self._receive_task, timeout=1.0) self._receive_task = None self._disconnecting = False except Exception as e: @@ -332,40 +328,32 @@ class OpenAIRealtimeBetaLLMService(LLMService): # async def _receive_task_handler(self): - try: - async for message in self._websocket: - evt = events.parse_server_event(message) - if evt.type == "session.created": - await self._handle_evt_session_created(evt) - elif evt.type == "session.updated": - await self._handle_evt_session_updated(evt) - elif evt.type == "response.audio.delta": - await self._handle_evt_audio_delta(evt) - elif evt.type == "response.audio.done": - await self._handle_evt_audio_done(evt) - elif evt.type == "conversation.item.created": - await self._handle_evt_conversation_item_created(evt) - elif evt.type == "conversation.item.input_audio_transcription.completed": - await self.handle_evt_input_audio_transcription_completed(evt) - elif evt.type == "response.done": - await self._handle_evt_response_done(evt) - elif evt.type == "input_audio_buffer.speech_started": - await self._handle_evt_speech_started(evt) - elif evt.type == "input_audio_buffer.speech_stopped": - await self._handle_evt_speech_stopped(evt) - elif evt.type == "response.audio_transcript.delta": - await self._handle_evt_audio_transcript_delta(evt) - elif evt.type == "error": - await self._handle_evt_error(evt) - # errors are fatal, so exit the receive loop - return - - else: - pass - except asyncio.CancelledError: - logger.debug("websocket receive task cancelled") - except Exception as e: - logger.error(f"{self} exception: {e}") + async for message in self._websocket: + evt = events.parse_server_event(message) + if evt.type == "session.created": + await self._handle_evt_session_created(evt) + elif evt.type == "session.updated": + await self._handle_evt_session_updated(evt) + elif evt.type == "response.audio.delta": + await self._handle_evt_audio_delta(evt) + elif evt.type == "response.audio.done": + await self._handle_evt_audio_done(evt) + elif evt.type == "conversation.item.created": + await self._handle_evt_conversation_item_created(evt) + elif evt.type == "conversation.item.input_audio_transcription.completed": + await self.handle_evt_input_audio_transcription_completed(evt) + elif evt.type == "response.done": + await self._handle_evt_response_done(evt) + elif evt.type == "input_audio_buffer.speech_started": + await self._handle_evt_speech_started(evt) + elif evt.type == "input_audio_buffer.speech_stopped": + await self._handle_evt_speech_stopped(evt) + elif evt.type == "response.audio_transcript.delta": + await self._handle_evt_audio_transcript_delta(evt) + elif evt.type == "error": + await self._handle_evt_error(evt) + # errors are fatal, so exit the receive loop + return async def _handle_evt_session_created(self, evt): # session.created is received right after connecting. Send a message diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 46b724a4d..07e78ae52 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -165,16 +165,13 @@ class PlayHTTTSService(TTSService, WebsocketService): async def _connect(self): await self._connect_websocket() - self._receive_task = self.get_event_loop().create_task( - self._receive_task_handler(self.push_error) - ) + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): await self._disconnect_websocket() if self._receive_task: - self._receive_task.cancel() - await self._receive_task + await self.cancel_task(self._receive_task) self._receive_task = None async def _connect_websocket(self): diff --git a/src/pipecat/services/riva.py b/src/pipecat/services/riva.py index 662ac21f5..cac1946d5 100644 --- a/src/pipecat/services/riva.py +++ b/src/pipecat/services/riva.py @@ -202,8 +202,8 @@ class ParakeetSTTService(STTService): async def start(self, frame: StartFrame): await super().start(frame) - self._thread_task = self.get_event_loop().create_task(self._thread_task_handler()) - self._response_task = self.get_event_loop().create_task(self._response_task_handler()) + self._thread_task = self.create_task(self._thread_task_handler()) + self._response_task = self.create_task(self._response_task_handler()) self._response_queue = asyncio.Queue() async def stop(self, frame: EndFrame): @@ -215,10 +215,8 @@ class ParakeetSTTService(STTService): await self._stop_tasks() async def _stop_tasks(self): - self._thread_task.cancel() - await self._thread_task - self._response_task.cancel() - await self._response_task + await self.cancel_task(self._thread_task) + await self.cancel_task(self._response_task) def _response_handler(self): responses = self._asr_service.streaming_response_generator( @@ -238,7 +236,7 @@ class ParakeetSTTService(STTService): await asyncio.to_thread(self._response_handler) except asyncio.CancelledError: self._thread_running = False - pass + raise async def _handle_response(self, response): for result in response.results: @@ -260,11 +258,8 @@ class ParakeetSTTService(STTService): async def _response_task_handler(self): while True: - try: - response = await self._response_queue.get() - await self._handle_response(response) - except asyncio.CancelledError: - break + response = await self._response_queue.get() + await self._handle_response(response) async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: await self._queue.put(audio) diff --git a/src/pipecat/services/simli.py b/src/pipecat/services/simli.py index be7edfdbd..5a6aecbe4 100644 --- a/src/pipecat/services/simli.py +++ b/src/pipecat/services/simli.py @@ -49,45 +49,33 @@ class SimliVideoService(FrameProcessor): async def _start_connection(self): await self._simli_client.Initialize() # Create task to consume and process audio and video - self._audio_task = asyncio.create_task(self._consume_and_process_audio()) - self._video_task = asyncio.create_task(self._consume_and_process_video()) + self._audio_task = self.create_task(self._consume_and_process_audio()) + self._video_task = self.create_task(self._consume_and_process_video()) async def _consume_and_process_audio(self): - try: - await self._pipecat_resampler_event.wait() - async for audio_frame in self._simli_client.getAudioStreamIterator(): - resampled_frames = self._pipecat_resampler.resample(audio_frame) - for resampled_frame in resampled_frames: - await self.push_frame( - TTSAudioRawFrame( - audio=resampled_frame.to_ndarray().tobytes(), - sample_rate=self._pipecat_resampler.rate, - num_channels=1, - ), - ) - except Exception as e: - logger.exception(f"{self} exception: {e}") - except asyncio.CancelledError: - pass + await self._pipecat_resampler_event.wait() + async for audio_frame in self._simli_client.getAudioStreamIterator(): + resampled_frames = self._pipecat_resampler.resample(audio_frame) + for resampled_frame in resampled_frames: + await self.push_frame( + TTSAudioRawFrame( + audio=resampled_frame.to_ndarray().tobytes(), + sample_rate=self._pipecat_resampler.rate, + num_channels=1, + ), + ) async def _consume_and_process_video(self): - try: - await self._pipecat_resampler_event.wait() - async for video_frame in self._simli_client.getVideoStreamIterator( - targetFormat="rgb24" - ): - # Process the video frame - convertedFrame: OutputImageRawFrame = OutputImageRawFrame( - image=video_frame.to_rgb().to_image().tobytes(), - size=(video_frame.width, video_frame.height), - format="RGB", - ) - convertedFrame.pts = video_frame.pts - await self.push_frame(convertedFrame) - except Exception as e: - logger.exception(f"{self} exception: {e}") - except asyncio.CancelledError: - pass + await self._pipecat_resampler_event.wait() + async for video_frame in self._simli_client.getVideoStreamIterator(targetFormat="rgb24"): + # Process the video frame + convertedFrame: OutputImageRawFrame = OutputImageRawFrame( + image=video_frame.to_rgb().to_image().tobytes(), + size=(video_frame.width, video_frame.height), + format="RGB", + ) + convertedFrame.pts = video_frame.pts + await self.push_frame(convertedFrame) async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -128,8 +116,6 @@ class SimliVideoService(FrameProcessor): async def _stop(self): await self._simli_client.stop() if self._audio_task: - self._audio_task.cancel() - await self._audio_task + await self.cancel_task(self._audio_task) if self._video_task: - self._video_task.cancel() - await self._video_task + await self.cancel_task(self._video_task) diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index 365f5a7c8..7e480f2b7 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -85,10 +85,6 @@ class WebsocketService(ABC): await self._receive_messages() logger.debug(f"{self} connection established successfully") retry_count = 0 # Reset counter on successful message receive - - except asyncio.CancelledError: - break - except Exception as e: retry_count += 1 if retry_count >= MAX_RETRIES: diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 93442f90a..192101e62 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -50,13 +50,12 @@ class BaseInputTransport(FrameProcessor): # Create audio input queue and task if needed. if self._params.audio_in_enabled or self._params.vad_enabled: self._audio_in_queue = asyncio.Queue() - self._audio_task = self.get_event_loop().create_task(self._audio_task_handler()) + self._audio_task = self.create_task(self._audio_task_handler()) async def stop(self, frame: EndFrame): # Cancel and wait for the audio input task to finish. if self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled): - self._audio_task.cancel() - await self._audio_task + await self.cancel_task(self._audio_task) self._audio_task = None # Stop audio filter. if self._params.audio_in_filter: @@ -65,8 +64,7 @@ class BaseInputTransport(FrameProcessor): async def cancel(self, frame: CancelFrame): # Cancel and wait for the audio input task to finish. if self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled): - self._audio_task.cancel() - await self._audio_task + await self.cancel_task(self._audio_task) self._audio_task = None def vad_analyzer(self) -> VADAnalyzer | None: @@ -173,27 +171,22 @@ class BaseInputTransport(FrameProcessor): async def _audio_task_handler(self): vad_state: VADState = VADState.QUIET while True: - try: - frame: InputAudioRawFrame = await self._audio_in_queue.get() + frame: InputAudioRawFrame = await self._audio_in_queue.get() - audio_passthrough = True + audio_passthrough = True - # If an audio filter is available, run it before VAD. - if self._params.audio_in_filter: - frame.audio = await self._params.audio_in_filter.filter(frame.audio) + # If an audio filter is available, run it before VAD. + if self._params.audio_in_filter: + frame.audio = await self._params.audio_in_filter.filter(frame.audio) - # Check VAD and push event if necessary. We just care about - # changes from QUIET to SPEAKING and vice versa. - if self._params.vad_enabled: - vad_state = await self._handle_vad(frame, vad_state) - audio_passthrough = self._params.vad_audio_passthrough + # Check VAD and push event if necessary. We just care about + # changes from QUIET to SPEAKING and vice versa. + if self._params.vad_enabled: + vad_state = await self._handle_vad(frame, vad_state) + audio_passthrough = self._params.vad_audio_passthrough - # Push audio downstream if passthrough. - if audio_passthrough: - await self.push_frame(frame) + # Push audio downstream if passthrough. + if audio_passthrough: + await self.push_frame(frame) - self._audio_in_queue.task_done() - except asyncio.CancelledError: - break - except Exception as e: - logger.exception(f"{self} error reading audio frames: {e}") + self._audio_in_queue.task_done() diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index ef073e22a..5181914ed 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -217,22 +217,19 @@ class BaseOutputTransport(FrameProcessor): # def _create_sink_tasks(self): - loop = self.get_event_loop() self._sink_queue = asyncio.Queue() - self._sink_task = loop.create_task(self._sink_task_handler()) self._sink_clock_queue = asyncio.PriorityQueue() - self._sink_clock_task = loop.create_task(self._sink_clock_task_handler()) + self._sink_task = self.create_task(self._sink_task_handler()) + self._sink_clock_task = self.create_task(self._sink_clock_task_handler()) async def _cancel_sink_tasks(self): # Stop sink tasks. if self._sink_task: - self._sink_task.cancel() - await self._sink_task + await self.cancel_task(self._sink_task) self._sink_task = None # Stop sink clock tasks. if self._sink_clock_task: - self._sink_clock_task.cancel() - await self._sink_clock_task + await self.cancel_task(self._sink_clock_task) self._sink_clock_task = None async def _sink_frame_handler(self, frame: Frame): @@ -269,7 +266,7 @@ class BaseOutputTransport(FrameProcessor): self._sink_clock_queue.task_done() except asyncio.CancelledError: - break + raise except Exception as e: logger.exception(f"{self} error processing sink clock queue: {e}") @@ -317,49 +314,42 @@ class BaseOutputTransport(FrameProcessor): return without_mixer(vad_stop_secs) async def _sink_task_handler(self): - try: - async for frame in self._next_frame(): - # Notify the bot started speaking upstream if necessary and that - # it's actually speaking. - if isinstance(frame, TTSAudioRawFrame): - await self._bot_started_speaking() - await self.push_frame(BotSpeakingFrame()) - await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM) + async for frame in self._next_frame(): + # Notify the bot started speaking upstream if necessary and that + # it's actually speaking. + if isinstance(frame, TTSAudioRawFrame): + await self._bot_started_speaking() + await self.push_frame(BotSpeakingFrame()) + await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM) - # No need to push EndFrame, it's pushed from process_frame(). - if isinstance(frame, EndFrame): - break + # No need to push EndFrame, it's pushed from process_frame(). + if isinstance(frame, EndFrame): + break - # Handle frame. - await self._sink_frame_handler(frame) + # Handle frame. + await self._sink_frame_handler(frame) - # Also, push frame downstream in case anyone else needs it. - await self.push_frame(frame) + # Also, push frame downstream in case anyone else needs it. + await self.push_frame(frame) - # Send audio. - if isinstance(frame, OutputAudioRawFrame): - await self.write_raw_audio_frames(frame.audio) - except asyncio.CancelledError: - pass - except Exception as e: - logger.exception(f"{self} error writing to microphone: {e}") + # Send audio. + if isinstance(frame, OutputAudioRawFrame): + await self.write_raw_audio_frames(frame.audio) # # Camera task # def _create_camera_task(self): - loop = self.get_event_loop() # Create camera output queue and task if needed. if self._params.camera_out_enabled: self._camera_out_queue = asyncio.Queue() - self._camera_out_task = loop.create_task(self._camera_out_task_handler()) + self._camera_out_task = self.create_task(self._camera_out_task_handler()) async def _cancel_camera_task(self): # Stop camera output task. if self._camera_out_task and self._params.camera_out_enabled: - self._camera_out_task.cancel() - await self._camera_out_task + await self.cancel_task(self._camera_out_task) self._camera_out_task = None async def _draw_image(self, frame: OutputImageRawFrame): @@ -387,19 +377,14 @@ class BaseOutputTransport(FrameProcessor): self._camera_out_frame_duration = 1 / self._params.camera_out_framerate self._camera_out_frame_reset = self._camera_out_frame_duration * 5 while True: - try: - if self._params.camera_out_is_live: - await self._camera_out_is_live_handler() - elif self._camera_images: - image = next(self._camera_images) - await self._draw_image(image) - await asyncio.sleep(self._camera_out_frame_duration) - else: - await asyncio.sleep(self._camera_out_frame_duration) - except asyncio.CancelledError: - break - except Exception as e: - logger.exception(f"{self} error writing to camera: {e}") + if self._params.camera_out_is_live: + await self._camera_out_is_live_handler() + elif self._camera_images: + image = next(self._camera_images) + await self._draw_image(image) + await asyncio.sleep(self._camera_out_frame_duration) + else: + await asyncio.sleep(self._camera_out_frame_duration) async def _camera_out_is_live_handler(self): image = await self._camera_out_queue.get() diff --git a/src/pipecat/transports/network/fastapi_websocket.py b/src/pipecat/transports/network/fastapi_websocket.py index fc5804611..42f1d3171 100644 --- a/src/pipecat/transports/network/fastapi_websocket.py +++ b/src/pipecat/transports/network/fastapi_websocket.py @@ -68,11 +68,9 @@ class FastAPIWebsocketInputTransport(BaseInputTransport): async def start(self, frame: StartFrame): await super().start(frame) if self._params.session_timeout: - self._monitor_websocket_task = self.get_event_loop().create_task( - self._monitor_websocket() - ) + self._monitor_websocket_task = self.create_task(self._monitor_websocket()) await self._callbacks.on_client_connected(self._websocket) - self._receive_task = self.get_event_loop().create_task(self._receive_messages()) + self._receive_task = self.create_task(self._receive_messages()) def _iter_data(self) -> typing.AsyncIterator[bytes | str]: if self._params.serializer.type == FrameSerializerType.BINARY: @@ -96,11 +94,8 @@ class FastAPIWebsocketInputTransport(BaseInputTransport): async def _monitor_websocket(self): """Wait for self._params.session_timeout seconds, if the websocket is still open, trigger timeout event.""" - try: - await asyncio.sleep(self._params.session_timeout) - await self._callbacks.on_session_timeout(self._websocket) - except asyncio.CancelledError: - logger.info(f"Monitoring task cancelled for: {self._websocket}") + await asyncio.sleep(self._params.session_timeout) + await self._callbacks.on_session_timeout(self._websocket) class FastAPIWebsocketOutputTransport(BaseOutputTransport): diff --git a/src/pipecat/transports/network/websocket_server.py b/src/pipecat/transports/network/websocket_server.py index 9bd344cb1..6fe5f91ef 100644 --- a/src/pipecat/transports/network/websocket_server.py +++ b/src/pipecat/transports/network/websocket_server.py @@ -71,7 +71,7 @@ class WebsocketServerInputTransport(BaseInputTransport): async def start(self, frame: StartFrame): await super().start(frame) - self._server_task = self.get_event_loop().create_task(self._server_task_handler()) + self._server_task = self.create_task(self._server_task_handler()) async def stop(self, frame: EndFrame): await super().stop(frame) @@ -131,6 +131,7 @@ class WebsocketServerInputTransport(BaseInputTransport): await self._callbacks.on_session_timeout(websocket) except asyncio.CancelledError: logger.info(f"Monitoring task cancelled for: {websocket.remote_address}") + raise class WebsocketServerOutputTransport(BaseOutputTransport): diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 47575bcce..f3915f662 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -46,6 +46,7 @@ from pipecat.transcriptions.language import Language from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.utils.utils import cancel_task, create_task try: from daily import CallClient, Daily, EventHandler @@ -218,7 +219,9 @@ class DailyTransportClient(EventHandler): # future) we will deadlock because completions use event handlers (which # are holding the GIL). self._callback_queue = asyncio.Queue() - self._callback_task = self._loop.create_task(self._callback_task_handler()) + self._callback_task = create_task( + self._loop, self._callback_task_handler(), "DailyTransportClient::callback_task" + ) self._camera: VirtualCameraDevice | None = None if self._params.camera_out_enabled: @@ -469,8 +472,7 @@ class DailyTransportClient(EventHandler): return await asyncio.wait_for(future, timeout=10) async def cleanup(self): - self._callback_task.cancel() - await self._callback_task + await cancel_task(self._callback_task) # Make sure we don't block the event loop in case `client.release()` # takes extra time. await self._loop.run_in_executor(self._executor, self._cleanup) @@ -687,11 +689,8 @@ class DailyTransportClient(EventHandler): async def _callback_task_handler(self): while True: - try: - (callback, *args) = await self._callback_queue.get() - await callback(*args) - except asyncio.CancelledError: - break + (callback, *args) = await self._callback_queue.get() + await callback(*args) class DailyInputTransport(BaseInputTransport): @@ -721,7 +720,7 @@ class DailyInputTransport(BaseInputTransport): # Create audio task. It reads audio frames from Daily and push them # internally for VAD processing. if self._params.audio_in_enabled or self._params.vad_enabled: - self._audio_in_task = self.get_event_loop().create_task(self._audio_in_task_handler()) + self._audio_in_task = self.create_task(self._audio_in_task_handler()) async def stop(self, frame: EndFrame): # Parent stop. @@ -730,8 +729,7 @@ class DailyInputTransport(BaseInputTransport): await self._client.leave() # Stop audio thread. if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): - self._audio_in_task.cancel() - await self._audio_in_task + await self.cancel_task(self._audio_in_task) self._audio_in_task = None async def cancel(self, frame: CancelFrame): @@ -741,8 +739,7 @@ class DailyInputTransport(BaseInputTransport): await self._client.leave() # Stop audio thread. if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): - self._audio_in_task.cancel() - await self._audio_in_task + await self.cancel_task(self._audio_in_task) self._audio_in_task = None async def cleanup(self): @@ -779,12 +776,9 @@ class DailyInputTransport(BaseInputTransport): async def _audio_in_task_handler(self): while True: - try: - frame = await self._client.read_next_audio_frame() - if frame: - await self.push_audio_frame(frame) - except asyncio.CancelledError: - break + frame = await self._client.read_next_audio_frame() + if frame: + await self.push_audio_frame(frame) # # Camera in diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index 0b036cecb..be916e239 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -28,6 +28,7 @@ from pipecat.processors.frame_processor import FrameDirection from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.utils.utils import create_task try: from livekit import rtc @@ -215,10 +216,18 @@ class LiveKitTransportClient: # Wrapper methods for event handlers def _on_participant_connected_wrapper(self, participant: rtc.RemoteParticipant): - asyncio.create_task(self._async_on_participant_connected(participant)) + create_task( + self._loop, + self._async_on_participant_connected(participant), + "LiveKitTransportClient::_async_on_participant_connected", + ) def _on_participant_disconnected_wrapper(self, participant: rtc.RemoteParticipant): - asyncio.create_task(self._async_on_participant_disconnected(participant)) + create_task( + self._loop, + self._async_on_participant_disconnected(participant), + "LiveKitTransportClient::_async_on_participant_disconnected", + ) def _on_track_subscribed_wrapper( self, @@ -226,7 +235,11 @@ class LiveKitTransportClient: publication: rtc.RemoteTrackPublication, participant: rtc.RemoteParticipant, ): - asyncio.create_task(self._async_on_track_subscribed(track, publication, participant)) + create_task( + self._loop, + self._async_on_track_subscribed(track, publication, participant), + "LiveKitTransportClient::_async_on_track_subscribed", + ) def _on_track_unsubscribed_wrapper( self, @@ -234,16 +247,30 @@ class LiveKitTransportClient: publication: rtc.RemoteTrackPublication, participant: rtc.RemoteParticipant, ): - asyncio.create_task(self._async_on_track_unsubscribed(track, publication, participant)) + create_task( + self._loop, + self._async_on_track_unsubscribed(track, publication, participant), + "LiveKitTransportClient::_async_on_track_unsubscribed", + ) def _on_data_received_wrapper(self, data: rtc.DataPacket): - asyncio.create_task(self._async_on_data_received(data)) + create_task( + self._loop, + self._async_on_data_received(data), + "LiveKitTransportClient::_async_on_data_received", + ) def _on_connected_wrapper(self): - asyncio.create_task(self._async_on_connected()) + create_task( + self._loop, self._async_on_connected(), "LiveKitTransportClient::_async_on_connected" + ) def _on_disconnected_wrapper(self): - asyncio.create_task(self._async_on_disconnected()) + create_task( + self._loop, + self._async_on_disconnected(), + "LiveKitTransportClient::_async_on_disconnected", + ) # Async methods for event handling async def _async_on_participant_connected(self, participant: rtc.RemoteParticipant): @@ -269,7 +296,11 @@ class LiveKitTransportClient: logger.info(f"Audio track subscribed: {track.sid} from participant {participant.sid}") self._audio_tracks[participant.sid] = track audio_stream = rtc.AudioStream(track) - asyncio.create_task(self._process_audio_stream(audio_stream, participant.sid)) + create_task( + self._loop, + self._process_audio_stream(audio_stream, participant.sid), + "LiveKitTransportClient::_process_audio_stream", + ) async def _async_on_track_unsubscribed( self, @@ -319,23 +350,21 @@ class LiveKitInputTransport(BaseInputTransport): await super().start(frame) await self._client.connect() if self._params.audio_in_enabled or self._params.vad_enabled: - self._audio_in_task = asyncio.create_task(self._audio_in_task_handler()) + self._audio_in_task = self.create_task(self._audio_in_task_handler()) logger.info("LiveKitInputTransport started") async def stop(self, frame: EndFrame): await super().stop(frame) await self._client.disconnect() if self._audio_in_task: - self._audio_in_task.cancel() - await self._audio_in_task + await self.cancel_task(self._audio_in_task) logger.info("LiveKitInputTransport stopped") async def cancel(self, frame: CancelFrame): await super().cancel(frame) await self._client.disconnect() if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): - self._audio_in_task.cancel() - await self._audio_in_task + await self.cancel_task(self._audio_in_task) def vad_analyzer(self) -> VADAnalyzer | None: return self._vad_analyzer @@ -347,22 +376,16 @@ class LiveKitInputTransport(BaseInputTransport): async def _audio_in_task_handler(self): logger.info("Audio input task started") while True: - try: - audio_data = await self._client.get_next_audio_frame() - if audio_data: - audio_frame_event, participant_id = audio_data - pipecat_audio_frame = self._convert_livekit_audio_to_pipecat(audio_frame_event) - input_audio_frame = InputAudioRawFrame( - audio=pipecat_audio_frame.audio, - sample_rate=pipecat_audio_frame.sample_rate, - num_channels=pipecat_audio_frame.num_channels, - ) - await self.push_audio_frame(input_audio_frame) - except asyncio.CancelledError: - logger.info("Audio input task cancelled") - break - except Exception as e: - logger.error(f"Error in audio input task: {e}") + audio_data = await self._client.get_next_audio_frame() + if audio_data: + audio_frame_event, participant_id = audio_data + pipecat_audio_frame = self._convert_livekit_audio_to_pipecat(audio_frame_event) + input_audio_frame = InputAudioRawFrame( + audio=pipecat_audio_frame.audio, + sample_rate=pipecat_audio_frame.sample_rate, + num_channels=pipecat_audio_frame.num_channels, + ) + await self.push_audio_frame(input_audio_frame) def _convert_livekit_audio_to_pipecat( self, audio_frame_event: rtc.AudioFrameEvent diff --git a/src/pipecat/utils/utils.py b/src/pipecat/utils/utils.py index 5470e63a5..59266601b 100644 --- a/src/pipecat/utils/utils.py +++ b/src/pipecat/utils/utils.py @@ -3,8 +3,13 @@ # # SPDX-License-Identifier: BSD 2-Clause License # + +import asyncio import collections import itertools +from typing import Coroutine, Optional + +from loguru import logger _COUNTS = collections.defaultdict(itertools.count) _ID = itertools.count() @@ -35,3 +40,37 @@ def obj_count(obj) -> int: 0 """ return next(_COUNTS[obj.__class__.__name__]) + + +def create_task(loop: asyncio.AbstractEventLoop, coroutine: Coroutine, name: str) -> asyncio.Task: + async def run_coroutine(): + try: + await coroutine + except asyncio.CancelledError: + logger.trace(f"{name}: cancelling task") + # Re-raise the exception to ensure the task is cancelled. + raise + except Exception as e: + logger.exception(f"{name}: unexpected exception: {e}") + + task = loop.create_task(run_coroutine()) + task.set_name(name) + logger.trace(f"{name}: task created") + return task + + +async def cancel_task(task: asyncio.Task, timeout: Optional[float] = None): + name = task.get_name() + task.cancel() + try: + if timeout: + await asyncio.wait_for(task, timeout=timeout) + else: + await task + except asyncio.TimeoutError: + logger.warning(f"{name}: timed out waiting for task to finish") + except asyncio.CancelledError: + # Here are sure the task is cancelled properly. + logger.trace(f"{name}: task cancelled") + except Exception as e: + logger.exception(f"{name}: unexpected exception while cancelling task: {e}") From 57b186cde8ca0b374196ee419df30c070dc54703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 16:33:34 -0800 Subject: [PATCH 02/28] base_transport: add name and id fields --- src/pipecat/transports/base_transport.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py index bcc13c4fd..00e269359 100644 --- a/src/pipecat/transports/base_transport.py +++ b/src/pipecat/transports/base_transport.py @@ -16,6 +16,7 @@ from pipecat.audio.filters.base_audio_filter import BaseAudioFilter from pipecat.audio.mixers.base_audio_mixer import BaseAudioMixer from pipecat.audio.vad.vad_analyzer import VADAnalyzer from pipecat.processors.frame_processor import FrameProcessor +from pipecat.utils.utils import obj_count, obj_id class TransportParams(BaseModel): @@ -46,10 +47,14 @@ class TransportParams(BaseModel): class BaseTransport(ABC): def __init__( self, - input_name: str | None = None, - output_name: str | None = None, - loop: asyncio.AbstractEventLoop | None = None, + *, + name: Optional[str] = None, + input_name: Optional[str] = None, + output_name: Optional[str] = None, + loop: Optional[asyncio.AbstractEventLoop] = None, ): + self.id: int = obj_id() + self.name = name or f"{self.__class__.__name__}#{obj_count(self)}" self._input_name = input_name self._output_name = output_name self._loop = loop or asyncio.get_running_loop() @@ -89,3 +94,6 @@ class BaseTransport(ABC): handler(self, *args, **kwargs) except Exception as e: logger.exception(f"Exception in event handler {event_name}: {e}") + + def __str__(self): + return self.name From 5885fcc230f4aaee984d54b04699fb4516278cb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 16:36:36 -0800 Subject: [PATCH 03/28] add id and name properties --- src/pipecat/pipeline/task.py | 12 ++++++++++-- src/pipecat/pipeline/task_observer.py | 13 +++++++++++-- src/pipecat/processors/frame_processor.py | 12 ++++++++++-- src/pipecat/transports/base_transport.py | 12 ++++++++++-- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 34a7bb5eb..9149a41a2 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -94,8 +94,8 @@ class PipelineTask(BaseTask): params: PipelineParams = PipelineParams(), clock: BaseClock = SystemClock(), ): - self.id: int = obj_id() - self.name: str = f"{self.__class__.__name__}#{obj_count(self)}" + self._id: int = obj_id() + self._name: str = f"{self.__class__.__name__}#{obj_count(self)}" self._pipeline = pipeline self._clock = clock @@ -123,6 +123,14 @@ class PipelineTask(BaseTask): self._observer = TaskObserver(params.observers) + @property + def id(self) -> int: + return self._id + + @property + def name(self) -> str: + return self._name + def has_finished(self) -> bool: """Indicates whether the tasks has finished. That is, all processors have stopped. diff --git a/src/pipecat/pipeline/task_observer.py b/src/pipecat/pipeline/task_observer.py index 1bf4ff0f9..da5a6818f 100644 --- a/src/pipecat/pipeline/task_observer.py +++ b/src/pipecat/pipeline/task_observer.py @@ -12,7 +12,7 @@ from attr import dataclass from pipecat.frames.frames import Frame from pipecat.observers.base_observer import BaseObserver from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.utils.utils import cancel_task, create_task, obj_count +from pipecat.utils.utils import cancel_task, create_task, obj_count, obj_id @dataclass @@ -55,9 +55,18 @@ class TaskObserver(BaseObserver): """ def __init__(self, observers: List[BaseObserver] = []): - self.name: str = f"{self.__class__.__name__}#{obj_count(self)}" + self._id: int = obj_id() + self._name: str = f"{self.__class__.__name__}#{obj_count(self)}" self._proxies: List[Proxy] = self._create_proxies(observers) + @property + def id(self) -> int: + return self._id + + @property + def name(self) -> str: + return self._name + async def stop(self): """Stops all proxy observer tasks.""" for proxy in self._proxies: diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 1895147dd..6a8252677 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -41,8 +41,8 @@ class FrameProcessor: loop: asyncio.AbstractEventLoop | None = None, **kwargs, ): - self.id: int = obj_id() - self.name = name or f"{self.__class__.__name__}#{obj_count(self)}" + self._id: int = obj_id() + self._name = name or f"{self.__class__.__name__}#{obj_count(self)}" self._parent: "FrameProcessor" | None = None self._prev: "FrameProcessor" | None = None self._next: "FrameProcessor" | None = None @@ -83,6 +83,14 @@ class FrameProcessor: # the exception to this rule. This create this task. self.__create_push_task() + @property + def id(self) -> int: + return self._id + + @property + def name(self) -> str: + return self._name + @property def interruptions_allowed(self): return self._allow_interruptions diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py index 00e269359..fd14033f2 100644 --- a/src/pipecat/transports/base_transport.py +++ b/src/pipecat/transports/base_transport.py @@ -53,13 +53,21 @@ class BaseTransport(ABC): output_name: Optional[str] = None, loop: Optional[asyncio.AbstractEventLoop] = None, ): - self.id: int = obj_id() - self.name = name or f"{self.__class__.__name__}#{obj_count(self)}" + self._id: int = obj_id() + self._name = name or f"{self.__class__.__name__}#{obj_count(self)}" self._input_name = input_name self._output_name = output_name self._loop = loop or asyncio.get_running_loop() self._event_handlers: dict = {} + @property + def id(self) -> int: + return self._id + + @property + def name(self) -> str: + return self._name + @abstractmethod def input(self) -> FrameProcessor: raise NotImplementedError From ff0bcec33a1c2b23df775fa6b68f8d8c82af840b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 16:37:00 -0800 Subject: [PATCH 04/28] transports: improve task naming --- src/pipecat/transports/local/audio.py | 1 + src/pipecat/transports/local/tk.py | 1 + src/pipecat/transports/services/daily.py | 8 ++++++-- src/pipecat/transports/services/livekit.py | 22 +++++++++++++--------- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/pipecat/transports/local/audio.py b/src/pipecat/transports/local/audio.py index c65279be9..52681900d 100644 --- a/src/pipecat/transports/local/audio.py +++ b/src/pipecat/transports/local/audio.py @@ -98,6 +98,7 @@ class LocalAudioOutputTransport(BaseOutputTransport): class LocalAudioTransport(BaseTransport): def __init__(self, params: TransportParams): + super().__init__() self._params = params self._pyaudio = pyaudio.PyAudio() diff --git a/src/pipecat/transports/local/tk.py b/src/pipecat/transports/local/tk.py index 842eac2df..44ca45b33 100644 --- a/src/pipecat/transports/local/tk.py +++ b/src/pipecat/transports/local/tk.py @@ -127,6 +127,7 @@ class TkOutputTransport(BaseOutputTransport): class TkLocalTransport(BaseTransport): def __init__(self, tk_root: tk.Tk, params: TransportParams): + super().__init__() self._tk_root = tk_root self._params = params self._pyaudio = pyaudio.PyAudio() diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index f3915f662..bccc54316 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -181,6 +181,7 @@ class DailyTransportClient(EventHandler): params: DailyParams, callbacks: DailyCallbacks, loop: asyncio.AbstractEventLoop, + transport_name: str, ): super().__init__() @@ -194,6 +195,7 @@ class DailyTransportClient(EventHandler): self._params: DailyParams = params self._callbacks = callbacks self._loop = loop + self._transport_name = transport_name self._participant_id: str = "" self._video_renderers = {} @@ -220,7 +222,9 @@ class DailyTransportClient(EventHandler): # are holding the GIL). self._callback_queue = asyncio.Queue() self._callback_task = create_task( - self._loop, self._callback_task_handler(), "DailyTransportClient::callback_task" + self._loop, + self._callback_task_handler(), + f"{self._transport_name}::DailyTransportClient::callback_task", ) self._camera: VirtualCameraDevice | None = None @@ -907,7 +911,7 @@ class DailyTransport(BaseTransport): self._params = params self._client = DailyTransportClient( - room_url, token, bot_name, params, callbacks, self._loop + room_url, token, bot_name, params, callbacks, self._loop, self.name ) self._input: DailyInputTransport | None = None self._output: DailyOutputTransport | None = None diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index be916e239..f0a2add0e 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -73,6 +73,7 @@ class LiveKitTransportClient: params: LiveKitParams, callbacks: LiveKitCallbacks, loop: asyncio.AbstractEventLoop, + transport_name: str, ): self._url = url self._token = token @@ -80,6 +81,7 @@ class LiveKitTransportClient: self._params = params self._callbacks = callbacks self._loop = loop + self._transport_name = transport_name self._room = rtc.Room(loop=loop) self._participant_id: str = "" self._connected = False @@ -219,14 +221,14 @@ class LiveKitTransportClient: create_task( self._loop, self._async_on_participant_connected(participant), - "LiveKitTransportClient::_async_on_participant_connected", + f"{self._transport_name}::LiveKitTransportClient::_async_on_participant_connected", ) def _on_participant_disconnected_wrapper(self, participant: rtc.RemoteParticipant): create_task( self._loop, self._async_on_participant_disconnected(participant), - "LiveKitTransportClient::_async_on_participant_disconnected", + f"{self._transport_name}::LiveKitTransportClient::_async_on_participant_disconnected", ) def _on_track_subscribed_wrapper( @@ -238,7 +240,7 @@ class LiveKitTransportClient: create_task( self._loop, self._async_on_track_subscribed(track, publication, participant), - "LiveKitTransportClient::_async_on_track_subscribed", + f"{self._transport_name}::LiveKitTransportClient::_async_on_track_subscribed", ) def _on_track_unsubscribed_wrapper( @@ -250,26 +252,28 @@ class LiveKitTransportClient: create_task( self._loop, self._async_on_track_unsubscribed(track, publication, participant), - "LiveKitTransportClient::_async_on_track_unsubscribed", + f"{self._transport_name}::LiveKitTransportClient::_async_on_track_unsubscribed", ) def _on_data_received_wrapper(self, data: rtc.DataPacket): create_task( self._loop, self._async_on_data_received(data), - "LiveKitTransportClient::_async_on_data_received", + f"{self._transport_name}::LiveKitTransportClient::_async_on_data_received", ) def _on_connected_wrapper(self): create_task( - self._loop, self._async_on_connected(), "LiveKitTransportClient::_async_on_connected" + self._loop, + self._async_on_connected(), + f"{self._transport_name}::LiveKitTransportClient::_async_on_connected", ) def _on_disconnected_wrapper(self): create_task( self._loop, self._async_on_disconnected(), - "LiveKitTransportClient::_async_on_disconnected", + f"{self._transport_name}::LiveKitTransportClient::_async_on_disconnected", ) # Async methods for event handling @@ -299,7 +303,7 @@ class LiveKitTransportClient: create_task( self._loop, self._process_audio_stream(audio_stream, participant.sid), - "LiveKitTransportClient::_process_audio_stream", + f"{self._transport_name}::LiveKitTransportClient::_process_audio_stream", ) async def _async_on_track_unsubscribed( @@ -474,7 +478,7 @@ class LiveKitTransport(BaseTransport): self._params = params self._client = LiveKitTransportClient( - url, token, room_name, self._params, callbacks, self._loop + url, token, room_name, self._params, callbacks, self._loop, self.name ) self._input: LiveKitInputTransport | None = None self._output: LiveKitOutputTransport | None = None From cb93f6b3683bdf838a6e184a89f031abe12f6979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 16:37:14 -0800 Subject: [PATCH 05/28] utils: store created tasks and add current_tasks() --- src/pipecat/utils/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/pipecat/utils/utils.py b/src/pipecat/utils/utils.py index 59266601b..6a708843d 100644 --- a/src/pipecat/utils/utils.py +++ b/src/pipecat/utils/utils.py @@ -7,12 +7,13 @@ import asyncio import collections import itertools -from typing import Coroutine, Optional +from typing import Coroutine, Optional, Set from loguru import logger _COUNTS = collections.defaultdict(itertools.count) _ID = itertools.count() +_TASKS: Set[asyncio.Task] = set() def obj_id() -> int: @@ -55,6 +56,7 @@ def create_task(loop: asyncio.AbstractEventLoop, coroutine: Coroutine, name: str task = loop.create_task(run_coroutine()) task.set_name(name) + _TASKS.add(task) logger.trace(f"{name}: task created") return task @@ -72,5 +74,10 @@ async def cancel_task(task: asyncio.Task, timeout: Optional[float] = None): except asyncio.CancelledError: # Here are sure the task is cancelled properly. logger.trace(f"{name}: task cancelled") + _TASKS.remove(task) except Exception as e: logger.exception(f"{name}: unexpected exception while cancelling task: {e}") + + +def current_tasks() -> Set[asyncio.Task]: + return _TASKS From aff6e24560883fe8499ea1e6a0a1e1f58db5e143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 18:06:19 -0800 Subject: [PATCH 06/28] pipeline: fix pipeline cleanup --- src/pipecat/pipeline/pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/pipeline/pipeline.py b/src/pipecat/pipeline/pipeline.py index e6b874ce9..270e100bd 100644 --- a/src/pipecat/pipeline/pipeline.py +++ b/src/pipecat/pipeline/pipeline.py @@ -71,6 +71,7 @@ class Pipeline(BasePipeline): # async def cleanup(self): + await super().cleanup() await self._cleanup_processors() async def process_frame(self, frame: Frame, direction: FrameDirection): From 80a7f1b1e7c346e6e44311684d5ddd9ae493677c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 18:08:44 -0800 Subject: [PATCH 07/28] runner: improve signal handler task cancellation --- src/pipecat/pipeline/runner.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py index 34f7a0a8c..62e9e5e6f 100644 --- a/src/pipecat/pipeline/runner.py +++ b/src/pipecat/pipeline/runner.py @@ -19,6 +19,7 @@ class PipelineRunner: self.name: str = name or f"{self.__class__.__name__}#{obj_count(self)}" self._tasks = {} + self._sig_task = None if handle_sigint: self._setup_sigint() @@ -28,6 +29,10 @@ class PipelineRunner: self._tasks[task.name] = task await task.run() del self._tasks[task.name] + # If we are cancelling through a signal, make sure we wait for it so + # everything gets cleaned up nicely. + if self._sig_task: + await self._sig_task logger.debug(f"Runner {self} finished running {task}") async def stop_when_done(self): @@ -40,14 +45,14 @@ class PipelineRunner: def _setup_sigint(self): loop = asyncio.get_running_loop() - loop.add_signal_handler( - signal.SIGINT, lambda *args: asyncio.create_task(self._sig_handler()) - ) - loop.add_signal_handler( - signal.SIGTERM, lambda *args: asyncio.create_task(self._sig_handler()) - ) + loop.add_signal_handler(signal.SIGINT, lambda *args: self._sig_handler()) + loop.add_signal_handler(signal.SIGTERM, lambda *args: self._sig_handler()) - async def _sig_handler(self): + def _sig_handler(self): + if not self._sig_task: + self._sig_task = asyncio.create_task(self._sig_cancel()) + + async def _sig_cancel(self): logger.warning(f"Interruption detected. Canceling runner {self}") await self.cancel() From c2d8a45a07be3e2742c7b04f610aec96c37404f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 18:13:18 -0800 Subject: [PATCH 08/28] runner: warn about remaining dangling tasks --- src/pipecat/pipeline/runner.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py index 62e9e5e6f..8cdfc3eaa 100644 --- a/src/pipecat/pipeline/runner.py +++ b/src/pipecat/pipeline/runner.py @@ -10,7 +10,7 @@ import signal from loguru import logger from pipecat.pipeline.task import PipelineTask -from pipecat.utils.utils import obj_count, obj_id +from pipecat.utils.utils import current_tasks, obj_count, obj_id class PipelineRunner: @@ -33,6 +33,7 @@ class PipelineRunner: # everything gets cleaned up nicely. if self._sig_task: await self._sig_task + self._print_dangling_tasks() logger.debug(f"Runner {self} finished running {task}") async def stop_when_done(self): @@ -56,5 +57,10 @@ class PipelineRunner: logger.warning(f"Interruption detected. Canceling runner {self}") await self.cancel() + def _print_dangling_tasks(self): + tasks = [t.get_name() for t in current_tasks()] + if tasks: + logger.warning(f"Dangling tasks detected: {tasks}") + def __str__(self): return self.name From 6bca8396d37d555de47a73506de3e4ecda52d0fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 19:29:08 -0800 Subject: [PATCH 09/28] utils: error if we try to cancel the same task multiple times --- src/pipecat/utils/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pipecat/utils/utils.py b/src/pipecat/utils/utils.py index 6a708843d..045308c57 100644 --- a/src/pipecat/utils/utils.py +++ b/src/pipecat/utils/utils.py @@ -74,7 +74,10 @@ async def cancel_task(task: asyncio.Task, timeout: Optional[float] = None): except asyncio.CancelledError: # Here are sure the task is cancelled properly. logger.trace(f"{name}: task cancelled") - _TASKS.remove(task) + try: + _TASKS.remove(task) + except KeyError as e: + logger.error(f"{name}: error removing task (already removed?): {e}") except Exception as e: logger.exception(f"{name}: unexpected exception while cancelling task: {e}") From e48c0e52ef4bdea0b5e761bee3b697670462b24d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 19:29:35 -0800 Subject: [PATCH 10/28] transports(daily): avoid canceling task more than once --- src/pipecat/transports/services/daily.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index bccc54316..70df2f05a 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -476,7 +476,9 @@ class DailyTransportClient(EventHandler): return await asyncio.wait_for(future, timeout=10) async def cleanup(self): - await cancel_task(self._callback_task) + if self._callback_task: + await cancel_task(self._callback_task) + self._callback_task = None # Make sure we don't block the event loop in case `client.release()` # takes extra time. await self._loop.run_in_executor(self._executor, self._cleanup) From 0a9daa2f56c9639c5b5aab3db21903364282c876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 19:29:59 -0800 Subject: [PATCH 11/28] task: avoid canceling tasks more than once --- src/pipecat/pipeline/task.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 9149a41a2..8d659bb90 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -155,7 +155,8 @@ class PipelineTask(BaseTask): # out-of-band from the main streaming task which is what we want since # we want to cancel right away. await self._source.push_frame(CancelFrame()) - await self._cancel_tasks(True) + # Only cancel the push task. Everything else will be cancelled in run(). + await cancel_task(self._process_push_task) await self._cleanup() async def run(self): @@ -171,7 +172,7 @@ class PipelineTask(BaseTask): # well, because you get a CancelledError in every place you are # awaiting a task. pass - await self._cancel_tasks(False) + await self._cancel_tasks() self._finished = True async def queue_frame(self, frame: Frame): @@ -215,12 +216,9 @@ class PipelineTask(BaseTask): loop, self._heartbeat_monitor_handler(), f"{self}::_heartbeat_monitor_handler" ) - async def _cancel_tasks(self, cancel_push: bool): + async def _cancel_tasks(self): await self._maybe_cancel_heartbeat_tasks() - if cancel_push: - await cancel_task(self._process_push_task) - await cancel_task(self._process_up_task) await cancel_task(self._process_down_task) From af90b8b4fa573bdf556bd9cec82db61707dd6579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 23:39:18 -0800 Subject: [PATCH 12/28] utils: add wait_for_task() --- src/pipecat/pipeline/task.py | 4 ++-- src/pipecat/transports/base_output.py | 5 +++-- src/pipecat/utils/utils.py | 31 ++++++++++++++++++++++----- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 8d659bb90..3a814e15d 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -30,7 +30,7 @@ from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.pipeline.base_task import BaseTask from pipecat.pipeline.task_observer import TaskObserver from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.utils.utils import cancel_task, create_task, obj_count, obj_id +from pipecat.utils.utils import cancel_task, create_task, obj_count, obj_id, wait_for_task HEARTBEAT_SECONDS = 1.0 HEARTBEAT_MONITOR_SECONDS = HEARTBEAT_SECONDS * 5 @@ -165,7 +165,7 @@ class PipelineTask(BaseTask): """ try: push_task = self._create_tasks() - await asyncio.gather(push_task) + await wait_for_task(push_task) except asyncio.CancelledError: # We are awaiting on the push task and it might be cancelled # (e.g. Ctrl-C). This means we will get a CancelledError here as diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 5181914ed..1ec5ea7fa 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -36,6 +36,7 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.transports.base_transport import TransportParams from pipecat.utils.time import nanoseconds_to_seconds +from pipecat.utils.utils import wait_for_task class BaseOutputTransport(FrameProcessor): @@ -87,9 +88,9 @@ class BaseOutputTransport(FrameProcessor): # for these tasks before cancelling the camera and audio tasks below # because they might be still rendering. if self._sink_task: - await self._sink_task + await wait_for_task(self._sink_task) if self._sink_clock_task: - await self._sink_clock_task + await wait_for_task(self._sink_clock_task) # We can now cancel the camera task. await self._cancel_camera_task() diff --git a/src/pipecat/utils/utils.py b/src/pipecat/utils/utils.py index 045308c57..12703c37c 100644 --- a/src/pipecat/utils/utils.py +++ b/src/pipecat/utils/utils.py @@ -48,7 +48,7 @@ def create_task(loop: asyncio.AbstractEventLoop, coroutine: Coroutine, name: str try: await coroutine except asyncio.CancelledError: - logger.trace(f"{name}: cancelling task") + logger.trace(f"{name}: task cancelled") # Re-raise the exception to ensure the task is cancelled. raise except Exception as e: @@ -61,6 +61,26 @@ def create_task(loop: asyncio.AbstractEventLoop, coroutine: Coroutine, name: str return task +async def wait_for_task(task: asyncio.Task, timeout: Optional[float] = None): + name = task.get_name() + try: + if timeout: + await asyncio.wait_for(task, timeout=timeout) + else: + await task + except asyncio.TimeoutError: + logger.warning(f"{name}: timed out waiting for task to finish") + except asyncio.CancelledError: + logger.error(f"{name}: unexpected task cancellation") + except Exception as e: + logger.exception(f"{name}: unexpected exception while stopping task: {e}") + finally: + try: + _TASKS.remove(task) + except KeyError as e: + logger.error(f"{name}: error removing task (already removed?): {e}") + + async def cancel_task(task: asyncio.Task, timeout: Optional[float] = None): name = task.get_name() task.cancel() @@ -70,16 +90,17 @@ async def cancel_task(task: asyncio.Task, timeout: Optional[float] = None): else: await task except asyncio.TimeoutError: - logger.warning(f"{name}: timed out waiting for task to finish") + logger.warning(f"{name}: timed out waiting for task to cancel") except asyncio.CancelledError: # Here are sure the task is cancelled properly. - logger.trace(f"{name}: task cancelled") + pass + except Exception as e: + logger.exception(f"{name}: unexpected exception while cancelling task: {e}") + finally: try: _TASKS.remove(task) except KeyError as e: logger.error(f"{name}: error removing task (already removed?): {e}") - except Exception as e: - logger.exception(f"{name}: unexpected exception while cancelling task: {e}") def current_tasks() -> Set[asyncio.Task]: From c03d0352b171139958c80198aa4560e5774906af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 23:48:26 -0800 Subject: [PATCH 13/28] utils/tasks: added new documentation --- src/pipecat/pipeline/base_task.py | 12 ++++++++++ src/pipecat/pipeline/task.py | 2 ++ src/pipecat/utils/utils.py | 38 +++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/src/pipecat/pipeline/base_task.py b/src/pipecat/pipeline/base_task.py index 51971e8ce..714c721e3 100644 --- a/src/pipecat/pipeline/base_task.py +++ b/src/pipecat/pipeline/base_task.py @@ -11,6 +11,18 @@ from pipecat.frames.frames import Frame class BaseTask(ABC): + @property + @abstractmethod + def id(self) -> int: + """Returns the unique indetifier for this task.""" + pass + + @property + @abstractmethod + def name(self) -> str: + """Returns the name of this task.""" + pass + @abstractmethod def has_finished(self) -> bool: """Indicates whether the tasks has finished. That is, all processors diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 3a814e15d..58935ff06 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -125,10 +125,12 @@ class PipelineTask(BaseTask): @property def id(self) -> int: + """Returns the unique indetifier for this task.""" return self._id @property def name(self) -> str: + """Returns the name of this task.""" return self._name def has_finished(self) -> bool: diff --git a/src/pipecat/utils/utils.py b/src/pipecat/utils/utils.py index 12703c37c..bc1bb5ec2 100644 --- a/src/pipecat/utils/utils.py +++ b/src/pipecat/utils/utils.py @@ -44,6 +44,20 @@ def obj_count(obj) -> int: def create_task(loop: asyncio.AbstractEventLoop, coroutine: Coroutine, name: str) -> asyncio.Task: + """ + Creates and schedules a new asyncio Task that runs the given coroutine. + + The task is added to a global set of created tasks. + + Args: + loop (asyncio.AbstractEventLoop): The event loop to use for creating the task. + coroutine (Coroutine): The coroutine to be executed within the task. + name (str): The name to assign to the task for identification. + + Returns: + asyncio.Task: The created task object. + """ + async def run_coroutine(): try: await coroutine @@ -62,6 +76,18 @@ def create_task(loop: asyncio.AbstractEventLoop, coroutine: Coroutine, name: str async def wait_for_task(task: asyncio.Task, timeout: Optional[float] = None): + """Wait for an asyncio.Task to complete with optional timeout handling. + + This function awaits the specified asyncio.Task and handles scenarios for + timeouts, cancellations, and other exceptions. It also ensures that the task + is removed from the set of registered tasks upon completion or failure. + + Args: + task (asyncio.Task): The asyncio Task to wait for. + timeout (Optional[float], optional): The maximum number of seconds + to wait for the task to complete. If None, waits indefinitely. + Defaults to None. + """ name = task.get_name() try: if timeout: @@ -82,6 +108,17 @@ async def wait_for_task(task: asyncio.Task, timeout: Optional[float] = None): async def cancel_task(task: asyncio.Task, timeout: Optional[float] = None): + """Cancels the given asyncio Task and awaits its completion with an + optional timeout. + + This function removes the task from the set of registered tasks upon + completion or failure. + + Args: + task (asyncio.Task): The task to be cancelled. + timeout (Optional[float]): The optional timeout in seconds to wait for the task to cancel. + + """ name = task.get_name() task.cancel() try: @@ -104,4 +141,5 @@ async def cancel_task(task: asyncio.Task, timeout: Optional[float] = None): def current_tasks() -> Set[asyncio.Task]: + """Returns the list of currently created/registered tasks.""" return _TASKS From 9374bed878d9fbd1821eb6bfa20e1873f69d1d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 23:52:06 -0800 Subject: [PATCH 14/28] tests: langchain fixes --- tests/test_langchain.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_langchain.py b/tests/test_langchain.py index b09eee86e..c94bc1c01 100644 --- a/tests/test_langchain.py +++ b/tests/test_langchain.py @@ -32,8 +32,7 @@ from pipecat.processors.frameworks.langchain import LangchainProcessor class TestLangchain(unittest.IsolatedAsyncioTestCase): class MockProcessor(FrameProcessor): def __init__(self, name): - super().__init__() - self.name = name + super().__init__(name=name) self.token: list[str] = [] # Start collecting tokens when we see the start frame self.start_collecting = False From 2eccd1b1e964ce97ed74e78fcd42ab998f2a3666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 24 Jan 2025 23:59:06 -0800 Subject: [PATCH 15/28] utils: update some logging levels --- src/pipecat/utils/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/utils/utils.py b/src/pipecat/utils/utils.py index bc1bb5ec2..435131684 100644 --- a/src/pipecat/utils/utils.py +++ b/src/pipecat/utils/utils.py @@ -97,14 +97,14 @@ async def wait_for_task(task: asyncio.Task, timeout: Optional[float] = None): except asyncio.TimeoutError: logger.warning(f"{name}: timed out waiting for task to finish") except asyncio.CancelledError: - logger.error(f"{name}: unexpected task cancellation") + logger.trace(f"{name}: unexpected task cancellation (maybe Ctrl-C?)") except Exception as e: logger.exception(f"{name}: unexpected exception while stopping task: {e}") finally: try: _TASKS.remove(task) except KeyError as e: - logger.error(f"{name}: error removing task (already removed?): {e}") + logger.trace(f"{name}: unable to remove task (already removed?): {e}") async def cancel_task(task: asyncio.Task, timeout: Optional[float] = None): @@ -137,7 +137,7 @@ async def cancel_task(task: asyncio.Task, timeout: Optional[float] = None): try: _TASKS.remove(task) except KeyError as e: - logger.error(f"{name}: error removing task (already removed?): {e}") + logger.trace(f"{name}: unable to remove task (already removed?): {e}") def current_tasks() -> Set[asyncio.Task]: From bf5ced18b2f93ea276d27722d32713d3a642314a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sat, 25 Jan 2025 00:15:21 -0800 Subject: [PATCH 16/28] fix parallel pipelines cleanup --- src/pipecat/pipeline/parallel_pipeline.py | 9 +++++---- src/pipecat/pipeline/sync_parallel_pipeline.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/pipecat/pipeline/parallel_pipeline.py b/src/pipecat/pipeline/parallel_pipeline.py index 2d5bd7761..f77d09cc6 100644 --- a/src/pipecat/pipeline/parallel_pipeline.py +++ b/src/pipecat/pipeline/parallel_pipeline.py @@ -23,7 +23,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -class Source(FrameProcessor): +class ParallelPipelineSource(FrameProcessor): def __init__( self, upstream_queue: asyncio.Queue, @@ -46,7 +46,7 @@ class Source(FrameProcessor): await self.push_frame(frame, direction) -class Sink(FrameProcessor): +class ParallelPipelineSink(FrameProcessor): def __init__( self, downstream_queue: asyncio.Queue, @@ -92,8 +92,8 @@ class ParallelPipeline(BasePipeline): raise TypeError(f"ParallelPipeline argument {processors} is not a list") # We will add a source before the pipeline and a sink after. - source = Source(self._up_queue, self._parallel_push_frame) - sink = Sink(self._down_queue, self._parallel_push_frame) + source = ParallelPipelineSource(self._up_queue, self._parallel_push_frame) + sink = ParallelPipelineSink(self._down_queue, self._parallel_push_frame) self._sources.append(source) self._sinks.append(sink) @@ -117,6 +117,7 @@ class ParallelPipeline(BasePipeline): # async def cleanup(self): + await super().cleanup() await asyncio.gather(*[s.cleanup() for s in self._sources]) await asyncio.gather(*[p.cleanup() for p in self._pipelines]) await asyncio.gather(*[s.cleanup() for s in self._sinks]) diff --git a/src/pipecat/pipeline/sync_parallel_pipeline.py b/src/pipecat/pipeline/sync_parallel_pipeline.py index 78ddff074..7fa3577a5 100644 --- a/src/pipecat/pipeline/sync_parallel_pipeline.py +++ b/src/pipecat/pipeline/sync_parallel_pipeline.py @@ -24,7 +24,7 @@ class SyncFrame(ControlFrame): pass -class Source(FrameProcessor): +class SyncParallelPipelineSource(FrameProcessor): def __init__(self, upstream_queue: asyncio.Queue): super().__init__() self._up_queue = upstream_queue @@ -39,7 +39,7 @@ class Source(FrameProcessor): await self.push_frame(frame, direction) -class Sink(FrameProcessor): +class SyncParallelPipelineSink(FrameProcessor): def __init__(self, downstream_queue: asyncio.Queue): super().__init__() self._down_queue = downstream_queue @@ -76,8 +76,8 @@ class SyncParallelPipeline(BasePipeline): # We add a source at the beginning of the pipeline and a sink at the end. up_queue = asyncio.Queue() down_queue = asyncio.Queue() - source = Source(up_queue) - sink = Sink(down_queue) + source = SyncParallelPipelineSource(up_queue) + sink = SyncParallelPipelineSink(down_queue) processors: List[FrameProcessor] = [source] + processors + [sink] # Keep track of sources and sinks. We also keep the output queue of @@ -101,6 +101,10 @@ class SyncParallelPipeline(BasePipeline): # Frame processor # + async def cleanup(self): + await super().cleanup() + await asyncio.gather(*[p.cleanup() for p in self._pipelines]) + async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) From a3a6adbd17a21b281317ee2a63f5817b100c5848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sat, 25 Jan 2025 00:27:51 -0800 Subject: [PATCH 17/28] user_idle_processor: add missing parent cleanup() --- src/pipecat/processors/user_idle_processor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/processors/user_idle_processor.py b/src/pipecat/processors/user_idle_processor.py index 3aadf0793..e65bd437f 100644 --- a/src/pipecat/processors/user_idle_processor.py +++ b/src/pipecat/processors/user_idle_processor.py @@ -156,6 +156,7 @@ class UserIdleProcessor(FrameProcessor): async def cleanup(self) -> None: """Cleans up resources when processor is shutting down.""" + await super().cleanup() if self._idle_task: # Only stop if task exists await self._stop() From 2a2928d96c1116c035a0b68606e8ecb9386adb15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 26 Jan 2025 15:07:14 -0800 Subject: [PATCH 18/28] gemini: create transcribe tasks only once --- src/pipecat/services/ai_services.py | 8 +-- .../services/gemini_multimodal_live/gemini.py | 71 ++++++++++++------- 2 files changed, 48 insertions(+), 31 deletions(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 2b0485fdd..bee15ceab 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -8,7 +8,7 @@ import asyncio import io import wave from abc import abstractmethod -from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple +from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple from loguru import logger @@ -69,7 +69,7 @@ class AIService(FrameProcessor): async def cancel(self, frame: CancelFrame): pass - async def _update_settings(self, settings: Dict[str, Any]): + async def _update_settings(self, settings: Mapping[str, Any]): from pipecat.services.openai_realtime_beta.events import ( SessionProperties, ) @@ -267,7 +267,7 @@ class TTSService(AIService): await self.cancel_task(self._stop_frame_task) self._stop_frame_task = None - async def _update_settings(self, settings: Dict[str, Any]): + async def _update_settings(self, settings: Mapping[str, Any]): for key, value in settings.items(): if key in self._settings: logger.info(f"Updating TTS setting {key} to: [{value}]") @@ -468,7 +468,7 @@ class STTService(AIService): """Returns transcript as a string""" pass - async def _update_settings(self, settings: Dict[str, Any]): + async def _update_settings(self, settings: Mapping[str, Any]): logger.info(f"Updating STT settings: {self._settings}") for key, value in settings.items(): if key in self._settings: diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index ef509e4ae..c2fc103b7 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -51,6 +51,7 @@ from pipecat.services.openai import ( OpenAIUserContextAggregator, ) from pipecat.utils.time import time_now_iso8601 +from pipecat.utils.utils import wait_for_task from . import events from .audio_transcriber import AudioTranscriber @@ -182,9 +183,13 @@ class GeminiMultimodalLiveLLMService(LLMService): self._audio_input_paused = start_audio_paused self._video_input_paused = start_video_paused + self._context = None self._websocket = None self._receive_task = None - self._context = None + self._transcribe_audio_task = None + self._transcribe_model_audio_task = None + self._transcribe_audio_queue = asyncio.Queue() + self._transcribe_model_audio_queue = asyncio.Queue() self._disconnecting = False self._api_session_ready = False @@ -275,7 +280,7 @@ class GeminiMultimodalLiveLLMService(LLMService): ) await self.send_client_event(evt) if self._transcribe_user_audio and self._context: - self.create_task(self._handle_transcribe_user_audio(audio, self._context)) + await self._transcribe_audio_queue.put(audio) async def _handle_transcribe_user_audio(self, audio, context): text = await self._transcribe_audio(audio, context) @@ -392,6 +397,10 @@ class GeminiMultimodalLiveLLMService(LLMService): logger.info(f"Connecting to {uri}") self._websocket = await websockets.connect(uri=uri) self._receive_task = self.create_task(self._receive_task_handler()) + self._transcribe_audio_task = self.create_task(self._transcribe_audio_handler()) + self._transcribe_model_audio_task = self.create_task( + self._transcribe_model_audio_handler() + ) config = events.Config.model_validate( { "setup": { @@ -443,6 +452,12 @@ class GeminiMultimodalLiveLLMService(LLMService): if self._receive_task: await self.cancel_task(self._receive_task, timeout=1.0) self._receive_task = None + if self._transcribe_audio_task: + await self.cancel_task(self._transcribe_audio_task) + self._transcribe_audio_task = None + if self._transcribe_model_audio_task: + await self.cancel_task(self._transcribe_model_audio_task) + self._transcribe_model_audio_task = None self._disconnecting = False except Exception as e: logger.error(f"{self} error disconnecting: {e}") @@ -469,33 +484,35 @@ class GeminiMultimodalLiveLLMService(LLMService): # async def _receive_task_handler(self): - try: - async for message in self._websocket: - evt = events.parse_server_event(message) - # logger.debug(f"Received event: {message[:500]}") - # logger.debug(f"Received event: {evt}") + async for message in self._websocket: + evt = events.parse_server_event(message) + # logger.debug(f"Received event: {message[:500]}") + # logger.debug(f"Received event: {evt}") - if evt.setupComplete: - await self._handle_evt_setup_complete(evt) - elif evt.serverContent and evt.serverContent.modelTurn: - await self._handle_evt_model_turn(evt) - elif evt.serverContent and evt.serverContent.turnComplete: - await self._handle_evt_turn_complete(evt) - elif evt.toolCall: - await self._handle_evt_tool_call(evt) + if evt.setupComplete: + await self._handle_evt_setup_complete(evt) + elif evt.serverContent and evt.serverContent.modelTurn: + await self._handle_evt_model_turn(evt) + elif evt.serverContent and evt.serverContent.turnComplete: + await self._handle_evt_turn_complete(evt) + elif evt.toolCall: + await self._handle_evt_tool_call(evt) + elif False: # !!! todo: error events? + await self._handle_evt_error(evt) + # errors are fatal, so exit the receive loop + return + else: + pass - elif False: # !!! todo: error events? - await self._handle_evt_error(evt) - # errors are fatal, so exit the receive loop - return + async def _transcribe_audio_handler(self): + while True: + audio = await self._transcribe_audio_queue.get() + await self._handle_transcribe_user_audio(audio, self._context) - else: - pass - except asyncio.CancelledError: - logger.debug("websocket receive task cancelled") - raise - except Exception as e: - logger.error(f"{self} exception: {e}") + async def _transcribe_model_audio_handler(self): + while True: + audio = await self._transcribe_model_audio_queue.get() + await self._handle_transcribe_model_audio(audio, self._context) # # @@ -676,7 +693,7 @@ class GeminiMultimodalLiveLLMService(LLMService): self._bot_text_buffer = "" if audio and self._transcribe_model_audio and self._context: - self.create_task(self._handle_transcribe_model_audio(audio, self._context)) + await self._transcribe_model_audio.put(audio) elif text: await self.push_frame(LLMFullResponseEndFrame()) From 6cc01bc5b0a11a310054890d474e92c874e589cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 26 Jan 2025 15:40:15 -0800 Subject: [PATCH 19/28] examples: update 14 series with TTSSpeakFrame --- examples/foundational/14-function-calling.py | 4 ++-- examples/foundational/14c-function-calling-together.py | 4 ++-- examples/foundational/14f-function-calling-groq.py | 4 ++-- examples/foundational/14g-function-calling-grok.py | 4 ++-- examples/foundational/14h-function-calling-azure.py | 4 ++-- examples/foundational/14i-function-calling-fireworks.py | 4 ++-- examples/foundational/14j-function-calling-nim.py | 4 ++-- examples/foundational/14k-function-calling-cerebras.py | 6 +++--- examples/foundational/14l-function-calling-deepseek.py | 6 +++--- examples/foundational/14m-function-calling-openrouter.py | 4 ++-- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 013164c67..5ce8693cc 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -15,7 +15,7 @@ from openai.types.chat import ChatCompletionToolParam from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TextFrame +from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -31,7 +31,7 @@ logger.add(sys.stderr, level="DEBUG") async def start_fetch_weather(function_name, llm, context): """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TextFrame("Let me check on that.")) + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index 41baba457..9c981f7bc 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -15,7 +15,7 @@ from openai.types.chat import ChatCompletionToolParam from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TextFrame +from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask @@ -32,7 +32,7 @@ logger.add(sys.stderr, level="DEBUG") async def start_fetch_weather(function_name, llm, context): """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TextFrame("Let me check on that.")) + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index 2b927637c..ee36b513f 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -15,7 +15,7 @@ from openai.types.chat import ChatCompletionToolParam from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TextFrame +from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -32,7 +32,7 @@ logger.add(sys.stderr, level="DEBUG") async def start_fetch_weather(function_name, llm, context): """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TextFrame("Let me check on that.")) + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 40c77c323..49b3d6831 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -15,7 +15,7 @@ from openai.types.chat import ChatCompletionToolParam from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TextFrame +from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -32,7 +32,7 @@ logger.add(sys.stderr, level="DEBUG") async def start_fetch_weather(function_name, llm, context): """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TextFrame("Let me check on that.")) + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index f280e6de9..cd426366c 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -15,7 +15,7 @@ from openai.types.chat import ChatCompletionToolParam from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TextFrame +from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -32,7 +32,7 @@ logger.add(sys.stderr, level="DEBUG") async def start_fetch_weather(function_name, llm, context): """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TextFrame("Let me check on that.")) + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 241df5cf7..331e1c850 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -15,7 +15,7 @@ from openai.types.chat import ChatCompletionToolParam from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TextFrame +from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -32,7 +32,7 @@ logger.add(sys.stderr, level="DEBUG") async def start_fetch_weather(function_name, llm, context): """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TextFrame("Let me check on that.")) + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 08060fa0b..6a2d7d061 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -15,7 +15,7 @@ from openai.types.chat import ChatCompletionToolParam from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TextFrame +from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -32,7 +32,7 @@ logger.add(sys.stderr, level="DEBUG") async def start_fetch_weather(function_name, llm, context): """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TextFrame("Let me check on that.")) + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index 176511a46..05f7d2a94 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -15,7 +15,7 @@ from openai.types.chat import ChatCompletionToolParam from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TextFrame +from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -32,7 +32,7 @@ logger.add(sys.stderr, level="DEBUG") async def start_fetch_weather(function_name, llm, context): """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TextFrame("Let me check on that.")) + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") @@ -93,7 +93,7 @@ async def main(): messages = [ { "role": "system", - "content": """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. + "content": """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. You have one functions available: diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index 50421a83b..8170c7d0f 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -15,7 +15,7 @@ from openai.types.chat import ChatCompletionToolParam from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TextFrame +from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -32,7 +32,7 @@ logger.add(sys.stderr, level="DEBUG") async def start_fetch_weather(function_name, llm, context): """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TextFrame("Let me check on that.")) + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") @@ -93,7 +93,7 @@ async def main(): messages = [ { "role": "system", - "content": """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. + "content": """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. You have one functions available: diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index 5252c9a5d..3325ab240 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -15,7 +15,7 @@ from openai.types.chat import ChatCompletionToolParam from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TextFrame +from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -32,7 +32,7 @@ logger.add(sys.stderr, level="DEBUG") async def start_fetch_weather(function_name, llm, context): """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TextFrame("Let me check on that.")) + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") From 8b5228a105ddf452d5ff220217403598d76c4bd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 26 Jan 2025 18:49:55 -0800 Subject: [PATCH 20/28] utils: move task functions to asyncio module --- src/pipecat/pipeline/runner.py | 3 +- src/pipecat/pipeline/task.py | 3 +- src/pipecat/pipeline/task_observer.py | 3 +- src/pipecat/processors/frame_processor.py | 3 +- .../services/gemini_multimodal_live/gemini.py | 1 - src/pipecat/transports/base_output.py | 2 +- src/pipecat/transports/services/daily.py | 2 +- src/pipecat/transports/services/livekit.py | 2 +- src/pipecat/utils/asyncio.py | 114 ++++++++++++++++++ src/pipecat/utils/utils.py | 107 ---------------- 10 files changed, 125 insertions(+), 115 deletions(-) create mode 100644 src/pipecat/utils/asyncio.py diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py index 8cdfc3eaa..9963454b6 100644 --- a/src/pipecat/pipeline/runner.py +++ b/src/pipecat/pipeline/runner.py @@ -10,7 +10,8 @@ import signal from loguru import logger from pipecat.pipeline.task import PipelineTask -from pipecat.utils.utils import current_tasks, obj_count, obj_id +from pipecat.utils.asyncio import current_tasks +from pipecat.utils.utils import obj_count, obj_id class PipelineRunner: diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 58935ff06..45596980f 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -30,7 +30,8 @@ from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.pipeline.base_task import BaseTask from pipecat.pipeline.task_observer import TaskObserver from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.utils.utils import cancel_task, create_task, obj_count, obj_id, wait_for_task +from pipecat.utils.asyncio import cancel_task, create_task, wait_for_task +from pipecat.utils.utils import obj_count, obj_id HEARTBEAT_SECONDS = 1.0 HEARTBEAT_MONITOR_SECONDS = HEARTBEAT_SECONDS * 5 diff --git a/src/pipecat/pipeline/task_observer.py b/src/pipecat/pipeline/task_observer.py index da5a6818f..c9816f7d3 100644 --- a/src/pipecat/pipeline/task_observer.py +++ b/src/pipecat/pipeline/task_observer.py @@ -12,7 +12,8 @@ from attr import dataclass from pipecat.frames.frames import Frame from pipecat.observers.base_observer import BaseObserver from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.utils.utils import cancel_task, create_task, obj_count, obj_id +from pipecat.utils.asyncio import cancel_task, create_task +from pipecat.utils.utils import obj_count, obj_id @dataclass diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 6a8252677..c986d7025 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -24,7 +24,8 @@ from pipecat.frames.frames import ( ) from pipecat.metrics.metrics import LLMTokenUsage, MetricsData from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics -from pipecat.utils.utils import cancel_task, create_task, obj_count, obj_id +from pipecat.utils.asyncio import cancel_task, create_task +from pipecat.utils.utils import obj_count, obj_id class FrameDirection(Enum): diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index c2fc103b7..770b64711 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -51,7 +51,6 @@ from pipecat.services.openai import ( OpenAIUserContextAggregator, ) from pipecat.utils.time import time_now_iso8601 -from pipecat.utils.utils import wait_for_task from . import events from .audio_transcriber import AudioTranscriber diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 1ec5ea7fa..5182fc790 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -35,8 +35,8 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.transports.base_transport import TransportParams +from pipecat.utils.asyncio import wait_for_task from pipecat.utils.time import nanoseconds_to_seconds -from pipecat.utils.utils import wait_for_task class BaseOutputTransport(FrameProcessor): diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 70df2f05a..6d32786d9 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -46,7 +46,7 @@ from pipecat.transcriptions.language import Language from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.utils.utils import cancel_task, create_task +from pipecat.utils.asyncio import cancel_task, create_task try: from daily import CallClient, Daily, EventHandler diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index f0a2add0e..eff892ac0 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -28,7 +28,7 @@ from pipecat.processors.frame_processor import FrameDirection from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.utils.utils import create_task +from pipecat.utils.asyncio import create_task try: from livekit import rtc diff --git a/src/pipecat/utils/asyncio.py b/src/pipecat/utils/asyncio.py new file mode 100644 index 000000000..d10895758 --- /dev/null +++ b/src/pipecat/utils/asyncio.py @@ -0,0 +1,114 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +from typing import Coroutine, Optional, Set + +from loguru import logger + +_TASKS: Set[asyncio.Task] = set() + + +def create_task(loop: asyncio.AbstractEventLoop, coroutine: Coroutine, name: str) -> asyncio.Task: + """ + Creates and schedules a new asyncio Task that runs the given coroutine. + + The task is added to a global set of created tasks. + + Args: + loop (asyncio.AbstractEventLoop): The event loop to use for creating the task. + coroutine (Coroutine): The coroutine to be executed within the task. + name (str): The name to assign to the task for identification. + + Returns: + asyncio.Task: The created task object. + """ + + async def run_coroutine(): + try: + await coroutine + except asyncio.CancelledError: + logger.trace(f"{name}: task cancelled") + # Re-raise the exception to ensure the task is cancelled. + raise + except Exception as e: + logger.exception(f"{name}: unexpected exception: {e}") + + task = loop.create_task(run_coroutine()) + task.set_name(name) + _TASKS.add(task) + logger.trace(f"{name}: task created") + return task + + +async def wait_for_task(task: asyncio.Task, timeout: Optional[float] = None): + """Wait for an asyncio.Task to complete with optional timeout handling. + + This function awaits the specified asyncio.Task and handles scenarios for + timeouts, cancellations, and other exceptions. It also ensures that the task + is removed from the set of registered tasks upon completion or failure. + + Args: + task (asyncio.Task): The asyncio Task to wait for. + timeout (Optional[float], optional): The maximum number of seconds + to wait for the task to complete. If None, waits indefinitely. + Defaults to None. + """ + name = task.get_name() + try: + if timeout: + await asyncio.wait_for(task, timeout=timeout) + else: + await task + except asyncio.TimeoutError: + logger.warning(f"{name}: timed out waiting for task to finish") + except asyncio.CancelledError: + logger.trace(f"{name}: unexpected task cancellation (maybe Ctrl-C?)") + except Exception as e: + logger.exception(f"{name}: unexpected exception while stopping task: {e}") + finally: + try: + _TASKS.remove(task) + except KeyError as e: + logger.trace(f"{name}: unable to remove task (already removed?): {e}") + + +async def cancel_task(task: asyncio.Task, timeout: Optional[float] = None): + """Cancels the given asyncio Task and awaits its completion with an + optional timeout. + + This function removes the task from the set of registered tasks upon + completion or failure. + + Args: + task (asyncio.Task): The task to be cancelled. + timeout (Optional[float]): The optional timeout in seconds to wait for the task to cancel. + + """ + name = task.get_name() + task.cancel() + try: + if timeout: + await asyncio.wait_for(task, timeout=timeout) + else: + await task + except asyncio.TimeoutError: + logger.warning(f"{name}: timed out waiting for task to cancel") + except asyncio.CancelledError: + # Here are sure the task is cancelled properly. + pass + except Exception as e: + logger.exception(f"{name}: unexpected exception while cancelling task: {e}") + finally: + try: + _TASKS.remove(task) + except KeyError as e: + logger.trace(f"{name}: unable to remove task (already removed?): {e}") + + +def current_tasks() -> Set[asyncio.Task]: + """Returns the list of currently created/registered tasks.""" + return _TASKS diff --git a/src/pipecat/utils/utils.py b/src/pipecat/utils/utils.py index 435131684..4cdb73050 100644 --- a/src/pipecat/utils/utils.py +++ b/src/pipecat/utils/utils.py @@ -4,16 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import collections import itertools -from typing import Coroutine, Optional, Set - -from loguru import logger _COUNTS = collections.defaultdict(itertools.count) _ID = itertools.count() -_TASKS: Set[asyncio.Task] = set() def obj_id() -> int: @@ -41,105 +36,3 @@ def obj_count(obj) -> int: 0 """ return next(_COUNTS[obj.__class__.__name__]) - - -def create_task(loop: asyncio.AbstractEventLoop, coroutine: Coroutine, name: str) -> asyncio.Task: - """ - Creates and schedules a new asyncio Task that runs the given coroutine. - - The task is added to a global set of created tasks. - - Args: - loop (asyncio.AbstractEventLoop): The event loop to use for creating the task. - coroutine (Coroutine): The coroutine to be executed within the task. - name (str): The name to assign to the task for identification. - - Returns: - asyncio.Task: The created task object. - """ - - async def run_coroutine(): - try: - await coroutine - except asyncio.CancelledError: - logger.trace(f"{name}: task cancelled") - # Re-raise the exception to ensure the task is cancelled. - raise - except Exception as e: - logger.exception(f"{name}: unexpected exception: {e}") - - task = loop.create_task(run_coroutine()) - task.set_name(name) - _TASKS.add(task) - logger.trace(f"{name}: task created") - return task - - -async def wait_for_task(task: asyncio.Task, timeout: Optional[float] = None): - """Wait for an asyncio.Task to complete with optional timeout handling. - - This function awaits the specified asyncio.Task and handles scenarios for - timeouts, cancellations, and other exceptions. It also ensures that the task - is removed from the set of registered tasks upon completion or failure. - - Args: - task (asyncio.Task): The asyncio Task to wait for. - timeout (Optional[float], optional): The maximum number of seconds - to wait for the task to complete. If None, waits indefinitely. - Defaults to None. - """ - name = task.get_name() - try: - if timeout: - await asyncio.wait_for(task, timeout=timeout) - else: - await task - except asyncio.TimeoutError: - logger.warning(f"{name}: timed out waiting for task to finish") - except asyncio.CancelledError: - logger.trace(f"{name}: unexpected task cancellation (maybe Ctrl-C?)") - except Exception as e: - logger.exception(f"{name}: unexpected exception while stopping task: {e}") - finally: - try: - _TASKS.remove(task) - except KeyError as e: - logger.trace(f"{name}: unable to remove task (already removed?): {e}") - - -async def cancel_task(task: asyncio.Task, timeout: Optional[float] = None): - """Cancels the given asyncio Task and awaits its completion with an - optional timeout. - - This function removes the task from the set of registered tasks upon - completion or failure. - - Args: - task (asyncio.Task): The task to be cancelled. - timeout (Optional[float]): The optional timeout in seconds to wait for the task to cancel. - - """ - name = task.get_name() - task.cancel() - try: - if timeout: - await asyncio.wait_for(task, timeout=timeout) - else: - await task - except asyncio.TimeoutError: - logger.warning(f"{name}: timed out waiting for task to cancel") - except asyncio.CancelledError: - # Here are sure the task is cancelled properly. - pass - except Exception as e: - logger.exception(f"{name}: unexpected exception while cancelling task: {e}") - finally: - try: - _TASKS.remove(task) - except KeyError as e: - logger.trace(f"{name}: unable to remove task (already removed?): {e}") - - -def current_tasks() -> Set[asyncio.Task]: - """Returns the list of currently created/registered tasks.""" - return _TASKS From 737e4fa3bdcb80597c95667e0620caf05c514375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 26 Jan 2025 21:11:17 -0800 Subject: [PATCH 21/28] gemini(multimodal live): connect on StartFrame --- .../foundational/26-gemini-multimodal-live.py | 16 ++++++++++++++++ .../services/gemini_multimodal_live/gemini.py | 16 ++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py index 21c4b66f7..5f41833c8 100644 --- a/examples/foundational/26-gemini-multimodal-live.py +++ b/examples/foundational/26-gemini-multimodal-live.py @@ -15,6 +15,7 @@ from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMMessagesAppendFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -71,6 +72,21 @@ async def main(): ), ) + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await task.queue_frames( + [ + LLMMessagesAppendFrame( + messages=[ + { + "role": "assistant", + "content": "Greet the user.", + } + ] + ) + ] + ) + runner = PipelineRunner() await runner.run(task) diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 770b64711..5fee17478 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -248,6 +248,7 @@ class GeminiMultimodalLiveLLMService(LLMService): async def start(self, frame: StartFrame): await super().start(frame) + await self._connect() async def stop(self, frame: EndFrame): await super().stop(frame) @@ -385,13 +386,13 @@ class GeminiMultimodalLiveLLMService(LLMService): await self._ws_send(event.model_dump(exclude_none=True)) async def _connect(self): + if self._websocket: + # Here we assume that if we have a websocket, we are connected. We + # handle disconnections in the send/recv code paths. + return + logger.info("Connecting to Gemini service") try: - if self._websocket: - # Here we assume that if we have a websocket, we are connected. We - # handle disconnections in the send/recv code paths. - return - uri = f"wss://{self.base_url}/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key={self.api_key}" logger.info(f"Connecting to {uri}") self._websocket = await websockets.connect(uri=uri) @@ -464,9 +465,8 @@ class GeminiMultimodalLiveLLMService(LLMService): async def _ws_send(self, message): # logger.debug(f"Sending message to websocket: {message}") try: - if not self._websocket: - await self._connect() - await self._websocket.send(json.dumps(message)) + if self._websocket: + await self._websocket.send(json.dumps(message)) except Exception as e: if self._disconnecting: return From 509f143e1bff4a5e3ef3d2ed72d3fdfce5265896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 27 Jan 2025 07:05:32 -0800 Subject: [PATCH 22/28] update CHANGELOG.md --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff51f936b..76ad41a12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- In order to create tasks in Pipecat it is now recommended to use + `utils.asyncio.create_task()`. It takes care of uncaught exceptions, task + cancellation handling and task management. To cancel or wait for a task there + is `utils.asyncio.cancel_task()` and `utils.asyncio.wait_for_task()`. All of + Pipecat processors have been updated accordingly. Also, when a pipeline runner + finishes, a warning about dangling tasks might appear, which indicates if any + of the created tasks was never cancelled or awaited for (using these new + functions). + - It is now possible to specify the period of the `PipelineTask` heartbeat frames with `heartbeats_period_secs`. @@ -41,6 +50,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an `GeminiMultimodalLiveLLMService` issue that was preventing the user + to push initial LLM assistant messages (using `LLMMessagesAppendFrame`). + +- Added missing `FrameProcessor.cleanup()` calls to `Pipeline`, + `ParallelPipeline` and `UserIdleProcessor`. + - Fixed a type error when using `voice_settings` in `ElevenLabsHttpTTSService`. - Fixed an issue where `OpenAIRealtimeBetaLLMService` function calling resulted From 498805a34c1592e54cbe2711d8d79f2160feaac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 27 Jan 2025 13:34:32 -0800 Subject: [PATCH 23/28] FrameProcessor: add wait_for_task() --- src/pipecat/processors/frame_processor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index c986d7025..3b228e639 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -24,7 +24,7 @@ from pipecat.frames.frames import ( ) from pipecat.metrics.metrics import LLMTokenUsage, MetricsData from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics -from pipecat.utils.asyncio import cancel_task, create_task +from pipecat.utils.asyncio import cancel_task, create_task, wait_for_task from pipecat.utils.utils import obj_count, obj_id @@ -157,6 +157,9 @@ class FrameProcessor: async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None): await cancel_task(task, timeout) + async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None): + await wait_for_task(task, timeout) + async def cleanup(self): await self.__cancel_input_task() await self.__cancel_push_task() From 10202dc529517a7ada39fd5c0a332a16810e10a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 27 Jan 2025 13:34:51 -0800 Subject: [PATCH 24/28] transports(websockets): cancel or wait for tasks to finish --- src/pipecat/transports/network/fastapi_websocket.py | 11 +++++++++++ src/pipecat/transports/network/websocket_server.py | 5 ++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/pipecat/transports/network/fastapi_websocket.py b/src/pipecat/transports/network/fastapi_websocket.py index 42f1d3171..bcfbbc8cd 100644 --- a/src/pipecat/transports/network/fastapi_websocket.py +++ b/src/pipecat/transports/network/fastapi_websocket.py @@ -16,6 +16,8 @@ from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ( + CancelFrame, + EndFrame, Frame, InputAudioRawFrame, OutputAudioRawFrame, @@ -27,6 +29,7 @@ from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializer from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.utils.asyncio import cancel_task try: from fastapi import WebSocket @@ -72,6 +75,14 @@ class FastAPIWebsocketInputTransport(BaseInputTransport): await self._callbacks.on_client_connected(self._websocket) self._receive_task = self.create_task(self._receive_messages()) + async def stop(self, frame: EndFrame): + await super().stop(frame) + await cancel_task(self._receive_task) + + async def cancel(self, frame: CancelFrame): + await super().cancel(frame) + await cancel_task(self._receive_task) + def _iter_data(self) -> typing.AsyncIterator[bytes | str]: if self._params.serializer.type == FrameSerializerType.BINARY: return self._websocket.iter_bytes() diff --git a/src/pipecat/transports/network/websocket_server.py b/src/pipecat/transports/network/websocket_server.py index 6fe5f91ef..3eb06dc6b 100644 --- a/src/pipecat/transports/network/websocket_server.py +++ b/src/pipecat/transports/network/websocket_server.py @@ -76,12 +76,11 @@ class WebsocketServerInputTransport(BaseInputTransport): async def stop(self, frame: EndFrame): await super().stop(frame) self._stop_server_event.set() - await self._server_task + await self.wait_for_task(self._server_task) async def cancel(self, frame: CancelFrame): await super().cancel(frame) - self._stop_server_event.set() - await self._server_task + await self.cancel_task(self._server_task) async def _server_task_handler(self): logger.info(f"Starting websocket server on {self._host}:{self._port}") From f55f78e70e57d08c99df2c237e0a70c9e714e2e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 27 Jan 2025 13:37:31 -0800 Subject: [PATCH 25/28] update CHANGELOG.md --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76ad41a12..bba54de5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- In order to create tasks in Pipecat it is now recommended to use - `utils.asyncio.create_task()`. It takes care of uncaught exceptions, task +- In order to create tasks in Pipecat frame processors it is now recommended to + use `FrameProcessor.create_task()` (which uses the new + `utils.asyncio.create_task()`). It takes care of uncaught exceptions, task cancellation handling and task management. To cancel or wait for a task there - is `utils.asyncio.cancel_task()` and `utils.asyncio.wait_for_task()`. All of + is `FrameProcessor.cancel_task()` and `FrameProcessor.wait_for_task()`. All of Pipecat processors have been updated accordingly. Also, when a pipeline runner finishes, a warning about dangling tasks might appear, which indicates if any of the created tasks was never cancelled or awaited for (using these new From 1489e447404f4e59d67603e84e637a58ba5a14a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 27 Jan 2025 13:46:18 -0800 Subject: [PATCH 26/28] gemini(multimodal live): fix model audio queue variable --- src/pipecat/services/gemini_multimodal_live/gemini.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 5fee17478..9f355d1ba 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -692,7 +692,7 @@ class GeminiMultimodalLiveLLMService(LLMService): self._bot_text_buffer = "" if audio and self._transcribe_model_audio and self._context: - await self._transcribe_model_audio.put(audio) + await self._transcribe_model_audio_queue.put(audio) elif text: await self.push_frame(LLMFullResponseEndFrame()) From cca241a2b7dc57ad6231d8c08501878808fdd757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 27 Jan 2025 14:27:43 -0800 Subject: [PATCH 27/28] examples(22c): fix cancel_task call --- examples/foundational/22c-natural-conversation-mixed-llms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index ff53cd838..d93a40d4d 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -374,7 +374,7 @@ class OutputGate(FrameProcessor): self._gate_task = self.get_event_loop().create_task(self._gate_task_handler()) async def _stop(self): - await cancel_task(self._gate_task) + await self.cancel_task(self._gate_task) async def _gate_task_handler(self): while True: From f09f4b8fc4fcfddf8ce8033988aa7fee923ee507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 27 Jan 2025 14:36:30 -0800 Subject: [PATCH 28/28] services(tavus): fix EndFrame and CancelFrame processing --- src/pipecat/services/tavus.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/tavus.py b/src/pipecat/services/tavus.py index 7b43f6176..b1dfef9e4 100644 --- a/src/pipecat/services/tavus.py +++ b/src/pipecat/services/tavus.py @@ -75,6 +75,14 @@ class TavusVideoService(AIService): logger.debug(f"TavusVideoService persona grabbed {response_json}") return response_json["persona_name"] + async def stop(self, frame: EndFrame): + await super().stop(frame) + await self._end_conversation() + + async def cancel(self, frame: CancelFrame): + await super().cancel(frame) + await self._end_conversation() + async def _end_conversation(self) -> None: url = f"https://tavusapi.com/v2/conversations/{self._conversation_id}/end" headers = {"Content-Type": "application/json", "x-api-key": self._api_key} @@ -105,8 +113,6 @@ class TavusVideoService(AIService): await self.stop_processing_metrics() elif isinstance(frame, StartInterruptionFrame): await self._send_interrupt_message() - elif isinstance(frame, (EndFrame, CancelFrame)): - await self._end_conversation() else: await self.push_frame(frame, direction)