From 7e31b2a795bd3be3ad97339bbd01be382afb29cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 25 Jul 2024 14:47:56 -0700 Subject: [PATCH 1/2] processors(user_idle): use user speaking instead of interruption frames --- src/pipecat/processors/user_idle_processor.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/pipecat/processors/user_idle_processor.py b/src/pipecat/processors/user_idle_processor.py index 20e8e7be6..7deda2555 100644 --- a/src/pipecat/processors/user_idle_processor.py +++ b/src/pipecat/processors/user_idle_processor.py @@ -8,7 +8,12 @@ import asyncio from typing import Awaitable, Callable -from pipecat.frames.frames import BotSpeakingFrame, Frame, StartInterruptionFrame, StopInterruptionFrame, SystemFrame +from pipecat.frames.frames import ( + BotSpeakingFrame, + Frame, + SystemFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame) from pipecat.processors.async_frame_processor import AsyncFrameProcessor from pipecat.processors.frame_processor import FrameDirection @@ -47,10 +52,10 @@ class UserIdleProcessor(AsyncFrameProcessor): await self.queue_frame(frame, direction) # We shouldn't call the idle callback if the user or the bot are speaking. - if isinstance(frame, StartInterruptionFrame): + if isinstance(frame, UserStartedSpeakingFrame): self._interrupted = True self._idle_event.set() - elif isinstance(frame, StopInterruptionFrame): + elif isinstance(frame, UserStoppedSpeakingFrame): self._interrupted = False self._idle_event.set() elif isinstance(frame, BotSpeakingFrame): From 76aca32f2e345cb8cdd2bd1494c79bef0ed7210c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 25 Jul 2024 14:48:43 -0700 Subject: [PATCH 2/2] transport(output): emit new bot start|stop speaking frames --- CHANGELOG.md | 6 +++++ src/pipecat/frames/frames.py | 38 +++++++++++++++++++-------- src/pipecat/transports/base_output.py | 10 +++++++ 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a74e0fc82..b12a6f236 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `BotStartedSpeakingFrame` and `BotStoppedSpeakingFrame` control + frames. These frames are pushed upstream and they should wrap + `BotSpeakingFrame`. + - Transports now allow you to register event handlers without decorators. ### Changed +- `BotSpeakingFrame` is now a control frame. + - `StartFrame` is now a control frame similar to `EndFrame`. - `DeepgramTTSService` now is more customizable. You can adjust the encoding and diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index f1b0fc7b1..270a8304e 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -270,17 +270,6 @@ class BotInterruptionFrame(SystemFrame): pass -@dataclass -class BotSpeakingFrame(SystemFrame): - """Emitted by transport outputs while the bot is still speaking. This can be - used, for example, to detect when a user is idle. That is, while the bot is - speaking we don't want to trigger any user idle timeout since the user might - be listening. - - """ - pass - - @dataclass class MetricsFrame(SystemFrame): """Emitted by processor that can compute metrics like latencies. @@ -348,6 +337,33 @@ class UserStoppedSpeakingFrame(ControlFrame): pass +@dataclass +class BotStartedSpeakingFrame(ControlFrame): + """Emitted upstream by transport outputs to indicate the bot started speaking. + + """ + pass + + +@dataclass +class BotStoppedSpeakingFrame(ControlFrame): + """Emitted upstream by transport outputs to indicate the bot stopped speaking. + + """ + pass + + +@dataclass +class BotSpeakingFrame(ControlFrame): + """Emitted upstream by transport outputs while the bot is still + speaking. This can be used, for example, to detect when a user is idle. That + is, while the bot is speaking we don't want to trigger any user idle timeout + since the user might be listening. + + """ + pass + + @dataclass class TTSStartedFrame(ControlFrame): """Used to indicate the beginning of a TTS response. Following diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 355008f89..8a99d94a8 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -15,6 +15,8 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.frames.frames import ( AudioRawFrame, BotSpeakingFrame, + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, CancelFrame, MetricsFrame, SpriteFrame, @@ -25,6 +27,8 @@ from pipecat.frames.frames import ( StartInterruptionFrame, StopInterruptionFrame, SystemFrame, + TTSStartedFrame, + TTSStoppedFrame, TransportMessageFrame) from pipecat.transports.base_transport import TransportParams @@ -172,6 +176,12 @@ class BaseOutputTransport(FrameProcessor): await self._set_camera_images(frame.images) elif isinstance(frame, TransportMessageFrame): await self.send_message(frame) + elif isinstance(frame, TTSStartedFrame): + await self._internal_push_frame(BotStartedSpeakingFrame(), FrameDirection.UPSTREAM) + await self._internal_push_frame(frame) + elif isinstance(frame, TTSStoppedFrame): + await self._internal_push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM) + await self._internal_push_frame(frame) else: await self._internal_push_frame(frame)