Merge pull request #327 from pipecat-ai/aleix/bot-start-stop-speaking-frames

bot start stop speaking frames

CHANGELOG.md

@@ -9,10 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added new `BotStartedSpeakingFrame` and `BotStoppedSpeakingFrame` control
+  frames. These frames are pushed upstream and they should wrap
+  `BotSpeakingFrame`.
+
 - Transports now allow you to register event handlers without decorators.
 
 ### Changed
 
+- `BotSpeakingFrame` is now a control frame.
+
 - `StartFrame` is now a control frame similar to `EndFrame`.
 
 - `DeepgramTTSService` now is more customizable. You can adjust the encoding and

src/pipecat/frames/frames.py

@@ -270,17 +270,6 @@ class BotInterruptionFrame(SystemFrame):
     pass
 
 
-@dataclass
-class BotSpeakingFrame(SystemFrame):
-    """Emitted by transport outputs while the bot is still speaking. This can be
-    used, for example, to detect when a user is idle. That is, while the bot is
-    speaking we don't want to trigger any user idle timeout since the user might
-    be listening.
-
-    """
-    pass
-
-
 @dataclass
 class MetricsFrame(SystemFrame):
     """Emitted by processor that can compute metrics like latencies.
@@ -348,6 +337,33 @@ class UserStoppedSpeakingFrame(ControlFrame):
     pass
 
 
+@dataclass
+class BotStartedSpeakingFrame(ControlFrame):
+    """Emitted upstream by transport outputs to indicate the bot started speaking.
+
+    """
+    pass
+
+
+@dataclass
+class BotStoppedSpeakingFrame(ControlFrame):
+    """Emitted upstream by transport outputs to indicate the bot stopped speaking.
+
+    """
+    pass
+
+
+@dataclass
+class BotSpeakingFrame(ControlFrame):
+    """Emitted upstream by transport outputs while the bot is still
+    speaking. This can be used, for example, to detect when a user is idle. That
+    is, while the bot is speaking we don't want to trigger any user idle timeout
+    since the user might be listening.
+
+    """
+    pass
+
+
 @dataclass
 class TTSStartedFrame(ControlFrame):
     """Used to indicate the beginning of a TTS response. Following

src/pipecat/processors/user_idle_processor.py

@@ -8,7 +8,12 @@ import asyncio
 
 from typing import Awaitable, Callable
 
-from pipecat.frames.frames import BotSpeakingFrame, Frame, StartInterruptionFrame, StopInterruptionFrame, SystemFrame
+from pipecat.frames.frames import (
+    BotSpeakingFrame,
+    Frame,
+    SystemFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame)
 from pipecat.processors.async_frame_processor import AsyncFrameProcessor
 from pipecat.processors.frame_processor import FrameDirection
 
@@ -47,10 +52,10 @@ class UserIdleProcessor(AsyncFrameProcessor):
             await self.queue_frame(frame, direction)
 
         # We shouldn't call the idle callback if the user or the bot are speaking.
-        if isinstance(frame, StartInterruptionFrame):
+        if isinstance(frame, UserStartedSpeakingFrame):
             self._interrupted = True
             self._idle_event.set()
-        elif isinstance(frame, StopInterruptionFrame):
+        elif isinstance(frame, UserStoppedSpeakingFrame):
             self._interrupted = False
             self._idle_event.set()
         elif isinstance(frame, BotSpeakingFrame):

src/pipecat/transports/base_output.py

@@ -15,6 +15,8 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.frames.frames import (
     AudioRawFrame,
     BotSpeakingFrame,
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
     CancelFrame,
     MetricsFrame,
     SpriteFrame,
@@ -25,6 +27,8 @@ from pipecat.frames.frames import (
     StartInterruptionFrame,
     StopInterruptionFrame,
     SystemFrame,
+    TTSStartedFrame,
+    TTSStoppedFrame,
     TransportMessageFrame)
 from pipecat.transports.base_transport import TransportParams
 
@@ -172,6 +176,12 @@ class BaseOutputTransport(FrameProcessor):
                     await self._set_camera_images(frame.images)
                 elif isinstance(frame, TransportMessageFrame):
                     await self.send_message(frame)
+                elif isinstance(frame, TTSStartedFrame):
+                    await self._internal_push_frame(BotStartedSpeakingFrame(), FrameDirection.UPSTREAM)
+                    await self._internal_push_frame(frame)
+                elif isinstance(frame, TTSStoppedFrame):
+                    await self._internal_push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM)
+                    await self._internal_push_frame(frame)
                 else:
                     await self._internal_push_frame(frame)