diff --git a/changelog/3687.added.md b/changelog/3687.added.md new file mode 100644 index 000000000..38a10fb75 --- /dev/null +++ b/changelog/3687.added.md @@ -0,0 +1 @@ +- Added `UserMuteStartedFrame` and `UserMuteStoppedFrame` system frames, and corresponding `user-mute-started` / `user-mute-stopped` RTVI messages, so clients can observe when mute strategies activate or deactivate. diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index b7be8045e..cb0dd0cee 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -1208,6 +1208,28 @@ class UserStoppedSpeakingFrame(SystemFrame): emulated: bool = False +@dataclass +class UserMuteStartedFrame(SystemFrame): + """Frame indicating that the user has been muted. + + Emitted when a mute strategy activates, suppressing user frames (audio, + transcription, interruption) from propagating through the pipeline. + """ + + pass + + +@dataclass +class UserMuteStoppedFrame(SystemFrame): + """Frame indicating that the user has been unmuted. + + Emitted when a mute strategy deactivates, allowing user frames to + propagate through the pipeline again. + """ + + pass + + @dataclass class UserSpeakingFrame(SystemFrame): """Frame indicating the user is speaking. diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index cf65c6443..05090dcc0 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -53,6 +53,8 @@ from pipecat.frames.frames import ( TextFrame, TranscriptionFrame, UserImageRawFrame, + UserMuteStartedFrame, + UserMuteStoppedFrame, UserSpeakingFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, @@ -569,8 +571,10 @@ class LLMUserAggregator(LLMContextAggregator): # Emit mute state change events if self._user_is_muted: await self._call_event_handler("on_user_mute_started") + await self.broadcast_frame(UserMuteStartedFrame) else: await self._call_event_handler("on_user_mute_stopped") + await self.broadcast_frame(UserMuteStoppedFrame) return should_mute_frame diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 61c3ef57e..cbb0d706f 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -67,6 +67,8 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, TTSTextFrame, + UserMuteStartedFrame, + UserMuteStoppedFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -891,6 +893,20 @@ class RTVIUserStoppedSpeakingMessage(BaseModel): type: Literal["user-stopped-speaking"] = "user-stopped-speaking" +class RTVIUserMuteStartedMessage(BaseModel): + """Message indicating user has been muted.""" + + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL + type: Literal["user-mute-started"] = "user-mute-started" + + +class RTVIUserMuteStoppedMessage(BaseModel): + """Message indicating user has been unmuted.""" + + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL + type: Literal["user-mute-stopped"] = "user-mute-stopped" + + class RTVIBotStartedSpeakingMessage(BaseModel): """Message indicating bot has started speaking.""" @@ -1043,6 +1059,7 @@ class RTVIObserverParams: bot_audio_level_enabled: bool = False user_llm_enabled: bool = True user_speaking_enabled: bool = True + user_mute_enabled: bool = True user_transcription_enabled: bool = True user_audio_level_enabled: bool = False metrics_enabled: bool = True @@ -1212,6 +1229,11 @@ class RTVIObserver(BaseObserver): and self._params.user_speaking_enabled ): await self._handle_interruptions(frame) + elif ( + isinstance(frame, (UserMuteStartedFrame, UserMuteStoppedFrame)) + and self._params.user_mute_enabled + ): + await self._handle_user_mute(frame) elif ( isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame)) and self._params.bot_speaking_enabled @@ -1343,6 +1365,17 @@ class RTVIObserver(BaseObserver): if message: await self.send_rtvi_message(message) + async def _handle_user_mute(self, frame: Frame): + """Handle user mute/unmute frames.""" + message = None + if isinstance(frame, UserMuteStartedFrame): + message = RTVIUserMuteStartedMessage() + elif isinstance(frame, UserMuteStoppedFrame): + message = RTVIUserMuteStoppedMessage() + + if message: + await self.send_rtvi_message(message) + async def _handle_bot_speaking(self, frame: Frame): """Handle bot speaking event frames.""" message = None