From a5f5e418a85bbf5fe6552fb8b54eab845f701c27 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 7 Aug 2025 13:46:58 -0400 Subject: [PATCH] feat: Add option to ignore emulated user speech while the bot is speaking --- CHANGELOG.md | 6 ++++++ .../processors/aggregators/llm_response.py | 20 ++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 533f5f65d..3df3620a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `enable_emulated_vad_interruptions` to `LLMUserAggregatorParams`. + When user speech is emulated (e.g. when a transcription is received but + VAD doesn't detect speech), this parameter controls whether the emulated + speech can interrupt the bot. Default is False (emulated speech is ignored + while the bot is speaking). + - Added new `handle_sigint` and `handle_sigterm` to `RunnerArguments`. This allows applications to know what settings they should use for the environment they are running on. Also, added `pipeline_idle_timeout_secs` to be able to diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 483b12059..3b1215473 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -73,10 +73,14 @@ class LLMUserAggregatorParams: turn_emulated_vad_timeout: Maximum time in seconds to wait for emulated VAD when using turn-based analysis. Applied when transcription is received but VAD didn't detect speech (e.g., whispered utterances). + enable_emulated_vad_interruptions: When True, allows emulated VAD events + to interrupt the bot when it's speaking. When False, emulated speech + is ignored while the bot is speaking. """ aggregation_timeout: float = 0.5 turn_emulated_vad_timeout: float = 0.8 + enable_emulated_vad_interruptions: bool = False @dataclass @@ -686,26 +690,24 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): """Maybe emulate user speaking based on transcription. Emulate user speaking if we got a transcription but it was not - detected by VAD. Only do that if the bot is not speaking. + detected by VAD. Behavior when bot is speaking depends on the + enable_emulated_vad_interruptions parameter. """ # Check if we received a transcription but VAD was not able to detect # voice (e.g. when you whisper a short utterance). In that case, we need - # to emulate VAD (i.e. user start/stopped speaking), but we do it only - # if the bot is not speaking. If the bot is speaking and we really have - # a short utterance we don't really want to interrupt the bot. + # to emulate VAD (i.e. user start/stopped speaking). if ( not self._user_speaking and not self._waiting_for_aggregation and len(self._aggregation) > 0 ): - if self._bot_speaking: - # If we reached this case and the bot is speaking, let's ignore - # what the user said. + if self._bot_speaking and not self._params.enable_emulated_vad_interruptions: + # If emulated VAD interruptions are disabled and bot is speaking, ignore logger.debug("Ignoring user speaking emulation, bot is speaking.") await self.reset() else: - # The bot is not speaking so, let's trigger user speaking - # emulation. + # Either bot is not speaking, or emulated VAD interruptions are enabled + # - trigger user speaking emulation. await self.push_frame(EmulateUserStartedSpeakingFrame(), FrameDirection.UPSTREAM) self._emulating_vad = True