From 5ab00e01aab5ad53d5d0e5c6d0c9c1012227d970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 25 Oct 2024 12:10:24 -0700 Subject: [PATCH 1/3] transports(base_output): fix constant bot started/stopped speaking frames --- CHANGELOG.md | 3 ++ src/pipecat/transports/base_output.py | 42 ++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea107042d..5b8771a7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue that was generating constant bot started/stopped speaking + frames for HTTP TTS services. + - Fixed an issue that was causing stuttering with AWS TTS service. ## [0.0.47] - 2024-10-22 diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index e61bcc448..525dc37fa 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -70,9 +70,14 @@ class BaseOutputTransport(FrameProcessor): self._stopped_event = asyncio.Event() # Indicates if the bot is currently speaking. This is useful when we - # have an interruption since all the queued messages will be thrown - # away and we would lose the TTSStoppedFrame. + # have an interruption since all the queued messages will be thrown away + # and we would lose the TTSStoppedFrame. We also keep a scheduler handle + # which is used to schedule pushing bot stopped speaking frames. With + # some services (HTTP TTS services) we will get a bunch of + # TTSStartedFrame and TTSStoppedFrame and we don't want to constantly + # generate bot started/stopped speaking frames. self._bot_speaking = False + self._bot_stopped_speaking_handle = None # Create sink frame task. This is the task that will actually write # audio or video frames. We write audio/video in a task so we can keep @@ -320,14 +325,35 @@ class BaseOutputTransport(FrameProcessor): logger.exception(f"{self} error processing sink clock queue: {e}") async def _bot_started_speaking(self): - logger.debug("Bot started speaking") - self._bot_speaking = True - await self.push_frame(BotStartedSpeakingFrame(), FrameDirection.UPSTREAM) + # If we scheduled bot stopped speaking, cancel it. + if self._bot_stopped_speaking_handle: + self._bot_stopped_speaking_handle.cancel() + self._bot_stopped_speaking_handle = None + + if not self._bot_speaking: + logger.debug("Bot started speaking") + await self.push_frame(BotStartedSpeakingFrame(), FrameDirection.UPSTREAM) + self._bot_speaking = True async def _bot_stopped_speaking(self): - logger.debug("Bot stopped speaking") - self._bot_speaking = False - await self.push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM) + async def push_bot_stopped_speaking_async(): + try: + logger.debug("Bot stopped speaking") + await self.push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM) + self._bot_speaking = False + except asyncio.CancelledError: + logger.debug("We've been cancelled") + pass + + def push_bot_stopped_speaking(): + self.get_event_loop().create_task(push_bot_stopped_speaking_async()) + + # Schedule a bot stopped speaking to be pushed in half a second (unless + # we get another bot started speaking). + if not self._bot_stopped_speaking_handle: + self._bot_stopped_speaking_handle = self.get_event_loop().call_later( + 0.5, push_bot_stopped_speaking + ) # # Camera out From 42835578942abf6cf515ddd6c6e3970c6663a6d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 25 Oct 2024 12:55:34 -0700 Subject: [PATCH 2/3] audio(vad): expose params property --- src/pipecat/audio/vad/vad_analyzer.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/pipecat/audio/vad/vad_analyzer.py b/src/pipecat/audio/vad/vad_analyzer.py index 3387a1746..17bce6543 100644 --- a/src/pipecat/audio/vad/vad_analyzer.py +++ b/src/pipecat/audio/vad/vad_analyzer.py @@ -12,6 +12,11 @@ from pydantic.main import BaseModel from pipecat.audio.utils import calculate_audio_volume, exp_smoothing +VAD_CONFIDENCE = 0.7 +VAD_START_SECS = 0.2 +VAD_STOP_SECS = 0.8 +VAD_MIN_VOLUME = 0.6 + class VADState(Enum): QUIET = 1 @@ -21,10 +26,10 @@ class VADState(Enum): class VADParams(BaseModel): - confidence: float = 0.7 - start_secs: float = 0.2 - stop_secs: float = 0.8 - min_volume: float = 0.6 + confidence: float = VAD_CONFIDENCE + start_secs: float = VAD_START_SECS + stop_secs: float = VAD_STOP_SECS + min_volume: float = VAD_MIN_VOLUME class VADAnalyzer: @@ -41,13 +46,17 @@ class VADAnalyzer: self._prev_volume = 0 @property - def sample_rate(self): + def sample_rate(self) -> int: return self._sample_rate @property - def num_channels(self): + def num_channels(self) -> int: return self._num_channels + @property + def params(self) -> VADParams: + return self._params + @abstractmethod def num_frames_required(self) -> int: pass From 55026898f676044d54953b253b46de712ac11fd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 25 Oct 2024 12:55:48 -0700 Subject: [PATCH 3/3] transports(base_output): use vad stop secs for bot stopped speaking --- src/pipecat/transports/base_output.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 525dc37fa..18faca582 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -13,6 +13,7 @@ from typing import List from loguru import logger from PIL import Image +from pipecat.audio.vad.vad_analyzer import VAD_STOP_SECS from pipecat.frames.frames import ( BotSpeakingFrame, BotStartedSpeakingFrame, @@ -348,11 +349,16 @@ class BaseOutputTransport(FrameProcessor): def push_bot_stopped_speaking(): self.get_event_loop().create_task(push_bot_stopped_speaking_async()) - # Schedule a bot stopped speaking to be pushed in half a second (unless - # we get another bot started speaking). + # Schedule a bot stopped speaking to be pushed after VAD stop secs + # (unless we get another bot started speaking). + wait_time = ( + self._params.vad_analyzer.params.stop_secs + if self._params.vad_analyzer + else VAD_STOP_SECS + ) if not self._bot_stopped_speaking_handle: self._bot_stopped_speaking_handle = self.get_event_loop().call_later( - 0.5, push_bot_stopped_speaking + wait_time, push_bot_stopped_speaking ) #