diff --git a/CHANGELOG.md b/CHANGELOG.md index 16c6075d7..7c4ec65df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added TTFB metrics for `HeyGenVideoService` and `TavusVideoService`. + - Added `endpoint_id` parameter to `AzureSTTService`. ([Custom EndpointId](https://docs.azure.cn/en-us/ai-services/speech-service/how-to-recognize-speech?pivots=programming-language-python#use-a-custom-endpoint)) ### Changed diff --git a/src/pipecat/services/heygen/video.py b/src/pipecat/services/heygen/video.py index f32c8f392..96c684641 100644 --- a/src/pipecat/services/heygen/video.py +++ b/src/pipecat/services/heygen/video.py @@ -20,6 +20,7 @@ from loguru import logger from pipecat.audio.utils import create_stream_resampler from pipecat.frames.frames import ( AudioRawFrame, + BotStartedSpeakingFrame, CancelFrame, EndFrame, Frame, @@ -30,6 +31,7 @@ from pipecat.frames.frames import ( SpeechOutputAudioRawFrame, StartFrame, TTSAudioRawFrame, + TTSStartedFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -232,9 +234,24 @@ class HeyGenVideoService(AIService): await self.push_frame(frame, direction) elif isinstance(frame, TTSAudioRawFrame): await self._handle_audio_frame(frame) + elif isinstance(frame, TTSStartedFrame): + await self.start_ttfb_metrics() + elif isinstance(frame, BotStartedSpeakingFrame): + # We constantly receive audio through WebRTC, but most of the time it is silence. + # As soon as we receive actual audio, the base output transport will create a + # BotStartedSpeakingFrame, which we can use as a signal for the TTFB metrics. + await self.stop_ttfb_metrics() else: await self.push_frame(frame, direction) + def can_generate_metrics(self) -> bool: + """Check if the service can generate metrics. + + Returns: + True if metrics generation is supported. + """ + return True + async def _handle_user_started_speaking(self): """Handle the event when a user starts speaking. diff --git a/src/pipecat/services/tavus/video.py b/src/pipecat/services/tavus/video.py index 684bd5659..e3c83bb89 100644 --- a/src/pipecat/services/tavus/video.py +++ b/src/pipecat/services/tavus/video.py @@ -19,6 +19,7 @@ from loguru import logger from pipecat.audio.utils import create_stream_resampler from pipecat.frames.frames import ( + BotStartedSpeakingFrame, CancelFrame, EndFrame, Frame, @@ -29,6 +30,7 @@ from pipecat.frames.frames import ( StartFrame, StartInterruptionFrame, TTSAudioRawFrame, + TTSStartedFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessorSetup from pipecat.services.ai_service import AIService @@ -229,6 +231,13 @@ class TavusVideoService(AIService): elif isinstance(frame, OutputTransportReadyFrame): self._transport_ready = True await self.push_frame(frame, direction) + elif isinstance(frame, TTSStartedFrame): + await self.start_ttfb_metrics() + elif isinstance(frame, BotStartedSpeakingFrame): + # We constantly receive audio through WebRTC, but most of the time it is silence. + # As soon as we receive actual audio, the base output transport will create a + # BotStartedSpeakingFrame, which we can use as a signal for the TTFB metrics. + await self.stop_ttfb_metrics() else: await self.push_frame(frame, direction)