From 12b5c5a646307d1be13cc159316e52f62f52833e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 6 Jan 2025 15:36:57 -0500 Subject: [PATCH] Fix truncation timing of OpenAIRealtimeBetaLLMService --- CHANGELOG.md | 5 ++++ .../services/openai_realtime_beta/openai.py | 23 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb080735d..4fc32ae72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `29-livekit-audio-chat.py`, as a new foundational examples for `LiveKitTransportLayer`. +### Fixed + +- Fixed an issue where `OpenAIRealtimeBetaLLMService` audio chunks were hitting + an error when truncating audio content. + ## [0.0.52] - 2024-12-24 ### Added diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index ea5d4c41e..8bb05e6f0 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -152,17 +152,38 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_bot_stopped_speaking(self): self._current_audio_response = None + def _calculate_audio_duration_ms( + self, total_bytes: int, sample_rate: int = 24000, bytes_per_sample: int = 2 + ) -> int: + """Calculate audio duration in milliseconds based on PCM audio parameters.""" + samples = total_bytes / bytes_per_sample + duration_seconds = samples / sample_rate + return int(duration_seconds * 1000) + async def _truncate_current_audio_response(self): + """Truncates the current audio response at the appropriate duration. + + Calculates the actual duration of the audio content and truncates at the shorter of + either the wall clock time or the actual audio duration to prevent invalid truncation + requests. + """ # if the bot is still speaking, truncate the last message if self._current_audio_response: current = self._current_audio_response self._current_audio_response = None + + # Calculate actual audio duration instead of using wall clock time + audio_duration_ms = self._calculate_audio_duration_ms(current.total_size) + + # Use the shorter of wall clock time or actual audio duration elapsed_ms = int(time.time() * 1000 - current.start_time_ms) + truncate_ms = min(elapsed_ms, audio_duration_ms) + await self.send_client_event( events.ConversationItemTruncateEvent( item_id=current.item_id, content_index=current.content_index, - audio_end_ms=elapsed_ms, + audio_end_ms=truncate_ms, ) )