From 2b4debec1116883d520f038b611329b3fb80b9c7 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 16 Mar 2025 18:23:30 -0700 Subject: [PATCH] add support for conversation.item.input_audio_transcription.delta --- src/pipecat/services/openai_realtime_beta/events.py | 8 ++++++++ src/pipecat/services/openai_realtime_beta/openai.py | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 17ce0a6d4..caa7e964c 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -193,6 +193,13 @@ class ConversationItemCreated(ServerEvent): item: ConversationItem +class ConversationItemInputAudioTranscriptionDelta(ServerEvent): + type: Literal["conversation.item.input_audio_transcription.delta"] + item_id: str + content_index: int + delta: str + + class ConversationItemInputAudioTranscriptionCompleted(ServerEvent): type: Literal["conversation.item.input_audio_transcription.completed"] item_id: str @@ -400,6 +407,7 @@ _server_event_types = { "input_audio_buffer.speech_started": InputAudioBufferSpeechStarted, "input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped, "conversation.item.created": ConversationItemCreated, + "conversation.item.input_audio_transcription.delta": ConversationItemInputAudioTranscriptionDelta, "conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted, "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed, "conversation.item.truncated": ConversationItemTruncated, diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 321c66826..6f0edd67c 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -29,6 +29,7 @@ from pipecat.frames.frames import ( EndFrame, ErrorFrame, Frame, + InterimTranscriptionFrame, InputAudioRawFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, @@ -354,6 +355,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self._handle_evt_audio_done(evt) elif evt.type == "conversation.item.created": await self._handle_evt_conversation_item_created(evt) + elif evt.type == "conversation.item.input_audio_transcription.delta": + await self._handle_evt_input_audio_transcription_delta(evt) elif evt.type == "conversation.item.input_audio_transcription.completed": await self.handle_evt_input_audio_transcription_completed(evt) elif evt.type == "response.done": @@ -425,6 +428,13 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._current_assistant_response = evt.item await self.push_frame(LLMFullResponseStartFrame()) + async def _handle_evt_input_audio_transcription_delta(self, evt): + if self._send_transcription_frames: + await self.push_frame( + # no way to get a language code? + InterimTranscriptionFrame(evt.delta, "", time_now_iso8601()) + ) + async def handle_evt_input_audio_transcription_completed(self, evt): if self._send_transcription_frames: await self.push_frame(