wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add support for conversation.item.input_audio_transcription.delta

Browse Source
This commit is contained in:
Kwindla Hultman Kramer
2025-03-16 18:23:30 -07:00
committed by Paul Kompfner
parent f4626287cd
commit 2b4debec11
2 changed files with 18 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/pipecat/services/openai_realtime_beta/events.py
View File

@@ -193,6 +193,13 @@ class ConversationItemCreated(ServerEvent):
item: ConversationItem item: ConversationItem
class ConversationItemInputAudioTranscriptionDelta(ServerEvent):
type: Literal["conversation.item.input_audio_transcription.delta"]
item_id: str
content_index: int
delta: str
class ConversationItemInputAudioTranscriptionCompleted(ServerEvent): class ConversationItemInputAudioTranscriptionCompleted(ServerEvent):
type: Literal["conversation.item.input_audio_transcription.completed"] type: Literal["conversation.item.input_audio_transcription.completed"]
item_id: str item_id: str
@@ -400,6 +407,7 @@ _server_event_types = {
"input_audio_buffer.speech_started": InputAudioBufferSpeechStarted, "input_audio_buffer.speech_started": InputAudioBufferSpeechStarted,
"input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped, "input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped,
"conversation.item.created": ConversationItemCreated, "conversation.item.created": ConversationItemCreated,
"conversation.item.input_audio_transcription.delta": ConversationItemInputAudioTranscriptionDelta,
"conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted, "conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted,
"conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed, "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed,
"conversation.item.truncated": ConversationItemTruncated, "conversation.item.truncated": ConversationItemTruncated,

10
src/pipecat/services/openai_realtime_beta/openai.py
View File

@@ -29,6 +29,7 @@ from pipecat.frames.frames import (
EndFrame, EndFrame,
ErrorFrame, ErrorFrame,
Frame, Frame,
InterimTranscriptionFrame,
InputAudioRawFrame, InputAudioRawFrame,
LLMFullResponseEndFrame, LLMFullResponseEndFrame,
LLMFullResponseStartFrame, LLMFullResponseStartFrame,
@@ -354,6 +355,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
await self._handle_evt_audio_done(evt) await self._handle_evt_audio_done(evt)
elif evt.type == "conversation.item.created": elif evt.type == "conversation.item.created":
await self._handle_evt_conversation_item_created(evt) await self._handle_evt_conversation_item_created(evt)
elif evt.type == "conversation.item.input_audio_transcription.delta":
await self._handle_evt_input_audio_transcription_delta(evt)
elif evt.type == "conversation.item.input_audio_transcription.completed": elif evt.type == "conversation.item.input_audio_transcription.completed":
await self.handle_evt_input_audio_transcription_completed(evt) await self.handle_evt_input_audio_transcription_completed(evt)
elif evt.type == "response.done": elif evt.type == "response.done":
@@ -425,6 +428,13 @@ class OpenAIRealtimeBetaLLMService(LLMService):
self._current_assistant_response = evt.item self._current_assistant_response = evt.item
await self.push_frame(LLMFullResponseStartFrame()) await self.push_frame(LLMFullResponseStartFrame())
async def _handle_evt_input_audio_transcription_delta(self, evt):
if self._send_transcription_frames:
await self.push_frame(
# no way to get a language code?
InterimTranscriptionFrame(evt.delta, "", time_now_iso8601())
)
async def handle_evt_input_audio_transcription_completed(self, evt): async def handle_evt_input_audio_transcription_completed(self, evt):
if self._send_transcription_frames: if self._send_transcription_frames:
await self.push_frame( await self.push_frame(