From 2b4debec1116883d520f038b611329b3fb80b9c7 Mon Sep 17 00:00:00 2001
From: Kwindla Hultman Kramer <kwindla@daily.co>
Date: Sun, 16 Mar 2025 18:23:30 -0700
Subject: [PATCH 01/28] add support for
 conversation.item.input_audio_transcription.delta

---
 src/pipecat/services/openai_realtime_beta/events.py |  8 ++++++++
 src/pipecat/services/openai_realtime_beta/openai.py | 10 ++++++++++
 2 files changed, 18 insertions(+)

diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py
index 17ce0a6d4..caa7e964c 100644
--- a/src/pipecat/services/openai_realtime_beta/events.py
+++ b/src/pipecat/services/openai_realtime_beta/events.py
@@ -193,6 +193,13 @@ class ConversationItemCreated(ServerEvent):
     item: ConversationItem
 
 
+class ConversationItemInputAudioTranscriptionDelta(ServerEvent):
+    type: Literal["conversation.item.input_audio_transcription.delta"]
+    item_id: str
+    content_index: int
+    delta: str
+
+
 class ConversationItemInputAudioTranscriptionCompleted(ServerEvent):
     type: Literal["conversation.item.input_audio_transcription.completed"]
     item_id: str
@@ -400,6 +407,7 @@ _server_event_types = {
     "input_audio_buffer.speech_started": InputAudioBufferSpeechStarted,
     "input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped,
     "conversation.item.created": ConversationItemCreated,
+    "conversation.item.input_audio_transcription.delta": ConversationItemInputAudioTranscriptionDelta,
     "conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted,
     "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed,
     "conversation.item.truncated": ConversationItemTruncated,
diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index 321c66826..6f0edd67c 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -29,6 +29,7 @@ from pipecat.frames.frames import (
     EndFrame,
     ErrorFrame,
     Frame,
+    InterimTranscriptionFrame,
     InputAudioRawFrame,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
@@ -354,6 +355,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
                 await self._handle_evt_audio_done(evt)
             elif evt.type == "conversation.item.created":
                 await self._handle_evt_conversation_item_created(evt)
+            elif evt.type == "conversation.item.input_audio_transcription.delta":
+                await self._handle_evt_input_audio_transcription_delta(evt)
             elif evt.type == "conversation.item.input_audio_transcription.completed":
                 await self.handle_evt_input_audio_transcription_completed(evt)
             elif evt.type == "response.done":
@@ -425,6 +428,13 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             self._current_assistant_response = evt.item
             await self.push_frame(LLMFullResponseStartFrame())
 
+    async def _handle_evt_input_audio_transcription_delta(self, evt):
+        if self._send_transcription_frames:
+            await self.push_frame(
+                # no way to get a language code?
+                InterimTranscriptionFrame(evt.delta, "", time_now_iso8601())
+            )
+
     async def handle_evt_input_audio_transcription_completed(self, evt):
         if self._send_transcription_frames:
             await self.push_frame(

From bfdf52bd69428faea7cb8eb6077b69dd5b8b7be0 Mon Sep 17 00:00:00 2001
From: Kwindla Hultman Kramer <kwindla@daily.co>
Date: Sun, 16 Mar 2025 18:27:17 -0700
Subject: [PATCH 02/28] change examples/foundational/19-openai-realtime-beta.py
 to use the new preview model

---
 examples/foundational/19-openai-realtime-beta.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py
index 8796e0141..8027b0669 100644
--- a/examples/foundational/19-openai-realtime-beta.py
+++ b/examples/foundational/19-openai-realtime-beta.py
@@ -114,6 +114,7 @@ Remember, your responses should be short. Just one or two sentences, usually."""
 
         llm = OpenAIRealtimeBetaLLMService(
             api_key=os.getenv("OPENAI_API_KEY"),
+            model="gpt-4o-realtime-preview-latest",
             session_properties=session_properties,
             start_audio_paused=False,
         )

From 4449e9a25bcf79b7659afec00937aebfcd5a06dd Mon Sep 17 00:00:00 2001
From: Kwindla Hultman Kramer <kwindla@daily.co>
Date: Sun, 16 Mar 2025 18:40:19 -0700
Subject: [PATCH 03/28] add response.done status=failed error

---
 examples/foundational/19-openai-realtime-beta.py    | 1 -
 src/pipecat/services/openai_realtime_beta/openai.py | 7 ++++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py
index 8027b0669..8796e0141 100644
--- a/examples/foundational/19-openai-realtime-beta.py
+++ b/examples/foundational/19-openai-realtime-beta.py
@@ -114,7 +114,6 @@ Remember, your responses should be short. Just one or two sentences, usually."""
 
         llm = OpenAIRealtimeBetaLLMService(
             api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o-realtime-preview-latest",
             session_properties=session_properties,
             start_audio_paused=False,
         )
diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index 6f0edd67c..f4c99a73f 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -87,7 +87,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         self,
         *,
         api_key: str,
-        model: str = "gpt-4o-realtime-preview-2024-12-17",
+        # model: str = "gpt-4o-realtime-preview-2024-12-17",
+        model: str = "gpt-4o-realtime-preview-latest",
         base_url: str = "wss://api.openai.com/v1/realtime",
         session_properties: events.SessionProperties = events.SessionProperties(),
         start_audio_paused: bool = False,
@@ -465,6 +466,10 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         await self.stop_processing_metrics()
         await self.push_frame(LLMFullResponseEndFrame())
         self._current_assistant_response = None
+        # error handling
+        if evt.response.status == "failed":
+            await self.push_error(ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True))
+            return
         # response content
         pair = self._user_and_response_message_tuple
         if pair:

From 16accafa6debc58db7cbd7c5fff40360eda2ca10 Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Mon, 17 Mar 2025 10:47:00 -0400
Subject: [PATCH 04/28] formatting fix

---
 src/pipecat/services/openai_realtime_beta/openai.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index f4c99a73f..879d96e6f 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -468,7 +468,9 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         self._current_assistant_response = None
         # error handling
         if evt.response.status == "failed":
-            await self.push_error(ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True))
+            await self.push_error(
+                ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True)
+            )
             return
         # response content
         pair = self._user_and_response_message_tuple

From 214c8f79eb2b5f372df0b118c1f389bf5d7f433a Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Mon, 17 Mar 2025 10:50:04 -0400
Subject: [PATCH 05/28] linter fix

---
 src/pipecat/services/openai_realtime_beta/openai.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index 879d96e6f..68dde4d32 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -29,8 +29,8 @@ from pipecat.frames.frames import (
     EndFrame,
     ErrorFrame,
     Frame,
-    InterimTranscriptionFrame,
     InputAudioRawFrame,
+    InterimTranscriptionFrame,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
     LLMMessagesAppendFrame,

From e80bfe22de734d9449b53fd6d0224159fd8cc6ab Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Mon, 17 Mar 2025 12:15:14 -0400
Subject: [PATCH 06/28] Add new GPT-4o transcription option to
 OpenAIRealtimeBetaLLMService

---
 examples/foundational/19-openai-realtime-beta.py    |  4 ++--
 examples/foundational/19a-azure-realtime-beta.py    |  4 ++--
 .../20b-persistent-context-openai-realtime.py       |  4 ++--
 .../services/openai_realtime_beta/__init__.py       |  2 +-
 src/pipecat/services/openai_realtime_beta/events.py | 13 +++++++++++--
 5 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py
index 8796e0141..c39c0f570 100644
--- a/examples/foundational/19-openai-realtime-beta.py
+++ b/examples/foundational/19-openai-realtime-beta.py
@@ -21,7 +21,7 @@ from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai_realtime_beta import (
-    InputAudioTranscription,
+    InputAudioTranscriptionModels,
     OpenAIRealtimeBetaLLMService,
     SessionProperties,
     TurnDetection,
@@ -89,7 +89,7 @@ async def main():
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscription(),
+            input_audio_transcription=InputAudioTranscriptionModels.Whisper1(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
             turn_detection=TurnDetection(silence_duration_ms=1000),
diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py
index 14d034836..12847ee33 100644
--- a/examples/foundational/19a-azure-realtime-beta.py
+++ b/examples/foundational/19a-azure-realtime-beta.py
@@ -23,7 +23,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai_realtime_beta import (
     AzureRealtimeBetaLLMService,
-    InputAudioTranscription,
+    InputAudioTranscriptionModels,
     SessionProperties,
     TurnDetection,
 )
@@ -90,7 +90,7 @@ async def main():
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscription(),
+            input_audio_transcription=InputAudioTranscriptionModels.Whisper1(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
             # turn_detection=TurnDetection(silence_duration_ms=1000),
diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py
index ef82bd567..26a6448e9 100644
--- a/examples/foundational/20b-persistent-context-openai-realtime.py
+++ b/examples/foundational/20b-persistent-context-openai-realtime.py
@@ -25,7 +25,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
     OpenAILLMContext,
 )
 from pipecat.services.openai_realtime_beta import (
-    InputAudioTranscription,
+    InputAudioTranscriptionModels,
     OpenAIRealtimeBetaLLMService,
     SessionProperties,
     TurnDetection,
@@ -186,7 +186,7 @@ async def main():
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscription(),
+            input_audio_transcription=InputAudioTranscriptionModels.Whisper1(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
             turn_detection=TurnDetection(silence_duration_ms=1000),
diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py
index 52b00f6c8..fdda648c4 100644
--- a/src/pipecat/services/openai_realtime_beta/__init__.py
+++ b/src/pipecat/services/openai_realtime_beta/__init__.py
@@ -1,3 +1,3 @@
 from .azure import AzureRealtimeBetaLLMService
-from .events import InputAudioTranscription, SessionProperties, TurnDetection
+from .events import InputAudioTranscriptionModels, SessionProperties, TurnDetection
 from .openai import OpenAIRealtimeBetaLLMService
diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py
index caa7e964c..9cc946807 100644
--- a/src/pipecat/services/openai_realtime_beta/events.py
+++ b/src/pipecat/services/openai_realtime_beta/events.py
@@ -16,8 +16,17 @@ from pydantic import BaseModel, Field
 #
 
 
-class InputAudioTranscription(BaseModel):
-    model: Optional[str] = "whisper-1"
+class InputAudioTranscriptionModels:
+    class Whisper1(BaseModel):
+        model: Optional[Literal["whisper-1"]] = "whisper-1"
+
+    class GPT4o(BaseModel):
+        model: Optional[Literal["gpt-4o-transcribe-latest"]] = "gpt-4o-transcribe-latest"
+        language: Optional[str] = None
+        prompt: Optional[str] = None
+
+
+InputAudioTranscription = Union[InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o]
 
 
 class TurnDetection(BaseModel):

From be2cf6d556aacf8b2233f7e83907657b9414d1ae Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Mon, 17 Mar 2025 12:28:11 -0400
Subject: [PATCH 07/28] formatting fix

---
 src/pipecat/services/openai_realtime_beta/events.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py
index 9cc946807..adfc34133 100644
--- a/src/pipecat/services/openai_realtime_beta/events.py
+++ b/src/pipecat/services/openai_realtime_beta/events.py
@@ -26,7 +26,9 @@ class InputAudioTranscriptionModels:
         prompt: Optional[str] = None
 
 
-InputAudioTranscription = Union[InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o]
+InputAudioTranscription = Union[
+    InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o
+]
 
 
 class TurnDetection(BaseModel):

From fe5fc302114586b6eeeaed55fd59c2711319345a Mon Sep 17 00:00:00 2001
From: kompfner <kompfner@gmail.com>
Date: Mon, 17 Mar 2025 12:36:31 -0400
Subject: [PATCH 08/28] Revert "Add new GPT-4o transcription option to
 OpenAIRealtimeBetaLLMService"

---
 examples/foundational/19-openai-realtime-beta.py  |  4 ++--
 examples/foundational/19a-azure-realtime-beta.py  |  4 ++--
 .../20b-persistent-context-openai-realtime.py     |  4 ++--
 .../services/openai_realtime_beta/__init__.py     |  2 +-
 .../services/openai_realtime_beta/events.py       | 15 ++-------------
 5 files changed, 9 insertions(+), 20 deletions(-)

diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py
index c39c0f570..8796e0141 100644
--- a/examples/foundational/19-openai-realtime-beta.py
+++ b/examples/foundational/19-openai-realtime-beta.py
@@ -21,7 +21,7 @@ from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai_realtime_beta import (
-    InputAudioTranscriptionModels,
+    InputAudioTranscription,
     OpenAIRealtimeBetaLLMService,
     SessionProperties,
     TurnDetection,
@@ -89,7 +89,7 @@ async def main():
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscriptionModels.Whisper1(),
+            input_audio_transcription=InputAudioTranscription(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
             turn_detection=TurnDetection(silence_duration_ms=1000),
diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py
index 12847ee33..14d034836 100644
--- a/examples/foundational/19a-azure-realtime-beta.py
+++ b/examples/foundational/19a-azure-realtime-beta.py
@@ -23,7 +23,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai_realtime_beta import (
     AzureRealtimeBetaLLMService,
-    InputAudioTranscriptionModels,
+    InputAudioTranscription,
     SessionProperties,
     TurnDetection,
 )
@@ -90,7 +90,7 @@ async def main():
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscriptionModels.Whisper1(),
+            input_audio_transcription=InputAudioTranscription(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
             # turn_detection=TurnDetection(silence_duration_ms=1000),
diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py
index 26a6448e9..ef82bd567 100644
--- a/examples/foundational/20b-persistent-context-openai-realtime.py
+++ b/examples/foundational/20b-persistent-context-openai-realtime.py
@@ -25,7 +25,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
     OpenAILLMContext,
 )
 from pipecat.services.openai_realtime_beta import (
-    InputAudioTranscriptionModels,
+    InputAudioTranscription,
     OpenAIRealtimeBetaLLMService,
     SessionProperties,
     TurnDetection,
@@ -186,7 +186,7 @@ async def main():
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscriptionModels.Whisper1(),
+            input_audio_transcription=InputAudioTranscription(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
             turn_detection=TurnDetection(silence_duration_ms=1000),
diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py
index fdda648c4..52b00f6c8 100644
--- a/src/pipecat/services/openai_realtime_beta/__init__.py
+++ b/src/pipecat/services/openai_realtime_beta/__init__.py
@@ -1,3 +1,3 @@
 from .azure import AzureRealtimeBetaLLMService
-from .events import InputAudioTranscriptionModels, SessionProperties, TurnDetection
+from .events import InputAudioTranscription, SessionProperties, TurnDetection
 from .openai import OpenAIRealtimeBetaLLMService
diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py
index adfc34133..caa7e964c 100644
--- a/src/pipecat/services/openai_realtime_beta/events.py
+++ b/src/pipecat/services/openai_realtime_beta/events.py
@@ -16,19 +16,8 @@ from pydantic import BaseModel, Field
 #
 
 
-class InputAudioTranscriptionModels:
-    class Whisper1(BaseModel):
-        model: Optional[Literal["whisper-1"]] = "whisper-1"
-
-    class GPT4o(BaseModel):
-        model: Optional[Literal["gpt-4o-transcribe-latest"]] = "gpt-4o-transcribe-latest"
-        language: Optional[str] = None
-        prompt: Optional[str] = None
-
-
-InputAudioTranscription = Union[
-    InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o
-]
+class InputAudioTranscription(BaseModel):
+    model: Optional[str] = "whisper-1"
 
 
 class TurnDetection(BaseModel):

From d009b804383a5dfc70bc3c6fcd655c6d5cde42cd Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Mon, 17 Mar 2025 12:54:32 -0400
Subject: [PATCH 09/28] Add new GPT-4o transcription option to
 OpenAIRealtimeBetaLLMService

---
 .../services/openai_realtime_beta/events.py     | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py
index caa7e964c..fcc330a6e 100644
--- a/src/pipecat/services/openai_realtime_beta/events.py
+++ b/src/pipecat/services/openai_realtime_beta/events.py
@@ -14,10 +14,25 @@ from pydantic import BaseModel, Field
 #
 # session properties
 #
+InputAudioTranscriptionModelArg = Optional[Literal["whisper-1", "gpt-4o-transcribe-latest"]]
 
 
 class InputAudioTranscription(BaseModel):
-    model: Optional[str] = "whisper-1"
+    model: InputAudioTranscriptionModelArg
+    language: Optional[str]
+    prompt: Optional[str]
+
+    def __init__(
+        self,
+        model: InputAudioTranscriptionModelArg = "whisper-1",
+        language: Optional[str] = None,
+        prompt: Optional[str] = None,
+    ):
+        super().__init__(model=model, language=language, prompt=prompt)
+        if self.model != "gpt-4o-transcribe-latest" and (self.language or self.prompt):
+            raise ValueError(
+                "Fields 'language' and 'prompt' are only supported when model is 'gpt-4o-transcribe-latest'"
+            )
 
 
 class TurnDetection(BaseModel):

From 1a20d9bed7c57d993c6a08d0b3ec747839190584 Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Mon, 17 Mar 2025 13:02:55 -0400
Subject: [PATCH 10/28] Add new input_audio_noise_reduction option to
 OpenAIRealtimeBetaLLMService

---
 src/pipecat/services/openai_realtime_beta/__init__.py | 7 ++++++-
 src/pipecat/services/openai_realtime_beta/events.py   | 5 +++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py
index 52b00f6c8..a4c102609 100644
--- a/src/pipecat/services/openai_realtime_beta/__init__.py
+++ b/src/pipecat/services/openai_realtime_beta/__init__.py
@@ -1,3 +1,8 @@
 from .azure import AzureRealtimeBetaLLMService
-from .events import InputAudioTranscription, SessionProperties, TurnDetection
+from .events import (
+    InputAudioTranscription,
+    InputAudioNoiseReduction,
+    SessionProperties,
+    TurnDetection,
+)
 from .openai import OpenAIRealtimeBetaLLMService
diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py
index fcc330a6e..26fbc0578 100644
--- a/src/pipecat/services/openai_realtime_beta/events.py
+++ b/src/pipecat/services/openai_realtime_beta/events.py
@@ -42,6 +42,10 @@ class TurnDetection(BaseModel):
     silence_duration_ms: Optional[int] = 800
 
 
+class InputAudioNoiseReduction(BaseModel):
+    type: Optional[Literal["near_field", "far_field"]]
+
+
 class SessionProperties(BaseModel):
     modalities: Optional[List[Literal["text", "audio"]]] = None
     instructions: Optional[str] = None
@@ -49,6 +53,7 @@ class SessionProperties(BaseModel):
     input_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None
     output_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None
     input_audio_transcription: Optional[InputAudioTranscription] = None
+    input_audio_noise_reduction: Optional[InputAudioNoiseReduction] = None
     # set turn_detection to False to disable turn detection
     turn_detection: Optional[Union[TurnDetection, bool]] = Field(default=None)
     tools: Optional[List[Dict]] = None

From e91610c69eb1439679cdeb369b565f01276633c5 Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Mon, 17 Mar 2025 13:18:52 -0400
Subject: [PATCH 11/28] linter fix

---
 src/pipecat/services/openai_realtime_beta/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py
index a4c102609..d54640919 100644
--- a/src/pipecat/services/openai_realtime_beta/__init__.py
+++ b/src/pipecat/services/openai_realtime_beta/__init__.py
@@ -1,7 +1,7 @@
 from .azure import AzureRealtimeBetaLLMService
 from .events import (
-    InputAudioTranscription,
     InputAudioNoiseReduction,
+    InputAudioTranscription,
     SessionProperties,
     TurnDetection,
 )

From 1075c25055873344f63d7b9f89669aa3c069b1ab Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Mon, 17 Mar 2025 15:28:31 -0400
Subject: [PATCH 12/28] Add new semantic turn detection option to
 OpenAIRealtimeBetaLLMService

---
 src/pipecat/services/openai_realtime_beta/__init__.py |  1 +
 src/pipecat/services/openai_realtime_beta/events.py   | 11 ++++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py
index d54640919..595105d7f 100644
--- a/src/pipecat/services/openai_realtime_beta/__init__.py
+++ b/src/pipecat/services/openai_realtime_beta/__init__.py
@@ -2,6 +2,7 @@ from .azure import AzureRealtimeBetaLLMService
 from .events import (
     InputAudioNoiseReduction,
     InputAudioTranscription,
+    SemanticTurnDetection,
     SessionProperties,
     TurnDetection,
 )
diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py
index 26fbc0578..c8a7c383f 100644
--- a/src/pipecat/services/openai_realtime_beta/events.py
+++ b/src/pipecat/services/openai_realtime_beta/events.py
@@ -42,6 +42,13 @@ class TurnDetection(BaseModel):
     silence_duration_ms: Optional[int] = 800
 
 
+class SemanticTurnDetection(BaseModel):
+    type: Optional[Literal["semantic_vad"]] = "semantic_vad"
+    eagerness: Optional[Literal["low", "medium", "high", "auto"]] = None
+    create_response: Optional[bool] = None
+    interrupt_response: Optional[bool] = None
+
+
 class InputAudioNoiseReduction(BaseModel):
     type: Optional[Literal["near_field", "far_field"]]
 
@@ -55,7 +62,9 @@ class SessionProperties(BaseModel):
     input_audio_transcription: Optional[InputAudioTranscription] = None
     input_audio_noise_reduction: Optional[InputAudioNoiseReduction] = None
     # set turn_detection to False to disable turn detection
-    turn_detection: Optional[Union[TurnDetection, bool]] = Field(default=None)
+    turn_detection: Optional[Union[TurnDetection, SemanticTurnDetection, bool]] = Field(
+        default=None
+    )
     tools: Optional[List[Dict]] = None
     tool_choice: Optional[Literal["auto", "none", "required"]] = None
     temperature: Optional[float] = None

From 9840abd85b828043bed7dfb4dab2c23dee19b546 Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Mon, 17 Mar 2025 14:51:19 -0400
Subject: [PATCH 13/28] Make it so you specifying `model=None` when creating a
 `InputAudioTranscription` results in a validation error

---
 src/pipecat/services/openai_realtime_beta/events.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py
index c8a7c383f..0f42a30b0 100644
--- a/src/pipecat/services/openai_realtime_beta/events.py
+++ b/src/pipecat/services/openai_realtime_beta/events.py
@@ -14,17 +14,17 @@ from pydantic import BaseModel, Field
 #
 # session properties
 #
-InputAudioTranscriptionModelArg = Optional[Literal["whisper-1", "gpt-4o-transcribe-latest"]]
+InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe-latest"]
 
 
 class InputAudioTranscription(BaseModel):
-    model: InputAudioTranscriptionModelArg
+    model: InputAudioTranscriptionModel
     language: Optional[str]
     prompt: Optional[str]
 
     def __init__(
         self,
-        model: InputAudioTranscriptionModelArg = "whisper-1",
+        model: Optional[InputAudioTranscriptionModel] = "whisper-1",
         language: Optional[str] = None,
         prompt: Optional[str] = None,
     ):

From 39ca607bbbe72a481c664c6d90055c23454ec9fc Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Mon, 17 Mar 2025 16:52:53 -0400
Subject: [PATCH 14/28] Add `on_conversation_item_created` and
 `on_conversation_item_updated` events to OpenAIRealtimeBetaLLMService. The
 hope is that this will expose to the user conversation item ids at relevant
 times for them to use with the new `conversation.item.retrieve` introspection
 message.

---
 src/pipecat/services/openai_realtime_beta/openai.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index 68dde4d32..4a3a14cf8 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -117,6 +117,9 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         self._messages_added_manually = {}
         self._user_and_response_message_tuple = None
 
+        self._register_event_handler("on_conversation_item_created")
+        self._register_event_handler("on_conversation_item_updated")
+
     def can_generate_metrics(self) -> bool:
         return True
 
@@ -413,6 +416,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             # receive a BotStoppedSpeakingFrame from the output transport.
 
     async def _handle_evt_conversation_item_created(self, evt):
+        await self._call_event_handler("on_conversation_item_created", evt.item.id, evt.item)
+
         # This will get sent from the server every time a new "message" is added
         # to the server's conversation state, whether we create it via the API
         # or the server creates it from LLM output.
@@ -437,6 +442,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             )
 
     async def handle_evt_input_audio_transcription_completed(self, evt):
+        await self._call_event_handler("on_conversation_item_updated", evt.item_id, None)
+
         if self._send_transcription_frames:
             await self.push_frame(
                 # no way to get a language code?
@@ -473,6 +480,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             )
             return
         # response content
+        for item in evt.response.output:
+            await self._call_event_handler("on_conversation_item_updated", item.id, item)
         pair = self._user_and_response_message_tuple
         if pair:
             user, assistant = pair

From f693a3c70f442ac410d15bed39a0fa9be0f3578b Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Mon, 17 Mar 2025 21:54:56 -0400
Subject: [PATCH 15/28] Add `retrieve_conversation_item()` method to
 OpenAIRealtimeBetaLLMService, using the new `conversation.item.retrieve`
 introspection message.

---
 .../services/openai_realtime_beta/events.py     | 11 +++++++++++
 .../services/openai_realtime_beta/openai.py     | 17 +++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py
index 0f42a30b0..88638b5fd 100644
--- a/src/pipecat/services/openai_realtime_beta/events.py
+++ b/src/pipecat/services/openai_realtime_beta/events.py
@@ -179,6 +179,11 @@ class ConversationItemDeleteEvent(ClientEvent):
     item_id: str
 
 
+class ConversationItemRetrieveEvent(ClientEvent):
+    type: Literal["conversation.item.retrieve"] = "conversation.item.retrieve"
+    item_id: str
+
+
 class ResponseCreateEvent(ClientEvent):
     type: Literal["response.create"] = "response.create"
     response: Optional[ResponseProperties] = None
@@ -255,6 +260,11 @@ class ConversationItemDeleted(ServerEvent):
     item_id: str
 
 
+class ConversationItemRetrieved(ServerEvent):
+    type: Literal["conversation.item.retrieved"]
+    item: ConversationItem
+
+
 class ResponseCreated(ServerEvent):
     type: Literal["response.created"]
     response: "Response"
@@ -441,6 +451,7 @@ _server_event_types = {
     "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed,
     "conversation.item.truncated": ConversationItemTruncated,
     "conversation.item.deleted": ConversationItemDeleted,
+    "conversation.item.retrieved": ConversationItemRetrieved,
     "response.created": ResponseCreated,
     "response.done": ResponseDone,
     "response.output_item.added": ResponseOutputItemAdded,
diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index 4a3a14cf8..922f6f2ff 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -119,6 +119,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
 
         self._register_event_handler("on_conversation_item_created")
         self._register_event_handler("on_conversation_item_updated")
+        self._retrieve_conversation_item_futures = {}
 
     def can_generate_metrics(self) -> bool:
         return True
@@ -126,6 +127,12 @@ class OpenAIRealtimeBetaLLMService(LLMService):
     def set_audio_input_paused(self, paused: bool):
         self._audio_input_paused = paused
 
+    async def retrieve_conversation_item(self, item_id: str):
+        future = self.get_event_loop().create_future()
+        self._retrieve_conversation_item_futures[item_id] = future
+        await self.send_client_event(events.ConversationItemRetrieveEvent(item_id=item_id))
+        return await future
+
     #
     # standard AIService frame handling
     #
@@ -363,6 +370,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
                 await self._handle_evt_input_audio_transcription_delta(evt)
             elif evt.type == "conversation.item.input_audio_transcription.completed":
                 await self.handle_evt_input_audio_transcription_completed(evt)
+            elif evt.type == "conversation.item.retrieved":
+                await self._handle_conversation_item_retrieved(evt)
             elif evt.type == "response.done":
                 await self._handle_evt_response_done(evt)
             elif evt.type == "input_audio_buffer.speech_started":
@@ -461,6 +470,14 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             # User message without preceding conversation.item.created. Bug?
             logger.warning(f"Transcript for unknown user message: {evt}")
 
+    async def _handle_conversation_item_retrieved(self, evt: events.ConversationItemRetrieved):
+        future = self._retrieve_conversation_item_futures.get(evt.item.id)
+        if future:
+            # print(f"[pk] setting result: {evt.item}")
+            future.set_result(evt.item)
+            # TODO: handle error
+            # TODO: what happens if we try to receive bogus item id?
+
     async def _handle_evt_response_done(self, evt):
         # todo: figure out whether there's anything we need to do for "cancelled" events
         # usage metrics

From 31317ce77d51abd060556b67a02a44c1fa5870ae Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Tue, 18 Mar 2025 09:42:20 -0400
Subject: [PATCH 16/28] Add error handling to the
 `retrieve_conversation_item()` method of the OpenAIRealtimeBetaLLMService

---
 .../services/openai_realtime_beta/openai.py   | 31 +++++++++++++++----
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index 922f6f2ff..52a03ba94 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -23,6 +23,8 @@ except ModuleNotFoundError as e:
     )
     raise Exception(f"Missing module: {e}")
 
+import re
+
 from pipecat.frames.frames import (
     BotStoppedSpeakingFrame,
     CancelFrame,
@@ -381,9 +383,10 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             elif evt.type == "response.audio_transcript.delta":
                 await self._handle_evt_audio_transcript_delta(evt)
             elif evt.type == "error":
-                await self._handle_evt_error(evt)
-                # errors are fatal, so exit the receive loop
-                return
+                if not await self._maybe_handle_evt_retrieve_conversation_item_error(evt):
+                    await self._handle_evt_error(evt)
+                    # errors are fatal, so exit the receive loop
+                    return
 
     async def _handle_evt_session_created(self, evt):
         # session.created is received right after connecting. Send a message
@@ -471,12 +474,10 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             logger.warning(f"Transcript for unknown user message: {evt}")
 
     async def _handle_conversation_item_retrieved(self, evt: events.ConversationItemRetrieved):
-        future = self._retrieve_conversation_item_futures.get(evt.item.id)
+        future = self._retrieve_conversation_item_futures.pop(evt.item.id, None)
         if future:
             # print(f"[pk] setting result: {evt.item}")
             future.set_result(evt.item)
-            # TODO: handle error
-            # TODO: what happens if we try to receive bogus item id?
 
     async def _handle_evt_response_done(self, evt):
         # todo: figure out whether there's anything we need to do for "cancelled" events
@@ -530,6 +531,24 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         await self.push_frame(StopInterruptionFrame())
         await self.push_frame(UserStoppedSpeakingFrame())
 
+    async def _maybe_handle_evt_retrieve_conversation_item_error(self, evt: events.ErrorEvent):
+        """If the given error event is an error retrieving a conversation item:
+        - set an exception on the future that retrieve_conversation_item() is waiting on
+        - return true
+        Otherwise:
+        - return false
+        """
+        match = re.match(
+            r"^Error retrieving item: the item with id '(.*)' does not exist\.$", evt.error.message
+        )
+        if match:
+            item_id = match.group(1)
+            future = self._retrieve_conversation_item_futures.pop(item_id, None)
+            if future:
+                future.set_exception(Exception(evt.error.message))
+            return True
+        return False
+
     async def _handle_evt_error(self, evt):
         # Errors are fatal to this connection. Send an ErrorFrame.
         await self.push_error(ErrorFrame(error=f"Error: {evt}", fatal=True))

From 7b594093ddb7b586c0c3c521f082bfee2a362a64 Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Tue, 18 Mar 2025 10:05:41 -0400
Subject: [PATCH 17/28] Handle the possibility of multiple concurrent calls to
 `retrieve_conversation_item()` in the OpenAIRealtimeBetaLLMService

---
 .../services/openai_realtime_beta/openai.py   | 25 ++++++++++++-------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index 52a03ba94..4b993c28a 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -131,8 +131,14 @@ class OpenAIRealtimeBetaLLMService(LLMService):
 
     async def retrieve_conversation_item(self, item_id: str):
         future = self.get_event_loop().create_future()
-        self._retrieve_conversation_item_futures[item_id] = future
-        await self.send_client_event(events.ConversationItemRetrieveEvent(item_id=item_id))
+        retrieval_in_progress = False
+        if not self._retrieve_conversation_item_futures.get(item_id):
+            self._retrieve_conversation_item_futures[item_id] = []
+        else:
+            retrieval_in_progress = True
+        self._retrieve_conversation_item_futures[item_id].append(future)
+        if not retrieval_in_progress:
+            await self.send_client_event(events.ConversationItemRetrieveEvent(item_id=item_id))
         return await future
 
     #
@@ -474,10 +480,10 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             logger.warning(f"Transcript for unknown user message: {evt}")
 
     async def _handle_conversation_item_retrieved(self, evt: events.ConversationItemRetrieved):
-        future = self._retrieve_conversation_item_futures.pop(evt.item.id, None)
-        if future:
-            # print(f"[pk] setting result: {evt.item}")
-            future.set_result(evt.item)
+        futures = self._retrieve_conversation_item_futures.pop(evt.item.id, None)
+        if futures:
+            for future in futures:
+                future.set_result(evt.item)
 
     async def _handle_evt_response_done(self, evt):
         # todo: figure out whether there's anything we need to do for "cancelled" events
@@ -543,9 +549,10 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         )
         if match:
             item_id = match.group(1)
-            future = self._retrieve_conversation_item_futures.pop(item_id, None)
-            if future:
-                future.set_exception(Exception(evt.error.message))
+            futures = self._retrieve_conversation_item_futures.pop(item_id, None)
+            if futures:
+                for future in futures:
+                    future.set_exception(Exception(evt.error.message))
             return True
         return False
 

From e707efbffa84cf77dab967fa3c58d5ac2fa68a6f Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Tue, 18 Mar 2025 12:30:19 -0400
Subject: [PATCH 18/28] Update changelog with slate of recent updates to
 OpenAIRealtimeBetaLLMService

---
 CHANGELOG.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6d55b97c3..42fad1c44 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -128,8 +128,60 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   Gemini models. Added foundational example
   `14p-function-calling-gemini-vertex-ai.py`.
 
+- Added support in `OpenAIRealtimeBetaLLMService` for the
+  `conversation.item.input_audio_transcription.delta` server message.
+
+- Added error handling in `OpenAIRealtimeBetaLLMService` for the
+  `response.done` server message reporting a failure.
+
+- Added support in `OpenAIRealtimeBetaLLMService` for the new
+  `gpt-4o-transcribe-latest` input audio transcription model.
+
+- Added support in `OpenAIRealtimeBetaLLMService` for the new
+  `input_audio_noise_reduction` session property.
+
+  ```python
+  session_properties = SessionProperties(
+    # ...
+    input_audio_noise_reduction=InputAudioNoiseReduction(
+      type="near_field" # also supported: "far_field"
+    )
+    # ...
+  )
+  ```
+
+- Added support in `OpenAIRealtimeBetaLLMService` for the new
+  `semantic_vad` `turn_detection` session property, which is a more
+  sophisticated model for detecting when the user has stopped speaking.
+
+- Added `on_conversation_item_created` and `on_conversation_item_updated`
+  events to `OpenAIRealtimeBetaLLMService`.
+
+  ```python
+  @llm.event_handler("on_conversation_item_created")
+  async def on_conversation_item_created(llm, item_id, item):
+    # ...
+
+  @llm.event_handler("on_conversation_item_updated")
+  async def on_conversation_item_updated(llm, item_id, item):
+    # `item` may not always be available here
+    # ...
+  ```
+
+- Added `retrieve_conversation_item(item_id)` to `OpenAIRealtimeBetaLLMService`.
+
+  ```python
+  item = await llm.retrieve_conversation_item(item_id)
+  ```
+
 ### Changed
 
+- Updated the default model for `CartesiaTTSService` and
+  `CartesiaHttpTTSService` to `sonic-2`.
+
+- Updated the default model for `OpenAIRealtimeBetaLLMService` to
+  `gpt-4o-realtime-preview-latest`.
+
 - Function calls are now executed in tasks. This means that the pipeline will
   not be blocked while the function call is being executed.
 
@@ -216,6 +268,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed an issue in `RimeTTSService` where the last line of text sent didn't
   result in an audio output being generated.
 
+- Fixed `OpenAIRealtimeBetaLLMService` by adding support for the
+  `conversation.item.input_audio_transcription.delta` server message, which was
+  added server-side at some point and not handled client-side.
+
 ### Other
 
 - Add foundational example `07w-interruptible-fal.py`, showing `FalSTTService`.

From 3dd4ef72303c45c439eebcfd9a4e7c1d1a955047 Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Tue, 18 Mar 2025 15:52:11 -0400
Subject: [PATCH 19/28] Tweak changelog entries describing slate of recent
 updates to OpenAIRealtimeBetaLLMService

---
 CHANGELOG.md | 73 ++++++++++++++++++++++++----------------------------
 1 file changed, 33 insertions(+), 40 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 42fad1c44..fe531bf7f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -128,51 +128,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   Gemini models. Added foundational example
   `14p-function-calling-gemini-vertex-ai.py`.
 
-- Added support in `OpenAIRealtimeBetaLLMService` for the
-  `conversation.item.input_audio_transcription.delta` server message.
+- Added support in `OpenAIRealtimeBetaLLMService` for a slate of new features:
 
-- Added error handling in `OpenAIRealtimeBetaLLMService` for the
-  `response.done` server message reporting a failure.
+  - The `'gpt-4o-transcribe-latest'` input audio transcription model.
+  - The `input_audio_noise_reduction` session property.
 
-- Added support in `OpenAIRealtimeBetaLLMService` for the new
-  `gpt-4o-transcribe-latest` input audio transcription model.
-
-- Added support in `OpenAIRealtimeBetaLLMService` for the new
-  `input_audio_noise_reduction` session property.
-
-  ```python
-  session_properties = SessionProperties(
-    # ...
-    input_audio_noise_reduction=InputAudioNoiseReduction(
-      type="near_field" # also supported: "far_field"
+    ```python
+    session_properties = SessionProperties(
+      # ...
+      input_audio_noise_reduction=InputAudioNoiseReduction(
+        type="near_field" # also supported: "far_field"
+      )
+      # ...
     )
-    # ...
-  )
-  ```
+    ```
 
-- Added support in `OpenAIRealtimeBetaLLMService` for the new
-  `semantic_vad` `turn_detection` session property, which is a more
-  sophisticated model for detecting when the user has stopped speaking.
+  - The `'semantic_vad'` `turn_detection` session property value, a more
+    sophisticated model for detecting when the user has stopped speaking.
+  - `on_conversation_item_created` and `on_conversation_item_updated`
+    events to `OpenAIRealtimeBetaLLMService`.
 
-- Added `on_conversation_item_created` and `on_conversation_item_updated`
-  events to `OpenAIRealtimeBetaLLMService`.
+    ```python
+    @llm.event_handler("on_conversation_item_created")
+    async def on_conversation_item_created(llm, item_id, item):
+      # ...
 
-  ```python
-  @llm.event_handler("on_conversation_item_created")
-  async def on_conversation_item_created(llm, item_id, item):
-    # ...
+    @llm.event_handler("on_conversation_item_updated")
+    async def on_conversation_item_updated(llm, item_id, item):
+      # `item` may not always be available here
+      # ...
+    ```
 
-  @llm.event_handler("on_conversation_item_updated")
-  async def on_conversation_item_updated(llm, item_id, item):
-    # `item` may not always be available here
-    # ...
-  ```
+  - The `retrieve_conversation_item(item_id)` method for introspecting a
+    conversation item on the server.
 
-- Added `retrieve_conversation_item(item_id)` to `OpenAIRealtimeBetaLLMService`.
-
-  ```python
-  item = await llm.retrieve_conversation_item(item_id)
-  ```
+    ```python
+    item = await llm.retrieve_conversation_item(item_id)
+    ```
 
 ### Changed
 
@@ -268,9 +260,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed an issue in `RimeTTSService` where the last line of text sent didn't
   result in an audio output being generated.
 
-- Fixed `OpenAIRealtimeBetaLLMService` by adding support for the
-  `conversation.item.input_audio_transcription.delta` server message, which was
-  added server-side at some point and not handled client-side.
+- Fixed `OpenAIRealtimeBetaLLMService` by adding proper handling for:
+  - The `conversation.item.input_audio_transcription.delta` server message,
+    which was added server-side at some point and not handled client-side.
+  - Errors reported by the `response.done` server message.
 
 ### Other
 

From f94a099111a7b23952be85e7b0d13e9a09158117 Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Wed, 19 Mar 2025 12:00:19 -0400
Subject: [PATCH 20/28] Revert the default model to be
 "gpt-4o-realtime-preview-2024-12-17" In OpenAIRealtimeBetaLLMService

---
 CHANGELOG.md                                        | 6 ++----
 src/pipecat/services/openai_realtime_beta/openai.py | 3 +--
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fe531bf7f..660cf5c82 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -130,7 +130,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Added support in `OpenAIRealtimeBetaLLMService` for a slate of new features:
 
-  - The `'gpt-4o-transcribe-latest'` input audio transcription model.
+  - The `'gpt-4o-transcribe-latest'` input audio transcription model, along
+    with new `language` and `prompt` options specific to that model.
   - The `input_audio_noise_reduction` session property.
 
     ```python
@@ -171,9 +172,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Updated the default model for `CartesiaTTSService` and
   `CartesiaHttpTTSService` to `sonic-2`.
 
-- Updated the default model for `OpenAIRealtimeBetaLLMService` to
-  `gpt-4o-realtime-preview-latest`.
-
 - Function calls are now executed in tasks. This means that the pipeline will
   not be blocked while the function call is being executed.
 
diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index 4b993c28a..ab78a4451 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -89,8 +89,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         self,
         *,
         api_key: str,
-        # model: str = "gpt-4o-realtime-preview-2024-12-17",
-        model: str = "gpt-4o-realtime-preview-latest",
+        model: str = "gpt-4o-realtime-preview-2024-12-17",
         base_url: str = "wss://api.openai.com/v1/realtime",
         session_properties: events.SessionProperties = events.SessionProperties(),
         start_audio_paused: bool = False,

From 0d74bcacb74388724e4b36bcf764869813d3698d Mon Sep 17 00:00:00 2001
From: Chad Bailey <chadbailey@gmail.com>
Date: Thu, 20 Mar 2025 01:12:13 +0000
Subject: [PATCH 21/28] updated models in the 07g example

---
 examples/foundational/07g-interruptible-openai.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py
index f9cac5910..fe4b2a53c 100644
--- a/examples/foundational/07g-interruptible-openai.py
+++ b/examples/foundational/07g-interruptible-openai.py
@@ -51,9 +51,11 @@ async def main():
         #     api_key="gsk_***",
         #     model="whisper-large-v3",
         # )
-        stt = OpenAISTTService(api_key=os.getenv("OPENAI_API_KEY"), model="whisper-1")
+        stt = OpenAISTTService(
+            api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-transcribe-latest"
+        )
 
-        tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY"), voice="alloy")
+        tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini-tts-latest")
 
         llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
 

From 2ae5bdd8a96fe3f3333814846e8863758b4ed9f0 Mon Sep 17 00:00:00 2001
From: Chad Bailey <chadbailey@gmail.com>
Date: Thu, 20 Mar 2025 01:20:07 +0000
Subject: [PATCH 22/28] lets talk about dogs

---
 examples/foundational/07g-interruptible-openai.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py
index fe4b2a53c..b66d0346a 100644
--- a/examples/foundational/07g-interruptible-openai.py
+++ b/examples/foundational/07g-interruptible-openai.py
@@ -52,7 +52,9 @@ async def main():
         #     model="whisper-large-v3",
         # )
         stt = OpenAISTTService(
-            api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-transcribe-latest"
+            api_key=os.getenv("OPENAI_API_KEY"),
+            model="gpt-4o-transcribe-latest",
+            prompt="Expect words related to dogs, such as breed names.",
         )
 
         tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini-tts-latest")
@@ -62,7 +64,7 @@ async def main():
         messages = [
             {
                 "role": "system",
-                "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+                "content": "You are very knowledgable about dogs. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
             },
         ]
 

From f0774268cc83145da2b92786b7646a386c801fe9 Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Wed, 19 Mar 2025 21:58:46 -0400
Subject: [PATCH 23/28] Rename gpt-4o-transcribe-latest to gpt-4o-transcribe in
 OpenAIRealtimeBetaLLMService

---
 CHANGELOG.md                                        | 2 +-
 src/pipecat/services/openai_realtime_beta/events.py | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 660cf5c82..6825a62f5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -130,7 +130,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Added support in `OpenAIRealtimeBetaLLMService` for a slate of new features:
 
-  - The `'gpt-4o-transcribe-latest'` input audio transcription model, along
+  - The `'gpt-4o-transcribe'` input audio transcription model, along
     with new `language` and `prompt` options specific to that model.
   - The `input_audio_noise_reduction` session property.
 
diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py
index 88638b5fd..3a18833d2 100644
--- a/src/pipecat/services/openai_realtime_beta/events.py
+++ b/src/pipecat/services/openai_realtime_beta/events.py
@@ -14,7 +14,7 @@ from pydantic import BaseModel, Field
 #
 # session properties
 #
-InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe-latest"]
+InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe"]
 
 
 class InputAudioTranscription(BaseModel):
@@ -29,9 +29,9 @@ class InputAudioTranscription(BaseModel):
         prompt: Optional[str] = None,
     ):
         super().__init__(model=model, language=language, prompt=prompt)
-        if self.model != "gpt-4o-transcribe-latest" and (self.language or self.prompt):
+        if self.model != "gpt-4o-transcribe" and (self.language or self.prompt):
             raise ValueError(
-                "Fields 'language' and 'prompt' are only supported when model is 'gpt-4o-transcribe-latest'"
+                "Fields 'language' and 'prompt' are only supported when model is 'gpt-4o-transcribe'"
             )
 
 

From 70dbf0d6fc952e594b09506744eca9c57142172c Mon Sep 17 00:00:00 2001
From: Mark Backman <mark@daily.co>
Date: Thu, 20 Mar 2025 11:48:06 -0400
Subject: [PATCH 24/28] Updated default models for OpenAISTTService and
 OpenAITTSService to gpt-4o based models

---
 CHANGELOG.md                   |  5 +++++
 src/pipecat/services/openai.py | 12 ++++++------
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6825a62f5..ac0a6ef3c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -169,6 +169,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Updated `OpenAISTTService` to use `gpt-4o-transcribe` as the default
+  transcription model.
+
+- Updated `OpenAITTSService` to use `gpt-4o-mini-tts` as the default TTS model.
+
 - Updated the default model for `CartesiaTTSService` and
   `CartesiaHttpTTSService` to `sonic-2`.
 
diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py
index b5f5d9d54..4d3e1d6d7 100644
--- a/src/pipecat/services/openai.py
+++ b/src/pipecat/services/openai.py
@@ -409,13 +409,13 @@ class OpenAIImageGenService(ImageGenService):
 
 
 class OpenAISTTService(BaseWhisperSTTService):
-    """OpenAI Whisper speech-to-text service.
+    """OpenAI Speech-to-Text service that generates text from audio.
 
-    Uses OpenAI's Whisper API to convert audio to text. Requires an OpenAI API key
+    Uses OpenAI's transcription API to convert audio to text. Requires an OpenAI API key
     set via the api_key parameter or OPENAI_API_KEY environment variable.
 
     Args:
-        model: Whisper model to use. Defaults to "whisper-1".
+        model: Model to use — either gpt-4o or Whisper. Defaults to "gpt-4o-transcribe".
         api_key: OpenAI API key. Defaults to None.
         base_url: API base URL. Defaults to None.
         language: Language of the audio input. Defaults to English.
@@ -427,7 +427,7 @@ class OpenAISTTService(BaseWhisperSTTService):
     def __init__(
         self,
         *,
-        model: str = "whisper-1",
+        model: str = "gpt-4o-transcribe",
         api_key: Optional[str] = None,
         base_url: Optional[str] = None,
         language: Optional[Language] = Language.EN,
@@ -472,7 +472,7 @@ class OpenAITTSService(TTSService):
     Args:
         api_key: OpenAI API key. Defaults to None.
         voice: Voice ID to use. Defaults to "alloy".
-        model: TTS model to use. Defaults to "tts-1".
+        model: TTS model to use. Defaults to "gpt-4o-mini-tts".
         sample_rate: Output audio sample rate in Hz. Defaults to None.
         **kwargs: Additional keyword arguments passed to TTSService.
 
@@ -487,7 +487,7 @@ class OpenAITTSService(TTSService):
         *,
         api_key: Optional[str] = None,
         voice: str = "alloy",
-        model: str = "tts-1",
+        model: str = "gpt-4o-mini-tts",
         sample_rate: Optional[int] = None,
         **kwargs,
     ):

From ada68f0699e09323f5d05b0ff5ce461754437f4e Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Wed, 19 Mar 2025 22:25:21 -0400
Subject: [PATCH 25/28] More robust handling of conversation item retrieval
 errors in OpenAIRealtimeBetaLLMService

---
 .../services/openai_realtime_beta/events.py   |  1 +
 .../services/openai_realtime_beta/openai.py   | 23 ++++++++++---------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py
index 3a18833d2..f4133766b 100644
--- a/src/pipecat/services/openai_realtime_beta/events.py
+++ b/src/pipecat/services/openai_realtime_beta/events.py
@@ -122,6 +122,7 @@ class RealtimeError(BaseModel):
     code: Optional[str] = ""
     message: str
     param: Optional[str] = None
+    event_id: Optional[str] = None
 
 
 #
diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index ab78a4451..324e5653a 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -23,8 +23,6 @@ except ModuleNotFoundError as e:
     )
     raise Exception(f"Missing module: {e}")
 
-import re
-
 from pipecat.frames.frames import (
     BotStoppedSpeakingFrame,
     CancelFrame,
@@ -130,14 +128,20 @@ class OpenAIRealtimeBetaLLMService(LLMService):
 
     async def retrieve_conversation_item(self, item_id: str):
         future = self.get_event_loop().create_future()
-        retrieval_in_progress = False
+        retrieval_in_flight = False
         if not self._retrieve_conversation_item_futures.get(item_id):
             self._retrieve_conversation_item_futures[item_id] = []
         else:
-            retrieval_in_progress = True
+            retrieval_in_flight = True
         self._retrieve_conversation_item_futures[item_id].append(future)
-        if not retrieval_in_progress:
-            await self.send_client_event(events.ConversationItemRetrieveEvent(item_id=item_id))
+        if not retrieval_in_flight:
+            await self.send_client_event(
+                # Set event_id to "rci_{item_id}" so that we can identiy an
+                # error later if the retrieval fails. We don't need a UUID
+                # suffix to the event_id because we're ensuring only one
+                # in-flight retrieval per item_id
+                events.ConversationItemRetrieveEvent(item_id=item_id, event_id=f"rci_{item_id}")
+            )
         return await future
 
     #
@@ -543,11 +547,8 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         Otherwise:
         - return false
         """
-        match = re.match(
-            r"^Error retrieving item: the item with id '(.*)' does not exist\.$", evt.error.message
-        )
-        if match:
-            item_id = match.group(1)
+        if evt.error.code == "item_retrieve_invalid_item_id":
+            item_id = evt.error.event_id.split("_", 1)[1]  # event_id is of the form "rci_{item_id}"
             futures = self._retrieve_conversation_item_futures.pop(item_id, None)
             if futures:
                 for future in futures:

From 721ee75887441827c46819b4efff4e4895df93c1 Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Thu, 20 Mar 2025 12:41:40 -0400
Subject: [PATCH 26/28] Comment tweak

---
 src/pipecat/services/openai_realtime_beta/openai.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index 324e5653a..c8f1f597a 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -136,10 +136,11 @@ class OpenAIRealtimeBetaLLMService(LLMService):
         self._retrieve_conversation_item_futures[item_id].append(future)
         if not retrieval_in_flight:
             await self.send_client_event(
-                # Set event_id to "rci_{item_id}" so that we can identiy an
+                # Set event_id to "rci_{item_id}" so that we can identify an
                 # error later if the retrieval fails. We don't need a UUID
                 # suffix to the event_id because we're ensuring only one
-                # in-flight retrieval per item_id
+                # in-flight retrieval per item_id. (Note: "rci" = "retrieve
+                # conversation item")
                 events.ConversationItemRetrieveEvent(item_id=item_id, event_id=f"rci_{item_id}")
             )
         return await future

From 44380bc8c0370dffe9685b3542f679b5d2b4ada0 Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Thu, 20 Mar 2025 13:51:16 -0400
Subject: [PATCH 27/28] Remove duplicate changelog entry due to rebase mistake

---
 CHANGELOG.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ac0a6ef3c..065a5ced6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -174,9 +174,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Updated `OpenAITTSService` to use `gpt-4o-mini-tts` as the default TTS model.
 
-- Updated the default model for `CartesiaTTSService` and
-  `CartesiaHttpTTSService` to `sonic-2`.
-
 - Function calls are now executed in tasks. This means that the pipeline will
   not be blocked while the function call is being executed.
 

From efa5f133d747783c43c11ff5b1aad929a1f01484 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= <aleix@daily.co>
Date: Thu, 20 Mar 2025 11:14:59 -0700
Subject: [PATCH 28/28] openai_realtime: fix and update function calling

---
 .../services/openai_realtime_beta/context.py  | 72 +++----------------
 .../services/openai_realtime_beta/openai.py   |  4 +-
 2 files changed, 11 insertions(+), 65 deletions(-)

diff --git a/src/pipecat/services/openai_realtime_beta/context.py b/src/pipecat/services/openai_realtime_beta/context.py
index 31639dc6b..c8381976f 100644
--- a/src/pipecat/services/openai_realtime_beta/context.py
+++ b/src/pipecat/services/openai_realtime_beta/context.py
@@ -12,6 +12,7 @@ from loguru import logger
 
 from pipecat.frames.frames import (
     Frame,
+    FunctionCallResultFrame,
     FunctionCallResultProperties,
     LLMMessagesUpdateFrame,
     LLMSetToolsFrame,
@@ -174,67 +175,12 @@ class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator):
 
 
 class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator):
-    async def push_aggregation(self):
-        # the only thing we implement here is function calling. in all other cases, messages
-        # are added to the context when we receive openai realtime api events
-        if not self._function_call_result:
-            return
+    async def handle_function_call_result(self, frame: FunctionCallResultFrame):
+        await super().handle_function_call_result(frame)
 
-        properties: Optional[FunctionCallResultProperties] = None
-
-        self.reset()
-        try:
-            run_llm = True
-            frame = self._function_call_result
-            properties = frame.properties
-            self._function_call_result = None
-            if frame.result:
-                # The "tool_call" message from the LLM that triggered the function call
-                self._context.add_message(
-                    {
-                        "role": "assistant",
-                        "tool_calls": [
-                            {
-                                "id": frame.tool_call_id,
-                                "function": {
-                                    "name": frame.function_name,
-                                    "arguments": json.dumps(frame.arguments),
-                                },
-                                "type": "function",
-                            }
-                        ],
-                    }
-                )
-                # The result of the function call. Need to add this both to our context here and to
-                # the openai realtime api context.
-                result_message = {
-                    "role": "tool",
-                    "content": json.dumps(frame.result),
-                    "tool_call_id": frame.tool_call_id,
-                }
-
-                self._context.add_message(result_message)
-                # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself,
-                # so we didn't have a chance to add the result to the openai realtime api context. Let's push a
-                # special frame to do that.
-                await self.push_frame(
-                    RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM
-                )
-                if properties and properties.run_llm is not None:
-                    # If the tool call result has a run_llm property, use it
-                    run_llm = properties.run_llm
-                else:
-                    # Default behavior is to run the LLM if there are no function calls in progress
-                    run_llm = not bool(self._function_calls_in_progress)
-
-            if run_llm:
-                await self.push_context_frame(FrameDirection.UPSTREAM)
-
-            # Emit the on_context_updated callback once the function call result is added to the context
-            if properties and properties.on_context_updated is not None:
-                await properties.on_context_updated()
-
-            await self.push_context_frame()
-
-        except Exception as e:
-            logger.error(f"Error processing frame: {e}")
+        # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself,
+        # so we didn't have a chance to add the result to the openai realtime api context. Let's push a
+        # special frame to do that.
+        await self.push_frame(
+            RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM
+        )
diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py
index c8f1f597a..a1f4bc731 100644
--- a/src/pipecat/services/openai_realtime_beta/openai.py
+++ b/src/pipecat/services/openai_realtime_beta/openai.py
@@ -579,7 +579,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
             arguments = json.loads(item.arguments)
             if self.has_function(function_name):
                 run_llm = index == total_items - 1
-                if function_name in self._callbacks.keys():
+                if function_name in self._functions.keys():
                     await self.call_function(
                         context=self._context,
                         tool_call_id=tool_id,
@@ -587,7 +587,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
                         arguments=arguments,
                         run_llm=run_llm,
                     )
-                elif None in self._callbacks.keys():
+                elif None in self._functions.keys():
                     await self.call_function(
                         context=self._context,
                         tool_call_id=tool_id,