From e80bfe22de734d9449b53fd6d0224159fd8cc6ab Mon Sep 17 00:00:00 2001
From: Paul Kompfner <paul@daily.co>
Date: Mon, 17 Mar 2025 12:15:14 -0400
Subject: [PATCH] Add new GPT-4o transcription option to
 OpenAIRealtimeBetaLLMService

---
 examples/foundational/19-openai-realtime-beta.py    |  4 ++--
 examples/foundational/19a-azure-realtime-beta.py    |  4 ++--
 .../20b-persistent-context-openai-realtime.py       |  4 ++--
 .../services/openai_realtime_beta/__init__.py       |  2 +-
 src/pipecat/services/openai_realtime_beta/events.py | 13 +++++++++++--
 5 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py
index 8796e0141..c39c0f570 100644
--- a/examples/foundational/19-openai-realtime-beta.py
+++ b/examples/foundational/19-openai-realtime-beta.py
@@ -21,7 +21,7 @@ from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai_realtime_beta import (
-    InputAudioTranscription,
+    InputAudioTranscriptionModels,
     OpenAIRealtimeBetaLLMService,
     SessionProperties,
     TurnDetection,
@@ -89,7 +89,7 @@ async def main():
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscription(),
+            input_audio_transcription=InputAudioTranscriptionModels.Whisper1(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
             turn_detection=TurnDetection(silence_duration_ms=1000),
diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py
index 14d034836..12847ee33 100644
--- a/examples/foundational/19a-azure-realtime-beta.py
+++ b/examples/foundational/19a-azure-realtime-beta.py
@@ -23,7 +23,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai_realtime_beta import (
     AzureRealtimeBetaLLMService,
-    InputAudioTranscription,
+    InputAudioTranscriptionModels,
     SessionProperties,
     TurnDetection,
 )
@@ -90,7 +90,7 @@ async def main():
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscription(),
+            input_audio_transcription=InputAudioTranscriptionModels.Whisper1(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
             # turn_detection=TurnDetection(silence_duration_ms=1000),
diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py
index ef82bd567..26a6448e9 100644
--- a/examples/foundational/20b-persistent-context-openai-realtime.py
+++ b/examples/foundational/20b-persistent-context-openai-realtime.py
@@ -25,7 +25,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
     OpenAILLMContext,
 )
 from pipecat.services.openai_realtime_beta import (
-    InputAudioTranscription,
+    InputAudioTranscriptionModels,
     OpenAIRealtimeBetaLLMService,
     SessionProperties,
     TurnDetection,
@@ -186,7 +186,7 @@ async def main():
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscription(),
+            input_audio_transcription=InputAudioTranscriptionModels.Whisper1(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
             turn_detection=TurnDetection(silence_duration_ms=1000),
diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py
index 52b00f6c8..fdda648c4 100644
--- a/src/pipecat/services/openai_realtime_beta/__init__.py
+++ b/src/pipecat/services/openai_realtime_beta/__init__.py
@@ -1,3 +1,3 @@
 from .azure import AzureRealtimeBetaLLMService
-from .events import InputAudioTranscription, SessionProperties, TurnDetection
+from .events import InputAudioTranscriptionModels, SessionProperties, TurnDetection
 from .openai import OpenAIRealtimeBetaLLMService
diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py
index caa7e964c..9cc946807 100644
--- a/src/pipecat/services/openai_realtime_beta/events.py
+++ b/src/pipecat/services/openai_realtime_beta/events.py
@@ -16,8 +16,17 @@ from pydantic import BaseModel, Field
 #
 
 
-class InputAudioTranscription(BaseModel):
-    model: Optional[str] = "whisper-1"
+class InputAudioTranscriptionModels:
+    class Whisper1(BaseModel):
+        model: Optional[Literal["whisper-1"]] = "whisper-1"
+
+    class GPT4o(BaseModel):
+        model: Optional[Literal["gpt-4o-transcribe-latest"]] = "gpt-4o-transcribe-latest"
+        language: Optional[str] = None
+        prompt: Optional[str] = None
+
+
+InputAudioTranscription = Union[InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o]
 
 
 class TurnDetection(BaseModel):