Add new GPT-4o transcription option to OpenAIRealtimeBetaLLMService

examples/foundational/19-openai-realtime-beta.py

@@ -21,7 +21,7 @@ from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai_realtime_beta import (
-    InputAudioTranscription,
+    InputAudioTranscriptionModels,
     OpenAIRealtimeBetaLLMService,
     SessionProperties,
     TurnDetection,
@@ -89,7 +89,7 @@ async def main():
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscription(),
+            input_audio_transcription=InputAudioTranscriptionModels.Whisper1(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
             turn_detection=TurnDetection(silence_duration_ms=1000),

examples/foundational/19a-azure-realtime-beta.py

@@ -23,7 +23,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai_realtime_beta import (
     AzureRealtimeBetaLLMService,
-    InputAudioTranscription,
+    InputAudioTranscriptionModels,
     SessionProperties,
     TurnDetection,
 )
@@ -90,7 +90,7 @@ async def main():
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscription(),
+            input_audio_transcription=InputAudioTranscriptionModels.Whisper1(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
             # turn_detection=TurnDetection(silence_duration_ms=1000),

examples/foundational/20b-persistent-context-openai-realtime.py

@@ -25,7 +25,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
     OpenAILLMContext,
 )
 from pipecat.services.openai_realtime_beta import (
-    InputAudioTranscription,
+    InputAudioTranscriptionModels,
     OpenAIRealtimeBetaLLMService,
     SessionProperties,
     TurnDetection,
@@ -186,7 +186,7 @@ async def main():
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscription(),
+            input_audio_transcription=InputAudioTranscriptionModels.Whisper1(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
             turn_detection=TurnDetection(silence_duration_ms=1000),

src/pipecat/services/openai_realtime_beta/__init__.py

@@ -1,3 +1,3 @@
 from .azure import AzureRealtimeBetaLLMService
-from .events import InputAudioTranscription, SessionProperties, TurnDetection
+from .events import InputAudioTranscriptionModels, SessionProperties, TurnDetection
 from .openai import OpenAIRealtimeBetaLLMService

src/pipecat/services/openai_realtime_beta/events.py

@@ -16,8 +16,17 @@ from pydantic import BaseModel, Field
 #
 
 
-class InputAudioTranscription(BaseModel):
+class InputAudioTranscriptionModels:
-    model: Optional[str] = "whisper-1"
+    class Whisper1(BaseModel):
+        model: Optional[Literal["whisper-1"]] = "whisper-1"
+
+    class GPT4o(BaseModel):
+        model: Optional[Literal["gpt-4o-transcribe-latest"]] = "gpt-4o-transcribe-latest"
+        language: Optional[str] = None
+        prompt: Optional[str] = None
+
+
+InputAudioTranscription = Union[InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o]
 
 
 class TurnDetection(BaseModel):