Merge pull request #1417 from pipecat-ai/mb/update-realtime-transcription

Update InputAudioTranscription to use gpt-4o-transcribe model, update…

CHANGELOG.md

@@ -11,6 +11,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Added `default_headers` parameter to `BaseOpenAILLMService` constructor.
 
+### Changed
+
+- Changed the default `InputAudioTranscription` model to `gpt-4o-transcribe`
+  for `OpenAIRealtimeBetaLLMService`.
+
+### Other
+
+- Update the `19-openai-realtime-beta.py` and `19a-azure-realtime-beta.py`
+  examples to use the FunctionSchema format.
+
 ## [0.0.59] - 2025-03-20
 
 ### Added

examples/foundational/19-openai-realtime-beta.py

@@ -14,6 +14,8 @@ from dotenv import load_dotenv
 from loguru import logger
 from runner import configure
 
+from pipecat.adapters.schemas.function_schema import FunctionSchema
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
 from pipecat.audio.vad.silero import SileroVADAnalyzer
 from pipecat.audio.vad.vad_analyzer import VADParams
 from pipecat.pipeline.pipeline import Pipeline
@@ -21,10 +23,11 @@ from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.openai_realtime_beta import (
+    InputAudioNoiseReduction,
     InputAudioTranscription,
     OpenAIRealtimeBetaLLMService,
+    SemanticTurnDetection,
     SessionProperties,
-    TurnDetection,
 )
 from pipecat.transports.services.daily import DailyParams, DailyTransport
 
@@ -46,28 +49,25 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context
     )
 
 
-tools = [
-    {
-        "type": "function",
-        "name": "get_current_weather",
-        "description": "Get the current weather",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "location": {
-                    "type": "string",
-                    "description": "The city and state, e.g. San Francisco, CA",
-                },
-                "format": {
-                    "type": "string",
-                    "enum": ["celsius", "fahrenheit"],
-                    "description": "The temperature unit to use. Infer this from the users location.",
-                },
-            },
-            "required": ["location", "format"],
+weather_function = FunctionSchema(
+    name="get_current_weather",
+    description="Get the current weather",
+    properties={
+        "location": {
+            "type": "string",
+            "description": "The city and state, e.g. San Francisco, CA",
         },
-    }
-]
+        "format": {
+            "type": "string",
+            "enum": ["celsius", "fahrenheit"],
+            "description": "The temperature unit to use. Infer this from the users location.",
+        },
+    },
+    required=["location", "format"],
+)
+
+# Create tools schema
+tools = ToolsSchema(standard_tools=[weather_function])
 
 
 async def main():
@@ -92,9 +92,10 @@ async def main():
             input_audio_transcription=InputAudioTranscription(),
             # Set openai TurnDetection parameters. Not setting this at all will turn it
             # on by default
-            turn_detection=TurnDetection(silence_duration_ms=1000),
+            turn_detection=SemanticTurnDetection(),
             # Or set to False to disable openai turn detection and use transport VAD
             # turn_detection=False,
+            input_audio_noise_reduction=InputAudioNoiseReduction(type="near_field"),
             # tools=tools,
             instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI.
 

examples/foundational/19a-azure-realtime-beta.py

@@ -10,11 +10,12 @@ import sys
 from datetime import datetime
 
 import aiohttp
-import websockets
 from dotenv import load_dotenv
 from loguru import logger
 from runner import configure
 
+from pipecat.adapters.schemas.function_schema import FunctionSchema
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
 from pipecat.audio.vad.silero import SileroVADAnalyzer
 from pipecat.audio.vad.vad_analyzer import VADParams
 from pipecat.pipeline.pipeline import Pipeline
@@ -25,7 +26,6 @@ from pipecat.services.openai_realtime_beta import (
     AzureRealtimeBetaLLMService,
     InputAudioTranscription,
     SessionProperties,
-    TurnDetection,
 )
 from pipecat.transports.services.daily import DailyParams, DailyTransport
 
@@ -47,28 +47,26 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context
     )
 
 
-tools = [
-    {
-        "type": "function",
-        "name": "get_current_weather",
-        "description": "Get the current weather",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "location": {
-                    "type": "string",
-                    "description": "The city and state, e.g. San Francisco, CA",
-                },
-                "format": {
-                    "type": "string",
-                    "enum": ["celsius", "fahrenheit"],
-                    "description": "The temperature unit to use. Infer this from the users location.",
-                },
-            },
-            "required": ["location", "format"],
+# Define weather function using standardized schema
+weather_function = FunctionSchema(
+    name="get_current_weather",
+    description="Get the current weather",
+    properties={
+        "location": {
+            "type": "string",
+            "description": "The city and state, e.g. San Francisco, CA",
         },
-    }
-]
+        "format": {
+            "type": "string",
+            "enum": ["celsius", "fahrenheit"],
+            "description": "The temperature unit to use. Infer this from the users location.",
+        },
+    },
+    required=["location", "format"],
+)
+
+# Create tools schema
+tools = ToolsSchema(standard_tools=[weather_function])
 
 
 async def main():

src/pipecat/services/openai.py

@@ -129,10 +129,23 @@ class BaseOpenAILLMService(LLMService):
         }
         self.set_model_name(model)
         self._client = self.create_client(
-            api_key=api_key, base_url=base_url, organization=organization, project=project, default_headers=default_headers, **kwargs
+            api_key=api_key,
+            base_url=base_url,
+            organization=organization,
+            project=project,
+            default_headers=default_headers,
+            **kwargs,
         )
 
-    def create_client(self, api_key=None, base_url=None, organization=None, project=None, default_headers=None, **kwargs):
+    def create_client(
+        self,
+        api_key=None,
+        base_url=None,
+        organization=None,
+        project=None,
+        default_headers=None,
+        **kwargs,
+    ):
         return AsyncOpenAI(
             api_key=api_key,
             base_url=base_url,
@@ -143,7 +156,7 @@ class BaseOpenAILLMService(LLMService):
                     max_keepalive_connections=100, max_connections=1000, keepalive_expiry=None
                 )
             ),
-            default_headers=default_headers
+            default_headers=default_headers,
         )
 
     def can_generate_metrics(self) -> bool:

src/pipecat/services/openai_realtime_beta/events.py

@@ -14,17 +14,24 @@ from pydantic import BaseModel, Field
 #
 # session properties
 #
-InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe"]
 
 
 class InputAudioTranscription(BaseModel):
-    model: InputAudioTranscriptionModel
+    """Configuration for audio transcription settings.
+
+    Attributes:
+        model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1").
+        language: Optional language code for transcription.
+        prompt: Optional transcription hint text.
+    """
+
+    model: str = "gpt-4o-transcribe"
     language: Optional[str]
     prompt: Optional[str]
 
     def __init__(
         self,
-        model: Optional[InputAudioTranscriptionModel] = "whisper-1",
+        model: Optional[str] = "gpt-4o-transcribe",
         language: Optional[str] = None,
         prompt: Optional[str] = None,
     ):