From 541a4b6063140e7d92c9f7982cbb492f4581a8be Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Mar 2025 15:12:43 -0400 Subject: [PATCH 1/3] Update InputAudioTranscription to use gpt-4o-transcribe model, update 19 examples to use FunctionSchema --- CHANGELOG.md | 10 +++++ .../foundational/19-openai-realtime-beta.py | 41 +++++++++--------- .../foundational/19a-azure-realtime-beta.py | 43 +++++++++---------- .../services/openai_realtime_beta/events.py | 13 ++++-- 4 files changed, 61 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a0524e29..63885acbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `default_headers` parameter to `BaseOpenAILLMService` constructor. +### Changed + +- Changed the default `InputAudioTranscription` model to `gpt-4o-transcribe` + for `OpenAIRealtimeBetaLLMService`. + +### Other + +- Update the `19-openai-realtime-beta.py` and `19a-azure-realtime-beta.py` + examples to use the FunctionSchema format. + ## [0.0.59] - 2025-03-20 ### Added diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 8796e0141..bb62466a7 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -14,6 +14,8 @@ from dotenv import load_dotenv from loguru import logger from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline @@ -46,28 +48,25 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context ) -tools = [ - { - "type": "function", - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - } -] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + required=["location", "format"], +) + +# Create tools schema +tools = ToolsSchema(standard_tools=[weather_function]) async def main(): diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 14d034836..7ec1d195a 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -10,11 +10,12 @@ import sys from datetime import datetime import aiohttp -import websockets from dotenv import load_dotenv from loguru import logger from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline @@ -47,28 +48,26 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context ) -tools = [ - { - "type": "function", - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], +# Define weather function using standardized schema +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - } -] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + required=["location", "format"], +) + +# Create tools schema +tools = ToolsSchema(standard_tools=[weather_function]) async def main(): diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index f4133766b..c2dcb7f09 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -14,17 +14,24 @@ from pydantic import BaseModel, Field # # session properties # -InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe"] class InputAudioTranscription(BaseModel): - model: InputAudioTranscriptionModel + """Configuration for audio transcription settings. + + Attributes: + model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). + language: Optional language code for transcription. + prompt: Optional transcription hint text. + """ + + model: str = "gpt-4o-transcribe" language: Optional[str] prompt: Optional[str] def __init__( self, - model: Optional[InputAudioTranscriptionModel] = "whisper-1", + model: Optional[str] = "gpt-4o-transcribe", language: Optional[str] = None, prompt: Optional[str] = None, ): From 41688205be87f443cbc73e08b215150befdbafa4 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 20 Mar 2025 15:23:57 -0400 Subject: [PATCH 2/3] Provide new settings in OpenAI Realtime example --- examples/foundational/19-openai-realtime-beta.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index bb62466a7..f2349f6d2 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -26,7 +26,8 @@ from pipecat.services.openai_realtime_beta import ( InputAudioTranscription, OpenAIRealtimeBetaLLMService, SessionProperties, - TurnDetection, + SemanticTurnDetection, + InputAudioNoiseReduction ) from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -91,9 +92,10 @@ async def main(): input_audio_transcription=InputAudioTranscription(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default - turn_detection=TurnDetection(silence_duration_ms=1000), + turn_detection=SemanticTurnDetection(), # Or set to False to disable openai turn detection and use transport VAD # turn_detection=False, + input_audio_noise_reduction=InputAudioNoiseReduction(type='near_field'), # tools=tools, instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. From 2ac8f2ec2d5acd26527a2e37510ddf17a521f5a0 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Mar 2025 15:50:51 -0400 Subject: [PATCH 3/3] Fix linting --- .../foundational/19-openai-realtime-beta.py | 6 +++--- .../foundational/19a-azure-realtime-beta.py | 1 - src/pipecat/services/openai.py | 19 ++++++++++++++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index f2349f6d2..3aff14e65 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -23,11 +23,11 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai_realtime_beta import ( + InputAudioNoiseReduction, InputAudioTranscription, OpenAIRealtimeBetaLLMService, - SessionProperties, SemanticTurnDetection, - InputAudioNoiseReduction + SessionProperties, ) from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -95,7 +95,7 @@ async def main(): turn_detection=SemanticTurnDetection(), # Or set to False to disable openai turn detection and use transport VAD # turn_detection=False, - input_audio_noise_reduction=InputAudioNoiseReduction(type='near_field'), + input_audio_noise_reduction=InputAudioNoiseReduction(type="near_field"), # tools=tools, instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 7ec1d195a..2eefd4ec9 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -26,7 +26,6 @@ from pipecat.services.openai_realtime_beta import ( AzureRealtimeBetaLLMService, InputAudioTranscription, SessionProperties, - TurnDetection, ) from pipecat.transports.services.daily import DailyParams, DailyTransport diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 48b108ad0..ff7bc0442 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -129,10 +129,23 @@ class BaseOpenAILLMService(LLMService): } self.set_model_name(model) self._client = self.create_client( - api_key=api_key, base_url=base_url, organization=organization, project=project, default_headers=default_headers, **kwargs + api_key=api_key, + base_url=base_url, + organization=organization, + project=project, + default_headers=default_headers, + **kwargs, ) - def create_client(self, api_key=None, base_url=None, organization=None, project=None, default_headers=None, **kwargs): + def create_client( + self, + api_key=None, + base_url=None, + organization=None, + project=None, + default_headers=None, + **kwargs, + ): return AsyncOpenAI( api_key=api_key, base_url=base_url, @@ -143,7 +156,7 @@ class BaseOpenAILLMService(LLMService): max_keepalive_connections=100, max_connections=1000, keepalive_expiry=None ) ), - default_headers=default_headers + default_headers=default_headers, ) def can_generate_metrics(self) -> bool: