diff --git a/examples/function-calling/function-calling-openai-async.py b/examples/function-calling/function-calling-openai-async.py index 47039711f..de0d1a014 100644 --- a/examples/function-calling/function-calling-openai-async.py +++ b/examples/function-calling/function-calling-openai-async.py @@ -29,7 +29,7 @@ from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService -from pipecat.services.openai.stt import OpenAISTTService +from pipecat.services.openai.stt import OpenAIRealtimeSTTService from pipecat.services.openai.tts import OpenAITTSService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -69,13 +69,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = OpenAISTTService( - api_key=os.environ["OPENAI_API_KEY"], - settings=OpenAISTTService.Settings( - model="gpt-4o-transcribe", - prompt="Expect words related weather, such as temperature and conditions. And restaurant names.", - ), - ) + stt = OpenAIRealtimeSTTService(api_key=os.environ["OPENAI_API_KEY"]) tts = OpenAITTSService( api_key=os.environ["OPENAI_API_KEY"], diff --git a/examples/function-calling/function-calling-openai.py b/examples/function-calling/function-calling-openai.py index 2a9779742..38c9d6385 100644 --- a/examples/function-calling/function-calling-openai.py +++ b/examples/function-calling/function-calling-openai.py @@ -25,7 +25,7 @@ from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService -from pipecat.services.openai.stt import OpenAISTTService +from pipecat.services.openai.stt import OpenAIRealtimeSTTService from pipecat.services.openai.tts import OpenAITTSService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -63,13 +63,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = OpenAISTTService( - api_key=os.environ["OPENAI_API_KEY"], - settings=OpenAISTTService.Settings( - model="gpt-4o-transcribe", - prompt="Expect words related weather, such as temperature and conditions. And restaurant names.", - ), - ) + stt = OpenAIRealtimeSTTService(api_key=os.environ["OPENAI_API_KEY"]) tts = OpenAITTSService( api_key=os.environ["OPENAI_API_KEY"], diff --git a/examples/transcription/transcription-openai.py b/examples/transcription/transcription-openai.py index 5f3072177..47228ce8b 100644 --- a/examples/transcription/transcription-openai.py +++ b/examples/transcription/transcription-openai.py @@ -49,13 +49,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = OpenAIRealtimeSTTService( - api_key=os.environ["OPENAI_API_KEY"], - settings=OpenAIRealtimeSTTService.Settings( - model="gpt-4o-transcribe", - prompt="Expect words related to dogs, such as breed names.", - ), - ) + stt = OpenAIRealtimeSTTService(api_key=os.environ["OPENAI_API_KEY"]) tl = TranscriptionLogger() vad_processor = VADProcessor(vad_analyzer=SileroVADAnalyzer()) diff --git a/examples/voice/voice-openai.py b/examples/voice/voice-openai.py index 8f162671f..f5c30d681 100644 --- a/examples/voice/voice-openai.py +++ b/examples/voice/voice-openai.py @@ -25,7 +25,6 @@ from pipecat.runner.utils import create_transport from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.stt import OpenAIRealtimeSTTService from pipecat.services.openai.tts import OpenAITTSService -from pipecat.transcriptions.language import Language from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams @@ -53,14 +52,7 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - stt = OpenAIRealtimeSTTService( - api_key=os.environ["OPENAI_API_KEY"], - settings=OpenAIRealtimeSTTService.Settings( - model="gpt-4o-transcribe", - prompt="Expect words related to dogs, such as breed names.", - language=Language.EN, - ), - ) + stt = OpenAIRealtimeSTTService(api_key=os.environ["OPENAI_API_KEY"]) tts = OpenAITTSService( api_key=os.environ["OPENAI_API_KEY"], @@ -72,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): llm = OpenAILLMService( api_key=os.environ["OPENAI_API_KEY"], settings=OpenAILLMService.Settings( - system_instruction="You are very knowledgable about dogs. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.", + system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.", ), ) diff --git a/src/pipecat/services/openai/realtime/events.py b/src/pipecat/services/openai/realtime/events.py index 57b53c565..42d3b7e4d 100644 --- a/src/pipecat/services/openai/realtime/events.py +++ b/src/pipecat/services/openai/realtime/events.py @@ -18,6 +18,8 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema # session properties # +GPT_REALTIME_WHISPER_MODEL = "gpt-realtime-whisper" + class AudioFormat(BaseModel): """Base class for audio format configuration.""" @@ -60,20 +62,21 @@ class PCMAAudioFormat(AudioFormat): class InputAudioTranscription(BaseModel): """Configuration for audio transcription settings.""" - model: str = "gpt-4o-transcribe" + model: str = GPT_REALTIME_WHISPER_MODEL language: str | None prompt: str | None def __init__( self, - model: str | None = "gpt-4o-transcribe", + model: str | None = GPT_REALTIME_WHISPER_MODEL, language: str | None = None, prompt: str | None = None, ): """Initialize InputAudioTranscription. Args: - model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). + model: Transcription model to use (e.g., "gpt-realtime-whisper", + "gpt-4o-transcribe", "whisper-1"). language: Optional language code for transcription. prompt: Optional transcription hint text. """ diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 8519fddd8..e39299dc0 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -287,6 +287,8 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]): if settings is not None: default_settings.apply_update(settings) + self._omit_unsupported_input_audio_transcription_prompt(default_settings.session_properties) + # Build WebSocket URL with model query parameter # Source: https://platform.openai.com/docs/guides/realtime-websocket full_url = f"{base_url}?model={default_settings.model}" @@ -323,6 +325,29 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]): self._register_event_handler("on_conversation_item_updated") self._retrieve_conversation_item_futures = {} + @staticmethod + def _omit_unsupported_input_audio_transcription_prompt( + session_properties: events.SessionProperties, + ) -> bool: + """Drop input transcription prompt settings unsupported by the selected model.""" + transcription = ( + session_properties.audio.input.transcription + if session_properties.audio + and session_properties.audio.input + and session_properties.audio.input.transcription + else None + ) + if transcription and transcription.model == events.GPT_REALTIME_WHISPER_MODEL: + if transcription.prompt: + transcription.prompt = None + logger.warning( + f"{events.GPT_REALTIME_WHISPER_MODEL} does not support the prompt " + "parameter; omitting prompt from OpenAI Realtime input audio " + "transcription settings." + ) + return True + return False + def can_generate_metrics(self) -> bool: """Check if the service can generate usage metrics. @@ -649,8 +674,11 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]): async def _update_settings(self, delta): """Apply a settings delta, sending a session update when needed.""" changed = await super()._update_settings(delta) + prompt_omitted = self._omit_unsupported_input_audio_transcription_prompt( + assert_given(self._settings.session_properties) + ) handled = {"session_properties", "system_instruction"} - if changed.keys() & handled: + if changed.keys() & handled or prompt_omitted: await self._send_session_update() self._warn_unhandled_updated_settings(changed.keys() - handled) return changed diff --git a/src/pipecat/services/openai/stt.py b/src/pipecat/services/openai/stt.py index 6f635639a..8fb82b8df 100644 --- a/src/pipecat/services/openai/stt.py +++ b/src/pipecat/services/openai/stt.py @@ -179,6 +179,7 @@ class OpenAISTTService(BaseWhisperSTTService): _OPENAI_SAMPLE_RATE = 24000 +_OPENAI_REALTIME_WHISPER_MODEL = "gpt-realtime-whisper" @dataclass @@ -186,7 +187,8 @@ class OpenAIRealtimeSTTSettings(STTSettings): """Settings for OpenAIRealtimeSTTService. Parameters: - prompt: Optional prompt text to guide transcription style. + prompt: Optional prompt text to guide transcription style. Not supported by + ``"gpt-realtime-whisper"``. noise_reduction: Noise reduction mode. ``"near_field"`` for close microphones, ``"far_field"`` for distant microphones, or ``None`` to disable. @@ -227,7 +229,7 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): stt = OpenAIRealtimeSTTService( api_key="sk-...", settings=OpenAIRealtimeSTTService.Settings( - model="gpt-4o-transcribe", + model="gpt-realtime-whisper", noise_reduction="near_field", ), ) @@ -255,7 +257,9 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): Args: api_key: OpenAI API key for authentication. - model: Transcription model. Supported values are + model: Transcription model. For low-latency streaming + transcription, use ``"gpt-realtime-whisper"``. Other + supported transcription models include ``"gpt-4o-transcribe"`` and ``"gpt-4o-mini-transcribe"``. .. deprecated:: 0.0.105 @@ -269,7 +273,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): Use ``settings=OpenAIRealtimeSTTService.Settings(language=...)`` instead. prompt: Optional prompt text to guide transcription style - or provide keyword hints. + or provide keyword hints. Not supported by + ``"gpt-realtime-whisper"``. .. deprecated:: 0.0.105 Use ``settings=OpenAIRealtimeSTTService.Settings(prompt=...)`` instead. @@ -303,7 +308,7 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): # --- 1. Hardcoded defaults --- default_settings = self.Settings( - model="gpt-4o-transcribe", + model=_OPENAI_REALTIME_WHISPER_MODEL, language=Language.EN, prompt=None, noise_reduction=None, @@ -329,6 +334,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): if settings is not None: default_settings.apply_update(settings) + self._omit_unsupported_prompt(default_settings) + super().__init__( ttfs_p99_latency=ttfs_p99_latency, settings=default_settings, @@ -349,6 +356,19 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): # Set to None or a dict to enable server-side VAD. self._server_vad_enabled = turn_detection is not False + @staticmethod + def _omit_unsupported_prompt(settings: OpenAIRealtimeSTTSettings) -> dict[str, Any]: + """Drop prompt settings that are not accepted by the selected model.""" + if settings.model == _OPENAI_REALTIME_WHISPER_MODEL and settings.prompt: + old_prompt = settings.prompt + settings.prompt = None + logger.warning( + f"{_OPENAI_REALTIME_WHISPER_MODEL} does not support the prompt parameter; " + "omitting prompt from OpenAI Realtime transcription session." + ) + return {"prompt": old_prompt} + return {} + @staticmethod def _language_to_code(language: Language) -> str: """Convert a Language enum value to an ISO-639-1 code. @@ -382,6 +402,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): Dict mapping changed field names to their previous values. """ changed = await super()._update_settings(delta) + for field, previous_value in self._omit_unsupported_prompt(self._settings).items(): + changed.setdefault(field, previous_value) if changed and self._session_ready: await self._send_session_update() @@ -676,9 +698,9 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): async def _handle_transcription_delta(self, evt: dict): """Handle incremental transcription text. - For ``gpt-4o-transcribe`` and ``gpt-4o-mini-transcribe``, deltas - contain streaming partial text. For ``whisper-1``, each delta - contains the full turn transcript. + For ``gpt-realtime-whisper``, ``gpt-4o-transcribe``, and + ``gpt-4o-mini-transcribe``, deltas contain low-latency streaming + partial text. Args: evt: The delta event from the server. diff --git a/tests/test_settings.py b/tests/test_settings.py index 6c45d53f5..78c1f30b8 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -13,7 +13,10 @@ from pipecat.services.deepgram.stt import DeepgramSTTService, DeepgramSTTSetting from pipecat.services.inworld.realtime import events as inworld_events from pipecat.services.inworld.realtime.llm import InworldRealtimeLLMSettings from pipecat.services.openai.realtime import events -from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMSettings +from pipecat.services.openai.realtime.llm import ( + OpenAIRealtimeLLMService, + OpenAIRealtimeLLMSettings, +) from pipecat.services.settings import ( NOT_GIVEN, LLMSettings, @@ -747,6 +750,48 @@ class TestOpenAIRealtimeSettingsApplyUpdate: assert store.session_properties.instructions == "Keep me." +class TestOpenAIRealtimeSessionProperties: + def test_realtime_whisper_prompt_is_omitted(self): + """gpt-realtime-whisper does not support input transcription prompt.""" + session_properties = events.SessionProperties( + audio=events.AudioConfiguration( + input=events.AudioInput( + transcription=events.InputAudioTranscription( + model=events.GPT_REALTIME_WHISPER_MODEL, + prompt="Keywords: metoprolol", + ) + ) + ) + ) + + changed = OpenAIRealtimeLLMService._omit_unsupported_input_audio_transcription_prompt( + session_properties + ) + + assert changed is True + assert session_properties.audio.input.transcription.prompt is None + + def test_supported_transcription_model_keeps_prompt(self): + """Other input transcription models can keep prompt settings.""" + session_properties = events.SessionProperties( + audio=events.AudioConfiguration( + input=events.AudioInput( + transcription=events.InputAudioTranscription( + model="gpt-4o-transcribe", + prompt="Keywords: metoprolol", + ) + ) + ) + ) + + changed = OpenAIRealtimeLLMService._omit_unsupported_input_audio_transcription_prompt( + session_properties + ) + + assert changed is False + assert session_properties.audio.input.transcription.prompt == "Keywords: metoprolol" + + # --------------------------------------------------------------------------- # OpenAIRealtimeLLMSettings: from_mapping # ---------------------------------------------------------------------------