wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update OpenAI realtime transcription default

Browse Source
This commit is contained in:
Mark Backman
2026-05-08 13:56:32 -07:00
parent 94a94ee28c
commit abd28e2ac1
8 changed files with 118 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/pipecat/services/openai/realtime/events.py
View File

@@ -18,6 +18,8 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
# session properties
#
GPT_REALTIME_WHISPER_MODEL = "gpt-realtime-whisper"
class AudioFormat(BaseModel):
"""Base class for audio format configuration."""
@@ -60,20 +62,21 @@ class PCMAAudioFormat(AudioFormat):
class InputAudioTranscription(BaseModel):
"""Configuration for audio transcription settings."""
model: str = "gpt-4o-transcribe"
model: str = GPT_REALTIME_WHISPER_MODEL
language: str | None
prompt: str | None
def __init__(
self,
model: str | None = "gpt-4o-transcribe",
model: str | None = GPT_REALTIME_WHISPER_MODEL,
language: str | None = None,
prompt: str | None = None,
):
"""Initialize InputAudioTranscription.
Args:
model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1").
model: Transcription model to use (e.g., "gpt-realtime-whisper",
"gpt-4o-transcribe", "whisper-1").
language: Optional language code for transcription.
prompt: Optional transcription hint text.
"""

30
src/pipecat/services/openai/realtime/llm.py
View File

@@ -287,6 +287,8 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]):
if settings is not None:
default_settings.apply_update(settings)
self._omit_unsupported_input_audio_transcription_prompt(default_settings.session_properties)
# Build WebSocket URL with model query parameter
# Source: https://platform.openai.com/docs/guides/realtime-websocket
full_url = f"{base_url}?model={default_settings.model}"
@@ -323,6 +325,29 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]):
self._register_event_handler("on_conversation_item_updated")
self._retrieve_conversation_item_futures = {}
@staticmethod
def _omit_unsupported_input_audio_transcription_prompt(
session_properties: events.SessionProperties,
) -> bool:
"""Drop input transcription prompt settings unsupported by the selected model."""
transcription = (
session_properties.audio.input.transcription
if session_properties.audio
and session_properties.audio.input
and session_properties.audio.input.transcription
else None
)
if transcription and transcription.model == events.GPT_REALTIME_WHISPER_MODEL:
if transcription.prompt:
transcription.prompt = None
logger.warning(
f"{events.GPT_REALTIME_WHISPER_MODEL} does not support the prompt "
"parameter; omitting prompt from OpenAI Realtime input audio "
"transcription settings."
)
return True
return False
def can_generate_metrics(self) -> bool:
"""Check if the service can generate usage metrics.
@@ -649,8 +674,11 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]):
async def _update_settings(self, delta):
"""Apply a settings delta, sending a session update when needed."""
changed = await super()._update_settings(delta)
prompt_omitted = self._omit_unsupported_input_audio_transcription_prompt(
assert_given(self._settings.session_properties)
)
handled = {"session_properties", "system_instruction"}
if changed.keys() & handled:
if changed.keys() & handled or prompt_omitted:
await self._send_session_update()
self._warn_unhandled_updated_settings(changed.keys() - handled)
return changed

38
src/pipecat/services/openai/stt.py
View File

@@ -179,6 +179,7 @@ class OpenAISTTService(BaseWhisperSTTService):
_OPENAI_SAMPLE_RATE = 24000
_OPENAI_REALTIME_WHISPER_MODEL = "gpt-realtime-whisper"
@dataclass
@@ -186,7 +187,8 @@ class OpenAIRealtimeSTTSettings(STTSettings):
"""Settings for OpenAIRealtimeSTTService.
Parameters:
prompt: Optional prompt text to guide transcription style.
prompt: Optional prompt text to guide transcription style. Not supported by
``"gpt-realtime-whisper"``.
noise_reduction: Noise reduction mode. ``"near_field"`` for close
microphones, ``"far_field"`` for distant microphones, or ``None``
to disable.
@@ -227,7 +229,7 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
stt = OpenAIRealtimeSTTService(
api_key="sk-...",
settings=OpenAIRealtimeSTTService.Settings(
model="gpt-4o-transcribe",
model="gpt-realtime-whisper",
noise_reduction="near_field",
),
)
@@ -255,7 +257,9 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
Args:
api_key: OpenAI API key for authentication.
model: Transcription model. Supported values are
model: Transcription model. For low-latency streaming
transcription, use ``"gpt-realtime-whisper"``. Other
supported transcription models include
``"gpt-4o-transcribe"`` and ``"gpt-4o-mini-transcribe"``.
.. deprecated:: 0.0.105
@@ -269,7 +273,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
Use ``settings=OpenAIRealtimeSTTService.Settings(language=...)`` instead.
prompt: Optional prompt text to guide transcription style
or provide keyword hints.
or provide keyword hints. Not supported by
``"gpt-realtime-whisper"``.
.. deprecated:: 0.0.105
Use ``settings=OpenAIRealtimeSTTService.Settings(prompt=...)`` instead.
@@ -303,7 +308,7 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
# --- 1. Hardcoded defaults ---
default_settings = self.Settings(
model="gpt-4o-transcribe",
model=_OPENAI_REALTIME_WHISPER_MODEL,
language=Language.EN,
prompt=None,
noise_reduction=None,
@@ -329,6 +334,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
if settings is not None:
default_settings.apply_update(settings)
self._omit_unsupported_prompt(default_settings)
super().__init__(
ttfs_p99_latency=ttfs_p99_latency,
settings=default_settings,
@@ -349,6 +356,19 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
# Set to None or a dict to enable server-side VAD.
self._server_vad_enabled = turn_detection is not False
@staticmethod
def _omit_unsupported_prompt(settings: OpenAIRealtimeSTTSettings) -> dict[str, Any]:
"""Drop prompt settings that are not accepted by the selected model."""
if settings.model == _OPENAI_REALTIME_WHISPER_MODEL and settings.prompt:
old_prompt = settings.prompt
settings.prompt = None
logger.warning(
f"{_OPENAI_REALTIME_WHISPER_MODEL} does not support the prompt parameter; "
"omitting prompt from OpenAI Realtime transcription session."
)
return {"prompt": old_prompt}
return {}
@staticmethod
def _language_to_code(language: Language) -> str:
"""Convert a Language enum value to an ISO-639-1 code.
@@ -382,6 +402,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
for field, previous_value in self._omit_unsupported_prompt(self._settings).items():
changed.setdefault(field, previous_value)
if changed and self._session_ready:
await self._send_session_update()
@@ -676,9 +698,9 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
async def _handle_transcription_delta(self, evt: dict):
"""Handle incremental transcription text.
For ``gpt-4o-transcribe`` and ``gpt-4o-mini-transcribe``, deltas
contain streaming partial text. For ``whisper-1``, each delta
contains the full turn transcript.
For ``gpt-realtime-whisper``, ``gpt-4o-transcribe``, and
``gpt-4o-mini-transcribe``, deltas contain low-latency streaming
partial text.
Args:
evt: The delta event from the server.