wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update OpenAI realtime transcription default

Browse Source
This commit is contained in:
Mark Backman
2026-05-08 13:56:32 -07:00
parent 94a94ee28c
commit abd28e2ac1
8 changed files with 118 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
examples/function-calling/function-calling-openai-async.py
View File

@@ -29,7 +29,7 @@ from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
from pipecat.services.llm_service import FunctionCallParams
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.services.openai.stt import OpenAISTTService
from pipecat.services.openai.stt import OpenAIRealtimeSTTService
from pipecat.services.openai.tts import OpenAITTSService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
@@ -69,13 +69,7 @@ transport_params = {
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot")
stt = OpenAISTTService(
api_key=os.environ["OPENAI_API_KEY"],
settings=OpenAISTTService.Settings(
model="gpt-4o-transcribe",
prompt="Expect words related weather, such as temperature and conditions. And restaurant names.",
),
)
stt = OpenAIRealtimeSTTService(api_key=os.environ["OPENAI_API_KEY"])
tts = OpenAITTSService(
api_key=os.environ["OPENAI_API_KEY"],

10
examples/function-calling/function-calling-openai.py
View File

@@ -25,7 +25,7 @@ from pipecat.runner.types import RunnerArguments
from pipecat.runner.utils import create_transport
from pipecat.services.llm_service import FunctionCallParams
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.services.openai.stt import OpenAISTTService
from pipecat.services.openai.stt import OpenAIRealtimeSTTService
from pipecat.services.openai.tts import OpenAITTSService
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
@@ -63,13 +63,7 @@ transport_params = {
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot")
stt = OpenAISTTService(
api_key=os.environ["OPENAI_API_KEY"],
settings=OpenAISTTService.Settings(
model="gpt-4o-transcribe",
prompt="Expect words related weather, such as temperature and conditions. And restaurant names.",
),
)
stt = OpenAIRealtimeSTTService(api_key=os.environ["OPENAI_API_KEY"])
tts = OpenAITTSService(
api_key=os.environ["OPENAI_API_KEY"],

8
examples/transcription/transcription-openai.py
View File

@@ -49,13 +49,7 @@ transport_params = {
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot")
stt = OpenAIRealtimeSTTService(
api_key=os.environ["OPENAI_API_KEY"],
settings=OpenAIRealtimeSTTService.Settings(
model="gpt-4o-transcribe",
prompt="Expect words related to dogs, such as breed names.",
),
)
stt = OpenAIRealtimeSTTService(api_key=os.environ["OPENAI_API_KEY"])
tl = TranscriptionLogger()
vad_processor = VADProcessor(vad_analyzer=SileroVADAnalyzer())

12
examples/voice/voice-openai.py
View File

@@ -25,7 +25,6 @@ from pipecat.runner.utils import create_transport
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.services.openai.stt import OpenAIRealtimeSTTService
from pipecat.services.openai.tts import OpenAITTSService
from pipecat.transcriptions.language import Language
from pipecat.transports.base_transport import BaseTransport, TransportParams
from pipecat.transports.daily.transport import DailyParams
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
@@ -53,14 +52,7 @@ transport_params = {
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
logger.info(f"Starting bot")
stt = OpenAIRealtimeSTTService(
api_key=os.environ["OPENAI_API_KEY"],
settings=OpenAIRealtimeSTTService.Settings(
model="gpt-4o-transcribe",
prompt="Expect words related to dogs, such as breed names.",
language=Language.EN,
),
)
stt = OpenAIRealtimeSTTService(api_key=os.environ["OPENAI_API_KEY"])
tts = OpenAITTSService(
api_key=os.environ["OPENAI_API_KEY"],
@@ -72,7 +64,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
llm = OpenAILLMService(
api_key=os.environ["OPENAI_API_KEY"],
settings=OpenAILLMService.Settings(
system_instruction="You are very knowledgable about dogs. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.",
system_instruction="You are a helpful assistant in a voice conversation. Your responses will be spoken aloud, so avoid emojis, bullet points, or other formatting that can't be spoken. Respond to what the user said in a creative, helpful, and brief way.",
),
)

9
src/pipecat/services/openai/realtime/events.py
View File

@@ -18,6 +18,8 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
# session properties
#
GPT_REALTIME_WHISPER_MODEL = "gpt-realtime-whisper"
class AudioFormat(BaseModel):
"""Base class for audio format configuration."""
@@ -60,20 +62,21 @@ class PCMAAudioFormat(AudioFormat):
class InputAudioTranscription(BaseModel):
"""Configuration for audio transcription settings."""
model: str = "gpt-4o-transcribe"
model: str = GPT_REALTIME_WHISPER_MODEL
language: str | None
prompt: str | None
def __init__(
self,
model: str | None = "gpt-4o-transcribe",
model: str | None = GPT_REALTIME_WHISPER_MODEL,
language: str | None = None,
prompt: str | None = None,
):
"""Initialize InputAudioTranscription.
Args:
model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1").
model: Transcription model to use (e.g., "gpt-realtime-whisper",
"gpt-4o-transcribe", "whisper-1").
language: Optional language code for transcription.
prompt: Optional transcription hint text.
"""

30
src/pipecat/services/openai/realtime/llm.py
View File

@@ -287,6 +287,8 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]):
if settings is not None:
default_settings.apply_update(settings)
self._omit_unsupported_input_audio_transcription_prompt(default_settings.session_properties)
# Build WebSocket URL with model query parameter
# Source: https://platform.openai.com/docs/guides/realtime-websocket
full_url = f"{base_url}?model={default_settings.model}"
@@ -323,6 +325,29 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]):
self._register_event_handler("on_conversation_item_updated")
self._retrieve_conversation_item_futures = {}
@staticmethod
def _omit_unsupported_input_audio_transcription_prompt(
session_properties: events.SessionProperties,
) -> bool:
"""Drop input transcription prompt settings unsupported by the selected model."""
transcription = (
session_properties.audio.input.transcription
if session_properties.audio
and session_properties.audio.input
and session_properties.audio.input.transcription
else None
)
if transcription and transcription.model == events.GPT_REALTIME_WHISPER_MODEL:
if transcription.prompt:
transcription.prompt = None
logger.warning(
f"{events.GPT_REALTIME_WHISPER_MODEL} does not support the prompt "
"parameter; omitting prompt from OpenAI Realtime input audio "
"transcription settings."
)
return True
return False
def can_generate_metrics(self) -> bool:
"""Check if the service can generate usage metrics.
@@ -649,8 +674,11 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]):
async def _update_settings(self, delta):
"""Apply a settings delta, sending a session update when needed."""
changed = await super()._update_settings(delta)
prompt_omitted = self._omit_unsupported_input_audio_transcription_prompt(
assert_given(self._settings.session_properties)
)
handled = {"session_properties", "system_instruction"}
if changed.keys() & handled:
if changed.keys() & handled or prompt_omitted:
await self._send_session_update()
self._warn_unhandled_updated_settings(changed.keys() - handled)
return changed

38
src/pipecat/services/openai/stt.py
View File

@@ -179,6 +179,7 @@ class OpenAISTTService(BaseWhisperSTTService):
_OPENAI_SAMPLE_RATE = 24000
_OPENAI_REALTIME_WHISPER_MODEL = "gpt-realtime-whisper"
@dataclass
@@ -186,7 +187,8 @@ class OpenAIRealtimeSTTSettings(STTSettings):
"""Settings for OpenAIRealtimeSTTService.
Parameters:
prompt: Optional prompt text to guide transcription style.
prompt: Optional prompt text to guide transcription style. Not supported by
``"gpt-realtime-whisper"``.
noise_reduction: Noise reduction mode. ``"near_field"`` for close
microphones, ``"far_field"`` for distant microphones, or ``None``
to disable.
@@ -227,7 +229,7 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
stt = OpenAIRealtimeSTTService(
api_key="sk-...",
settings=OpenAIRealtimeSTTService.Settings(
model="gpt-4o-transcribe",
model="gpt-realtime-whisper",
noise_reduction="near_field",
),
)
@@ -255,7 +257,9 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
Args:
api_key: OpenAI API key for authentication.
model: Transcription model. Supported values are
model: Transcription model. For low-latency streaming
transcription, use ``"gpt-realtime-whisper"``. Other
supported transcription models include
``"gpt-4o-transcribe"`` and ``"gpt-4o-mini-transcribe"``.
.. deprecated:: 0.0.105
@@ -269,7 +273,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
Use ``settings=OpenAIRealtimeSTTService.Settings(language=...)`` instead.
prompt: Optional prompt text to guide transcription style
or provide keyword hints.
or provide keyword hints. Not supported by
``"gpt-realtime-whisper"``.
.. deprecated:: 0.0.105
Use ``settings=OpenAIRealtimeSTTService.Settings(prompt=...)`` instead.
@@ -303,7 +308,7 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
# --- 1. Hardcoded defaults ---
default_settings = self.Settings(
model="gpt-4o-transcribe",
model=_OPENAI_REALTIME_WHISPER_MODEL,
language=Language.EN,
prompt=None,
noise_reduction=None,
@@ -329,6 +334,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
if settings is not None:
default_settings.apply_update(settings)
self._omit_unsupported_prompt(default_settings)
super().__init__(
ttfs_p99_latency=ttfs_p99_latency,
settings=default_settings,
@@ -349,6 +356,19 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
# Set to None or a dict to enable server-side VAD.
self._server_vad_enabled = turn_detection is not False
@staticmethod
def _omit_unsupported_prompt(settings: OpenAIRealtimeSTTSettings) -> dict[str, Any]:
"""Drop prompt settings that are not accepted by the selected model."""
if settings.model == _OPENAI_REALTIME_WHISPER_MODEL and settings.prompt:
old_prompt = settings.prompt
settings.prompt = None
logger.warning(
f"{_OPENAI_REALTIME_WHISPER_MODEL} does not support the prompt parameter; "
"omitting prompt from OpenAI Realtime transcription session."
)
return {"prompt": old_prompt}
return {}
@staticmethod
def _language_to_code(language: Language) -> str:
"""Convert a Language enum value to an ISO-639-1 code.
@@ -382,6 +402,8 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
Dict mapping changed field names to their previous values.
"""
changed = await super()._update_settings(delta)
for field, previous_value in self._omit_unsupported_prompt(self._settings).items():
changed.setdefault(field, previous_value)
if changed and self._session_ready:
await self._send_session_update()
@@ -676,9 +698,9 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
async def _handle_transcription_delta(self, evt: dict):
"""Handle incremental transcription text.
For ``gpt-4o-transcribe`` and ``gpt-4o-mini-transcribe``, deltas
contain streaming partial text. For ``whisper-1``, each delta
contains the full turn transcript.
For ``gpt-realtime-whisper``, ``gpt-4o-transcribe``, and
``gpt-4o-mini-transcribe``, deltas contain low-latency streaming
partial text.
Args:
evt: The delta event from the server.

47
tests/test_settings.py
View File

@@ -13,7 +13,10 @@ from pipecat.services.deepgram.stt import DeepgramSTTService, DeepgramSTTSetting
from pipecat.services.inworld.realtime import events as inworld_events
from pipecat.services.inworld.realtime.llm import InworldRealtimeLLMSettings
from pipecat.services.openai.realtime import events
from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMSettings
from pipecat.services.openai.realtime.llm import (
OpenAIRealtimeLLMService,
OpenAIRealtimeLLMSettings,
)
from pipecat.services.settings import (
NOT_GIVEN,
LLMSettings,
@@ -747,6 +750,48 @@ class TestOpenAIRealtimeSettingsApplyUpdate:
assert store.session_properties.instructions == "Keep me."
class TestOpenAIRealtimeSessionProperties:
def test_realtime_whisper_prompt_is_omitted(self):
"""gpt-realtime-whisper does not support input transcription prompt."""
session_properties = events.SessionProperties(
audio=events.AudioConfiguration(
input=events.AudioInput(
transcription=events.InputAudioTranscription(
model=events.GPT_REALTIME_WHISPER_MODEL,
prompt="Keywords: metoprolol",
)
)
)
)
changed = OpenAIRealtimeLLMService._omit_unsupported_input_audio_transcription_prompt(
session_properties
)
assert changed is True
assert session_properties.audio.input.transcription.prompt is None
def test_supported_transcription_model_keeps_prompt(self):
"""Other input transcription models can keep prompt settings."""
session_properties = events.SessionProperties(
audio=events.AudioConfiguration(
input=events.AudioInput(
transcription=events.InputAudioTranscription(
model="gpt-4o-transcribe",
prompt="Keywords: metoprolol",
)
)
)
)
changed = OpenAIRealtimeLLMService._omit_unsupported_input_audio_transcription_prompt(
session_properties
)
assert changed is False
assert session_properties.audio.input.transcription.prompt == "Keywords: metoprolol"
# ---------------------------------------------------------------------------
# OpenAIRealtimeLLMSettings: from_mapping
# ---------------------------------------------------------------------------