wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Centralize OpenAI audio constants

Browse Source
This commit is contained in:
Mark Backman
2026-05-12 15:38:20 -04:00
parent e2bfa6352f
commit 644030584f
6 changed files with 32 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/pipecat/services/openai/_constants.py Normal file
View File

@@ -0,0 +1,10 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Internal constants for OpenAI service integrations."""
OPENAI_SAMPLE_RATE = 24000
OPENAI_REALTIME_WHISPER_MODEL = "gpt-realtime-whisper"

9
src/pipecat/services/openai/realtime/events.py
View File

@@ -13,13 +13,12 @@ from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field from pydantic import BaseModel, ConfigDict, Field
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.services.openai._constants import OPENAI_REALTIME_WHISPER_MODEL, OPENAI_SAMPLE_RATE
# #
# session properties # session properties
# #
GPT_REALTIME_WHISPER_MODEL = "gpt-realtime-whisper"
class AudioFormat(BaseModel): class AudioFormat(BaseModel):
"""Base class for audio format configuration.""" """Base class for audio format configuration."""
@@ -36,7 +35,7 @@ class PCMAudioFormat(AudioFormat):
""" """
type: Literal["audio/pcm"] = "audio/pcm" type: Literal["audio/pcm"] = "audio/pcm"
rate: Literal[24000] = 24000 rate: Literal[24000] = OPENAI_SAMPLE_RATE
class PCMUAudioFormat(AudioFormat): class PCMUAudioFormat(AudioFormat):
@@ -62,13 +61,13 @@ class PCMAAudioFormat(AudioFormat):
class InputAudioTranscription(BaseModel): class InputAudioTranscription(BaseModel):
"""Configuration for audio transcription settings.""" """Configuration for audio transcription settings."""
model: str = GPT_REALTIME_WHISPER_MODEL model: str = OPENAI_REALTIME_WHISPER_MODEL
language: str | None language: str | None
prompt: str | None prompt: str | None
def __init__( def __init__(
self, self,
model: str | None = GPT_REALTIME_WHISPER_MODEL, model: str | None = OPENAI_REALTIME_WHISPER_MODEL,
language: str | None = None, language: str | None = None,
prompt: str | None = None, prompt: str | None = None,
): ):

9
src/pipecat/services/openai/realtime/llm.py
View File

@@ -51,6 +51,7 @@ from pipecat.metrics.metrics import LLMTokenUsage
from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
from pipecat.services.openai._constants import OPENAI_REALTIME_WHISPER_MODEL, OPENAI_SAMPLE_RATE
from pipecat.services.settings import ( from pipecat.services.settings import (
NOT_GIVEN, NOT_GIVEN,
LLMSettings, LLMSettings,
@@ -337,11 +338,11 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]):
and session_properties.audio.input.transcription and session_properties.audio.input.transcription
else None else None
) )
if transcription and transcription.model == events.GPT_REALTIME_WHISPER_MODEL: if transcription and transcription.model == OPENAI_REALTIME_WHISPER_MODEL:
if transcription.prompt: if transcription.prompt:
transcription.prompt = None transcription.prompt = None
logger.warning( logger.warning(
f"{events.GPT_REALTIME_WHISPER_MODEL} does not support the prompt " f"{OPENAI_REALTIME_WHISPER_MODEL} does not support the prompt "
"parameter; omitting prompt from OpenAI Realtime input audio " "parameter; omitting prompt from OpenAI Realtime input audio "
"transcription settings." "transcription settings."
) )
@@ -505,7 +506,7 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]):
self._current_audio_response = None self._current_audio_response = None
def _calculate_audio_duration_ms( def _calculate_audio_duration_ms(
self, total_bytes: int, sample_rate: int = 24000, bytes_per_sample: int = 2 self, total_bytes: int, sample_rate: int = OPENAI_SAMPLE_RATE, bytes_per_sample: int = 2
) -> int: ) -> int:
"""Calculate audio duration in milliseconds based on PCM audio parameters.""" """Calculate audio duration in milliseconds based on PCM audio parameters."""
samples = total_bytes / bytes_per_sample samples = total_bytes / bytes_per_sample
@@ -797,7 +798,7 @@ class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]):
self._current_audio_response.total_size += len(audio) self._current_audio_response.total_size += len(audio)
frame = TTSAudioRawFrame( frame = TTSAudioRawFrame(
audio=audio, audio=audio,
sample_rate=24000, sample_rate=OPENAI_SAMPLE_RATE,
num_channels=1, num_channels=1,
) )
await self.push_frame(frame) await self.push_frame(frame)

15
src/pipecat/services/openai/stt.py
View File

@@ -36,6 +36,7 @@ from pipecat.frames.frames import (
VADUserStoppedSpeakingFrame, VADUserStoppedSpeakingFrame,
) )
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.openai._constants import OPENAI_REALTIME_WHISPER_MODEL, OPENAI_SAMPLE_RATE
from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, assert_given from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven, assert_given
from pipecat.services.stt_latency import OPENAI_REALTIME_TTFS_P99, OPENAI_TTFS_P99 from pipecat.services.stt_latency import OPENAI_REALTIME_TTFS_P99, OPENAI_TTFS_P99
from pipecat.services.stt_service import WebsocketSTTService from pipecat.services.stt_service import WebsocketSTTService
@@ -178,10 +179,6 @@ class OpenAISTTService(BaseWhisperSTTService):
return await self._client.audio.transcriptions.create(**kwargs) return await self._client.audio.transcriptions.create(**kwargs)
_OPENAI_SAMPLE_RATE = 24000
_OPENAI_REALTIME_WHISPER_MODEL = "gpt-realtime-whisper"
@dataclass @dataclass
class OpenAIRealtimeSTTSettings(STTSettings): class OpenAIRealtimeSTTSettings(STTSettings):
"""Settings for OpenAIRealtimeSTTService. """Settings for OpenAIRealtimeSTTService.
@@ -308,7 +305,7 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
# --- 1. Hardcoded defaults --- # --- 1. Hardcoded defaults ---
default_settings = self.Settings( default_settings = self.Settings(
model=_OPENAI_REALTIME_WHISPER_MODEL, model=OPENAI_REALTIME_WHISPER_MODEL,
language=Language.EN, language=Language.EN,
prompt=None, prompt=None,
noise_reduction=None, noise_reduction=None,
@@ -359,11 +356,11 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
@staticmethod @staticmethod
def _omit_unsupported_prompt(settings: OpenAIRealtimeSTTSettings) -> dict[str, Any]: def _omit_unsupported_prompt(settings: OpenAIRealtimeSTTSettings) -> dict[str, Any]:
"""Drop prompt settings that are not accepted by the selected model.""" """Drop prompt settings that are not accepted by the selected model."""
if settings.model == _OPENAI_REALTIME_WHISPER_MODEL and settings.prompt: if settings.model == OPENAI_REALTIME_WHISPER_MODEL and settings.prompt:
old_prompt = settings.prompt old_prompt = settings.prompt
settings.prompt = None settings.prompt = None
logger.warning( logger.warning(
f"{_OPENAI_REALTIME_WHISPER_MODEL} does not support the prompt parameter; " f"{OPENAI_REALTIME_WHISPER_MODEL} does not support the prompt parameter; "
"omitting prompt from OpenAI Realtime transcription session." "omitting prompt from OpenAI Realtime transcription session."
) )
return {"prompt": old_prompt} return {"prompt": old_prompt}
@@ -572,7 +569,7 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
input_audio: dict = { input_audio: dict = {
"format": { "format": {
"type": "audio/pcm", "type": "audio/pcm",
"rate": _OPENAI_SAMPLE_RATE, "rate": OPENAI_SAMPLE_RATE,
}, },
"transcription": transcription, "transcription": transcription,
} }
@@ -609,7 +606,7 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
Args: Args:
audio: Raw audio bytes at the pipeline sample rate. audio: Raw audio bytes at the pipeline sample rate.
""" """
audio = await self._resampler.resample(audio, self.sample_rate, _OPENAI_SAMPLE_RATE) audio = await self._resampler.resample(audio, self.sample_rate, OPENAI_SAMPLE_RATE)
if not audio: if not audio:
return return
payload = base64.b64encode(audio).decode("utf-8") payload = base64.b64encode(audio).decode("utf-8")

11
src/pipecat/services/openai/tts.py
View File

@@ -24,6 +24,7 @@ from pipecat.frames.frames import (
StartFrame, StartFrame,
TTSAudioRawFrame, TTSAudioRawFrame,
) )
from pipecat.services.openai._constants import OPENAI_SAMPLE_RATE
from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, assert_given from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven, assert_given
from pipecat.services.tts_service import TTSService from pipecat.services.tts_service import TTSService
from pipecat.utils.tracing.service_decorators import traced_tts from pipecat.utils.tracing.service_decorators import traced_tts
@@ -85,8 +86,6 @@ class OpenAITTSService(TTSService):
Settings = OpenAITTSSettings Settings = OpenAITTSSettings
_settings: Settings _settings: Settings
OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for OpenAI TTS configuration. """Input parameters for OpenAI TTS configuration.
@@ -150,9 +149,9 @@ class OpenAITTSService(TTSService):
parameters, ``settings`` values take precedence. parameters, ``settings`` values take precedence.
**kwargs: Additional keyword arguments passed to TTSService. **kwargs: Additional keyword arguments passed to TTSService.
""" """
if sample_rate and sample_rate != self.OPENAI_SAMPLE_RATE: if sample_rate and sample_rate != OPENAI_SAMPLE_RATE:
logger.warning( logger.warning(
f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. " f"OpenAI TTS only supports {OPENAI_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {sample_rate}Hz may cause issues." f"Current rate of {sample_rate}Hz may cause issues."
) )
@@ -217,9 +216,9 @@ class OpenAITTSService(TTSService):
frame: The start frame containing initialization parameters. frame: The start frame containing initialization parameters.
""" """
await super().start(frame) await super().start(frame)
if self.sample_rate != self.OPENAI_SAMPLE_RATE: if self.sample_rate != OPENAI_SAMPLE_RATE:
logger.warning( logger.warning(
f"OpenAI TTS requires {self.OPENAI_SAMPLE_RATE}Hz sample rate. " f"OpenAI TTS requires {OPENAI_SAMPLE_RATE}Hz sample rate. "
f"Current rate of {self.sample_rate}Hz may cause issues." f"Current rate of {self.sample_rate}Hz may cause issues."
) )

3
tests/test_settings.py
View File

@@ -12,6 +12,7 @@ from pipecat.services.deepgram.sagemaker.stt import DeepgramSageMakerSTTSettings
from pipecat.services.deepgram.stt import DeepgramSTTService, DeepgramSTTSettings from pipecat.services.deepgram.stt import DeepgramSTTService, DeepgramSTTSettings
from pipecat.services.inworld.realtime import events as inworld_events from pipecat.services.inworld.realtime import events as inworld_events
from pipecat.services.inworld.realtime.llm import InworldRealtimeLLMSettings from pipecat.services.inworld.realtime.llm import InworldRealtimeLLMSettings
from pipecat.services.openai._constants import OPENAI_REALTIME_WHISPER_MODEL
from pipecat.services.openai.realtime import events from pipecat.services.openai.realtime import events
from pipecat.services.openai.realtime.llm import ( from pipecat.services.openai.realtime.llm import (
OpenAIRealtimeLLMService, OpenAIRealtimeLLMService,
@@ -757,7 +758,7 @@ class TestOpenAIRealtimeSessionProperties:
audio=events.AudioConfiguration( audio=events.AudioConfiguration(
input=events.AudioInput( input=events.AudioInput(
transcription=events.InputAudioTranscription( transcription=events.InputAudioTranscription(
model=events.GPT_REALTIME_WHISPER_MODEL, model=OPENAI_REALTIME_WHISPER_MODEL,
prompt="Keywords: metoprolol", prompt="Keywords: metoprolol",
) )
) )