From 775b1539f388ca2689f33993a6545f0b0cfefa1a Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 5 Mar 2026 15:33:41 -0500 Subject: [PATCH] Add `system_instruction` to realtime service settings Add `system_instruction=None` to `default_settings` for OpenAIRealtimeLLMService, GrokRealtimeLLMService, UltravoxRealtimeLLMService, AWSNovaSonicLLMService (Azure inherits from OpenAI), and OpenAIRealtimeBetaLLMService (Azure Beta inherits from OpenAI Beta). Deprecate `system_instruction` init arg in AWSNovaSonicLLMService in favor of `settings=AWSNovaSonicLLMSettings(system_instruction=...)`. Use `self._settings.system_instruction` directly instead of storing a separate `self._system_instruction`. Deprecation of `params` and `session_properties` in favor of `settings` for realtime services will be tackled in future work. --- src/pipecat/services/aws/nova_sonic/llm.py | 14 +++++++++++--- src/pipecat/services/grok/realtime/llm.py | 1 + src/pipecat/services/openai/realtime/llm.py | 1 + .../services/openai_realtime_beta/openai.py | 1 + src/pipecat/services/ultravox/llm.py | 1 + 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 9103ebdfd..fd13fe56e 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -261,6 +261,9 @@ class AWSNovaSonicLLMService(LLMService): deprecated top-level parameters, the ``settings`` values take precedence. system_instruction: System-level instruction for the model. + + .. deprecated:: 0.0.105 + Use ``settings=AWSNovaSonicLLMSettings(system_instruction=...)`` instead. tools: Available tools/functions for the model to use. send_transcription_frames: Whether to emit transcription frames. @@ -273,6 +276,7 @@ class AWSNovaSonicLLMService(LLMService): # 1. Initialize default_settings with hardcoded defaults default_settings = AWSNovaSonicLLMSettings( model="amazon.nova-2-sonic-v1:0", + system_instruction=None, voice="matthew", temperature=0.7, max_tokens=1024, @@ -293,6 +297,11 @@ class AWSNovaSonicLLMService(LLMService): if voice_id != "matthew": _warn_deprecated_param("voice_id", AWSNovaSonicLLMSettings, "voice") default_settings.voice = voice_id + if system_instruction is not None: + _warn_deprecated_param( + "system_instruction", AWSNovaSonicLLMSettings, "system_instruction" + ) + default_settings.system_instruction = system_instruction # 3. Apply params overrides — only if settings not provided if params is not None: @@ -325,7 +334,6 @@ class AWSNovaSonicLLMService(LLMService): self._output_sample_rate = _audio_params.output_sample_rate self._output_sample_size = _audio_params.output_sample_size self._output_channel_count = _audio_params.output_channel_count - self._system_instruction = system_instruction self._tools = tools # Validate endpointing_sensitivity parameter @@ -625,11 +633,11 @@ class AWSNovaSonicLLMService(LLMService): await self._send_prompt_start_event(tools) # Send system instruction. - # Instruction from context takes priority over self._system_instruction. + # Instruction from context takes priority over self._settings.system_instruction. system_instruction = ( llm_connection_params["system_instruction"] if llm_connection_params["system_instruction"] - else self._system_instruction + else self._settings.system_instruction ) logger.debug(f"Using system instruction: {system_instruction}") if system_instruction: diff --git a/src/pipecat/services/grok/realtime/llm.py b/src/pipecat/services/grok/realtime/llm.py index 638477b5e..074b44c5d 100644 --- a/src/pipecat/services/grok/realtime/llm.py +++ b/src/pipecat/services/grok/realtime/llm.py @@ -144,6 +144,7 @@ class GrokRealtimeLLMService(LLMService): # 1. Initialize default_settings with hardcoded defaults default_settings = GrokRealtimeLLMSettings( model=None, + system_instruction=None, temperature=None, max_tokens=None, top_p=None, diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 761d7244b..b5d9dc8e2 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -170,6 +170,7 @@ class OpenAIRealtimeLLMService(LLMService): # 1. Initialize default_settings with hardcoded defaults default_settings = OpenAIRealtimeLLMSettings( model="gpt-realtime-1.5", + system_instruction=None, temperature=None, max_tokens=None, top_p=None, diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 67a892e6f..a3f8e47fc 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -158,6 +158,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): # 1. Initialize default_settings with hardcoded defaults default_settings = OpenAIRealtimeBetaLLMSettings( model="gpt-4o-realtime-preview-2025-06-03", + system_instruction=None, temperature=None, max_tokens=None, top_p=None, diff --git a/src/pipecat/services/ultravox/llm.py b/src/pipecat/services/ultravox/llm.py index 094d22827..eccc1a864 100644 --- a/src/pipecat/services/ultravox/llm.py +++ b/src/pipecat/services/ultravox/llm.py @@ -189,6 +189,7 @@ class UltravoxRealtimeLLMService(LLMService): # 1. Initialize default_settings with hardcoded defaults default_settings = UltravoxRealtimeLLMSettings( model=None, + system_instruction=None, temperature=None, max_tokens=None, top_p=None,