From 56a56a417482cf8d866a32f87d11910de40406d9 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 24 Mar 2026 17:06:56 -0400 Subject: [PATCH 1/6] Prefer init-provided system instruction in Gemini Live Pass self._system_instruction_from_init to the adapter's get_llm_invocation_params(), which calls _resolve_system_instruction() to prefer init-provided over context-provided system instructions and warn on conflicts. Previously context-provided took precedence. Also fix the reconnect check to only reconnect when the resolved system instruction actually differs from what the initial connection used, avoiding unnecessary reconnects. --- .../services/google/gemini_live/llm.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 3e282e7ee..bb916a7cf 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -1078,28 +1078,26 @@ class GeminiLiveLLMService(LLMService): # We got our initial context self._context = context - # If context contains system instruction or tools, reconnect in - # order to apply them. - # (Context-provided system instruction and tools take precedence - # over the ones provided at initialization time. Note that we could - # do more sophisticated comparisons here, but for now this is - # sufficient: we'll assume folks won't mean to provide these - # settings both in the context and at initialization time. In a - # future change, we could/should implement the ability to swap - # these settings at any point). + # Reconnect if context changes the effective system instruction + # or tools compared to the initial connection (which used the + # init-provided values). Note that the determination of "effective" + # system instruction is delegated to the adapter, which still + # chooses the init-provided value if there is one. adapter: GeminiLLMAdapter = self.get_llm_adapter() - params = adapter.get_llm_invocation_params(self._context) + params = adapter.get_llm_invocation_params( + self._context, system_instruction=self._system_instruction_from_init + ) system_instruction = params["system_instruction"] tools = params["tools"] - if system_instruction and self._system_instruction_from_init: - logger.warning( - "System instruction provided both at init time and in context; using context-provided value." - ) + system_instruction_changed = system_instruction != self._system_instruction_from_init if tools and self._tools_from_init: logger.warning( "Tools provided both at init time and in context; using context-provided value." ) - if system_instruction or tools: + # For tools we simply check presence rather than diffing against + # init-provided tools, assuming that if context provides tools + # they warrant a reconnect. + if system_instruction_changed or tools: await self._reconnect() # Initialize our bookkeeping of already-completed tool calls in @@ -1281,10 +1279,12 @@ class GeminiLiveLLMService(LLMService): system_instruction = None tools = None if self._context: - params = adapter.get_llm_invocation_params(self._context) + params = adapter.get_llm_invocation_params( + self._context, system_instruction=self._system_instruction_from_init + ) system_instruction = params["system_instruction"] tools = params["tools"] - if not system_instruction: + else: system_instruction = self._system_instruction_from_init if not tools: tools = adapter.from_standard_tools(self._tools_from_init) From 39329aaddba24f73c347de35d3e686d84408817c Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 24 Mar 2026 17:18:44 -0400 Subject: [PATCH 2/6] Prefer init-provided system instruction in Nova Sonic Add system_instruction parameter to the Nova Sonic adapter's get_llm_invocation_params() and call _resolve_system_instruction() to prefer init-provided over context-provided system instructions and warn on conflicts. Previously context-provided took precedence. Remove the service-side fallback logic, as the adapter now handles resolution. --- .../adapters/services/aws_nova_sonic_adapter.py | 14 ++++++++++---- src/pipecat/services/aws/nova_sonic/llm.py | 13 ++++++------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py index 657c83bf9..492e02db6 100644 --- a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py +++ b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py @@ -72,20 +72,26 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): """Get the identifier used in LLMSpecificMessage instances for AWS Nova Sonic.""" return "aws-nova-sonic" - def get_llm_invocation_params(self, context: LLMContext) -> AWSNovaSonicLLMInvocationParams: + def get_llm_invocation_params( + self, context: LLMContext, *, system_instruction: Optional[str] = None + ) -> AWSNovaSonicLLMInvocationParams: """Get AWS Nova Sonic-specific LLM invocation parameters from a universal LLM context. - This is a placeholder until support for universal LLMContext machinery is added for AWS Nova Sonic. - Args: context: The LLM context containing messages, tools, etc. + system_instruction: Optional system instruction from service settings. Returns: Dictionary of parameters for invoking AWS Nova Sonic's LLM API. """ messages = self._from_universal_context_messages(self.get_messages(context)) + effective_system = self._resolve_system_instruction( + messages.system_instruction, + system_instruction, + discard_context_system=True, + ) return { - "system_instruction": messages.system_instruction, + "system_instruction": effective_system, "messages": messages.messages, # NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN) "tools": self.from_standard_tools(context.tools) or [], diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index ffcbb5e5d..3541946c7 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -629,7 +629,9 @@ class AWSNovaSonicLLMService(LLMService): # Read context adapter: AWSNovaSonicLLMAdapter = self.get_llm_adapter() - llm_connection_params = adapter.get_llm_invocation_params(self._context) + llm_connection_params = adapter.get_llm_invocation_params( + self._context, system_instruction=self._settings.system_instruction + ) # Send prompt start event, specifying tools. # Tools from context take priority over self._tools. @@ -642,12 +644,9 @@ class AWSNovaSonicLLMService(LLMService): await self._send_prompt_start_event(tools) # Send system instruction. - # Instruction from context takes priority over self._settings.system_instruction. - system_instruction = ( - llm_connection_params["system_instruction"] - if llm_connection_params["system_instruction"] - else self._settings.system_instruction - ) + # The adapter resolves conflicts between init-provided and + # context-provided system instructions (preferring init-provided). + system_instruction = llm_connection_params["system_instruction"] logger.debug(f"Using system instruction: {system_instruction}") if system_instruction: await self._send_text_event(text=system_instruction, role=Role.SYSTEM) From e7dd84b552e87b3dd06c0e1639ee9d8290c57b28 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 24 Mar 2026 17:21:53 -0400 Subject: [PATCH 3/6] Prefer init-provided system instruction in OpenAI Realtime Add system_instruction parameter to the OpenAI Realtime adapter's get_llm_invocation_params() and call _resolve_system_instruction() to prefer init-provided over context-provided system instructions and warn on conflicts. Previously context-provided took precedence. --- .../adapters/services/open_ai_realtime_adapter.py | 14 ++++++++++---- src/pipecat/services/openai/realtime/llm.py | 8 +++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/pipecat/adapters/services/open_ai_realtime_adapter.py b/src/pipecat/adapters/services/open_ai_realtime_adapter.py index 6bea0e2ed..3c394d99b 100644 --- a/src/pipecat/adapters/services/open_ai_realtime_adapter.py +++ b/src/pipecat/adapters/services/open_ai_realtime_adapter.py @@ -43,20 +43,26 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): """Get the identifier used in LLMSpecificMessage instances for OpenAI Realtime.""" return "openai-realtime" - def get_llm_invocation_params(self, context: LLMContext) -> OpenAIRealtimeLLMInvocationParams: + def get_llm_invocation_params( + self, context: LLMContext, *, system_instruction: Optional[str] = None + ) -> OpenAIRealtimeLLMInvocationParams: """Get OpenAI Realtime-specific LLM invocation parameters from a universal LLM context. - This is a placeholder until support for universal LLMContext machinery is added for OpenAI Realtime. - Args: context: The LLM context containing messages, tools, etc. + system_instruction: Optional system instruction from service settings. Returns: Dictionary of parameters for invoking OpenAI Realtime's API. """ messages = self._from_universal_context_messages(self.get_messages(context)) + effective_system = self._resolve_system_instruction( + messages.system_instruction, + system_instruction, + discard_context_system=True, + ) return { - "system_instruction": messages.system_instruction, + "system_instruction": effective_system, "messages": messages.messages, # NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN) "tools": self.from_standard_tools(context.tools) or [], diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index bd31369ae..293ee0d87 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -687,14 +687,16 @@ class OpenAIRealtimeLLMService(LLMService): adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter() if self._context: - llm_invocation_params = adapter.get_llm_invocation_params(self._context) + llm_invocation_params = adapter.get_llm_invocation_params( + self._context, system_instruction=self._settings.system_instruction + ) # tools given in the context override the tools in the session properties if llm_invocation_params["tools"]: settings.tools = llm_invocation_params["tools"] - # instructions in the context come from an initial "system" message in the - # messages list, and override instructions in the session properties + # The adapter resolves conflicts between init-provided and + # context-provided system instructions (preferring init-provided). if llm_invocation_params["system_instruction"]: settings.instructions = llm_invocation_params["system_instruction"] From ac2b1ecd4788bce2f608f6bcf67055dbf708a618 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 24 Mar 2026 17:29:19 -0400 Subject: [PATCH 4/6] Prefer init-provided system instruction in Grok Realtime Add system_instruction parameter to the Grok Realtime adapter's get_llm_invocation_params() and call _resolve_system_instruction() to prefer init-provided over context-provided system instructions and warn on conflicts. Previously context-provided took precedence. Update the Grok Realtime example to use settings.system_instruction instead of session_properties.instructions. --- examples/foundational/51-grok-realtime.py | 33 +++++++++---------- .../services/grok_realtime_adapter.py | 12 +++++-- src/pipecat/services/grok/realtime/llm.py | 6 +++- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/examples/foundational/51-grok-realtime.py b/examples/foundational/51-grok-realtime.py index a233b85bc..8784359f0 100644 --- a/examples/foundational/51-grok-realtime.py +++ b/examples/foundational/51-grok-realtime.py @@ -172,23 +172,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): session_properties = SessionProperties( # Voice options: Ara, Rex, Sal, Eve, Leo voice="Ara", - # System instructions - instructions="""You are a helpful and friendly AI assistant powered by Grok. - -You have access to several tools: -- Weather information -- Current time -- Restaurant recommendations -- Web search (built-in) -- X/Twitter search (built-in) - -Your voice and personality should be warm and engaging. Keep your responses -concise and conversational since this is a voice interaction. - -If the user asks about current events or news, use web search. -If they ask about what people are saying on social media, use X search. - -Always be helpful and proactive in offering assistance.""", # Grok-specific built-in tools can be added here: # tools=[ # WebSearchTool(), # Enable web search @@ -200,6 +183,22 @@ Always be helpful and proactive in offering assistance.""", llm = GrokRealtimeLLMService( api_key=os.getenv("GROK_API_KEY"), settings=GrokRealtimeLLMService.Settings( + system_instruction="""You are a helpful and friendly AI assistant powered by Grok. + + You have access to several tools: + - Weather information + - Current time + - Restaurant recommendations + - Web search (built-in) + - X/Twitter search (built-in) + + Your voice and personality should be warm and engaging. Keep your responses + concise and conversational since this is a voice interaction. + + If the user asks about current events or news, use web search. + If they ask about what people are saying on social media, use X search. + + Always be helpful and proactive in offering assistance.""", session_properties=session_properties, ), ) diff --git a/src/pipecat/adapters/services/grok_realtime_adapter.py b/src/pipecat/adapters/services/grok_realtime_adapter.py index 07b1de843..b95efb62c 100644 --- a/src/pipecat/adapters/services/grok_realtime_adapter.py +++ b/src/pipecat/adapters/services/grok_realtime_adapter.py @@ -50,18 +50,26 @@ class GrokRealtimeLLMAdapter(BaseLLMAdapter): """Get the identifier used in LLMSpecificMessage instances for Grok Realtime.""" return "grok-realtime" - def get_llm_invocation_params(self, context: LLMContext) -> GrokRealtimeLLMInvocationParams: + def get_llm_invocation_params( + self, context: LLMContext, *, system_instruction: Optional[str] = None + ) -> GrokRealtimeLLMInvocationParams: """Get Grok Realtime-specific LLM invocation parameters from a universal LLM context. Args: context: The LLM context containing messages, tools, etc. + system_instruction: Optional system instruction from service settings. Returns: Dictionary of parameters for invoking Grok's Voice Agent API. """ messages = self._from_universal_context_messages(self.get_messages(context)) + effective_system = self._resolve_system_instruction( + messages.system_instruction, + system_instruction, + discard_context_system=True, + ) return { - "system_instruction": messages.system_instruction, + "system_instruction": effective_system, "messages": messages.messages, "tools": self.from_standard_tools(context.tools) or [], } diff --git a/src/pipecat/services/grok/realtime/llm.py b/src/pipecat/services/grok/realtime/llm.py index 6e37d21d2..1317e7269 100644 --- a/src/pipecat/services/grok/realtime/llm.py +++ b/src/pipecat/services/grok/realtime/llm.py @@ -607,11 +607,15 @@ class GrokRealtimeLLMService(LLMService): adapter: GrokRealtimeLLMAdapter = self.get_llm_adapter() if self._context: - llm_invocation_params = adapter.get_llm_invocation_params(self._context) + llm_invocation_params = adapter.get_llm_invocation_params( + self._context, system_instruction=self._settings.system_instruction + ) if llm_invocation_params["tools"]: settings.tools = llm_invocation_params["tools"] + # The adapter resolves conflicts between init-provided and + # context-provided system instructions (preferring init-provided). if llm_invocation_params["system_instruction"]: settings.instructions = llm_invocation_params["system_instruction"] From bb33045389fea76d47afaca9730cd09a8f43604e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 24 Mar 2026 17:30:35 -0400 Subject: [PATCH 5/6] Add system instruction conflict resolution tests for realtime adapters Test that OpenAI Realtime, Grok Realtime, and Nova Sonic adapters prefer init-provided system_instruction over context-provided, warn on conflicts, and don't warn for developer messages. --- tests/test_get_llm_invocation_params.py | 126 ++++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/tests/test_get_llm_invocation_params.py b/tests/test_get_llm_invocation_params.py index 883c60631..6f0015ada 100644 --- a/tests/test_get_llm_invocation_params.py +++ b/tests/test_get_llm_invocation_params.py @@ -2085,6 +2085,48 @@ class TestOpenAIRealtimeGetLLMInvocationParams(unittest.TestCase): self.assertEqual(params["messages"], []) self.assertIsNone(params["system_instruction"]) + def test_both_system_instruction_and_system_message_warns(self): + """system_instruction + initial system message warns and uses system_instruction.""" + messages: list[LLMStandardMessage] = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello"}, + ] + context = LLMContext(messages=messages) + + with patch("pipecat.adapters.base_llm_adapter.logger") as mock_logger: + params = self.adapter.get_llm_invocation_params( + context, system_instruction="Be concise." + ) + mock_logger.warning.assert_called_once() + + self.assertEqual(params["system_instruction"], "Be concise.") + + def test_both_system_instruction_and_developer_message_no_warning(self): + """system_instruction + initial developer message: no warning, developer becomes user.""" + messages: list[LLMStandardMessage] = [ + {"role": "developer", "content": "Extra context."}, + {"role": "user", "content": "Hello"}, + ] + context = LLMContext(messages=messages) + + with patch("pipecat.adapters.base_llm_adapter.logger") as mock_logger: + params = self.adapter.get_llm_invocation_params( + context, system_instruction="Be concise." + ) + mock_logger.warning.assert_not_called() + + self.assertEqual(params["system_instruction"], "Be concise.") + + def test_system_instruction_only(self): + """system_instruction without context system message returns system_instruction.""" + messages: list[LLMStandardMessage] = [ + {"role": "user", "content": "Hello"}, + ] + context = LLMContext(messages=messages) + params = self.adapter.get_llm_invocation_params(context, system_instruction="Be concise.") + + self.assertEqual(params["system_instruction"], "Be concise.") + class TestGrokRealtimeGetLLMInvocationParams(unittest.TestCase): def setUp(self) -> None: @@ -2135,6 +2177,48 @@ class TestGrokRealtimeGetLLMInvocationParams(unittest.TestCase): self.assertEqual(params["messages"], []) self.assertIsNone(params["system_instruction"]) + def test_both_system_instruction_and_system_message_warns(self): + """system_instruction + initial system message warns and uses system_instruction.""" + messages: list[LLMStandardMessage] = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello"}, + ] + context = LLMContext(messages=messages) + + with patch("pipecat.adapters.base_llm_adapter.logger") as mock_logger: + params = self.adapter.get_llm_invocation_params( + context, system_instruction="Be concise." + ) + mock_logger.warning.assert_called_once() + + self.assertEqual(params["system_instruction"], "Be concise.") + + def test_both_system_instruction_and_developer_message_no_warning(self): + """system_instruction + initial developer message: no warning, developer becomes user.""" + messages: list[LLMStandardMessage] = [ + {"role": "developer", "content": "Extra context."}, + {"role": "user", "content": "Hello"}, + ] + context = LLMContext(messages=messages) + + with patch("pipecat.adapters.base_llm_adapter.logger") as mock_logger: + params = self.adapter.get_llm_invocation_params( + context, system_instruction="Be concise." + ) + mock_logger.warning.assert_not_called() + + self.assertEqual(params["system_instruction"], "Be concise.") + + def test_system_instruction_only(self): + """system_instruction without context system message returns system_instruction.""" + messages: list[LLMStandardMessage] = [ + {"role": "user", "content": "Hello"}, + ] + context = LLMContext(messages=messages) + params = self.adapter.get_llm_invocation_params(context, system_instruction="Be concise.") + + self.assertEqual(params["system_instruction"], "Be concise.") + class TestAWSNovaSonicGetLLMInvocationParams(unittest.TestCase): def setUp(self) -> None: @@ -2179,6 +2263,48 @@ class TestAWSNovaSonicGetLLMInvocationParams(unittest.TestCase): # Developer becomes user, plus assistant self.assertEqual(len(params["messages"]), 2) + def test_both_system_instruction_and_system_message_warns(self): + """system_instruction + initial system message warns and uses system_instruction.""" + messages: list[LLMStandardMessage] = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello"}, + ] + context = LLMContext(messages=messages) + + with patch("pipecat.adapters.base_llm_adapter.logger") as mock_logger: + params = self.adapter.get_llm_invocation_params( + context, system_instruction="Be concise." + ) + mock_logger.warning.assert_called_once() + + self.assertEqual(params["system_instruction"], "Be concise.") + + def test_both_system_instruction_and_developer_message_no_warning(self): + """system_instruction + initial developer message: no warning, developer becomes user.""" + messages: list[LLMStandardMessage] = [ + {"role": "developer", "content": "Extra context."}, + {"role": "user", "content": "Hello"}, + ] + context = LLMContext(messages=messages) + + with patch("pipecat.adapters.base_llm_adapter.logger") as mock_logger: + params = self.adapter.get_llm_invocation_params( + context, system_instruction="Be concise." + ) + mock_logger.warning.assert_not_called() + + self.assertEqual(params["system_instruction"], "Be concise.") + + def test_system_instruction_only(self): + """system_instruction without context system message returns system_instruction.""" + messages: list[LLMStandardMessage] = [ + {"role": "user", "content": "Hello"}, + ] + context = LLMContext(messages=messages) + params = self.adapter.get_llm_invocation_params(context, system_instruction="Be concise.") + + self.assertEqual(params["system_instruction"], "Be concise.") + class TestBaseLLMAdapterHelpers(unittest.TestCase): """Tests for the shared helper methods on BaseLLMAdapter.""" From 4bdfe1cf31f5322f6e4bd39e60f817061b7f7319 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 24 Mar 2026 17:34:50 -0400 Subject: [PATCH 6/6] Add changelog for realtime system instruction preference change --- changelog/4130.changed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/4130.changed.md diff --git a/changelog/4130.changed.md b/changelog/4130.changed.md new file mode 100644 index 000000000..66b38ffb7 --- /dev/null +++ b/changelog/4130.changed.md @@ -0,0 +1 @@ +- ⚠️ Realtime services (Gemini Live, OpenAI Realtime, Grok Realtime, Nova Sonic) now prefer `system_instruction` from service settings over an initial system message in the LLM context, matching the behavior of non-realtime services. Previously, context-provided system instructions took precedence. A warning is now logged when both are set.