wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4130 from pipecat-ai/pk/realtime-services-init-v-context-system-instructions-cleanup

Browse Source 
Prefer init-provided system instructions in realtime services
This commit is contained in:
kompfner
2026-03-25 09:13:52 -04:00
committed by GitHub
parent b80328e038 4bdfe1cf31
commit 86e086c6b5
10 changed files with 206 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/pipecat/adapters/services/aws_nova_sonic_adapter.py
View File

@@ -72,20 +72,26 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]):
"""Get the identifier used in LLMSpecificMessage instances for AWS Nova Sonic."""
return "aws-nova-sonic"
def get_llm_invocation_params(self, context: LLMContext) -> AWSNovaSonicLLMInvocationParams:
def get_llm_invocation_params(
self, context: LLMContext, *, system_instruction: Optional[str] = None
) -> AWSNovaSonicLLMInvocationParams:
"""Get AWS Nova Sonic-specific LLM invocation parameters from a universal LLM context.
This is a placeholder until support for universal LLMContext machinery is added for AWS Nova Sonic.
Args:
context: The LLM context containing messages, tools, etc.
system_instruction: Optional system instruction from service settings.
Returns:
Dictionary of parameters for invoking AWS Nova Sonic's LLM API.
"""
messages = self._from_universal_context_messages(self.get_messages(context))
effective_system = self._resolve_system_instruction(
messages.system_instruction,
system_instruction,
discard_context_system=True,
)
return {
"system_instruction": messages.system_instruction,
"system_instruction": effective_system,
"messages": messages.messages,
# NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
"tools": self.from_standard_tools(context.tools) or [],

12
src/pipecat/adapters/services/grok_realtime_adapter.py
View File

@@ -50,18 +50,26 @@ class GrokRealtimeLLMAdapter(BaseLLMAdapter):
"""Get the identifier used in LLMSpecificMessage instances for Grok Realtime."""
return "grok-realtime"
def get_llm_invocation_params(self, context: LLMContext) -> GrokRealtimeLLMInvocationParams:
def get_llm_invocation_params(
self, context: LLMContext, *, system_instruction: Optional[str] = None
) -> GrokRealtimeLLMInvocationParams:
"""Get Grok Realtime-specific LLM invocation parameters from a universal LLM context.
Args:
context: The LLM context containing messages, tools, etc.
system_instruction: Optional system instruction from service settings.
Returns:
Dictionary of parameters for invoking Grok's Voice Agent API.
"""
messages = self._from_universal_context_messages(self.get_messages(context))
effective_system = self._resolve_system_instruction(
messages.system_instruction,
system_instruction,
discard_context_system=True,
)
return {
"system_instruction": messages.system_instruction,
"system_instruction": effective_system,
"messages": messages.messages,
"tools": self.from_standard_tools(context.tools) or [],
}

14
src/pipecat/adapters/services/open_ai_realtime_adapter.py
View File

@@ -43,20 +43,26 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
"""Get the identifier used in LLMSpecificMessage instances for OpenAI Realtime."""
return "openai-realtime"
def get_llm_invocation_params(self, context: LLMContext) -> OpenAIRealtimeLLMInvocationParams:
def get_llm_invocation_params(
self, context: LLMContext, *, system_instruction: Optional[str] = None
) -> OpenAIRealtimeLLMInvocationParams:
"""Get OpenAI Realtime-specific LLM invocation parameters from a universal LLM context.
This is a placeholder until support for universal LLMContext machinery is added for OpenAI Realtime.
Args:
context: The LLM context containing messages, tools, etc.
system_instruction: Optional system instruction from service settings.
Returns:
Dictionary of parameters for invoking OpenAI Realtime's API.
"""
messages = self._from_universal_context_messages(self.get_messages(context))
effective_system = self._resolve_system_instruction(
messages.system_instruction,
system_instruction,
discard_context_system=True,
)
return {
"system_instruction": messages.system_instruction,
"system_instruction": effective_system,
"messages": messages.messages,
# NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
"tools": self.from_standard_tools(context.tools) or [],

13
src/pipecat/services/aws/nova_sonic/llm.py
View File

@@ -629,7 +629,9 @@ class AWSNovaSonicLLMService(LLMService):
# Read context
adapter: AWSNovaSonicLLMAdapter = self.get_llm_adapter()
llm_connection_params = adapter.get_llm_invocation_params(self._context)
llm_connection_params = adapter.get_llm_invocation_params(
self._context, system_instruction=self._settings.system_instruction
)
# Send prompt start event, specifying tools.
# Tools from context take priority over self._tools.
@@ -642,12 +644,9 @@ class AWSNovaSonicLLMService(LLMService):
await self._send_prompt_start_event(tools)
# Send system instruction.
# Instruction from context takes priority over self._settings.system_instruction.
system_instruction = (
llm_connection_params["system_instruction"]
if llm_connection_params["system_instruction"]
else self._settings.system_instruction
)
# The adapter resolves conflicts between init-provided and
# context-provided system instructions (preferring init-provided).
system_instruction = llm_connection_params["system_instruction"]
logger.debug(f"Using system instruction: {system_instruction}")
if system_instruction:
await self._send_text_event(text=system_instruction, role=Role.SYSTEM)

34
src/pipecat/services/google/gemini_live/llm.py
View File

@@ -1080,28 +1080,26 @@ class GeminiLiveLLMService(LLMService):
# We got our initial context
self._context = context
# If context contains system instruction or tools, reconnect in
# order to apply them.
# (Context-provided system instruction and tools take precedence
# over the ones provided at initialization time. Note that we could
# do more sophisticated comparisons here, but for now this is
# sufficient: we'll assume folks won't mean to provide these
# settings both in the context and at initialization time. In a
# future change, we could/should implement the ability to swap
# these settings at any point).
# Reconnect if context changes the effective system instruction
# or tools compared to the initial connection (which used the
# init-provided values). Note that the determination of "effective"
# system instruction is delegated to the adapter, which still
# chooses the init-provided value if there is one.
adapter: GeminiLLMAdapter = self.get_llm_adapter()
params = adapter.get_llm_invocation_params(self._context)
params = adapter.get_llm_invocation_params(
self._context, system_instruction=self._system_instruction_from_init
)
system_instruction = params["system_instruction"]
tools = params["tools"]
if system_instruction and self._system_instruction_from_init:
logger.warning(
"System instruction provided both at init time and in context; using context-provided value."
)
system_instruction_changed = system_instruction != self._system_instruction_from_init
if tools and self._tools_from_init:
logger.warning(
"Tools provided both at init time and in context; using context-provided value."
)
if system_instruction or tools:
# For tools we simply check presence rather than diffing against
# init-provided tools, assuming that if context provides tools
# they warrant a reconnect.
if system_instruction_changed or tools:
await self._reconnect()
# Initialize our bookkeeping of already-completed tool calls in
@@ -1322,10 +1320,12 @@ class GeminiLiveLLMService(LLMService):
system_instruction = None
tools = None
if self._context:
params = adapter.get_llm_invocation_params(self._context)
params = adapter.get_llm_invocation_params(
self._context, system_instruction=self._system_instruction_from_init
)
system_instruction = params["system_instruction"]
tools = params["tools"]
if not system_instruction:
else:
system_instruction = self._system_instruction_from_init
if not tools:
tools = adapter.from_standard_tools(self._tools_from_init)

6
src/pipecat/services/grok/realtime/llm.py
View File

@@ -607,11 +607,15 @@ class GrokRealtimeLLMService(LLMService):
adapter: GrokRealtimeLLMAdapter = self.get_llm_adapter()
if self._context:
llm_invocation_params = adapter.get_llm_invocation_params(self._context)
llm_invocation_params = adapter.get_llm_invocation_params(
self._context, system_instruction=self._settings.system_instruction
)
if llm_invocation_params["tools"]:
settings.tools = llm_invocation_params["tools"]
# The adapter resolves conflicts between init-provided and
# context-provided system instructions (preferring init-provided).
if llm_invocation_params["system_instruction"]:
settings.instructions = llm_invocation_params["system_instruction"]

8
src/pipecat/services/openai/realtime/llm.py
View File

@@ -687,14 +687,16 @@ class OpenAIRealtimeLLMService(LLMService):
adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter()
if self._context:
llm_invocation_params = adapter.get_llm_invocation_params(self._context)
llm_invocation_params = adapter.get_llm_invocation_params(
self._context, system_instruction=self._settings.system_instruction
)
# tools given in the context override the tools in the session properties
if llm_invocation_params["tools"]:
settings.tools = llm_invocation_params["tools"]
# instructions in the context come from an initial "system" message in the
# messages list, and override instructions in the session properties
# The adapter resolves conflicts between init-provided and
# context-provided system instructions (preferring init-provided).
if llm_invocation_params["system_instruction"]:
settings.instructions = llm_invocation_params["system_instruction"]