wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add system_instruction parameter to run_inference (#3968)

Browse Source 
* Add system_instruction parameter to run_inference

Allow callers to provide a custom system instruction directly when calling
run_inference, without having to construct provider-specific context objects.

For OpenAI, the instruction is prepended as a system message (preserving
existing messages). For Anthropic, Google, and AWS Bedrock, it overrides the
single system field with a warning when an existing system instruction is
present in the context.

* Use system_instruction parameter in _generate_summary

Pass the summarization prompt via run_inference's system_instruction
parameter instead of embedding it as a system message in the context.

* Add changelog for #3968
This commit is contained in:
Mark Backman
2026-03-10 12:57:23 -04:00
committed by GitHub
parent 0817a57f4c
commit 912f1be31c
8 changed files with 332 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/pipecat/pipeline/llm_switcher.py
View File

@@ -52,17 +52,19 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
"""
return self.strategy.active_service
async def run_inference(self, context: LLMContext) -> Optional[str]:
async def run_inference(self, context: LLMContext, **kwargs) -> Optional[str]:
"""Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context, using the currently active LLM.
Args:
context: The LLM context containing conversation history.
**kwargs: Additional arguments forwarded to the active LLM's run_inference
(e.g. max_tokens, system_instruction).
Returns:
The LLM's response as a string, or None if no response is generated.
"""
if self.active_llm:
return await self.active_llm.run_inference(context=context)
return await self.active_llm.run_inference(context=context, **kwargs)
return None
def register_function(

16
src/pipecat/services/anthropic/llm.py
View File

@@ -346,7 +346,10 @@ class AnthropicLLMService(LLMService):
return response
async def run_inference(
self, context: LLMContext | OpenAILLMContext, max_tokens: Optional[int] = None
self,
context: LLMContext | OpenAILLMContext,
max_tokens: Optional[int] = None,
system_instruction: Optional[str] = None,
) -> Optional[str]:
"""Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context.
@@ -354,6 +357,8 @@ class AnthropicLLMService(LLMService):
context: The LLM context containing conversation history.
max_tokens: Optional maximum number of tokens to generate. If provided,
overrides the service's default max_tokens setting.
system_instruction: Optional system instruction to use for this inference.
If provided, overrides any system instruction in the context.
Returns:
The LLM's response as a string, or None if no response is generated.
@@ -375,6 +380,15 @@ class AnthropicLLMService(LLMService):
system = getattr(context, "system", NOT_GIVEN)
tools = context.tools or []
# Override system instruction if provided
if system_instruction is not None:
if system and system is not NOT_GIVEN:
logger.warning(
f"{self}: Both system_instruction and a system message in context are set."
" Using system_instruction."
)
system = system_instruction
# Build params using the same method as streaming completions
params = {
"model": self._settings.model,

16
src/pipecat/services/aws/llm.py
View File

@@ -923,7 +923,10 @@ class AWSBedrockLLMService(LLMService):
return inference_config
async def run_inference(
self, context: LLMContext | OpenAILLMContext, max_tokens: Optional[int] = None
self,
context: LLMContext | OpenAILLMContext,
max_tokens: Optional[int] = None,
system_instruction: Optional[str] = None,
) -> Optional[str]:
"""Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context.
@@ -931,6 +934,8 @@ class AWSBedrockLLMService(LLMService):
context: The LLM context containing conversation history.
max_tokens: Optional maximum number of tokens to generate. If provided,
overrides the service's default max_tokens setting.
system_instruction: Optional system instruction to use for this inference.
If provided, overrides any system instruction in the context.
Returns:
The LLM's response as a string, or None if no response is generated.
@@ -947,6 +952,15 @@ class AWSBedrockLLMService(LLMService):
messages = context.messages
system = getattr(context, "system", None) # [{"text": "system message"}]
# Override system instruction if provided
if system_instruction is not None:
if system:
logger.warning(
f"{self}: Both system_instruction and a system message in context are set."
" Using system_instruction."
)
system = [{"text": system_instruction}]
# Prepare request parameters using the same method as streaming
inference_config = self._build_inference_config()

16
src/pipecat/services/google/llm.py
View File

@@ -881,7 +881,10 @@ class GoogleLLMService(LLMService):
self._client = genai.Client(api_key=self._api_key, http_options=self._http_options)
async def run_inference(
self, context: LLMContext | OpenAILLMContext, max_tokens: Optional[int] = None
self,
context: LLMContext | OpenAILLMContext,
max_tokens: Optional[int] = None,
system_instruction: Optional[str] = None,
) -> Optional[str]:
"""Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context.
@@ -889,6 +892,8 @@ class GoogleLLMService(LLMService):
context: The LLM context containing conversation history.
max_tokens: Optional maximum number of tokens to generate. If provided,
overrides the service's default max_tokens setting.
system_instruction: Optional system instruction to use for this inference.
If provided, overrides any system instruction in the context.
Returns:
The LLM's response as a string, or None if no response is generated.
@@ -908,6 +913,15 @@ class GoogleLLMService(LLMService):
system = getattr(context, "system_message", None)
tools = context.tools or []
# Override system instruction if provided
if system_instruction is not None:
if system:
logger.warning(
f"{self}: Both system_instruction and a system message in context are set."
" Using system_instruction."
)
system = system_instruction
# Build generation config using the same method as streaming
generation_params = self._build_generation_params(
system_instruction=system, tools=tools if tools else None

25
src/pipecat/services/llm_service.py
View File

@@ -244,7 +244,10 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
return self.get_llm_adapter().create_llm_specific_message(message)
async def run_inference(
self, context: LLMContext | OpenAILLMContext, max_tokens: Optional[int] = None
self,
context: LLMContext | OpenAILLMContext,
max_tokens: Optional[int] = None,
system_instruction: Optional[str] = None,
) -> Optional[str]:
"""Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context.
@@ -254,6 +257,8 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
context: The LLM context containing conversation history.
max_tokens: Optional maximum number of tokens to generate. If provided,
overrides the service's default max_tokens/max_completion_tokens setting.
system_instruction: Optional system instruction to use for this inference.
If provided, overrides any system instruction in the context.
Returns:
The LLM's response as a string, or None if no response is generated.
@@ -535,23 +540,17 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
# Create summary context
transcript = LLMContextSummarizationUtil.format_messages_for_summary(result.messages)
prompt_messages = [
{
"role": "system",
"content": frame.summarization_prompt,
},
{
"role": "user",
"content": f"Conversation history:\n{transcript}",
},
]
summary_context = LLMContext(messages=prompt_messages)
summary_context = LLMContext(
messages=[{"role": "user", "content": f"Conversation history:\n{transcript}"}]
)
# Generate summary using run_inference
# This will be overridden by each LLM service implementation
try:
summary_text = await self.run_inference(
summary_context, max_tokens=frame.target_context_tokens
summary_context,
max_tokens=frame.target_context_tokens,
system_instruction=frame.summarization_prompt,
)
except NotImplementedError:
raise RuntimeError(

17
src/pipecat/services/openai/base_llm.py
View File

@@ -342,7 +342,10 @@ class BaseOpenAILLMService(LLMService):
return params
async def run_inference(
self, context: LLMContext | OpenAILLMContext, max_tokens: Optional[int] = None
self,
context: LLMContext | OpenAILLMContext,
max_tokens: Optional[int] = None,
system_instruction: Optional[str] = None,
) -> Optional[str]:
"""Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context.
@@ -350,6 +353,8 @@ class BaseOpenAILLMService(LLMService):
context: The LLM context containing conversation history.
max_tokens: Optional maximum number of tokens to generate. If provided,
overrides the service's default max_tokens/max_completion_tokens setting.
system_instruction: Optional system instruction to use for this inference.
If provided, overrides any system instruction in the context.
Returns:
The LLM's response as a string, or None if no response is generated.
@@ -371,6 +376,16 @@ class BaseOpenAILLMService(LLMService):
params["stream"] = False
params.pop("stream_options", None)
# Prepend system instruction if provided
if system_instruction is not None:
messages = params.get("messages", [])
if messages and messages[0].get("role") == "system":
logger.warning(
f"{self}: Both system_instruction and a system message in context are set."
" Using system_instruction."
)
params["messages"] = [{"role": "system", "content": system_instruction}] + messages
# Override max_tokens if provided
if max_tokens is not None:
# Use max_completion_tokens for newer models, fallback to max_tokens