wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move system_instruction into LLMSettings

Browse Source 
Add `system_instruction` field to `LLMSettings` so it is runtime-updatable via settings.
For Google (GoogleLLMService, GoogleVertexLLMService), deprecate the init-time arg since it was already shipped. For Anthropic, AWS Bedrock, and OpenAI, remove the init-time arg entirely since it was never shipped.

Still need to handle realtime services (OpenAI Realtime, Grok Realtime, Gemini Live).
This commit is contained in:
Paul Kompfner
2026-03-05 14:03:32 -05:00
parent ee2895a783
commit 78deaa735d
223 changed files with 860 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/pipecat/services/anthropic/llm.py
View File

@@ -223,7 +223,6 @@ class AnthropicLLMService(LLMService):
client=None,
retry_timeout_secs: Optional[float] = 5.0,
retry_on_timeout: Optional[bool] = False,
system_instruction: Optional[str] = None,
**kwargs,
):
"""Initialize the Anthropic LLM service.
@@ -246,12 +245,12 @@ class AnthropicLLMService(LLMService):
client: Optional custom Anthropic client instance.
retry_timeout_secs: Request timeout in seconds for retry logic.
retry_on_timeout: Whether to retry the request once if it times out.
system_instruction: Optional system instruction to use as the system prompt.
**kwargs: Additional arguments passed to parent LLMService.
"""
# 1. Initialize default_settings with hardcoded defaults
default_settings = AnthropicLLMSettings(
model="claude-sonnet-4-6",
system_instruction=None,
max_tokens=4096,
enable_prompt_caching=False,
temperature=NOT_GIVEN,
@@ -309,9 +308,8 @@ class AnthropicLLMService(LLMService):
) # if the client is provided, use it and remove it, otherwise create a new one
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self._system_instruction = system_instruction
if self._system_instruction:
logger.debug(f"{self}: Using system instruction: {self._system_instruction}")
if self._settings.system_instruction:
logger.debug(f"{self}: Using system instruction: {self._settings.system_instruction}")
def can_generate_metrics(self) -> bool:
"""Check if this service can generate usage metrics.
@@ -445,13 +443,13 @@ class AnthropicLLMService(LLMService):
params: AnthropicLLMInvocationParams = adapter.get_llm_invocation_params(
context, enable_prompt_caching=self._settings.enable_prompt_caching
)
if self._system_instruction:
if self._settings.system_instruction:
if params["system"] is not NOT_GIVEN:
logger.warning(
f"{self}: Both system_instruction and a system message in context are"
" set. Using system_instruction."
)
params["system"] = self._system_instruction
params["system"] = self._settings.system_instruction
return params
# Anthropic-specific context

12
src/pipecat/services/aws/llm.py
View File

@@ -788,7 +788,6 @@ class AWSBedrockLLMService(LLMService):
client_config: Optional[Config] = None,
retry_timeout_secs: Optional[float] = 5.0,
retry_on_timeout: Optional[bool] = False,
system_instruction: Optional[str] = None,
**kwargs,
):
"""Initialize the AWS Bedrock LLM service.
@@ -819,12 +818,12 @@ class AWSBedrockLLMService(LLMService):
client_config: Custom boto3 client configuration.
retry_timeout_secs: Request timeout in seconds for retry logic.
retry_on_timeout: Whether to retry the request once if it times out.
system_instruction: Optional system instruction to use as the system prompt.
**kwargs: Additional arguments passed to parent LLMService.
"""
# 1. Initialize default_settings with hardcoded defaults
default_settings = AWSBedrockLLMSettings(
model="us.amazon.nova-lite-v1:0",
system_instruction=None,
max_tokens=None,
temperature=None,
top_p=None,
@@ -889,11 +888,10 @@ class AWSBedrockLLMService(LLMService):
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self._system_instruction = system_instruction
logger.info(f"Using AWS Bedrock model: {self._settings.model}")
if self._system_instruction:
logger.debug(f"{self}: Using system instruction: {self._system_instruction}")
if self._settings.system_instruction:
logger.debug(f"{self}: Using system instruction: {self._settings.system_instruction}")
def can_generate_metrics(self) -> bool:
"""Check if the service can generate usage metrics.
@@ -1074,13 +1072,13 @@ class AWSBedrockLLMService(LLMService):
if isinstance(context, LLMContext):
adapter: AWSBedrockLLMAdapter = self.get_llm_adapter()
params: AWSBedrockLLMInvocationParams = adapter.get_llm_invocation_params(context)
if self._system_instruction:
if self._settings.system_instruction:
if params["system"]:
logger.warning(
f"{self}: Both system_instruction and a system message in context are"
" set. Using system_instruction."
)
params["system"] = [{"text": self._system_instruction}]
params["system"] = [{"text": self._settings.system_instruction}]
return params
# AWS Bedrock-specific context

8
src/pipecat/services/cerebras/llm.py
View File

@@ -111,4 +111,12 @@ class CerebrasLLMService(OpenAILLMService):
params.update(params_from_context)
params.update(self._settings.extra)
# Prepend system instruction if set
if self._settings.system_instruction:
messages = params.get("messages", [])
params["messages"] = [
{"role": "system", "content": self._settings.system_instruction}
] + messages
return params

8
src/pipecat/services/fireworks/llm.py
View File

@@ -112,4 +112,12 @@ class FireworksLLMService(OpenAILLMService):
params.update(params_from_context)
params.update(self._settings.extra)
# Prepend system instruction if set
if self._settings.system_instruction:
messages = params.get("messages", [])
params["messages"] = [
{"role": "system", "content": self._settings.system_instruction}
] + messages
return params

16
src/pipecat/services/google/llm.py
View File

@@ -809,6 +809,9 @@ class GoogleLLMService(LLMService):
deprecated parameters and *settings* are provided, *settings*
values take precedence.
system_instruction: System instruction/prompt for the model.
.. deprecated:: 0.0.105
Use ``settings=GoogleLLMSettings(system_instruction=...)`` instead.
tools: List of available tools/functions.
tool_config: Configuration for tool usage.
http_options: HTTP options for the client.
@@ -817,6 +820,7 @@ class GoogleLLMService(LLMService):
# 1. Initialize default_settings with hardcoded defaults
default_settings = GoogleLLMSettings(
model="gemini-2.5-flash",
system_instruction=None,
max_tokens=4096,
temperature=None,
top_k=None,
@@ -834,6 +838,9 @@ class GoogleLLMService(LLMService):
if model is not None:
_warn_deprecated_param("model", GoogleLLMSettings, "model")
default_settings.model = model
if system_instruction is not None:
_warn_deprecated_param("system_instruction", GoogleLLMSettings, "system_instruction")
default_settings.system_instruction = system_instruction
# 3. Apply params overrides — only if settings not provided
if params is not None:
@@ -854,7 +861,6 @@ class GoogleLLMService(LLMService):
super().__init__(settings=default_settings, **kwargs)
self._api_key = api_key
self._system_instruction = system_instruction
self._http_options = update_google_client_http_options(http_options)
self._tools = tools
self._tool_config = tool_config
@@ -993,10 +999,10 @@ class GoogleLLMService(LLMService):
messages = params_from_context["messages"]
if (
params_from_context["system_instruction"]
and self._system_instruction != params_from_context["system_instruction"]
and self._settings.system_instruction != params_from_context["system_instruction"]
):
logger.debug(f"System instruction changed: {params_from_context['system_instruction']}")
self._system_instruction = params_from_context["system_instruction"]
self._settings.system_instruction = params_from_context["system_instruction"]
tools = []
if params_from_context["tools"]:
@@ -1009,7 +1015,9 @@ class GoogleLLMService(LLMService):
# Build generation parameters
generation_params = self._build_generation_params(
system_instruction=self._system_instruction, tools=tools, tool_config=tool_config
system_instruction=self._settings.system_instruction,
tools=tools,
tool_config=tool_config,
)
# possibly modify generation_params (in place) to set thinking to off by default

10
src/pipecat/services/google/llm_vertex.py
View File

@@ -142,6 +142,9 @@ class GoogleVertexLLMService(GoogleLLMService):
deprecated parameters and *settings* are provided, *settings*
values take precedence.
system_instruction: System instruction/prompt for the model.
.. deprecated:: 0.0.105
Use ``settings=GoogleVertexLLMSettings(system_instruction=...)`` instead.
tools: List of available tools/functions.
tool_config: Configuration for tool usage.
http_options: HTTP options for the client.
@@ -195,6 +198,7 @@ class GoogleVertexLLMService(GoogleLLMService):
# 1. Initialize default_settings with hardcoded defaults
default_settings = GoogleVertexLLMSettings(
model="gemini-2.5-flash",
system_instruction=None,
max_tokens=4096,
temperature=None,
top_k=None,
@@ -212,6 +216,11 @@ class GoogleVertexLLMService(GoogleLLMService):
if model is not None:
_warn_deprecated_param("model", GoogleVertexLLMSettings, "model")
default_settings.model = model
if system_instruction is not None:
_warn_deprecated_param(
"system_instruction", GoogleVertexLLMSettings, "system_instruction"
)
default_settings.system_instruction = system_instruction
# 3. Apply params overrides — only if settings not provided
if params is not None:
@@ -234,7 +243,6 @@ class GoogleVertexLLMService(GoogleLLMService):
super().__init__(
api_key="dummy",
settings=default_settings,
system_instruction=system_instruction,
tools=tools,
tool_config=tool_config,
http_options=http_options,

7
src/pipecat/services/mistral/llm.py
View File

@@ -231,4 +231,11 @@ class MistralLLMService(OpenAILLMService):
# Add any extra parameters
params.update(self._settings.extra)
# Prepend system instruction if set
if self._settings.system_instruction:
messages = params.get("messages", [])
params["messages"] = [
{"role": "system", "content": self._settings.system_instruction}
] + messages
return params

12
src/pipecat/services/openai/base_llm.py
View File

@@ -121,7 +121,6 @@ class BaseOpenAILLMService(LLMService):
settings: Optional[OpenAILLMSettings] = None,
retry_timeout_secs: Optional[float] = 5.0,
retry_on_timeout: Optional[bool] = False,
system_instruction: Optional[str] = None,
**kwargs,
):
"""Initialize the BaseOpenAILLMService.
@@ -147,12 +146,12 @@ class BaseOpenAILLMService(LLMService):
parameters, ``settings`` values take precedence.
retry_timeout_secs: Request timeout in seconds. Defaults to 5.0 seconds.
retry_on_timeout: Whether to retry the request once if it times out.
system_instruction: Optional system instruction to prepend to messages.
**kwargs: Additional arguments passed to the parent LLMService.
"""
# 1. Initialize default_settings with hardcoded defaults
default_settings = OpenAILLMSettings(
model="gpt-4o",
system_instruction=None,
frequency_penalty=NOT_GIVEN,
presence_penalty=NOT_GIVEN,
seed=NOT_GIVEN,
@@ -193,7 +192,6 @@ class BaseOpenAILLMService(LLMService):
self._service_tier = service_tier
self._retry_timeout_secs = retry_timeout_secs
self._retry_on_timeout = retry_on_timeout
self._system_instruction = system_instruction
self._full_model_name: str = ""
self._client = self.create_client(
api_key=api_key,
@@ -204,8 +202,8 @@ class BaseOpenAILLMService(LLMService):
**kwargs,
)
if self._system_instruction:
logger.debug(f"{self}: Using system instruction: {self._system_instruction}")
if self._settings.system_instruction:
logger.debug(f"{self}: Using system instruction: {self._settings.system_instruction}")
def create_client(
self,
@@ -329,7 +327,7 @@ class BaseOpenAILLMService(LLMService):
params.update(self._settings.extra)
# Prepend system instruction from constructor, replacing any context system message
if self._system_instruction:
if self._settings.system_instruction:
messages = params.get("messages", [])
if messages and messages[0].get("role") == "system":
logger.warning(
@@ -337,7 +335,7 @@ class BaseOpenAILLMService(LLMService):
" Using system_instruction."
)
params["messages"] = [
{"role": "system", "content": self._system_instruction}
{"role": "system", "content": self._settings.system_instruction}
] + messages
return params

1
src/pipecat/services/openai/llm.py
View File

@@ -102,6 +102,7 @@ class OpenAILLMService(BaseOpenAILLMService):
# 1. Initialize default_settings with hardcoded defaults
default_settings = OpenAILLMSettings(
model="gpt-4.1",
system_instruction=None,
frequency_penalty=NOT_GIVEN,
presence_penalty=NOT_GIVEN,
seed=NOT_GIVEN,

7
src/pipecat/services/perplexity/llm.py
View File

@@ -114,6 +114,13 @@ class PerplexityLLMService(OpenAILLMService):
if self._settings.max_tokens is not None:
params["max_tokens"] = self._settings.max_tokens
# Prepend system instruction if set
if self._settings.system_instruction:
messages = params.get("messages", [])
params["messages"] = [
{"role": "system", "content": self._settings.system_instruction}
] + messages
return params
async def _process_context(self, context: OpenAILLMContext | LLMContext):

8
src/pipecat/services/sambanova/llm.py
View File

@@ -128,6 +128,14 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore
params.update(params_from_context)
params.update(self._settings.extra)
# Prepend system instruction if set
if self._settings.system_instruction:
messages = params.get("messages", [])
params["messages"] = [
{"role": "system", "content": self._settings.system_instruction}
] + messages
return params
@traced_llm # type: ignore

2
src/pipecat/services/settings.py
View File

@@ -395,6 +395,7 @@ class LLMSettings(ServiceSettings):
Parameters:
model: LLM model identifier.
system_instruction: System instruction/prompt for the model.
temperature: Sampling temperature.
max_tokens: Maximum tokens to generate.
top_p: Nucleus sampling probability.
@@ -411,6 +412,7 @@ class LLMSettings(ServiceSettings):
and prompts for incomplete turns.
"""
system_instruction: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
max_tokens: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
top_p: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)