wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update foundational examples to use "user" role

Browse Source 
Use system_instruction on LLM service constructors instead of adding
system messages to LLMContext. Messages added to context now use
"user" role.
This commit is contained in:
Aleix Conchillo Flaqué
2026-03-05 16:17:32 -08:00
parent 18494658c3
commit 593b75bc8b
179 changed files with 271 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
scripts/evals/eval.py
View File

@@ -51,6 +51,7 @@ from pipecat.runner.types import RunnerArguments
from pipecat.services.cartesia.tts import CartesiaTTSService, CartesiaTTSSettings
from pipecat.services.deepgram.stt import DeepgramSTTService, DeepgramSTTSettings
from pipecat.services.llm_service import FunctionCallParams
from pipecat.services.openai.base_llm import OpenAILLMSettings
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.daily.transport import DailyParams, DailyTransport
@@ -256,30 +257,6 @@ async def run_eval_pipeline(
),
)
# Load example prompt depending on image.
example_prompt = ""
example_image: Optional[ImageFile] = None
if isinstance(eval_config.prompt, str):
example_prompt = eval_config.prompt
elif isinstance(eval_config.prompt, tuple):
example_prompt, example_image = eval_config.prompt
common_system_prompt = (
"You should only call the eval function if:\n"
"- The user explicitly attempts to answer the question, AND\n"
f"- Their answer can be cleanly evaluated using: {eval_config.eval}\n"
"Ignore greetings, comments, non-answers, or requests for clarification.\n"
"Numerical word answers are allowed (e.g., 'five' is the same as '5').\n"
)
if eval_config.eval_speaks_first:
system_prompt = f"You are an evaluation agent, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}"
else:
system_prompt = f"You are an evaluation agent, be extremly brief. First, ask one question: {example_prompt}. {common_system_prompt}"
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), system_instruction=system_prompt)
llm.register_function("eval_function", eval_runner.function_assert_eval)
eval_function = FunctionSchema(
name="eval_function",
description=(
@@ -303,6 +280,33 @@ async def run_eval_pipeline(
)
tools = ToolsSchema(standard_tools=[eval_function])
# Load example prompt depending on image.
example_prompt = ""
example_image: Optional[ImageFile] = None
if isinstance(eval_config.prompt, str):
example_prompt = eval_config.prompt
elif isinstance(eval_config.prompt, tuple):
example_prompt, example_image = eval_config.prompt
common_system_prompt = (
"You should only call the eval function if:\n"
"- The user explicitly attempts to answer the question, AND\n"
f"- Their answer can be cleanly evaluated using: {eval_config.eval}\n"
"Ignore greetings, comments, non-answers, or requests for clarification.\n"
"Numerical word answers are allowed (e.g., 'five' is the same as '5').\n"
)
if eval_config.eval_speaks_first:
system_prompt = f"You are an evaluation agent, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}"
else:
system_prompt = f"You are an evaluation agent, be extremly brief. First, ask one question: {example_prompt}. {common_system_prompt}"
llm = OpenAILLMService(
api_key=os.getenv("OPENAI_API_KEY"),
settings=OpenAILLMSettings(system_instruction=system_prompt),
)
llm.register_function("eval_function", eval_runner.function_assert_eval)
context = LLMContext(tools=tools)
context_aggregator = LLMContextAggregatorPair(
context,