wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2607 from pipecat-ai/aleix/scripts-evals-improve-eval-prompt

Browse Source 
scripts(evals): allow user to talk and only eval when needed
This commit is contained in:
Aleix Conchillo Flaqué
2025-09-07 20:49:43 -07:00
committed by GitHub
parent 6edb6fed41 46deaada70
commit 6a6a63a532
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
scripts/evals/eval.py
View File

@@ -47,7 +47,7 @@ from pipecat.transports.daily.transport import DailyParams, DailyTransport
SCRIPT_DIR = Path(__file__).resolve().parent SCRIPT_DIR = Path(__file__).resolve().parent
PIPELINE_IDLE_TIMEOUT_SECS = 60 PIPELINE_IDLE_TIMEOUT_SECS = 60
EVAL_TIMEOUT_SECS = 90 EVAL_TIMEOUT_SECS = 120
EvalPrompt = str | Tuple[str, ImageFile] EvalPrompt = str | Tuple[str, ImageFile]
@@ -266,8 +266,11 @@ async def run_eval_pipeline(
elif isinstance(prompt, tuple): elif isinstance(prompt, tuple):
example_prompt, example_image = prompt example_prompt, example_image = prompt
eval_prompt = f"The answer is correct if it's appropriate for the context and matches: {eval}." eval_prompt = f"The answer is correct if it matches: {eval}."
common_system_prompt = f"Call the eval function with your assessment only if the user answers the question. {eval_prompt}" common_system_prompt = (
"The user might say things other than the answer and that's allowed. "
f"You should only call the eval function with your assessment when the user actually answers the question. {eval_prompt}"
)
if user_speaks_first: if user_speaks_first:
system_prompt = f"You are an LLM eval, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" system_prompt = f"You are an LLM eval, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}"
else: else: