wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scripts(evals): fix runner eval cancellation

Browse Source 
We need to call asyncio.gather() just once, not for every cancelled task.
This commit is contained in:
Aleix Conchillo Flaqué
2025-08-06 19:36:42 -07:00
parent 0b2b9f5f1b
commit ed8b0655a8
1 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
scripts/evals/eval.py
View File

@@ -46,7 +46,8 @@ from pipecat.transports.services.daily import DailyParams, DailyTransport
SCRIPT_DIR = Path(__file__).resolve().parent SCRIPT_DIR = Path(__file__).resolve().parent
PIPELINE_IDLE_TIMEOUT_SECS = 30 PIPELINE_IDLE_TIMEOUT_SECS = 60
EVAL_TIMEOUT_SECS = 90
class EvalRunner: class EvalRunner:
@@ -105,12 +106,16 @@ class EvalRunner:
asyncio.create_task(run_example_pipeline(script_path)), asyncio.create_task(run_example_pipeline(script_path)),
asyncio.create_task(run_eval_pipeline(self, example_file, prompt, eval)), asyncio.create_task(run_eval_pipeline(self, example_file, prompt, eval)),
] ]
_, pending = await asyncio.wait(tasks, timeout=90) _, pending = await asyncio.wait(tasks, timeout=EVAL_TIMEOUT_SECS)
if pending: if pending:
logger.error(f"ERROR: Eval timeout expired, cancelling pending tasks...") logger.error(f"ERROR: Eval timeout expired, cancelling pending tasks...")
# Both pipeline idle timeouts should have worked and both tasks
# should have exited already, but if we got here something went
# wrong so we perform an abrupt asyncio task cancellation, which
# will not cleanup things nicely.
for task in pending: for task in pending:
task.cancel() task.cancel()
await asyncio.gather(*pending, return_exceptions=True) await asyncio.gather(*pending, return_exceptions=True)
except Exception as e: except Exception as e:
logger.error(f"ERROR: Unable to run {example_file}: {e}") logger.error(f"ERROR: Unable to run {example_file}: {e}")