introduce release evals
This is an initial attempt to implement evals for all (or most) of our foundational examples. Before we release, we want to make sure all of them work and reply properly. Until now this has been done manually, hopefully this will be useful to speed up our release process.
This commit is contained in:
Aleix Conchillo Flaqué
138
scripts/release/run-release-evals.py
Normal file
138
scripts/release/run-release-evals.py
Normal file
@@ -0,0 +1,138 @@
|
||||
#
|
||||
# Copyright (c) 2024-2025 Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from eval import EvalRunner
|
||||
from loguru import logger
|
||||
from utils import check_env_variables
|
||||
|
||||
load_dotenv(override=True)
|
||||
|
||||
# Math
|
||||
PROMPT_SIMPLE_MATH = "A simple math addition."
|
||||
|
||||
# Weather
|
||||
PROMPT_WEATHER = "What's the weather in San Francisco?"
|
||||
EVAL_WEATHER = (
|
||||
"Something specific about the current weather in San Francisco, including the degrees."
|
||||
)
|
||||
|
||||
# Online search
|
||||
PROMPT_ONLINE_SEARCH = "What's the date right now in London?"
|
||||
EVAL_ONLINE_SEARCH = f"Today is {datetime.now(timezone.utc).strftime('%B %d, %Y')}."
|
||||
|
||||
TESTS_07 = [
|
||||
# 07 series
|
||||
("07-interruptible.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07-interruptible-cartesia-http.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07b-interruptible-langchain.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07c-interruptible-deepgram.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07d-interruptible-elevenlabs.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07d-interruptible-elevenlabs-http.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07e-interruptible-playht.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07e-interruptible-playht-http.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07f-interruptible-azure.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07g-interruptible-openai.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07h-interruptible-openpipe.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07j-interruptible-gladia.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07k-interruptible-lmnt.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07l-interruptible-groq.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07m-interruptible-aws.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07n-interruptible-google.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07o-interruptible-assemblyai.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07q-interruptible-rime.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07q-interruptible-rime-http.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07r-interruptible-riva-nim.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07s-interruptible-google-audio-in.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07t-interruptible-fish.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07v-interruptible-neuphonic.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07v-interruptible-neuphonic-http.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07w-interruptible-fal.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07y-interruptible-minimax.py", PROMPT_SIMPLE_MATH, None),
|
||||
("07z-interruptible-sarvam.py", PROMPT_SIMPLE_MATH, None),
|
||||
# Needs a local XTTS docker instance running.
|
||||
# ("07i-interruptible-xtts.py", PROMPT_SIMPLE_MATH, None),
|
||||
# Needs a Krisp license.
|
||||
# ("07p-interruptible-krisp.py", PROMPT_SIMPLE_MATH, None),
|
||||
# Needs GPU resources.
|
||||
# ("07u-interruptible-ultravox.py", PROMPT_SIMPLE_MATH, None),
|
||||
]
|
||||
|
||||
TESTS_14 = [
|
||||
("14-function-calling.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("14a-function-calling-anthropic.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("14b-function-calling-anthropic-video.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("14d-function-calling-video.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("14e-function-calling-gemini.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("14f-function-calling-groq.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("14g-function-calling-grok.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("14h-function-calling-azure.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("14i-function-calling-fireworks.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("14j-function-calling-nim.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("14n-function-calling-perplexity.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("14q-function-calling-qwen.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("14r-function-calling-aws.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
# Currently not working.
|
||||
# ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
# ("14j-function-calling-nim.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
# ("14k-function-calling-cerebras.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
# ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
# ("14m-function-calling-openrouter.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
# ("14o-function-calling-gemini-openai-format.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
# ("14p-function-calling-gemini-vertex-ai.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
]
|
||||
|
||||
TESTS_19 = [
|
||||
("19-openai-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("19a-azure-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
]
|
||||
|
||||
TESTS_26 = [
|
||||
("26-gemini-multimodal-live.py", PROMPT_SIMPLE_MATH, None),
|
||||
("26a-gemini-multimodal-live-transcription.py", PROMPT_SIMPLE_MATH, None),
|
||||
("26b-gemini-multimodal-live-function-calling.py", PROMPT_WEATHER, EVAL_WEATHER),
|
||||
("26c-gemini-multimodal-live-video.py", PROMPT_SIMPLE_MATH, None),
|
||||
("26e-gemini-multimodal-google-search.py", PROMPT_ONLINE_SEARCH, EVAL_ONLINE_SEARCH),
|
||||
# Currently not working.
|
||||
# ("26d-gemini-multimodal-live-text.py", PROMPT_SIMPLE_MATH, None),
|
||||
]
|
||||
|
||||
TESTS = [
|
||||
*TESTS_07,
|
||||
*TESTS_14,
|
||||
*TESTS_19,
|
||||
*TESTS_26,
|
||||
]
|
||||
|
||||
|
||||
async def main(args: argparse.Namespace):
|
||||
if not check_env_variables():
|
||||
return
|
||||
|
||||
runner = EvalRunner(args.pattern)
|
||||
for test, prompt, eval in TESTS:
|
||||
await runner.run_eval(test, prompt, eval)
|
||||
|
||||
runner.print_results()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Pipecat Eval Runner")
|
||||
parser.add_argument("--pattern", "-p", help="Only run tests that match the pattern")
|
||||
parser.add_argument("--verbose", "-v", action="count", default=0)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Log level
|
||||
logger.remove(0)
|
||||
if args.verbose:
|
||||
logger.add(sys.stderr, level="TRACE" if args.verbose >= 2 else "DEBUG")
|
||||
|
||||
asyncio.run(main(args))
|
||||
Reference in New Issue
Block a user