2994448036
This is an initial attempt to implement evals for all (or most) of our foundational examples. Before we release, we want to make sure all of them work and reply properly. Until now this has been done manually, hopefully this will be useful to speed up our release process.
139 lines
5.6 KiB
Python
139 lines
5.6 KiB
Python
#
|
|
# Copyright (c) 2024-2025 Daily
|
|
#
|
|
# SPDX-License-Identifier: BSD 2-Clause License
|
|
#
|
|
|
|
import argparse
|
|
import asyncio
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
from dotenv import load_dotenv
|
|
from eval import EvalRunner
|
|
from loguru import logger
|
|
from utils import check_env_variables
|
|
|
|
load_dotenv(override=True)
|
|
|
|
# Math
|
|
PROMPT_SIMPLE_MATH = "A simple math addition."
|
|
|
|
# Weather
|
|
PROMPT_WEATHER = "What's the weather in San Francisco?"
|
|
EVAL_WEATHER = (
|
|
"Something specific about the current weather in San Francisco, including the degrees."
|
|
)
|
|
|
|
# Online search
|
|
PROMPT_ONLINE_SEARCH = "What's the date right now in London?"
|
|
EVAL_ONLINE_SEARCH = f"Today is {datetime.now(timezone.utc).strftime('%B %d, %Y')}."
|
|
|
|
TESTS_07 = [
|
|
# 07 series
|
|
("07-interruptible.py", PROMPT_SIMPLE_MATH, None),
|
|
("07-interruptible-cartesia-http.py", PROMPT_SIMPLE_MATH, None),
|
|
("07b-interruptible-langchain.py", PROMPT_SIMPLE_MATH, None),
|
|
("07c-interruptible-deepgram.py", PROMPT_SIMPLE_MATH, None),
|
|
("07d-interruptible-elevenlabs.py", PROMPT_SIMPLE_MATH, None),
|
|
("07d-interruptible-elevenlabs-http.py", PROMPT_SIMPLE_MATH, None),
|
|
("07e-interruptible-playht.py", PROMPT_SIMPLE_MATH, None),
|
|
("07e-interruptible-playht-http.py", PROMPT_SIMPLE_MATH, None),
|
|
("07f-interruptible-azure.py", PROMPT_SIMPLE_MATH, None),
|
|
("07g-interruptible-openai.py", PROMPT_SIMPLE_MATH, None),
|
|
("07h-interruptible-openpipe.py", PROMPT_SIMPLE_MATH, None),
|
|
("07j-interruptible-gladia.py", PROMPT_SIMPLE_MATH, None),
|
|
("07k-interruptible-lmnt.py", PROMPT_SIMPLE_MATH, None),
|
|
("07l-interruptible-groq.py", PROMPT_SIMPLE_MATH, None),
|
|
("07m-interruptible-aws.py", PROMPT_SIMPLE_MATH, None),
|
|
("07n-interruptible-google.py", PROMPT_SIMPLE_MATH, None),
|
|
("07o-interruptible-assemblyai.py", PROMPT_SIMPLE_MATH, None),
|
|
("07q-interruptible-rime.py", PROMPT_SIMPLE_MATH, None),
|
|
("07q-interruptible-rime-http.py", PROMPT_SIMPLE_MATH, None),
|
|
("07r-interruptible-riva-nim.py", PROMPT_SIMPLE_MATH, None),
|
|
("07s-interruptible-google-audio-in.py", PROMPT_SIMPLE_MATH, None),
|
|
("07t-interruptible-fish.py", PROMPT_SIMPLE_MATH, None),
|
|
("07v-interruptible-neuphonic.py", PROMPT_SIMPLE_MATH, None),
|
|
("07v-interruptible-neuphonic-http.py", PROMPT_SIMPLE_MATH, None),
|
|
("07w-interruptible-fal.py", PROMPT_SIMPLE_MATH, None),
|
|
("07y-interruptible-minimax.py", PROMPT_SIMPLE_MATH, None),
|
|
("07z-interruptible-sarvam.py", PROMPT_SIMPLE_MATH, None),
|
|
# Needs a local XTTS docker instance running.
|
|
# ("07i-interruptible-xtts.py", PROMPT_SIMPLE_MATH, None),
|
|
# Needs a Krisp license.
|
|
# ("07p-interruptible-krisp.py", PROMPT_SIMPLE_MATH, None),
|
|
# Needs GPU resources.
|
|
# ("07u-interruptible-ultravox.py", PROMPT_SIMPLE_MATH, None),
|
|
]
|
|
|
|
TESTS_14 = [
|
|
("14-function-calling.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("14a-function-calling-anthropic.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("14b-function-calling-anthropic-video.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("14d-function-calling-video.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("14e-function-calling-gemini.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("14f-function-calling-groq.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("14g-function-calling-grok.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("14h-function-calling-azure.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("14i-function-calling-fireworks.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("14j-function-calling-nim.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("14n-function-calling-perplexity.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("14q-function-calling-qwen.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("14r-function-calling-aws.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
# Currently not working.
|
|
# ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
# ("14j-function-calling-nim.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
# ("14k-function-calling-cerebras.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
# ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
# ("14m-function-calling-openrouter.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
# ("14o-function-calling-gemini-openai-format.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
# ("14p-function-calling-gemini-vertex-ai.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
]
|
|
|
|
TESTS_19 = [
|
|
("19-openai-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("19a-azure-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
]
|
|
|
|
TESTS_26 = [
|
|
("26-gemini-multimodal-live.py", PROMPT_SIMPLE_MATH, None),
|
|
("26a-gemini-multimodal-live-transcription.py", PROMPT_SIMPLE_MATH, None),
|
|
("26b-gemini-multimodal-live-function-calling.py", PROMPT_WEATHER, EVAL_WEATHER),
|
|
("26c-gemini-multimodal-live-video.py", PROMPT_SIMPLE_MATH, None),
|
|
("26e-gemini-multimodal-google-search.py", PROMPT_ONLINE_SEARCH, EVAL_ONLINE_SEARCH),
|
|
# Currently not working.
|
|
# ("26d-gemini-multimodal-live-text.py", PROMPT_SIMPLE_MATH, None),
|
|
]
|
|
|
|
TESTS = [
|
|
*TESTS_07,
|
|
*TESTS_14,
|
|
*TESTS_19,
|
|
*TESTS_26,
|
|
]
|
|
|
|
|
|
async def main(args: argparse.Namespace):
|
|
if not check_env_variables():
|
|
return
|
|
|
|
runner = EvalRunner(args.pattern)
|
|
for test, prompt, eval in TESTS:
|
|
await runner.run_eval(test, prompt, eval)
|
|
|
|
runner.print_results()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description="Pipecat Eval Runner")
|
|
parser.add_argument("--pattern", "-p", help="Only run tests that match the pattern")
|
|
parser.add_argument("--verbose", "-v", action="count", default=0)
|
|
args = parser.parse_args()
|
|
|
|
# Log level
|
|
logger.remove(0)
|
|
if args.verbose:
|
|
logger.add(sys.stderr, level="TRACE" if args.verbose >= 2 else "DEBUG")
|
|
|
|
asyncio.run(main(args))
|