diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index 5a164ad84..550b50809 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -10,9 +10,10 @@ import os import re import time import wave +from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import List, Optional, Tuple +from typing import Any, List, Optional, Tuple import aiofiles from deepgram import LiveOptions @@ -53,6 +54,14 @@ EVAL_TIMEOUT_SECS = 120 EvalPrompt = str | Tuple[str, ImageFile] +@dataclass +class EvalConfig: + prompt: EvalPrompt + eval: str + eval_speaks_first: bool = False + runner_args_body: Optional[Any] = None + + class EvalRunner: def __init__( self, @@ -93,9 +102,7 @@ class EvalRunner: async def run_eval( self, example_file: str, - prompt: EvalPrompt, - eval: str, - user_speaks_first: bool = False, + eval_config: EvalConfig, ): if not re.match(self._pattern, example_file): return @@ -112,10 +119,8 @@ class EvalRunner: try: tasks = [ - asyncio.create_task(run_example_pipeline(script_path)), - asyncio.create_task( - run_eval_pipeline(self, example_file, prompt, eval, user_speaks_first) - ), + asyncio.create_task(run_example_pipeline(script_path, eval_config)), + asyncio.create_task(run_eval_pipeline(self, example_file, eval_config)), ] _, pending = await asyncio.wait(tasks, timeout=EVAL_TIMEOUT_SECS) if pending: @@ -177,7 +182,7 @@ class EvalRunner: return os.path.join(self._recordings_dir, f"{base_name}.wav") -async def run_example_pipeline(script_path: Path): +async def run_example_pipeline(script_path: Path, eval_config: EvalConfig): room_url = os.getenv("DAILY_SAMPLE_ROOM_URL") module = load_module_from_path(script_path) @@ -196,6 +201,7 @@ async def run_example_pipeline(script_path: Path): runner_args = RunnerArguments() runner_args.pipeline_idle_timeout_secs = PIPELINE_IDLE_TIMEOUT_SECS + runner_args.body = eval_config.runner_args_body await module.run_bot(transport, runner_args) @@ -203,9 +209,7 @@ async def run_example_pipeline(script_path: Path): async def run_eval_pipeline( eval_runner: EvalRunner, example_file: str, - prompt: EvalPrompt, - eval: str, - user_speaks_first: bool = False, + eval_config: EvalConfig, ): logger.info(f"Starting eval bot") @@ -262,17 +266,17 @@ async def run_eval_pipeline( # Load example prompt depending on image. example_prompt = "" example_image: Optional[ImageFile] = None - if isinstance(prompt, str): - example_prompt = prompt - elif isinstance(prompt, tuple): - example_prompt, example_image = prompt + if isinstance(eval_config.prompt, str): + example_prompt = eval_config.prompt + elif isinstance(eval_config.prompt, tuple): + example_prompt, example_image = eval_config.prompt eval_prompt = f"The answer is correct if it matches: {eval}." common_system_prompt = ( "The user might say things other than the answer and that's allowed. " f"You should only call the eval function with your assessment when the user actually answers the question. {eval_prompt}" ) - if user_speaks_first: + if eval_config.eval_speaks_first: system_prompt = f"You are an LLM eval, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" else: system_prompt = f"You are an LLM eval, be extremly brief. Your goal is to first ask one question: {example_prompt}. {common_system_prompt}" @@ -330,9 +334,9 @@ async def run_eval_pipeline( # Default behavior is for the bot to speak first # If the eval bot speaks first, we append the prompt to the messages - if user_speaks_first: + if eval_config.eval_speaks_first: messages.append( - {"role": "user", "content": f"Start by saying this exactly: '{prompt}'"} + {"role": "user", "content": f"Start by saying this exactly: '{eval_config.prompt}'"} ) await task.queue_frames([LLMRunFrame()]) diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index ce380158b..2f6038c14 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -11,7 +11,7 @@ from datetime import datetime, timezone from pathlib import Path from dotenv import load_dotenv -from eval import EvalRunner +from eval import EvalConfig, EvalRunner from loguru import logger from PIL import Image from utils import check_env_variables @@ -24,190 +24,184 @@ ASSETS_DIR = SCRIPT_DIR / "assets" FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational" -# Speaking order constants -USER_SPEAKS_FIRST = True -BOT_SPEAKS_FIRST = False - -# Math -PROMPT_SIMPLE_MATH = "A simple math addition." -EVAL_SIMPLE_MATH = "Correct math addition." - -# Weather -PROMPT_WEATHER = "What's the weather in San Francisco?" -EVAL_WEATHER = ( - "Something specific about the current weather in San Francisco, including the degrees." +EVAL_SIMPLE_MATH = EvalConfig( + prompt="A simple math addition.", + eval="Correct math addition.", ) -# Online search -PROMPT_ONLINE_SEARCH = "What's the date right now in London?" -EVAL_ONLINE_SEARCH = f"Today is {datetime.now(timezone.utc).strftime('%B %d, %Y')}." +EVAL_WEATHER = EvalConfig( + prompt="What's the weather in San Francisco?", + eval="Something specific about the current weather in San Francisco, including the degrees.", +) -# Switch language -PROMPT_SWITCH_LANGUAGE = "Say something in Spanish." -EVAL_SWITCH_LANGUAGE = "The user is now talking in Spanish." +EVAL_ONLINE_SEARCH = EvalConfig( + prompt="What's the date right now in London?", + eval=f"Today is {datetime.now(timezone.utc).strftime('%B %d, %Y')}.", +) -# Vision -PROMPT_VISION = ("Briefly describe what you see.", Image.open(ASSETS_DIR / "cat.jpg")) -EVAL_VISION = "A cat description." +EVAL_SWITCH_LANGUAGE = EvalConfig( + prompt="Say something in Spanish.", + eval="The user is now talking in Spanish.", +) + +EVAL_VISION_CAMERA = EvalConfig( + prompt=("Briefly describe what you see.", Image.open(ASSETS_DIR / "cat.jpg")), + eval="A cat description.", +) + +EVAL_VISION_IMAGE = EvalConfig( + prompt="Briefly describe this image.", + eval="A cat description.", + eval_speaks_first=True, + runner_args_body={ + "image_path": ASSETS_DIR / "cat.jpg", + "question": "Briefly describe this image.", + }, +) + +EVAL_VOICEMAIL = EvalConfig( + prompt="Please leave a message after the beep.", + eval="Assess the conversation and determine if it is a voicemail.", + eval_speaks_first=True, +) + +EVAL_CONVERSATION = EvalConfig( + prompt="Hello, this is Mark.", + eval="A start of a conversation, not a voicemail.", + eval_speaks_first=True, +) -# Voicemail -PROMPT_VOICEMAIL = "Please leave a message after the beep." -EVAL_VOICEMAIL = "Assess the conversation and determine if it is a voicemail." -PROMPT_CONVERSATION = "Hello, this is Mark." -EVAL_CONVERSATION = "A start of a conversation, not a voicemail." TESTS_07 = [ # 07 series - ("07-interruptible.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07-interruptible-cartesia-http.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07a-interruptible-speechmatics.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07aa-interruptible-soniox.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07ab-interruptible-inworld-http.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07ac-interruptible-asyncai.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07ac-interruptible-asyncai-http.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07b-interruptible-langchain.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07c-interruptible-deepgram.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07c-interruptible-deepgram-flux.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07d-interruptible-elevenlabs.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ( - "07d-interruptible-elevenlabs-http.py", - PROMPT_SIMPLE_MATH, - EVAL_SIMPLE_MATH, - BOT_SPEAKS_FIRST, - ), - ("07f-interruptible-azure.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07g-interruptible-openai.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07h-interruptible-openpipe.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07j-interruptible-gladia.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07k-interruptible-lmnt.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07l-interruptible-groq.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07m-interruptible-aws.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07m-interruptible-aws-strands.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("07n-interruptible-gemini.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07n-interruptible-google.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07o-interruptible-assemblyai.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07q-interruptible-rime.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07q-interruptible-rime-http.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07r-interruptible-riva-nim.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ( - "07s-interruptible-google-audio-in.py", - PROMPT_SIMPLE_MATH, - EVAL_SIMPLE_MATH, - BOT_SPEAKS_FIRST, - ), - ("07t-interruptible-fish.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07v-interruptible-neuphonic.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07v-interruptible-neuphonic-http.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07w-interruptible-fal.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07y-interruptible-minimax.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07z-interruptible-sarvam.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07ae-interruptible-hume.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("07-interruptible.py", EVAL_SIMPLE_MATH), + ("07-interruptible-cartesia-http.py", EVAL_SIMPLE_MATH), + ("07a-interruptible-speechmatics.py", EVAL_SIMPLE_MATH), + ("07aa-interruptible-soniox.py", EVAL_SIMPLE_MATH), + ("07ab-interruptible-inworld-http.py", EVAL_SIMPLE_MATH), + ("07ac-interruptible-asyncai.py", EVAL_SIMPLE_MATH), + ("07ac-interruptible-asyncai-http.py", EVAL_SIMPLE_MATH), + ("07b-interruptible-langchain.py", EVAL_SIMPLE_MATH), + ("07c-interruptible-deepgram.py", EVAL_SIMPLE_MATH), + ("07c-interruptible-deepgram-flux.py", EVAL_SIMPLE_MATH), + ("07d-interruptible-elevenlabs.py", EVAL_SIMPLE_MATH), + ("07d-interruptible-elevenlabs-http.py", EVAL_SIMPLE_MATH), + ("07f-interruptible-azure.py", EVAL_SIMPLE_MATH), + ("07g-interruptible-openai.py", EVAL_SIMPLE_MATH), + ("07h-interruptible-openpipe.py", EVAL_SIMPLE_MATH), + ("07j-interruptible-gladia.py", EVAL_SIMPLE_MATH), + ("07k-interruptible-lmnt.py", EVAL_SIMPLE_MATH), + ("07l-interruptible-groq.py", EVAL_SIMPLE_MATH), + ("07m-interruptible-aws.py", EVAL_SIMPLE_MATH), + ("07m-interruptible-aws-strands.py", EVAL_WEATHER), + ("07n-interruptible-gemini.py", EVAL_SIMPLE_MATH), + ("07n-interruptible-google.py", EVAL_SIMPLE_MATH), + ("07o-interruptible-assemblyai.py", EVAL_SIMPLE_MATH), + ("07q-interruptible-rime.py", EVAL_SIMPLE_MATH), + ("07q-interruptible-rime-http.py", EVAL_SIMPLE_MATH), + ("07r-interruptible-riva-nim.py", EVAL_SIMPLE_MATH), + ("07s-interruptible-google-audio-in.py", EVAL_SIMPLE_MATH), + ("07t-interruptible-fish.py", EVAL_SIMPLE_MATH), + ("07v-interruptible-neuphonic.py", EVAL_SIMPLE_MATH), + ("07v-interruptible-neuphonic-http.py", EVAL_SIMPLE_MATH), + ("07w-interruptible-fal.py", EVAL_SIMPLE_MATH), + ("07y-interruptible-minimax.py", EVAL_SIMPLE_MATH), + ("07z-interruptible-sarvam.py", EVAL_SIMPLE_MATH), + ("07ae-interruptible-hume.py", EVAL_SIMPLE_MATH), # Needs a local XTTS docker instance running. - # ("07i-interruptible-xtts.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + # ("07i-interruptible-xtts.py", EVAL_SIMPLE_MATH), # Needs a Krisp license. - # ("07p-interruptible-krisp.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + # ("07p-interruptible-krisp.py", EVAL_SIMPLE_MATH), # Needs GPU resources. - # ("07u-interruptible-ultravox.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + # ("07u-interruptible-ultravox.py", EVAL_SIMPLE_MATH), +] + +TESTS_12 = [ + ("12-describe-image-openai.py", EVAL_VISION_IMAGE), + ("12a-describe-image-anthropic.py", EVAL_VISION_IMAGE), + ("12b-describe-image-aws.py", EVAL_VISION_IMAGE), + ("12c-describe-image-gemini-flash.py", EVAL_VISION_IMAGE), ] TESTS_14 = [ - ("14-function-calling.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14a-function-calling-anthropic.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14e-function-calling-google.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14f-function-calling-groq.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14g-function-calling-grok.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14h-function-calling-azure.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14i-function-calling-fireworks.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14j-function-calling-nim.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14k-function-calling-cerebras.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14m-function-calling-openrouter.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14n-function-calling-perplexity.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14p-function-calling-gemini-vertex-ai.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14q-function-calling-qwen.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14r-function-calling-aws.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14v-function-calling-openai.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14w-function-calling-mistral.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14x-function-calling-openpipe.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14-function-calling.py", EVAL_WEATHER), + ("14a-function-calling-anthropic.py", EVAL_WEATHER), + ("14e-function-calling-google.py", EVAL_WEATHER), + ("14f-function-calling-groq.py", EVAL_WEATHER), + ("14g-function-calling-grok.py", EVAL_WEATHER), + ("14h-function-calling-azure.py", EVAL_WEATHER), + ("14i-function-calling-fireworks.py", EVAL_WEATHER), + ("14j-function-calling-nim.py", EVAL_WEATHER), + ("14k-function-calling-cerebras.py", EVAL_WEATHER), + ("14m-function-calling-openrouter.py", EVAL_WEATHER), + ("14n-function-calling-perplexity.py", EVAL_WEATHER), + ("14p-function-calling-gemini-vertex-ai.py", EVAL_WEATHER), + ("14q-function-calling-qwen.py", EVAL_WEATHER), + ("14r-function-calling-aws.py", EVAL_WEATHER), + ("14v-function-calling-openai.py", EVAL_WEATHER), + ("14w-function-calling-mistral.py", EVAL_WEATHER), + ("14x-function-calling-openpipe.py", EVAL_WEATHER), # Video - ("14d-function-calling-anthropic-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), - ("14d-function-calling-aws-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), - ("14d-function-calling-gemini-flash-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), - ("14d-function-calling-moondream-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), - ("14d-function-calling-openai-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), + ("14d-function-calling-anthropic-video.py", EVAL_VISION_CAMERA), + ("14d-function-calling-aws-video.py", EVAL_VISION_CAMERA), + ("14d-function-calling-gemini-flash-video.py", EVAL_VISION_CAMERA), + ("14d-function-calling-moondream-video.py", EVAL_VISION_CAMERA), + ("14d-function-calling-openai-video.py", EVAL_VISION_CAMERA), # Currently not working. - # ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - # ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - # ("14o-function-calling-gemini-openai-format.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + # ("14c-function-calling-together.py", EVAL_WEATHER), + # ("14l-function-calling-deepseek.py", EVAL_WEATHER), + # ("14o-function-calling-gemini-openai-format.py", EVAL_WEATHER), ] TESTS_15 = [ - ("15a-switch-languages.py", PROMPT_SWITCH_LANGUAGE, EVAL_SWITCH_LANGUAGE, BOT_SPEAKS_FIRST), + ("15a-switch-languages.py", EVAL_SWITCH_LANGUAGE), ] TESTS_19 = [ - ("19-openai-realtime.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("19-openai-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("19-openai-realtime.py", EVAL_WEATHER), + ("19-openai-realtime-beta.py", EVAL_WEATHER), # OpenAI Realtime not released on Azure yet - # ("19a-azure-realtime.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("19a-azure-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("19b-openai-realtime-text.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("19b-openai-realtime-beta-text.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + # ("19a-azure-realtime.py", EVAL_WEATHER), + ("19a-azure-realtime-beta.py", EVAL_WEATHER), + ("19b-openai-realtime-text.py", EVAL_WEATHER), + ("19b-openai-realtime-beta-text.py", EVAL_WEATHER), ] TESTS_21 = [ - ("21a-tavus-video-service.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("21a-tavus-video-service.py", EVAL_SIMPLE_MATH), ] TESTS_26 = [ - ("26-gemini-multimodal-live.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ( - "26a-gemini-live-transcription.py", - PROMPT_SIMPLE_MATH, - EVAL_SIMPLE_MATH, - BOT_SPEAKS_FIRST, - ), - ( - "26b-gemini-live-function-calling.py", - PROMPT_WEATHER, - EVAL_WEATHER, - BOT_SPEAKS_FIRST, - ), - ("26c-gemini-live-video.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ( - "26e-gemini-multimodal-google-search.py", - PROMPT_ONLINE_SEARCH, - EVAL_ONLINE_SEARCH, - BOT_SPEAKS_FIRST, - ), + ("26-gemini-multimodal-live.py", EVAL_SIMPLE_MATH), + ("26a-gemini-live-transcription.py", EVAL_SIMPLE_MATH), + ("26b-gemini-live-function-calling.py", EVAL_WEATHER), + ("26c-gemini-live-video.py", EVAL_SIMPLE_MATH), + ("26e-gemini-multimodal-google-search.py", EVAL_ONLINE_SEARCH), + ("26h-gemini-live-vertex-function-calling.py", EVAL_WEATHER), # Currently not working. - # ("26d-gemini-live-text.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ( - "26h-gemini-live-vertex-function-calling.py", - PROMPT_WEATHER, - EVAL_WEATHER, - BOT_SPEAKS_FIRST, - ), + # ("26d-gemini-live-text.py", EVAL_SIMPLE_MATH), ] TESTS_27 = [ - ("27-simli-layer.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("27-simli-layer.py", EVAL_SIMPLE_MATH), ] TESTS_40 = [ - ("40-aws-nova-sonic.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("40-aws-nova-sonic.py", EVAL_SIMPLE_MATH), ] TESTS_43 = [ - ("43a-heygen-video-service.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("43a-heygen-video-service.py", EVAL_SIMPLE_MATH), ] TESTS_44 = [ - ("44-voicemail-detection.py", PROMPT_VOICEMAIL, EVAL_VOICEMAIL, USER_SPEAKS_FIRST), - ("44-voicemail-detection.py", PROMPT_CONVERSATION, EVAL_CONVERSATION, USER_SPEAKS_FIRST), + ("44-voicemail-detection.py", EVAL_VOICEMAIL), + ("44-voicemail-detection.py", EVAL_CONVERSATION), ] TESTS = [ *TESTS_07, + *TESTS_12, *TESTS_14, *TESTS_15, *TESTS_19, @@ -240,9 +234,9 @@ async def main(args: argparse.Namespace): # Parse test config: (test, prompt, eval, user_speaks_first) for test_config in TESTS: - test, prompt, eval, user_speaks_first = test_config + test, eval_config = test_config - await runner.run_eval(test, prompt, eval, user_speaks_first) + await runner.run_eval(test, eval_config) runner.print_results()