diff --git a/examples/foundational/49-ultravox-realtime.py b/examples/foundational/50-ultravox-realtime.py similarity index 100% rename from examples/foundational/49-ultravox-realtime.py rename to examples/foundational/50-ultravox-realtime.py diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 68600706f..19df05da9 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -79,6 +79,12 @@ EVAL_FLIGHT_STATUS = EvalConfig( eval="The user says something about the status of flight AA100, such as whether it's on time or delayed.", ) +EVAL_ORDER = EvalConfig( + prompt="I'd like to order a chocolate iced doughnut and a regular brewed coffee.", + eval="The user acknowledges the order of a chocolate iced doughnut and regular brewed coffee.", + eval_speaks_first=True, +) + TESTS_07 = [ # 07 series @@ -215,6 +221,11 @@ TESTS_49 = [ ("49d-thinking-functions-google.py", EVAL_FLIGHT_STATUS), ] + +TESTS_50 = [ + ("50-ultravox-realtime.py", EVAL_ORDER), +] + TESTS = [ *TESTS_07, *TESTS_12, @@ -228,6 +239,7 @@ TESTS = [ *TESTS_43, *TESTS_44, *TESTS_49, + *TESTS_50, ]