diff --git a/CHANGELOG.md b/CHANGELOG.md index 57df4e03f..e1eadf147 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance +- Improving the latency of the `HeyGenVideoService`. + - Improved some frame processors performance by using the new frame processor direct mode. In direct mode a frame processor will process frames right away avoiding the need for internal queues and tasks. This is useful for some @@ -46,7 +48,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other -- Improving the latency of the `HeyGenVideoService`. +- Added foundational example `15a-switch-languages.py` to release evals. It is + able to detect if we switched the language properly. - Updated foundational examples to show how to enclose complex logic (e.g. `ParallelPipeline`) into a single processor so the main pipeline becomes diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index f4518d7ce..c12f75fa5 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -210,7 +210,10 @@ async def run_eval_pipeline( # 5" (in audio) this can be converted to "32 is 5". stt = DeepgramSTTService( api_key=os.getenv("DEEPGRAM_API_KEY"), - live_options=LiveOptions(smart_format=False), + live_options=LiveOptions( + language="multi", + smart_format=False, + ), ) tts = CartesiaTTSService( diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 6b5bbcd25..5aac70c6b 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -35,6 +35,10 @@ EVAL_WEATHER = ( PROMPT_ONLINE_SEARCH = "What's the date right now in London?" EVAL_ONLINE_SEARCH = f"Today is {datetime.now(timezone.utc).strftime('%B %d, %Y')}." +# Switch language +PROMPT_SWITCH_LANGUAGE = "Say something in Spanish." +EVAL_SWITCH_LANGUAGE = "Check if the user is now talking in Spanish." + TESTS_07 = [ # 07 series ("07-interruptible.py", PROMPT_SIMPLE_MATH, None), @@ -101,6 +105,10 @@ TESTS_14 = [ # ("14o-function-calling-gemini-openai-format.py", PROMPT_WEATHER, EVAL_WEATHER), ] +TESTS_15 = [ + ("15a-switch-languages.py", PROMPT_SWITCH_LANGUAGE, EVAL_SWITCH_LANGUAGE), +] + TESTS_19 = [ ("19-openai-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER), ("19a-azure-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER), @@ -135,6 +143,7 @@ TESTS_43 = [ TESTS = [ *TESTS_07, *TESTS_14, + *TESTS_15, *TESTS_19, *TESTS_21, *TESTS_26,