Merge pull request #3601 from pipecat-ai/mb/add-22-release-evals

Add 22 foundational to release evals

examples/foundational/22-filter-incomplete-turns.py

@@ -49,24 +49,20 @@ from pipecat.turns.user_turn_strategies import UserTurnStrategies
 load_dotenv(override=True)
 
 
-# We store functions so objects (e.g. SileroVADAnalyzer) don't get
-# instantiated. The function will be called when the desired transport gets
-# selected.
+# We use lambdas to defer transport parameter creation until the transport
+# type is selected at runtime.
 transport_params = {
     "daily": lambda: DailyParams(
         audio_in_enabled=True,
         audio_out_enabled=True,
-        vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
     ),
     "twilio": lambda: FastAPIWebsocketParams(
         audio_in_enabled=True,
         audio_out_enabled=True,
-        vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
     ),
     "webrtc": lambda: TransportParams(
         audio_in_enabled=True,
         audio_out_enabled=True,
-        vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
     ),
 }
 
@@ -97,6 +93,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
             user_turn_strategies=UserTurnStrategies(
                 stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=LocalSmartTurnAnalyzerV3())]
             ),
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
             # Enable turn completion filtering - the LLM will output:
             # ✓ = complete (respond normally)
             # ○ = incomplete short (wait 5s, then prompt)

scripts/evals/eval.py

@@ -313,7 +313,7 @@ async def run_eval_pipeline(
     context_aggregator = LLMContextAggregatorPair(
         context,
         user_params=LLMUserAggregatorParams(
-            vad_analyzer=SileroVADAnalyzer(),
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=2.0)),
         ),
     )
 

scripts/evals/run-release-evals.py

@@ -90,6 +90,11 @@ EVAL_ORDER = EvalConfig(
     eval_speaks_first=True,
 )
 
+EVAL_COMPLETE_TURN = EvalConfig(
+    prompt="I would go to Japan because I love the culture and want to try authentic ramen.",
+    eval="The user provides a relevant response about Japan or travel, showing the conversation continues normally.",
+)
+
 
 TESTS_07 = [
     # 07 series
@@ -210,6 +215,10 @@ TESTS_21 = [
     ("21a-tavus-video-service.py", EVAL_SIMPLE_MATH),
 ]
 
+TESTS_22 = [
+    ("22-filter-incomplete-turns.py", EVAL_COMPLETE_TURN),
+]
+
 TESTS_26 = [
     ("26-gemini-live.py", EVAL_SIMPLE_MATH),
     ("26a-gemini-live-transcription.py", EVAL_SIMPLE_MATH),
@@ -266,6 +275,7 @@ TESTS = [
     *TESTS_15,
     *TESTS_19,
     *TESTS_21,
+    *TESTS_22,
     *TESTS_26,
     *TESTS_27,
     *TESTS_40,