scripts: allow storing logs for release evals

scripts/release/eval.py

@@ -55,23 +55,29 @@ PIPELINE_IDLE_TIMEOUT_SECS = 30
 
 
 class EvalRunner:
-    def __init__(self, *, pattern: str = "", record_audio: bool = False):
+    def __init__(self, *, pattern: str = "", record_audio: bool = False, log_level: str = "DEBUG"):
         self._pattern = f".*{pattern}.*" if pattern else ""
         self._record_audio = record_audio
+        self._log_level = log_level
         self._total_success = 0
         self._tests: List[EvalResult] = []
         self._queue = asyncio.Queue()
 
-    @property
+        # We to save runner files.
-    def record_audio(self):
+        self._runs_dir = os.path.join(
-        return self._record_audio
+            SCRIPT_DIR, "test-runs", f"{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        )
+        self._logs_dir = os.path.join(self._runs_dir, "logs")
+        self._recordings_dir = os.path.join(self._runs_dir, "recordings")
+        os.makedirs(self._logs_dir, exist_ok=True)
+        os.makedirs(self._recordings_dir, exist_ok=True)
 
     async def assert_eval(self, params: FunctionCallParams):
         reasoning = params.arguments["reasoning"]
         logger.debug(f"🧠 EVAL REASONING: {reasoning}")
         await self._queue.put(params.arguments["result"])
-        await params.llm.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM)
         await params.result_callback(None)
+        await params.llm.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM)
 
     async def assert_eval_false(self):
         await self._queue.put(False)
@@ -80,6 +86,10 @@ class EvalRunner:
         if not re.match(self._pattern, example_file):
             return
 
+        # Store logs
+        filename = self._log_file_name(example_file)
+        log_file_id = logger.add(filename, level=self._log_level)
+
         print_begin_test(example_file)
 
         start_time = time.time()
@@ -111,8 +121,37 @@ class EvalRunner:
 
         print_end_test(example_file, result, eval_time)
 
+        logger.remove(log_file_id)
+
     def print_results(self):
-        print_test_results(self._tests, self._total_success)
+        print_test_results(self._tests, self._total_success, self._runs_dir)
+
+    async def save_audio(self, name: str, audio: bytes, sample_rate: int, num_channels: int):
+        if len(audio) > 0:
+            filename = self._recording_file_name(name)
+            with io.BytesIO() as buffer:
+                with wave.open(buffer, "wb") as wf:
+                    wf.setsampwidth(2)
+                    wf.setnchannels(num_channels)
+                    wf.setframerate(sample_rate)
+                    wf.writeframes(audio)
+                async with aiofiles.open(filename, "wb") as file:
+                    await file.write(buffer.getvalue())
+            logger.debug(f"Saving {name} audio to {filename}")
+        else:
+            logger.warning(f"There's no audio to save for {name}")
+
+    def _base_file_name(self, example_file: str):
+        base_name = os.path.splitext(example_file)[0]
+        return f"{base_name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+
+    def _log_file_name(self, example_file: str):
+        base_name = self._base_file_name(example_file)
+        return os.path.join(self._logs_dir, f"{base_name}.log")
+
+    def _recording_file_name(self, example_file: str):
+        base_name = self._base_file_name(example_file)
+        return os.path.join(self._recordings_dir, f"{base_name}.wav")
 
 
 async def run_example_pipeline(example_file: str):
@@ -136,28 +175,6 @@ async def run_example_pipeline(example_file: str):
     await module.run_example(transport, argparse.Namespace(), True)
 
 
-async def save_audio(audio: bytes, sample_rate: int, num_channels: int, name: str):
-    if len(audio) > 0:
-        recordings_dir = os.path.join(SCRIPT_DIR, "recordings")
-        base_name = os.path.splitext(name)[0]
-        os.makedirs(recordings_dir, exist_ok=True)
-        filename = os.path.join(
-            recordings_dir,
-            f"{base_name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.wav",
-        )
-        with io.BytesIO() as buffer:
-            with wave.open(buffer, "wb") as wf:
-                wf.setsampwidth(2)
-                wf.setnchannels(num_channels)
-                wf.setframerate(sample_rate)
-                wf.writeframes(audio)
-            async with aiofiles.open(filename, "wb") as file:
-                await file.write(buffer.getvalue())
-        logger.debug(f"Saving {name} audio to {filename}")
-    else:
-        logger.warning(f"There's no audio to save for {name}")
-
-
 async def run_eval_pipeline(
     eval_runner: EvalRunner, example_file: str, prompt: str, eval: Optional[str]
 ):
@@ -212,7 +229,7 @@ async def run_eval_pipeline(
     messages = [
         {
             "role": "system",
-            "content": f"You are an LLM eval, be extremly brief. Your goal is to only ask one question: {prompt}. Tell the user to simply say the answer. Call the eval function only if the user answers the question and check if the answer is correct (words as numbers are valid). {eval_prompt}",
+            "content": f"You are an LLM eval, be extremly brief. Your goal is to only ask one question: {prompt}. Call the eval function only if the user answers the question and check if the answer is correct (words as numbers are valid). {eval_prompt}",
         },
     ]
 
@@ -246,8 +263,7 @@ async def run_eval_pipeline(
 
     @audio_buffer.event_handler("on_audio_data")
     async def on_audio_data(buffer, audio, sample_rate, num_channels):
-        if eval_runner.record_audio:
+        await eval_runner.save_audio(example_file, audio, sample_rate, num_channels)
-            await save_audio(audio, sample_rate, num_channels, example_file)
 
     @transport.event_handler("on_client_connected")
     async def on_client_connected(transport, client):

scripts/release/run-release-evals.py

@@ -117,7 +117,13 @@ async def main(args: argparse.Namespace):
     if not check_env_variables():
         return
 
-    runner = EvalRunner(pattern=args.pattern, record_audio=args.audio)
+    # Log level
+    logger.remove(0)
+    log_level = "TRACE" if args.verbose >= 2 else "DEBUG"
+    if args.verbose:
+        logger.add(sys.stderr, level=log_level)
+
+    runner = EvalRunner(pattern=args.pattern, record_audio=args.audio, log_level=log_level)
     for test, prompt, eval in TESTS:
         await runner.run_eval(test, prompt, eval)
 
@@ -131,9 +137,4 @@ if __name__ == "__main__":
     parser.add_argument("--verbose", "-v", action="count", default=0)
     args = parser.parse_args()
 
-    # Log level
-    logger.remove(0)
-    if args.verbose:
-        logger.add(sys.stderr, level="TRACE" if args.verbose >= 2 else "DEBUG")
-
     asyncio.run(main(args))

scripts/release/utils.py

@@ -47,7 +47,7 @@ def print_end_test(example_file: str, passed: bool, time: float):
     print(f"{example_file:<55} {status} ({time:.2f}s){CLEAR}")
 
 
-def print_test_results(tests: Sequence[EvalResult], total_success: int):
+def print_test_results(tests: Sequence[EvalResult], total_success: int, location: str):
     total_count = len(tests)
 
     bar = "=" * 80
@@ -70,6 +70,8 @@ def print_test_results(tests: Sequence[EvalResult], total_success: int):
         f"{GREEN}SUCCESS{RESET}: {total_success} | {RED}FAIL{RESET}: {total_fail} | TOTAL TIME: {total_time:.2f}s"
     )
     print(f"{GREEN}{bar}{RESET}")
+    print()
+    print(f"Tests output: {location}")
 
 
 def load_module_from_path(path: str | Path):