scripts: allow specifying a name for the test run

scripts/release/eval.py

@@ -55,7 +55,14 @@ PIPELINE_IDLE_TIMEOUT_SECS = 30
 
 
 class EvalRunner:
-    def __init__(self, *, pattern: str = "", record_audio: bool = False, log_level: str = "DEBUG"):
+    def __init__(
+        self,
+        *,
+        pattern: str = "",
+        record_audio: bool = False,
+        name: Optional[str] = None,
+        log_level: str = "DEBUG",
+    ):
         self._pattern = f".*{pattern}.*" if pattern else ""
         self._record_audio = record_audio
         self._log_level = log_level
@@ -64,9 +71,8 @@ class EvalRunner:
         self._queue = asyncio.Queue()
 
         # We to save runner files.
-        self._runs_dir = os.path.join(
-            SCRIPT_DIR, "test-runs", f"{datetime.now().strftime('%Y%m%d_%H%M%S')}"
-        )
+        name = name or f"{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        self._runs_dir = os.path.join(SCRIPT_DIR, "test-runs", name)
         self._logs_dir = os.path.join(self._runs_dir, "logs")
         self._recordings_dir = os.path.join(self._runs_dir, "recordings")
         os.makedirs(self._logs_dir, exist_ok=True)

scripts/release/run-release-evals.py

@@ -123,7 +123,13 @@ async def main(args: argparse.Namespace):
     if args.verbose:
         logger.add(sys.stderr, level=log_level)
 
-    runner = EvalRunner(pattern=args.pattern, record_audio=args.audio, log_level=log_level)
+    runner = EvalRunner(
+        name=args.name,
+        pattern=args.pattern,
+        record_audio=args.audio,
+        log_level=log_level,
+    )
+
     for test, prompt, eval in TESTS:
         await runner.run_eval(test, prompt, eval)
 
@@ -133,6 +139,7 @@ async def main(args: argparse.Namespace):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Pipecat Eval Runner")
     parser.add_argument("--audio", "-a", action="store_true", help="Record audio for each test")
+    parser.add_argument("--name", "-n", help="Name for the current runner (e.g. 'v.0.0.68')")
     parser.add_argument("--pattern", "-p", help="Only run tests that match the pattern")
     parser.add_argument("--verbose", "-v", action="count", default=0)
     args = parser.parse_args()