wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scripts: allow storing logs for release evals

Browse Source
This commit is contained in:
Aleix Conchillo Flaqué
2025-05-27 20:45:31 -07:00
parent 736c7f1f30
commit 356f4039e4
3 changed files with 57 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
scripts/release/eval.py
View File

@@ -55,23 +55,29 @@ PIPELINE_IDLE_TIMEOUT_SECS = 30
class EvalRunner:
def __init__(self, *, pattern: str = "", record_audio: bool = False):
def __init__(self, *, pattern: str = "", record_audio: bool = False, log_level: str = "DEBUG"):
self._pattern = f".*{pattern}.*" if pattern else ""
self._record_audio = record_audio
self._log_level = log_level
self._total_success = 0
self._tests: List[EvalResult] = []
self._queue = asyncio.Queue()
@property
def record_audio(self):
return self._record_audio
# We to save runner files.
self._runs_dir = os.path.join(
SCRIPT_DIR, "test-runs", f"{datetime.now().strftime('%Y%m%d_%H%M%S')}"
)
self._logs_dir = os.path.join(self._runs_dir, "logs")
self._recordings_dir = os.path.join(self._runs_dir, "recordings")
os.makedirs(self._logs_dir, exist_ok=True)
os.makedirs(self._recordings_dir, exist_ok=True)
async def assert_eval(self, params: FunctionCallParams):
reasoning = params.arguments["reasoning"]
logger.debug(f"🧠 EVAL REASONING: {reasoning}")
await self._queue.put(params.arguments["result"])
await params.llm.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM)
await params.result_callback(None)
await params.llm.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM)
async def assert_eval_false(self):
await self._queue.put(False)
@@ -80,6 +86,10 @@ class EvalRunner:
if not re.match(self._pattern, example_file):
return
# Store logs
filename = self._log_file_name(example_file)
log_file_id = logger.add(filename, level=self._log_level)
print_begin_test(example_file)
start_time = time.time()
@@ -111,8 +121,37 @@ class EvalRunner:
print_end_test(example_file, result, eval_time)
logger.remove(log_file_id)
def print_results(self):
print_test_results(self._tests, self._total_success)
print_test_results(self._tests, self._total_success, self._runs_dir)
async def save_audio(self, name: str, audio: bytes, sample_rate: int, num_channels: int):
if len(audio) > 0:
filename = self._recording_file_name(name)
with io.BytesIO() as buffer:
with wave.open(buffer, "wb") as wf:
wf.setsampwidth(2)
wf.setnchannels(num_channels)
wf.setframerate(sample_rate)
wf.writeframes(audio)
async with aiofiles.open(filename, "wb") as file:
await file.write(buffer.getvalue())
logger.debug(f"Saving {name} audio to {filename}")
else:
logger.warning(f"There's no audio to save for {name}")
def _base_file_name(self, example_file: str):
base_name = os.path.splitext(example_file)[0]
return f"{base_name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
def _log_file_name(self, example_file: str):
base_name = self._base_file_name(example_file)
return os.path.join(self._logs_dir, f"{base_name}.log")
def _recording_file_name(self, example_file: str):
base_name = self._base_file_name(example_file)
return os.path.join(self._recordings_dir, f"{base_name}.wav")
async def run_example_pipeline(example_file: str):
@@ -136,28 +175,6 @@ async def run_example_pipeline(example_file: str):
await module.run_example(transport, argparse.Namespace(), True)
async def save_audio(audio: bytes, sample_rate: int, num_channels: int, name: str):
if len(audio) > 0:
recordings_dir = os.path.join(SCRIPT_DIR, "recordings")
base_name = os.path.splitext(name)[0]
os.makedirs(recordings_dir, exist_ok=True)
filename = os.path.join(
recordings_dir,
f"{base_name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.wav",
)
with io.BytesIO() as buffer:
with wave.open(buffer, "wb") as wf:
wf.setsampwidth(2)
wf.setnchannels(num_channels)
wf.setframerate(sample_rate)
wf.writeframes(audio)
async with aiofiles.open(filename, "wb") as file:
await file.write(buffer.getvalue())
logger.debug(f"Saving {name} audio to {filename}")
else:
logger.warning(f"There's no audio to save for {name}")
async def run_eval_pipeline(
eval_runner: EvalRunner, example_file: str, prompt: str, eval: Optional[str]
):
@@ -212,7 +229,7 @@ async def run_eval_pipeline(
messages = [
{
"role": "system",
"content": f"You are an LLM eval, be extremly brief. Your goal is to only ask one question: {prompt}. Tell the user to simply say the answer. Call the eval function only if the user answers the question and check if the answer is correct (words as numbers are valid). {eval_prompt}",
"content": f"You are an LLM eval, be extremly brief. Your goal is to only ask one question: {prompt}. Call the eval function only if the user answers the question and check if the answer is correct (words as numbers are valid). {eval_prompt}",
},
]
@@ -246,8 +263,7 @@ async def run_eval_pipeline(
@audio_buffer.event_handler("on_audio_data")
async def on_audio_data(buffer, audio, sample_rate, num_channels):
if eval_runner.record_audio:
await save_audio(audio, sample_rate, num_channels, example_file)
await eval_runner.save_audio(example_file, audio, sample_rate, num_channels)
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):

13
scripts/release/run-release-evals.py
View File

@@ -117,7 +117,13 @@ async def main(args: argparse.Namespace):
if not check_env_variables():
return
runner = EvalRunner(pattern=args.pattern, record_audio=args.audio)
# Log level
logger.remove(0)
log_level = "TRACE" if args.verbose >= 2 else "DEBUG"
if args.verbose:
logger.add(sys.stderr, level=log_level)
runner = EvalRunner(pattern=args.pattern, record_audio=args.audio, log_level=log_level)
for test, prompt, eval in TESTS:
await runner.run_eval(test, prompt, eval)
@@ -131,9 +137,4 @@ if __name__ == "__main__":
parser.add_argument("--verbose", "-v", action="count", default=0)
args = parser.parse_args()
# Log level
logger.remove(0)
if args.verbose:
logger.add(sys.stderr, level="TRACE" if args.verbose >= 2 else "DEBUG")
asyncio.run(main(args))

4
scripts/release/utils.py
View File

@@ -47,7 +47,7 @@ def print_end_test(example_file: str, passed: bool, time: float):
print(f"{example_file:<55} {status} ({time:.2f}s){CLEAR}")
def print_test_results(tests: Sequence[EvalResult], total_success: int):
def print_test_results(tests: Sequence[EvalResult], total_success: int, location: str):
total_count = len(tests)
bar = "=" * 80
@@ -70,6 +70,8 @@ def print_test_results(tests: Sequence[EvalResult], total_success: int):
f"{GREEN}SUCCESS{RESET}: {total_success} | {RED}FAIL{RESET}: {total_fail} | TOTAL TIME: {total_time:.2f}s"
)
print(f"{GREEN}{bar}{RESET}")
print()
print(f"Tests output: {location}")
def load_module_from_path(path: str | Path):