From 4e621e2f0c834969ac3cab12d1083b3be8406902 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Wed, 29 Jul 2026 11:13:07 +0800 Subject: [PATCH] Enhance evaluation runner with transcript handling and new case formats - Updated the evaluation runner to support live dialogue printing and transcript saving in both Markdown and JSON formats. - Introduced new data classes for managing transcripts and their components, improving structure and readability. - Modified existing YAML case files to align with the new direct client mode and updated stage codes. - Added new test cases for the direct client mode, ensuring comprehensive coverage of user interactions. --- .gitignore | 3 +- evals/README.md | 42 ++++- evals/cases/browser_addon/initial.yaml | 60 +++++++ evals/cases/direct/initial.yaml | 98 +++++++++- evals/run.py | 237 ++++++++++++++++++++++--- 5 files changed, 406 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index e72cd47..da3f06c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ __pycache__/ logs/ *.log -.env \ No newline at end of file +.env +evals/results/ \ No newline at end of file diff --git a/evals/README.md b/evals/README.md index 50fe18b..406d3eb 100644 --- a/evals/README.md +++ b/evals/README.md @@ -2,27 +2,29 @@ The evaluation runner calls a live FastAPI service backed by FastGPT. For now, each conversation step checks only the SSE `stage_code.nextStageCode` value. +During the run it also prints the live dialogue and saves transcripts. ## Case format Add YAML files anywhere under `evals/cases/`: ```yaml -name: browser_addon_example -clientMode: browser_addon +name: direct_initial_stage +clientMode: direct steps: - - input: hi + - input: 新对话 expect: - stageCode: browser_addon.1001 + stageCode: "1001" - - input: 继续 + - input: 继续办理 expect: - stageCode: browser_addon.1002 + stageCode: "1002" ``` -Steps in one file share the same generated `sessionId`. Different cases always -use different session IDs. +`clientMode` in the case file is the API request setting (`direct` or +`browser_addon`). Steps in one file share the same generated `sessionId`. +Different cases always use different session IDs. ## Run @@ -41,8 +43,32 @@ python evals/run.py --base-url http://192.168.1.10:8000 Run selected cases: ```bash +python evals/run.py --case direct python evals/run.py --case browser_addon ``` +## Dialogue and transcripts + +While a case runs, each step prints the user input, bot reply (`text_delta`), +and stage-code check result. + +By default, after the run finishes, transcripts are written under +`evals/results//`: + +- `*.md` — readable conversation log +- `*.json` — structured step results + +Save to a specific directory: + +```bash +python evals/run.py --output-dir evals/results/manual-run --case direct +``` + +Print live dialogue only, without writing files: + +```bash +python evals/run.py --no-save --case direct +``` + The process exits with code `1` when an expectation fails and code `2` when case configuration is invalid. diff --git a/evals/cases/browser_addon/initial.yaml b/evals/cases/browser_addon/initial.yaml index 034adfd..7a6ddde 100644 --- a/evals/cases/browser_addon/initial.yaml +++ b/evals/cases/browser_addon/initial.yaml @@ -5,3 +5,63 @@ steps: - input: hi expect: stageCode: browser_addon.1001 + + - input: 继续办理 + expect: + stageCode: browser_addon.1002 + + - input: 十分钟之前两车追尾 + expect: + stageCode: browser_addon.1002 + + - input: 没有 + expect: + stageCode: browser_addon.1002 + + - input: 没有 + expect: + stageCode: browser_addon.1002 + + - input: 十分钟之前 + expect: + stageCode: browser_addon.1002 + + - input: 对 + expect: + stageCode: browser_addon.1002 + + - input: 是的 + expect: + stageCode: browser_addon.1002 + + - input: 对 + expect: + stageCode: browser_addon.2010 + + - input: 【拍摄完成】 + expect: + stageCode: browser_addon.2011 + + - input: 【拍摄完成】 + expect: + stageCode: browser_addon.2012 + + - input: 【拍摄完成】 + expect: + stageCode: browser_addon.2013 + + - input: 【拍摄完成】 + expect: + stageCode: browser_addon.2014 + + - input: 【拍摄完成】 + expect: + stageCode: browser_addon.2015 + + - input: 【拍摄完成】 + expect: + stageCode: browser_addon.2016 + + - input: 没错 + expect: + stageCode: browser_addon.0000 \ No newline at end of file diff --git a/evals/cases/direct/initial.yaml b/evals/cases/direct/initial.yaml index 6758917..fc1a7c2 100644 --- a/evals/cases/direct/initial.yaml +++ b/evals/cases/direct/initial.yaml @@ -2,6 +2,102 @@ name: direct_initial_stage clientMode: direct steps: - - input: hi + - input: 新对话 expect: stageCode: "1001" + + - input: 继续办理 + expect: + stageCode: "1002" + + - input: 十分钟之前两车追尾 + expect: + stageCode: "1002" + + - input: 没有 + expect: + stageCode: "1002" + + - input: 没有 + expect: + stageCode: "1002" + + - input: 十分钟之前 + expect: + stageCode: "1002" + + - input: 对 + expect: + stageCode: "1002" + + - input: 是的 + expect: + stageCode: "1002" + + - input: 对 + expect: + stageCode: "2010" + + - input: 【拍摄完成】 + expect: + stageCode: "2011" + + - input: 【拍摄完成】 + expect: + stageCode: "2012" + + - input: 【拍摄完成】 + expect: + stageCode: "2013" + + - input: 【拍摄完成】 + expect: + stageCode: "2014" + + - input: 【拍摄完成】 + expect: + stageCode: "2015" + + - input: 【拍摄完成】 + expect: + stageCode: "2016" + + - input: 没错 + expect: + stageCode: "1002" + + - input: 对 + expect: + stageCode: "1002" + + - input: 李四 + expect: + stageCode: "1002" + + - input: 对 + expect: + stageCode: "1002" + + - input: 对 + expect: + stageCode: "1002" + + - input: 移交完毕 + expect: + stageCode: "1002" + + - input: 对 + expect: + stageCode: "1002" + + - input: 刘云 + expect: + stageCode: "1002" + + - input: 对的 + expect: + stageCode: "1002" + + - input: 是 + expect: + stageCode: "0000" diff --git a/evals/run.py b/evals/run.py index 7f59826..52eaee8 100644 --- a/evals/run.py +++ b/evals/run.py @@ -6,7 +6,7 @@ import json import os import re import sys -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from uuid import uuid4 @@ -17,18 +17,47 @@ import yaml EVALS_DIR = Path(__file__).resolve().parent DEFAULT_CASES_DIR = EVALS_DIR / "cases" +DEFAULT_OUTPUT_DIR = EVALS_DIR / "results" @dataclass class ChatResult: stage_codes: list[str] errors: list[dict] + text: str = "" + + +@dataclass +class TranscriptTurn: + step: int + input: str + expected_stage_code: str + actual_stage_codes: list[str] + reply: str + errors: list[dict] + ok: bool + error: str | None = None + + +@dataclass +class CaseTranscript: + name: str + path: str + client_mode: str + session_id: str + passed: bool + turns: list[TranscriptTurn] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) def timestamp() -> str: return datetime.now(timezone.utc).isoformat() +def local_run_stamp() -> str: + return datetime.now().strftime("%Y%m%d-%H%M%S") + + def load_cases(cases_dir: Path, name_filters: list[str]) -> list[dict]: cases = [] for path in sorted(cases_dir.rglob("*.yaml")): @@ -73,6 +102,7 @@ def load_cases(cases_dir: Path, name_filters: list[str]) -> list[dict]: def parse_sse(response: httpx.Response) -> ChatResult: stage_codes: list[str] = [] errors: list[dict] = [] + text_parts: list[str] = [] event_name = "" data_lines: list[str] = [] @@ -82,6 +112,10 @@ def parse_sse(response: httpx.Response) -> ChatResult: payload = json.loads("\n".join(data_lines)) if event_name == "stage_code": stage_codes.append(payload["nextStageCode"]) + elif event_name == "text_delta": + chunk = payload.get("text") + if isinstance(chunk, str) and chunk: + text_parts.append(chunk) elif event_name == "error": errors.append(payload) event_name = "" @@ -96,7 +130,11 @@ def parse_sse(response: httpx.Response) -> ChatResult: data_lines.append(line.removeprefix("data:").strip()) consume_event() - return ChatResult(stage_codes=stage_codes, errors=errors) + return ChatResult( + stage_codes=stage_codes, + errors=errors, + text="".join(text_parts), + ) def run_chat_step( @@ -144,41 +182,169 @@ def session_id_for(case_name: str) -> str: return f"eval-{slug[:24]}-{uuid4().hex[:12]}" -def run_case(client: httpx.Client, case: dict) -> tuple[bool, list[str]]: +def print_turn(turn: TranscriptTurn) -> None: + status = "OK" if turn.ok else "FAIL" + print(f" [{status}] step {turn.step}") + print(f" user: {turn.input}") + if turn.reply: + print(f" bot: {turn.reply}") + else: + print(" bot: ") + print( + f" stage: expected={turn.expected_stage_code} " + f"actual={turn.actual_stage_codes}" + ) + if turn.error: + print(f" error: {turn.error}") + if turn.errors: + print(f" sse_errors: {turn.errors}") + + +def transcript_to_markdown(transcript: CaseTranscript) -> str: + lines = [ + f"# {transcript.name}", + "", + f"- path: `{transcript.path}`", + f"- clientMode: `{transcript.client_mode}`", + f"- sessionId: `{transcript.session_id}`", + f"- result: `{'PASS' if transcript.passed else 'FAIL'}`", + "", + ] + for turn in transcript.turns: + status = "OK" if turn.ok else "FAIL" + lines.extend( + [ + f"## Step {turn.step} ({status})", + "", + f"**User:** {turn.input}", + "", + f"**Bot:** {turn.reply or ''}", + "", + f"- expected stageCode: `{turn.expected_stage_code}`", + f"- actual stageCodes: `{turn.actual_stage_codes}`", + ] + ) + if turn.error: + lines.append(f"- error: `{turn.error}`") + if turn.errors: + lines.append(f"- sse_errors: `{turn.errors}`") + lines.append("") + if transcript.warnings: + lines.append("## Warnings") + lines.append("") + for warning in transcript.warnings: + lines.append(f"- {warning}") + lines.append("") + return "\n".join(lines) + + +def transcript_to_json(transcript: CaseTranscript) -> dict: + return { + "name": transcript.name, + "path": transcript.path, + "clientMode": transcript.client_mode, + "sessionId": transcript.session_id, + "passed": transcript.passed, + "warnings": transcript.warnings, + "turns": [ + { + "step": turn.step, + "input": turn.input, + "expectedStageCode": turn.expected_stage_code, + "actualStageCodes": turn.actual_stage_codes, + "reply": turn.reply, + "errors": turn.errors, + "ok": turn.ok, + "error": turn.error, + } + for turn in transcript.turns + ], + } + + +def safe_case_filename(name: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", name).strip("-") + return slug or "case" + + +def save_transcript(output_dir: Path, transcript: CaseTranscript) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + base = safe_case_filename(transcript.name) + json_path = output_dir / f"{base}.json" + md_path = output_dir / f"{base}.md" + json_path.write_text( + json.dumps(transcript_to_json(transcript), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + md_path.write_text(transcript_to_markdown(transcript), encoding="utf-8") + return md_path + + +def run_case(client: httpx.Client, case: dict) -> CaseTranscript: session_id = session_id_for(case["name"]) - messages = [] - passed = True + transcript = CaseTranscript( + name=case["name"], + path=case["_path"], + client_mode=case["clientMode"], + session_id=session_id, + passed=True, + ) + + print( + f"\n=== {case['name']} clientMode={case['clientMode']} " + f"({case['_path']}) session={session_id} ===" + ) try: for step_number, step in enumerate(case["steps"], start=1): expected = step["expect"]["stageCode"] + user_input = step["input"] try: result = run_chat_step( client, session_id=session_id, client_mode=case["clientMode"], - text=step["input"], + text=user_input, ) except (httpx.HTTPError, json.JSONDecodeError, KeyError) as exc: - messages.append(f"step {step_number}: request error: {exc}") - passed = False - break - - if result.stage_codes != [expected]: - messages.append( - f"step {step_number}: expected={[expected]} " - f"actual={result.stage_codes} sse_errors={result.errors}" + turn = TranscriptTurn( + step=step_number, + input=user_input, + expected_stage_code=expected, + actual_stage_codes=[], + reply="", + errors=[], + ok=False, + error=str(exc), ) - passed = False + transcript.turns.append(turn) + transcript.passed = False + print_turn(turn) break - messages.append(f"step {step_number}: stageCode={expected}") + ok = result.stage_codes == [expected] + turn = TranscriptTurn( + step=step_number, + input=user_input, + expected_stage_code=expected, + actual_stage_codes=result.stage_codes, + reply=result.text, + errors=result.errors, + ok=ok, + error=None if ok else "stageCode mismatch", + ) + transcript.turns.append(turn) + print_turn(turn) + if not ok: + transcript.passed = False + break finally: cleanup_warning = cleanup_session(client, session_id) if cleanup_warning: - messages.append(f"warning: {cleanup_warning}") + transcript.warnings.append(cleanup_warning) + print(f" warning: {cleanup_warning}") - return passed, messages + return transcript def parse_args() -> argparse.Namespace: @@ -202,6 +368,20 @@ def parse_args() -> argparse.Namespace: default=[], help="Run cases whose names contain this value; repeatable", ) + parser.add_argument( + "--output-dir", + type=Path, + default=None, + help=( + "Directory for conversation transcripts " + f"(default: {DEFAULT_OUTPUT_DIR}/)" + ), + ) + parser.add_argument( + "--no-save", + action="store_true", + help="Print dialogue live but do not write transcript files", + ) parser.add_argument("--timeout", type=float, default=90.0) parser.add_argument( "--insecure", @@ -219,6 +399,12 @@ def main() -> int: print(f"ERROR: {exc}", file=sys.stderr) return 2 + output_dir: Path | None = None + if not args.no_save: + output_dir = args.output_dir or (DEFAULT_OUTPUT_DIR / local_run_stamp()) + output_dir.mkdir(parents=True, exist_ok=True) + print(f"Transcripts will be saved under: {output_dir}") + passed_count = 0 failed_count = 0 with httpx.Client( @@ -227,12 +413,13 @@ def main() -> int: verify=not args.insecure, ) as client: for case in cases: - passed, messages = run_case(client, case) - status = "PASS" if passed else "FAIL" - print(f"{status} {case['name']} ({case['_path']})") - for message in messages: - print(f" {message}") - if passed: + transcript = run_case(client, case) + status = "PASS" if transcript.passed else "FAIL" + print(f"{status} {transcript.name} clientMode={transcript.client_mode}") + if output_dir is not None: + saved = save_transcript(output_dir, transcript) + print(f" saved: {saved}") + if transcript.passed: passed_count += 1 else: failed_count += 1 @@ -241,6 +428,8 @@ def main() -> int: f"\nSummary: total={len(cases)} " f"passed={passed_count} failed={failed_count}" ) + if output_dir is not None: + print(f"Transcripts: {output_dir}") return 1 if failed_count else 0