Enhance evaluation runner with transcript handling and new case formats

- Updated the evaluation runner to support live dialogue printing and transcript saving in both Markdown and JSON formats. - Introduced new data classes for managing transcripts and their components, improving structure and readability. - Modified existing YAML case files to align with the new direct client mode and updated stage codes. - Added new test cases for the direct client mode, ensuring comprehensive coverage of user interactions.
2026-07-29 11:13:07 +08:00
parent e6437ab332
commit 4e621e2f0c
5 changed files with 406 additions and 34 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,5 @@
 __pycache__/
 logs/
 *.log
-.env
+.env
+evals/results/
--- a/evals/README.md
+++ b/evals/README.md
@@ -2,27 +2,29 @@

 The evaluation runner calls a live FastAPI service backed by FastGPT. For now,
 each conversation step checks only the SSE `stage_code.nextStageCode` value.
+During the run it also prints the live dialogue and saves transcripts.

 ## Case format

 Add YAML files anywhere under `evals/cases/`:

 ```yaml
-name: browser_addon_example
-clientMode: browser_addon
+name: direct_initial_stage
+clientMode: direct

 steps:
-  - input: hi
+  - input: 新对话
    expect:
-      stageCode: browser_addon.1001
+      stageCode: "1001"

-  - input: 继续
+  - input: 继续办理
    expect:
-      stageCode: browser_addon.1002
+      stageCode: "1002"
 ```

-Steps in one file share the same generated `sessionId`. Different cases always
-use different session IDs.
+`clientMode` in the case file is the API request setting (`direct` or
+`browser_addon`). Steps in one file share the same generated `sessionId`.
+Different cases always use different session IDs.

 ## Run

@@ -41,8 +43,32 @@ python evals/run.py --base-url http://192.168.1.10:8000
 Run selected cases:

 ```bash
+python evals/run.py --case direct
 python evals/run.py --case browser_addon
 ```

+## Dialogue and transcripts
+
+While a case runs, each step prints the user input, bot reply (`text_delta`),
+and stage-code check result.
+
+By default, after the run finishes, transcripts are written under
+`evals/results/<timestamp>/`:
+
+- `*.md` — readable conversation log
+- `*.json` — structured step results
+
+Save to a specific directory:
+
+```bash
+python evals/run.py --output-dir evals/results/manual-run --case direct
+```
+
+Print live dialogue only, without writing files:
+
+```bash
+python evals/run.py --no-save --case direct
+```
+
 The process exits with code `1` when an expectation fails and code `2` when
 case configuration is invalid.
--- a/evals/cases/browser_addon/initial.yaml
+++ b/evals/cases/browser_addon/initial.yaml
@@ -5,3 +5,63 @@ steps:
  - input: hi
    expect:
      stageCode: browser_addon.1001
+
+  - input: 继续办理
+    expect:
+      stageCode: browser_addon.1002
+
+  - input: 十分钟之前两车追尾
+    expect:
+      stageCode: browser_addon.1002
+
+  - input: 没有
+    expect:
+      stageCode: browser_addon.1002
+
+  - input: 没有
+    expect:
+      stageCode: browser_addon.1002
+
+  - input: 十分钟之前
+    expect:
+      stageCode: browser_addon.1002
+
+  - input: 对
+    expect:
+      stageCode: browser_addon.1002
+
+  - input: 是的
+    expect:
+      stageCode: browser_addon.1002
+
+  - input: 对
+    expect:
+      stageCode: browser_addon.2010
+
+  - input: 【拍摄完成】
+    expect:
+      stageCode: browser_addon.2011
+
+  - input: 【拍摄完成】
+    expect:
+      stageCode: browser_addon.2012
+
+  - input: 【拍摄完成】
+    expect:
+      stageCode: browser_addon.2013
+
+  - input: 【拍摄完成】
+    expect:
+      stageCode: browser_addon.2014
+
+  - input: 【拍摄完成】
+    expect:
+      stageCode: browser_addon.2015
+
+  - input: 【拍摄完成】
+    expect:
+      stageCode: browser_addon.2016
+
+  - input: 没错
+    expect:
+      stageCode: browser_addon.0000
--- a/evals/cases/direct/initial.yaml
+++ b/evals/cases/direct/initial.yaml
@@ -2,6 +2,102 @@ name: direct_initial_stage
 clientMode: direct

 steps:
-  - input: hi
+  - input: 新对话
    expect:
      stageCode: "1001"
+
+  - input: 继续办理
+    expect:
+      stageCode: "1002"
+
+  - input: 十分钟之前两车追尾
+    expect:
+      stageCode: "1002"
+
+  - input: 没有
+    expect:
+      stageCode: "1002"
+
+  - input: 没有
+    expect:
+      stageCode: "1002"
+
+  - input: 十分钟之前
+    expect:
+      stageCode: "1002"
+
+  - input: 对
+    expect:
+      stageCode: "1002"
+
+  - input: 是的
+    expect:
+      stageCode: "1002"
+
+  - input: 对
+    expect:
+      stageCode: "2010"
+
+  - input: 【拍摄完成】
+    expect:
+      stageCode: "2011"
+
+  - input: 【拍摄完成】
+    expect:
+      stageCode: "2012"
+
+  - input: 【拍摄完成】
+    expect:
+      stageCode: "2013"
+
+  - input: 【拍摄完成】
+    expect:
+      stageCode: "2014"
+
+  - input: 【拍摄完成】
+    expect:
+      stageCode: "2015"
+
+  - input: 【拍摄完成】
+    expect:
+      stageCode: "2016"
+
+  - input: 没错
+    expect:
+      stageCode: "1002"
+
+  - input: 对
+    expect:
+      stageCode: "1002"
+
+  - input: 李四
+    expect:
+      stageCode: "1002"
+
+  - input: 对
+    expect:
+      stageCode: "1002"
+
+  - input: 对
+    expect:
+      stageCode: "1002"
+
+  - input: 移交完毕
+    expect:
+      stageCode: "1002"
+
+  - input: 对
+    expect:
+      stageCode: "1002"
+
+  - input: 刘云
+    expect:
+      stageCode: "1002"
+
+  - input: 对的
+    expect:
+      stageCode: "1002"
+
+  - input: 是
+    expect:
+      stageCode: "0000"
--- a/evals/run.py
+++ b/evals/run.py
@@ -6,7 +6,7 @@ import json
 import os
 import re
 import sys
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from pathlib import Path
 from uuid import uuid4
@@ -17,18 +17,47 @@ import yaml

 EVALS_DIR = Path(__file__).resolve().parent
 DEFAULT_CASES_DIR = EVALS_DIR / "cases"
+DEFAULT_OUTPUT_DIR = EVALS_DIR / "results"


@dataclass
 class ChatResult:
    stage_codes: list[str]
    errors: list[dict]
+    text: str = ""
+
+
+@dataclass
+class TranscriptTurn:
+    step: int
+    input: str
+    expected_stage_code: str
+    actual_stage_codes: list[str]
+    reply: str
+    errors: list[dict]
+    ok: bool
+    error: str | None = None
+
+
+@dataclass
+class CaseTranscript:
+    name: str
+    path: str
+    client_mode: str
+    session_id: str
+    passed: bool
+    turns: list[TranscriptTurn] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)


 def timestamp() -> str:
    return datetime.now(timezone.utc).isoformat()


+def local_run_stamp() -> str:
+    return datetime.now().strftime("%Y%m%d-%H%M%S")
+
+
 def load_cases(cases_dir: Path, name_filters: list[str]) -> list[dict]:
    cases = []
    for path in sorted(cases_dir.rglob("*.yaml")):
@@ -73,6 +102,7 @@ def load_cases(cases_dir: Path, name_filters: list[str]) -> list[dict]:
 def parse_sse(response: httpx.Response) -> ChatResult:
    stage_codes: list[str] = []
    errors: list[dict] = []
+    text_parts: list[str] = []
    event_name = ""
    data_lines: list[str] = []

@@ -82,6 +112,10 @@ def parse_sse(response: httpx.Response) -> ChatResult:
            payload = json.loads("\n".join(data_lines))
            if event_name == "stage_code":
                stage_codes.append(payload["nextStageCode"])
+            elif event_name == "text_delta":
+                chunk = payload.get("text")
+                if isinstance(chunk, str) and chunk:
+                    text_parts.append(chunk)
            elif event_name == "error":
                errors.append(payload)
        event_name = ""
@@ -96,7 +130,11 @@ def parse_sse(response: httpx.Response) -> ChatResult:
            data_lines.append(line.removeprefix("data:").strip())

    consume_event()
-    return ChatResult(stage_codes=stage_codes, errors=errors)
+    return ChatResult(
+        stage_codes=stage_codes,
+        errors=errors,
+        text="".join(text_parts),
+    )


 def run_chat_step(
@@ -144,41 +182,169 @@ def session_id_for(case_name: str) -> str:
    return f"eval-{slug[:24]}-{uuid4().hex[:12]}"


-def run_case(client: httpx.Client, case: dict) -> tuple[bool, list[str]]:
+def print_turn(turn: TranscriptTurn) -> None:
+    status = "OK" if turn.ok else "FAIL"
+    print(f"  [{status}] step {turn.step}")
+    print(f"    user: {turn.input}")
+    if turn.reply:
+        print(f"    bot:  {turn.reply}")
+    else:
+        print("    bot:  <empty>")
+    print(
+        f"    stage: expected={turn.expected_stage_code} "
+        f"actual={turn.actual_stage_codes}"
+    )
+    if turn.error:
+        print(f"    error: {turn.error}")
+    if turn.errors:
+        print(f"    sse_errors: {turn.errors}")
+
+
+def transcript_to_markdown(transcript: CaseTranscript) -> str:
+    lines = [
+        f"# {transcript.name}",
+        "",
+        f"- path: `{transcript.path}`",
+        f"- clientMode: `{transcript.client_mode}`",
+        f"- sessionId: `{transcript.session_id}`",
+        f"- result: `{'PASS' if transcript.passed else 'FAIL'}`",
+        "",
+    ]
+    for turn in transcript.turns:
+        status = "OK" if turn.ok else "FAIL"
+        lines.extend(
+            [
+                f"## Step {turn.step} ({status})",
+                "",
+                f"**User:** {turn.input}",
+                "",
+                f"**Bot:** {turn.reply or '<empty>'}",
+                "",
+                f"- expected stageCode: `{turn.expected_stage_code}`",
+                f"- actual stageCodes: `{turn.actual_stage_codes}`",
+            ]
+        )
+        if turn.error:
+            lines.append(f"- error: `{turn.error}`")
+        if turn.errors:
+            lines.append(f"- sse_errors: `{turn.errors}`")
+        lines.append("")
+    if transcript.warnings:
+        lines.append("## Warnings")
+        lines.append("")
+        for warning in transcript.warnings:
+            lines.append(f"- {warning}")
+        lines.append("")
+    return "\n".join(lines)
+
+
+def transcript_to_json(transcript: CaseTranscript) -> dict:
+    return {
+        "name": transcript.name,
+        "path": transcript.path,
+        "clientMode": transcript.client_mode,
+        "sessionId": transcript.session_id,
+        "passed": transcript.passed,
+        "warnings": transcript.warnings,
+        "turns": [
+            {
+                "step": turn.step,
+                "input": turn.input,
+                "expectedStageCode": turn.expected_stage_code,
+                "actualStageCodes": turn.actual_stage_codes,
+                "reply": turn.reply,
+                "errors": turn.errors,
+                "ok": turn.ok,
+                "error": turn.error,
+            }
+            for turn in transcript.turns
+        ],
+    }
+
+
+def safe_case_filename(name: str) -> str:
+    slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", name).strip("-")
+    return slug or "case"
+
+
+def save_transcript(output_dir: Path, transcript: CaseTranscript) -> Path:
+    output_dir.mkdir(parents=True, exist_ok=True)
+    base = safe_case_filename(transcript.name)
+    json_path = output_dir / f"{base}.json"
+    md_path = output_dir / f"{base}.md"
+    json_path.write_text(
+        json.dumps(transcript_to_json(transcript), ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+    md_path.write_text(transcript_to_markdown(transcript), encoding="utf-8")
+    return md_path
+
+
+def run_case(client: httpx.Client, case: dict) -> CaseTranscript:
    session_id = session_id_for(case["name"])
-    messages = []
-    passed = True
+    transcript = CaseTranscript(
+        name=case["name"],
+        path=case["_path"],
+        client_mode=case["clientMode"],
+        session_id=session_id,
+        passed=True,
+    )
+
+    print(
+        f"\n=== {case['name']} clientMode={case['clientMode']} "
+        f"({case['_path']}) session={session_id} ==="
+    )

    try:
        for step_number, step in enumerate(case["steps"], start=1):
            expected = step["expect"]["stageCode"]
+            user_input = step["input"]
            try:
                result = run_chat_step(
                    client,
                    session_id=session_id,
                    client_mode=case["clientMode"],
-                    text=step["input"],
+                    text=user_input,
                )
            except (httpx.HTTPError, json.JSONDecodeError, KeyError) as exc:
-                messages.append(f"step {step_number}: request error: {exc}")
-                passed = False
-                break
-
-            if result.stage_codes != [expected]:
-                messages.append(
-                    f"step {step_number}: expected={[expected]} "
-                    f"actual={result.stage_codes} sse_errors={result.errors}"
+                turn = TranscriptTurn(
+                    step=step_number,
+                    input=user_input,
+                    expected_stage_code=expected,
+                    actual_stage_codes=[],
+                    reply="",
+                    errors=[],
+                    ok=False,
+                    error=str(exc),
                )
-                passed = False
+                transcript.turns.append(turn)
+                transcript.passed = False
+                print_turn(turn)
                break

-            messages.append(f"step {step_number}: stageCode={expected}")
+            ok = result.stage_codes == [expected]
+            turn = TranscriptTurn(
+                step=step_number,
+                input=user_input,
+                expected_stage_code=expected,
+                actual_stage_codes=result.stage_codes,
+                reply=result.text,
+                errors=result.errors,
+                ok=ok,
+                error=None if ok else "stageCode mismatch",
+            )
+            transcript.turns.append(turn)
+            print_turn(turn)
+            if not ok:
+                transcript.passed = False
+                break
    finally:
        cleanup_warning = cleanup_session(client, session_id)
        if cleanup_warning:
-            messages.append(f"warning: {cleanup_warning}")
+            transcript.warnings.append(cleanup_warning)
+            print(f"  warning: {cleanup_warning}")

-    return passed, messages
+    return transcript


 def parse_args() -> argparse.Namespace:
@@ -202,6 +368,20 @@ def parse_args() -> argparse.Namespace:
        default=[],
        help="Run cases whose names contain this value; repeatable",
    )
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=None,
+        help=(
+            "Directory for conversation transcripts "
+            f"(default: {DEFAULT_OUTPUT_DIR}/<timestamp>)"
+        ),
+    )
+    parser.add_argument(
+        "--no-save",
+        action="store_true",
+        help="Print dialogue live but do not write transcript files",
+    )
    parser.add_argument("--timeout", type=float, default=90.0)
    parser.add_argument(
        "--insecure",
@@ -219,6 +399,12 @@ def main() -> int:
        print(f"ERROR: {exc}", file=sys.stderr)
        return 2

+    output_dir: Path | None = None
+    if not args.no_save:
+        output_dir = args.output_dir or (DEFAULT_OUTPUT_DIR / local_run_stamp())
+        output_dir.mkdir(parents=True, exist_ok=True)
+        print(f"Transcripts will be saved under: {output_dir}")
+
    passed_count = 0
    failed_count = 0
    with httpx.Client(
@@ -227,12 +413,13 @@ def main() -> int:
        verify=not args.insecure,
    ) as client:
        for case in cases:
-            passed, messages = run_case(client, case)
-            status = "PASS" if passed else "FAIL"
-            print(f"{status} {case['name']} ({case['_path']})")
-            for message in messages:
-                print(f"  {message}")
-            if passed:
+            transcript = run_case(client, case)
+            status = "PASS" if transcript.passed else "FAIL"
+            print(f"{status} {transcript.name} clientMode={transcript.client_mode}")
+            if output_dir is not None:
+                saved = save_transcript(output_dir, transcript)
+                print(f"  saved: {saved}")
+            if transcript.passed:
                passed_count += 1
            else:
                failed_count += 1
@@ -241,6 +428,8 @@ def main() -> int:
        f"\nSummary: total={len(cases)} "
        f"passed={passed_count} failed={failed_count}"
    )
+    if output_dir is not None:
+        print(f"Transcripts: {output_dir}")
    return 1 if failed_count else 0