#!/usr/bin/env python3 """Run live FastGPT workflow evaluations against the FastAPI service.""" import argparse import json import os import re import sys from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from uuid import uuid4 import httpx import yaml EVALS_DIR = Path(__file__).resolve().parent DEFAULT_CASES_DIR = EVALS_DIR / "cases" @dataclass class ChatResult: stage_codes: list[str] errors: list[dict] def timestamp() -> str: return datetime.now(timezone.utc).isoformat() def load_cases(cases_dir: Path, name_filters: list[str]) -> list[dict]: cases = [] for path in sorted(cases_dir.rglob("*.yaml")): case = yaml.safe_load(path.read_text(encoding="utf-8")) if not isinstance(case, dict): raise ValueError(f"{path}: case must be a YAML object") name = case.get("name") client_mode = case.get("clientMode") steps = case.get("steps") if not isinstance(name, str) or not name: raise ValueError(f"{path}: name is required") if client_mode not in {"direct", "browser_addon"}: raise ValueError(f"{path}: unsupported clientMode={client_mode!r}") if not isinstance(steps, list) or not steps: raise ValueError(f"{path}: at least one step is required") for index, step in enumerate(steps, start=1): expected_code = ( step.get("expect", {}).get("stageCode") if isinstance(step, dict) else None ) if not isinstance(step, dict) or not isinstance(step.get("input"), str): raise ValueError(f"{path}: step {index} input must be a string") if not isinstance(expected_code, str) or not expected_code: raise ValueError( f"{path}: step {index} expect.stageCode is required" ) if name_filters and not any(value in name for value in name_filters): continue case["_path"] = str(path.relative_to(EVALS_DIR)) cases.append(case) if not cases: raise ValueError(f"No evaluation cases found under {cases_dir}") return cases def parse_sse(response: httpx.Response) -> ChatResult: stage_codes: list[str] = [] errors: list[dict] = [] event_name = "" data_lines: list[str] = [] def consume_event() -> None: nonlocal event_name, data_lines if data_lines: payload = json.loads("\n".join(data_lines)) if event_name == "stage_code": stage_codes.append(payload["nextStageCode"]) elif event_name == "error": errors.append(payload) event_name = "" data_lines = [] for line in response.iter_lines(): if not line: consume_event() elif line.startswith("event:"): event_name = line.removeprefix("event:").strip() elif line.startswith("data:"): data_lines.append(line.removeprefix("data:").strip()) consume_event() return ChatResult(stage_codes=stage_codes, errors=errors) def run_chat_step( client: httpx.Client, *, session_id: str, client_mode: str, text: str, ) -> ChatResult: with client.stream( "POST", "/chat", params={"stream": "true"}, json={ "sessionId": session_id, "timeStamp": timestamp(), "text": text, "clientMode": client_mode, "needFormUpdate": False, "useTextChunk": False, }, ) as response: response.raise_for_status() return parse_sse(response) def cleanup_session(client: httpx.Client, session_id: str) -> str | None: try: response = client.request( "DELETE", "/delete_session", json={"sessionId": session_id, "timeStamp": timestamp()}, ) response.raise_for_status() payload = response.json() if payload.get("code") != "200": return f"cleanup response={payload}" except (httpx.HTTPError, ValueError) as exc: return f"cleanup error={exc}" return None def session_id_for(case_name: str) -> str: slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", case_name).strip("-") return f"eval-{slug[:24]}-{uuid4().hex[:12]}" def run_case(client: httpx.Client, case: dict) -> tuple[bool, list[str]]: session_id = session_id_for(case["name"]) messages = [] passed = True try: for step_number, step in enumerate(case["steps"], start=1): expected = step["expect"]["stageCode"] try: result = run_chat_step( client, session_id=session_id, client_mode=case["clientMode"], text=step["input"], ) except (httpx.HTTPError, json.JSONDecodeError, KeyError) as exc: messages.append(f"step {step_number}: request error: {exc}") passed = False break if result.stage_codes != [expected]: messages.append( f"step {step_number}: expected={[expected]} " f"actual={result.stage_codes} sse_errors={result.errors}" ) passed = False break messages.append(f"step {step_number}: stageCode={expected}") finally: cleanup_warning = cleanup_session(client, session_id) if cleanup_warning: messages.append(f"warning: {cleanup_warning}") return passed, messages def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Run live workflow evaluations and assert stage codes." ) parser.add_argument( "--base-url", default=os.getenv("EVAL_BASE_URL", "http://127.0.0.1:8000"), help="FastAPI base URL (default: EVAL_BASE_URL or http://127.0.0.1:8000)", ) parser.add_argument( "--cases-dir", type=Path, default=DEFAULT_CASES_DIR, help="Directory containing YAML cases", ) parser.add_argument( "--case", action="append", default=[], help="Run cases whose names contain this value; repeatable", ) parser.add_argument("--timeout", type=float, default=90.0) parser.add_argument( "--insecure", action="store_true", help="Disable TLS certificate verification", ) return parser.parse_args() def main() -> int: args = parse_args() try: cases = load_cases(args.cases_dir, args.case) except (OSError, ValueError, yaml.YAMLError) as exc: print(f"ERROR: {exc}", file=sys.stderr) return 2 passed_count = 0 failed_count = 0 with httpx.Client( base_url=args.base_url.rstrip("/"), timeout=args.timeout, verify=not args.insecure, ) as client: for case in cases: passed, messages = run_case(client, case) status = "PASS" if passed else "FAIL" print(f"{status} {case['name']} ({case['_path']})") for message in messages: print(f" {message}") if passed: passed_count += 1 else: failed_count += 1 print( f"\nSummary: total={len(cases)} " f"passed={passed_count} failed={failed_count}" ) return 1 if failed_count else 0 if __name__ == "__main__": raise SystemExit(main())