Move workflow evaluations to standalone runner
This commit is contained in:
Xin Wang
248
evals/run.py
Normal file
248
evals/run.py
Normal file
@@ -0,0 +1,248 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run live FastGPT workflow evaluations against the FastAPI service."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import yaml
|
||||
|
||||
|
||||
EVALS_DIR = Path(__file__).resolve().parent
|
||||
DEFAULT_CASES_DIR = EVALS_DIR / "cases"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatResult:
|
||||
stage_codes: list[str]
|
||||
errors: list[dict]
|
||||
|
||||
|
||||
def timestamp() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def load_cases(cases_dir: Path, name_filters: list[str]) -> list[dict]:
|
||||
cases = []
|
||||
for path in sorted(cases_dir.rglob("*.yaml")):
|
||||
case = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(case, dict):
|
||||
raise ValueError(f"{path}: case must be a YAML object")
|
||||
|
||||
name = case.get("name")
|
||||
client_mode = case.get("clientMode")
|
||||
steps = case.get("steps")
|
||||
if not isinstance(name, str) or not name:
|
||||
raise ValueError(f"{path}: name is required")
|
||||
if client_mode not in {"direct", "browser_addon"}:
|
||||
raise ValueError(f"{path}: unsupported clientMode={client_mode!r}")
|
||||
if not isinstance(steps, list) or not steps:
|
||||
raise ValueError(f"{path}: at least one step is required")
|
||||
|
||||
for index, step in enumerate(steps, start=1):
|
||||
expected_code = (
|
||||
step.get("expect", {}).get("stageCode")
|
||||
if isinstance(step, dict)
|
||||
else None
|
||||
)
|
||||
if not isinstance(step, dict) or not isinstance(step.get("input"), str):
|
||||
raise ValueError(f"{path}: step {index} input must be a string")
|
||||
if not isinstance(expected_code, str) or not expected_code:
|
||||
raise ValueError(
|
||||
f"{path}: step {index} expect.stageCode is required"
|
||||
)
|
||||
|
||||
if name_filters and not any(value in name for value in name_filters):
|
||||
continue
|
||||
|
||||
case["_path"] = str(path.relative_to(EVALS_DIR))
|
||||
cases.append(case)
|
||||
|
||||
if not cases:
|
||||
raise ValueError(f"No evaluation cases found under {cases_dir}")
|
||||
return cases
|
||||
|
||||
|
||||
def parse_sse(response: httpx.Response) -> ChatResult:
|
||||
stage_codes: list[str] = []
|
||||
errors: list[dict] = []
|
||||
event_name = ""
|
||||
data_lines: list[str] = []
|
||||
|
||||
def consume_event() -> None:
|
||||
nonlocal event_name, data_lines
|
||||
if data_lines:
|
||||
payload = json.loads("\n".join(data_lines))
|
||||
if event_name == "stage_code":
|
||||
stage_codes.append(payload["nextStageCode"])
|
||||
elif event_name == "error":
|
||||
errors.append(payload)
|
||||
event_name = ""
|
||||
data_lines = []
|
||||
|
||||
for line in response.iter_lines():
|
||||
if not line:
|
||||
consume_event()
|
||||
elif line.startswith("event:"):
|
||||
event_name = line.removeprefix("event:").strip()
|
||||
elif line.startswith("data:"):
|
||||
data_lines.append(line.removeprefix("data:").strip())
|
||||
|
||||
consume_event()
|
||||
return ChatResult(stage_codes=stage_codes, errors=errors)
|
||||
|
||||
|
||||
def run_chat_step(
|
||||
client: httpx.Client,
|
||||
*,
|
||||
session_id: str,
|
||||
client_mode: str,
|
||||
text: str,
|
||||
) -> ChatResult:
|
||||
with client.stream(
|
||||
"POST",
|
||||
"/chat",
|
||||
params={"stream": "true"},
|
||||
json={
|
||||
"sessionId": session_id,
|
||||
"timeStamp": timestamp(),
|
||||
"text": text,
|
||||
"clientMode": client_mode,
|
||||
"needFormUpdate": False,
|
||||
"useTextChunk": False,
|
||||
},
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
return parse_sse(response)
|
||||
|
||||
|
||||
def cleanup_session(client: httpx.Client, session_id: str) -> str | None:
|
||||
try:
|
||||
response = client.request(
|
||||
"DELETE",
|
||||
"/delete_session",
|
||||
json={"sessionId": session_id, "timeStamp": timestamp()},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
if payload.get("code") != "200":
|
||||
return f"cleanup response={payload}"
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
return f"cleanup error={exc}"
|
||||
return None
|
||||
|
||||
|
||||
def session_id_for(case_name: str) -> str:
|
||||
slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", case_name).strip("-")
|
||||
return f"eval-{slug[:24]}-{uuid4().hex[:12]}"
|
||||
|
||||
|
||||
def run_case(client: httpx.Client, case: dict) -> tuple[bool, list[str]]:
|
||||
session_id = session_id_for(case["name"])
|
||||
messages = []
|
||||
passed = True
|
||||
|
||||
try:
|
||||
for step_number, step in enumerate(case["steps"], start=1):
|
||||
expected = step["expect"]["stageCode"]
|
||||
try:
|
||||
result = run_chat_step(
|
||||
client,
|
||||
session_id=session_id,
|
||||
client_mode=case["clientMode"],
|
||||
text=step["input"],
|
||||
)
|
||||
except (httpx.HTTPError, json.JSONDecodeError, KeyError) as exc:
|
||||
messages.append(f"step {step_number}: request error: {exc}")
|
||||
passed = False
|
||||
break
|
||||
|
||||
if result.stage_codes != [expected]:
|
||||
messages.append(
|
||||
f"step {step_number}: expected={[expected]} "
|
||||
f"actual={result.stage_codes} sse_errors={result.errors}"
|
||||
)
|
||||
passed = False
|
||||
break
|
||||
|
||||
messages.append(f"step {step_number}: stageCode={expected}")
|
||||
finally:
|
||||
cleanup_warning = cleanup_session(client, session_id)
|
||||
if cleanup_warning:
|
||||
messages.append(f"warning: {cleanup_warning}")
|
||||
|
||||
return passed, messages
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run live workflow evaluations and assert stage codes."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base-url",
|
||||
default=os.getenv("EVAL_BASE_URL", "http://127.0.0.1:8000"),
|
||||
help="FastAPI base URL (default: EVAL_BASE_URL or http://127.0.0.1:8000)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cases-dir",
|
||||
type=Path,
|
||||
default=DEFAULT_CASES_DIR,
|
||||
help="Directory containing YAML cases",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--case",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Run cases whose names contain this value; repeatable",
|
||||
)
|
||||
parser.add_argument("--timeout", type=float, default=90.0)
|
||||
parser.add_argument(
|
||||
"--insecure",
|
||||
action="store_true",
|
||||
help="Disable TLS certificate verification",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
cases = load_cases(args.cases_dir, args.case)
|
||||
except (OSError, ValueError, yaml.YAMLError) as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
passed_count = 0
|
||||
failed_count = 0
|
||||
with httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
timeout=args.timeout,
|
||||
verify=not args.insecure,
|
||||
) as client:
|
||||
for case in cases:
|
||||
passed, messages = run_case(client, case)
|
||||
status = "PASS" if passed else "FAIL"
|
||||
print(f"{status} {case['name']} ({case['_path']})")
|
||||
for message in messages:
|
||||
print(f" {message}")
|
||||
if passed:
|
||||
passed_count += 1
|
||||
else:
|
||||
failed_count += 1
|
||||
|
||||
print(
|
||||
f"\nSummary: total={len(cases)} "
|
||||
f"passed={passed_count} failed={failed_count}"
|
||||
)
|
||||
return 1 if failed_count else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user