249 lines
7.4 KiB
Python
249 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Run live FastGPT workflow evaluations against the FastAPI service."""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
import yaml
|
|
|
|
|
|
EVALS_DIR = Path(__file__).resolve().parent
|
|
DEFAULT_CASES_DIR = EVALS_DIR / "cases"
|
|
|
|
|
|
@dataclass
|
|
class ChatResult:
|
|
stage_codes: list[str]
|
|
errors: list[dict]
|
|
|
|
|
|
def timestamp() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def load_cases(cases_dir: Path, name_filters: list[str]) -> list[dict]:
|
|
cases = []
|
|
for path in sorted(cases_dir.rglob("*.yaml")):
|
|
case = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
if not isinstance(case, dict):
|
|
raise ValueError(f"{path}: case must be a YAML object")
|
|
|
|
name = case.get("name")
|
|
client_mode = case.get("clientMode")
|
|
steps = case.get("steps")
|
|
if not isinstance(name, str) or not name:
|
|
raise ValueError(f"{path}: name is required")
|
|
if client_mode not in {"direct", "browser_addon"}:
|
|
raise ValueError(f"{path}: unsupported clientMode={client_mode!r}")
|
|
if not isinstance(steps, list) or not steps:
|
|
raise ValueError(f"{path}: at least one step is required")
|
|
|
|
for index, step in enumerate(steps, start=1):
|
|
expected_code = (
|
|
step.get("expect", {}).get("stageCode")
|
|
if isinstance(step, dict)
|
|
else None
|
|
)
|
|
if not isinstance(step, dict) or not isinstance(step.get("input"), str):
|
|
raise ValueError(f"{path}: step {index} input must be a string")
|
|
if not isinstance(expected_code, str) or not expected_code:
|
|
raise ValueError(
|
|
f"{path}: step {index} expect.stageCode is required"
|
|
)
|
|
|
|
if name_filters and not any(value in name for value in name_filters):
|
|
continue
|
|
|
|
case["_path"] = str(path.relative_to(EVALS_DIR))
|
|
cases.append(case)
|
|
|
|
if not cases:
|
|
raise ValueError(f"No evaluation cases found under {cases_dir}")
|
|
return cases
|
|
|
|
|
|
def parse_sse(response: httpx.Response) -> ChatResult:
|
|
stage_codes: list[str] = []
|
|
errors: list[dict] = []
|
|
event_name = ""
|
|
data_lines: list[str] = []
|
|
|
|
def consume_event() -> None:
|
|
nonlocal event_name, data_lines
|
|
if data_lines:
|
|
payload = json.loads("\n".join(data_lines))
|
|
if event_name == "stage_code":
|
|
stage_codes.append(payload["nextStageCode"])
|
|
elif event_name == "error":
|
|
errors.append(payload)
|
|
event_name = ""
|
|
data_lines = []
|
|
|
|
for line in response.iter_lines():
|
|
if not line:
|
|
consume_event()
|
|
elif line.startswith("event:"):
|
|
event_name = line.removeprefix("event:").strip()
|
|
elif line.startswith("data:"):
|
|
data_lines.append(line.removeprefix("data:").strip())
|
|
|
|
consume_event()
|
|
return ChatResult(stage_codes=stage_codes, errors=errors)
|
|
|
|
|
|
def run_chat_step(
|
|
client: httpx.Client,
|
|
*,
|
|
session_id: str,
|
|
client_mode: str,
|
|
text: str,
|
|
) -> ChatResult:
|
|
with client.stream(
|
|
"POST",
|
|
"/chat",
|
|
params={"stream": "true"},
|
|
json={
|
|
"sessionId": session_id,
|
|
"timeStamp": timestamp(),
|
|
"text": text,
|
|
"clientMode": client_mode,
|
|
"needFormUpdate": False,
|
|
"useTextChunk": False,
|
|
},
|
|
) as response:
|
|
response.raise_for_status()
|
|
return parse_sse(response)
|
|
|
|
|
|
def cleanup_session(client: httpx.Client, session_id: str) -> str | None:
|
|
try:
|
|
response = client.request(
|
|
"DELETE",
|
|
"/delete_session",
|
|
json={"sessionId": session_id, "timeStamp": timestamp()},
|
|
)
|
|
response.raise_for_status()
|
|
payload = response.json()
|
|
if payload.get("code") != "200":
|
|
return f"cleanup response={payload}"
|
|
except (httpx.HTTPError, ValueError) as exc:
|
|
return f"cleanup error={exc}"
|
|
return None
|
|
|
|
|
|
def session_id_for(case_name: str) -> str:
|
|
slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", case_name).strip("-")
|
|
return f"eval-{slug[:24]}-{uuid4().hex[:12]}"
|
|
|
|
|
|
def run_case(client: httpx.Client, case: dict) -> tuple[bool, list[str]]:
|
|
session_id = session_id_for(case["name"])
|
|
messages = []
|
|
passed = True
|
|
|
|
try:
|
|
for step_number, step in enumerate(case["steps"], start=1):
|
|
expected = step["expect"]["stageCode"]
|
|
try:
|
|
result = run_chat_step(
|
|
client,
|
|
session_id=session_id,
|
|
client_mode=case["clientMode"],
|
|
text=step["input"],
|
|
)
|
|
except (httpx.HTTPError, json.JSONDecodeError, KeyError) as exc:
|
|
messages.append(f"step {step_number}: request error: {exc}")
|
|
passed = False
|
|
break
|
|
|
|
if result.stage_codes != [expected]:
|
|
messages.append(
|
|
f"step {step_number}: expected={[expected]} "
|
|
f"actual={result.stage_codes} sse_errors={result.errors}"
|
|
)
|
|
passed = False
|
|
break
|
|
|
|
messages.append(f"step {step_number}: stageCode={expected}")
|
|
finally:
|
|
cleanup_warning = cleanup_session(client, session_id)
|
|
if cleanup_warning:
|
|
messages.append(f"warning: {cleanup_warning}")
|
|
|
|
return passed, messages
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Run live workflow evaluations and assert stage codes."
|
|
)
|
|
parser.add_argument(
|
|
"--base-url",
|
|
default=os.getenv("EVAL_BASE_URL", "http://127.0.0.1:8000"),
|
|
help="FastAPI base URL (default: EVAL_BASE_URL or http://127.0.0.1:8000)",
|
|
)
|
|
parser.add_argument(
|
|
"--cases-dir",
|
|
type=Path,
|
|
default=DEFAULT_CASES_DIR,
|
|
help="Directory containing YAML cases",
|
|
)
|
|
parser.add_argument(
|
|
"--case",
|
|
action="append",
|
|
default=[],
|
|
help="Run cases whose names contain this value; repeatable",
|
|
)
|
|
parser.add_argument("--timeout", type=float, default=90.0)
|
|
parser.add_argument(
|
|
"--insecure",
|
|
action="store_true",
|
|
help="Disable TLS certificate verification",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
try:
|
|
cases = load_cases(args.cases_dir, args.case)
|
|
except (OSError, ValueError, yaml.YAMLError) as exc:
|
|
print(f"ERROR: {exc}", file=sys.stderr)
|
|
return 2
|
|
|
|
passed_count = 0
|
|
failed_count = 0
|
|
with httpx.Client(
|
|
base_url=args.base_url.rstrip("/"),
|
|
timeout=args.timeout,
|
|
verify=not args.insecure,
|
|
) as client:
|
|
for case in cases:
|
|
passed, messages = run_case(client, case)
|
|
status = "PASS" if passed else "FAIL"
|
|
print(f"{status} {case['name']} ({case['_path']})")
|
|
for message in messages:
|
|
print(f" {message}")
|
|
if passed:
|
|
passed_count += 1
|
|
else:
|
|
failed_count += 1
|
|
|
|
print(
|
|
f"\nSummary: total={len(cases)} "
|
|
f"passed={passed_count} failed={failed_count}"
|
|
)
|
|
return 1 if failed_count else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|