feat: add persisted batch text testing

This commit is contained in:
Xin Wang
2026-08-10 13:49:24 +08:00
parent 5b8b9fd097
commit 19e8c8c108
27 changed files with 3855 additions and 1450 deletions

View File

@@ -66,15 +66,10 @@ export type DebugTestCase = {
failAtTurn?: number;
};
/** 测试集Debug Drawer 只按「当前助手关联」浏览,不做管理 */
/** 测试集Debug Drawer 只浏览和选择,不做管理 */
export type DebugTestSuite = {
id: string;
name: string;
/**
* 关联助手 id。MVP 用 `"*"` 表示「当前正在调试的助手都可见」;
* 接上真实 API 后改成具体 assistantId 列表。
*/
assistantIds: string[];
caseIds: string[];
};
@@ -295,15 +290,11 @@ export const MOCK_TEST_CASES: DebugTestCase[] = [
),
];
/**
* 当前助手关联的测试集MVP mock
* `"*"` = 任意正在调试的助手都可见,避免把「系统全部测试集」甩进 Drawer。
*/
/** 测试集MVP mock测试集本身不绑定助手。 */
export const MOCK_TEST_SUITES: DebugTestSuite[] = [
{
id: "suite-accident-basic",
name: "事故快处基础流程",
assistantIds: ["*"],
caseIds: [
"tc-dual-car",
"tc-no-injury",
@@ -314,7 +305,6 @@ export const MOCK_TEST_SUITES: DebugTestSuite[] = [
{
id: "suite-clarify",
name: "异常输入与澄清",
assistantIds: ["*"],
caseIds: [
"tc-hello",
"tc-noise",
@@ -326,13 +316,11 @@ export const MOCK_TEST_SUITES: DebugTestSuite[] = [
{
id: "suite-tools",
name: "工具与系统联动",
assistantIds: ["*"],
caseIds: ["tc-injury-transfer", "tc-transfer-tool", "tc-end-call-tool"],
},
{
id: "suite-realtime-voice",
name: "实时语音交互",
assistantIds: ["*"],
caseIds: ["tc-hello", "tc-noise", "tc-barge-in", "tc-long-silence"],
},
];
@@ -347,15 +335,6 @@ function findSuite(id: string | null): DebugTestSuite | null {
return MOCK_TEST_SUITES.find((item) => item.id === id) ?? null;
}
/** 只返回当前助手关联的测试集 */
function suitesForAssistant(assistantId: string | null): DebugTestSuite[] {
return MOCK_TEST_SUITES.filter((suite) => {
if (suite.assistantIds.includes("*")) return true;
if (!assistantId) return false;
return suite.assistantIds.includes(assistantId);
});
}
function casesInSuites(suites: DebugTestSuite[]): DebugTestCase[] {
const seen = new Set<string>();
const cases: DebugTestCase[] = [];
@@ -657,23 +636,18 @@ function TestCasePickerPopover({
onOpenChange,
selectedId,
onSelect,
assistantId,
trigger,
}: {
open: boolean;
onOpenChange: (open: boolean) => void;
selectedId: string | null;
onSelect: (id: string) => void;
assistantId: string | null;
trigger: React.ReactNode;
}) {
const [query, setQuery] = useState("");
const [activeSuiteId, setActiveSuiteId] = useState<string | null>(null);
const suites = useMemo(
() => suitesForAssistant(assistantId),
[assistantId],
);
const suites = MOCK_TEST_SUITES;
const scopedCases = useMemo(() => casesInSuites(suites), [suites]);
const activeSuite = findSuite(activeSuiteId);
@@ -810,7 +784,7 @@ function TestCasePickerPopover({
</div>
{suites.length === 0 ? (
<div className="rounded-xl border border-dashed border-hairline-strong px-3 py-6 text-center text-xs text-muted-foreground">
</div>
) : (
suites.map((suite) => (
@@ -851,7 +825,6 @@ function TestCasePickerPopover({
export function AutoTestRunBar({
state,
assistantId,
onSelectCase,
onClearCase,
onStart,
@@ -859,7 +832,6 @@ export function AutoTestRunBar({
onRerun,
}: {
state: AutoTestRunState;
assistantId: string | null;
onSelectCase: (id: string) => void;
onClearCase: () => void;
onStart: () => void;
@@ -877,7 +849,6 @@ export function AutoTestRunBar({
onOpenChange={setPickerOpen}
selectedId={state.selectedId}
onSelect={onSelectCase}
assistantId={assistantId}
trigger={
<button
type="button"
@@ -985,7 +956,6 @@ export function AutoTestRunBar({
onOpenChange={setPickerOpen}
selectedId={state.selectedId}
onSelect={onSelectCase}
assistantId={assistantId}
trigger={
<button
type="button"

View File

@@ -334,7 +334,7 @@ export function DebugDrawer({
)}
</div>
{debugMode === "auto" ? (
<AutoTestPanel assistantId={assistantId} autoTest={autoTest} />
<AutoTestPanel autoTest={autoTest} />
) : (
<DebugVoicePanel
view={view}
@@ -355,10 +355,8 @@ export function DebugDrawer({
}
function AutoTestPanel({
assistantId,
autoTest,
}: {
assistantId: string | null;
autoTest: ReturnType<typeof useAutoTestRunner>;
}) {
const {
@@ -379,7 +377,6 @@ function AutoTestPanel({
<div className="shrink-0 border-t border-hairline bg-card p-3">
<AutoTestRunBar
state={state}
assistantId={assistantId}
onSelectCase={selectCase}
onClearCase={clearCase}
onStart={start}

View File

@@ -3,7 +3,6 @@
/**
* 批量测试用例详情面板。
* 与 prompt mode DebugDrawer overlay 相同:作为右侧 flex 半屏,无遮罩/模糊。
* 当前展示前端 mock 数据,后续可直接替换为真实执行事件与校验结果。
*/
import {
@@ -79,7 +78,7 @@ export function BatchCaseDetailDrawer({
{item.name}
</h2>
<p className="truncate text-xs text-muted-soft">
{assistantName} · Mock
{assistantName} · Pipeline
</p>
</div>
<BatchCaseStatusBadge status={item.status} />
@@ -99,12 +98,7 @@ export function BatchCaseDetailDrawer({
<section className="rounded-2xl border border-hairline bg-canvas-soft px-4 py-3.5">
<p className="text-xs font-medium text-foreground"></p>
<p className="mt-1 text-xs leading-5 text-muted-soft">
{item.turns.length} ·{" "}
{item.turns.reduce(
(sum, turn) => sum + turn.evaluations.length,
0,
) + item.overallCriteria.length}{" "}
·
</p>
</section>
<ExecutionTimeline status={item.status} />
@@ -288,7 +282,7 @@ function ToolCallCard({ toolCall }: { toolCall: BatchToolCallRecord }) {
</div>
<div className="mt-2 grid gap-2 sm:grid-cols-2">
<CodeResult label="参数" value={toolCall.argumentsJson} />
<CodeResult label="Mock 返回" value={toolCall.resultJson} />
<CodeResult label="工具返回" value={toolCall.resultJson} />
</div>
</div>
);
@@ -401,7 +395,7 @@ function ExecutionTimeline({ status }: { status: "waiting" | "running" }) {
},
{
label: "请求助手响应",
detail: isRunning ? "生成中Mock" : "尚未开始",
detail: isRunning ? "Pipeline 生成中" : "尚未开始",
state: isRunning ? "active" : "pending",
},
{

View File

@@ -1,16 +1,13 @@
"use client";
/**
* 批量测试 — 已完成结果视图MVP mock
*/
/** 批量测试已完成结果视图。 */
import { ChevronRight } from "lucide-react";
import { ChevronDown, ChevronRight, Settings2 } from "lucide-react";
import { useRef, useState } from "react";
import { BatchCaseDetailDrawer } from "@/components/batch-test/batch-case-detail-drawer";
import {
BatchCaseStatusBadge,
BatchPhasePill,
getBatchCaseStatusLabel,
} from "@/components/batch-test/batch-case-status";
import {
@@ -50,13 +47,8 @@ export function BatchRunCompletedView({
const detailOpen = Boolean(selectedCase);
const counts = countByStatus(run.cases);
const judged = counts.pass + counts.fail + counts.error;
const total = run.cases.length;
const passRate = judged === 0 ? 0 : Math.round((counts.pass / judged) * 100);
const failRate =
judged === 0 ? 0 : Math.round((counts.fail / judged) * 100);
const errorRate =
judged === 0 ? 0 : Math.round((counts.error / judged) * 100);
const outcome = getRunOutcome(run, counts);
const visibleCases =
filter === "fail"
? run.cases.filter((item) => item.status === "fail")
@@ -93,26 +85,96 @@ export function BatchRunCompletedView({
>
{/* 结果总览 */}
<section className="rounded-2xl border border-hairline bg-card p-5 shadow-sm">
<div className="flex flex-wrap items-center gap-2">
<h2 className="text-base font-medium text-foreground">
{run.title}
</h2>
<BatchPhasePill phase="completed" />
<div className="flex flex-wrap items-start justify-between gap-3">
<div>
<h2 className="text-base font-medium text-foreground">
{run.title}
</h2>
{run.finishedAt && (
<p className="mt-1.5 text-xs text-muted-soft">
{formatRunTime(run.finishedAt)}
{run.stopReason ? ` · ${stopReasonLabel(run)}` : ""}
</p>
)}
</div>
<span
className={cn(
"inline-flex shrink-0 items-center rounded-full px-3 py-1 text-xs font-medium",
outcome.className,
)}
>
{outcome.label}
</span>
</div>
{run.finishedAt && (
<p className="mt-1.5 text-xs text-muted-soft">
{formatRunTime(run.finishedAt)}
{run.stopReason ? ` · ${stopReasonLabel(run)}` : ""}
</p>
)}
<dl className="mt-4 grid gap-3 rounded-xl border border-hairline bg-canvas-soft/60 px-4 py-3 text-sm sm:grid-cols-2 lg:grid-cols-4">
<div className="mt-5 rounded-xl border border-hairline bg-canvas-soft/60 p-4">
<div className="flex flex-col gap-4 sm:flex-row sm:items-end sm:justify-between">
<div>
<p className="text-xs font-medium tracking-wide text-muted-soft uppercase">
</p>
<p className="mt-1 font-display text-3xl text-ink">
<span className="tabular-nums">{counts.pass}</span>
<span className="mx-1.5 text-muted-soft">/</span>
<span className="tabular-nums">{total}</span>
<span className="ml-2 text-2xl"></span>
</p>
</div>
<div className="grid grid-cols-3 gap-x-5 gap-y-2 text-sm sm:shrink-0">
<ResultCount tone="success" label="通过" count={counts.pass} />
<ResultCount
tone="destructive"
label="断言失败"
count={counts.fail}
/>
<ResultCount
tone="warning"
label="执行错误"
count={counts.error}
/>
{counts.skipped > 0 && (
<ResultCount
tone="muted"
label="未执行"
count={counts.skipped}
/>
)}
</div>
</div>
<ResultDistribution counts={counts} total={total} />
</div>
<dl
className={cn(
"mt-4 grid gap-x-6 gap-y-4 text-sm sm:grid-cols-2",
!detailOpen && "lg:grid-cols-4",
)}
>
<div>
<dt className="text-xs text-muted-foreground"></dt>
<dd className="mt-1 truncate font-medium text-foreground">
<dd className="mt-1 break-words font-medium text-foreground">
{run.assistantName}
</dd>
</div>
<div className="min-w-0">
<dt className="text-xs text-muted-foreground"></dt>
<dd
className="mt-1 break-words font-medium text-foreground"
title={run.config.evaluatorModelResourceName}
>
{run.config.evaluatorModelResourceName}
</dd>
{run.config.evaluatorModel && (
<dd
className="mt-0.5 break-all font-mono text-[11px] text-muted-soft"
title={run.config.evaluatorModel}
>
{run.config.evaluatorModel}
</dd>
)}
</div>
<div>
<dt className="text-xs text-muted-foreground"></dt>
<dd className="mt-1 font-medium text-foreground">
@@ -122,70 +184,34 @@ export function BatchRunCompletedView({
<div>
<dt className="text-xs text-muted-foreground"></dt>
<dd className="mt-1 font-medium text-foreground">
{run.config.concurrency} · {run.config.timeoutSecs}
</dd>
</div>
<div>
<dt className="text-xs text-muted-foreground"> / </dt>
<dd className="mt-1 text-xs font-medium leading-5 text-foreground">
{formatRunDuration(run.startedAt, run.finishedAt)} ·
{BATCH_FAILURE_STRATEGY_LABEL[run.config.failureStrategy]}
<br />
{run.config.errorRetryCount}
{BATCH_ERROR_STRATEGY_LABEL[run.config.errorStrategy]}
{run.config.concurrency} · {run.config.timeoutSecs}
<span className="mx-1.5 text-muted-soft">·</span>
{formatRunDuration(run.startedAt, run.finishedAt)}
</dd>
</div>
</dl>
<div className="mt-5 flex flex-col items-center gap-6 sm:flex-row sm:justify-between sm:gap-8">
<div className="text-center sm:text-left">
<div className="font-display text-3xl text-ink">
{counts.pass} / {judged || total}
</div>
<div className="mt-1 text-sm font-medium text-success">
{passRate}%
</div>
<details className="group mt-4 border-t border-hairline pt-3">
<summary className="flex w-fit cursor-pointer list-none items-center gap-2 rounded-full px-2 py-1 text-xs font-medium text-muted-foreground transition-colors hover:bg-canvas-soft hover:text-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/40 [&::-webkit-details-marker]:hidden">
<Settings2 size={13} />
<ChevronDown
size={13}
className="transition-transform group-open:rotate-180"
/>
</summary>
<div className="mt-3 grid gap-3 rounded-xl border border-hairline bg-canvas-soft/50 px-4 py-3 text-xs leading-5 text-muted-foreground sm:grid-cols-2">
<p>
<span className="font-medium text-foreground"></span>
{BATCH_FAILURE_STRATEGY_LABEL[run.config.failureStrategy]}
</p>
<p>
<span className="font-medium text-foreground"></span>
{run.config.errorRetryCount}
{BATCH_ERROR_STRATEGY_LABEL[run.config.errorStrategy]}
</p>
</div>
<PassRateDonut
pass={counts.pass}
fail={counts.fail}
error={counts.error}
/>
<div className="space-y-2 text-sm">
<LegendRow
tone="success"
label="通过"
count={counts.pass}
percent={passRate}
/>
<LegendRow
tone="destructive"
label="断言失败"
count={counts.fail}
percent={failRate}
/>
<LegendRow
tone="warning"
label="执行错误"
count={counts.error}
percent={errorRate}
/>
{counts.skipped > 0 && (
<LegendRow
tone="muted"
label="未执行"
count={counts.skipped}
percent={
total === 0
? 0
: Math.round((counts.skipped / total) * 100)
}
/>
)}
</div>
</div>
</details>
</section>
{/* 用例列表 */}
@@ -319,106 +345,53 @@ export function BatchRunCompletedView({
);
}
function PassRateDonut({
pass,
fail,
error,
}: {
pass: number;
fail: number;
error: number;
}) {
const total = pass + fail + error;
const passRatio = total === 0 ? 0 : pass / total;
const size = 112;
const stroke = 14;
const radius = (size - stroke) / 2;
const circumference = 2 * Math.PI * radius;
const passLength = circumference * passRatio;
const failLength = circumference * (total === 0 ? 0 : fail / total);
const errorLength = circumference * (total === 0 ? 0 : error / total);
type BatchCounts = ReturnType<typeof countByStatus>;
return (
<div
className="relative shrink-0"
style={{ width: size, height: size }}
aria-hidden
>
<svg width={size} height={size} className="-rotate-90">
<circle
cx={size / 2}
cy={size / 2}
r={radius}
fill="none"
stroke="currentColor"
strokeWidth={stroke}
className="text-surface-strong"
/>
{total > 0 && (
<>
{pass > 0 && (
<circle
cx={size / 2}
cy={size / 2}
r={radius}
fill="none"
stroke="currentColor"
strokeWidth={stroke}
strokeDasharray={`${passLength} ${circumference}`}
strokeLinecap="butt"
className="text-success"
/>
)}
{fail > 0 && (
<circle
cx={size / 2}
cy={size / 2}
r={radius}
fill="none"
stroke="currentColor"
strokeWidth={stroke}
strokeDasharray={`${failLength} ${circumference}`}
strokeDashoffset={-passLength}
strokeLinecap="butt"
className="text-destructive"
/>
)}
{error > 0 && (
<circle
cx={size / 2}
cy={size / 2}
r={radius}
fill="none"
stroke="currentColor"
strokeWidth={stroke}
strokeDasharray={`${errorLength} ${circumference}`}
strokeDashoffset={-(passLength + failLength)}
strokeLinecap="butt"
className="text-amber-500"
/>
)}
</>
)}
</svg>
<div className="absolute inset-0 flex flex-col items-center justify-center">
<span className="text-lg font-medium tabular-nums text-foreground">
{total === 0 ? 0 : Math.round((pass / total) * 100)}%
</span>
</div>
</div>
);
function getRunOutcome(run: BatchRunSnapshot, counts: BatchCounts) {
if (run.status === "cancelled") {
return {
label: "运行已停止",
className: "bg-surface-strong text-muted-foreground",
};
}
if (counts.error > 0) {
return {
label: "存在执行错误",
className: "bg-amber-500/10 text-amber-600 dark:text-amber-400",
};
}
if (counts.fail > 0) {
return {
label: "存在断言失败",
className: "bg-destructive/10 text-destructive",
};
}
if (counts.skipped > 0) {
return {
label: "部分未执行",
className: "bg-surface-strong text-muted-foreground",
};
}
if (counts.pass > 0) {
return {
label: "全部通过",
className: "bg-success/10 text-success",
};
}
return {
label: "暂无结果",
className: "bg-surface-strong text-muted-foreground",
};
}
function LegendRow({
function ResultCount({
tone,
label,
count,
percent,
}: {
tone: "success" | "destructive" | "warning" | "muted";
label: string;
count: number;
percent: number;
}) {
const dotClass = {
success: "bg-success",
@@ -428,11 +401,48 @@ function LegendRow({
}[tone];
return (
<div className="flex items-center gap-2 text-muted-foreground">
<span className={cn("size-2.5 rounded-full", dotClass)} />
<span>
{label} {count}{percent}%
<div className="flex min-w-0 items-center gap-2">
<span
className={cn("size-2 shrink-0 rounded-full", dotClass)}
aria-hidden
/>
<span className="truncate text-muted-foreground">{label}</span>
<span className="ml-auto font-medium tabular-nums text-foreground">
{count}
</span>
</div>
);
}
function ResultDistribution({
counts,
total,
}: {
counts: BatchCounts;
total: number;
}) {
const segments = [
{ key: "pass", count: counts.pass, className: "bg-success" },
{ key: "fail", count: counts.fail, className: "bg-destructive" },
{ key: "error", count: counts.error, className: "bg-amber-500" },
{ key: "skipped", count: counts.skipped, className: "bg-muted-soft" },
].filter((item) => item.count > 0);
return (
<div
className="mt-4 flex h-2 overflow-hidden rounded-full bg-surface-strong"
role="img"
aria-label={`结果分布:通过 ${counts.pass},断言失败 ${counts.fail},执行错误 ${counts.error},未执行 ${counts.skipped}`}
>
{total > 0 &&
segments.map((segment) => (
<span
key={segment.key}
className={segment.className}
style={{ flexBasis: 0, flexGrow: segment.count }}
aria-hidden
/>
))}
</div>
);
}

View File

@@ -1,8 +1,6 @@
"use client";
/**
* 批量测试 — 运行中视图MVP mock
*/
/** 批量测试运行中的持久化快照视图。 */
import { ChevronRight } from "lucide-react";
import { useRef, useState } from "react";
@@ -67,6 +65,10 @@ export function BatchRunRunningView({ run }: { run: BatchRunSnapshot }) {
</h2>
<BatchPhasePill phase="running" />
</div>
<p className="text-xs text-muted-soft">
{run.assistantName} ·
{run.config.evaluatorModelResourceName}
</p>
<div className="flex items-center gap-3">
<div className="h-2.5 min-w-0 flex-1 overflow-hidden rounded-full bg-surface-strong">

View File

@@ -2,7 +2,7 @@
/**
* 批量测试页MVP
* 三阶段:配置 → 运行中 → 已完成。执行进度为前端 mock后续可换真实引擎
* 三阶段:配置 → 运行中 → 已完成。运行快照由后端持久化并轮询更新
*/
import {
@@ -35,21 +35,21 @@ import {
import {
BATCH_ERROR_STRATEGY_LABEL,
BATCH_FAILURE_STRATEGY_LABEL,
createMockBatchRun,
hasEvaluationFailure,
type BatchErrorStrategy,
type BatchFailureStrategy,
type BatchMockExecutionPlan,
type BatchRunPhase,
type BatchRunSnapshot,
} from "@/data/batch-run";
import type { TestCase, TestSuite } from "@/data/test-suites";
import {
listTestCases,
listTestSuites,
type TestCase,
type TestSuite,
} from "@/data/test-suites";
import { assistantsApi, type Assistant } from "@/lib/api";
assistantsApi,
batchRunsApi,
modelResourcesApi,
testCasesApi,
testSuitesApi,
type Assistant,
type ModelResource,
} from "@/lib/api";
const CONCURRENCY_OPTIONS = ["1", "2", "3", "5", "10"] as const;
@@ -82,7 +82,7 @@ const BATCH_SECTIONS = [
{ id: "settings", label: "运行设置" },
] as const;
const TICK_MS = 900;
const RUN_POLL_MS = 1000;
function getAppScrollContainer(): HTMLElement | null {
return document.querySelector<HTMLElement>(".app-content");
@@ -104,29 +104,38 @@ function buildRunTitle(
return "批量测试";
}
function loadTestScope() {
const suites = listTestSuites();
function groupCasesBySuite(suites: TestSuite[], cases: TestCase[]) {
const casesBySuite: Record<string, TestCase[]> = {};
for (const suite of suites) {
const items = listTestCases(suite.id);
casesBySuite[suite.id] = items;
casesBySuite[suite.id] = cases
.filter((item) => item.suiteId === suite.id)
.sort((left, right) => left.sortOrder - right.sortOrder);
}
return { suites, casesBySuite };
return casesBySuite;
}
export function BatchTestPage() {
const router = useRouter();
const [phase, setPhase] = useState<BatchRunPhase>("config");
const [run, setRun] = useState<BatchRunSnapshot | null>(null);
const mockExecutionPlanRef = useRef<BatchMockExecutionPlan>({});
const [runError, setRunError] = useState("");
const [starting, setStarting] = useState(false);
const [stopping, setStopping] = useState(false);
const [assistants, setAssistants] = useState<Assistant[]>([]);
const [loadingAssistants, setLoadingAssistants] = useState(true);
const [assistantId, setAssistantId] = useState("");
const [evaluatorModels, setEvaluatorModels] = useState<ModelResource[]>([]);
const [loadingEvaluatorModels, setLoadingEvaluatorModels] = useState(true);
const [evaluatorModelResourceId, setEvaluatorModelResourceId] = useState("");
const [{ suites, casesBySuite }] = useState(loadTestScope);
const [suites, setSuites] = useState<TestSuite[]>([]);
const [casesBySuite, setCasesBySuite] = useState<Record<string, TestCase[]>>(
{},
);
const [loadingScope, setLoadingScope] = useState(true);
const [scopeError, setScopeError] = useState("");
const [selectedCaseIds, setSelectedCaseIds] = useState<Set<string>>(
() => new Set(),
);
@@ -249,102 +258,84 @@ export function BatchTestPage() {
})();
}, []);
// mock 执行引擎:按并发推进用例状态
useEffect(() => {
if (phase !== "running" || !run?.startedAt) return;
function tick(settleRunning: boolean) {
setRun((prev) => {
if (!prev || prev.finishedAt) return prev;
let cases = prev.cases.map((item) => ({ ...item }));
// 1) 结算上一拍仍在运行的用例(首拍只启动,不结算)
let sawFail = false;
let sawError = false;
if (settleRunning) {
for (const item of cases) {
if (item.status !== "running") continue;
const plannedError = mockExecutionPlanRef.current[item.id];
if (plannedError) {
if (
plannedError.retryable &&
item.attemptCount < item.maxAttempts
) {
item.status = "waiting";
} else {
item.status = "error";
item.executionError = plannedError;
sawError = true;
}
continue;
}
item.status = hasEvaluationFailure(item) ? "fail" : "pass";
if (item.status === "fail") sawFail = true;
}
}
const stopReason =
sawError && prev.config.errorStrategy === "stop_on_error"
? "execution_error"
: sawFail && prev.config.failureStrategy === "stop_on_fail"
? "assertion_failure"
: null;
// 配置为停止时,剩余等待/运行中的用例统一标记为未执行。
if (stopReason) {
cases = cases.map((item) =>
item.status === "waiting" || item.status === "running"
? { ...item, status: "skipped" as const }
: item,
);
return {
...prev,
cases,
finishedAt: new Date().toISOString(),
stopReason,
};
}
// 2) 按并发补齐运行中
const limit = prev.config.concurrency;
let running = cases.filter((item) => item.status === "running").length;
for (const item of cases) {
if (running >= limit) break;
if (item.status !== "waiting") continue;
item.status = "running";
item.attemptCount += 1;
running += 1;
}
const pending = cases.some(
(item) => item.status === "waiting" || item.status === "running",
void (async () => {
try {
const resources = (await modelResourcesApi.list()).filter(
(item) => item.capability === "LLM" && item.enabled,
);
if (!pending) {
return {
...prev,
cases,
finishedAt: new Date().toISOString(),
stopReason: null,
};
}
setEvaluatorModels(resources);
const defaultResource =
resources.find((item) => item.isDefault) ?? resources[0];
if (defaultResource) setEvaluatorModelResourceId(defaultResource.id);
} catch {
// 模型资源为空或加载失败时,运行前检查会阻止启动。
} finally {
setLoadingEvaluatorModels(false);
}
})();
}, []);
return { ...prev, cases };
});
useEffect(() => {
void (async () => {
try {
setScopeError("");
const [loadedSuites, loadedCases] = await Promise.all([
testSuitesApi.list(),
testCasesApi.list(),
]);
setSuites(loadedSuites);
setCasesBySuite(groupCasesBySuite(loadedSuites, loadedCases));
} catch (error) {
setScopeError(
error instanceof Error ? error.message : "加载测试范围失败",
);
} finally {
setLoadingScope(false);
}
})();
}, []);
useEffect(() => {
if (phase !== "running" || !run?.id) return;
let disposed = false;
let timer = 0;
async function poll() {
let finished = false;
try {
const snapshot = await batchRunsApi.get(run!.id);
if (disposed) return;
setRun(snapshot);
setRunError("");
if (
snapshot.finishedAt ||
snapshot.status === "completed" ||
snapshot.status === "cancelled"
) {
finished = true;
setStopping(false);
setPhase("completed");
}
} catch (error) {
if (!disposed) {
setRunError(
error instanceof Error ? error.message : "刷新运行结果失败",
);
}
} finally {
if (!disposed && !finished) {
timer = window.setTimeout(() => void poll(), RUN_POLL_MS);
}
}
}
// 立刻拉起第一批,再按节拍结算/推进
tick(false);
const timer = window.setInterval(() => tick(true), TICK_MS);
return () => window.clearInterval(timer);
}, [phase, run?.startedAt]);
// 执行引擎只负责写入完成快照;页面阶段在快照稳定后统一切换。
useEffect(() => {
if (phase !== "running" || !run?.finishedAt) return;
const timer = window.setTimeout(() => setPhase("completed"), 0);
return () => window.clearTimeout(timer);
}, [phase, run?.finishedAt]);
void poll();
return () => {
disposed = true;
window.clearTimeout(timer);
};
}, [phase, run?.id]);
const selectedCases = useMemo(() => {
const ordered: TestCase[] = [];
@@ -364,9 +355,13 @@ export function BatchTestPage() {
const canStart =
Boolean(assistantId) &&
Boolean(evaluatorModelResourceId) &&
selectedCount > 0 &&
timeoutValid &&
!loadingAssistants &&
!loadingEvaluatorModels &&
!loadingScope &&
!starting &&
phase === "config";
const runReadiness: {
@@ -375,60 +370,90 @@ export function BatchTestPage() {
actionable?: boolean;
} = loadingAssistants
? { label: "正在加载助手…" }
: assistants.length === 0
? { label: "暂无可用助手", target: "target", actionable: true }
: !assistantId
? { label: "请选择被测助手", target: "target", actionable: true }
: selectedCount === 0
? { label: "请选择测试用例", target: "scope", actionable: true }
: !timeoutValid
? {
label: "请修正超时时间",
target: "settings",
actionable: true,
}
: {
label: `${selectedCount} 个用例 · ${concurrency} 并发 · 超时 ${timeoutValue}`,
};
: loadingEvaluatorModels
? { label: "正在加载评估模型…" }
: loadingScope
? { label: "正在加载测试范围…" }
: scopeError
? { label: "测试范围加载失败", target: "scope", actionable: true }
: assistants.length === 0
? { label: "暂无可用助手", target: "target", actionable: true }
: !assistantId
? { label: "请选择被测助手", target: "target", actionable: true }
: evaluatorModels.length === 0
? { label: "暂无可用评估模型", target: "target", actionable: true }
: !evaluatorModelResourceId
? { label: "请选择评估模型", target: "target", actionable: true }
: selectedCount === 0
? { label: "请选择测试用例", target: "scope", actionable: true }
: !timeoutValid
? {
label: "请修正超时时间",
target: "settings",
actionable: true,
}
: {
label: `${selectedCount} 个用例 · ${concurrency} 并发 · 超时 ${timeoutValue}`,
};
function startRun(cases: TestCase[]) {
const assistantName =
assistants.find((item) => item.id === assistantId)?.name ?? "助手";
const { snapshot, executionPlan } = createMockBatchRun({
title: buildRunTitle(cases, suites),
assistantName,
cases,
concurrency: Number(concurrency) || 3,
timeoutSecs: timeoutValue,
failureStrategy: failStrategy,
errorRetryCount: errorRetryValue,
errorStrategy,
});
mockExecutionPlanRef.current = executionPlan;
setRun(snapshot);
setPhase("running");
async function startRun(cases: TestCase[]) {
if (
!assistantId ||
!evaluatorModelResourceId ||
cases.length === 0 ||
starting
) {
return;
}
setStarting(true);
setRunError("");
try {
const snapshot = await batchRunsApi.create({
assistantId,
evaluatorModelResourceId,
caseIds: cases.map((item) => item.id),
title: buildRunTitle(cases, suites),
config: {
concurrency: Number(concurrency) || 3,
timeoutSecs: timeoutValue,
failureStrategy: failStrategy,
errorRetryCount: errorRetryValue,
errorStrategy,
},
});
setRun(snapshot);
setPhase("running");
} catch (error) {
setRunError(error instanceof Error ? error.message : "启动批量测试失败");
} finally {
setStarting(false);
}
}
function handleStart() {
if (!canStart) return;
startRun(selectedCases);
void startRun(selectedCases);
}
function handleStop() {
setRun((prev) => {
if (!prev) return prev;
return {
...prev,
stopReason: "manual",
finishedAt: new Date().toISOString(),
cases: prev.cases.map((item) =>
item.status === "waiting" || item.status === "running"
? { ...item, status: "skipped" }
: item,
),
};
});
setPhase("completed");
async function handleStop() {
if (!run || stopping) return;
setStopping(true);
setRunError("");
try {
const snapshot = await batchRunsApi.cancel(run.id);
setRun(snapshot);
if (
snapshot.finishedAt ||
snapshot.status === "completed" ||
snapshot.status === "cancelled"
) {
setPhase("completed");
setStopping(false);
}
} catch (error) {
setRunError(error instanceof Error ? error.message : "停止批量测试失败");
setStopping(false);
}
}
function sourceCasesByIds(ids: string[]): TestCase[] {
@@ -445,7 +470,7 @@ export function BatchTestPage() {
setRun(null);
return;
}
startRun(cases);
void startRun(cases);
}
function handleRerunFailed() {
@@ -457,12 +482,12 @@ export function BatchTestPage() {
)
.map((item) => item.id),
);
if (failedCases.length > 0) startRun(failedCases);
if (failedCases.length > 0) void startRun(failedCases);
}
function handleRerunCase(caseId: string) {
const [item] = sourceCasesByIds([caseId]);
if (item) startRun([item]);
if (item) void startRun([item]);
}
function handleEditCase(caseId: string) {
@@ -474,6 +499,7 @@ export function BatchTestPage() {
function handleBackToConfig() {
setPhase("config");
setRun(null);
setRunError("");
}
if (phase === "running" && run) {
@@ -485,13 +511,26 @@ export function BatchTestPage() {
<Button
variant="outline"
className="gap-2 rounded-full border-destructive/40 text-destructive hover:bg-destructive/10 hover:text-destructive"
onClick={handleStop}
disabled={stopping}
onClick={() => void handleStop()}
>
<Square size={14} className="fill-current" />
{stopping ? (
<Loader2 size={14} className="animate-spin" />
) : (
<Square size={14} className="fill-current" />
)}
{stopping ? "正在停止…" : "停止运行"}
</Button>
}
>
{runError && (
<p
className="border-b border-destructive/20 bg-destructive/5 px-5 py-2 text-sm text-destructive"
role="alert"
>
{runError}
</p>
)}
<BatchRunRunningView run={run} />
</ListPageLayout>
);
@@ -517,7 +556,7 @@ export function BatchTestPage() {
<Button
variant="outline"
className="gap-2 rounded-full border-hairline-strong"
disabled={failedCount === 0}
disabled={failedCount === 0 || starting}
onClick={handleRerunFailed}
>
<RotateCcw size={15} />
@@ -525,14 +564,27 @@ export function BatchTestPage() {
</Button>
<Button
className="gap-2 rounded-full px-4"
disabled={starting}
onClick={handleRerunAll}
>
<RotateCcw size={15} />
{starting ? (
<Loader2 size={15} className="animate-spin" />
) : (
<RotateCcw size={15} />
)}
{starting ? "正在启动…" : "重跑全部"}
</Button>
</div>
}
>
{runError && (
<p
className="border-b border-destructive/20 bg-destructive/5 px-5 py-2 text-sm text-destructive"
role="alert"
>
{runError}
</p>
)}
<BatchRunCompletedView
run={run}
onEditCase={handleEditCase}
@@ -545,7 +597,7 @@ export function BatchTestPage() {
return (
<ListPageLayout
title="批量测试"
description="配置一次批量运行:选择被测助手、测试范围与基础运行参数。"
description="配置一次批量运行:选择被测助手、评估模型、测试范围与运行参数。"
className="max-w-[880px]"
topbarAction={
<div className="flex items-center gap-3">
@@ -578,8 +630,12 @@ export function BatchTestPage() {
aria-describedby="batch-run-readiness"
onClick={handleStart}
>
<Play size={16} />
{starting ? (
<Loader2 size={16} className="animate-spin" />
) : (
<Play size={16} />
)}
{starting ? "正在启动…" : "开始批量测试"}
</Button>
</div>
}
@@ -593,6 +649,15 @@ export function BatchTestPage() {
}
/>
{runError && (
<p
className="rounded-xl border border-destructive/20 bg-destructive/5 px-3 py-2 text-sm text-destructive"
role="alert"
>
{runError}
</p>
)}
<div className="space-y-4">
<section
ref={(element) => {
@@ -603,36 +668,77 @@ export function BatchTestPage() {
<SectionCard
icon={<Target size={15} />}
title="运行目标"
description="选择本次批量测试要对齐的助手配置"
description="分别选择生成回复的助手和判断结果的 LLM"
>
<label className="block space-y-2">
<span className="text-sm font-medium text-foreground">
</span>
{loadingAssistants ? (
<div className="flex items-center gap-2 text-sm text-muted-foreground">
<Loader2 size={16} className="animate-spin" />
</div>
) : assistants.length === 0 ? (
<p className="rounded-xl border border-dashed border-hairline-strong bg-canvas-soft/50 px-3 py-4 text-sm text-muted-foreground">
</p>
) : (
<Select value={assistantId} onValueChange={setAssistantId}>
<SelectTrigger className="w-full border-hairline-strong bg-background">
<SelectValue placeholder="选择被测助手" />
</SelectTrigger>
<SelectContent>
{assistants.map((item) => (
<SelectItem key={item.id} value={item.id}>
{item.name}
</SelectItem>
))}
</SelectContent>
</Select>
)}
</label>
<div className="grid gap-5 sm:grid-cols-2">
<label className="block space-y-2">
<span className="text-sm font-medium text-foreground">
</span>
{loadingAssistants ? (
<div className="flex items-center gap-2 text-sm text-muted-foreground">
<Loader2 size={16} className="animate-spin" />
</div>
) : assistants.length === 0 ? (
<p className="rounded-xl border border-dashed border-hairline-strong bg-canvas-soft/50 px-3 py-4 text-sm text-muted-foreground">
</p>
) : (
<Select value={assistantId} onValueChange={setAssistantId}>
<SelectTrigger className="w-full border-hairline-strong bg-background">
<SelectValue placeholder="选择被测助手" />
</SelectTrigger>
<SelectContent>
{assistants.map((item) => (
<SelectItem key={item.id} value={item.id}>
{item.name}
</SelectItem>
))}
</SelectContent>
</Select>
)}
</label>
<label className="block space-y-2">
<span className="text-sm font-medium text-foreground">
</span>
{loadingEvaluatorModels ? (
<div className="flex items-center gap-2 text-sm text-muted-foreground">
<Loader2 size={16} className="animate-spin" />
</div>
) : evaluatorModels.length === 0 ? (
<p className="rounded-xl border border-dashed border-hairline-strong bg-canvas-soft/50 px-3 py-4 text-sm text-muted-foreground">
LLM
</p>
) : (
<Select
value={evaluatorModelResourceId}
onValueChange={setEvaluatorModelResourceId}
>
<SelectTrigger className="w-full border-hairline-strong bg-background">
<SelectValue placeholder="选择评估模型" />
</SelectTrigger>
<SelectContent>
{evaluatorModels.map((item) => (
<SelectItem key={item.id} value={item.id}>
{item.name}
{typeof item.values.modelId === "string" &&
item.values.modelId
? ` · ${item.values.modelId}`
: ""}
</SelectItem>
))}
</SelectContent>
</Select>
)}
<span className="block text-xs leading-5 text-muted-soft">
LLM
</span>
</label>
</div>
</SectionCard>
</section>
@@ -647,12 +753,26 @@ export function BatchTestPage() {
title="选择测试范围"
description="搜索测试集或用例,按需组合本次运行范围"
>
<TestScopeSelector
suites={suites}
casesBySuite={casesBySuite}
selectedCaseIds={selectedCaseIds}
onSelectionChange={setSelectedCaseIds}
/>
{loadingScope ? (
<div className="flex items-center gap-2 py-6 text-sm text-muted-foreground">
<Loader2 size={16} className="animate-spin" />
</div>
) : scopeError ? (
<p
className="rounded-xl border border-destructive/20 bg-destructive/5 px-3 py-3 text-sm text-destructive"
role="alert"
>
{scopeError}
</p>
) : (
<TestScopeSelector
suites={suites}
casesBySuite={casesBySuite}
selectedCaseIds={selectedCaseIds}
onSelectionChange={setSelectedCaseIds}
/>
)}
</SectionCard>
</section>

View File

@@ -48,41 +48,24 @@ import {
import { Input } from "@/components/ui/input";
import { ListToolbar } from "@/components/ui/list-toolbar";
import { SearchInput } from "@/components/ui/search-input";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import {
cloneOverallCriteria,
cloneTurns,
createEmptyFixedInputTurn,
createTestCase,
createTestSuite,
DEFAULT_INPUT_MODE,
duplicateTestCase,
duplicateTestSuite,
getTestCaseValidationMessage,
getTestSuite,
listTestCases,
listTestSuites,
normalizeOverallCriteria,
removeTestCase,
removeTestCases,
removeTestSuite,
reorderTestCases,
suiteCaseStats,
TEST_CASE_INPUT_MODE_LABEL,
TEST_CASE_INPUT_MODE_SHORT_LABEL,
updateTestCase,
updateTestSuite,
type TestCase,
type TestCaseInputMode,
type TestSuite,
} from "@/data/test-suites";
import { assistantsApi, type Assistant } from "@/lib/api";
import {
testCasesApi,
testSuitesApi,
type TestCaseWrite,
} from "@/lib/api";
// 路由驱动:
// /test/cases → list
@@ -109,16 +92,33 @@ export function TestCasesPage(props: TestCasesPageProps) {
function SuiteListView() {
const router = useRouter();
const [suites, setSuites] = useState<TestSuite[]>(() => listTestSuites());
const [suites, setSuites] = useState<TestSuite[]>([]);
const [loading, setLoading] = useState(true);
const [loadError, setLoadError] = useState("");
const [search, setSearch] = useState("");
const [currentPage, setCurrentPage] = useState(1);
const [deletingId, setDeletingId] = useState<string | null>(null);
async function reloadSuites() {
try {
setLoadError("");
setSuites(await testSuitesApi.list());
} catch (error) {
setLoadError(error instanceof Error ? error.message : "加载测试集失败");
} finally {
setLoading(false);
}
}
useEffect(() => {
void reloadSuites();
}, []);
const filtered = useMemo(() => {
const keyword = search.trim().toLowerCase();
return suites.filter((suite) => {
if (!keyword) return true;
return [suite.name, suite.assistantName, suite.id]
return [suite.name, suite.description, suite.id]
.join(" ")
.toLowerCase()
.includes(keyword);
@@ -136,13 +136,16 @@ function SuiteListView() {
router.push(`/test/cases/${suite.id}`);
}
function duplicateSuite(suite: TestSuite) {
const copied = duplicateTestSuite(suite.id);
if (!copied) return;
setSuites(listTestSuites());
async function duplicateSuite(suite: TestSuite) {
try {
await testSuitesApi.duplicate(suite.id);
await reloadSuites();
} catch (error) {
setLoadError(error instanceof Error ? error.message : "复制测试集失败");
}
}
function removeSuite(suite: TestSuite) {
async function removeSuite(suite: TestSuite) {
if (
!window.confirm(
`确定删除测试集“${suite.name}”及其全部测试用例吗?`,
@@ -151,9 +154,14 @@ function SuiteListView() {
return;
}
setDeletingId(suite.id);
removeTestSuite(suite.id);
setSuites(listTestSuites());
setDeletingId(null);
try {
await testSuitesApi.remove(suite.id);
await reloadSuites();
} catch (error) {
setLoadError(error instanceof Error ? error.message : "删除测试集失败");
} finally {
setDeletingId(null);
}
}
return (
@@ -186,14 +194,26 @@ function SuiteListView() {
}
/>
{loadError && (
<p className="mb-3 text-sm text-destructive" role="alert">
{loadError}
</p>
)}
<DataList<TestSuite>
rows={paginated}
rowKey={(suite) => suite.id}
onRowClick={openSuite}
empty={{
title: suites.length === 0 ? "暂无测试集" : "未找到匹配的测试集",
title: loading
? "正在加载测试集…"
: suites.length === 0
? "暂无测试集"
: "未找到匹配的测试集",
description:
suites.length === 0
loading
? "请稍候。"
: suites.length === 0
? "点击右上角「新建测试集」开始。"
: "请调整关键词后再试。",
}}
@@ -222,19 +242,12 @@ function SuiteListView() {
</>
),
},
{
key: "assistant",
header: "关联助手",
width: "md:w-[160px]",
cellClassName: "text-muted-foreground",
cell: (suite) => suite.assistantName || "—",
},
{
key: "caseCount",
header: "用例数",
width: "md:w-[96px]",
cellClassName: "tabular-nums text-muted-foreground",
cell: (suite) => suiteCaseStats(suite.id).total,
cell: (suite) => suite.caseCount ?? 0,
},
{
key: "actions",
@@ -313,42 +326,31 @@ function SuiteListView() {
function SuiteCreateView() {
const router = useRouter();
const [assistants, setAssistants] = useState<Assistant[]>([]);
const [loadingAssistants, setLoadingAssistants] = useState(true);
const [name, setName] = useState("");
const [assistantName, setAssistantName] = useState("");
const [creating, setCreating] = useState(false);
const [createError, setCreateError] = useState("");
useEffect(() => {
void (async () => {
try {
const list = await assistantsApi.list();
setAssistants(list);
if (list[0]) setAssistantName(list[0].name);
} catch {
setAssistantName("视频快处助手");
} finally {
setLoadingAssistants(false);
}
})();
}, []);
function confirmCreate() {
async function confirmCreate() {
if (!name.trim() || creating) return;
setCreating(true);
const saved = createTestSuite({
name,
description: "",
assistantName,
});
router.push(`/test/cases/${saved.id}`);
setCreateError("");
try {
const saved = await testSuitesApi.create({
name: name.trim(),
description: "",
});
router.push(`/test/cases/${saved.id}`);
} catch (error) {
setCreateError(error instanceof Error ? error.message : "创建测试集失败");
setCreating(false);
}
}
return (
<ListPageLayout
title="新建测试集"
className="max-w-[1180px]"
description="测试集固定输入用例的业务分组。确认后进入用例编辑。"
description="测试集只保存固定输入和预期;被测助手在批量运行时选择。"
action={
<Button
variant="outline"
@@ -375,39 +377,11 @@ function SuiteCreateView() {
</label>
</ListPageSection>
<ListPageSection>
<div>
<div className="mb-2 text-sm font-medium text-foreground">
</div>
{loadingAssistants ? (
<div className="flex items-center gap-2 text-sm text-muted-foreground">
<Loader2 size={16} className="animate-spin" />
</div>
) : assistants.length > 0 ? (
<Select value={assistantName} onValueChange={setAssistantName}>
<SelectTrigger className="w-full border-hairline-strong bg-background">
<SelectValue placeholder="选择关联助手" />
</SelectTrigger>
<SelectContent>
{assistants.map((item) => (
<SelectItem key={item.id} value={item.name}>
{item.name}
</SelectItem>
))}
</SelectContent>
</Select>
) : (
<Input
value={assistantName}
onChange={(event) => setAssistantName(event.target.value)}
placeholder="助手名称"
className="border-hairline-strong bg-background text-foreground placeholder:text-muted-soft"
/>
)}
</div>
</ListPageSection>
{createError && (
<p className="text-sm text-destructive" role="alert">
{createError}
</p>
)}
<div className="flex items-center justify-end gap-3">
<Button
@@ -476,64 +450,67 @@ function SuiteDetailView({
initialCaseId?: string;
}) {
const router = useRouter();
const [initial] = useState(() => {
const initialCases = listTestCases(suiteId);
const initialCase =
initialCases.find((item) => item.id === initialCaseId) ??
initialCases[0] ??
null;
const initialDraft = initialCase ? caseToDraft(initialCase) : null;
return {
suite: getTestSuite(suiteId),
cases: initialCases,
selectedId: initialCase?.id ?? null,
draft: initialDraft,
};
});
const [suite, setSuite] = useState<TestSuite | null>(initial.suite);
const [cases, setCases] = useState<TestCase[]>(initial.cases);
const [selectedId, setSelectedId] = useState<string | null>(
initial.selectedId,
);
const [suite, setSuite] = useState<TestSuite | null>(null);
const [cases, setCases] = useState<TestCase[]>([]);
const [selectedId, setSelectedId] = useState<string | null>(null);
const [search, setSearch] = useState("");
const [draft, setDraft] = useState<CaseEditorDraft | null>(initial.draft);
const [savedSnapshot, setSavedSnapshot] = useState<CaseEditorDraft | null>(
initial.draft,
);
const [draft, setDraft] = useState<CaseEditorDraft | null>(null);
const [savedSnapshot, setSavedSnapshot] = useState<CaseEditorDraft | null>(null);
const [statusMessage, setStatusMessage] = useState("");
const [selectionMode, setSelectionMode] = useState(false);
const [checkedIds, setCheckedIds] = useState<Set<string>>(new Set());
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
function reload(preferId?: string | null) {
const nextSuite = getTestSuite(suiteId);
const nextCases = listTestCases(suiteId);
setSuite(nextSuite);
setCases(nextCases);
async function reload(preferId?: string | null) {
try {
const [nextSuite, nextCases] = await Promise.all([
testSuitesApi.get(suiteId),
testCasesApi.list(suiteId),
]);
setSuite(nextSuite);
setCases(nextCases);
const nextSelected =
(preferId && nextCases.some((item) => item.id === preferId)
? preferId
: null) ??
(selectedId && nextCases.some((item) => item.id === selectedId)
? selectedId
: null) ??
nextCases[0]?.id ??
null;
const nextSelected =
(preferId && nextCases.some((item) => item.id === preferId)
? preferId
: null) ??
(selectedId && nextCases.some((item) => item.id === selectedId)
? selectedId
: null) ??
nextCases[0]?.id ??
null;
setSelectedId(nextSelected);
if (nextSelected) {
const item = nextCases.find((caseItem) => caseItem.id === nextSelected);
if (item) {
const nextDraft = caseToDraft(item);
setDraft(nextDraft);
setSavedSnapshot(nextDraft);
setSelectedId(nextSelected);
if (nextSelected) {
const item = nextCases.find((caseItem) => caseItem.id === nextSelected);
if (item) {
const nextDraft = caseToDraft(item);
setDraft(nextDraft);
setSavedSnapshot(nextDraft);
}
} else {
setDraft(null);
setSavedSnapshot(null);
}
} else {
setStatusMessage("");
} catch (error) {
setSuite(null);
setCases([]);
setDraft(null);
setSavedSnapshot(null);
setStatusMessage(error instanceof Error ? error.message : "加载测试集失败");
} finally {
setLoading(false);
}
}
useEffect(() => {
void reload(initialCaseId ?? null);
// suiteId changes by remount; the initial request only runs once per view.
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [suiteId, initialCaseId]);
const filtered = useMemo(() => {
const keyword = search.trim().toLowerCase();
return cases.filter((item) => {
@@ -558,7 +535,10 @@ function SuiteDetailView({
? getTestCaseValidationMessage(draft)
: "请先选择测试用例";
const canSave =
Boolean(draft?.name.trim()) && dirty && validationMessage === null;
Boolean(draft?.name.trim()) &&
dirty &&
validationMessage === null &&
!saving;
useEffect(() => {
if (!dirty) return;
@@ -629,39 +609,36 @@ function SuiteDetailView({
setStatusMessage("");
}
function handleSave() {
if (!draft || validationMessage) return;
const patch = {
async function handleSave() {
if (!draft || validationMessage || saving) return;
const body: TestCaseWrite = {
name: draft.name.trim() || "未命名用例",
description:
cases.find((item) => item.id === selectedId)?.description ?? "",
inputMode: draft.inputMode,
contextTurns: draft.contextTurns,
turns: draft.turns,
overallCriteria: normalizeOverallCriteria(draft.overallCriteria),
};
let saved: TestCase | null;
if (selectedId) {
saved = updateTestCase(selectedId, patch);
} else {
const created = createTestCase({
suiteId,
name: patch.name,
});
saved = created ? updateTestCase(created.id, patch) : null;
setSaving(true);
setStatusMessage("保存中…");
try {
const saved = selectedId
? await testCasesApi.update(selectedId, body)
: await testCasesApi.create(suiteId, body);
await reload(saved.id);
setStatusMessage("已保存");
window.setTimeout(() => setStatusMessage(""), 2000);
} catch (error) {
setStatusMessage(error instanceof Error ? error.message : "保存测试用例失败");
} finally {
setSaving(false);
}
if (!saved) return;
const nextDraft = caseToDraft(saved);
setSelectedId(saved.id);
setCases(listTestCases(suiteId));
setSuite(getTestSuite(suiteId));
setDraft(nextDraft);
setSavedSnapshot(nextDraft);
setStatusMessage("已保存");
window.setTimeout(() => setStatusMessage(""), 2000);
}
function handleCancelNewCase() {
reload();
void reload();
setStatusMessage("");
}
@@ -672,28 +649,34 @@ function SuiteDetailView({
router.push("/test/cases");
}
function handleDuplicateCase(item: TestCase) {
async function handleDuplicateCase(item: TestCase) {
if (dirty && !window.confirm("当前用例有未保存修改,复制将丢弃。继续?")) {
return;
}
const copied = duplicateTestCase(item.id);
if (!copied) return;
reload(copied.id);
setStatusMessage("");
try {
const copied = await testCasesApi.duplicate(item.id);
await reload(copied.id);
} catch (error) {
setStatusMessage(error instanceof Error ? error.message : "复制测试用例失败");
}
}
function handleDeleteCase(item: TestCase) {
async function handleDeleteCase(item: TestCase) {
if (!window.confirm(`确定删除测试用例“${item.name}”吗?`)) return;
removeTestCase(item.id);
setCheckedIds((prev) => {
const next = new Set(prev);
next.delete(item.id);
return next;
});
reload(selectedId === item.id ? null : selectedId);
try {
await testCasesApi.remove(item.id);
setCheckedIds((prev) => {
const next = new Set(prev);
next.delete(item.id);
return next;
});
await reload(selectedId === item.id ? null : selectedId);
} catch (error) {
setStatusMessage(error instanceof Error ? error.message : "删除测试用例失败");
}
}
function handleDeleteSelected() {
async function handleDeleteSelected() {
if (!selectedId || !draft) return;
if (
!window.confirm(
@@ -702,8 +685,12 @@ function SuiteDetailView({
) {
return;
}
removeTestCase(selectedId);
reload(null);
try {
await testCasesApi.remove(selectedId);
await reload(null);
} catch (error) {
setStatusMessage(error instanceof Error ? error.message : "删除测试用例失败");
}
}
function handleEnterSelectionMode() {
@@ -743,32 +730,66 @@ function SuiteDetailView({
});
}
function handleBulkDelete() {
async function handleBulkDelete() {
const ids = filtered
.filter((item) => checkedIds.has(item.id))
.map((item) => item.id);
if (ids.length === 0) return;
if (!window.confirm(`确定删除选中的 ${ids.length} 个测试用例吗?`)) return;
removeTestCases(ids);
handleExitSelectionMode();
reload(selectedId && ids.includes(selectedId) ? null : selectedId);
try {
await testCasesApi.bulkRemove(ids);
handleExitSelectionMode();
await reload(selectedId && ids.includes(selectedId) ? null : selectedId);
} catch (error) {
setStatusMessage(error instanceof Error ? error.message : "批量删除测试用例失败");
}
}
function handleReorder(orderedIds: string[]) {
reorderTestCases(suiteId, orderedIds);
setCases(listTestCases(suiteId));
async function handleReorder(orderedIds: string[]) {
const order = new Map(orderedIds.map((id, index) => [id, index]));
setCases((current) =>
[...current].sort(
(left, right) =>
(order.get(left.id) ?? Number.MAX_SAFE_INTEGER) -
(order.get(right.id) ?? Number.MAX_SAFE_INTEGER),
),
);
try {
await testCasesApi.reorder(suiteId, orderedIds);
} catch (error) {
setStatusMessage(error instanceof Error ? error.message : "调整用例顺序失败");
await reload(selectedId);
}
}
function renameSuite(nextName: string) {
async function renameSuite(nextName: string) {
if (!suite || nextName === suite.name) return;
const saved = updateTestSuite(suite.id, { name: nextName });
if (saved) setSuite(saved);
try {
const saved = await testSuitesApi.update(suite.id, {
name: nextName,
description: suite.description,
});
setSuite(saved);
} catch (error) {
setStatusMessage(error instanceof Error ? error.message : "重命名测试集失败");
}
}
if (loading) {
return (
<div className="flex h-full items-center justify-center gap-2 text-sm text-muted-foreground">
<Loader2 size={16} className="animate-spin" />
</div>
);
}
if (!suite) {
return (
<div className="mx-auto flex w-full max-w-[1280px] flex-col gap-4 py-16">
<div className="font-medium text-destructive"></div>
<div className="font-medium text-destructive">
{statusMessage || "测试集不存在"}
</div>
<Button
variant="outline"
size="sm"
@@ -791,7 +812,7 @@ function SuiteDetailView({
/>
<EditableTitle
value={suite.name}
onChange={renameSuite}
onChange={(value) => void renameSuite(value)}
placeholder="未命名测试集"
editLabel="测试集名称"
/>
@@ -813,14 +834,14 @@ function SuiteDetailView({
onSearchChange={setSearch}
onSelectCase={selectCase}
onCreateCase={handleCreateCase}
onDuplicateCase={handleDuplicateCase}
onDeleteCase={handleDeleteCase}
onDuplicateCase={(item) => void handleDuplicateCase(item)}
onDeleteCase={(item) => void handleDeleteCase(item)}
onEnterSelectionMode={handleEnterSelectionMode}
onExitSelectionMode={handleExitSelectionMode}
onToggleChecked={handleToggleChecked}
onToggleSelectAll={handleToggleSelectAll}
onBulkDelete={handleBulkDelete}
onReorder={handleReorder}
onBulkDelete={() => void handleBulkDelete()}
onReorder={(ids) => void handleReorder(ids)}
/>
<main className="flex min-h-0 min-w-0 flex-1 flex-col overflow-hidden bg-background">
@@ -868,7 +889,7 @@ function SuiteDetailView({
className="gap-1.5"
disabled={!canSave}
title={validationMessage ?? undefined}
onClick={handleSave}
onClick={() => void handleSave()}
>
<Save size={14} />
@@ -882,7 +903,9 @@ function SuiteDetailView({
: "border-hairline-strong text-muted-foreground hover:text-foreground"
}
onClick={
selectedId ? handleDeleteSelected : handleCancelNewCase
selectedId
? () => void handleDeleteSelected()
: handleCancelNewCase
}
>
{selectedId ? <Trash2 size={14} /> : <X size={14} />}

View File

@@ -122,25 +122,6 @@ function editorSections(): {
];
}
/** API 不可用时的兜底工具,保证下拉可选 */
const FALLBACK_TOOLS: EditorToolOption[] = [
{
id: "mock_transfer_to_human",
functionName: "transfer_to_human",
label: "转人工",
parameters: [
{ name: "reason", description: "转人工原因" },
{ name: "target", description: "转接目标" },
],
},
{
id: "mock_end_conversation",
functionName: "end_conversation",
label: "结束会话",
parameters: [{ name: "reason", description: "结束原因" }],
},
];
function toolParametersFromDefinition(tool: Tool): EditorToolOption["parameters"] {
const definition = tool.definition;
if (definition.type === "http" || definition.type === "client") {
@@ -208,9 +189,8 @@ export function NextReplyEditorBody({
const sections = editorSections();
const tools = useMemo(() => {
if (availableTools && availableTools.length > 0) return availableTools;
if (loadedTools.length > 0) return loadedTools;
return FALLBACK_TOOLS;
if (availableTools) return availableTools;
return loadedTools;
}, [availableTools, loadedTools]);
const sensors = useSensors(
@@ -221,16 +201,15 @@ export function NextReplyEditorBody({
);
useEffect(() => {
if (availableTools && availableTools.length > 0) return;
if (availableTools) return;
let cancelled = false;
void (async () => {
try {
const list = await toolsApi.list();
if (cancelled) return;
const mapped = list.map(toEditorToolOption);
setLoadedTools(mapped.length > 0 ? mapped : FALLBACK_TOOLS);
setLoadedTools(list.map(toEditorToolOption));
} catch {
if (!cancelled) setLoadedTools(FALLBACK_TOOLS);
if (!cancelled) setLoadedTools([]);
}
})();
return () => {
@@ -1464,14 +1443,20 @@ function ToolCallBehaviorEditor({
<SelectValue placeholder="选择工具…" />
</SelectTrigger>
<SelectContent>
{tools.map((tool) => (
<SelectItem key={tool.id} value={tool.id}>
{tool.functionName}
{tool.label && tool.label !== tool.functionName
? ` · ${tool.label}`
: ""}
</SelectItem>
))}
{tools.length === 0 ? (
<div className="px-2 py-3 text-xs text-muted-foreground">
</div>
) : (
tools.map((tool) => (
<SelectItem key={tool.id} value={tool.id}>
{tool.functionName}
{tool.label && tool.label !== tool.functionName
? ` · ${tool.label}`
: ""}
</SelectItem>
))
)}
</SelectContent>
</Select>
</div>

View File

@@ -1,14 +1,4 @@
/**
* 批量测试运行 — 前端 mock。
* 真实执行引擎接入前,用本地状态模拟进度与结果。
*/
import type {
ExpectedBehavior,
ReplyExpectedBehavior,
TestCase,
ToolCallExpectedBehavior,
} from "@/data/test-suites";
/** 批量测试运行的前后端共享结果契约与纯展示辅助函数。 */
export type BatchCaseStatus =
| "waiting"
@@ -34,24 +24,13 @@ export const BATCH_ERROR_STRATEGY_LABEL: Record<BatchErrorStrategy, string> = {
stop_on_error: "标记错误并停止",
};
type BatchExecutionError = {
export type BatchExecutionError = {
code: string;
message: string;
stage: "pipeline" | "model" | "tool" | "evaluation";
retryable: boolean;
};
export type BatchRunCase = {
id: string;
name: string;
status: BatchCaseStatus;
turns: BatchTurnResult[];
overallCriteria: BatchEvaluationResult[];
attemptCount: number;
maxAttempts: number;
executionError: BatchExecutionError | null;
};
export type BatchEvaluationResult = {
id: string;
label: string;
@@ -80,13 +59,30 @@ export type BatchTurnResult = {
evaluations: BatchEvaluationResult[];
};
export type BatchRunCase = {
id: string;
name: string;
status: BatchCaseStatus;
turns: BatchTurnResult[];
overallCriteria: BatchEvaluationResult[];
attemptCount: number;
maxAttempts: number;
executionError: BatchExecutionError | null;
};
export type BatchRunPhase = "config" | "running" | "completed";
export type BatchRunStatus = "queued" | "running" | "completed" | "cancelled";
export type BatchRunSnapshot = {
id: string;
status: BatchRunStatus;
title: string;
assistantName: string;
config: {
suiteCount: number;
evaluatorModelResourceId: string;
evaluatorModelResourceName: string;
evaluatorModel: string;
concurrency: number;
timeoutSecs: number;
failureStrategy: BatchFailureStrategy;
@@ -99,246 +95,6 @@ export type BatchRunSnapshot = {
stopReason: "manual" | "assertion_failure" | "execution_error" | null;
};
export type BatchMockExecutionPlan = Record<string, BatchExecutionError>;
/** 预定部分用例失败,方便演示失败展开态 */
function shouldFail(index: number, item: TestCase): boolean {
if (item.lastResult === "fail") return true;
return index % 4 === 2;
}
/** 稳定产生少量执行错误,便于检查重试和继续/停止流程。 */
function shouldError(index: number): boolean {
return index % 5 === 3;
}
function replyExpectation(behavior: ReplyExpectedBehavior): string {
if (behavior.assertionType === "llm") return behavior.llmCriteria;
const relation = behavior.negateKeywords
? "不应包含"
: behavior.keywordMatchMode === "all"
? "应包含全部"
: "应至少包含其一";
return `${relation}${behavior.keywords.join("、")}`;
}
function toolExpectation(behavior: ToolCallExpectedBehavior): string {
if (behavior.expectation === "not_called") {
return `不应调用 ${behavior.functionName}`;
}
const count =
behavior.maxCalls === null
? `${behavior.minCalls} 次以上`
: behavior.minCalls === behavior.maxCalls
? `${behavior.minCalls}`
: `${behavior.minCalls}${behavior.maxCalls}`;
return `应调用 ${behavior.functionName} ${count}`;
}
function mockAssistantReply(behaviors: ExpectedBehavior[]): string {
const positiveKeyword = behaviors.find(
(behavior): behavior is ReplyExpectedBehavior =>
behavior.type === "reply" &&
behavior.assertionType === "keyword" &&
!behavior.negateKeywords &&
behavior.keywords.length > 0,
);
if (positiveKeyword) {
return `好的,我已了解。请继续说明${positiveKeyword.keywords.slice(0, 2).join("和")}`;
}
return "好的,我已记录当前信息,我们继续处理。";
}
function mockToolArguments(behavior: ToolCallExpectedBehavior): string {
const entries = behavior.paramAssertions.map((item) => [
item.name,
item.matchMode === "exact" ? item.value : `mock_${item.name}`,
]);
return JSON.stringify(Object.fromEntries(entries), null, 2);
}
function createBehaviorEvaluation(
behavior: ExpectedBehavior,
fail: boolean,
): {
evaluation: BatchEvaluationResult;
toolCalls: BatchToolCallRecord[];
} {
if (behavior.type === "reply") {
const reason = fail
? behavior.assertionType === "keyword"
? "实际回复未满足关键词规则。"
: "LLM 判断认为回复未达到该项语义要求。"
: "";
return {
evaluation: {
id: behavior.id,
label:
behavior.assertionType === "keyword"
? "回复 · 关键词"
: "回复 · LLM 判断",
kind: "reply",
status: fail ? "fail" : "pass",
expected: replyExpectation(behavior),
actual: fail
? "已为您转接人工处理。"
: "实际回复满足本条内容要求。",
reason,
},
toolCalls: [],
};
}
const shouldRecordCall =
behavior.expectation === "called" ? !fail : fail;
const toolCalls: BatchToolCallRecord[] = shouldRecordCall
? [
{
id: `call_${behavior.id}`,
functionName: behavior.functionName,
argumentsJson: mockToolArguments(behavior),
resultJson: behavior.mockResponse.body,
outcome: behavior.mockResponse.outcome,
durationMs: behavior.mockResponse.delayMs + 84,
},
]
: [];
const actual = shouldRecordCall
? `实际调用 1 次 ${behavior.functionName}`
: `实际调用 0 次 ${behavior.functionName}`;
const reason = fail
? behavior.expectation === "not_called"
? "检测到本轮不应发生的工具调用。"
: "未检测到满足次数要求的工具调用。"
: "";
return {
evaluation: {
id: behavior.id,
label: `工具 · ${behavior.functionName}`,
kind: "tool_call",
status: fail ? "fail" : "pass",
expected: toolExpectation(behavior),
actual,
reason,
},
toolCalls,
};
}
function createCaseResult(
item: TestCase,
caseIndex: number,
maxAttempts: number,
): { result: BatchRunCase; plannedError: BatchExecutionError | null } {
const plannedError = shouldError(caseIndex);
const plannedFailure = !plannedError && shouldFail(caseIndex, item);
let failureAssigned = false;
const turns = item.turns.map((turn, turnIndex) => {
const toolCalls: BatchToolCallRecord[] = [];
const evaluations = turn.behaviors.map((behavior) => {
const fail = plannedFailure && !failureAssigned;
if (fail) failureAssigned = true;
const result = createBehaviorEvaluation(behavior, fail);
toolCalls.push(...result.toolCalls);
return result.evaluation;
});
return {
id: turn.id,
index: turnIndex,
userInput: turn.userInput,
assistantReply: mockAssistantReply(turn.behaviors),
toolCalls,
evaluations,
};
});
const overallCriteria = item.overallCriteria.map((criterion) => {
const fail = plannedFailure && !failureAssigned;
if (fail) failureAssigned = true;
return {
id: criterion.id,
label: criterion.name,
kind: "overall" as const,
status: fail ? ("fail" as const) : ("pass" as const),
expected: criterion.criteria,
actual: fail
? "整段对话未完整达到该业务目标。"
: "整段对话达到该业务目标。",
reason: fail ? "LLM 判断认为整段对话未满足该项标准。" : "",
};
});
return {
result: {
id: item.id,
name: item.name,
status: "waiting",
turns,
overallCriteria,
attemptCount: 0,
maxAttempts,
executionError: null,
},
plannedError: plannedError
? {
code: "PIPELINE_TIMEOUT",
message: "等待 pipeline 完成回复时超过单用例超时时间。",
stage: "pipeline",
retryable: true,
}
: null,
};
}
export function createMockBatchRun(input: {
title: string;
assistantName: string;
cases: TestCase[];
concurrency: number;
timeoutSecs: number;
failureStrategy: BatchFailureStrategy;
errorRetryCount: number;
errorStrategy: BatchErrorStrategy;
}): {
snapshot: BatchRunSnapshot;
executionPlan: BatchMockExecutionPlan;
} {
const preparedCases = input.cases.map((item, index) =>
createCaseResult(item, index, input.errorRetryCount + 1),
);
const executionPlan = Object.fromEntries(
preparedCases.flatMap(({ result, plannedError }) =>
plannedError ? [[result.id, plannedError]] : [],
),
);
return {
snapshot: {
title: input.title,
assistantName: input.assistantName,
config: {
suiteCount: new Set(input.cases.map((item) => item.suiteId)).size,
concurrency: input.concurrency,
timeoutSecs: input.timeoutSecs,
failureStrategy: input.failureStrategy,
errorRetryCount: input.errorRetryCount,
errorStrategy: input.errorStrategy,
},
startedAt: new Date().toISOString(),
finishedAt: null,
stopReason: null,
cases: preparedCases.map(({ result }) => result),
},
executionPlan,
};
}
export function hasEvaluationFailure(item: BatchRunCase): boolean {
const evaluations = [
...item.turns.flatMap((turn) => turn.evaluations),
...item.overallCriteria,
];
return evaluations.some((evaluation) => evaluation.status === "fail");
}
export function firstEvaluationFailureReason(item: BatchRunCase): string {
const evaluations = [
...item.turns.flatMap((turn) => turn.evaluations),
@@ -358,15 +114,13 @@ export function countByStatus(cases: BatchRunCase[]) {
waiting: 0,
skipped: 0,
};
for (const item of cases) {
counts[item.status] += 1;
}
for (const item of cases) counts[item.status] += 1;
return counts;
}
export function formatRunTime(iso: string): string {
const date = new Date(iso);
const pad = (n: number) => String(n).padStart(2, "0");
const pad = (value: number) => String(value).padStart(2, "0");
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`;
}

View File

@@ -1,5 +1,5 @@
/**
* 测试集 / 测试用例 — 管理页用的本地 mock
* 测试集 / 测试用例的前后端共享契约与编辑器辅助函数
* 两层Test Suite → Test Case。
* MVP 仅运行固定文字脚本;其它输入模式保留为产品路线提示。
*/
@@ -12,7 +12,7 @@ export type TestCaseInputMode =
| "user_sim_text"
| "user_sim_voice";
type TestCaseResult = "pass" | "fail" | "not_run";
export type TestCaseResult = "pass" | "fail" | "not_run";
export type AssertionType = "keyword" | "llm";
@@ -105,8 +105,9 @@ export type TestSuite = {
id: string;
name: string;
description: string;
/** 关联助手展示名MVP 直接存文案,接 API 后可改成 assistantId */
assistantName: string;
caseCount: number;
passedCount: number;
runCount: number;
updatedAt: string;
};
@@ -160,12 +161,60 @@ export function getTestCaseValidationMessage(
}
if (context.role === "tool_call" || context.role === "tool_result") {
try {
JSON.parse(context.content);
const payload = JSON.parse(context.content);
if (
context.role === "tool_call" &&
(payload === null || Array.isArray(payload) || typeof payload !== "object")
) {
return `上下文第 ${index + 1} 条 Tool Call 参数必须是 JSON 对象`;
}
} catch {
return `上下文第 ${index + 1} 条工具数据必须是有效 JSON`;
}
}
}
const pendingToolCalls: Array<{ id: string | null; toolName: string }> = [];
const seenToolCallIds = new Set<string>();
for (let index = 0; index < item.contextTurns.length; index += 1) {
const context = item.contextTurns[index];
if (context.role === "tool_call") {
const id = context.toolCallId?.trim() || null;
if (id && seenToolCallIds.has(id)) {
return `上下文第 ${index + 1} 条 Tool Call ID 重复`;
}
if (id) seenToolCallIds.add(id);
pendingToolCalls.push({ id, toolName: context.toolName?.trim() ?? "" });
continue;
}
if (context.role !== "tool_result") continue;
const resultId = context.toolCallId?.trim() || null;
let matchedIndex = -1;
for (
let pendingIndex = pendingToolCalls.length - 1;
pendingIndex >= 0;
pendingIndex -= 1
) {
const pending = pendingToolCalls[pendingIndex];
if (
resultId ? pending.id === resultId : pending.toolName === context.toolName?.trim()
) {
matchedIndex = pendingIndex;
break;
}
}
if (matchedIndex < 0) {
return `上下文第 ${index + 1} 条 Tool Result 没有匹配的 Tool Call`;
}
const [matched] = pendingToolCalls.splice(matchedIndex, 1);
if (matched.toolName !== context.toolName?.trim()) {
return `上下文第 ${index + 1} 条 Tool Result 的工具名称不匹配`;
}
}
if (pendingToolCalls.length > 0) {
return `上下文 Tool Call 缺少对应的 Tool Result${pendingToolCalls
.map((item) => item.toolName)
.join("、")}`;
}
const emptyTurnIndex = item.turns.findIndex(
(turn) => !turn.userInput.trim(),
);
@@ -311,26 +360,19 @@ export function getExpectedBehaviorValidationMessage(
return null;
}
let turnSeq = 1;
let behaviorSeq = 1;
let criterionSeq = 1;
function nextTurnId() {
const id = `turn_${String(turnSeq).padStart(4, "0")}`;
turnSeq += 1;
return id;
function newEditorId(prefix: string) {
const randomPart = globalThis.crypto?.randomUUID
? globalThis.crypto.randomUUID().replaceAll("-", "")
: `${Date.now().toString(36)}${Math.random().toString(36).slice(2)}`;
return `${prefix}_${randomPart}`;
}
function nextBehaviorId() {
const id = `beh_${String(behaviorSeq).padStart(4, "0")}`;
behaviorSeq += 1;
return id;
return newEditorId("beh");
}
function nextCriterionId() {
const id = `criterion_${String(criterionSeq).padStart(4, "0")}`;
criterionSeq += 1;
return id;
return newEditorId("criterion");
}
export function createOverallCriterion(
@@ -365,7 +407,7 @@ export function createEmptyFixedInputTurn(
userInput = "",
): FixedInputTurn {
return {
id: nextTurnId(),
id: newEditorId("turn"),
userInput,
behaviors: [],
};
@@ -435,10 +477,6 @@ function cloneBehavior(behavior: ExpectedBehavior): ExpectedBehavior {
};
}
function cloneBehaviorWithNewId(behavior: ExpectedBehavior): ExpectedBehavior {
return { ...cloneBehavior(behavior), id: nextBehaviorId() };
}
export function cloneTurns(turns: FixedInputTurn[]): FixedInputTurn[] {
return turns.map((turn) => ({
id: turn.id,
@@ -447,480 +485,6 @@ export function cloneTurns(turns: FixedInputTurn[]): FixedInputTurn[] {
}));
}
/** 新建用例时复制轮次并分配新 id */
function cloneTurnsWithNewIds(turns: FixedInputTurn[]): FixedInputTurn[] {
return turns.map((turn) => ({
id: nextTurnId(),
userInput: turn.userInput,
behaviors: turn.behaviors.map(cloneBehaviorWithNewId),
}));
}
const INITIAL_SUITES: TestSuite[] = [
{
id: "suite_001",
name: "事故基础流程",
description: "核心业务流程和正常事故处理",
assistantName: "视频快处助手",
updatedAt: "2026-08-05T10:20:00+08:00",
},
{
id: "suite_002",
name: "异常输入与澄清",
description: "模糊表达、误打断、主动唤醒等",
assistantName: "视频快处助手",
updatedAt: "2026-08-04T16:40:00+08:00",
},
{
id: "suite_003",
name: "实时语音交互",
description: "打断、延迟、VAD、语音链路等",
assistantName: "视频快处助手",
updatedAt: "2026-08-03T09:15:00+08:00",
},
];
type RawCaseSeed = Omit<TestCase, "sortOrder" | "inputMode" | "overallCriteria"> & {
inputMode?: TestCaseInputMode;
overallCriteria?: OverallCriterion[];
};
function seedReplyTurn(
seedId: string,
userInput: string,
expectation: Omit<ReplyExpectedBehavior, "id" | "type">,
): FixedInputTurn {
return {
id: `turn_seed_${seedId}`,
userInput,
behaviors: [
{
id: `beh_seed_${seedId}`,
type: "reply",
...expectation,
},
],
};
}
const RAW_CASE_SEEDS: RawCaseSeed[] = [
{
id: "tc_001",
suiteId: "suite_001",
name: "正常双车事故开场",
description: "开场后用户补充事故经过",
lastResult: "pass",
contextTurns: [],
turns: [
seedReplyTurn("001", "我这里刚刚撞了一下。", {
assertionType: "keyword",
keywords: ["经过", "描述", "受伤"],
keywordMatchMode: "any",
negateKeywords: false,
llmCriteria: "",
}),
],
updatedAt: "2026-08-05T10:18:00+08:00",
},
{
id: "tc_002",
suiteId: "suite_001",
name: "有人伤转人工",
description: "用户提到人伤时应引导转人工",
lastResult: "pass",
contextTurns: [],
turns: [
seedReplyTurn("002", "有人受伤了,流血不止。", {
assertionType: "llm",
keywords: [],
keywordMatchMode: "any",
negateKeywords: false,
llmCriteria:
"Agent 应识别人员受伤风险,明确告知将转接人工,并避免继续推进普通快处流程。",
}),
],
overallCriteria: [
{
id: "criterion_seed_001",
type: "llm",
name: "人伤流程正确",
criteria:
"整段对话应正确识别人员受伤场景,及时并准确转接人工处理。",
},
{
id: "criterion_seed_002",
type: "llm",
name: "不推进普通快处",
criteria:
"确认存在人员受伤后,不应继续引导用户进入普通事故快处流程。",
},
],
updatedAt: "2026-08-05T10:12:00+08:00",
},
{
id: "tc_101",
suiteId: "suite_002",
name: "用户说“喂”",
description: "验证主动唤醒回复且业务状态不推进",
lastResult: "pass",
contextTurns: [],
turns: [
seedReplyTurn("101", "喂", {
assertionType: "keyword",
keywords: ["我在", "请说", "继续"],
keywordMatchMode: "any",
negateKeywords: false,
llmCriteria: "",
}),
],
updatedAt: "2026-08-04T16:35:00+08:00",
},
{
id: "tc_102",
suiteId: "suite_002",
name: "用户只说“嗯”",
description: "短促确认不应误推进流程",
lastResult: "fail",
contextTurns: [],
turns: [
seedReplyTurn("102", "嗯", {
assertionType: "llm",
keywords: [],
keywordMatchMode: "any",
negateKeywords: false,
llmCriteria:
"Agent 应当回应用户的主动唤醒,同时继续引导用户描述事故情况,不应该无响应。",
}),
],
updatedAt: "2026-08-04T16:20:00+08:00",
},
{
id: "tc_103",
suiteId: "suite_002",
name: "模糊事故描述",
description: "地点含糊时应主动澄清",
lastResult: "not_run",
contextTurns: [],
turns: [
seedReplyTurn("103", "就在那边……撞了一下。", {
assertionType: "keyword",
keywords: ["哪里", "路口", "路名", "再说"],
keywordMatchMode: "any",
negateKeywords: false,
llmCriteria: "",
}),
],
updatedAt: "2026-08-04T15:30:00+08:00",
},
{
id: "tc_201",
suiteId: "suite_003",
name: "用户打断播报",
description: "播报中打断后正确切换聆听并承接",
lastResult: "pass",
contextTurns: [],
turns: [
seedReplyTurn("201", "等一下,对方走了。", {
assertionType: "llm",
keywords: [],
keywordMatchMode: "any",
negateKeywords: false,
llmCriteria:
"Agent 应立即停止原播报思路,确认已听到用户新信息,并围绕「对方离开」继续询问。",
}),
],
updatedAt: "2026-08-03T09:10:00+08:00",
},
];
/** 按套件出现顺序写入 sortOrder并补齐可选字段。 */
const INITIAL_CASES: TestCase[] = (() => {
const counters = new Map<string, number>();
return RAW_CASE_SEEDS.map((item) => {
const order = counters.get(item.suiteId) ?? 0;
counters.set(item.suiteId, order + 1);
return {
...item,
inputMode: item.inputMode ?? DEFAULT_INPUT_MODE,
turns: cloneTurns(item.turns),
overallCriteria: cloneOverallCriteria(item.overallCriteria),
sortOrder: order,
};
});
})();
/** 会话内可变的 mock 仓库(刷新页面会重置) */
let suites = [...INITIAL_SUITES];
let cases = structuredClone(INITIAL_CASES);
let suiteSeq = 4;
let caseSeq = 300;
function nowIso() {
return new Date().toISOString();
}
function nextSuiteId() {
const id = `suite_${String(suiteSeq).padStart(3, "0")}`;
suiteSeq += 1;
return id;
}
function nextCaseId() {
const id = `tc_${String(caseSeq).padStart(3, "0")}`;
caseSeq += 1;
return id;
}
export function listTestSuites(): TestSuite[] {
return [...suites].sort(
(a, b) => new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime(),
);
}
export function getTestSuite(id: string): TestSuite | null {
return suites.find((item) => item.id === id) ?? null;
}
function getTestCase(id: string): TestCase | null {
return cases.find((item) => item.id === id) ?? null;
}
export function listTestCases(suiteId: string): TestCase[] {
return cases
.filter((item) => item.suiteId === suiteId)
.sort(
(a, b) =>
a.sortOrder - b.sortOrder || a.name.localeCompare(b.name, "zh-CN"),
);
}
export function suiteCaseStats(suiteId: string): {
total: number;
passed: number;
run: number;
} {
const items = cases.filter((item) => item.suiteId === suiteId);
const run = items.filter((item) => item.lastResult !== "not_run");
const passed = run.filter((item) => item.lastResult === "pass");
return { total: items.length, passed: passed.length, run: run.length };
}
export function createTestSuite(input: {
name: string;
description: string;
assistantName: string;
}): TestSuite {
const suite: TestSuite = {
id: nextSuiteId(),
name: input.name.trim(),
description: input.description.trim(),
assistantName: input.assistantName.trim() || "未关联助手",
updatedAt: nowIso(),
};
suites = [suite, ...suites];
return suite;
}
export function updateTestSuite(
id: string,
patch: Partial<Pick<TestSuite, "name" | "description" | "assistantName">>,
): TestSuite | null {
const index = suites.findIndex((item) => item.id === id);
if (index < 0) return null;
const next = {
...suites[index],
...patch,
updatedAt: nowIso(),
};
suites = [...suites.slice(0, index), next, ...suites.slice(index + 1)];
return next;
}
export function removeTestSuite(id: string): boolean {
const before = suites.length;
suites = suites.filter((item) => item.id !== id);
cases = cases.filter((item) => item.suiteId !== id);
return suites.length < before;
}
/** 复制测试集及其全部用例;名称加「(副本)」 */
export function duplicateTestSuite(id: string): TestSuite | null {
const source = getTestSuite(id);
if (!source) return null;
const copied = createTestSuite({
name: `${source.name}(副本)`,
description: source.description,
assistantName: source.assistantName,
});
const sourceCases = listTestCases(id);
const clonedCases: TestCase[] = sourceCases.map((item, index) => {
const turns = cloneTurnsWithNewIds(item.turns);
return {
id: nextCaseId(),
suiteId: copied.id,
name: item.name,
description: item.description,
inputMode: item.inputMode,
lastResult: "not_run" as const,
contextTurns: item.contextTurns.map((turn) => ({ ...turn })),
turns,
overallCriteria: cloneOverallCriteria(item.overallCriteria),
sortOrder: index,
updatedAt: nowIso(),
};
});
cases = [...cases, ...clonedCases];
return copied;
}
export function createTestCase(input: {
suiteId: string;
name?: string;
description?: string;
}): TestCase | null {
if (!getTestSuite(input.suiteId)) return null;
const maxOrder = cases
.filter((item) => item.suiteId === input.suiteId)
.reduce((max, item) => Math.max(max, item.sortOrder), -1);
const turns = [createEmptyFixedInputTurn()];
const item: TestCase = {
id: nextCaseId(),
suiteId: input.suiteId,
name: (input.name ?? "未命名用例").trim() || "未命名用例",
description: (input.description ?? "").trim(),
inputMode: DEFAULT_INPUT_MODE,
lastResult: "not_run",
contextTurns: [],
turns,
overallCriteria: [],
sortOrder: maxOrder + 1,
updatedAt: nowIso(),
};
cases = [...cases, item];
updateTestSuite(input.suiteId, {});
return item;
}
/** 复制单条用例;名称加「(副本)」,排在同套件末尾 */
export function duplicateTestCase(id: string): TestCase | null {
const source = getTestCase(id);
if (!source || !getTestSuite(source.suiteId)) return null;
const maxOrder = cases
.filter((item) => item.suiteId === source.suiteId)
.reduce((max, item) => Math.max(max, item.sortOrder), -1);
const turns = cloneTurnsWithNewIds(source.turns);
const copied: TestCase = {
id: nextCaseId(),
suiteId: source.suiteId,
name: `${source.name}(副本)`,
description: source.description,
inputMode: source.inputMode,
lastResult: "not_run",
contextTurns: source.contextTurns.map((turn) => ({ ...turn })),
turns,
overallCriteria: cloneOverallCriteria(source.overallCriteria),
sortOrder: maxOrder + 1,
updatedAt: nowIso(),
};
cases = [...cases, copied];
updateTestSuite(source.suiteId, {});
return copied;
}
type TestCasePatch = Partial<
Pick<
TestCase,
| "name"
| "description"
| "inputMode"
| "contextTurns"
| "turns"
| "overallCriteria"
| "lastResult"
>
>;
export function updateTestCase(
id: string,
patch: TestCasePatch,
): TestCase | null {
const index = cases.findIndex((item) => item.id === id);
if (index < 0) return null;
const current = cases[index];
const nextTurns =
patch.turns !== undefined
? cloneTurns(
patch.turns.length > 0 ? patch.turns : [createEmptyFixedInputTurn()],
)
: current.turns;
const nextOverallCriteria =
patch.overallCriteria !== undefined
? cloneOverallCriteria(patch.overallCriteria)
: cloneOverallCriteria(current.overallCriteria);
const next: TestCase = {
...current,
...patch,
turns: nextTurns,
overallCriteria: nextOverallCriteria,
updatedAt: nowIso(),
};
cases = [...cases.slice(0, index), next, ...cases.slice(index + 1)];
updateTestSuite(next.suiteId, {});
return next;
}
export function removeTestCase(id: string): boolean {
const existing = cases.find((item) => item.id === id);
if (!existing) return false;
cases = cases.filter((item) => item.id !== id);
renumberSortOrder(existing.suiteId);
updateTestSuite(existing.suiteId, {});
return true;
}
/** 批量删除;返回受影响的 suiteId若有 */
export function removeTestCases(ids: string[]): string | null {
const idSet = new Set(ids);
const affected = cases.find((item) => idSet.has(item.id));
if (!affected) return null;
const suiteId = affected.suiteId;
cases = cases.filter((item) => !idSet.has(item.id));
renumberSortOrder(suiteId);
updateTestSuite(suiteId, {});
return suiteId;
}
/**
* 按给定 id 顺序写回 sortOrder应包含该套件全部用例 id
* 用于拖拽结束后持久化顺序。
*/
export function reorderTestCases(suiteId: string, orderedIds: string[]): void {
const orderMap = new Map(orderedIds.map((id, index) => [id, index]));
cases = cases.map((item) => {
if (item.suiteId !== suiteId) return item;
const nextOrder = orderMap.get(item.id);
if (nextOrder === undefined) return item;
return { ...item, sortOrder: nextOrder };
});
updateTestSuite(suiteId, {});
}
function renumberSortOrder(suiteId: string) {
const ordered = cases
.filter((item) => item.suiteId === suiteId)
.sort((a, b) => a.sortOrder - b.sortOrder);
const orderMap = new Map(ordered.map((item, index) => [item.id, index]));
cases = cases.map((item) => {
if (item.suiteId !== suiteId) return item;
const nextOrder = orderMap.get(item.id);
return nextOrder === undefined ? item : { ...item, sortOrder: nextOrder };
});
}
export function formatUpdatedAt(value?: string | null) {
if (!value) return "—";
const date = new Date(value);

View File

@@ -6,6 +6,15 @@
*/
import { loginPathWithReturnTo } from "@/lib/auth-redirect";
import type { BatchRunSnapshot } from "@/data/batch-run";
import type {
ContextTurn,
FixedInputTurn,
OverallCriterion,
TestCase,
TestCaseInputMode,
TestSuite,
} from "@/data/test-suites";
const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "[::1]"]);
@@ -769,3 +778,94 @@ export const webrtcApi = {
iceServers: () =>
request<{ iceServers: IceServerConfig[] }>("/api/webrtc/ice-servers"),
};
// ---------- 测试用例与批量运行 ----------
export type TestSuiteWrite = {
name: string;
description: string;
};
export type TestCaseWrite = {
name: string;
description: string;
inputMode: TestCaseInputMode;
contextTurns: ContextTurn[];
turns: FixedInputTurn[];
overallCriteria: OverallCriterion[];
};
export const testSuitesApi = {
list: () => request<TestSuite[]>("/api/test-suites"),
get: (id: string) => request<TestSuite>(`/api/test-suites/${id}`),
create: (body: TestSuiteWrite) =>
request<TestSuite>("/api/test-suites", {
method: "POST",
body: JSON.stringify(body),
}),
update: (id: string, body: TestSuiteWrite) =>
request<TestSuite>(`/api/test-suites/${id}`, {
method: "PUT",
body: JSON.stringify(body),
}),
duplicate: (id: string) =>
request<TestSuite>(`/api/test-suites/${id}/duplicate`, { method: "POST" }),
remove: (id: string) =>
request<{ ok: boolean }>(`/api/test-suites/${id}`, { method: "DELETE" }),
};
export const testCasesApi = {
list: (suiteId?: string) =>
request<TestCase[]>(
`/api/test-cases${suiteId ? `?suiteId=${encodeURIComponent(suiteId)}` : ""}`,
),
get: (id: string) => request<TestCase>(`/api/test-cases/${id}`),
create: (suiteId: string, body: TestCaseWrite) =>
request<TestCase>(`/api/test-suites/${suiteId}/cases`, {
method: "POST",
body: JSON.stringify(body),
}),
update: (id: string, body: TestCaseWrite) =>
request<TestCase>(`/api/test-cases/${id}`, {
method: "PUT",
body: JSON.stringify(body),
}),
duplicate: (id: string) =>
request<TestCase>(`/api/test-cases/${id}/duplicate`, { method: "POST" }),
remove: (id: string) =>
request<{ ok: boolean }>(`/api/test-cases/${id}`, { method: "DELETE" }),
bulkRemove: (caseIds: string[]) =>
request<{ ok: boolean; deleted: number }>("/api/test-cases/bulk-delete", {
method: "POST",
body: JSON.stringify({ caseIds }),
}),
reorder: (suiteId: string, caseIds: string[]) =>
request<{ ok: boolean }>(`/api/test-suites/${suiteId}/case-order`, {
method: "PUT",
body: JSON.stringify({ caseIds }),
}),
};
export const batchRunsApi = {
create: (body: {
assistantId: string;
evaluatorModelResourceId: string;
caseIds: string[];
title?: string;
config: {
concurrency: number;
timeoutSecs: number;
failureStrategy: "continue" | "stop_on_fail";
errorRetryCount: number;
errorStrategy: "continue" | "stop_on_error";
};
}) =>
request<BatchRunSnapshot>("/api/test-runs", {
method: "POST",
body: JSON.stringify(body),
}),
get: (id: string) => request<BatchRunSnapshot>(`/api/test-runs/${id}`),
cancel: (id: string) =>
request<BatchRunSnapshot>(`/api/test-runs/${id}/cancel`, {
method: "POST",
}),
};