From 2ab56f48d7dfcce0774e580e50d3b7beabf8a380 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Tue, 15 Sep 2026 02:27:48 +0800 Subject: [PATCH] Add annotated fixed-horizon floor evaluation --- PROJECT_CONTEXT.md | 2 + docs/README.md | 1 + docs/easycom_annotated_floor_evaluation.md | 101 ++++ evaluate_annotated_floor_events.py | 668 +++++++++++++++++++++ manuscript/main/REVIEWER_CHECKLIST.md | 7 +- manuscript/main/main.tex | 88 +-- test_evaluate_annotated_floor_events.py | 270 +++++++++ 7 files changed, 1099 insertions(+), 38 deletions(-) create mode 100644 docs/easycom_annotated_floor_evaluation.md create mode 100644 evaluate_annotated_floor_events.py create mode 100644 test_evaluate_annotated_floor_events.py diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md index 8e70e32..2c0c064 100644 --- a/PROJECT_CONTEXT.md +++ b/PROJECT_CONTEXT.md @@ -45,6 +45,8 @@ result. - Encoder-only A/V/AV and `G`, `G+A`, `G+V`, `G+A+V` baselines. - Ego4D batch and 8-GPU local-feature launchers, with strict frame audits. - Timing-derived wearer floor-state and event diagnostics. +- Human-annotated fixed-horizon entry/INT evaluator with point positives, + region-level hard negatives and training-side threshold calibration. ### Measured screening result diff --git a/docs/README.md b/docs/README.md index eae0f5a..d26162d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,6 +34,7 @@ tests. | [`easycom_competing_risk_floor_evaluation.md`](easycom_competing_risk_floor_evaluation.md) | Running and auditing the competing-risk entry/yield evaluator with training-side threshold calibration | | [`easycom_floor_event_diagnostics.md`](easycom_floor_event_diagnostics.md) | Separating boundary observability, event-objective mismatch and addressee information with remote commands | | [`easycom_event_review_tool.md`](easycom_event_review_tool.md) | Reviewing automatically mined EasyCom floor-event candidates against local video and transcripts | +| [`easycom_annotated_floor_evaluation.md`](easycom_annotated_floor_evaluation.md) | Scoring human-confirmed point events and hard-negative opportunity regions at fixed EgoSpeak horizons | ## Design history and implementation options diff --git a/docs/easycom_annotated_floor_evaluation.md b/docs/easycom_annotated_floor_evaluation.md new file mode 100644 index 0000000..4e5bf82 --- /dev/null +++ b/docs/easycom_annotated_floor_evaluation.md @@ -0,0 +1,101 @@ +# EasyCom human-annotated fixed-horizon floor-event evaluation + +This protocol evaluates saved EgoSpeak-layout dense predictions against +human-confirmed `WEARER_ENTRY`, `INT_YIELD`, and hard-negative opportunities. +It is separate from the timing-derived proxy in +[`evaluate_floor_state_changes.py`](../evaluate_floor_state_changes.py). + +## Scoring contract + +At each nominal 0.2--2.0 second horizon: + +- a positive entry is one corrected 5 Hz behavior-anchor bin, scored by wearer + activity probability; +- a positive INT-yield is one corrected 5 Hz behavior-anchor bin, scored by one + minus wearer activity probability; +- a hard negative is one opportunity sample, scored by the maximum erroneous + action probability over its annotated half-open region; +- a positive timestamp is never expanded into a tolerance window; +- an excluded or unreviewed candidate never enters scoring. + +The prediction layout is `[target_time, class, future_horizon]`. Event frames +are rasterized from the original 20 Hz annotation with the same `frame // 4` +rule used by the EasyCom event derivation. A human `-0.2/0/+0.2` second boundary +correction shifts the resulting target bin; for hard negatives it shifts the +whole region. + +The reported horizon is the nominal EgoSpeak forecast-bin horizon. Because an +acoustic boundary can occur anywhere inside a 200 ms target bin, do not +reinterpret it as an exact continuous-time lead without separately reporting +the quantization convention. + +## Development smoke run + +Use only training-side sessions while the annotation and region rules are +being audited: + +```bash +python3 evaluate_annotated_floor_events.py \ + --reviews reports/easycom_event_reviews/reviews.jsonl \ + --run egospeak=/path/to/egospeak/result \ + --sessions Session_12 \ + --matched-recalls 0.7 0.8 0.9 \ + --print-event-examples 3 \ + --output reports/easycom_annotated_floor_events/session12_summary.json +``` + +This selects thresholds on the same development events and labels the threshold +source as exploratory. It is not a confirmatory test command. + +## Confirmatory test run + +Thresholds must be selected independently for entry and INT at every horizon +from training-side out-of-fold predictions: + +```bash +python3 evaluate_annotated_floor_events.py \ + --reviews reports/easycom_event_reviews/test_reviews.jsonl \ + --run baseline=/path/to/baseline/test_result \ + --run candidate=/path/to/candidate/test_result \ + --sessions Session_3 Session_2 Session_1 \ + --calibration-reviews reports/easycom_event_reviews/oof_train_reviews.jsonl \ + --calibration-run baseline=/path/to/baseline/oof_result \ + --calibration-run candidate=/path/to/candidate/oof_result \ + --calibration-sessions Session_4 Session_5 Session_6 Session_7 Session_8 Session_9 Session_10 Session_11 Session_12 \ + --matched-recalls 0.7 0.8 0.9 \ + --print-event-examples 3 \ + --output results/easycom_annotated_floor_events_YYYY-MM-DD/summary.json +``` + +The CLI rejects evaluation involving official Sessions 1--3 unless calibration +reviews and calibration sessions are supplied. The +`--allow-in-sample-thresholds` override is exploratory only. + +Outputs: + +- `summary.json`: protocol metadata, counts, ten horizons per family, AP and + operating points; +- `summary_metrics.csv`: one row per run, family, horizon, and requested recall; +- `summary_events.csv`: one scored row per run, annotated opportunity, and + horizon for alignment audits and paired analysis. + +Before comparing models, verify identical positive, hard-negative, and invalid +counts. The headline manuscript operating point is the nominal 0.2 second bin +at matched recall 0.8; ten-horizon entry/INT AP and all horizon curves are +supporting results. + +## Required audit before freezing + +1. Print examples and verify the 20 Hz boundary, 5 Hz target row, horizon, and + valid mask against the source timeline. +2. Confirm that every positive occupies one target row. +3. Confirm that each hard-negative region contributes one max-pooled score, + not one negative per frame. +4. Audit continuous timeline segments without candidate hints to estimate + candidate recall. +5. Freeze the annotation guide, region endpoints, boundary-correction policy, + and out-of-fold calibration procedure before inspecting Sessions 1--3. + +This evaluator does not emit committed actions or detection latency. Those +require a separately frozen online controller with threshold, persistence, +refractory period, matching interval, and competing-risk rules. diff --git a/evaluate_annotated_floor_events.py b/evaluate_annotated_floor_events.py new file mode 100644 index 0000000..eaab58c --- /dev/null +++ b/evaluate_annotated_floor_events.py @@ -0,0 +1,668 @@ +#!/usr/bin/env python3 +"""Evaluate human-adjudicated wearer entry and INT-yield opportunities. + +The forecasting model emits dense EgoSpeak-compatible probabilities rather +than event detections. At each nominal horizon, a confirmed positive event is +therefore represented by exactly one score at its corrected behavior-anchor +bin. A confirmed hard negative is represented by the maximum erroneous-action +score inside its labeled half-open opportunity region. Positives are never +expanded into tolerance windows. + +Thresholds for recall-matched false-positive rates must come from disjoint +training-side calibration predictions when Sessions 1--3 are evaluated. +""" + +from __future__ import annotations + +import argparse +import csv +from dataclasses import dataclass +import json +import math +from pathlib import Path +from typing import Any, Iterable, Literal + +import numpy as np + +from evaluate_floor_state_changes import ( + CLASS_NAMES, + NUM_CLASSES, + NUM_HORIZONS, + OFFICIAL_TRAIN_SESSIONS, + OFFICIAL_TEST_SESSIONS, + STEP_SECONDS, + WEARER_INDEX, + _operating_points, + _safe_ap, + parse_runs, + validate_recalls, +) + + +SOURCE_FPS = 20.0 +Family = Literal["entry", "int"] +FAMILIES: tuple[Family, ...] = ("entry", "int") + + +@dataclass(frozen=True) +class AnnotatedOpportunity: + uid: str + session: str + family: Family + positive: bool + subtype: str + start_row: int + end_row_exclusive: int + anchor_seconds: float + region_start_seconds: float + region_end_seconds: float + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--reviews", + type=Path, + action="append", + required=True, + help="Human review JSONL. Repeat to combine independently stored files.", + ) + parser.add_argument( + "--run", + action="append", + required=True, + metavar="NAME=RESULT_DIR", + help="Inference directory containing pred/ and valid/. Repeat per method.", + ) + parser.add_argument( + "--sessions", + nargs="+", + default=list(OFFICIAL_TEST_SESSIONS), + ) + parser.add_argument( + "--calibration-reviews", + type=Path, + action="append", + default=None, + help="Disjoint training-side human review JSONL used for thresholds.", + ) + parser.add_argument( + "--calibration-run", + action="append", + default=None, + metavar="NAME=RESULT_DIR", + help="Calibration inference directory for each --run name.", + ) + parser.add_argument( + "--calibration-sessions", + nargs="+", + default=None, + ) + parser.add_argument( + "--matched-recalls", + nargs="+", + type=float, + default=(0.7, 0.8, 0.9), + ) + parser.add_argument( + "--allow-in-sample-thresholds", + action="store_true", + help="Exploratory only: select thresholds on the evaluated events.", + ) + parser.add_argument( + "--output", + type=Path, + default=Path("reports/easycom_annotated_floor_events/summary.json"), + ) + parser.add_argument( + "--csv-output", + type=Path, + default=None, + ) + parser.add_argument( + "--event-csv-output", + type=Path, + default=None, + help="Optional scored-opportunity CSV (default: OUTPUT stem + _events.csv).", + ) + parser.add_argument( + "--print-event-examples", + type=int, + default=0, + metavar="N", + help="Print up to N scored examples per family for the first run.", + ) + return parser.parse_args() + + +def _read_jsonl(paths: Iterable[Path]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for path in paths: + if not path.is_file(): + raise FileNotFoundError(path) + with path.open("r", encoding="utf-8") as handle: + for line_number, raw_line in enumerate(handle, 1): + line = raw_line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError(f"Invalid JSON at {path}:{line_number}: {error}") from error + if not isinstance(row, dict): + raise ValueError(f"Expected object at {path}:{line_number}") + rows.append(row) + return rows + + +def _integer_frame(value: Any, *, field: str, uid: str) -> int: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{uid}: {field} must be a numeric 20-Hz frame") + rounded = int(round(float(value))) + if not math.isclose(float(value), rounded, abs_tol=1e-6): + raise ValueError(f"{uid}: {field} must be integral, got {value!r}") + if rounded < 0: + raise ValueError(f"{uid}: {field} must be non-negative") + return rounded + + +def _seconds(value: Any, *, field: str, uid: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{uid}: {field} must be numeric") + result = float(value) + if not math.isfinite(result) or result < 0: + raise ValueError(f"{uid}: {field} must be finite and non-negative") + return result + + +def _frame_to_row(frame: int) -> int: + """Use the same 20-Hz-to-5-Hz rasterization as EasyCom event derivation.""" + + return frame // int(round(SOURCE_FPS * STEP_SECONDS)) + + +def _seconds_to_start_row(seconds: float) -> int: + return int(math.floor(seconds / STEP_SECONDS + 1e-7)) + + +def _seconds_to_end_row_exclusive(seconds: float) -> int: + return int(math.ceil(seconds / STEP_SECONDS - 1e-7)) + + +def _review_payload(row: dict[str, Any], uid: str) -> tuple[dict[str, Any], dict[str, Any]]: + review = row.get("review") + candidate = row.get("candidate") + if not isinstance(review, dict) or not isinstance(candidate, dict): + raise ValueError( + f"{uid}: expected review_floor_events.py JSONL with nested review and candidate objects" + ) + if review.get("human_status") != "REVIEWED": + raise ValueError(f"{uid}: human_status must be REVIEWED") + return review, candidate + + +def load_annotated_opportunities( + paths: Iterable[Path], + sessions: Iterable[str], +) -> tuple[list[AnnotatedOpportunity], dict[str, int]]: + selected_sessions = set(sessions) + opportunities: list[AnnotatedOpportunity] = [] + seen: set[str] = set() + excluded = 0 + + for row_number, row in enumerate(_read_jsonl(paths), 1): + uid = str(row.get("candidate_id") or row.get("candidate_uid") or f"row-{row_number}") + if uid in seen: + raise ValueError(f"Duplicate reviewed candidate: {uid}") + seen.add(uid) + session = str(row.get("session", "")) + if session not in selected_sessions: + continue + review, candidate = _review_payload(row, uid) + verdict = review.get("human_verdict") + if verdict == "EXCLUDE": + excluded += 1 + continue + if verdict not in {"POSITIVE", "HARD_NEGATIVE"}: + raise ValueError(f"{uid}: invalid human_verdict {verdict!r}") + + family_value = row.get("candidate_family") + if family_value == "entry": + family: Family = "entry" + elif family_value == "int": + family = "int" + else: + raise ValueError(f"{uid}: scoring row has invalid candidate_family {family_value!r}") + + positive = verdict == "POSITIVE" + event_type = review.get("human_event_type") + subtype = review.get("human_subtype") + expected_event_type = "WEARER_ENTRY" if family == "entry" else "INT_YIELD" + if positive: + if event_type != expected_event_type or subtype is not None: + raise ValueError(f"{uid}: positive {family} must be {expected_event_type}") + subtype_text = expected_event_type + else: + prefix = "ENTRY_" if family == "entry" else "INT_" + if not isinstance(subtype, str) or ( + subtype != "OTHER_HARD_NEGATIVE" and not subtype.startswith(prefix) + ): + raise ValueError(f"{uid}: hard-negative subtype does not match {family}") + subtype_text = subtype + + correction = review.get("boundary_correction_seconds") or 0.0 + correction = float(correction) + correction_rows = int(round(correction / STEP_SECONDS)) + if not math.isclose(correction, correction_rows * STEP_SECONDS, abs_tol=1e-7): + raise ValueError(f"{uid}: boundary correction is not on the 5-Hz grid") + + evidence = candidate.get("temporal_evidence") + evidence = evidence if isinstance(evidence, dict) else {} + if positive: + raw_frame = evidence.get("behavior_anchor_frame", candidate.get("anchor_dense_frame")) + frame = _integer_frame(raw_frame, field="behavior_anchor_frame", uid=uid) + row_index = _frame_to_row(frame) + correction_rows + if row_index < 0: + raise ValueError(f"{uid}: corrected behavior anchor precedes the session") + anchor_seconds = frame / SOURCE_FPS + correction + start_row = row_index + end_row_exclusive = row_index + 1 + region_start = anchor_seconds + region_end = anchor_seconds + else: + raw_start = _seconds( + candidate.get("region_start_seconds", candidate.get("anchor_dense_seconds")), + field="region_start_seconds", + uid=uid, + ) + correction + raw_end = _seconds( + candidate.get("region_end_seconds", candidate.get("anchor_dense_seconds")), + field="region_end_seconds", + uid=uid, + ) + correction + if raw_start < 0 or raw_end < raw_start: + raise ValueError(f"{uid}: invalid corrected hard-negative region") + start_row = _seconds_to_start_row(raw_start) + end_row_exclusive = max(start_row + 1, _seconds_to_end_row_exclusive(raw_end)) + anchor_seconds = _seconds( + candidate.get("anchor_dense_seconds", raw_start - correction), + field="anchor_dense_seconds", + uid=uid, + ) + correction + region_start, region_end = raw_start, raw_end + + opportunities.append( + AnnotatedOpportunity( + uid=uid, + session=session, + family=family, + positive=positive, + subtype=subtype_text, + start_row=start_row, + end_row_exclusive=end_row_exclusive, + anchor_seconds=anchor_seconds, + region_start_seconds=region_start, + region_end_seconds=region_end, + ) + ) + + diagnostics = { + "num_review_rows": len(seen), + "num_scoring_eligible": len(opportunities), + "num_excluded": excluded, + } + return opportunities, diagnostics + + +def load_session_predictions( + run_dir: Path, + sessions: Iterable[str], +) -> dict[str, tuple[np.ndarray, np.ndarray]]: + result: dict[str, tuple[np.ndarray, np.ndarray]] = {} + for session in sessions: + pred_path = run_dir / "pred" / f"{session}.npy" + valid_path = run_dir / "valid" / f"{session}.npy" + if not pred_path.is_file(): + raise FileNotFoundError(pred_path) + if not valid_path.is_file(): + raise FileNotFoundError(valid_path) + prediction = np.asarray(np.load(pred_path, allow_pickle=False), dtype=np.float32) + valid = np.asarray(np.load(valid_path, allow_pickle=False), dtype=bool) + if prediction.ndim != 3 or prediction.shape[1:] != (NUM_CLASSES, NUM_HORIZONS): + raise ValueError(f"Expected [T,{NUM_CLASSES},{NUM_HORIZONS}], got {prediction.shape}: {pred_path}") + if valid.shape != (len(prediction), NUM_HORIZONS): + raise ValueError(f"Expected {(len(prediction), NUM_HORIZONS)}, got {valid.shape}: {valid_path}") + if not np.isfinite(prediction).all() or (prediction < -1e-6).any() or (prediction > 1 + 1e-6).any(): + raise ValueError(f"Prediction must be finite and in [0,1]: {pred_path}") + result[session] = (np.clip(prediction, 0.0, 1.0), valid) + return result + + +def score_opportunity( + opportunity: AnnotatedOpportunity, + arrays: dict[str, tuple[np.ndarray, np.ndarray]], + horizon_index: int, +) -> float | None: + prediction, valid = arrays[opportunity.session] + start = opportunity.start_row + end = opportunity.end_row_exclusive + if start < 0 or start >= end or end > len(prediction): + return None + usable = valid[start:end, horizon_index] + if not usable.all(): + return None + scores = prediction[start:end, WEARER_INDEX, horizon_index] + if opportunity.family == "int": + scores = 1.0 - scores + return float(np.max(scores)) + + +def evaluate_family_horizon( + opportunities: list[AnnotatedOpportunity], + arrays: dict[str, tuple[np.ndarray, np.ndarray]], + family: Family, + horizon_index: int, + recalls: list[float], + *, + fixed_thresholds: dict[float, float | None] | None = None, + threshold_source: str = "evaluated_events", +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + rows: list[dict[str, Any]] = [] + invalid = 0 + for opportunity in opportunities: + if opportunity.family != family: + continue + score = score_opportunity(opportunity, arrays, horizon_index) + if score is None: + invalid += 1 + continue + rows.append( + { + "uid": opportunity.uid, + "session": opportunity.session, + "family": family, + "positive": opportunity.positive, + "subtype": opportunity.subtype, + "score": score, + "anchor_seconds": opportunity.anchor_seconds, + "region_start_seconds": opportunity.region_start_seconds, + "region_end_seconds": opportunity.region_end_seconds, + "target_start_row": opportunity.start_row, + "target_end_row_exclusive": opportunity.end_row_exclusive, + "source_start_row": opportunity.start_row - horizon_index - 1, + "source_end_row_exclusive": opportunity.end_row_exclusive - horizon_index - 1, + "nominal_horizon_seconds": (horizon_index + 1) * STEP_SECONDS, + } + ) + + scores = np.asarray([row["score"] for row in rows], dtype=np.float64) + positive = np.asarray([row["positive"] for row in rows], dtype=bool) + negative = ~positive + subtype_names = sorted({str(row["subtype"]) for row in rows if not row["positive"]}) + extra_groups = { + f"fpr_{name.lower()}": np.asarray( + [not row["positive"] and row["subtype"] == name for row in rows], + dtype=bool, + ) + for name in subtype_names + } + labels = positive.astype(np.int8) + metric = { + "family": family, + "horizon_index": horizon_index + 1, + "nominal_seconds": (horizon_index + 1) * STEP_SECONDS, + "num_positive": int(positive.sum()), + "num_hard_negative": int(negative.sum()), + "num_invalid": invalid, + "average_precision": _safe_ap(labels, scores), + "operating_points": _operating_points( + scores, + positive, + negative, + recalls, + extra_groups=extra_groups, + fixed_thresholds=fixed_thresholds, + threshold_source=threshold_source, + ), + } + return metric, rows + + +def thresholds_from_metric(metric: dict[str, Any]) -> dict[float, float | None]: + return { + float(item["requested_positive_recall"]): item["threshold"] + for item in metric["operating_points"] + } + + +def _run_names(runs: list[tuple[str, Path]]) -> list[str]: + return [name for name, _ in runs] + + +def _validate_protocol_args(args: argparse.Namespace, runs: list[tuple[str, Path]]) -> tuple[list[tuple[str, Path]], bool]: + official_test = bool(set(args.sessions) & set(OFFICIAL_TEST_SESSIONS)) + if bool(args.calibration_reviews) != bool(args.calibration_sessions): + raise ValueError( + "--calibration-reviews and --calibration-sessions must be supplied together" + ) + has_calibration = bool(args.calibration_reviews and args.calibration_sessions) + if has_calibration: + unknown = sorted(set(args.calibration_sessions) - set(OFFICIAL_TRAIN_SESSIONS)) + if unknown: + raise ValueError( + f"Calibration must use EasyCom Sessions 4--12, got {unknown}" + ) + overlap = sorted(set(args.sessions) & set(args.calibration_sessions)) + if overlap: + raise ValueError(f"Evaluation and calibration sessions overlap: {overlap}") + if official_test and not has_calibration and not args.allow_in_sample_thresholds: + raise ValueError( + "Official Sessions 1--3 require --calibration-reviews and " + "--calibration-sessions; use --allow-in-sample-thresholds only for exploration" + ) + if args.calibration_run: + calibration_runs = parse_runs(args.calibration_run) + if _run_names(calibration_runs) != _run_names(runs): + raise ValueError("--calibration-run names/order must match --run") + else: + calibration_runs = runs + return calibration_runs, has_calibration + + +def write_csv(path: Path, run_results: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fieldnames = [ + "run", "family", "horizon_index", "nominal_seconds", "num_positive", + "num_hard_negative", "num_invalid", "average_precision", "requested_recall", + "threshold", "threshold_source", "achieved_recall", "false_positive_rate", + "false_positive_count", + ] + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + for run in run_results: + for metric in run["horizons"]: + for point in metric["operating_points"]: + writer.writerow( + { + "run": run["name"], + "family": metric["family"], + "horizon_index": metric["horizon_index"], + "nominal_seconds": metric["nominal_seconds"], + "num_positive": metric["num_positive"], + "num_hard_negative": metric["num_hard_negative"], + "num_invalid": metric["num_invalid"], + "average_precision": metric["average_precision"], + "requested_recall": point["requested_positive_recall"], + "threshold": point["threshold"], + "threshold_source": point["threshold_source"], + "achieved_recall": point["achieved_positive_recall"], + "false_positive_rate": point["negative_false_positive_rate"], + "false_positive_count": point["negative_false_positive_count"], + } + ) + + +def write_event_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fieldnames = [ + "run", + "uid", + "session", + "family", + "positive", + "subtype", + "horizon_index", + "nominal_horizon_seconds", + "score", + "anchor_seconds", + "region_start_seconds", + "region_end_seconds", + "target_start_row", + "target_end_row_exclusive", + "source_start_row", + "source_end_row_exclusive", + ] + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def print_examples(rows: list[dict[str, Any]], limit: int) -> None: + if limit <= 0: + return + print("\nAnnotated fixed-horizon examples (first run, 0.2-s nominal horizon)") + for family in FAMILIES: + print(f"{family}:") + shown = 0 + for row in rows: + if row["family"] != family: + continue + label = "POS" if row["positive"] else "HN" + print( + f" {label} {row['session']} {row['uid']} " + f"target_rows=[{row['target_start_row']},{row['target_end_row_exclusive']}) " + f"source_rows=[{row['source_start_row']},{row['source_end_row_exclusive']}) " + f"region={row['region_start_seconds']:.2f}--{row['region_end_seconds']:.2f}s " + f"score={row['score']:.4f} subtype={row['subtype']}" + ) + shown += 1 + if shown >= limit: + break + + +def main() -> None: + args = parse_args() + recalls = validate_recalls(args.matched_recalls) + runs = parse_runs(args.run) + calibration_runs, has_calibration = _validate_protocol_args(args, runs) + + opportunities, review_diagnostics = load_annotated_opportunities(args.reviews, args.sessions) + if not opportunities: + raise ValueError("No scoring-eligible reviewed opportunities for evaluated sessions") + + calibration_opportunities: list[AnnotatedOpportunity] | None = None + calibration_diagnostics: dict[str, int] | None = None + if has_calibration: + calibration_opportunities, calibration_diagnostics = load_annotated_opportunities( + args.calibration_reviews, + args.calibration_sessions, + ) + if not calibration_opportunities: + raise ValueError("No scoring-eligible calibration opportunities") + + all_results: list[dict[str, Any]] = [] + all_event_rows: list[dict[str, Any]] = [] + first_rows: list[dict[str, Any]] = [] + for run_index, ((name, run_dir), (_, calibration_dir)) in enumerate(zip(runs, calibration_runs)): + arrays = load_session_predictions(run_dir, args.sessions) + calibration_arrays = ( + None + if calibration_opportunities is None + else load_session_predictions(calibration_dir, args.calibration_sessions) + ) + horizons: list[dict[str, Any]] = [] + event_rows: list[dict[str, Any]] = [] + for horizon_index in range(NUM_HORIZONS): + for family in FAMILIES: + fixed_thresholds = None + threshold_source = "evaluated_events_exploratory" + if calibration_opportunities is not None and calibration_arrays is not None: + calibration_metric, _ = evaluate_family_horizon( + calibration_opportunities, + calibration_arrays, + family, + horizon_index, + recalls, + threshold_source="training_side_calibration", + ) + fixed_thresholds = thresholds_from_metric(calibration_metric) + threshold_source = "training_side_calibration" + metric, rows = evaluate_family_horizon( + opportunities, + arrays, + family, + horizon_index, + recalls, + fixed_thresholds=fixed_thresholds, + threshold_source=threshold_source, + ) + horizons.append(metric) + event_rows.extend( + {"run": name, **row, "horizon_index": horizon_index + 1} + for row in rows + ) + if run_index == 0 and horizon_index == 0: + first_rows.extend(rows) + all_event_rows.extend(event_rows) + all_results.append( + { + "name": name, + "result_dir": str(run_dir), + "horizons": horizons, + "mean_entry_ap": float(np.mean([ + item["average_precision"] for item in horizons + if item["family"] == "entry" and item["average_precision"] is not None + ])) if any(item["family"] == "entry" and item["average_precision"] is not None for item in horizons) else None, + "mean_int_ap": float(np.mean([ + item["average_precision"] for item in horizons + if item["family"] == "int" and item["average_precision"] is not None + ])) if any(item["family"] == "int" and item["average_precision"] is not None for item in horizons) else None, + } + ) + + payload = { + "schema_version": 1, + "protocol": "human-adjudicated-fixed-horizon-floor-events-v1", + "semantic_scope": ( + "Human-confirmed wearer entry, INT-yield, and hard-negative opportunities; " + "positive events remain single 5-Hz target bins." + ), + "prediction_layout": "[target_time, class, future_horizon]", + "class_names": list(CLASS_NAMES), + "sessions": list(args.sessions), + "calibration_sessions": args.calibration_sessions, + "matched_recalls": recalls, + "positive_score": "one corrected behavior-anchor bin at each nominal horizon", + "hard_negative_score": "maximum erroneous-action score over the labeled half-open region at each nominal horizon", + "review_diagnostics": review_diagnostics, + "calibration_review_diagnostics": calibration_diagnostics, + "runs": all_results, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + csv_path = args.csv_output or args.output.with_name(args.output.stem + "_metrics.csv") + event_csv_path = args.event_csv_output or args.output.with_name( + args.output.stem + "_events.csv" + ) + write_csv(csv_path, all_results) + write_event_csv(event_csv_path, all_event_rows) + print_examples(first_rows, args.print_event_examples) + print(f"Saved {args.output}") + print(f"Saved {csv_path}") + print(f"Saved {event_csv_path}") + + +if __name__ == "__main__": + main() diff --git a/manuscript/main/REVIEWER_CHECKLIST.md b/manuscript/main/REVIEWER_CHECKLIST.md index a72c695..4bd8864 100644 --- a/manuscript/main/REVIEWER_CHECKLIST.md +++ b/manuscript/main/REVIEWER_CHECKLIST.md @@ -145,8 +145,11 @@ retrain test-time counterfactuals. - [ ] Three seeds, mean ± sample standard deviation. - [ ] Paired cluster bootstrap by session/conversation block for 95% intervals. - [ ] Operating points at wearer-onset recall 0.7, 0.8, and 0.9; headline 0.8. -- [ ] False wearer entries per multi-other opportunity minute, onset latency, - wearer-yield recall/latency, and original ten-horizon AP/mAP. +- [ ] Fixed-horizon entry/INT event AP, recall-matched hard-negative + opportunity FPR, subtype FPR, and original ten-horizon dense AP/mAP. +- [ ] Treat one-to-one event matching and signed latency as secondary; report + them only after freezing the online threshold, persistence, refractory + period, matching interval, and competing-risk rule. - [ ] Error strata: visibility, number of visible people, overlap, offscreen activity, and transition proxy. Add track age/fragmentation only if a track-aware method is selected. diff --git a/manuscript/main/main.tex b/manuscript/main/main.tex index ecc59da..d63c2ac 100644 --- a/manuscript/main/main.tex +++ b/manuscript/main/main.tex @@ -33,8 +33,8 @@ not explicitly trained to expose sub-second, wearer-relative floor evidence. We therefore study a candidate causal local floor adapter that combines the global Omni state with short-window mixed audio and an anonymous set of visible-person observations, without assuming persistent speaker identity. On EasyCom, -\todo{report the final three-seed dense and committed-event -results: wearer-entry and INT-yield recall, false-positive rate, latency, +\todo{report the final three-seed dense and fixed-horizon behavioral-event +results: wearer-entry and INT-yield AP, recall-matched false-positive rates, confidence intervals, and oracle-gap recovery}. Controlled \todo{time-shift/remove-vision/same-rate or capacity-matched} interventions show that \todo{state the supported mechanism, without claiming addressee or @@ -102,9 +102,9 @@ corresponding TODO gates pass: \item We evaluate a wearer-relative causal local floor adapter that combines long-context Omni information with short-window audio and anonymous visible-person evidence. - \item We evaluate whether the adapter reduces erroneous wearer commits - during multiparty floor transfers under an entry/INT event protocol with - explicitly annotated hard-negative regions. + \item We evaluate whether the adapter assigns lower erroneous-action scores + during multiparty floor transfers under an entry/INT protocol with point + positives and explicitly annotated hard-negative opportunity regions. \end{itemize} \begin{figure}[t] @@ -136,11 +136,13 @@ Wearer--other overlap is allowed. The model outputs \in[0,1]^{K\times3}. \label{eq:task} \end{equation} -The wearer-activity forecast also provides a causal control score. When the -wearer is silent, $\hat y^{\mathrm w}_{t,1}$ acts as a near-term take-turn -score; while the wearer is active, $1-\hat y^{\mathrm w}_{t,1}$ acts as a -yield score. These scores are trained only from future activity targets. -Committed wearer-entry and INT-yield events used for behavioral evaluation are +The wearer-activity forecast also provides causal behavioral scores at every +horizon. At decision time $t$, if the wearer is silent, +$\hat y^{\mathrm w}_{t,k}$ acts as a take-turn score for future bin $k$; if +the wearer is active, $1-\hat y^{\mathrm w}_{t,k}$ acts as a yield score. +These scores are trained only from future activity targets and are not event +hazards or committed controller actions. +Adjudicated wearer-entry and INT-yield events used for behavioral evaluation are defined separately from participant-resolved annotations in Sec.~\ref{sec:experiments}. @@ -229,7 +231,7 @@ the training/development pool and Sessions 1--3 only for final evaluation converted into two parallel representations. First, they are downsampled to 5~Hz to construct the EgoSpeak-compatible ten-horizon targets for background, camera-wearer, and aggregate non-wearer activity. Second, the original -participant timeline is retained to derive committed interaction events for +participant timeline is retained to derive adjudicated interaction events for behavioral evaluation. \textbf{Behavioral event annotation.} @@ -326,24 +328,38 @@ configuration, trainable parameter counts, and inference latency.} We first retain dense future-activity evaluation for direct comparison with EgoSpeak, reporting AP at the 0.2--2.0~s horizons and their mean mAP. -Behavioral evaluation follows a committed-event protocol. For both -wearer-entry and INT-yield, each gold event is matched to at most one committed -prediction and is scored by recall and signed detection latency. +Behavioral evaluation uses fixed-horizon event classification. For both +wearer-entry and INT-yield, we align the dense forecast to each adjudicated +item at each of the ten nominal horizons. A positive remains a single corrected +5-Hz behavior-anchor bin: its entry score is wearer activity probability and +its INT-yield score is one minus wearer activity probability. We do not expand +the positive timestamp into a tolerance window. A hard-negative opportunity is +one sample whose score is the maximum erroneous-action probability over its +labeled half-open region at the same fixed horizon. Thus, each positive event +or hard-negative opportunity contributes exactly one score per horizon. \textbf{False-positive scoring rule.} -A false positive is defined by the model committing inside a labeled -hard-negative region, regardless of whether the wearer subsequently exhibits -an onset or offset. Thus, false-positive evaluation includes conversational -situations in which the correct wearer behavior is to remain silent or retain -the floor. +A false positive is a hard-negative opportunity whose region-level score +exceeds the frozen threshold, regardless of whether the wearer subsequently +exhibits an onset or offset. Region max-pooling means that multiple high-scoring +frames still count as one false-positive opportunity. Thus, false-positive +evaluation includes conversational situations in which the correct wearer +behavior is to remain silent or retain the floor. -We report the recall--false-positive trade-off and a common operating point at -matched recall $R=0.8$. The primary behavioral quantities are therefore entry -false-positive rate, INT false-positive rate, and signed latency at the frozen -operating point. We additionally report performance on the other-to-other -subset to test whether local visual evidence specifically reduces erroneous -wearer turn-taking during multiparty floor transfers. \todo{Freeze the FPR -denominators and report signed median latency separately for entry and INT.} +For every horizon we report event AP and the recall--false-positive trade-off. +The headline behavioral operating point uses the first, nominal 0.2-s forecast +bin at matched recall $R=0.8$; thresholds are selected independently for entry +and INT from training-side out-of-fold predictions. We additionally report the +complete 0.2--2.0-s curves and the other-to-other hard-negative subset to test +whether local visual evidence specifically reduces erroneous wearer action +scores during multiparty floor transfers. \todo{Freeze the hard-negative +region endpoints and verify identical event denominators across methods.} + +A deployment-style committed-event analysis with one-to-one matching and +signed latency is secondary and is omitted unless a threshold, persistence, +refractory period, matching interval, and competing-risk rule are all frozen +before test evaluation. The fixed-horizon protocol itself makes no detection- +latency claim. \section{Results and Analysis} @@ -361,9 +377,9 @@ frozen before test evaluation. \todo{Freeze all baseline implementations and verify identical targets, causal validity, and evaluation.} \begin{table}[t] -\caption{Confirmatory EasyCom results. Entry and INT false-positive rates are -measured at matched recall $R=0.8$. E/I Lat. gives signed median entry/INT -latency in seconds; values closer to zero are better.} +\caption{Confirmatory EasyCom results. Entry and INT false-positive rates use +the nominal 0.2-s forecast bin at matched recall $R=0.8$; E/I AP averages each +event family over all ten fixed horizons.} \label{tab:main} \centering \small @@ -371,7 +387,7 @@ latency in seconds; values closer to zero are better.} \begin{tabular}{lccccc} \hline Method & 0.2-s mAP $\uparrow$ & Mean mAP $\uparrow$ -& Entry FPR@$R=.8$ $\downarrow$ & INT FPR@$R=.8$ $\downarrow$ & E/I Lat.\\ +& Entry FPR@$R=.8$ $\downarrow$ & INT FPR@$R=.8$ $\downarrow$ & E/I AP $\uparrow$\\ \hline EgoSpeak reproduction & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--/--}\\ Omni-3B frozen $G$ & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--/--}\\ @@ -392,13 +408,13 @@ The gain is largest at \todo{short prediction horizons / transition-adjacent frames / specify actual observation}, indicating that local audiovisual evidence primarily improves \todo{the interpretation supported by the data}. -\textbf{Committed behavior.} +\textbf{Annotated behavior.} At matched wearer-entry recall $R=0.8$, the proposed adapter reduces entry FPR from \todo{$X$} to \todo{$Y$}; at matched INT-yield recall, it changes INT FPR -from \todo{$A$} to \todo{$B$}. Its signed median entry/INT latencies are -\todo{$+0.XX/+0.XX$~s}. These recall-matched error rates and signed latencies -test whether fewer erroneous commits arise from better floor decisions rather -than indefinite waiting. +from \todo{$A$} to \todo{$B$}. Its ten-horizon mean entry/INT event APs are +\todo{$E/I$}. These recall-matched opportunity error rates test whether lower +scores on confusing floor situations are obtained without suppressing +legitimate entry or yield events. \textbf{Comparison with adaptation baselines.} Matched PEFT yields \todo{result}, whereas adding recent local audio alone @@ -544,7 +560,7 @@ for causal floor forecasting of an invisible egocentric wearer. The candidate method separates long-context Omni understanding from short-window local floor evidence without assuming persistent speaker identity. \todo{State the final learned-method improvement, matched-recall entry/INT false-positive rates, -signed latency, causal-control evidence, and main limitation. If no learned +fixed-horizon event AP, causal-control evidence, and main limitation. If no learned variant passes these tests, reframe the submission as an information audit or do not make the method claim.} diff --git a/test_evaluate_annotated_floor_events.py b/test_evaluate_annotated_floor_events.py new file mode 100644 index 0000000..b516258 --- /dev/null +++ b/test_evaluate_annotated_floor_events.py @@ -0,0 +1,270 @@ +from __future__ import annotations + +import argparse +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import numpy as np + +import evaluate_annotated_floor_events as evaluator +from evaluate_annotated_floor_events import ( + AnnotatedOpportunity, + _validate_protocol_args, + evaluate_family_horizon, + load_annotated_opportunities, + score_opportunity, + write_event_csv, +) +from evaluate_floor_state_changes import NUM_CLASSES, NUM_HORIZONS + + +def reviewed_row( + uid: str, + *, + family: str, + verdict: str, + anchor_frame: int, + subtype: str | None = None, + correction: float = 0.0, + region_start: float | None = None, + region_end: float | None = None, +) -> dict[str, object]: + event_type = None + if verdict == "POSITIVE": + event_type = "WEARER_ENTRY" if family == "entry" else "INT_YIELD" + anchor_seconds = anchor_frame / 20.0 + return { + "candidate_id": uid, + "candidate_uid": uid, + "session": "Session_4", + "candidate_family": family, + "review": { + "human_status": "REVIEWED", + "human_verdict": verdict, + "human_event_type": event_type, + "human_subtype": subtype, + "boundary_correction_seconds": correction, + }, + "candidate": { + "anchor_dense_frame": anchor_frame, + "anchor_dense_seconds": anchor_seconds, + "region_start_seconds": anchor_seconds if region_start is None else region_start, + "region_end_seconds": anchor_seconds if region_end is None else region_end, + "temporal_evidence": { + "behavior_anchor_frame": anchor_frame, + "behavior_anchor_seconds": anchor_seconds, + }, + }, + } + + +class AnnotatedFloorEventTests(unittest.TestCase): + def write_reviews(self, rows: list[dict[str, object]]) -> tuple[tempfile.TemporaryDirectory[str], Path]: + temporary = tempfile.TemporaryDirectory() + path = Path(temporary.name) / "reviews.jsonl" + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + return temporary, path + + def test_positive_stays_one_bin_and_boundary_correction_shifts_it(self) -> None: + temporary, path = self.write_reviews([ + reviewed_row( + "entry-positive", + family="entry", + verdict="POSITIVE", + anchor_frame=445, # 22.25 s -> EasyCom 5-Hz row 111 + correction=0.2, + ) + ]) + self.addCleanup(temporary.cleanup) + opportunities, diagnostics = load_annotated_opportunities([path], ["Session_4"]) + self.assertEqual(diagnostics["num_scoring_eligible"], 1) + self.assertEqual(opportunities[0].start_row, 112) + self.assertEqual(opportunities[0].end_row_exclusive, 113) + + def test_hard_negative_is_one_region_sample_with_max_score(self) -> None: + temporary, path = self.write_reviews([ + reviewed_row( + "entry-negative", + family="entry", + verdict="HARD_NEGATIVE", + subtype="ENTRY_OTHER_TO_OTHER", + anchor_frame=40, + region_start=2.0, + region_end=2.6, + ) + ]) + self.addCleanup(temporary.cleanup) + opportunities, _ = load_annotated_opportunities([path], ["Session_4"]) + self.assertEqual((opportunities[0].start_row, opportunities[0].end_row_exclusive), (10, 13)) + prediction = np.zeros((20, NUM_CLASSES, NUM_HORIZONS), dtype=np.float32) + prediction[10:13, 1, 0] = [0.2, 0.9, 0.4] + arrays = {"Session_4": (prediction, np.ones((20, NUM_HORIZONS), dtype=bool))} + self.assertAlmostEqual(score_opportunity(opportunities[0], arrays, 0), 0.9) + + def test_yield_uses_one_minus_wearer_probability(self) -> None: + opportunity = AnnotatedOpportunity( + uid="yield", + session="Session_4", + family="int", + positive=True, + subtype="INT_YIELD", + start_row=5, + end_row_exclusive=6, + anchor_seconds=1.0, + region_start_seconds=1.0, + region_end_seconds=1.0, + ) + prediction = np.zeros((10, NUM_CLASSES, NUM_HORIZONS), dtype=np.float32) + prediction[5, 1, 2] = 0.15 + arrays = {"Session_4": (prediction, np.ones((10, NUM_HORIZONS), dtype=bool))} + self.assertAlmostEqual(score_opportunity(opportunity, arrays, 2), 0.85) + + def test_partial_valid_hard_negative_region_is_omitted(self) -> None: + opportunity = AnnotatedOpportunity( + "negative", + "Session_4", + "entry", + False, + "ENTRY_OTHER_TO_OTHER", + 5, + 8, + 1.0, + 1.0, + 1.6, + ) + prediction = np.zeros((10, NUM_CLASSES, NUM_HORIZONS), dtype=np.float32) + valid = np.ones((10, NUM_HORIZONS), dtype=bool) + valid[6, 0] = False + self.assertIsNone( + score_opportunity(opportunity, {"Session_4": (prediction, valid)}, 0) + ) + + def test_event_ap_and_false_positive_use_one_row_per_opportunity(self) -> None: + opportunities = [ + AnnotatedOpportunity("p", "Session_4", "entry", True, "WEARER_ENTRY", 4, 5, 0.8, 0.8, 0.8), + AnnotatedOpportunity("n", "Session_4", "entry", False, "ENTRY_OTHER_TO_OTHER", 8, 10, 1.6, 1.6, 2.0), + ] + prediction = np.zeros((12, NUM_CLASSES, NUM_HORIZONS), dtype=np.float32) + prediction[4, 1, 0] = 0.9 + prediction[8:10, 1, 0] = [0.1, 0.2] + arrays = {"Session_4": (prediction, np.ones((12, NUM_HORIZONS), dtype=bool))} + metric, rows = evaluate_family_horizon(opportunities, arrays, "entry", 0, [0.8]) + self.assertEqual(len(rows), 2) + self.assertEqual(metric["num_positive"], 1) + self.assertEqual(metric["num_hard_negative"], 1) + self.assertAlmostEqual(metric["average_precision"], 1.0) + self.assertAlmostEqual(metric["operating_points"][0]["negative_false_positive_rate"], 0.0) + + def test_official_test_requires_disjoint_calibration(self) -> None: + args = argparse.Namespace( + sessions=["Session_1"], + calibration_reviews=None, + calibration_sessions=None, + calibration_run=None, + allow_in_sample_thresholds=False, + ) + with self.assertRaisesRegex(ValueError, "Official Sessions"): + _validate_protocol_args(args, [("model", Path("result"))]) + + def test_calibration_must_be_training_side_and_disjoint(self) -> None: + args = argparse.Namespace( + sessions=["Session_1"], + calibration_reviews=[Path("calibration.jsonl")], + calibration_sessions=["Session_1"], + calibration_run=None, + allow_in_sample_thresholds=False, + ) + with self.assertRaisesRegex(ValueError, "Sessions 4--12"): + _validate_protocol_args(args, [("model", Path("result"))]) + + def test_event_csv_keeps_one_scored_opportunity_row(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "events.csv" + row = { + "run": "model", + "uid": "entry-positive", + "session": "Session_4", + "family": "entry", + "positive": True, + "subtype": "WEARER_ENTRY", + "horizon_index": 1, + "nominal_horizon_seconds": 0.2, + "score": 0.9, + "anchor_seconds": 10.0, + "region_start_seconds": 10.0, + "region_end_seconds": 10.0, + "target_start_row": 50, + "target_end_row_exclusive": 51, + "source_start_row": 49, + "source_end_row_exclusive": 50, + } + write_event_csv(path, [row]) + text = path.read_text(encoding="utf-8") + self.assertIn("entry-positive", text) + self.assertEqual(len(text.splitlines()), 2) + + def test_main_writes_summary_metric_and_event_outputs(self) -> None: + rows = [ + reviewed_row("ep", family="entry", verdict="POSITIVE", anchor_frame=40), + reviewed_row( + "en", + family="entry", + verdict="HARD_NEGATIVE", + subtype="ENTRY_OTHER_TO_OTHER", + anchor_frame=80, + region_start=4.0, + region_end=4.4, + ), + reviewed_row("ip", family="int", verdict="POSITIVE", anchor_frame=120), + reviewed_row( + "in", + family="int", + verdict="HARD_NEGATIVE", + subtype="INT_OTHER_BACKCHANNEL", + anchor_frame=160, + region_start=8.0, + region_end=8.4, + ), + ] + temporary, reviews = self.write_reviews(rows) + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + run_dir = root / "run" + (run_dir / "pred").mkdir(parents=True) + (run_dir / "valid").mkdir() + prediction = np.full((50, NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32) + np.save(run_dir / "pred" / "Session_4.npy", prediction) + np.save( + run_dir / "valid" / "Session_4.npy", + np.ones((50, NUM_HORIZONS), dtype=bool), + ) + output = root / "summary.json" + args = argparse.Namespace( + reviews=[reviews], + run=[f"model={run_dir}"], + sessions=["Session_4"], + calibration_reviews=None, + calibration_run=None, + calibration_sessions=None, + matched_recalls=[0.8], + allow_in_sample_thresholds=False, + output=output, + csv_output=None, + event_csv_output=None, + print_event_examples=0, + ) + with mock.patch.object(evaluator, "parse_args", return_value=args): + evaluator.main() + self.assertTrue(output.is_file()) + self.assertTrue((root / "summary_metrics.csv").is_file()) + self.assertTrue((root / "summary_events.csv").is_file()) + payload = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(payload["protocol"], "human-adjudicated-fixed-horizon-floor-events-v1") + self.assertEqual(len(payload["runs"][0]["horizons"]), 20) + + +if __name__ == "__main__": + unittest.main()