#!/usr/bin/env python3 """Evaluate human-adjudicated wearer entry and INT-yield opportunities. The forecasting model emits dense EgoSpeak-compatible probabilities rather than event detections. At each nominal horizon, a confirmed positive event is therefore represented by exactly one score at its corrected behavior-anchor bin. A confirmed hard negative is represented by the maximum erroneous-action score inside its labeled half-open opportunity region. Positives are never expanded into tolerance windows. Thresholds for recall-matched false-positive rates must come from disjoint training-side calibration predictions when Sessions 1--3 are evaluated. """ from __future__ import annotations import argparse import csv from dataclasses import dataclass import json import math from pathlib import Path from typing import Any, Iterable, Literal import numpy as np from evaluate_floor_state_changes import ( CLASS_NAMES, NUM_CLASSES, NUM_HORIZONS, OFFICIAL_TRAIN_SESSIONS, OFFICIAL_TEST_SESSIONS, STEP_SECONDS, WEARER_INDEX, _operating_points, _safe_ap, parse_runs, validate_recalls, ) SOURCE_FPS = 20.0 Family = Literal["entry", "int"] FAMILIES: tuple[Family, ...] = ("entry", "int") @dataclass(frozen=True) class AnnotatedOpportunity: uid: str session: str family: Family positive: bool subtype: str start_row: int end_row_exclusive: int anchor_seconds: float region_start_seconds: float region_end_seconds: float def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--reviews", type=Path, action="append", required=True, help="Human review JSONL. Repeat to combine independently stored files.", ) parser.add_argument( "--run", action="append", required=True, metavar="NAME=RESULT_DIR", help="Inference directory containing pred/ and valid/. Repeat per method.", ) parser.add_argument( "--sessions", nargs="+", default=list(OFFICIAL_TEST_SESSIONS), ) parser.add_argument( "--calibration-reviews", type=Path, action="append", default=None, help="Disjoint training-side human review JSONL used for thresholds.", ) parser.add_argument( "--calibration-run", action="append", default=None, metavar="NAME=RESULT_DIR", help="Calibration inference directory for each --run name.", ) parser.add_argument( "--calibration-sessions", nargs="+", default=None, ) parser.add_argument( "--matched-recalls", nargs="+", type=float, default=(0.7, 0.8, 0.9), ) parser.add_argument( "--allow-in-sample-thresholds", action="store_true", help="Exploratory only: select thresholds on the evaluated events.", ) parser.add_argument( "--output", type=Path, default=Path("reports/easycom_annotated_floor_events/summary.json"), ) parser.add_argument( "--csv-output", type=Path, default=None, ) parser.add_argument( "--event-csv-output", type=Path, default=None, help="Optional scored-opportunity CSV (default: OUTPUT stem + _events.csv).", ) parser.add_argument( "--print-event-examples", type=int, default=0, metavar="N", help="Print up to N scored examples per family for the first run.", ) return parser.parse_args() def _read_jsonl(paths: Iterable[Path]) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] for path in paths: if not path.is_file(): raise FileNotFoundError(path) with path.open("r", encoding="utf-8") as handle: for line_number, raw_line in enumerate(handle, 1): line = raw_line.strip() if not line: continue try: row = json.loads(line) except json.JSONDecodeError as error: raise ValueError(f"Invalid JSON at {path}:{line_number}: {error}") from error if not isinstance(row, dict): raise ValueError(f"Expected object at {path}:{line_number}") rows.append(row) return rows def _integer_frame(value: Any, *, field: str, uid: str) -> int: if isinstance(value, bool) or not isinstance(value, (int, float)): raise ValueError(f"{uid}: {field} must be a numeric 20-Hz frame") rounded = int(round(float(value))) if not math.isclose(float(value), rounded, abs_tol=1e-6): raise ValueError(f"{uid}: {field} must be integral, got {value!r}") if rounded < 0: raise ValueError(f"{uid}: {field} must be non-negative") return rounded def _seconds(value: Any, *, field: str, uid: str) -> float: if isinstance(value, bool) or not isinstance(value, (int, float)): raise ValueError(f"{uid}: {field} must be numeric") result = float(value) if not math.isfinite(result) or result < 0: raise ValueError(f"{uid}: {field} must be finite and non-negative") return result def _frame_to_row(frame: int) -> int: """Use the same 20-Hz-to-5-Hz rasterization as EasyCom event derivation.""" return frame // int(round(SOURCE_FPS * STEP_SECONDS)) def _seconds_to_start_row(seconds: float) -> int: return int(math.floor(seconds / STEP_SECONDS + 1e-7)) def _seconds_to_end_row_exclusive(seconds: float) -> int: return int(math.ceil(seconds / STEP_SECONDS - 1e-7)) def _review_payload(row: dict[str, Any], uid: str) -> tuple[dict[str, Any], dict[str, Any]]: review = row.get("review") candidate = row.get("candidate") if not isinstance(review, dict) or not isinstance(candidate, dict): raise ValueError( f"{uid}: expected review_floor_events.py JSONL with nested review and candidate objects" ) if review.get("human_status") != "REVIEWED": raise ValueError(f"{uid}: human_status must be REVIEWED") return review, candidate def load_annotated_opportunities( paths: Iterable[Path], sessions: Iterable[str], ) -> tuple[list[AnnotatedOpportunity], dict[str, int]]: selected_sessions = set(sessions) opportunities: list[AnnotatedOpportunity] = [] seen: set[str] = set() excluded = 0 for row_number, row in enumerate(_read_jsonl(paths), 1): uid = str(row.get("candidate_id") or row.get("candidate_uid") or f"row-{row_number}") if uid in seen: raise ValueError(f"Duplicate reviewed candidate: {uid}") seen.add(uid) session = str(row.get("session", "")) if session not in selected_sessions: continue review, candidate = _review_payload(row, uid) verdict = review.get("human_verdict") if verdict == "EXCLUDE": excluded += 1 continue if verdict not in {"POSITIVE", "HARD_NEGATIVE"}: raise ValueError(f"{uid}: invalid human_verdict {verdict!r}") family_value = row.get("candidate_family") if family_value == "entry": family: Family = "entry" elif family_value == "int": family = "int" else: raise ValueError(f"{uid}: scoring row has invalid candidate_family {family_value!r}") positive = verdict == "POSITIVE" event_type = review.get("human_event_type") subtype = review.get("human_subtype") expected_event_type = "WEARER_ENTRY" if family == "entry" else "INT_YIELD" if positive: if event_type != expected_event_type or subtype is not None: raise ValueError(f"{uid}: positive {family} must be {expected_event_type}") subtype_text = expected_event_type else: prefix = "ENTRY_" if family == "entry" else "INT_" if not isinstance(subtype, str) or ( subtype != "OTHER_HARD_NEGATIVE" and not subtype.startswith(prefix) ): raise ValueError(f"{uid}: hard-negative subtype does not match {family}") subtype_text = subtype correction = review.get("boundary_correction_seconds") or 0.0 correction = float(correction) correction_rows = int(round(correction / STEP_SECONDS)) if not math.isclose(correction, correction_rows * STEP_SECONDS, abs_tol=1e-7): raise ValueError(f"{uid}: boundary correction is not on the 5-Hz grid") evidence = candidate.get("temporal_evidence") evidence = evidence if isinstance(evidence, dict) else {} if positive: raw_frame = evidence.get("behavior_anchor_frame", candidate.get("anchor_dense_frame")) frame = _integer_frame(raw_frame, field="behavior_anchor_frame", uid=uid) row_index = _frame_to_row(frame) + correction_rows if row_index < 0: raise ValueError(f"{uid}: corrected behavior anchor precedes the session") anchor_seconds = frame / SOURCE_FPS + correction start_row = row_index end_row_exclusive = row_index + 1 region_start = anchor_seconds region_end = anchor_seconds else: raw_start = _seconds( candidate.get("region_start_seconds", candidate.get("anchor_dense_seconds")), field="region_start_seconds", uid=uid, ) + correction raw_end = _seconds( candidate.get("region_end_seconds", candidate.get("anchor_dense_seconds")), field="region_end_seconds", uid=uid, ) + correction if raw_start < 0 or raw_end < raw_start: raise ValueError(f"{uid}: invalid corrected hard-negative region") start_row = _seconds_to_start_row(raw_start) end_row_exclusive = max(start_row + 1, _seconds_to_end_row_exclusive(raw_end)) anchor_seconds = _seconds( candidate.get("anchor_dense_seconds", raw_start - correction), field="anchor_dense_seconds", uid=uid, ) + correction region_start, region_end = raw_start, raw_end opportunities.append( AnnotatedOpportunity( uid=uid, session=session, family=family, positive=positive, subtype=subtype_text, start_row=start_row, end_row_exclusive=end_row_exclusive, anchor_seconds=anchor_seconds, region_start_seconds=region_start, region_end_seconds=region_end, ) ) diagnostics = { "num_review_rows": len(seen), "num_scoring_eligible": len(opportunities), "num_excluded": excluded, } return opportunities, diagnostics def load_session_predictions( run_dir: Path, sessions: Iterable[str], ) -> dict[str, tuple[np.ndarray, np.ndarray]]: result: dict[str, tuple[np.ndarray, np.ndarray]] = {} for session in sessions: pred_path = run_dir / "pred" / f"{session}.npy" valid_path = run_dir / "valid" / f"{session}.npy" if not pred_path.is_file(): raise FileNotFoundError(pred_path) if not valid_path.is_file(): raise FileNotFoundError(valid_path) prediction = np.asarray(np.load(pred_path, allow_pickle=False), dtype=np.float32) valid = np.asarray(np.load(valid_path, allow_pickle=False), dtype=bool) if prediction.ndim != 3 or prediction.shape[1:] != (NUM_CLASSES, NUM_HORIZONS): raise ValueError(f"Expected [T,{NUM_CLASSES},{NUM_HORIZONS}], got {prediction.shape}: {pred_path}") if valid.shape != (len(prediction), NUM_HORIZONS): raise ValueError(f"Expected {(len(prediction), NUM_HORIZONS)}, got {valid.shape}: {valid_path}") if not np.isfinite(prediction).all() or (prediction < -1e-6).any() or (prediction > 1 + 1e-6).any(): raise ValueError(f"Prediction must be finite and in [0,1]: {pred_path}") result[session] = (np.clip(prediction, 0.0, 1.0), valid) return result def score_opportunity( opportunity: AnnotatedOpportunity, arrays: dict[str, tuple[np.ndarray, np.ndarray]], horizon_index: int, ) -> float | None: prediction, valid = arrays[opportunity.session] start = opportunity.start_row end = opportunity.end_row_exclusive if start < 0 or start >= end or end > len(prediction): return None usable = valid[start:end, horizon_index] if not usable.all(): return None scores = prediction[start:end, WEARER_INDEX, horizon_index] if opportunity.family == "int": scores = 1.0 - scores return float(np.max(scores)) def evaluate_family_horizon( opportunities: list[AnnotatedOpportunity], arrays: dict[str, tuple[np.ndarray, np.ndarray]], family: Family, horizon_index: int, recalls: list[float], *, fixed_thresholds: dict[float, float | None] | None = None, threshold_source: str = "evaluated_events", ) -> tuple[dict[str, Any], list[dict[str, Any]]]: rows: list[dict[str, Any]] = [] invalid = 0 for opportunity in opportunities: if opportunity.family != family: continue score = score_opportunity(opportunity, arrays, horizon_index) if score is None: invalid += 1 continue rows.append( { "uid": opportunity.uid, "session": opportunity.session, "family": family, "positive": opportunity.positive, "subtype": opportunity.subtype, "score": score, "anchor_seconds": opportunity.anchor_seconds, "region_start_seconds": opportunity.region_start_seconds, "region_end_seconds": opportunity.region_end_seconds, "target_start_row": opportunity.start_row, "target_end_row_exclusive": opportunity.end_row_exclusive, "source_start_row": opportunity.start_row - horizon_index - 1, "source_end_row_exclusive": opportunity.end_row_exclusive - horizon_index - 1, "nominal_horizon_seconds": (horizon_index + 1) * STEP_SECONDS, } ) scores = np.asarray([row["score"] for row in rows], dtype=np.float64) positive = np.asarray([row["positive"] for row in rows], dtype=bool) negative = ~positive subtype_names = sorted({str(row["subtype"]) for row in rows if not row["positive"]}) extra_groups = { f"fpr_{name.lower()}": np.asarray( [not row["positive"] and row["subtype"] == name for row in rows], dtype=bool, ) for name in subtype_names } labels = positive.astype(np.int8) metric = { "family": family, "horizon_index": horizon_index + 1, "nominal_seconds": (horizon_index + 1) * STEP_SECONDS, "num_positive": int(positive.sum()), "num_hard_negative": int(negative.sum()), "num_invalid": invalid, "average_precision": _safe_ap(labels, scores), "operating_points": _operating_points( scores, positive, negative, recalls, extra_groups=extra_groups, fixed_thresholds=fixed_thresholds, threshold_source=threshold_source, ), } return metric, rows def thresholds_from_metric(metric: dict[str, Any]) -> dict[float, float | None]: return { float(item["requested_positive_recall"]): item["threshold"] for item in metric["operating_points"] } def _run_names(runs: list[tuple[str, Path]]) -> list[str]: return [name for name, _ in runs] def _validate_protocol_args(args: argparse.Namespace, runs: list[tuple[str, Path]]) -> tuple[list[tuple[str, Path]], bool]: official_test = bool(set(args.sessions) & set(OFFICIAL_TEST_SESSIONS)) if bool(args.calibration_reviews) != bool(args.calibration_sessions): raise ValueError( "--calibration-reviews and --calibration-sessions must be supplied together" ) has_calibration = bool(args.calibration_reviews and args.calibration_sessions) if has_calibration: unknown = sorted(set(args.calibration_sessions) - set(OFFICIAL_TRAIN_SESSIONS)) if unknown: raise ValueError( f"Calibration must use EasyCom Sessions 4--12, got {unknown}" ) overlap = sorted(set(args.sessions) & set(args.calibration_sessions)) if overlap: raise ValueError(f"Evaluation and calibration sessions overlap: {overlap}") if official_test and not has_calibration and not args.allow_in_sample_thresholds: raise ValueError( "Official Sessions 1--3 require --calibration-reviews and " "--calibration-sessions; use --allow-in-sample-thresholds only for exploration" ) if args.calibration_run: calibration_runs = parse_runs(args.calibration_run) if _run_names(calibration_runs) != _run_names(runs): raise ValueError("--calibration-run names/order must match --run") else: calibration_runs = runs return calibration_runs, has_calibration def write_csv(path: Path, run_results: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) fieldnames = [ "run", "family", "horizon_index", "nominal_seconds", "num_positive", "num_hard_negative", "num_invalid", "average_precision", "requested_recall", "threshold", "threshold_source", "achieved_recall", "false_positive_rate", "false_positive_count", ] with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() for run in run_results: for metric in run["horizons"]: for point in metric["operating_points"]: writer.writerow( { "run": run["name"], "family": metric["family"], "horizon_index": metric["horizon_index"], "nominal_seconds": metric["nominal_seconds"], "num_positive": metric["num_positive"], "num_hard_negative": metric["num_hard_negative"], "num_invalid": metric["num_invalid"], "average_precision": metric["average_precision"], "requested_recall": point["requested_positive_recall"], "threshold": point["threshold"], "threshold_source": point["threshold_source"], "achieved_recall": point["achieved_positive_recall"], "false_positive_rate": point["negative_false_positive_rate"], "false_positive_count": point["negative_false_positive_count"], } ) def write_event_csv(path: Path, rows: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) fieldnames = [ "run", "uid", "session", "family", "positive", "subtype", "horizon_index", "nominal_horizon_seconds", "score", "anchor_seconds", "region_start_seconds", "region_end_seconds", "target_start_row", "target_end_row_exclusive", "source_start_row", "source_end_row_exclusive", ] with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() writer.writerows(rows) def print_examples(rows: list[dict[str, Any]], limit: int) -> None: if limit <= 0: return print("\nAnnotated fixed-horizon examples (first run, 0.2-s nominal horizon)") for family in FAMILIES: print(f"{family}:") shown = 0 for row in rows: if row["family"] != family: continue label = "POS" if row["positive"] else "HN" print( f" {label} {row['session']} {row['uid']} " f"target_rows=[{row['target_start_row']},{row['target_end_row_exclusive']}) " f"source_rows=[{row['source_start_row']},{row['source_end_row_exclusive']}) " f"region={row['region_start_seconds']:.2f}--{row['region_end_seconds']:.2f}s " f"score={row['score']:.4f} subtype={row['subtype']}" ) shown += 1 if shown >= limit: break def main() -> None: args = parse_args() recalls = validate_recalls(args.matched_recalls) runs = parse_runs(args.run) calibration_runs, has_calibration = _validate_protocol_args(args, runs) opportunities, review_diagnostics = load_annotated_opportunities(args.reviews, args.sessions) if not opportunities: raise ValueError("No scoring-eligible reviewed opportunities for evaluated sessions") calibration_opportunities: list[AnnotatedOpportunity] | None = None calibration_diagnostics: dict[str, int] | None = None if has_calibration: calibration_opportunities, calibration_diagnostics = load_annotated_opportunities( args.calibration_reviews, args.calibration_sessions, ) if not calibration_opportunities: raise ValueError("No scoring-eligible calibration opportunities") all_results: list[dict[str, Any]] = [] all_event_rows: list[dict[str, Any]] = [] first_rows: list[dict[str, Any]] = [] for run_index, ((name, run_dir), (_, calibration_dir)) in enumerate(zip(runs, calibration_runs)): arrays = load_session_predictions(run_dir, args.sessions) calibration_arrays = ( None if calibration_opportunities is None else load_session_predictions(calibration_dir, args.calibration_sessions) ) horizons: list[dict[str, Any]] = [] event_rows: list[dict[str, Any]] = [] for horizon_index in range(NUM_HORIZONS): for family in FAMILIES: fixed_thresholds = None threshold_source = "evaluated_events_exploratory" if calibration_opportunities is not None and calibration_arrays is not None: calibration_metric, _ = evaluate_family_horizon( calibration_opportunities, calibration_arrays, family, horizon_index, recalls, threshold_source="training_side_calibration", ) fixed_thresholds = thresholds_from_metric(calibration_metric) threshold_source = "training_side_calibration" metric, rows = evaluate_family_horizon( opportunities, arrays, family, horizon_index, recalls, fixed_thresholds=fixed_thresholds, threshold_source=threshold_source, ) horizons.append(metric) event_rows.extend( {"run": name, **row, "horizon_index": horizon_index + 1} for row in rows ) if run_index == 0 and horizon_index == 0: first_rows.extend(rows) all_event_rows.extend(event_rows) all_results.append( { "name": name, "result_dir": str(run_dir), "horizons": horizons, "mean_entry_ap": float(np.mean([ item["average_precision"] for item in horizons if item["family"] == "entry" and item["average_precision"] is not None ])) if any(item["family"] == "entry" and item["average_precision"] is not None for item in horizons) else None, "mean_int_ap": float(np.mean([ item["average_precision"] for item in horizons if item["family"] == "int" and item["average_precision"] is not None ])) if any(item["family"] == "int" and item["average_precision"] is not None for item in horizons) else None, } ) payload = { "schema_version": 1, "protocol": "human-adjudicated-fixed-horizon-floor-events-v1", "semantic_scope": ( "Human-confirmed wearer entry, INT-yield, and hard-negative opportunities; " "positive events remain single 5-Hz target bins." ), "prediction_layout": "[target_time, class, future_horizon]", "class_names": list(CLASS_NAMES), "sessions": list(args.sessions), "calibration_sessions": args.calibration_sessions, "matched_recalls": recalls, "positive_score": "one corrected behavior-anchor bin at each nominal horizon", "hard_negative_score": "maximum erroneous-action score over the labeled half-open region at each nominal horizon", "review_diagnostics": review_diagnostics, "calibration_review_diagnostics": calibration_diagnostics, "runs": all_results, } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") csv_path = args.csv_output or args.output.with_name(args.output.stem + "_metrics.csv") event_csv_path = args.event_csv_output or args.output.with_name( args.output.stem + "_events.csv" ) write_csv(csv_path, all_results) write_event_csv(event_csv_path, all_event_rows) print_examples(first_rows, args.print_event_examples) print(f"Saved {args.output}") print(f"Saved {csv_path}") print(f"Saved {event_csv_path}") if __name__ == "__main__": main()