Add provisional LLM review materializer

evaluate_annotated_floor_events.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Evaluate human-adjudicated wearer entry and INT-yield opportunities.
+"""Evaluate reviewed wearer entry and INT-yield opportunities.
 
 The forecasting model emits dense EgoSpeak-compatible probabilities rather
 than event detections.  At each nominal horizon, a confirmed positive event is
@@ -42,6 +42,7 @@ from evaluate_floor_state_changes import (
 SOURCE_FPS = 20.0
 Family = Literal["entry", "int"]
 FAMILIES: tuple[Family, ...] = ("entry", "int")
+PROVISIONAL_LLM_SOURCE = "PROVISIONAL_LLM_AUTO_ACCEPT"
 
 
 @dataclass(frozen=True)
@@ -65,7 +66,7 @@ def parse_args() -> argparse.Namespace:
         type=Path,
         action="append",
         required=True,
-        help="Human review JSONL. Repeat to combine independently stored files.",
+        help="Review JSONL. Repeat to combine independently stored files.",
     )
     parser.add_argument(
         "--run",
@@ -84,7 +85,7 @@ def parse_args() -> argparse.Namespace:
         type=Path,
         action="append",
         default=None,
-        help="Disjoint training-side human review JSONL used for thresholds.",
+        help="Disjoint training-side review JSONL used for thresholds.",
     )
     parser.add_argument(
         "--calibration-run",
@@ -132,6 +133,14 @@ def parse_args() -> argparse.Namespace:
         metavar="N",
         help="Print up to N scored examples per family for the first run.",
     )
+    parser.add_argument(
+        "--allow-provisional-llm-reviews",
+        action="store_true",
+        help=(
+            "Explicitly allow auto-accepted LLM labels. Outputs remain marked "
+            "provisional and must not be reported as human-adjudicated ground truth."
+        ),
+    )
     return parser.parse_args()
 
 
@@ -189,7 +198,9 @@ def _seconds_to_end_row_exclusive(seconds: float) -> int:
     return int(math.ceil(seconds / STEP_SECONDS - 1e-7))
 
 
-def _review_payload(row: dict[str, Any], uid: str) -> tuple[dict[str, Any], dict[str, Any]]:
+def _review_payload(
+    row: dict[str, Any], uid: str, *, allow_provisional_llm: bool
+) -> tuple[dict[str, Any], dict[str, Any]]:
     review = row.get("review")
     candidate = row.get("candidate")
     if not isinstance(review, dict) or not isinstance(candidate, dict):
@@ -198,17 +209,26 @@ def _review_payload(row: dict[str, Any], uid: str) -> tuple[dict[str, Any], dict
         )
     if review.get("human_status") != "REVIEWED":
         raise ValueError(f"{uid}: human_status must be REVIEWED")
+    review_source = row.get("review_source", review.get("review_source"))
+    if review_source == PROVISIONAL_LLM_SOURCE and not allow_provisional_llm:
+        raise ValueError(
+            f"{uid}: provisional LLM auto-accept requires "
+            "--allow-provisional-llm-reviews"
+        )
     return review, candidate
 
 
 def load_annotated_opportunities(
     paths: Iterable[Path],
     sessions: Iterable[str],
+    *,
+    allow_provisional_llm: bool = False,
 ) -> tuple[list[AnnotatedOpportunity], dict[str, int]]:
     selected_sessions = set(sessions)
     opportunities: list[AnnotatedOpportunity] = []
     seen: set[str] = set()
     excluded = 0
+    provisional_llm = 0
 
     for row_number, row in enumerate(_read_jsonl(paths), 1):
         uid = str(row.get("candidate_id") or row.get("candidate_uid") or f"row-{row_number}")
@@ -218,7 +238,12 @@ def load_annotated_opportunities(
         session = str(row.get("session", ""))
         if session not in selected_sessions:
             continue
-        review, candidate = _review_payload(row, uid)
+        review, candidate = _review_payload(
+            row, uid, allow_provisional_llm=allow_provisional_llm
+        )
+        review_source = row.get("review_source", review.get("review_source"))
+        if review_source == PROVISIONAL_LLM_SOURCE:
+            provisional_llm += 1
         verdict = review.get("human_verdict")
         if verdict == "EXCLUDE":
             excluded += 1
@@ -310,6 +335,7 @@ def load_annotated_opportunities(
         "num_review_rows": len(seen),
         "num_scoring_eligible": len(opportunities),
         "num_excluded": excluded,
+        "num_provisional_llm": provisional_llm,
     }
     return opportunities, diagnostics
 
@@ -558,7 +584,12 @@ def main() -> None:
     runs = parse_runs(args.run)
     calibration_runs, has_calibration = _validate_protocol_args(args, runs)
 
-    opportunities, review_diagnostics = load_annotated_opportunities(args.reviews, args.sessions)
+    allow_provisional = bool(getattr(args, "allow_provisional_llm_reviews", False))
+    opportunities, review_diagnostics = load_annotated_opportunities(
+        args.reviews,
+        args.sessions,
+        allow_provisional_llm=allow_provisional,
+    )
     if not opportunities:
         raise ValueError("No scoring-eligible reviewed opportunities for evaluated sessions")
 
@@ -568,6 +599,7 @@ def main() -> None:
         calibration_opportunities, calibration_diagnostics = load_annotated_opportunities(
             args.calibration_reviews,
             args.calibration_sessions,
+            allow_provisional_llm=allow_provisional,
         )
         if not calibration_opportunities:
             raise ValueError("No scoring-eligible calibration opportunities")
@@ -632,13 +664,25 @@ def main() -> None:
             }
         )
 
+    provisional_count = review_diagnostics["num_provisional_llm"] + (
+        0 if calibration_diagnostics is None
+        else calibration_diagnostics["num_provisional_llm"]
+    )
     payload = {
         "schema_version": 1,
-        "protocol": "human-adjudicated-fixed-horizon-floor-events-v1",
-        "semantic_scope": (
-            "Human-confirmed wearer entry, INT-yield, and hard-negative opportunities; "
-            "positive events remain single 5-Hz target bins."
+        "protocol": (
+            "provisional-llm-fixed-horizon-floor-events-v1"
+            if provisional_count
+            else "human-adjudicated-fixed-horizon-floor-events-v1"
         ),
+        "semantic_scope": (
+            "Provisional LLM auto-accepted wearer entry, INT-yield, and hard-negative "
+            "opportunities; no human review is implied."
+            if provisional_count
+            else "Human-confirmed wearer entry, INT-yield, and hard-negative "
+            "opportunities; positive events remain single 5-Hz target bins."
+        ),
+        "provisional_llm": bool(provisional_count),
         "prediction_layout": "[target_time, class, future_horizon]",
         "class_names": list(CLASS_NAMES),
         "sessions": list(args.sessions),