Add annotated fixed-horizon floor evaluation
This commit is contained in:
Xin Wang
668
evaluate_annotated_floor_events.py
Normal file
668
evaluate_annotated_floor_events.py
Normal file
@@ -0,0 +1,668 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate human-adjudicated wearer entry and INT-yield opportunities.
|
||||
|
||||
The forecasting model emits dense EgoSpeak-compatible probabilities rather
|
||||
than event detections. At each nominal horizon, a confirmed positive event is
|
||||
therefore represented by exactly one score at its corrected behavior-anchor
|
||||
bin. A confirmed hard negative is represented by the maximum erroneous-action
|
||||
score inside its labeled half-open opportunity region. Positives are never
|
||||
expanded into tolerance windows.
|
||||
|
||||
Thresholds for recall-matched false-positive rates must come from disjoint
|
||||
training-side calibration predictions when Sessions 1--3 are evaluated.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Literal
|
||||
|
||||
import numpy as np
|
||||
|
||||
from evaluate_floor_state_changes import (
|
||||
CLASS_NAMES,
|
||||
NUM_CLASSES,
|
||||
NUM_HORIZONS,
|
||||
OFFICIAL_TRAIN_SESSIONS,
|
||||
OFFICIAL_TEST_SESSIONS,
|
||||
STEP_SECONDS,
|
||||
WEARER_INDEX,
|
||||
_operating_points,
|
||||
_safe_ap,
|
||||
parse_runs,
|
||||
validate_recalls,
|
||||
)
|
||||
|
||||
|
||||
SOURCE_FPS = 20.0
|
||||
Family = Literal["entry", "int"]
|
||||
FAMILIES: tuple[Family, ...] = ("entry", "int")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AnnotatedOpportunity:
|
||||
uid: str
|
||||
session: str
|
||||
family: Family
|
||||
positive: bool
|
||||
subtype: str
|
||||
start_row: int
|
||||
end_row_exclusive: int
|
||||
anchor_seconds: float
|
||||
region_start_seconds: float
|
||||
region_end_seconds: float
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--reviews",
|
||||
type=Path,
|
||||
action="append",
|
||||
required=True,
|
||||
help="Human review JSONL. Repeat to combine independently stored files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--run",
|
||||
action="append",
|
||||
required=True,
|
||||
metavar="NAME=RESULT_DIR",
|
||||
help="Inference directory containing pred/ and valid/. Repeat per method.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sessions",
|
||||
nargs="+",
|
||||
default=list(OFFICIAL_TEST_SESSIONS),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--calibration-reviews",
|
||||
type=Path,
|
||||
action="append",
|
||||
default=None,
|
||||
help="Disjoint training-side human review JSONL used for thresholds.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--calibration-run",
|
||||
action="append",
|
||||
default=None,
|
||||
metavar="NAME=RESULT_DIR",
|
||||
help="Calibration inference directory for each --run name.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--calibration-sessions",
|
||||
nargs="+",
|
||||
default=None,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--matched-recalls",
|
||||
nargs="+",
|
||||
type=float,
|
||||
default=(0.7, 0.8, 0.9),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-in-sample-thresholds",
|
||||
action="store_true",
|
||||
help="Exploratory only: select thresholds on the evaluated events.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=Path("reports/easycom_annotated_floor_events/summary.json"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--csv-output",
|
||||
type=Path,
|
||||
default=None,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--event-csv-output",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional scored-opportunity CSV (default: OUTPUT stem + _events.csv).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--print-event-examples",
|
||||
type=int,
|
||||
default=0,
|
||||
metavar="N",
|
||||
help="Print up to N scored examples per family for the first run.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _read_jsonl(paths: Iterable[Path]) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for path in paths:
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(path)
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
for line_number, raw_line in enumerate(handle, 1):
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError(f"Invalid JSON at {path}:{line_number}: {error}") from error
|
||||
if not isinstance(row, dict):
|
||||
raise ValueError(f"Expected object at {path}:{line_number}")
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _integer_frame(value: Any, *, field: str, uid: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError(f"{uid}: {field} must be a numeric 20-Hz frame")
|
||||
rounded = int(round(float(value)))
|
||||
if not math.isclose(float(value), rounded, abs_tol=1e-6):
|
||||
raise ValueError(f"{uid}: {field} must be integral, got {value!r}")
|
||||
if rounded < 0:
|
||||
raise ValueError(f"{uid}: {field} must be non-negative")
|
||||
return rounded
|
||||
|
||||
|
||||
def _seconds(value: Any, *, field: str, uid: str) -> float:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError(f"{uid}: {field} must be numeric")
|
||||
result = float(value)
|
||||
if not math.isfinite(result) or result < 0:
|
||||
raise ValueError(f"{uid}: {field} must be finite and non-negative")
|
||||
return result
|
||||
|
||||
|
||||
def _frame_to_row(frame: int) -> int:
|
||||
"""Use the same 20-Hz-to-5-Hz rasterization as EasyCom event derivation."""
|
||||
|
||||
return frame // int(round(SOURCE_FPS * STEP_SECONDS))
|
||||
|
||||
|
||||
def _seconds_to_start_row(seconds: float) -> int:
|
||||
return int(math.floor(seconds / STEP_SECONDS + 1e-7))
|
||||
|
||||
|
||||
def _seconds_to_end_row_exclusive(seconds: float) -> int:
|
||||
return int(math.ceil(seconds / STEP_SECONDS - 1e-7))
|
||||
|
||||
|
||||
def _review_payload(row: dict[str, Any], uid: str) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
review = row.get("review")
|
||||
candidate = row.get("candidate")
|
||||
if not isinstance(review, dict) or not isinstance(candidate, dict):
|
||||
raise ValueError(
|
||||
f"{uid}: expected review_floor_events.py JSONL with nested review and candidate objects"
|
||||
)
|
||||
if review.get("human_status") != "REVIEWED":
|
||||
raise ValueError(f"{uid}: human_status must be REVIEWED")
|
||||
return review, candidate
|
||||
|
||||
|
||||
def load_annotated_opportunities(
|
||||
paths: Iterable[Path],
|
||||
sessions: Iterable[str],
|
||||
) -> tuple[list[AnnotatedOpportunity], dict[str, int]]:
|
||||
selected_sessions = set(sessions)
|
||||
opportunities: list[AnnotatedOpportunity] = []
|
||||
seen: set[str] = set()
|
||||
excluded = 0
|
||||
|
||||
for row_number, row in enumerate(_read_jsonl(paths), 1):
|
||||
uid = str(row.get("candidate_id") or row.get("candidate_uid") or f"row-{row_number}")
|
||||
if uid in seen:
|
||||
raise ValueError(f"Duplicate reviewed candidate: {uid}")
|
||||
seen.add(uid)
|
||||
session = str(row.get("session", ""))
|
||||
if session not in selected_sessions:
|
||||
continue
|
||||
review, candidate = _review_payload(row, uid)
|
||||
verdict = review.get("human_verdict")
|
||||
if verdict == "EXCLUDE":
|
||||
excluded += 1
|
||||
continue
|
||||
if verdict not in {"POSITIVE", "HARD_NEGATIVE"}:
|
||||
raise ValueError(f"{uid}: invalid human_verdict {verdict!r}")
|
||||
|
||||
family_value = row.get("candidate_family")
|
||||
if family_value == "entry":
|
||||
family: Family = "entry"
|
||||
elif family_value == "int":
|
||||
family = "int"
|
||||
else:
|
||||
raise ValueError(f"{uid}: scoring row has invalid candidate_family {family_value!r}")
|
||||
|
||||
positive = verdict == "POSITIVE"
|
||||
event_type = review.get("human_event_type")
|
||||
subtype = review.get("human_subtype")
|
||||
expected_event_type = "WEARER_ENTRY" if family == "entry" else "INT_YIELD"
|
||||
if positive:
|
||||
if event_type != expected_event_type or subtype is not None:
|
||||
raise ValueError(f"{uid}: positive {family} must be {expected_event_type}")
|
||||
subtype_text = expected_event_type
|
||||
else:
|
||||
prefix = "ENTRY_" if family == "entry" else "INT_"
|
||||
if not isinstance(subtype, str) or (
|
||||
subtype != "OTHER_HARD_NEGATIVE" and not subtype.startswith(prefix)
|
||||
):
|
||||
raise ValueError(f"{uid}: hard-negative subtype does not match {family}")
|
||||
subtype_text = subtype
|
||||
|
||||
correction = review.get("boundary_correction_seconds") or 0.0
|
||||
correction = float(correction)
|
||||
correction_rows = int(round(correction / STEP_SECONDS))
|
||||
if not math.isclose(correction, correction_rows * STEP_SECONDS, abs_tol=1e-7):
|
||||
raise ValueError(f"{uid}: boundary correction is not on the 5-Hz grid")
|
||||
|
||||
evidence = candidate.get("temporal_evidence")
|
||||
evidence = evidence if isinstance(evidence, dict) else {}
|
||||
if positive:
|
||||
raw_frame = evidence.get("behavior_anchor_frame", candidate.get("anchor_dense_frame"))
|
||||
frame = _integer_frame(raw_frame, field="behavior_anchor_frame", uid=uid)
|
||||
row_index = _frame_to_row(frame) + correction_rows
|
||||
if row_index < 0:
|
||||
raise ValueError(f"{uid}: corrected behavior anchor precedes the session")
|
||||
anchor_seconds = frame / SOURCE_FPS + correction
|
||||
start_row = row_index
|
||||
end_row_exclusive = row_index + 1
|
||||
region_start = anchor_seconds
|
||||
region_end = anchor_seconds
|
||||
else:
|
||||
raw_start = _seconds(
|
||||
candidate.get("region_start_seconds", candidate.get("anchor_dense_seconds")),
|
||||
field="region_start_seconds",
|
||||
uid=uid,
|
||||
) + correction
|
||||
raw_end = _seconds(
|
||||
candidate.get("region_end_seconds", candidate.get("anchor_dense_seconds")),
|
||||
field="region_end_seconds",
|
||||
uid=uid,
|
||||
) + correction
|
||||
if raw_start < 0 or raw_end < raw_start:
|
||||
raise ValueError(f"{uid}: invalid corrected hard-negative region")
|
||||
start_row = _seconds_to_start_row(raw_start)
|
||||
end_row_exclusive = max(start_row + 1, _seconds_to_end_row_exclusive(raw_end))
|
||||
anchor_seconds = _seconds(
|
||||
candidate.get("anchor_dense_seconds", raw_start - correction),
|
||||
field="anchor_dense_seconds",
|
||||
uid=uid,
|
||||
) + correction
|
||||
region_start, region_end = raw_start, raw_end
|
||||
|
||||
opportunities.append(
|
||||
AnnotatedOpportunity(
|
||||
uid=uid,
|
||||
session=session,
|
||||
family=family,
|
||||
positive=positive,
|
||||
subtype=subtype_text,
|
||||
start_row=start_row,
|
||||
end_row_exclusive=end_row_exclusive,
|
||||
anchor_seconds=anchor_seconds,
|
||||
region_start_seconds=region_start,
|
||||
region_end_seconds=region_end,
|
||||
)
|
||||
)
|
||||
|
||||
diagnostics = {
|
||||
"num_review_rows": len(seen),
|
||||
"num_scoring_eligible": len(opportunities),
|
||||
"num_excluded": excluded,
|
||||
}
|
||||
return opportunities, diagnostics
|
||||
|
||||
|
||||
def load_session_predictions(
|
||||
run_dir: Path,
|
||||
sessions: Iterable[str],
|
||||
) -> dict[str, tuple[np.ndarray, np.ndarray]]:
|
||||
result: dict[str, tuple[np.ndarray, np.ndarray]] = {}
|
||||
for session in sessions:
|
||||
pred_path = run_dir / "pred" / f"{session}.npy"
|
||||
valid_path = run_dir / "valid" / f"{session}.npy"
|
||||
if not pred_path.is_file():
|
||||
raise FileNotFoundError(pred_path)
|
||||
if not valid_path.is_file():
|
||||
raise FileNotFoundError(valid_path)
|
||||
prediction = np.asarray(np.load(pred_path, allow_pickle=False), dtype=np.float32)
|
||||
valid = np.asarray(np.load(valid_path, allow_pickle=False), dtype=bool)
|
||||
if prediction.ndim != 3 or prediction.shape[1:] != (NUM_CLASSES, NUM_HORIZONS):
|
||||
raise ValueError(f"Expected [T,{NUM_CLASSES},{NUM_HORIZONS}], got {prediction.shape}: {pred_path}")
|
||||
if valid.shape != (len(prediction), NUM_HORIZONS):
|
||||
raise ValueError(f"Expected {(len(prediction), NUM_HORIZONS)}, got {valid.shape}: {valid_path}")
|
||||
if not np.isfinite(prediction).all() or (prediction < -1e-6).any() or (prediction > 1 + 1e-6).any():
|
||||
raise ValueError(f"Prediction must be finite and in [0,1]: {pred_path}")
|
||||
result[session] = (np.clip(prediction, 0.0, 1.0), valid)
|
||||
return result
|
||||
|
||||
|
||||
def score_opportunity(
|
||||
opportunity: AnnotatedOpportunity,
|
||||
arrays: dict[str, tuple[np.ndarray, np.ndarray]],
|
||||
horizon_index: int,
|
||||
) -> float | None:
|
||||
prediction, valid = arrays[opportunity.session]
|
||||
start = opportunity.start_row
|
||||
end = opportunity.end_row_exclusive
|
||||
if start < 0 or start >= end or end > len(prediction):
|
||||
return None
|
||||
usable = valid[start:end, horizon_index]
|
||||
if not usable.all():
|
||||
return None
|
||||
scores = prediction[start:end, WEARER_INDEX, horizon_index]
|
||||
if opportunity.family == "int":
|
||||
scores = 1.0 - scores
|
||||
return float(np.max(scores))
|
||||
|
||||
|
||||
def evaluate_family_horizon(
|
||||
opportunities: list[AnnotatedOpportunity],
|
||||
arrays: dict[str, tuple[np.ndarray, np.ndarray]],
|
||||
family: Family,
|
||||
horizon_index: int,
|
||||
recalls: list[float],
|
||||
*,
|
||||
fixed_thresholds: dict[float, float | None] | None = None,
|
||||
threshold_source: str = "evaluated_events",
|
||||
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
invalid = 0
|
||||
for opportunity in opportunities:
|
||||
if opportunity.family != family:
|
||||
continue
|
||||
score = score_opportunity(opportunity, arrays, horizon_index)
|
||||
if score is None:
|
||||
invalid += 1
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"uid": opportunity.uid,
|
||||
"session": opportunity.session,
|
||||
"family": family,
|
||||
"positive": opportunity.positive,
|
||||
"subtype": opportunity.subtype,
|
||||
"score": score,
|
||||
"anchor_seconds": opportunity.anchor_seconds,
|
||||
"region_start_seconds": opportunity.region_start_seconds,
|
||||
"region_end_seconds": opportunity.region_end_seconds,
|
||||
"target_start_row": opportunity.start_row,
|
||||
"target_end_row_exclusive": opportunity.end_row_exclusive,
|
||||
"source_start_row": opportunity.start_row - horizon_index - 1,
|
||||
"source_end_row_exclusive": opportunity.end_row_exclusive - horizon_index - 1,
|
||||
"nominal_horizon_seconds": (horizon_index + 1) * STEP_SECONDS,
|
||||
}
|
||||
)
|
||||
|
||||
scores = np.asarray([row["score"] for row in rows], dtype=np.float64)
|
||||
positive = np.asarray([row["positive"] for row in rows], dtype=bool)
|
||||
negative = ~positive
|
||||
subtype_names = sorted({str(row["subtype"]) for row in rows if not row["positive"]})
|
||||
extra_groups = {
|
||||
f"fpr_{name.lower()}": np.asarray(
|
||||
[not row["positive"] and row["subtype"] == name for row in rows],
|
||||
dtype=bool,
|
||||
)
|
||||
for name in subtype_names
|
||||
}
|
||||
labels = positive.astype(np.int8)
|
||||
metric = {
|
||||
"family": family,
|
||||
"horizon_index": horizon_index + 1,
|
||||
"nominal_seconds": (horizon_index + 1) * STEP_SECONDS,
|
||||
"num_positive": int(positive.sum()),
|
||||
"num_hard_negative": int(negative.sum()),
|
||||
"num_invalid": invalid,
|
||||
"average_precision": _safe_ap(labels, scores),
|
||||
"operating_points": _operating_points(
|
||||
scores,
|
||||
positive,
|
||||
negative,
|
||||
recalls,
|
||||
extra_groups=extra_groups,
|
||||
fixed_thresholds=fixed_thresholds,
|
||||
threshold_source=threshold_source,
|
||||
),
|
||||
}
|
||||
return metric, rows
|
||||
|
||||
|
||||
def thresholds_from_metric(metric: dict[str, Any]) -> dict[float, float | None]:
|
||||
return {
|
||||
float(item["requested_positive_recall"]): item["threshold"]
|
||||
for item in metric["operating_points"]
|
||||
}
|
||||
|
||||
|
||||
def _run_names(runs: list[tuple[str, Path]]) -> list[str]:
|
||||
return [name for name, _ in runs]
|
||||
|
||||
|
||||
def _validate_protocol_args(args: argparse.Namespace, runs: list[tuple[str, Path]]) -> tuple[list[tuple[str, Path]], bool]:
|
||||
official_test = bool(set(args.sessions) & set(OFFICIAL_TEST_SESSIONS))
|
||||
if bool(args.calibration_reviews) != bool(args.calibration_sessions):
|
||||
raise ValueError(
|
||||
"--calibration-reviews and --calibration-sessions must be supplied together"
|
||||
)
|
||||
has_calibration = bool(args.calibration_reviews and args.calibration_sessions)
|
||||
if has_calibration:
|
||||
unknown = sorted(set(args.calibration_sessions) - set(OFFICIAL_TRAIN_SESSIONS))
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"Calibration must use EasyCom Sessions 4--12, got {unknown}"
|
||||
)
|
||||
overlap = sorted(set(args.sessions) & set(args.calibration_sessions))
|
||||
if overlap:
|
||||
raise ValueError(f"Evaluation and calibration sessions overlap: {overlap}")
|
||||
if official_test and not has_calibration and not args.allow_in_sample_thresholds:
|
||||
raise ValueError(
|
||||
"Official Sessions 1--3 require --calibration-reviews and "
|
||||
"--calibration-sessions; use --allow-in-sample-thresholds only for exploration"
|
||||
)
|
||||
if args.calibration_run:
|
||||
calibration_runs = parse_runs(args.calibration_run)
|
||||
if _run_names(calibration_runs) != _run_names(runs):
|
||||
raise ValueError("--calibration-run names/order must match --run")
|
||||
else:
|
||||
calibration_runs = runs
|
||||
return calibration_runs, has_calibration
|
||||
|
||||
|
||||
def write_csv(path: Path, run_results: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fieldnames = [
|
||||
"run", "family", "horizon_index", "nominal_seconds", "num_positive",
|
||||
"num_hard_negative", "num_invalid", "average_precision", "requested_recall",
|
||||
"threshold", "threshold_source", "achieved_recall", "false_positive_rate",
|
||||
"false_positive_count",
|
||||
]
|
||||
with path.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
for run in run_results:
|
||||
for metric in run["horizons"]:
|
||||
for point in metric["operating_points"]:
|
||||
writer.writerow(
|
||||
{
|
||||
"run": run["name"],
|
||||
"family": metric["family"],
|
||||
"horizon_index": metric["horizon_index"],
|
||||
"nominal_seconds": metric["nominal_seconds"],
|
||||
"num_positive": metric["num_positive"],
|
||||
"num_hard_negative": metric["num_hard_negative"],
|
||||
"num_invalid": metric["num_invalid"],
|
||||
"average_precision": metric["average_precision"],
|
||||
"requested_recall": point["requested_positive_recall"],
|
||||
"threshold": point["threshold"],
|
||||
"threshold_source": point["threshold_source"],
|
||||
"achieved_recall": point["achieved_positive_recall"],
|
||||
"false_positive_rate": point["negative_false_positive_rate"],
|
||||
"false_positive_count": point["negative_false_positive_count"],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def write_event_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fieldnames = [
|
||||
"run",
|
||||
"uid",
|
||||
"session",
|
||||
"family",
|
||||
"positive",
|
||||
"subtype",
|
||||
"horizon_index",
|
||||
"nominal_horizon_seconds",
|
||||
"score",
|
||||
"anchor_seconds",
|
||||
"region_start_seconds",
|
||||
"region_end_seconds",
|
||||
"target_start_row",
|
||||
"target_end_row_exclusive",
|
||||
"source_start_row",
|
||||
"source_end_row_exclusive",
|
||||
]
|
||||
with path.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def print_examples(rows: list[dict[str, Any]], limit: int) -> None:
|
||||
if limit <= 0:
|
||||
return
|
||||
print("\nAnnotated fixed-horizon examples (first run, 0.2-s nominal horizon)")
|
||||
for family in FAMILIES:
|
||||
print(f"{family}:")
|
||||
shown = 0
|
||||
for row in rows:
|
||||
if row["family"] != family:
|
||||
continue
|
||||
label = "POS" if row["positive"] else "HN"
|
||||
print(
|
||||
f" {label} {row['session']} {row['uid']} "
|
||||
f"target_rows=[{row['target_start_row']},{row['target_end_row_exclusive']}) "
|
||||
f"source_rows=[{row['source_start_row']},{row['source_end_row_exclusive']}) "
|
||||
f"region={row['region_start_seconds']:.2f}--{row['region_end_seconds']:.2f}s "
|
||||
f"score={row['score']:.4f} subtype={row['subtype']}"
|
||||
)
|
||||
shown += 1
|
||||
if shown >= limit:
|
||||
break
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
recalls = validate_recalls(args.matched_recalls)
|
||||
runs = parse_runs(args.run)
|
||||
calibration_runs, has_calibration = _validate_protocol_args(args, runs)
|
||||
|
||||
opportunities, review_diagnostics = load_annotated_opportunities(args.reviews, args.sessions)
|
||||
if not opportunities:
|
||||
raise ValueError("No scoring-eligible reviewed opportunities for evaluated sessions")
|
||||
|
||||
calibration_opportunities: list[AnnotatedOpportunity] | None = None
|
||||
calibration_diagnostics: dict[str, int] | None = None
|
||||
if has_calibration:
|
||||
calibration_opportunities, calibration_diagnostics = load_annotated_opportunities(
|
||||
args.calibration_reviews,
|
||||
args.calibration_sessions,
|
||||
)
|
||||
if not calibration_opportunities:
|
||||
raise ValueError("No scoring-eligible calibration opportunities")
|
||||
|
||||
all_results: list[dict[str, Any]] = []
|
||||
all_event_rows: list[dict[str, Any]] = []
|
||||
first_rows: list[dict[str, Any]] = []
|
||||
for run_index, ((name, run_dir), (_, calibration_dir)) in enumerate(zip(runs, calibration_runs)):
|
||||
arrays = load_session_predictions(run_dir, args.sessions)
|
||||
calibration_arrays = (
|
||||
None
|
||||
if calibration_opportunities is None
|
||||
else load_session_predictions(calibration_dir, args.calibration_sessions)
|
||||
)
|
||||
horizons: list[dict[str, Any]] = []
|
||||
event_rows: list[dict[str, Any]] = []
|
||||
for horizon_index in range(NUM_HORIZONS):
|
||||
for family in FAMILIES:
|
||||
fixed_thresholds = None
|
||||
threshold_source = "evaluated_events_exploratory"
|
||||
if calibration_opportunities is not None and calibration_arrays is not None:
|
||||
calibration_metric, _ = evaluate_family_horizon(
|
||||
calibration_opportunities,
|
||||
calibration_arrays,
|
||||
family,
|
||||
horizon_index,
|
||||
recalls,
|
||||
threshold_source="training_side_calibration",
|
||||
)
|
||||
fixed_thresholds = thresholds_from_metric(calibration_metric)
|
||||
threshold_source = "training_side_calibration"
|
||||
metric, rows = evaluate_family_horizon(
|
||||
opportunities,
|
||||
arrays,
|
||||
family,
|
||||
horizon_index,
|
||||
recalls,
|
||||
fixed_thresholds=fixed_thresholds,
|
||||
threshold_source=threshold_source,
|
||||
)
|
||||
horizons.append(metric)
|
||||
event_rows.extend(
|
||||
{"run": name, **row, "horizon_index": horizon_index + 1}
|
||||
for row in rows
|
||||
)
|
||||
if run_index == 0 and horizon_index == 0:
|
||||
first_rows.extend(rows)
|
||||
all_event_rows.extend(event_rows)
|
||||
all_results.append(
|
||||
{
|
||||
"name": name,
|
||||
"result_dir": str(run_dir),
|
||||
"horizons": horizons,
|
||||
"mean_entry_ap": float(np.mean([
|
||||
item["average_precision"] for item in horizons
|
||||
if item["family"] == "entry" and item["average_precision"] is not None
|
||||
])) if any(item["family"] == "entry" and item["average_precision"] is not None for item in horizons) else None,
|
||||
"mean_int_ap": float(np.mean([
|
||||
item["average_precision"] for item in horizons
|
||||
if item["family"] == "int" and item["average_precision"] is not None
|
||||
])) if any(item["family"] == "int" and item["average_precision"] is not None for item in horizons) else None,
|
||||
}
|
||||
)
|
||||
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"protocol": "human-adjudicated-fixed-horizon-floor-events-v1",
|
||||
"semantic_scope": (
|
||||
"Human-confirmed wearer entry, INT-yield, and hard-negative opportunities; "
|
||||
"positive events remain single 5-Hz target bins."
|
||||
),
|
||||
"prediction_layout": "[target_time, class, future_horizon]",
|
||||
"class_names": list(CLASS_NAMES),
|
||||
"sessions": list(args.sessions),
|
||||
"calibration_sessions": args.calibration_sessions,
|
||||
"matched_recalls": recalls,
|
||||
"positive_score": "one corrected behavior-anchor bin at each nominal horizon",
|
||||
"hard_negative_score": "maximum erroneous-action score over the labeled half-open region at each nominal horizon",
|
||||
"review_diagnostics": review_diagnostics,
|
||||
"calibration_review_diagnostics": calibration_diagnostics,
|
||||
"runs": all_results,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
csv_path = args.csv_output or args.output.with_name(args.output.stem + "_metrics.csv")
|
||||
event_csv_path = args.event_csv_output or args.output.with_name(
|
||||
args.output.stem + "_events.csv"
|
||||
)
|
||||
write_csv(csv_path, all_results)
|
||||
write_event_csv(event_csv_path, all_event_rows)
|
||||
print_examples(first_rows, args.print_event_examples)
|
||||
print(f"Saved {args.output}")
|
||||
print(f"Saved {csv_path}")
|
||||
print(f"Saved {event_csv_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user