#!/usr/bin/env python3 """Materialize unanimous LLM pre-labels as explicitly provisional review rows. The output is schema-compatible with ``review_floor_events.py`` and ``evaluate_annotated_floor_events.py``, but it is not evidence of human review. Every row carries machine-readable provenance stating that no human reviewed the label. This command is intended for pipeline smoke tests and preliminary analysis only. """ from __future__ import annotations import argparse from collections import Counter from datetime import datetime, timezone import json from pathlib import Path from typing import Any, Iterable from review_floor_events import ( REVIEW_SCHEMA_VERSION, ReviewStore, expand_candidate_paths, load_candidates, ) LABEL_SOURCE = "PROVISIONAL_LLM_AUTO_ACCEPT" def _confidence(value: Any) -> str: if isinstance(value, bool) or not isinstance(value, (int, float)): return "low" if value >= 0.8: return "high" if value >= 0.5: return "medium" return "low" def materialize( event_paths: Iterable[Path], output_path: Path, *, include_nonunanimous: bool = False, ) -> tuple[list[dict[str, Any]], dict[str, Any]]: candidate_paths = expand_candidate_paths(event_paths, output_path=output_path) candidates = load_candidates(candidate_paths) store = ReviewStore(output_path, candidates) records: list[dict[str, Any]] = [] skipped: Counter[str] = Counter() counts: Counter[str] = Counter() timestamp = datetime.now(timezone.utc).isoformat() for candidate in candidates: suggestion = candidate.payload.get("llm_suggestion") if not isinstance(suggestion, dict) or not suggestion.get("verdict"): if candidate.payload.get("llm_adjudication_failure"): skipped["pending_llm_failure"] += 1 else: skipped["missing_llm_suggestion"] += 1 continue if suggestion.get("unanimous") is not True and not include_nonunanimous: skipped["nonunanimous"] += 1 continue review = store._validated_review( candidate.candidate_id, { "human_action": "accept", "confidence": _confidence(suggestion.get("confidence")), "boundary_correction_seconds": 0, "reviewer": "", "notes": ( "Automatically accepted from an LLM pre-label for a provisional " "pipeline smoke test; no human reviewed this item." ), }, ) review["review_source"] = LABEL_SOURCE review["human_reviewed"] = False record = { "schema_version": REVIEW_SCHEMA_VERSION, "candidate_id": candidate.candidate_id, "candidate_uid": candidate.uid, "source_file": candidate.source_path.name, "source_row": candidate.source_row, "session": candidate.session, "split": candidate.split, "candidate_family": candidate.family, "task": candidate.task, "automatic_label": candidate.automatic_label, "automatic_protocol_label": candidate.payload.get("automatic_protocol_label"), "automatic_hard_negative_type": candidate.payload.get( "automatic_hard_negative_type" ), "ai_verdict": candidate.payload.get("ai_verdict"), "ai_event_type": candidate.payload.get("ai_event_type"), "ai_subtype": candidate.payload.get("ai_subtype"), "ai_exclude_reason": candidate.payload.get("ai_exclude_reason"), "ai_exclude_reason_group": candidate.payload.get("ai_exclude_reason_group"), "review_source": LABEL_SOURCE, "human_reviewed": False, "review": review, "updated_at": timestamp, "candidate": candidate.payload, } records.append(record) counts[f"session/{candidate.session}"] += 1 counts[f"family/{candidate.family}"] += 1 counts[f"verdict/{review['human_verdict']}"] += 1 if review.get("human_subtype"): counts[f"subtype/{review['human_subtype']}"] += 1 if review.get("exclude_reason_group"): counts[f"exclude_group/{review['exclude_reason_group']}"] += 1 store.reviews = {str(row["candidate_id"]): row for row in records} store._write_locked() summary = { "schema_version": 1, "label_source": LABEL_SOURCE, "human_reviewed": False, "warning": "Provisional LLM auto-accepts are not human-adjudicated ground truth.", "input_files": [str(path) for path in candidate_paths], "output": str(output_path), "num_input_candidates": len(candidates), "num_materialized": len(records), "num_skipped": sum(skipped.values()), "skipped": dict(sorted(skipped.items())), "counts": dict(sorted(counts.items())), "include_nonunanimous": include_nonunanimous, "created_at": timestamp, } summary_path = output_path.with_suffix(".summary.json") summary_path.write_text( json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) return records, summary def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--events", type=Path, nargs="+", required=True, help="LLM-enriched candidate JSONL files or candidate directories.", ) parser.add_argument("--output", type=Path, required=True) parser.add_argument( "--include-nonunanimous", action="store_true", help="Also accept pass-disagreement fallbacks; discouraged even for smoke tests.", ) parser.add_argument( "--acknowledge-provisional-llm-auto-accept", action="store_true", help="Required acknowledgement that the output is not human-reviewed ground truth.", ) parser.add_argument("--overwrite", action="store_true") return parser.parse_args() def main() -> None: args = parse_args() if not args.acknowledge_provisional_llm_auto_accept: raise PermissionError( "Pass --acknowledge-provisional-llm-auto-accept; generated rows are " "for provisional smoke tests, not human-reviewed ground truth." ) output = args.output.expanduser().resolve() if output.exists() and not args.overwrite: raise FileExistsError(f"Refusing to overwrite {output}; pass --overwrite") records, summary = materialize( args.events, output, include_nonunanimous=args.include_nonunanimous, ) print(f"Saved {len(records)} provisional review rows to {output}") print(f"CSV: {output.with_suffix('.csv')}") print(f"Summary: {output.with_suffix('.summary.json')}") print("Skipped:", summary["skipped"]) print("WARNING: no row in this output was human reviewed.") if __name__ == "__main__": main()