Update llm assisted annotation

adjudicate_easycom_eot_int_candidates.py

@@ -4,7 +4,7 @@
 The model receives the candidate source, structured participant timing and the
 transcript excerpt.  It cannot change any source timestamp.  Its output is
 stored as an auditable suggestion beside the untouched rule candidate and must
-still be accepted, relabeled, excluded, or sent to second review by a human.
+still be accepted, relabeled, or excluded by a human.
 """
 
 from __future__ import annotations
@@ -17,13 +17,16 @@ import os
 from pathlib import Path
 import random
 import re
+import sys
 import time
-from typing import Any, Sequence
+from typing import Any, Sequence, TextIO
 import urllib.error
 import urllib.request
 
+from prepare_easycom_eot_int_review import requires_semantic_adjudication
 
-PROMPT_VERSION = "easycom-wearer-centric-eot-int-prelabel-v2"
+
+PROMPT_VERSION = "easycom-wearer-centric-eot-int-prelabel-v6"
 DEFAULT_MODEL = "deepseek-v4-flash"
 DEFAULT_BASE_URL = "https://api.deepseek.com"
 VERDICTS = {"POSITIVE", "HARD_NEGATIVE", "NOT_APPLICABLE", "EXCLUDE", "UNSURE"}
@@ -38,6 +41,51 @@ HARD_NEGATIVE_TYPES = {
     "OTHER_HARD_NEGATIVE",
 }
 
+
+def _progress_line(
+    completed: int, total: int, *, label: str, elapsed_seconds: float, width: int = 26
+) -> str:
+    fraction = 1.0 if total == 0 else min(1.0, max(0.0, completed / total))
+    filled = round(width * fraction)
+    bar = "█" * filled + "░" * (width - filled)
+    minutes, seconds = divmod(max(0, round(elapsed_seconds)), 60)
+    return (
+        f"DeepSeek [{bar}] {completed}/{total} ({fraction * 100:5.1f}%) "
+        f"{label} · {minutes:02d}:{seconds:02d}"
+    )
+
+
+class _ProgressBar:
+    """Dependency-free terminal progress with readable redirected logs."""
+
+    def __init__(self, total: int, *, stream: TextIO | None = None) -> None:
+        self.total = total
+        self.stream = stream or sys.stderr
+        self.started = time.monotonic()
+        self.is_tty = bool(getattr(self.stream, "isatty", lambda: False)())
+        self.last_logged = -1
+
+    def update(self, completed: int, label: str, *, final: bool = False) -> None:
+        line = _progress_line(
+            completed,
+            self.total,
+            label=label,
+            elapsed_seconds=time.monotonic() - self.started,
+        )
+        if self.is_tty:
+            self.stream.write("\r" + line)
+            if final:
+                self.stream.write("\n")
+            self.stream.flush()
+            return
+        # Avoid producing one permanent log line per API batch in redirected jobs.
+        step = max(1, self.total // 10)
+        if final or self.last_logged < 0 or completed - self.last_logged >= step:
+            self.stream.write(line + "\n")
+            self.stream.flush()
+            self.last_logged = completed
+
+
 SYSTEM_PROMPT = """You conservatively pre-label candidate regions for a scientific study of wearer-relative conversational floor control.
 
 Evidence rules:
@@ -45,20 +93,27 @@ Evidence rules:
 2. Never alter or propose an event timestamp. Do not infer prosody, gaze, intention, or acoustic cues that were not supplied.
 3. A wearer onset is not automatically a floor take; a wearer offset is not automatically a yield; an other onset is not automatically a takeover. A pause is not automatically EOT, overlap is not automatically interruption, and words such as "yeah" are not automatically backchannels. Check who speaks sustainably afterward and who resumes.
 4. Use UNSURE when the evidence does not establish conversational function. Never force a label for coverage.
+5. Every supplied WEARER_ONSET has two immutable anchors: a non-wearer release opportunity and a later floor-level wearer entry. Do not reject it solely because response_delay_seconds is long. Decide whether the release remained available, whether the wearer contribution is a genuine response, or whether it is a semantic hard negative such as feedback.
+6. Every supplied WEARER_OFFSET has already passed both deterministic INT prerequisites: a new non-wearer onset occurs strictly inside the current wearer floor episode (t_w < t_o < t_y), and that same non-wearer interval crosses the wearer yield boundary within the recorded annotation tolerance. Decide only whether it is a genuine takeover or a semantic hard negative. Pre-existing other activity, post-completion other onset, within-turn wearer pauses, and completion-to-silence are controls outside this queue.
+7. Transcript event markers are evidence, not lexical turns. In EasyCom, a transcription consisting of [L] denotes laughter-only vocalization. Do not treat [L] as a sustained floor take, an EOT response, or an interruption takeover merely because its activity interval creates an onset, offset, or overlap candidate. Specifically:
+   - wearer [L] during a non-wearer's retained turn is normally HARD_NEGATIVE / EOT_WEARER_BACKCHANNEL, not EOT_RESPONSE;
+   - non-wearer [L] overlapping a wearer's retained turn is normally HARD_NEGATIVE / INT_OTHER_BACKCHANNEL, not INT_YIELD;
+   - wearer [L] ending while a non-wearer continues does not establish that the wearer held and yielded the floor.
+   Use UNSURE if the surrounding timing does not establish whose floor was retained. Do not EXCLUDE a valid laughter-only annotation merely because it is non-lexical, and do not infer that a visible smile occurred: [L] supplies laughter evidence only. Do not assign semantics to other bracketed markers unless the supplied context establishes them.
 
 Top-level verdicts:
 - POSITIVE: the system should commit.
 - HARD_NEGATIVE: a valid, interpretable region where the system should not commit.
 - NOT_APPLICABLE: a real speech transition outside the committed EOT/INT task, such as a wearer starting after long silence or naturally ending into silence.
 - EXCLUDE: the region cannot be scored reliably because of annotation corruption, a recording boundary, missing context, a non-speech artifact, timestamp corruption, or another unresolvable defect.
-- UNSURE: evidence is insufficient and a second human review is needed.
+- UNSURE: evidence is insufficient for a reliable automatic pre-label and needs human judgment.
 
 POSITIVE event_type must be exactly EOT_RESPONSE or INT_YIELD. EOT_RESPONSE means a non-wearer genuinely releases the floor and the wearer then takes it with a sustained response. INT_YIELD means a non-wearer genuinely takes over during wearer speech and the wearer relinquishes the floor.
 
 HARD_NEGATIVE subtype must be exactly one of EOT_MIDTURN_PAUSE, EOT_OTHER_TO_OTHER, EOT_WEARER_BACKCHANNEL, INT_OTHER_BACKCHANNEL, INT_NON_TAKEOVER_OVERLAP, INT_WEARER_HOLDS_FLOOR, or OTHER_HARD_NEGATIVE.
 
 Return exactly one JSON object with this shape and no markdown:
-{"items":[{"uid":"...","verdict":"UNSURE","event_type":null,"subtype":null,"confidence":0.25,"reason":"brief evidence-based sentence","needs_human_review":true}]}
+{"items":[{"uid":"...","verdict":"UNSURE","event_type":null,"subtype":null,"confidence":0.25,"reason":"brief evidence-based sentence"}]}
 Return every supplied uid exactly once. Copy uid byte-for-byte and never add a backslash before an underscore. confidence must be a number from 0 to 1."""
 
 
@@ -171,7 +226,6 @@ def validate_decisions(payload: Any, expected_uids: Sequence[str]) -> list[dict[
         subtype = item.get("subtype")
         confidence = item.get("confidence")
         reason = item.get("reason")
-        needs_human_review = item.get("needs_human_review")
         if uid not in expected or uid in decisions:
             raise ValueError(f"Unexpected or duplicate uid: {raw_uid!r}")
         if verdict not in VERDICTS:
@@ -188,8 +242,6 @@ def validate_decisions(payload: Any, expected_uids: Sequence[str]) -> list[dict[
             raise ValueError(f"confidence must be in [0, 1] for {uid}")
         if not isinstance(reason, str) or not reason.strip():
             raise ValueError(f"Missing reason for {uid}")
-        if not isinstance(needs_human_review, bool):
-            raise ValueError(f"needs_human_review must be boolean for {uid}")
         decisions[str(uid)] = {
             "uid": str(uid),
             "verdict": str(verdict),
@@ -197,7 +249,10 @@ def validate_decisions(payload: Any, expected_uids: Sequence[str]) -> list[dict[
             "subtype": subtype,
             "confidence": float(confidence),
             "reason": reason.strip(),
-            "needs_human_review": needs_human_review,
+            # Every AI decision enters the human queue. This is deliberately
+            # local rather than model-generated, so harmless JSON type drift
+            # cannot invalidate and retry an otherwise valid batch.
+            "needs_human_review": True,
         }
     if set(decisions) != expected:
         raise ValueError(f"Response uid mismatch; missing={sorted(expected - set(decisions))}")
@@ -276,28 +331,47 @@ def add_suggestions(
     usage: Counter[str] = Counter()
     if batch_size <= 0:
         raise ValueError("batch_size must be positive")
-    for pass_index in range(passes):
-        for start in range(0, len(rows), batch_size):
-            batch = rows[start:start + batch_size]
-            transported, _ = _transport_rows(batch)
-            prompt_hash = hashlib.sha256(
-                (SYSTEM_PROMPT + "\n" + build_user_prompt(transported, pass_index)).encode("utf-8")
-            ).hexdigest()
-            decisions, pass_usage = _post(
-                rows=batch,
-                pass_index=pass_index,
-                api_key=api_key,
-                model=model,
-                base_url=base_url,
-                timeout_seconds=timeout_seconds,
-                max_retries=max_retries,
-            )
-            for decision in decisions:
-                by_uid[decision["uid"]].append(decision)
-                prompt_hashes[decision["uid"]].append(prompt_hash)
-            for key, value in pass_usage.items():
-                if isinstance(value, int):
-                    usage[key] += value
+    total_work = len(rows) * passes
+    batches_per_pass = (len(rows) + batch_size - 1) // batch_size
+    completed_work = 0
+    progress = _ProgressBar(total_work)
+    progress.update(0, f"pass 1/{passes} · waiting")
+    try:
+        for pass_index in range(passes):
+            for batch_index, start in enumerate(range(0, len(rows), batch_size), 1):
+                batch = rows[start:start + batch_size]
+                progress.update(
+                    completed_work,
+                    f"pass {pass_index + 1}/{passes} · batch {batch_index}/{batches_per_pass}",
+                )
+                transported, _ = _transport_rows(batch)
+                prompt_hash = hashlib.sha256(
+                    (SYSTEM_PROMPT + "\n" + build_user_prompt(transported, pass_index)).encode("utf-8")
+                ).hexdigest()
+                decisions, pass_usage = _post(
+                    rows=batch,
+                    pass_index=pass_index,
+                    api_key=api_key,
+                    model=model,
+                    base_url=base_url,
+                    timeout_seconds=timeout_seconds,
+                    max_retries=max_retries,
+                )
+                for decision in decisions:
+                    by_uid[decision["uid"]].append(decision)
+                    prompt_hashes[decision["uid"]].append(prompt_hash)
+                for key, value in pass_usage.items():
+                    if isinstance(value, int):
+                        usage[key] += value
+                completed_work += len(batch)
+                progress.update(
+                    completed_work,
+                    f"pass {pass_index + 1}/{passes} · batch {batch_index}/{batches_per_pass}",
+                )
+    except Exception:
+        progress.update(completed_work, "failed", final=True)
+        raise
+    progress.update(completed_work, "complete", final=True)
 
     output: list[dict[str, Any]] = []
     for source in rows:
@@ -370,14 +444,19 @@ def main() -> None:
     args = parse_args()
     if args.passes <= 0 or args.batch_size <= 0 or args.max_retries < 0:
         raise ValueError("passes/batch-size must be positive and max-retries non-negative")
-    rows = read_candidates(args.input.expanduser().resolve())
+    input_rows = read_candidates(args.input.expanduser().resolve())
+    rows = [row for row in input_rows if requires_semantic_adjudication(row)]
+    skipped_controls = len(input_rows) - len(rows)
     if args.max_events is not None:
         if args.max_events < 0:
             raise ValueError("--max-events must be non-negative")
         rng = random.Random(args.selection_seed)
         rows = rng.sample(rows, min(args.max_events, len(rows)))
         rows.sort(key=lambda row: row["uid"])
-    print(f"Candidates={len(rows)} model={args.model} passes={args.passes}")
+    print(
+        f"Candidates={len(rows)} skipped_deterministic_controls={skipped_controls} "
+        f"model={args.model} passes={args.passes}"
+    )
     if args.dry_run:
         print("DRY RUN: no API request will be made.\n")
         print(build_user_prompt(rows, 0))