Add floor-event timing objective and addressee diagnostics

prepare_easycom_oracle_features.py

@@ -46,8 +46,14 @@ ORACLE_CONDITIONS = (
     "per-person",
     "shuffled-id",
     "delayed",
+    # Keep new analysis conditions after the historical five so their addition
+    # cannot change the established shuffled-id generation seeds.
+    "aggregate-visible-address-control",
+    "aggregate-visible-addressee",
 )
 UNKNOWN_PERSON_IDS = {"", "-1", "none", "null", "unknown"}
+UNKNOWN_ADDRESSEE_VALUES = {"", "-1", "none", "null", "unknown"}
+GROUP_ADDRESSEE_VALUES = {"all", "everyone", "group"}
 
 
 def exact_steps(seconds: float, name: str) -> int:
@@ -59,6 +65,33 @@ def exact_steps(seconds: float, name: str) -> int:
     return int(steps)
 
 
+def _sampled_row_interval(row: Any, num_steps: int) -> tuple[int, int, int]:
+    """Map one transcript interval to the released EgoSpeak 5 Hz grid."""
+
+    dense_start = int(row.dense_start_frame) + 1
+    dense_end = int(row.dense_end_frame_exclusive)
+    grid_frames = num_steps * FRAMES_PER_TASK_STEP
+    overrun = max(0, dense_end - grid_frames)
+    if overrun > FRAMES_PER_TASK_STEP - 1:
+        raise ValueError(
+            f"Transcript exceeds the final 5 Hz sample by {overrun} frames: "
+            f"{row.uid}"
+        )
+    if dense_start >= dense_end:
+        return 0, 0, overrun
+
+    first = (dense_start + FRAMES_PER_TASK_STEP - 1) // FRAMES_PER_TASK_STEP - 1
+    stop = (dense_end + FRAMES_PER_TASK_STEP - 1) // FRAMES_PER_TASK_STEP - 1
+    first = max(0, first)
+    stop = min(num_steps, stop)
+    if first > num_steps or stop < 0:
+        raise ValueError(
+            f"Transcript interval leaves target grid: {row.uid}, "
+            f"rows=[{first},{stop}), target_steps={num_steps}"
+        )
+    return first, stop, overrun
+
+
 def participant_activity(
     utterances: Sequence[Any],
     *,
@@ -78,33 +111,9 @@ def participant_activity(
     activity: dict[int, torch.BoolTensor] = {}
     wearer = torch.zeros(num_steps, dtype=torch.bool)
     terminal_frames_after_last_sample = 0
-    grid_frames = num_steps * FRAMES_PER_TASK_STEP
     for row in utterances:
-        # Recover the source annotation marker from the zero-origin interval.
-        dense_start = int(row.dense_start_frame) + 1
-        dense_end = int(row.dense_end_frame_exclusive)
-        if dense_end > grid_frames:
-            overrun = dense_end - grid_frames
-            if overrun > FRAMES_PER_TASK_STEP - 1:
-                raise ValueError(
-                    f"Transcript exceeds the final 5 Hz sample by "
-                    f"{overrun} frames: {row.uid}"
-                )
-            terminal_frames_after_last_sample += overrun
-        if dense_start >= dense_end:
-            continue
-
-        # Sample number k (one-origin) observes marker 4*k and maps to row k-1.
-        # The exclusive stop follows directly from 4*k < dense_end.
-        first = (dense_start + FRAMES_PER_TASK_STEP - 1) // FRAMES_PER_TASK_STEP - 1
-        stop = (dense_end + FRAMES_PER_TASK_STEP - 1) // FRAMES_PER_TASK_STEP - 1
-        first = max(0, first)
-        stop = min(num_steps, stop)
-        if first > num_steps or stop < 0:
-            raise ValueError(
-                f"Transcript interval leaves target grid: {row.uid}, "
-                f"rows=[{first},{stop}), target_steps={num_steps}"
-            )
+        first, stop, overrun = _sampled_row_interval(row, num_steps)
+        terminal_frames_after_last_sample += overrun
         if first >= stop:
             continue
         destination = (
@@ -124,6 +133,71 @@ def participant_activity(
     }
 
 
+def target_matches_participant(
+    targets: Sequence[int | str], participant_id: int
+) -> bool:
+    """Match only an explicit numeric participant target."""
+
+    for value in targets:
+        if isinstance(value, bool):
+            continue
+        if isinstance(value, int) and value == participant_id:
+            return True
+        text = str(value).strip()
+        if text.lstrip("+-").isdigit() and int(text) == participant_id:
+            return True
+    return False
+
+
+def participant_direct_addressee_activity(
+    utterances: Sequence[Any],
+    *,
+    num_steps: int,
+    wearer_id: int,
+) -> tuple[dict[int, torch.BoolTensor], dict[str, Any]]:
+    """Return non-wearer speech bins explicitly addressed to the wearer.
+
+    Group and unknown targets are audited but are not coerced to either a
+    positive or a trustworthy negative semantic label.  The output is an
+    analysis-only privileged input derived from annotation.
+    """
+
+    directed: dict[int, torch.BoolTensor] = {}
+    category_counts = {"direct_wearer": 0, "group": 0, "other": 0, "unknown": 0}
+    direct_active_bins = 0
+    for row in utterances:
+        if int(row.participant_id) == wearer_id:
+            continue
+        targets = tuple(row.target_of_speech)
+        normalized = {str(value).strip().lower() for value in targets}
+        if target_matches_participant(targets, wearer_id):
+            category = "direct_wearer"
+        elif normalized & GROUP_ADDRESSEE_VALUES:
+            category = "group"
+        elif not normalized or normalized.issubset(UNKNOWN_ADDRESSEE_VALUES):
+            category = "unknown"
+        else:
+            category = "other"
+        category_counts[category] += 1
+        if category != "direct_wearer":
+            continue
+        first, stop, _overrun = _sampled_row_interval(row, num_steps)
+        if first >= stop:
+            continue
+        sequence = directed.setdefault(
+            int(row.participant_id),
+            torch.zeros(num_steps, dtype=torch.bool),
+        )
+        before = int(sequence[first:stop].sum())
+        sequence[first:stop] = True
+        direct_active_bins += int(sequence[first:stop].sum()) - before
+    return directed, {
+        "target_of_speech_utterance_counts": category_counts,
+        "direct_wearer_addressed_person_bins": direct_active_bins,
+        "target_of_speech_group_unknown_not_coerced": True,
+    }
+
+
 def causal_history(
     sequence: torch.BoolTensor,
     *,
@@ -215,6 +289,7 @@ def oracle_person_rows(
     history_steps: int,
     delay_steps: int,
     seed: int,
+    direct_addressee_activity: dict[int, torch.BoolTensor] | None = None,
 ) -> tuple[torch.Tensor, torch.LongTensor, list[dict[str, Any]], dict[str, int]]:
     """Create a 5 Hz ragged set of causal oracle histories."""
 
@@ -222,6 +297,12 @@ def oracle_person_rows(
         raise ValueError(f"Unknown oracle condition: {condition}")
     if len(visible_ids) != len(source_records):
         raise ValueError("Visible ID/record row counts differ")
+    address_condition = condition in {
+        "aggregate-visible-address-control",
+        "aggregate-visible-addressee",
+    }
+    if condition == "aggregate-visible-addressee" and direct_addressee_activity is None:
+        raise ValueError("Addressee condition requires direct addressee activity")
     num_steps = len(visible_ids)
     zero_sequence = torch.zeros(num_steps, dtype=torch.bool)
     rng = np.random.default_rng(seed)
@@ -251,17 +332,49 @@ def oracle_person_rows(
         if condition == "visibility-only":
             output = base.clone()
             output[:, :history_steps] = 0
-        elif condition == "aggregate-visible":
+        elif condition in {
+            "aggregate-visible",
+            "aggregate-visible-address-control",
+            "aggregate-visible-addressee",
+        }:
             if len(base):
-                output = torch.cat(
+                aggregate = torch.cat(
                     (
                         base[:, :history_steps].amax(dim=0, keepdim=True),
                         base[:, history_steps:].amax(dim=0, keepdim=True),
                     ),
                     dim=-1,
                 )
+                if address_condition:
+                    if condition == "aggregate-visible-addressee":
+                        assert direct_addressee_activity is not None
+                        addressed_rows = torch.stack(
+                            [
+                                causal_history(
+                                    direct_addressee_activity.get(
+                                        person_id, zero_sequence
+                                    ),
+                                    row=row,
+                                    history_steps=history_steps,
+                                    observed_sequence=visibility.get(
+                                        person_id, zero_sequence
+                                    ),
+                                )[:history_steps]
+                                for person_id in person_ids
+                            ],
+                            dim=0,
+                        )
+                        addressed = addressed_rows.amax(dim=0, keepdim=True)
+                    else:
+                        addressed = torch.zeros(
+                            (1, history_steps), dtype=torch.float32
+                        )
+                    output = torch.cat((aggregate, addressed), dim=-1)
+                else:
+                    output = aggregate
             else:
-                output = base
+                feature_dim = (3 if address_condition else 2) * history_steps
+                output = torch.empty((0, feature_dim), dtype=torch.float32)
         elif condition == "shuffled-id":
             output = base.clone()
             # Independently reassign each historical time bin across the
@@ -282,10 +395,10 @@ def oracle_person_rows(
             output = base
 
         feature_rows.extend(output.unbind(dim=0))
-        if condition == "aggregate-visible" and person_ids:
+        if condition.startswith("aggregate-visible") and person_ids:
             records.append(
                 {
-                    "id": "aggregate-visible",
+                    "id": condition,
                     "source_person_ids": person_ids,
                     "source_visual_times": [
                         float(item.get("source_time", float("nan")))
@@ -308,7 +421,7 @@ def oracle_person_rows(
                 )
         offsets.append(offsets[-1] + len(output))
 
-    feature_dim = 2 * history_steps
+    feature_dim = (3 if address_condition else 2) * history_steps
     features = (
         torch.stack(feature_rows)
         if feature_rows
@@ -402,6 +515,13 @@ def main() -> None:
             num_steps=len(times),
             wearer_id=args.camera_wearer_id,
         )
+        direct_addressee_activity, addressee_audit = (
+            participant_direct_addressee_activity(
+                utterances,
+                num_steps=len(times),
+                wearer_id=args.camera_wearer_id,
+            )
+        )
         all_other_activity = torch.zeros(len(times), dtype=torch.bool)
         for sequence in activity.values():
             all_other_activity |= sequence
@@ -503,8 +623,10 @@ def main() -> None:
                 history_steps=history_steps,
                 delay_steps=delay_steps,
                 seed=condition_seed,
+                direct_addressee_activity=direct_addressee_activity,
             )
-            visual = torch.zeros((len(times), 2 * history_steps))
+            visual_dim = int(features.shape[1])
+            visual = torch.zeros((len(times), visual_dim))
             visual_valid = torch.zeros(len(times), dtype=torch.bool)
             for row in range(len(times)):
                 start = int(frame_offsets[row])
@@ -515,6 +637,8 @@ def main() -> None:
             config = {
                 "oracle_analysis": True,
                 "uses_speech_labels": True,
+                "uses_target_of_speech": condition == "aggregate-visible-addressee",
+                "online_observable": False,
                 "causal_policy": "egospeak-right-edge-current-state",
                 "oracle_condition": condition,
                 "oracle_scope": "currently-visible-nonwearer-participants",
@@ -526,9 +650,19 @@ def main() -> None:
                 ),
                 "shuffle_seed": args.shuffle_seed if condition == "shuffled-id" else None,
                 "speaker_activity_source": "EasyCom Speech_Transcriptions inclusive 20Hz intervals",
+                "addressee_source": (
+                    "EasyCom Target_of_Speech explicit wearer ID"
+                    if condition == "aggregate-visible-addressee"
+                    else None
+                ),
+                "addressee_semantics": (
+                    "direct-wearer-only; Group/None/unknown are not coerced"
+                    if condition == "aggregate-visible-addressee"
+                    else None
+                ),
                 "visibility_source": "provided face boxes in source local sidecar",
                 "camera_wearer_id": args.camera_wearer_id,
-                "visual_dim": 2 * history_steps,
+                "visual_dim": visual_dim,
                 "audio_dim": audio_dim,
                 "source_local_contract": source_contract,
             }
@@ -556,6 +690,9 @@ def main() -> None:
                 "source_local_features": str(source_local_path.resolve()),
                 "audit": {
                     "strictly_past_or_current_bins_only": True,
+                    "target_of_speech_is_privileged_utterance_annotation": (
+                        condition == "aggregate-visible-addressee"
+                    ),
                     "transcript_target_mismatch_rows": mismatch_union,
                     "wearer_mismatch_rows": wearer_mismatch,
                     "others_mismatch_rows": other_mismatch,
@@ -566,6 +703,7 @@ def main() -> None:
                     "person_observations": len(features),
                     "generation_seed": condition_seed,
                     **transcript_audit,
+                    **addressee_audit,
                     **corruption,
                 },
             }