From 2632402a70ed623637d9b694a7e6d833c9ddcc22 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Mon, 7 Sep 2026 10:22:05 +0800 Subject: [PATCH] Match EasyCom oracle labels to EgoSpeak sampling --- docs/easycom_oracle_information.md | 15 ++++--- local_turn_features.py | 18 ++++++--- prepare_easycom_oracle_features.py | 65 ++++++++++++++++++------------ test_easycom_oracle_features.py | 26 +++++++----- 4 files changed, 78 insertions(+), 46 deletions(-) diff --git a/docs/easycom_oracle_information.md b/docs/easycom_oracle_information.md index 00121a3..5bb4999 100644 --- a/docs/easycom_oracle_information.md +++ b/docs/easycom_oracle_information.md @@ -10,8 +10,11 @@ This is an information upper-bound experiment, not a deployable input setting. ## Causal contract - The task grid remains 5 Hz. State row `t` has cutoff `(t+1)/5` seconds. -- Oracle bin `t` summarizes transcript frames in - `[t/5,(t+1)/5)` and is therefore completely observed at that cutoff. +- Oracle state `t` follows EgoSpeak's released-target conversion exactly: + source-frame markers `4, 8, 12, ...` are sampled at the corresponding + `0.2, 0.4, 0.6, ...` second cutoffs, with + `Start_Frame <= marker < End_Frame`. This is right-edge point sampling, not + an any-overlap reduction over a 200 ms bin. - Each person vector stores the last 2 seconds of binary activity followed by a visibility/availability mask. Activity is exposed only in bins where that participant was visible according to the provided boxes. No future bin, @@ -19,8 +22,8 @@ This is an information upper-bound experiment, not a deployable input setting. - Only non-wearer participant IDs present in the latest provided-box frame strictly before the cutoff receive slots. IDs route annotations during preparation but are not exposed as embeddings to the head. -- Transcript-derived wearer/others activity must match `target_perframe` by - default. Any mismatch fails closed instead of being silently tolerated. +- Transcript-derived wearer/others activity must match `target_perframe` + exactly. Any mismatch fails closed; there is no tolerance flag. - Oracle sidecars use `easycom-oracle-person-history-v1` and are accepted only by the explicit `oracle-frame-set-attention` mode. Normal local-feature loading rejects them because they use speech labels. @@ -65,8 +68,8 @@ dense AP/mAP screen. Existing compatible oracle sidecars are validated and reused, so interrupted preparation can resume. It deliberately does not run Sessions 1--3. -If preparation reports any transcript/target mismatch, stop and inspect it. -Do not raise `--max-label-mismatch-rows` merely to make the run continue. +If preparation reports any transcript/target mismatch, stop and inspect it; +the preparation command has no mismatch-tolerance option. ## Follow-up diff --git a/local_turn_features.py b/local_turn_features.py index d357ad1..9feefe8 100644 --- a/local_turn_features.py +++ b/local_turn_features.py @@ -301,7 +301,7 @@ def _load_validated_payload( if oracle_analysis: if ( dataset != "easycom" - or config.get("causal_policy") != "fully-observed-half-open-5hz-bins" + or config.get("causal_policy") != "egospeak-right-edge-current-state" or config.get("uses_speech_labels") is not True or config.get("oracle_analysis") is not True ): @@ -441,13 +441,21 @@ def load_frame_set_features( if len(frame_times) and float(frame_times[-1]) > float(times[-1]) + 1e-8: raise ValueError(f"Person frame timeline extends beyond the reference: {path}") - # Match the sidecar's half-open policy: a frame stamped exactly at the - # state cutoff is not yet part of that state's observed prefix. - row_to_frame = torch.searchsorted(frame_times, times, right=False) - 1 + # Normal visual observations must be strictly earlier than the cutoff. + # Oracle speech state is defined and sampled at the cutoff itself, so its + # explicit analysis-only format uses an inclusive current-state lookup. + row_to_frame = torch.searchsorted( + frame_times, + times, + right=oracle_analysis, + ) - 1 chosen = torch.full(times.shape, float("nan"), dtype=torch.float64) has_frame = row_to_frame >= 0 chosen[has_frame] = frame_times[row_to_frame[has_frame]] - if bool((chosen[has_frame] >= times[has_frame]).any()): + if oracle_analysis: + if bool((chosen[has_frame] > times[has_frame]).any()): + raise ValueError(f"Oracle person mapping selected future evidence: {path}") + elif bool((chosen[has_frame] >= times[has_frame]).any()): raise ValueError(f"Person frame mapping selected future visual evidence: {path}") previous_observation = None if track_max_gap_seconds is not None: diff --git a/prepare_easycom_oracle_features.py b/prepare_easycom_oracle_features.py index a3eb293..24d0573 100644 --- a/prepare_easycom_oracle_features.py +++ b/prepare_easycom_oracle_features.py @@ -64,36 +64,49 @@ def participant_activity( *, num_steps: int, wearer_id: int, -) -> tuple[dict[int, torch.BoolTensor], torch.BoolTensor, dict[str, int]]: - """Rasterize inclusive 20 Hz transcript spans into observed 5 Hz bins.""" +) -> tuple[dict[int, torch.BoolTensor], torch.BoolTensor, dict[str, Any]]: + """Sample transcript activity on EgoSpeak's 5 Hz right-edge grid. + + EasyCom stores one-based ``Start_Frame``/``End_Frame`` markers. EgoSpeak's + released targets sample source-frame markers 4, 8, 12, ... and use + ``Start_Frame <= marker < End_Frame``. ``load_utterances`` represents the + corresponding interval as ``[Start_Frame - 1, End_Frame)`` on its + zero-origin dense timeline, hence the ``+1`` below. This is deliberately + point sampling, not an any-overlap reduction over a 200 ms bin. + """ activity: dict[int, torch.BoolTensor] = {} wearer = torch.zeros(num_steps, dtype=torch.bool) - discarded_terminal_frames = 0 + terminal_frames_after_last_sample = 0 grid_frames = num_steps * FRAMES_PER_TASK_STEP for row in utterances: - dense_start = int(row.dense_start_frame) + # Recover the source annotation marker from the zero-origin interval. + dense_start = int(row.dense_start_frame) + 1 dense_end = int(row.dense_end_frame_exclusive) if dense_end > grid_frames: overrun = dense_end - grid_frames if overrun > FRAMES_PER_TASK_STEP - 1: raise ValueError( - f"Transcript exceeds the discarded terminal sub-step by " + f"Transcript exceeds the final 5 Hz sample by " f"{overrun} frames: {row.uid}" ) - discarded_terminal_frames += max( - 0, dense_end - max(dense_start, grid_frames) - ) - dense_end = grid_frames + terminal_frames_after_last_sample += overrun if dense_start >= dense_end: continue - first = dense_start // FRAMES_PER_TASK_STEP - last = (dense_end - 1) // FRAMES_PER_TASK_STEP - if first < 0 or last < first or last >= num_steps: + + # Sample number k (one-origin) observes marker 4*k and maps to row k-1. + # The exclusive stop follows directly from 4*k < dense_end. + first = (dense_start + FRAMES_PER_TASK_STEP - 1) // FRAMES_PER_TASK_STEP - 1 + stop = (dense_end + FRAMES_PER_TASK_STEP - 1) // FRAMES_PER_TASK_STEP - 1 + first = max(0, first) + stop = min(num_steps, stop) + if first > num_steps or stop < 0: raise ValueError( f"Transcript interval leaves target grid: {row.uid}, " - f"bins=[{first},{last}], target_steps={num_steps}" + f"rows=[{first},{stop}), target_steps={num_steps}" ) + if first >= stop: + continue destination = ( wearer if int(row.participant_id) == wearer_id @@ -102,9 +115,12 @@ def participant_activity( torch.zeros(num_steps, dtype=torch.bool), ) ) - destination[first : last + 1] = True + destination[first:stop] = True return activity, wearer, { - "discarded_terminal_transcript_frames": discarded_terminal_frames + "target_sampling_policy": "egospeak-right-edge-start-inclusive-end-exclusive", + "terminal_transcript_frames_after_last_5hz_sample": ( + terminal_frames_after_last_sample + ), } @@ -323,7 +339,6 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--delay-seconds", type=float, default=1.0) parser.add_argument("--camera-wearer-id", type=int, default=CAMERA_WEARER_ID) parser.add_argument("--shuffle-seed", type=int, default=2027) - parser.add_argument("--max-label-mismatch-rows", type=int, default=0) parser.add_argument("--expected-mode", default="open-stream-replay") parser.add_argument("--reuse-existing", action="store_true") parser.add_argument("--overwrite", action="store_true") @@ -335,8 +350,6 @@ def main() -> None: args = parse_args() history_steps = exact_steps(args.history_seconds, "--history-seconds") delay_steps = exact_steps(args.delay_seconds, "--delay-seconds") - if args.max_label_mismatch_rows < 0: - raise ValueError("--max-label-mismatch-rows cannot be negative") sessions = [normalize_sample_name(value, "easycom") for value in args.sessions] unknown = sorted(set(sessions) - set(ALL_SESSIONS)) if unknown: @@ -405,12 +418,13 @@ def main() -> None: | (all_other_activity != target_tensor[:, 2].bool()) ).sum() ) - if mismatch_union > args.max_label_mismatch_rows: + if mismatch_union: raise ValueError( f"{session}: transcript/target activity differs on " f"{mismatch_union} rows (wearer={wearer_mismatch}, " - f"others={other_mismatch}); inspect alignment or explicitly " - "raise --max-label-mismatch-rows" + f"others={other_mismatch}) under the EgoSpeak right-edge " + "sampling rule; inspect transcript ordering and dense-timeline " + "alignment. Oracle preparation never tolerates label mismatch." ) visible_ids, source_records = visible_people_by_row( @@ -501,7 +515,7 @@ def main() -> None: config = { "oracle_analysis": True, "uses_speech_labels": True, - "causal_policy": "fully-observed-half-open-5hz-bins", + "causal_policy": "egospeak-right-edge-current-state", "oracle_condition": condition, "oracle_scope": "currently-visible-nonwearer-participants", "history_steps": history_steps, @@ -531,10 +545,9 @@ def main() -> None: "audio_valid": audio_valid, "persons": { "features": features, - # Bin k covers [k/5,(k+1)/5); its feature is available at - # the matching state cutoff (k+1)/5. - "frame_times": torch.arange(len(times), dtype=torch.float64) - * STEP_SECONDS, + # This oracle state is sampled exactly at the matching + # reference cutoff and is available at that cutoff. + "frame_times": times.clone(), "frame_offsets": frame_offsets, "records": records, }, diff --git a/test_easycom_oracle_features.py b/test_easycom_oracle_features.py index dec1d9a..e5ffc93 100644 --- a/test_easycom_oracle_features.py +++ b/test_easycom_oracle_features.py @@ -23,27 +23,33 @@ class OracleFeatureTests(unittest.TestCase): def setUp(self) -> None: torch.set_num_threads(1) - def test_transcript_intervals_are_rasterized_into_half_open_bins(self) -> None: + def test_transcript_intervals_match_egospeak_right_edge_sampling(self) -> None: rows = [ SimpleNamespace( uid="a", participant_id=3, - dense_start_frame=3, - dense_end_frame_exclusive=5, + # EasyCom annotation Start_Frame=1, End_Frame=8. Samples at + # source markers 4,8,... therefore include only row 0. + dense_start_frame=0, + dense_end_frame_exclusive=8, ), SimpleNamespace( uid="b", participant_id=2, - dense_start_frame=8, - dense_end_frame_exclusive=12, + # Start_Frame=5, End_Frame=13 includes markers 8 and 12. + dense_start_frame=4, + dense_end_frame_exclusive=13, ), ] activity, wearer, audit = participant_activity( rows, num_steps=4, wearer_id=2 ) - self.assertEqual(activity[3].tolist(), [True, True, False, False]) - self.assertEqual(wearer.tolist(), [False, False, True, False]) - self.assertEqual(audit["discarded_terminal_transcript_frames"], 0) + self.assertEqual(activity[3].tolist(), [True, False, False, False]) + self.assertEqual(wearer.tolist(), [False, True, True, False]) + self.assertEqual( + audit["target_sampling_policy"], + "egospeak-right-edge-start-inclusive-end-exclusive", + ) def test_history_at_row_does_not_read_later_activity(self) -> None: first = torch.tensor([0, 1, 0, 0, 0], dtype=torch.bool) @@ -152,7 +158,7 @@ class OracleFeatureTests(unittest.TestCase): "config": { "oracle_analysis": True, "uses_speech_labels": True, - "causal_policy": "fully-observed-half-open-5hz-bins", + "causal_policy": "egospeak-right-edge-current-state", "oracle_condition": "per-person", "visual_dim": 6, "audio_dim": 5, @@ -179,6 +185,8 @@ class OracleFeatureTests(unittest.TestCase): self.assertEqual(audio.shape, (n, 6)) self.assertEqual(people.dimension, 6) self.assertEqual(contract["format"], ORACLE_FORMAT) + self.assertTrue(torch.equal(people.row_to_frame, torch.arange(n))) + self.assertTrue(torch.allclose(people.chosen_frame_times, times)) head = TurnPredictionHead( hidden_size=8, num_horizons=2,