wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Match EasyCom oracle labels to EgoSpeak sampling

Browse Source
This commit is contained in:
Xin Wang
2026-09-07 10:22:05 +08:00
parent 70e3aaaaf6
commit 2632402a70
4 changed files with 78 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
docs/easycom_oracle_information.md
View File

@@ -10,8 +10,11 @@ This is an information upper-bound experiment, not a deployable input setting.
## Causal contract
- The task grid remains 5 Hz. State row `t` has cutoff `(t+1)/5` seconds.
- Oracle bin `t` summarizes transcript frames in
`[t/5,(t+1)/5)` and is therefore completely observed at that cutoff.
- Oracle state `t` follows EgoSpeak's released-target conversion exactly:
source-frame markers `4, 8, 12, ...` are sampled at the corresponding
`0.2, 0.4, 0.6, ...` second cutoffs, with
`Start_Frame <= marker < End_Frame`. This is right-edge point sampling, not
an any-overlap reduction over a 200 ms bin.
- Each person vector stores the last 2 seconds of binary activity followed by
a visibility/availability mask. Activity is exposed only in bins where that
participant was visible according to the provided boxes. No future bin,
@@ -19,8 +22,8 @@ This is an information upper-bound experiment, not a deployable input setting.
- Only non-wearer participant IDs present in the latest provided-box frame
strictly before the cutoff receive slots. IDs route annotations during
preparation but are not exposed as embeddings to the head.
- Transcript-derived wearer/others activity must match `target_perframe` by
default. Any mismatch fails closed instead of being silently tolerated.
- Transcript-derived wearer/others activity must match `target_perframe`
exactly. Any mismatch fails closed; there is no tolerance flag.
- Oracle sidecars use `easycom-oracle-person-history-v1` and are accepted only
by the explicit `oracle-frame-set-attention` mode. Normal local-feature
loading rejects them because they use speech labels.
@@ -65,8 +68,8 @@ dense AP/mAP screen. Existing compatible oracle sidecars are validated and
reused, so interrupted preparation can resume. It deliberately does not run
Sessions 1--3.
If preparation reports any transcript/target mismatch, stop and inspect it.
Do not raise `--max-label-mismatch-rows` merely to make the run continue.
If preparation reports any transcript/target mismatch, stop and inspect it;
the preparation command has no mismatch-tolerance option.
## Follow-up

18
local_turn_features.py
View File

@@ -301,7 +301,7 @@ def _load_validated_payload(
if oracle_analysis:
if (
dataset != "easycom"
or config.get("causal_policy") != "fully-observed-half-open-5hz-bins"
or config.get("causal_policy") != "egospeak-right-edge-current-state"
or config.get("uses_speech_labels") is not True
or config.get("oracle_analysis") is not True
):
@@ -441,13 +441,21 @@ def load_frame_set_features(
if len(frame_times) and float(frame_times[-1]) > float(times[-1]) + 1e-8:
raise ValueError(f"Person frame timeline extends beyond the reference: {path}")
# Match the sidecar's half-open policy: a frame stamped exactly at the
# state cutoff is not yet part of that state's observed prefix.
row_to_frame = torch.searchsorted(frame_times, times, right=False) - 1
# Normal visual observations must be strictly earlier than the cutoff.
# Oracle speech state is defined and sampled at the cutoff itself, so its
# explicit analysis-only format uses an inclusive current-state lookup.
row_to_frame = torch.searchsorted(
frame_times,
times,
right=oracle_analysis,
) - 1
chosen = torch.full(times.shape, float("nan"), dtype=torch.float64)
has_frame = row_to_frame >= 0
chosen[has_frame] = frame_times[row_to_frame[has_frame]]
if bool((chosen[has_frame] >= times[has_frame]).any()):
if oracle_analysis:
if bool((chosen[has_frame] > times[has_frame]).any()):
raise ValueError(f"Oracle person mapping selected future evidence: {path}")
elif bool((chosen[has_frame] >= times[has_frame]).any()):
raise ValueError(f"Person frame mapping selected future visual evidence: {path}")
previous_observation = None
if track_max_gap_seconds is not None:

65
prepare_easycom_oracle_features.py
View File

@@ -64,36 +64,49 @@ def participant_activity(
*,
num_steps: int,
wearer_id: int,
) -> tuple[dict[int, torch.BoolTensor], torch.BoolTensor, dict[str, int]]:
"""Rasterize inclusive 20 Hz transcript spans into observed 5 Hz bins."""
) -> tuple[dict[int, torch.BoolTensor], torch.BoolTensor, dict[str, Any]]:
"""Sample transcript activity on EgoSpeak's 5 Hz right-edge grid.
EasyCom stores one-based ``Start_Frame``/``End_Frame`` markers. EgoSpeak's
released targets sample source-frame markers 4, 8, 12, ... and use
``Start_Frame <= marker < End_Frame``. ``load_utterances`` represents the
corresponding interval as ``[Start_Frame - 1, End_Frame)`` on its
zero-origin dense timeline, hence the ``+1`` below. This is deliberately
point sampling, not an any-overlap reduction over a 200 ms bin.
"""
activity: dict[int, torch.BoolTensor] = {}
wearer = torch.zeros(num_steps, dtype=torch.bool)
discarded_terminal_frames = 0
terminal_frames_after_last_sample = 0
grid_frames = num_steps * FRAMES_PER_TASK_STEP
for row in utterances:
dense_start = int(row.dense_start_frame)
# Recover the source annotation marker from the zero-origin interval.
dense_start = int(row.dense_start_frame) + 1
dense_end = int(row.dense_end_frame_exclusive)
if dense_end > grid_frames:
overrun = dense_end - grid_frames
if overrun > FRAMES_PER_TASK_STEP - 1:
raise ValueError(
f"Transcript exceeds the discarded terminal sub-step by "
f"Transcript exceeds the final 5 Hz sample by "
f"{overrun} frames: {row.uid}"
)
discarded_terminal_frames += max(
0, dense_end - max(dense_start, grid_frames)
)
dense_end = grid_frames
terminal_frames_after_last_sample += overrun
if dense_start >= dense_end:
continue
first = dense_start // FRAMES_PER_TASK_STEP
last = (dense_end - 1) // FRAMES_PER_TASK_STEP
if first < 0 or last < first or last >= num_steps:
# Sample number k (one-origin) observes marker 4*k and maps to row k-1.
# The exclusive stop follows directly from 4*k < dense_end.
first = (dense_start + FRAMES_PER_TASK_STEP - 1) // FRAMES_PER_TASK_STEP - 1
stop = (dense_end + FRAMES_PER_TASK_STEP - 1) // FRAMES_PER_TASK_STEP - 1
first = max(0, first)
stop = min(num_steps, stop)
if first > num_steps or stop < 0:
raise ValueError(
f"Transcript interval leaves target grid: {row.uid}, "
f"bins=[{first},{last}], target_steps={num_steps}"
f"rows=[{first},{stop}), target_steps={num_steps}"
)
if first >= stop:
continue
destination = (
wearer
if int(row.participant_id) == wearer_id
@@ -102,9 +115,12 @@ def participant_activity(
torch.zeros(num_steps, dtype=torch.bool),
)
)
destination[first : last + 1] = True
destination[first:stop] = True
return activity, wearer, {
"discarded_terminal_transcript_frames": discarded_terminal_frames
"target_sampling_policy": "egospeak-right-edge-start-inclusive-end-exclusive",
"terminal_transcript_frames_after_last_5hz_sample": (
terminal_frames_after_last_sample
),
}
@@ -323,7 +339,6 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--delay-seconds", type=float, default=1.0)
parser.add_argument("--camera-wearer-id", type=int, default=CAMERA_WEARER_ID)
parser.add_argument("--shuffle-seed", type=int, default=2027)
parser.add_argument("--max-label-mismatch-rows", type=int, default=0)
parser.add_argument("--expected-mode", default="open-stream-replay")
parser.add_argument("--reuse-existing", action="store_true")
parser.add_argument("--overwrite", action="store_true")
@@ -335,8 +350,6 @@ def main() -> None:
args = parse_args()
history_steps = exact_steps(args.history_seconds, "--history-seconds")
delay_steps = exact_steps(args.delay_seconds, "--delay-seconds")
if args.max_label_mismatch_rows < 0:
raise ValueError("--max-label-mismatch-rows cannot be negative")
sessions = [normalize_sample_name(value, "easycom") for value in args.sessions]
unknown = sorted(set(sessions) - set(ALL_SESSIONS))
if unknown:
@@ -405,12 +418,13 @@ def main() -> None:
| (all_other_activity != target_tensor[:, 2].bool())
).sum()
)
if mismatch_union > args.max_label_mismatch_rows:
if mismatch_union:
raise ValueError(
f"{session}: transcript/target activity differs on "
f"{mismatch_union} rows (wearer={wearer_mismatch}, "
f"others={other_mismatch}); inspect alignment or explicitly "
"raise --max-label-mismatch-rows"
f"others={other_mismatch}) under the EgoSpeak right-edge "
"sampling rule; inspect transcript ordering and dense-timeline "
"alignment. Oracle preparation never tolerates label mismatch."
)
visible_ids, source_records = visible_people_by_row(
@@ -501,7 +515,7 @@ def main() -> None:
config = {
"oracle_analysis": True,
"uses_speech_labels": True,
"causal_policy": "fully-observed-half-open-5hz-bins",
"causal_policy": "egospeak-right-edge-current-state",
"oracle_condition": condition,
"oracle_scope": "currently-visible-nonwearer-participants",
"history_steps": history_steps,
@@ -531,10 +545,9 @@ def main() -> None:
"audio_valid": audio_valid,
"persons": {
"features": features,
# Bin k covers [k/5,(k+1)/5); its feature is available at
# the matching state cutoff (k+1)/5.
"frame_times": torch.arange(len(times), dtype=torch.float64)
* STEP_SECONDS,
# This oracle state is sampled exactly at the matching
# reference cutoff and is available at that cutoff.
"frame_times": times.clone(),
"frame_offsets": frame_offsets,
"records": records,
},

26
test_easycom_oracle_features.py
View File

@@ -23,27 +23,33 @@ class OracleFeatureTests(unittest.TestCase):
def setUp(self) -> None:
torch.set_num_threads(1)
def test_transcript_intervals_are_rasterized_into_half_open_bins(self) -> None:
def test_transcript_intervals_match_egospeak_right_edge_sampling(self) -> None:
rows = [
SimpleNamespace(
uid="a",
participant_id=3,
dense_start_frame=3,
dense_end_frame_exclusive=5,
# EasyCom annotation Start_Frame=1, End_Frame=8. Samples at
# source markers 4,8,... therefore include only row 0.
dense_start_frame=0,
dense_end_frame_exclusive=8,
),
SimpleNamespace(
uid="b",
participant_id=2,
dense_start_frame=8,
dense_end_frame_exclusive=12,
# Start_Frame=5, End_Frame=13 includes markers 8 and 12.
dense_start_frame=4,
dense_end_frame_exclusive=13,
),
]
activity, wearer, audit = participant_activity(
rows, num_steps=4, wearer_id=2
)
self.assertEqual(activity[3].tolist(), [True, True, False, False])
self.assertEqual(wearer.tolist(), [False, False, True, False])
self.assertEqual(audit["discarded_terminal_transcript_frames"], 0)
self.assertEqual(activity[3].tolist(), [True, False, False, False])
self.assertEqual(wearer.tolist(), [False, True, True, False])
self.assertEqual(
audit["target_sampling_policy"],
"egospeak-right-edge-start-inclusive-end-exclusive",
)
def test_history_at_row_does_not_read_later_activity(self) -> None:
first = torch.tensor([0, 1, 0, 0, 0], dtype=torch.bool)
@@ -152,7 +158,7 @@ class OracleFeatureTests(unittest.TestCase):
"config": {
"oracle_analysis": True,
"uses_speech_labels": True,
"causal_policy": "fully-observed-half-open-5hz-bins",
"causal_policy": "egospeak-right-edge-current-state",
"oracle_condition": "per-person",
"visual_dim": 6,
"audio_dim": 5,
@@ -179,6 +185,8 @@ class OracleFeatureTests(unittest.TestCase):
self.assertEqual(audio.shape, (n, 6))
self.assertEqual(people.dimension, 6)
self.assertEqual(contract["format"], ORACLE_FORMAT)
self.assertTrue(torch.equal(people.row_to_frame, torch.arange(n)))
self.assertTrue(torch.allclose(people.chosen_frame_times, times))
head = TurnPredictionHead(
hidden_size=8,
num_horizons=2,