wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Match EasyCom oracle labels to EgoSpeak sampling

Browse Source
This commit is contained in:
Xin Wang
2026-09-07 10:22:05 +08:00
parent 70e3aaaaf6
commit 2632402a70
4 changed files with 78 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
prepare_easycom_oracle_features.py
View File

@@ -64,36 +64,49 @@ def participant_activity(
*,
num_steps: int,
wearer_id: int,
) -> tuple[dict[int, torch.BoolTensor], torch.BoolTensor, dict[str, int]]:
"""Rasterize inclusive 20 Hz transcript spans into observed 5 Hz bins."""
) -> tuple[dict[int, torch.BoolTensor], torch.BoolTensor, dict[str, Any]]:
"""Sample transcript activity on EgoSpeak's 5 Hz right-edge grid.
EasyCom stores one-based ``Start_Frame``/``End_Frame`` markers. EgoSpeak's
released targets sample source-frame markers 4, 8, 12, ... and use
``Start_Frame <= marker < End_Frame``. ``load_utterances`` represents the
corresponding interval as ``[Start_Frame - 1, End_Frame)`` on its
zero-origin dense timeline, hence the ``+1`` below. This is deliberately
point sampling, not an any-overlap reduction over a 200 ms bin.
"""
activity: dict[int, torch.BoolTensor] = {}
wearer = torch.zeros(num_steps, dtype=torch.bool)
discarded_terminal_frames = 0
terminal_frames_after_last_sample = 0
grid_frames = num_steps * FRAMES_PER_TASK_STEP
for row in utterances:
dense_start = int(row.dense_start_frame)
# Recover the source annotation marker from the zero-origin interval.
dense_start = int(row.dense_start_frame) + 1
dense_end = int(row.dense_end_frame_exclusive)
if dense_end > grid_frames:
overrun = dense_end - grid_frames
if overrun > FRAMES_PER_TASK_STEP - 1:
raise ValueError(
f"Transcript exceeds the discarded terminal sub-step by "
f"Transcript exceeds the final 5 Hz sample by "
f"{overrun} frames: {row.uid}"
)
discarded_terminal_frames += max(
0, dense_end - max(dense_start, grid_frames)
)
dense_end = grid_frames
terminal_frames_after_last_sample += overrun
if dense_start >= dense_end:
continue
first = dense_start // FRAMES_PER_TASK_STEP
last = (dense_end - 1) // FRAMES_PER_TASK_STEP
if first < 0 or last < first or last >= num_steps:
# Sample number k (one-origin) observes marker 4*k and maps to row k-1.
# The exclusive stop follows directly from 4*k < dense_end.
first = (dense_start + FRAMES_PER_TASK_STEP - 1) // FRAMES_PER_TASK_STEP - 1
stop = (dense_end + FRAMES_PER_TASK_STEP - 1) // FRAMES_PER_TASK_STEP - 1
first = max(0, first)
stop = min(num_steps, stop)
if first > num_steps or stop < 0:
raise ValueError(
f"Transcript interval leaves target grid: {row.uid}, "
f"bins=[{first},{last}], target_steps={num_steps}"
f"rows=[{first},{stop}), target_steps={num_steps}"
)
if first >= stop:
continue
destination = (
wearer
if int(row.participant_id) == wearer_id
@@ -102,9 +115,12 @@ def participant_activity(
torch.zeros(num_steps, dtype=torch.bool),
)
)
destination[first : last + 1] = True
destination[first:stop] = True
return activity, wearer, {
"discarded_terminal_transcript_frames": discarded_terminal_frames
"target_sampling_policy": "egospeak-right-edge-start-inclusive-end-exclusive",
"terminal_transcript_frames_after_last_5hz_sample": (
terminal_frames_after_last_sample
),
}
@@ -323,7 +339,6 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--delay-seconds", type=float, default=1.0)
parser.add_argument("--camera-wearer-id", type=int, default=CAMERA_WEARER_ID)
parser.add_argument("--shuffle-seed", type=int, default=2027)
parser.add_argument("--max-label-mismatch-rows", type=int, default=0)
parser.add_argument("--expected-mode", default="open-stream-replay")
parser.add_argument("--reuse-existing", action="store_true")
parser.add_argument("--overwrite", action="store_true")
@@ -335,8 +350,6 @@ def main() -> None:
args = parse_args()
history_steps = exact_steps(args.history_seconds, "--history-seconds")
delay_steps = exact_steps(args.delay_seconds, "--delay-seconds")
if args.max_label_mismatch_rows < 0:
raise ValueError("--max-label-mismatch-rows cannot be negative")
sessions = [normalize_sample_name(value, "easycom") for value in args.sessions]
unknown = sorted(set(sessions) - set(ALL_SESSIONS))
if unknown:
@@ -405,12 +418,13 @@ def main() -> None:
| (all_other_activity != target_tensor[:, 2].bool())
).sum()
)
if mismatch_union > args.max_label_mismatch_rows:
if mismatch_union:
raise ValueError(
f"{session}: transcript/target activity differs on "
f"{mismatch_union} rows (wearer={wearer_mismatch}, "
f"others={other_mismatch}); inspect alignment or explicitly "
"raise --max-label-mismatch-rows"
f"others={other_mismatch}) under the EgoSpeak right-edge "
"sampling rule; inspect transcript ordering and dense-timeline "
"alignment. Oracle preparation never tolerates label mismatch."
)
visible_ids, source_records = visible_people_by_row(
@@ -501,7 +515,7 @@ def main() -> None:
config = {
"oracle_analysis": True,
"uses_speech_labels": True,
"causal_policy": "fully-observed-half-open-5hz-bins",
"causal_policy": "egospeak-right-edge-current-state",
"oracle_condition": condition,
"oracle_scope": "currently-visible-nonwearer-participants",
"history_steps": history_steps,
@@ -531,10 +545,9 @@ def main() -> None:
"audio_valid": audio_valid,
"persons": {
"features": features,
# Bin k covers [k/5,(k+1)/5); its feature is available at
# the matching state cutoff (k+1)/5.
"frame_times": torch.arange(len(times), dtype=torch.float64)
* STEP_SECONDS,
# This oracle state is sampled exactly at the matching
# reference cutoff and is available at that cutoff.
"frame_times": times.clone(),
"frame_offsets": frame_offsets,
"records": records,
},