From 0f6a72cfdfd5743bbb00ec780559144feb2bde00 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Fri, 28 Aug 2026 15:03:31 +0800 Subject: [PATCH] Fix Ego4D target compatibility in local baselines --- docs/ego4d_local_baselines.md | 7 +++++ easycom_head.py | 53 +++++++++++++++++++++++++++++++---- run_ego4d_local_baselines.py | 24 ++++++++++++++++ test_ego4d_local_workflow.py | 31 ++++++++++++++++++++ train_turn_head.py | 35 ++++++++++++++++------- 5 files changed, 134 insertions(+), 16 deletions(-) diff --git a/docs/ego4d_local_baselines.md b/docs/ego4d_local_baselines.md index 4b07e6b..8229c82 100644 --- a/docs/ego4d_local_baselines.md +++ b/docs/ego4d_local_baselines.md @@ -291,6 +291,13 @@ python run_ego4d_local_baselines.py "${EXP_ARGS[@]}" --seeds 1 2 如果原 stacked baseline 使用 fixed-mean，可在四组统一加 `--layer-fusion fixed-mean`。 不要拿不同 epoch/layer-fusion 的旧 baseline 直接计算新增分支的增量。 +Ego4D target 由 EgoSpeak/LSTR 的 `MCE` 当作 binary/multi-positive 权重直接读取； +因此代码对 `dataset=ego4d` 保留其原始二值行，不强行套用 EasyCom 的四状态约束， +也绝不把 `[0,0,0]` 等行擅自改成 `[1,0,0]` background。预检会汇总并打印所有 +非 canonical pattern，便于核对数据。EasyCom 仍严格限制为四个 activity states。 +当前四组实验关闭 transition objective，因此不会从非 canonical 行伪造 transition +标签；如果以后启用 transition auxiliary，需要先定义语义和显式 valid mask。 + 预检不会一次把 433 条全局特征全部装进 RAM；实际训练仍沿用原训练器，会加载整个 train split。 last4 特征可能需要数十 GB 主机内存。多卡并行 head 训练也会复制主机内存，不是只看 GPU 是否空闲。 diff --git a/easycom_head.py b/easycom_head.py index 7dd97d4..95cd950 100644 --- a/easycom_head.py +++ b/easycom_head.py @@ -278,8 +278,18 @@ def _validate_loss_inputs( ) -def _validated_binary_activity_targets(targets: torch.Tensor) -> torch.Tensor: - """Validate the four legal EasyCom/EgoSpeak three-label activity rows.""" +def _validated_binary_activity_targets( + targets: torch.Tensor, + *, + allow_noncanonical_binary: bool = False, +) -> torch.Tensor: + """Validate three-label activity rows. + + EasyCom uses four explicit activity states. Ego4D's released target files + are consumed by EgoSpeak's MCE objective as binary/multi-positive weights, + so the compatibility path preserves any binary row instead of silently + rewriting it to an EasyCom state. + """ if targets.ndim < 1 or targets.shape[-1] != NUM_CLASSES: raise ValueError( @@ -291,11 +301,22 @@ def _validated_binary_activity_targets(targets: torch.Tensor) -> torch.Tensor: rounded = targets.round() if not torch.allclose(targets, rounded, atol=1e-6, rtol=0.0): raise ValueError("Activity targets must contain only binary 0/1 values") - expected_background = (rounded[..., 1:].sum(dim=-1) == 0).to(rounded.dtype) - if not torch.equal(rounded[..., 0], expected_background): + expected_background = rounded[..., 1:].sum(dim=-1) == 0 + valid = rounded[..., 0].bool() == expected_background + if allow_noncanonical_binary: + return rounded + if not bool(valid.all()): + invalid = torch.unique(rounded[~valid].reshape(-1, NUM_CLASSES), dim=0) + invalid_rows = ", ".join( + str([int(value) for value in row.tolist()]) + for row in invalid[:8] + ) + if len(invalid) > 8: + invalid_rows += ", ..." raise ValueError( "Activity targets must be one of [1,0,0], [0,1,0], " - "[0,0,1], or [0,1,1]" + "[0,0,1], or [0,1,1]; observed invalid " + f"row(s): {invalid_rows}" ) return rounded @@ -799,7 +820,13 @@ def load_session_data( f"{target_path}" ) targets = torch.from_numpy(np.asarray(target_array, dtype=np.float32)) - _validated_binary_activity_targets(targets) + try: + _validated_binary_activity_targets( + targets, + allow_noncanonical_binary=dataset == "ego4d", + ) + except ValueError as error: + raise ValueError(f"{error}: {target_path}") from error if len(label_indices) and int(label_indices[-1]) >= len(targets): raise ValueError( f"Last label index {int(label_indices[-1])} exceeds target length " @@ -823,6 +850,20 @@ def load_session_data( "video_sampling_grid", ) } + expected_background = targets[:, 1:].sum(dim=-1) == 0 + canonical_rows = targets[:, 0].bool() == expected_background + unique_rows, row_counts = torch.unique( + targets.to(dtype=torch.int64), + dim=0, + return_counts=True, + ) + metadata["num_noncanonical_target_rows"] = int( + (~canonical_rows).sum().item() + ) + metadata["target_row_pattern_counts"] = { + "".join(str(int(value)) for value in row.tolist()): int(count) + for row, count in zip(unique_rows, row_counts) + } local = None if local_feature_root is not None: from local_turn_features import load_local_features diff --git a/run_ego4d_local_baselines.py b/run_ego4d_local_baselines.py index c4d619e..4694403 100644 --- a/run_ego4d_local_baselines.py +++ b/run_ego4d_local_baselines.py @@ -93,6 +93,8 @@ def preflight(args) -> str: shape, contract = None, None counts = {"visual": [0, 0], "audio": [0, 0]} total = 0 + noncanonical_target_rows = 0 + target_row_pattern_counts: dict[str, int] = {} for index, name in enumerate(names, 1): data = load_session_data( args.feature_root, args.target_root, name, dataset="ego4d", @@ -117,6 +119,15 @@ def preflight(args) -> str: counts[modality][1] += len(data.states) offset += width + 1 total += len(data.states) + noncanonical_target_rows += int( + data.feature_metadata.get("num_noncanonical_target_rows", 0) + ) + for pattern, count in data.feature_metadata.get( + "target_row_pattern_counts", {} + ).items(): + target_row_pattern_counts[pattern] = ( + target_row_pattern_counts.get(pattern, 0) + int(count) + ) del data # Validate one clip at a time, including all test clips. if not args.no_progress and (index == 1 or index % 25 == 0 or index == len(names)): print(f"preflight: {index}/{len(names)} clips", flush=True) @@ -128,6 +139,19 @@ def preflight(args) -> str: if fusion != "single" and len(shape) != 2: raise ValueError(f"--layer-fusion {fusion} requires stacked [T,L,D] global states") print(f"Validated {total} states; global shape per row={shape}; fusion={fusion}", flush=True) + if noncanonical_target_rows: + canonical_patterns = {"100", "010", "001", "011"} + details = ", ".join( + f"[{','.join(pattern)}]={count}" + for pattern, count in sorted(target_row_pattern_counts.items()) + if pattern not in canonical_patterns + ) + print( + "Ego4D binary target rows outside the EasyCom four-state " + f"constraint: {noncanonical_target_rows}/{total} ({details}); " + "preserved exactly for the official LSTR MCE objective.", + flush=True, + ) for modality, (valid, count) in counts.items(): if count: note = "; provided boxes, not an online detector" if modality == "visual" else "" diff --git a/test_ego4d_local_workflow.py b/test_ego4d_local_workflow.py index d538a8e..879e3b1 100644 --- a/test_ego4d_local_workflow.py +++ b/test_ego4d_local_workflow.py @@ -181,6 +181,30 @@ class Ego4DLocalWorkflowTests(unittest.TestCase): if name != "baseline_seed0": self.assertEqual(train[train.index("--local-modalities") + 1], name.split("_")[0]) + def test_ego4d_noncanonical_binary_rows_are_preserved(self): + target_path = self.root / "targets" / f"{self.names[0]}.npy" + targets = np.load(target_path) + targets[3] = 0 + targets[4] = [1, 1, 0] + np.save(target_path, targets) + + data = load_session_data( + self.root / "features", + self.root / "targets", + self.names[0], + dataset="ego4d", + expected_mode="open-stream-replay", + ) + self.assertTrue(torch.equal(data.targets[3], torch.zeros(3))) + self.assertTrue( + torch.equal(data.targets[4], torch.tensor([1.0, 1.0, 0.0])) + ) + self.assertEqual(data.feature_metadata["num_noncanonical_target_rows"], 2) + output = io.StringIO() + with contextlib.redirect_stdout(output): + runner.preflight(runner.parse_args(self.runner_args(["--variants", "baseline"]))) + self.assertIn("constraint: 2/60 ([0,0,0]=1, [1,1,0]=1)", output.getvalue()) + def test_family_preflight_catches_missing_test_sidecar_before_training(self): (self.root / "local" / f"{self.names[1]}.pt").unlink() with patch("run_ego4d_local_baselines.subprocess.run") as run: @@ -219,6 +243,13 @@ class Ego4DLocalWorkflowTests(unittest.TestCase): self.assertFalse((self.root / "checkpoints").exists()) def test_full_cpu_family_round_trip_and_summary(self): + # EgoSpeak's LSTR MCE consumes the released Ego4D rows as binary + # weights. Preserve noncanonical rows, and do not derive transitions + # when that auxiliary objective is disabled. + target_path = self.root / "targets" / f"{self.names[0]}.npy" + targets = np.load(target_path) + targets[3] = 0 + np.save(target_path, targets) self.split_feature_layout() env = dict(os.environ, OMP_NUM_THREADS="1", MKL_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1") completed = subprocess.run([sys.executable, str(runner.REPO / "run_ego4d_local_baselines.py"), diff --git a/train_turn_head.py b/train_turn_head.py index 8e640ed..9c850d4 100644 --- a/train_turn_head.py +++ b/train_turn_head.py @@ -224,6 +224,7 @@ def _load_examples( local_feature_root: Path | None = None, local_modalities: str = "av", local_contract: dict[str, Any] | None = None, + compute_transition_targets: bool = True, ) -> tuple[ torch.Tensor, torch.Tensor, @@ -259,11 +260,19 @@ def _load_examples( if data.local_features is not None else torch.empty(len(states), 0) ) - activity_sequence = torch.cat( - (data.targets[current_indices, None, :], targets), - dim=1, - ) - transition_targets = transition_targets_from_activity(activity_sequence) + if compute_transition_targets: + activity_sequence = torch.cat( + (data.targets[current_indices, None, :], targets), + dim=1, + ) + transition_targets = transition_targets_from_activity( + activity_sequence + ) + else: + transition_targets = torch.zeros( + targets.shape[:-1], + dtype=targets.dtype, + ) dense_example_count = len(states) if selected_indices is not None: if name not in selected_indices: @@ -605,6 +614,7 @@ def main() -> None: sparse_anchor_indices, args.local_feature_root, args.local_modalities, + compute_transition_targets=args.transition_objective != "none", ) local_contract = feature_metadata[0].get("local_contract") if not len(states): @@ -613,11 +623,15 @@ def main() -> None: f"training examples: {len(states)}, feature_dim={hidden_size}, " f"input_layers={num_input_layers}" ) - print( - "transition targets: " - f"{int(transition_targets.sum().item())}/{transition_targets.numel()} " - f"positive ({float(transition_targets.mean()) * 100:.3f}%)" - ) + if args.transition_objective == "none": + print("transition targets: disabled (not derived or loaded)") + else: + print( + "transition targets: " + f"{int(transition_targets.sum().item())}/" + f"{transition_targets.numel()} positive " + f"({float(transition_targets.mean()) * 100:.3f}%)" + ) if num_input_layers > 1 and args.layer_fusion == "single": raise ValueError( f"Features contain {num_input_layers} stacked layers; pass " @@ -650,6 +664,7 @@ def main() -> None: args.local_feature_root, args.local_modalities, local_contract, + compute_transition_targets=args.transition_objective != "none", ) if dev_hidden_size != hidden_size: raise ValueError(