wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow explicitly audited Ego4D bbox tails

Browse Source
This commit is contained in:
Xin Wang
2026-08-28 12:17:02 +08:00
parent 4011ba9b2a
commit 4abbde4f52
4 changed files with 180 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
test_ego4d_box_frames.py
View File

@@ -1,4 +1,4 @@
"""CPU regressions for verified one/two-frame Ego4D terminal bbox overruns."""
"""CPU regressions for bounded, explicitly authorized Ego4D bbox tails."""
from __future__ import annotations
@@ -24,15 +24,19 @@ from test_local_turn_features import fixture
UID = "31488ff6-05ef-4362-9786-35981b06f307"
TWO_FRAME_UID = "ac7d3d16-d74d-4fe3-a1e0-cab0b1adf492"
THIRTEEN_FRAME_UID = "d2cb4045-8ef1-4288-8398-ab6d4d6a0709"
def terminal_clip(*, uid=UID, count=8999, tail_frames=1):
num_steps = count // 6 # 30 FPS source, 5 Hz targets.
num_video_frames = (num_steps * 2 + 4) // 5 # ceil(duration * 2 FPS).
last_grid_frame = (num_video_frames - 1) * 15
return {
"clip_uid": uid, "clip_start_frame": 0, "clip_end_frame": count,
"clip_start_sec": 0, "clip_end_sec": count / 30.,
"persons": [{"person_id": "p1", "tracking_paths": [{"track": [
{"frame": frame, "clip_frame": None, "x": 0, "y": 0, "width": 2, "height": 2}
for frame in (0, 8985, count - 1, *range(count, count + tail_frames))
for frame in (0, last_grid_frame, count - 1, *range(count, count + tail_frames))
]}]}],
}
@@ -123,13 +127,70 @@ class Ego4DBoxFrameTests(unittest.TestCase):
extract.validate_ego4d_box_frames(
UID, terminal_annotation(), SimpleNamespace(num_frames=8999, source_fps=fps))
def test_thirteen_frame_tail_requires_explicit_limit_and_keeps_valid_boxes(self):
annotation = terminal_annotation(uid=THIRTEEN_FRAME_UID, count=8987, tail_frames=13)
original = copy.deepcopy(annotation)
source = SimpleNamespace(num_frames=8987, source_fps=30.)
with self.assertRaisesRegex(ValueError, "terminal_overrun_frames=13"):
extract.validate_ego4d_box_frames(THIRTEEN_FRAME_UID, annotation, source)
with contextlib.redirect_stdout(io.StringIO()):
frames, audit = extract.validate_ego4d_box_frames(
THIRTEEN_FRAME_UID, annotation, source, max_terminal_frame_overrun=13)
self.assertEqual(annotation, original)
self.assertEqual(frames, {frame: boxes for frame, boxes in original["frames"].items() if frame < 8987})
self.assertIn(8986, frames) # Never discard/replace the last valid frame.
self.assertEqual(audit["ignored_frame_indices"], list(range(8987, 9000)))
self.assertEqual(audit["ignored_box_count"], 13)
self.assertEqual(audit["max_terminal_frame_overrun"], 13)
self.assertEqual(audit["annotation_terminal_overrun_frames"], 13)
def test_explicit_limit_does_not_bypass_endpoint_duration_or_far_outliers(self):
source = SimpleNamespace(num_frames=8987, source_fps=30.)
for key, value in (("clip_start_frame", 1), ("clip_end_frame", 9000),
("clip_start_sec", .1), ("clip_end_sec", 300.),
("clip_end_sec", float("nan"))):
annotation = terminal_annotation(count=8987, tail_frames=13)
annotation["bounds"][key] = value
with self.subTest(key=key, value=value), self.assertRaises(ValueError):
extract.validate_ego4d_box_frames(
THIRTEEN_FRAME_UID, annotation, source, max_terminal_frame_overrun=13)
for frame in (-1, 9000, 12000):
annotation = terminal_annotation(count=8987, tail_frames=0)
annotation["frames"][frame] = []
with self.subTest(frame=frame), self.assertRaises(ValueError):
extract.validate_ego4d_box_frames(
THIRTEEN_FRAME_UID, annotation, source, max_terminal_frame_overrun=13)
with self.assertRaises(ValueError):
extract.validate_ego4d_box_frames(
THIRTEEN_FRAME_UID, terminal_annotation(count=8987, tail_frames=13),
SimpleNamespace(num_frames=8986, source_fps=30.), max_terminal_frame_overrun=13)
def test_zero_limit_disables_tail_compatibility_and_invalid_limits_fail(self):
with self.assertRaisesRegex(ValueError, "Current tail limit is 0"):
extract.validate_ego4d_box_frames(
UID, terminal_annotation(), self.source, max_terminal_frame_overrun=0)
annotation = terminal_annotation(tail_frames=0)
frames, audit = extract.validate_ego4d_box_frames(
UID, annotation, self.source, max_terminal_frame_overrun=0)
self.assertIs(frames, annotation["frames"])
self.assertEqual(audit["ignored_frame_indices"], [])
for limit in (-1, 1.5, True):
with self.subTest(limit=limit), self.assertRaises(ValueError):
extract.validate_ego4d_box_frames(
UID, annotation, self.source, max_terminal_frame_overrun=limit)
def test_full_extraction_keeps_1499_rows_saves_audit_and_reuses(self):
self._assert_full_extraction_and_reuse(UID, 8999, 1)
def test_two_frame_tail_extraction_matches_manual_removal_of_unsampled_boxes(self):
self._assert_full_extraction_and_reuse(TWO_FRAME_UID, 8998, 2)
def _assert_full_extraction_and_reuse(self, uid, count, tail_frames):
def test_thirteen_frame_tail_extraction_keeps_target_grid_and_reuses(self):
self._assert_full_extraction_and_reuse(THIRTEEN_FRAME_UID, 8987, 13, explicit_limit=13)
def _assert_full_extraction_and_reuse(self, uid, count, tail_frames, explicit_limit=None):
num_steps = count // 6
last_video_time = ((num_steps * 2 + 4) // 5 - 1) / 2
image = Image.new("RGB", (2, 2))
source = SimpleNamespace(num_frames=count, source_fps=30., crop=None, dense_start=0.,
path=Path(f"{uid}.mp4"), width=2, height=2)
@@ -150,7 +211,7 @@ class Ego4DBoxFrameTests(unittest.TestCase):
root = Path(temporary)
for folder in ("global", "targets", "v1/clips", "v1/annotations"):
(root / folder).mkdir(parents=True)
reference, _, targets = fixture(n=1499, name=uid, dataset="ego4d")
reference, _, targets = fixture(n=num_steps, name=uid, dataset="ego4d")
torch.save(reference, root / "global" / f"{uid}.pt")
np.save(root / "targets" / f"{uid}.npy", targets)
(root / "v1/clips" / f"{uid}.mp4").write_bytes(b"fake-video")
@@ -162,6 +223,10 @@ class Ego4DBoxFrameTests(unittest.TestCase):
argv = ["--dataset", "ego4d", "--data-root", str(root), "--feature-root", str(root / "global"),
"--target-root", str(root / "targets"), "--output-root", str(root / "local"),
"--samples", uid, "--no-progress"]
if explicit_limit is not None:
before = extract.requested_local_config(extract.parse_args(argv))
argv += ["--ego4d-max-terminal-overrun-frames", str(explicit_limit)]
self.assertEqual(extract.requested_local_config(extract.parse_args(argv)), before)
with patch.dict(sys.modules, {"transformers": SimpleNamespace(__version__="4.57.6"),
"streaming_turn_model_directly": fake_module}), \
patch.object(extract, "FrozenQwenLocalEncoder", return_value=encoder) as model, \
@@ -170,16 +235,17 @@ class Ego4DBoxFrameTests(unittest.TestCase):
model.assert_called_once()
destination = root / "local" / f"{uid}.pt"
result = torch.load(destination, weights_only=False)
self.assertEqual(result["visual"].shape, (1499, 6))
self.assertEqual(result["audio"].shape, (1499, 5))
self.assertEqual(result["visual"].shape, (num_steps, 6))
self.assertEqual(result["audio"].shape, (num_steps, 5))
self.assertTrue(torch.equal(result["times"], reference["times"]))
self.assertTrue(torch.equal(result["label_indices"], reference["label_indices"]))
self.assertAlmostEqual(result["times"][-1].item(), 299.8)
self.assertAlmostEqual(result["times"][-1].item(), num_steps / 5)
self.assertLessEqual(num_steps / 5, count / 30)
self.assertFalse(result["partial"])
ignored = list(range(count, count + tail_frames))
self.assertEqual(result["box_frame_audit"]["ignored_frame_indices"], ignored)
self.assertEqual(result["persons"]["frame_times"][-1].item(), 299.5)
self.assertEqual(result["persons"]["records"][-1]["source_frame_index"], 8985)
self.assertEqual(result["persons"]["frame_times"][-1].item(), last_video_time)
self.assertEqual(result["persons"]["records"][-1]["source_frame_index"], int(last_video_time * 30))
load_local_features(destination, reference, sample_id=uid, dataset="ego4d", modalities="av")
# Previously completed valid sidecars have no audit. Their
# vectors/config remain reusable: only unsampled boxes changed.