wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow explicitly audited Ego4D bbox tails

Browse Source
This commit is contained in:
Xin Wang
2026-08-28 12:17:02 +08:00
parent 4011ba9b2a
commit 4abbde4f52
4 changed files with 180 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
extract_turn_local_features.py
View File

@@ -27,8 +27,9 @@ from local_turn_features import (
TESTED_TRANSFORMERS_VERSIONS = ("4.52.3", "4.57.6")
# Verified Ego4D exports can retain one or two bbox frame indices after the
# media endpoint. This bounds frame positions, not the number of people/boxes.
# Conservative default for previously verified exports. Longer, audited tails
# require an explicit CLI limit; endpoint/duration checks remain mandatory.
# This bounds frame positions, not the number of people/boxes.
EGO4D_MAX_TERMINAL_FRAME_OVERRUN = 2
@@ -193,14 +194,23 @@ def load_ego4d_boxes(paths: list[Path], names: list[str]) -> dict[str, Any]:
return result
def validate_ego4d_box_frames(name: str, annotation: dict[str, Any], source):
def validate_ego4d_box_frames(
name: str, annotation: dict[str, Any], source, *,
max_terminal_frame_overrun: int = EGO4D_MAX_TERMINAL_FRAME_OVERRUN,
):
"""Ignore a bounded, boundary-confirmed tail; never shift/clamp boxes."""
if (isinstance(max_terminal_frame_overrun, bool)
or not isinstance(max_terminal_frame_overrun, int)
or max_terminal_frame_overrun < 0):
raise ValueError("max_terminal_frame_overrun must be a nonnegative integer")
frames, bounds = annotation["frames"], annotation["bounds"]
count = source.num_frames
outside = sorted(frame for frame in frames if not 0 <= frame < count)
terminal_overrun = max(0, max(frames, default=-1) - count + 1)
audit = {
"policy": "clip-local-zero-based; ignore-confirmed-tail-up-to-2-frames",
"max_terminal_frame_overrun": EGO4D_MAX_TERMINAL_FRAME_OVERRUN,
"policy": f"clip-local-zero-based; ignore-confirmed-tail-up-to-{max_terminal_frame_overrun}-frames",
"max_terminal_frame_overrun": max_terminal_frame_overrun,
"annotation_terminal_overrun_frames": terminal_overrun,
"source_frames": count, "source_fps": source.source_fps,
"declared_bounds": dict(bounds),
"annotation_frame_range": [min(frames), max(frames)] if frames else None,
@@ -215,9 +225,10 @@ def validate_ego4d_box_frames(name: str, annotation: dict[str, Any], source):
and source.source_fps > 0 and abs(start_sec) <= 1e-6
and abs((end_sec - start_sec) - count / source.source_fps) <= 1e-4
)
# Allow only N and N+1. Counting outliers alone would accidentally permit
# sparse but arbitrarily distant boxes; the maximum index must be bounded.
small_terminal_tail = count <= outside[0] <= outside[-1] < count + EGO4D_MAX_TERMINAL_FRAME_OVERRUN
# The default allows N and N+1. An explicit larger limit authorizes only
# that bounded tail, never arbitrary outliers. Counting outliers alone
# would accidentally permit sparse but arbitrarily distant boxes.
small_terminal_tail = count <= outside[0] <= outside[-1] < count + max_terminal_frame_overrun
if (not small_terminal_tail or bounds.get("clip_start_frame") != 0
or bounds.get("clip_end_frame") != count or not duration_matches):
raise ValueError(
@@ -225,10 +236,13 @@ def validate_ego4d_box_frames(name: str, annotation: dict[str, Any], source):
f"source_frames={count}, source_fps={source.source_fps}, "
f"annotation_range={audit['annotation_frame_range']}, "
f"out_of_range_count={len(outside)}, first_10={outside[:10]}, "
f"declared_bounds={bounds}. Only terminal indices in "
f"[{count}, {count + EGO4D_MAX_TERMINAL_FRAME_OVERRUN - 1}] "
"with source_frames == clip_end_frame, zero origin and matching duration are allowed; "
"check annotation/video alignment instead of clamping or skipping the clip."
f"declared_bounds={bounds}; terminal_overrun_frames={terminal_overrun}. "
f"Current tail limit is {max_terminal_frame_overrun} frame(s): require "
f"0 <= frame - {count} < {max_terminal_frame_overrun}, "
"source_frames == clip_end_frame, zero origin and matching duration. "
"A larger audited tail requires explicit --ego4d-max-terminal-overrun-frames; "
"this never bypasses endpoint/duration checks. Check annotation/video alignment "
"instead of clamping or skipping the clip."
)
ignored_boxes = sum(len(frames[frame]) for frame in outside)
audit.update(ignored_frame_indices=outside, ignored_box_count=ignored_boxes)
@@ -264,7 +278,10 @@ def build_media_and_boxes(args: argparse.Namespace, name: str, end: float, ego_b
raise ValueError(f"Box/video frame inventory mismatch: {path}")
annotations.append(mapping)
else:
mapping, audit = validate_ego4d_box_frames(name, ego_boxes[name], media.clips[0])
mapping, audit = validate_ego4d_box_frames(
name, ego_boxes[name], media.clips[0],
max_terminal_frame_overrun=args.ego4d_max_terminal_overrun_frames,
)
annotations = [mapping]
ego_boxes[name]["frame_audit"] = audit
return media, annotations
@@ -517,6 +534,12 @@ def parse_args(argv=None):
p.add_argument("--num-shards", type=int, default=1, help="Disjoint round-robin shards after all sample selection")
p.add_argument("--shard-index", type=int, default=0, help="Zero-based shard index")
p.add_argument("--ego4d-annotations", type=Path, nargs="+")
p.add_argument(
"--ego4d-max-terminal-overrun-frames", type=int,
default=EGO4D_MAX_TERMINAL_FRAME_OVERRUN,
help=("Maximum audited bbox tail beyond the decoded Ego4D endpoint (default: 2; 0: reject all). "
"Requires matching declared frame/time bounds; never shifts boxes or removes target rows."),
)
p.add_argument("--model-id", default="Qwen/Qwen2.5-Omni-3B")
p.add_argument("--encoder-id", help="Stable checkpoint identity for relocated copies; default: model-id")
p.add_argument("--device", default="cuda:0")
@@ -551,6 +574,11 @@ def parse_args(argv=None):
p.add_argument("--dry-run", action="store_true", help="Validate references/targets and list work; no model, media decode or writes")
p.add_argument("--no-progress", action="store_true")
args = p.parse_args(argv)
if args.ego4d_max_terminal_overrun_frames < 0:
p.error("--ego4d-max-terminal-overrun-frames must be nonnegative")
if (args.dataset != "ego4d"
and args.ego4d_max_terminal_overrun_frames != EGO4D_MAX_TERMINAL_FRAME_OVERRUN):
p.error("--ego4d-max-terminal-overrun-frames is only supported with --dataset ego4d")
if args.num_shards <= 0 or not 0 <= args.shard_index < args.num_shards:
p.error("Require --num-shards > 0 and 0 <= --shard-index < --num-shards")
if args.manifest is not None and args.dataset != "ego4d":