Handle verified Ego4D terminal bounding-box records

docs/ego4d_local_baselines.md

@@ -173,6 +173,24 @@ worker 及其进程组，不杀无关训练。已完成的 `.pt` 保留，重跑
 8 workers 也会增加 CPU、主机 RAM、磁盘 I/O 和标注 JSON 的解析开销；不要预设能达到 8 倍速度。
 主机内存不足时改为 `--gpus 0 1 2 3` 并重跑，完成的文件仍可复用；不同次分片也不会改写已完成特征。
 
+#### 已核实的末尾 bbox 记录
+
+2026-08-28 在 `31488ff6-05ef-4362-9786-35981b06f307` 上核对到：
+原视频 8999 帧、30 FPS，有效索引为 `0..8998`；标注从 0 开始，
+`clip_end_frame=8999`、`clip_end_sec=299.9666666666666`，唯一越界索引是 `8999`。
+这是无对应解码帧的末尾记录，不应把该框移到 `8998`，也不应将所有框减 1。
+
+提取器只在以下条件同时满足时忽略末尾记录：唯一越界索引为 `N`，
+`N=视频帧数=clip_end_frame`，标注帧起点为 0、时间起点在 1 微秒内为 0，
+且标注时长与 `N / source_fps` 相差不超过 0.1 毫秒。
+缺少确认边界、较大越界或时长不匹配仍会报错，并打印具体范围；不自动跳过 clip。
+
+日志显示忽略的末尾索引和框数；新 `.pt` 的 `box_frame_audit` 与预览目录的
+`box_frame_audit.json` 保存原始范围、视频帧数、标注边界和忽略记录。
+不修改原视频、JSON、target 或已完成的特征。本例仍输出 1499 行，对应截止时间
+`0.2..299.8s`，2 FPS 的最后一个输入采样时刻仍为 `299.5s`。
+该处理不改编码器/池化配置，之前完成且通过校验的特征可以继续复用；同步修复后重跑原 8 卡命令即可。
+
 提取完成后，原训练入口也直接用同一个原始全局特征根，无需 flat 索引：
 
 ```bash
@@ -247,10 +265,11 @@ python summarize_local_baselines.py \
 ## 回归测试
 
 ```bash
-python -m unittest -q test_easycom_head_objectives test_local_turn_features test_ego4d_local_workflow test_ego4d_local_multigpu
+python -m unittest -q test_easycom_head_objectives test_local_turn_features test_ego4d_local_workflow test_ego4d_local_multigpu test_ego4d_box_frames
 ```
 
 CPU 回归包括 manifest 顺序/防交叉重复、完整批量提取的模型复用、断点文件校验、
 单层/多层全局特征、测试集输入预检，以及合成 Ego4D 上四组各 4 epochs 的训练/推理与增量汇总。
 另测试原 train/test 目录、433 clips 的 8 卡完整互斥分片、GPU mask、独立日志、失败/中断时清理自有进程。
+末尾框回归覆盖严格边界验证、其它越界拒绝、1499 行完整提取、审计保存和旧特征续跑。
 真实 Ego4D 像素、提供框和 3B/L20 提取仍需按上面的 preview/smoke 步骤验证。

extract_turn_local_features.py

@@ -175,7 +175,14 @@ def load_ego4d_boxes(paths: list[Path], names: list[str]) -> dict[str, Any]:
                 if uid in selected:
                     if uid in result:
                         raise ValueError(f"Duplicate Ego4D annotation UID: {uid}")
-                    result[uid] = ego4d_clip_boxes(clip)
+                    # Retain declared bounds so a terminal record can be
+                    # distinguished from a truncated video or frame mismatch.
+                    result[uid] = {
+                        "frames": ego4d_clip_boxes(clip),
+                        "bounds": {key: clip.get(key) for key in (
+                            "clip_start_frame", "clip_end_frame", "clip_start_sec", "clip_end_sec",
+                        )},
+                    }
         del payload
     missing = set(names) - result.keys()
     if missing:
@@ -183,6 +190,45 @@ def load_ego4d_boxes(paths: list[Path], names: list[str]) -> dict[str, Any]:
     return result
 
 
+def validate_ego4d_box_frames(name: str, annotation: dict[str, Any], source):
+    """Ignore only a confirmed exclusive-end record; never shift/clamp boxes."""
+    frames, bounds = annotation["frames"], annotation["bounds"]
+    count = source.num_frames
+    outside = sorted(frame for frame in frames if not 0 <= frame < count)
+    audit = {
+        "policy": "clip-local-zero-based; ignore-confirmed-exclusive-end-only",
+        "source_frames": count, "source_fps": source.source_fps,
+        "declared_bounds": dict(bounds),
+        "annotation_frame_range": [min(frames), max(frames)] if frames else None,
+        "ignored_frame_indices": [], "ignored_box_count": 0,
+    }
+    if not outside:
+        return frames, audit
+    start_sec, end_sec = bounds.get("clip_start_sec"), bounds.get("clip_end_sec")
+    duration_matches = (
+        isinstance(start_sec, (int, float)) and isinstance(end_sec, (int, float))
+        and np.isfinite([start_sec, end_sec, source.source_fps]).all()
+        and source.source_fps > 0 and abs(start_sec) <= 1e-6
+        and abs((end_sec - start_sec) - count / source.source_fps) <= 1e-4
+    )
+    if (outside != [count] or bounds.get("clip_start_frame") != 0
+            or bounds.get("clip_end_frame") != count or not duration_matches):
+        raise ValueError(
+            f"Ego4D annotation frame exceeds source clip: {name}; "
+            f"source_frames={count}, source_fps={source.source_fps}, "
+            f"annotation_range={audit['annotation_frame_range']}, "
+            f"out_of_range_count={len(outside)}, first_10={outside[:10]}, "
+            f"declared_bounds={bounds}. Only frame == source_frames == clip_end_frame "
+            "with zero-origin, matching duration is an allowed terminal record; "
+            "check annotation/video alignment instead of clamping or skipping the clip."
+        )
+    audit.update(ignored_frame_indices=[count], ignored_box_count=len(frames[count]))
+    print(f"{name}: ignoring exclusive-end bbox record at frame {count} "
+          f"({len(frames[count])} boxes); decoded frames are 0..{count - 1}. "
+          "No frame shift or feature/target row removal.", flush=True)
+    return {frame: boxes for frame, boxes in frames.items() if frame != count}, audit
+
+
 def build_media_and_boxes(args: argparse.Namespace, name: str, end: float, ego_boxes: dict[str, Any]):
     # Lazy imports allow --help and pure tests without Transformers/decoders.
     from streaming_turn_model_directly import CausalSourceAV, SourceClipSpec, _easycom_directory_specs
@@ -209,9 +255,9 @@ def build_media_and_boxes(args: argparse.Namespace, name: str, end: float, ego_b
                 raise ValueError(f"Box/video frame inventory mismatch: {path}")
             annotations.append(mapping)
     else:
-        annotations = [ego_boxes[name]]
-        if annotations[0] and max(annotations[0]) >= media.clips[0].num_frames:
-            raise ValueError(f"Ego4D annotation frame exceeds source clip: {name}")
+        mapping, audit = validate_ego4d_box_frames(name, ego_boxes[name], media.clips[0])
+        annotations = [mapping]
+        ego_boxes[name]["frame_audit"] = audit
     return media, annotations
 
 
@@ -622,6 +668,9 @@ def main(argv=None) -> None:
                   "annotations use --box-coordinate-space encoded; otherwise boxes shift right.", flush=True)
         if args.preview_only:
             preview_boxes(media, annotations, args, name, len(times) / 5)
+            if args.dataset == "ego4d":
+                audit_path = args.output_root / f"{name}_preview" / "box_frame_audit.json"
+                audit_path.write_text(json.dumps(ego_boxes[name]["frame_audit"], indent=2) + "\n")
             ego_boxes.pop(name, None)
             del media, annotations, reference
             continue
@@ -643,6 +692,10 @@ def main(argv=None) -> None:
             "reference_features": str(path.resolve()), "direct_media": media.metadata(),
             "config": config,
         })
+        if args.dataset == "ego4d":
+            # Per-clip diagnostics, not encoder config: existing valid sidecars
+            # retain the same reuse contract and need not be re-extracted.
+            output["box_frame_audit"] = ego_boxes[name]["frame_audit"]
         destination = args.output_root / f"{name}.pt"
         # Atomic replacement: a failed extraction never looks like a ready file.
         with tempfile.NamedTemporaryFile(dir=args.output_root, prefix=f".{name}.", suffix=".tmp", delete=False) as handle:

test_ego4d_box_frames.py

@@ -0,0 +1,163 @@
+"""CPU regressions for a verified Ego4D exclusive-end bbox record."""
+
+from __future__ import annotations
+
+import contextlib
+import copy
+import io
+import json
+from pathlib import Path
+import sys
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+import numpy as np
+from PIL import Image
+import torch
+
+import extract_turn_local_features as extract
+from local_turn_features import load_local_features
+from test_local_turn_features import fixture
+
+
+UID = "31488ff6-05ef-4362-9786-35981b06f307"
+
+
+def terminal_clip():
+    return {
+        "clip_uid": UID, "clip_start_frame": 0, "clip_end_frame": 8999,
+        "clip_start_sec": 0, "clip_end_sec": 299.9666666666666,
+        "persons": [{"person_id": "p1", "tracking_paths": [{"track": [
+            {"frame": frame, "clip_frame": None, "x": 0, "y": 0, "width": 2, "height": 2}
+            for frame in (0, 8985, 8998, 8999)
+        ]}]}],
+    }
+
+
+def terminal_annotation():
+    clip = terminal_clip()
+    return {"frames": extract.ego4d_clip_boxes(clip),
+            "bounds": {key: clip[key] for key in (
+                "clip_start_frame", "clip_end_frame", "clip_start_sec", "clip_end_sec",
+            )}}
+
+
+class Ego4DBoxFrameTests(unittest.TestCase):
+    def setUp(self):
+        torch.set_num_threads(1)
+        self.source = SimpleNamespace(num_frames=8999, source_fps=30.)
+
+    def test_confirmed_terminal_only_is_removed_without_mutating_input(self):
+        annotation = terminal_annotation()
+        original = copy.deepcopy(annotation)
+        with contextlib.redirect_stdout(io.StringIO()) as log:
+            frames, audit = extract.validate_ego4d_box_frames(UID, annotation, self.source)
+        self.assertEqual(frames, {frame: original["frames"][frame] for frame in (0, 8985, 8998)})
+        self.assertEqual(annotation, original)
+        self.assertEqual(audit["ignored_frame_indices"], [8999])
+        self.assertEqual(audit["ignored_box_count"], 1)
+        self.assertEqual(audit["annotation_frame_range"], [0, 8999])
+        self.assertIn("No frame shift or feature/target row removal", log.getvalue())
+
+    def test_valid_and_empty_mappings_are_unchanged_even_without_bounds(self):
+        for frames in ({}, {0: [], 8998: [{"id": "valid"}]}):
+            with self.subTest(frames=frames):
+                result, audit = extract.validate_ego4d_box_frames(
+                    UID, {"frames": frames, "bounds": {}}, self.source)
+                self.assertIs(result, frames)
+                self.assertEqual(audit["ignored_frame_indices"], [])
+
+    def test_larger_overrun_negative_and_truncated_source_still_fail(self):
+        for extra in (-1, 9000, 12000):
+            annotation = terminal_annotation()
+            annotation["frames"][extra] = []
+            with self.subTest(frame=extra), self.assertRaisesRegex(ValueError, "source_frames=8999"):
+                extract.validate_ego4d_box_frames(UID, annotation, self.source)
+        with self.assertRaisesRegex(ValueError, "source_frames=8998"):
+            extract.validate_ego4d_box_frames(
+                UID, terminal_annotation(), SimpleNamespace(num_frames=8998, source_fps=30.))
+
+    def test_terminal_requires_zero_origin_matching_declared_end_and_duration(self):
+        for key, value in (("clip_start_frame", 1), ("clip_end_frame", 9000),
+                           ("clip_start_sec", .1), ("clip_end_sec", 300.),
+                           ("clip_end_sec", float("nan"))):
+            annotation = terminal_annotation()
+            annotation["bounds"][key] = value
+            with self.subTest(key=key, value=value), self.assertRaisesRegex(ValueError, "declared_bounds"):
+                extract.validate_ego4d_box_frames(UID, annotation, self.source)
+        for key in terminal_annotation()["bounds"]:
+            annotation = terminal_annotation()
+            del annotation["bounds"][key]
+            with self.subTest(missing=key), self.assertRaises(ValueError):
+                extract.validate_ego4d_box_frames(UID, annotation, self.source)
+        for fps in (29.97, 15., 0., float("nan")):
+            with self.subTest(fps=fps), self.assertRaises(ValueError):
+                extract.validate_ego4d_box_frames(
+                    UID, terminal_annotation(), SimpleNamespace(num_frames=8999, source_fps=fps))
+
+    def test_full_extraction_keeps_1499_rows_saves_audit_and_reuses(self):
+        image = Image.new("RGB", (2, 2))
+        source = SimpleNamespace(num_frames=8999, source_fps=30., crop=None, dense_start=0.,
+                                 path=Path(f"{UID}.mp4"), width=2, height=2)
+        media = SimpleNamespace(
+            clips=[source], metadata=lambda: {},
+            _source_frame_for_time=lambda t: (0, int(t * 30 + 1e-8), int(t * 30 + 1e-8) / 30),
+            frame_at_or_before=lambda t: image,
+            audio_slice=lambda start, end: np.array([start, end]),
+        )
+        fake_module = SimpleNamespace(CausalSourceAV=lambda *a, **kw: media,
+                                      SourceClipSpec=lambda path: path, _easycom_directory_specs=None)
+        encoder = SimpleNamespace(
+            visual_dim=6, audio_dim=5, transformers_version="4.57.6", model_config_sha256="test",
+            images=lambda crops: torch.ones(len(crops), 6),
+            audio=lambda waveform: torch.full((5,), float(waveform[-1])),
+        )
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            for folder in ("global", "targets", "v1/clips", "v1/annotations"):
+                (root / folder).mkdir(parents=True)
+            reference, _, targets = fixture(n=1499, name=UID, dataset="ego4d")
+            torch.save(reference, root / "global" / f"{UID}.pt")
+            np.save(root / "targets" / f"{UID}.npy", targets)
+            (root / "v1/clips" / f"{UID}.mp4").write_bytes(b"fake-video")
+            for split, clips in (("train", [terminal_clip()]), ("val", [])):
+                (root / f"v1/annotations/av_{split}.json").write_text(json.dumps({"videos": [{"clips": clips}]}))
+            loaded = extract.load_ego4d_boxes([root / "v1/annotations/av_train.json"], [UID])
+            self.assertEqual(loaded[UID], terminal_annotation())
+            argv = ["--dataset", "ego4d", "--data-root", str(root), "--feature-root", str(root / "global"),
+                    "--target-root", str(root / "targets"), "--output-root", str(root / "local"),
+                    "--samples", UID, "--no-progress"]
+            with patch.dict(sys.modules, {"transformers": SimpleNamespace(__version__="4.57.6"),
+                                          "streaming_turn_model_directly": fake_module}), \
+                 patch.object(extract, "FrozenQwenLocalEncoder", return_value=encoder) as model, \
+                 contextlib.redirect_stdout(io.StringIO()):
+                extract.main(argv)
+                model.assert_called_once()
+                destination = root / "local" / f"{UID}.pt"
+                result = torch.load(destination, weights_only=False)
+                self.assertEqual(result["visual"].shape, (1499, 6))
+                self.assertEqual(result["audio"].shape, (1499, 5))
+                self.assertTrue(torch.equal(result["times"], reference["times"]))
+                self.assertTrue(torch.equal(result["label_indices"], reference["label_indices"]))
+                self.assertAlmostEqual(result["times"][-1].item(), 299.8)
+                self.assertFalse(result["partial"])
+                self.assertEqual(result["box_frame_audit"]["ignored_frame_indices"], [8999])
+                self.assertEqual(result["persons"]["frame_times"][-1].item(), 299.5)
+                self.assertEqual(result["persons"]["records"][-1]["source_frame_index"], 8985)
+                load_local_features(destination, reference, sample_id=UID, dataset="ego4d", modalities="av")
+                # Previously completed valid sidecars have no audit. Their
+                # vectors/config remain reusable: only unsampled boxes changed.
+                del result["box_frame_audit"]
+                torch.save(result, destination)
+                model.reset_mock()
+                extract.main(argv + ["--reuse-existing"])
+                model.assert_not_called()
+                extract.main(argv + ["--preview-only", "--preview-count", "1"])
+                audit = json.loads((root / "local" / f"{UID}_preview/box_frame_audit.json").read_text())
+                self.assertEqual(audit["ignored_frame_indices"], [8999])
+
+
+if __name__ == "__main__":
+    unittest.main()

test_ego4d_local_workflow.py

@@ -138,9 +138,14 @@ class Ego4DLocalWorkflowTests(unittest.TestCase):
                                        model_config_sha256="same-checkpoint")
         _, side, _ = fixture(dataset="ego4d")
         media = SimpleNamespace(clips=[], metadata=lambda: {})
+
+        def fake_build(args, name, end, annotations):
+            annotations[name]["frame_audit"] = {"ignored_frame_indices": []}
+            return media, []
+
         with patch.dict(sys.modules, {"transformers": SimpleNamespace(__version__="4.57.6")}), \
              patch("extract_turn_local_features.FrozenQwenLocalEncoder", return_value=fake_encoder) as encoder, \
-             patch("extract_turn_local_features.build_media_and_boxes", return_value=(media, [])) as build, \
+             patch("extract_turn_local_features.build_media_and_boxes", side_effect=fake_build) as build, \
              patch("extract_turn_local_features.extract_local", side_effect=lambda *a: copy.deepcopy(side)):
             extract.main(self.extraction_args())
             encoder.assert_called_once()