From 9bd204b468cfbc426c54f378f8691b0a31802575 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Fri, 28 Aug 2026 00:46:05 +0800 Subject: [PATCH] Add more preview --- docs/ego4d_local_baselines.md | 24 +++++++++ extract_turn_local_features.py | 87 +++++++++++++++++++++++++++--- test_local_turn_features.py | 96 +++++++++++++++++++++++++++++++++- 3 files changed, 200 insertions(+), 7 deletions(-) diff --git a/docs/ego4d_local_baselines.md b/docs/ego4d_local_baselines.md index 39d8c25..74a31cc 100644 --- a/docs/ego4d_local_baselines.md +++ b/docs/ego4d_local_baselines.md @@ -77,6 +77,30 @@ python extract_turn_local_features.py "${LOCAL_ARGS[@]}" \ 4s smoke 应得到 V/A 各 `[20,2048]`、`partial=True`；此目录不能用于正式训练。 V-valid 不必是 100%，Ego4D 有无可见人物和缺框的时刻；若有明显人物却长期无有效框，先停下来检查标注。 +### 前 4s 没有人物：增加预览范围和数量 + +旧预览固定最多 4 张；`--max-steps 20` 又把范围限制到前 4s。 +现在可指定数量和时间范围，并仅从有有效人物框的网格帧中均匀抽样： + +```bash +# 前三条 clip：在整条 clip 中各取最多 40 张有效带框图片。 +# 不要再带 --max-steps 20；不传 preview-end 就使用完整可用时间范围。 +python extract_turn_local_features.py "${LOCAL_ARGS[@]}" \ + --limit 3 --preview-only --preview-count 40 --preview-with-boxes \ + --output-root features/ego4d_local_preview_people + +# 也可聚焦第一条 clip 的 30–120s；实际不足 120s 时截到 clip 结束。 +python extract_turn_local_features.py "${LOCAL_ARGS[@]}" \ + --limit 1 --preview-only --preview-count 60 \ + --preview-start 30 --preview-end 120 --preview-with-boxes \ + --output-root features/ego4d_local_preview_30_120s +``` + +每张图标注时间、源帧号和有效人数；`preview.json` 记录候选帧数、有效帧数、实际输出数， +`boxes.json` 记录当前这次输出的图片和框。无有效框时明确报告 0 张，可换片段或核对帧号对齐， +不会伪造框或强行当作成功。`--preview-with-boxes` 只筛选预览，**不删除特征提取或评测中的无框帧**。 +默认仍最多 4 张。改变范围时建议换输出目录；旧 JPG 不会自动删除，以当前 JSON 清单为准。 + ## 3. 提取完整 manifest：前台、模型只加载一次 ```bash diff --git a/extract_turn_local_features.py b/extract_turn_local_features.py index 34cf926..3be96c4 100644 --- a/extract_turn_local_features.py +++ b/extract_turn_local_features.py @@ -264,15 +264,57 @@ def validate_source_timeline(reference: dict[str, Any], media) -> None: raise ValueError(f"Local/reference source timeline differs at {new['path']}") +def preview_time_bounds(args: argparse.Namespace, available_end: float) -> tuple[float, float]: + """Preview-only range on the original timeline, never a rebased feature grid.""" + start = float(getattr(args, "preview_start", 0.)) + requested_end = getattr(args, "preview_end", None) + end = min(available_end, requested_end) if requested_end is not None else available_end + if start >= end: + raise ValueError(f"Preview start {start:g}s is outside the available prefix ending at {end:g}s; " + "remove/increase --max-steps or choose an earlier --preview-start") + return start, end + + +def preview_ticks(media, annotations, args: argparse.Namespace, end: float) -> tuple[list[int], int, int]: + start, stop = preview_time_bounds(args, end) + ticks = [tick for tick in causal_frame_ticks(stop, args.video_fps) + if tick / args.video_fps >= start - 1e-8] + grid_count = len(ticks) + if getattr(args, "preview_with_boxes", False): + eligible = [] + # Scan annotation metadata, not every RGB frame. Use the same source + # frame mapping, coordinate correction and crop validity as extraction. + for tick in ticks: + clip_index, frame_index, _ = media._source_frame_for_time(tick / args.video_fps) + clip = media.clips[clip_index] + width, height = clip.crop[2:] if clip.crop is not None else (clip.width, clip.height) + offset_x, offset_y = box_coordinate_offset(clip, args.box_coordinate_space) + if any(crop_box(person["xyxy"], width, height, offset_x=offset_x, offset_y=offset_y, + expansion=args.crop_expansion) is not None + for person in annotations[clip_index].get(frame_index, [])): + eligible.append(tick) + ticks = eligible + eligible_count = len(ticks) + count = min(getattr(args, "preview_count", 4), eligible_count) + indices = np.linspace(0, eligible_count - 1, count, dtype=int) if count else [] + return [ticks[index] for index in indices], grid_count, eligible_count + + def preview_boxes(media, annotations, args: argparse.Namespace, name: str, end: float) -> None: from PIL import ImageDraw folder = args.output_root / f"{name}_preview" folder.mkdir(parents=True, exist_ok=True) - ticks = list(causal_frame_ticks(end, args.video_fps)) + start, stop = preview_time_bounds(args, end) + ticks, grid_count, eligible_count = preview_ticks(media, annotations, args, end) + print(f"Preview [{start:g},{stop:g})s: {grid_count} grid frames, {eligible_count} eligible, " + f"saving {len(ticks)} images", flush=True) + if not ticks: + print("No eligible preview frames. Try a longer range or another clip. " + "If people are visible but no valid boxes exist, check annotation/frame alignment.", flush=True) audit = [] - for index in np.linspace(0, len(ticks) - 1, min(4, len(ticks)), dtype=int): - time = ticks[index] / args.video_fps + for tick in ticks: + time = tick / args.video_fps image, _, records = frame_crops(media, annotations, time, args) clip_index, frame_index, _ = media._source_frame_for_time(time) clip = media.clips[clip_index] @@ -281,7 +323,10 @@ def preview_boxes(media, annotations, args: argparse.Namespace, name: str, end: for record in records: draw.rectangle(record["box"], outline="lime", width=4) draw.text(record["box"][:2], record["id"], fill="red") - filename = f"frame_{ticks[index]:06d}.jpg" + caption = f"t={time:.2f}s | source frame={frame_index} | people={len(records)}" + draw.rectangle(draw.textbbox((8, 8), caption), fill="black") + draw.text((8, 8), caption, fill="white") + filename = f"frame_{tick:06d}.jpg" image.save(folder / filename) audit.append({ "image": filename, "time": time, "source_clip": str(clip.path), @@ -291,9 +336,18 @@ def preview_boxes(media, annotations, args: argparse.Namespace, name: str, end: "coordinate_offset": list(offset), "boxes": records, }) print(f"preview t={time:g}s: {clip.width}x{clip.height} -> {image.width}x{image.height}; " - f"boxes={args.box_coordinate_space}, subtract offset={offset}") + f"boxes={args.box_coordinate_space}, subtract offset={offset}, people={len(records)}") (folder / "boxes.json").write_text(json.dumps(audit, indent=2) + "\n") + (folder / "preview.json").write_text(json.dumps({ + "start": start, "end": stop, "video_fps": args.video_fps, + "with_boxes_only": getattr(args, "preview_with_boxes", False), + "requested_count": getattr(args, "preview_count", 4), + "grid_frame_count": grid_count, "eligible_frame_count": eligible_count, + "saved_count": len(audit), "saved_with_boxes": sum(bool(row["boxes"]) for row in audit), + "images": [row["image"] for row in audit], + }, indent=2) + "\n") print(f"Box previews: {folder} (inspect before extracting all sessions)") + print("boxes.json lists this run's images; older images in the directory are not removed.") def extract_local(media, annotations, encoder, times: torch.Tensor, args: argparse.Namespace) -> dict[str, Any]: @@ -425,6 +479,11 @@ def parse_args(argv=None): p.add_argument("--decoder-gpu-id", type=int, default=0, help="Visible-device index, not physical GPU number") p.add_argument("--max-steps", type=int, help="Smoke test only; partial sidecars are rejected by training") p.add_argument("--preview-only", action="store_true", help="Save box overlays without loading the model") + p.add_argument("--preview-count", type=int, default=4, help="Maximum images per clip (preview only)") + p.add_argument("--preview-start", type=float, default=0., help="Start time in seconds on the clip/dense timeline") + p.add_argument("--preview-end", type=float, help="Exclusive end time; default: entire available prefix") + p.add_argument("--preview-with-boxes", action="store_true", + help="Sample only grid frames with valid provided crops; preview selection only, not feature filtering") existing = p.add_mutually_exclusive_group() existing.add_argument("--overwrite", action="store_true") existing.add_argument("--reuse-existing", action="store_true", help="Validate and skip complete matching sidecars") @@ -444,6 +503,19 @@ def parse_args(argv=None): p.error("--reuse-existing is for full extraction, not smoke tests or previews") if args.dry_run and args.preview_only: p.error("--dry-run and --preview-only are separate checks") + if args.preview_count <= 0: + p.error("--preview-count must be positive") + if not np.isfinite(args.preview_start) or args.preview_start < 0: + p.error("--preview-start must be finite and nonnegative") + if args.preview_end is not None and (not np.isfinite(args.preview_end) or args.preview_end <= args.preview_start): + p.error("--preview-end must be finite and greater than --preview-start") + if not args.preview_only and (args.preview_count != 4 or args.preview_start != 0. + or args.preview_end is not None or args.preview_with_boxes): + p.error("Preview selection options require --preview-only") + if args.preview_only and args.max_steps is not None: + if args.preview_start >= args.max_steps / 5 or (args.preview_end is not None + and args.preview_end > args.max_steps / 5): + p.error("Preview range exceeds --max-steps; remove --max-steps to inspect later frames") if args.box_coordinate_space is None: args.box_coordinate_space = "encoded" if args.dataset == "easycom" else "camera" for name in ("video_fps", "audio_window_seconds", "visual_window_seconds", "crop_expansion"): @@ -523,7 +595,10 @@ def main(argv=None) -> None: full_steps = len(times) if args.max_steps is not None: times, indices = times[:args.max_steps], indices[:args.max_steps] - media, annotations = build_media_and_boxes(args, name, len(times) / 5, ego_boxes) + media_end = len(times) / 5 + if args.preview_only: + _, media_end = preview_time_bounds(args, media_end) + media, annotations = build_media_and_boxes(args, name, media_end, ego_boxes) validate_source_timeline(reference, media) if (args.dataset == "easycom" and args.box_coordinate_space == "camera" and any(clip.crop is not None and clip.crop[0] for clip in media.clips)): diff --git a/test_local_turn_features.py b/test_local_turn_features.py index c45b604..788bcf5 100644 --- a/test_local_turn_features.py +++ b/test_local_turn_features.py @@ -23,7 +23,7 @@ from extract_turn_local_features import ( FrozenQwenLocalEncoder, ego4d_clip_boxes, extract_local, validate_source_timeline, model_config_digest, - box_coordinate_offset, frame_crops, parse_args, preview_boxes, + box_coordinate_offset, frame_crops, parse_args, preview_boxes, preview_ticks, preview_time_bounds, TESTED_TRANSFORMERS_VERSIONS, validate_local_transformers_version, main as extract_main, ) @@ -160,6 +160,100 @@ class LocalTests(unittest.TestCase): self.assertEqual(audit[0]["boxes"][0]["box"], [100, 80, 160, 140]) self.assertTrue((folder / audit[0]["image"]).is_file()) + def test_preview_can_save_more_than_four_images_in_a_later_range(self): + clip = SimpleNamespace(path=Path("source.mp4"), width=640, height=360, + crop=None, dense_start=0.) + media = SimpleNamespace(clips=[clip], + _source_frame_for_time=lambda t: (0, round(t * 2), t), + frame_at_or_before=lambda t: Image.new("RGB", (640, 360))) + with tempfile.TemporaryDirectory() as directory: + args = SimpleNamespace(box_coordinate_space="camera", crop_expansion=1., video_fps=2., + output_root=Path(directory), preview_count=8, + preview_start=2., preview_end=6., preview_with_boxes=False) + preview_boxes(media, [{}], args, "clip", 10.) + folder = Path(directory) / "clip_preview" + audit = json.loads((folder / "boxes.json").read_text()) + self.assertEqual([row["time"] for row in audit], [2., 2.5, 3., 3.5, 4., 4.5, 5., 5.5]) + self.assertEqual(audit[0]["image"], "frame_000004.jpg") + self.assertEqual(len(list(folder.glob("*.jpg"))), 8) + report = json.loads((folder / "preview.json").read_text()) + self.assertEqual(report["saved_count"], 8) + self.assertEqual(report["saved_with_boxes"], 0) + + def test_boxed_preview_filters_valid_corrected_crops_without_decoding(self): + # Encoded boxes are corrected by the crop origin, as in feature extraction. + clip = SimpleNamespace(crop=(204, 0, 640, 360), width=844, height=360) + media = SimpleNamespace(clips=[clip], _source_frame_for_time=lambda t: (0, round(t * 2), t)) + valid = {"id": "p", "xyxy": [304, 80, 364, 140]} + invalid = {"id": "p", "xyxy": [4, 80, 64, 140]} # entirely inside removed banner + annotations = [{1: [valid], 3: [invalid], 5: [valid], 7: [valid]}] + args = SimpleNamespace(box_coordinate_space="encoded", crop_expansion=1., video_fps=2., + preview_count=2, preview_start=0., preview_end=None, preview_with_boxes=True) + selected, grid_count, eligible_count = preview_ticks(media, annotations, args, 4.) + self.assertEqual(selected, [1, 7]) # uniform among valid frames, not first N + self.assertEqual((grid_count, eligible_count), (8, 3)) + # No RGB reader is needed by the scan, and annotations are not changed. + self.assertEqual(len(annotations[0]), 4) + + def test_empty_boxed_preview_writes_an_explicit_empty_report(self): + clip = SimpleNamespace(crop=None, width=640, height=360) + media = SimpleNamespace(clips=[clip], _source_frame_for_time=lambda t: (0, round(t * 2), t)) + with tempfile.TemporaryDirectory() as directory: + args = SimpleNamespace(box_coordinate_space="camera", crop_expansion=1., video_fps=2., + output_root=Path(directory), preview_count=40, + preview_start=0., preview_end=None, preview_with_boxes=True) + preview_boxes(media, [{}], args, "empty_clip", 4.) + folder = Path(directory) / "empty_clip_preview" + self.assertEqual(json.loads((folder / "boxes.json").read_text()), []) + report = json.loads((folder / "preview.json").read_text()) + self.assertEqual(report["eligible_frame_count"], 0) + self.assertEqual(report["saved_count"], 0) + self.assertEqual(list(folder.glob("*.jpg")), []) + + def test_preview_range_clips_to_available_end_and_excludes_endpoint(self): + args = SimpleNamespace(video_fps=2., preview_start=.1, preview_end=100., preview_count=100) + self.assertEqual(preview_time_bounds(args, 1.), (.1, 1.)) + self.assertEqual(preview_ticks(None, [], args, 1.), ([1], 1, 1)) + args.preview_start = 2. + with self.assertRaisesRegex(ValueError, "outside the available prefix"): + preview_time_bounds(args, 1.) + + def test_preview_cli_validation(self): + import contextlib + import io + + required = ["--data-root", "/raw", "--feature-root", "/global", "--target-root", "/targets", + "--output-root", "/preview", "--samples", "4"] + valid = parse_args(required + ["--preview-only", "--preview-count", "40", "--preview-start", "30", + "--preview-end", "120", "--preview-with-boxes"]) + self.assertEqual((valid.preview_count, valid.preview_start, valid.preview_end), (40, 30., 120.)) + for extra in (["--preview-only", "--preview-count", "0"], + ["--preview-only", "--preview-start", "-1"], + ["--preview-only", "--preview-end", "nan"], + ["--preview-only", "--preview-start", "10", "--preview-end", "10"], + ["--preview-only", "--max-steps", "20", "--preview-end", "60"], + ["--preview-count", "40"], ["--preview-with-boxes"]): + with self.subTest(extra=extra), contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + parse_args(required + extra) + + def test_preview_end_limits_media_reader_without_loading_encoder(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + reference, _, labels = fixture() + torch.save(reference, root / "Session_4.pt") + np.save(root / "Session_4.npy", labels) + args = ["--data-root", str(root), "--feature-root", str(root), "--target-root", str(root), + "--output-root", str(root / "preview"), "--samples", "4", + "--preview-only", "--preview-end", "2", "--preview-count", "20"] + media = SimpleNamespace(clips=[], metadata=lambda: {}) + with patch("extract_turn_local_features.build_media_and_boxes", return_value=(media, [])) as build, \ + patch("extract_turn_local_features.preview_boxes") as preview, \ + patch("extract_turn_local_features.FrozenQwenLocalEncoder") as encoder: + extract_main(args) + self.assertEqual(build.call_args.args[2], 2.) + preview.assert_called_once() + encoder.assert_not_called() + def test_sidecar_alignment_contract_and_modalities(self): reference, side, _ = fixture() with tempfile.TemporaryDirectory() as directory: