Add extract_ego4d_batch.py for efficient EgoSpeak clip extraction. This script loads the Qwen model once and processes selected manifest rows sequentially, ensuring each clip is handled independently. Update README with usage instructions and details on the extraction process, including handling of completed outputs and memory management.

This commit is contained in:
Xin Wang
2026-08-18 22:22:37 +08:00
parent 609b2781d4
commit 5fa61eed23
3 changed files with 514 additions and 11 deletions

View File

@@ -59,7 +59,7 @@ import subprocess
from collections import OrderedDict
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any
from typing import Any, Sequence
import numpy as np
import torch
@@ -778,7 +778,7 @@ def _parse_crop(value: str | None) -> tuple[int, int, int, int] | None:
return values
def parse_args() -> argparse.Namespace:
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
source = parser.add_mutually_exclusive_group(required=True)
source.add_argument("--video", type=Path, help="One original video clip")
@@ -873,7 +873,7 @@ def parse_args() -> argparse.Namespace:
action="store_true",
help="Validate/index direct media and exit before loading Qwen",
)
return parser.parse_args()
return parser.parse_args(argv)
def _resolve_duration_and_target(
@@ -941,9 +941,23 @@ def _print_media_summary(reader: CausalSourceAV, start: float, end: float) -> No
)
def main() -> None:
args = parse_args()
def main(
argv: Sequence[str] | None = None,
*,
model: Any | None = None,
processor: Any | None = None,
) -> None:
"""Run one direct extraction, optionally reusing loaded Qwen objects.
``model`` and ``processor`` must be supplied together. Reusing them is
safe across independent clips because this function constructs a new media
reader, extractor, and temporal cache on every call.
"""
args = parse_args(argv)
core._configure_qwen_logging()
if (model is None) != (processor is None):
raise ValueError("model and processor must be supplied together")
if args.verify_every < 0 or args.verify_continuous_every < 0:
raise SystemExit("verification intervals must be >= 0")
if args.frame_cache_size < 0:
@@ -1055,12 +1069,15 @@ def main() -> None:
)
config.validate()
model, processor = core.load_qwen_omni(
args.model_id,
dtype=args.dtype,
device_map={"": args.device},
use_flash_attention_2=args.flash_attention_2,
)
if model is None:
model, processor = core.load_qwen_omni(
args.model_id,
dtype=args.dtype,
device_map={"": args.device},
use_flash_attention_2=args.flash_attention_2,
)
else:
model.eval()
model.requires_grad_(False)
media_path_for_api = reader.clips[0].path