Add multi-GPU Ego4D local feature extraction without symlinks

This commit is contained in:
Xin Wang
2026-08-28 01:01:25 +08:00
parent 9bd204b468
commit 5fd285e369
8 changed files with 552 additions and 16 deletions

View File

@@ -638,6 +638,30 @@ def normalize_sample_name(
raise ValueError(f"Invalid Ego4D clip UUID: {value}") from error
def resolve_feature_path(
feature_root: str | Path,
sample: str | int,
*,
dataset: DatasetName = "easycom",
) -> Path:
"""Read flat features or Ego4D train/test subdirectories without link farms.
Manifest membership, not directory discovery, determines the data split.
Multiple copies are rejected rather than silently choosing stale features.
"""
name = normalize_sample_name(sample, dataset)
root = Path(feature_root)
candidates = [root / f"{name}.pt"]
if dataset == "ego4d":
candidates += [root / split / f"{name}.pt" for split in ("train", "test")]
found = [path for path in candidates if path.is_file()]
if len(found) > 1:
raise ValueError(f"Ambiguous feature copies for {name}: {found}; select one feature root")
if not found:
raise FileNotFoundError(f"No features for {name}; checked: " + ", ".join(map(str, candidates)))
return found[0]
def load_ego4d_manifest_split(
manifest: str | Path,
split: Literal["train", "test", "all"],
@@ -727,7 +751,7 @@ def load_session_data(
"""Load and validate one saved Omni feature payload and target array."""
name = normalize_sample_name(session, dataset)
feature_path = Path(feature_root) / f"{name}.pt"
feature_path = resolve_feature_path(feature_root, name, dataset=dataset)
target_path = Path(target_root) / f"{name}.npy"
if not feature_path.is_file():
raise FileNotFoundError(feature_path)
@@ -804,7 +828,7 @@ def load_session_data(
from local_turn_features import load_local_features
local, contract = load_local_features(
Path(local_feature_root) / f"{name}.pt",
resolve_feature_path(local_feature_root, name, dataset=dataset),
payload,
sample_id=name,
dataset=dataset,