Make oracle preparation resumable
This commit is contained in:
@@ -61,7 +61,9 @@ CUDA_VISIBLE_DEVICES=0 bash run_easycom_oracle_information.sh all 0
|
||||
|
||||
`all` prepares the five oracle conditions, trains on Sessions 4--11, selects
|
||||
the checkpoint on Session 12, runs Session-12 inference and prints the standard
|
||||
dense AP/mAP screen. It deliberately does not run Sessions 1--3.
|
||||
dense AP/mAP screen. Existing compatible oracle sidecars are validated and
|
||||
reused, so interrupted preparation can resume. It deliberately does not run
|
||||
Sessions 1--3.
|
||||
|
||||
If preparation reports any transcript/target mismatch, stop and inspect it.
|
||||
Do not raise `--max-label-mismatch-rows` merely to make the run continue.
|
||||
|
||||
@@ -325,6 +325,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--shuffle-seed", type=int, default=2027)
|
||||
parser.add_argument("--max-label-mismatch-rows", type=int, default=0)
|
||||
parser.add_argument("--expected-mode", default="open-stream-replay")
|
||||
parser.add_argument("--reuse-existing", action="store_true")
|
||||
parser.add_argument("--overwrite", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
return parser.parse_args()
|
||||
@@ -437,9 +438,42 @@ def main() -> None:
|
||||
|
||||
for condition in conditions:
|
||||
destination = args.output_root / condition / f"{session}.pt"
|
||||
if destination.exists() and args.reuse_existing and not args.overwrite:
|
||||
_, existing_people, existing_contract = load_frame_set_features(
|
||||
destination,
|
||||
reference,
|
||||
sample_id=session,
|
||||
dataset="easycom",
|
||||
oracle_analysis=True,
|
||||
)
|
||||
existing_config = existing_contract["config"]
|
||||
expected_existing = {
|
||||
"oracle_condition": condition,
|
||||
"history_steps": history_steps,
|
||||
"delay_steps": delay_steps if condition == "delayed" else 0,
|
||||
"camera_wearer_id": args.camera_wearer_id,
|
||||
"audio_dim": audio_dim,
|
||||
"source_local_contract": source_contract,
|
||||
}
|
||||
mismatches = {
|
||||
key: (existing_config.get(key), value)
|
||||
for key, value in expected_existing.items()
|
||||
if existing_config.get(key) != value
|
||||
}
|
||||
if mismatches:
|
||||
raise ValueError(
|
||||
f"Cannot reuse incompatible oracle sidecar {destination}: "
|
||||
f"{mismatches}"
|
||||
)
|
||||
print(
|
||||
f" {condition}: reuse {destination}; "
|
||||
f"observations={len(existing_people.features)}"
|
||||
)
|
||||
continue
|
||||
if destination.exists() and not args.overwrite:
|
||||
raise FileExistsError(
|
||||
f"Refusing to overwrite {destination}; pass --overwrite"
|
||||
f"Refusing to overwrite {destination}; pass --overwrite "
|
||||
"or --reuse-existing"
|
||||
)
|
||||
condition_seed = (
|
||||
args.shuffle_seed
|
||||
|
||||
@@ -51,7 +51,8 @@ prepare_oracles() {
|
||||
--conditions "${oracle_conditions[@]}" \
|
||||
--history-seconds 2.0 \
|
||||
--delay-seconds 1.0 \
|
||||
--expected-mode "${EXPECTED_MODE}"
|
||||
--expected-mode "${EXPECTED_MODE}" \
|
||||
--reuse-existing
|
||||
}
|
||||
|
||||
variant_local_args() {
|
||||
|
||||
Reference in New Issue
Block a user