From 3cc0c3da8535a6c188fafa0cb29dea291e4ca63b Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Mon, 24 Aug 2026 01:07:42 +0800 Subject: [PATCH] Add frozen Thinker cache-path control --- README.md | 32 ++++++++++++++++++++++++++ eval_omni_turn_lora.py | 52 ++++++++++++++++++++++++++++++------------ 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6b60b3e..acd8b73 100644 --- a/README.md +++ b/README.md @@ -1072,6 +1072,38 @@ and `valid/`, and writes the aggregate metrics to `metrics.json`. Add Treat Sessions 1, 2, and 3 as the official test set: inspect their metrics only after the training recipe and hyperparameters have been fixed. +To isolate whether a score change comes from LoRA or from the dense-cache +reconstruction path, run the same evaluator without `--adapter-checkpoint` and +load the original frozen 8-second head: + +```bash +FROZEN_HEAD=checkpoints/easycom_open_stream_2fps_8s_head_seed0.pth + +CUDA_VISIBLE_DEVICES=0,1,2 \ +OMP_NUM_THREADS=1 \ +MKL_NUM_THREADS=1 \ +OPENBLAS_NUM_THREADS=1 \ +PYTORCH_ALLOC_CONF=expandable_segments:True \ +TORCH_NCCL_ASYNC_ERROR_HANDLING=1 \ +torchrun --standalone --nproc_per_node=3 eval_omni_turn_lora.py \ + --model-id /data1/yfzx/wangx/Models/Qwen2.5-Omni-3B \ + --cache-dir /data1/yfzx/wangx/Datasets/easycom/omni_lora_cache_dense_8s/Session_3 \ + --cache-dir /data1/yfzx/wangx/Datasets/easycom/omni_lora_cache_dense_8s/Session_2 \ + --cache-dir /data1/yfzx/wangx/Datasets/easycom/omni_lora_cache_dense_8s/Session_1 \ + --head-checkpoint "${FROZEN_HEAD}" \ + --output-dir results/easycom_frozen_8s_dense_cache_control \ + --batch-size 2 \ + --block-cache-size 32 \ + --dtype bfloat16 \ + --flash-attention-2 \ + --ddp-timeout-minutes 60 +``` + +This control should approach the previously reported frozen 8-second score of +58.51937 full-timeline mean mAP. A material mismatch indicates a representation +difference between dense-cache reconstruction and the earlier open-stream +feature files; a close match isolates the regression to LoRA/head training. + For a speed-only alternative, `train_omni_turn_lora.py` supports a two-GPU producer-consumer benchmark. The producer GPU runs the frozen AV towers and places current-tail embeddings in a bounded CPU queue while the consumer GPU diff --git a/eval_omni_turn_lora.py b/eval_omni_turn_lora.py index 9306d44..af0e7de 100644 --- a/eval_omni_turn_lora.py +++ b/eval_omni_turn_lora.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Evaluate Thinker LoRA + turn head from dense Omni AV caches. +"""Evaluate a frozen or LoRA-adapted Thinker from dense Omni AV caches. The script accepts one dense cache directory per EasyCom session. With ``torchrun``, complete sessions are assigned round-robin to ranks; rank zero @@ -55,7 +55,14 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: required=True, help="Dense cache directory from cache_omni_av_pieces.py; repeat per session", ) - parser.add_argument("--adapter-checkpoint", type=Path, required=True) + parser.add_argument( + "--adapter-checkpoint", + type=Path, + help=( + "PEFT adapter directory. Omit it to evaluate the frozen base " + "Thinker as a cache-path control." + ), + ) parser.add_argument("--head-checkpoint", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--batch-size", type=int, default=2) @@ -267,18 +274,30 @@ def _run(args: argparse.Namespace, run: DistributedRun) -> None: base_model, processor = _load_base_model(args, run) base_model.requires_grad_(False) - try: - from peft import PeftModel - except ImportError as exc: - raise RuntimeError("PEFT is required to evaluate the LoRA adapter") from exc - adapter_path = args.adapter_checkpoint.expanduser().resolve() - if not (adapter_path / "adapter_config.json").is_file(): - raise FileNotFoundError(f"Missing adapter config: {adapter_path}") - language_model = PeftModel.from_pretrained( - base_model.model, - str(adapter_path), - is_trainable=False, - ).eval() + adapter_path = None + if args.adapter_checkpoint is None: + language_model = base_model.model.eval() + model_variant = "frozen_base_thinker" + print( + f"[rank {run.rank}] frozen control: no LoRA adapter loaded", + flush=True, + ) + else: + try: + from peft import PeftModel + except ImportError as exc: + raise RuntimeError( + "PEFT is required to evaluate the LoRA adapter" + ) from exc + adapter_path = args.adapter_checkpoint.expanduser().resolve() + if not (adapter_path / "adapter_config.json").is_file(): + raise FileNotFoundError(f"Missing adapter config: {adapter_path}") + language_model = PeftModel.from_pretrained( + base_model.model, + str(adapter_path), + is_trainable=False, + ).eval() + model_variant = "thinker_lora" language_model.config.use_cache = False model_dtype = next(language_model.parameters()).dtype head, head_payload = _load_head(args.head_checkpoint, run.device) @@ -336,7 +355,10 @@ def _run(args: argparse.Namespace, run: DistributedRun) -> None: "classes": list(CLASS_NAMES), "prediction_layout": "[target_time, class, future_horizon]", "model_id": args.model_id, - "adapter_checkpoint": str(adapter_path), + "model_variant": model_variant, + "adapter_checkpoint": ( + None if adapter_path is None else str(adapter_path) + ), "head_checkpoint": str(args.head_checkpoint.expanduser().resolve()), "head_training_step": head_payload.get("training_step"), "cache_dirs": [str(item.cache_dir) for item in sessions],