Add frozen Thinker cache-path control

README.md

@@ -1072,6 +1072,38 @@ and `valid/`, and writes the aggregate metrics to `metrics.json`.  Add
 Treat Sessions 1, 2, and 3 as the official test set: inspect their metrics only
 after the training recipe and hyperparameters have been fixed.
 
+To isolate whether a score change comes from LoRA or from the dense-cache
+reconstruction path, run the same evaluator without `--adapter-checkpoint` and
+load the original frozen 8-second head:
+
+```bash
+FROZEN_HEAD=checkpoints/easycom_open_stream_2fps_8s_head_seed0.pth
+
+CUDA_VISIBLE_DEVICES=0,1,2 \
+OMP_NUM_THREADS=1 \
+MKL_NUM_THREADS=1 \
+OPENBLAS_NUM_THREADS=1 \
+PYTORCH_ALLOC_CONF=expandable_segments:True \
+TORCH_NCCL_ASYNC_ERROR_HANDLING=1 \
+torchrun --standalone --nproc_per_node=3 eval_omni_turn_lora.py \
+  --model-id /data1/yfzx/wangx/Models/Qwen2.5-Omni-3B \
+  --cache-dir /data1/yfzx/wangx/Datasets/easycom/omni_lora_cache_dense_8s/Session_3 \
+  --cache-dir /data1/yfzx/wangx/Datasets/easycom/omni_lora_cache_dense_8s/Session_2 \
+  --cache-dir /data1/yfzx/wangx/Datasets/easycom/omni_lora_cache_dense_8s/Session_1 \
+  --head-checkpoint "${FROZEN_HEAD}" \
+  --output-dir results/easycom_frozen_8s_dense_cache_control \
+  --batch-size 2 \
+  --block-cache-size 32 \
+  --dtype bfloat16 \
+  --flash-attention-2 \
+  --ddp-timeout-minutes 60
+```
+
+This control should approach the previously reported frozen 8-second score of
+58.51937 full-timeline mean mAP.  A material mismatch indicates a representation
+difference between dense-cache reconstruction and the earlier open-stream
+feature files; a close match isolates the regression to LoRA/head training.
+
 For a speed-only alternative, `train_omni_turn_lora.py` supports a two-GPU
 producer-consumer benchmark.  The producer GPU runs the frozen AV towers and
 places current-tail embeddings in a bounded CPU queue while the consumer GPU

eval_omni_turn_lora.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Evaluate Thinker LoRA + turn head from dense Omni AV caches.
+"""Evaluate a frozen or LoRA-adapted Thinker from dense Omni AV caches.
 
 The script accepts one dense cache directory per EasyCom session.  With
 ``torchrun``, complete sessions are assigned round-robin to ranks; rank zero
@@ -55,7 +55,14 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
         required=True,
         help="Dense cache directory from cache_omni_av_pieces.py; repeat per session",
     )
-    parser.add_argument("--adapter-checkpoint", type=Path, required=True)
+    parser.add_argument(
+        "--adapter-checkpoint",
+        type=Path,
+        help=(
+            "PEFT adapter directory. Omit it to evaluate the frozen base "
+            "Thinker as a cache-path control."
+        ),
+    )
     parser.add_argument("--head-checkpoint", type=Path, required=True)
     parser.add_argument("--output-dir", type=Path, required=True)
     parser.add_argument("--batch-size", type=int, default=2)
@@ -267,10 +274,21 @@ def _run(args: argparse.Namespace, run: DistributedRun) -> None:
 
     base_model, processor = _load_base_model(args, run)
     base_model.requires_grad_(False)
+    adapter_path = None
+    if args.adapter_checkpoint is None:
+        language_model = base_model.model.eval()
+        model_variant = "frozen_base_thinker"
+        print(
+            f"[rank {run.rank}] frozen control: no LoRA adapter loaded",
+            flush=True,
+        )
+    else:
         try:
             from peft import PeftModel
         except ImportError as exc:
-        raise RuntimeError("PEFT is required to evaluate the LoRA adapter") from exc
+            raise RuntimeError(
+                "PEFT is required to evaluate the LoRA adapter"
+            ) from exc
         adapter_path = args.adapter_checkpoint.expanduser().resolve()
         if not (adapter_path / "adapter_config.json").is_file():
             raise FileNotFoundError(f"Missing adapter config: {adapter_path}")
@@ -279,6 +297,7 @@ def _run(args: argparse.Namespace, run: DistributedRun) -> None:
             str(adapter_path),
             is_trainable=False,
         ).eval()
+        model_variant = "thinker_lora"
     language_model.config.use_cache = False
     model_dtype = next(language_model.parameters()).dtype
     head, head_payload = _load_head(args.head_checkpoint, run.device)
@@ -336,7 +355,10 @@ def _run(args: argparse.Namespace, run: DistributedRun) -> None:
                 "classes": list(CLASS_NAMES),
                 "prediction_layout": "[target_time, class, future_horizon]",
                 "model_id": args.model_id,
-                "adapter_checkpoint": str(adapter_path),
+                "model_variant": model_variant,
+                "adapter_checkpoint": (
+                    None if adapter_path is None else str(adapter_path)
+                ),
                 "head_checkpoint": str(args.head_checkpoint.expanduser().resolve()),
                 "head_training_step": head_payload.get("training_step"),
                 "cache_dirs": [str(item.cache_dir) for item in sessions],