diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md index 1d4bcdb..4ce8dc5 100644 --- a/PROJECT_CONTEXT.md +++ b/PROJECT_CONTEXT.md @@ -167,6 +167,10 @@ The detailed architecture, controls and stopping gates for P3 are frozen in - EasyCom targets are dense `[T, 3]`; predictions are `[target_time, class, future_horizon]`. - EasyCom split: train Session 4--12; test Session 1--3. +- Locked final head protocol: exactly 4 epochs on all Sessions 4--12, no + development checkpoint selection, followed by one evaluation on Sessions + 1--3. Historical Session-12 screens remain development diagnostics and are + not the final training protocol. ### Global feature sidecar diff --git a/docs/easycom_oracle_information.md b/docs/easycom_oracle_information.md index 5bb4999..5f34ab3 100644 --- a/docs/easycom_oracle_information.md +++ b/docs/easycom_oracle_information.md @@ -71,17 +71,39 @@ Sessions 1--3. If preparation reports any transcript/target mismatch, stop and inspect it; the preparation command has no mismatch-tolerance option. +## Locked four-epoch train-all protocol + +The final protocol fixes the training budget at four epochs, trains on all +official EasyCom training sessions (Sessions 4--12), performs no development +checkpoint selection, and evaluates Sessions 1--3 once: + +```bash +CUDA_VISIBLE_DEVICES=0 bash run_easycom_oracle_information.sh final 0 1 2 +``` + +The `final` stage enforces `EPOCHS=4` and exactly Sessions 4--12, clears the +development-session list internally, and saves the last epoch. Its artifacts +use separate `_train4_12_fixed4` checkpoint/result roots by default, so they +cannot overwrite the historical Session-12 development screen. Override those +roots only with `FINAL_CHECKPOINT_ROOT` and `FINAL_RESULT_ROOT`. + +mAP itself does not require threshold calibration. Nevertheless, the official +test results must not be used to choose a variant, epoch, loss, fusion design, +or any later hyperparameter. Run `final` only after those decisions are frozen; +repeatedly inspecting Sessions 1--3 and modifying the method would contaminate +the official test split. + ## Follow-up -If `per-person` is consistently better on Session 12, add seeds 1 and 2 using -the already prepared sidecars: +To reproduce the historical Session-12 development screen for seeds 1 and 2, +use the already prepared sidecars: ```bash CUDA_VISIBLE_DEVICES=0 bash run_easycom_oracle_information.sh screen 1 2 ``` -Only after freezing the method and event evaluator should selected variants be -run once on Sessions 1--3: +The older two-stage equivalent of final test inference is retained for +reproducibility, but the locked protocol above is preferred for new final runs: ```bash VARIANTS="audio aggregate-visible per-person shuffled-id delayed" \ diff --git a/run_easycom_oracle_information.sh b/run_easycom_oracle_information.sh index 0a19aae..8fae88f 100755 --- a/run_easycom_oracle_information.sh +++ b/run_easycom_oracle_information.sh @@ -72,18 +72,13 @@ variant_local_args() { fi } -train_and_screen() { +train_variants() { mkdir -p "${CHECKPOINT_ROOT}" "${RESULT_ROOT}" for seed in "${seeds[@]}"; do for variant in "${variants[@]}"; do checkpoint="${CHECKPOINT_ROOT}/${variant}_seed${seed}.pth" - result="${RESULT_ROOT}/${variant}_seed${seed}" - local_feature_root="${ORACLE_ROOT}/${variant}" - if [[ "${variant}" == "audio" ]]; then - local_feature_root="${LOCAL_ROOT}" - fi - if [[ -e "${checkpoint}" || -e "${result}" ]]; then - echo "Refusing to overwrite ${checkpoint} or ${result}" >&2 + if [[ -e "${checkpoint}" ]]; then + echo "Refusing to overwrite ${checkpoint}" >&2 exit 1 fi local_args=() @@ -111,7 +106,28 @@ train_and_screen() { --expected-mode "${EXPECTED_MODE}" \ --seed "${seed}" \ --device cuda:0 + done + done +} +screen_variants() { + mkdir -p "${RESULT_ROOT}" + for seed in "${seeds[@]}"; do + for variant in "${variants[@]}"; do + checkpoint="${CHECKPOINT_ROOT}/${variant}_seed${seed}.pth" + result="${RESULT_ROOT}/${variant}_seed${seed}" + local_feature_root="${ORACLE_ROOT}/${variant}" + if [[ "${variant}" == "audio" ]]; then + local_feature_root="${LOCAL_ROOT}" + fi + if [[ ! -f "${checkpoint}" ]]; then + echo "Missing checkpoint: ${checkpoint}" >&2 + exit 1 + fi + if [[ -e "${result}" ]]; then + echo "Refusing to overwrite ${result}" >&2 + exit 1 + fi echo "$(date): screening ${variant}, seed=${seed} on ${screen_sessions[*]}" "${PYTHON_BIN}" infer_turn_head.py \ --feature-root "${FEATURE_ROOT}" \ @@ -130,6 +146,30 @@ train_and_screen() { --protocol causal_valid_only } +train_all_fixed_and_test() { + if [[ "${EPOCHS}" != "4" ]]; then + echo "The final protocol requires EPOCHS=4, got ${EPOCHS}" >&2 + exit 2 + fi + + normalized_train=() + for session in "${train_sessions[@]}"; do + normalized_train+=("${session#Session_}") + done + if [[ "${normalized_train[*]}" != "4 5 6 7 8 9 10 11 12" ]]; then + echo "The final protocol requires exactly EasyCom Sessions 4--12 for training" >&2 + exit 2 + fi + + # No development checkpoint selection: the final epoch is fixed in advance. + # Keep these artifacts separate from the historical Session-12 screen. + dev_sessions=() + CHECKPOINT_ROOT="${FINAL_CHECKPOINT_ROOT:-${CHECKPOINT_ROOT}_train4_12_fixed4}" + RESULT_ROOT="${FINAL_RESULT_ROOT:-${RESULT_ROOT}_train4_12_fixed4}" + train_variants + test_selected +} + test_selected() { mkdir -p "${RESULT_ROOT}_test" for seed in "${seeds[@]}"; do @@ -170,11 +210,17 @@ case "${stage}" in prepare_oracles ;; screen) - train_and_screen + train_variants + screen_variants ;; all) prepare_oracles - train_and_screen + train_variants + screen_variants + ;; + final) + prepare_oracles + train_all_fixed_and_test ;; test) test_selected @@ -186,7 +232,7 @@ case "${stage}" in --protocol causal_valid_only ;; *) - echo "Usage: $0 {prepare|screen|all|test|summarize} [seed ...]" >&2 + echo "Usage: $0 {prepare|screen|all|final|test|summarize} [seed ...]" >&2 exit 2 ;; esac