From 1925ff71a7d93b81b95bc97ee7db29ddf4ac7360 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Thu, 27 Aug 2026 09:25:08 +0800 Subject: [PATCH] Record preliminary transition auxiliary results --- README.md | 8 + .../README.md | 122 +++++++++++++ .../metrics.csv | 8 + .../raw_console.txt | 34 ++++ .../summary.json | 170 ++++++++++++++++++ 5 files changed, 342 insertions(+) create mode 100644 results/easycom_transition_auxiliary_4ep_2026-08-27/README.md create mode 100644 results/easycom_transition_auxiliary_4ep_2026-08-27/metrics.csv create mode 100644 results/easycom_transition_auxiliary_4ep_2026-08-27/raw_console.txt create mode 100644 results/easycom_transition_auxiliary_4ep_2026-08-27/summary.json diff --git a/README.md b/README.md index ec463b6..d02ffb9 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,14 @@ same four-epoch budget. The primary paper outcome remains activity mAP; the transition metrics diagnose whether an activity improvement is genuinely associated with better boundary prediction. +The preliminary seed-0/1 results are archived in +[results/easycom_transition_auxiliary_4ep_2026-08-27/](results/easycom_transition_auxiliary_4ep_2026-08-27/README.md). +Neither R1 nor R2 establishes a meaningful activity-mAP gain. At 0.2 seconds, +transition prevalence is 11.21718%, while transition AP is approximately +15.3%, indicating weak but non-random ranking. R2 improves activity mean mAP +over R1 by only 0.01162 points and transition AP by 0.16660 points. Seed 2 is +still required before freezing this as a complete null ablation. + ## Current Ego4D context-length results The following single-seed snapshot was recorded on 2026-08-22. Both variants diff --git a/results/easycom_transition_auxiliary_4ep_2026-08-27/README.md b/results/easycom_transition_auxiliary_4ep_2026-08-27/README.md new file mode 100644 index 0000000..09b22a8 --- /dev/null +++ b/results/easycom_transition_auxiliary_4ep_2026-08-27/README.md @@ -0,0 +1,122 @@ +# EasyCom transition-aware auxiliary-head ablation + +Recorded on 2026-08-27 from user-reported remote evaluation output. This is a +preliminary two-seed snapshot: seed 2 has not been reported, so these numbers +must not yet be used as a final multi-seed paper result. + +## Experiment contract + +- Dataset: EasyCom. +- Train split: Sessions 4 through 12, with no development set. +- Test split: EgoSpeak official Sessions 3, 2, and 1. +- Backbone: frozen Qwen2.5-Omni-3B Thinker. +- Representation: 8-second open-stream replay, 2 FPS video, 5 Hz states, final + four query layers stacked. +- Readout: concat-MLP followed by the activity MLP. +- Activity objective: LSTR-MCE. +- Training budget: 4 epochs, batch size 512, learning rate 7e-5, weight decay + 5e-5, dropout 0.2, 1.6 warmup epochs, warmup factor 0.3. +- Completed seeds: 0 and 1. Missing seed: 2. +- Primary evaluation protocol: causal-valid-only activity AP/mAP. + +A positive transition at future horizon h means that the complete three-label +activity state changes between target rows t+h-1 and t+h. + +- R1: activity loss + 0.2 times weighted transition BCE, with positive weight + 3. +- R2: R1 loss - 0.1 times exact expected weighted transition correctness. + +## Individual results + +All values below are percentage points. + +| Variant | Seed | 0.2s mAP | BG AP | Wearer AP | Others AP | Mean mAP | Transition 0.2s AP | Transition 0.2s F1@0.5 | Transition mean AP | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| R1 | 0 | 73.51590 | 56.84019 | 77.92448 | 85.78303 | 60.86360 | 15.19153 | 0.46172 | 13.04354 | +| R1 | 1 | 73.78692 | 56.92102 | 78.63090 | 85.80884 | 60.82023 | 15.32492 | 1.14068 | 13.06174 | +| R2 | 0 | 73.53288 | 56.89701 | 77.92732 | 85.77432 | 60.86528 | 15.37608 | 0.00000 | 13.11854 | +| R2 | 1 | 73.82947 | 56.97474 | 78.69340 | 85.82027 | 60.84178 | 15.47356 | 0.84227 | 13.13820 | + +## Two-seed mean and sample standard deviation + +| Variant | 0.2s mAP | Wearer AP | Others AP | Mean mAP | Transition 0.2s AP | Transition mean AP | +|---|---:|---:|---:|---:|---:|---:| +| R1 | 73.65141 +/- 0.19164 | 78.27769 +/- 0.49951 | 85.79594 +/- 0.01825 | 60.84192 +/- 0.03067 | 15.25823 +/- 0.09432 | 13.05264 +/- 0.01287 | +| R2 | 73.68118 +/- 0.20972 | 78.31036 +/- 0.54169 | 85.79729 +/- 0.03249 | 60.85353 +/- 0.01662 | 15.42482 +/- 0.06893 | 13.12837 +/- 0.01390 | + +## Matched-seed comparison with LSTR-MCE + +The baseline values are taken from the existing four-epoch loss-structure +ablation. Using only the same completed seeds 0 and 1 avoids comparing a +two-seed transition result with a three-seed baseline mean. + +| Comparison | Delta 0.2s mAP | Delta BG AP | Delta wearer AP | Delta others AP | Delta mean mAP | +|---|---:|---:|---:|---:|---:| +| R1 minus LSTR-MCE | +0.04975 | +0.02633 | +0.05942 | +0.06350 | +0.04949 | +| R2 minus LSTR-MCE | +0.07951 | +0.08160 | +0.09209 | +0.06486 | +0.06110 | +| R2 minus R1 | +0.02977 | +0.05527 | +0.03267 | +0.00136 | +0.01162 | + +All activity changes are much smaller than the observed seed variation. The +current evidence therefore does not establish a meaningful activity +anticipation improvement. + +## Transition diagnostic + +The 0.2-second transition prevalence is 11.21718%, which is also the expected +AP of a random ranking. R1 reaches 15.19--15.32% AP, or 1.35--1.37 times the +random baseline. R2 reaches 15.38--15.47%, or 1.37--1.38 times random. The +auxiliary head therefore learns weak but non-random transition ranking. + +F1 at threshold 0.5 is nearly zero because recall is only 0--0.58%. This fixed +threshold is not a reliable model-quality measure under weighted BCE: positive +weight 3 changes probability calibration and remains below the approximately +7.9 negative-to-positive ratio implied by the test prevalence. Threshold-free +AP is the appropriate diagnostic here. Any deployable threshold must be +selected on training-held-out data, not the test set. + +R2 improves transition AP over R1 by only 0.16660 points at 0.2 seconds and +0.07573 points on the ten-horizon mean. Its activity mean-mAP improvement over +R1 is only 0.01162 points. Exact expected correctness supplies almost the same +gradient direction as weighted BCE, so this result is consistent with the two +terms being largely redundant rather than R2 learning a new sequential policy. + +## Current interpretation + +This is a preliminary null ablation: + +> Exact 200-ms transition supervision provides weak non-random boundary +> ranking but no meaningful improvement in the primary activity-anticipation +> metric. Adding exact expected correctness produces negligible further gain. + +This experiment should not be described as interactive reinforcement learning. +R2 computes the binary-action expectation analytically on fixed offline labels; +it is a low-variance supervised reward control. + +## Required follow-up + +1. Complete and report seed 2 before freezing the table. +2. Do not select a transition threshold or tune positive/reward weights on the + test sessions. +3. If the transition line is continued, replace the sparse exact-boundary + target with a pre-specified cumulative transition-hazard target: whether at + least one transition occurs within the next h steps. +4. If that structured target also fails to improve activity mAP, treat + transition auxiliary learning on frozen states as a rejected direction and + move experiment budget to representation or temporal modeling. + +## Provenance + +- Feature root: + /data1/yfzx/wangx/Datasets/easycom/omni_features/open_stream_2fps_8s_last4_stack +- Target root: + /data1/yfzx/wangx/Datasets/easycom/easycom_target_frame +- Remote result roots: + results/easycom-transition-r1-4ep-seed-N and + results/easycom-transition-r2-4ep-seed-N +- Source: user-reported summary and diagnostic console output. +- Raw remote metrics JSON archived locally: no. +- Full per-horizon activity and transition curves archived locally: no. + +Machine-readable records are in summary.json and metrics.csv. The received +console values are preserved in raw_console.txt. + diff --git a/results/easycom_transition_auxiliary_4ep_2026-08-27/metrics.csv b/results/easycom_transition_auxiliary_4ep_2026-08-27/metrics.csv new file mode 100644 index 0000000..08b6177 --- /dev/null +++ b/results/easycom_transition_auxiliary_4ep_2026-08-27/metrics.csv @@ -0,0 +1,8 @@ +variant,seed,horizon_0.2_map,horizon_0.2_background_ap,horizon_0.2_wearer_ap,horizon_0.2_others_ap,ten_horizon_mean_map,transition_positive_rate,transition_horizon_0.2_ap,transition_horizon_0.2_f1_at_0.5,transition_ten_horizon_mean_ap +lstr-mce,0,73.44637,56.76438,77.86650,85.70824,60.84861,,,, +lstr-mce,1,73.75696,56.94418,78.57004,85.75664,60.73626,,,, +r1-supervised,0,73.51590,56.84019,77.92448,85.78303,60.86360,11.21718,15.19153,0.46172,13.04354 +r1-supervised,1,73.78692,56.92102,78.63090,85.80884,60.82023,11.21718,15.32492,1.14068,13.06174 +r2-expected-reward,0,73.53288,56.89701,77.92732,85.77432,60.86528,11.21718,15.37608,0.00000,13.11854 +r2-expected-reward,1,73.82947,56.97474,78.69340,85.82027,60.84178,11.21718,15.47356,0.84227,13.13820 + diff --git a/results/easycom_transition_auxiliary_4ep_2026-08-27/raw_console.txt b/results/easycom_transition_auxiliary_4ep_2026-08-27/raw_console.txt new file mode 100644 index 0000000..ceec7a2 --- /dev/null +++ b/results/easycom_transition_auxiliary_4ep_2026-08-27/raw_console.txt @@ -0,0 +1,34 @@ +r1 seed=0: 0.2s_mAP=73.51590, BG_AP=56.84019, wearer_AP=77.92448, others_AP=85.78303, mean_mAP=60.86360, transition_0.2s_AP=15.19153, transition_0.2s_F1=0.46172, transition_mean_AP=13.04354 +r1 seed=1: 0.2s_mAP=73.78692, BG_AP=56.92102, wearer_AP=78.63090, others_AP=85.80884, mean_mAP=60.82023, transition_0.2s_AP=15.32492, transition_0.2s_F1=1.14068, transition_mean_AP=13.06174 +r2 seed=0: 0.2s_mAP=73.53288, BG_AP=56.89701, wearer_AP=77.92732, others_AP=85.77432, mean_mAP=60.86528, transition_0.2s_AP=15.37608, transition_0.2s_F1=0.00000, transition_mean_AP=13.11854 +r2 seed=1: 0.2s_mAP=73.82947, BG_AP=56.97474, wearer_AP=78.69340, others_AP=85.82027, mean_mAP=60.84178, transition_0.2s_AP=15.47356, transition_0.2s_F1=0.84227, transition_mean_AP=13.13820 + +Mean +/- sample standard deviation + +r1: + 0.2s_mAP 73.65141 +/- 0.19164 + BG_AP 56.88061 +/- 0.05716 + wearer_AP 78.27769 +/- 0.49951 + others_AP 85.79594 +/- 0.01825 + mean_mAP 60.84192 +/- 0.03067 + transition_0.2s_AP 15.25823 +/- 0.09432 + transition_0.2s_F1 0.80120 +/- 0.48010 + transition_mean_AP 13.05264 +/- 0.01287 + +r2: + 0.2s_mAP 73.68118 +/- 0.20972 + BG_AP 56.93587 +/- 0.05496 + wearer_AP 78.31036 +/- 0.54169 + others_AP 85.79729 +/- 0.03249 + mean_mAP 60.85353 +/- 0.01662 + transition_0.2s_AP 15.42482 +/- 0.06893 + transition_0.2s_F1 0.42113 +/- 0.59557 + transition_mean_AP 13.12837 +/- 0.01390 + +Transition prevalence diagnostic: + +r1 seed=0: positive_rate=11.21718%, AP=15.19153%, AP/random=1.35x, P@0.5=42.85714%, R@0.5=0.23211%, F1@0.5=0.46172% +r1 seed=1: positive_rate=11.21718%, AP=15.32492%, AP/random=1.37x, P@0.5=33.33333%, R@0.5=0.58027%, F1@0.5=1.14068% +r2 seed=0: positive_rate=11.21718%, AP=15.37608%, AP/random=1.37x, P@0.5=0.00000%, R@0.5=0.00000%, F1@0.5=0.00000% +r2 seed=1: positive_rate=11.21718%, AP=15.47356%, AP/random=1.38x, P@0.5=40.74074%, R@0.5=0.42553%, F1@0.5=0.84227% + diff --git a/results/easycom_transition_auxiliary_4ep_2026-08-27/summary.json b/results/easycom_transition_auxiliary_4ep_2026-08-27/summary.json new file mode 100644 index 0000000..300f2c2 --- /dev/null +++ b/results/easycom_transition_auxiliary_4ep_2026-08-27/summary.json @@ -0,0 +1,170 @@ +{ + "schema_version": 1, + "recorded_date": "2026-08-27", + "status": "preliminary_two_seed_null_ablation", + "missing_seed": 2, + "dataset": "EasyCom", + "split": { + "train_sessions": [4, 5, 6, 7, 8, 9, 10, 11, 12], + "dev_sessions": [], + "test_sessions": [3, 2, 1] + }, + "model": { + "backbone": "Qwen2.5-Omni-3B", + "thinker_frozen": true, + "streaming_mode": "open-stream-replay", + "memory_seconds": 8, + "video_fps": 2, + "prediction_hz": 5, + "selected_layers": "last_4_query_states", + "layer_fusion": "concat-mlp" + }, + "training": { + "epochs": 4, + "batch_size": 512, + "learning_rate": 0.00007, + "weight_decay": 0.00005, + "warmup_epochs": 1.6, + "warmup_factor": 0.3, + "dropout": 0.2, + "activity_loss": "lstr-mce", + "completed_seeds": [0, 1], + "checkpoint_selection": "final_epoch_no_dev" + }, + "transition_definition": { + "positive": "activity[t+h] differs from activity[t+h-1]", + "r1": { + "objective": "activity + 0.2 * weighted_transition_bce", + "positive_weight": 3.0 + }, + "r2": { + "objective": "activity + 0.2 * transition - 0.1 * expected_transition_correctness", + "positive_weight": 3.0, + "reward_weight": 0.1, + "expectation": "analytic_binary_action_expectation_without_sampling" + } + }, + "evaluation": { + "activity_protocol": "causal_valid_only", + "transition_horizon_0.2_positive_rate_percent": 11.21718, + "transition_random_ap_percent": 11.21718 + }, + "individual_run_summary_percent": [ + { + "variant": "r1-supervised", + "seed": 0, + "horizon_0.2_map": 73.51590, + "horizon_0.2_background_ap": 56.84019, + "horizon_0.2_wearer_ap": 77.92448, + "horizon_0.2_others_ap": 85.78303, + "ten_horizon_mean_map": 60.86360, + "transition_horizon_0.2_ap": 15.19153, + "transition_horizon_0.2_f1_at_0.5": 0.46172, + "transition_ten_horizon_mean_ap": 13.04354, + "transition_ap_over_random": 1.35 + }, + { + "variant": "r1-supervised", + "seed": 1, + "horizon_0.2_map": 73.78692, + "horizon_0.2_background_ap": 56.92102, + "horizon_0.2_wearer_ap": 78.63090, + "horizon_0.2_others_ap": 85.80884, + "ten_horizon_mean_map": 60.82023, + "transition_horizon_0.2_ap": 15.32492, + "transition_horizon_0.2_f1_at_0.5": 1.14068, + "transition_ten_horizon_mean_ap": 13.06174, + "transition_ap_over_random": 1.37 + }, + { + "variant": "r2-expected-reward", + "seed": 0, + "horizon_0.2_map": 73.53288, + "horizon_0.2_background_ap": 56.89701, + "horizon_0.2_wearer_ap": 77.92732, + "horizon_0.2_others_ap": 85.77432, + "ten_horizon_mean_map": 60.86528, + "transition_horizon_0.2_ap": 15.37608, + "transition_horizon_0.2_f1_at_0.5": 0.00000, + "transition_ten_horizon_mean_ap": 13.11854, + "transition_ap_over_random": 1.37 + }, + { + "variant": "r2-expected-reward", + "seed": 1, + "horizon_0.2_map": 73.82947, + "horizon_0.2_background_ap": 56.97474, + "horizon_0.2_wearer_ap": 78.69340, + "horizon_0.2_others_ap": 85.82027, + "ten_horizon_mean_map": 60.84178, + "transition_horizon_0.2_ap": 15.47356, + "transition_horizon_0.2_f1_at_0.5": 0.84227, + "transition_ten_horizon_mean_ap": 13.13820, + "transition_ap_over_random": 1.38 + } + ], + "two_seed_summary_percent": { + "r1-supervised": { + "horizon_0.2_map": {"mean": 73.65141, "sample_std": 0.19164}, + "horizon_0.2_background_ap": {"mean": 56.88061, "sample_std": 0.05716}, + "horizon_0.2_wearer_ap": {"mean": 78.27769, "sample_std": 0.49951}, + "horizon_0.2_others_ap": {"mean": 85.79594, "sample_std": 0.01825}, + "ten_horizon_mean_map": {"mean": 60.84192, "sample_std": 0.03067}, + "transition_horizon_0.2_ap": {"mean": 15.25823, "sample_std": 0.09432}, + "transition_horizon_0.2_f1_at_0.5": {"mean": 0.80120, "sample_std": 0.48010}, + "transition_ten_horizon_mean_ap": {"mean": 13.05264, "sample_std": 0.01287} + }, + "r2-expected-reward": { + "horizon_0.2_map": {"mean": 73.68118, "sample_std": 0.20972}, + "horizon_0.2_background_ap": {"mean": 56.93587, "sample_std": 0.05496}, + "horizon_0.2_wearer_ap": {"mean": 78.31036, "sample_std": 0.54169}, + "horizon_0.2_others_ap": {"mean": 85.79729, "sample_std": 0.03249}, + "ten_horizon_mean_map": {"mean": 60.85353, "sample_std": 0.01662}, + "transition_horizon_0.2_ap": {"mean": 15.42482, "sample_std": 0.06893}, + "transition_horizon_0.2_f1_at_0.5": {"mean": 0.42113, "sample_std": 0.59557}, + "transition_ten_horizon_mean_ap": {"mean": 13.12837, "sample_std": 0.01390} + } + }, + "paired_mean_deltas_percent": { + "r1_minus_lstr_mce": { + "horizon_0.2_map": 0.04975, + "horizon_0.2_background_ap": 0.02633, + "horizon_0.2_wearer_ap": 0.05942, + "horizon_0.2_others_ap": 0.06350, + "ten_horizon_mean_map": 0.04949 + }, + "r2_minus_lstr_mce": { + "horizon_0.2_map": 0.07951, + "horizon_0.2_background_ap": 0.08160, + "horizon_0.2_wearer_ap": 0.09209, + "horizon_0.2_others_ap": 0.06486, + "ten_horizon_mean_map": 0.06110 + }, + "r2_minus_r1": { + "horizon_0.2_map": 0.02977, + "horizon_0.2_background_ap": 0.05527, + "horizon_0.2_wearer_ap": 0.03267, + "horizon_0.2_others_ap": 0.00136, + "ten_horizon_mean_map": 0.01162, + "transition_horizon_0.2_ap": 0.16660, + "transition_ten_horizon_mean_ap": 0.07573 + } + }, + "interpretation": { + "activity_improvement_established": false, + "transition_ranking": "weak_but_above_random", + "threshold_0.5_is_calibrated": false, + "r2_additional_gain": "negligible", + "r2_is_interactive_rl": false, + "recommended_next_target": "cumulative_transition_hazard", + "required_before_final_claim": "complete_seed_2" + }, + "provenance": { + "source": "user_reported_remote_summary_and_diagnostic_console", + "remote_feature_root": "/data1/yfzx/wangx/Datasets/easycom/omni_features/open_stream_2fps_8s_last4_stack", + "remote_target_root": "/data1/yfzx/wangx/Datasets/easycom/easycom_target_frame", + "raw_remote_metrics_archived": false, + "full_horizon_metrics_archived": false + } +} +