Files
icassp2027-experiment/results/easycom_encoder_only_1s_4ep_2026-08-28/summary.json

1319 lines
42 KiB
JSON

{
"schema_version": 1,
"recorded_date": "2026-08-28",
"dataset": "easycom",
"status": "preliminary_single_seed_user_reported_remote_evaluation",
"units": {
"ap_and_map": "percent",
"deltas": "percentage_points",
"printed_precision_decimals": 5
},
"primary_protocol": "causal_valid_only",
"full_horizon_protocol_received": "egospeak_full_timeline",
"evidence": {
"current_user_attachments": 3,
"earlier_user_attachments": 2,
"full_horizon_records_received": 50,
"remote_metrics_json_received": false,
"remote_predictions_received": false,
"remote_checkpoint_payload_received": false,
"training_summary_json_received": false,
"training_curves_received": false,
"checkpoint_epoch_independently_verified": false,
"raw_media_or_feature_causality_reaudited": false,
"remote_run_git_commit_verified": false,
"seed_variance_available": false,
"note": "Attachments show inference excerpts and paths to checkpoints/training summaries, not their contents. No experiment was re-run locally."
},
"experiment": {
"new_variants": [
"encoder_audio",
"encoder_visual",
"encoder_av"
],
"reported_seed": 0,
"missing_seeds": [
1,
2
],
"planned_epochs": 4,
"train_sessions": [
4,
5,
6,
7,
8,
9,
10,
11,
12
],
"dev_sessions": [],
"test_sessions_observed": [
"Session_3",
"Session_2",
"Session_1"
],
"split_and_training_budget_source": "User run instructions and local runner at 321238a; not remote checkpoint inspection.",
"loss_observed": "lstr-mce",
"output_space_observed": "three-class-softmax",
"configured_training": {
"batch_size": 512,
"learning_rate": 0.00007,
"weight_decay": 0.00005,
"warmup_epochs": 1.6,
"warmup_factor": 0.3,
"dropout": 0.2,
"transition_objective": "none",
"selection": "final_epoch_no_dev"
},
"configured_encoder": {
"checkpoint": "Qwen2.5-Omni-3B",
"frozen": true,
"reported_transformers": "4.57.6",
"audio_window_seconds": 1,
"visual_window_seconds": 1,
"video_fps": 2,
"state_hz": 5,
"native_audio_dim": 2048,
"native_visual_dim": 2048,
"visual_readout": "mean tokens, equal-person mean, then trailing-window mean over valid frames",
"audio_readout": "pooled valid encoder outputs from half-open trailing mixed-audio window",
"provided_person_boxes": true,
"online_detector_evaluated": false,
"uses_thinker_hidden_states": false
},
"prior_G_definition": "Frozen 3B Thinker, open-stream-replay, 8s memory, 2fps, last-four-layer stack with concat-MLP",
"remote_paths": {
"project": "/data1/yfzx/wangx/Code/icassp2027-experiment",
"source_local_features_confirmed_by_user": "/data1/yfzx/wangx/Datasets/easycom/omni_features/local_qwen_1s_encoded_tf4576",
"target_root": "/data1/yfzx/wangx/Datasets/easycom/easycom_target_frame",
"exported_features_template": "/data1/yfzx/wangx/Datasets/easycom/omni_features/encoder_only_{audio,av,visual}_1s",
"checkpoint_root": "checkpoints/easycom_encoder_only_1s",
"result_root": "results/easycom_encoder_only_1s",
"prior_checkpoint_root": "checkpoints/easycom_local_b1_encoded_tf4576"
},
"trainable_parameter_counts_from_local_code": {
"assumption": "Native A/V=2048D; G=4x2048 with concat-MLP; 256D B1 local projections; LSTR-MCE; no transition head. Counts not read from remote checkpoints.",
"source": "easycom_head.TurnPredictionHead at 321238a",
"counts": {
"encoder_audio": 2134048,
"encoder_visual": 2134048,
"encoder_av": 8468547,
"thinker": 18928670,
"thinker_audio": 19720992,
"thinker_visual": 19720992,
"thinker_av": 20513314
}
}
},
"evaluation": {
"horizon_semantics": "Nominal EgoSpeak labels, 0.2 to 2.0 seconds. Physical input-to-target lead and online latency are not inferred from metric names.",
"full_timeline_note": "Includes unavailable initial h target rows per session as zero-score predictions.",
"causal_valid_only_note": "Excludes the first h target rows per session for horizon h; not an independent audit of raw input causality.",
"session_outputs_observed": [
{
"sample_id": "Session_3",
"shape": [
8400,
3,
10
],
"causal_valid_counts": [
8399,
8398,
8397,
8396,
8395,
8394,
8393,
8392,
8391,
8390
]
},
{
"sample_id": "Session_2",
"shape": [
7706,
3,
10
],
"causal_valid_counts": [
7705,
7704,
7703,
7702,
7701,
7700,
7699,
7698,
7697,
7696
]
},
{
"sample_id": "Session_1",
"shape": [
6942,
3,
10
],
"causal_valid_counts": [
6941,
6940,
6939,
6938,
6937,
6936,
6935,
6934,
6933,
6932
]
}
],
"full_timeline_num_samples_each_horizon": 23048,
"causal_valid_num_samples_each_horizon": [
23045,
23042,
23039,
23036,
23033,
23030,
23027,
23024,
23021,
23018
]
},
"causal_valid_only": {
"availability": "Only the earlier reported first-horizon class AP/mAP and ten-horizon mean; full curves not received.",
"runs": [
{
"variant": "thinker",
"input": "G",
"seed": 0,
"horizon_0_2": {
"nominal_seconds": 0.2,
"map_percent": 73.44637,
"background_ap_percent": 56.76438,
"wearer_ap_percent": 77.8665,
"others_ap_percent": 85.70824
},
"ten_horizon_mean_map_percent": 60.84861,
"full_horizon_metrics_available": false,
"source_file": "reported_causal_summary.txt"
},
{
"variant": "encoder_audio",
"input": "A",
"seed": 0,
"horizon_0_2": {
"nominal_seconds": 0.2,
"map_percent": 79.69221,
"background_ap_percent": 63.69477,
"wearer_ap_percent": 84.09982,
"others_ap_percent": 91.28205
},
"ten_horizon_mean_map_percent": 60.50771,
"full_horizon_metrics_available": false,
"source_file": "reported_causal_summary.txt"
},
{
"variant": "encoder_visual",
"input": "V",
"seed": 0,
"horizon_0_2": {
"nominal_seconds": 0.2,
"map_percent": 48.38434,
"background_ap_percent": 30.16284,
"wearer_ap_percent": 41.68834,
"others_ap_percent": 73.30186
},
"ten_horizon_mean_map_percent": 46.2525,
"full_horizon_metrics_available": false,
"source_file": "reported_causal_summary.txt"
},
{
"variant": "encoder_av",
"input": "A+V",
"seed": 0,
"horizon_0_2": {
"nominal_seconds": 0.2,
"map_percent": 79.00727,
"background_ap_percent": 63.67397,
"wearer_ap_percent": 82.69832,
"others_ap_percent": 90.64952
},
"ten_horizon_mean_map_percent": 60.76303,
"full_horizon_metrics_available": false,
"source_file": "reported_causal_summary.txt"
},
{
"variant": "thinker_audio",
"input": "G+A",
"seed": 0,
"horizon_0_2": {
"nominal_seconds": 0.2,
"map_percent": 81.72257,
"background_ap_percent": 68.15248,
"wearer_ap_percent": 85.06554,
"others_ap_percent": 91.94969
},
"ten_horizon_mean_map_percent": 65.10895,
"full_horizon_metrics_available": false,
"source_file": "reported_causal_summary.txt"
},
{
"variant": "thinker_visual",
"input": "G+V",
"seed": 0,
"horizon_0_2": {
"nominal_seconds": 0.2,
"map_percent": 73.47412,
"background_ap_percent": 56.67089,
"wearer_ap_percent": 78.60954,
"others_ap_percent": 85.14194
},
"ten_horizon_mean_map_percent": 60.99238,
"full_horizon_metrics_available": false,
"source_file": "reported_causal_summary.txt"
},
{
"variant": "thinker_av",
"input": "G+A+V",
"seed": 0,
"horizon_0_2": {
"nominal_seconds": 0.2,
"map_percent": 82.02661,
"background_ap_percent": 68.93423,
"wearer_ap_percent": 85.08583,
"others_ap_percent": 92.05978
},
"ten_horizon_mean_map_percent": 65.39978,
"full_horizon_metrics_available": false,
"source_file": "reported_causal_summary.txt"
}
],
"comparisons": [
{
"contrast": "encoder_audio_minus_thinker",
"protocol": "causal_valid_only",
"horizon_0_2_delta_pp": {
"map": 6.24584,
"background_ap": 6.93039,
"wearer_ap": 6.23332,
"others_ap": 5.57381
},
"ten_horizon_mean_delta_pp": -0.3409
},
{
"contrast": "encoder_av_minus_encoder_audio",
"protocol": "causal_valid_only",
"horizon_0_2_delta_pp": {
"map": -0.68494,
"background_ap": -0.0208,
"wearer_ap": -1.4015,
"others_ap": -0.63253
},
"ten_horizon_mean_delta_pp": 0.25532
},
{
"contrast": "thinker_audio_minus_encoder_audio",
"protocol": "causal_valid_only",
"horizon_0_2_delta_pp": {
"map": 2.03036,
"background_ap": 4.45771,
"wearer_ap": 0.96572,
"others_ap": 0.66764
},
"ten_horizon_mean_delta_pp": 4.60124
},
{
"contrast": "thinker_av_minus_encoder_av",
"protocol": "causal_valid_only",
"horizon_0_2_delta_pp": {
"map": 3.01934,
"background_ap": 5.26026,
"wearer_ap": 2.38751,
"others_ap": 1.41026
},
"ten_horizon_mean_delta_pp": 4.63675
},
{
"contrast": "thinker_av_minus_thinker_audio",
"protocol": "causal_valid_only",
"horizon_0_2_delta_pp": {
"map": 0.30404,
"background_ap": 0.78175,
"wearer_ap": 0.02029,
"others_ap": 0.11009
},
"ten_horizon_mean_delta_pp": 0.29083
}
]
},
"egospeak_full_timeline": {
"availability": "Parsed from 3 current attachments and 2 earlier attachments; not original metrics.json.",
"runs": [
{
"variant": "encoder_visual",
"input": "V",
"seed": 0,
"source_file": "raw_encoder_visual_seed0.txt",
"source_kind": "current_user_attachment",
"horizons": [
{
"nominal_seconds": 0.2,
"map_percent": 48.38326,
"background_ap_percent": 30.16284,
"wearer_ap_percent": 41.68834,
"others_ap_percent": 73.29862,
"num_samples": 23048
},
{
"nominal_seconds": 0.4,
"map_percent": 47.68478,
"background_ap_percent": 29.82215,
"wearer_ap_percent": 40.98323,
"others_ap_percent": 72.24896,
"num_samples": 23048
},
{
"nominal_seconds": 0.6,
"map_percent": 47.26191,
"background_ap_percent": 29.28129,
"wearer_ap_percent": 40.74974,
"others_ap_percent": 71.75469,
"num_samples": 23048
},
{
"nominal_seconds": 0.8,
"map_percent": 46.626,
"background_ap_percent": 28.24103,
"wearer_ap_percent": 40.4737,
"others_ap_percent": 71.16328,
"num_samples": 23048
},
{
"nominal_seconds": 1,
"map_percent": 46.00582,
"background_ap_percent": 27.73638,
"wearer_ap_percent": 39.50369,
"others_ap_percent": 70.7774,
"num_samples": 23048
},
{
"nominal_seconds": 1.2,
"map_percent": 45.75002,
"background_ap_percent": 27.39577,
"wearer_ap_percent": 39.53354,
"others_ap_percent": 70.32075,
"num_samples": 23048
},
{
"nominal_seconds": 1.4,
"map_percent": 45.41812,
"background_ap_percent": 27.0492,
"wearer_ap_percent": 39.3556,
"others_ap_percent": 69.84957,
"num_samples": 23048
},
{
"nominal_seconds": 1.6,
"map_percent": 45.56259,
"background_ap_percent": 27.36515,
"wearer_ap_percent": 39.16056,
"others_ap_percent": 70.16205,
"num_samples": 23048
},
{
"nominal_seconds": 1.8,
"map_percent": 44.79454,
"background_ap_percent": 26.64061,
"wearer_ap_percent": 38.50155,
"others_ap_percent": 69.24145,
"num_samples": 23048
},
{
"nominal_seconds": 2,
"map_percent": 44.97339,
"background_ap_percent": 27.3481,
"wearer_ap_percent": 37.96646,
"others_ap_percent": 69.6056,
"num_samples": 23048
}
],
"ten_horizon_mean_map_percent": 46.24604,
"ten_horizon_class_mean_ap_percent": {
"background_ap_percent": 28.10425,
"wearer_ap_percent": 39.79164,
"others_ap_percent": 70.84224
}
},
{
"variant": "encoder_av",
"input": "A+V",
"seed": 0,
"source_file": "raw_encoder_av_seed0.txt",
"source_kind": "current_user_attachment",
"horizons": [
{
"nominal_seconds": 0.2,
"map_percent": 79.00491,
"background_ap_percent": 63.67397,
"wearer_ap_percent": 82.69832,
"others_ap_percent": 90.64244,
"num_samples": 23048
},
{
"nominal_seconds": 0.4,
"map_percent": 72.06793,
"background_ap_percent": 53.18404,
"wearer_ap_percent": 75.84449,
"others_ap_percent": 87.17528,
"num_samples": 23048
},
{
"nominal_seconds": 0.6,
"map_percent": 66.16179,
"background_ap_percent": 45.02921,
"wearer_ap_percent": 69.82908,
"others_ap_percent": 83.62708,
"num_samples": 23048
},
{
"nominal_seconds": 0.8,
"map_percent": 61.62137,
"background_ap_percent": 40.05126,
"wearer_ap_percent": 64.4192,
"others_ap_percent": 80.39364,
"num_samples": 23048
},
{
"nominal_seconds": 1,
"map_percent": 58.49237,
"background_ap_percent": 36.31889,
"wearer_ap_percent": 61.01987,
"others_ap_percent": 78.13836,
"num_samples": 23048
},
{
"nominal_seconds": 1.2,
"map_percent": 56.07422,
"background_ap_percent": 33.93638,
"wearer_ap_percent": 57.97721,
"others_ap_percent": 76.30907,
"num_samples": 23048
},
{
"nominal_seconds": 1.4,
"map_percent": 54.61933,
"background_ap_percent": 32.99248,
"wearer_ap_percent": 55.68891,
"others_ap_percent": 75.17659,
"num_samples": 23048
},
{
"nominal_seconds": 1.6,
"map_percent": 53.7818,
"background_ap_percent": 32.58516,
"wearer_ap_percent": 54.19289,
"others_ap_percent": 74.56735,
"num_samples": 23048
},
{
"nominal_seconds": 1.8,
"map_percent": 52.91556,
"background_ap_percent": 32.18203,
"wearer_ap_percent": 52.7506,
"others_ap_percent": 73.81406,
"num_samples": 23048
},
{
"nominal_seconds": 2,
"map_percent": 52.77341,
"background_ap_percent": 32.53903,
"wearer_ap_percent": 52.15913,
"others_ap_percent": 73.62206,
"num_samples": 23048
}
],
"ten_horizon_mean_map_percent": 60.75127,
"ten_horizon_class_mean_ap_percent": {
"background_ap_percent": 40.24924,
"wearer_ap_percent": 62.65797,
"others_ap_percent": 79.34659
}
},
{
"variant": "encoder_audio",
"input": "A",
"seed": 0,
"source_file": "raw_encoder_audio_seed0.txt",
"source_kind": "current_user_attachment",
"horizons": [
{
"nominal_seconds": 0.2,
"map_percent": 79.68981,
"background_ap_percent": 63.69477,
"wearer_ap_percent": 84.09982,
"others_ap_percent": 91.27483,
"num_samples": 23048
},
{
"nominal_seconds": 0.4,
"map_percent": 72.41576,
"background_ap_percent": 52.4778,
"wearer_ap_percent": 77.43606,
"others_ap_percent": 87.3334,
"num_samples": 23048
},
{
"nominal_seconds": 0.6,
"map_percent": 66.33666,
"background_ap_percent": 44.20066,
"wearer_ap_percent": 71.41221,
"others_ap_percent": 83.3971,
"num_samples": 23048
},
{
"nominal_seconds": 0.8,
"map_percent": 61.60584,
"background_ap_percent": 38.60179,
"wearer_ap_percent": 66.08811,
"others_ap_percent": 80.12762,
"num_samples": 23048
},
{
"nominal_seconds": 1,
"map_percent": 58.28586,
"background_ap_percent": 34.75843,
"wearer_ap_percent": 62.53734,
"others_ap_percent": 77.56181,
"num_samples": 23048
},
{
"nominal_seconds": 1.2,
"map_percent": 55.93708,
"background_ap_percent": 32.93325,
"wearer_ap_percent": 59.29096,
"others_ap_percent": 75.58705,
"num_samples": 23048
},
{
"nominal_seconds": 1.4,
"map_percent": 54.22585,
"background_ap_percent": 31.56977,
"wearer_ap_percent": 56.79707,
"others_ap_percent": 74.31072,
"num_samples": 23048
},
{
"nominal_seconds": 1.6,
"map_percent": 52.83014,
"background_ap_percent": 30.67884,
"wearer_ap_percent": 54.61175,
"others_ap_percent": 73.19983,
"num_samples": 23048
},
{
"nominal_seconds": 1.8,
"map_percent": 51.97218,
"background_ap_percent": 30.3703,
"wearer_ap_percent": 52.80894,
"others_ap_percent": 72.73728,
"num_samples": 23048
},
{
"nominal_seconds": 2,
"map_percent": 51.66256,
"background_ap_percent": 30.86998,
"wearer_ap_percent": 51.49671,
"others_ap_percent": 72.62098,
"num_samples": 23048
}
],
"ten_horizon_mean_map_percent": 60.49617,
"ten_horizon_class_mean_ap_percent": {
"background_ap_percent": 39.01556,
"wearer_ap_percent": 63.6579,
"others_ap_percent": 78.81506
}
},
{
"variant": "thinker_av",
"input": "G+A+V",
"seed": 0,
"source_file": "raw_previous_thinker_av_seed0.txt",
"source_kind": "earlier_user_attachment",
"horizons": [
{
"nominal_seconds": 0.2,
"map_percent": 82.02415,
"background_ap_percent": 68.93423,
"wearer_ap_percent": 85.08583,
"others_ap_percent": 92.05239,
"num_samples": 23048
},
{
"nominal_seconds": 0.4,
"map_percent": 75.99169,
"background_ap_percent": 60.58903,
"wearer_ap_percent": 78.83427,
"others_ap_percent": 88.55177,
"num_samples": 23048
},
{
"nominal_seconds": 0.6,
"map_percent": 70.92665,
"background_ap_percent": 53.84208,
"wearer_ap_percent": 73.45503,
"others_ap_percent": 85.48283,
"num_samples": 23048
},
{
"nominal_seconds": 0.8,
"map_percent": 66.79615,
"background_ap_percent": 49.21382,
"wearer_ap_percent": 68.56072,
"others_ap_percent": 82.6139,
"num_samples": 23048
},
{
"nominal_seconds": 1,
"map_percent": 63.86856,
"background_ap_percent": 45.62073,
"wearer_ap_percent": 65.27034,
"others_ap_percent": 80.71461,
"num_samples": 23048
},
{
"nominal_seconds": 1.2,
"map_percent": 61.5628,
"background_ap_percent": 43.4667,
"wearer_ap_percent": 62.05277,
"others_ap_percent": 79.16893,
"num_samples": 23048
},
{
"nominal_seconds": 1.4,
"map_percent": 60.3407,
"background_ap_percent": 42.54682,
"wearer_ap_percent": 60.2054,
"others_ap_percent": 78.26988,
"num_samples": 23048
},
{
"nominal_seconds": 1.6,
"map_percent": 58.83407,
"background_ap_percent": 41.15952,
"wearer_ap_percent": 58.18522,
"others_ap_percent": 77.15747,
"num_samples": 23048
},
{
"nominal_seconds": 1.8,
"map_percent": 57.39446,
"background_ap_percent": 39.94176,
"wearer_ap_percent": 55.99271,
"others_ap_percent": 76.2489,
"num_samples": 23048
},
{
"nominal_seconds": 2,
"map_percent": 56.12449,
"background_ap_percent": 38.87001,
"wearer_ap_percent": 53.82707,
"others_ap_percent": 75.67639,
"num_samples": 23048
}
],
"ten_horizon_mean_map_percent": 65.38637,
"ten_horizon_class_mean_ap_percent": {
"background_ap_percent": 48.41847,
"wearer_ap_percent": 66.14694,
"others_ap_percent": 81.59371
}
},
{
"variant": "thinker_audio",
"input": "G+A",
"seed": 0,
"source_file": "raw_previous_thinker_audio_seed0.txt",
"source_kind": "earlier_user_attachment",
"horizons": [
{
"nominal_seconds": 0.2,
"map_percent": 81.72011,
"background_ap_percent": 68.15248,
"wearer_ap_percent": 85.06554,
"others_ap_percent": 91.94232,
"num_samples": 23048
},
{
"nominal_seconds": 0.4,
"map_percent": 75.91398,
"background_ap_percent": 59.90715,
"wearer_ap_percent": 79.22465,
"others_ap_percent": 88.61015,
"num_samples": 23048
},
{
"nominal_seconds": 0.6,
"map_percent": 70.37189,
"background_ap_percent": 52.4848,
"wearer_ap_percent": 73.21461,
"others_ap_percent": 85.41626,
"num_samples": 23048
},
{
"nominal_seconds": 0.8,
"map_percent": 66.45047,
"background_ap_percent": 47.92066,
"wearer_ap_percent": 68.779,
"others_ap_percent": 82.65175,
"num_samples": 23048
},
{
"nominal_seconds": 1,
"map_percent": 63.79981,
"background_ap_percent": 45.59262,
"wearer_ap_percent": 65.0989,
"others_ap_percent": 80.70791,
"num_samples": 23048
},
{
"nominal_seconds": 1.2,
"map_percent": 61.3512,
"background_ap_percent": 43.15384,
"wearer_ap_percent": 61.86698,
"others_ap_percent": 79.03278,
"num_samples": 23048
},
{
"nominal_seconds": 1.4,
"map_percent": 59.98279,
"background_ap_percent": 41.83907,
"wearer_ap_percent": 60.22346,
"others_ap_percent": 77.88584,
"num_samples": 23048
},
{
"nominal_seconds": 1.6,
"map_percent": 58.29734,
"background_ap_percent": 40.44538,
"wearer_ap_percent": 57.66189,
"others_ap_percent": 76.78474,
"num_samples": 23048
},
{
"nominal_seconds": 1.8,
"map_percent": 56.93221,
"background_ap_percent": 39.04677,
"wearer_ap_percent": 55.70067,
"others_ap_percent": 76.04919,
"num_samples": 23048
},
{
"nominal_seconds": 2,
"map_percent": 56.13656,
"background_ap_percent": 38.77318,
"wearer_ap_percent": 54.02582,
"others_ap_percent": 75.61069,
"num_samples": 23048
}
],
"ten_horizon_mean_map_percent": 65.09564,
"ten_horizon_class_mean_ap_percent": {
"background_ap_percent": 47.73159,
"wearer_ap_percent": 66.08615,
"others_ap_percent": 81.46916
}
}
],
"comparisons": [
{
"contrast": "encoder_av_minus_encoder_audio",
"protocol": "egospeak_full_timeline",
"ten_horizon_mean_delta_pp": 0.2551,
"per_horizon": [
{
"nominal_seconds": 0.2,
"map_delta_pp": -0.6849,
"background_ap_delta_pp": -0.0208,
"wearer_ap_delta_pp": -1.4015,
"others_ap_delta_pp": -0.63239
},
{
"nominal_seconds": 0.4,
"map_delta_pp": -0.34783,
"background_ap_delta_pp": 0.70624,
"wearer_ap_delta_pp": -1.59157,
"others_ap_delta_pp": -0.15812
},
{
"nominal_seconds": 0.6,
"map_delta_pp": -0.17487,
"background_ap_delta_pp": 0.82855,
"wearer_ap_delta_pp": -1.58313,
"others_ap_delta_pp": 0.22998
},
{
"nominal_seconds": 0.8,
"map_delta_pp": 0.01553,
"background_ap_delta_pp": 1.44947,
"wearer_ap_delta_pp": -1.66891,
"others_ap_delta_pp": 0.26602
},
{
"nominal_seconds": 1,
"map_delta_pp": 0.20651,
"background_ap_delta_pp": 1.56046,
"wearer_ap_delta_pp": -1.51747,
"others_ap_delta_pp": 0.57655
},
{
"nominal_seconds": 1.2,
"map_delta_pp": 0.13714,
"background_ap_delta_pp": 1.00313,
"wearer_ap_delta_pp": -1.31375,
"others_ap_delta_pp": 0.72202
},
{
"nominal_seconds": 1.4,
"map_delta_pp": 0.39348,
"background_ap_delta_pp": 1.42271,
"wearer_ap_delta_pp": -1.10816,
"others_ap_delta_pp": 0.86587
},
{
"nominal_seconds": 1.6,
"map_delta_pp": 0.95166,
"background_ap_delta_pp": 1.90632,
"wearer_ap_delta_pp": -0.41886,
"others_ap_delta_pp": 1.36752
},
{
"nominal_seconds": 1.8,
"map_delta_pp": 0.94338,
"background_ap_delta_pp": 1.81173,
"wearer_ap_delta_pp": -0.05834,
"others_ap_delta_pp": 1.07678
},
{
"nominal_seconds": 2,
"map_delta_pp": 1.11085,
"background_ap_delta_pp": 1.66905,
"wearer_ap_delta_pp": 0.66242,
"others_ap_delta_pp": 1.00108
}
],
"positive_map_horizons": [
0.8,
1,
1.2,
1.4,
1.6,
1.8,
2
],
"negative_wearer_horizons": [
0.2,
0.4,
0.6,
0.8,
1,
1.2,
1.4,
1.6,
1.8
],
"ten_horizon_mean_class_delta_pp": {
"background_ap_delta_pp": 1.23369,
"wearer_ap_delta_pp": -0.99993,
"others_ap_delta_pp": 0.53153
}
},
{
"contrast": "thinker_audio_minus_encoder_audio",
"protocol": "egospeak_full_timeline",
"ten_horizon_mean_delta_pp": 4.59947,
"per_horizon": [
{
"nominal_seconds": 0.2,
"map_delta_pp": 2.0303,
"background_ap_delta_pp": 4.45771,
"wearer_ap_delta_pp": 0.96572,
"others_ap_delta_pp": 0.66749
},
{
"nominal_seconds": 0.4,
"map_delta_pp": 3.49822,
"background_ap_delta_pp": 7.42935,
"wearer_ap_delta_pp": 1.78859,
"others_ap_delta_pp": 1.27675
},
{
"nominal_seconds": 0.6,
"map_delta_pp": 4.03523,
"background_ap_delta_pp": 8.28414,
"wearer_ap_delta_pp": 1.8024,
"others_ap_delta_pp": 2.01916
},
{
"nominal_seconds": 0.8,
"map_delta_pp": 4.84463,
"background_ap_delta_pp": 9.31887,
"wearer_ap_delta_pp": 2.69089,
"others_ap_delta_pp": 2.52413
},
{
"nominal_seconds": 1,
"map_delta_pp": 5.51395,
"background_ap_delta_pp": 10.83419,
"wearer_ap_delta_pp": 2.56156,
"others_ap_delta_pp": 3.1461
},
{
"nominal_seconds": 1.2,
"map_delta_pp": 5.41412,
"background_ap_delta_pp": 10.22059,
"wearer_ap_delta_pp": 2.57602,
"others_ap_delta_pp": 3.44573
},
{
"nominal_seconds": 1.4,
"map_delta_pp": 5.75694,
"background_ap_delta_pp": 10.2693,
"wearer_ap_delta_pp": 3.42639,
"others_ap_delta_pp": 3.57512
},
{
"nominal_seconds": 1.6,
"map_delta_pp": 5.4672,
"background_ap_delta_pp": 9.76654,
"wearer_ap_delta_pp": 3.05014,
"others_ap_delta_pp": 3.58491
},
{
"nominal_seconds": 1.8,
"map_delta_pp": 4.96003,
"background_ap_delta_pp": 8.67647,
"wearer_ap_delta_pp": 2.89173,
"others_ap_delta_pp": 3.31191
},
{
"nominal_seconds": 2,
"map_delta_pp": 4.474,
"background_ap_delta_pp": 7.9032,
"wearer_ap_delta_pp": 2.52911,
"others_ap_delta_pp": 2.98971
}
],
"positive_map_horizons": [
0.2,
0.4,
0.6,
0.8,
1,
1.2,
1.4,
1.6,
1.8,
2
],
"negative_wearer_horizons": [],
"ten_horizon_mean_class_delta_pp": {
"background_ap_delta_pp": 8.71604,
"wearer_ap_delta_pp": 2.42825,
"others_ap_delta_pp": 2.6541
}
},
{
"contrast": "thinker_av_minus_encoder_av",
"protocol": "egospeak_full_timeline",
"ten_horizon_mean_delta_pp": 4.6351,
"per_horizon": [
{
"nominal_seconds": 0.2,
"map_delta_pp": 3.01924,
"background_ap_delta_pp": 5.26026,
"wearer_ap_delta_pp": 2.38751,
"others_ap_delta_pp": 1.40995
},
{
"nominal_seconds": 0.4,
"map_delta_pp": 3.92376,
"background_ap_delta_pp": 7.40499,
"wearer_ap_delta_pp": 2.98978,
"others_ap_delta_pp": 1.37649
},
{
"nominal_seconds": 0.6,
"map_delta_pp": 4.76486,
"background_ap_delta_pp": 8.81287,
"wearer_ap_delta_pp": 3.62595,
"others_ap_delta_pp": 1.85575
},
{
"nominal_seconds": 0.8,
"map_delta_pp": 5.17478,
"background_ap_delta_pp": 9.16256,
"wearer_ap_delta_pp": 4.14152,
"others_ap_delta_pp": 2.22026
},
{
"nominal_seconds": 1,
"map_delta_pp": 5.37619,
"background_ap_delta_pp": 9.30184,
"wearer_ap_delta_pp": 4.25047,
"others_ap_delta_pp": 2.57625
},
{
"nominal_seconds": 1.2,
"map_delta_pp": 5.48858,
"background_ap_delta_pp": 9.53032,
"wearer_ap_delta_pp": 4.07556,
"others_ap_delta_pp": 2.85986
},
{
"nominal_seconds": 1.4,
"map_delta_pp": 5.72137,
"background_ap_delta_pp": 9.55434,
"wearer_ap_delta_pp": 4.51649,
"others_ap_delta_pp": 3.09329
},
{
"nominal_seconds": 1.6,
"map_delta_pp": 5.05227,
"background_ap_delta_pp": 8.57436,
"wearer_ap_delta_pp": 3.99233,
"others_ap_delta_pp": 2.59012
},
{
"nominal_seconds": 1.8,
"map_delta_pp": 4.4789,
"background_ap_delta_pp": 7.75973,
"wearer_ap_delta_pp": 3.24211,
"others_ap_delta_pp": 2.43484
},
{
"nominal_seconds": 2,
"map_delta_pp": 3.35108,
"background_ap_delta_pp": 6.33098,
"wearer_ap_delta_pp": 1.66794,
"others_ap_delta_pp": 2.05433
}
],
"positive_map_horizons": [
0.2,
0.4,
0.6,
0.8,
1,
1.2,
1.4,
1.6,
1.8,
2
],
"negative_wearer_horizons": [],
"ten_horizon_mean_class_delta_pp": {
"background_ap_delta_pp": 8.16923,
"wearer_ap_delta_pp": 3.48897,
"others_ap_delta_pp": 2.24711
}
},
{
"contrast": "thinker_av_minus_thinker_audio",
"protocol": "egospeak_full_timeline",
"ten_horizon_mean_delta_pp": 0.29073,
"per_horizon": [
{
"nominal_seconds": 0.2,
"map_delta_pp": 0.30404,
"background_ap_delta_pp": 0.78175,
"wearer_ap_delta_pp": 0.02029,
"others_ap_delta_pp": 0.11007
},
{
"nominal_seconds": 0.4,
"map_delta_pp": 0.07771,
"background_ap_delta_pp": 0.68188,
"wearer_ap_delta_pp": -0.39038,
"others_ap_delta_pp": -0.05838
},
{
"nominal_seconds": 0.6,
"map_delta_pp": 0.55476,
"background_ap_delta_pp": 1.35728,
"wearer_ap_delta_pp": 0.24042,
"others_ap_delta_pp": 0.06657
},
{
"nominal_seconds": 0.8,
"map_delta_pp": 0.34568,
"background_ap_delta_pp": 1.29316,
"wearer_ap_delta_pp": -0.21828,
"others_ap_delta_pp": -0.03785
},
{
"nominal_seconds": 1,
"map_delta_pp": 0.06875,
"background_ap_delta_pp": 0.02811,
"wearer_ap_delta_pp": 0.17144,
"others_ap_delta_pp": 0.0067
},
{
"nominal_seconds": 1.2,
"map_delta_pp": 0.2116,
"background_ap_delta_pp": 0.31286,
"wearer_ap_delta_pp": 0.18579,
"others_ap_delta_pp": 0.13615
},
{
"nominal_seconds": 1.4,
"map_delta_pp": 0.35791,
"background_ap_delta_pp": 0.70775,
"wearer_ap_delta_pp": -0.01806,
"others_ap_delta_pp": 0.38404
},
{
"nominal_seconds": 1.6,
"map_delta_pp": 0.53673,
"background_ap_delta_pp": 0.71414,
"wearer_ap_delta_pp": 0.52333,
"others_ap_delta_pp": 0.37273
},
{
"nominal_seconds": 1.8,
"map_delta_pp": 0.46225,
"background_ap_delta_pp": 0.89499,
"wearer_ap_delta_pp": 0.29204,
"others_ap_delta_pp": 0.19971
},
{
"nominal_seconds": 2,
"map_delta_pp": -0.01207,
"background_ap_delta_pp": 0.09683,
"wearer_ap_delta_pp": -0.19875,
"others_ap_delta_pp": 0.0657
}
],
"positive_map_horizons": [
0.2,
0.4,
0.6,
0.8,
1,
1.2,
1.4,
1.6,
1.8
],
"negative_wearer_horizons": [
0.4,
0.8,
1.4,
2
],
"ten_horizon_mean_class_delta_pp": {
"background_ap_delta_pp": 0.68687,
"wearer_ap_delta_pp": 0.06078,
"others_ap_delta_pp": 0.12454
}
}
]
},
"protocol_differences": [
{
"variant": "encoder_visual",
"causal_minus_full_horizon_0_2_map_pp": 0.00108,
"causal_minus_full_ten_horizon_mean_map_pp": 0.00646
},
{
"variant": "encoder_av",
"causal_minus_full_horizon_0_2_map_pp": 0.00236,
"causal_minus_full_ten_horizon_mean_map_pp": 0.01176
},
{
"variant": "encoder_audio",
"causal_minus_full_horizon_0_2_map_pp": 0.0024,
"causal_minus_full_ten_horizon_mean_map_pp": 0.01154
},
{
"variant": "thinker_av",
"causal_minus_full_horizon_0_2_map_pp": 0.00246,
"causal_minus_full_ten_horizon_mean_map_pp": 0.01341
},
{
"variant": "thinker_audio",
"causal_minus_full_horizon_0_2_map_pp": 0.00246,
"causal_minus_full_ten_horizon_mean_map_pp": 0.01331
}
],
"interpretation": {
"observation_1": "In this seed, G+A beats A at every full-timeline horizon and class; G+AV likewise beats AV. This supports a useful combined system, not isolated decoder causation.",
"observation_2": "Encoder AV trails A in mAP at 0.2/0.4/0.6, turns slightly positive at 0.8, and gains about one pp at 1.6-2.0. Wearer AP declines at 9/10 horizons.",
"observation_3": "Adding local V to G+A yields small macro gains mainly from background; mean wearer AP change is only +0.06078 pp.",
"unproven_explanations": [
"Longer history contribution versus head capacity or nonlinear fusion",
"Whether person/time pooling suppresses speaker-specific information",
"Whether a different fusion or training budget would avoid negative wearer transfer"
],
"limitations": [
"One reported seed",
"Different head capacities and visual fields/history budgets",
"No training curves, remote model metadata, or original prediction arrays",
"Overall AP does not establish multi-person interaction understanding or boundary anticipation",
"No significance from treating correlated horizons as independent replicates",
"No feature-extraction or end-to-end speed claim from inference batch/s"
],
"next_steps": [
"Collect raw metrics.json and checkpoint/training metadata, plus causal-valid curves",
"Complete seeds 1/2 with fixed epoch-4/no-dev protocol",
"Run architecture/parameter and raw-visible-history controlled modality comparisons before attributing gains to Thinker",
"Only then test audio-conditioned per-person fusion and pre-specified multi-person/overlap reporting; avoid test-driven epoch/threshold selection"
]
},
"provenance": {
"normalization": "Console archives preserve all lines and wrapped paths; trailing horizontal whitespace removed and one final newline added. Numeric values unchanged.",
"sources": [
{
"variant": "encoder_visual",
"source_kind": "current_user_attachment",
"original_attachment_path": "/Users/wx44wx/.codex/attachments/a58e2943-0d9a-4d93-8b20-94ecd405c632/pasted-text.txt",
"archived_file": "raw_encoder_visual_seed0.txt"
},
{
"variant": "encoder_av",
"source_kind": "current_user_attachment",
"original_attachment_path": "/Users/wx44wx/.codex/attachments/ce28f77e-e131-497e-b9aa-83d9b1dc9171/pasted-text.txt",
"archived_file": "raw_encoder_av_seed0.txt"
},
{
"variant": "encoder_audio",
"source_kind": "current_user_attachment",
"original_attachment_path": "/Users/wx44wx/.codex/attachments/094440a9-5a37-4737-a32b-e905cdefbf3d/pasted-text.txt",
"archived_file": "raw_encoder_audio_seed0.txt"
},
{
"variant": "thinker_av",
"source_kind": "earlier_user_attachment",
"original_attachment_path": "/Users/wx44wx/.codex/attachments/e3e61fca-d5d2-452f-b040-c156f3d415e2/pasted-text.txt",
"archived_file": "raw_previous_thinker_av_seed0.txt"
},
{
"variant": "thinker_audio",
"source_kind": "earlier_user_attachment",
"original_attachment_path": "/Users/wx44wx/.codex/attachments/4a1649ae-1881-4c79-a7c4-0b0f56140eb8/pasted-text.txt",
"archived_file": "raw_previous_thinker_audio_seed0.txt"
}
],
"supplementary_source": "Earlier user summary tables and run commands in this thread, transcribed in reported_causal_summary.txt",
"locally_available_artifact": "Derived research record only, not a replacement for remote metrics.json"
}
}