diff --git a/.gitignore b/.gitignore index 4f06fce..ecbb082 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ __pycache__/ /outputs/ /target_frame/ /third_party/EgoSpeak/ +/third_party/MuVAP/ # Local editors and environments. .idea/ diff --git a/docs/global_local_omni_paper_position_2026-09-07.md b/docs/global_local_omni_paper_position_2026-09-07.md new file mode 100644 index 0000000..71bed07 --- /dev/null +++ b/docs/global_local_omni_paper_position_2026-09-07.md @@ -0,0 +1,78 @@ +# Global Omni + causal local AV:当前论文定位(2026-09-07) + +## 决策 + +当前论文主线写成: + +> 面向真实多人场景中的 egocentric conversational agent,用 human camera wearer 的未来 +> 说话行为作为 agent floor action 的 imitation target;研究全局 Omni 表示是否遗漏了 +> 决定何时开口/保持倾听所需的及时局部视听 floor evidence,并用一个严格因果的局部分支 +> 补充这种信息。 + +模型分解为两部分: + +- **Global Omni context**:保留场景、长时对话和联合音视频语境; +- **Causal local AV evidence**:从最近 mixed audio 和可变数量的 visible-person observations + 中恢复短时 floor activity,再与全局状态融合。 + +人物框/track 是组织局部观测的接口。只有在 correct track 稳定优于 aggregate、no-ID、 +shuffled/broken-ID、错人和错时控制后,track continuity 才能成为方法贡献。在此之前不能 +声称模型学会了 speaker identity、addressee 或 other-to-other 语义理解。 + +## 为什么这个主线比“直接用 person track”更符合现有证据 + +EasyCom Session 12 的 single-seed causal oracle screen 显示: + +- `aggregate-visible - audio`:0.2 秒 mAP `+6.96649`,十 horizon mean mAP `+2.91047`; +- visibility-only 和延迟 1 秒的 activity 接近 audio baseline; +- `aggregate-visible` 高于 `per-person`; +- `per-person - shuffled-id`:0.2 秒 `-0.05826`,mean mAP `-0.07725`。 + +因此已支持的命题是“及时的局部 visible-other activity 有信息价值”。尚未支持的命题是 +“人物 ID 连续性本身有额外价值”。第一版 track-GRU 又因 zero scalar gate 阻塞早期分支 +梯度而不是有效的阴性实验,不能据此断言 track 无用。 + +完整数值和协议见 +[`EasyCom oracle result record`](../results/easycom_oracle_information_4ep_2026-09-07/README.md)。 + +## 论文中的 claim ladder + +### 现在可以写 + +1. 在当前 Qwen2.5-Omni-3B causal readout 下,全局状态与最近 mixed audio 尚未包含全部及时 + visible-other floor information。 +2. analysis-only oracle 证明 local aggregate activity 是有意义的互补信号。 +3. 提出并比较从真实因果 AV 恢复该信号的 global--local 模型族。 +4. provided boxes/tracks 是方法隔离条件;部署时可由 online detector/tracker 提供,但在 + 运行该前端前不能称为完整 online-track 系统。 + +### 现在不能写 + +1. 通用 Omni 模型都缺乏这种能力;目前证据只来自 Qwen2.5-Omni-3B 和当前 readout。 +2. person track、speaker identity 或跨时身份连续性提高了 floor action。 +3. 模型理解了谁对谁说话、语义 EOT/INT 或 addressee。 +4. dense mAP 的提高已经减少实际错误抢话;这必须由冻结后的事件协议证明。 + +## 方法选择顺序 + +1. seeds 1--2 复核 oracle 六条件,保持 Session 12 为 development screen。 +2. 训练 causal visible-person activity/floor estimator,先尝试恢复 aggregate oracle gap。 +3. 比较 `G+A`、pooled local AV、Frame-set Attention、参数匹配分支,以及 + past-shift/wrong-person/wrong-time/NULL 控制。 +4. 只有 person-organized feature 通过这些控制,才继续修复 track residual 并测试 + broken continuity;否则最终模型保持更简单的 aggregate local branch。 +5. 在主结果前冻结 competing-risk-censored event evaluator,报告 matched wearer-onset + recall 下的 multi-other false entry/min,同时确认 valid entry 和 yield 不退化。 +6. 架构和协议冻结后再一次性运行 EasyCom Sessions 1--3;Ego4D 只报告它的标注确实支持的 + activity/floor 指标。 + +## 稿件组织 + +- **问题**:真实多人环境中的 agent floor action,而不是预测 camera wearer 本身作为最终 + 应用;wearer 行为只是可监督的 imitation target。 +- **诊断**:global Omni 与 local evidence 的信息分解,以及 oracle 的正/负控制。 +- **方法**:最终只描述通过 gate 的 learned local variant;未通过的 track/adapter 放消融。 +- **评价**:EgoSpeak-compatible dense AP 用于可比性,event metrics 用于验证行为价值。 +- **结论边界**:aggregate local activity 的价值已得到初步支持;identity continuity 尚未 + 得到支持。若 learned branch 不能恢复 oracle gap,论文只能定位为 information audit, + 不能包装成 track-based solution。 diff --git a/manuscript/main/main.tex b/manuscript/main/main.tex index 1f91e44..b63cdfa 100644 --- a/manuscript/main/main.tex +++ b/manuscript/main/main.tex @@ -10,8 +10,7 @@ \newcommand{\vect}[1]{\boldsymbol{#1}} \setlength{\emergencystretch}{2em} -\title{GLOBAL OMNI CONTEXT AND CAUSAL LOCAL AUDIOVISUAL EVIDENCE FOR -EGOCENTRIC MULTIPARTY FLOOR-ACTION FORECASTING} +\title{Adapting Omni Models for Egocentric Multiparty Floor-Entry Forecasting} % TODO: Insert the final author list and affiliations after the ICASSP 2027 % review policy and author order are finalized. diff --git a/results/easycom_oracle_information_4ep_2026-09-07/README.md b/results/easycom_oracle_information_4ep_2026-09-07/README.md new file mode 100644 index 0000000..ce06d5a --- /dev/null +++ b/results/easycom_oracle_information_4ep_2026-09-07/README.md @@ -0,0 +1,86 @@ +# EasyCom causal oracle-information screen (4 epochs, 2026-09-07) + +## Question + +Does perfectly timed, non-wearer speech activity contain information that is +missing from the frozen Qwen2.5-Omni global state and the existing recent +mixed-audio branch? If so, is the useful information aggregate activity or +person-continuous activity? + +This is an information upper bound, not a deployable feature condition. The +oracle inputs use EasyCom transcript-derived speech labels at or before the +current cutoff; no future wearer target, transcript text, target-of-speech, or +close-microphone signal is exposed. + +## Protocol and provenance + +- Dataset: EasyCom using the EgoSpeak target and split. +- Model development: Sessions 4--11 for training and Session 12 for checkpoint + selection and this screen. Sessions 1--3 were not evaluated. +- Seed: 0 only; three-seed confirmation is pending. +- Training: 4 epochs, LSTR-MCE activity loss, frozen Qwen2.5-Omni-3B global + states, `concat-mlp` layer fusion, and the same recent local-audio branch. +- Evaluation: `causal_valid_only`; AP/mAP values below are percentages. +- Oracle history: 10 samples at 5 Hz (2 seconds). The delayed control shifts + person activity by 5 samples (1 second). +- Target alignment: EgoSpeak right-edge sampling, + `Start_Frame <= 4*(t+1) < End_Frame`; all twelve sessions must have zero + transcript/target mismatch before any sidecar is written. +- Raw run directory on the L20 host: + `results/easycom_oracle_information_4ep/`. + +## Results + +| Condition | 0.2-s mAP | BG AP | wearer AP | others AP | 10-horizon mean mAP | +|---|---:|---:|---:|---:|---:| +| audio | 72.57922 | 59.84555 | 66.82464 | 91.06747 | 58.77336 | +| visibility-only | 71.95065 | 57.85640 | 66.73511 | 91.26045 | 58.86491 | +| delayed | 72.02265 | 57.86302 | 66.88550 | 91.31943 | 58.89590 | +| aggregate-visible | **79.54571** | **69.79448** | **72.65278** | **96.18987** | **61.68383** | +| per-person | 78.80474 | 69.74061 | 71.00535 | 95.66825 | 61.10890 | +| shuffled-id | 78.86300 | 68.70055 | 71.85510 | 96.03333 | 61.18615 | + +Important paired descriptive deltas for this single seed: + +- `aggregate-visible - audio`: +6.96649 at 0.2 seconds and +2.91047 mean mAP. +- `aggregate-visible - delayed`: +7.52306 at 0.2 seconds and +2.78793 mean mAP. +- `aggregate-visible - per-person`: +0.74097 at 0.2 seconds and +0.57493 mean mAP. +- `per-person - shuffled-id`: -0.05826 at 0.2 seconds and -0.07725 mean mAP. +- `visibility-only - audio`: -0.62857 at 0.2 seconds and +0.09155 mean mAP. + +## Interpretation + +The positive result is an information-value result: accurately aligned current +activity from visible non-wearers supplies substantial information beyond the +frozen global Omni state and recent mixed audio. The visibility-only and +one-second-delayed controls are close to the audio baseline, so the gain is +not explained by the number of slots, extra parameters, or stale activity. + +The person-continuity hypothesis is not supported by this run. Aggregate +visible activity is stronger than the person-resolved representation, and +independently shuffling historical activity across current person slots does +not hurt. The present dense aggregate-`others` objective can exploit per-lag +activity counts without identifying who produced them, and the simpler +aggregate representation is also easier for the head to consume. Therefore +this result cannot support claims of speaker identity, speaker continuity, +addressee understanding, or other-to-other interaction understanding. + +The defensible method hypothesis is now **global Omni context plus causal local +audiovisual floor evidence organized around visible people**. A learned causal +active-speaker/floor branch should first attempt to recover the aggregate +oracle gain. Anonymous track continuity becomes part of the selected method +only if correct tracks beat no-ID, broken-continuity, time-shifted, and +capacity-matched controls and improve the frozen event protocol. Otherwise the +paper must use the simpler global-plus-local-floor-evidence formulation. + +## Required follow-up + +1. Repeat the six conditions with seeds 1 and 2 on Session 12. +2. Do not use Sessions 1--3 until the architecture and event protocol are + frozen. +3. Train a causal learned visible-person activity estimator from deployable + audio/video inputs; report how much of the oracle gap it recovers. +4. Freeze competing-risk-censored entry/yield evaluation. Dense mAP does not + demonstrate fewer erroneous agent entries. +5. Test person continuity only with matched no-ID/broken-track controls; stop + the track claim if they do not separate. diff --git a/results/easycom_oracle_information_4ep_2026-09-07/metrics.csv b/results/easycom_oracle_information_4ep_2026-09-07/metrics.csv new file mode 100644 index 0000000..f72f710 --- /dev/null +++ b/results/easycom_oracle_information_4ep_2026-09-07/metrics.csv @@ -0,0 +1,7 @@ +condition,seed,split,protocol,map_0p2,bg_ap_0p2,wearer_ap_0p2,others_ap_0p2,mean_map +audio,0,Session_12,causal_valid_only,72.57922,59.84555,66.82464,91.06747,58.77336 +visibility-only,0,Session_12,causal_valid_only,71.95065,57.85640,66.73511,91.26045,58.86491 +delayed,0,Session_12,causal_valid_only,72.02265,57.86302,66.88550,91.31943,58.89590 +aggregate-visible,0,Session_12,causal_valid_only,79.54571,69.79448,72.65278,96.18987,61.68383 +per-person,0,Session_12,causal_valid_only,78.80474,69.74061,71.00535,95.66825,61.10890 +shuffled-id,0,Session_12,causal_valid_only,78.86300,68.70055,71.85510,96.03333,61.18615 diff --git a/results/easycom_oracle_information_4ep_2026-09-07/summary.json b/results/easycom_oracle_information_4ep_2026-09-07/summary.json new file mode 100644 index 0000000..cf53349 --- /dev/null +++ b/results/easycom_oracle_information_4ep_2026-09-07/summary.json @@ -0,0 +1,43 @@ +{ + "record": "easycom_oracle_information_4ep_2026-09-07", + "status": "preliminary_single_seed_development_screen", + "dataset": "easycom", + "train_sessions": [4, 5, 6, 7, 8, 9, 10, 11], + "development_sessions": [12], + "test_sessions_used": [], + "seed": 0, + "epochs": 4, + "protocol": "causal_valid_only", + "units": "percentage_points", + "oracle_history_steps": 10, + "oracle_history_seconds": 2.0, + "delay_steps": 5, + "delay_seconds": 1.0, + "target_sampling_policy": "egospeak-right-edge-start-inclusive-end-exclusive", + "metrics": { + "audio": [72.57922, 59.84555, 66.82464, 91.06747, 58.77336], + "visibility-only": [71.95065, 57.85640, 66.73511, 91.26045, 58.86491], + "delayed": [72.02265, 57.86302, 66.88550, 91.31943, 58.89590], + "aggregate-visible": [79.54571, 69.79448, 72.65278, 96.18987, 61.68383], + "per-person": [78.80474, 69.74061, 71.00535, 95.66825, 61.10890], + "shuffled-id": [78.86300, 68.70055, 71.85510, 96.03333, 61.18615] + }, + "metric_order": [ + "map_0p2", + "bg_ap_0p2", + "wearer_ap_0p2", + "others_ap_0p2", + "mean_map" + ], + "descriptive_deltas": { + "aggregate-visible_minus_audio": [6.96649, 9.94893, 5.82814, 5.12240, 2.91047], + "aggregate-visible_minus_delayed": [7.52306, 11.93146, 5.76728, 4.87044, 2.78793], + "aggregate-visible_minus_per-person": [0.74097, 0.05387, 1.64743, 0.52162, 0.57493], + "per-person_minus_shuffled-id": [-0.05826, 1.04006, -0.84975, -0.36508, -0.07725] + }, + "interpretation": { + "supported": "timely visible-other aggregate activity has information value beyond global Omni plus recent mixed audio", + "not_supported": "person identity or track continuity provides additional value", + "restriction": "oracle inputs are analysis-only ground-truth current/past speech activity" + } +}