From 4011ba9b2ab4f408e8c8ec54be79a3fec473bea0 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Fri, 28 Aug 2026 11:15:09 +0800 Subject: [PATCH] Record EasyCom encoder-only results and horizon analysis --- README.md | 6 + docs/easycom_encoder_only.md | 8 + .../README.md | 242 +++ .../raw_encoder_audio_seed0.txt | 40 + .../raw_encoder_av_seed0.txt | 39 + .../raw_encoder_visual_seed0.txt | 40 + .../raw_previous_thinker_audio_seed0.txt | 36 + .../raw_previous_thinker_av_seed0.txt | 37 + .../reported_causal_summary.txt | 15 + .../summary.json | 1318 +++++++++++++++++ 10 files changed, 1781 insertions(+) create mode 100644 results/easycom_encoder_only_1s_4ep_2026-08-28/README.md create mode 100644 results/easycom_encoder_only_1s_4ep_2026-08-28/raw_encoder_audio_seed0.txt create mode 100644 results/easycom_encoder_only_1s_4ep_2026-08-28/raw_encoder_av_seed0.txt create mode 100644 results/easycom_encoder_only_1s_4ep_2026-08-28/raw_encoder_visual_seed0.txt create mode 100644 results/easycom_encoder_only_1s_4ep_2026-08-28/raw_previous_thinker_audio_seed0.txt create mode 100644 results/easycom_encoder_only_1s_4ep_2026-08-28/raw_previous_thinker_av_seed0.txt create mode 100644 results/easycom_encoder_only_1s_4ep_2026-08-28/reported_causal_summary.txt create mode 100644 results/easycom_encoder_only_1s_4ep_2026-08-28/summary.json diff --git a/README.md b/README.md index 85d59b8..cb12c46 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,12 @@ For **EasyCom encoder-only A / V / AV** with no Thinker states, start with They reuse the verified local sidecars, train on Session 4–12, test on 3/2/1, and keep four epochs with no dev set. +The user-reported EasyCom encoder-only seed-0 results and complete supplied +horizon curves are archived in +[the 2026-08-28 result record](results/easycom_encoder_only_1s_4ep_2026-08-28/README.md). +It keeps causal-valid summaries separate from full-timeline curves and records +the head-capacity/context caveats for comparisons with G+A and G+A+V. + The main experiment is deliberately simple: ```text diff --git a/docs/easycom_encoder_only.md b/docs/easycom_encoder_only.md index 8f79ac4..7c90d2e 100644 --- a/docs/easycom_encoder_only.md +++ b/docs/easycom_encoder_only.md @@ -7,8 +7,16 @@ 固定训练 Session **4–12(升序,与原 B1 一致)**,测试 **3/2/1**;4 epochs、无 dev, 评测最后一轮。沿用原来的 5Hz 行索引、十个 horizon、LSTR-MCE 和 AP 协议,不改变标签。 +已收到的 seed 0 结果与完整日志见 +[2026-08-28 实验记录](../results/easycom_encoder_only_1s_4ep_2026-08-28/README.md)。 +其中完整曲线为 `egospeak_full_timeline`,此前主表为 `causal_valid_only`,两者分开记录。 + ## 1. 确定已经验证过的局部特征目录 +用户已确认本次使用的实际目录是 +`/data1/yfzx/wangx/Datasets/easycom/omni_features/local_qwen_1s_encoded_tf4576`, +不是早期默认的 `local_qwen_1s`;以下 checkpoint 读取方法可用于再次核对。 + 在 L20 机器更新本次代码后执行。下面从你之前跑完的 `av_seed0.pth` 读取实际路径, 避免猜错目录或混入 bbox 修正前的旧视觉特征;checkpoint 仅用于找路径,不作为新模型输入。 diff --git a/results/easycom_encoder_only_1s_4ep_2026-08-28/README.md b/results/easycom_encoder_only_1s_4ep_2026-08-28/README.md new file mode 100644 index 0000000..b5358c3 --- /dev/null +++ b/results/easycom_encoder_only_1s_4ep_2026-08-28/README.md @@ -0,0 +1,242 @@ +# EasyCom encoder-only 1s:seed 0 完整曲线记录与分析 + +记录日期:2026-08-28。来源是用户在 L20 远程机器运行后提供的控制台输出; +本地没有重新训练或执行真实数据推理。本记录仍是**单种子初步结果**,标准差为 n/a, +不是 0;不能据此报告统计显著性。 + +## 1. 收到了什么,尚未收到什么 + +- 本次三份附件:encoder A、V、AV 的 `egospeak_full_timeline` 十个 horizon 的三类 AP、mAP 和均值。 +- 此前两份附件:G+A、G+A+V 同协议的完整曲线;在本目录一并归档用于同协议比较。 +- 此前用户汇总表:七组 `causal_valid_only` 的标称 0.2s 三类 AP/mAP 和十-horizon 平均。 +- 本地没有收到原始 `metrics.json`、预测数组、checkpoint 内容或训练摘要 JSON。 + 附件中的 `training summary: ...pth.json` 只是路径,不是文件内容;没有训练曲线。 +- 本次附件在 full-timeline 均值之后结束,**没有包含 causal-valid 的完整曲线**。 + 不将前者改名为后者,也不根据均值补造未提供的 horizon。 +- 以下四轮训练、训练 split、窗口和 head 设置来自已约定命令与代码; + 尚未从远程 checkpoint 元数据独立核验 epoch、seed、参数量或运行 commit。 + +G = 全局 Thinker 特征;A = 近期混合音频编码器向量;V = 逐人局部视觉编码器向量的池化。 +旧 B1 中名为 `audio / visual / av` 的运行实际输入为 G+A / G+V / G+A+V, +不是本次 encoder-only 的 A / V / AV。 + +## 2. 实验配置与来源路径 + +约定配置: + +- EasyCom:训练 Session 4–12 升序、无 dev、测试 Session 3/2/1。 +- 冻结 Qwen2.5-Omni-3B 编码器;本次实验名为 seed 0、4 epochs、最后一轮评测。 +- A/V 原生各 2048D;A 是最近 1s 混合音频的有效编码器 token pooling。 +- V 使用提供的人物框;2FPS 空间 token mean → 同帧逐人等权 mean → 最近 1s 有效帧 mean。 + 这不是全帧视觉,不是逐人 attention,也没有评测在线人脸检测器。 +- 每 200ms 一行,输入窗口按 `[max(0,t-1),t)` 约定,标签与原实验相同。 +- encoder-only 不输入 Thinker 状态;将选定原生向量和 validity 位拼接后输入 MLP。 +- 对照 G:8s open-stream-replay、2FPS、Thinker last4 stack + concat-MLP。 +- LSTR-MCE(三路 softmax),batch 512、lr 7e-5、weight decay 5e-5、 + warmup 1.6 epochs、warmup factor 0.3、dropout 0.2、无 transition objective。 +- 局部提取版本按先前实验报告为 Transformers 4.57.6;新附件不是编码器提取日志, + 没有再次核验所有 sidecar 的实际 config。 + +用户已通过目录列表确认局部源特征在: + +```text +/data1/yfzx/wangx/Datasets/easycom/omni_features/local_qwen_1s_encoded_tf4576 +``` + +新导出目录为同一 `omni_features` 下的 +`encoder_only_audio_1s`、`encoder_only_visual_1s`、`encoder_only_av_1s`。 +这些是已有向量的重新打包,不是重新运行 Qwen。 + +其他路径: + +- 项目:`/data1/yfzx/wangx/Code/icassp2027-experiment` +- target:`/data1/yfzx/wangx/Datasets/easycom/easycom_target_frame` +- 新 checkpoint:`checkpoints/easycom_encoder_only_1s/encoder__seed0.pth` +- 新评测:`results/easycom_encoder_only_1s/encoder__seed0/metrics.json` +- 旧 B1 checkpoint:`checkpoints/easycom_local_b1_encoded_tf4576/{audio,av}_seed0.pth` +- 实验入口与命令:[EasyCom encoder-only 指南](../../docs/easycom_encoder_only.md); + 本地入口版本为 `321238a`,不是对远程运行版本的证明。 + +## 3. 两种评测协议必须分开 + +三个新运行及两个旧对照都打印了相同预测形状: +Session_3 = (8400,3,10)、Session_2 = (7706,3,10)、Session_1 = (6942,3,10), +总计 23048 行。 + +`egospeak_full_timeline` 每个 horizon 都评测 23048 行, +包含每个 session 开头没有历史预测的 h 行,数组中这些分数初始化为零。 +`causal_valid_only` 排除这部分行,因此标称 0.2s 为 23045 行, +依次减少,到 2.0s 为 23018 行。这里 h 指 1–10 的 horizon 索引。 + +新日志和此前汇总值的小差异与这两种协议一致,不应当成新的训练结果或抄录误差: + +| 输入 | causal − full:0.2s mAP | causal − full:十-horizon 均值 | +|---|---:|---:| +| V | +0.00108 | +0.00646 | +| A+V | +0.00236 | +0.01176 | +| A | +0.00240 | +0.01154 | + +数值是 AP/mAP 百分数;差值单位是百分点。代码依据: +[`infer_turn_head.py`](../../infer_turn_head.py) 的 `_align_to_egospeak_timeline` 与 `_evaluate`。 +`causal_valid_only` 是有效预测行的评测掩码,**不是原始输入没有泄漏的独立证据**。 +本次没有重新审计标签时间戳、原视频时钟或在线延迟;表中 0.2–2.0s 一律指协议标称 horizon, +不直接当作已证实的物理提前量。 + +## 4. 主协议汇总:causal_valid_only,seed 0 + +来自此前用户直接提供的汇总,不是由 full-timeline 反推。仅有首个 horizon 和均值。 + +| 输入 | 标称 0.2s mAP | BG AP | wearer AP | others AP | 十-horizon 平均 mAP | +|---|---:|---:|---:|---:|---:| +| G | 73.44637 | 56.76438 | 77.86650 | 85.70824 | 60.84861 | +| A | 79.69221 | 63.69477 | 84.09982 | 91.28205 | 60.50771 | +| V | 48.38434 | 30.16284 | 41.68834 | 73.30186 | 46.25250 | +| A+V | 79.00727 | 63.67397 | 82.69832 | 90.64952 | 60.76303 | +| G+A | 81.72257 | 68.15248 | 85.06554 | 91.94969 | 65.10895 | +| G+V | 73.47412 | 56.67089 | 78.60954 | 85.14194 | 60.99238 | +| G+A+V | 82.02661 | 68.93423 | 85.08583 | 92.05978 | 65.39978 | + +来源转录:[reported_causal_summary.txt](reported_causal_summary.txt)。 +不要将这里的 G 行与下一节 full-timeline 的逐 horizon 行混算差值。 + +## 5. 完整 mAP 曲线:egospeak_full_timeline,seed 0 + +A/V/AV 为本次三份附件;G+A/G+A+V 为此前两份附件。所有列是同一协议。 + +| 标称 horizon / s | A | V | A+V | G+A | G+A+V | +|---|---:|---:|---:|---:|---:| +| 0.2 | 79.68981 | 48.38326 | 79.00491 | 81.72011 | 82.02415 | +| 0.4 | 72.41576 | 47.68478 | 72.06793 | 75.91398 | 75.99169 | +| 0.6 | 66.33666 | 47.26191 | 66.16179 | 70.37189 | 70.92665 | +| 0.8 | 61.60584 | 46.62600 | 61.62137 | 66.45047 | 66.79615 | +| 1.0 | 58.28586 | 46.00582 | 58.49237 | 63.79981 | 63.86856 | +| 1.2 | 55.93708 | 45.75002 | 56.07422 | 61.35120 | 61.56280 | +| 1.4 | 54.22585 | 45.41812 | 54.61933 | 59.98279 | 60.34070 | +| 1.6 | 52.83014 | 45.56259 | 53.78180 | 58.29734 | 58.83407 | +| 1.8 | 51.97218 | 44.79454 | 52.91556 | 56.93221 | 57.39446 | +| 2.0 | 51.66256 | 44.97339 | 52.77341 | 56.13656 | 56.12449 | +| 十个 horizon 平均 | 60.49617 | 46.24604 | 60.75127 | 65.09564 | 65.38637 | + +所有 50 个 horizon 的三类 AP、样本数和 mAP 都保存在 +[summary.json](summary.json) 的 `egospeak_full_timeline.runs`; +不只保存表格中的 mAP。 + +## 6. 视觉不是“全程无效”:需要分 horizon、分类别看 + +下表是 **A+V − A**,全部使用 full-timeline,单位为百分点: + +| 标称 horizon / s | ΔmAP | ΔBG AP | Δwearer AP | Δothers AP | +|---|---:|---:|---:|---:| +| 0.2 | -0.68490 | -0.02080 | -1.40150 | -0.63239 | +| 0.4 | -0.34783 | +0.70624 | -1.59157 | -0.15812 | +| 0.6 | -0.17487 | +0.82855 | -1.58313 | +0.22998 | +| 0.8 | +0.01553 | +1.44947 | -1.66891 | +0.26602 | +| 1.0 | +0.20651 | +1.56046 | -1.51747 | +0.57655 | +| 1.2 | +0.13714 | +1.00313 | -1.31375 | +0.72202 | +| 1.4 | +0.39348 | +1.42271 | -1.10816 | +0.86587 | +| 1.6 | +0.95166 | +1.90632 | -0.41886 | +1.36752 | +| 1.8 | +0.94338 | +1.81173 | -0.05834 | +1.07678 | +| 2.0 | +1.11085 | +1.66905 | +0.66242 | +1.00108 | + +观察: + +- mAP 在 0.2/0.4/0.6s 下降;0.8s 的 +0.01553 只是很小的数值差。 + 1.6/1.8/2.0s 分别 +0.95166/+0.94338/+1.11085,十-horizon 均值为 +0.25510。 +- wearer AP 在前 9 个 horizon 均下降,直到 2.0s 才提高。 + 十-horizon 的 BG、wearer、others AP 平均变化分别为 + **+1.23369 / −0.99993 / +0.53153**。 +- 因此“AV 的平均 mAP 略好”掩盖了 wearer 分支的退步;也不能再笼统说视觉在所有 horizon 都有害。 +- 这些 horizon 来自同一 seed、同一批序列,彼此相关;7/10 个正差不能当成七次独立复现实验。 + 正负号是描述,不是显著性检验。 + +V-only 的 mAP 从 48.38326 到 44.97339,整体较低;但 AP 不是分类准确率, +未给出测试标签 prevalence 或随机排序基准,不能仅凭 48% 宣称“随机水平”。 +可见人物框不直接提供 wearer 的脸,且当前逐人/时间平均不保留显式交互关系; +这可作为解释假设,尚不能证明退步确由视觉信息不足或池化导致。 + +## 7. 加入 G 的组合增益:在完整曲线上不只是均值现象 + +以下仍是 full-timeline。各类别列为十个 horizon 的 AP 差值平均: + +| 对照 | Δ十-horizon mAP | ΔBG AP 均值 | Δwearer AP 均值 | Δothers AP 均值 | mAP 正差 horizon | +|---|---:|---:|---:|---:|---:| +| G+A − A | +4.59947 | +8.71604 | +2.42825 | +2.65410 | 10/10 | +| G+A+V − A+V | +4.63510 | +8.16923 | +3.48897 | +2.24711 | 10/10 | +| G+A+V − G+A | +0.29073 | +0.68687 | +0.06078 | +0.12454 | 9/10 | + +- **G+A 相比 A 在全部 10 个 horizon、每个类别上都更高。** + mAP 的增益从 0.2s 的 +2.03030 增至 1.4s 的 +5.75694, + 在 2.0s 仍为 +4.47400。完整曲线支持组合方案具有互补价值, + 不是仅由某个 horizon 抬高均值。 +- **G+A+V 相比 A+V 同样在所有 horizon、所有类别上更高。** + 十-horizon mAP 为 +4.63510。 +- **G+A+V 相比 G+A 的收益仍小且偏向 BG。** + 十-horizon mAP +0.29073;wearer AP 均值仅 +0.06078; + 0.2s wearer AP 仅 +0.02029,2.0s 的 mAP 反而 −0.01207。 + 因此不能把整体增益描述为已解决 wearer 发言预测或多说话人视觉理解。 +- 主协议对应的均值差是 G+A − A = +4.60124、 + G+A+V − A+V = +4.63675、G+A+V − G+A = +0.29083。 + 这些数值与 full-timeline 接近,但仍分别标注,不互相替代。 + +## 8. 归因限制:这还不是等容量、等上下文的 decoder 消融 + +按当前 `TurnPredictionHead` 和约定维度计算的可训练参数量: + +| 输入 | 参数量 | +|---|---:| +| A 或 V | 2,134,048 | +| A+V | 8,468,547 | +| G | 18,928,670 | +| G+A 或 G+V | 19,720,992 | +| G+A+V | 20,513,314 | + +来源是本地代码实例化计算,不是远程 checkpoint 读取。A/V 单模态原生向量+有效性位 +为 2049D;AV 为 4098D,MLP 隐藏宽度分别 1024 和 2049。 +G 使用 last4 concat-MLP;G+A/G+V/G+AV 还使用各路 256D 投影。 +因此 A→AV 同时改变参数量和优化问题;G+A→A 还改变了 history、视野和融合结构。 + +这组结果不能单独证明: + +- Thinker 的语言推理或长时记忆是增益的唯一原因; +- 模型已学到多人互动、speaker identity 或 turn transition; +- AV 退步一定是过拟合,或一定由跨人/跨时间 pooling 引起; +- 同为四轮就代表不同模型同样充分收敛; +- encoder-only 或某一模型端到端更快。附件的 batch/s 不包含特征提取, + 也没有同步计时、预热和统一基准,不能当作系统速度对照。 + +## 9. 下一步实验优先级与论文表述 + +1. **先补证据**:收集三组原始 `metrics.json`、checkpoint epoch/head 配置和训练摘要, + 补全 causal-valid 曲线;在固定四轮/无 dev 的既定协议下补 seed 1/2。 + 不根据这份测试曲线重新选 epoch、阈值或只挑有利 horizon。 +2. **先做受控融合对照**:固定分支投影与后端 MLP 架构,以明确的模态屏蔽比较 A/AV, + 并报告总参数和实际参与学习的参数;用预先定义的相同原始可见时间范围控制 history。 + 再比较 G 的增量,避免把更大 head 或更长历史完全归给 decoder。 +3. **再测试多人视觉假设**:优先复用已经保存的逐人向量,比较平均池化与音频条件的逐人加权, + 保留音频分支作为对照。事先定义多可见人物、重叠说话等分析子集及样本量, + 同时报告 BG/wearer/others,而不只看总体 mAP。提供框的结果与在线检测器场景分开说明。 + +当前可以写成: + +> 在本次单种子设置中,近期音频编码器提供强近时预测信号,加入全局 Thinker 表示的组合 +> 在所有评测 horizon 上优于对应的 encoder-only 方案。局部视觉的影响依赖类别和 horizon: +> 部分较后 horizon 的宏平均指标改善,但 wearer AP 多数下降。等容量、等上下文控制和 +> 多种子复现仍是隔离各组件贡献的必要步骤。 + +这支持继续研究“近期感知与上下文表示如何互补、多人视觉如何有效融入”, +但尚不足以把“多人视觉调优成功”写成已经证实的贡献。 + +## 10. 归档文件与校验范围 + +- [summary.json](summary.json):两种协议分区、完整 AP 曲线、计算差值、配置来源与证据边界。 +- [reported_causal_summary.txt](reported_causal_summary.txt):此前主协议汇总的转录。 +- [raw_encoder_audio_seed0.txt](raw_encoder_audio_seed0.txt)、 + [raw_encoder_visual_seed0.txt](raw_encoder_visual_seed0.txt)、 + [raw_encoder_av_seed0.txt](raw_encoder_av_seed0.txt):本次三份控制台附件。 +- [raw_previous_thinker_audio_seed0.txt](raw_previous_thinker_audio_seed0.txt)、 + [raw_previous_thinker_av_seed0.txt](raw_previous_thinker_av_seed0.txt):此前两个对照附件。 + +控制台归档仅移除了行尾水平空白、统一最后的换行;保留换行折断的路径和全部数值。 +本地核对了 50 个 horizon 的 AP→mAP、十-horizon 均值、样本数、主协议转录与差值; +四舍五入允许 1e-5 的百分数误差。没有从原始预测重新计算 AP, +这些文件不是对远程 `metrics.json` 或真实训练产物的替代。 diff --git a/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_encoder_audio_seed0.txt b/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_encoder_audio_seed0.txt new file mode 100644 index 0000000..c490b24 --- /dev/null +++ b/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_encoder_audio_seed0.txt @@ -0,0 +1,40 @@ +checkpoint: /data1/yfzx/wangx/Code/icassp2027-experiment/checkpoints/easycom_encoder_only_1s/encoder_audio_seed0.pth +training summary: /data1/yfzx/wangx/Code/icassp2027-experiment/checkpoints/easycom_encoder_only_1s/encoder_audio_seed0.pth.json +/usr/local/bin/python /data1/yfzx/wangx/Code/icassp2027-experiment/infer_turn_head.py --dataset easycom --feature-root /data1/yfzx/wangx/Datasets/easycom/omni_features/encoder_only_audio_1s --target-root /data1/yfzx/wangx/Datasets/easyco +m/easycom_target_frame --expected-mode encoder-only-audio --device cuda:0 --checkpoint /data1/yfzx/wangx/Code/icassp2027-experiment/checkpoints/easycom_encoder_only_1s/encoder_audio_seed0.pth --output-dir /data1/yfzx/wangx/Code/icassp202 +7-experiment/results/easycom_encoder_only_1s/encoder_audio_seed0 --sessions Session_3 Session_2 Session_1 +/usr/local/lib/python3.11/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the + package that installed pynvml for you. + import pynvml # type: ignore[import] +device: cuda:0 +loss: lstr-mce; output_space: three-class-softmax +sessions: Session_3, Session_2, Session_1 +infer Session_3: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00, 21.47batch/s] +Session_3: saved pred (8400, 3, 10), causal-valid counts=[8399, 8398, 8397, 8396, 8395, 8394, 8393, 8392, 8391, 8390] +infer Session_2: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:00<00:00, 1441.65batch/s] +Session_2: saved pred (7706, 3, 10), causal-valid counts=[7705, 7704, 7703, 7702, 7701, 7700, 7699, 7698, 7697, 7696] +infer Session_1: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 1407.96batch/s] +Session_1: saved pred (6942, 3, 10), causal-valid counts=[6941, 6940, 6939, 6938, 6937, 6936, 6935, 6934, 6933, 6932] + +egospeak_full_timeline: +Action anticipation (0.20s) perframe mAP: 79.68981 + AP: background=63.69477, camera_wearer_speaking=84.09982, other_people_speaking=91.27483; n=23048 +Action anticipation (0.40s) perframe mAP: 72.41576 + AP: background=52.47780, camera_wearer_speaking=77.43606, other_people_speaking=87.33340; n=23048 +Action anticipation (0.60s) perframe mAP: 66.33666 + AP: background=44.20066, camera_wearer_speaking=71.41221, other_people_speaking=83.39710; n=23048 +Action anticipation (0.80s) perframe mAP: 61.60584 + AP: background=38.60179, camera_wearer_speaking=66.08811, other_people_speaking=80.12762; n=23048 +Action anticipation (1.00s) perframe mAP: 58.28586 + AP: background=34.75843, camera_wearer_speaking=62.53734, other_people_speaking=77.56181; n=23048 +Action anticipation (1.20s) perframe mAP: 55.93708 + AP: background=32.93325, camera_wearer_speaking=59.29096, other_people_speaking=75.58705; n=23048 +Action anticipation (1.40s) perframe mAP: 54.22585 + AP: background=31.56977, camera_wearer_speaking=56.79707, other_people_speaking=74.31072; n=23048 +Action anticipation (1.60s) perframe mAP: 52.83014 + AP: background=30.67884, camera_wearer_speaking=54.61175, other_people_speaking=73.19983; n=23048 +Action anticipation (1.80s) perframe mAP: 51.97218 + AP: background=30.37030, camera_wearer_speaking=52.80894, other_people_speaking=72.73728; n=23048 +Action anticipation (2.00s) perframe mAP: 51.66256 + AP: background=30.86998, camera_wearer_speaking=51.49671, other_people_speaking=72.62098; n=23048 +Action anticipation (mean) perframe mAP: 60.49617 diff --git a/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_encoder_av_seed0.txt b/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_encoder_av_seed0.txt new file mode 100644 index 0000000..cb129ec --- /dev/null +++ b/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_encoder_av_seed0.txt @@ -0,0 +1,39 @@ +training summary: /data1/yfzx/wangx/Code/icassp2027-experiment/checkpoints/easycom_encoder_only_1s/encoder_av_seed0.pth.json +/usr/local/bin/python /data1/yfzx/wangx/Code/icassp2027-experiment/infer_turn_head.py --dataset easycom --feature-root /data1/yfzx/wangx/Datasets/easycom/omni_features/encoder_only_av_1s --target-root /data1/yfzx/wangx/Datasets/easycom/e +asycom_target_frame --expected-mode encoder-only-av --device cuda:0 --checkpoint /data1/yfzx/wangx/Code/icassp2027-experiment/checkpoints/easycom_encoder_only_1s/encoder_av_seed0.pth --output-dir /data1/yfzx/wangx/Code/icassp2027-experim +ent/results/easycom_encoder_only_1s/encoder_av_seed0 --sessions Session_3 Session_2 Session_1 +/usr/local/lib/python3.11/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the + package that installed pynvml for you. + import pynvml # type: ignore[import] +device: cuda:0 +loss: lstr-mce; output_space: three-class-softmax +sessions: Session_3, Session_2, Session_1 +infer Session_3: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00, 69.00batch/s] +Session_3: saved pred (8400, 3, 10), causal-valid counts=[8399, 8398, 8397, 8396, 8395, 8394, 8393, 8392, 8391, 8390] +infer Session_2: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:00<00:00, 683.75batch/s] +Session_2: saved pred (7706, 3, 10), causal-valid counts=[7705, 7704, 7703, 7702, 7701, 7700, 7699, 7698, 7697, 7696] +infer Session_1: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 634.13batch/s] +Session_1: saved pred (6942, 3, 10), causal-valid counts=[6941, 6940, 6939, 6938, 6937, 6936, 6935, 6934, 6933, 6932] + +egospeak_full_timeline: +Action anticipation (0.20s) perframe mAP: 79.00491 + AP: background=63.67397, camera_wearer_speaking=82.69832, other_people_speaking=90.64244; n=23048 +Action anticipation (0.40s) perframe mAP: 72.06793 + AP: background=53.18404, camera_wearer_speaking=75.84449, other_people_speaking=87.17528; n=23048 +Action anticipation (0.60s) perframe mAP: 66.16179 + AP: background=45.02921, camera_wearer_speaking=69.82908, other_people_speaking=83.62708; n=23048 +Action anticipation (0.80s) perframe mAP: 61.62137 + AP: background=40.05126, camera_wearer_speaking=64.41920, other_people_speaking=80.39364; n=23048 +Action anticipation (1.00s) perframe mAP: 58.49237 + AP: background=36.31889, camera_wearer_speaking=61.01987, other_people_speaking=78.13836; n=23048 +Action anticipation (1.20s) perframe mAP: 56.07422 + AP: background=33.93638, camera_wearer_speaking=57.97721, other_people_speaking=76.30907; n=23048 +Action anticipation (1.40s) perframe mAP: 54.61933 + AP: background=32.99248, camera_wearer_speaking=55.68891, other_people_speaking=75.17659; n=23048 +Action anticipation (1.60s) perframe mAP: 53.78180 + AP: background=32.58516, camera_wearer_speaking=54.19289, other_people_speaking=74.56735; n=23048 +Action anticipation (1.80s) perframe mAP: 52.91556 + AP: background=32.18203, camera_wearer_speaking=52.75060, other_people_speaking=73.81406; n=23048 +Action anticipation (2.00s) perframe mAP: 52.77341 + AP: background=32.53903, camera_wearer_speaking=52.15913, other_people_speaking=73.62206; n=23048 +Action anticipation (mean) perframe mAP: 60.75127 diff --git a/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_encoder_visual_seed0.txt b/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_encoder_visual_seed0.txt new file mode 100644 index 0000000..1d6a4cf --- /dev/null +++ b/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_encoder_visual_seed0.txt @@ -0,0 +1,40 @@ +checkpoint: /data1/yfzx/wangx/Code/icassp2027-experiment/checkpoints/easycom_encoder_only_1s/encoder_visual_seed0.pth +training summary: /data1/yfzx/wangx/Code/icassp2027-experiment/checkpoints/easycom_encoder_only_1s/encoder_visual_seed0.pth.json +/usr/local/bin/python /data1/yfzx/wangx/Code/icassp2027-experiment/infer_turn_head.py --dataset easycom --feature-root /data1/yfzx/wangx/Datasets/easycom/omni_features/encoder_only_visual_1s --target-root /data1/yfzx/wangx/Datasets/easyc +om/easycom_target_frame --expected-mode encoder-only-visual --device cuda:0 --checkpoint /data1/yfzx/wangx/Code/icassp2027-experiment/checkpoints/easycom_encoder_only_1s/encoder_visual_seed0.pth --output-dir /data1/yfzx/wangx/Code/icassp +2027-experiment/results/easycom_encoder_only_1s/encoder_visual_seed0 --sessions Session_3 Session_2 Session_1 +/usr/local/lib/python3.11/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the + package that installed pynvml for you. + import pynvml # type: ignore[import] +device: cuda:0 +loss: lstr-mce; output_space: three-class-softmax +sessions: Session_3, Session_2, Session_1 +infer Session_3: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00, 71.22batch/s] +Session_3: saved pred (8400, 3, 10), causal-valid counts=[8399, 8398, 8397, 8396, 8395, 8394, 8393, 8392, 8391, 8390] +infer Session_2: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:00<00:00, 1487.47batch/s] +Session_2: saved pred (7706, 3, 10), causal-valid counts=[7705, 7704, 7703, 7702, 7701, 7700, 7699, 7698, 7697, 7696] +infer Session_1: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 1409.24batch/s] +Session_1: saved pred (6942, 3, 10), causal-valid counts=[6941, 6940, 6939, 6938, 6937, 6936, 6935, 6934, 6933, 6932] + +egospeak_full_timeline: +Action anticipation (0.20s) perframe mAP: 48.38326 + AP: background=30.16284, camera_wearer_speaking=41.68834, other_people_speaking=73.29862; n=23048 +Action anticipation (0.40s) perframe mAP: 47.68478 + AP: background=29.82215, camera_wearer_speaking=40.98323, other_people_speaking=72.24896; n=23048 +Action anticipation (0.60s) perframe mAP: 47.26191 + AP: background=29.28129, camera_wearer_speaking=40.74974, other_people_speaking=71.75469; n=23048 +Action anticipation (0.80s) perframe mAP: 46.62600 + AP: background=28.24103, camera_wearer_speaking=40.47370, other_people_speaking=71.16328; n=23048 +Action anticipation (1.00s) perframe mAP: 46.00582 + AP: background=27.73638, camera_wearer_speaking=39.50369, other_people_speaking=70.77740; n=23048 +Action anticipation (1.20s) perframe mAP: 45.75002 + AP: background=27.39577, camera_wearer_speaking=39.53354, other_people_speaking=70.32075; n=23048 +Action anticipation (1.40s) perframe mAP: 45.41812 + AP: background=27.04920, camera_wearer_speaking=39.35560, other_people_speaking=69.84957; n=23048 +Action anticipation (1.60s) perframe mAP: 45.56259 + AP: background=27.36515, camera_wearer_speaking=39.16056, other_people_speaking=70.16205; n=23048 +Action anticipation (1.80s) perframe mAP: 44.79454 + AP: background=26.64061, camera_wearer_speaking=38.50155, other_people_speaking=69.24145; n=23048 +Action anticipation (2.00s) perframe mAP: 44.97339 + AP: background=27.34810, camera_wearer_speaking=37.96646, other_people_speaking=69.60560; n=23048 +Action anticipation (mean) perframe mAP: 46.24604 diff --git a/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_previous_thinker_audio_seed0.txt b/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_previous_thinker_audio_seed0.txt new file mode 100644 index 0000000..7b11610 --- /dev/null +++ b/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_previous_thinker_audio_seed0.txt @@ -0,0 +1,36 @@ +training summary: checkpoints/easycom_local_b1_encoded_tf4576/audio_seed0.pth.json +/usr/local/lib/python3.11/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml d +irectly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +device: cuda:0 +loss: lstr-mce; output_space: three-class-softmax +sessions: Session_3, Session_2, Session_1 +infer Session_3: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00, 39.03batch/s] +Session_3: saved pred (8400, 3, 10), causal-valid counts=[8399, 8398, 8397, 8396, 8395, 8394, 8393, 8392, 8391, 8390] +infer Session_2: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:00<00:00, 142.52batch/s] +Session_2: saved pred (7706, 3, 10), causal-valid counts=[7705, 7704, 7703, 7702, 7701, 7700, 7699, 7698, 7697, 7696] +infer Session_1: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 144.02batch/s] +Session_1: saved pred (6942, 3, 10), causal-valid counts=[6941, 6940, 6939, 6938, 6937, 6936, 6935, 6934, 6933, 6932] + +egospeak_full_timeline: +Action anticipation (0.20s) perframe mAP: 81.72011 + AP: background=68.15248, camera_wearer_speaking=85.06554, other_people_speaking=91.94232; n=23048 +Action anticipation (0.40s) perframe mAP: 75.91398 + AP: background=59.90715, camera_wearer_speaking=79.22465, other_people_speaking=88.61015; n=23048 +Action anticipation (0.60s) perframe mAP: 70.37189 + AP: background=52.48480, camera_wearer_speaking=73.21461, other_people_speaking=85.41626; n=23048 +Action anticipation (0.80s) perframe mAP: 66.45047 + AP: background=47.92066, camera_wearer_speaking=68.77900, other_people_speaking=82.65175; n=23048 +Action anticipation (1.00s) perframe mAP: 63.79981 + AP: background=45.59262, camera_wearer_speaking=65.09890, other_people_speaking=80.70791; n=23048 +Action anticipation (1.20s) perframe mAP: 61.35120 + AP: background=43.15384, camera_wearer_speaking=61.86698, other_people_speaking=79.03278; n=23048 +Action anticipation (1.40s) perframe mAP: 59.98279 + AP: background=41.83907, camera_wearer_speaking=60.22346, other_people_speaking=77.88584; n=23048 +Action anticipation (1.60s) perframe mAP: 58.29734 + AP: background=40.44538, camera_wearer_speaking=57.66189, other_people_speaking=76.78474; n=23048 +Action anticipation (1.80s) perframe mAP: 56.93221 + AP: background=39.04677, camera_wearer_speaking=55.70067, other_people_speaking=76.04919; n=23048 +Action anticipation (2.00s) perframe mAP: 56.13656 + AP: background=38.77318, camera_wearer_speaking=54.02582, other_people_speaking=75.61069; n=23048 +Action anticipation (mean) perframe mAP: 65.09564 diff --git a/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_previous_thinker_av_seed0.txt b/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_previous_thinker_av_seed0.txt new file mode 100644 index 0000000..ff9eaa8 --- /dev/null +++ b/results/easycom_encoder_only_1s_4ep_2026-08-28/raw_previous_thinker_av_seed0.txt @@ -0,0 +1,37 @@ +checkpoint: checkpoints/easycom_local_b1_encoded_tf4576/av_seed0.pth +training summary: checkpoints/easycom_local_b1_encoded_tf4576/av_seed0.pth.json +/usr/local/lib/python3.11/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml d +irectly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +device: cuda:0 +loss: lstr-mce; output_space: three-class-softmax +sessions: Session_3, Session_2, Session_1 +infer Session_3: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [00:00<00:00, 40.61batch/s] +Session_3: saved pred (8400, 3, 10), causal-valid counts=[8399, 8398, 8397, 8396, 8395, 8394, 8393, 8392, 8391, 8390] +infer Session_2: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:00<00:00, 143.92batch/s] +Session_2: saved pred (7706, 3, 10), causal-valid counts=[7705, 7704, 7703, 7702, 7701, 7700, 7699, 7698, 7697, 7696] +infer Session_1: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 160.27batch/s] +Session_1: saved pred (6942, 3, 10), causal-valid counts=[6941, 6940, 6939, 6938, 6937, 6936, 6935, 6934, 6933, 6932] + +egospeak_full_timeline: +Action anticipation (0.20s) perframe mAP: 82.02415 + AP: background=68.93423, camera_wearer_speaking=85.08583, other_people_speaking=92.05239; n=23048 +Action anticipation (0.40s) perframe mAP: 75.99169 + AP: background=60.58903, camera_wearer_speaking=78.83427, other_people_speaking=88.55177; n=23048 +Action anticipation (0.60s) perframe mAP: 70.92665 + AP: background=53.84208, camera_wearer_speaking=73.45503, other_people_speaking=85.48283; n=23048 +Action anticipation (0.80s) perframe mAP: 66.79615 + AP: background=49.21382, camera_wearer_speaking=68.56072, other_people_speaking=82.61390; n=23048 +Action anticipation (1.00s) perframe mAP: 63.86856 + AP: background=45.62073, camera_wearer_speaking=65.27034, other_people_speaking=80.71461; n=23048 +Action anticipation (1.20s) perframe mAP: 61.56280 + AP: background=43.46670, camera_wearer_speaking=62.05277, other_people_speaking=79.16893; n=23048 +Action anticipation (1.40s) perframe mAP: 60.34070 + AP: background=42.54682, camera_wearer_speaking=60.20540, other_people_speaking=78.26988; n=23048 +Action anticipation (1.60s) perframe mAP: 58.83407 + AP: background=41.15952, camera_wearer_speaking=58.18522, other_people_speaking=77.15747; n=23048 +Action anticipation (1.80s) perframe mAP: 57.39446 + AP: background=39.94176, camera_wearer_speaking=55.99271, other_people_speaking=76.24890; n=23048 +Action anticipation (2.00s) perframe mAP: 56.12449 + AP: background=38.87001, camera_wearer_speaking=53.82707, other_people_speaking=75.67639; n=23048 +Action anticipation (mean) perframe mAP: 65.38637 diff --git a/results/easycom_encoder_only_1s_4ep_2026-08-28/reported_causal_summary.txt b/results/easycom_encoder_only_1s_4ep_2026-08-28/reported_causal_summary.txt new file mode 100644 index 0000000..5a28384 --- /dev/null +++ b/results/easycom_encoder_only_1s_4ep_2026-08-28/reported_causal_summary.txt @@ -0,0 +1,15 @@ +User-reported causal_valid_only summaries from earlier messages in this thread. +Not a dump of remote metrics.json; only the first horizon and ten-horizon mean were supplied. +G=Thinker; A/V=local encoders. All runs below are seed 0. AP/mAP values are percent. + +variant,input,seed,nominal_0.2s_map,background_ap,wearer_ap,others_ap,ten_horizon_mean_map +thinker,G,0,73.44637,56.76438,77.86650,85.70824,60.84861 +encoder_audio,A,0,79.69221,63.69477,84.09982,91.28205,60.50771 +encoder_visual,V,0,48.38434,30.16284,41.68834,73.30186,46.25250 +encoder_av,A+V,0,79.00727,63.67397,82.69832,90.64952,60.76303 +thinker_audio,G+A,0,81.72257,68.15248,85.06554,91.94969,65.10895 +thinker_visual,G+V,0,73.47412,56.67089,78.60954,85.14194,60.99238 +thinker_av,G+A+V,0,82.02661,68.93423,85.08583,92.05978,65.39978 + +The newly supplied full logs use egospeak_full_timeline, not this protocol. +The reported sample standard deviation for each encoder-only run was n/a, not zero. diff --git a/results/easycom_encoder_only_1s_4ep_2026-08-28/summary.json b/results/easycom_encoder_only_1s_4ep_2026-08-28/summary.json new file mode 100644 index 0000000..953affd --- /dev/null +++ b/results/easycom_encoder_only_1s_4ep_2026-08-28/summary.json @@ -0,0 +1,1318 @@ +{ + "schema_version": 1, + "recorded_date": "2026-08-28", + "dataset": "easycom", + "status": "preliminary_single_seed_user_reported_remote_evaluation", + "units": { + "ap_and_map": "percent", + "deltas": "percentage_points", + "printed_precision_decimals": 5 + }, + "primary_protocol": "causal_valid_only", + "full_horizon_protocol_received": "egospeak_full_timeline", + "evidence": { + "current_user_attachments": 3, + "earlier_user_attachments": 2, + "full_horizon_records_received": 50, + "remote_metrics_json_received": false, + "remote_predictions_received": false, + "remote_checkpoint_payload_received": false, + "training_summary_json_received": false, + "training_curves_received": false, + "checkpoint_epoch_independently_verified": false, + "raw_media_or_feature_causality_reaudited": false, + "remote_run_git_commit_verified": false, + "seed_variance_available": false, + "note": "Attachments show inference excerpts and paths to checkpoints/training summaries, not their contents. No experiment was re-run locally." + }, + "experiment": { + "new_variants": [ + "encoder_audio", + "encoder_visual", + "encoder_av" + ], + "reported_seed": 0, + "missing_seeds": [ + 1, + 2 + ], + "planned_epochs": 4, + "train_sessions": [ + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12 + ], + "dev_sessions": [], + "test_sessions_observed": [ + "Session_3", + "Session_2", + "Session_1" + ], + "split_and_training_budget_source": "User run instructions and local runner at 321238a; not remote checkpoint inspection.", + "loss_observed": "lstr-mce", + "output_space_observed": "three-class-softmax", + "configured_training": { + "batch_size": 512, + "learning_rate": 0.00007, + "weight_decay": 0.00005, + "warmup_epochs": 1.6, + "warmup_factor": 0.3, + "dropout": 0.2, + "transition_objective": "none", + "selection": "final_epoch_no_dev" + }, + "configured_encoder": { + "checkpoint": "Qwen2.5-Omni-3B", + "frozen": true, + "reported_transformers": "4.57.6", + "audio_window_seconds": 1, + "visual_window_seconds": 1, + "video_fps": 2, + "state_hz": 5, + "native_audio_dim": 2048, + "native_visual_dim": 2048, + "visual_readout": "mean tokens, equal-person mean, then trailing-window mean over valid frames", + "audio_readout": "pooled valid encoder outputs from half-open trailing mixed-audio window", + "provided_person_boxes": true, + "online_detector_evaluated": false, + "uses_thinker_hidden_states": false + }, + "prior_G_definition": "Frozen 3B Thinker, open-stream-replay, 8s memory, 2fps, last-four-layer stack with concat-MLP", + "remote_paths": { + "project": "/data1/yfzx/wangx/Code/icassp2027-experiment", + "source_local_features_confirmed_by_user": "/data1/yfzx/wangx/Datasets/easycom/omni_features/local_qwen_1s_encoded_tf4576", + "target_root": "/data1/yfzx/wangx/Datasets/easycom/easycom_target_frame", + "exported_features_template": "/data1/yfzx/wangx/Datasets/easycom/omni_features/encoder_only_{audio,av,visual}_1s", + "checkpoint_root": "checkpoints/easycom_encoder_only_1s", + "result_root": "results/easycom_encoder_only_1s", + "prior_checkpoint_root": "checkpoints/easycom_local_b1_encoded_tf4576" + }, + "trainable_parameter_counts_from_local_code": { + "assumption": "Native A/V=2048D; G=4x2048 with concat-MLP; 256D B1 local projections; LSTR-MCE; no transition head. Counts not read from remote checkpoints.", + "source": "easycom_head.TurnPredictionHead at 321238a", + "counts": { + "encoder_audio": 2134048, + "encoder_visual": 2134048, + "encoder_av": 8468547, + "thinker": 18928670, + "thinker_audio": 19720992, + "thinker_visual": 19720992, + "thinker_av": 20513314 + } + } + }, + "evaluation": { + "horizon_semantics": "Nominal EgoSpeak labels, 0.2 to 2.0 seconds. Physical input-to-target lead and online latency are not inferred from metric names.", + "full_timeline_note": "Includes unavailable initial h target rows per session as zero-score predictions.", + "causal_valid_only_note": "Excludes the first h target rows per session for horizon h; not an independent audit of raw input causality.", + "session_outputs_observed": [ + { + "sample_id": "Session_3", + "shape": [ + 8400, + 3, + 10 + ], + "causal_valid_counts": [ + 8399, + 8398, + 8397, + 8396, + 8395, + 8394, + 8393, + 8392, + 8391, + 8390 + ] + }, + { + "sample_id": "Session_2", + "shape": [ + 7706, + 3, + 10 + ], + "causal_valid_counts": [ + 7705, + 7704, + 7703, + 7702, + 7701, + 7700, + 7699, + 7698, + 7697, + 7696 + ] + }, + { + "sample_id": "Session_1", + "shape": [ + 6942, + 3, + 10 + ], + "causal_valid_counts": [ + 6941, + 6940, + 6939, + 6938, + 6937, + 6936, + 6935, + 6934, + 6933, + 6932 + ] + } + ], + "full_timeline_num_samples_each_horizon": 23048, + "causal_valid_num_samples_each_horizon": [ + 23045, + 23042, + 23039, + 23036, + 23033, + 23030, + 23027, + 23024, + 23021, + 23018 + ] + }, + "causal_valid_only": { + "availability": "Only the earlier reported first-horizon class AP/mAP and ten-horizon mean; full curves not received.", + "runs": [ + { + "variant": "thinker", + "input": "G", + "seed": 0, + "horizon_0_2": { + "nominal_seconds": 0.2, + "map_percent": 73.44637, + "background_ap_percent": 56.76438, + "wearer_ap_percent": 77.8665, + "others_ap_percent": 85.70824 + }, + "ten_horizon_mean_map_percent": 60.84861, + "full_horizon_metrics_available": false, + "source_file": "reported_causal_summary.txt" + }, + { + "variant": "encoder_audio", + "input": "A", + "seed": 0, + "horizon_0_2": { + "nominal_seconds": 0.2, + "map_percent": 79.69221, + "background_ap_percent": 63.69477, + "wearer_ap_percent": 84.09982, + "others_ap_percent": 91.28205 + }, + "ten_horizon_mean_map_percent": 60.50771, + "full_horizon_metrics_available": false, + "source_file": "reported_causal_summary.txt" + }, + { + "variant": "encoder_visual", + "input": "V", + "seed": 0, + "horizon_0_2": { + "nominal_seconds": 0.2, + "map_percent": 48.38434, + "background_ap_percent": 30.16284, + "wearer_ap_percent": 41.68834, + "others_ap_percent": 73.30186 + }, + "ten_horizon_mean_map_percent": 46.2525, + "full_horizon_metrics_available": false, + "source_file": "reported_causal_summary.txt" + }, + { + "variant": "encoder_av", + "input": "A+V", + "seed": 0, + "horizon_0_2": { + "nominal_seconds": 0.2, + "map_percent": 79.00727, + "background_ap_percent": 63.67397, + "wearer_ap_percent": 82.69832, + "others_ap_percent": 90.64952 + }, + "ten_horizon_mean_map_percent": 60.76303, + "full_horizon_metrics_available": false, + "source_file": "reported_causal_summary.txt" + }, + { + "variant": "thinker_audio", + "input": "G+A", + "seed": 0, + "horizon_0_2": { + "nominal_seconds": 0.2, + "map_percent": 81.72257, + "background_ap_percent": 68.15248, + "wearer_ap_percent": 85.06554, + "others_ap_percent": 91.94969 + }, + "ten_horizon_mean_map_percent": 65.10895, + "full_horizon_metrics_available": false, + "source_file": "reported_causal_summary.txt" + }, + { + "variant": "thinker_visual", + "input": "G+V", + "seed": 0, + "horizon_0_2": { + "nominal_seconds": 0.2, + "map_percent": 73.47412, + "background_ap_percent": 56.67089, + "wearer_ap_percent": 78.60954, + "others_ap_percent": 85.14194 + }, + "ten_horizon_mean_map_percent": 60.99238, + "full_horizon_metrics_available": false, + "source_file": "reported_causal_summary.txt" + }, + { + "variant": "thinker_av", + "input": "G+A+V", + "seed": 0, + "horizon_0_2": { + "nominal_seconds": 0.2, + "map_percent": 82.02661, + "background_ap_percent": 68.93423, + "wearer_ap_percent": 85.08583, + "others_ap_percent": 92.05978 + }, + "ten_horizon_mean_map_percent": 65.39978, + "full_horizon_metrics_available": false, + "source_file": "reported_causal_summary.txt" + } + ], + "comparisons": [ + { + "contrast": "encoder_audio_minus_thinker", + "protocol": "causal_valid_only", + "horizon_0_2_delta_pp": { + "map": 6.24584, + "background_ap": 6.93039, + "wearer_ap": 6.23332, + "others_ap": 5.57381 + }, + "ten_horizon_mean_delta_pp": -0.3409 + }, + { + "contrast": "encoder_av_minus_encoder_audio", + "protocol": "causal_valid_only", + "horizon_0_2_delta_pp": { + "map": -0.68494, + "background_ap": -0.0208, + "wearer_ap": -1.4015, + "others_ap": -0.63253 + }, + "ten_horizon_mean_delta_pp": 0.25532 + }, + { + "contrast": "thinker_audio_minus_encoder_audio", + "protocol": "causal_valid_only", + "horizon_0_2_delta_pp": { + "map": 2.03036, + "background_ap": 4.45771, + "wearer_ap": 0.96572, + "others_ap": 0.66764 + }, + "ten_horizon_mean_delta_pp": 4.60124 + }, + { + "contrast": "thinker_av_minus_encoder_av", + "protocol": "causal_valid_only", + "horizon_0_2_delta_pp": { + "map": 3.01934, + "background_ap": 5.26026, + "wearer_ap": 2.38751, + "others_ap": 1.41026 + }, + "ten_horizon_mean_delta_pp": 4.63675 + }, + { + "contrast": "thinker_av_minus_thinker_audio", + "protocol": "causal_valid_only", + "horizon_0_2_delta_pp": { + "map": 0.30404, + "background_ap": 0.78175, + "wearer_ap": 0.02029, + "others_ap": 0.11009 + }, + "ten_horizon_mean_delta_pp": 0.29083 + } + ] + }, + "egospeak_full_timeline": { + "availability": "Parsed from 3 current attachments and 2 earlier attachments; not original metrics.json.", + "runs": [ + { + "variant": "encoder_visual", + "input": "V", + "seed": 0, + "source_file": "raw_encoder_visual_seed0.txt", + "source_kind": "current_user_attachment", + "horizons": [ + { + "nominal_seconds": 0.2, + "map_percent": 48.38326, + "background_ap_percent": 30.16284, + "wearer_ap_percent": 41.68834, + "others_ap_percent": 73.29862, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.4, + "map_percent": 47.68478, + "background_ap_percent": 29.82215, + "wearer_ap_percent": 40.98323, + "others_ap_percent": 72.24896, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.6, + "map_percent": 47.26191, + "background_ap_percent": 29.28129, + "wearer_ap_percent": 40.74974, + "others_ap_percent": 71.75469, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.8, + "map_percent": 46.626, + "background_ap_percent": 28.24103, + "wearer_ap_percent": 40.4737, + "others_ap_percent": 71.16328, + "num_samples": 23048 + }, + { + "nominal_seconds": 1, + "map_percent": 46.00582, + "background_ap_percent": 27.73638, + "wearer_ap_percent": 39.50369, + "others_ap_percent": 70.7774, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.2, + "map_percent": 45.75002, + "background_ap_percent": 27.39577, + "wearer_ap_percent": 39.53354, + "others_ap_percent": 70.32075, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.4, + "map_percent": 45.41812, + "background_ap_percent": 27.0492, + "wearer_ap_percent": 39.3556, + "others_ap_percent": 69.84957, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.6, + "map_percent": 45.56259, + "background_ap_percent": 27.36515, + "wearer_ap_percent": 39.16056, + "others_ap_percent": 70.16205, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.8, + "map_percent": 44.79454, + "background_ap_percent": 26.64061, + "wearer_ap_percent": 38.50155, + "others_ap_percent": 69.24145, + "num_samples": 23048 + }, + { + "nominal_seconds": 2, + "map_percent": 44.97339, + "background_ap_percent": 27.3481, + "wearer_ap_percent": 37.96646, + "others_ap_percent": 69.6056, + "num_samples": 23048 + } + ], + "ten_horizon_mean_map_percent": 46.24604, + "ten_horizon_class_mean_ap_percent": { + "background_ap_percent": 28.10425, + "wearer_ap_percent": 39.79164, + "others_ap_percent": 70.84224 + } + }, + { + "variant": "encoder_av", + "input": "A+V", + "seed": 0, + "source_file": "raw_encoder_av_seed0.txt", + "source_kind": "current_user_attachment", + "horizons": [ + { + "nominal_seconds": 0.2, + "map_percent": 79.00491, + "background_ap_percent": 63.67397, + "wearer_ap_percent": 82.69832, + "others_ap_percent": 90.64244, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.4, + "map_percent": 72.06793, + "background_ap_percent": 53.18404, + "wearer_ap_percent": 75.84449, + "others_ap_percent": 87.17528, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.6, + "map_percent": 66.16179, + "background_ap_percent": 45.02921, + "wearer_ap_percent": 69.82908, + "others_ap_percent": 83.62708, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.8, + "map_percent": 61.62137, + "background_ap_percent": 40.05126, + "wearer_ap_percent": 64.4192, + "others_ap_percent": 80.39364, + "num_samples": 23048 + }, + { + "nominal_seconds": 1, + "map_percent": 58.49237, + "background_ap_percent": 36.31889, + "wearer_ap_percent": 61.01987, + "others_ap_percent": 78.13836, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.2, + "map_percent": 56.07422, + "background_ap_percent": 33.93638, + "wearer_ap_percent": 57.97721, + "others_ap_percent": 76.30907, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.4, + "map_percent": 54.61933, + "background_ap_percent": 32.99248, + "wearer_ap_percent": 55.68891, + "others_ap_percent": 75.17659, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.6, + "map_percent": 53.7818, + "background_ap_percent": 32.58516, + "wearer_ap_percent": 54.19289, + "others_ap_percent": 74.56735, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.8, + "map_percent": 52.91556, + "background_ap_percent": 32.18203, + "wearer_ap_percent": 52.7506, + "others_ap_percent": 73.81406, + "num_samples": 23048 + }, + { + "nominal_seconds": 2, + "map_percent": 52.77341, + "background_ap_percent": 32.53903, + "wearer_ap_percent": 52.15913, + "others_ap_percent": 73.62206, + "num_samples": 23048 + } + ], + "ten_horizon_mean_map_percent": 60.75127, + "ten_horizon_class_mean_ap_percent": { + "background_ap_percent": 40.24924, + "wearer_ap_percent": 62.65797, + "others_ap_percent": 79.34659 + } + }, + { + "variant": "encoder_audio", + "input": "A", + "seed": 0, + "source_file": "raw_encoder_audio_seed0.txt", + "source_kind": "current_user_attachment", + "horizons": [ + { + "nominal_seconds": 0.2, + "map_percent": 79.68981, + "background_ap_percent": 63.69477, + "wearer_ap_percent": 84.09982, + "others_ap_percent": 91.27483, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.4, + "map_percent": 72.41576, + "background_ap_percent": 52.4778, + "wearer_ap_percent": 77.43606, + "others_ap_percent": 87.3334, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.6, + "map_percent": 66.33666, + "background_ap_percent": 44.20066, + "wearer_ap_percent": 71.41221, + "others_ap_percent": 83.3971, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.8, + "map_percent": 61.60584, + "background_ap_percent": 38.60179, + "wearer_ap_percent": 66.08811, + "others_ap_percent": 80.12762, + "num_samples": 23048 + }, + { + "nominal_seconds": 1, + "map_percent": 58.28586, + "background_ap_percent": 34.75843, + "wearer_ap_percent": 62.53734, + "others_ap_percent": 77.56181, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.2, + "map_percent": 55.93708, + "background_ap_percent": 32.93325, + "wearer_ap_percent": 59.29096, + "others_ap_percent": 75.58705, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.4, + "map_percent": 54.22585, + "background_ap_percent": 31.56977, + "wearer_ap_percent": 56.79707, + "others_ap_percent": 74.31072, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.6, + "map_percent": 52.83014, + "background_ap_percent": 30.67884, + "wearer_ap_percent": 54.61175, + "others_ap_percent": 73.19983, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.8, + "map_percent": 51.97218, + "background_ap_percent": 30.3703, + "wearer_ap_percent": 52.80894, + "others_ap_percent": 72.73728, + "num_samples": 23048 + }, + { + "nominal_seconds": 2, + "map_percent": 51.66256, + "background_ap_percent": 30.86998, + "wearer_ap_percent": 51.49671, + "others_ap_percent": 72.62098, + "num_samples": 23048 + } + ], + "ten_horizon_mean_map_percent": 60.49617, + "ten_horizon_class_mean_ap_percent": { + "background_ap_percent": 39.01556, + "wearer_ap_percent": 63.6579, + "others_ap_percent": 78.81506 + } + }, + { + "variant": "thinker_av", + "input": "G+A+V", + "seed": 0, + "source_file": "raw_previous_thinker_av_seed0.txt", + "source_kind": "earlier_user_attachment", + "horizons": [ + { + "nominal_seconds": 0.2, + "map_percent": 82.02415, + "background_ap_percent": 68.93423, + "wearer_ap_percent": 85.08583, + "others_ap_percent": 92.05239, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.4, + "map_percent": 75.99169, + "background_ap_percent": 60.58903, + "wearer_ap_percent": 78.83427, + "others_ap_percent": 88.55177, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.6, + "map_percent": 70.92665, + "background_ap_percent": 53.84208, + "wearer_ap_percent": 73.45503, + "others_ap_percent": 85.48283, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.8, + "map_percent": 66.79615, + "background_ap_percent": 49.21382, + "wearer_ap_percent": 68.56072, + "others_ap_percent": 82.6139, + "num_samples": 23048 + }, + { + "nominal_seconds": 1, + "map_percent": 63.86856, + "background_ap_percent": 45.62073, + "wearer_ap_percent": 65.27034, + "others_ap_percent": 80.71461, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.2, + "map_percent": 61.5628, + "background_ap_percent": 43.4667, + "wearer_ap_percent": 62.05277, + "others_ap_percent": 79.16893, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.4, + "map_percent": 60.3407, + "background_ap_percent": 42.54682, + "wearer_ap_percent": 60.2054, + "others_ap_percent": 78.26988, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.6, + "map_percent": 58.83407, + "background_ap_percent": 41.15952, + "wearer_ap_percent": 58.18522, + "others_ap_percent": 77.15747, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.8, + "map_percent": 57.39446, + "background_ap_percent": 39.94176, + "wearer_ap_percent": 55.99271, + "others_ap_percent": 76.2489, + "num_samples": 23048 + }, + { + "nominal_seconds": 2, + "map_percent": 56.12449, + "background_ap_percent": 38.87001, + "wearer_ap_percent": 53.82707, + "others_ap_percent": 75.67639, + "num_samples": 23048 + } + ], + "ten_horizon_mean_map_percent": 65.38637, + "ten_horizon_class_mean_ap_percent": { + "background_ap_percent": 48.41847, + "wearer_ap_percent": 66.14694, + "others_ap_percent": 81.59371 + } + }, + { + "variant": "thinker_audio", + "input": "G+A", + "seed": 0, + "source_file": "raw_previous_thinker_audio_seed0.txt", + "source_kind": "earlier_user_attachment", + "horizons": [ + { + "nominal_seconds": 0.2, + "map_percent": 81.72011, + "background_ap_percent": 68.15248, + "wearer_ap_percent": 85.06554, + "others_ap_percent": 91.94232, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.4, + "map_percent": 75.91398, + "background_ap_percent": 59.90715, + "wearer_ap_percent": 79.22465, + "others_ap_percent": 88.61015, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.6, + "map_percent": 70.37189, + "background_ap_percent": 52.4848, + "wearer_ap_percent": 73.21461, + "others_ap_percent": 85.41626, + "num_samples": 23048 + }, + { + "nominal_seconds": 0.8, + "map_percent": 66.45047, + "background_ap_percent": 47.92066, + "wearer_ap_percent": 68.779, + "others_ap_percent": 82.65175, + "num_samples": 23048 + }, + { + "nominal_seconds": 1, + "map_percent": 63.79981, + "background_ap_percent": 45.59262, + "wearer_ap_percent": 65.0989, + "others_ap_percent": 80.70791, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.2, + "map_percent": 61.3512, + "background_ap_percent": 43.15384, + "wearer_ap_percent": 61.86698, + "others_ap_percent": 79.03278, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.4, + "map_percent": 59.98279, + "background_ap_percent": 41.83907, + "wearer_ap_percent": 60.22346, + "others_ap_percent": 77.88584, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.6, + "map_percent": 58.29734, + "background_ap_percent": 40.44538, + "wearer_ap_percent": 57.66189, + "others_ap_percent": 76.78474, + "num_samples": 23048 + }, + { + "nominal_seconds": 1.8, + "map_percent": 56.93221, + "background_ap_percent": 39.04677, + "wearer_ap_percent": 55.70067, + "others_ap_percent": 76.04919, + "num_samples": 23048 + }, + { + "nominal_seconds": 2, + "map_percent": 56.13656, + "background_ap_percent": 38.77318, + "wearer_ap_percent": 54.02582, + "others_ap_percent": 75.61069, + "num_samples": 23048 + } + ], + "ten_horizon_mean_map_percent": 65.09564, + "ten_horizon_class_mean_ap_percent": { + "background_ap_percent": 47.73159, + "wearer_ap_percent": 66.08615, + "others_ap_percent": 81.46916 + } + } + ], + "comparisons": [ + { + "contrast": "encoder_av_minus_encoder_audio", + "protocol": "egospeak_full_timeline", + "ten_horizon_mean_delta_pp": 0.2551, + "per_horizon": [ + { + "nominal_seconds": 0.2, + "map_delta_pp": -0.6849, + "background_ap_delta_pp": -0.0208, + "wearer_ap_delta_pp": -1.4015, + "others_ap_delta_pp": -0.63239 + }, + { + "nominal_seconds": 0.4, + "map_delta_pp": -0.34783, + "background_ap_delta_pp": 0.70624, + "wearer_ap_delta_pp": -1.59157, + "others_ap_delta_pp": -0.15812 + }, + { + "nominal_seconds": 0.6, + "map_delta_pp": -0.17487, + "background_ap_delta_pp": 0.82855, + "wearer_ap_delta_pp": -1.58313, + "others_ap_delta_pp": 0.22998 + }, + { + "nominal_seconds": 0.8, + "map_delta_pp": 0.01553, + "background_ap_delta_pp": 1.44947, + "wearer_ap_delta_pp": -1.66891, + "others_ap_delta_pp": 0.26602 + }, + { + "nominal_seconds": 1, + "map_delta_pp": 0.20651, + "background_ap_delta_pp": 1.56046, + "wearer_ap_delta_pp": -1.51747, + "others_ap_delta_pp": 0.57655 + }, + { + "nominal_seconds": 1.2, + "map_delta_pp": 0.13714, + "background_ap_delta_pp": 1.00313, + "wearer_ap_delta_pp": -1.31375, + "others_ap_delta_pp": 0.72202 + }, + { + "nominal_seconds": 1.4, + "map_delta_pp": 0.39348, + "background_ap_delta_pp": 1.42271, + "wearer_ap_delta_pp": -1.10816, + "others_ap_delta_pp": 0.86587 + }, + { + "nominal_seconds": 1.6, + "map_delta_pp": 0.95166, + "background_ap_delta_pp": 1.90632, + "wearer_ap_delta_pp": -0.41886, + "others_ap_delta_pp": 1.36752 + }, + { + "nominal_seconds": 1.8, + "map_delta_pp": 0.94338, + "background_ap_delta_pp": 1.81173, + "wearer_ap_delta_pp": -0.05834, + "others_ap_delta_pp": 1.07678 + }, + { + "nominal_seconds": 2, + "map_delta_pp": 1.11085, + "background_ap_delta_pp": 1.66905, + "wearer_ap_delta_pp": 0.66242, + "others_ap_delta_pp": 1.00108 + } + ], + "positive_map_horizons": [ + 0.8, + 1, + 1.2, + 1.4, + 1.6, + 1.8, + 2 + ], + "negative_wearer_horizons": [ + 0.2, + 0.4, + 0.6, + 0.8, + 1, + 1.2, + 1.4, + 1.6, + 1.8 + ], + "ten_horizon_mean_class_delta_pp": { + "background_ap_delta_pp": 1.23369, + "wearer_ap_delta_pp": -0.99993, + "others_ap_delta_pp": 0.53153 + } + }, + { + "contrast": "thinker_audio_minus_encoder_audio", + "protocol": "egospeak_full_timeline", + "ten_horizon_mean_delta_pp": 4.59947, + "per_horizon": [ + { + "nominal_seconds": 0.2, + "map_delta_pp": 2.0303, + "background_ap_delta_pp": 4.45771, + "wearer_ap_delta_pp": 0.96572, + "others_ap_delta_pp": 0.66749 + }, + { + "nominal_seconds": 0.4, + "map_delta_pp": 3.49822, + "background_ap_delta_pp": 7.42935, + "wearer_ap_delta_pp": 1.78859, + "others_ap_delta_pp": 1.27675 + }, + { + "nominal_seconds": 0.6, + "map_delta_pp": 4.03523, + "background_ap_delta_pp": 8.28414, + "wearer_ap_delta_pp": 1.8024, + "others_ap_delta_pp": 2.01916 + }, + { + "nominal_seconds": 0.8, + "map_delta_pp": 4.84463, + "background_ap_delta_pp": 9.31887, + "wearer_ap_delta_pp": 2.69089, + "others_ap_delta_pp": 2.52413 + }, + { + "nominal_seconds": 1, + "map_delta_pp": 5.51395, + "background_ap_delta_pp": 10.83419, + "wearer_ap_delta_pp": 2.56156, + "others_ap_delta_pp": 3.1461 + }, + { + "nominal_seconds": 1.2, + "map_delta_pp": 5.41412, + "background_ap_delta_pp": 10.22059, + "wearer_ap_delta_pp": 2.57602, + "others_ap_delta_pp": 3.44573 + }, + { + "nominal_seconds": 1.4, + "map_delta_pp": 5.75694, + "background_ap_delta_pp": 10.2693, + "wearer_ap_delta_pp": 3.42639, + "others_ap_delta_pp": 3.57512 + }, + { + "nominal_seconds": 1.6, + "map_delta_pp": 5.4672, + "background_ap_delta_pp": 9.76654, + "wearer_ap_delta_pp": 3.05014, + "others_ap_delta_pp": 3.58491 + }, + { + "nominal_seconds": 1.8, + "map_delta_pp": 4.96003, + "background_ap_delta_pp": 8.67647, + "wearer_ap_delta_pp": 2.89173, + "others_ap_delta_pp": 3.31191 + }, + { + "nominal_seconds": 2, + "map_delta_pp": 4.474, + "background_ap_delta_pp": 7.9032, + "wearer_ap_delta_pp": 2.52911, + "others_ap_delta_pp": 2.98971 + } + ], + "positive_map_horizons": [ + 0.2, + 0.4, + 0.6, + 0.8, + 1, + 1.2, + 1.4, + 1.6, + 1.8, + 2 + ], + "negative_wearer_horizons": [], + "ten_horizon_mean_class_delta_pp": { + "background_ap_delta_pp": 8.71604, + "wearer_ap_delta_pp": 2.42825, + "others_ap_delta_pp": 2.6541 + } + }, + { + "contrast": "thinker_av_minus_encoder_av", + "protocol": "egospeak_full_timeline", + "ten_horizon_mean_delta_pp": 4.6351, + "per_horizon": [ + { + "nominal_seconds": 0.2, + "map_delta_pp": 3.01924, + "background_ap_delta_pp": 5.26026, + "wearer_ap_delta_pp": 2.38751, + "others_ap_delta_pp": 1.40995 + }, + { + "nominal_seconds": 0.4, + "map_delta_pp": 3.92376, + "background_ap_delta_pp": 7.40499, + "wearer_ap_delta_pp": 2.98978, + "others_ap_delta_pp": 1.37649 + }, + { + "nominal_seconds": 0.6, + "map_delta_pp": 4.76486, + "background_ap_delta_pp": 8.81287, + "wearer_ap_delta_pp": 3.62595, + "others_ap_delta_pp": 1.85575 + }, + { + "nominal_seconds": 0.8, + "map_delta_pp": 5.17478, + "background_ap_delta_pp": 9.16256, + "wearer_ap_delta_pp": 4.14152, + "others_ap_delta_pp": 2.22026 + }, + { + "nominal_seconds": 1, + "map_delta_pp": 5.37619, + "background_ap_delta_pp": 9.30184, + "wearer_ap_delta_pp": 4.25047, + "others_ap_delta_pp": 2.57625 + }, + { + "nominal_seconds": 1.2, + "map_delta_pp": 5.48858, + "background_ap_delta_pp": 9.53032, + "wearer_ap_delta_pp": 4.07556, + "others_ap_delta_pp": 2.85986 + }, + { + "nominal_seconds": 1.4, + "map_delta_pp": 5.72137, + "background_ap_delta_pp": 9.55434, + "wearer_ap_delta_pp": 4.51649, + "others_ap_delta_pp": 3.09329 + }, + { + "nominal_seconds": 1.6, + "map_delta_pp": 5.05227, + "background_ap_delta_pp": 8.57436, + "wearer_ap_delta_pp": 3.99233, + "others_ap_delta_pp": 2.59012 + }, + { + "nominal_seconds": 1.8, + "map_delta_pp": 4.4789, + "background_ap_delta_pp": 7.75973, + "wearer_ap_delta_pp": 3.24211, + "others_ap_delta_pp": 2.43484 + }, + { + "nominal_seconds": 2, + "map_delta_pp": 3.35108, + "background_ap_delta_pp": 6.33098, + "wearer_ap_delta_pp": 1.66794, + "others_ap_delta_pp": 2.05433 + } + ], + "positive_map_horizons": [ + 0.2, + 0.4, + 0.6, + 0.8, + 1, + 1.2, + 1.4, + 1.6, + 1.8, + 2 + ], + "negative_wearer_horizons": [], + "ten_horizon_mean_class_delta_pp": { + "background_ap_delta_pp": 8.16923, + "wearer_ap_delta_pp": 3.48897, + "others_ap_delta_pp": 2.24711 + } + }, + { + "contrast": "thinker_av_minus_thinker_audio", + "protocol": "egospeak_full_timeline", + "ten_horizon_mean_delta_pp": 0.29073, + "per_horizon": [ + { + "nominal_seconds": 0.2, + "map_delta_pp": 0.30404, + "background_ap_delta_pp": 0.78175, + "wearer_ap_delta_pp": 0.02029, + "others_ap_delta_pp": 0.11007 + }, + { + "nominal_seconds": 0.4, + "map_delta_pp": 0.07771, + "background_ap_delta_pp": 0.68188, + "wearer_ap_delta_pp": -0.39038, + "others_ap_delta_pp": -0.05838 + }, + { + "nominal_seconds": 0.6, + "map_delta_pp": 0.55476, + "background_ap_delta_pp": 1.35728, + "wearer_ap_delta_pp": 0.24042, + "others_ap_delta_pp": 0.06657 + }, + { + "nominal_seconds": 0.8, + "map_delta_pp": 0.34568, + "background_ap_delta_pp": 1.29316, + "wearer_ap_delta_pp": -0.21828, + "others_ap_delta_pp": -0.03785 + }, + { + "nominal_seconds": 1, + "map_delta_pp": 0.06875, + "background_ap_delta_pp": 0.02811, + "wearer_ap_delta_pp": 0.17144, + "others_ap_delta_pp": 0.0067 + }, + { + "nominal_seconds": 1.2, + "map_delta_pp": 0.2116, + "background_ap_delta_pp": 0.31286, + "wearer_ap_delta_pp": 0.18579, + "others_ap_delta_pp": 0.13615 + }, + { + "nominal_seconds": 1.4, + "map_delta_pp": 0.35791, + "background_ap_delta_pp": 0.70775, + "wearer_ap_delta_pp": -0.01806, + "others_ap_delta_pp": 0.38404 + }, + { + "nominal_seconds": 1.6, + "map_delta_pp": 0.53673, + "background_ap_delta_pp": 0.71414, + "wearer_ap_delta_pp": 0.52333, + "others_ap_delta_pp": 0.37273 + }, + { + "nominal_seconds": 1.8, + "map_delta_pp": 0.46225, + "background_ap_delta_pp": 0.89499, + "wearer_ap_delta_pp": 0.29204, + "others_ap_delta_pp": 0.19971 + }, + { + "nominal_seconds": 2, + "map_delta_pp": -0.01207, + "background_ap_delta_pp": 0.09683, + "wearer_ap_delta_pp": -0.19875, + "others_ap_delta_pp": 0.0657 + } + ], + "positive_map_horizons": [ + 0.2, + 0.4, + 0.6, + 0.8, + 1, + 1.2, + 1.4, + 1.6, + 1.8 + ], + "negative_wearer_horizons": [ + 0.4, + 0.8, + 1.4, + 2 + ], + "ten_horizon_mean_class_delta_pp": { + "background_ap_delta_pp": 0.68687, + "wearer_ap_delta_pp": 0.06078, + "others_ap_delta_pp": 0.12454 + } + } + ] + }, + "protocol_differences": [ + { + "variant": "encoder_visual", + "causal_minus_full_horizon_0_2_map_pp": 0.00108, + "causal_minus_full_ten_horizon_mean_map_pp": 0.00646 + }, + { + "variant": "encoder_av", + "causal_minus_full_horizon_0_2_map_pp": 0.00236, + "causal_minus_full_ten_horizon_mean_map_pp": 0.01176 + }, + { + "variant": "encoder_audio", + "causal_minus_full_horizon_0_2_map_pp": 0.0024, + "causal_minus_full_ten_horizon_mean_map_pp": 0.01154 + }, + { + "variant": "thinker_av", + "causal_minus_full_horizon_0_2_map_pp": 0.00246, + "causal_minus_full_ten_horizon_mean_map_pp": 0.01341 + }, + { + "variant": "thinker_audio", + "causal_minus_full_horizon_0_2_map_pp": 0.00246, + "causal_minus_full_ten_horizon_mean_map_pp": 0.01331 + } + ], + "interpretation": { + "observation_1": "In this seed, G+A beats A at every full-timeline horizon and class; G+AV likewise beats AV. This supports a useful combined system, not isolated decoder causation.", + "observation_2": "Encoder AV trails A in mAP at 0.2/0.4/0.6, turns slightly positive at 0.8, and gains about one pp at 1.6-2.0. Wearer AP declines at 9/10 horizons.", + "observation_3": "Adding local V to G+A yields small macro gains mainly from background; mean wearer AP change is only +0.06078 pp.", + "unproven_explanations": [ + "Longer history contribution versus head capacity or nonlinear fusion", + "Whether person/time pooling suppresses speaker-specific information", + "Whether a different fusion or training budget would avoid negative wearer transfer" + ], + "limitations": [ + "One reported seed", + "Different head capacities and visual fields/history budgets", + "No training curves, remote model metadata, or original prediction arrays", + "Overall AP does not establish multi-person interaction understanding or boundary anticipation", + "No significance from treating correlated horizons as independent replicates", + "No feature-extraction or end-to-end speed claim from inference batch/s" + ], + "next_steps": [ + "Collect raw metrics.json and checkpoint/training metadata, plus causal-valid curves", + "Complete seeds 1/2 with fixed epoch-4/no-dev protocol", + "Run architecture/parameter and raw-visible-history controlled modality comparisons before attributing gains to Thinker", + "Only then test audio-conditioned per-person fusion and pre-specified multi-person/overlap reporting; avoid test-driven epoch/threshold selection" + ] + }, + "provenance": { + "normalization": "Console archives preserve all lines and wrapped paths; trailing horizontal whitespace removed and one final newline added. Numeric values unchanged.", + "sources": [ + { + "variant": "encoder_visual", + "source_kind": "current_user_attachment", + "original_attachment_path": "/Users/wx44wx/.codex/attachments/a58e2943-0d9a-4d93-8b20-94ecd405c632/pasted-text.txt", + "archived_file": "raw_encoder_visual_seed0.txt" + }, + { + "variant": "encoder_av", + "source_kind": "current_user_attachment", + "original_attachment_path": "/Users/wx44wx/.codex/attachments/ce28f77e-e131-497e-b9aa-83d9b1dc9171/pasted-text.txt", + "archived_file": "raw_encoder_av_seed0.txt" + }, + { + "variant": "encoder_audio", + "source_kind": "current_user_attachment", + "original_attachment_path": "/Users/wx44wx/.codex/attachments/094440a9-5a37-4737-a32b-e905cdefbf3d/pasted-text.txt", + "archived_file": "raw_encoder_audio_seed0.txt" + }, + { + "variant": "thinker_av", + "source_kind": "earlier_user_attachment", + "original_attachment_path": "/Users/wx44wx/.codex/attachments/e3e61fca-d5d2-452f-b040-c156f3d415e2/pasted-text.txt", + "archived_file": "raw_previous_thinker_av_seed0.txt" + }, + { + "variant": "thinker_audio", + "source_kind": "earlier_user_attachment", + "original_attachment_path": "/Users/wx44wx/.codex/attachments/4a1649ae-1881-4c79-a7c4-0b0f56140eb8/pasted-text.txt", + "archived_file": "raw_previous_thinker_audio_seed0.txt" + } + ], + "supplementary_source": "Earlier user summary tables and run commands in this thread, transcribed in reported_causal_summary.txt", + "locally_available_artifact": "Derived research record only, not a replacement for remote metrics.json" + } +}