From 383a3b72c6e0c6d4717ef1ae4d3f7f7456301895 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Mon, 31 Aug 2026 00:04:52 +0800 Subject: [PATCH] docs: draft ICASSP abstract and introduction --- manuscript/main.tex | 110 ++++++++++++++++++++++++++++++++++++++ manuscript/references.bib | 42 +++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 manuscript/main.tex create mode 100644 manuscript/references.bib diff --git a/manuscript/main.tex b/manuscript/main.tex new file mode 100644 index 0000000..cef284c --- /dev/null +++ b/manuscript/main.tex @@ -0,0 +1,110 @@ +% Draft ICASSP 2027 manuscript. +% Compile from this directory with: +% pdflatex main && bibtex main && pdflatex main && pdflatex main +\documentclass{article} +\usepackage{ICASSP2026_Paper_Templates/spconf} +\usepackage{amsmath,graphicx,hyperref} + +\title{NOT EVERY TURN IS MINE: ADDRESSEE-GROUNDED FLOOR-ENTRY PROJECTION IN EGOCENTRIC MULTIPARTY CONVERSATION} + +% TODO: Replace these placeholders after the ICASSP 2027 review policy and +% author list are finalized. +\name{Author Name(s)} +\address{Author Affiliation(s)} + +\begin{document} +\ninept +\maketitle + +\begin{abstract} +Egocentric conversational agents must decide not only when a speaking +opportunity arises, but also whether that opportunity belongs to them. +Existing speech-initiation forecasting collapses all non-wearer speakers, so +strong frame-level accuracy can coexist with intrusive entries while two +bystanders converse. We study addressee-grounded floor-entry projection in +causal multiparty audio-video streams. Our formulation retains dense forecasts +of wearer and non-wearer activity at 200-ms resolution over a two-second +horizon, while grounding the wearer-entry decision in person-aligned visual +context. Using EasyCom target-of-speech annotations, we introduce an event-level +protocol that separates addressed-to-wearer responses from third-party +exchanges and measures false entries at matched response recall, together with +decision latency. Wrong-person and time-shifted visual controls test whether +any improvement follows correct social grounding rather than additional model +capacity. Experiments on EasyCom, with Ego4D as an activity-forecasting +generalization set, show \textbf{[TODO: insert the final matched-recall and +latency results]}. +\end{abstract} + +\begin{keywords} +turn-taking, speech initiation, addressee detection, egocentric video, +audio-visual learning +\end{keywords} + +\section{Introduction} +\label{sec:intro} + +Low-latency conversational agents must continuously decide whether to speak or +to keep listening. In a dyadic exchange, a pause or an apparent end of turn is +often treated as an invitation to respond. This assumption fails in +multiparty interaction: one participant may finish an utterance addressed to +another participant, and an egocentric agent that reacts only to acoustic +availability will enter a floor that was never offered to it. The resulting +error is socially more disruptive than a small framewise prediction error. +Accordingly, a useful model must distinguish \emph{when the floor may change} +from \emph{whether the change is directed to the camera wearer}. + +Voice Activity Projection (VAP) models future joint speech activity and derives +turn-taking events without manually labeling every event +\cite{ekstedt2022vap}. EgoSpeak extends future activity prediction to causal, +untrimmed egocentric video, forecasting wearer and non-wearer activity over the +next two seconds \cite{kim2025egospeak}. Its role-relative output is attractive +for a wearable agent, but aggregating all visible participants into a single +``others'' channel removes the distinction between an utterance addressed to +the wearer and an exchange among bystanders. Consequently, conventional +per-frame average precision can be dominated by ordinary speech activity and +can hide precisely the false-entry behavior that visual social cues should +prevent. Recent event-oriented evaluation likewise emphasizes the joint +trade-off among recall, false positives, and latency rather than a single +frame-level score \cite{jiang2026turnbench}. + +Related work provides two complementary pieces of the problem but does not +unify them. Egocentric talking-to-me detection predicts whether a completed +utterance addresses the camera wearer and benefits from long-term social +context \cite{kong2024sicnet}. Multiparty VAP grounds future activity in +visible face tracks to predict floor shifts and next speakers +\cite{qi2026muvap}. The former is generally evaluated as utterance-level +addressee classification rather than continuous anticipation, whereas the +latter selects among visible participants and does not ask whether an +invisible camera wearer should enter. We target the intersection: causal +floor-entry forecasting for an egocentric agent whose decision depends on the +speaker--addressee relation among the people in view. + +We formulate this problem as \emph{addressee-conditioned egocentric activity +projection}. A global streaming audio-video representation predicts future +wearer and non-wearer activity, while a person-aligned visual pathway preserves +participant identity and recent social context. The final wearer-entry score +can therefore represent both floor availability and evidence that the current +utterance is directed to the wearer. EasyCom is particularly suitable for this +analysis because it provides synchronized egocentric audio-video, per-person +tracks, voice activity, and target-of-speech annotations +\cite{donley2021easycom}. Target annotations are used for training-side +auxiliary supervision and evaluation strata, never as model input at test +time. + +Our contributions are threefold. First, we cast speech initiation as +recipient-aware floor-entry projection, connecting dense egocentric activity +forecasting with multiparty addressee grounding. Second, we propose an +event-level evaluation that contrasts addressed-to-wearer response events with +third-party exchanges and reports false entries at matched response recall, +alongside latency and the original EgoSpeak metrics. Third, we use +wrong-person, temporally shifted, and visibility-conditioned visual controls to +test whether improvements arise from the correct social evidence instead of +extra feature capacity or globally conservative predictions. + +% TODO: Add Method, Experimental Setup, Results, and Conclusion after the +% event protocol and person-aligned model are frozen. + +\bibliographystyle{ICASSP2026_Paper_Templates/IEEEbib} +\bibliography{references} + +\end{document} diff --git a/manuscript/references.bib b/manuscript/references.bib new file mode 100644 index 0000000..fb37077 --- /dev/null +++ b/manuscript/references.bib @@ -0,0 +1,42 @@ +@inproceedings{ekstedt2022vap, + author = {Erik Ekstedt and Gabriel Skantze}, + title = {Voice Activity Projection: Self-Supervised Learning of Turn-Taking Events}, + booktitle = {Proc. Interspeech}, + year = {2022} +} + +@inproceedings{kim2025egospeak, + author = {Junhyeok Kim and Min Soo Kim and Jiwan Chung and Jungbin Cho and Jisoo Kim and Sungwoong Kim and Gyeongbo Sim and Youngjae Yu}, + title = {{EgoSpeak}: Learning When to Speak for Egocentric Conversational Agents in the Wild}, + booktitle = {Findings of the Association for Computational Linguistics: NAACL}, + pages = {2990--3005}, + year = {2025} +} + +@article{jiang2026turnbench, + author = {Freeman Jiang and others}, + title = {{TurnBench}: A Multi-Domain Benchmark for Turn-Taking Dynamics in Spoken Dialogue}, + journal = {arXiv preprint arXiv:2608.25218}, + year = {2026} +} + +@inproceedings{kong2024sicnet, + author = {Deqian Kong and Furqan Khan and Xu Zhang and Prateek Singhal and Ying Nian Wu}, + title = {Long-Term Social Interaction Context: The Key to Egocentric Addressee Detection}, + booktitle = {Proc. IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, + year = {2024} +} + +@article{qi2026muvap, + author = {Haotian Qi and Gabriel Skantze}, + title = {{MuVAP}: Multimodal Multiparty Voice Activity Projection for Turn-Taking Prediction in the Wild}, + journal = {arXiv preprint arXiv:2606.16731}, + year = {2026} +} + +@article{donley2021easycom, + author = {Jacob Donley and Vladimir Tourbabin and Jung-Suk Lee and Mark Broyles and Hao Jiang and Jie Shen and Maja Pantic and Vamsi Krishna Ithapu and Ravish Mehra}, + title = {{EasyCom}: An Augmented Reality Dataset to Support Algorithms for Easy Communication in Noisy Environments}, + journal = {arXiv preprint arXiv:2107.04174}, + year = {2021} +}