diff --git a/manuscript/main/Makefile b/manuscript/main/Makefile index e671c8d..aa235be 100644 --- a/manuscript/main/Makefile +++ b/manuscript/main/Makefile @@ -1,10 +1,16 @@ LATEXMK ?= latexmk -.PHONY: all clean +.PHONY: all clean figure -all: +all: figures/method_architecture.pdf mkdir -p build $(LATEXMK) -pdf -interaction=nonstopmode -halt-on-error -outdir=build main.tex +figure: figures/method_architecture.pdf + +figures/method_architecture.pdf: figures/method_architecture.tex + $(LATEXMK) -pdf -interaction=nonstopmode -halt-on-error -outdir=figures $< + clean: $(LATEXMK) -C -outdir=build main.tex + $(LATEXMK) -c -outdir=figures figures/method_architecture.tex diff --git a/manuscript/main/README.md b/manuscript/main/README.md index b5ec23f..72ff270 100644 --- a/manuscript/main/README.md +++ b/manuscript/main/README.md @@ -26,13 +26,18 @@ Current source layout: main/ main.tex references.bib + figures/ + method_architecture.tex # reproducible TikZ source + method_architecture.pdf # manuscript-ready vector figure latexmkrc Makefile README.md build/ # generated, ignored ``` -The manuscript is intentionally a content skeleton with visible TODOs. Do not +Running `make` regenerates the method figure when its TikZ source changes +before compiling the manuscript. The manuscript is intentionally a content +skeleton with visible TODOs. Do not turn planned implementations, controls, or results into completed claims. The frozen paper story is documented in diff --git a/manuscript/main/figures/.gitignore b/manuscript/main/figures/.gitignore new file mode 100644 index 0000000..dd57b0b --- /dev/null +++ b/manuscript/main/figures/.gitignore @@ -0,0 +1,4 @@ +method_architecture.aux +method_architecture.fdb_latexmk +method_architecture.fls +method_architecture.log diff --git a/manuscript/main/figures/method_architecture.pdf b/manuscript/main/figures/method_architecture.pdf new file mode 100644 index 0000000..5796102 Binary files /dev/null and b/manuscript/main/figures/method_architecture.pdf differ diff --git a/manuscript/main/figures/method_architecture.tex b/manuscript/main/figures/method_architecture.tex new file mode 100644 index 0000000..c305782 --- /dev/null +++ b/manuscript/main/figures/method_architecture.tex @@ -0,0 +1,97 @@ +\documentclass[tikz,border=2pt]{standalone} +\usepackage{amsmath} +\usetikzlibrary{arrows.meta,calc,positioning} + +\definecolor{navy}{HTML}{17324D} +\definecolor{bluefill}{HTML}{EAF2F8} +\definecolor{teal}{HTML}{137C8B} +\definecolor{tealfill}{HTML}{E8F6F5} +\definecolor{orange}{HTML}{D97706} +\definecolor{orangefill}{HTML}{FFF3E0} +\definecolor{grayline}{HTML}{73808C} + +\begin{document} +\begin{tikzpicture}[ + font=\sffamily\scriptsize, + >=Latex, + flow/.style={-{Latex[length=2.0mm]}, line width=0.65pt, draw=navy}, + aux/.style={-{Latex[length=1.8mm]}, line width=0.6pt, draw=teal}, + box/.style={draw=navy, rounded corners=1.5pt, line width=0.65pt, + fill=bluefill, minimum height=7mm, align=center, inner sep=2.3pt}, + localbox/.style={box, draw=teal, fill=tealfill}, + residualbox/.style={box, draw=orange, fill=orangefill}, + labelbox/.style={align=center, text=navy}, + note/.style={font=\sffamily\tiny, text=grayline, align=center} +] + +\node[labelbox] (av) at (0,2.65) {causal full-scene AV\\prefix $\mathcal X_{\leq t_n}$}; +\node[box, right=7mm of av, minimum width=24mm] (qwen) {frozen\\Qwen2.5-Omni}; +\node[box, right=8mm of qwen, minimum width=12mm] (g) {global\\$G_{t_n}$}; + +\node[labelbox] (audio) at (0,1.45) {mixed audio\\$\leq t_n$}; +\node[localbox, right=7mm of audio, minimum width=24mm] (cpc) {frozen causal CPC\\25-Hz processing}; +\node[localbox, right=8mm of cpc, minimum width=12mm] (u) {shared\\$U_{t_n}$}; + +\node[labelbox] (faces) at (0,0.15) {visible face crops\\$\leq t_n$}; +\node[localbox, right=7mm of faces, minimum width=24mm] (asd) {frozen causal AV/ASD\\25-Hz processing}; +\node[localbox, right=8mm of asd, minimum width=20mm] (persons) {anonymous set\\$\mathcal P_{t_n}=\{[V_i;s_i]\}$}; + +\node[box, minimum width=21mm] (base) at (8.25,2.05) {$G+U$ base head\\$Z^{\rm base}_{t_n}$}; +\node[localbox, minimum width=22mm] (attn) at (8.25,0.70) {masked person MHA\\query: $G_{t_n}+U_{t_n}$\\\scriptsize memory: $\mathcal P_{t_n}+$ NULL}; +\node[residualbox, minimum width=19mm] (res) at (10.65,0.70) {zero-init residual\\$\Delta Z_{t_n}$}; +\node[circle, draw=navy, fill=white, line width=0.7pt, + minimum size=5.5mm, inner sep=0pt] (sum) at (11.55,2.05) {$+$}; +\node[box, minimum width=21mm] (out) at (13.15,2.05) {$10\!\times\!3$ future activity\\$0.2$--$2.0$ s}; + +\draw[flow] (av) -- (qwen); +\draw[flow] (qwen) -- (g); +\draw[aux] (audio) -- (cpc); +\draw[aux] (cpc) -- (u); +\draw[aux] (faces) -- (asd); +\draw[aux] (asd) -- (persons); +\draw[flow] (g.east) -| ($(base.west)+(-0.25,0.12)$) -- (base.west); +\draw[aux] (u.east) -| ($(base.west)+(-0.25,-0.12)$) -- (base.west); +\draw[flow] (g.east) -- ++(0.3,0) |- (attn.north west); +\draw[aux] (u.east) -- ++(0.5,0) |- (attn.west); +\draw[aux] (persons.east) -- (attn.west); +\draw[flow] (base) -- (sum); +\draw[aux] (attn) -- (res); +\draw[flow, draw=orange] (res.north) |- (sum.south); +\draw[flow] (sum) -- (out); + +\node[note, below=1.5mm of base] {trained first, then frozen}; +\node[note, below=1.2mm of res] {trainable adaptation}; + +\draw[grayline, line width=0.45pt] (-0.15,-0.62) -- (14.35,-0.62); +\node[anchor=west, font=\sffamily\scriptsize\bfseries, text=navy] + at (-0.05,-0.92) {Causal streaming at cutoff $t_n$}; + +\node[anchor=east, text=navy] at (2.25,-1.45) {Omni input}; +\draw[fill=bluefill, draw=navy, line width=0.55pt] (2.45,-1.67) rectangle (4.35,-1.23); +\draw[fill=bluefill, draw=navy, line width=0.55pt] (4.35,-1.67) rectangle (6.25,-1.23); +\node[text=navy] at (3.40,-1.45) {cached 2-s blocks}; +\node[text=navy] at (5.30,-1.45) {$\cdots$}; +\draw[fill=orangefill, draw=orange, line width=0.65pt] (6.25,-1.67) rectangle (9.15,-1.23); +\node[text=orange] at (7.70,-1.45) {current partial block}; +\draw[dashed, draw=orange, line width=0.8pt] (9.15,-2.90) -- (9.15,-1.02); +\node[anchor=south, text=orange, font=\sffamily\tiny\bfseries] at (9.15,-1.02) {$t_n$}; +\node[note, anchor=west] at (9.40,-1.45) {latest observed input included\\up to 8-s rolling budget}; + +\node[anchor=east, text=teal] at (2.25,-2.15) {local encoder}; +\foreach \x in {2.55,2.85,...,8.85} { + \draw[draw=teal, line width=0.55pt] (\x,-2.33) -- (\x,-1.97); +} +\draw[draw=teal, line width=0.55pt] (2.55,-2.15) -- (9.15,-2.15); +\node[note, anchor=west, text=teal] at (9.40,-2.15) {25 Hz internal states}; + +\node[anchor=east, text=navy] at (2.25,-2.72) {forecast grid}; +\draw[draw=navy, line width=0.65pt] (2.55,-2.72) -- (9.15,-2.72); +\foreach \x/\lab in {3.15/0.2,4.65/0.4,6.15/0.6,7.65/0.8,9.15/$t_n$} { + \draw[draw=navy, line width=0.75pt] (\x,-2.92) -- (\x,-2.52); + \node[below=0.5mm, font=\sffamily\tiny, text=navy] at (\x,-2.92) {\lab}; + \draw[-{Latex[length=1.4mm]}, draw=teal, line width=0.55pt] (\x,-2.32) -- (\x,-2.50); +} +\node[note, anchor=west, text=navy] at (9.40,-2.72) {5 Hz fusion and prediction\\every fifth local right edge}; + +\end{tikzpicture} +\end{document} diff --git a/manuscript/main/main.tex b/manuscript/main/main.tex index ce1926c..0d4a062 100644 --- a/manuscript/main/main.tex +++ b/manuscript/main/main.tex @@ -107,117 +107,125 @@ corresponding TODO gates pass: positives and explicitly annotated hard-negative opportunity regions. \end{itemize} -\begin{figure}[t] +\begin{figure*}[t] \centering -\fbox{\parbox[c][1.00in][c]{0.94\columnwidth}{\centering -\todo{Draw only the selected method: causal AV to the long-context Omni path; -the short-window mixed-audio and anonymous person-set path; residual fusion; -the common $10\times3$ head; and the separate wearer-entry/INT-yield evaluation. -Mark physical cutoffs, trainable modules, person masks, and NULL/OFFSCREEN. -Do not draw oracle labels, rejected track variants, or multiple backbones.}}} -\caption{Long-context Omni understanding and short-window local floor evidence -for wearer-relative future-activity prediction.} +\includegraphics[width=0.98\textwidth]{figures/method_architecture.pdf} +\caption{Our strictly causal global--local architecture (top) and streaming +alignment (bottom). At every 5-Hz cutoff $t_n$, the global state includes the +latest partial 2-s Omni block, while every fifth causal 25-Hz local state +provides shared audio and an anonymous visible-person set. Masked person +attention contributes only through a zero-initialized residual to the matched +$G+U$ baseline.} \label{fig:method} -\end{figure} +\end{figure*} \section{Method} \label{sec:method} -\subsection{Causal wearer-relative projection} +\subsection{Model Architecture} +\label{sec:model_arch} -At decision time $t$, the model observes only the stream -$\mathcal X_{\leq t}$ and predicts $K=10$ future bins at $\Delta=0.2$~s. For -bin $k$, the multilabel target +At decision time $t$, the model observes only the audiovisual stream +$\mathcal X_{\leq t}$ and predicts $K=10$ future bins spaced by +$\Delta=0.2$~s. For horizon $k$, the multilabel target $\vect y_{t,k}=[y^{\mathrm{bg}},y^{\mathrm{w}},y^{\mathrm{o}}]$ denotes -background, camera-wearer activity, and activity by one or more other people. -Wearer--other overlap is allowed. The model outputs +background, camera-wearer activity, and activity by one or more other people; +wearer--other overlap is allowed. The output is \begin{equation} \hat{\vect Y}_t=f_\theta(\mathcal X_{\leq t}) \in[0,1]^{K\times3}. \label{eq:task} \end{equation} -The wearer-activity forecast also provides causal behavioral scores at every -horizon. At decision time $t$, if the wearer is silent, -$\hat y^{\mathrm w}_{t,k}$ acts as a take-turn score for future bin $k$; if -the wearer is active, $1-\hat y^{\mathrm w}_{t,k}$ acts as a yield score. -These scores are trained only from future activity targets and are not event -hazards or committed controller actions. -Adjudicated wearer-entry and INT-yield events used for behavioral evaluation are -defined separately from participant-resolved annotations in -Sec.~\ref{sec:experiments}. -\subsection{Global Omni baseline} - -Let $\vect g_t\in\mathbb R^{d_g}$ be the causal global readout of -Qwen2.5-Omni at the physical cutoff $t$ \cite{xu2025qwenomni}. The frozen -baseline trains only a dimension-matched prediction head. A matched PEFT/LoRA -baseline updates a pre-registered subset of the Omni model under the same -examples, context, optimizer budget, head, and evaluator. Its purpose is to -test whether generic parameter adaptation is sufficient; one negative recipe -is not interpreted as a general limitation of LoRA or all Omni models. - -\todo{Freeze the exact Qwen2.5-Omni-3B revision, processor, input rates, -streaming/cache semantics, readout layer, context, dtype, LoRA targets/rank, -trainable parameter budget, and cutoff audit. Add Qwen2.5-Omni-7B only as a -capacity-scaling control. MiniCPM-o 4.5 is restricted to the diagnostic -streaming/cache ablation below and does not enter the core method or support a -cross-family superiority claim.} - -\subsection{Causal local floor adapter} - -The candidate local path operates on the most recent $L_\ell$ seconds. At local instant -$\tau\leq t$, a causal audio encoder produces $\vect a_\tau$ from mixed audio. -For the anonymous visible-person set -$\mathcal P_\tau=\{(\vect x_{\tau,i},\vect b_{\tau,i})\}_{i=1}^{N_\tau}$, -a shared visual encoder and masked set operator produce +Our architecture complements globally integrated Omni context with explicitly +accessible local floor evidence (Fig.~\ref{fig:method}). Let +$\vect g_t\in\mathbb R^{d_g}$ denote the causal Qwen2.5-Omni Thinker readout +at $t$ \cite{xu2025qwenomni}. In parallel, the frozen MuVAP ASD frontend +\cite{qi2026muvap} produces one shared mixed-audio CPC state $\vect u_t$ and a +variable-size set of visible-person states \begin{equation} - \vect v_\tau=\mathrm{SetPool} - \left(\{E_v(\vect x_{\tau,i},\vect b_{\tau,i})\}_{i=1}^{N_\tau} - \cup\{\vect v_\varnothing\}\right), - \label{eq:set} + \mathcal P_t=\{\vect p_{t,i}\}_{i=1}^{N_t},\qquad + \vect p_{t,i}=[\vect v_{t,i};s_{t,i}], + \label{eq:person_state} \end{equation} -where $\vect v_\varnothing$ handles an empty or offscreen visible set. The -operator is permutation invariant and uses no participant-ID embedding or -cross-time identity. A causal short-window encoder summarizes local dynamics, +where $\vect v_{t,i}$ is a causal visual state and $s_{t,i}$ is one +audio--visual speaking logit. The shared audio state is not duplicated in +each person token. The prediction model receives neither participant IDs nor +cross-time track identity; provided causal face boxes are used only to isolate +the representation and fusion question. + +A matched global--audio head first maps $[\vect g_t;\vect u_t]$ to baseline +logits $\vect Z_t^{\mathrm{base}}\in\mathbb R^{K\times3}$ and an intermediate +state $\vect h_t$. The same inputs query the current person set: \begin{equation} - \vect \ell_t=F_\ell - \left(\{[\vect a_\tau;\vect v_\tau]\}_{\tau\in(t-L_\ell,t]}\right). - \label{eq:local} + \begin{aligned} + \vect q_t &= W_g\vect g_t+W_a\vect u_t,\\ + \mathcal M_t &= \{W_p\vect p_{t,i}\}_{i=1}^{N_t} + \cup\{\vect p_\varnothing\},\\ + \vect c_t &= \mathrm{MHA}(\vect q_t,\mathcal M_t,\mathcal M_t), + \end{aligned} + \label{eq:person_attention} \end{equation} -The adapter preserves the global initialization through a zero-initialized -residual output, +where masking supports variable cardinality and the learned null token +$\vect p_\varnothing$ makes the attention well defined for an empty set. If +$m_t$ indicates that at least one valid person observation exists, the person +branch supplies only a residual logit correction, \begin{equation} - \vect z_t=\vect g_t+W_o\, - \sigma\!\left(W_g\vect g_t+W_\ell\vect\ell_t\right), - \qquad W_o\vert_{\mathrm{init}}=0, - \label{eq:fusion} + \vect Z_t=\vect Z_t^{\mathrm{base}} + {}+m_t\,\mathrm{reshape}\!\left(W_r[\vect h_t;\vect c_t;m_t]\right), + \qquad W_r\vert_{\mathrm{init}}=0, + \label{eq:residual} \end{equation} -and the common head maps $\vect z_t$ to Eq.~\eqref{eq:task}. This implements an -evidence-access hypothesis rather than adding an unconstrained second model: -the global path retains broad context, while the local path preserves timely -floor evidence. +and $\hat{\vect Y}_t=\mathrm{softmax}_c(\vect Z_t)$. Thus, after copying and +freezing the trained $G+U$ head, the complete model initially reproduces that +baseline exactly; only the person projection, query, masked attention, and +residual readout are trained in the adaptation stage. -\todo{Freeze $L_\ell$, local audio/video rates, crop source and resolution, -set pooling, causal temporal encoder, masks, reset rules, parameter count, and -latency. If the local path runs faster than the 5-Hz global/output grid, compare -it with a same-parameter 5-Hz variant before using the term ``dual-rate.'' Use -provided causal boxes only as a clearly labeled method-isolation condition; -online detection/tracking is not a paper contribution unless actually run.} - -\subsection{Training} - -The primary objective is class-balanced binary cross entropy over valid -horizons and activity channels, +We use the EgoSpeak/LSTR multi-positive cross-entropy over causal-valid rows, \begin{equation} - \mathcal L_{\mathrm{act}}= - \sum_{t,k,c}m_{t,k}w_c\, - \mathrm{BCE}(\hat y^c_{t,k},y^c_{t,k}). + \begin{aligned} + \mathcal L&=-\frac{1}{M}\sum_{t,k,c}m_{t,k}y^c_{t,k} + \log\mathrm{softmax}_c(\vect Z_{t,k}),\\ + M&=\sum_{t,k}m_{t,k}. + \end{aligned} \label{eq:loss} \end{equation} -No transcript, target-of-speech label, close microphone, future frame, or -per-person speech label enters the deployable model. \todo{Add an active-speaker -or event auxiliary loss only if it improves the frozen primary protocol and -its label/input boundary is fully reported.} +For an overlap target, both active channels contribute a log-probability term. +No transcript, target-of-speech annotation, close-microphone signal, speech +label, or future observation is supplied to the forecasting model. The +wearer forecast is interpreted only at evaluation time: it is an entry proxy +when the wearer is silent and its complement is a yield proxy when the wearer +is active, not a trained event hazard or committed controller action. + +\subsection{Causal Streaming Architecture} +\label{sec:causal_streaming} + +We distinguish encoder processing granularity from the common forecasting +rate. Predictions are emitted at $t_n=n\Delta$, $n\geq1$, while every +representation used at $t_n$ is restricted to media available by that physical +cutoff. + +The global path uses an 8-s rolling causal presentation budget with +source-aligned 2-s audiovisual blocks. Completed blocks are cached and +replayed, while the currently incomplete block is a temporary tail. At every +200-ms cutoff, the tail is rebuilt with all newly observed audio and video and +included in computing $\vect g_{t_n}$; when it reaches 2~s, it is committed to +the reusable history. Consequently, the Omni state is not held constant for +2~s: it includes the latest available content at every decision cutoff without +observing media after $t_n$. We retain the final four Thinker query-layer +states and fuse them in the matched head with a concatenation MLP. + +The local frontend operates causally at 25~Hz. State $j$ consumes the +half-open interval $[j/25,(j+1)/25)$ and is timestamped at its right edge. +Mixed audio is encoded once as a continuous shared CPC stream, while each +available face crop is processed by the causal visual and audio--visual +speaking modules. Ten-second extraction chunks may prepend up to 2~s of past +context, never future context. At a 5-Hz forecasting cutoff, we select the +local state whose right edge equals $t_n$; hence every fifth 25-Hz state aligns +with one Omni prediction row. The two paths therefore describe the same +causal prefix: the Omni path preserves integrated conversational context, +whereas the local path exposes fine-grained audio and visible-person floor +evidence. Fusion and prediction occur only on the common 5-Hz grid. \section{Experiments} @@ -301,8 +309,8 @@ matched PEFT/LoRA adaptation, global features with recent local audio, a parameter- and history-matched temporal local branch, and the proposed causal local audio--visual adapter. Qwen2.5-Omni-7B is used as a capacity-scaling control. All learned variants use identical causal examples, prediction -horizons, prediction heads, and the class-balanced binary cross-entropy -objective in Eq.~\eqref{eq:loss}. +horizons, prediction heads, and the multi-positive cross-entropy objective in +Eq.~\eqref{eq:loss}. The diagnostic streaming ablation additionally uses MiniCPM-o 4.5 \cite{cui2026minicpmo45}. Its native cached stream is evaluated at its native