wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add floor-event timing objective and addressee diagnostics

Browse Source
This commit is contained in:
Xin Wang
2026-09-11 00:15:34 +08:00
parent 5937971222
commit a57e827606
14 changed files with 2144 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

553
evaluate_floor_state_changes.py
View File

@@ -16,10 +16,11 @@ from __future__ import annotations
import argparse
import csv
from dataclasses import dataclass
import json
import math
from pathlib import Path
from typing import Any, Iterable
from typing import Any, Iterable, Literal
import numpy as np
@@ -39,6 +40,24 @@ NUM_CLASSES = 3
WEARER_INDEX = 1
OTHERS_INDEX = 2
DECISION_TIMINGS = ("pre-boundary", "post-boundary")
EventType = Literal["response", "yield"]
@dataclass(frozen=True)
class FloorEventCandidate:
"""One observable boundary-conditioned event example.
``source_row`` is the row whose model state is allowed to make the
decision. It is the last pre-transition row in the primary anticipation
protocol and the first post-transition row in the reactive diagnostic.
"""
source_row: int
label: bool
effective_steps: int
competing_row: int | None
short_resume_hard_negative: bool = False
def normalize_session_name(value: str | int) -> str:
@@ -125,6 +144,16 @@ def parse_args() -> argparse.Namespace:
"short-resume hard-negative subgroup (default: 2.0)."
),
)
parser.add_argument(
"--decision-timing",
choices=DECISION_TIMINGS,
default="pre-boundary",
help=(
"pre-boundary keeps the primary anticipation protocol; "
"post-boundary is a reactive diagnostic whose source has already "
"observed the aggregate-other boundary (default: pre-boundary)."
),
)
parser.add_argument(
"--print-framewise",
action="store_true",
@@ -523,6 +552,7 @@ def _competing_risk_window(
horizon_steps: int,
*,
competing_rising: bool,
first_competing_step: int = 2,
) -> tuple[int, int | None, bool] | None:
"""Return the observable half-open event window after ``source_row``.
@@ -550,10 +580,15 @@ def _competing_risk_window(
):
available_last_row -= 1
# Row source+1 contains the anchor's post-transition state. A competing
# transition can occur no earlier than source+2.
if first_competing_step not in {1, 2}:
raise ValueError("first_competing_step must be 1 or 2")
# In the primary pre-boundary protocol, row source+1 contains the anchor's
# post-transition state and a competing transition can occur no earlier
# than source+2. In the post-boundary diagnostic, source is already the
# post-transition row, so a competitor may occur at source+1.
competing_row: int | None = None
for row in range(source_row + 2, available_last_row + 1):
for row in range(source_row + first_competing_step, available_last_row + 1):
before = bool(competing_state[row - 1])
after = bool(competing_state[row])
is_competing = ((not before) and after) if competing_rising else (
@@ -565,14 +600,140 @@ def _competing_risk_window(
if competing_row is not None:
effective_steps = competing_row - source_row - 1
if effective_steps < 1:
raise AssertionError("Competing-risk window must contain an observable step")
return effective_steps, competing_row, True
nominal_complete = available_last_row >= nominal_last_row
return available_last_row - source_row, None, nominal_complete
def collect_floor_event_candidates(
target: np.ndarray,
session_ids: np.ndarray,
horizon_index: int,
*,
event_type: EventType,
decision_timing: str = "pre-boundary",
midturn_pause_max_steps: int = NUM_HORIZONS,
) -> tuple[list[FloorEventCandidate], dict[str, int]]:
"""Build label-only event examples without reading model predictions.
This is shared by the dense-activity evaluator and the direct event-head
diagnostic so their anchors, horizons, competing risks and censoring are
identical. The post-boundary mode excludes a wearer transition that is
simultaneous with the observed aggregate-other boundary: that outcome has
already happened before the reactive decision source.
"""
if decision_timing not in DECISION_TIMINGS:
raise ValueError(f"Unknown decision timing: {decision_timing}")
if event_type not in {"response", "yield"}:
raise ValueError(f"Unknown event type: {event_type}")
if horizon_index < 0:
raise ValueError("horizon_index must be non-negative")
if midturn_pause_max_steps <= 0:
raise ValueError("midturn_pause_max_steps must be positive")
if target.ndim != 2 or target.shape[1] != NUM_CLASSES:
raise ValueError(f"Expected target [T,{NUM_CLASSES}], got {target.shape}")
if session_ids.shape != (len(target),):
raise ValueError("session_ids must be [T]")
horizon_steps = horizon_index + 1
wearer = target[:, WEARER_INDEX].astype(bool)
others = target[:, OTHERS_INDEX].astype(bool)
if event_type == "response":
pre_sources = np.flatnonzero(others[:-1] & ~others[1:])
required_wearer_state = False
wearer_rising = True
competing_rising = True
else:
pre_sources = np.flatnonzero(~others[:-1] & others[1:])
required_wearer_state = True
wearer_rising = False
competing_rising = False
candidates: list[FloorEventCandidate] = []
source_boundaries = 0
right_censored = 0
simultaneous_boundary_outcomes_excluded = 0
zero_window_competing_risk = 0
for raw_pre_source in pre_sources:
pre_source = int(raw_pre_source)
boundary_row = pre_source + 1
if session_ids[pre_source] != session_ids[boundary_row]:
continue
if bool(wearer[pre_source]) != required_wearer_state:
continue
source_boundaries += 1
source_row = pre_source
first_competing_step = 2
if decision_timing == "post-boundary":
source_row = boundary_row
first_competing_step = 1
if bool(wearer[source_row]) != required_wearer_state:
simultaneous_boundary_outcomes_excluded += 1
continue
window = _competing_risk_window(
others,
session_ids,
source_row,
horizon_steps,
competing_rising=competing_rising,
first_competing_step=first_competing_step,
)
if window is None:
right_censored += 1
continue
effective_steps, competing_row, nominal_complete = window
if effective_steps == 0:
# The floor changed again in the first post-decision bin. There is
# no attributable target row before that competing boundary.
zero_window_competing_risk += 1
continue
outcome_row = _first_transition_row(
wearer,
source_row,
effective_steps,
rising=wearer_rising,
)
if outcome_row is None and competing_row is None and not nominal_complete:
right_censored += 1
continue
label = outcome_row is not None
simultaneous_competing_outcome = bool(
competing_row is not None
and bool(wearer[competing_row - 1]) == required_wearer_state
and bool(wearer[competing_row]) != required_wearer_state
)
short_resume = bool(
event_type == "response"
and not label
and competing_row is not None
and competing_row <= boundary_row + midturn_pause_max_steps
and not simultaneous_competing_outcome
)
candidates.append(
FloorEventCandidate(
source_row=source_row,
label=label,
effective_steps=effective_steps,
competing_row=competing_row,
short_resume_hard_negative=short_resume,
)
)
return candidates, {
"num_source_boundaries": source_boundaries,
"num_right_censored": right_censored,
"num_simultaneous_boundary_outcomes_excluded": (
simultaneous_boundary_outcomes_excluded
),
"num_zero_window_competing_risk": zero_window_competing_risk,
}
def _event_metric(
scores: list[float],
labels: list[bool],
@@ -611,6 +772,8 @@ def _event_window_diagnostics(
competing_censored: int,
right_censored: int,
invalid_score: int,
simultaneous_boundary_outcomes_excluded: int = 0,
zero_window_competing_risk: int = 0,
) -> dict[str, Any]:
steps = np.asarray(effective_steps, dtype=np.float64)
evaluated = int(len(steps))
@@ -623,6 +786,10 @@ def _event_window_diagnostics(
),
"num_right_censored": int(right_censored),
"num_invalid_score_window": int(invalid_score),
"num_simultaneous_boundary_outcomes_excluded": int(
simultaneous_boundary_outcomes_excluded
),
"num_zero_window_competing_risk": int(zero_window_competing_risk),
"mean_effective_window_seconds": (
float(steps.mean() * STEP_SECONDS) if evaluated else None
),
@@ -644,6 +811,7 @@ def evaluate_event_horizon(
recalls: list[float],
*,
midturn_pause_max_steps: int,
decision_timing: str = "pre-boundary",
operating_thresholds: dict[str, dict[float, float | None]] | None = None,
threshold_source: str = "evaluated_sessions",
) -> dict[str, Any]:
@@ -659,126 +827,78 @@ def evaluate_event_horizon(
"""
horizon_steps = horizon_index + 1
wearer = target[:, WEARER_INDEX].astype(bool)
others = target[:, OTHERS_INDEX].astype(bool)
response_candidates, response_diagnostics = collect_floor_event_candidates(
target,
session_ids,
horizon_index,
event_type="response",
decision_timing=decision_timing,
midturn_pause_max_steps=midturn_pause_max_steps,
)
yield_candidates, yield_diagnostics = collect_floor_event_candidates(
target,
session_ids,
horizon_index,
event_type="yield",
decision_timing=decision_timing,
midturn_pause_max_steps=midturn_pause_max_steps,
)
response_scores: list[float] = []
response_labels: list[bool] = []
short_resume_groups: list[bool] = []
response_effective_steps: list[int] = []
response_competing_censored = 0
response_right_censored = 0
response_invalid_score = 0
response_source_boundaries = 0
offset_sources = np.flatnonzero(others[:-1] & ~others[1:])
for source_row in offset_sources:
if session_ids[source_row] != session_ids[source_row + 1]:
continue
if wearer[source_row]:
continue
response_source_boundaries += 1
window = _competing_risk_window(
others,
session_ids,
int(source_row),
horizon_steps,
competing_rising=True,
)
if window is None:
response_right_censored += 1
continue
effective_steps, competing_row, nominal_complete = window
response_row = _first_transition_row(
wearer,
int(source_row),
def score_candidates(
candidates: list[FloorEventCandidate],
*,
offset: bool,
) -> tuple[list[float], list[bool], list[int], list[bool], int, int]:
scores: list[float] = []
labels: list[bool] = []
effective_steps: list[int] = []
short_resume: list[bool] = []
competing_censored = 0
invalid_score = 0
for candidate in candidates:
score = _same_source_path_score(
prediction,
valid,
session_ids,
candidate.source_row,
candidate.effective_steps,
offset=offset,
)
if score is None:
invalid_score += 1
continue
scores.append(score)
labels.append(candidate.label)
effective_steps.append(candidate.effective_steps)
short_resume.append(candidate.short_resume_hard_negative)
competing_censored += int(
candidate.competing_row is not None and not candidate.label
)
return (
scores,
labels,
effective_steps,
rising=True,
)
if response_row is None and competing_row is None and not nominal_complete:
response_right_censored += 1
continue
score = _same_source_path_score(
prediction,
valid,
session_ids,
int(source_row),
effective_steps,
offset=False,
)
if score is None:
response_invalid_score += 1
continue
response = response_row is not None
response_scores.append(score)
response_labels.append(response)
response_effective_steps.append(effective_steps)
response_competing_censored += int(
competing_row is not None and response_row is None
)
simultaneous_wearer_entry = bool(
competing_row is not None
and not wearer[competing_row - 1]
and wearer[competing_row]
)
short_resume_groups.append(
(not response)
and competing_row is not None
and competing_row <= source_row + 1 + midturn_pause_max_steps
and not simultaneous_wearer_entry
short_resume,
competing_censored,
invalid_score,
)
yield_scores: list[float] = []
yield_labels: list[bool] = []
yield_effective_steps: list[int] = []
yield_competing_censored = 0
yield_right_censored = 0
yield_invalid_score = 0
yield_source_boundaries = 0
onset_sources = np.flatnonzero(~others[:-1] & others[1:])
for source_row in onset_sources:
if session_ids[source_row] != session_ids[source_row + 1]:
continue
if not wearer[source_row]:
continue
yield_source_boundaries += 1
window = _competing_risk_window(
others,
session_ids,
int(source_row),
horizon_steps,
competing_rising=False,
)
if window is None:
yield_right_censored += 1
continue
effective_steps, competing_row, nominal_complete = window
yield_row = _first_transition_row(
wearer,
int(source_row),
effective_steps,
rising=False,
)
if yield_row is None and competing_row is None and not nominal_complete:
yield_right_censored += 1
continue
score = _same_source_path_score(
prediction,
valid,
session_ids,
int(source_row),
effective_steps,
offset=True,
)
if score is None:
yield_invalid_score += 1
continue
yielded = yield_row is not None
yield_scores.append(score)
yield_labels.append(yielded)
yield_effective_steps.append(effective_steps)
yield_competing_censored += int(
competing_row is not None and yield_row is None
)
(
response_scores,
response_labels,
response_effective_steps,
short_resume_groups,
response_competing_censored,
response_invalid_score,
) = score_candidates(response_candidates, offset=False)
(
yield_scores,
yield_labels,
yield_effective_steps,
_yield_short_resume,
yield_competing_censored,
yield_invalid_score,
) = score_candidates(yield_candidates, offset=True)
response_metric = _event_metric(
response_scores,
@@ -794,11 +914,17 @@ def evaluate_event_horizon(
)
response_metric.update(
_event_window_diagnostics(
source_boundaries=response_source_boundaries,
source_boundaries=response_diagnostics["num_source_boundaries"],
effective_steps=response_effective_steps,
competing_censored=response_competing_censored,
right_censored=response_right_censored,
right_censored=response_diagnostics["num_right_censored"],
invalid_score=response_invalid_score,
simultaneous_boundary_outcomes_excluded=response_diagnostics[
"num_simultaneous_boundary_outcomes_excluded"
],
zero_window_competing_risk=response_diagnostics[
"num_zero_window_competing_risk"
],
)
)
yield_metric = _event_metric(
@@ -812,17 +938,24 @@ def evaluate_event_horizon(
)
yield_metric.update(
_event_window_diagnostics(
source_boundaries=yield_source_boundaries,
source_boundaries=yield_diagnostics["num_source_boundaries"],
effective_steps=yield_effective_steps,
competing_censored=yield_competing_censored,
right_censored=yield_right_censored,
right_censored=yield_diagnostics["num_right_censored"],
invalid_score=yield_invalid_score,
simultaneous_boundary_outcomes_excluded=yield_diagnostics[
"num_simultaneous_boundary_outcomes_excluded"
],
zero_window_competing_risk=yield_diagnostics[
"num_zero_window_competing_risk"
],
)
)
return {
"horizon_index": horizon_steps,
"nominal_seconds": horizon_steps * STEP_SECONDS,
"decision_timing": decision_timing,
"other_offset_to_wearer_response_event": response_metric,
"other_onset_to_wearer_yield_event": yield_metric,
}
@@ -931,6 +1064,7 @@ def print_event_timeline_examples(
*,
examples_per_type: int,
horizon_steps: int = NUM_HORIZONS,
decision_timing: str = "pre-boundary",
) -> None:
"""Print causal event windows for manual attribution inspection."""
@@ -942,51 +1076,30 @@ def print_event_timeline_examples(
session_id: int(np.flatnonzero(session_ids == session_id)[0])
for session_id in np.unique(session_ids)
}
specifications = (
(
"response",
np.flatnonzero(others[:-1] & ~others[1:]),
False,
True,
True,
False,
),
(
"yield",
np.flatnonzero(~others[:-1] & others[1:]),
True,
False,
False,
True,
),
specifications: tuple[tuple[EventType, bool, bool], ...] = (
("response", True, False),
("yield", False, True),
)
print(
"\nCompeting-risk event timeline audit "
f"(first run, {horizon_steps * STEP_SECONDS:g}-s horizon, "
f"{decision_timing})"
)
print("\nCompeting-risk event timeline audit (first run, 2-s horizon)")
print("C=competing boundary excluded from the earlier event; E=wearer outcome")
for (
event_name,
sources,
required_wearer_state,
competing_rising,
wearer_rising,
offset_score,
) in specifications:
for event_name, wearer_rising, offset_score in specifications:
printed = 0
for source_value in sources:
source_row = int(source_value)
if session_ids[source_row] != session_ids[source_row + 1]:
continue
if bool(wearer[source_row]) != required_wearer_state:
continue
window = _competing_risk_window(
others,
session_ids,
source_row,
horizon_steps,
competing_rising=competing_rising,
)
if window is None:
continue
effective_steps, competing_row, nominal_complete = window
candidates, _ = collect_floor_event_candidates(
target,
session_ids,
horizon_steps - 1,
event_type=event_name,
decision_timing=decision_timing,
midturn_pause_max_steps=NUM_HORIZONS,
)
for candidate in candidates:
source_row = candidate.source_row
effective_steps = candidate.effective_steps
competing_row = candidate.competing_row
outcome_row = _first_transition_row(
wearer,
source_row,
@@ -997,37 +1110,44 @@ def print_event_timeline_examples(
status = "positive"
elif competing_row is not None:
status = "competing-censored negative"
elif nominal_complete:
status = "negative"
else:
status = "right-censored (excluded)"
event_score = (
None
if status.startswith("right-censored")
else _same_source_path_score(
prediction,
valid,
session_ids,
source_row,
effective_steps,
offset=offset_score,
)
status = "negative"
event_score = _same_source_path_score(
prediction,
valid,
session_ids,
source_row,
effective_steps,
offset=offset_score,
)
session_id = int(session_ids[source_row])
local_source = source_row - session_starts[session_id]
display_start = source_row
if (
decision_timing == "post-boundary"
and source_row > 0
and session_ids[source_row - 1] == session_ids[source_row]
):
display_start = source_row - 1
display_last = source_row + effective_steps
if competing_row is not None:
display_last = max(display_last, competing_row)
print(
f"\n{event_name} | {session_names[session_id]} | "
f"source={local_source} ({local_source * STEP_SECONDS:.1f}s) | "
f"source={local_source} "
f"({(local_source + 1) * STEP_SECONDS:.1f}s cutoff) | "
f"status={status} | score={event_score}"
)
print("local_row | time_s | wearer | others | fixed-source score | mark")
for row in range(source_row, display_last + 1):
print(
"local_row | cutoff_s | wearer | others | "
"fixed-source score | mark"
)
for row in range(display_start, display_last + 1):
local_row = row - session_starts[session_id]
step = row - source_row
if step == 0:
if step < 0:
score_text = "pre-boundary"
elif step == 0:
score_text = "source"
elif row == competing_row:
score_text = "excluded"
@@ -1038,13 +1158,19 @@ def print_event_timeline_examples(
score_text = "invalid"
marks: list[str] = []
if row == source_row:
marks.append("anchor")
marks.append(
"decision/observed-boundary"
if decision_timing == "post-boundary"
else "anchor"
)
elif row == source_row - 1 and decision_timing == "post-boundary":
marks.append("pre-boundary")
if row == outcome_row:
marks.append("E")
if row == competing_row:
marks.append("C/excluded")
print(
f"{local_row:9d} | {local_row * STEP_SECONDS:6.1f} | "
f"{local_row:9d} | {(local_row + 1) * STEP_SECONDS:6.1f} | "
f"{int(wearer[row]):6d} | {int(others[row]):6d} | "
f"{score_text:18s} | {','.join(marks)}"
)
@@ -1059,12 +1185,19 @@ def print_results(payload: dict[str, Any], *, print_framewise: bool) -> None:
report_recall = min(
payload["matched_recalls"], key=lambda value: abs(value - 0.8)
)
print("\nWearer floor-event evaluation (primary event-level protocol)")
timing = payload.get("decision_timing", "pre-boundary")
label = (
"primary anticipation protocol"
if timing == "pre-boundary"
else "post-boundary reactive diagnostic"
)
print(f"\nWearer floor-event evaluation ({label})")
print(
"Each aggregate-other boundary is counted once. Response windows stop "
"before the next other onset; yield windows stop before the next other "
"offset. The competing boundary itself is excluded. AP/rates are "
"percentages; events remain timing proxies, not semantic EOT/INT."
"percentages; events remain timing proxies, not semantic EOT/INT. "
f"Decision timing: {timing}."
)
if payload["calibration_sessions"]:
print(
@@ -1231,6 +1364,7 @@ def write_event_csv(path: Path, payload: dict[str, Any]) -> None:
with path.open("w", newline="") as file:
fieldnames = [
"run",
"decision_timing",
"horizon_index",
"nominal_seconds",
"response_event_ap",
@@ -1241,6 +1375,8 @@ def write_event_csv(path: Path, payload: dict[str, Any]) -> None:
"response_competing_censored_count",
"response_right_censored_count",
"response_invalid_score_window_count",
"response_simultaneous_boundary_outcome_excluded_count",
"response_zero_window_competing_risk_count",
"response_mean_effective_window_seconds",
"yield_event_ap",
"yield_count",
@@ -1249,6 +1385,8 @@ def write_event_csv(path: Path, payload: dict[str, Any]) -> None:
"yield_competing_censored_count",
"yield_right_censored_count",
"yield_invalid_score_window_count",
"yield_simultaneous_boundary_outcome_excluded_count",
"yield_zero_window_competing_risk_count",
"yield_mean_effective_window_seconds",
]
for recall in payload["matched_recalls"]:
@@ -1274,6 +1412,9 @@ def write_event_csv(path: Path, payload: dict[str, Any]) -> None:
yield_metric = item["other_onset_to_wearer_yield_event"]
row: dict[str, Any] = {
"run": name,
"decision_timing": payload.get(
"decision_timing", "pre-boundary"
),
"horizon_index": item["horizon_index"],
"nominal_seconds": item["nominal_seconds"],
"response_event_ap": response["average_precision"],
@@ -1294,6 +1435,12 @@ def write_event_csv(path: Path, payload: dict[str, Any]) -> None:
"response_invalid_score_window_count": response[
"num_invalid_score_window"
],
"response_simultaneous_boundary_outcome_excluded_count": response[
"num_simultaneous_boundary_outcomes_excluded"
],
"response_zero_window_competing_risk_count": response[
"num_zero_window_competing_risk"
],
"response_mean_effective_window_seconds": response[
"mean_effective_window_seconds"
],
@@ -1312,6 +1459,12 @@ def write_event_csv(path: Path, payload: dict[str, Any]) -> None:
"yield_invalid_score_window_count": yield_metric[
"num_invalid_score_window"
],
"yield_simultaneous_boundary_outcome_excluded_count": yield_metric[
"num_simultaneous_boundary_outcomes_excluded"
],
"yield_zero_window_competing_risk_count": yield_metric[
"num_zero_window_competing_risk"
],
"yield_mean_effective_window_seconds": yield_metric[
"mean_effective_window_seconds"
],
@@ -1423,7 +1576,12 @@ def main() -> None:
)
payload: dict[str, Any] = {
"protocol": "wearer_floor_event_competing_risk_v2",
"protocol": (
"wearer_floor_event_competing_risk_v2"
if args.decision_timing == "pre-boundary"
else "wearer_floor_event_post_boundary_diagnostic_v1"
),
"decision_timing": args.decision_timing,
"sessions": sessions,
"calibration_sessions": calibration_sessions,
"classes": list(CLASS_NAMES),
@@ -1442,10 +1600,12 @@ def main() -> None:
"event_protocol": (
"Each aggregate-others 1->0 boundary with wearer initially off is one "
"response candidate; each aggregate-others 0->1 boundary with wearer "
"initially on is one yield candidate. A response window is half-open and "
"ends before the next aggregate-others onset; a yield window ends before "
"the next aggregate-others offset. A simultaneous wearer change at that "
"competing boundary is excluded from the earlier event. Labels are "
"initially on is one yield candidate. The decision source is the last "
"pre-boundary row for pre-boundary anticipation and the first post-boundary "
"row for the reactive diagnostic. A wearer change simultaneous with the "
"anchor is excluded from post-boundary evaluation because it precedes that "
"decision. A response window ends before the next aggregate-others onset; "
"a yield window ends before the next aggregate-others offset. Labels are "
"observed wearer transitions within the remaining window. Fixed-source "
"future activity scores are max-pooled over the same remaining window "
"only as ranking scores. Candidates without enough follow-up and without "
@@ -1467,6 +1627,7 @@ def main() -> None:
session_ids,
sessions,
examples_per_type=args.print_event_timelines,
decision_timing=args.decision_timing,
)
printed_timelines = True
calibration_thresholds: list[dict[str, Any]] | None = None
@@ -1517,6 +1678,7 @@ def main() -> None:
horizon_index,
recalls,
midturn_pause_max_steps=midturn_pause_max_steps,
decision_timing=args.decision_timing,
)
thresholds = _event_operating_thresholds(
calibration_event, recalls
@@ -1570,6 +1732,7 @@ def main() -> None:
horizon_index,
recalls,
midturn_pause_max_steps=midturn_pause_max_steps,
decision_timing=args.decision_timing,
operating_thresholds=fixed_event_thresholds,
threshold_source=threshold_source,
)