Add competing-risk evaluation and align ICASSP story
This commit is contained in:
Xin Wang
@@ -1,8 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import numpy as np
|
||||
|
||||
@@ -12,7 +17,9 @@ from evaluate_floor_state_changes import (
|
||||
evaluate_event_horizon,
|
||||
evaluate_horizon,
|
||||
load_arrays,
|
||||
main,
|
||||
parse_runs,
|
||||
print_event_timeline_examples,
|
||||
validate_recalls,
|
||||
)
|
||||
|
||||
@@ -22,6 +29,385 @@ def activity(wearer: int, others: int) -> list[int]:
|
||||
|
||||
|
||||
class FloorStateMetricTests(unittest.TestCase):
|
||||
def test_response_window_stops_before_next_other_onset(self) -> None:
|
||||
target = np.asarray(
|
||||
[
|
||||
activity(0, 1), # session 0: other offset anchor
|
||||
activity(0, 0),
|
||||
activity(0, 1), # competing other onset; window ends before here
|
||||
activity(1, 1), # must not count as response to row-1 offset
|
||||
activity(0, 1), # session 1: other offset anchor
|
||||
activity(0, 0),
|
||||
activity(1, 0), # valid response before any competing onset
|
||||
activity(1, 1), # later competitor; response has already won
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
prediction = np.full(
|
||||
(len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32
|
||||
)
|
||||
prediction[1, 1, 0] = 0.1
|
||||
prediction[2, 1, 1] = 0.2
|
||||
prediction[3, 1, 2] = 0.99 # excluded by competing-risk censoring
|
||||
prediction[5, 1, 0] = 0.2
|
||||
prediction[6, 1, 1] = 0.9
|
||||
prediction[7, 1, 2] = 0.9
|
||||
valid = np.ones((len(target), NUM_HORIZONS), dtype=bool)
|
||||
session_ids = np.asarray([0, 0, 0, 0, 1, 1, 1, 1], dtype=np.int32)
|
||||
|
||||
result = evaluate_event_horizon(
|
||||
target,
|
||||
prediction,
|
||||
valid,
|
||||
session_ids,
|
||||
2,
|
||||
[0.8],
|
||||
midturn_pause_max_steps=10,
|
||||
)
|
||||
response = result["other_offset_to_wearer_response_event"]
|
||||
self.assertEqual(response["num_positive"], 1)
|
||||
self.assertEqual(response["num_negative"], 1)
|
||||
self.assertEqual(response["num_competing_risk_censored"], 1)
|
||||
self.assertEqual(response["num_short_resume_hard_negative"], 1)
|
||||
self.assertAlmostEqual(response["average_precision"], 1.0)
|
||||
self.assertAlmostEqual(response["min_effective_window_seconds"], 0.2)
|
||||
self.assertAlmostEqual(response["max_effective_window_seconds"], 0.4)
|
||||
|
||||
def test_yield_window_stops_before_next_other_offset(self) -> None:
|
||||
target = np.asarray(
|
||||
[
|
||||
activity(1, 0), # session 0: other onset anchor
|
||||
activity(1, 1),
|
||||
activity(1, 0), # competing other offset; window ends before here
|
||||
activity(0, 0), # must not count as yield to row-1 onset
|
||||
activity(1, 0), # session 1: other onset anchor
|
||||
activity(1, 1),
|
||||
activity(0, 1), # valid yield while other remains active
|
||||
activity(0, 0), # later competitor; yield has already won
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
prediction = np.full(
|
||||
(len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32
|
||||
)
|
||||
prediction[1, 1, 0] = 0.9
|
||||
prediction[2, 1, 1] = 0.2
|
||||
prediction[3, 1, 2] = 0.01 # excluded by competing-risk censoring
|
||||
prediction[5, 1, 0] = 0.8
|
||||
prediction[6, 1, 1] = 0.1
|
||||
prediction[7, 1, 2] = 0.1
|
||||
valid = np.ones((len(target), NUM_HORIZONS), dtype=bool)
|
||||
session_ids = np.asarray([0, 0, 0, 0, 1, 1, 1, 1], dtype=np.int32)
|
||||
|
||||
result = evaluate_event_horizon(
|
||||
target,
|
||||
prediction,
|
||||
valid,
|
||||
session_ids,
|
||||
2,
|
||||
[0.8],
|
||||
midturn_pause_max_steps=10,
|
||||
)
|
||||
yield_metric = result["other_onset_to_wearer_yield_event"]
|
||||
self.assertEqual(yield_metric["num_positive"], 1)
|
||||
self.assertEqual(yield_metric["num_negative"], 1)
|
||||
self.assertEqual(yield_metric["num_competing_risk_censored"], 1)
|
||||
self.assertAlmostEqual(yield_metric["average_precision"], 1.0)
|
||||
point = yield_metric["operating_points"][0]
|
||||
self.assertEqual(point["negative_false_positive_count"], 0)
|
||||
|
||||
def test_observed_response_before_recording_end_is_not_right_censored(self) -> None:
|
||||
target = np.asarray(
|
||||
[
|
||||
activity(0, 1),
|
||||
activity(0, 0),
|
||||
activity(1, 0), # observed response, but nominal 0.6-s end is absent
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
prediction = np.full(
|
||||
(len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32
|
||||
)
|
||||
valid = np.ones((len(target), NUM_HORIZONS), dtype=bool)
|
||||
session_ids = np.zeros(len(target), dtype=np.int32)
|
||||
result = evaluate_event_horizon(
|
||||
target,
|
||||
prediction,
|
||||
valid,
|
||||
session_ids,
|
||||
2,
|
||||
[0.8],
|
||||
midturn_pause_max_steps=10,
|
||||
)
|
||||
response = result["other_offset_to_wearer_response_event"]
|
||||
self.assertEqual(response["num_positive"], 1)
|
||||
self.assertEqual(response["num_right_censored"], 0)
|
||||
|
||||
def test_unresolved_event_at_recording_end_is_right_censored(self) -> None:
|
||||
target = np.asarray(
|
||||
[
|
||||
activity(0, 1),
|
||||
activity(0, 0),
|
||||
activity(0, 0),
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
prediction = np.full(
|
||||
(len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32
|
||||
)
|
||||
valid = np.ones((len(target), NUM_HORIZONS), dtype=bool)
|
||||
session_ids = np.zeros(len(target), dtype=np.int32)
|
||||
result = evaluate_event_horizon(
|
||||
target,
|
||||
prediction,
|
||||
valid,
|
||||
session_ids,
|
||||
2,
|
||||
[0.8],
|
||||
midturn_pause_max_steps=10,
|
||||
)
|
||||
response = result["other_offset_to_wearer_response_event"]
|
||||
self.assertEqual(response["num_evaluated"], 0)
|
||||
self.assertEqual(response["num_right_censored"], 1)
|
||||
|
||||
def test_simultaneous_wearer_entry_at_competing_onset_is_excluded(self) -> None:
|
||||
target = np.asarray(
|
||||
[
|
||||
activity(0, 1),
|
||||
activity(0, 0),
|
||||
activity(1, 1), # wearer and others restart simultaneously
|
||||
activity(1, 1),
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
prediction = np.full(
|
||||
(len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32
|
||||
)
|
||||
valid = np.ones((len(target), NUM_HORIZONS), dtype=bool)
|
||||
session_ids = np.zeros(len(target), dtype=np.int32)
|
||||
result = evaluate_event_horizon(
|
||||
target,
|
||||
prediction,
|
||||
valid,
|
||||
session_ids,
|
||||
2,
|
||||
[0.8],
|
||||
midturn_pause_max_steps=10,
|
||||
)
|
||||
response = result["other_offset_to_wearer_response_event"]
|
||||
self.assertEqual(response["num_positive"], 0)
|
||||
self.assertEqual(response["num_negative"], 1)
|
||||
self.assertEqual(response["num_competing_risk_censored"], 1)
|
||||
self.assertEqual(response["num_short_resume_hard_negative"], 0)
|
||||
|
||||
def test_timeline_marks_competing_boundary_as_excluded(self) -> None:
|
||||
target = np.asarray(
|
||||
[
|
||||
activity(0, 1),
|
||||
activity(0, 0),
|
||||
activity(0, 1),
|
||||
activity(1, 1),
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
prediction = np.full(
|
||||
(len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32
|
||||
)
|
||||
valid = np.ones((len(target), NUM_HORIZONS), dtype=bool)
|
||||
session_ids = np.zeros(len(target), dtype=np.int32)
|
||||
stream = io.StringIO()
|
||||
with redirect_stdout(stream):
|
||||
print_event_timeline_examples(
|
||||
target,
|
||||
prediction,
|
||||
valid,
|
||||
session_ids,
|
||||
["Session_1"],
|
||||
examples_per_type=1,
|
||||
horizon_steps=3,
|
||||
)
|
||||
output = stream.getvalue()
|
||||
self.assertIn("status=competing-censored negative", output)
|
||||
self.assertIn("C/excluded", output)
|
||||
|
||||
def test_fixed_calibration_threshold_is_not_reselected(self) -> None:
|
||||
target = np.asarray(
|
||||
[
|
||||
activity(0, 1),
|
||||
activity(0, 0),
|
||||
activity(1, 0),
|
||||
activity(1, 0),
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
prediction = np.full(
|
||||
(len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32
|
||||
)
|
||||
valid = np.ones((len(target), NUM_HORIZONS), dtype=bool)
|
||||
session_ids = np.zeros(len(target), dtype=np.int32)
|
||||
result = evaluate_event_horizon(
|
||||
target,
|
||||
prediction,
|
||||
valid,
|
||||
session_ids,
|
||||
1,
|
||||
[0.8],
|
||||
midturn_pause_max_steps=10,
|
||||
operating_thresholds={
|
||||
"response": {0.8: 0.75},
|
||||
"yield": {0.8: 0.25},
|
||||
},
|
||||
threshold_source="calibration_sessions:Session_12",
|
||||
)
|
||||
point = result["other_offset_to_wearer_response_event"][
|
||||
"operating_points"
|
||||
][0]
|
||||
self.assertEqual(point["threshold"], 0.75)
|
||||
self.assertEqual(
|
||||
point["threshold_source"], "calibration_sessions:Session_12"
|
||||
)
|
||||
self.assertEqual(point["achieved_positive_recall"], 0.0)
|
||||
|
||||
def test_event_score_uses_full_observable_window(self) -> None:
|
||||
target = np.asarray(
|
||||
[
|
||||
activity(0, 1),
|
||||
activity(0, 0),
|
||||
activity(1, 0), # response occurs here
|
||||
activity(1, 0),
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
prediction = np.full(
|
||||
(len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32
|
||||
)
|
||||
prediction[1, 1, 0] = 0.1
|
||||
prediction[2, 1, 1] = 0.2
|
||||
prediction[3, 1, 2] = 0.9 # still part of the by-horizon event score
|
||||
valid = np.ones((len(target), NUM_HORIZONS), dtype=bool)
|
||||
session_ids = np.zeros(len(target), dtype=np.int32)
|
||||
result = evaluate_event_horizon(
|
||||
target,
|
||||
prediction,
|
||||
valid,
|
||||
session_ids,
|
||||
2,
|
||||
[0.8],
|
||||
midturn_pause_max_steps=10,
|
||||
)
|
||||
response = result["other_offset_to_wearer_response_event"]
|
||||
self.assertAlmostEqual(response["max_effective_window_seconds"], 0.6)
|
||||
self.assertAlmostEqual(response["operating_points"][0]["threshold"], 0.9)
|
||||
|
||||
def test_main_calibrates_on_disjoint_training_session(self) -> None:
|
||||
target = np.asarray(
|
||||
[
|
||||
activity(0, 1),
|
||||
activity(0, 0), # response anchor
|
||||
activity(1, 0), # response
|
||||
activity(1, 0),
|
||||
activity(1, 1), # yield anchor
|
||||
activity(0, 1), # yield
|
||||
activity(0, 1),
|
||||
activity(0, 0), # response anchor
|
||||
activity(0, 0),
|
||||
activity(0, 1), # competing onset, no response
|
||||
activity(1, 0), # response anchor and response
|
||||
activity(1, 0),
|
||||
activity(1, 1), # yield anchor
|
||||
activity(1, 1),
|
||||
activity(1, 0), # competing offset, no yield
|
||||
]
|
||||
+ [activity(0, 1)] * 15,
|
||||
dtype=np.float32,
|
||||
)
|
||||
prediction = np.full(
|
||||
(len(target), NUM_CLASSES, NUM_HORIZONS), 0.5, dtype=np.float32
|
||||
)
|
||||
valid = np.ones((len(target), NUM_HORIZONS), dtype=bool)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
target_root = root / "target"
|
||||
run_root = root / "run"
|
||||
(run_root / "pred").mkdir(parents=True)
|
||||
(run_root / "valid").mkdir(parents=True)
|
||||
target_root.mkdir()
|
||||
for session in ("Session_1", "Session_12"):
|
||||
np.save(target_root / f"{session}.npy", target)
|
||||
np.save(run_root / "pred" / f"{session}.npy", prediction)
|
||||
np.save(run_root / "valid" / f"{session}.npy", valid)
|
||||
|
||||
output = root / "report.json"
|
||||
argv = [
|
||||
"evaluate_floor_state_changes.py",
|
||||
"--target-root",
|
||||
str(target_root),
|
||||
"--run",
|
||||
f"model={run_root}",
|
||||
"--sessions",
|
||||
"Session_1",
|
||||
"--calibration-sessions",
|
||||
"Session_12",
|
||||
"--matched-recalls",
|
||||
"0.8",
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
with mock.patch.object(sys, "argv", argv), redirect_stdout(io.StringIO()):
|
||||
main()
|
||||
|
||||
payload = json.loads(output.read_text())
|
||||
self.assertEqual(
|
||||
payload["protocol"], "wearer_floor_event_competing_risk_v2"
|
||||
)
|
||||
self.assertEqual(payload["calibration_sessions"], ["Session_12"])
|
||||
point = payload["runs"]["model"]["event_horizons"][1][
|
||||
"other_offset_to_wearer_response_event"
|
||||
]["operating_points"][0]
|
||||
self.assertEqual(
|
||||
point["threshold_source"],
|
||||
"calibration_sessions:Session_12",
|
||||
)
|
||||
framewise_point = payload["runs"]["model"]["horizons"][1][
|
||||
"eot_response_vs_other_hold_proxy"
|
||||
]["operating_points"][0]
|
||||
self.assertEqual(
|
||||
framewise_point["threshold_source"],
|
||||
"calibration_sessions:Session_12",
|
||||
)
|
||||
self.assertTrue(output.with_name("report_events.csv").is_file())
|
||||
|
||||
def test_main_refuses_official_test_without_calibration(self) -> None:
|
||||
argv = [
|
||||
"evaluate_floor_state_changes.py",
|
||||
"--target-root",
|
||||
"unused",
|
||||
"--run",
|
||||
"model=unused",
|
||||
"--sessions",
|
||||
"Session_1",
|
||||
]
|
||||
with mock.patch.object(sys, "argv", argv):
|
||||
with self.assertRaisesRegex(ValueError, "requires --calibration-sessions"):
|
||||
main()
|
||||
|
||||
def test_main_refuses_test_session_as_official_calibration(self) -> None:
|
||||
argv = [
|
||||
"evaluate_floor_state_changes.py",
|
||||
"--target-root",
|
||||
"unused",
|
||||
"--run",
|
||||
"model=unused",
|
||||
"--sessions",
|
||||
"Session_1",
|
||||
"--calibration-sessions",
|
||||
"Session_2",
|
||||
]
|
||||
with mock.patch.object(sys, "argv", argv):
|
||||
with self.assertRaisesRegex(ValueError, "Sessions 4--12"):
|
||||
main()
|
||||
|
||||
def test_event_level_counts_each_boundary_once(self) -> None:
|
||||
target = np.asarray(
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user