Add manuscript-aligned EasyCom EOT INT review workflow
This commit is contained in:
Xin Wang
571
prepare_easycom_eot_int_review.py
Normal file
571
prepare_easycom_eot_int_review.py
Normal file
@@ -0,0 +1,571 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare manuscript-aligned EasyCom EOT/INT regions for human review.
|
||||
|
||||
The released EasyCom annotations provide participant-resolved speech intervals,
|
||||
not semantic EOT/INT ground truth. This script therefore creates auditable
|
||||
candidate regions from the original 20 Hz timeline. It keeps the two wearer-
|
||||
relative directions distinct:
|
||||
|
||||
* EOT-response: an aggregate non-wearer offset while the wearer is silent;
|
||||
* INT-yield: an aggregate non-wearer onset while the wearer is speaking.
|
||||
|
||||
Observed wearer transitions form timing-positive candidates. Competing other
|
||||
transitions and fully observed no-transition windows form hard-negative
|
||||
candidates. Source-boundary cases are retained as exclusions. Every output
|
||||
row remains an automatic candidate until it is independently human reviewed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import asdict, dataclass
|
||||
import json
|
||||
from pathlib import Path
|
||||
import random
|
||||
from typing import Any, Iterable, Sequence
|
||||
|
||||
from derive_easycom_interaction_events import (
|
||||
CAMERA_WEARER_ID,
|
||||
SOURCE_FPS,
|
||||
Clip,
|
||||
Rules,
|
||||
Utterance,
|
||||
build_clips,
|
||||
is_lexical_feedback,
|
||||
is_pure_nonverbal,
|
||||
load_utterances,
|
||||
)
|
||||
from easycom_manifest import ALL_SESSIONS, TEST_SESSIONS
|
||||
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
PROTOCOL_VERSION = "easycom-manuscript-eot-int-v1"
|
||||
DEFAULT_WINDOW_SECONDS = 2.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProtocolCandidate:
|
||||
schema_version: int
|
||||
protocol_version: str
|
||||
uid: str
|
||||
session: str
|
||||
split: str
|
||||
task: str
|
||||
event_type: str
|
||||
automatic_protocol_label: str
|
||||
automatic_hard_negative_type: str
|
||||
automatic_confidence: str
|
||||
anchor_type: str
|
||||
anchor_dense_frame: int
|
||||
anchor_dense_seconds: float
|
||||
behavior_anchor_type: str | None
|
||||
behavior_dense_frame: int | None
|
||||
behavior_seconds: float | None
|
||||
region_start_seconds: float
|
||||
region_end_seconds: float
|
||||
region_end_reason: str
|
||||
previous_other_participant_ids: tuple[int, ...]
|
||||
incoming_other_participant_ids: tuple[int, ...]
|
||||
clip_stem: str
|
||||
source_start_frame: int
|
||||
context: tuple[dict[str, Any], ...]
|
||||
automatic_reason: str
|
||||
|
||||
|
||||
def _fill_state(
|
||||
rows: Sequence[Utterance],
|
||||
*,
|
||||
segment_start: int,
|
||||
segment_end: int,
|
||||
wearer_id: int,
|
||||
) -> tuple[bytearray, bytearray]:
|
||||
length = segment_end - segment_start
|
||||
wearer = bytearray(length)
|
||||
others = bytearray(length)
|
||||
for row in rows:
|
||||
left = max(segment_start, row.dense_start_frame) - segment_start
|
||||
right = min(segment_end, row.dense_end_frame_exclusive) - segment_start
|
||||
if right <= left:
|
||||
continue
|
||||
target = wearer if row.participant_id == wearer_id else others
|
||||
target[left:right] = b"\x01" * (right - left)
|
||||
return wearer, others
|
||||
|
||||
|
||||
def _first_transition(
|
||||
state: bytearray,
|
||||
*,
|
||||
start: int,
|
||||
stop: int,
|
||||
rising: bool,
|
||||
) -> int | None:
|
||||
for index in range(max(1, start), min(stop, len(state))):
|
||||
before = bool(state[index - 1])
|
||||
after = bool(state[index])
|
||||
if (not before and after) if rising else (before and not after):
|
||||
return index
|
||||
return None
|
||||
|
||||
|
||||
def _active_other_ids(
|
||||
rows: Sequence[Utterance],
|
||||
dense_frame: int,
|
||||
*,
|
||||
wearer_id: int,
|
||||
) -> tuple[int, ...]:
|
||||
return tuple(sorted({
|
||||
row.participant_id
|
||||
for row in rows
|
||||
if row.participant_id != wearer_id
|
||||
and row.dense_start_frame <= dense_frame < row.dense_end_frame_exclusive
|
||||
}))
|
||||
|
||||
|
||||
def _clip_for_frame(clips: Sequence[Clip], dense_frame: int) -> Clip | None:
|
||||
return next(
|
||||
(
|
||||
clip
|
||||
for clip in clips
|
||||
if clip.dense_start_frame <= dense_frame < clip.dense_start_frame + clip.num_frames
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _context_rows(
|
||||
rows: Sequence[Utterance],
|
||||
*,
|
||||
anchor_frame: int,
|
||||
context_frames: int,
|
||||
wearer_id: int,
|
||||
) -> tuple[dict[str, Any], ...]:
|
||||
left = anchor_frame - context_frames
|
||||
right = anchor_frame + context_frames
|
||||
selected = []
|
||||
for row in rows:
|
||||
if row.dense_end_frame_exclusive <= left or row.dense_start_frame >= right:
|
||||
continue
|
||||
selected.append(
|
||||
{
|
||||
"uid": row.uid,
|
||||
"participant_id": row.participant_id,
|
||||
"participant_role": "wearer" if row.participant_id == wearer_id else "other",
|
||||
"start_seconds": row.dense_start_seconds,
|
||||
"end_seconds": row.dense_end_seconds,
|
||||
"transcription": row.transcription,
|
||||
"target_of_speech": list(row.target_of_speech),
|
||||
}
|
||||
)
|
||||
return tuple(selected)
|
||||
|
||||
|
||||
def _incoming_is_backchannel(
|
||||
rows: Sequence[Utterance],
|
||||
*,
|
||||
dense_frame: int,
|
||||
wearer_id: int,
|
||||
max_seconds: float,
|
||||
) -> bool:
|
||||
incoming = [
|
||||
row
|
||||
for row in rows
|
||||
if row.participant_id != wearer_id
|
||||
and row.dense_start_frame <= dense_frame < row.dense_end_frame_exclusive
|
||||
and row.dense_start_frame >= dense_frame - 1
|
||||
]
|
||||
return bool(incoming) and all(row.duration_seconds <= max_seconds for row in incoming) and any(
|
||||
is_lexical_feedback(row.transcription) or is_pure_nonverbal(row.transcription)
|
||||
for row in incoming
|
||||
)
|
||||
|
||||
|
||||
def _candidate(
|
||||
*,
|
||||
session: str,
|
||||
task: str,
|
||||
automatic_protocol_label: str,
|
||||
automatic_hard_negative_type: str,
|
||||
anchor_frame: int,
|
||||
behavior_frame: int | None,
|
||||
region_end_frame: int,
|
||||
region_end_reason: str,
|
||||
previous_ids: tuple[int, ...],
|
||||
incoming_ids: tuple[int, ...],
|
||||
clips: Sequence[Clip],
|
||||
rows: Sequence[Utterance],
|
||||
wearer_id: int,
|
||||
context_frames: int,
|
||||
reason: str,
|
||||
) -> ProtocolCandidate:
|
||||
clip = _clip_for_frame(clips, anchor_frame)
|
||||
if clip is None:
|
||||
raise ValueError(f"No source clip for {session} dense frame {anchor_frame}")
|
||||
event_type = "eot_response_candidate" if task == "eot_response" else "int_yield_candidate"
|
||||
anchor_type = "other_offset" if task == "eot_response" else "other_onset"
|
||||
behavior_type = None
|
||||
if behavior_frame is not None:
|
||||
behavior_type = "wearer_onset" if task == "eot_response" else "wearer_offset"
|
||||
return ProtocolCandidate(
|
||||
schema_version=SCHEMA_VERSION,
|
||||
protocol_version=PROTOCOL_VERSION,
|
||||
uid=f"{session}/{task}/{anchor_frame:08d}",
|
||||
session=session,
|
||||
split="test" if session in TEST_SESSIONS else "train",
|
||||
task=task,
|
||||
event_type=event_type,
|
||||
automatic_protocol_label=automatic_protocol_label,
|
||||
automatic_hard_negative_type=automatic_hard_negative_type,
|
||||
automatic_confidence="timing_candidate",
|
||||
anchor_type=anchor_type,
|
||||
anchor_dense_frame=anchor_frame,
|
||||
anchor_dense_seconds=anchor_frame / SOURCE_FPS,
|
||||
behavior_anchor_type=behavior_type,
|
||||
behavior_dense_frame=behavior_frame,
|
||||
behavior_seconds=None if behavior_frame is None else behavior_frame / SOURCE_FPS,
|
||||
region_start_seconds=anchor_frame / SOURCE_FPS,
|
||||
region_end_seconds=region_end_frame / SOURCE_FPS,
|
||||
region_end_reason=region_end_reason,
|
||||
previous_other_participant_ids=previous_ids,
|
||||
incoming_other_participant_ids=incoming_ids,
|
||||
clip_stem=clip.stem,
|
||||
source_start_frame=anchor_frame - clip.dense_start_frame + 1,
|
||||
context=_context_rows(
|
||||
rows,
|
||||
anchor_frame=anchor_frame,
|
||||
context_frames=context_frames,
|
||||
wearer_id=wearer_id,
|
||||
),
|
||||
automatic_reason=reason,
|
||||
)
|
||||
|
||||
|
||||
def derive_protocol_candidates(
|
||||
utterances: Sequence[Utterance],
|
||||
clips: Sequence[Clip],
|
||||
*,
|
||||
session: str,
|
||||
wearer_id: int = CAMERA_WEARER_ID,
|
||||
window_seconds: float = DEFAULT_WINDOW_SECONDS,
|
||||
context_seconds: float = 3.0,
|
||||
) -> list[ProtocolCandidate]:
|
||||
"""Construct timing candidates with manuscript-aligned directionality."""
|
||||
|
||||
if window_seconds <= 0 or context_seconds < 0:
|
||||
raise ValueError("window_seconds must be positive and context_seconds non-negative")
|
||||
window_frames = max(1, round(window_seconds * SOURCE_FPS))
|
||||
context_frames = max(0, round(context_seconds * SOURCE_FPS))
|
||||
rows_by_segment: dict[int, list[Utterance]] = defaultdict(list)
|
||||
clips_by_segment: dict[int, list[Clip]] = defaultdict(list)
|
||||
for row in utterances:
|
||||
rows_by_segment[row.segment_id].append(row)
|
||||
for clip in clips:
|
||||
clips_by_segment[clip.segment_id].append(clip)
|
||||
|
||||
candidates: list[ProtocolCandidate] = []
|
||||
for segment_id, segment_clips in sorted(clips_by_segment.items()):
|
||||
segment_clips.sort(key=lambda clip: clip.dense_start_frame)
|
||||
segment_rows = sorted(
|
||||
rows_by_segment.get(segment_id, []),
|
||||
key=lambda row: (row.dense_start_frame, row.dense_end_frame_exclusive, row.uid),
|
||||
)
|
||||
segment_start = segment_clips[0].dense_start_frame
|
||||
segment_end = segment_clips[-1].dense_start_frame + segment_clips[-1].num_frames
|
||||
wearer, others = _fill_state(
|
||||
segment_rows,
|
||||
segment_start=segment_start,
|
||||
segment_end=segment_end,
|
||||
wearer_id=wearer_id,
|
||||
)
|
||||
|
||||
for index in range(1, len(others)):
|
||||
dense_frame = segment_start + index
|
||||
if others[index - 1] and not others[index] and not wearer[index - 1]:
|
||||
competing = _first_transition(
|
||||
others,
|
||||
start=index + 1,
|
||||
stop=min(len(others), index + window_frames + 1),
|
||||
rising=True,
|
||||
)
|
||||
nominal_end = index + window_frames
|
||||
observable_end = min(nominal_end, len(others))
|
||||
effective_end = min(observable_end, competing if competing is not None else observable_end)
|
||||
outcome = _first_transition(
|
||||
wearer,
|
||||
start=index,
|
||||
stop=effective_end,
|
||||
rising=True,
|
||||
)
|
||||
previous_ids = _active_other_ids(
|
||||
segment_rows, dense_frame - 1, wearer_id=wearer_id
|
||||
)
|
||||
incoming_ids = (
|
||||
()
|
||||
if competing is None
|
||||
else _active_other_ids(
|
||||
segment_rows, segment_start + competing, wearer_id=wearer_id
|
||||
)
|
||||
)
|
||||
if outcome is not None:
|
||||
label = "positive"
|
||||
subtype = "not_applicable"
|
||||
end_index = outcome
|
||||
end_reason = "wearer_onset"
|
||||
reason = "wearer onset observed after other offset and before competing other onset"
|
||||
elif competing is not None:
|
||||
label = "hard_negative"
|
||||
subtype = (
|
||||
"eot_midturn_pause"
|
||||
if set(previous_ids) & set(incoming_ids)
|
||||
else "eot_other_to_other"
|
||||
)
|
||||
end_index = competing
|
||||
end_reason = "competing_other_onset"
|
||||
reason = "wearer remains silent until another non-wearer onset"
|
||||
elif nominal_end <= len(others):
|
||||
label = "hard_negative"
|
||||
subtype = "eot_no_response"
|
||||
end_index = nominal_end
|
||||
end_reason = "full_window"
|
||||
reason = "wearer remains silent for the full observable response window"
|
||||
else:
|
||||
label = "exclude"
|
||||
subtype = "other"
|
||||
end_index = len(others)
|
||||
end_reason = "right_censored"
|
||||
reason = "recording segment ends before a wearer outcome or full window"
|
||||
candidates.append(_candidate(
|
||||
session=session,
|
||||
task="eot_response",
|
||||
automatic_protocol_label=label,
|
||||
automatic_hard_negative_type=subtype,
|
||||
anchor_frame=dense_frame,
|
||||
behavior_frame=None if outcome is None else segment_start + outcome,
|
||||
region_end_frame=segment_start + end_index,
|
||||
region_end_reason=end_reason,
|
||||
previous_ids=previous_ids,
|
||||
incoming_ids=incoming_ids,
|
||||
clips=segment_clips,
|
||||
rows=segment_rows,
|
||||
wearer_id=wearer_id,
|
||||
context_frames=context_frames,
|
||||
reason=reason,
|
||||
))
|
||||
|
||||
if not others[index - 1] and others[index] and wearer[index - 1]:
|
||||
competing = _first_transition(
|
||||
others,
|
||||
start=index + 1,
|
||||
stop=min(len(others), index + window_frames + 1),
|
||||
rising=False,
|
||||
)
|
||||
nominal_end = index + window_frames
|
||||
observable_end = min(nominal_end, len(others))
|
||||
effective_end = min(observable_end, competing if competing is not None else observable_end)
|
||||
outcome = _first_transition(
|
||||
wearer,
|
||||
start=index,
|
||||
stop=effective_end,
|
||||
rising=False,
|
||||
)
|
||||
previous_ids = _active_other_ids(
|
||||
segment_rows, dense_frame - 1, wearer_id=wearer_id
|
||||
)
|
||||
incoming_ids = _active_other_ids(
|
||||
segment_rows, dense_frame, wearer_id=wearer_id
|
||||
)
|
||||
if outcome is not None:
|
||||
label = "positive"
|
||||
subtype = "not_applicable"
|
||||
end_index = outcome
|
||||
end_reason = "wearer_offset"
|
||||
reason = "wearer offset observed after other onset and before competing other offset"
|
||||
elif competing is not None:
|
||||
label = "hard_negative"
|
||||
subtype = (
|
||||
"int_backchannel"
|
||||
if _incoming_is_backchannel(
|
||||
segment_rows,
|
||||
dense_frame=dense_frame,
|
||||
wearer_id=wearer_id,
|
||||
max_seconds=1.0,
|
||||
)
|
||||
else "int_non_takeover_overlap"
|
||||
)
|
||||
end_index = competing
|
||||
end_reason = "competing_other_offset"
|
||||
reason = "wearer retains the floor until the incoming other activity ends"
|
||||
elif nominal_end <= len(others):
|
||||
label = "hard_negative"
|
||||
subtype = "int_retain_floor"
|
||||
end_index = nominal_end
|
||||
end_reason = "full_window"
|
||||
reason = "wearer remains active through the full observable yield window"
|
||||
else:
|
||||
label = "exclude"
|
||||
subtype = "other"
|
||||
end_index = len(others)
|
||||
end_reason = "right_censored"
|
||||
reason = "recording segment ends before a wearer outcome or full window"
|
||||
candidates.append(_candidate(
|
||||
session=session,
|
||||
task="int_yield",
|
||||
automatic_protocol_label=label,
|
||||
automatic_hard_negative_type=subtype,
|
||||
anchor_frame=dense_frame,
|
||||
behavior_frame=None if outcome is None else segment_start + outcome,
|
||||
region_end_frame=segment_start + end_index,
|
||||
region_end_reason=end_reason,
|
||||
previous_ids=previous_ids,
|
||||
incoming_ids=incoming_ids,
|
||||
clips=segment_clips,
|
||||
rows=segment_rows,
|
||||
wearer_id=wearer_id,
|
||||
context_frames=context_frames,
|
||||
reason=reason,
|
||||
))
|
||||
|
||||
return sorted(candidates, key=lambda row: (row.anchor_dense_frame, row.task, row.uid))
|
||||
|
||||
|
||||
def stratified_limit(
|
||||
rows: Sequence[ProtocolCandidate],
|
||||
limit: int | None,
|
||||
*,
|
||||
seed: int,
|
||||
) -> list[ProtocolCandidate]:
|
||||
if limit is None or limit >= len(rows):
|
||||
return list(rows)
|
||||
if limit < 0:
|
||||
raise ValueError("--max-candidates must be non-negative")
|
||||
rng = random.Random(seed)
|
||||
buckets: dict[tuple[str, str, str], list[ProtocolCandidate]] = defaultdict(list)
|
||||
for row in rows:
|
||||
buckets[(row.task, row.automatic_protocol_label, row.automatic_hard_negative_type)].append(row)
|
||||
keys = sorted(buckets)
|
||||
for key in keys:
|
||||
rng.shuffle(buckets[key])
|
||||
rng.shuffle(keys)
|
||||
selected: list[ProtocolCandidate] = []
|
||||
while len(selected) < limit:
|
||||
progressed = False
|
||||
for key in keys:
|
||||
if buckets[key]:
|
||||
selected.append(buckets[key].pop())
|
||||
progressed = True
|
||||
if len(selected) == limit:
|
||||
break
|
||||
if not progressed:
|
||||
break
|
||||
return sorted(selected, key=lambda row: (row.session, row.anchor_dense_frame, row.task))
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: Iterable[ProtocolCandidate]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
for row in rows:
|
||||
handle.write(json.dumps(asdict(row), ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--dataset-root", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--sessions", nargs="+", default=list(ALL_SESSIONS))
|
||||
parser.add_argument("--camera-wearer-id", type=int, default=CAMERA_WEARER_ID)
|
||||
parser.add_argument("--window-seconds", type=float, default=DEFAULT_WINDOW_SECONDS)
|
||||
parser.add_argument("--context-seconds", type=float, default=3.0)
|
||||
parser.add_argument(
|
||||
"--tasks",
|
||||
nargs="+",
|
||||
choices=("eot_response", "int_yield"),
|
||||
default=("eot_response", "int_yield"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--automatic-labels",
|
||||
nargs="+",
|
||||
choices=("positive", "hard_negative", "exclude"),
|
||||
default=("positive", "hard_negative", "exclude"),
|
||||
)
|
||||
parser.add_argument("--max-candidates", type=int)
|
||||
parser.add_argument("--selection-seed", type=int, default=2027)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
unknown = sorted(set(args.sessions) - set(ALL_SESSIONS))
|
||||
if unknown:
|
||||
raise ValueError(f"Unknown EasyCom sessions: {unknown}")
|
||||
dataset_root = args.dataset_root.expanduser().resolve()
|
||||
output_dir = args.output_dir.expanduser().resolve()
|
||||
all_rows: list[ProtocolCandidate] = []
|
||||
rules = Rules()
|
||||
for session in args.sessions:
|
||||
clips = build_clips(dataset_root, session, rules)
|
||||
utterances = load_utterances(session, clips)
|
||||
rows = derive_protocol_candidates(
|
||||
utterances,
|
||||
clips,
|
||||
session=session,
|
||||
wearer_id=args.camera_wearer_id,
|
||||
window_seconds=args.window_seconds,
|
||||
context_seconds=args.context_seconds,
|
||||
)
|
||||
all_rows.extend(rows)
|
||||
counts = Counter((row.task, row.automatic_protocol_label) for row in rows)
|
||||
print(f"{session}: candidates={len(rows)} counts={dict(sorted(counts.items()))}")
|
||||
|
||||
eligible = [
|
||||
row
|
||||
for row in all_rows
|
||||
if row.task in set(args.tasks)
|
||||
and row.automatic_protocol_label in set(args.automatic_labels)
|
||||
]
|
||||
selected = stratified_limit(eligible, args.max_candidates, seed=args.selection_seed)
|
||||
output_path = output_dir / "easycom_eot_int_candidates.jsonl"
|
||||
_write_jsonl(output_path, selected)
|
||||
counts = Counter(
|
||||
(row.split, row.task, row.automatic_protocol_label, row.automatic_hard_negative_type)
|
||||
for row in selected
|
||||
)
|
||||
summary = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"protocol_version": PROTOCOL_VERSION,
|
||||
"dataset": "EasyCom",
|
||||
"semantic_warning": (
|
||||
"Automatic labels are timing candidates for independent human audit; "
|
||||
"they are not semantic EOT/INT ground truth."
|
||||
),
|
||||
"sessions": list(args.sessions),
|
||||
"camera_wearer_id": args.camera_wearer_id,
|
||||
"window_seconds": args.window_seconds,
|
||||
"context_seconds": args.context_seconds,
|
||||
"selection_seed": args.selection_seed,
|
||||
"max_candidates": args.max_candidates,
|
||||
"tasks": list(args.tasks),
|
||||
"automatic_labels": list(args.automatic_labels),
|
||||
"num_candidates_before_filter": len(all_rows),
|
||||
"num_candidates_before_limit": len(eligible),
|
||||
"num_candidates_written": len(selected),
|
||||
"counts": [
|
||||
{
|
||||
"split": key[0],
|
||||
"task": key[1],
|
||||
"automatic_protocol_label": key[2],
|
||||
"automatic_hard_negative_type": key[3],
|
||||
"count": value,
|
||||
}
|
||||
for key, value in sorted(counts.items())
|
||||
],
|
||||
}
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "summary.json").write_text(
|
||||
json.dumps(summary, indent=2, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"Saved {len(selected)} manuscript-aligned review candidates to {output_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user