Revise wearer entry and yield annotation protocol
This commit is contained in:
Xin Wang
@@ -304,7 +304,7 @@ Do not reorganize them physically without compatibility wrappers.
|
||||
| `derive_easycom_interaction_events.py` | Deterministic transcript/timing proxies |
|
||||
| `adjudicate_easycom_interaction_events.py` | Optional ambiguous-event adjudication |
|
||||
| `audit_easycom_floor_events.py` | Count EOT/takeover/backchannel candidates |
|
||||
| `prepare_easycom_eot_int_review.py` | Group short non-competed wearer resumptions into floor episodes; start from wearer onsets/offsets, retrieve constrained preceding release options for EOT, require `wearer_floor_start < other_onset < wearer_offset` plus boundary crossing for INT, preserve ineligible boundaries as controls, and scan independent hard negatives; semantic labels and the selected EOT trigger remain AI/human review outputs |
|
||||
| `prepare_easycom_eot_int_review.py` | Group short non-competed wearer resumptions into floor episodes; retain every resulting wearer onset as an entry candidate with optional preceding-release metadata, require `wearer_floor_start < other_onset < wearer_offset` plus boundary crossing for INT, preserve ineligible raw boundaries as controls, and scan independent hard negatives; semantic labels remain AI/human review outputs |
|
||||
| `adjudicate_easycom_eot_int_candidates.py` | Add optional DeepSeek transcript/timing suggestions; never human ground truth |
|
||||
| `review_floor_events.py` | Serve the local video/transcript workspace and persist human event reviews |
|
||||
| `prepare_easycom_oracle_features.py` | Build explicit past-only oracle person-activity sidecars and analysis-only addressee controls |
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Add conservative DeepSeek pre-labels to wearer-centric EOT/INT candidates.
|
||||
"""Add conservative DeepSeek pre-labels to wearer entry/INT candidates.
|
||||
|
||||
The model receives the candidate source, structured participant timing and the
|
||||
transcript excerpt. It cannot change any source timestamp. Its output is
|
||||
@@ -26,15 +26,15 @@ import urllib.request
|
||||
from prepare_easycom_eot_int_review import requires_semantic_adjudication
|
||||
|
||||
|
||||
PROMPT_VERSION = "easycom-wearer-anchor-first-eot-int-prelabel-v7"
|
||||
PROMPT_VERSION = "easycom-wearer-entry-int-prelabel-v12"
|
||||
DEFAULT_MODEL = "deepseek-v4-flash"
|
||||
DEFAULT_BASE_URL = "https://api.deepseek.com"
|
||||
VERDICTS = {"POSITIVE", "HARD_NEGATIVE", "NOT_APPLICABLE", "EXCLUDE", "UNSURE"}
|
||||
EVENT_TYPES = {"EOT_RESPONSE", "INT_YIELD"}
|
||||
VERDICTS = {"POSITIVE", "HARD_NEGATIVE", "EXCLUDE"}
|
||||
EVENT_TYPES = {"WEARER_ENTRY", "INT_YIELD"}
|
||||
HARD_NEGATIVE_TYPES = {
|
||||
"EOT_MIDTURN_PAUSE",
|
||||
"EOT_OTHER_TO_OTHER",
|
||||
"EOT_WEARER_BACKCHANNEL",
|
||||
"ENTRY_OTHER_MIDTURN_PAUSE",
|
||||
"ENTRY_OTHER_TO_OTHER",
|
||||
"ENTRY_WEARER_BACKCHANNEL",
|
||||
"INT_OTHER_BACKCHANNEL",
|
||||
"INT_NON_TAKEOVER_OVERLAP",
|
||||
"INT_WEARER_HOLDS_FLOOR",
|
||||
@@ -89,31 +89,47 @@ class _ProgressBar:
|
||||
SYSTEM_PROMPT = """You conservatively pre-label candidate regions for a scientific study of wearer-relative conversational floor control.
|
||||
|
||||
Evidence rules:
|
||||
1. Use candidate source, structured participant timing, per-speaker activity, and transcript together. Timing evidence has priority over a speculative transcript interpretation.
|
||||
1. Reconstruct the event chronologically before assigning semantics. Use exact start/end timestamps to identify (a) the proposed release or competing onset, (b) the wearer behavior anchor, (c) which same participant continues across the anchor, and (d) who produces the next substantive turn. Then use transcript content and Target_of_Speech, when supplied, to decide conversational function. Timing has priority over speculative transcript interpretation, but duration alone never determines floor function.
|
||||
2. Never alter or propose an event timestamp. Do not infer prosody, gaze, intention, or acoustic cues that were not supplied.
|
||||
3. A wearer onset is not automatically a floor take; a wearer offset is not automatically a yield; an other onset is not automatically a takeover. A pause is not automatically EOT, overlap is not automatically interruption, and words such as "yeah" are not automatically backchannels. Check who speaks sustainably afterward and who resumes.
|
||||
4. Use UNSURE when the evidence does not establish conversational function. Never force a label for coverage.
|
||||
5. Every supplied WEARER_ONSET was found by starting from an observed floor-level wearer entry and looking backward. Its immutable behavior anchor is fixed. Select the true preceding release only from temporal_evidence.opportunity_options; never invent a trigger or timestamp. A POSITIVE EOT_RESPONSE must return that option's uid as selected_opportunity_uid. Do not reject an option solely because its response delay is long. If none is a defensible trigger, return another verdict and null.
|
||||
6. Every supplied WEARER_OFFSET has already passed both deterministic INT prerequisites: a new non-wearer onset occurs strictly inside the current wearer floor episode (t_w < t_o < t_y), and that same non-wearer interval crosses the wearer yield boundary within the recorded annotation tolerance. Decide only whether it is a genuine takeover or a semantic hard negative. Pre-existing other activity, post-completion other onset, within-turn wearer pauses, and completion-to-silence are controls outside this queue.
|
||||
7. Transcript event markers are evidence, not lexical turns. In EasyCom, a transcription consisting of [L] denotes laughter-only vocalization. Do not treat [L] as a sustained floor take, an EOT response, or an interruption takeover merely because its activity interval creates an onset, offset, or overlap candidate. Specifically:
|
||||
- wearer [L] during a non-wearer's retained turn is normally HARD_NEGATIVE / EOT_WEARER_BACKCHANNEL, not EOT_RESPONSE;
|
||||
3. A raw wearer onset is not automatically a floor entry; a wearer offset is not automatically a yield; an other onset is not automatically a takeover. A pause is not automatically a release, overlap is not automatically interruption, and words such as "yeah" are not automatically backchannels. Determine what the utterance does in context, not only how long it lasts.
|
||||
4. EXCLUDE is the sole non-scoring outcome. Use it both for insufficient/unresolvable evidence and for clear out-of-scope non-events that should not enter evaluation, including a normal wearer completion that merely coincides with laughter or feedback. Never force such a region into HARD_NEGATIVE for coverage.
|
||||
5. Every supplied WEARER_ONSET is an observed floor-level onset after deterministic within-turn pause merging. Decide whether the wearer genuinely enters or re-enters the conversational floor. A POSITIVE WEARER_ENTRY does not require a preceding non-wearer release: it may occur after a release, from silence, or while another speaker is active. temporal_evidence.opportunity_options and entry_context_timing are context metadata only. If one supplied release clearly prompted the entry, selected_opportunity_uid may contain its uid; otherwise return null. Never invent a trigger or timestamp. A direct, semantically complete answer can be a genuine entry even when short. Do not call it a backchannel merely because it lasts less than one second or because another participant speaks later. For example, "Nope." beginning 0.15 s after an addressed question ends is normally WEARER_ENTRY, not ENTRY_WEARER_BACKCHANNEL. Later conversation does not retroactively cancel a completed entry.
|
||||
6. Every supplied WEARER_OFFSET has already passed both deterministic INT prerequisites: a new non-wearer onset occurs strictly inside the current wearer floor episode (t_w < t_o < t_y), and that same non-wearer interval crosses the wearer yield boundary within the recorded annotation tolerance. This is only a retrieval gate. Verify that the same incoming non-wearer actually takes and sustains the next floor after the wearer offset. A different participant's later substantive turn is not evidence that the incoming speaker took over. Do not label INT_WEARER_HOLDS_FLOOR merely because the wearer speaks again later: if the competing other establishes an intervening substantive floor and the wearer has an intervening silent interval, the later wearer onset is a new WEARER_ENTRY and does not invalidate the earlier INT_YIELD. If the wearer reaches a semantically complete natural ending while the overlapping contribution is only [L] or feedback, and that contributor does not take the next substantive floor, return EXCLUDE: out-of-scope normal completion, not INT_YIELD and not a scored hard negative. Pre-existing other activity, post-completion other onset, within-turn wearer pauses, and completion-to-silence are controls outside this queue.
|
||||
7. Transcript event markers are evidence, not lexical turns. In EasyCom, a transcription consisting of [L] denotes laughter-only vocalization. Do not treat [L] as a sustained floor entry or interruption takeover merely because its activity interval creates an onset, offset, or overlap candidate. Specifically:
|
||||
- wearer [L] during a non-wearer's retained turn is normally HARD_NEGATIVE / ENTRY_WEARER_BACKCHANNEL, not WEARER_ENTRY;
|
||||
- non-wearer [L] overlapping a wearer's retained turn is normally HARD_NEGATIVE / INT_OTHER_BACKCHANNEL, not INT_YIELD;
|
||||
- wearer [L] ending while a non-wearer continues does not establish that the wearer held and yielded the floor.
|
||||
Use UNSURE if the surrounding timing does not establish whose floor was retained. Do not EXCLUDE a valid laughter-only annotation merely because it is non-lexical, and do not infer that a visible smile occurred: [L] supplies laughter evidence only. Do not assign semantics to other bracketed markers unless the supplied context establishes them.
|
||||
Use EXCLUDE if the surrounding timing does not establish whose floor was retained. Do not EXCLUDE a valid laughter-only annotation merely because it is non-lexical, and do not infer that a visible smile occurred: [L] supplies laughter evidence only. Do not assign semantics to other bracketed markers unless the supplied context establishes them.
|
||||
|
||||
Top-level verdicts:
|
||||
- POSITIVE: the system should commit.
|
||||
- HARD_NEGATIVE: a valid, interpretable region where the system should not commit.
|
||||
- NOT_APPLICABLE: a real speech transition outside the committed EOT/INT task, such as a wearer starting after long silence or naturally ending into silence.
|
||||
- EXCLUDE: the region cannot be scored reliably because of annotation corruption, a recording boundary, missing context, a non-speech artifact, timestamp corruption, or another unresolvable defect.
|
||||
- UNSURE: evidence is insufficient for a reliable automatic pre-label and needs human judgment.
|
||||
- POSITIVE: a target WEARER_ENTRY or INT_YIELD occurs and the system should commit.
|
||||
- HARD_NEGATIVE: a deliberately retained, valid false-positive opportunity matching one of the behavioral counterexamples below; it enters evaluation as a negative.
|
||||
- EXCLUDE: a non-scoring region. This includes clear out-of-scope/non-target opportunities such as natural wearer completion coincident with feedback, as well as insufficient evidence, annotation corruption, a recording boundary, missing context, a non-speech artifact, timestamp corruption, or another unresolvable defect.
|
||||
|
||||
POSITIVE event_type must be exactly EOT_RESPONSE or INT_YIELD. EOT_RESPONSE means a non-wearer genuinely releases the floor and the wearer then takes it with a sustained response. INT_YIELD means a non-wearer genuinely takes over during wearer speech and the wearer relinquishes the floor.
|
||||
POSITIVE event_type must be exactly WEARER_ENTRY or INT_YIELD. WEARER_ENTRY means the wearer genuinely enters or re-enters the floor with a contextually complete response or substantive contribution; no prior release is required and the contribution need not be long. INT_YIELD means a non-wearer genuinely takes over during wearer speech and the wearer relinquishes the floor. A later WEARER_ENTRY after an intervening other floor is a new event and must not retroactively invalidate the preceding INT_YIELD.
|
||||
|
||||
HARD_NEGATIVE subtype must be exactly one of EOT_MIDTURN_PAUSE, EOT_OTHER_TO_OTHER, EOT_WEARER_BACKCHANNEL, INT_OTHER_BACKCHANNEL, INT_NON_TAKEOVER_OVERLAP, INT_WEARER_HOLDS_FLOOR, or OTHER_HARD_NEGATIVE.
|
||||
HARD_NEGATIVE subtypes are behavioral counterexamples, not data-quality labels:
|
||||
- ENTRY_OTHER_MIDTURN_PAUSE: a non-wearer merely pauses inside that same speaker's continuing floor episode; the same non-wearer resumes or clearly retains the turn, while the wearer does not enter.
|
||||
- ENTRY_OTHER_TO_OTHER: one non-wearer releases and a different non-wearer takes the next floor; the wearer does not enter.
|
||||
- ENTRY_WEARER_BACKCHANNEL: the wearer vocalizes, but only as brief feedback, acknowledgement, laughter, or another non-floor-taking response; the non-wearer retains or resumes the floor.
|
||||
- INT_OTHER_BACKCHANNEL: the competing non-wearer contribution during wearer speech is brief feedback, acknowledgement, laughter, or another backchannel, and the wearer clearly continues or resumes the same floor episode. If the wearer instead naturally completes at the overlap boundary and the feedback speaker does not take the next substantive floor, use EXCLUDE.
|
||||
- INT_WEARER_HOLDS_FLOOR: the competing contribution is not best characterized as a backchannel, but the wearer demonstrably continues or promptly resumes the same floor episode after the overlap, so no yield occurs.
|
||||
- INT_NON_TAKEOVER_OVERLAP: the competing contribution is not merely a backchannel, yet it still fails to establish a sustained takeover (for example, a collision, abandoned attempt, or overlap that dies out); use this only when wearer continuation is not the clearer defining evidence for INT_WEARER_HOLDS_FLOOR.
|
||||
- OTHER_HARD_NEGATIVE: a valid, interpretable false-positive opportunity in the correct ENTRY or INT family that does not fit any named subtype. State the unmatched behavior explicitly. This is not an uncertainty fallback; use EXCLUDE when the evidence is insufficient or unresolvable.
|
||||
|
||||
Subtype precedence when descriptions overlap:
|
||||
1. Use the family matching the candidate task: ENTRY candidates take only ENTRY_* subtypes and INT candidates take only INT_* subtypes.
|
||||
2. For ENTRY, prefer ENTRY_WEARER_BACKCHANNEL when an actual wearer vocalization is feedback rather than a floor entry; otherwise distinguish same-other resumption (ENTRY_OTHER_MIDTURN_PAUSE) from a different-other takeover (ENTRY_OTHER_TO_OTHER).
|
||||
3. For INT, use INT_OTHER_BACKCHANNEL when the other contribution is feedback and wearer continuation/resumption makes it a scored false-positive opportunity. If feedback merely coincides with a natural wearer completion, use EXCLUDE. Otherwise use INT_WEARER_HOLDS_FLOOR when wearer continuation/resumption clearly demonstrates retained floor; use INT_NON_TAKEOVER_OVERLAP for the remaining interpretable failed-takeover overlaps.
|
||||
4. Use OTHER_HARD_NEGATIVE only after the named subtypes have been ruled out. Never use it merely because confidence is low.
|
||||
|
||||
Return exactly one JSON object with this shape and no markdown:
|
||||
{"items":[{"uid":"...","verdict":"UNSURE","event_type":null,"subtype":null,"selected_opportunity_uid":null,"confidence":0.25,"reason":"brief evidence-based sentence"}]}
|
||||
{"items":[{"uid":"...","verdict":"EXCLUDE","event_type":null,"subtype":null,"selected_opportunity_uid":null,"confidence":0.25,"reason":"brief evidence-based sentence"}]}
|
||||
Field combinations are strict:
|
||||
- POSITIVE / WEARER_ENTRY: subtype=null; selected_opportunity_uid is one supplied opportunity uid or null.
|
||||
- POSITIVE / INT_YIELD: subtype=null; selected_opportunity_uid=null.
|
||||
- HARD_NEGATIVE: event_type=null; selected_opportunity_uid=null; subtype is one valid subtype from the candidate's ENTRY or INT family.
|
||||
- EXCLUDE: event_type=null; subtype=null; selected_opportunity_uid=null.
|
||||
Return every supplied uid exactly once. Copy uid byte-for-byte and never add a backslash before an underscore. confidence must be a number from 0 to 1."""
|
||||
|
||||
|
||||
@@ -191,7 +207,7 @@ def build_user_prompt(rows: Sequence[dict[str, Any]], pass_index: int) -> str:
|
||||
instruction = (
|
||||
"Primary pass: classify conservatively."
|
||||
if pass_index == 0
|
||||
else "Independent skeptical pass: seek alternative explanations and choose uncertain when semantics are unsupported."
|
||||
else "Independent skeptical pass: seek alternative explanations and choose EXCLUDE when semantics are unsupported."
|
||||
)
|
||||
return (
|
||||
instruction
|
||||
@@ -215,6 +231,7 @@ def validate_decisions(
|
||||
payload: Any,
|
||||
expected_uids: Sequence[str],
|
||||
allowed_opportunities: dict[str, set[str]] | None = None,
|
||||
expected_families: dict[str, str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("items"), list):
|
||||
raise ValueError("Response must be an object with an items list")
|
||||
@@ -236,21 +253,31 @@ def validate_decisions(
|
||||
if verdict not in VERDICTS:
|
||||
raise ValueError(f"Invalid verdict for {uid}: {verdict!r}")
|
||||
if verdict == "POSITIVE" and event_type not in EVENT_TYPES:
|
||||
raise ValueError(f"POSITIVE requires EOT_RESPONSE or INT_YIELD for {uid}")
|
||||
raise ValueError(f"POSITIVE requires WEARER_ENTRY or INT_YIELD for {uid}")
|
||||
if verdict != "POSITIVE" and event_type is not None:
|
||||
raise ValueError(f"Only POSITIVE may have event_type for {uid}")
|
||||
if verdict == "HARD_NEGATIVE" and subtype not in HARD_NEGATIVE_TYPES:
|
||||
raise ValueError(f"HARD_NEGATIVE requires a valid subtype for {uid}")
|
||||
if verdict != "HARD_NEGATIVE" and subtype is not None:
|
||||
raise ValueError(f"Only HARD_NEGATIVE may have subtype for {uid}")
|
||||
family = None if expected_families is None else expected_families.get(str(uid))
|
||||
if verdict == "POSITIVE" and family in {"ENTRY", "INT"}:
|
||||
required_event_type = "WEARER_ENTRY" if family == "ENTRY" else "INT_YIELD"
|
||||
if event_type != required_event_type:
|
||||
raise ValueError(f"{family} candidate requires {required_event_type} for {uid}")
|
||||
if verdict == "HARD_NEGATIVE" and family in {"ENTRY", "INT"}:
|
||||
if subtype != "OTHER_HARD_NEGATIVE" and not str(subtype).startswith(f"{family}_"):
|
||||
raise ValueError(f"{family} candidate requires an {family}_* hard-negative subtype for {uid}")
|
||||
allowed = set() if allowed_opportunities is None else allowed_opportunities.get(str(uid), set())
|
||||
if verdict == "POSITIVE" and event_type == "EOT_RESPONSE" and allowed:
|
||||
if selected_opportunity_uid not in allowed:
|
||||
raise ValueError(
|
||||
f"EOT_RESPONSE must select a supplied opportunity uid for {uid}"
|
||||
)
|
||||
if verdict != "POSITIVE" or event_type != "WEARER_ENTRY":
|
||||
# This field is auxiliary attribution metadata, not part of the
|
||||
# semantic label. DeepSeek occasionally copies an opportunity uid
|
||||
# into an INT/HN/EXCLUDE result despite the output contract. Drop
|
||||
# that irrelevant value instead of retrying and losing the batch.
|
||||
selected_opportunity_uid = None
|
||||
elif selected_opportunity_uid is not None:
|
||||
raise ValueError(f"selected_opportunity_uid is only valid for EOT_RESPONSE: {uid}")
|
||||
if selected_opportunity_uid not in allowed:
|
||||
raise ValueError(f"WEARER_ENTRY may select only a supplied opportunity uid for {uid}")
|
||||
if isinstance(confidence, bool) or not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1:
|
||||
raise ValueError(f"confidence must be in [0, 1] for {uid}")
|
||||
if not isinstance(reason, str) or not reason.strip():
|
||||
@@ -310,6 +337,7 @@ def _post(
|
||||
content = envelope["choices"][0]["message"]["content"]
|
||||
parsed = json.loads(content)
|
||||
allowed_opportunities = {}
|
||||
expected_families = {}
|
||||
for row in transported:
|
||||
evidence = row.get("temporal_evidence") or {}
|
||||
options = evidence.get("opportunity_options") or []
|
||||
@@ -320,8 +348,13 @@ def _post(
|
||||
}
|
||||
if not allowed_opportunities[row["uid"]] and evidence.get("opportunity_source_uid"):
|
||||
allowed_opportunities[row["uid"]].add(str(evidence["opportunity_source_uid"]))
|
||||
task = str(row.get("task", "")).lower()
|
||||
if task.startswith("wearer_entry") or task.startswith("eot"):
|
||||
expected_families[row["uid"]] = "ENTRY"
|
||||
elif task.startswith("int"):
|
||||
expected_families[row["uid"]] = "INT"
|
||||
decisions = validate_decisions(
|
||||
parsed, list(original_by_transport), allowed_opportunities
|
||||
parsed, list(original_by_transport), allowed_opportunities, expected_families
|
||||
)
|
||||
for decision in decisions:
|
||||
decision["uid"] = original_by_transport[decision["uid"]]
|
||||
@@ -404,17 +437,21 @@ def add_suggestions(
|
||||
for source in rows:
|
||||
decisions = by_uid[source["uid"]]
|
||||
labels = {
|
||||
(d["verdict"], d["event_type"], d["subtype"], d["selected_opportunity_uid"])
|
||||
(d["verdict"], d["event_type"], d["subtype"])
|
||||
for d in decisions
|
||||
}
|
||||
unanimous = len(labels) == 1
|
||||
if unanimous:
|
||||
verdict, event_type, subtype, selected_opportunity_uid = next(iter(labels))
|
||||
verdict, event_type, subtype = next(iter(labels))
|
||||
selected_options = {d["selected_opportunity_uid"] for d in decisions}
|
||||
selected_opportunity_uid = (
|
||||
next(iter(selected_options)) if len(selected_options) == 1 else None
|
||||
)
|
||||
confidence = min(float(item["confidence"]) for item in decisions)
|
||||
reason = " | ".join(item["reason"] for item in decisions)
|
||||
needs_human_review = any(item["needs_human_review"] for item in decisions)
|
||||
else:
|
||||
verdict, event_type, subtype, selected_opportunity_uid = "UNSURE", None, None, None
|
||||
verdict, event_type, subtype, selected_opportunity_uid = "EXCLUDE", None, None, None
|
||||
confidence = min(float(item["confidence"]) for item in decisions)
|
||||
reason = "The independent DeepSeek passes disagreed."
|
||||
needs_human_review = True
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# EasyCom wearer-centric EOT/INT pre-label and review pipeline
|
||||
# EasyCom wearer-entry/INT pre-label and review pipeline
|
||||
|
||||
The workflow has three explicit stages. Speech activity timestamps are
|
||||
read-only throughout; reviewed events never modify the dense 5 Hz activity
|
||||
@@ -24,9 +24,9 @@ python3 prepare_easycom_eot_int_review.py \
|
||||
|
||||
The generator emits two independent files:
|
||||
|
||||
- `easycom_eot_int_candidates.jsonl` contains floor-level wearer `0->1`
|
||||
behavior anchors with one or more admissible preceding non-wearer releases
|
||||
(`WEARER_ONSET`, `EOT_OPPORTUNITY_CANDIDATE`), only episode-ending wearer
|
||||
- `easycom_eot_int_candidates.jsonl` (legacy-compatible filename) contains every
|
||||
floor-level wearer `0->1` behavior anchor (`WEARER_ONSET`,
|
||||
`WEARER_ENTRY_CANDIDATE`), only episode-ending wearer
|
||||
`1->0` boundaries where a new non-wearer onset satisfies
|
||||
`wearer_floor_start < other_onset < wearer_offset` and the same activity
|
||||
crosses the offset (`WEARER_OFFSET`, `YIELD_CANDIDATE`), and hard-negative
|
||||
@@ -34,8 +34,7 @@ The generator emits two independent files:
|
||||
`SAME_SPEAKER_PAUSE`, `OTHER_TO_OTHER`, `OTHER_BACKCHANNEL`, and
|
||||
`NON_TAKEOVER_OVERLAP`;
|
||||
- `easycom_eot_int_controls.jsonl` contains short within-turn raw onsets
|
||||
(`WEARER_RESUME`), floor-level wearer entries with no surviving preceding
|
||||
release opportunity (`WEARER_ONSET_CONTROL`), and wearer offsets that fail
|
||||
(`WEARER_RESUME`) and wearer offsets that fail
|
||||
the strict INT ordering/crossing prerequisite (`WEARER_OFFSET_CONTROL`). They
|
||||
are retained for audits, but are not sent to DeepSeek or placed in the
|
||||
primary human-review queue.
|
||||
@@ -49,18 +48,20 @@ continues across the resumed onset conservatively preserves a new wearer-entry
|
||||
candidate. Every decision is stored in `temporal_evidence`, including the raw
|
||||
gap, feedback IDs, competing IDs, threshold, and `floor_episode_merged` flag.
|
||||
|
||||
EOT construction iterates floor-level wearer onsets, then searches backward for
|
||||
preceding non-wearer releases. A release is an admissible option only when the
|
||||
Entry construction retains every floor-level wearer onset, then searches backward
|
||||
for preceding non-wearer releases as optional context. A release is an admissible option only when the
|
||||
wearer stays silent and no new non-wearer onset or already-active other floor
|
||||
invalidates the interval before the wearer onset. Silence length alone does not
|
||||
invalidate an option. Each wearer onset produces at most one primary row. Its
|
||||
`opportunity_options` list contains the only release anchors that DeepSeek or a
|
||||
human may select; neither may invent a timestamp. The selected release is the
|
||||
opportunity anchor, the wearer onset is the behavior anchor, and their interval
|
||||
deterministically defines the valid EOT prediction interval.
|
||||
invalidate an option. Each wearer onset produces exactly one primary row. A
|
||||
missing release does not demote an entry to a control: an entry may occur from
|
||||
silence or while another speaker is active. `entry_context_timing` records
|
||||
`AFTER_OTHER_RELEASE`, `OTHER_ACTIVE`, or `FROM_SILENCE_OR_UNRESOLVED` as
|
||||
metadata. When present, `opportunity_options` contains the only release anchors
|
||||
that DeepSeek or a human may optionally select; neither may invent a timestamp.
|
||||
|
||||
Paired wearer onsets and strictly qualified offsets are initially `unsure`;
|
||||
timing alone is not a semantic positive label. The INT prerequisite is
|
||||
Floor-level wearer onsets and strictly qualified offsets carry the rule-stage value
|
||||
`candidate`; timing alone is not a semantic positive label. Deterministic rows
|
||||
outside the primary queue carry `control`, never a semantic verdict. The INT prerequisite is
|
||||
`wearer_floor_start < other.start < wearer_offset < other.end`, with a
|
||||
configurable one-frame (50 ms) tolerance only at the offset boundary.
|
||||
Pre-existing other activity and raw offsets inside a merged wearer episode stay
|
||||
@@ -89,15 +90,17 @@ timeline; it is not sent to DeepSeek.
|
||||
|
||||
## 2. Add DeepSeek pre-labels
|
||||
|
||||
Prompt v7 treats the EasyCom transcript marker `[L]` as laughter-only evidence,
|
||||
Prompt v11 first reconstructs the exact participant-resolved chronology and
|
||||
then uses transcript content and `Target_of_Speech` to determine conversational
|
||||
function. It treats the EasyCom transcript marker `[L]` as laughter-only evidence,
|
||||
not as a lexical floor take. Wearer laughter during another participant's
|
||||
retained turn is directed toward `EOT_WEARER_BACKCHANNEL`; non-wearer laughter
|
||||
retained turn is directed toward `ENTRY_WEARER_BACKCHANNEL`; non-wearer laughter
|
||||
overlapping a retained wearer turn is directed toward `INT_OTHER_BACKCHANNEL`.
|
||||
Ambiguous laughter remains `UNSURE`, and `[L]` alone is neither an exclusion
|
||||
reason nor evidence of a visible smile. The prompt also treats the two EOT
|
||||
anchor options and strict INT ordering as deterministic prerequisites. For a
|
||||
positive EOT response it must return `selected_opportunity_uid` from the supplied
|
||||
options; all other decisions use `null`.
|
||||
Ambiguous laughter falls back to `EXCLUDE`, and `[L]` alone is neither evidence
|
||||
of a floor take nor evidence of a visible smile. Preceding release options are
|
||||
optional entry metadata, whereas strict INT ordering remains a deterministic
|
||||
retrieval prerequisite. A positive `WEARER_ENTRY` may optionally select one
|
||||
supplied release; entries from silence or during other activity use `null`.
|
||||
|
||||
`needs_human_review` is not requested from DeepSeek: every pre-label enters the
|
||||
human queue by protocol, so the adjudicator assigns that boolean locally. This
|
||||
@@ -127,12 +130,26 @@ python3 adjudicate_easycom_eot_int_candidates.py \
|
||||
```
|
||||
|
||||
DeepSeek sees candidate source, structured timing, per-speaker activity, and
|
||||
transcript together. It returns one of `POSITIVE`, `HARD_NEGATIVE`,
|
||||
`NOT_APPLICABLE`, `EXCLUDE`, or `UNSURE`; positives additionally receive
|
||||
`EOT_RESPONSE` or `INT_YIELD`, and hard negatives receive a named subtype. It
|
||||
transcript together. It returns one of `POSITIVE`, `HARD_NEGATIVE`, or
|
||||
`EXCLUDE`; positives additionally receive
|
||||
`WEARER_ENTRY` or `INT_YIELD`, and hard negatives receive a named subtype. It
|
||||
cannot modify timestamps. The output preserves the original rule candidate,
|
||||
the selected EOT opportunity UID when applicable, and the complete AI
|
||||
decision/provenance.
|
||||
the selected preceding-release UID when applicable, and the complete AI
|
||||
decision/provenance. `EXCLUDE` is the sole non-scoring outcome. It covers
|
||||
out-of-scope non-events (including a normal wearer completion that merely
|
||||
coincides with feedback), insufficient evidence, unresolvable data quality, or
|
||||
disagreement between the two model passes.
|
||||
|
||||
Hard negatives use behavior-specific subtypes. ENTRY distinguishes a same-other
|
||||
mid-turn pause, a different-other transition, and a wearer backchannel that
|
||||
does not take the floor. INT first distinguishes an other backchannel, then a
|
||||
non-backchannel overlap after which the wearer demonstrably holds or resumes
|
||||
the floor, and finally another interpretable overlap that fails to become a
|
||||
sustained takeover. `OTHER_HARD_NEGATIVE` is reserved for valid false-positive
|
||||
opportunities outside these named behaviors; it is never an uncertainty
|
||||
fallback. A short but complete answer to an addressed question remains a
|
||||
positive wearer entry; duration alone and later resumption by another speaker
|
||||
do not turn it into a wearer backchannel.
|
||||
|
||||
During live adjudication, the CLI displays a dependency-free progress bar over
|
||||
all candidate/pass work, including the current pass, batch, percentage, and
|
||||
@@ -160,10 +177,10 @@ question, then presents the exact AI verdict/subtype with a plain-language
|
||||
meaning; the Accept control repeats that exact label so its effect is explicit.
|
||||
The common path is one operation:
|
||||
|
||||
- **Accept** keeps the AI verdict/type and its selected EOT opportunity anchor;
|
||||
- **Relabel** expands verdict and subtype controls and, for a positive EOT,
|
||||
requires one of the supplied release anchors;
|
||||
- **Exclude** requires a concrete data-quality reason.
|
||||
- **Accept** keeps the AI verdict/type and optional preceding-release metadata;
|
||||
- **Relabel** expands verdict and subtype controls; a positive entry may
|
||||
optionally select one of the supplied release anchors;
|
||||
- **Exclude** requires a concrete out-of-scope or data-quality reason.
|
||||
|
||||
The current primary-review UI deliberately has no second-review action. Older
|
||||
saved `second_review` records remain readable for backward compatibility.
|
||||
@@ -185,8 +202,9 @@ Protocol wording for the paper appendix:
|
||||
|
||||
> Raw wearer speech onsets are first grouped into floor-level speech episodes
|
||||
> by merging short within-turn resumptions that are not separated by a
|
||||
> competing floor takeover. Only the resulting floor-level 0→1 transitions are
|
||||
> proposed as EOT-response candidates.
|
||||
> competing floor takeover. Every resulting floor-level 0→1 transition is
|
||||
> proposed as a wearer-entry candidate; a preceding non-wearer release is
|
||||
> retained as context rather than required for positive status.
|
||||
|
||||
> The original speech intervals are preserved unchanged for dense forecasting
|
||||
> and auditability; this grouping is used only for behavioral event annotation.
|
||||
|
||||
@@ -230,10 +230,6 @@ textarea { resize: vertical; line-height: 1.45; }
|
||||
.ai-rationale p { margin: 0; color: #d7e6f5; font-size: .7rem; line-height: 1.3; display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; }
|
||||
.ai-decision-card[data-verdict="POSITIVE"] { border-color: rgba(52,211,153,.48); box-shadow: inset 3px 0 0 var(--success); }
|
||||
.ai-decision-card[data-verdict="POSITIVE"] .ai-verdict { color: #86efac; background: rgba(52,211,153,.13); }
|
||||
.ai-decision-card[data-verdict="NOT_APPLICABLE"] { border-color: var(--line-strong); box-shadow: inset 3px 0 0 #71829a; }
|
||||
.ai-decision-card[data-verdict="NOT_APPLICABLE"] .ai-verdict { color: #c4d0df; background: rgba(113,130,154,.14); }
|
||||
.ai-decision-card[data-verdict="UNSURE"] { border-color: rgba(167,139,250,.48); box-shadow: inset 3px 0 0 #a78bfa; }
|
||||
.ai-decision-card[data-verdict="UNSURE"] .ai-verdict { color: #c4b5fd; background: rgba(167,139,250,.13); }
|
||||
.ai-decision-card[data-verdict="EXCLUDE"] { border-color: rgba(251,113,133,.48); box-shadow: inset 3px 0 0 var(--danger); }
|
||||
.ai-decision-card[data-verdict="EXCLUDE"] .ai-verdict { color: #fda4af; background: rgba(251,113,133,.13); }
|
||||
.event-position { color: var(--muted); font-size: 0.8rem; }
|
||||
|
||||
@@ -31,7 +31,7 @@ const elements = Object.fromEntries(
|
||||
);
|
||||
|
||||
const familyNames = {
|
||||
eot: "EOT-RESPONSE",
|
||||
entry: "WEARER-ENTRY",
|
||||
int: "INT-YIELD",
|
||||
onset: "WEARER ONSET",
|
||||
offset: "LEGACY EOT PROXY",
|
||||
@@ -39,7 +39,7 @@ const familyNames = {
|
||||
};
|
||||
|
||||
const reviewQuestions = {
|
||||
eot: "Other 释放 floor 后，wearer 是否真正持续接话？",
|
||||
entry: "这次 floor-level wearer onset 是否是真正进入或重新进入 floor？",
|
||||
int: "Other 是否在 wearer 发言中接管，使 wearer 让出 floor？",
|
||||
onset: "这次 wearer onset 是否代表新的 floor entry？",
|
||||
offset: "这次 wearer offset 是否代表让出 floor？",
|
||||
@@ -47,22 +47,22 @@ const reviewQuestions = {
|
||||
};
|
||||
|
||||
const decisionMeanings = {
|
||||
EOT_RESPONSE: "Other 释放 floor 后，wearer 持续接话。",
|
||||
WEARER_ENTRY: "Wearer 真正进入或重新进入 floor；不要求此前 other 已释放。",
|
||||
INT_YIELD: "Other 在 wearer 发言中接管，wearer 随后让出 floor。",
|
||||
EOT_MIDTURN_PAUSE: "Other 只是短暂停顿后继续，没有释放 floor 给 wearer。",
|
||||
EOT_OTHER_TO_OTHER: "这是 non-wearer 之间的换人，不是 wearer 接话。",
|
||||
EOT_WEARER_BACKCHANNEL: "Wearer 只是反馈、笑声或短回应，没有接过 floor。",
|
||||
INT_OTHER_BACKCHANNEL: "Other 只是反馈或笑声，没有接管 wearer 的 floor。",
|
||||
INT_NON_TAKEOVER_OVERLAP: "虽然发生重叠，但 other 没有形成持续接管。",
|
||||
INT_WEARER_HOLDS_FLOOR: "Wearer 在重叠后仍持有或继续自己的 floor。",
|
||||
OTHER_HARD_NEGATIVE: "这是有效困难负例，但不属于更具体的负例类型。",
|
||||
NOT_APPLICABLE: "这是真实活动边界，但不属于当前 EOT/INT 评测问题。",
|
||||
UNSURE: "现有时间和转写证据不足，需要人工直接判断标签。",
|
||||
EXCLUDE: "该区域存在数据质量问题，不应进入可靠评分。",
|
||||
ENTRY_OTHER_MIDTURN_PAUSE: "同一个 other 只是在自己的 turn 内暂停后继续；wearer 没有进入 floor。",
|
||||
ENTRY_OTHER_TO_OTHER: "一个 other 结束后由另一个 other 接话；wearer 没有进入 floor。",
|
||||
ENTRY_WEARER_BACKCHANNEL: "Wearer 只有附和或反馈，没有形成真实 floor entry；other 保持或恢复 floor。",
|
||||
INT_OTHER_BACKCHANNEL: "Other 只是反馈、笑声或附和，且 wearer 明确继续或恢复原 turn。若 wearer 正常说完，应 Exclude。",
|
||||
INT_NON_TAKEOVER_OVERLAP: "Other 不是单纯 backchannel，但重叠仍未形成持续接管。",
|
||||
INT_WEARER_HOLDS_FLOOR: "Other 不是单纯 backchannel，且 wearer 在重叠后明确继续或恢复原 turn。",
|
||||
OTHER_HARD_NEGATIVE: "这是可解释的困难负例，但不符合任何命名子类型；不能用于替代 Exclude。",
|
||||
EXCLUDE: "该区域是无需参评的非目标事件，或存在证据/数据问题，不进入评分。",
|
||||
};
|
||||
|
||||
const currentAiVerdicts = new Set(["POSITIVE", "HARD_NEGATIVE", "EXCLUDE"]);
|
||||
|
||||
function suggestionView(suggestion) {
|
||||
const verdict = suggestion.verdict || suggestion.protocol_disposition || "UNSURE";
|
||||
const verdict = suggestion.verdict || suggestion.protocol_disposition || "—";
|
||||
const label = suggestion.event_type || suggestion.subtype || suggestion.hard_negative_type || verdict;
|
||||
const rawConfidence = suggestion.confidence ?? suggestion.final_confidence;
|
||||
const numericConfidence = Number(rawConfidence);
|
||||
@@ -266,7 +266,7 @@ function renderDetail(detail) {
|
||||
elements.opportunitySelect.replaceChildren();
|
||||
const emptyOption = document.createElement("option");
|
||||
emptyOption.value = "";
|
||||
emptyOption.textContent = "请选择 preceding release";
|
||||
emptyOption.textContent = "无/不确定（可选）";
|
||||
elements.opportunitySelect.append(emptyOption);
|
||||
for (const option of opportunityOptions) {
|
||||
const control = document.createElement("option");
|
||||
@@ -288,12 +288,21 @@ function renderDetail(detail) {
|
||||
elements.aiLabel.textContent = view.label;
|
||||
elements.aiMeaning.textContent = view.meaning;
|
||||
elements.aiRationale.textContent = view.rationale;
|
||||
elements.acceptRadio.disabled = false;
|
||||
elements.acceptTitle.textContent = `接受 ${view.verdict.replaceAll("_", " ")}`;
|
||||
const familyEvent = detail.family === "entry" ? "WEARER_ENTRY" : detail.family === "int" ? "INT_YIELD" : null;
|
||||
const familyPrefix = detail.family === "entry" ? "ENTRY_" : detail.family === "int" ? "INT_" : null;
|
||||
const canAccept = currentAiVerdicts.has(view.verdict)
|
||||
&& (view.verdict !== "POSITIVE" || !familyEvent || suggestion.event_type === familyEvent)
|
||||
&& (view.verdict !== "HARD_NEGATIVE" || !familyPrefix || suggestion.subtype === "OTHER_HARD_NEGATIVE" || String(suggestion.subtype || "").startsWith(familyPrefix));
|
||||
elements.acceptRadio.disabled = !canAccept;
|
||||
elements.acceptTitle.textContent = canAccept
|
||||
? `接受 ${view.verdict.replaceAll("_", " ")}`
|
||||
: `旧标签 ${view.verdict.replaceAll("_", " ")}`;
|
||||
const selected = suggestion.selected_opportunity_uid;
|
||||
const selectedOption = opportunityOptions.find((option) => option.uid === selected);
|
||||
const anchorText = selectedOption ? ` · trigger ${formatTime(selectedOption.anchor_seconds)} P${selectedOption.participant_id}` : "";
|
||||
elements.acceptSummary.textContent = `${view.label}${anchorText} · ${view.meaning}`;
|
||||
elements.acceptSummary.textContent = canAccept
|
||||
? `${view.label}${anchorText} · ${view.meaning}`
|
||||
: "该标签已移除，请改标为正例/硬负例，或选择 Exclude。";
|
||||
} else {
|
||||
elements.acceptRadio.disabled = true;
|
||||
elements.acceptTitle.textContent = "没有 AI 结论";
|
||||
@@ -537,7 +546,6 @@ function validateReview(review) {
|
||||
if (review.human_action === "relabel" && !review.human_verdict) missing.push("一级 verdict");
|
||||
if (review.human_action === "relabel" && review.human_verdict === "POSITIVE" && !review.human_event_type) missing.push("positive event");
|
||||
const options = state.currentDetail?.candidate?.temporal_evidence?.opportunity_options || [];
|
||||
if (review.human_action === "relabel" && review.human_verdict === "POSITIVE" && review.human_event_type === "EOT_RESPONSE" && options.length && !review.selected_opportunity_uid) missing.push("EOT opportunity anchor");
|
||||
if (review.human_action === "relabel" && review.human_verdict === "HARD_NEGATIVE" && !review.human_subtype) missing.push("hard-negative subtype");
|
||||
if (review.human_action === "exclude" && !review.exclude_reason) missing.push("排除原因");
|
||||
if (review.human_action === "accept" && !state.currentDetail?.candidate?.ai_verdict && !state.currentDetail?.llm_suggestion?.verdict) missing.push("AI 预标注（尚未运行，需 Relabel）");
|
||||
@@ -552,7 +560,7 @@ function updateConditionalFields() {
|
||||
elements.relabelFields.hidden = action !== "relabel";
|
||||
elements.excludeFields.hidden = action !== "exclude";
|
||||
elements.eventTypeField.hidden = action !== "relabel" || verdict !== "POSITIVE";
|
||||
elements.opportunityField.hidden = action !== "relabel" || verdict !== "POSITIVE" || eventType !== "EOT_RESPONSE" || !hasOpportunityOptions;
|
||||
elements.opportunityField.hidden = action !== "relabel" || verdict !== "POSITIVE" || eventType !== "WEARER_ENTRY" || !hasOpportunityOptions;
|
||||
elements.subtypeField.hidden = action !== "relabel" || verdict !== "HARD_NEGATIVE";
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
<section class="filters" aria-label="候选筛选">
|
||||
<label><span>Session</span><select id="sessionFilter"><option value="all">全部</option></select></label>
|
||||
<label><span>Split</span><select id="splitFilter"><option value="all">全部</option><option value="train">train</option><option value="test">test</option></select></label>
|
||||
<label><span>任务</span><select id="familyFilter"><option value="all">全部</option><option value="eot">EOT / ENTRY</option><option value="int">INT / YIELD</option></select></label>
|
||||
<label><span>任务</span><select id="familyFilter"><option value="all">全部</option><option value="entry">WEARER ENTRY</option><option value="int">INT / YIELD</option></select></label>
|
||||
<label><span>审核状态</span><select id="statusFilter"><option value="unreviewed">未审核</option><option value="all">全部</option><option value="reviewed">已审核</option></select></label>
|
||||
<label class="search-field"><span>搜索</span><input id="searchFilter" type="search" placeholder="事件来源、AI 标签、session" /></label>
|
||||
<button id="clearFilters" class="button ghost" type="button">清除筛选</button>
|
||||
@@ -77,18 +77,18 @@
|
||||
<div class="choice-grid two action-grid">
|
||||
<label class="accept-option"><input id="acceptRadio" type="radio" name="human_action" value="accept" /><span><em>AI 建议</em><b id="acceptTitle">接受 AI 结论</b><small id="acceptSummary">载入候选后显示具体标签</small></span></label>
|
||||
<label><input type="radio" name="human_action" value="relabel" /><span><b>Relabel</b><small>展开并选择正确标签</small></span></label>
|
||||
<label><input type="radio" name="human_action" value="exclude" /><span><b>Exclude</b><small>数据不可可靠评分</small></span></label>
|
||||
<label><input type="radio" name="human_action" value="exclude" /><span><b>Exclude</b><small>非目标事件或不可可靠评分</small></span></label>
|
||||
</div>
|
||||
</fieldset>
|
||||
|
||||
<section id="relabelFields" class="conditional-fields" hidden>
|
||||
<label><span>一级 verdict</span><select name="human_verdict"><option value="">请选择</option><option>POSITIVE</option><option>HARD_NEGATIVE</option><option>NOT_APPLICABLE</option><option>UNSURE</option></select></label>
|
||||
<label id="eventTypeField" hidden><span>Positive event</span><select name="human_event_type"><option value="">请选择</option><option>EOT_RESPONSE</option><option>INT_YIELD</option></select></label>
|
||||
<label id="opportunityField" hidden><span>EOT opportunity anchor</span><select id="opportunitySelect" name="selected_opportunity_uid"><option value="">请选择 preceding release</option></select></label>
|
||||
<label id="subtypeField" hidden><span>Hard-negative subtype</span><select name="human_subtype"><option value="">请选择</option><option>EOT_MIDTURN_PAUSE</option><option>EOT_OTHER_TO_OTHER</option><option>EOT_WEARER_BACKCHANNEL</option><option>INT_OTHER_BACKCHANNEL</option><option>INT_NON_TAKEOVER_OVERLAP</option><option>INT_WEARER_HOLDS_FLOOR</option><option>OTHER_HARD_NEGATIVE</option></select></label>
|
||||
<label><span>一级 verdict</span><select name="human_verdict"><option value="">请选择</option><option>POSITIVE</option><option>HARD_NEGATIVE</option></select></label>
|
||||
<label id="eventTypeField" hidden><span>Positive event</span><select name="human_event_type"><option value="">请选择</option><option>WEARER_ENTRY</option><option>INT_YIELD</option></select></label>
|
||||
<label id="opportunityField" hidden><span>前序 release（可选 metadata）</span><select id="opportunitySelect" name="selected_opportunity_uid"><option value="">无/不确定</option></select></label>
|
||||
<label id="subtypeField" hidden><span>Hard-negative subtype</span><select name="human_subtype"><option value="">请选择</option><option>ENTRY_OTHER_MIDTURN_PAUSE</option><option>ENTRY_OTHER_TO_OTHER</option><option>ENTRY_WEARER_BACKCHANNEL</option><option>INT_OTHER_BACKCHANNEL</option><option>INT_NON_TAKEOVER_OVERLAP</option><option>INT_WEARER_HOLDS_FLOOR</option><option>OTHER_HARD_NEGATIVE</option></select></label>
|
||||
</section>
|
||||
|
||||
<section id="excludeFields" class="conditional-fields" hidden><label><span>排除原因</span><select name="exclude_reason"><option value="">请选择</option><option>ANNOTATION_ERROR</option><option>RECORDING_BOUNDARY</option><option>MISSING_CONTEXT</option><option>NON_SPEECH_ARTIFACT</option><option>TIMESTAMP_CORRUPTION</option><option>UNRESOLVABLE</option></select></label></section>
|
||||
<section id="excludeFields" class="conditional-fields" hidden><label><span>排除原因</span><select name="exclude_reason"><option value="">请选择</option><option value="OUT_OF_SCOPE_NORMAL_COMPLETION">OUT_OF_SCOPE_NORMAL_COMPLETION — wearer 正常说完</option><option value="OUT_OF_SCOPE_NON_EVENT">OUT_OF_SCOPE_NON_EVENT — 其他无需参评事件</option><option>ANNOTATION_ERROR</option><option>RECORDING_BOUNDARY</option><option>MISSING_CONTEXT</option><option>NON_SPEECH_ARTIFACT</option><option>TIMESTAMP_CORRUPTION</option><option>UNRESOLVABLE</option></select></label></section>
|
||||
|
||||
<details class="boundary-details"><summary>仅在原 annotation 明显错误时校正边界</summary><label><span>可选校正（原时间始终保留）</span><select name="boundary_correction_seconds"><option value="">不校正</option><option value="-0.2">−0.2 s</option><option value="0.2">+0.2 s</option></select></label></details>
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ We therefore study a candidate causal local floor adapter that combines the glob
|
||||
Omni state with short-window mixed audio and an anonymous set of visible-person
|
||||
observations, without assuming persistent speaker identity. On EasyCom,
|
||||
\todo{report the final three-seed dense and committed-event
|
||||
results: EOT-response and INT-yield recall, false-positive rate, latency,
|
||||
results: wearer-entry and INT-yield recall, false-positive rate, latency,
|
||||
confidence intervals, and oracle-gap recovery}. Controlled
|
||||
\todo{time-shift/remove-vision/same-rate or capacity-matched} interventions
|
||||
show that \todo{state the supported mechanism, without claiming addressee or
|
||||
@@ -103,7 +103,7 @@ corresponding TODO gates pass:
|
||||
long-context Omni information with short-window audio and anonymous
|
||||
visible-person evidence.
|
||||
\item We evaluate whether the adapter reduces erroneous wearer commits
|
||||
during multiparty floor transfers under an EOT/INT event protocol with
|
||||
during multiparty floor transfers under an entry/INT event protocol with
|
||||
explicitly annotated hard-negative regions.
|
||||
\end{itemize}
|
||||
|
||||
@@ -112,7 +112,7 @@ corresponding TODO gates pass:
|
||||
\fbox{\parbox[c][1.00in][c]{0.94\columnwidth}{\centering
|
||||
\todo{Draw only the selected method: causal AV to the long-context Omni path;
|
||||
the short-window mixed-audio and anonymous person-set path; residual fusion;
|
||||
the common $10\times3$ head; and the separate EOT-response/INT-yield evaluation.
|
||||
the common $10\times3$ head; and the separate wearer-entry/INT-yield evaluation.
|
||||
Mark physical cutoffs, trainable modules, person masks, and NULL/OFFSCREEN.
|
||||
Do not draw oracle labels, rejected track variants, or multiple backbones.}}}
|
||||
\caption{Long-context Omni understanding and short-window local floor evidence
|
||||
@@ -140,7 +140,7 @@ The wearer-activity forecast also provides a causal control score. When the
|
||||
wearer is silent, $\hat y^{\mathrm w}_{t,1}$ acts as a near-term take-turn
|
||||
score; while the wearer is active, $1-\hat y^{\mathrm w}_{t,1}$ acts as a
|
||||
yield score. These scores are trained only from future activity targets.
|
||||
Committed EOT-response and INT-yield events used for behavioral evaluation are
|
||||
Committed wearer-entry and INT-yield events used for behavioral evaluation are
|
||||
defined separately from participant-resolved annotations in
|
||||
Sec.~\ref{sec:experiments}.
|
||||
|
||||
@@ -235,15 +235,16 @@ behavioral evaluation.
|
||||
\textbf{Behavioral event annotation.}
|
||||
We use floor-level wearer transitions as high-recall anchors for positive-event
|
||||
annotation. Each wearer onset is examined retrospectively for a preceding
|
||||
non-wearer floor release and labeled as an EOT-response only when the subsequent
|
||||
wearer speech constitutes a genuine floor take. Similarly, each wearer offset
|
||||
non-wearer floor release as contextual metadata and labeled as a wearer-entry
|
||||
whenever the subsequent wearer speech constitutes a genuine floor take,
|
||||
including re-entry while another speaker is active. Similarly, each wearer offset
|
||||
is examined for a new competing non-wearer onset during the current wearer turn
|
||||
and labeled as INT-yield only when the competing speaker genuinely takes over
|
||||
the floor. We use permissive temporal retrieval and let an LLM jointly inspect
|
||||
participant-resolved timing and transcripts to provide a semantic pre-label,
|
||||
followed by mandatory human adjudication. Separately, we mine regions with no
|
||||
wearer response that nevertheless constitute false-positive opportunities:
|
||||
same-speaker pauses and other-to-other transitions for EOT, and backchannels or
|
||||
same-speaker pauses and other-to-other transitions for entry, and backchannels or
|
||||
non-takeover overlaps for INT. Only human-confirmed positives and hard negatives
|
||||
enter evaluation. Appendix~\ref{app:event_annotation} gives the full protocol.
|
||||
|
||||
@@ -259,8 +260,8 @@ only.}
|
||||
\hline
|
||||
Annotation outcome & Train/dev & Test \\
|
||||
\hline
|
||||
\multicolumn{3}{l}{\emph{EOT-response}}\\
|
||||
Positive: EOT-response & \todo{--} & \todo{--} \\
|
||||
\multicolumn{3}{l}{\emph{Wearer entry}}\\
|
||||
Positive: wearer entry & \todo{--} & \todo{--} \\
|
||||
HN: same-speaker pause & \todo{--} & \todo{--} \\
|
||||
HN: other-to-other transition & \todo{--} & \todo{--} \\
|
||||
HN: wearer backchannel & \todo{--} & \todo{--} \\
|
||||
@@ -271,9 +272,7 @@ HN: other backchannel & \todo{--} & \todo{--} \\
|
||||
HN: non-takeover overlap & \todo{--} & \todo{--} \\
|
||||
HN: wearer holds floor & \todo{--} & \todo{--} \\
|
||||
\hline
|
||||
Not applicable & \todo{--} & \todo{--} \\
|
||||
Excluded / unscorable & \todo{--} & \todo{--} \\
|
||||
Unresolved & \todo{--} & \todo{--} \\
|
||||
Excluded / fallback & \todo{--} & \todo{--} \\
|
||||
\textbf{Scoring-eligible total} & \todo{--} & \todo{--} \\
|
||||
\hline
|
||||
\end{tabular}
|
||||
@@ -283,10 +282,10 @@ Participant identity is used only to construct these evaluation annotations
|
||||
and is never provided to the prediction model. Provided person boxes are used
|
||||
only for the causal visible-person input; transcripts, target-of-speech labels,
|
||||
close-microphone audio, and future observations are unavailable to the model.
|
||||
\todo{Report duration and event statistics: EOT positives/hard negatives,
|
||||
\todo{Report duration and event statistics: entry positives/hard negatives,
|
||||
INT positives/hard negatives, excluded candidates, and audited-region counts
|
||||
for train/development and test. Freeze the event-construction and region-audit
|
||||
protocol before reporting semantic EOT/INT results; until then, retain the
|
||||
protocol before reporting semantic entry/INT results; until then, retain the
|
||||
timing-derived proxy terminology.}
|
||||
|
||||
\subsection{Model Configuration and Training}
|
||||
@@ -324,7 +323,7 @@ We first retain dense future-activity evaluation for direct comparison with
|
||||
EgoSpeak, reporting AP at the 0.2--2.0~s horizons and their mean mAP.
|
||||
|
||||
Behavioral evaluation follows a committed-event protocol. For both
|
||||
EOT-response and INT-yield, each gold event is matched to at most one committed
|
||||
wearer-entry and INT-yield, each gold event is matched to at most one committed
|
||||
prediction and is scored by recall and signed detection latency.
|
||||
|
||||
\textbf{False-positive scoring rule.}
|
||||
@@ -335,12 +334,12 @@ situations in which the correct wearer behavior is to remain silent or retain
|
||||
the floor.
|
||||
|
||||
We report the recall--false-positive trade-off and a common operating point at
|
||||
matched recall $R=0.8$. The primary behavioral quantities are therefore EOT
|
||||
matched recall $R=0.8$. The primary behavioral quantities are therefore entry
|
||||
false-positive rate, INT false-positive rate, and signed latency at the frozen
|
||||
operating point. We additionally report performance on the other-to-other
|
||||
subset to test whether local visual evidence specifically reduces erroneous
|
||||
wearer turn-taking during multiparty floor transfers. \todo{Freeze the FPR
|
||||
denominators and report signed median latency separately for EOT and INT.}
|
||||
denominators and report signed median latency separately for entry and INT.}
|
||||
|
||||
|
||||
\section{Results and Analysis}
|
||||
@@ -358,8 +357,8 @@ frozen before test evaluation. \todo{Freeze all baseline implementations and
|
||||
verify identical targets, causal validity, and evaluation.}
|
||||
|
||||
\begin{table}[t]
|
||||
\caption{Confirmatory EasyCom results. EOT and INT false-positive rates are
|
||||
measured at matched recall $R=0.8$. E/I Lat. gives signed median EOT/INT
|
||||
\caption{Confirmatory EasyCom results. Entry and INT false-positive rates are
|
||||
measured at matched recall $R=0.8$. E/I Lat. gives signed median entry/INT
|
||||
latency in seconds; values closer to zero are better.}
|
||||
\label{tab:main}
|
||||
\centering
|
||||
@@ -368,7 +367,7 @@ latency in seconds; values closer to zero are better.}
|
||||
\begin{tabular}{lccccc}
|
||||
\hline
|
||||
Method & 0.2-s mAP $\uparrow$ & Mean mAP $\uparrow$
|
||||
& EOT FPR@$R=.8$ $\downarrow$ & INT FPR@$R=.8$ $\downarrow$ & E/I Lat.\\
|
||||
& Entry FPR@$R=.8$ $\downarrow$ & INT FPR@$R=.8$ $\downarrow$ & E/I Lat.\\
|
||||
\hline
|
||||
EgoSpeak reproduction & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--/--}\\
|
||||
Omni-3B frozen $G$ & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--/--}\\
|
||||
@@ -390,9 +389,9 @@ frames / specify actual observation}, indicating that local audiovisual
|
||||
evidence primarily improves \todo{the interpretation supported by the data}.
|
||||
|
||||
\textbf{Committed behavior.}
|
||||
At matched EOT-response recall $R=0.8$, the proposed adapter reduces EOT FPR
|
||||
At matched wearer-entry recall $R=0.8$, the proposed adapter reduces entry FPR
|
||||
from \todo{$X$} to \todo{$Y$}; at matched INT-yield recall, it changes INT FPR
|
||||
from \todo{$A$} to \todo{$B$}. Its signed median EOT/INT latencies are
|
||||
from \todo{$A$} to \todo{$B$}. Its signed median entry/INT latencies are
|
||||
\todo{$+0.XX/+0.XX$~s}. These recall-matched error rates and signed latencies
|
||||
test whether fewer erroneous commits arise from better floor decisions rather
|
||||
than indefinite waiting.
|
||||
@@ -450,7 +449,7 @@ Cross-family raw scores are diagnostic.}
|
||||
\resizebox{\columnwidth}{!}{%
|
||||
\begin{tabular}{lcccc}
|
||||
\hline
|
||||
Condition & Mean mAP $\uparrow$ & EOT FPR $\downarrow$
|
||||
Condition & Mean mAP $\uparrow$ & Entry FPR $\downarrow$
|
||||
& O$\rightarrow$O FPR $\downarrow$ & INT FPR $\downarrow$\\
|
||||
\hline
|
||||
\multicolumn{5}{l}{\emph{Local-evidence controls (Qwen2.5-Omni-3B)}}\\
|
||||
@@ -472,7 +471,7 @@ MiniCPM-o, audited cached 5-Hz branch
|
||||
\end{tabular}}
|
||||
\end{table}
|
||||
|
||||
Removing local vision changes \todo{EOT/O$\rightarrow$O/INT FPR by $X/Y/Z$},
|
||||
Removing local vision changes \todo{entry/O$\rightarrow$O/INT FPR by $X/Y/Z$},
|
||||
while shifting the visual stream by \todo{$\Delta t$} produces \todo{result}.
|
||||
Replacing person-local evidence with full-frame vision produces \todo{result}.
|
||||
If aligned local AV improves estimates of surrounding floor occupancy, its
|
||||
@@ -489,7 +488,7 @@ not describe lower false entry as success.}
|
||||
\subsection{Scaling and Streaming Controls}
|
||||
|
||||
Increasing the frozen Omni backbone from 3B to 7B yields
|
||||
\todo{$X$ points} in 0.2-s mAP and \todo{EOT/INT FPR of $Y/Z$} at matched
|
||||
\todo{$X$ points} in 0.2-s mAP and \todo{entry/INT FPR of $Y/Z$} at matched
|
||||
recall.
|
||||
The proposed 3B adapter \todo{remains better than / approaches / is surpassed
|
||||
by} the 7B scaling control.
|
||||
@@ -540,7 +539,7 @@ This study asks whether a general-purpose Omni representation can be adapted
|
||||
for causal floor forecasting of an invisible egocentric wearer. The candidate
|
||||
method separates long-context Omni understanding from short-window local floor
|
||||
evidence without assuming persistent speaker identity. \todo{State the final
|
||||
learned-method improvement, matched-recall EOT/INT false-positive rates,
|
||||
learned-method improvement, matched-recall entry/INT false-positive rates,
|
||||
signed latency, causal-control evidence, and main limitation. If no learned
|
||||
variant passes these tests, reframe the submission as an information audit or
|
||||
do not make the method claim.}
|
||||
@@ -566,16 +565,20 @@ non-wearer activity. Short feedback-like activity does not by itself split the
|
||||
wearer episode. This grouping changes only the behavioral-event layer, not the
|
||||
source intervals or dense targets.
|
||||
|
||||
\textbf{EOT-response.}
|
||||
Starting from a floor-level wearer onset at $t_b$, we search backward for
|
||||
plausible non-wearer floor releases. A release at $t_r$ is retained as an option
|
||||
\textbf{Wearer entry.}
|
||||
Every floor-level wearer onset at $t_b$ is proposed as a wearer-entry candidate.
|
||||
We additionally search backward for plausible non-wearer floor releases. A
|
||||
release at $t_r$ is retained as a contextual option
|
||||
only when the interval from $t_r$ to $t_b$ contains neither intervening wearer
|
||||
speech nor a competing non-wearer onset. Each wearer onset contributes at most
|
||||
one review item: $t_b$ is its immutable \emph{behavior anchor}, and the item
|
||||
stores every admissible \emph{opportunity-anchor} option and response delay.
|
||||
Silence duration alone does not reject an option. A positive EOT annotation
|
||||
must select one supplied release; neither the LLM nor the human can invent or
|
||||
move a trigger timestamp. Unpaired wearer onsets remain audit controls.
|
||||
Silence duration alone does not reject an option. A preceding release is not
|
||||
required for a positive entry: the wearer may enter after a release, from
|
||||
silence, or while another speaker remains active. When a release clearly
|
||||
prompts the entry, the LLM or human may select one supplied option; neither may
|
||||
invent or move a trigger timestamp. The release relation is metadata for
|
||||
response-oriented analyses, not part of the positive entry label.
|
||||
|
||||
\textbf{INT-yield.}
|
||||
Starting from an episode-ending wearer offset $t_y$, we search backward inside
|
||||
@@ -594,7 +597,7 @@ the competitor genuinely takes over and the wearer yields.
|
||||
\subsection{False-positive opportunity mining}
|
||||
|
||||
Hard-negative regions are retrieved independently of positive wearer
|
||||
transitions. For EOT, a same-speaker pause followed by resumption and an
|
||||
transitions. For entry, a same-speaker pause followed by resumption and an
|
||||
other-to-other handover while the wearer stays silent are false-entry
|
||||
opportunities. For INT, a short non-wearer backchannel or a non-takeover overlap
|
||||
during which the wearer retains and continues the floor is a false-yield
|
||||
@@ -606,26 +609,30 @@ labels.
|
||||
The annotation assistant jointly receives immutable timestamps,
|
||||
participant-resolved activity, neighboring boundaries, gap and overlap
|
||||
measurements, transcripts, and target-of-speech metadata when available. It is
|
||||
instructed not to equate onset with entry, offset with yield, silence with EOT,
|
||||
instructed not to equate raw onset with floor entry, offset with yield, or silence
|
||||
with an entry opportunity,
|
||||
or overlap with interruption, and not to invent unavailable prosody, gaze, or
|
||||
intent. EasyCom's \texttt{[L]} marker supplies laughter evidence only and is not
|
||||
by itself a sustained floor take or takeover. A positive EOT pre-label must
|
||||
also select one of the supplied opportunity-anchor options.
|
||||
by itself a sustained floor take or takeover. Preceding release options are
|
||||
optional metadata for a positive wearer-entry pre-label.
|
||||
|
||||
We use \texttt{deepseek-v4-flash} at temperature zero with a primary and a
|
||||
skeptical pass. Disagreement on verdict, event type, or hard-negative subtype
|
||||
produces an \textsc{Unsure} suggestion. Every candidate then receives mandatory
|
||||
human adjudication: accept the complete pre-label and selected EOT trigger,
|
||||
relabel it (selecting a displayed trigger for a positive EOT), or exclude an
|
||||
unscorable region. Only human-confirmed \textsc{EOT-Response},
|
||||
produces an \textsc{Exclude} suggestion. Every candidate then receives mandatory
|
||||
human adjudication: accept the complete pre-label and optional release metadata,
|
||||
relabel it, or exclude an
|
||||
unscorable region. Only human-confirmed \textsc{Wearer-Entry},
|
||||
\textsc{INT-Yield}, and hard-negative opportunities enter evaluation;
|
||||
\textsc{Not-Applicable}, \textsc{Exclude}, and unresolved candidates do not.
|
||||
\textsc{Exclude} candidates do not. Thus \textsc{Exclude} is the sole
|
||||
non-scoring outcome for out-of-scope non-events (e.g., natural wearer completion
|
||||
that merely coincides with feedback), insufficient evidence, unresolvable data
|
||||
quality, or pre-label disagreement.
|
||||
Annotation-only transcripts, identities, target-of-speech metadata, LLM output,
|
||||
and future observations are never supplied to the evaluated forecasting model.
|
||||
|
||||
\todo{Before freezing this annotation protocol, audit randomly sampled
|
||||
continuous timeline segments without candidate hints to estimate candidate
|
||||
recall for EOT, INT, and each hard-negative family. Independently double-label
|
||||
recall for entry, INT, and each hard-negative family. Independently double-label
|
||||
a pre-registered subset, report its size and agreement statistic, freeze all
|
||||
candidate parameters and prompt/model identifiers on Sessions 4--12, and
|
||||
report final candidate, control, exclusion, and adjudicated-label counts.}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate wearer-centric EasyCom EOT/INT candidates for AI and human review.
|
||||
"""Generate wearer-centric EasyCom entry/INT candidates for AI and human review.
|
||||
|
||||
This stage never changes EasyCom speech boundaries and never claims semantic
|
||||
ground truth. It groups short wearer resumptions into floor-level episodes,
|
||||
starts EOT candidates from observed wearer entries and retrieves preceding
|
||||
non-wearer release options, requires a new
|
||||
starts entry candidates from every observed floor-level wearer onset and
|
||||
retains preceding non-wearer release options as context, requires a new
|
||||
non-wearer onset inside the wearer floor episode for INT candidates, and keeps
|
||||
unpaired or ineligible raw boundaries as deterministic controls. All timing
|
||||
ineligible raw boundaries as deterministic controls. All timing
|
||||
thresholds are CLI parameters, and raw evidence is retained for DeepSeek
|
||||
pre-labeling and human correction.
|
||||
"""
|
||||
@@ -28,8 +28,8 @@ from derive_easycom_interaction_events import (
|
||||
from easycom_manifest import ALL_SESSIONS, TEST_SESSIONS
|
||||
|
||||
|
||||
SCHEMA_VERSION = 4
|
||||
PROTOCOL_VERSION = "easycom-wearer-anchor-first-eot-int-candidates-v6"
|
||||
SCHEMA_VERSION = 5
|
||||
PROTOCOL_VERSION = "easycom-wearer-entry-int-candidates-v7"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -88,7 +88,7 @@ class ProtocolCandidate:
|
||||
def requires_semantic_adjudication(row: dict[str, Any]) -> bool:
|
||||
"""Return whether a stored row belongs in the primary AI/human queue.
|
||||
|
||||
Explicit v3--v5 control sources are authoritative. For old v2 candidate
|
||||
Explicit legacy control sources are authoritative. For old v2 candidate
|
||||
files, reconstruct the same-speaker crossing requirement from the stored
|
||||
activity IDs. Missing legacy evidence remains reviewable rather than being
|
||||
silently discarded.
|
||||
@@ -354,8 +354,8 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[
|
||||
|
||||
# A: classify raw wearer onsets before proposing any behavioral event.
|
||||
# Short uncompeted gaps are internal to a floor episode. All other raw
|
||||
# onsets begin a new floor episode, but become EOT candidates only when
|
||||
# they can be paired with a preceding non-wearer release opportunity.
|
||||
# onsets begin a new floor episode and are all entry candidates. A
|
||||
# preceding non-wearer release is optional context, not a gate.
|
||||
onset_records: dict[int, dict[str, Any]] = {}
|
||||
raw_offsets: list[int] = []
|
||||
for index in range(1, len(wearer)):
|
||||
@@ -412,10 +412,10 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[
|
||||
for frame in sorted(resume_onsets):
|
||||
record = onset_records[frame]
|
||||
add(_candidate(
|
||||
session=session, task="eot_response_control", source="WEARER_RESUME",
|
||||
session=session, task="wearer_entry_control", source="WEARER_RESUME",
|
||||
rule_type="WITHIN_TURN_RESUME_CONTROL", anchor_type="wearer_onset", frame=frame,
|
||||
region_start=frame, region_end=min(segment_end, frame + relation_frames),
|
||||
label="not_applicable", subtype=None,
|
||||
label="control", subtype=None,
|
||||
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
|
||||
incoming_ids=_active_other_ids(rows, frame, wearer_id=wearer_id),
|
||||
evidence=record["evidence"], clips=segment_clips, rows=rows,
|
||||
@@ -423,11 +423,9 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[
|
||||
reason="short raw wearer pause without sustained competing other takeover; merged into the existing floor episode",
|
||||
), ("WEARER_RESUME", frame, ()))
|
||||
|
||||
# A1: start from each observed floor-level wearer entry, then search
|
||||
# backward for release anchors that survive deterministic timing checks.
|
||||
# This yields at most one review item per wearer behavior. The semantic
|
||||
# annotator may select only one of the stored options; it cannot invent
|
||||
# a trigger or move either timestamp. Long silence alone is not a veto.
|
||||
# A1: every observed floor-level wearer onset is an entry candidate.
|
||||
# Search backward for release anchors only to preserve response context;
|
||||
# no preceding release is required for a positive WEARER_ENTRY.
|
||||
for onset_frame in floor_onsets:
|
||||
release_options: list[tuple[Utterance, int]] = []
|
||||
for release in other_rows:
|
||||
@@ -481,19 +479,24 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[
|
||||
minimum_response_delay_seconds=(onset_frame - latest_opportunity_frame) / SOURCE_FPS,
|
||||
maximum_response_delay_seconds=(onset_frame - earliest_opportunity_frame) / SOURCE_FPS,
|
||||
competing_other_onsets_before_behavior=[],
|
||||
eot_opportunity_paired=True,
|
||||
preceding_release_paired=True,
|
||||
entry_context_timing=(
|
||||
"OTHER_ACTIVE"
|
||||
if _active_other_ids(rows, onset_frame, wearer_id=wearer_id)
|
||||
else "AFTER_OTHER_RELEASE"
|
||||
),
|
||||
)
|
||||
previous_ids = tuple(sorted({row.participant_id for row, _ in release_options}))
|
||||
add(_candidate(
|
||||
session=session, task="eot_response", source="WEARER_ONSET",
|
||||
rule_type="EOT_OPPORTUNITY_CANDIDATE", anchor_type="wearer_onset",
|
||||
session=session, task="wearer_entry", source="WEARER_ONSET",
|
||||
rule_type="WEARER_ENTRY_CANDIDATE", anchor_type="wearer_onset",
|
||||
frame=onset_frame, region_start=earliest_opportunity_frame,
|
||||
region_end=min(segment_end, onset_frame + relation_frames),
|
||||
label="unsure", subtype=None, previous_ids=previous_ids,
|
||||
label="candidate", subtype=None, previous_ids=previous_ids,
|
||||
incoming_ids=_active_other_ids(rows, onset_frame, wearer_id=wearer_id),
|
||||
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
||||
context_frames=context_frames,
|
||||
reason="floor-level wearer entry with one or more surviving preceding release options",
|
||||
reason="floor-level wearer onset; preceding release options retained as context",
|
||||
), ("WEARER_ONSET", onset_frame, previous_ids))
|
||||
continue
|
||||
|
||||
@@ -507,19 +510,26 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[
|
||||
behavior_anchor_type="wearer_floor_onset",
|
||||
behavior_anchor_frame=frame,
|
||||
behavior_anchor_seconds=frame / SOURCE_FPS,
|
||||
eot_opportunity_paired=False,
|
||||
preceding_release_paired=False,
|
||||
entry_context_timing=(
|
||||
"OTHER_ACTIVE"
|
||||
if _active_other_ids(rows, frame, wearer_id=wearer_id)
|
||||
else "FROM_SILENCE_OR_UNRESOLVED"
|
||||
),
|
||||
)
|
||||
previous_ids = _active_other_ids(rows, frame - 1, wearer_id=wearer_id)
|
||||
add(_candidate(
|
||||
session=session, task="eot_response_control", source="WEARER_ONSET_CONTROL",
|
||||
rule_type="UNPAIRED_WEARER_ENTRY_CONTROL", anchor_type="wearer_onset", frame=frame,
|
||||
region_start=frame, region_end=min(segment_end, frame + relation_frames),
|
||||
label="not_applicable", subtype=None,
|
||||
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
|
||||
session=session, task="wearer_entry", source="WEARER_ONSET",
|
||||
rule_type="WEARER_ENTRY_CANDIDATE", anchor_type="wearer_onset", frame=frame,
|
||||
region_start=max(segment_start, frame - relation_frames),
|
||||
region_end=min(segment_end, frame + relation_frames),
|
||||
label="candidate", subtype=None,
|
||||
previous_ids=previous_ids,
|
||||
incoming_ids=_active_other_ids(rows, frame, wearer_id=wearer_id),
|
||||
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
||||
context_frames=context_frames,
|
||||
reason="floor-level wearer entry has no surviving preceding non-wearer release opportunity",
|
||||
), ("WEARER_ONSET_CONTROL", frame, ()))
|
||||
reason="floor-level wearer onset without a surviving preceding release option",
|
||||
), ("WEARER_ONSET", frame, previous_ids))
|
||||
|
||||
# A2: a raw wearer offset is an INT candidate only at the end of a
|
||||
# floor-level wearer episode and only if a new non-wearer onset obeys
|
||||
@@ -604,7 +614,7 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[
|
||||
anchor_type="wearer_offset", frame=frame,
|
||||
region_start=max(segment_start, frame - relation_frames),
|
||||
region_end=min(segment_end, frame + relation_frames),
|
||||
label="unsure" if is_int_candidate else "not_applicable", subtype=None,
|
||||
label="candidate" if is_int_candidate else "control", subtype=None,
|
||||
previous_ids=_active_other_ids(rows, frame - 1, wearer_id=wearer_id),
|
||||
incoming_ids=eligible_ids,
|
||||
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
||||
@@ -636,10 +646,10 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[
|
||||
"original_timestamp_seconds": frame / SOURCE_FPS,
|
||||
}
|
||||
add(_candidate(
|
||||
session=session, task="eot_response", source="SAME_SPEAKER_PAUSE",
|
||||
rule_type="EOT_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame,
|
||||
session=session, task="wearer_entry", source="SAME_SPEAKER_PAUSE",
|
||||
rule_type="ENTRY_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame,
|
||||
region_start=frame, region_end=resumed.dense_start_frame,
|
||||
label="hard_negative", subtype="EOT_MIDTURN_PAUSE",
|
||||
label="hard_negative", subtype="ENTRY_OTHER_MIDTURN_PAUSE",
|
||||
previous_ids=(participant_id,), incoming_ids=(participant_id,), evidence=evidence,
|
||||
clips=segment_clips, rows=rows, wearer_id=wearer_id, context_frames=context_frames,
|
||||
reason="same participant resumes after a configurable pause while wearer stays silent",
|
||||
@@ -669,10 +679,10 @@ def derive_protocol_candidates(utterances: Sequence[Utterance], clips: Sequence[
|
||||
"wearer_active_during_handover": False, "original_timestamp_seconds": frame / SOURCE_FPS,
|
||||
}
|
||||
add(_candidate(
|
||||
session=session, task="eot_response", source="OTHER_TO_OTHER",
|
||||
rule_type="EOT_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame,
|
||||
session=session, task="wearer_entry", source="OTHER_TO_OTHER",
|
||||
rule_type="ENTRY_HARD_NEGATIVE_REGION", anchor_type="other_offset", frame=frame,
|
||||
region_start=frame, region_end=incoming.dense_start_frame,
|
||||
label="hard_negative", subtype="EOT_OTHER_TO_OTHER",
|
||||
label="hard_negative", subtype="ENTRY_OTHER_TO_OTHER",
|
||||
previous_ids=(first.participant_id,), incoming_ids=(incoming.participant_id,),
|
||||
evidence=evidence, clips=segment_clips, rows=rows, wearer_id=wearer_id,
|
||||
context_frames=context_frames,
|
||||
|
||||
@@ -34,10 +34,10 @@ SOURCE_FPS = 20.0
|
||||
STATIC_ROOT = Path(__file__).resolve().parent / "floor_event_review_web"
|
||||
VIDEO_ROOT_PARTS = ("Main", "Video_Compressed")
|
||||
TRANSCRIPT_ROOT_PARTS = ("Main", "Speech_Transcriptions")
|
||||
REVIEW_SCHEMA_VERSION = 4
|
||||
REVIEW_SCHEMA_VERSION = 5
|
||||
MAX_REQUEST_BYTES = 128 * 1024
|
||||
|
||||
PROTOCOL_DISPOSITION_VALUES = {"positive", "hard_negative", "exclude", "uncertain"}
|
||||
PROTOCOL_DISPOSITION_VALUES = {"positive", "hard_negative", "exclude"}
|
||||
HARD_NEGATIVE_TYPE_VALUES = {
|
||||
"eot_midturn_pause",
|
||||
"eot_other_to_other",
|
||||
@@ -46,17 +46,13 @@ HARD_NEGATIVE_TYPE_VALUES = {
|
||||
"int_non_takeover_overlap",
|
||||
"int_retain_floor",
|
||||
"other",
|
||||
"not_applicable",
|
||||
"uncertain",
|
||||
}
|
||||
VALIDITY_VALUES = {"valid", "artifact", "uncertain"}
|
||||
VALIDITY_VALUES = {"valid", "artifact"}
|
||||
ONSET_FUNCTION_VALUES = {
|
||||
"floor_taking",
|
||||
"backchannel",
|
||||
"within_turn_resume",
|
||||
"nonverbal",
|
||||
"not_applicable",
|
||||
"uncertain",
|
||||
}
|
||||
CONTEXT_VALUES = {"addressed_wearer", "third_party", "group", "unknown"}
|
||||
OFFSET_OUTCOME_VALUES = {
|
||||
@@ -65,21 +61,20 @@ OFFSET_OUTCOME_VALUES = {
|
||||
"temporary_pause",
|
||||
"backchannel_end",
|
||||
"cutoff",
|
||||
"not_applicable",
|
||||
"uncertain",
|
||||
}
|
||||
CONFIDENCE_VALUES = {"high", "medium", "low"}
|
||||
HUMAN_ACTION_VALUES = {"accept", "relabel", "exclude", "second_review"}
|
||||
HUMAN_VERDICT_VALUES = {"POSITIVE", "HARD_NEGATIVE", "NOT_APPLICABLE", "EXCLUDE", "UNSURE"}
|
||||
HUMAN_EVENT_TYPE_VALUES = {"EOT_RESPONSE", "INT_YIELD"}
|
||||
HUMAN_ACTION_VALUES = {"accept", "relabel", "exclude"}
|
||||
HUMAN_VERDICT_VALUES = {"POSITIVE", "HARD_NEGATIVE", "EXCLUDE"}
|
||||
HUMAN_EVENT_TYPE_VALUES = {"WEARER_ENTRY", "INT_YIELD"}
|
||||
HUMAN_SUBTYPE_VALUES = {
|
||||
"EOT_MIDTURN_PAUSE", "EOT_OTHER_TO_OTHER", "EOT_WEARER_BACKCHANNEL",
|
||||
"ENTRY_OTHER_MIDTURN_PAUSE", "ENTRY_OTHER_TO_OTHER", "ENTRY_WEARER_BACKCHANNEL",
|
||||
"INT_OTHER_BACKCHANNEL", "INT_NON_TAKEOVER_OVERLAP", "INT_WEARER_HOLDS_FLOOR",
|
||||
"OTHER_HARD_NEGATIVE",
|
||||
}
|
||||
EXCLUDE_REASON_VALUES = {
|
||||
"ANNOTATION_ERROR", "RECORDING_BOUNDARY", "MISSING_CONTEXT", "NON_SPEECH_ARTIFACT",
|
||||
"TIMESTAMP_CORRUPTION", "UNRESOLVABLE",
|
||||
"TIMESTAMP_CORRUPTION", "OUT_OF_SCOPE_NORMAL_COMPLETION", "OUT_OF_SCOPE_NON_EVENT",
|
||||
"UNRESOLVABLE",
|
||||
}
|
||||
|
||||
|
||||
@@ -123,12 +118,12 @@ class Candidate:
|
||||
|
||||
@property
|
||||
def family(self) -> str:
|
||||
if self.payload.get("candidate_source"):
|
||||
return "eot" if self.task == "eot_response" else "int"
|
||||
if self.task == "eot_response":
|
||||
return "eot"
|
||||
if self.task == "int_yield":
|
||||
if self.task in {"wearer_entry", "wearer_entry_control", "eot_response", "eot_response_control"}:
|
||||
return "entry"
|
||||
if self.task in {"int_yield", "int_yield_control"}:
|
||||
return "int"
|
||||
if self.payload.get("candidate_source") == "WEARER_ONSET":
|
||||
return "entry"
|
||||
if "holder_uid" in self.payload or self.automatic_label.startswith("eot_"):
|
||||
return "offset"
|
||||
if self.payload.get("participant_role") == "wearer":
|
||||
@@ -454,7 +449,8 @@ class ReviewStore:
|
||||
normalized[key] = value
|
||||
|
||||
action = normalized["human_action"]
|
||||
candidate = self.candidates[candidate_id].payload
|
||||
candidate_record = self.candidates[candidate_id]
|
||||
candidate = candidate_record.payload
|
||||
if action == "accept":
|
||||
verdict = candidate.get("ai_verdict") or (candidate.get("llm_suggestion") or {}).get("verdict")
|
||||
event_type = candidate.get("ai_event_type") or (candidate.get("llm_suggestion") or {}).get("event_type")
|
||||
@@ -467,22 +463,28 @@ class ReviewStore:
|
||||
elif action == "exclude":
|
||||
verdict, event_type, subtype = "EXCLUDE", None, None
|
||||
selected_opportunity_uid = None
|
||||
elif action == "second_review":
|
||||
verdict, event_type, subtype = "UNSURE", None, None
|
||||
selected_opportunity_uid = None
|
||||
else:
|
||||
verdict, event_type, subtype = payload.get("human_verdict"), payload.get("human_event_type"), payload.get("human_subtype")
|
||||
selected_opportunity_uid = payload.get("selected_opportunity_uid")
|
||||
if verdict not in HUMAN_VERDICT_VALUES:
|
||||
raise ValueError(f"Invalid human_verdict: {verdict!r}")
|
||||
if verdict == "POSITIVE" and event_type not in HUMAN_EVENT_TYPE_VALUES:
|
||||
raise ValueError("POSITIVE requires EOT_RESPONSE or INT_YIELD")
|
||||
raise ValueError("POSITIVE requires WEARER_ENTRY or INT_YIELD")
|
||||
if verdict != "POSITIVE" and event_type is not None:
|
||||
raise ValueError("Only POSITIVE may have a human_event_type")
|
||||
if verdict == "HARD_NEGATIVE" and subtype not in HUMAN_SUBTYPE_VALUES:
|
||||
raise ValueError("HARD_NEGATIVE requires a valid human_subtype")
|
||||
if verdict != "HARD_NEGATIVE" and subtype is not None:
|
||||
raise ValueError("Only HARD_NEGATIVE may have a human_subtype")
|
||||
family = candidate_record.family
|
||||
if verdict == "POSITIVE" and family in {"entry", "int"}:
|
||||
required_event_type = "WEARER_ENTRY" if family == "entry" else "INT_YIELD"
|
||||
if event_type != required_event_type:
|
||||
raise ValueError(f"{family.upper()} candidate requires {required_event_type}")
|
||||
if verdict == "HARD_NEGATIVE" and family in {"entry", "int"}:
|
||||
prefix = family.upper() + "_"
|
||||
if subtype != "OTHER_HARD_NEGATIVE" and not str(subtype).startswith(prefix):
|
||||
raise ValueError(f"{family.upper()} candidate requires {prefix}* hard-negative subtype")
|
||||
evidence = candidate.get("temporal_evidence") or {}
|
||||
opportunity_options = evidence.get("opportunity_options") or []
|
||||
allowed_opportunity_uids = {
|
||||
@@ -490,11 +492,11 @@ class ReviewStore:
|
||||
for option in opportunity_options
|
||||
if isinstance(option, dict) and option.get("uid") is not None
|
||||
}
|
||||
if verdict == "POSITIVE" and event_type == "EOT_RESPONSE" and allowed_opportunity_uids:
|
||||
if selected_opportunity_uid is not None:
|
||||
if verdict != "POSITIVE" or event_type != "WEARER_ENTRY":
|
||||
raise ValueError("selected_opportunity_uid is only valid for WEARER_ENTRY")
|
||||
if selected_opportunity_uid not in allowed_opportunity_uids:
|
||||
raise ValueError("EOT_RESPONSE requires one supplied opportunity anchor")
|
||||
elif selected_opportunity_uid is not None:
|
||||
raise ValueError("selected_opportunity_uid is only valid for EOT_RESPONSE")
|
||||
raise ValueError("WEARER_ENTRY may select only a supplied opportunity anchor")
|
||||
selected_opportunity = next(
|
||||
(
|
||||
option for option in opportunity_options
|
||||
@@ -521,7 +523,7 @@ class ReviewStore:
|
||||
if correction not in {-0.2, 0, 0.2, None}:
|
||||
raise ValueError("boundary_correction_seconds must be -0.2, 0, 0.2, or null")
|
||||
normalized.update(
|
||||
human_status="SECOND_REVIEW" if action == "second_review" else "REVIEWED",
|
||||
human_status="REVIEWED",
|
||||
human_verdict=verdict,
|
||||
human_event_type=event_type,
|
||||
human_subtype=subtype,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Tests for EOT/INT DeepSeek suggestion validation without network calls."""
|
||||
"""Tests for wearer-entry/INT DeepSeek suggestion validation without network calls."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -13,7 +13,7 @@ from adjudicate_easycom_eot_int_candidates import (
|
||||
)
|
||||
|
||||
|
||||
class EOTINTAdjudicationTests(unittest.TestCase):
|
||||
class EntryINTAdjudicationTests(unittest.TestCase):
|
||||
def test_progress_line_reports_items_percent_pass_and_elapsed_time(self) -> None:
|
||||
line = _progress_line(25, 100, label="pass 1/2 · batch 4/13", elapsed_seconds=65)
|
||||
self.assertIn("25/100 ( 25.0%)", line)
|
||||
@@ -22,7 +22,7 @@ class EOTINTAdjudicationTests(unittest.TestCase):
|
||||
|
||||
def test_prompt_contains_task_and_context(self) -> None:
|
||||
prompt = build_user_prompt(
|
||||
[{"uid": "u1", "candidate_source": "WEARER_ONSET", "task": "eot_response",
|
||||
[{"uid": "u1", "candidate_source": "WEARER_ONSET", "task": "wearer_entry",
|
||||
"temporal_evidence": {"wearer_duration": 2.0},
|
||||
"context": [{"transcription": "Yes."}]}],
|
||||
0,
|
||||
@@ -32,13 +32,26 @@ class EOTINTAdjudicationTests(unittest.TestCase):
|
||||
self.assertIn("Yes.", prompt)
|
||||
|
||||
def test_system_prompt_treats_laughter_as_nonlexical_floor_feedback(self) -> None:
|
||||
self.assertEqual(PROMPT_VERSION, "easycom-wearer-anchor-first-eot-int-prelabel-v7")
|
||||
self.assertEqual(PROMPT_VERSION, "easycom-wearer-entry-int-prelabel-v12")
|
||||
self.assertIn("opportunity_options", SYSTEM_PROMPT)
|
||||
self.assertIn("t_w < t_o < t_y", SYSTEM_PROMPT)
|
||||
self.assertIn("a transcription consisting of [L] denotes laughter-only vocalization", SYSTEM_PROMPT)
|
||||
self.assertIn("EOT_WEARER_BACKCHANNEL", SYSTEM_PROMPT)
|
||||
self.assertIn("ENTRY_WEARER_BACKCHANNEL", SYSTEM_PROMPT)
|
||||
self.assertIn("INT_OTHER_BACKCHANNEL", SYSTEM_PROMPT)
|
||||
self.assertIn("Subtype precedence", SYSTEM_PROMPT)
|
||||
self.assertIn("same non-wearer resumes", SYSTEM_PROMPT)
|
||||
self.assertIn("different non-wearer takes", SYSTEM_PROMPT)
|
||||
self.assertIn("This is not an uncertainty fallback", SYSTEM_PROMPT)
|
||||
self.assertIn('"Nope." beginning 0.15 s after an addressed question ends', SYSTEM_PROMPT)
|
||||
self.assertIn("does not require a preceding non-wearer release", SYSTEM_PROMPT)
|
||||
self.assertIn("duration alone never determines floor function", SYSTEM_PROMPT)
|
||||
self.assertIn("out-of-scope normal completion", SYSTEM_PROMPT)
|
||||
self.assertIn("same incoming non-wearer actually takes", SYSTEM_PROMPT)
|
||||
self.assertIn("must not retroactively invalidate the preceding INT_YIELD", SYSTEM_PROMPT)
|
||||
self.assertIn("do not infer that a visible smile occurred", SYSTEM_PROMPT)
|
||||
self.assertIn("POSITIVE / INT_YIELD: subtype=null; selected_opportunity_uid=null", SYSTEM_PROMPT)
|
||||
self.assertNotIn("NOT_APPLICABLE", SYSTEM_PROMPT)
|
||||
self.assertNotIn("UNSURE", SYSTEM_PROMPT)
|
||||
self.assertNotIn('"needs_human_review"', SYSTEM_PROMPT)
|
||||
|
||||
def test_decisions_require_exact_uid_and_vocabulary(self) -> None:
|
||||
@@ -64,7 +77,7 @@ class EOTINTAdjudicationTests(unittest.TestCase):
|
||||
def test_model_needs_human_review_type_drift_is_ignored(self) -> None:
|
||||
payload = {
|
||||
"items": [{
|
||||
"uid": "u1", "verdict": "UNSURE", "event_type": None,
|
||||
"uid": "u1", "verdict": "EXCLUDE", "event_type": None,
|
||||
"subtype": None, "confidence": 0.4, "reason": "Ambiguous context.",
|
||||
"needs_human_review": "true",
|
||||
}]
|
||||
@@ -72,9 +85,18 @@ class EOTINTAdjudicationTests(unittest.TestCase):
|
||||
decision = validate_decisions(payload, ["u1"])[0]
|
||||
self.assertTrue(decision["needs_human_review"])
|
||||
|
||||
def test_eot_positive_must_select_one_supplied_release(self) -> None:
|
||||
def test_removed_fallback_verdicts_are_rejected(self) -> None:
|
||||
for verdict in ("UNSURE", "NOT_APPLICABLE"):
|
||||
payload = {"items": [{
|
||||
"uid": "u1", "verdict": verdict, "event_type": None,
|
||||
"subtype": None, "confidence": 0.4, "reason": "Ambiguous context.",
|
||||
}]}
|
||||
with self.subTest(verdict=verdict), self.assertRaisesRegex(ValueError, "Invalid verdict"):
|
||||
validate_decisions(payload, ["u1"])
|
||||
|
||||
def test_entry_positive_may_select_only_a_supplied_release(self) -> None:
|
||||
item = {
|
||||
"uid": "u1", "verdict": "POSITIVE", "event_type": "EOT_RESPONSE",
|
||||
"uid": "u1", "verdict": "POSITIVE", "event_type": "WEARER_ENTRY",
|
||||
"subtype": None, "selected_opportunity_uid": "release-2",
|
||||
"confidence": 0.9, "reason": "The wearer answers the second release.",
|
||||
}
|
||||
@@ -82,18 +104,66 @@ class EOTINTAdjudicationTests(unittest.TestCase):
|
||||
{"items": [item]}, ["u1"], {"u1": {"release-1", "release-2"}}
|
||||
)[0]
|
||||
self.assertEqual(decision["selected_opportunity_uid"], "release-2")
|
||||
with self.assertRaisesRegex(ValueError, "select a supplied opportunity"):
|
||||
with self.assertRaisesRegex(ValueError, "only a supplied opportunity"):
|
||||
validate_decisions(
|
||||
{"items": [{**item, "selected_opportunity_uid": "invented"}]},
|
||||
["u1"],
|
||||
{"u1": {"release-1", "release-2"}},
|
||||
)
|
||||
optional = validate_decisions(
|
||||
{"items": [{**item, "selected_opportunity_uid": None}]},
|
||||
["u1"], {"u1": {"release-1", "release-2"}},
|
||||
)[0]
|
||||
self.assertIsNone(optional["selected_opportunity_uid"])
|
||||
|
||||
def test_irrelevant_selected_opportunity_is_cleared_without_retrying_label(self) -> None:
|
||||
for verdict, event_type, subtype in (
|
||||
("POSITIVE", "INT_YIELD", None),
|
||||
("HARD_NEGATIVE", None, "INT_OTHER_BACKCHANNEL"),
|
||||
("EXCLUDE", None, None),
|
||||
):
|
||||
with self.subTest(verdict=verdict):
|
||||
item = {
|
||||
"uid": "u1",
|
||||
"verdict": verdict,
|
||||
"event_type": event_type,
|
||||
"subtype": subtype,
|
||||
"selected_opportunity_uid": "copied-release",
|
||||
"confidence": 0.8,
|
||||
"reason": "Evidence-based decision.",
|
||||
}
|
||||
decision = validate_decisions(
|
||||
{"items": [item]}, ["u1"],
|
||||
{"u1": {"copied-release"}},
|
||||
{"u1": "INT"},
|
||||
)[0]
|
||||
self.assertIsNone(decision["selected_opportunity_uid"])
|
||||
|
||||
def test_event_and_hard_negative_subtypes_match_candidate_family(self) -> None:
|
||||
positive = {
|
||||
"uid": "u1", "verdict": "POSITIVE", "event_type": "INT_YIELD",
|
||||
"subtype": None, "selected_opportunity_uid": None,
|
||||
"confidence": 0.8, "reason": "Wrong family.",
|
||||
}
|
||||
with self.assertRaisesRegex(ValueError, "ENTRY candidate requires WEARER_ENTRY"):
|
||||
validate_decisions(
|
||||
{"items": [positive]}, ["u1"], expected_families={"u1": "ENTRY"}
|
||||
)
|
||||
hard_negative = {
|
||||
"uid": "u1", "verdict": "HARD_NEGATIVE", "event_type": None,
|
||||
"subtype": "ENTRY_OTHER_MIDTURN_PAUSE", "selected_opportunity_uid": None,
|
||||
"confidence": 0.8, "reason": "Wrong family.",
|
||||
}
|
||||
with self.assertRaisesRegex(ValueError, "INT candidate requires an INT_"):
|
||||
validate_decisions(
|
||||
{"items": [hard_negative]}, ["u1"], expected_families={"u1": "INT"}
|
||||
)
|
||||
|
||||
def test_markdown_escaped_uid_is_normalized_but_still_unique(self) -> None:
|
||||
item = {
|
||||
"uid": "Session\\_4/wearer\\_onset/00000024",
|
||||
"verdict": "POSITIVE",
|
||||
"event_type": "EOT_RESPONSE",
|
||||
"event_type": "WEARER_ENTRY",
|
||||
"subtype": None,
|
||||
"confidence": 0.9,
|
||||
"reason": "Sustained wearer response after the other speaker stops.",
|
||||
@@ -108,7 +178,7 @@ class EOTINTAdjudicationTests(unittest.TestCase):
|
||||
def test_duplicate_rule_uids_are_disambiguated_with_provenance(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
path = Path(temporary) / "candidates.jsonl"
|
||||
base = {"uid": "Session_4/other_to_other/00003600", "task": "eot_response"}
|
||||
base = {"uid": "Session_4/other_to_other/00003600", "task": "wearer_entry"}
|
||||
rows = [
|
||||
{**base, "previous_other_participant_ids": [6], "incoming_other_participant_ids": [4]},
|
||||
{**base, "previous_other_participant_ids": [3], "incoming_other_participant_ids": [6]},
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Tests for manuscript-aligned EasyCom EOT/INT review regions."""
|
||||
"""Tests for manuscript-aligned EasyCom wearer-entry/INT review regions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -46,18 +46,19 @@ def clip(seconds: float = 5.0) -> Clip:
|
||||
)
|
||||
|
||||
|
||||
class EOTINTReviewCandidateTests(unittest.TestCase):
|
||||
class EntryINTReviewCandidateTests(unittest.TestCase):
|
||||
def derive(self, rows: list[Utterance]):
|
||||
return derive_protocol_candidates(rows, [clip()], session="Session_4")
|
||||
|
||||
def test_eot_candidate_pairs_release_and_behavior_anchors(self) -> None:
|
||||
def test_entry_candidate_retains_release_and_behavior_anchors(self) -> None:
|
||||
rows = [
|
||||
utterance("other", 4, 0.0, 1.0, "Your turn."),
|
||||
utterance("wearer", 2, 1.2, 2.0, "My response."),
|
||||
]
|
||||
event = next(row for row in self.derive(rows) if row.candidate_source == "WEARER_ONSET")
|
||||
self.assertEqual(event.automatic_protocol_label, "unsure")
|
||||
self.assertEqual(event.rule_candidate_type, "EOT_OPPORTUNITY_CANDIDATE")
|
||||
self.assertEqual(event.automatic_protocol_label, "candidate")
|
||||
self.assertEqual(event.task, "wearer_entry")
|
||||
self.assertEqual(event.rule_candidate_type, "WEARER_ENTRY_CANDIDATE")
|
||||
self.assertEqual(event.anchor_type, "wearer_onset")
|
||||
self.assertAlmostEqual(event.event_time, 1.2)
|
||||
self.assertIsNone(event.temporal_evidence["opportunity_anchor_seconds"])
|
||||
@@ -68,7 +69,7 @@ class EOTINTReviewCandidateTests(unittest.TestCase):
|
||||
self.assertAlmostEqual(event.temporal_evidence["minimum_response_delay_seconds"], 0.2)
|
||||
self.assertAlmostEqual(event.temporal_evidence["wearer_speech_duration_after_onset_seconds"], 0.8)
|
||||
|
||||
def test_long_silence_does_not_cancel_eot_opportunity(self) -> None:
|
||||
def test_long_silence_does_not_cancel_entry_release_metadata(self) -> None:
|
||||
rows = [
|
||||
utterance("other", 4, 0.0, 0.5, "Your turn."),
|
||||
utterance("wearer", 2, 3.5, 4.5, "Delayed response."),
|
||||
@@ -91,7 +92,7 @@ class EOTINTReviewCandidateTests(unittest.TestCase):
|
||||
self.assertEqual({option["uid"] for option in options}, {"other-4", "other-3"})
|
||||
self.assertEqual(events[0].temporal_evidence["behavior_anchor_seconds"], 1.4)
|
||||
|
||||
def test_competing_other_onset_terminates_eot_opportunity(self) -> None:
|
||||
def test_entry_while_other_active_remains_a_primary_candidate(self) -> None:
|
||||
rows = [
|
||||
utterance("release", 4, 0.0, 1.0, "First turn."),
|
||||
utterance("competitor", 3, 1.5, 3.0, "I take the floor."),
|
||||
@@ -101,8 +102,11 @@ class EOTINTReviewCandidateTests(unittest.TestCase):
|
||||
row for row in self.derive(rows)
|
||||
if row.anchor_dense_seconds == 2.5 and row.anchor_type == "wearer_onset"
|
||||
)
|
||||
self.assertEqual(onset.candidate_source, "WEARER_ONSET_CONTROL")
|
||||
self.assertEqual(onset.automatic_protocol_label, "not_applicable")
|
||||
self.assertEqual(onset.candidate_source, "WEARER_ONSET")
|
||||
self.assertEqual(onset.task, "wearer_entry")
|
||||
self.assertEqual(onset.automatic_protocol_label, "candidate")
|
||||
self.assertEqual(onset.temporal_evidence["entry_context_timing"], "OTHER_ACTIVE")
|
||||
self.assertFalse(onset.temporal_evidence["preceding_release_paired"])
|
||||
|
||||
def test_short_wearer_pause_without_other_takeover_becomes_resume_control(self) -> None:
|
||||
rows = [
|
||||
@@ -111,9 +115,9 @@ class EOTINTReviewCandidateTests(unittest.TestCase):
|
||||
]
|
||||
derived = self.derive(rows)
|
||||
resume = next(row for row in derived if row.candidate_source == "WEARER_RESUME")
|
||||
self.assertEqual(resume.task, "eot_response_control")
|
||||
self.assertEqual(resume.task, "wearer_entry_control")
|
||||
self.assertEqual(resume.rule_candidate_type, "WITHIN_TURN_RESUME_CONTROL")
|
||||
self.assertEqual(resume.automatic_protocol_label, "not_applicable")
|
||||
self.assertEqual(resume.automatic_protocol_label, "control")
|
||||
self.assertTrue(resume.temporal_evidence["floor_episode_merged"])
|
||||
self.assertAlmostEqual(resume.temporal_evidence["previous_wearer_silence_duration_seconds"], 0.5)
|
||||
self.assertFalse(any(
|
||||
@@ -131,21 +135,29 @@ class EOTINTReviewCandidateTests(unittest.TestCase):
|
||||
self.assertEqual(resume.temporal_evidence["feedback_like_other_ids_during_gap"], [4])
|
||||
self.assertFalse(resume.temporal_evidence["sustained_other_takeover_during_gap"])
|
||||
|
||||
def test_short_gap_with_sustained_other_takeover_preserves_floor_reentry_but_not_eot(self) -> None:
|
||||
def test_short_gap_with_sustained_other_takeover_preserves_positive_entry_candidate(self) -> None:
|
||||
rows = [
|
||||
utterance("wearer-1", 2, 0.2, 1.0, "First wearer turn."),
|
||||
utterance("other", 4, 1.0, 2.2, "I have the floor now."),
|
||||
utterance("other", 4, 0.8, 2.2, "I have the floor now."),
|
||||
utterance("wearer-2", 2, 1.6, 2.5, "Wearer re-enters."),
|
||||
]
|
||||
derived = self.derive(rows)
|
||||
reentry = next(
|
||||
row for row in self.derive(rows)
|
||||
if row.candidate_source == "WEARER_ONSET_CONTROL" and row.anchor_dense_seconds == 1.6
|
||||
row for row in derived
|
||||
if row.candidate_source == "WEARER_ONSET" and row.anchor_dense_seconds == 1.6
|
||||
)
|
||||
self.assertFalse(reentry.temporal_evidence["floor_episode_merged"])
|
||||
self.assertTrue(reentry.temporal_evidence["sustained_other_takeover_during_gap"])
|
||||
self.assertEqual(reentry.temporal_evidence["sustained_competing_other_ids_during_gap"], [4])
|
||||
self.assertEqual(reentry.temporal_evidence["entry_context_timing"], "OTHER_ACTIVE")
|
||||
self.assertEqual(reentry.automatic_protocol_label, "candidate")
|
||||
preceding_yield = next(
|
||||
row for row in derived
|
||||
if row.candidate_source == "WEARER_OFFSET" and row.anchor_dense_seconds == 1.0
|
||||
)
|
||||
self.assertTrue(preceding_yield.temporal_evidence["int_ordering_satisfied"])
|
||||
|
||||
def test_same_other_resume_is_eot_midturn_hard_negative(self) -> None:
|
||||
def test_same_other_resume_is_entry_midturn_hard_negative(self) -> None:
|
||||
rows = [
|
||||
utterance("first", 4, 0.0, 1.0, "First clause."),
|
||||
utterance("resume", 4, 1.3, 2.5, "Same turn continues."),
|
||||
@@ -156,10 +168,10 @@ class EOTINTReviewCandidateTests(unittest.TestCase):
|
||||
if row.candidate_source == "SAME_SPEAKER_PAUSE" and row.anchor_dense_seconds == 1.0
|
||||
)
|
||||
self.assertEqual(event.automatic_protocol_label, "hard_negative")
|
||||
self.assertEqual(event.automatic_hard_negative_type, "EOT_MIDTURN_PAUSE")
|
||||
self.assertEqual(event.automatic_hard_negative_type, "ENTRY_OTHER_MIDTURN_PAUSE")
|
||||
self.assertAlmostEqual(event.temporal_evidence["resume_delay_seconds"], 0.3)
|
||||
|
||||
def test_other_to_other_transfer_is_eot_hard_negative(self) -> None:
|
||||
def test_other_to_other_transfer_is_entry_hard_negative(self) -> None:
|
||||
rows = [
|
||||
utterance("first", 4, 0.0, 1.0, "First speaker."),
|
||||
utterance("next", 3, 1.2, 2.5, "Second speaker."),
|
||||
@@ -170,7 +182,7 @@ class EOTINTReviewCandidateTests(unittest.TestCase):
|
||||
if row.candidate_source == "OTHER_TO_OTHER" and row.anchor_dense_seconds == 1.0
|
||||
)
|
||||
self.assertEqual(event.automatic_protocol_label, "hard_negative")
|
||||
self.assertEqual(event.automatic_hard_negative_type, "EOT_OTHER_TO_OTHER")
|
||||
self.assertEqual(event.automatic_hard_negative_type, "ENTRY_OTHER_TO_OTHER")
|
||||
self.assertNotEqual(
|
||||
event.temporal_evidence["previous_speaker_id"],
|
||||
event.temporal_evidence["incoming_speaker_id"],
|
||||
@@ -182,7 +194,7 @@ class EOTINTReviewCandidateTests(unittest.TestCase):
|
||||
utterance("other", 4, 1.0, 3.0, "I will take over now."),
|
||||
]
|
||||
event = next(row for row in self.derive(rows) if row.candidate_source == "WEARER_OFFSET")
|
||||
self.assertEqual(event.automatic_protocol_label, "unsure")
|
||||
self.assertEqual(event.automatic_protocol_label, "candidate")
|
||||
self.assertEqual(event.rule_candidate_type, "YIELD_CANDIDATE")
|
||||
self.assertEqual(event.anchor_type, "wearer_offset")
|
||||
self.assertAlmostEqual(event.event_time, 2.0)
|
||||
@@ -211,7 +223,7 @@ class EOTINTReviewCandidateTests(unittest.TestCase):
|
||||
event = next(row for row in self.derive(rows) if row.anchor_type == "wearer_offset")
|
||||
self.assertEqual(event.candidate_source, "WEARER_OFFSET_CONTROL")
|
||||
self.assertEqual(event.rule_candidate_type, "WEARER_OFFSET_CONTROL")
|
||||
self.assertEqual(event.automatic_protocol_label, "not_applicable")
|
||||
self.assertEqual(event.automatic_protocol_label, "control")
|
||||
self.assertFalse(event.temporal_evidence["other_crosses_wearer_offset"])
|
||||
|
||||
def test_one_frame_boundary_tolerance_is_configurable(self) -> None:
|
||||
|
||||
@@ -59,8 +59,9 @@ class FloorEventReviewTests(unittest.TestCase):
|
||||
"dense_start_seconds": 1.2,
|
||||
"event_time": 1.2,
|
||||
"candidate_source": "WEARER_ONSET",
|
||||
"task": "wearer_entry",
|
||||
"ai_verdict": "POSITIVE",
|
||||
"ai_event_type": "EOT_RESPONSE",
|
||||
"ai_event_type": "WEARER_ENTRY",
|
||||
"ai_subtype": None,
|
||||
"speaker_timeline": [
|
||||
{
|
||||
@@ -129,7 +130,7 @@ class FloorEventReviewTests(unittest.TestCase):
|
||||
resumed = ReviewStore(output, candidates)
|
||||
self.assertEqual(
|
||||
resumed.reviews[candidates[0].candidate_id]["review"]["human_event_type"],
|
||||
"EOT_RESPONSE",
|
||||
"WEARER_ENTRY",
|
||||
)
|
||||
self.assertFalse(resumed.reviews[candidates[0].candidate_id]["review"]["relabeled"])
|
||||
self.assertEqual(resumed.reviews[candidates[0].candidate_id]["review"]["reviewer"], "")
|
||||
@@ -162,7 +163,7 @@ class FloorEventReviewTests(unittest.TestCase):
|
||||
{
|
||||
"uid": "Session_4/wearer_onset/00000338",
|
||||
"candidate_source": "WEARER_ONSET",
|
||||
"task": "eot_response",
|
||||
"task": "wearer_entry",
|
||||
},
|
||||
]
|
||||
path.write_text(
|
||||
@@ -183,18 +184,83 @@ class FloorEventReviewTests(unittest.TestCase):
|
||||
"human_action": "relabel",
|
||||
"human_verdict": "HARD_NEGATIVE",
|
||||
"human_event_type": None,
|
||||
"human_subtype": "EOT_WEARER_BACKCHANNEL",
|
||||
"human_subtype": "ENTRY_WEARER_BACKCHANNEL",
|
||||
"exclude_reason": None,
|
||||
"boundary_correction_seconds": None,
|
||||
"confidence": "medium",
|
||||
"reviewer": "r2",
|
||||
"notes": "brief acknowledgement",
|
||||
})
|
||||
self.assertEqual(record["ai_event_type"], "EOT_RESPONSE")
|
||||
self.assertEqual(record["review"]["human_subtype"], "EOT_WEARER_BACKCHANNEL")
|
||||
self.assertEqual(record["ai_event_type"], "WEARER_ENTRY")
|
||||
self.assertEqual(record["review"]["human_subtype"], "ENTRY_WEARER_BACKCHANNEL")
|
||||
self.assertTrue(record["review"]["relabeled"])
|
||||
|
||||
def test_eot_review_selects_only_a_supplied_opportunity_anchor(self) -> None:
|
||||
def test_removed_verdicts_cannot_be_accepted_or_reintroduced(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
_, events = self._fixture(root)
|
||||
candidates = load_candidates([events])
|
||||
candidate = candidates[0]
|
||||
store = ReviewStore(root / "reviews.jsonl", candidates)
|
||||
base = {
|
||||
"human_event_type": None, "human_subtype": None,
|
||||
"selected_opportunity_uid": None, "exclude_reason": None,
|
||||
"boundary_correction_seconds": None, "confidence": "medium",
|
||||
"notes": "legacy fallback",
|
||||
}
|
||||
for verdict in ("UNSURE", "NOT_APPLICABLE"):
|
||||
with self.subTest(action="relabel", verdict=verdict), self.assertRaisesRegex(
|
||||
ValueError, "Invalid human_verdict"
|
||||
):
|
||||
store.save(candidate.candidate_id, {
|
||||
**base, "human_action": "relabel", "human_verdict": verdict,
|
||||
})
|
||||
|
||||
candidate.payload["ai_verdict"] = verdict
|
||||
with self.subTest(action="accept", verdict=verdict), self.assertRaisesRegex(
|
||||
ValueError, "Accept requires an AI pre-label"
|
||||
):
|
||||
store.save(candidate.candidate_id, {
|
||||
**base, "human_action": "accept", "human_verdict": None,
|
||||
})
|
||||
|
||||
candidate.payload["ai_verdict"] = "POSITIVE"
|
||||
|
||||
def test_relabel_subtype_must_match_candidate_family(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
_, events = self._fixture(root)
|
||||
candidates = load_candidates([events])
|
||||
store = ReviewStore(root / "reviews.jsonl", candidates)
|
||||
with self.assertRaisesRegex(ValueError, "ENTRY candidate requires ENTRY_"):
|
||||
store.save(candidates[0].candidate_id, {
|
||||
"human_action": "relabel", "human_verdict": "HARD_NEGATIVE",
|
||||
"human_event_type": None, "human_subtype": "INT_OTHER_BACKCHANNEL",
|
||||
"selected_opportunity_uid": None, "exclude_reason": None,
|
||||
"boundary_correction_seconds": None, "confidence": "medium",
|
||||
"notes": "wrong family",
|
||||
})
|
||||
|
||||
def test_normal_completion_can_be_excluded_as_out_of_scope(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
_, events = self._fixture(root)
|
||||
candidates = load_candidates([events])
|
||||
store = ReviewStore(root / "reviews.jsonl", candidates)
|
||||
record = store.save(candidates[0].candidate_id, {
|
||||
"human_action": "exclude", "human_verdict": None,
|
||||
"human_event_type": None, "human_subtype": None,
|
||||
"selected_opportunity_uid": None,
|
||||
"exclude_reason": "OUT_OF_SCOPE_NORMAL_COMPLETION",
|
||||
"boundary_correction_seconds": None, "confidence": "high",
|
||||
"notes": "wearer completed; laughter did not take over",
|
||||
})
|
||||
self.assertEqual(record["review"]["human_verdict"], "EXCLUDE")
|
||||
self.assertEqual(
|
||||
record["review"]["exclude_reason"], "OUT_OF_SCOPE_NORMAL_COMPLETION"
|
||||
)
|
||||
|
||||
def test_entry_review_may_select_only_a_supplied_opportunity_anchor(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
_, events = self._fixture(root)
|
||||
@@ -208,7 +274,7 @@ class FloorEventReviewTests(unittest.TestCase):
|
||||
store = ReviewStore(root / "reviews.jsonl", candidates)
|
||||
base = {
|
||||
"human_action": "relabel", "human_verdict": "POSITIVE",
|
||||
"human_event_type": "EOT_RESPONSE", "human_subtype": None,
|
||||
"human_event_type": "WEARER_ENTRY", "human_subtype": None,
|
||||
"exclude_reason": None, "boundary_correction_seconds": None,
|
||||
"confidence": "high", "notes": "clear response",
|
||||
}
|
||||
@@ -219,11 +285,15 @@ class FloorEventReviewTests(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
record["review"]["valid_prediction_interval_seconds"], [1.0, 1.2]
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "requires one supplied"):
|
||||
with self.assertRaisesRegex(ValueError, "only a supplied"):
|
||||
store.save(
|
||||
candidate.candidate_id,
|
||||
{**base, "selected_opportunity_uid": "invented"},
|
||||
)
|
||||
optional = store.save(
|
||||
candidate.candidate_id, {**base, "selected_opportunity_uid": None}
|
||||
)
|
||||
self.assertIsNone(optional["review"]["selected_opportunity_uid"])
|
||||
|
||||
def test_byte_ranges(self) -> None:
|
||||
self.assertEqual(parse_byte_range("bytes=2-5", 10), (2, 5))
|
||||
|
||||
Reference in New Issue
Block a user