diff --git a/adjudicate_easycom_eot_int_candidates.py b/adjudicate_easycom_eot_int_candidates.py index 22c1e83..9dc7a96 100644 --- a/adjudicate_easycom_eot_int_candidates.py +++ b/adjudicate_easycom_eot_int_candidates.py @@ -27,7 +27,7 @@ from easycom_event_taxonomy import EXCLUDE_REASONS, exclude_reason_group from prepare_easycom_eot_int_review import requires_semantic_adjudication -PROMPT_VERSION = "easycom-wearer-boundary-semantic-prelabel-v15" +PROMPT_VERSION = "easycom-wearer-boundary-semantic-prelabel-v16" DEFAULT_MODEL = "deepseek-v4-flash" DEFAULT_BASE_URL = "https://api.deepseek.com" VERDICTS = {"POSITIVE", "HARD_NEGATIVE", "EXCLUDE"} @@ -107,7 +107,11 @@ Evidence rules: Apply this adjacency-pair rule before using brevity or later speaker activity as evidence: when a participant addresses the wearer with a question, request, proposal, or other first-pair part and the wearer supplies its contextually complete second-pair part, classify the wearer contribution as POSITIVE / WEARER_ENTRY. A polar-question answer such as "yes", "no", or "Nope." is complete even if it is one word and lasts less than one second. Reciprocal Target_of_Speech evidence (question targets wearer; answer targets questioner) strengthens this reading. Floor entry here means making the committed conversational response; it does not require monopolizing or retaining the floor afterward. Therefore, the questioner or another participant speaking after the answer does not turn that completed answer into a backchannel. Use HARD_NEGATIVE / ENTRY_WEARER_BACKCHANNEL only when the wearer vocalization functions merely as feedback on another participant's still-ongoing contribution and does not answer a question, accept or reject a proposal, provide requested information, or otherwise complete an adjacency pair. Lexical form alone is insufficient: "yeah" may answer a yes/no question or may be a continuer, depending on the preceding utterance and addressee evidence. For the supplied audit pattern, participant 3 asks the wearer "Have you ever analyzed documents?" and releases at 54.00 s; the wearer answers "Nope." at 54.15 s. This is POSITIVE / WEARER_ENTRY, not ENTRY_WEARER_BACKCHANNEL, even though it is brief and participant 3 later speaks again. -6. For every WEARER_OFFSET, answer the central question: did the wearer stop because a non-wearer genuinely took over the conversational floor? Reconstruct the wearer activity-run start and offset, every nearby other onset and offset, which others are active immediately before and after the offset, whether each other is pre-existing or starts during wearer speech, whether it crosses the offset, overlap duration, who speaks next, transcript content, and Target_of_Speech. All temporal booleans are descriptive evidence, not prerequisite labels. A valid INT_YIELD may involve a newly entering or pre-existing competitor if the chronology and semantics establish genuine takeover and wearer relinquishment. Conversely, crossing the offset or satisfying t_w < t_o < t_y never proves takeover by itself. Verify that the relevant non-wearer actually takes and sustains the next floor. A different participant's later substantive turn is not evidence that the proposed competitor took over. Do not label INT_WEARER_HOLDS_FLOOR merely because the wearer speaks again later: if a competitor establishes an intervening substantive floor and the wearer has an intervening silent interval, the later wearer onset is a new WEARER_ENTRY and does not invalidate the earlier INT_YIELD. Natural completion, completion-to-silence, a within-turn wearer pause, or a post-completion other onset that could not have caused the stop is normally EXCLUDE unless the region is a valid named hard-negative opportunity. If the wearer reaches a semantically complete natural ending while overlapping activity is only [L] or feedback and that contributor does not take the next substantive floor, return EXCLUDE. +6. For every WEARER_OFFSET, answer the central question: did the wearer stop because a non-wearer genuinely took over the conversational floor? Reconstruct the wearer activity-run start and offset, every nearby other onset and offset, which others are active immediately before and after the offset, whether each other is pre-existing or starts during wearer speech, whether it crosses the offset, overlap duration, who speaks next, transcript content, and Target_of_Speech. All temporal booleans are descriptive evidence, not prerequisite labels. A valid INT_YIELD may involve a newly entering or pre-existing competitor if the chronology and semantics establish genuine takeover and wearer relinquishment. Conversely, crossing the offset or satisfying t_w < t_o < t_y never proves takeover by itself. Verify that the relevant non-wearer actually takes and sustains the next floor. A different participant's later substantive turn is not evidence that the proposed competitor took over. Do not label INT_WEARER_HOLDS_FLOOR merely because the wearer speaks again later: if a competitor establishes an intervening substantive floor and the wearer has an intervening silent interval, the later wearer onset is a new WEARER_ENTRY and does not invalidate the earlier INT_YIELD. + + Before deciding an offset, determine from transcript and chronology whether the wearer activity run ending there ever held the floor. If it is only a wearer backchannel embedded in a non-wearer's already ongoing turn, then the non-wearer's continuation across its offset is neither a takeover nor evidence that the wearer held the floor. Return EXCLUDE / OUT_OF_SCOPE_WEARER_BACKCHANNEL_OFFSET. The backchannel is scored once, at its wearer-onset ENTRY candidate, as HARD_NEGATIVE / ENTRY_WEARER_BACKCHANNEL; do not count its trailing offset again as an INT hard negative. For the supplied audit pattern, participant 1 speaks from 320.25 to 325.50 s while the wearer says only "Awesome." from 323.05 to 323.80 s. The wearer onset is an ENTRY_WEARER_BACKCHANNEL opportunity, while the wearer offset is OUT_OF_SCOPE_WEARER_BACKCHANNEL_OFFSET, not INT_WEARER_HOLDS_FLOOR and not INT_YIELD. + + Natural completion, completion-to-silence, a within-turn wearer pause, or a post-completion other onset that could not have caused the stop is normally EXCLUDE unless the region is a valid named hard-negative opportunity. If the wearer reaches a semantically complete natural ending while overlapping activity is only [L] or feedback and that contributor does not take the next substantive floor, return EXCLUDE. 7. Transcript event markers are evidence, not lexical turns. In EasyCom, a transcription consisting of [L] denotes laughter-only vocalization. Do not treat [L] as a sustained floor entry or interruption takeover merely because its activity interval creates an onset, offset, or overlap candidate. Specifically: - wearer [L] during a non-wearer's retained turn is normally HARD_NEGATIVE / ENTRY_WEARER_BACKCHANNEL, not WEARER_ENTRY; - non-wearer [L] overlapping a wearer's retained turn is normally HARD_NEGATIVE / INT_OTHER_BACKCHANNEL, not INT_YIELD; @@ -121,6 +125,7 @@ Top-level verdicts: EXCLUDE requires exactly one exclude_reason from this closed taxonomy: - OUT_OF_SCOPE_NORMAL_COMPLETION: the wearer naturally completes; no interruption-yield target occurs. +- OUT_OF_SCOPE_WEARER_BACKCHANNEL_OFFSET: the wearer activity run is only a backchannel inside a non-wearer's retained turn; its onset carries the ENTRY hard-negative opportunity, while this trailing offset is not scored again as INT. - OUT_OF_SCOPE_WITHIN_TURN_CONTINUATION: a raw boundary is only a pause/resumption within the same wearer contribution. - OUT_OF_SCOPE_POST_COMPLETION_ACTIVITY: another contribution begins only after wearer completion and cannot have caused the boundary. - OUT_OF_SCOPE_NON_EVENT: another clear, interpretable non-target boundary not covered above. diff --git a/docs/easycom_event_review_tool.md b/docs/easycom_event_review_tool.md index c84a5b5..ec9af71 100644 --- a/docs/easycom_event_review_tool.md +++ b/docs/easycom_event_review_tool.md @@ -164,7 +164,7 @@ non-backchannel overlap after which the wearer demonstrably holds or resumes the floor, and finally another interpretable overlap that fails to become a sustained takeover. `OTHER_HARD_NEGATIVE` is reserved for valid false-positive opportunities outside these named behaviors; it is never an uncertainty -fallback. Prompt v15 gives adjacency-pair completion priority over duration: +fallback. Prompt v16 gives adjacency-pair completion priority over duration: a short but complete answer to an addressed question remains a positive wearer entry, and floor entry does not require the wearer to retain the next turn. Later resumption by the questioner or another speaker therefore does not turn @@ -173,6 +173,15 @@ reserved for feedback that does not answer a question, accept or reject a proposal, provide requested information, or otherwise complete an adjacency pair. +The onset and offset of one wearer vocalization remain separate raw boundary +candidates, but they are not automatically two scoring opportunities. If the +wearer activity run is a backchannel inside a non-wearer's retained turn, its +onset may be retained once as `ENTRY_WEARER_BACKCHANNEL`; its trailing offset +is excluded as `OUT_OF_SCOPE_WEARER_BACKCHANNEL_OFFSET`, rather than being +counted again as an INT hard negative. This episode-level interpretation is +made by the LLM and human reviewer from timing and transcript evidence, not by +candidate-generation gating. + During live adjudication, the CLI displays a dependency-free progress bar over all candidate/pass work, including the current pass, batch, percentage, and elapsed time. Redirected logs emit periodic progress lines instead of carriage diff --git a/easycom_event_taxonomy.py b/easycom_event_taxonomy.py index 72eb336..b5011b5 100644 --- a/easycom_event_taxonomy.py +++ b/easycom_event_taxonomy.py @@ -5,6 +5,7 @@ from __future__ import annotations EXCLUDE_REASONS = { "OUT_OF_SCOPE_NORMAL_COMPLETION", + "OUT_OF_SCOPE_WEARER_BACKCHANNEL_OFFSET", "OUT_OF_SCOPE_WITHIN_TURN_CONTINUATION", "OUT_OF_SCOPE_POST_COMPLETION_ACTIVITY", "OUT_OF_SCOPE_NON_EVENT", diff --git a/floor_event_review_web/app.js b/floor_event_review_web/app.js index 494f886..49f9901 100644 --- a/floor_event_review_web/app.js +++ b/floor_event_review_web/app.js @@ -58,6 +58,7 @@ const decisionMeanings = { OTHER_HARD_NEGATIVE: "这是可解释的困难负例，但不符合任何命名子类型；不能用于替代 Exclude。", EXCLUDE: "该区域是无需参评的非目标事件，或存在证据/数据问题，不进入评分。", OUT_OF_SCOPE_NORMAL_COMPLETION: "有效语境中的自然结束，不是 interruption yield。", + OUT_OF_SCOPE_WEARER_BACKCHANNEL_OFFSET: "该 wearer 发声只是 backchannel；其 onset 已作为 ENTRY 困难负例审查，尾部 offset 不再重复计为 INT。", OUT_OF_SCOPE_WITHIN_TURN_CONTINUATION: "这是同一 wearer turn 内的暂停或继续，不是新的 floor event。", OUT_OF_SCOPE_POST_COMPLETION_ACTIVITY: "Other 在 wearer 已结束后才开始，不能构成导致停止的 takeover。", OUT_OF_SCOPE_NON_EVENT: "数据有效且可解释，但该边界不属于目标行为。", diff --git a/floor_event_review_web/index.html b/floor_event_review_web/index.html index 85b353a..45aad7e 100644 --- a/floor_event_review_web/index.html +++ b/floor_event_review_web/index.html @@ -88,7 +88,7 @@ - +
仅在原 annotation 明显错误时校正边界
diff --git a/manuscript/main/main.tex b/manuscript/main/main.tex index d63c2ac..9eded13 100644 --- a/manuscript/main/main.tex +++ b/manuscript/main/main.tex @@ -609,6 +609,9 @@ admit nor reject a candidate and do not establish takeover. The LLM and human annotator answer whether the wearer stopped because a non-wearer genuinely took over the floor. Natural completion, post-completion activity, and within-turn pauses are therefore resolved semantically rather than pre-filtered. +When a wearer activity run is itself only a backchannel within a non-wearer's +retained turn, its onset supplies the false-entry opportunity; its trailing +offset is excluded rather than counted again as a false-yield opportunity. \subsection{False-positive opportunity mining} diff --git a/reports/README.md b/reports/README.md index 95b6f4f..877cc76 100644 --- a/reports/README.md +++ b/reports/README.md @@ -7,6 +7,10 @@ Current artifact: - [`egospeak_easycom_feature_audit.json`](egospeak_easycom_feature_audit.json): released EgoSpeak EasyCom feature-layout audit. +- [`easycom_event_ai_prelabel_stats_v15/`](easycom_event_ai_prelabel_stats_v15/): + descriptive DeepSeek v15 pre-label counts for EasyCom Sessions 1--4. These + rows are all unreviewed and must not replace the manuscript's + human-adjudicated annotation statistics. When adding a report, record the generating script, command, source dataset version and timestamp in the payload or an adjacent README. Derived timing or diff --git a/reports/easycom_event_ai_prelabel_stats_v15/README.md b/reports/easycom_event_ai_prelabel_stats_v15/README.md new file mode 100644 index 0000000..3cf5d4c --- /dev/null +++ b/reports/easycom_event_ai_prelabel_stats_v15/README.md @@ -0,0 +1,32 @@ +# EasyCom event AI pre-label audit + +> These are DeepSeek pre-labels awaiting mandatory human adjudication, not ground truth and not final manuscript annotation counts. + +Inputs contain 2,128 unique candidates from Session_1, Session_2, Session_3, Session_4. Protocol: `easycom-wearer-entry-int-candidates-v8`; prompt: `easycom-wearer-boundary-semantic-prelabel-v15`. + +## Cohort overview + +| Cohort | Candidates | Positive | Hard negative | Exclude | Prelabel scoring-eligible | Two-pass semantic/full agreement | Human reviewed | +|---|---:|---:|---:|---:|---:|---:|---:| +| Session_1 | 496 | 116 | 182 | 198 | 298 (60.08%) | 458/496 (92.34%) / 453/496 (91.33%) | 0 | +| Session_2 | 490 | 71 | 252 | 167 | 323 (65.92%) | 454/490 (92.65%) / 452/490 (92.24%) | 0 | +| Session_3 | 545 | 117 | 184 | 244 | 301 (55.23%) | 489/545 (89.72%) / 486/545 (89.17%) | 0 | +| Session_4 | 597 | 84 | 381 | 132 | 465 (77.89%) | 575/597 (96.31%) / 571/597 (95.64%) | 0 | +| Test pooled (Sessions 1-3) | 1531 | 304 | 618 | 609 | 922 (60.22%) | 1401/1531 (91.51%) / 1391/1531 (90.86%) | 0 | +| Session 4 audit (train split) | 597 | 84 | 381 | 132 | 465 (77.89%) | 575/597 (96.31%) / 571/597 (95.64%) | 0 | +| All supplied (descriptive only) | 2128 | 388 | 999 | 741 | 1387 (65.18%) | 1976/2128 (92.86%) / 1962/2128 (92.20%) | 0 | + +The official EasyCom test sessions (1--3) are pooled only for descriptive reporting; these AI pre-labels must not be used to tune thresholds, taxonomy, prompts, or model choices. Session 4 is shown separately as a train-split audit and is not the complete Sessions 4--12 training corpus. + +Detailed counts are in `taxonomy_counts.csv` and `candidate_source_counts.csv`. `manuscript_ai_prelabel_table.tex` mirrors the manuscript row order but is deliberately labeled as an AI audit; retain the manuscript TODOs until human adjudication is frozen. + +## Reproduce + +```bash +python3 summarize_easycom_event_prelabels.py \ + --input reports/easycom_entry_int_session1_v14/easycom_eot_int_deepseek_official.jsonl \ + --input reports/easycom_entry_int_session2_v14/easycom_eot_int_deepseek_official.jsonl \ + --input reports/easycom_entry_int_session3_v14/easycom_eot_int_deepseek_official.jsonl \ + --input reports/easycom_entry_int_session4_v14/easycom_eot_int_deepseek_official.jsonl \ + --output-dir reports/easycom_event_ai_prelabel_stats_v15 +``` diff --git a/reports/easycom_event_ai_prelabel_stats_v15/candidate_source_counts.csv b/reports/easycom_event_ai_prelabel_stats_v15/candidate_source_counts.csv new file mode 100644 index 0000000..e59aa54 --- /dev/null +++ b/reports/easycom_event_ai_prelabel_stats_v15/candidate_source_counts.csv @@ -0,0 +1,43 @@ +cohort,candidate_source,count,percent_of_all_candidates +Session_1,NON_TAKEOVER_OVERLAP,21,4.23 +Session_1,OTHER_BACKCHANNEL,20,4.03 +Session_1,OTHER_TO_OTHER,56,11.29 +Session_1,SAME_SPEAKER_PAUSE,60,12.1 +Session_1,WEARER_OFFSET,170,34.27 +Session_1,WEARER_ONSET,169,34.07 +Session_2,NON_TAKEOVER_OVERLAP,14,2.86 +Session_2,OTHER_BACKCHANNEL,15,3.06 +Session_2,OTHER_TO_OTHER,123,25.1 +Session_2,SAME_SPEAKER_PAUSE,84,17.14 +Session_2,WEARER_OFFSET,127,25.92 +Session_2,WEARER_ONSET,127,25.92 +Session_3,NON_TAKEOVER_OVERLAP,22,4.04 +Session_3,OTHER_BACKCHANNEL,19,3.49 +Session_3,OTHER_TO_OTHER,49,8.99 +Session_3,SAME_SPEAKER_PAUSE,63,11.56 +Session_3,WEARER_OFFSET,196,35.96 +Session_3,WEARER_ONSET,196,35.96 +Session_4,NON_TAKEOVER_OVERLAP,12,2.01 +Session_4,OTHER_BACKCHANNEL,3,0.5 +Session_4,OTHER_TO_OTHER,255,42.71 +Session_4,SAME_SPEAKER_PAUSE,105,17.59 +Session_4,WEARER_OFFSET,111,18.59 +Session_4,WEARER_ONSET,111,18.59 +Test pooled (Sessions 1-3),NON_TAKEOVER_OVERLAP,57,3.72 +Test pooled (Sessions 1-3),OTHER_BACKCHANNEL,54,3.53 +Test pooled (Sessions 1-3),OTHER_TO_OTHER,228,14.89 +Test pooled (Sessions 1-3),SAME_SPEAKER_PAUSE,207,13.52 +Test pooled (Sessions 1-3),WEARER_OFFSET,493,32.2 +Test pooled (Sessions 1-3),WEARER_ONSET,492,32.14 +Session 4 audit (train split),NON_TAKEOVER_OVERLAP,12,2.01 +Session 4 audit (train split),OTHER_BACKCHANNEL,3,0.5 +Session 4 audit (train split),OTHER_TO_OTHER,255,42.71 +Session 4 audit (train split),SAME_SPEAKER_PAUSE,105,17.59 +Session 4 audit (train split),WEARER_OFFSET,111,18.59 +Session 4 audit (train split),WEARER_ONSET,111,18.59 +All supplied (descriptive only),NON_TAKEOVER_OVERLAP,69,3.24 +All supplied (descriptive only),OTHER_BACKCHANNEL,57,2.68 +All supplied (descriptive only),OTHER_TO_OTHER,483,22.7 +All supplied (descriptive only),SAME_SPEAKER_PAUSE,312,14.66 +All supplied (descriptive only),WEARER_OFFSET,604,28.38 +All supplied (descriptive only),WEARER_ONSET,603,28.34 diff --git a/reports/easycom_event_ai_prelabel_stats_v15/cohort_summary.csv b/reports/easycom_event_ai_prelabel_stats_v15/cohort_summary.csv new file mode 100644 index 0000000..6533bdc --- /dev/null +++ b/reports/easycom_event_ai_prelabel_stats_v15/cohort_summary.csv @@ -0,0 +1,8 @@ +cohort,sessions,num_candidates,num_entry_family,num_int_family,num_positive,num_hard_negative,num_exclude,num_scoring_eligible_prelabels,scoring_eligible_prelabel_percent,mean_ai_confidence,median_ai_confidence,num_rows_with_two_passes,num_two_pass_semantic_agreement,two_pass_semantic_agreement_percent,num_two_pass_full_decision_agreement,two_pass_full_decision_agreement_percent,num_human_reviewed +Session_1,['Session_1'],496,285,211,116,182,198,298,60.08,0.7012,0.7,496,458,92.34,453,91.33,0 +Session_2,['Session_2'],490,334,156,71,252,167,323,65.92,0.7143,0.7,490,454,92.65,452,92.24,0 +Session_3,['Session_3'],545,308,237,117,184,244,301,55.23,0.7256,0.75,545,489,89.72,486,89.17,0 +Session_4,['Session_4'],597,471,126,84,381,132,465,77.89,0.7348,0.75,597,575,96.31,571,95.64,0 +Test pooled (Sessions 1-3),"['Session_1', 'Session_2', 'Session_3']",1531,927,604,304,618,609,922,60.22,0.7141,0.7,1531,1401,91.51,1391,90.86,0 +Session 4 audit (train split),['Session_4'],597,471,126,84,381,132,465,77.89,0.7348,0.75,597,575,96.31,571,95.64,0 +All supplied (descriptive only),"['Session_1', 'Session_2', 'Session_3', 'Session_4']",2128,1398,730,388,999,741,1387,65.18,0.7199,0.7,2128,1976,92.86,1962,92.2,0 diff --git a/reports/easycom_event_ai_prelabel_stats_v15/manuscript_ai_prelabel_table.tex b/reports/easycom_event_ai_prelabel_stats_v15/manuscript_ai_prelabel_table.tex new file mode 100644 index 0000000..912595b --- /dev/null +++ b/reports/easycom_event_ai_prelabel_stats_v15/manuscript_ai_prelabel_table.tex @@ -0,0 +1,21 @@ +% AI pre-label audit only; do not substitute for the human-adjudicated manuscript table. +\begin{tabular}{lrr} +\hline +AI pre-label outcome & Session 4 audit & Test Sessions 1--3 \\ +\hline +Positive: wearer entry & 75 & 265 \\ +HN: same-speaker pause & 104 & 189 \\ +HN: other-to-other transition & 251 & 228 \\ +HN: wearer backchannel & 11 & 95 \\ +Positive: INT-yield & 9 & 39 \\ +HN: other backchannel & 9 & 71 \\ +HN: non-takeover overlap & 6 & 23 \\ +HN: wearer holds floor & 0 & 12 \\ +HN: other & 0 & 0 \\ +Excluded: valid but out of scope & 110 & 479 \\ +Excluded: insufficient evidence & 22 & 130 \\ +Excluded: data quality & 0 & 0 \\ +Scoring-eligible total & 465 & 922 \\ +All candidates & 597 & 1531 \\ +\hline +\end{tabular} diff --git a/reports/easycom_event_ai_prelabel_stats_v15/summary.json b/reports/easycom_event_ai_prelabel_stats_v15/summary.json new file mode 100644 index 0000000..b0cf0c1 --- /dev/null +++ b/reports/easycom_event_ai_prelabel_stats_v15/summary.json @@ -0,0 +1,191 @@ +{ + "artifact_type": "AI_PRELABEL_DESCRIPTIVE_STATISTICS_NOT_GROUND_TRUTH", + "warning": "All supplied rows require human adjudication. Do not place these counts in the manuscript table labeled after human adjudication.", + "inputs": [ + "reports/easycom_entry_int_session1_v14/easycom_eot_int_deepseek_official.jsonl", + "reports/easycom_entry_int_session2_v14/easycom_eot_int_deepseek_official.jsonl", + "reports/easycom_entry_int_session3_v14/easycom_eot_int_deepseek_official.jsonl", + "reports/easycom_entry_int_session4_v14/easycom_eot_int_deepseek_official.jsonl" + ], + "metadata": { + "protocol_versions": [ + "easycom-wearer-entry-int-candidates-v8" + ], + "schema_versions": [ + "6" + ], + "prompt_versions": [ + "easycom-wearer-boundary-semantic-prelabel-v15" + ], + "models": [ + "deepseek-flash" + ], + "passes": [ + "2" + ], + "human_statuses": { + "UNREVIEWED": 2128 + } + }, + "cohorts": [ + { + "cohort": "Session_1", + "sessions": [ + "Session_1" + ], + "num_candidates": 496, + "num_entry_family": 285, + "num_int_family": 211, + "num_positive": 116, + "num_hard_negative": 182, + "num_exclude": 198, + "num_scoring_eligible_prelabels": 298, + "scoring_eligible_prelabel_percent": 60.08, + "mean_ai_confidence": 0.7012, + "median_ai_confidence": 0.7, + "num_rows_with_two_passes": 496, + "num_two_pass_semantic_agreement": 458, + "two_pass_semantic_agreement_percent": 92.34, + "num_two_pass_full_decision_agreement": 453, + "two_pass_full_decision_agreement_percent": 91.33, + "num_human_reviewed": 0 + }, + { + "cohort": "Session_2", + "sessions": [ + "Session_2" + ], + "num_candidates": 490, + "num_entry_family": 334, + "num_int_family": 156, + "num_positive": 71, + "num_hard_negative": 252, + "num_exclude": 167, + "num_scoring_eligible_prelabels": 323, + "scoring_eligible_prelabel_percent": 65.92, + "mean_ai_confidence": 0.7143, + "median_ai_confidence": 0.7, + "num_rows_with_two_passes": 490, + "num_two_pass_semantic_agreement": 454, + "two_pass_semantic_agreement_percent": 92.65, + "num_two_pass_full_decision_agreement": 452, + "two_pass_full_decision_agreement_percent": 92.24, + "num_human_reviewed": 0 + }, + { + "cohort": "Session_3", + "sessions": [ + "Session_3" + ], + "num_candidates": 545, + "num_entry_family": 308, + "num_int_family": 237, + "num_positive": 117, + "num_hard_negative": 184, + "num_exclude": 244, + "num_scoring_eligible_prelabels": 301, + "scoring_eligible_prelabel_percent": 55.23, + "mean_ai_confidence": 0.7256, + "median_ai_confidence": 0.75, + "num_rows_with_two_passes": 545, + "num_two_pass_semantic_agreement": 489, + "two_pass_semantic_agreement_percent": 89.72, + "num_two_pass_full_decision_agreement": 486, + "two_pass_full_decision_agreement_percent": 89.17, + "num_human_reviewed": 0 + }, + { + "cohort": "Session_4", + "sessions": [ + "Session_4" + ], + "num_candidates": 597, + "num_entry_family": 471, + "num_int_family": 126, + "num_positive": 84, + "num_hard_negative": 381, + "num_exclude": 132, + "num_scoring_eligible_prelabels": 465, + "scoring_eligible_prelabel_percent": 77.89, + "mean_ai_confidence": 0.7348, + "median_ai_confidence": 0.75, + "num_rows_with_two_passes": 597, + "num_two_pass_semantic_agreement": 575, + "two_pass_semantic_agreement_percent": 96.31, + "num_two_pass_full_decision_agreement": 571, + "two_pass_full_decision_agreement_percent": 95.64, + "num_human_reviewed": 0 + }, + { + "cohort": "Test pooled (Sessions 1-3)", + "sessions": [ + "Session_1", + "Session_2", + "Session_3" + ], + "num_candidates": 1531, + "num_entry_family": 927, + "num_int_family": 604, + "num_positive": 304, + "num_hard_negative": 618, + "num_exclude": 609, + "num_scoring_eligible_prelabels": 922, + "scoring_eligible_prelabel_percent": 60.22, + "mean_ai_confidence": 0.7141, + "median_ai_confidence": 0.7, + "num_rows_with_two_passes": 1531, + "num_two_pass_semantic_agreement": 1401, + "two_pass_semantic_agreement_percent": 91.51, + "num_two_pass_full_decision_agreement": 1391, + "two_pass_full_decision_agreement_percent": 90.86, + "num_human_reviewed": 0 + }, + { + "cohort": "Session 4 audit (train split)", + "sessions": [ + "Session_4" + ], + "num_candidates": 597, + "num_entry_family": 471, + "num_int_family": 126, + "num_positive": 84, + "num_hard_negative": 381, + "num_exclude": 132, + "num_scoring_eligible_prelabels": 465, + "scoring_eligible_prelabel_percent": 77.89, + "mean_ai_confidence": 0.7348, + "median_ai_confidence": 0.75, + "num_rows_with_two_passes": 597, + "num_two_pass_semantic_agreement": 575, + "two_pass_semantic_agreement_percent": 96.31, + "num_two_pass_full_decision_agreement": 571, + "two_pass_full_decision_agreement_percent": 95.64, + "num_human_reviewed": 0 + }, + { + "cohort": "All supplied (descriptive only)", + "sessions": [ + "Session_1", + "Session_2", + "Session_3", + "Session_4" + ], + "num_candidates": 2128, + "num_entry_family": 1398, + "num_int_family": 730, + "num_positive": 388, + "num_hard_negative": 999, + "num_exclude": 741, + "num_scoring_eligible_prelabels": 1387, + "scoring_eligible_prelabel_percent": 65.18, + "mean_ai_confidence": 0.7199, + "median_ai_confidence": 0.7, + "num_rows_with_two_passes": 2128, + "num_two_pass_semantic_agreement": 1976, + "two_pass_semantic_agreement_percent": 92.86, + "num_two_pass_full_decision_agreement": 1962, + "two_pass_full_decision_agreement_percent": 92.2, + "num_human_reviewed": 0 + } + ] +} diff --git a/reports/easycom_event_ai_prelabel_stats_v15/taxonomy_counts.csv b/reports/easycom_event_ai_prelabel_stats_v15/taxonomy_counts.csv new file mode 100644 index 0000000..8f49c34 --- /dev/null +++ b/reports/easycom_event_ai_prelabel_stats_v15/taxonomy_counts.csv @@ -0,0 +1,125 @@ +cohort,dimension,label,count,percent_of_all_candidates +Session_1,verdict,EXCLUDE,198,39.92 +Session_1,verdict,HARD_NEGATIVE,182,36.69 +Session_1,verdict,POSITIVE,116,23.39 +Session_1,positive_event,INT_YIELD,20,4.03 +Session_1,positive_event,WEARER_ENTRY,96,19.35 +Session_1,hard_negative_subtype,ENTRY_OTHER_MIDTURN_PAUSE,58,11.69 +Session_1,hard_negative_subtype,ENTRY_OTHER_TO_OTHER,56,11.29 +Session_1,hard_negative_subtype,ENTRY_WEARER_BACKCHANNEL,28,5.65 +Session_1,hard_negative_subtype,INT_NON_TAKEOVER_OVERLAP,10,2.02 +Session_1,hard_negative_subtype,INT_OTHER_BACKCHANNEL,29,5.85 +Session_1,hard_negative_subtype,INT_WEARER_HOLDS_FLOOR,1,0.2 +Session_1,exclude_reason,INSUFFICIENT_EVIDENCE,38,7.66 +Session_1,exclude_reason,OUT_OF_SCOPE_NON_EVENT,8,1.61 +Session_1,exclude_reason,OUT_OF_SCOPE_NORMAL_COMPLETION,104,20.97 +Session_1,exclude_reason,OUT_OF_SCOPE_POST_COMPLETION_ACTIVITY,21,4.23 +Session_1,exclude_reason,OUT_OF_SCOPE_WITHIN_TURN_CONTINUATION,27,5.44 +Session_1,exclude_group,INSUFFICIENT_EVIDENCE,38,7.66 +Session_1,exclude_group,OUT_OF_SCOPE,160,32.26 +Session_2,verdict,EXCLUDE,167,34.08 +Session_2,verdict,HARD_NEGATIVE,252,51.43 +Session_2,verdict,POSITIVE,71,14.49 +Session_2,positive_event,INT_YIELD,5,1.02 +Session_2,positive_event,WEARER_ENTRY,66,13.47 +Session_2,hard_negative_subtype,ENTRY_OTHER_MIDTURN_PAUSE,76,15.51 +Session_2,hard_negative_subtype,ENTRY_OTHER_TO_OTHER,123,25.1 +Session_2,hard_negative_subtype,ENTRY_WEARER_BACKCHANNEL,27,5.51 +Session_2,hard_negative_subtype,INT_NON_TAKEOVER_OVERLAP,7,1.43 +Session_2,hard_negative_subtype,INT_OTHER_BACKCHANNEL,15,3.06 +Session_2,hard_negative_subtype,INT_WEARER_HOLDS_FLOOR,4,0.82 +Session_2,exclude_reason,INSUFFICIENT_EVIDENCE,36,7.35 +Session_2,exclude_reason,OUT_OF_SCOPE_NON_EVENT,3,0.61 +Session_2,exclude_reason,OUT_OF_SCOPE_NORMAL_COMPLETION,97,19.8 +Session_2,exclude_reason,OUT_OF_SCOPE_POST_COMPLETION_ACTIVITY,4,0.82 +Session_2,exclude_reason,OUT_OF_SCOPE_WITHIN_TURN_CONTINUATION,27,5.51 +Session_2,exclude_group,INSUFFICIENT_EVIDENCE,36,7.35 +Session_2,exclude_group,OUT_OF_SCOPE,131,26.73 +Session_3,verdict,EXCLUDE,244,44.77 +Session_3,verdict,HARD_NEGATIVE,184,33.76 +Session_3,verdict,POSITIVE,117,21.47 +Session_3,positive_event,INT_YIELD,14,2.57 +Session_3,positive_event,WEARER_ENTRY,103,18.9 +Session_3,hard_negative_subtype,ENTRY_OTHER_MIDTURN_PAUSE,55,10.09 +Session_3,hard_negative_subtype,ENTRY_OTHER_TO_OTHER,49,8.99 +Session_3,hard_negative_subtype,ENTRY_WEARER_BACKCHANNEL,40,7.34 +Session_3,hard_negative_subtype,INT_NON_TAKEOVER_OVERLAP,6,1.1 +Session_3,hard_negative_subtype,INT_OTHER_BACKCHANNEL,27,4.95 +Session_3,hard_negative_subtype,INT_WEARER_HOLDS_FLOOR,7,1.28 +Session_3,exclude_reason,INSUFFICIENT_EVIDENCE,56,10.28 +Session_3,exclude_reason,OUT_OF_SCOPE_NON_EVENT,23,4.22 +Session_3,exclude_reason,OUT_OF_SCOPE_NORMAL_COMPLETION,99,18.17 +Session_3,exclude_reason,OUT_OF_SCOPE_POST_COMPLETION_ACTIVITY,25,4.59 +Session_3,exclude_reason,OUT_OF_SCOPE_WITHIN_TURN_CONTINUATION,41,7.52 +Session_3,exclude_group,INSUFFICIENT_EVIDENCE,56,10.28 +Session_3,exclude_group,OUT_OF_SCOPE,188,34.5 +Session_4,verdict,EXCLUDE,132,22.11 +Session_4,verdict,HARD_NEGATIVE,381,63.82 +Session_4,verdict,POSITIVE,84,14.07 +Session_4,positive_event,INT_YIELD,9,1.51 +Session_4,positive_event,WEARER_ENTRY,75,12.56 +Session_4,hard_negative_subtype,ENTRY_OTHER_MIDTURN_PAUSE,104,17.42 +Session_4,hard_negative_subtype,ENTRY_OTHER_TO_OTHER,251,42.04 +Session_4,hard_negative_subtype,ENTRY_WEARER_BACKCHANNEL,11,1.84 +Session_4,hard_negative_subtype,INT_NON_TAKEOVER_OVERLAP,6,1.01 +Session_4,hard_negative_subtype,INT_OTHER_BACKCHANNEL,9,1.51 +Session_4,exclude_reason,INSUFFICIENT_EVIDENCE,22,3.69 +Session_4,exclude_reason,OUT_OF_SCOPE_NON_EVENT,4,0.67 +Session_4,exclude_reason,OUT_OF_SCOPE_NORMAL_COMPLETION,90,15.08 +Session_4,exclude_reason,OUT_OF_SCOPE_POST_COMPLETION_ACTIVITY,3,0.5 +Session_4,exclude_reason,OUT_OF_SCOPE_WITHIN_TURN_CONTINUATION,13,2.18 +Session_4,exclude_group,INSUFFICIENT_EVIDENCE,22,3.69 +Session_4,exclude_group,OUT_OF_SCOPE,110,18.43 +Test pooled (Sessions 1-3),verdict,EXCLUDE,609,39.78 +Test pooled (Sessions 1-3),verdict,HARD_NEGATIVE,618,40.37 +Test pooled (Sessions 1-3),verdict,POSITIVE,304,19.86 +Test pooled (Sessions 1-3),positive_event,INT_YIELD,39,2.55 +Test pooled (Sessions 1-3),positive_event,WEARER_ENTRY,265,17.31 +Test pooled (Sessions 1-3),hard_negative_subtype,ENTRY_OTHER_MIDTURN_PAUSE,189,12.34 +Test pooled (Sessions 1-3),hard_negative_subtype,ENTRY_OTHER_TO_OTHER,228,14.89 +Test pooled (Sessions 1-3),hard_negative_subtype,ENTRY_WEARER_BACKCHANNEL,95,6.21 +Test pooled (Sessions 1-3),hard_negative_subtype,INT_NON_TAKEOVER_OVERLAP,23,1.5 +Test pooled (Sessions 1-3),hard_negative_subtype,INT_OTHER_BACKCHANNEL,71,4.64 +Test pooled (Sessions 1-3),hard_negative_subtype,INT_WEARER_HOLDS_FLOOR,12,0.78 +Test pooled (Sessions 1-3),exclude_reason,INSUFFICIENT_EVIDENCE,130,8.49 +Test pooled (Sessions 1-3),exclude_reason,OUT_OF_SCOPE_NON_EVENT,34,2.22 +Test pooled (Sessions 1-3),exclude_reason,OUT_OF_SCOPE_NORMAL_COMPLETION,300,19.6 +Test pooled (Sessions 1-3),exclude_reason,OUT_OF_SCOPE_POST_COMPLETION_ACTIVITY,50,3.27 +Test pooled (Sessions 1-3),exclude_reason,OUT_OF_SCOPE_WITHIN_TURN_CONTINUATION,95,6.21 +Test pooled (Sessions 1-3),exclude_group,INSUFFICIENT_EVIDENCE,130,8.49 +Test pooled (Sessions 1-3),exclude_group,OUT_OF_SCOPE,479,31.29 +Session 4 audit (train split),verdict,EXCLUDE,132,22.11 +Session 4 audit (train split),verdict,HARD_NEGATIVE,381,63.82 +Session 4 audit (train split),verdict,POSITIVE,84,14.07 +Session 4 audit (train split),positive_event,INT_YIELD,9,1.51 +Session 4 audit (train split),positive_event,WEARER_ENTRY,75,12.56 +Session 4 audit (train split),hard_negative_subtype,ENTRY_OTHER_MIDTURN_PAUSE,104,17.42 +Session 4 audit (train split),hard_negative_subtype,ENTRY_OTHER_TO_OTHER,251,42.04 +Session 4 audit (train split),hard_negative_subtype,ENTRY_WEARER_BACKCHANNEL,11,1.84 +Session 4 audit (train split),hard_negative_subtype,INT_NON_TAKEOVER_OVERLAP,6,1.01 +Session 4 audit (train split),hard_negative_subtype,INT_OTHER_BACKCHANNEL,9,1.51 +Session 4 audit (train split),exclude_reason,INSUFFICIENT_EVIDENCE,22,3.69 +Session 4 audit (train split),exclude_reason,OUT_OF_SCOPE_NON_EVENT,4,0.67 +Session 4 audit (train split),exclude_reason,OUT_OF_SCOPE_NORMAL_COMPLETION,90,15.08 +Session 4 audit (train split),exclude_reason,OUT_OF_SCOPE_POST_COMPLETION_ACTIVITY,3,0.5 +Session 4 audit (train split),exclude_reason,OUT_OF_SCOPE_WITHIN_TURN_CONTINUATION,13,2.18 +Session 4 audit (train split),exclude_group,INSUFFICIENT_EVIDENCE,22,3.69 +Session 4 audit (train split),exclude_group,OUT_OF_SCOPE,110,18.43 +All supplied (descriptive only),verdict,EXCLUDE,741,34.82 +All supplied (descriptive only),verdict,HARD_NEGATIVE,999,46.95 +All supplied (descriptive only),verdict,POSITIVE,388,18.23 +All supplied (descriptive only),positive_event,INT_YIELD,48,2.26 +All supplied (descriptive only),positive_event,WEARER_ENTRY,340,15.98 +All supplied (descriptive only),hard_negative_subtype,ENTRY_OTHER_MIDTURN_PAUSE,293,13.77 +All supplied (descriptive only),hard_negative_subtype,ENTRY_OTHER_TO_OTHER,479,22.51 +All supplied (descriptive only),hard_negative_subtype,ENTRY_WEARER_BACKCHANNEL,106,4.98 +All supplied (descriptive only),hard_negative_subtype,INT_NON_TAKEOVER_OVERLAP,29,1.36 +All supplied (descriptive only),hard_negative_subtype,INT_OTHER_BACKCHANNEL,80,3.76 +All supplied (descriptive only),hard_negative_subtype,INT_WEARER_HOLDS_FLOOR,12,0.56 +All supplied (descriptive only),exclude_reason,INSUFFICIENT_EVIDENCE,152,7.14 +All supplied (descriptive only),exclude_reason,OUT_OF_SCOPE_NON_EVENT,38,1.79 +All supplied (descriptive only),exclude_reason,OUT_OF_SCOPE_NORMAL_COMPLETION,390,18.33 +All supplied (descriptive only),exclude_reason,OUT_OF_SCOPE_POST_COMPLETION_ACTIVITY,53,2.49 +All supplied (descriptive only),exclude_reason,OUT_OF_SCOPE_WITHIN_TURN_CONTINUATION,108,5.08 +All supplied (descriptive only),exclude_group,INSUFFICIENT_EVIDENCE,152,7.14 +All supplied (descriptive only),exclude_group,OUT_OF_SCOPE,589,27.68 diff --git a/summarize_easycom_event_prelabels.py b/summarize_easycom_event_prelabels.py new file mode 100644 index 0000000..5eb0530 --- /dev/null +++ b/summarize_easycom_event_prelabels.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +"""Summarize EasyCom AI event pre-labels without treating them as ground truth.""" + +from __future__ import annotations + +import argparse +import csv +import json +import statistics +from collections import Counter +from pathlib import Path +from typing import Any, Iterable + + +MANUSCRIPT_ROWS = [ + ("Positive: wearer entry", "event", "WEARER_ENTRY"), + ("HN: same-speaker pause", "subtype", "ENTRY_OTHER_MIDTURN_PAUSE"), + ("HN: other-to-other transition", "subtype", "ENTRY_OTHER_TO_OTHER"), + ("HN: wearer backchannel", "subtype", "ENTRY_WEARER_BACKCHANNEL"), + ("Positive: INT-yield", "event", "INT_YIELD"), + ("HN: other backchannel", "subtype", "INT_OTHER_BACKCHANNEL"), + ("HN: non-takeover overlap", "subtype", "INT_NON_TAKEOVER_OVERLAP"), + ("HN: wearer holds floor", "subtype", "INT_WEARER_HOLDS_FLOOR"), + ("HN: other", "subtype", "OTHER_HARD_NEGATIVE"), + ("Excluded: valid but out of scope", "exclude_group", "OUT_OF_SCOPE"), + ("Excluded: insufficient evidence", "exclude_group", "INSUFFICIENT_EVIDENCE"), + ("Excluded: data quality", "exclude_group", "DATA_QUALITY"), + ("Scoring-eligible total", "derived", "SCORING_ELIGIBLE"), + ("All candidates", "derived", "ALL"), +] + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + with path.open(encoding="utf-8") as handle: + for line_number, line in enumerate(handle, 1): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError(f"Invalid JSON at {path}:{line_number}: {error}") from error + row["_source_file"] = str(path) + rows.append(row) + return rows + + +def _exclude_group(row: dict[str, Any]) -> str | None: + group = row.get("ai_exclude_reason_group") + if group: + return str(group) + reason = row.get("ai_exclude_reason") + if not reason: + return None + if str(reason).startswith("OUT_OF_SCOPE_"): + return "OUT_OF_SCOPE" + if str(reason).startswith("DATA_"): + return "DATA_QUALITY" + return str(reason) + + +def _decision_key(decision: dict[str, Any], *, include_opportunity: bool) -> tuple[Any, ...]: + keys = ["verdict", "event_type", "subtype", "exclude_reason"] + if include_opportunity: + keys.append("selected_opportunity_uid") + return tuple( + decision.get(key) for key in keys + ) + + +def _count(rows: Iterable[dict[str, Any]], dimension: str, value: str) -> int: + rows = list(rows) + if dimension == "event": + return sum(row.get("ai_verdict") == "POSITIVE" and row.get("ai_event_type") == value for row in rows) + if dimension == "subtype": + return sum(row.get("ai_verdict") == "HARD_NEGATIVE" and row.get("ai_subtype") == value for row in rows) + if dimension == "exclude_group": + return sum(row.get("ai_verdict") == "EXCLUDE" and _exclude_group(row) == value for row in rows) + if value == "SCORING_ELIGIBLE": + return sum(row.get("ai_verdict") in {"POSITIVE", "HARD_NEGATIVE"} for row in rows) + if value == "ALL": + return len(rows) + raise ValueError((dimension, value)) + + +def _cohort_summary(name: str, rows: list[dict[str, Any]]) -> dict[str, Any]: + confidences = [float(row["ai_confidence"]) for row in rows if row.get("ai_confidence") is not None] + pass_rows = 0 + semantic_agree = 0 + full_agree = 0 + for row in rows: + decisions = (row.get("llm_adjudication") or {}).get("decisions") or [] + if len(decisions) >= 2: + pass_rows += 1 + semantic_agree += len({_decision_key(decision, include_opportunity=False) for decision in decisions}) == 1 + full_agree += len({_decision_key(decision, include_opportunity=True) for decision in decisions}) == 1 + verdicts = Counter(str(row.get("ai_verdict")) for row in rows) + return { + "cohort": name, + "sessions": sorted({str(row.get("session")) for row in rows}), + "num_candidates": len(rows), + "num_entry_family": sum(row.get("task") == "wearer_entry" for row in rows), + "num_int_family": sum(row.get("task") == "int_yield" for row in rows), + "num_positive": verdicts["POSITIVE"], + "num_hard_negative": verdicts["HARD_NEGATIVE"], + "num_exclude": verdicts["EXCLUDE"], + "num_scoring_eligible_prelabels": verdicts["POSITIVE"] + verdicts["HARD_NEGATIVE"], + "scoring_eligible_prelabel_percent": round( + 100 * (verdicts["POSITIVE"] + verdicts["HARD_NEGATIVE"]) / len(rows), 2 + ) if rows else None, + "mean_ai_confidence": round(statistics.fmean(confidences), 4) if confidences else None, + "median_ai_confidence": round(statistics.median(confidences), 4) if confidences else None, + "num_rows_with_two_passes": pass_rows, + "num_two_pass_semantic_agreement": semantic_agree, + "two_pass_semantic_agreement_percent": round(100 * semantic_agree / pass_rows, 2) if pass_rows else None, + "num_two_pass_full_decision_agreement": full_agree, + "two_pass_full_decision_agreement_percent": round(100 * full_agree / pass_rows, 2) if pass_rows else None, + "num_human_reviewed": sum(row.get("human_status") != "UNREVIEWED" for row in rows), + } + + +def _write_csv(path: Path, fieldnames: list[str], rows: list[dict[str, Any]]) -> None: + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def summarize(paths: list[Path], output_dir: Path) -> dict[str, Any]: + all_rows: list[dict[str, Any]] = [] + for path in paths: + all_rows.extend(_read_jsonl(path)) + if not all_rows: + raise ValueError("No rows found") + + duplicate_uids = [uid for uid, count in Counter(row.get("uid") for row in all_rows).items() if count > 1] + if duplicate_uids: + raise ValueError(f"Duplicate uid across inputs: {duplicate_uids[:5]}") + + sessions = sorted({str(row.get("session")) for row in all_rows}) + by_session = {session: [row for row in all_rows if row.get("session") == session] for session in sessions} + test_rows = [row for row in all_rows if row.get("session") in {"Session_1", "Session_2", "Session_3"}] + session4_rows = by_session.get("Session_4", []) + cohorts: dict[str, list[dict[str, Any]]] = {**by_session} + if test_rows: + cohorts["Test pooled (Sessions 1-3)"] = test_rows + if session4_rows: + cohorts["Session 4 audit (train split)"] = session4_rows + cohorts["All supplied (descriptive only)"] = all_rows + + metadata = { + "protocol_versions": sorted({str(row.get("protocol_version")) for row in all_rows}), + "schema_versions": sorted({str(row.get("schema_version")) for row in all_rows}), + "prompt_versions": sorted({str((row.get("llm_adjudication") or {}).get("prompt_version")) for row in all_rows}), + "models": sorted({str((row.get("llm_adjudication") or {}).get("model")) for row in all_rows}), + "passes": sorted({str((row.get("llm_adjudication") or {}).get("passes")) for row in all_rows}), + "human_statuses": dict(sorted(Counter(str(row.get("human_status")) for row in all_rows).items())), + } + for key in ("protocol_versions", "schema_versions", "prompt_versions", "models", "passes"): + if len(metadata[key]) != 1: + raise ValueError(f"Cannot aggregate mixed {key}: {metadata[key]}") + summary = { + "artifact_type": "AI_PRELABEL_DESCRIPTIVE_STATISTICS_NOT_GROUND_TRUTH", + "warning": "All supplied rows require human adjudication. Do not place these counts in the manuscript table labeled after human adjudication.", + "inputs": [str(path) for path in paths], + "metadata": metadata, + "cohorts": [_cohort_summary(name, rows) for name, rows in cohorts.items()], + } + + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + cohort_rows = summary["cohorts"] + _write_csv(output_dir / "cohort_summary.csv", list(cohort_rows[0]), cohort_rows) + + taxonomy_rows: list[dict[str, Any]] = [] + source_rows: list[dict[str, Any]] = [] + for name, rows in cohorts.items(): + total = len(rows) + dimensions = { + "verdict": Counter(str(row.get("ai_verdict")) for row in rows), + "positive_event": Counter(str(row.get("ai_event_type")) for row in rows if row.get("ai_verdict") == "POSITIVE"), + "hard_negative_subtype": Counter(str(row.get("ai_subtype")) for row in rows if row.get("ai_verdict") == "HARD_NEGATIVE"), + "exclude_reason": Counter(str(row.get("ai_exclude_reason")) for row in rows if row.get("ai_verdict") == "EXCLUDE"), + "exclude_group": Counter(str(_exclude_group(row)) for row in rows if row.get("ai_verdict") == "EXCLUDE"), + } + for dimension, counts in dimensions.items(): + for label, count in sorted(counts.items()): + taxonomy_rows.append({ + "cohort": name, "dimension": dimension, "label": label, + "count": count, "percent_of_all_candidates": round(100 * count / total, 2), + }) + for source, count in sorted(Counter(str(row.get("candidate_source")) for row in rows).items()): + source_rows.append({ + "cohort": name, "candidate_source": source, "count": count, + "percent_of_all_candidates": round(100 * count / total, 2), + }) + _write_csv( + output_dir / "taxonomy_counts.csv", + ["cohort", "dimension", "label", "count", "percent_of_all_candidates"], + taxonomy_rows, + ) + _write_csv( + output_dir / "candidate_source_counts.csv", + ["cohort", "candidate_source", "count", "percent_of_all_candidates"], + source_rows, + ) + + train_label = "Session 4 audit (train split)" + test_label = "Test pooled (Sessions 1-3)" + table_lines = [ + "% AI pre-label audit only; do not substitute for the human-adjudicated manuscript table.", + "\\begin{tabular}{lrr}", + "\\hline", + "AI pre-label outcome & Session 4 audit & Test Sessions 1--3 \\\\", + "\\hline", + ] + for label, dimension, value in MANUSCRIPT_ROWS: + train_count = _count(cohorts.get(train_label, []), dimension, value) + test_count = _count(cohorts.get(test_label, []), dimension, value) + table_lines.append(f"{label} & {train_count} & {test_count} \\\\") + table_lines.extend(["\\hline", "\\end{tabular}", ""]) + (output_dir / "manuscript_ai_prelabel_table.tex").write_text("\n".join(table_lines), encoding="utf-8") + + report_lines = [ + "# EasyCom event AI pre-label audit", + "", + "> These are DeepSeek pre-labels awaiting mandatory human adjudication, not ground truth and not final manuscript annotation counts.", + "", + f"Inputs contain {len(all_rows):,} unique candidates from {', '.join(sessions)}. " + f"Protocol: `{', '.join(metadata['protocol_versions'])}`; prompt: `{', '.join(metadata['prompt_versions'])}`.", + "", + "## Cohort overview", + "", + "| Cohort | Candidates | Positive | Hard negative | Exclude | Prelabel scoring-eligible | Two-pass semantic/full agreement | Human reviewed |", + "|---|---:|---:|---:|---:|---:|---:|---:|", + ] + for item in cohort_rows: + report_lines.append( + f"| {item['cohort']} | {item['num_candidates']} | {item['num_positive']} | " + f"{item['num_hard_negative']} | {item['num_exclude']} | " + f"{item['num_scoring_eligible_prelabels']} ({item['scoring_eligible_prelabel_percent']:.2f}%) | " + f"{item['num_two_pass_semantic_agreement']}/{item['num_rows_with_two_passes']} " + f"({item['two_pass_semantic_agreement_percent']:.2f}%) / " + f"{item['num_two_pass_full_decision_agreement']}/{item['num_rows_with_two_passes']} " + f"({item['two_pass_full_decision_agreement_percent']:.2f}%) | {item['num_human_reviewed']} |" + ) + report_lines.extend([ + "", + "The official EasyCom test sessions (1--3) are pooled only for descriptive reporting; these AI pre-labels must not be used to tune thresholds, taxonomy, prompts, or model choices. Session 4 is shown separately as a train-split audit and is not the complete Sessions 4--12 training corpus.", + "", + "Detailed counts are in `taxonomy_counts.csv` and `candidate_source_counts.csv`. " + "`manuscript_ai_prelabel_table.tex` mirrors the manuscript row order but is deliberately labeled as an AI audit; retain the manuscript TODOs until human adjudication is frozen.", + "", + "## Reproduce", + "", + "```bash", + "python3 summarize_easycom_event_prelabels.py \\", + ]) + for index, path in enumerate(paths): + suffix = " \\" if index < len(paths) - 1 else " \\" + report_lines.append(f" --input {path}{suffix}") + report_lines.extend([f" --output-dir {output_dir}", "```", ""]) + (output_dir / "README.md").write_text("\n".join(report_lines), encoding="utf-8") + return summary + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", action="append", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + summarize(args.input, args.output_dir) + + +if __name__ == "__main__": + main() diff --git a/test_adjudicate_easycom_eot_int_candidates.py b/test_adjudicate_easycom_eot_int_candidates.py index 4130233..ad59863 100644 --- a/test_adjudicate_easycom_eot_int_candidates.py +++ b/test_adjudicate_easycom_eot_int_candidates.py @@ -60,7 +60,7 @@ class EntryINTAdjudicationTests(unittest.TestCase): self.assertIn("Yes.", prompt) def test_system_prompt_treats_laughter_as_nonlexical_floor_feedback(self) -> None: - self.assertEqual(PROMPT_VERSION, "easycom-wearer-boundary-semantic-prelabel-v15") + self.assertEqual(PROMPT_VERSION, "easycom-wearer-boundary-semantic-prelabel-v16") self.assertIn("opportunity_options", SYSTEM_PROMPT) self.assertIn("t_w < t_o < t_y never proves takeover", SYSTEM_PROMPT) self.assertIn("Every supplied WEARER_ONSET and WEARER_OFFSET is a raw observed", SYSTEM_PROMPT) @@ -76,6 +76,9 @@ class EntryINTAdjudicationTests(unittest.TestCase): self.assertIn("does not answer a question, accept or reject a proposal", SYSTEM_PROMPT) self.assertIn('the wearer answers "Nope." at 54.15 s', SYSTEM_PROMPT) self.assertIn("participant 3 later speaks again", SYSTEM_PROMPT) + self.assertIn("whether the wearer activity run ending there ever held the floor", SYSTEM_PROMPT) + self.assertIn("OUT_OF_SCOPE_WEARER_BACKCHANNEL_OFFSET", SYSTEM_PROMPT) + self.assertIn("do not count its trailing offset again", SYSTEM_PROMPT) self.assertIn("does not require a preceding non-wearer release", SYSTEM_PROMPT) self.assertIn("duration alone never determines floor function", SYSTEM_PROMPT) self.assertIn("normal wearer completion", SYSTEM_PROMPT) @@ -182,6 +185,9 @@ class EntryINTAdjudicationTests(unittest.TestCase): self.assertEqual( exclude_reason_group("OUT_OF_SCOPE_NORMAL_COMPLETION"), "OUT_OF_SCOPE" ) + self.assertEqual( + exclude_reason_group("OUT_OF_SCOPE_WEARER_BACKCHANNEL_OFFSET"), "OUT_OF_SCOPE" + ) self.assertEqual( exclude_reason_group("INSUFFICIENT_EVIDENCE"), "INSUFFICIENT_EVIDENCE" ) diff --git a/test_summarize_easycom_event_prelabels.py b/test_summarize_easycom_event_prelabels.py new file mode 100644 index 0000000..d51abcf --- /dev/null +++ b/test_summarize_easycom_event_prelabels.py @@ -0,0 +1,52 @@ +"""Tests for descriptive EasyCom AI pre-label aggregation.""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from summarize_easycom_event_prelabels import summarize + + +class EventPrelabelSummaryTests(unittest.TestCase): + def test_summary_separates_ai_prelabels_and_two_pass_agreement(self) -> None: + base = { + "protocol_version": "p1", "schema_version": 6, + "candidate_source": "WEARER_ONSET", "task": "wearer_entry", + "human_status": "UNREVIEWED", "ai_confidence": 0.8, + "llm_adjudication": {"prompt_version": "v1", "model": "m", "passes": 2}, + } + rows = [ + {**base, "uid": "u1", "session": "Session_1", "ai_verdict": "POSITIVE", + "ai_event_type": "WEARER_ENTRY", "ai_subtype": None, + "llm_adjudication": {**base["llm_adjudication"], "decisions": [ + {"verdict": "POSITIVE", "event_type": "WEARER_ENTRY"}, + {"verdict": "POSITIVE", "event_type": "WEARER_ENTRY"}, + ]}}, + {**base, "uid": "u2", "session": "Session_4", "task": "int_yield", + "candidate_source": "WEARER_OFFSET", "ai_verdict": "EXCLUDE", + "ai_event_type": None, "ai_subtype": None, + "ai_exclude_reason": "OUT_OF_SCOPE_NORMAL_COMPLETION", + "ai_exclude_reason_group": "OUT_OF_SCOPE", + "llm_adjudication": {**base["llm_adjudication"], "decisions": [ + {"verdict": "EXCLUDE", "exclude_reason": "OUT_OF_SCOPE_NORMAL_COMPLETION"}, + {"verdict": "EXCLUDE", "exclude_reason": "INSUFFICIENT_EVIDENCE"}, + ]}}, + ] + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "input.jsonl" + source.write_text("".join(json.dumps(row) + "\n" for row in rows)) + summary = summarize([source], root / "out") + cohorts = {item["cohort"]: item for item in summary["cohorts"]} + self.assertEqual(cohorts["Test pooled (Sessions 1-3)"]["num_positive"], 1) + self.assertEqual(cohorts["Session 4 audit (train split)"]["num_exclude"], 1) + self.assertEqual(cohorts["All supplied (descriptive only)"]["num_two_pass_semantic_agreement"], 1) + self.assertEqual(cohorts["All supplied (descriptive only)"]["num_two_pass_full_decision_agreement"], 1) + self.assertIn("AI pre-label audit only", (root / "out/manuscript_ai_prelabel_table.tex").read_text()) + + +if __name__ == "__main__": + unittest.main()