192 lines
5.9 KiB
JSON
192 lines
5.9 KiB
JSON
{
|
|
"artifact_type": "AI_PRELABEL_DESCRIPTIVE_STATISTICS_NOT_GROUND_TRUTH",
|
|
"warning": "All supplied rows require human adjudication. Do not place these counts in the manuscript table labeled after human adjudication.",
|
|
"inputs": [
|
|
"reports/easycom_entry_int_session1_v16/easycom_eot_int_deepseek_official.jsonl",
|
|
"reports/easycom_entry_int_session2_v16/easycom_eot_int_deepseek_official.jsonl",
|
|
"reports/easycom_entry_int_session3_v16/easycom_eot_int_deepseek_official.jsonl",
|
|
"reports/easycom_entry_int_session4_v16/easycom_eot_int_deepseek_official.jsonl"
|
|
],
|
|
"metadata": {
|
|
"protocol_versions": [
|
|
"easycom-wearer-entry-int-candidates-v8"
|
|
],
|
|
"schema_versions": [
|
|
"6"
|
|
],
|
|
"prompt_versions": [
|
|
"easycom-wearer-boundary-semantic-prelabel-v16"
|
|
],
|
|
"models": [
|
|
"deepseek-flash"
|
|
],
|
|
"passes": [
|
|
"2"
|
|
],
|
|
"human_statuses": {
|
|
"UNREVIEWED": 2128
|
|
}
|
|
},
|
|
"cohorts": [
|
|
{
|
|
"cohort": "Session_1",
|
|
"sessions": [
|
|
"Session_1"
|
|
],
|
|
"num_candidates": 496,
|
|
"num_entry_family": 285,
|
|
"num_int_family": 211,
|
|
"num_positive": 113,
|
|
"num_hard_negative": 192,
|
|
"num_exclude": 191,
|
|
"num_scoring_eligible_prelabels": 305,
|
|
"scoring_eligible_prelabel_percent": 61.49,
|
|
"mean_ai_confidence": 0.6863,
|
|
"median_ai_confidence": 0.7,
|
|
"num_rows_with_two_passes": 496,
|
|
"num_two_pass_semantic_agreement": 458,
|
|
"two_pass_semantic_agreement_percent": 92.34,
|
|
"num_two_pass_full_decision_agreement": 456,
|
|
"two_pass_full_decision_agreement_percent": 91.94,
|
|
"num_human_reviewed": 0
|
|
},
|
|
{
|
|
"cohort": "Session_2",
|
|
"sessions": [
|
|
"Session_2"
|
|
],
|
|
"num_candidates": 490,
|
|
"num_entry_family": 334,
|
|
"num_int_family": 156,
|
|
"num_positive": 64,
|
|
"num_hard_negative": 260,
|
|
"num_exclude": 166,
|
|
"num_scoring_eligible_prelabels": 324,
|
|
"scoring_eligible_prelabel_percent": 66.12,
|
|
"mean_ai_confidence": 0.6966,
|
|
"median_ai_confidence": 0.7,
|
|
"num_rows_with_two_passes": 490,
|
|
"num_two_pass_semantic_agreement": 467,
|
|
"two_pass_semantic_agreement_percent": 95.31,
|
|
"num_two_pass_full_decision_agreement": 464,
|
|
"two_pass_full_decision_agreement_percent": 94.69,
|
|
"num_human_reviewed": 0
|
|
},
|
|
{
|
|
"cohort": "Session_3",
|
|
"sessions": [
|
|
"Session_3"
|
|
],
|
|
"num_candidates": 545,
|
|
"num_entry_family": 308,
|
|
"num_int_family": 237,
|
|
"num_positive": 112,
|
|
"num_hard_negative": 200,
|
|
"num_exclude": 233,
|
|
"num_scoring_eligible_prelabels": 312,
|
|
"scoring_eligible_prelabel_percent": 57.25,
|
|
"mean_ai_confidence": 0.7134,
|
|
"median_ai_confidence": 0.7,
|
|
"num_rows_with_two_passes": 545,
|
|
"num_two_pass_semantic_agreement": 507,
|
|
"two_pass_semantic_agreement_percent": 93.03,
|
|
"num_two_pass_full_decision_agreement": 504,
|
|
"two_pass_full_decision_agreement_percent": 92.48,
|
|
"num_human_reviewed": 0
|
|
},
|
|
{
|
|
"cohort": "Session_4",
|
|
"sessions": [
|
|
"Session_4"
|
|
],
|
|
"num_candidates": 597,
|
|
"num_entry_family": 471,
|
|
"num_int_family": 126,
|
|
"num_positive": 78,
|
|
"num_hard_negative": 365,
|
|
"num_exclude": 154,
|
|
"num_scoring_eligible_prelabels": 443,
|
|
"scoring_eligible_prelabel_percent": 74.2,
|
|
"mean_ai_confidence": 0.6997,
|
|
"median_ai_confidence": 0.7,
|
|
"num_rows_with_two_passes": 597,
|
|
"num_two_pass_semantic_agreement": 555,
|
|
"two_pass_semantic_agreement_percent": 92.96,
|
|
"num_two_pass_full_decision_agreement": 550,
|
|
"two_pass_full_decision_agreement_percent": 92.13,
|
|
"num_human_reviewed": 0
|
|
},
|
|
{
|
|
"cohort": "Test pooled (Sessions 1-3)",
|
|
"sessions": [
|
|
"Session_1",
|
|
"Session_2",
|
|
"Session_3"
|
|
],
|
|
"num_candidates": 1531,
|
|
"num_entry_family": 927,
|
|
"num_int_family": 604,
|
|
"num_positive": 289,
|
|
"num_hard_negative": 652,
|
|
"num_exclude": 590,
|
|
"num_scoring_eligible_prelabels": 941,
|
|
"scoring_eligible_prelabel_percent": 61.46,
|
|
"mean_ai_confidence": 0.6992,
|
|
"median_ai_confidence": 0.7,
|
|
"num_rows_with_two_passes": 1531,
|
|
"num_two_pass_semantic_agreement": 1432,
|
|
"two_pass_semantic_agreement_percent": 93.53,
|
|
"num_two_pass_full_decision_agreement": 1424,
|
|
"two_pass_full_decision_agreement_percent": 93.01,
|
|
"num_human_reviewed": 0
|
|
},
|
|
{
|
|
"cohort": "Session 4 audit (train split)",
|
|
"sessions": [
|
|
"Session_4"
|
|
],
|
|
"num_candidates": 597,
|
|
"num_entry_family": 471,
|
|
"num_int_family": 126,
|
|
"num_positive": 78,
|
|
"num_hard_negative": 365,
|
|
"num_exclude": 154,
|
|
"num_scoring_eligible_prelabels": 443,
|
|
"scoring_eligible_prelabel_percent": 74.2,
|
|
"mean_ai_confidence": 0.6997,
|
|
"median_ai_confidence": 0.7,
|
|
"num_rows_with_two_passes": 597,
|
|
"num_two_pass_semantic_agreement": 555,
|
|
"two_pass_semantic_agreement_percent": 92.96,
|
|
"num_two_pass_full_decision_agreement": 550,
|
|
"two_pass_full_decision_agreement_percent": 92.13,
|
|
"num_human_reviewed": 0
|
|
},
|
|
{
|
|
"cohort": "All supplied (descriptive only)",
|
|
"sessions": [
|
|
"Session_1",
|
|
"Session_2",
|
|
"Session_3",
|
|
"Session_4"
|
|
],
|
|
"num_candidates": 2128,
|
|
"num_entry_family": 1398,
|
|
"num_int_family": 730,
|
|
"num_positive": 367,
|
|
"num_hard_negative": 1017,
|
|
"num_exclude": 744,
|
|
"num_scoring_eligible_prelabels": 1384,
|
|
"scoring_eligible_prelabel_percent": 65.04,
|
|
"mean_ai_confidence": 0.6994,
|
|
"median_ai_confidence": 0.7,
|
|
"num_rows_with_two_passes": 2128,
|
|
"num_two_pass_semantic_agreement": 1987,
|
|
"two_pass_semantic_agreement_percent": 93.37,
|
|
"num_two_pass_full_decision_agreement": 1974,
|
|
"two_pass_full_decision_agreement_percent": 92.76,
|
|
"num_human_reviewed": 0
|
|
}
|
|
]
|
|
}
|