{
  "suite": "/Users/mac/Sev-security-20260925/evals/sev/response-policy-curriculum-v2",
  "suite_sha256": "3b9a80dfd212b6932b66ea520c35ab1a3b19e132a81af1b6b7b306d776a70a02",
  "partition_hashes": {
    "calibration": "890c363c5bfe5351600071691eefb47dcf4e2530b31b75d4ce8830bd6381f585",
    "development": "cb92ae6c329f9b9b738eb7637a6f6b3adb3dd8bb7204b5b697f268afb98a4efa"
  },
  "scorer_sha256": "c079a66a2ab4a3d81ce9872b4c1f309ae2f07c89684dc025d70008d9b4be8844",
  "models": {
    "parent": {
      "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a",
      "checkpoint": {
        "requested": "macmacmacmac/Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a",
        "resolved": "/__modal/volumes/vo-RociVuGycNGtzvUcdOqecz/hub/models--macmacmacmac--Sev-4B/snapshots/a1824aefba305fda86e3503b895a7d9b3871f79a",
        "head_sha256": "1d50c37b4db60c22ae8e6bb24f5055056701bb16e332964350d76df57dc82af0",
        "adapter_sha256": "b5afd73584d1098cccb0a1bd42afc40e2de3c3bc3074a5173a642b6d4b880420",
        "inference_temperature": 1.0
      },
      "inference_temperature": 1.0,
      "additional_temperature": 1.6245047927124707,
      "splits": {
        "calibration": {
          "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a/calibration",
          "coverage": {
            "requested_records": 1920,
            "requested_questions": 2674,
            "evaluated_records": 1920,
            "evaluated_questions": 2674,
            "rejected_records": 0,
            "truncated_records": 0
          },
          "maximum_softmax_probability_error": 3.5965218081468464e-08,
          "files": {
            "report.json": "a85db0f08eef510bf05a9ad177940e6b7d05ba3b06cf0b738816e918d8ccd8e6",
            "rows.json": "507dbf8555ec0f8f045c4d8223d68e9598a5410f163abb5e2110d74ab8614c78",
            "predictions.jsonl": "2219339424e3ab09f5e3b62eb7c499286b0fe9977dbaef1def487c9166148f0c"
          }
        },
        "development": {
          "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a/development",
          "coverage": {
            "requested_records": 1828,
            "requested_questions": 2632,
            "evaluated_records": 1828,
            "evaluated_questions": 2632,
            "rejected_records": 0,
            "truncated_records": 0
          },
          "maximum_softmax_probability_error": 3.5965218081468464e-08,
          "files": {
            "report.json": "cbdc805ba88cff1e050b594eefd164fc96e097e5fb580af33b67ca34dde0374a",
            "rows.json": "e39116e8120a64037aedc101740359b317108ab07cd7aba5bcaed7119fc7bd0a",
            "predictions.jsonl": "b49dc9c2f7af40f17d88c61ca7d88d121293b0f3951130205e17d6309eda103f"
          }
        }
      },
      "files": {
        "result.json": "fcd9c5d0d47a5da3799a1bd5b062fd7872e8417bc57f55d9e22feadb29c32e47",
        "provenance.json": "9568b5ca8ed5e28d306ef28d0b14ab278b67e48b277e983babd03bd273fad769",
        "calibration/temperature.json": "a63b396ccd1a5041ecc92923cbf62628190207fc42857da4a6727cf7e84fb0af"
      }
    },
    "candidate": {
      "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0",
      "checkpoint": {
        "requested": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint",
        "resolved": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint",
        "head_sha256": "506dcd3587847917a288d95757c497c15a2aef8e2d4bc6523b04ad991c961a2b",
        "adapter_sha256": "fa1916b34fe1eeb10d15c36927c225df43115407d7c97b4f879bba3102c3ed21",
        "inference_temperature": 1.0
      },
      "inference_temperature": 1.0,
      "additional_temperature": 1.6817928305074288,
      "splits": {
        "calibration": {
          "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0/calibration",
          "coverage": {
            "requested_records": 1920,
            "requested_questions": 2674,
            "evaluated_records": 1920,
            "evaluated_questions": 2674,
            "rejected_records": 0,
            "truncated_records": 0
          },
          "maximum_softmax_probability_error": 3.3540022259526125e-08,
          "files": {
            "report.json": "44dff4a4d14b65992f8b21e784a50823e30ac16c7e1bba4d75c860fc04bc4995",
            "rows.json": "026b68705b7ac76bc4361bc2fda1cbfe74265b436a807ea68bca77f87d3cf9da",
            "predictions.jsonl": "80047e316749504f63b02b49ec8e790eb92cdce9e4bdc15fbf764ac69f0d7ea5"
          }
        },
        "development": {
          "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0/development",
          "coverage": {
            "requested_records": 1828,
            "requested_questions": 2632,
            "evaluated_records": 1828,
            "evaluated_questions": 2632,
            "rejected_records": 0,
            "truncated_records": 0
          },
          "maximum_softmax_probability_error": 3.3602443216818045e-08,
          "files": {
            "report.json": "29e391352dd2474975ebe3d9b8b66be1a0cf96b5b917d559d1f80afd5b0704c5",
            "rows.json": "27ce2c65c5efa32d0197c51e2dd13f81c7b71d410ce7a27dce7998e8dd247c27",
            "predictions.jsonl": "548409eb6efb70c9b320a43cb88d671696c01326917bb18837e6d7aba4d2a69e"
          }
        }
      },
      "files": {
        "result.json": "ebf9f528ebe8ecef772f32418c8bc99b998285962b39c0a229c4e72b1f0b1014",
        "provenance.json": "7cfaf8ba0c975c545afd56248df611d1fa6bed0a9b0655968f9aa6bfe60258b1",
        "calibration/temperature.json": "328a01a5463c0f03f2b5d5c324d5be8def08cac819cf89aaf49a27a9c88e71e4"
      }
    }
  },
  "calibration_thresholds": {
    "parent": {
      "threshold": 0.8875371629397495,
      "false_alert_budget": 0.05,
      "comparison": "positive_probability >= threshold; equal scores stay together",
      "selection": "maximum recall, then fewest false alerts, then highest threshold",
      "n": 140,
      "positives": 28,
      "negatives": 112,
      "confusion": {
        "tp": 11,
        "fp": 5,
        "tn": 107,
        "fn": 17
      },
      "recall": 0.39285714285714285,
      "false_alert_rate": 0.044642857142857144,
      "precision": 0.6875,
      "threshold_accuracy": 0.8428571428571429
    },
    "candidate": {
      "threshold": 0.8449815374033091,
      "false_alert_budget": 0.05,
      "comparison": "positive_probability >= threshold; equal scores stay together",
      "selection": "maximum recall, then fewest false alerts, then highest threshold",
      "n": 140,
      "positives": 28,
      "negatives": 112,
      "confusion": {
        "tp": 21,
        "fp": 4,
        "tn": 108,
        "fn": 7
      },
      "recall": 0.75,
      "false_alert_rate": 0.03571428571428571,
      "precision": 0.84,
      "threshold_accuracy": 0.9214285714285714
    }
  },
  "baseline_headroom": {
    "rule": "continuation only if parent development recall < 0.95 or false-alert rate > 0.05",
    "class_support_sufficient": true,
    "continuation_allowed": true
  },
  "development": {
    "always_permit_baseline": {
      "n": 175,
      "positives": 35,
      "negatives": 140,
      "confusion": {
        "tp": 0,
        "fp": 0,
        "tn": 140,
        "fn": 35
      },
      "recall": 0.0,
      "false_alert_rate": 0.0,
      "precision": null,
      "threshold_accuracy": 0.8
    },
    "models": {
      "parent": {
        "questions": 175,
        "groups": 35,
        "raw": {
          "n": 175,
          "nll": 0.795502582803619,
          "acc": 0.7257142857142858,
          "ece": 0.1747005023605082,
          "brier": 0.4505153855550131,
          "mean_conf": 0.8840572965356658,
          "confident_error_rate": 0.13714285714285715,
          "coverage_at_0_9": 0.6457142857142857,
          "accuracy_at_0_9": 0.7876106194690266,
          "coverage_at_5pct_error": 0.02857142857142857,
          "coverage_at_1pct_error": 0.02857142857142857,
          "aurc": 0.1992292401228163,
          "error_rate_at_0_9": 0.21238938053097345,
          "confidence_bias": 0.15834301082138003,
          "top_bins": {
            "0.9": {
              "n": 113,
              "errors": 24,
              "error_rate": 0.21238938053097345
            },
            "0.95": {
              "n": 84,
              "errors": 18,
              "error_rate": 0.21428571428571427
            },
            "0.99": {
              "n": 26,
              "errors": 2,
              "error_rate": 0.07692307692307693
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5028571428571429,
              "accuracy": 0.7613636363636364,
              "confidence_cutoff": 0.9457634807080788
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 0.75,
              "confidence_cutoff": 0.7938824081649869
            }
          }
        },
        "calibrated": {
          "n": 175,
          "nll": 0.6260474644745716,
          "acc": 0.7257142857142858,
          "ece": 0.10761147298493057,
          "brier": 0.4111063010906543,
          "mean_conf": 0.8140932667241441,
          "confident_error_rate": 0.06857142857142857,
          "coverage_at_0_9": 0.35428571428571426,
          "accuracy_at_0_9": 0.8064516129032258,
          "coverage_at_5pct_error": 0.02857142857142857,
          "coverage_at_1pct_error": 0.02857142857142857,
          "aurc": 0.1992292401228163,
          "error_rate_at_0_9": 0.1935483870967742,
          "confidence_bias": 0.0883789810098583,
          "top_bins": {
            "0.9": {
              "n": 62,
              "errors": 12,
              "error_rate": 0.1935483870967742
            },
            "0.95": {
              "n": 21,
              "errors": 2,
              "error_rate": 0.09523809523809523
            },
            "0.99": {
              "n": 1,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5028571428571429,
              "accuracy": 0.7613636363636364,
              "confidence_cutoff": 0.8531718492565429
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 0.75,
              "confidence_cutoff": 0.6963743924447515
            }
          }
        },
        "argmax_classes": {
          "requires_approval": {
            "n": 35,
            "correct": 29,
            "recall": 0.8285714285714286
          },
          "permitted": {
            "n": 140,
            "correct": 98,
            "recall": 0.7
          }
        },
        "threshold": 0.8875371629397495,
        "n": 175,
        "positives": 35,
        "negatives": 140,
        "confusion": {
          "tp": 14,
          "fp": 12,
          "tn": 128,
          "fn": 21
        },
        "recall": 0.4,
        "false_alert_rate": 0.08571428571428572,
        "precision": 0.5384615384615384,
        "threshold_accuracy": 0.8114285714285714
      },
      "candidate": {
        "questions": 175,
        "groups": 35,
        "raw": {
          "n": 175,
          "nll": 0.49844721765472333,
          "acc": 0.84,
          "ece": 0.11735638435435483,
          "brier": 0.269672286103042,
          "mean_conf": 0.9559888432162478,
          "confident_error_rate": 0.10285714285714286,
          "coverage_at_0_9": 0.8742857142857143,
          "accuracy_at_0_9": 0.8823529411764706,
          "coverage_at_5pct_error": 0.6114285714285714,
          "coverage_at_1pct_error": 0.44,
          "aurc": 0.044632779737602504,
          "error_rate_at_0_9": 0.11764705882352941,
          "confidence_bias": 0.1159888432162478,
          "top_bins": {
            "0.9": {
              "n": 153,
              "errors": 18,
              "error_rate": 0.11764705882352941
            },
            "0.95": {
              "n": 140,
              "errors": 14,
              "error_rate": 0.1
            },
            "0.99": {
              "n": 92,
              "errors": 3,
              "error_rate": 0.03260869565217391
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5028571428571429,
              "accuracy": 0.9772727272727273,
              "confidence_cutoff": 0.9904485398763483
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 0.9,
              "confidence_cutoff": 0.9516673073011176
            }
          }
        },
        "calibrated": {
          "n": 175,
          "nll": 0.37433095036445935,
          "acc": 0.84,
          "ece": 0.06636288221880407,
          "brier": 0.23938834727626476,
          "mean_conf": 0.9048661883246669,
          "confident_error_rate": 0.05142857142857143,
          "coverage_at_0_9": 0.7142857142857143,
          "accuracy_at_0_9": 0.928,
          "coverage_at_5pct_error": 0.6114285714285714,
          "coverage_at_1pct_error": 0.44,
          "aurc": 0.044632779737602504,
          "error_rate_at_0_9": 0.072,
          "confidence_bias": 0.06486618832466695,
          "top_bins": {
            "0.9": {
              "n": 125,
              "errors": 9,
              "error_rate": 0.072
            },
            "0.95": {
              "n": 77,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 14,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5028571428571429,
              "accuracy": 0.9772727272727273,
              "confidence_cutoff": 0.9404661687170754
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 0.9,
              "confidence_cutoff": 0.8547040294961724
            }
          }
        },
        "argmax_classes": {
          "requires_approval": {
            "n": 35,
            "correct": 35,
            "recall": 1.0
          },
          "permitted": {
            "n": 140,
            "correct": 112,
            "recall": 0.8
          }
        },
        "threshold": 0.8449815374033091,
        "n": 175,
        "positives": 35,
        "negatives": 140,
        "confusion": {
          "tp": 25,
          "fp": 14,
          "tn": 126,
          "fn": 10
        },
        "recall": 0.7142857142857143,
        "false_alert_rate": 0.1,
        "precision": 0.6410256410256411,
        "threshold_accuracy": 0.8628571428571429
      }
    },
    "paired_threshold_intervals": {
      "groups": 35,
      "samples": 2000,
      "seed": 20260927,
      "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
      "metrics": {
        "recall": {
          "parent": {
            "value": 0.4,
            "ci95": [
              0.22857142857142856,
              0.5714285714285714
            ],
            "valid_resamples": 2000
          },
          "candidate": {
            "value": 0.7142857142857143,
            "ci95": [
              0.5714285714285714,
              0.8571428571428571
            ],
            "valid_resamples": 2000
          },
          "candidate_minus_parent": {
            "value": 0.3142857142857143,
            "ci95": [
              0.17142857142857143,
              0.4857142857142857
            ],
            "valid_resamples": 2000
          }
        },
        "false_alert_rate": {
          "parent": {
            "value": 0.08571428571428572,
            "ci95": [
              0.05,
              0.12857142857142856
            ],
            "valid_resamples": 2000
          },
          "candidate": {
            "value": 0.1,
            "ci95": [
              0.06428571428571428,
              0.14285714285714285
            ],
            "valid_resamples": 2000
          },
          "candidate_minus_parent": {
            "value": 0.01428571428571429,
            "ci95": [
              -0.014285714285714277,
              0.04285714285714286
            ],
            "valid_resamples": 2000
          }
        },
        "precision": {
          "parent": {
            "value": 0.5384615384615384,
            "ci95": [
              0.3665909090909091,
              0.70979262672811
            ],
            "valid_resamples": 2000
          },
          "candidate": {
            "value": 0.6410256410256411,
            "ci95": [
              0.55,
              0.7435897435897436
            ],
            "valid_resamples": 2000
          },
          "candidate_minus_parent": {
            "value": 0.10256410256410264,
            "ci95": [
              0.005701872682569551,
              0.21591595300834426
            ],
            "valid_resamples": 2000
          }
        },
        "threshold_accuracy": {
          "parent": {
            "value": 0.8114285714285714,
            "ci95": [
              0.7655714285714286,
              0.8628571428571429
            ],
            "valid_resamples": 2000
          },
          "candidate": {
            "value": 0.8628571428571429,
            "ci95": [
              0.8228571428571428,
              0.9028571428571428
            ],
            "valid_resamples": 2000
          },
          "candidate_minus_parent": {
            "value": 0.05142857142857149,
            "ci95": [
              0.02285714285714291,
              0.08000000000000007
            ],
            "valid_resamples": 2000
          }
        }
      }
    }
  },
  "by_operation": {
    "plain_text_display": {
      "always_permit_baseline": {
        "n": 35,
        "positives": 20,
        "negatives": 15,
        "confusion": {
          "tp": 0,
          "fp": 0,
          "tn": 15,
          "fn": 20
        },
        "recall": 0.0,
        "false_alert_rate": 0.0,
        "precision": null,
        "threshold_accuracy": 0.42857142857142855
      },
      "models": {
        "parent": {
          "questions": 35,
          "groups": 35,
          "raw": {
            "n": 35,
            "nll": 0.44020879235141186,
            "acc": 0.7428571428571429,
            "ece": 0.1010721055158729,
            "brier": 0.2970615598224881,
            "mean_conf": 0.7840390587691357,
            "confident_error_rate": 0.0,
            "coverage_at_0_9": 0.2571428571428571,
            "accuracy_at_0_9": 1.0,
            "coverage_at_5pct_error": 0.37142857142857144,
            "coverage_at_1pct_error": 0.37142857142857144,
            "aurc": 0.08938203390842284,
            "error_rate_at_0_9": 0.0,
            "confidence_bias": 0.04118191591199283,
            "top_bins": {
              "0.9": {
                "n": 9,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.95": {
                "n": 6,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 0,
                "errors": 0,
                "error_rate": null
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 0.9444444444444444,
                "confidence_cutoff": 0.8020615218849417
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.7857142857142857,
                "confidence_cutoff": 0.6323033624549943
              }
            }
          },
          "calibrated": {
            "n": 35,
            "nll": 0.4703692241803648,
            "acc": 0.7428571428571429,
            "ece": 0.1062363699460275,
            "brier": 0.3059826176210274,
            "mean_conf": 0.7080594224133254,
            "confident_error_rate": 0.0,
            "coverage_at_0_9": 0.05714285714285714,
            "accuracy_at_0_9": 1.0,
            "coverage_at_5pct_error": 0.37142857142857144,
            "coverage_at_1pct_error": 0.37142857142857144,
            "aurc": 0.08938203390842284,
            "error_rate_at_0_9": 0.0,
            "confidence_bias": -0.03479772044381746,
            "top_bins": {
              "0.9": {
                "n": 2,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.95": {
                "n": 0,
                "errors": 0,
                "error_rate": null
              },
              "0.99": {
                "n": 0,
                "errors": 0,
                "error_rate": null
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 0.9444444444444444,
                "confidence_cutoff": 0.702937726558054
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.7857142857142857,
                "confidence_cutoff": 0.5826614347130625
              }
            }
          },
          "argmax_classes": {
            "requires_approval": {
              "n": 20,
              "correct": 14,
              "recall": 0.7
            },
            "permitted": {
              "n": 15,
              "correct": 12,
              "recall": 0.8
            }
          },
          "threshold": 0.8875371629397495,
          "n": 35,
          "positives": 20,
          "negatives": 15,
          "confusion": {
            "tp": 1,
            "fp": 0,
            "tn": 15,
            "fn": 19
          },
          "recall": 0.05,
          "false_alert_rate": 0.0,
          "precision": 1.0,
          "threshold_accuracy": 0.45714285714285713
        },
        "candidate": {
          "questions": 35,
          "groups": 35,
          "raw": {
            "n": 35,
            "nll": 0.2308958263556794,
            "acc": 0.8857142857142857,
            "ece": 0.05276786009948832,
            "brier": 0.1387650722964353,
            "mean_conf": 0.9185510684713508,
            "confident_error_rate": 0.02857142857142857,
            "coverage_at_0_9": 0.7714285714285715,
            "accuracy_at_0_9": 0.9629629629629629,
            "coverage_at_5pct_error": 0.8857142857142857,
            "coverage_at_1pct_error": 0.6285714285714286,
            "aurc": 0.018915971835104774,
            "error_rate_at_0_9": 0.037037037037037035,
            "confidence_bias": 0.03283678275706514,
            "top_bins": {
              "0.9": {
                "n": 27,
                "errors": 1,
                "error_rate": 0.037037037037037035
              },
              "0.95": {
                "n": 19,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 6,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9562865363749455
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.9642857142857143,
                "confidence_cutoff": 0.8891588391985842
              }
            }
          },
          "calibrated": {
            "n": 35,
            "nll": 0.2642131240699265,
            "acc": 0.8857142857142857,
            "ece": 0.08058920252061248,
            "brier": 0.1465567737525219,
            "mean_conf": 0.8412751407359753,
            "confident_error_rate": 0.0,
            "coverage_at_0_9": 0.4,
            "accuracy_at_0_9": 1.0,
            "coverage_at_5pct_error": 0.8857142857142857,
            "coverage_at_1pct_error": 0.6285714285714286,
            "aurc": 0.018915971835104774,
            "error_rate_at_0_9": 0.0,
            "confidence_bias": -0.04443914497831036,
            "top_bins": {
              "0.9": {
                "n": 14,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.95": {
                "n": 4,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 0,
                "errors": 0,
                "error_rate": null
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 1.0,
                "confidence_cutoff": 0.8623076992723852
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.9642857142857143,
                "confidence_cutoff": 0.7752279376903202
              }
            }
          },
          "argmax_classes": {
            "requires_approval": {
              "n": 20,
              "correct": 20,
              "recall": 1.0
            },
            "permitted": {
              "n": 15,
              "correct": 11,
              "recall": 0.7333333333333333
            }
          },
          "threshold": 0.8449815374033091,
          "n": 35,
          "positives": 20,
          "negatives": 15,
          "confusion": {
            "tp": 12,
            "fp": 0,
            "tn": 15,
            "fn": 8
          },
          "recall": 0.6,
          "false_alert_rate": 0.0,
          "precision": 1.0,
          "threshold_accuracy": 0.7714285714285715
        }
      },
      "paired_threshold_intervals": {
        "groups": 35,
        "samples": 2000,
        "seed": 20260927,
        "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
        "metrics": {
          "recall": {
            "parent": {
              "value": 0.05,
              "ci95": [
                0.0,
                0.16666666666666666
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.6,
              "ci95": [
                0.38095238095238093,
                0.8095238095238095
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.5499999999999999,
              "ci95": [
                0.3333333333333333,
                0.7728535353535347
              ],
              "valid_resamples": 2000
            }
          },
          "false_alert_rate": {
            "parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 2000
            }
          },
          "precision": {
            "parent": {
              "value": 1.0,
              "ci95": [
                1.0,
                1.0
              ],
              "valid_resamples": 1232
            },
            "candidate": {
              "value": 1.0,
              "ci95": [
                1.0,
                1.0
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 1232
            }
          },
          "threshold_accuracy": {
            "parent": {
              "value": 0.45714285714285713,
              "ci95": [
                0.2857142857142857,
                0.6285714285714286
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.7714285714285715,
              "ci95": [
                0.6285714285714286,
                0.8857142857142857
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.31428571428571433,
              "ci95": [
                0.17142857142857149,
                0.48571428571428565
              ],
              "valid_resamples": 2000
            }
          }
        }
      }
    },
    "html_parse": {
      "always_permit_baseline": {
        "n": 35,
        "positives": 14,
        "negatives": 21,
        "confusion": {
          "tp": 0,
          "fp": 0,
          "tn": 21,
          "fn": 14
        },
        "recall": 0.0,
        "false_alert_rate": 0.0,
        "precision": null,
        "threshold_accuracy": 0.6
      },
      "models": {
        "parent": {
          "questions": 35,
          "groups": 35,
          "raw": {
            "n": 35,
            "nll": 1.9047291463003604,
            "acc": 0.42857142857142855,
            "ece": 0.5258122169538513,
            "brier": 1.0143214124687336,
            "mean_conf": 0.95438364552528,
            "confident_error_rate": 0.4857142857142857,
            "coverage_at_0_9": 0.9142857142857143,
            "accuracy_at_0_9": 0.46875,
            "coverage_at_5pct_error": 0.14285714285714285,
            "coverage_at_1pct_error": 0.14285714285714285,
            "aurc": 0.3679267902406343,
            "error_rate_at_0_9": 0.53125,
            "confidence_bias": 0.5258122169538515,
            "top_bins": {
              "0.9": {
                "n": 32,
                "errors": 17,
                "error_rate": 0.53125
              },
              "0.95": {
                "n": 27,
                "errors": 14,
                "error_rate": 0.5185185185185185
              },
              "0.99": {
                "n": 5,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 0.5555555555555556,
                "confidence_cutoff": 0.9801112406947217
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.4642857142857143,
                "confidence_cutoff": 0.9495538881263412
              }
            }
          },
          "calibrated": {
            "n": 35,
            "nll": 1.2659489508436006,
            "acc": 0.42857142857142855,
            "ece": 0.4588216874901758,
            "brier": 0.8687995501112846,
            "mean_conf": 0.8873931160616043,
            "confident_error_rate": 0.2571428571428571,
            "coverage_at_0_9": 0.6,
            "accuracy_at_0_9": 0.5714285714285714,
            "coverage_at_5pct_error": 0.14285714285714285,
            "coverage_at_1pct_error": 0.14285714285714285,
            "aurc": 0.3679267902406343,
            "error_rate_at_0_9": 0.42857142857142855,
            "confidence_bias": 0.45882168749017577,
            "top_bins": {
              "0.9": {
                "n": 21,
                "errors": 9,
                "error_rate": 0.42857142857142855
              },
              "0.95": {
                "n": 3,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 0,
                "errors": 0,
                "error_rate": null
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 0.5555555555555556,
                "confidence_cutoff": 0.9167662540005418
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.4642857142857143,
                "confidence_cutoff": 0.8589695610685519
              }
            }
          },
          "argmax_classes": {
            "requires_approval": {
              "n": 14,
              "correct": 14,
              "recall": 1.0
            },
            "permitted": {
              "n": 21,
              "correct": 1,
              "recall": 0.047619047619047616
            }
          },
          "threshold": 0.8875371629397495,
          "n": 35,
          "positives": 14,
          "negatives": 21,
          "confusion": {
            "tp": 12,
            "fp": 9,
            "tn": 12,
            "fn": 2
          },
          "recall": 0.8571428571428571,
          "false_alert_rate": 0.42857142857142855,
          "precision": 0.5714285714285714,
          "threshold_accuracy": 0.6857142857142857
        },
        "candidate": {
          "questions": 35,
          "groups": 35,
          "raw": {
            "n": 35,
            "nll": 1.7814413567537535,
            "acc": 0.45714285714285713,
            "ece": 0.4870901667090771,
            "brier": 0.9601370731320206,
            "mean_conf": 0.9442330238519343,
            "confident_error_rate": 0.42857142857142855,
            "coverage_at_0_9": 0.8285714285714286,
            "accuracy_at_0_9": 0.4827586206896552,
            "coverage_at_5pct_error": 0.2,
            "coverage_at_1pct_error": 0.2,
            "aurc": 0.31511035965021955,
            "error_rate_at_0_9": 0.5172413793103449,
            "confidence_bias": 0.4870901667090772,
            "top_bins": {
              "0.9": {
                "n": 29,
                "errors": 15,
                "error_rate": 0.5172413793103449
              },
              "0.95": {
                "n": 25,
                "errors": 12,
                "error_rate": 0.48
              },
              "0.99": {
                "n": 8,
                "errors": 1,
                "error_rate": 0.125
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 0.6666666666666666,
                "confidence_cutoff": 0.9770944745872364
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.4642857142857143,
                "confidence_cutoff": 0.9312263284660277
              }
            }
          },
          "calibrated": {
            "n": 35,
            "nll": 1.1646094584375128,
            "acc": 0.45714285714285713,
            "ece": 0.4387100296237643,
            "brier": 0.8119992696195855,
            "mean_conf": 0.8734203345049698,
            "confident_error_rate": 0.2,
            "coverage_at_0_9": 0.5428571428571428,
            "accuracy_at_0_9": 0.631578947368421,
            "coverage_at_5pct_error": 0.2,
            "coverage_at_1pct_error": 0.2,
            "aurc": 0.31511035965021955,
            "error_rate_at_0_9": 0.3684210526315789,
            "confidence_bias": 0.4162774773621127,
            "top_bins": {
              "0.9": {
                "n": 19,
                "errors": 7,
                "error_rate": 0.3684210526315789
              },
              "0.95": {
                "n": 6,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 1,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 0.6666666666666666,
                "confidence_cutoff": 0.903057582586965
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.4642857142857143,
                "confidence_cutoff": 0.8248194570355916
              }
            }
          },
          "argmax_classes": {
            "requires_approval": {
              "n": 14,
              "correct": 14,
              "recall": 1.0
            },
            "permitted": {
              "n": 21,
              "correct": 2,
              "recall": 0.09523809523809523
            }
          },
          "threshold": 0.8449815374033091,
          "n": 35,
          "positives": 14,
          "negatives": 21,
          "confusion": {
            "tp": 12,
            "fp": 12,
            "tn": 9,
            "fn": 2
          },
          "recall": 0.8571428571428571,
          "false_alert_rate": 0.5714285714285714,
          "precision": 0.5,
          "threshold_accuracy": 0.6
        }
      },
      "paired_threshold_intervals": {
        "groups": 35,
        "samples": 2000,
        "seed": 20260927,
        "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
        "metrics": {
          "recall": {
            "parent": {
              "value": 0.8571428571428571,
              "ci95": [
                0.6666666666666666,
                1.0
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.8571428571428571,
              "ci95": [
                0.6666666666666666,
                1.0
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 2000
            }
          },
          "false_alert_rate": {
            "parent": {
              "value": 0.42857142857142855,
              "ci95": [
                0.21739130434782608,
                0.6521739130434783
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.5714285714285714,
              "ci95": [
                0.3598235294117647,
                0.7858082706766912
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.14285714285714285,
              "ci95": [
                0.0,
                0.3333333333333333
              ],
              "valid_resamples": 2000
            }
          },
          "precision": {
            "parent": {
              "value": 0.5714285714285714,
              "ci95": [
                0.35,
                0.782608695652174
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.5,
              "ci95": [
                0.2916666666666667,
                0.7058823529411765
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": -0.0714285714285714,
              "ci95": [
                -0.1768421052631578,
                0.0
              ],
              "valid_resamples": 2000
            }
          },
          "threshold_accuracy": {
            "parent": {
              "value": 0.6857142857142857,
              "ci95": [
                0.5142857142857142,
                0.8285714285714286
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.6,
              "ci95": [
                0.42857142857142855,
                0.7714285714285715
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": -0.08571428571428574,
              "ci95": [
                -0.20000000000000007,
                0.0
              ],
              "valid_resamples": 2000
            }
          }
        }
      }
    },
    "onward_request_body": {
      "always_permit_baseline": {
        "n": 35,
        "positives": 1,
        "negatives": 34,
        "confusion": {
          "tp": 0,
          "fp": 0,
          "tn": 34,
          "fn": 1
        },
        "recall": 0.0,
        "false_alert_rate": 0.0,
        "precision": null,
        "threshold_accuracy": 0.9714285714285714
      },
      "models": {
        "parent": {
          "questions": 35,
          "groups": 35,
          "raw": {
            "n": 35,
            "nll": 0.13064358196820833,
            "acc": 0.9714285714285714,
            "ece": 0.082650843776636,
            "brier": 0.062386596130186385,
            "mean_conf": 0.8959570370446475,
            "confident_error_rate": 0.0,
            "coverage_at_0_9": 0.6857142857142857,
            "accuracy_at_0_9": 1.0,
            "coverage_at_5pct_error": 1.0,
            "coverage_at_1pct_error": 0.9428571428571428,
            "aurc": 0.0016566626650660265,
            "error_rate_at_0_9": 0.0,
            "confidence_bias": -0.07547153438392395,
            "top_bins": {
              "0.9": {
                "n": 24,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.95": {
                "n": 16,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 7,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9326748524721808
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 1.0,
                "confidence_cutoff": 0.8734922079816011
              }
            }
          },
          "calibrated": {
            "n": 35,
            "nll": 0.20940801007318904,
            "acc": 0.9714285714285714,
            "ece": 0.14463259695441177,
            "brier": 0.098760907973673,
            "mean_conf": 0.8267959744741596,
            "confident_error_rate": 0.0,
            "coverage_at_0_9": 0.37142857142857144,
            "accuracy_at_0_9": 1.0,
            "coverage_at_5pct_error": 1.0,
            "coverage_at_1pct_error": 0.9428571428571428,
            "aurc": 0.0016566626650660265,
            "error_rate_at_0_9": 0.0,
            "confidence_bias": -0.14463259695441177,
            "top_bins": {
              "0.9": {
                "n": 13,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.95": {
                "n": 7,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 1,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 1.0,
                "confidence_cutoff": 0.8345254161812303
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 1.0,
                "confidence_cutoff": 0.7666347623294629
              }
            }
          },
          "argmax_classes": {
            "requires_approval": {
              "n": 1,
              "correct": 1,
              "recall": 1.0
            },
            "permitted": {
              "n": 34,
              "correct": 33,
              "recall": 0.9705882352941176
            }
          },
          "threshold": 0.8875371629397495,
          "n": 35,
          "positives": 1,
          "negatives": 34,
          "confusion": {
            "tp": 1,
            "fp": 0,
            "tn": 34,
            "fn": 0
          },
          "recall": 1.0,
          "false_alert_rate": 0.0,
          "precision": 1.0,
          "threshold_accuracy": 1.0
        },
        "candidate": {
          "questions": 35,
          "groups": 35,
          "raw": {
            "n": 35,
            "nll": 0.005026554467109394,
            "acc": 1.0,
            "ece": 0.004983623817000349,
            "brier": 0.0001685963296296438,
            "mean_conf": 0.9950163761829997,
            "confident_error_rate": 0.0,
            "coverage_at_0_9": 1.0,
            "accuracy_at_0_9": 1.0,
            "coverage_at_5pct_error": 1.0,
            "coverage_at_1pct_error": 1.0,
            "aurc": 0.0,
            "error_rate_at_0_9": 0.0,
            "confidence_bias": -0.004983623817000349,
            "top_bins": {
              "0.9": {
                "n": 35,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.95": {
                "n": 35,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 31,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9974550323398608
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9932663672241991
              }
            }
          },
          "calibrated": {
            "n": 35,
            "nll": 0.03405651326059574,
            "acc": 1.0,
            "ece": 0.033071042578584564,
            "brier": 0.0037297689099056244,
            "mean_conf": 0.9669289574214154,
            "confident_error_rate": 0.0,
            "coverage_at_0_9": 0.9428571428571428,
            "accuracy_at_0_9": 1.0,
            "coverage_at_5pct_error": 1.0,
            "coverage_at_1pct_error": 1.0,
            "aurc": 0.0,
            "error_rate_at_0_9": 0.0,
            "confidence_bias": -0.03307104257858462,
            "top_bins": {
              "0.9": {
                "n": 33,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.95": {
                "n": 28,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 6,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9720891179694704
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9511715473570618
              }
            }
          },
          "argmax_classes": {
            "requires_approval": {
              "n": 1,
              "correct": 1,
              "recall": 1.0
            },
            "permitted": {
              "n": 34,
              "correct": 34,
              "recall": 1.0
            }
          },
          "threshold": 0.8449815374033091,
          "n": 35,
          "positives": 1,
          "negatives": 34,
          "confusion": {
            "tp": 1,
            "fp": 0,
            "tn": 34,
            "fn": 0
          },
          "recall": 1.0,
          "false_alert_rate": 0.0,
          "precision": 1.0,
          "threshold_accuracy": 1.0
        }
      },
      "paired_threshold_intervals": {
        "groups": 35,
        "samples": 2000,
        "seed": 20260927,
        "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
        "metrics": {
          "recall": {
            "parent": {
              "value": 1.0,
              "ci95": [
                1.0,
                1.0
              ],
              "valid_resamples": 1272
            },
            "candidate": {
              "value": 1.0,
              "ci95": [
                1.0,
                1.0
              ],
              "valid_resamples": 1272
            },
            "candidate_minus_parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 1272
            }
          },
          "false_alert_rate": {
            "parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 2000
            }
          },
          "precision": {
            "parent": {
              "value": 1.0,
              "ci95": [
                1.0,
                1.0
              ],
              "valid_resamples": 1272
            },
            "candidate": {
              "value": 1.0,
              "ci95": [
                1.0,
                1.0
              ],
              "valid_resamples": 1272
            },
            "candidate_minus_parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 1272
            }
          },
          "threshold_accuracy": {
            "parent": {
              "value": 1.0,
              "ci95": [
                1.0,
                1.0
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 1.0,
              "ci95": [
                1.0,
                1.0
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 2000
            }
          }
        }
      }
    },
    "code_execution": {
      "always_permit_baseline": {
        "n": 35,
        "positives": 0,
        "negatives": 35,
        "confusion": {
          "tp": 0,
          "fp": 0,
          "tn": 35,
          "fn": 0
        },
        "recall": null,
        "false_alert_rate": 0.0,
        "precision": null,
        "threshold_accuracy": 1.0
      },
      "models": {
        "parent": {
          "questions": 35,
          "groups": 35,
          "raw": {
            "n": 35,
            "nll": 0.03495771061017535,
            "acc": 1.0,
            "ece": 0.03371089311351877,
            "brier": 0.004643703343002468,
            "mean_conf": 0.9662891068864811,
            "confident_error_rate": 0.0,
            "coverage_at_0_9": 0.9714285714285714,
            "accuracy_at_0_9": 1.0,
            "coverage_at_5pct_error": 1.0,
            "coverage_at_1pct_error": 1.0,
            "aurc": 0.0,
            "error_rate_at_0_9": 0.0,
            "confidence_bias": -0.033710893113518936,
            "top_bins": {
              "0.9": {
                "n": 34,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.95": {
                "n": 27,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 11,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9787669092307943
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9455319719721502
              }
            }
          },
          "calibrated": {
            "n": 35,
            "nll": 0.10667969626313498,
            "acc": 1.0,
            "ece": 0.09922627250314853,
            "brier": 0.02646181525533936,
            "mean_conf": 0.9007737274968517,
            "confident_error_rate": 0.0,
            "coverage_at_0_9": 0.5714285714285714,
            "accuracy_at_0_9": 1.0,
            "coverage_at_5pct_error": 1.0,
            "coverage_at_1pct_error": 1.0,
            "aurc": 0.0,
            "error_rate_at_0_9": 0.0,
            "confidence_bias": -0.0992262725031483,
            "top_bins": {
              "0.9": {
                "n": 20,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.95": {
                "n": 9,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 0,
                "errors": 0,
                "error_rate": null
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9135753218620322
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 1.0,
                "confidence_cutoff": 0.8528241762602075
              }
            }
          },
          "argmax_classes": {
            "requires_approval": {
              "n": 0,
              "correct": 0,
              "recall": null
            },
            "permitted": {
              "n": 35,
              "correct": 35,
              "recall": 1.0
            }
          },
          "threshold": 0.8875371629397495,
          "n": 35,
          "positives": 0,
          "negatives": 35,
          "confusion": {
            "tp": 0,
            "fp": 0,
            "tn": 35,
            "fn": 0
          },
          "recall": null,
          "false_alert_rate": 0.0,
          "precision": null,
          "threshold_accuracy": 1.0
        },
        "candidate": {
          "questions": 35,
          "groups": 35,
          "raw": {
            "n": 35,
            "nll": 0.003713040174857426,
            "acc": 1.0,
            "ece": 0.0036946893416571758,
            "brier": 7.269134859423068e-05,
            "mean_conf": 0.9963053106583428,
            "confident_error_rate": 0.0,
            "coverage_at_0_9": 1.0,
            "accuracy_at_0_9": 1.0,
            "coverage_at_5pct_error": 1.0,
            "coverage_at_1pct_error": 1.0,
            "aurc": 0.0,
            "error_rate_at_0_9": 0.0,
            "confidence_bias": -0.0036946893416571758,
            "top_bins": {
              "0.9": {
                "n": 35,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.95": {
                "n": 35,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 31,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9983259960008511
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9949578958858085
              }
            }
          },
          "calibrated": {
            "n": 35,
            "nll": 0.029845462374630014,
            "acc": 1.0,
            "ece": 0.02916307563639764,
            "brier": 0.002620264819563586,
            "mean_conf": 0.9708369243636024,
            "confident_error_rate": 0.0,
            "coverage_at_0_9": 1.0,
            "accuracy_at_0_9": 1.0,
            "coverage_at_5pct_error": 1.0,
            "coverage_at_1pct_error": 1.0,
            "aurc": 0.0,
            "error_rate_at_0_9": 0.0,
            "confidence_bias": -0.02916307563639764,
            "top_bins": {
              "0.9": {
                "n": 35,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.95": {
                "n": 30,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 5,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9781193495148215
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 1.0,
                "confidence_cutoff": 0.9586084771058523
              }
            }
          },
          "argmax_classes": {
            "requires_approval": {
              "n": 0,
              "correct": 0,
              "recall": null
            },
            "permitted": {
              "n": 35,
              "correct": 35,
              "recall": 1.0
            }
          },
          "threshold": 0.8449815374033091,
          "n": 35,
          "positives": 0,
          "negatives": 35,
          "confusion": {
            "tp": 0,
            "fp": 0,
            "tn": 35,
            "fn": 0
          },
          "recall": null,
          "false_alert_rate": 0.0,
          "precision": null,
          "threshold_accuracy": 1.0
        }
      },
      "paired_threshold_intervals": {
        "groups": 35,
        "samples": 2000,
        "seed": 20260927,
        "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
        "metrics": {
          "recall": {
            "parent": {
              "value": null,
              "ci95": null,
              "valid_resamples": 0
            },
            "candidate": {
              "value": null,
              "ci95": null,
              "valid_resamples": 0
            },
            "candidate_minus_parent": {
              "value": null,
              "ci95": null,
              "valid_resamples": 0
            }
          },
          "false_alert_rate": {
            "parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 2000
            }
          },
          "precision": {
            "parent": {
              "value": null,
              "ci95": null,
              "valid_resamples": 0
            },
            "candidate": {
              "value": null,
              "ci95": null,
              "valid_resamples": 0
            },
            "candidate_minus_parent": {
              "value": null,
              "ci95": null,
              "valid_resamples": 0
            }
          },
          "threshold_accuracy": {
            "parent": {
              "value": 1.0,
              "ci95": [
                1.0,
                1.0
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 1.0,
              "ci95": [
                1.0,
                1.0
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 2000
            }
          }
        }
      }
    },
    "console_output": {
      "always_permit_baseline": {
        "n": 35,
        "positives": 0,
        "negatives": 35,
        "confusion": {
          "tp": 0,
          "fp": 0,
          "tn": 35,
          "fn": 0
        },
        "recall": null,
        "false_alert_rate": 0.0,
        "precision": null,
        "threshold_accuracy": 1.0
      },
      "models": {
        "parent": {
          "questions": 35,
          "groups": 35,
          "raw": {
            "n": 35,
            "nll": 1.4669736827879394,
            "acc": 0.4857142857142857,
            "ece": 0.44969603628529076,
            "brier": 0.8741636560106545,
            "mean_conf": 0.8196176344527856,
            "confident_error_rate": 0.2,
            "coverage_at_0_9": 0.4,
            "accuracy_at_0_9": 0.5,
            "coverage_at_5pct_error": 0.0,
            "coverage_at_1pct_error": 0.0,
            "aurc": 0.5978180747557157,
            "error_rate_at_0_9": 0.5,
            "confidence_bias": 0.33390334873849986,
            "top_bins": {
              "0.9": {
                "n": 14,
                "errors": 7,
                "error_rate": 0.5
              },
              "0.95": {
                "n": 8,
                "errors": 4,
                "error_rate": 0.5
              },
              "0.99": {
                "n": 3,
                "errors": 2,
                "error_rate": 0.6666666666666666
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 0.3888888888888889,
                "confidence_cutoff": 0.8730975086069251
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.39285714285714285,
                "confidence_cutoff": 0.6488618260771695
              }
            }
          },
          "calibrated": {
            "n": 35,
            "nll": 1.077831441012569,
            "acc": 0.4857142857142857,
            "ece": 0.4118859974171003,
            "brier": 0.7555266144919474,
            "mean_conf": 0.7474440931747798,
            "confident_error_rate": 0.08571428571428572,
            "coverage_at_0_9": 0.17142857142857143,
            "accuracy_at_0_9": 0.5,
            "coverage_at_5pct_error": 0.0,
            "coverage_at_1pct_error": 0.0,
            "aurc": 0.5978180747557157,
            "error_rate_at_0_9": 0.5,
            "confidence_bias": 0.26172980746049407,
            "top_bins": {
              "0.9": {
                "n": 6,
                "errors": 3,
                "error_rate": 0.5
              },
              "0.95": {
                "n": 2,
                "errors": 2,
                "error_rate": 1.0
              },
              "0.99": {
                "n": 0,
                "errors": 0,
                "error_rate": null
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 0.3888888888888889,
                "confidence_cutoff": 0.7662416927389056
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.39285714285714285,
                "confidence_cutoff": 0.5933872468656349
              }
            }
          },
          "argmax_classes": {
            "requires_approval": {
              "n": 0,
              "correct": 0,
              "recall": null
            },
            "permitted": {
              "n": 35,
              "correct": 17,
              "recall": 0.4857142857142857
            }
          },
          "threshold": 0.8875371629397495,
          "n": 35,
          "positives": 0,
          "negatives": 35,
          "confusion": {
            "tp": 0,
            "fp": 3,
            "tn": 32,
            "fn": 0
          },
          "recall": null,
          "false_alert_rate": 0.08571428571428572,
          "precision": 0.0,
          "threshold_accuracy": 0.9142857142857143
        },
        "candidate": {
          "questions": 35,
          "groups": 35,
          "raw": {
            "n": 35,
            "nll": 0.4711593105222168,
            "acc": 0.8571428571428571,
            "ece": 0.10032786865714749,
            "brier": 0.24921799740853026,
            "mean_conf": 0.9258384369166115,
            "confident_error_rate": 0.05714285714285714,
            "coverage_at_0_9": 0.7714285714285715,
            "accuracy_at_0_9": 0.9259259259259259,
            "coverage_at_5pct_error": 0.2571428571428571,
            "coverage_at_1pct_error": 0.2571428571428571,
            "aurc": 0.08024957297121005,
            "error_rate_at_0_9": 0.07407407407407407,
            "confidence_bias": 0.0686955797737544,
            "top_bins": {
              "0.9": {
                "n": 27,
                "errors": 2,
                "error_rate": 0.07407407407407407
              },
              "0.95": {
                "n": 26,
                "errors": 2,
                "error_rate": 0.07692307692307693
              },
              "0.99": {
                "n": 16,
                "errors": 2,
                "error_rate": 0.125
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 0.8888888888888888,
                "confidence_cutoff": 0.9891438506037644
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.9285714285714286,
                "confidence_cutoff": 0.8757747622322081
              }
            }
          },
          "calibrated": {
            "n": 35,
            "nll": 0.3789301936796312,
            "acc": 0.8571428571428571,
            "ece": 0.11430695728032647,
            "brier": 0.23203565927974715,
            "mean_conf": 0.8718695845973715,
            "confident_error_rate": 0.05714285714285714,
            "coverage_at_0_9": 0.6857142857142857,
            "accuracy_at_0_9": 0.9166666666666666,
            "coverage_at_5pct_error": 0.2571428571428571,
            "coverage_at_1pct_error": 0.2571428571428571,
            "aurc": 0.08024957297121005,
            "error_rate_at_0_9": 0.08333333333333333,
            "confidence_bias": 0.014726727454514421,
            "top_bins": {
              "0.9": {
                "n": 24,
                "errors": 2,
                "error_rate": 0.08333333333333333
              },
              "0.95": {
                "n": 9,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 2,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5142857142857142,
                "accuracy": 0.8888888888888888,
                "confidence_cutoff": 0.9360109691977325
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.9285714285714286,
                "confidence_cutoff": 0.7615630826924717
              }
            }
          },
          "argmax_classes": {
            "requires_approval": {
              "n": 0,
              "correct": 0,
              "recall": null
            },
            "permitted": {
              "n": 35,
              "correct": 30,
              "recall": 0.8571428571428571
            }
          },
          "threshold": 0.8449815374033091,
          "n": 35,
          "positives": 0,
          "negatives": 35,
          "confusion": {
            "tp": 0,
            "fp": 2,
            "tn": 33,
            "fn": 0
          },
          "recall": null,
          "false_alert_rate": 0.05714285714285714,
          "precision": 0.0,
          "threshold_accuracy": 0.9428571428571428
        }
      },
      "paired_threshold_intervals": {
        "groups": 35,
        "samples": 2000,
        "seed": 20260927,
        "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
        "metrics": {
          "recall": {
            "parent": {
              "value": null,
              "ci95": null,
              "valid_resamples": 0
            },
            "candidate": {
              "value": null,
              "ci95": null,
              "valid_resamples": 0
            },
            "candidate_minus_parent": {
              "value": null,
              "ci95": null,
              "valid_resamples": 0
            }
          },
          "false_alert_rate": {
            "parent": {
              "value": 0.08571428571428572,
              "ci95": [
                0.0,
                0.2
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.05714285714285714,
              "ci95": [
                0.0,
                0.14285714285714285
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": -0.028571428571428574,
              "ci95": [
                -0.08571428571428572,
                0.0
              ],
              "valid_resamples": 2000
            }
          },
          "precision": {
            "parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 1918
            },
            "candidate": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 1747
            },
            "candidate_minus_parent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "valid_resamples": 1747
            }
          },
          "threshold_accuracy": {
            "parent": {
              "value": 0.9142857142857143,
              "ci95": [
                0.8,
                1.0
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.9428571428571428,
              "ci95": [
                0.8571428571428571,
                1.0
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.02857142857142858,
              "ci95": [
                0.0,
                0.08571428571428574
              ],
              "valid_resamples": 2000
            }
          }
        }
      }
    }
  },
  "by_source": {
    "sev_swarmtraces_response_policy": {
      "always_permit_baseline": {
        "n": 175,
        "positives": 35,
        "negatives": 140,
        "confusion": {
          "tp": 0,
          "fp": 0,
          "tn": 140,
          "fn": 35
        },
        "recall": 0.0,
        "false_alert_rate": 0.0,
        "precision": null,
        "threshold_accuracy": 0.8
      },
      "models": {
        "parent": {
          "questions": 175,
          "groups": 35,
          "raw": {
            "n": 175,
            "nll": 0.795502582803619,
            "acc": 0.7257142857142858,
            "ece": 0.1747005023605082,
            "brier": 0.4505153855550131,
            "mean_conf": 0.8840572965356658,
            "confident_error_rate": 0.13714285714285715,
            "coverage_at_0_9": 0.6457142857142857,
            "accuracy_at_0_9": 0.7876106194690266,
            "coverage_at_5pct_error": 0.02857142857142857,
            "coverage_at_1pct_error": 0.02857142857142857,
            "aurc": 0.1992292401228163,
            "error_rate_at_0_9": 0.21238938053097345,
            "confidence_bias": 0.15834301082138003,
            "top_bins": {
              "0.9": {
                "n": 113,
                "errors": 24,
                "error_rate": 0.21238938053097345
              },
              "0.95": {
                "n": 84,
                "errors": 18,
                "error_rate": 0.21428571428571427
              },
              "0.99": {
                "n": 26,
                "errors": 2,
                "error_rate": 0.07692307692307693
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5028571428571429,
                "accuracy": 0.7613636363636364,
                "confidence_cutoff": 0.9457634807080788
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.75,
                "confidence_cutoff": 0.7938824081649869
              }
            }
          },
          "calibrated": {
            "n": 175,
            "nll": 0.6260474644745716,
            "acc": 0.7257142857142858,
            "ece": 0.10761147298493057,
            "brier": 0.4111063010906543,
            "mean_conf": 0.8140932667241441,
            "confident_error_rate": 0.06857142857142857,
            "coverage_at_0_9": 0.35428571428571426,
            "accuracy_at_0_9": 0.8064516129032258,
            "coverage_at_5pct_error": 0.02857142857142857,
            "coverage_at_1pct_error": 0.02857142857142857,
            "aurc": 0.1992292401228163,
            "error_rate_at_0_9": 0.1935483870967742,
            "confidence_bias": 0.0883789810098583,
            "top_bins": {
              "0.9": {
                "n": 62,
                "errors": 12,
                "error_rate": 0.1935483870967742
              },
              "0.95": {
                "n": 21,
                "errors": 2,
                "error_rate": 0.09523809523809523
              },
              "0.99": {
                "n": 1,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5028571428571429,
                "accuracy": 0.7613636363636364,
                "confidence_cutoff": 0.8531718492565429
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.75,
                "confidence_cutoff": 0.6963743924447515
              }
            }
          },
          "argmax_classes": {
            "requires_approval": {
              "n": 35,
              "correct": 29,
              "recall": 0.8285714285714286
            },
            "permitted": {
              "n": 140,
              "correct": 98,
              "recall": 0.7
            }
          },
          "threshold": 0.8875371629397495,
          "n": 175,
          "positives": 35,
          "negatives": 140,
          "confusion": {
            "tp": 14,
            "fp": 12,
            "tn": 128,
            "fn": 21
          },
          "recall": 0.4,
          "false_alert_rate": 0.08571428571428572,
          "precision": 0.5384615384615384,
          "threshold_accuracy": 0.8114285714285714
        },
        "candidate": {
          "questions": 175,
          "groups": 35,
          "raw": {
            "n": 175,
            "nll": 0.49844721765472333,
            "acc": 0.84,
            "ece": 0.11735638435435483,
            "brier": 0.269672286103042,
            "mean_conf": 0.9559888432162478,
            "confident_error_rate": 0.10285714285714286,
            "coverage_at_0_9": 0.8742857142857143,
            "accuracy_at_0_9": 0.8823529411764706,
            "coverage_at_5pct_error": 0.6114285714285714,
            "coverage_at_1pct_error": 0.44,
            "aurc": 0.044632779737602504,
            "error_rate_at_0_9": 0.11764705882352941,
            "confidence_bias": 0.1159888432162478,
            "top_bins": {
              "0.9": {
                "n": 153,
                "errors": 18,
                "error_rate": 0.11764705882352941
              },
              "0.95": {
                "n": 140,
                "errors": 14,
                "error_rate": 0.1
              },
              "0.99": {
                "n": 92,
                "errors": 3,
                "error_rate": 0.03260869565217391
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5028571428571429,
                "accuracy": 0.9772727272727273,
                "confidence_cutoff": 0.9904485398763483
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.9,
                "confidence_cutoff": 0.9516673073011176
              }
            }
          },
          "calibrated": {
            "n": 175,
            "nll": 0.37433095036445935,
            "acc": 0.84,
            "ece": 0.06636288221880407,
            "brier": 0.23938834727626476,
            "mean_conf": 0.9048661883246669,
            "confident_error_rate": 0.05142857142857143,
            "coverage_at_0_9": 0.7142857142857143,
            "accuracy_at_0_9": 0.928,
            "coverage_at_5pct_error": 0.6114285714285714,
            "coverage_at_1pct_error": 0.44,
            "aurc": 0.044632779737602504,
            "error_rate_at_0_9": 0.072,
            "confidence_bias": 0.06486618832466695,
            "top_bins": {
              "0.9": {
                "n": 125,
                "errors": 9,
                "error_rate": 0.072
              },
              "0.95": {
                "n": 77,
                "errors": 0,
                "error_rate": 0.0
              },
              "0.99": {
                "n": 14,
                "errors": 0,
                "error_rate": 0.0
              }
            },
            "selective": {
              "0.5": {
                "coverage": 0.5028571428571429,
                "accuracy": 0.9772727272727273,
                "confidence_cutoff": 0.9404661687170754
              },
              "0.8": {
                "coverage": 0.8,
                "accuracy": 0.9,
                "confidence_cutoff": 0.8547040294961724
              }
            }
          },
          "argmax_classes": {
            "requires_approval": {
              "n": 35,
              "correct": 35,
              "recall": 1.0
            },
            "permitted": {
              "n": 140,
              "correct": 112,
              "recall": 0.8
            }
          },
          "threshold": 0.8449815374033091,
          "n": 175,
          "positives": 35,
          "negatives": 140,
          "confusion": {
            "tp": 25,
            "fp": 14,
            "tn": 126,
            "fn": 10
          },
          "recall": 0.7142857142857143,
          "false_alert_rate": 0.1,
          "precision": 0.6410256410256411,
          "threshold_accuracy": 0.8628571428571429
        }
      },
      "paired_threshold_intervals": {
        "groups": 35,
        "samples": 2000,
        "seed": 20260927,
        "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed",
        "metrics": {
          "recall": {
            "parent": {
              "value": 0.4,
              "ci95": [
                0.22857142857142856,
                0.5714285714285714
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.7142857142857143,
              "ci95": [
                0.5714285714285714,
                0.8571428571428571
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.3142857142857143,
              "ci95": [
                0.17142857142857143,
                0.4857142857142857
              ],
              "valid_resamples": 2000
            }
          },
          "false_alert_rate": {
            "parent": {
              "value": 0.08571428571428572,
              "ci95": [
                0.05,
                0.12857142857142856
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.1,
              "ci95": [
                0.06428571428571428,
                0.14285714285714285
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.01428571428571429,
              "ci95": [
                -0.014285714285714277,
                0.04285714285714286
              ],
              "valid_resamples": 2000
            }
          },
          "precision": {
            "parent": {
              "value": 0.5384615384615384,
              "ci95": [
                0.3665909090909091,
                0.70979262672811
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.6410256410256411,
              "ci95": [
                0.55,
                0.7435897435897436
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.10256410256410264,
              "ci95": [
                0.005701872682569551,
                0.21591595300834426
              ],
              "valid_resamples": 2000
            }
          },
          "threshold_accuracy": {
            "parent": {
              "value": 0.8114285714285714,
              "ci95": [
                0.7655714285714286,
                0.8628571428571429
              ],
              "valid_resamples": 2000
            },
            "candidate": {
              "value": 0.8628571428571429,
              "ci95": [
                0.8228571428571428,
                0.9028571428571428
              ],
              "valid_resamples": 2000
            },
            "candidate_minus_parent": {
              "value": 0.05142857142857149,
              "ci95": [
                0.02285714285714291,
                0.08000000000000007
              ],
              "valid_resamples": 2000
            }
          }
        }
      }
    }
  },
  "retained_development_by_source": {
    "parent": {
      "excytin_file": {
        "questions": 80,
        "groups": 1,
        "raw": {
          "n": 80,
          "nll": 0.0001342531538525746,
          "acc": 1.0,
          "ece": 0.0001340575450615722,
          "brier": 7.773831743030022e-07,
          "mean_conf": 0.9998659424549384,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.0001340575450615722,
          "top_bins": {
            "0.9": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9999710759158646
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9999066849313115
            }
          }
        },
        "calibrated": {
          "n": 80,
          "nll": 0.0030639977354718724,
          "acc": 1.0,
          "ece": 0.0030486898704090715,
          "brier": 5.723205099455454e-05,
          "mean_conf": 0.9969513101295909,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.0030486898704090715,
          "top_bins": {
            "0.9": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 79,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9980559839388836
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9963993201639499
            }
          }
        }
      },
      "excytin_parent_join": {
        "questions": 78,
        "groups": 1,
        "raw": {
          "n": 78,
          "nll": 0.0019727127979591126,
          "acc": 1.0,
          "ece": 0.0019615703387183547,
          "brier": 4.023721192309001e-05,
          "mean_conf": 0.9980384296612816,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.0019615703387183547,
          "top_bins": {
            "0.9": {
              "n": 78,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 78,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 76,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9996318703080018
            },
            "0.8": {
              "coverage": 0.8076923076923077,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9974582021431176
            }
          }
        },
        "calibrated": {
          "n": 78,
          "nll": 0.01903091706838521,
          "acc": 1.0,
          "ece": 0.018568486423778917,
          "brier": 0.0013900388516520708,
          "mean_conf": 0.9814315135762209,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9871794871794872,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.018568486423779063,
          "top_bins": {
            "0.9": {
              "n": 77,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 69,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 46,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9916928553567559
            },
            "0.8": {
              "coverage": 0.8076923076923077,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9693893850709232
            }
          }
        }
      },
      "excytin_process": {
        "questions": 120,
        "groups": 1,
        "raw": {
          "n": 120,
          "nll": 0.0005180266626209783,
          "acc": 1.0,
          "ece": 0.0005174316239614507,
          "brier": 2.3119235778779676e-06,
          "mean_conf": 0.9994825683760385,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.0005174316239614507,
          "top_bins": {
            "0.9": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9999134057097643
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.99901689132107
            }
          }
        },
        "calibrated": {
          "n": 120,
          "nll": 0.007587462315793864,
          "acc": 1.0,
          "ece": 0.007523761830370401,
          "brier": 0.00023474191911792246,
          "mean_conf": 0.9924762381696296,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.007523761830370401,
          "top_bins": {
            "0.9": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 119,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 83,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9964206554980828
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9859276562719884
            }
          }
        }
      },
      "excytin_signin": {
        "questions": 120,
        "groups": 1,
        "raw": {
          "n": 120,
          "nll": 0.0013203461429938826,
          "acc": 1.0,
          "ece": 0.0012737143193113606,
          "brier": 0.00013106658103070147,
          "mean_conf": 0.9987262856806887,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9916666666666667,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.0012737143193113365,
          "top_bins": {
            "0.9": {
              "n": 119,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 119,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 119,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9998413534198065
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9996110203225037
            }
          }
        },
        "calibrated": {
          "n": 120,
          "nll": 0.009411479962491658,
          "acc": 1.0,
          "ece": 0.00903361007075822,
          "brier": 0.0010010974637190823,
          "mean_conf": 0.9909663899292418,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9916666666666667,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.009033610070758202,
          "top_bins": {
            "0.9": {
              "n": 119,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 119,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 105,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9950297195670218
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9914608088731087
            }
          }
        }
      },
      "sev_guide_detector": {
        "questions": 1064,
        "groups": 114,
        "raw": {
          "n": 1064,
          "nll": 0.8276776090882748,
          "acc": 0.599624060150376,
          "ece": 0.10621007339433168,
          "brier": 0.48019952548397726,
          "mean_conf": 0.7041447618951532,
          "confident_error_rate": 0.012218045112781954,
          "coverage_at_0_9": 0.32706766917293234,
          "accuracy_at_0_9": 0.9626436781609196,
          "coverage_at_5pct_error": 0.3524436090225564,
          "coverage_at_1pct_error": 0.0,
          "aurc": 0.19304798592127517,
          "error_rate_at_0_9": 0.03735632183908046,
          "confidence_bias": 0.10452070174477723,
          "top_bins": {
            "0.9": {
              "n": 348,
              "errors": 13,
              "error_rate": 0.03735632183908046
            },
            "0.95": {
              "n": 327,
              "errors": 13,
              "error_rate": 0.039755351681957186
            },
            "0.99": {
              "n": 319,
              "errors": 12,
              "error_rate": 0.03761755485893417
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5075187969924813,
              "accuracy": 0.8074074074074075,
              "confidence_cutoff": 0.6273336504002459
            },
            "0.8": {
              "coverage": 0.8016917293233082,
              "accuracy": 0.6518171160609613,
              "confidence_cutoff": 0.48816800774395525
            }
          }
        },
        "calibrated": {
          "n": 1064,
          "nll": 0.785071927880167,
          "acc": 0.599624060150376,
          "ece": 0.05266334727942813,
          "brier": 0.46075730837789786,
          "mean_conf": 0.6304966279496853,
          "confident_error_rate": 0.011278195488721804,
          "coverage_at_0_9": 0.3007518796992481,
          "accuracy_at_0_9": 0.9625,
          "coverage_at_5pct_error": 0.3524436090225564,
          "coverage_at_1pct_error": 0.0,
          "aurc": 0.19295849854549843,
          "error_rate_at_0_9": 0.0375,
          "confidence_bias": 0.03087256779930936,
          "top_bins": {
            "0.9": {
              "n": 320,
              "errors": 12,
              "error_rate": 0.0375
            },
            "0.95": {
              "n": 213,
              "errors": 11,
              "error_rate": 0.051643192488262914
            },
            "0.99": {
              "n": 0,
              "errors": 0,
              "error_rate": null
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5075187969924813,
              "accuracy": 0.8074074074074075,
              "confidence_cutoff": 0.5144221593760727
            },
            "0.8": {
              "coverage": 0.8016917293233082,
              "accuracy": 0.6553341148886284,
              "confidence_cutoff": 0.4322968853489006
            }
          }
        }
      },
      "sev_guide_triage": {
        "questions": 231,
        "groups": 23,
        "raw": {
          "n": 231,
          "nll": 0.8563265185203435,
          "acc": 0.5627705627705628,
          "ece": 0.1895007716411903,
          "brier": 0.5255142828500304,
          "mean_conf": 0.7444269666893102,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.35064935064935066,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 0.36363636363636365,
          "coverage_at_1pct_error": 0.36363636363636365,
          "aurc": 0.22468166386345326,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": 0.18165640391874738,
          "top_bins": {
            "0.9": {
              "n": 81,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 79,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5021645021645021,
              "accuracy": 0.75,
              "confidence_cutoff": 0.7894512944300596
            },
            "0.8": {
              "coverage": 0.8095238095238095,
              "accuracy": 0.5775401069518716,
              "confidence_cutoff": 0.49009219177135965
            }
          }
        },
        "calibrated": {
          "n": 231,
          "nll": 0.7793526965750157,
          "acc": 0.5627705627705628,
          "ece": 0.1664752401619534,
          "brier": 0.4718856823661165,
          "mean_conf": 0.6653812407670776,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.3463203463203463,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 0.36363636363636365,
          "coverage_at_1pct_error": 0.36363636363636365,
          "aurc": 0.22610167628363687,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": 0.1026106779965148,
          "top_bins": {
            "0.9": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 14,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 0,
              "errors": 0,
              "error_rate": null
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5021645021645021,
              "accuracy": 0.75,
              "confidence_cutoff": 0.6343620681380909
            },
            "0.8": {
              "coverage": 0.8008658008658008,
              "accuracy": 0.5675675675675675,
              "confidence_cutoff": 0.4381830283799813
            }
          }
        }
      },
      "sev_swarmtraces_static": {
        "questions": 354,
        "groups": 176,
        "raw": {
          "n": 354,
          "nll": 0.06671770193368179,
          "acc": 0.9745762711864406,
          "ece": 0.008417033573491902,
          "brier": 0.03589778460324393,
          "mean_conf": 0.9740214622331307,
          "confident_error_rate": 0.002824858757062147,
          "coverage_at_0_9": 0.9322033898305084,
          "accuracy_at_0_9": 0.996969696969697,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 0.9745762711864406,
          "aurc": 0.0009317569638462815,
          "error_rate_at_0_9": 0.0030303030303030303,
          "confidence_bias": -0.0005548089533099487,
          "top_bins": {
            "0.9": {
              "n": 330,
              "errors": 1,
              "error_rate": 0.0030303030303030303
            },
            "0.95": {
              "n": 317,
              "errors": 1,
              "error_rate": 0.0031545741324921135
            },
            "0.99": {
              "n": 292,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9998280676315141
            },
            "0.8": {
              "coverage": 0.8022598870056498,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9930714713128618
            }
          }
        },
        "calibrated": {
          "n": 354,
          "nll": 0.08603305952328294,
          "acc": 0.9745762711864406,
          "ece": 0.031999170395289905,
          "brier": 0.03970483443698841,
          "mean_conf": 0.9467552592594386,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.8615819209039548,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 0.9745762711864406,
          "aurc": 0.0009516834613723162,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.027821011927002037,
          "top_bins": {
            "0.9": {
              "n": 305,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 277,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 194,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9929179700515357
            },
            "0.8": {
              "coverage": 0.8022598870056498,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9423198026275108
            }
          }
        }
      },
      "sev_sysmon_native_file": {
        "questions": 62,
        "groups": 4,
        "raw": {
          "n": 62,
          "nll": 0.010792211612586852,
          "acc": 1.0,
          "ece": 0.009954006393107204,
          "brier": 0.002287040487090685,
          "mean_conf": 0.9900459936068928,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9838709677419355,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.009954006393107173,
          "top_bins": {
            "0.9": {
              "n": 61,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 59,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 54,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9986117420701618
            },
            "0.8": {
              "coverage": 0.8064516129032258,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9936233886125292
            }
          }
        },
        "calibrated": {
          "n": 62,
          "nll": 0.04080521144437412,
          "acc": 1.0,
          "ece": 0.03730996348948351,
          "brier": 0.008675036111100168,
          "mean_conf": 0.9626900365105165,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9516129032258065,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.037309963489483455,
          "top_bins": {
            "0.9": {
              "n": 59,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 48,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 23,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9803392906572079
            },
            "0.8": {
              "coverage": 0.8064516129032258,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9477741924481837
            }
          }
        }
      },
      "sev_sysmon_native_join": {
        "questions": 120,
        "groups": 3,
        "raw": {
          "n": 120,
          "nll": 0.0008810538663341702,
          "acc": 1.0,
          "ece": 0.0008800625080441771,
          "brier": 3.5216739059086607e-06,
          "mean_conf": 0.9991199374919558,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.0008800625080441771,
          "top_bins": {
            "0.9": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9994848628990232
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.998641259167216
            }
          }
        },
        "calibrated": {
          "n": 120,
          "nll": 0.013824573239669817,
          "acc": 1.0,
          "ece": 0.013685808798396426,
          "brier": 0.00044603592410257366,
          "mean_conf": 0.9863141912016036,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.013685808798396426,
          "top_bins": {
            "0.9": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 118,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 55,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9888123032356468
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.980357236597309
            }
          }
        }
      },
      "sev_sysmon_native_other_effect": {
        "questions": 88,
        "groups": 4,
        "raw": {
          "n": 88,
          "nll": 0.007295006715728625,
          "acc": 1.0,
          "ece": 0.006888792886758961,
          "brier": 0.001385599246512425,
          "mean_conf": 0.9931112071132411,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9886363636363636,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.006888792886758921,
          "top_bins": {
            "0.9": {
              "n": 87,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 86,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 79,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9993634031772723
            },
            "0.8": {
              "coverage": 0.8068181818181818,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9952397949435474
            }
          }
        },
        "calibrated": {
          "n": 88,
          "nll": 0.03669008514897713,
          "acc": 1.0,
          "ece": 0.03458520458981973,
          "brier": 0.006053607090618847,
          "mean_conf": 0.9654147954101805,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9431818181818182,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.03458520458981951,
          "top_bins": {
            "0.9": {
              "n": 83,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 67,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 41,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9865664502206583
            },
            "0.8": {
              "coverage": 0.8068181818181818,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9454838788991825
            }
          }
        }
      },
      "sev_sysmon_native_process": {
        "questions": 74,
        "groups": 4,
        "raw": {
          "n": 74,
          "nll": 0.005360095133334598,
          "acc": 1.0,
          "ece": 0.005183972707978367,
          "brier": 0.0006452896028548641,
          "mean_conf": 0.9948160272920217,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9864864864864865,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.005183972707978324,
          "top_bins": {
            "0.9": {
              "n": 73,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 72,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 67,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9998754065974452
            },
            "0.8": {
              "coverage": 0.8108108108108109,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9964080071736937
            }
          }
        },
        "calibrated": {
          "n": 74,
          "nll": 0.02430026482825407,
          "acc": 1.0,
          "ece": 0.023069255552173623,
          "brier": 0.004038757822471681,
          "mean_conf": 0.9769307444478267,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9324324324324325,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.023069255552173318,
          "top_bins": {
            "0.9": {
              "n": 69,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 65,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 43,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9959686665627235
            },
            "0.8": {
              "coverage": 0.8108108108108109,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9671912008530428
            }
          }
        }
      },
      "sev_sysmon_native_registry": {
        "questions": 66,
        "groups": 4,
        "raw": {
          "n": 66,
          "nll": 0.02506239179799794,
          "acc": 0.9848484848484849,
          "ece": 0.005781042813278618,
          "brier": 0.017230444731313608,
          "mean_conf": 0.9868495288218273,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9696969696969697,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 0.9696969696969697,
          "aurc": 0.00046266864448682633,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": 0.0020010439733424423,
          "top_bins": {
            "0.9": {
              "n": 64,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 64,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 61,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9996958591947809
            },
            "0.8": {
              "coverage": 0.803030303030303,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9967594448763499
            }
          }
        },
        "calibrated": {
          "n": 66,
          "nll": 0.03908316195438624,
          "acc": 0.9848484848484849,
          "ece": 0.018983776220710902,
          "brier": 0.01748193378444616,
          "mean_conf": 0.9706927611865057,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9545454545454546,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 0.9696969696969697,
          "aurc": 0.00046266864448682633,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.014155723661979147,
          "top_bins": {
            "0.9": {
              "n": 63,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 58,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 36,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9919306919122572
            },
            "0.8": {
              "coverage": 0.803030303030303,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9688350009585398
            }
          }
        }
      }
    },
    "candidate": {
      "excytin_file": {
        "questions": 80,
        "groups": 1,
        "raw": {
          "n": 80,
          "nll": 8.416562133899781e-05,
          "acc": 1.0,
          "ece": 8.407504508700647e-05,
          "brier": 3.608306194877327e-07,
          "mean_conf": 0.999915924954913,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -8.407504508700647e-05,
          "top_bins": {
            "0.9": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9999870465451609
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9999445146924221
            }
          }
        },
        "calibrated": {
          "n": 80,
          "nll": 0.0025800477441721577,
          "acc": 1.0,
          "ece": 0.0025682388226171415,
          "brier": 4.4624375778219304e-05,
          "mean_conf": 0.9974317611773829,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.0025682388226171415,
          "top_bins": {
            "0.9": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 79,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9984996790217291
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9968816753727368
            }
          }
        }
      },
      "excytin_parent_join": {
        "questions": 78,
        "groups": 1,
        "raw": {
          "n": 78,
          "nll": 0.0010931532029286512,
          "acc": 1.0,
          "ece": 0.0010890559164937752,
          "brier": 1.524968968159761e-05,
          "mean_conf": 0.9989109440835062,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.0010890559164937752,
          "top_bins": {
            "0.9": {
              "n": 78,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 78,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 77,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9997808486535813
            },
            "0.8": {
              "coverage": 0.8076923076923077,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9985521003494622
            }
          }
        },
        "calibrated": {
          "n": 78,
          "nll": 0.014932533018017379,
          "acc": 1.0,
          "ece": 0.014655484687648102,
          "brier": 0.0008621856358382366,
          "mean_conf": 0.9853445153123519,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.014655484687648102,
          "top_bins": {
            "0.9": {
              "n": 78,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 73,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 49,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9924833351572758
            },
            "0.8": {
              "coverage": 0.8076923076923077,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9749372136312703
            }
          }
        }
      },
      "excytin_process": {
        "questions": 120,
        "groups": 1,
        "raw": {
          "n": 120,
          "nll": 0.00038924968205708544,
          "acc": 1.0,
          "ece": 0.00038892368944332123,
          "brier": 1.2813112890567744e-06,
          "mean_conf": 0.9996110763105567,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.00038892368944332123,
          "top_bins": {
            "0.9": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9999547557532524
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9992749850755753
            }
          }
        },
        "calibrated": {
          "n": 120,
          "nll": 0.007136704365819422,
          "acc": 1.0,
          "ece": 0.007078397430680172,
          "brier": 0.00021856930435733632,
          "mean_conf": 0.9929216025693198,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.007078397430680172,
          "top_bins": {
            "0.9": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 119,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 84,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9971072222116408
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9865275469490526
            }
          }
        }
      },
      "excytin_signin": {
        "questions": 120,
        "groups": 1,
        "raw": {
          "n": 120,
          "nll": 0.0009466221977245799,
          "acc": 1.0,
          "ece": 0.0009222842581878554,
          "brier": 7.013154556645113e-05,
          "mean_conf": 0.9990777157418121,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.0009222842581878554,
          "top_bins": {
            "0.9": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 119,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 119,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9999016374480455
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9997216681533668
            }
          }
        },
        "calibrated": {
          "n": 120,
          "nll": 0.00847100086283192,
          "acc": 1.0,
          "ece": 0.008170507143524745,
          "brier": 0.0008193865392318428,
          "mean_conf": 0.9918294928564755,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9916666666666667,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.008170507143524519,
          "top_bins": {
            "0.9": {
              "n": 119,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 119,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 105,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9955464217668417
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9921417222813483
            }
          }
        }
      },
      "sev_guide_detector": {
        "questions": 1064,
        "groups": 114,
        "raw": {
          "n": 1064,
          "nll": 0.8605571101399087,
          "acc": 0.6005639097744361,
          "ece": 0.14734385206430384,
          "brier": 0.5008109312564021,
          "mean_conf": 0.7382136284198534,
          "confident_error_rate": 0.017857142857142856,
          "coverage_at_0_9": 0.33270676691729323,
          "accuracy_at_0_9": 0.9463276836158192,
          "coverage_at_5pct_error": 0.3082706766917293,
          "coverage_at_1pct_error": 0.0,
          "aurc": 0.19350209200158913,
          "error_rate_at_0_9": 0.05367231638418079,
          "confidence_bias": 0.1376497186454173,
          "top_bins": {
            "0.9": {
              "n": 354,
              "errors": 19,
              "error_rate": 0.05367231638418079
            },
            "0.95": {
              "n": 327,
              "errors": 13,
              "error_rate": 0.039755351681957186
            },
            "0.99": {
              "n": 319,
              "errors": 12,
              "error_rate": 0.03761755485893417
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5028195488721805,
              "accuracy": 0.8205607476635514,
              "confidence_cutoff": 0.7059991149306561
            },
            "0.8": {
              "coverage": 0.8035714285714286,
              "accuracy": 0.6596491228070176,
              "confidence_cutoff": 0.5473316703985754
            }
          }
        },
        "calibrated": {
          "n": 1064,
          "nll": 0.7966730081647482,
          "acc": 0.6005639097744361,
          "ece": 0.06373435814748656,
          "brier": 0.46843654552918773,
          "mean_conf": 0.6455031255616036,
          "confident_error_rate": 0.011278195488721804,
          "coverage_at_0_9": 0.299812030075188,
          "accuracy_at_0_9": 0.9623824451410659,
          "coverage_at_5pct_error": 0.3082706766917293,
          "coverage_at_1pct_error": 0.0,
          "aurc": 0.1938629191063446,
          "error_rate_at_0_9": 0.03761755485893417,
          "confidence_bias": 0.04493921578716753,
          "top_bins": {
            "0.9": {
              "n": 319,
              "errors": 12,
              "error_rate": 0.03761755485893417
            },
            "0.95": {
              "n": 0,
              "errors": 0,
              "error_rate": null
            },
            "0.99": {
              "n": 0,
              "errors": 0,
              "error_rate": null
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5028195488721805,
              "accuracy": 0.8205607476635514,
              "confidence_cutoff": 0.5612814391210313
            },
            "0.8": {
              "coverage": 0.8007518796992481,
              "accuracy": 0.6619718309859155,
              "confidence_cutoff": 0.4656043556130704
            }
          }
        }
      },
      "sev_guide_triage": {
        "questions": 231,
        "groups": 23,
        "raw": {
          "n": 231,
          "nll": 0.8396428451107271,
          "acc": 0.5670995670995671,
          "ece": 0.1889188281653689,
          "brier": 0.5162927524049745,
          "mean_conf": 0.7421494516023849,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.3593073593073593,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 0.3722943722943723,
          "coverage_at_1pct_error": 0.36363636363636365,
          "aurc": 0.20987091476964936,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": 0.17504988450281778,
          "top_bins": {
            "0.9": {
              "n": 83,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 80,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 14,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5627705627705628,
              "accuracy": 0.7153846153846154,
              "confidence_cutoff": 0.7564145922660828
            },
            "0.8": {
              "coverage": 0.8051948051948052,
              "accuracy": 0.5806451612903226,
              "confidence_cutoff": 0.51023405030021
            }
          }
        },
        "calibrated": {
          "n": 231,
          "nll": 0.7761249864817235,
          "acc": 0.5670995670995671,
          "ece": 0.16797778410789055,
          "brier": 0.46407372894941223,
          "mean_conf": 0.6510059283006658,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.341991341991342,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 0.37662337662337664,
          "coverage_at_1pct_error": 0.36363636363636365,
          "aurc": 0.20742662880964652,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": 0.08390636120109873,
          "top_bins": {
            "0.9": {
              "n": 79,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 14,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 0,
              "errors": 0,
              "error_rate": null
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5584415584415584,
              "accuracy": 0.7209302325581395,
              "confidence_cutoff": 0.6016800953517527
            },
            "0.8": {
              "coverage": 0.8095238095238095,
              "accuracy": 0.5989304812834224,
              "confidence_cutoff": 0.4481415540755688
            }
          }
        }
      },
      "sev_swarmtraces_static": {
        "questions": 354,
        "groups": 176,
        "raw": {
          "n": 354,
          "nll": 0.04944532768977841,
          "acc": 0.980225988700565,
          "ece": 0.01487544035693287,
          "brier": 0.024608535342050798,
          "mean_conf": 0.9802643779708107,
          "confident_error_rate": 0.002824858757062147,
          "coverage_at_0_9": 0.9463276836158192,
          "accuracy_at_0_9": 0.9970149253731343,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 0.9858757062146892,
          "aurc": 0.0005032049402250012,
          "error_rate_at_0_9": 0.0029850746268656717,
          "confidence_bias": 3.8389270245686724e-05,
          "top_bins": {
            "0.9": {
              "n": 335,
              "errors": 1,
              "error_rate": 0.0029850746268656717
            },
            "0.95": {
              "n": 330,
              "errors": 1,
              "error_rate": 0.0030303030303030303
            },
            "0.99": {
              "n": 308,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9999276209511099
            },
            "0.8": {
              "coverage": 0.8022598870056498,
              "accuracy": 1.0,
              "confidence_cutoff": 0.997796467038603
            }
          }
        },
        "calibrated": {
          "n": 354,
          "nll": 0.06588860080731399,
          "acc": 0.980225988700565,
          "ece": 0.029025934670617953,
          "brier": 0.028907916842541088,
          "mean_conf": 0.9580211141684397,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.8898305084745762,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 0.9887005649717514,
          "aurc": 0.0005463830895640532,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.022204874532125274,
          "top_bins": {
            "0.9": {
              "n": 315,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 297,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 221,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9952110216327251
            },
            "0.8": {
              "coverage": 0.8022598870056498,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9673839577051934
            }
          }
        }
      },
      "sev_sysmon_native_file": {
        "questions": 62,
        "groups": 4,
        "raw": {
          "n": 62,
          "nll": 0.008428780795422771,
          "acc": 1.0,
          "ece": 0.007810434776248104,
          "brier": 0.0017443201233063612,
          "mean_conf": 0.9921895652237519,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9838709677419355,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.0078104347762480986,
          "top_bins": {
            "0.9": {
              "n": 61,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 59,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 57,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9991686260052061
            },
            "0.8": {
              "coverage": 0.8064516129032258,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9962482122843107
            }
          }
        },
        "calibrated": {
          "n": 62,
          "nll": 0.03509983435601263,
          "acc": 1.0,
          "ece": 0.032160212545650484,
          "brier": 0.007419485561600889,
          "mean_conf": 0.9678397874543494,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9516129032258065,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.0321602125456506,
          "top_bins": {
            "0.9": {
              "n": 59,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 55,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 26,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9838915386931907
            },
            "0.8": {
              "coverage": 0.8064516129032258,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9583052893470861
            }
          }
        }
      },
      "sev_sysmon_native_join": {
        "questions": 120,
        "groups": 3,
        "raw": {
          "n": 120,
          "nll": 0.00039767294618876087,
          "acc": 1.0,
          "ece": 0.0003974740966455892,
          "brier": 7.310901229709924e-07,
          "mean_conf": 0.9996025259033544,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.0003974740966455892,
          "top_bins": {
            "0.9": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9997660582574378
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9994606035447323
            }
          }
        },
        "calibrated": {
          "n": 120,
          "nll": 0.009879536093519296,
          "acc": 1.0,
          "ece": 0.009811175359927704,
          "brier": 0.00022722202758135087,
          "mean_conf": 0.9901888246400723,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 1.0,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.009811175359927704,
          "top_bins": {
            "0.9": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 120,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 67,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9917343480609009
            },
            "0.8": {
              "coverage": 0.8,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9855433361281362
            }
          }
        }
      },
      "sev_sysmon_native_other_effect": {
        "questions": 88,
        "groups": 4,
        "raw": {
          "n": 88,
          "nll": 0.005466714567741097,
          "acc": 1.0,
          "ece": 0.005048372154544871,
          "brier": 0.0014126265728775101,
          "mean_conf": 0.994951627845455,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9886363636363636,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.005048372154545011,
          "top_bins": {
            "0.9": {
              "n": 87,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 86,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 83,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9997677646417501
            },
            "0.8": {
              "coverage": 0.8068181818181818,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9978176224669177
            }
          }
        },
        "calibrated": {
          "n": 88,
          "nll": 0.02789592785139411,
          "acc": 1.0,
          "ece": 0.026282565295571752,
          "brier": 0.004783993285232044,
          "mean_conf": 0.9737174347044284,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9545454545454546,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.026282565295571603,
          "top_bins": {
            "0.9": {
              "n": 84,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 75,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 46,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9913758403771589
            },
            "0.8": {
              "coverage": 0.8068181818181818,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9627441989296704
            }
          }
        }
      },
      "sev_sysmon_native_process": {
        "questions": 74,
        "groups": 4,
        "raw": {
          "n": 74,
          "nll": 0.004798899209615666,
          "acc": 1.0,
          "ece": 0.004644321057477101,
          "brier": 0.0005716684001551928,
          "mean_conf": 0.995355678942523,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9864864864864865,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.00464432105747703,
          "top_bins": {
            "0.9": {
              "n": 73,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 72,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 68,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9999268900168323
            },
            "0.8": {
              "coverage": 0.8108108108108109,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9973453419434639
            }
          }
        },
        "calibrated": {
          "n": 74,
          "nll": 0.023528453108139967,
          "acc": 1.0,
          "ece": 0.022317812691888844,
          "brier": 0.004027925361032504,
          "mean_conf": 0.9776821873081113,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9324324324324325,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 1.0,
          "aurc": 0.0,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.022317812691888705,
          "top_bins": {
            "0.9": {
              "n": 69,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 65,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 45,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9964760714345572
            },
            "0.8": {
              "coverage": 0.8108108108108109,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9703686880072409
            }
          }
        }
      },
      "sev_sysmon_native_registry": {
        "questions": 66,
        "groups": 4,
        "raw": {
          "n": 66,
          "nll": 0.024015813931122083,
          "acc": 0.9848484848484849,
          "ece": 0.016913465896023366,
          "brier": 0.01690246747159733,
          "mean_conf": 0.9866786138448916,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9696969696969697,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 0.9696969696969697,
          "aurc": 0.00046266864448682633,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": 0.0018301289964067768,
          "top_bins": {
            "0.9": {
              "n": 64,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 64,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 62,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9998656142719909
            },
            "0.8": {
              "coverage": 0.803030303030303,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9982153954384291
            }
          }
        },
        "calibrated": {
          "n": 66,
          "nll": 0.03582391955947498,
          "acc": 0.9848484848484849,
          "ece": 0.01540577138447916,
          "brier": 0.016990289728223678,
          "mean_conf": 0.9730644954445979,
          "confident_error_rate": 0.0,
          "coverage_at_0_9": 0.9696969696969697,
          "accuracy_at_0_9": 1.0,
          "coverage_at_5pct_error": 1.0,
          "coverage_at_1pct_error": 0.9696969696969697,
          "aurc": 0.00046266864448682633,
          "error_rate_at_0_9": 0.0,
          "confidence_bias": -0.011783989403886919,
          "top_bins": {
            "0.9": {
              "n": 64,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.95": {
              "n": 59,
              "errors": 0,
              "error_rate": 0.0
            },
            "0.99": {
              "n": 40,
              "errors": 0,
              "error_rate": 0.0
            }
          },
          "selective": {
            "0.5": {
              "coverage": 0.5,
              "accuracy": 1.0,
              "confidence_cutoff": 0.9940913990982823
            },
            "0.8": {
              "coverage": 0.803030303030303,
              "accuracy": 1.0,
              "confidence_cutoff": 0.975504713794386
            }
          }
        }
      }
    }
  },
  "policy": {
    "positive": "Requires approval under the supplied policy",
    "negative": "Permitted under the supplied policy",
    "threshold_split": "calibration",
    "temperature": "reuse saved calibration receipt, or benchmark inference temperature; no refit",
    "population": "all five conditional policy questions per admitted source program",
    "bootstrap": "percentile intervals conditional on the fixed calibration threshold; no threshold reselection"
  },
  "limitations": [
    "Authored permissions and static callback dataflow, not historical maliciousness or execution evidence.",
    "The selected conditional panel does not estimate deployment prevalence.",
    "Actor origin is not measured. No locked test is read and this report does not promote a model.",
    "The calibration false-alert budget is empirical, not a deployment guarantee.",
    "Rare-operation rates are null without the relevant class; bootstrap draws without that class are omitted."
  ]
}