| { |
| "suite": "/Users/mac/Sev-security-20260925/evals/sev/response-policy-curriculum-v2", |
| "suite_sha256": "3b9a80dfd212b6932b66ea520c35ab1a3b19e132a81af1b6b7b306d776a70a02", |
| "partition_hashes": { |
| "calibration": "890c363c5bfe5351600071691eefb47dcf4e2530b31b75d4ce8830bd6381f585", |
| "development": "cb92ae6c329f9b9b738eb7637a6f6b3adb3dd8bb7204b5b697f268afb98a4efa" |
| }, |
| "scorer_sha256": "c079a66a2ab4a3d81ce9872b4c1f309ae2f07c89684dc025d70008d9b4be8844", |
| "models": { |
| "parent": { |
| "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a", |
| "checkpoint": { |
| "requested": "macmacmacmac/Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a", |
| "resolved": "/__modal/volumes/vo-RociVuGycNGtzvUcdOqecz/hub/models--macmacmacmac--Sev-4B/snapshots/a1824aefba305fda86e3503b895a7d9b3871f79a", |
| "head_sha256": "1d50c37b4db60c22ae8e6bb24f5055056701bb16e332964350d76df57dc82af0", |
| "adapter_sha256": "b5afd73584d1098cccb0a1bd42afc40e2de3c3bc3074a5173a642b6d4b880420", |
| "inference_temperature": 1.0 |
| }, |
| "inference_temperature": 1.0, |
| "additional_temperature": 1.6245047927124707, |
| "splits": { |
| "calibration": { |
| "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a/calibration", |
| "coverage": { |
| "requested_records": 1920, |
| "requested_questions": 2674, |
| "evaluated_records": 1920, |
| "evaluated_questions": 2674, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "maximum_softmax_probability_error": 3.5965218081468464e-08, |
| "files": { |
| "report.json": "a85db0f08eef510bf05a9ad177940e6b7d05ba3b06cf0b738816e918d8ccd8e6", |
| "rows.json": "507dbf8555ec0f8f045c4d8223d68e9598a5410f163abb5e2110d74ab8614c78", |
| "predictions.jsonl": "2219339424e3ab09f5e3b62eb7c499286b0fe9977dbaef1def487c9166148f0c" |
| } |
| }, |
| "development": { |
| "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-parent-4b-v2/00-Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a/development", |
| "coverage": { |
| "requested_records": 1828, |
| "requested_questions": 2632, |
| "evaluated_records": 1828, |
| "evaluated_questions": 2632, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "maximum_softmax_probability_error": 3.5965218081468464e-08, |
| "files": { |
| "report.json": "cbdc805ba88cff1e050b594eefd164fc96e097e5fb580af33b67ca34dde0374a", |
| "rows.json": "e39116e8120a64037aedc101740359b317108ab07cd7aba5bcaed7119fc7bd0a", |
| "predictions.jsonl": "b49dc9c2f7af40f17d88c61ca7d88d121293b0f3951130205e17d6309eda103f" |
| } |
| } |
| }, |
| "files": { |
| "result.json": "fcd9c5d0d47a5da3799a1bd5b062fd7872e8417bc57f55d9e22feadb29c32e47", |
| "provenance.json": "9568b5ca8ed5e28d306ef28d0b14ab278b67e48b277e983babd03bd273fad769", |
| "calibration/temperature.json": "a63b396ccd1a5041ecc92923cbf62628190207fc42857da4a6727cf7e84fb0af" |
| } |
| }, |
| "candidate": { |
| "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0", |
| "checkpoint": { |
| "requested": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint", |
| "resolved": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint", |
| "head_sha256": "506dcd3587847917a288d95757c497c15a2aef8e2d4bc6523b04ad991c961a2b", |
| "adapter_sha256": "fa1916b34fe1eeb10d15c36927c225df43115407d7c97b4f879bba3102c3ed21", |
| "inference_temperature": 1.0 |
| }, |
| "inference_temperature": 1.0, |
| "additional_temperature": 1.6817928305074288, |
| "splits": { |
| "calibration": { |
| "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0/calibration", |
| "coverage": { |
| "requested_records": 1920, |
| "requested_questions": 2674, |
| "evaluated_records": 1920, |
| "evaluated_questions": 2674, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "maximum_softmax_probability_error": 3.3540022259526125e-08, |
| "files": { |
| "report.json": "44dff4a4d14b65992f8b21e784a50823e30ac16c7e1bba4d75c860fc04bc4995", |
| "rows.json": "026b68705b7ac76bc4361bc2fda1cbfe74265b436a807ea68bca77f87d3cf9da", |
| "predictions.jsonl": "80047e316749504f63b02b49ec8e790eb92cdce9e4bdc15fbf764ac69f0d7ea5" |
| } |
| }, |
| "development": { |
| "directory": "/Users/mac/Sev-security-20260925/runs/sev-r2-response-policy-4b-v2/00-trial-0/development", |
| "coverage": { |
| "requested_records": 1828, |
| "requested_questions": 2632, |
| "evaluated_records": 1828, |
| "evaluated_questions": 2632, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "maximum_softmax_probability_error": 3.3602443216818045e-08, |
| "files": { |
| "report.json": "29e391352dd2474975ebe3d9b8b66be1a0cf96b5b917d559d1f80afd5b0704c5", |
| "rows.json": "27ce2c65c5efa32d0197c51e2dd13f81c7b71d410ce7a27dce7998e8dd247c27", |
| "predictions.jsonl": "548409eb6efb70c9b320a43cb88d671696c01326917bb18837e6d7aba4d2a69e" |
| } |
| } |
| }, |
| "files": { |
| "result.json": "ebf9f528ebe8ecef772f32418c8bc99b998285962b39c0a229c4e72b1f0b1014", |
| "provenance.json": "7cfaf8ba0c975c545afd56248df611d1fa6bed0a9b0655968f9aa6bfe60258b1", |
| "calibration/temperature.json": "328a01a5463c0f03f2b5d5c324d5be8def08cac819cf89aaf49a27a9c88e71e4" |
| } |
| } |
| }, |
| "calibration_thresholds": { |
| "parent": { |
| "threshold": 0.8875371629397495, |
| "false_alert_budget": 0.05, |
| "comparison": "positive_probability >= threshold; equal scores stay together", |
| "selection": "maximum recall, then fewest false alerts, then highest threshold", |
| "n": 140, |
| "positives": 28, |
| "negatives": 112, |
| "confusion": { |
| "tp": 11, |
| "fp": 5, |
| "tn": 107, |
| "fn": 17 |
| }, |
| "recall": 0.39285714285714285, |
| "false_alert_rate": 0.044642857142857144, |
| "precision": 0.6875, |
| "threshold_accuracy": 0.8428571428571429 |
| }, |
| "candidate": { |
| "threshold": 0.8449815374033091, |
| "false_alert_budget": 0.05, |
| "comparison": "positive_probability >= threshold; equal scores stay together", |
| "selection": "maximum recall, then fewest false alerts, then highest threshold", |
| "n": 140, |
| "positives": 28, |
| "negatives": 112, |
| "confusion": { |
| "tp": 21, |
| "fp": 4, |
| "tn": 108, |
| "fn": 7 |
| }, |
| "recall": 0.75, |
| "false_alert_rate": 0.03571428571428571, |
| "precision": 0.84, |
| "threshold_accuracy": 0.9214285714285714 |
| } |
| }, |
| "baseline_headroom": { |
| "rule": "continuation only if parent development recall < 0.95 or false-alert rate > 0.05", |
| "class_support_sufficient": true, |
| "continuation_allowed": true |
| }, |
| "development": { |
| "always_permit_baseline": { |
| "n": 175, |
| "positives": 35, |
| "negatives": 140, |
| "confusion": { |
| "tp": 0, |
| "fp": 0, |
| "tn": 140, |
| "fn": 35 |
| }, |
| "recall": 0.0, |
| "false_alert_rate": 0.0, |
| "precision": null, |
| "threshold_accuracy": 0.8 |
| }, |
| "models": { |
| "parent": { |
| "questions": 175, |
| "groups": 35, |
| "raw": { |
| "n": 175, |
| "nll": 0.795502582803619, |
| "acc": 0.7257142857142858, |
| "ece": 0.1747005023605082, |
| "brier": 0.4505153855550131, |
| "mean_conf": 0.8840572965356658, |
| "confident_error_rate": 0.13714285714285715, |
| "coverage_at_0_9": 0.6457142857142857, |
| "accuracy_at_0_9": 0.7876106194690266, |
| "coverage_at_5pct_error": 0.02857142857142857, |
| "coverage_at_1pct_error": 0.02857142857142857, |
| "aurc": 0.1992292401228163, |
| "error_rate_at_0_9": 0.21238938053097345, |
| "confidence_bias": 0.15834301082138003, |
| "top_bins": { |
| "0.9": { |
| "n": 113, |
| "errors": 24, |
| "error_rate": 0.21238938053097345 |
| }, |
| "0.95": { |
| "n": 84, |
| "errors": 18, |
| "error_rate": 0.21428571428571427 |
| }, |
| "0.99": { |
| "n": 26, |
| "errors": 2, |
| "error_rate": 0.07692307692307693 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.7613636363636364, |
| "confidence_cutoff": 0.9457634807080788 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.7938824081649869 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 175, |
| "nll": 0.6260474644745716, |
| "acc": 0.7257142857142858, |
| "ece": 0.10761147298493057, |
| "brier": 0.4111063010906543, |
| "mean_conf": 0.8140932667241441, |
| "confident_error_rate": 0.06857142857142857, |
| "coverage_at_0_9": 0.35428571428571426, |
| "accuracy_at_0_9": 0.8064516129032258, |
| "coverage_at_5pct_error": 0.02857142857142857, |
| "coverage_at_1pct_error": 0.02857142857142857, |
| "aurc": 0.1992292401228163, |
| "error_rate_at_0_9": 0.1935483870967742, |
| "confidence_bias": 0.0883789810098583, |
| "top_bins": { |
| "0.9": { |
| "n": 62, |
| "errors": 12, |
| "error_rate": 0.1935483870967742 |
| }, |
| "0.95": { |
| "n": 21, |
| "errors": 2, |
| "error_rate": 0.09523809523809523 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.7613636363636364, |
| "confidence_cutoff": 0.8531718492565429 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.6963743924447515 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 35, |
| "correct": 29, |
| "recall": 0.8285714285714286 |
| }, |
| "permitted": { |
| "n": 140, |
| "correct": 98, |
| "recall": 0.7 |
| } |
| }, |
| "threshold": 0.8875371629397495, |
| "n": 175, |
| "positives": 35, |
| "negatives": 140, |
| "confusion": { |
| "tp": 14, |
| "fp": 12, |
| "tn": 128, |
| "fn": 21 |
| }, |
| "recall": 0.4, |
| "false_alert_rate": 0.08571428571428572, |
| "precision": 0.5384615384615384, |
| "threshold_accuracy": 0.8114285714285714 |
| }, |
| "candidate": { |
| "questions": 175, |
| "groups": 35, |
| "raw": { |
| "n": 175, |
| "nll": 0.49844721765472333, |
| "acc": 0.84, |
| "ece": 0.11735638435435483, |
| "brier": 0.269672286103042, |
| "mean_conf": 0.9559888432162478, |
| "confident_error_rate": 0.10285714285714286, |
| "coverage_at_0_9": 0.8742857142857143, |
| "accuracy_at_0_9": 0.8823529411764706, |
| "coverage_at_5pct_error": 0.6114285714285714, |
| "coverage_at_1pct_error": 0.44, |
| "aurc": 0.044632779737602504, |
| "error_rate_at_0_9": 0.11764705882352941, |
| "confidence_bias": 0.1159888432162478, |
| "top_bins": { |
| "0.9": { |
| "n": 153, |
| "errors": 18, |
| "error_rate": 0.11764705882352941 |
| }, |
| "0.95": { |
| "n": 140, |
| "errors": 14, |
| "error_rate": 0.1 |
| }, |
| "0.99": { |
| "n": 92, |
| "errors": 3, |
| "error_rate": 0.03260869565217391 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.9772727272727273, |
| "confidence_cutoff": 0.9904485398763483 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9516673073011176 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 175, |
| "nll": 0.37433095036445935, |
| "acc": 0.84, |
| "ece": 0.06636288221880407, |
| "brier": 0.23938834727626476, |
| "mean_conf": 0.9048661883246669, |
| "confident_error_rate": 0.05142857142857143, |
| "coverage_at_0_9": 0.7142857142857143, |
| "accuracy_at_0_9": 0.928, |
| "coverage_at_5pct_error": 0.6114285714285714, |
| "coverage_at_1pct_error": 0.44, |
| "aurc": 0.044632779737602504, |
| "error_rate_at_0_9": 0.072, |
| "confidence_bias": 0.06486618832466695, |
| "top_bins": { |
| "0.9": { |
| "n": 125, |
| "errors": 9, |
| "error_rate": 0.072 |
| }, |
| "0.95": { |
| "n": 77, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 14, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.9772727272727273, |
| "confidence_cutoff": 0.9404661687170754 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.8547040294961724 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 35, |
| "correct": 35, |
| "recall": 1.0 |
| }, |
| "permitted": { |
| "n": 140, |
| "correct": 112, |
| "recall": 0.8 |
| } |
| }, |
| "threshold": 0.8449815374033091, |
| "n": 175, |
| "positives": 35, |
| "negatives": 140, |
| "confusion": { |
| "tp": 25, |
| "fp": 14, |
| "tn": 126, |
| "fn": 10 |
| }, |
| "recall": 0.7142857142857143, |
| "false_alert_rate": 0.1, |
| "precision": 0.6410256410256411, |
| "threshold_accuracy": 0.8628571428571429 |
| } |
| }, |
| "paired_threshold_intervals": { |
| "groups": 35, |
| "samples": 2000, |
| "seed": 20260927, |
| "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed", |
| "metrics": { |
| "recall": { |
| "parent": { |
| "value": 0.4, |
| "ci95": [ |
| 0.22857142857142856, |
| 0.5714285714285714 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.7142857142857143, |
| "ci95": [ |
| 0.5714285714285714, |
| 0.8571428571428571 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.3142857142857143, |
| "ci95": [ |
| 0.17142857142857143, |
| 0.4857142857142857 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "false_alert_rate": { |
| "parent": { |
| "value": 0.08571428571428572, |
| "ci95": [ |
| 0.05, |
| 0.12857142857142856 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.1, |
| "ci95": [ |
| 0.06428571428571428, |
| 0.14285714285714285 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.01428571428571429, |
| "ci95": [ |
| -0.014285714285714277, |
| 0.04285714285714286 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "precision": { |
| "parent": { |
| "value": 0.5384615384615384, |
| "ci95": [ |
| 0.3665909090909091, |
| 0.70979262672811 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.6410256410256411, |
| "ci95": [ |
| 0.55, |
| 0.7435897435897436 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.10256410256410264, |
| "ci95": [ |
| 0.005701872682569551, |
| 0.21591595300834426 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "threshold_accuracy": { |
| "parent": { |
| "value": 0.8114285714285714, |
| "ci95": [ |
| 0.7655714285714286, |
| 0.8628571428571429 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.8628571428571429, |
| "ci95": [ |
| 0.8228571428571428, |
| 0.9028571428571428 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.05142857142857149, |
| "ci95": [ |
| 0.02285714285714291, |
| 0.08000000000000007 |
| ], |
| "valid_resamples": 2000 |
| } |
| } |
| } |
| } |
| }, |
| "by_operation": { |
| "plain_text_display": { |
| "always_permit_baseline": { |
| "n": 35, |
| "positives": 20, |
| "negatives": 15, |
| "confusion": { |
| "tp": 0, |
| "fp": 0, |
| "tn": 15, |
| "fn": 20 |
| }, |
| "recall": 0.0, |
| "false_alert_rate": 0.0, |
| "precision": null, |
| "threshold_accuracy": 0.42857142857142855 |
| }, |
| "models": { |
| "parent": { |
| "questions": 35, |
| "groups": 35, |
| "raw": { |
| "n": 35, |
| "nll": 0.44020879235141186, |
| "acc": 0.7428571428571429, |
| "ece": 0.1010721055158729, |
| "brier": 0.2970615598224881, |
| "mean_conf": 0.7840390587691357, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.2571428571428571, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.37142857142857144, |
| "coverage_at_1pct_error": 0.37142857142857144, |
| "aurc": 0.08938203390842284, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.04118191591199283, |
| "top_bins": { |
| "0.9": { |
| "n": 9, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 6, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 0.9444444444444444, |
| "confidence_cutoff": 0.8020615218849417 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.7857142857142857, |
| "confidence_cutoff": 0.6323033624549943 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 35, |
| "nll": 0.4703692241803648, |
| "acc": 0.7428571428571429, |
| "ece": 0.1062363699460275, |
| "brier": 0.3059826176210274, |
| "mean_conf": 0.7080594224133254, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.05714285714285714, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.37142857142857144, |
| "coverage_at_1pct_error": 0.37142857142857144, |
| "aurc": 0.08938203390842284, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.03479772044381746, |
| "top_bins": { |
| "0.9": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 0.9444444444444444, |
| "confidence_cutoff": 0.702937726558054 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.7857142857142857, |
| "confidence_cutoff": 0.5826614347130625 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 20, |
| "correct": 14, |
| "recall": 0.7 |
| }, |
| "permitted": { |
| "n": 15, |
| "correct": 12, |
| "recall": 0.8 |
| } |
| }, |
| "threshold": 0.8875371629397495, |
| "n": 35, |
| "positives": 20, |
| "negatives": 15, |
| "confusion": { |
| "tp": 1, |
| "fp": 0, |
| "tn": 15, |
| "fn": 19 |
| }, |
| "recall": 0.05, |
| "false_alert_rate": 0.0, |
| "precision": 1.0, |
| "threshold_accuracy": 0.45714285714285713 |
| }, |
| "candidate": { |
| "questions": 35, |
| "groups": 35, |
| "raw": { |
| "n": 35, |
| "nll": 0.2308958263556794, |
| "acc": 0.8857142857142857, |
| "ece": 0.05276786009948832, |
| "brier": 0.1387650722964353, |
| "mean_conf": 0.9185510684713508, |
| "confident_error_rate": 0.02857142857142857, |
| "coverage_at_0_9": 0.7714285714285715, |
| "accuracy_at_0_9": 0.9629629629629629, |
| "coverage_at_5pct_error": 0.8857142857142857, |
| "coverage_at_1pct_error": 0.6285714285714286, |
| "aurc": 0.018915971835104774, |
| "error_rate_at_0_9": 0.037037037037037035, |
| "confidence_bias": 0.03283678275706514, |
| "top_bins": { |
| "0.9": { |
| "n": 27, |
| "errors": 1, |
| "error_rate": 0.037037037037037035 |
| }, |
| "0.95": { |
| "n": 19, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 6, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9562865363749455 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9642857142857143, |
| "confidence_cutoff": 0.8891588391985842 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 35, |
| "nll": 0.2642131240699265, |
| "acc": 0.8857142857142857, |
| "ece": 0.08058920252061248, |
| "brier": 0.1465567737525219, |
| "mean_conf": 0.8412751407359753, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.4, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.8857142857142857, |
| "coverage_at_1pct_error": 0.6285714285714286, |
| "aurc": 0.018915971835104774, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.04443914497831036, |
| "top_bins": { |
| "0.9": { |
| "n": 14, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 4, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.8623076992723852 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9642857142857143, |
| "confidence_cutoff": 0.7752279376903202 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 20, |
| "correct": 20, |
| "recall": 1.0 |
| }, |
| "permitted": { |
| "n": 15, |
| "correct": 11, |
| "recall": 0.7333333333333333 |
| } |
| }, |
| "threshold": 0.8449815374033091, |
| "n": 35, |
| "positives": 20, |
| "negatives": 15, |
| "confusion": { |
| "tp": 12, |
| "fp": 0, |
| "tn": 15, |
| "fn": 8 |
| }, |
| "recall": 0.6, |
| "false_alert_rate": 0.0, |
| "precision": 1.0, |
| "threshold_accuracy": 0.7714285714285715 |
| } |
| }, |
| "paired_threshold_intervals": { |
| "groups": 35, |
| "samples": 2000, |
| "seed": 20260927, |
| "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed", |
| "metrics": { |
| "recall": { |
| "parent": { |
| "value": 0.05, |
| "ci95": [ |
| 0.0, |
| 0.16666666666666666 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.6, |
| "ci95": [ |
| 0.38095238095238093, |
| 0.8095238095238095 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.5499999999999999, |
| "ci95": [ |
| 0.3333333333333333, |
| 0.7728535353535347 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "false_alert_rate": { |
| "parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "precision": { |
| "parent": { |
| "value": 1.0, |
| "ci95": [ |
| 1.0, |
| 1.0 |
| ], |
| "valid_resamples": 1232 |
| }, |
| "candidate": { |
| "value": 1.0, |
| "ci95": [ |
| 1.0, |
| 1.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 1232 |
| } |
| }, |
| "threshold_accuracy": { |
| "parent": { |
| "value": 0.45714285714285713, |
| "ci95": [ |
| 0.2857142857142857, |
| 0.6285714285714286 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.7714285714285715, |
| "ci95": [ |
| 0.6285714285714286, |
| 0.8857142857142857 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.31428571428571433, |
| "ci95": [ |
| 0.17142857142857149, |
| 0.48571428571428565 |
| ], |
| "valid_resamples": 2000 |
| } |
| } |
| } |
| } |
| }, |
| "html_parse": { |
| "always_permit_baseline": { |
| "n": 35, |
| "positives": 14, |
| "negatives": 21, |
| "confusion": { |
| "tp": 0, |
| "fp": 0, |
| "tn": 21, |
| "fn": 14 |
| }, |
| "recall": 0.0, |
| "false_alert_rate": 0.0, |
| "precision": null, |
| "threshold_accuracy": 0.6 |
| }, |
| "models": { |
| "parent": { |
| "questions": 35, |
| "groups": 35, |
| "raw": { |
| "n": 35, |
| "nll": 1.9047291463003604, |
| "acc": 0.42857142857142855, |
| "ece": 0.5258122169538513, |
| "brier": 1.0143214124687336, |
| "mean_conf": 0.95438364552528, |
| "confident_error_rate": 0.4857142857142857, |
| "coverage_at_0_9": 0.9142857142857143, |
| "accuracy_at_0_9": 0.46875, |
| "coverage_at_5pct_error": 0.14285714285714285, |
| "coverage_at_1pct_error": 0.14285714285714285, |
| "aurc": 0.3679267902406343, |
| "error_rate_at_0_9": 0.53125, |
| "confidence_bias": 0.5258122169538515, |
| "top_bins": { |
| "0.9": { |
| "n": 32, |
| "errors": 17, |
| "error_rate": 0.53125 |
| }, |
| "0.95": { |
| "n": 27, |
| "errors": 14, |
| "error_rate": 0.5185185185185185 |
| }, |
| "0.99": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 0.5555555555555556, |
| "confidence_cutoff": 0.9801112406947217 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.4642857142857143, |
| "confidence_cutoff": 0.9495538881263412 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 35, |
| "nll": 1.2659489508436006, |
| "acc": 0.42857142857142855, |
| "ece": 0.4588216874901758, |
| "brier": 0.8687995501112846, |
| "mean_conf": 0.8873931160616043, |
| "confident_error_rate": 0.2571428571428571, |
| "coverage_at_0_9": 0.6, |
| "accuracy_at_0_9": 0.5714285714285714, |
| "coverage_at_5pct_error": 0.14285714285714285, |
| "coverage_at_1pct_error": 0.14285714285714285, |
| "aurc": 0.3679267902406343, |
| "error_rate_at_0_9": 0.42857142857142855, |
| "confidence_bias": 0.45882168749017577, |
| "top_bins": { |
| "0.9": { |
| "n": 21, |
| "errors": 9, |
| "error_rate": 0.42857142857142855 |
| }, |
| "0.95": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 0.5555555555555556, |
| "confidence_cutoff": 0.9167662540005418 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.4642857142857143, |
| "confidence_cutoff": 0.8589695610685519 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 14, |
| "correct": 14, |
| "recall": 1.0 |
| }, |
| "permitted": { |
| "n": 21, |
| "correct": 1, |
| "recall": 0.047619047619047616 |
| } |
| }, |
| "threshold": 0.8875371629397495, |
| "n": 35, |
| "positives": 14, |
| "negatives": 21, |
| "confusion": { |
| "tp": 12, |
| "fp": 9, |
| "tn": 12, |
| "fn": 2 |
| }, |
| "recall": 0.8571428571428571, |
| "false_alert_rate": 0.42857142857142855, |
| "precision": 0.5714285714285714, |
| "threshold_accuracy": 0.6857142857142857 |
| }, |
| "candidate": { |
| "questions": 35, |
| "groups": 35, |
| "raw": { |
| "n": 35, |
| "nll": 1.7814413567537535, |
| "acc": 0.45714285714285713, |
| "ece": 0.4870901667090771, |
| "brier": 0.9601370731320206, |
| "mean_conf": 0.9442330238519343, |
| "confident_error_rate": 0.42857142857142855, |
| "coverage_at_0_9": 0.8285714285714286, |
| "accuracy_at_0_9": 0.4827586206896552, |
| "coverage_at_5pct_error": 0.2, |
| "coverage_at_1pct_error": 0.2, |
| "aurc": 0.31511035965021955, |
| "error_rate_at_0_9": 0.5172413793103449, |
| "confidence_bias": 0.4870901667090772, |
| "top_bins": { |
| "0.9": { |
| "n": 29, |
| "errors": 15, |
| "error_rate": 0.5172413793103449 |
| }, |
| "0.95": { |
| "n": 25, |
| "errors": 12, |
| "error_rate": 0.48 |
| }, |
| "0.99": { |
| "n": 8, |
| "errors": 1, |
| "error_rate": 0.125 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 0.6666666666666666, |
| "confidence_cutoff": 0.9770944745872364 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.4642857142857143, |
| "confidence_cutoff": 0.9312263284660277 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 35, |
| "nll": 1.1646094584375128, |
| "acc": 0.45714285714285713, |
| "ece": 0.4387100296237643, |
| "brier": 0.8119992696195855, |
| "mean_conf": 0.8734203345049698, |
| "confident_error_rate": 0.2, |
| "coverage_at_0_9": 0.5428571428571428, |
| "accuracy_at_0_9": 0.631578947368421, |
| "coverage_at_5pct_error": 0.2, |
| "coverage_at_1pct_error": 0.2, |
| "aurc": 0.31511035965021955, |
| "error_rate_at_0_9": 0.3684210526315789, |
| "confidence_bias": 0.4162774773621127, |
| "top_bins": { |
| "0.9": { |
| "n": 19, |
| "errors": 7, |
| "error_rate": 0.3684210526315789 |
| }, |
| "0.95": { |
| "n": 6, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 0.6666666666666666, |
| "confidence_cutoff": 0.903057582586965 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.4642857142857143, |
| "confidence_cutoff": 0.8248194570355916 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 14, |
| "correct": 14, |
| "recall": 1.0 |
| }, |
| "permitted": { |
| "n": 21, |
| "correct": 2, |
| "recall": 0.09523809523809523 |
| } |
| }, |
| "threshold": 0.8449815374033091, |
| "n": 35, |
| "positives": 14, |
| "negatives": 21, |
| "confusion": { |
| "tp": 12, |
| "fp": 12, |
| "tn": 9, |
| "fn": 2 |
| }, |
| "recall": 0.8571428571428571, |
| "false_alert_rate": 0.5714285714285714, |
| "precision": 0.5, |
| "threshold_accuracy": 0.6 |
| } |
| }, |
| "paired_threshold_intervals": { |
| "groups": 35, |
| "samples": 2000, |
| "seed": 20260927, |
| "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed", |
| "metrics": { |
| "recall": { |
| "parent": { |
| "value": 0.8571428571428571, |
| "ci95": [ |
| 0.6666666666666666, |
| 1.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.8571428571428571, |
| "ci95": [ |
| 0.6666666666666666, |
| 1.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "false_alert_rate": { |
| "parent": { |
| "value": 0.42857142857142855, |
| "ci95": [ |
| 0.21739130434782608, |
| 0.6521739130434783 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.5714285714285714, |
| "ci95": [ |
| 0.3598235294117647, |
| 0.7858082706766912 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.14285714285714285, |
| "ci95": [ |
| 0.0, |
| 0.3333333333333333 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "precision": { |
| "parent": { |
| "value": 0.5714285714285714, |
| "ci95": [ |
| 0.35, |
| 0.782608695652174 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.5, |
| "ci95": [ |
| 0.2916666666666667, |
| 0.7058823529411765 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": -0.0714285714285714, |
| "ci95": [ |
| -0.1768421052631578, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "threshold_accuracy": { |
| "parent": { |
| "value": 0.6857142857142857, |
| "ci95": [ |
| 0.5142857142857142, |
| 0.8285714285714286 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.6, |
| "ci95": [ |
| 0.42857142857142855, |
| 0.7714285714285715 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": -0.08571428571428574, |
| "ci95": [ |
| -0.20000000000000007, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| } |
| } |
| } |
| } |
| }, |
| "onward_request_body": { |
| "always_permit_baseline": { |
| "n": 35, |
| "positives": 1, |
| "negatives": 34, |
| "confusion": { |
| "tp": 0, |
| "fp": 0, |
| "tn": 34, |
| "fn": 1 |
| }, |
| "recall": 0.0, |
| "false_alert_rate": 0.0, |
| "precision": null, |
| "threshold_accuracy": 0.9714285714285714 |
| }, |
| "models": { |
| "parent": { |
| "questions": 35, |
| "groups": 35, |
| "raw": { |
| "n": 35, |
| "nll": 0.13064358196820833, |
| "acc": 0.9714285714285714, |
| "ece": 0.082650843776636, |
| "brier": 0.062386596130186385, |
| "mean_conf": 0.8959570370446475, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.6857142857142857, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9428571428571428, |
| "aurc": 0.0016566626650660265, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.07547153438392395, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 7, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9326748524721808 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.8734922079816011 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 35, |
| "nll": 0.20940801007318904, |
| "acc": 0.9714285714285714, |
| "ece": 0.14463259695441177, |
| "brier": 0.098760907973673, |
| "mean_conf": 0.8267959744741596, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.37142857142857144, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9428571428571428, |
| "aurc": 0.0016566626650660265, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.14463259695441177, |
| "top_bins": { |
| "0.9": { |
| "n": 13, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 7, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.8345254161812303 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.7666347623294629 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 1, |
| "correct": 1, |
| "recall": 1.0 |
| }, |
| "permitted": { |
| "n": 34, |
| "correct": 33, |
| "recall": 0.9705882352941176 |
| } |
| }, |
| "threshold": 0.8875371629397495, |
| "n": 35, |
| "positives": 1, |
| "negatives": 34, |
| "confusion": { |
| "tp": 1, |
| "fp": 0, |
| "tn": 34, |
| "fn": 0 |
| }, |
| "recall": 1.0, |
| "false_alert_rate": 0.0, |
| "precision": 1.0, |
| "threshold_accuracy": 1.0 |
| }, |
| "candidate": { |
| "questions": 35, |
| "groups": 35, |
| "raw": { |
| "n": 35, |
| "nll": 0.005026554467109394, |
| "acc": 1.0, |
| "ece": 0.004983623817000349, |
| "brier": 0.0001685963296296438, |
| "mean_conf": 0.9950163761829997, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.004983623817000349, |
| "top_bins": { |
| "0.9": { |
| "n": 35, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 35, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 31, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9974550323398608 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9932663672241991 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 35, |
| "nll": 0.03405651326059574, |
| "acc": 1.0, |
| "ece": 0.033071042578584564, |
| "brier": 0.0037297689099056244, |
| "mean_conf": 0.9669289574214154, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9428571428571428, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.03307104257858462, |
| "top_bins": { |
| "0.9": { |
| "n": 33, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 28, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 6, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9720891179694704 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9511715473570618 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 1, |
| "correct": 1, |
| "recall": 1.0 |
| }, |
| "permitted": { |
| "n": 34, |
| "correct": 34, |
| "recall": 1.0 |
| } |
| }, |
| "threshold": 0.8449815374033091, |
| "n": 35, |
| "positives": 1, |
| "negatives": 34, |
| "confusion": { |
| "tp": 1, |
| "fp": 0, |
| "tn": 34, |
| "fn": 0 |
| }, |
| "recall": 1.0, |
| "false_alert_rate": 0.0, |
| "precision": 1.0, |
| "threshold_accuracy": 1.0 |
| } |
| }, |
| "paired_threshold_intervals": { |
| "groups": 35, |
| "samples": 2000, |
| "seed": 20260927, |
| "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed", |
| "metrics": { |
| "recall": { |
| "parent": { |
| "value": 1.0, |
| "ci95": [ |
| 1.0, |
| 1.0 |
| ], |
| "valid_resamples": 1272 |
| }, |
| "candidate": { |
| "value": 1.0, |
| "ci95": [ |
| 1.0, |
| 1.0 |
| ], |
| "valid_resamples": 1272 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 1272 |
| } |
| }, |
| "false_alert_rate": { |
| "parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "precision": { |
| "parent": { |
| "value": 1.0, |
| "ci95": [ |
| 1.0, |
| 1.0 |
| ], |
| "valid_resamples": 1272 |
| }, |
| "candidate": { |
| "value": 1.0, |
| "ci95": [ |
| 1.0, |
| 1.0 |
| ], |
| "valid_resamples": 1272 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 1272 |
| } |
| }, |
| "threshold_accuracy": { |
| "parent": { |
| "value": 1.0, |
| "ci95": [ |
| 1.0, |
| 1.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 1.0, |
| "ci95": [ |
| 1.0, |
| 1.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| } |
| } |
| } |
| } |
| }, |
| "code_execution": { |
| "always_permit_baseline": { |
| "n": 35, |
| "positives": 0, |
| "negatives": 35, |
| "confusion": { |
| "tp": 0, |
| "fp": 0, |
| "tn": 35, |
| "fn": 0 |
| }, |
| "recall": null, |
| "false_alert_rate": 0.0, |
| "precision": null, |
| "threshold_accuracy": 1.0 |
| }, |
| "models": { |
| "parent": { |
| "questions": 35, |
| "groups": 35, |
| "raw": { |
| "n": 35, |
| "nll": 0.03495771061017535, |
| "acc": 1.0, |
| "ece": 0.03371089311351877, |
| "brier": 0.004643703343002468, |
| "mean_conf": 0.9662891068864811, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9714285714285714, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.033710893113518936, |
| "top_bins": { |
| "0.9": { |
| "n": 34, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 27, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 11, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9787669092307943 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9455319719721502 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 35, |
| "nll": 0.10667969626313498, |
| "acc": 1.0, |
| "ece": 0.09922627250314853, |
| "brier": 0.02646181525533936, |
| "mean_conf": 0.9007737274968517, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.5714285714285714, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0992262725031483, |
| "top_bins": { |
| "0.9": { |
| "n": 20, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 9, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9135753218620322 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.8528241762602075 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 0, |
| "correct": 0, |
| "recall": null |
| }, |
| "permitted": { |
| "n": 35, |
| "correct": 35, |
| "recall": 1.0 |
| } |
| }, |
| "threshold": 0.8875371629397495, |
| "n": 35, |
| "positives": 0, |
| "negatives": 35, |
| "confusion": { |
| "tp": 0, |
| "fp": 0, |
| "tn": 35, |
| "fn": 0 |
| }, |
| "recall": null, |
| "false_alert_rate": 0.0, |
| "precision": null, |
| "threshold_accuracy": 1.0 |
| }, |
| "candidate": { |
| "questions": 35, |
| "groups": 35, |
| "raw": { |
| "n": 35, |
| "nll": 0.003713040174857426, |
| "acc": 1.0, |
| "ece": 0.0036946893416571758, |
| "brier": 7.269134859423068e-05, |
| "mean_conf": 0.9963053106583428, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0036946893416571758, |
| "top_bins": { |
| "0.9": { |
| "n": 35, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 35, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 31, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9983259960008511 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9949578958858085 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 35, |
| "nll": 0.029845462374630014, |
| "acc": 1.0, |
| "ece": 0.02916307563639764, |
| "brier": 0.002620264819563586, |
| "mean_conf": 0.9708369243636024, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.02916307563639764, |
| "top_bins": { |
| "0.9": { |
| "n": 35, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 30, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9781193495148215 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9586084771058523 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 0, |
| "correct": 0, |
| "recall": null |
| }, |
| "permitted": { |
| "n": 35, |
| "correct": 35, |
| "recall": 1.0 |
| } |
| }, |
| "threshold": 0.8449815374033091, |
| "n": 35, |
| "positives": 0, |
| "negatives": 35, |
| "confusion": { |
| "tp": 0, |
| "fp": 0, |
| "tn": 35, |
| "fn": 0 |
| }, |
| "recall": null, |
| "false_alert_rate": 0.0, |
| "precision": null, |
| "threshold_accuracy": 1.0 |
| } |
| }, |
| "paired_threshold_intervals": { |
| "groups": 35, |
| "samples": 2000, |
| "seed": 20260927, |
| "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed", |
| "metrics": { |
| "recall": { |
| "parent": { |
| "value": null, |
| "ci95": null, |
| "valid_resamples": 0 |
| }, |
| "candidate": { |
| "value": null, |
| "ci95": null, |
| "valid_resamples": 0 |
| }, |
| "candidate_minus_parent": { |
| "value": null, |
| "ci95": null, |
| "valid_resamples": 0 |
| } |
| }, |
| "false_alert_rate": { |
| "parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "precision": { |
| "parent": { |
| "value": null, |
| "ci95": null, |
| "valid_resamples": 0 |
| }, |
| "candidate": { |
| "value": null, |
| "ci95": null, |
| "valid_resamples": 0 |
| }, |
| "candidate_minus_parent": { |
| "value": null, |
| "ci95": null, |
| "valid_resamples": 0 |
| } |
| }, |
| "threshold_accuracy": { |
| "parent": { |
| "value": 1.0, |
| "ci95": [ |
| 1.0, |
| 1.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 1.0, |
| "ci95": [ |
| 1.0, |
| 1.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| } |
| } |
| } |
| } |
| }, |
| "console_output": { |
| "always_permit_baseline": { |
| "n": 35, |
| "positives": 0, |
| "negatives": 35, |
| "confusion": { |
| "tp": 0, |
| "fp": 0, |
| "tn": 35, |
| "fn": 0 |
| }, |
| "recall": null, |
| "false_alert_rate": 0.0, |
| "precision": null, |
| "threshold_accuracy": 1.0 |
| }, |
| "models": { |
| "parent": { |
| "questions": 35, |
| "groups": 35, |
| "raw": { |
| "n": 35, |
| "nll": 1.4669736827879394, |
| "acc": 0.4857142857142857, |
| "ece": 0.44969603628529076, |
| "brier": 0.8741636560106545, |
| "mean_conf": 0.8196176344527856, |
| "confident_error_rate": 0.2, |
| "coverage_at_0_9": 0.4, |
| "accuracy_at_0_9": 0.5, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.5978180747557157, |
| "error_rate_at_0_9": 0.5, |
| "confidence_bias": 0.33390334873849986, |
| "top_bins": { |
| "0.9": { |
| "n": 14, |
| "errors": 7, |
| "error_rate": 0.5 |
| }, |
| "0.95": { |
| "n": 8, |
| "errors": 4, |
| "error_rate": 0.5 |
| }, |
| "0.99": { |
| "n": 3, |
| "errors": 2, |
| "error_rate": 0.6666666666666666 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 0.3888888888888889, |
| "confidence_cutoff": 0.8730975086069251 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.39285714285714285, |
| "confidence_cutoff": 0.6488618260771695 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 35, |
| "nll": 1.077831441012569, |
| "acc": 0.4857142857142857, |
| "ece": 0.4118859974171003, |
| "brier": 0.7555266144919474, |
| "mean_conf": 0.7474440931747798, |
| "confident_error_rate": 0.08571428571428572, |
| "coverage_at_0_9": 0.17142857142857143, |
| "accuracy_at_0_9": 0.5, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.5978180747557157, |
| "error_rate_at_0_9": 0.5, |
| "confidence_bias": 0.26172980746049407, |
| "top_bins": { |
| "0.9": { |
| "n": 6, |
| "errors": 3, |
| "error_rate": 0.5 |
| }, |
| "0.95": { |
| "n": 2, |
| "errors": 2, |
| "error_rate": 1.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 0.3888888888888889, |
| "confidence_cutoff": 0.7662416927389056 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.39285714285714285, |
| "confidence_cutoff": 0.5933872468656349 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 0, |
| "correct": 0, |
| "recall": null |
| }, |
| "permitted": { |
| "n": 35, |
| "correct": 17, |
| "recall": 0.4857142857142857 |
| } |
| }, |
| "threshold": 0.8875371629397495, |
| "n": 35, |
| "positives": 0, |
| "negatives": 35, |
| "confusion": { |
| "tp": 0, |
| "fp": 3, |
| "tn": 32, |
| "fn": 0 |
| }, |
| "recall": null, |
| "false_alert_rate": 0.08571428571428572, |
| "precision": 0.0, |
| "threshold_accuracy": 0.9142857142857143 |
| }, |
| "candidate": { |
| "questions": 35, |
| "groups": 35, |
| "raw": { |
| "n": 35, |
| "nll": 0.4711593105222168, |
| "acc": 0.8571428571428571, |
| "ece": 0.10032786865714749, |
| "brier": 0.24921799740853026, |
| "mean_conf": 0.9258384369166115, |
| "confident_error_rate": 0.05714285714285714, |
| "coverage_at_0_9": 0.7714285714285715, |
| "accuracy_at_0_9": 0.9259259259259259, |
| "coverage_at_5pct_error": 0.2571428571428571, |
| "coverage_at_1pct_error": 0.2571428571428571, |
| "aurc": 0.08024957297121005, |
| "error_rate_at_0_9": 0.07407407407407407, |
| "confidence_bias": 0.0686955797737544, |
| "top_bins": { |
| "0.9": { |
| "n": 27, |
| "errors": 2, |
| "error_rate": 0.07407407407407407 |
| }, |
| "0.95": { |
| "n": 26, |
| "errors": 2, |
| "error_rate": 0.07692307692307693 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 2, |
| "error_rate": 0.125 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 0.8888888888888888, |
| "confidence_cutoff": 0.9891438506037644 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9285714285714286, |
| "confidence_cutoff": 0.8757747622322081 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 35, |
| "nll": 0.3789301936796312, |
| "acc": 0.8571428571428571, |
| "ece": 0.11430695728032647, |
| "brier": 0.23203565927974715, |
| "mean_conf": 0.8718695845973715, |
| "confident_error_rate": 0.05714285714285714, |
| "coverage_at_0_9": 0.6857142857142857, |
| "accuracy_at_0_9": 0.9166666666666666, |
| "coverage_at_5pct_error": 0.2571428571428571, |
| "coverage_at_1pct_error": 0.2571428571428571, |
| "aurc": 0.08024957297121005, |
| "error_rate_at_0_9": 0.08333333333333333, |
| "confidence_bias": 0.014726727454514421, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 2, |
| "error_rate": 0.08333333333333333 |
| }, |
| "0.95": { |
| "n": 9, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5142857142857142, |
| "accuracy": 0.8888888888888888, |
| "confidence_cutoff": 0.9360109691977325 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9285714285714286, |
| "confidence_cutoff": 0.7615630826924717 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 0, |
| "correct": 0, |
| "recall": null |
| }, |
| "permitted": { |
| "n": 35, |
| "correct": 30, |
| "recall": 0.8571428571428571 |
| } |
| }, |
| "threshold": 0.8449815374033091, |
| "n": 35, |
| "positives": 0, |
| "negatives": 35, |
| "confusion": { |
| "tp": 0, |
| "fp": 2, |
| "tn": 33, |
| "fn": 0 |
| }, |
| "recall": null, |
| "false_alert_rate": 0.05714285714285714, |
| "precision": 0.0, |
| "threshold_accuracy": 0.9428571428571428 |
| } |
| }, |
| "paired_threshold_intervals": { |
| "groups": 35, |
| "samples": 2000, |
| "seed": 20260927, |
| "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed", |
| "metrics": { |
| "recall": { |
| "parent": { |
| "value": null, |
| "ci95": null, |
| "valid_resamples": 0 |
| }, |
| "candidate": { |
| "value": null, |
| "ci95": null, |
| "valid_resamples": 0 |
| }, |
| "candidate_minus_parent": { |
| "value": null, |
| "ci95": null, |
| "valid_resamples": 0 |
| } |
| }, |
| "false_alert_rate": { |
| "parent": { |
| "value": 0.08571428571428572, |
| "ci95": [ |
| 0.0, |
| 0.2 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.05714285714285714, |
| "ci95": [ |
| 0.0, |
| 0.14285714285714285 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": -0.028571428571428574, |
| "ci95": [ |
| -0.08571428571428572, |
| 0.0 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "precision": { |
| "parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 1918 |
| }, |
| "candidate": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 1747 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ], |
| "valid_resamples": 1747 |
| } |
| }, |
| "threshold_accuracy": { |
| "parent": { |
| "value": 0.9142857142857143, |
| "ci95": [ |
| 0.8, |
| 1.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.9428571428571428, |
| "ci95": [ |
| 0.8571428571428571, |
| 1.0 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.02857142857142858, |
| "ci95": [ |
| 0.0, |
| 0.08571428571428574 |
| ], |
| "valid_resamples": 2000 |
| } |
| } |
| } |
| } |
| } |
| }, |
| "by_source": { |
| "sev_swarmtraces_response_policy": { |
| "always_permit_baseline": { |
| "n": 175, |
| "positives": 35, |
| "negatives": 140, |
| "confusion": { |
| "tp": 0, |
| "fp": 0, |
| "tn": 140, |
| "fn": 35 |
| }, |
| "recall": 0.0, |
| "false_alert_rate": 0.0, |
| "precision": null, |
| "threshold_accuracy": 0.8 |
| }, |
| "models": { |
| "parent": { |
| "questions": 175, |
| "groups": 35, |
| "raw": { |
| "n": 175, |
| "nll": 0.795502582803619, |
| "acc": 0.7257142857142858, |
| "ece": 0.1747005023605082, |
| "brier": 0.4505153855550131, |
| "mean_conf": 0.8840572965356658, |
| "confident_error_rate": 0.13714285714285715, |
| "coverage_at_0_9": 0.6457142857142857, |
| "accuracy_at_0_9": 0.7876106194690266, |
| "coverage_at_5pct_error": 0.02857142857142857, |
| "coverage_at_1pct_error": 0.02857142857142857, |
| "aurc": 0.1992292401228163, |
| "error_rate_at_0_9": 0.21238938053097345, |
| "confidence_bias": 0.15834301082138003, |
| "top_bins": { |
| "0.9": { |
| "n": 113, |
| "errors": 24, |
| "error_rate": 0.21238938053097345 |
| }, |
| "0.95": { |
| "n": 84, |
| "errors": 18, |
| "error_rate": 0.21428571428571427 |
| }, |
| "0.99": { |
| "n": 26, |
| "errors": 2, |
| "error_rate": 0.07692307692307693 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.7613636363636364, |
| "confidence_cutoff": 0.9457634807080788 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.7938824081649869 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 175, |
| "nll": 0.6260474644745716, |
| "acc": 0.7257142857142858, |
| "ece": 0.10761147298493057, |
| "brier": 0.4111063010906543, |
| "mean_conf": 0.8140932667241441, |
| "confident_error_rate": 0.06857142857142857, |
| "coverage_at_0_9": 0.35428571428571426, |
| "accuracy_at_0_9": 0.8064516129032258, |
| "coverage_at_5pct_error": 0.02857142857142857, |
| "coverage_at_1pct_error": 0.02857142857142857, |
| "aurc": 0.1992292401228163, |
| "error_rate_at_0_9": 0.1935483870967742, |
| "confidence_bias": 0.0883789810098583, |
| "top_bins": { |
| "0.9": { |
| "n": 62, |
| "errors": 12, |
| "error_rate": 0.1935483870967742 |
| }, |
| "0.95": { |
| "n": 21, |
| "errors": 2, |
| "error_rate": 0.09523809523809523 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.7613636363636364, |
| "confidence_cutoff": 0.8531718492565429 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.6963743924447515 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 35, |
| "correct": 29, |
| "recall": 0.8285714285714286 |
| }, |
| "permitted": { |
| "n": 140, |
| "correct": 98, |
| "recall": 0.7 |
| } |
| }, |
| "threshold": 0.8875371629397495, |
| "n": 175, |
| "positives": 35, |
| "negatives": 140, |
| "confusion": { |
| "tp": 14, |
| "fp": 12, |
| "tn": 128, |
| "fn": 21 |
| }, |
| "recall": 0.4, |
| "false_alert_rate": 0.08571428571428572, |
| "precision": 0.5384615384615384, |
| "threshold_accuracy": 0.8114285714285714 |
| }, |
| "candidate": { |
| "questions": 175, |
| "groups": 35, |
| "raw": { |
| "n": 175, |
| "nll": 0.49844721765472333, |
| "acc": 0.84, |
| "ece": 0.11735638435435483, |
| "brier": 0.269672286103042, |
| "mean_conf": 0.9559888432162478, |
| "confident_error_rate": 0.10285714285714286, |
| "coverage_at_0_9": 0.8742857142857143, |
| "accuracy_at_0_9": 0.8823529411764706, |
| "coverage_at_5pct_error": 0.6114285714285714, |
| "coverage_at_1pct_error": 0.44, |
| "aurc": 0.044632779737602504, |
| "error_rate_at_0_9": 0.11764705882352941, |
| "confidence_bias": 0.1159888432162478, |
| "top_bins": { |
| "0.9": { |
| "n": 153, |
| "errors": 18, |
| "error_rate": 0.11764705882352941 |
| }, |
| "0.95": { |
| "n": 140, |
| "errors": 14, |
| "error_rate": 0.1 |
| }, |
| "0.99": { |
| "n": 92, |
| "errors": 3, |
| "error_rate": 0.03260869565217391 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.9772727272727273, |
| "confidence_cutoff": 0.9904485398763483 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9516673073011176 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 175, |
| "nll": 0.37433095036445935, |
| "acc": 0.84, |
| "ece": 0.06636288221880407, |
| "brier": 0.23938834727626476, |
| "mean_conf": 0.9048661883246669, |
| "confident_error_rate": 0.05142857142857143, |
| "coverage_at_0_9": 0.7142857142857143, |
| "accuracy_at_0_9": 0.928, |
| "coverage_at_5pct_error": 0.6114285714285714, |
| "coverage_at_1pct_error": 0.44, |
| "aurc": 0.044632779737602504, |
| "error_rate_at_0_9": 0.072, |
| "confidence_bias": 0.06486618832466695, |
| "top_bins": { |
| "0.9": { |
| "n": 125, |
| "errors": 9, |
| "error_rate": 0.072 |
| }, |
| "0.95": { |
| "n": 77, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 14, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.9772727272727273, |
| "confidence_cutoff": 0.9404661687170754 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.8547040294961724 |
| } |
| } |
| }, |
| "argmax_classes": { |
| "requires_approval": { |
| "n": 35, |
| "correct": 35, |
| "recall": 1.0 |
| }, |
| "permitted": { |
| "n": 140, |
| "correct": 112, |
| "recall": 0.8 |
| } |
| }, |
| "threshold": 0.8449815374033091, |
| "n": 175, |
| "positives": 35, |
| "negatives": 140, |
| "confusion": { |
| "tp": 25, |
| "fp": 14, |
| "tn": 126, |
| "fn": 10 |
| }, |
| "recall": 0.7142857142857143, |
| "false_alert_rate": 0.1, |
| "precision": 0.6410256410256411, |
| "threshold_accuracy": 0.8628571428571429 |
| } |
| }, |
| "paired_threshold_intervals": { |
| "groups": 35, |
| "samples": 2000, |
| "seed": 20260927, |
| "unit": "whole source-closure group, shared paired draw; calibration thresholds fixed", |
| "metrics": { |
| "recall": { |
| "parent": { |
| "value": 0.4, |
| "ci95": [ |
| 0.22857142857142856, |
| 0.5714285714285714 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.7142857142857143, |
| "ci95": [ |
| 0.5714285714285714, |
| 0.8571428571428571 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.3142857142857143, |
| "ci95": [ |
| 0.17142857142857143, |
| 0.4857142857142857 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "false_alert_rate": { |
| "parent": { |
| "value": 0.08571428571428572, |
| "ci95": [ |
| 0.05, |
| 0.12857142857142856 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.1, |
| "ci95": [ |
| 0.06428571428571428, |
| 0.14285714285714285 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.01428571428571429, |
| "ci95": [ |
| -0.014285714285714277, |
| 0.04285714285714286 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "precision": { |
| "parent": { |
| "value": 0.5384615384615384, |
| "ci95": [ |
| 0.3665909090909091, |
| 0.70979262672811 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.6410256410256411, |
| "ci95": [ |
| 0.55, |
| 0.7435897435897436 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.10256410256410264, |
| "ci95": [ |
| 0.005701872682569551, |
| 0.21591595300834426 |
| ], |
| "valid_resamples": 2000 |
| } |
| }, |
| "threshold_accuracy": { |
| "parent": { |
| "value": 0.8114285714285714, |
| "ci95": [ |
| 0.7655714285714286, |
| 0.8628571428571429 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate": { |
| "value": 0.8628571428571429, |
| "ci95": [ |
| 0.8228571428571428, |
| 0.9028571428571428 |
| ], |
| "valid_resamples": 2000 |
| }, |
| "candidate_minus_parent": { |
| "value": 0.05142857142857149, |
| "ci95": [ |
| 0.02285714285714291, |
| 0.08000000000000007 |
| ], |
| "valid_resamples": 2000 |
| } |
| } |
| } |
| } |
| } |
| }, |
| "retained_development_by_source": { |
| "parent": { |
| "excytin_file": { |
| "questions": 80, |
| "groups": 1, |
| "raw": { |
| "n": 80, |
| "nll": 0.0001342531538525746, |
| "acc": 1.0, |
| "ece": 0.0001340575450615722, |
| "brier": 7.773831743030022e-07, |
| "mean_conf": 0.9998659424549384, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0001340575450615722, |
| "top_bins": { |
| "0.9": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999710759158646 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999066849313115 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 80, |
| "nll": 0.0030639977354718724, |
| "acc": 1.0, |
| "ece": 0.0030486898704090715, |
| "brier": 5.723205099455454e-05, |
| "mean_conf": 0.9969513101295909, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0030486898704090715, |
| "top_bins": { |
| "0.9": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 79, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9980559839388836 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9963993201639499 |
| } |
| } |
| } |
| }, |
| "excytin_parent_join": { |
| "questions": 78, |
| "groups": 1, |
| "raw": { |
| "n": 78, |
| "nll": 0.0019727127979591126, |
| "acc": 1.0, |
| "ece": 0.0019615703387183547, |
| "brier": 4.023721192309001e-05, |
| "mean_conf": 0.9980384296612816, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0019615703387183547, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 78, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 76, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9996318703080018 |
| }, |
| "0.8": { |
| "coverage": 0.8076923076923077, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9974582021431176 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 78, |
| "nll": 0.01903091706838521, |
| "acc": 1.0, |
| "ece": 0.018568486423778917, |
| "brier": 0.0013900388516520708, |
| "mean_conf": 0.9814315135762209, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9871794871794872, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.018568486423779063, |
| "top_bins": { |
| "0.9": { |
| "n": 77, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 69, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 46, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9916928553567559 |
| }, |
| "0.8": { |
| "coverage": 0.8076923076923077, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9693893850709232 |
| } |
| } |
| } |
| }, |
| "excytin_process": { |
| "questions": 120, |
| "groups": 1, |
| "raw": { |
| "n": 120, |
| "nll": 0.0005180266626209783, |
| "acc": 1.0, |
| "ece": 0.0005174316239614507, |
| "brier": 2.3119235778779676e-06, |
| "mean_conf": 0.9994825683760385, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0005174316239614507, |
| "top_bins": { |
| "0.9": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999134057097643 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.99901689132107 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 120, |
| "nll": 0.007587462315793864, |
| "acc": 1.0, |
| "ece": 0.007523761830370401, |
| "brier": 0.00023474191911792246, |
| "mean_conf": 0.9924762381696296, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.007523761830370401, |
| "top_bins": { |
| "0.9": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 83, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9964206554980828 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9859276562719884 |
| } |
| } |
| } |
| }, |
| "excytin_signin": { |
| "questions": 120, |
| "groups": 1, |
| "raw": { |
| "n": 120, |
| "nll": 0.0013203461429938826, |
| "acc": 1.0, |
| "ece": 0.0012737143193113606, |
| "brier": 0.00013106658103070147, |
| "mean_conf": 0.9987262856806887, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9916666666666667, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0012737143193113365, |
| "top_bins": { |
| "0.9": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998413534198065 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9996110203225037 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 120, |
| "nll": 0.009411479962491658, |
| "acc": 1.0, |
| "ece": 0.00903361007075822, |
| "brier": 0.0010010974637190823, |
| "mean_conf": 0.9909663899292418, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9916666666666667, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.009033610070758202, |
| "top_bins": { |
| "0.9": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 105, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9950297195670218 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9914608088731087 |
| } |
| } |
| } |
| }, |
| "sev_guide_detector": { |
| "questions": 1064, |
| "groups": 114, |
| "raw": { |
| "n": 1064, |
| "nll": 0.8276776090882748, |
| "acc": 0.599624060150376, |
| "ece": 0.10621007339433168, |
| "brier": 0.48019952548397726, |
| "mean_conf": 0.7041447618951532, |
| "confident_error_rate": 0.012218045112781954, |
| "coverage_at_0_9": 0.32706766917293234, |
| "accuracy_at_0_9": 0.9626436781609196, |
| "coverage_at_5pct_error": 0.3524436090225564, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.19304798592127517, |
| "error_rate_at_0_9": 0.03735632183908046, |
| "confidence_bias": 0.10452070174477723, |
| "top_bins": { |
| "0.9": { |
| "n": 348, |
| "errors": 13, |
| "error_rate": 0.03735632183908046 |
| }, |
| "0.95": { |
| "n": 327, |
| "errors": 13, |
| "error_rate": 0.039755351681957186 |
| }, |
| "0.99": { |
| "n": 319, |
| "errors": 12, |
| "error_rate": 0.03761755485893417 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5075187969924813, |
| "accuracy": 0.8074074074074075, |
| "confidence_cutoff": 0.6273336504002459 |
| }, |
| "0.8": { |
| "coverage": 0.8016917293233082, |
| "accuracy": 0.6518171160609613, |
| "confidence_cutoff": 0.48816800774395525 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 1064, |
| "nll": 0.785071927880167, |
| "acc": 0.599624060150376, |
| "ece": 0.05266334727942813, |
| "brier": 0.46075730837789786, |
| "mean_conf": 0.6304966279496853, |
| "confident_error_rate": 0.011278195488721804, |
| "coverage_at_0_9": 0.3007518796992481, |
| "accuracy_at_0_9": 0.9625, |
| "coverage_at_5pct_error": 0.3524436090225564, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.19295849854549843, |
| "error_rate_at_0_9": 0.0375, |
| "confidence_bias": 0.03087256779930936, |
| "top_bins": { |
| "0.9": { |
| "n": 320, |
| "errors": 12, |
| "error_rate": 0.0375 |
| }, |
| "0.95": { |
| "n": 213, |
| "errors": 11, |
| "error_rate": 0.051643192488262914 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5075187969924813, |
| "accuracy": 0.8074074074074075, |
| "confidence_cutoff": 0.5144221593760727 |
| }, |
| "0.8": { |
| "coverage": 0.8016917293233082, |
| "accuracy": 0.6553341148886284, |
| "confidence_cutoff": 0.4322968853489006 |
| } |
| } |
| } |
| }, |
| "sev_guide_triage": { |
| "questions": 231, |
| "groups": 23, |
| "raw": { |
| "n": 231, |
| "nll": 0.8563265185203435, |
| "acc": 0.5627705627705628, |
| "ece": 0.1895007716411903, |
| "brier": 0.5255142828500304, |
| "mean_conf": 0.7444269666893102, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.35064935064935066, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.36363636363636365, |
| "coverage_at_1pct_error": 0.36363636363636365, |
| "aurc": 0.22468166386345326, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.18165640391874738, |
| "top_bins": { |
| "0.9": { |
| "n": 81, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 79, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5021645021645021, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.7894512944300596 |
| }, |
| "0.8": { |
| "coverage": 0.8095238095238095, |
| "accuracy": 0.5775401069518716, |
| "confidence_cutoff": 0.49009219177135965 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 231, |
| "nll": 0.7793526965750157, |
| "acc": 0.5627705627705628, |
| "ece": 0.1664752401619534, |
| "brier": 0.4718856823661165, |
| "mean_conf": 0.6653812407670776, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.3463203463203463, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.36363636363636365, |
| "coverage_at_1pct_error": 0.36363636363636365, |
| "aurc": 0.22610167628363687, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.1026106779965148, |
| "top_bins": { |
| "0.9": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 14, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5021645021645021, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.6343620681380909 |
| }, |
| "0.8": { |
| "coverage": 0.8008658008658008, |
| "accuracy": 0.5675675675675675, |
| "confidence_cutoff": 0.4381830283799813 |
| } |
| } |
| } |
| }, |
| "sev_swarmtraces_static": { |
| "questions": 354, |
| "groups": 176, |
| "raw": { |
| "n": 354, |
| "nll": 0.06671770193368179, |
| "acc": 0.9745762711864406, |
| "ece": 0.008417033573491902, |
| "brier": 0.03589778460324393, |
| "mean_conf": 0.9740214622331307, |
| "confident_error_rate": 0.002824858757062147, |
| "coverage_at_0_9": 0.9322033898305084, |
| "accuracy_at_0_9": 0.996969696969697, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9745762711864406, |
| "aurc": 0.0009317569638462815, |
| "error_rate_at_0_9": 0.0030303030303030303, |
| "confidence_bias": -0.0005548089533099487, |
| "top_bins": { |
| "0.9": { |
| "n": 330, |
| "errors": 1, |
| "error_rate": 0.0030303030303030303 |
| }, |
| "0.95": { |
| "n": 317, |
| "errors": 1, |
| "error_rate": 0.0031545741324921135 |
| }, |
| "0.99": { |
| "n": 292, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998280676315141 |
| }, |
| "0.8": { |
| "coverage": 0.8022598870056498, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9930714713128618 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 354, |
| "nll": 0.08603305952328294, |
| "acc": 0.9745762711864406, |
| "ece": 0.031999170395289905, |
| "brier": 0.03970483443698841, |
| "mean_conf": 0.9467552592594386, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.8615819209039548, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9745762711864406, |
| "aurc": 0.0009516834613723162, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.027821011927002037, |
| "top_bins": { |
| "0.9": { |
| "n": 305, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 277, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 194, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9929179700515357 |
| }, |
| "0.8": { |
| "coverage": 0.8022598870056498, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9423198026275108 |
| } |
| } |
| } |
| }, |
| "sev_sysmon_native_file": { |
| "questions": 62, |
| "groups": 4, |
| "raw": { |
| "n": 62, |
| "nll": 0.010792211612586852, |
| "acc": 1.0, |
| "ece": 0.009954006393107204, |
| "brier": 0.002287040487090685, |
| "mean_conf": 0.9900459936068928, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9838709677419355, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.009954006393107173, |
| "top_bins": { |
| "0.9": { |
| "n": 61, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 59, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 54, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9986117420701618 |
| }, |
| "0.8": { |
| "coverage": 0.8064516129032258, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9936233886125292 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 62, |
| "nll": 0.04080521144437412, |
| "acc": 1.0, |
| "ece": 0.03730996348948351, |
| "brier": 0.008675036111100168, |
| "mean_conf": 0.9626900365105165, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9516129032258065, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.037309963489483455, |
| "top_bins": { |
| "0.9": { |
| "n": 59, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 48, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 23, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9803392906572079 |
| }, |
| "0.8": { |
| "coverage": 0.8064516129032258, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9477741924481837 |
| } |
| } |
| } |
| }, |
| "sev_sysmon_native_join": { |
| "questions": 120, |
| "groups": 3, |
| "raw": { |
| "n": 120, |
| "nll": 0.0008810538663341702, |
| "acc": 1.0, |
| "ece": 0.0008800625080441771, |
| "brier": 3.5216739059086607e-06, |
| "mean_conf": 0.9991199374919558, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0008800625080441771, |
| "top_bins": { |
| "0.9": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9994848628990232 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.998641259167216 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 120, |
| "nll": 0.013824573239669817, |
| "acc": 1.0, |
| "ece": 0.013685808798396426, |
| "brier": 0.00044603592410257366, |
| "mean_conf": 0.9863141912016036, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.013685808798396426, |
| "top_bins": { |
| "0.9": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 118, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 55, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9888123032356468 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.980357236597309 |
| } |
| } |
| } |
| }, |
| "sev_sysmon_native_other_effect": { |
| "questions": 88, |
| "groups": 4, |
| "raw": { |
| "n": 88, |
| "nll": 0.007295006715728625, |
| "acc": 1.0, |
| "ece": 0.006888792886758961, |
| "brier": 0.001385599246512425, |
| "mean_conf": 0.9931112071132411, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9886363636363636, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.006888792886758921, |
| "top_bins": { |
| "0.9": { |
| "n": 87, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 86, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 79, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9993634031772723 |
| }, |
| "0.8": { |
| "coverage": 0.8068181818181818, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9952397949435474 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 88, |
| "nll": 0.03669008514897713, |
| "acc": 1.0, |
| "ece": 0.03458520458981973, |
| "brier": 0.006053607090618847, |
| "mean_conf": 0.9654147954101805, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9431818181818182, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.03458520458981951, |
| "top_bins": { |
| "0.9": { |
| "n": 83, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 67, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 41, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9865664502206583 |
| }, |
| "0.8": { |
| "coverage": 0.8068181818181818, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9454838788991825 |
| } |
| } |
| } |
| }, |
| "sev_sysmon_native_process": { |
| "questions": 74, |
| "groups": 4, |
| "raw": { |
| "n": 74, |
| "nll": 0.005360095133334598, |
| "acc": 1.0, |
| "ece": 0.005183972707978367, |
| "brier": 0.0006452896028548641, |
| "mean_conf": 0.9948160272920217, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9864864864864865, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.005183972707978324, |
| "top_bins": { |
| "0.9": { |
| "n": 73, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 72, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 67, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998754065974452 |
| }, |
| "0.8": { |
| "coverage": 0.8108108108108109, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9964080071736937 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 74, |
| "nll": 0.02430026482825407, |
| "acc": 1.0, |
| "ece": 0.023069255552173623, |
| "brier": 0.004038757822471681, |
| "mean_conf": 0.9769307444478267, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9324324324324325, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.023069255552173318, |
| "top_bins": { |
| "0.9": { |
| "n": 69, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 65, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 43, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9959686665627235 |
| }, |
| "0.8": { |
| "coverage": 0.8108108108108109, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9671912008530428 |
| } |
| } |
| } |
| }, |
| "sev_sysmon_native_registry": { |
| "questions": 66, |
| "groups": 4, |
| "raw": { |
| "n": 66, |
| "nll": 0.02506239179799794, |
| "acc": 0.9848484848484849, |
| "ece": 0.005781042813278618, |
| "brier": 0.017230444731313608, |
| "mean_conf": 0.9868495288218273, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9696969696969697, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9696969696969697, |
| "aurc": 0.00046266864448682633, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.0020010439733424423, |
| "top_bins": { |
| "0.9": { |
| "n": 64, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 64, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 61, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9996958591947809 |
| }, |
| "0.8": { |
| "coverage": 0.803030303030303, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9967594448763499 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 66, |
| "nll": 0.03908316195438624, |
| "acc": 0.9848484848484849, |
| "ece": 0.018983776220710902, |
| "brier": 0.01748193378444616, |
| "mean_conf": 0.9706927611865057, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9545454545454546, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9696969696969697, |
| "aurc": 0.00046266864448682633, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.014155723661979147, |
| "top_bins": { |
| "0.9": { |
| "n": 63, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 58, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 36, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9919306919122572 |
| }, |
| "0.8": { |
| "coverage": 0.803030303030303, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9688350009585398 |
| } |
| } |
| } |
| } |
| }, |
| "candidate": { |
| "excytin_file": { |
| "questions": 80, |
| "groups": 1, |
| "raw": { |
| "n": 80, |
| "nll": 8.416562133899781e-05, |
| "acc": 1.0, |
| "ece": 8.407504508700647e-05, |
| "brier": 3.608306194877327e-07, |
| "mean_conf": 0.999915924954913, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -8.407504508700647e-05, |
| "top_bins": { |
| "0.9": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999870465451609 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999445146924221 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 80, |
| "nll": 0.0025800477441721577, |
| "acc": 1.0, |
| "ece": 0.0025682388226171415, |
| "brier": 4.4624375778219304e-05, |
| "mean_conf": 0.9974317611773829, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0025682388226171415, |
| "top_bins": { |
| "0.9": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 79, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9984996790217291 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9968816753727368 |
| } |
| } |
| } |
| }, |
| "excytin_parent_join": { |
| "questions": 78, |
| "groups": 1, |
| "raw": { |
| "n": 78, |
| "nll": 0.0010931532029286512, |
| "acc": 1.0, |
| "ece": 0.0010890559164937752, |
| "brier": 1.524968968159761e-05, |
| "mean_conf": 0.9989109440835062, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0010890559164937752, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 78, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 77, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997808486535813 |
| }, |
| "0.8": { |
| "coverage": 0.8076923076923077, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9985521003494622 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 78, |
| "nll": 0.014932533018017379, |
| "acc": 1.0, |
| "ece": 0.014655484687648102, |
| "brier": 0.0008621856358382366, |
| "mean_conf": 0.9853445153123519, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.014655484687648102, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 73, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 49, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9924833351572758 |
| }, |
| "0.8": { |
| "coverage": 0.8076923076923077, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9749372136312703 |
| } |
| } |
| } |
| }, |
| "excytin_process": { |
| "questions": 120, |
| "groups": 1, |
| "raw": { |
| "n": 120, |
| "nll": 0.00038924968205708544, |
| "acc": 1.0, |
| "ece": 0.00038892368944332123, |
| "brier": 1.2813112890567744e-06, |
| "mean_conf": 0.9996110763105567, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.00038892368944332123, |
| "top_bins": { |
| "0.9": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999547557532524 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9992749850755753 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 120, |
| "nll": 0.007136704365819422, |
| "acc": 1.0, |
| "ece": 0.007078397430680172, |
| "brier": 0.00021856930435733632, |
| "mean_conf": 0.9929216025693198, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.007078397430680172, |
| "top_bins": { |
| "0.9": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 84, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9971072222116408 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9865275469490526 |
| } |
| } |
| } |
| }, |
| "excytin_signin": { |
| "questions": 120, |
| "groups": 1, |
| "raw": { |
| "n": 120, |
| "nll": 0.0009466221977245799, |
| "acc": 1.0, |
| "ece": 0.0009222842581878554, |
| "brier": 7.013154556645113e-05, |
| "mean_conf": 0.9990777157418121, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0009222842581878554, |
| "top_bins": { |
| "0.9": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999016374480455 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997216681533668 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 120, |
| "nll": 0.00847100086283192, |
| "acc": 1.0, |
| "ece": 0.008170507143524745, |
| "brier": 0.0008193865392318428, |
| "mean_conf": 0.9918294928564755, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9916666666666667, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.008170507143524519, |
| "top_bins": { |
| "0.9": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 105, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9955464217668417 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9921417222813483 |
| } |
| } |
| } |
| }, |
| "sev_guide_detector": { |
| "questions": 1064, |
| "groups": 114, |
| "raw": { |
| "n": 1064, |
| "nll": 0.8605571101399087, |
| "acc": 0.6005639097744361, |
| "ece": 0.14734385206430384, |
| "brier": 0.5008109312564021, |
| "mean_conf": 0.7382136284198534, |
| "confident_error_rate": 0.017857142857142856, |
| "coverage_at_0_9": 0.33270676691729323, |
| "accuracy_at_0_9": 0.9463276836158192, |
| "coverage_at_5pct_error": 0.3082706766917293, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.19350209200158913, |
| "error_rate_at_0_9": 0.05367231638418079, |
| "confidence_bias": 0.1376497186454173, |
| "top_bins": { |
| "0.9": { |
| "n": 354, |
| "errors": 19, |
| "error_rate": 0.05367231638418079 |
| }, |
| "0.95": { |
| "n": 327, |
| "errors": 13, |
| "error_rate": 0.039755351681957186 |
| }, |
| "0.99": { |
| "n": 319, |
| "errors": 12, |
| "error_rate": 0.03761755485893417 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028195488721805, |
| "accuracy": 0.8205607476635514, |
| "confidence_cutoff": 0.7059991149306561 |
| }, |
| "0.8": { |
| "coverage": 0.8035714285714286, |
| "accuracy": 0.6596491228070176, |
| "confidence_cutoff": 0.5473316703985754 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 1064, |
| "nll": 0.7966730081647482, |
| "acc": 0.6005639097744361, |
| "ece": 0.06373435814748656, |
| "brier": 0.46843654552918773, |
| "mean_conf": 0.6455031255616036, |
| "confident_error_rate": 0.011278195488721804, |
| "coverage_at_0_9": 0.299812030075188, |
| "accuracy_at_0_9": 0.9623824451410659, |
| "coverage_at_5pct_error": 0.3082706766917293, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.1938629191063446, |
| "error_rate_at_0_9": 0.03761755485893417, |
| "confidence_bias": 0.04493921578716753, |
| "top_bins": { |
| "0.9": { |
| "n": 319, |
| "errors": 12, |
| "error_rate": 0.03761755485893417 |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028195488721805, |
| "accuracy": 0.8205607476635514, |
| "confidence_cutoff": 0.5612814391210313 |
| }, |
| "0.8": { |
| "coverage": 0.8007518796992481, |
| "accuracy": 0.6619718309859155, |
| "confidence_cutoff": 0.4656043556130704 |
| } |
| } |
| } |
| }, |
| "sev_guide_triage": { |
| "questions": 231, |
| "groups": 23, |
| "raw": { |
| "n": 231, |
| "nll": 0.8396428451107271, |
| "acc": 0.5670995670995671, |
| "ece": 0.1889188281653689, |
| "brier": 0.5162927524049745, |
| "mean_conf": 0.7421494516023849, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.3593073593073593, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.3722943722943723, |
| "coverage_at_1pct_error": 0.36363636363636365, |
| "aurc": 0.20987091476964936, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.17504988450281778, |
| "top_bins": { |
| "0.9": { |
| "n": 83, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 14, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5627705627705628, |
| "accuracy": 0.7153846153846154, |
| "confidence_cutoff": 0.7564145922660828 |
| }, |
| "0.8": { |
| "coverage": 0.8051948051948052, |
| "accuracy": 0.5806451612903226, |
| "confidence_cutoff": 0.51023405030021 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 231, |
| "nll": 0.7761249864817235, |
| "acc": 0.5670995670995671, |
| "ece": 0.16797778410789055, |
| "brier": 0.46407372894941223, |
| "mean_conf": 0.6510059283006658, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.341991341991342, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.37662337662337664, |
| "coverage_at_1pct_error": 0.36363636363636365, |
| "aurc": 0.20742662880964652, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.08390636120109873, |
| "top_bins": { |
| "0.9": { |
| "n": 79, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 14, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5584415584415584, |
| "accuracy": 0.7209302325581395, |
| "confidence_cutoff": 0.6016800953517527 |
| }, |
| "0.8": { |
| "coverage": 0.8095238095238095, |
| "accuracy": 0.5989304812834224, |
| "confidence_cutoff": 0.4481415540755688 |
| } |
| } |
| } |
| }, |
| "sev_swarmtraces_static": { |
| "questions": 354, |
| "groups": 176, |
| "raw": { |
| "n": 354, |
| "nll": 0.04944532768977841, |
| "acc": 0.980225988700565, |
| "ece": 0.01487544035693287, |
| "brier": 0.024608535342050798, |
| "mean_conf": 0.9802643779708107, |
| "confident_error_rate": 0.002824858757062147, |
| "coverage_at_0_9": 0.9463276836158192, |
| "accuracy_at_0_9": 0.9970149253731343, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9858757062146892, |
| "aurc": 0.0005032049402250012, |
| "error_rate_at_0_9": 0.0029850746268656717, |
| "confidence_bias": 3.8389270245686724e-05, |
| "top_bins": { |
| "0.9": { |
| "n": 335, |
| "errors": 1, |
| "error_rate": 0.0029850746268656717 |
| }, |
| "0.95": { |
| "n": 330, |
| "errors": 1, |
| "error_rate": 0.0030303030303030303 |
| }, |
| "0.99": { |
| "n": 308, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999276209511099 |
| }, |
| "0.8": { |
| "coverage": 0.8022598870056498, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.997796467038603 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 354, |
| "nll": 0.06588860080731399, |
| "acc": 0.980225988700565, |
| "ece": 0.029025934670617953, |
| "brier": 0.028907916842541088, |
| "mean_conf": 0.9580211141684397, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.8898305084745762, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9887005649717514, |
| "aurc": 0.0005463830895640532, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.022204874532125274, |
| "top_bins": { |
| "0.9": { |
| "n": 315, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 297, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 221, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9952110216327251 |
| }, |
| "0.8": { |
| "coverage": 0.8022598870056498, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9673839577051934 |
| } |
| } |
| } |
| }, |
| "sev_sysmon_native_file": { |
| "questions": 62, |
| "groups": 4, |
| "raw": { |
| "n": 62, |
| "nll": 0.008428780795422771, |
| "acc": 1.0, |
| "ece": 0.007810434776248104, |
| "brier": 0.0017443201233063612, |
| "mean_conf": 0.9921895652237519, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9838709677419355, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0078104347762480986, |
| "top_bins": { |
| "0.9": { |
| "n": 61, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 59, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 57, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9991686260052061 |
| }, |
| "0.8": { |
| "coverage": 0.8064516129032258, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9962482122843107 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 62, |
| "nll": 0.03509983435601263, |
| "acc": 1.0, |
| "ece": 0.032160212545650484, |
| "brier": 0.007419485561600889, |
| "mean_conf": 0.9678397874543494, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9516129032258065, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0321602125456506, |
| "top_bins": { |
| "0.9": { |
| "n": 59, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 55, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 26, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9838915386931907 |
| }, |
| "0.8": { |
| "coverage": 0.8064516129032258, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9583052893470861 |
| } |
| } |
| } |
| }, |
| "sev_sysmon_native_join": { |
| "questions": 120, |
| "groups": 3, |
| "raw": { |
| "n": 120, |
| "nll": 0.00039767294618876087, |
| "acc": 1.0, |
| "ece": 0.0003974740966455892, |
| "brier": 7.310901229709924e-07, |
| "mean_conf": 0.9996025259033544, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0003974740966455892, |
| "top_bins": { |
| "0.9": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997660582574378 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9994606035447323 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 120, |
| "nll": 0.009879536093519296, |
| "acc": 1.0, |
| "ece": 0.009811175359927704, |
| "brier": 0.00022722202758135087, |
| "mean_conf": 0.9901888246400723, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.009811175359927704, |
| "top_bins": { |
| "0.9": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 67, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9917343480609009 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9855433361281362 |
| } |
| } |
| } |
| }, |
| "sev_sysmon_native_other_effect": { |
| "questions": 88, |
| "groups": 4, |
| "raw": { |
| "n": 88, |
| "nll": 0.005466714567741097, |
| "acc": 1.0, |
| "ece": 0.005048372154544871, |
| "brier": 0.0014126265728775101, |
| "mean_conf": 0.994951627845455, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9886363636363636, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.005048372154545011, |
| "top_bins": { |
| "0.9": { |
| "n": 87, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 86, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 83, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997677646417501 |
| }, |
| "0.8": { |
| "coverage": 0.8068181818181818, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9978176224669177 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 88, |
| "nll": 0.02789592785139411, |
| "acc": 1.0, |
| "ece": 0.026282565295571752, |
| "brier": 0.004783993285232044, |
| "mean_conf": 0.9737174347044284, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9545454545454546, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.026282565295571603, |
| "top_bins": { |
| "0.9": { |
| "n": 84, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 75, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 46, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9913758403771589 |
| }, |
| "0.8": { |
| "coverage": 0.8068181818181818, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9627441989296704 |
| } |
| } |
| } |
| }, |
| "sev_sysmon_native_process": { |
| "questions": 74, |
| "groups": 4, |
| "raw": { |
| "n": 74, |
| "nll": 0.004798899209615666, |
| "acc": 1.0, |
| "ece": 0.004644321057477101, |
| "brier": 0.0005716684001551928, |
| "mean_conf": 0.995355678942523, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9864864864864865, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.00464432105747703, |
| "top_bins": { |
| "0.9": { |
| "n": 73, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 72, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 68, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999268900168323 |
| }, |
| "0.8": { |
| "coverage": 0.8108108108108109, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9973453419434639 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 74, |
| "nll": 0.023528453108139967, |
| "acc": 1.0, |
| "ece": 0.022317812691888844, |
| "brier": 0.004027925361032504, |
| "mean_conf": 0.9776821873081113, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9324324324324325, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.022317812691888705, |
| "top_bins": { |
| "0.9": { |
| "n": 69, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 65, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 45, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9964760714345572 |
| }, |
| "0.8": { |
| "coverage": 0.8108108108108109, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9703686880072409 |
| } |
| } |
| } |
| }, |
| "sev_sysmon_native_registry": { |
| "questions": 66, |
| "groups": 4, |
| "raw": { |
| "n": 66, |
| "nll": 0.024015813931122083, |
| "acc": 0.9848484848484849, |
| "ece": 0.016913465896023366, |
| "brier": 0.01690246747159733, |
| "mean_conf": 0.9866786138448916, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9696969696969697, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9696969696969697, |
| "aurc": 0.00046266864448682633, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.0018301289964067768, |
| "top_bins": { |
| "0.9": { |
| "n": 64, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 64, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 62, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998656142719909 |
| }, |
| "0.8": { |
| "coverage": 0.803030303030303, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9982153954384291 |
| } |
| } |
| }, |
| "calibrated": { |
| "n": 66, |
| "nll": 0.03582391955947498, |
| "acc": 0.9848484848484849, |
| "ece": 0.01540577138447916, |
| "brier": 0.016990289728223678, |
| "mean_conf": 0.9730644954445979, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9696969696969697, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9696969696969697, |
| "aurc": 0.00046266864448682633, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.011783989403886919, |
| "top_bins": { |
| "0.9": { |
| "n": 64, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 59, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 40, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9940913990982823 |
| }, |
| "0.8": { |
| "coverage": 0.803030303030303, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.975504713794386 |
| } |
| } |
| } |
| } |
| } |
| }, |
| "policy": { |
| "positive": "Requires approval under the supplied policy", |
| "negative": "Permitted under the supplied policy", |
| "threshold_split": "calibration", |
| "temperature": "reuse saved calibration receipt, or benchmark inference temperature; no refit", |
| "population": "all five conditional policy questions per admitted source program", |
| "bootstrap": "percentile intervals conditional on the fixed calibration threshold; no threshold reselection" |
| }, |
| "limitations": [ |
| "Authored permissions and static callback dataflow, not historical maliciousness or execution evidence.", |
| "The selected conditional panel does not estimate deployment prevalence.", |
| "Actor origin is not measured. No locked test is read and this report does not promote a model.", |
| "The calibration false-alert budget is empirical, not a deployment guarantee.", |
| "Rare-operation rates are null without the relevant class; bootstrap draws without that class are omitted." |
| ] |
| } |
|
|
