| { |
| "objective": -0.17643950559609828, |
| "paired_flip": null, |
| "unknowable": null, |
| "clean": { |
| "n": 2632, |
| "nll": 0.4626010690091542, |
| "acc": 0.7868541033434651, |
| "ece": 0.0822959829967011, |
| "brier": 0.2695405506463609, |
| "mean_conf": 0.8650301490049586, |
| "confident_error_rate": 0.014437689969604863, |
| "coverage_at_0_9": 0.6565349544072948, |
| "accuracy_at_0_9": 0.9780092592592593, |
| "coverage_at_5pct_error": 0.6808510638297872, |
| "coverage_at_1pct_error": 0.5828267477203647, |
| "aurc": 0.04539486327850754, |
| "error_rate_at_0_9": 0.02199074074074074, |
| "confidence_bias": 0.0781760456614935, |
| "top_bins": { |
| "0.9": { |
| "n": 1728, |
| "errors": 38, |
| "error_rate": 0.02199074074074074 |
| }, |
| "0.95": { |
| "n": 1675, |
| "errors": 28, |
| "error_rate": 0.01671641791044776 |
| }, |
| "0.99": { |
| "n": 1519, |
| "errors": 15, |
| "error_rate": 0.009874917709019092 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5155775075987842, |
| "accuracy": 0.9918938835666913, |
| "confidence_cutoff": 0.9953955617749282 |
| }, |
| "0.8": { |
| "coverage": 0.8028115501519757, |
| "accuracy": 0.8920965451964032, |
| "confidence_cutoff": 0.658356020724118 |
| } |
| } |
| }, |
| "tasks": { |
| "excytin_file": { |
| "n": 80, |
| "nll": 8.416562133899781e-05, |
| "acc": 1.0, |
| "ece": 8.407504508700647e-05, |
| "brier": 3.608306194877327e-07, |
| "mean_conf": 0.999915924954913, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -8.407504508700647e-05, |
| "top_bins": { |
| "0.9": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999870465451609 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999445146924221 |
| } |
| } |
| }, |
| "excytin_parent_join": { |
| "n": 78, |
| "nll": 0.0010931532029286512, |
| "acc": 1.0, |
| "ece": 0.0010890559164937752, |
| "brier": 1.524968968159761e-05, |
| "mean_conf": 0.9989109440835062, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0010890559164937752, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 78, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 77, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997808486535813 |
| }, |
| "0.8": { |
| "coverage": 0.8076923076923077, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9985521003494622 |
| } |
| } |
| }, |
| "excytin_process": { |
| "n": 120, |
| "nll": 0.00038924968205708544, |
| "acc": 1.0, |
| "ece": 0.00038892368944332123, |
| "brier": 1.2813112890567744e-06, |
| "mean_conf": 0.9996110763105567, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.00038892368944332123, |
| "top_bins": { |
| "0.9": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999547557532524 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9992749850755753 |
| } |
| } |
| }, |
| "excytin_signin": { |
| "n": 120, |
| "nll": 0.0009466221977245799, |
| "acc": 1.0, |
| "ece": 0.0009222842581878554, |
| "brier": 7.013154556645113e-05, |
| "mean_conf": 0.9990777157418121, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0009222842581878554, |
| "top_bins": { |
| "0.9": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 119, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999016374480455 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997216681533668 |
| } |
| } |
| }, |
| "sev_guide_detector": { |
| "n": 1064, |
| "nll": 0.8605571101399087, |
| "acc": 0.6005639097744361, |
| "ece": 0.14734385206430384, |
| "brier": 0.5008109312564021, |
| "mean_conf": 0.7382136284198534, |
| "confident_error_rate": 0.017857142857142856, |
| "coverage_at_0_9": 0.33270676691729323, |
| "accuracy_at_0_9": 0.9463276836158192, |
| "coverage_at_5pct_error": 0.3082706766917293, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.19350209200158913, |
| "error_rate_at_0_9": 0.05367231638418079, |
| "confidence_bias": 0.1376497186454173, |
| "top_bins": { |
| "0.9": { |
| "n": 354, |
| "errors": 19, |
| "error_rate": 0.05367231638418079 |
| }, |
| "0.95": { |
| "n": 327, |
| "errors": 13, |
| "error_rate": 0.039755351681957186 |
| }, |
| "0.99": { |
| "n": 319, |
| "errors": 12, |
| "error_rate": 0.03761755485893417 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028195488721805, |
| "accuracy": 0.8205607476635514, |
| "confidence_cutoff": 0.7059991149306561 |
| }, |
| "0.8": { |
| "coverage": 0.8035714285714286, |
| "accuracy": 0.6596491228070176, |
| "confidence_cutoff": 0.5473316703985754 |
| } |
| } |
| }, |
| "sev_guide_triage": { |
| "n": 231, |
| "nll": 0.8396428451107271, |
| "acc": 0.5670995670995671, |
| "ece": 0.1889188281653689, |
| "brier": 0.5162927524049745, |
| "mean_conf": 0.7421494516023849, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.3593073593073593, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.3722943722943723, |
| "coverage_at_1pct_error": 0.36363636363636365, |
| "aurc": 0.20987091476964936, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.17504988450281778, |
| "top_bins": { |
| "0.9": { |
| "n": 83, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 80, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 14, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5627705627705628, |
| "accuracy": 0.7153846153846154, |
| "confidence_cutoff": 0.7564145922660828 |
| }, |
| "0.8": { |
| "coverage": 0.8051948051948052, |
| "accuracy": 0.5806451612903226, |
| "confidence_cutoff": 0.51023405030021 |
| } |
| } |
| }, |
| "sev_swarmtraces_response_policy": { |
| "n": 175, |
| "nll": 0.49844721765472333, |
| "acc": 0.84, |
| "ece": 0.1173563845984423, |
| "brier": 0.26967228692055817, |
| "mean_conf": 0.955988843320762, |
| "confident_error_rate": 0.10285714285714286, |
| "coverage_at_0_9": 0.8742857142857143, |
| "accuracy_at_0_9": 0.8823529411764706, |
| "coverage_at_5pct_error": 0.6114285714285714, |
| "coverage_at_1pct_error": 0.44, |
| "aurc": 0.044632779737602504, |
| "error_rate_at_0_9": 0.11764705882352941, |
| "confidence_bias": 0.11598884332076198, |
| "top_bins": { |
| "0.9": { |
| "n": 153, |
| "errors": 18, |
| "error_rate": 0.11764705882352941 |
| }, |
| "0.95": { |
| "n": 140, |
| "errors": 14, |
| "error_rate": 0.1 |
| }, |
| "0.99": { |
| "n": 92, |
| "errors": 3, |
| "error_rate": 0.03260869565217391 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5028571428571429, |
| "accuracy": 0.9772727272727273, |
| "confidence_cutoff": 0.9904485381783863 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9516673045977523 |
| } |
| } |
| }, |
| "sev_swarmtraces_static": { |
| "n": 354, |
| "nll": 0.04944532768977841, |
| "acc": 0.980225988700565, |
| "ece": 0.01487544035693287, |
| "brier": 0.024608535342050798, |
| "mean_conf": 0.9802643779708107, |
| "confident_error_rate": 0.002824858757062147, |
| "coverage_at_0_9": 0.9463276836158192, |
| "accuracy_at_0_9": 0.9970149253731343, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9858757062146892, |
| "aurc": 0.0005032049402250012, |
| "error_rate_at_0_9": 0.0029850746268656717, |
| "confidence_bias": 3.8389270245686724e-05, |
| "top_bins": { |
| "0.9": { |
| "n": 335, |
| "errors": 1, |
| "error_rate": 0.0029850746268656717 |
| }, |
| "0.95": { |
| "n": 330, |
| "errors": 1, |
| "error_rate": 0.0030303030303030303 |
| }, |
| "0.99": { |
| "n": 308, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999276209511099 |
| }, |
| "0.8": { |
| "coverage": 0.8022598870056498, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.997796467038603 |
| } |
| } |
| }, |
| "sev_sysmon_native_file": { |
| "n": 62, |
| "nll": 0.008428780795422771, |
| "acc": 1.0, |
| "ece": 0.007810434776248104, |
| "brier": 0.0017443201233063612, |
| "mean_conf": 0.9921895652237519, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9838709677419355, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0078104347762480986, |
| "top_bins": { |
| "0.9": { |
| "n": 61, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 59, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 57, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9991686260052061 |
| }, |
| "0.8": { |
| "coverage": 0.8064516129032258, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9962482122843107 |
| } |
| } |
| }, |
| "sev_sysmon_native_join": { |
| "n": 120, |
| "nll": 0.00039767294618876087, |
| "acc": 1.0, |
| "ece": 0.0003974740966455892, |
| "brier": 7.310901229709924e-07, |
| "mean_conf": 0.9996025259033544, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0003974740966455892, |
| "top_bins": { |
| "0.9": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 120, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997660582574378 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9994606035447323 |
| } |
| } |
| }, |
| "sev_sysmon_native_other_effect": { |
| "n": 88, |
| "nll": 0.005466714567741097, |
| "acc": 1.0, |
| "ece": 0.005048372154544871, |
| "brier": 0.0014126265728775101, |
| "mean_conf": 0.994951627845455, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9886363636363636, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.005048372154545011, |
| "top_bins": { |
| "0.9": { |
| "n": 87, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 86, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 83, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997677646417501 |
| }, |
| "0.8": { |
| "coverage": 0.8068181818181818, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9978176224669177 |
| } |
| } |
| }, |
| "sev_sysmon_native_process": { |
| "n": 74, |
| "nll": 0.004798899209615666, |
| "acc": 1.0, |
| "ece": 0.004644321057477101, |
| "brier": 0.0005716684001551928, |
| "mean_conf": 0.995355678942523, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9864864864864865, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.00464432105747703, |
| "top_bins": { |
| "0.9": { |
| "n": 73, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 72, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 68, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999268900168323 |
| }, |
| "0.8": { |
| "coverage": 0.8108108108108109, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9973453419434639 |
| } |
| } |
| }, |
| "sev_sysmon_native_registry": { |
| "n": 66, |
| "nll": 0.024015813931122083, |
| "acc": 0.9848484848484849, |
| "ece": 0.016913465896023366, |
| "brier": 0.01690246747159733, |
| "mean_conf": 0.9866786138448916, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9696969696969697, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9696969696969697, |
| "aurc": 0.00046266864448682633, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.0018301289964067768, |
| "top_bins": { |
| "0.9": { |
| "n": 64, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 64, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 62, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998656142719909 |
| }, |
| "0.8": { |
| "coverage": 0.803030303030303, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9982153954384291 |
| } |
| } |
| } |
| }, |
| "variants": { |
| "clean": { |
| "n": 2632, |
| "nll": 0.4626010690091542, |
| "acc": 0.7868541033434651, |
| "ece": 0.0822959829967011, |
| "brier": 0.2695405506463609, |
| "mean_conf": 0.8650301490049586, |
| "confident_error_rate": 0.014437689969604863, |
| "coverage_at_0_9": 0.6565349544072948, |
| "accuracy_at_0_9": 0.9780092592592593, |
| "coverage_at_5pct_error": 0.6808510638297872, |
| "coverage_at_1pct_error": 0.5828267477203647, |
| "aurc": 0.04539486327850754, |
| "error_rate_at_0_9": 0.02199074074074074, |
| "confidence_bias": 0.0781760456614935, |
| "top_bins": { |
| "0.9": { |
| "n": 1728, |
| "errors": 38, |
| "error_rate": 0.02199074074074074 |
| }, |
| "0.95": { |
| "n": 1675, |
| "errors": 28, |
| "error_rate": 0.01671641791044776 |
| }, |
| "0.99": { |
| "n": 1519, |
| "errors": 15, |
| "error_rate": 0.009874917709019092 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5155775075987842, |
| "accuracy": 0.9918938835666913, |
| "confidence_cutoff": 0.9953955617749282 |
| }, |
| "0.8": { |
| "coverage": 0.8028115501519757, |
| "accuracy": 0.8920965451964032, |
| "confidence_cutoff": 0.658356020724118 |
| } |
| } |
| } |
| }, |
| "heldout_tasks": {}, |
| "permutation": { |
| "n": 0, |
| "mean_max_delta": null, |
| "flip_rate": null |
| }, |
| "temperature": 1.6817928305074288, |
| "calibrated_clean": { |
| "n": 2632, |
| "nll": 0.42892992382965894, |
| "acc": 0.7868541033434651, |
| "ece": 0.04404789769068498, |
| "brier": 0.25086130169182097, |
| "mean_conf": 0.8094973007371603, |
| "confident_error_rate": 0.007978723404255319, |
| "coverage_at_0_9": 0.6196808510638298, |
| "accuracy_at_0_9": 0.9871244635193133, |
| "coverage_at_5pct_error": 0.6918693009118541, |
| "coverage_at_1pct_error": 0.5395136778115501, |
| "aurc": 0.045402363700726256, |
| "error_rate_at_0_9": 0.012875536480686695, |
| "confidence_bias": 0.022643197393695247, |
| "top_bins": { |
| "0.9": { |
| "n": 1631, |
| "errors": 21, |
| "error_rate": 0.012875536480686695 |
| }, |
| "0.95": { |
| "n": 1153, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 776, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5208966565349544, |
| "accuracy": 0.9912472647702407, |
| "confidence_cutoff": 0.9486799539412789 |
| }, |
| "0.8": { |
| "coverage": 0.8111702127659575, |
| "accuracy": 0.8861826697892272, |
| "confidence_cutoff": 0.52766162421842 |
| } |
| } |
| }, |
| "metric_policy": { |
| "version": 2, |
| "selective_ties": "whole_confidence_groups", |
| "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", |
| "aurc": "right-step integral over whole confidence groups", |
| "confident_error_rate": "high-confidence errors divided by all questions", |
| "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", |
| "nll": "exact from logits when recorded; otherwise from floored probabilities", |
| "nll_floor": 1e-09, |
| "renormalize_returned_probabilities": true, |
| "raw_sums_outside_1e_5": 0, |
| "returned_zeros": 0 |
| }, |
| "coverage": { |
| "requested_records": 1828, |
| "requested_questions": 2632, |
| "evaluated_records": 1828, |
| "evaluated_questions": 2632, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "latency_ms": { |
| "median": 54.25747899994349, |
| "p95": 161.14951469993457 |
| }, |
| "calibration": { |
| "inference_temperature": 1.0, |
| "additional_temperature": 1.6817928305074288, |
| "logits_recorded": true |
| }, |
| "transfer": { |
| "objective": -0.4988755241804635, |
| "paired_flip": { |
| "pairs": 16, |
| "flip_rate": 0.9375, |
| "both_correct_rate": 0.9375, |
| "invariant_pairs": 8, |
| "invariance_rate": 1.0, |
| "invariant_both_correct_rate": 0.875 |
| }, |
| "unknowable": null, |
| "clean": { |
| "n": 243, |
| "nll": 0.6065943824685833, |
| "acc": 0.8683127572016461, |
| "ece": 0.09195653550207036, |
| "brier": 0.23499321102178952, |
| "mean_conf": 0.9525644631029061, |
| "confident_error_rate": 0.09053497942386832, |
| "coverage_at_0_9": 0.8600823045267489, |
| "accuracy_at_0_9": 0.8947368421052632, |
| "coverage_at_5pct_error": 0.6872427983539094, |
| "coverage_at_1pct_error": 0.42386831275720166, |
| "aurc": 0.034793809228677, |
| "error_rate_at_0_9": 0.10526315789473684, |
| "confidence_bias": 0.08425170590125997, |
| "top_bins": { |
| "0.9": { |
| "n": 209, |
| "errors": 22, |
| "error_rate": 0.10526315789473684 |
| }, |
| "0.95": { |
| "n": 193, |
| "errors": 13, |
| "error_rate": 0.06735751295336788 |
| }, |
| "0.99": { |
| "n": 154, |
| "errors": 4, |
| "error_rate": 0.025974025974025976 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5020576131687243, |
| "accuracy": 0.9836065573770492, |
| "confidence_cutoff": 0.9985267483650163 |
| }, |
| "0.8": { |
| "coverage": 0.8024691358024691, |
| "accuracy": 0.9230769230769231, |
| "confidence_cutoff": 0.946113674977154 |
| } |
| } |
| }, |
| "tasks": { |
| "composition_held_and_or": { |
| "n": 32, |
| "nll": 0.24171629359001318, |
| "acc": 0.90625, |
| "ece": 0.08267706695037225, |
| "brier": 0.1580607438572586, |
| "mean_conf": 0.9889270669503722, |
| "confident_error_rate": 0.09375, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 0.90625, |
| "coverage_at_5pct_error": 0.9375, |
| "coverage_at_1pct_error": 0.90625, |
| "aurc": 0.0069010416666666664, |
| "error_rate_at_0_9": 0.09375, |
| "confidence_bias": 0.08267706695037225, |
| "top_bins": { |
| "0.9": { |
| "n": 32, |
| "errors": 3, |
| "error_rate": 0.09375 |
| }, |
| "0.95": { |
| "n": 30, |
| "errors": 1, |
| "error_rate": 0.03333333333333333 |
| }, |
| "0.99": { |
| "n": 26, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999364856833449 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9996934954887139 |
| } |
| } |
| }, |
| "contrastive_authorization": { |
| "n": 16, |
| "nll": 1.502653141036028e-05, |
| "acc": 1.0, |
| "ece": 1.50264042420023e-05, |
| "brier": 5.139828295030656e-10, |
| "mean_conf": 0.999984973595758, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -1.50264042420023e-05, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999862890472545 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999813198967473 |
| } |
| } |
| }, |
| "emotion": { |
| "n": 8, |
| "nll": 1.4194273414243654, |
| "acc": 0.625, |
| "ece": 0.3420785503418361, |
| "brier": 0.6845960520300963, |
| "mean_conf": 0.967078550341836, |
| "confident_error_rate": 0.25, |
| "coverage_at_0_9": 0.875, |
| "accuracy_at_0_9": 0.7142857142857143, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.12842261904761904, |
| "error_rate_at_0_9": 0.2857142857142857, |
| "confidence_bias": 0.342078550341836, |
| "top_bins": { |
| "0.9": { |
| "n": 7, |
| "errors": 2, |
| "error_rate": 0.2857142857142857 |
| }, |
| "0.95": { |
| "n": 5, |
| "errors": 1, |
| "error_rate": 0.2 |
| }, |
| "0.99": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9869563807629563 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.7142857142857143, |
| "confidence_cutoff": 0.936063568486461 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 8, |
| "nll": 1.3129749769552814, |
| "acc": 0.625, |
| "ece": 0.15765322690405312, |
| "brier": 0.6484351711065982, |
| "mean_conf": 0.7567759888850003, |
| "confident_error_rate": 0.125, |
| "coverage_at_0_9": 0.5, |
| "accuracy_at_0_9": 0.75, |
| "coverage_at_5pct_error": 0.375, |
| "coverage_at_1pct_error": 0.375, |
| "aurc": 0.22336309523809525, |
| "error_rate_at_0_9": 0.25, |
| "confidence_bias": 0.1317759888850003, |
| "top_bins": { |
| "0.9": { |
| "n": 4, |
| "errors": 1, |
| "error_rate": 0.25 |
| }, |
| "0.95": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.9327223999584052 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.5714285714285714, |
| "confidence_cutoff": 0.485476400811401 |
| } |
| } |
| }, |
| "otrf_sysmon_file": { |
| "n": 16, |
| "nll": 0.008605175298168915, |
| "acc": 1.0, |
| "ece": 0.008437849080092708, |
| "brier": 0.0005211762382657875, |
| "mean_conf": 0.9915621509199073, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.008437849080092708, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 13, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9989837483137961 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9925042879134804 |
| } |
| } |
| }, |
| "otrf_sysmon_join": { |
| "n": 16, |
| "nll": 0.12967885980971613, |
| "acc": 0.9375, |
| "ece": 0.0857233928404766, |
| "brier": 0.08146985002971384, |
| "mean_conf": 0.946506108546294, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.6875, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.9375, |
| "coverage_at_1pct_error": 0.9375, |
| "aurc": 0.00390625, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.009006108546294023, |
| "top_bins": { |
| "0.9": { |
| "n": 11, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 11, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 9, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9970209747654696 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.8702734279589676 |
| } |
| } |
| }, |
| "otrf_sysmon_process": { |
| "n": 24, |
| "nll": 0.16156109121321785, |
| "acc": 0.9583333333333334, |
| "ece": 0.05273169620250518, |
| "brier": 0.0899182770612567, |
| "mean_conf": 0.9768730534919899, |
| "confident_error_rate": 0.041666666666666664, |
| "coverage_at_0_9": 0.9583333333333334, |
| "accuracy_at_0_9": 0.9565217391304348, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9166666666666666, |
| "aurc": 0.0035477053140096615, |
| "error_rate_at_0_9": 0.043478260869565216, |
| "confidence_bias": 0.01853972015865657, |
| "top_bins": { |
| "0.9": { |
| "n": 23, |
| "errors": 1, |
| "error_rate": 0.043478260869565216 |
| }, |
| "0.95": { |
| "n": 22, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 19, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9994396085787187 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9832554759400715 |
| } |
| } |
| }, |
| "otrf_sysmon_registry": { |
| "n": 8, |
| "nll": 0.010723606364118954, |
| "acc": 1.0, |
| "ece": 0.010542385937563847, |
| "brier": 0.0006959554565011166, |
| "mean_conf": 0.9894576140624362, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.010542385937563847, |
| "top_bins": { |
| "0.9": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9972651302471073 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9840692701748682 |
| } |
| } |
| }, |
| "paws": { |
| "n": 8, |
| "nll": 1.0368545966100273, |
| "acc": 0.75, |
| "ece": 0.22498828525853157, |
| "brier": 0.34163600039007924, |
| "mean_conf": 0.9122532529823204, |
| "confident_error_rate": 0.125, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.8333333333333334, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.09494047619047619, |
| "error_rate_at_0_9": 0.16666666666666666, |
| "confidence_bias": 0.1622532529823204, |
| "top_bins": { |
| "0.9": { |
| "n": 6, |
| "errors": 1, |
| "error_rate": 0.16666666666666666 |
| }, |
| "0.95": { |
| "n": 6, |
| "errors": 1, |
| "error_rate": 0.16666666666666666 |
| }, |
| "0.99": { |
| "n": 6, |
| "errors": 1, |
| "error_rate": 0.16666666666666666 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9992691391151949 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.8571428571428571, |
| "confidence_cutoff": 0.749059870895155 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 8, |
| "nll": 0.005181466224742151, |
| "acc": 1.0, |
| "ece": 0.005129334884136472, |
| "brier": 0.00020524982613720548, |
| "mean_conf": 0.9948706651158635, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.005129334884136472, |
| "top_bins": { |
| "0.9": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 6, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999952371295075 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.98872862957213 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 8, |
| "nll": 0.00023928943016782004, |
| "acc": 1.0, |
| "ece": 0.0002391650142747892, |
| "brier": 4.955209174865095e-07, |
| "mean_conf": 0.9997608349857252, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0002391650142747892, |
| "top_bins": { |
| "0.9": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999911919978735 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9994259298933852 |
| } |
| } |
| }, |
| "sev_swarmtraces_manual_credential": { |
| "n": 40, |
| "nll": 0.9530052125004369, |
| "acc": 0.825, |
| "ece": 0.17316335764162732, |
| "brier": 0.3580423587545788, |
| "mean_conf": 0.9202306277592299, |
| "confident_error_rate": 0.15, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.8, |
| "coverage_at_5pct_error": 0.225, |
| "coverage_at_1pct_error": 0.225, |
| "aurc": 0.10718612602417658, |
| "error_rate_at_0_9": 0.2, |
| "confidence_bias": 0.09523062775922997, |
| "top_bins": { |
| "0.9": { |
| "n": 30, |
| "errors": 6, |
| "error_rate": 0.2 |
| }, |
| "0.95": { |
| "n": 26, |
| "errors": 3, |
| "error_rate": 0.11538461538461539 |
| }, |
| "0.99": { |
| "n": 15, |
| "errors": 1, |
| "error_rate": 0.06666666666666667 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9774245184450835 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8125, |
| "confidence_cutoff": 0.8542392185801473 |
| } |
| } |
| }, |
| "sev_swarmtraces_manual_flow": { |
| "n": 40, |
| "nll": 1.485896746743676, |
| "acc": 0.75, |
| "ece": 0.21977756775377905, |
| "brier": 0.43299049842888415, |
| "mean_conf": 0.9214803040956191, |
| "confident_error_rate": 0.15, |
| "coverage_at_0_9": 0.725, |
| "accuracy_at_0_9": 0.7931034482758621, |
| "coverage_at_5pct_error": 0.1, |
| "coverage_at_1pct_error": 0.1, |
| "aurc": 0.14190456225170725, |
| "error_rate_at_0_9": 0.20689655172413793, |
| "confidence_bias": 0.1714803040956191, |
| "top_bins": { |
| "0.9": { |
| "n": 29, |
| "errors": 6, |
| "error_rate": 0.20689655172413793 |
| }, |
| "0.95": { |
| "n": 27, |
| "errors": 6, |
| "error_rate": 0.2222222222222222 |
| }, |
| "0.99": { |
| "n": 21, |
| "errors": 2, |
| "error_rate": 0.09523809523809523 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9901066734665328 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8125, |
| "confidence_cutoff": 0.8291658959545783 |
| } |
| } |
| }, |
| "sev_swarmtraces_manual_prerequisite": { |
| "n": 3, |
| "nll": 1.0331257533433365e-05, |
| "acc": 1.0, |
| "ece": 1.0331183196821137e-05, |
| "brier": 2.5195789327524225e-10, |
| "mean_conf": 0.9999896688168032, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -1.0331183196821137e-05, |
| "top_bins": { |
| "0.9": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.6666666666666666, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999897312178962 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999810314632929 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 8, |
| "nll": 0.7172428487540772, |
| "acc": 0.75, |
| "ece": 0.2179639808956667, |
| "brier": 0.44120571426799937, |
| "mean_conf": 0.9679639808956667, |
| "confident_error_rate": 0.25, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 0.75, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.09494047619047619, |
| "error_rate_at_0_9": 0.25, |
| "confidence_bias": 0.2179639808956667, |
| "top_bins": { |
| "0.9": { |
| "n": 8, |
| "errors": 2, |
| "error_rate": 0.25 |
| }, |
| "0.95": { |
| "n": 5, |
| "errors": 1, |
| "error_rate": 0.2 |
| }, |
| "0.99": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9897168920827885 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.8571428571428571, |
| "confidence_cutoff": 0.940377631626033 |
| } |
| } |
| } |
| }, |
| "variants": { |
| "authorized_dns_diagnostic": { |
| "n": 4, |
| "nll": 0.2732991829356235, |
| "acc": 0.75, |
| "ece": 0.2014060191596143, |
| "brier": 0.1826209474615037, |
| "mean_conf": 0.8245034642731555, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.5, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.75, |
| "coverage_at_1pct_error": 0.75, |
| "aurc": 0.0625, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.07450346427315546, |
| "top_bins": { |
| "0.9": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9941628768025105 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.5518189668655396 |
| } |
| } |
| }, |
| "cache_denied_no_reply": { |
| "n": 4, |
| "nll": 0.6290998127898555, |
| "acc": 0.5, |
| "ece": 0.19783432109801047, |
| "brier": 0.45073491547326267, |
| "mean_conf": 0.6842140294824197, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.25, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.25, |
| "coverage_at_1pct_error": 0.25, |
| "aurc": 0.41666666666666663, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.18421402948241972, |
| "top_bins": { |
| "0.9": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.5997726159351691 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.5818420334307344 |
| } |
| } |
| }, |
| "clean": { |
| "n": 243, |
| "nll": 0.6065943824685833, |
| "acc": 0.8683127572016461, |
| "ece": 0.09195653550207036, |
| "brier": 0.23499321102178952, |
| "mean_conf": 0.9525644631029061, |
| "confident_error_rate": 0.09053497942386832, |
| "coverage_at_0_9": 0.8600823045267489, |
| "accuracy_at_0_9": 0.8947368421052632, |
| "coverage_at_5pct_error": 0.6872427983539094, |
| "coverage_at_1pct_error": 0.42386831275720166, |
| "aurc": 0.034793809228677, |
| "error_rate_at_0_9": 0.10526315789473684, |
| "confidence_bias": 0.08425170590125997, |
| "top_bins": { |
| "0.9": { |
| "n": 209, |
| "errors": 22, |
| "error_rate": 0.10526315789473684 |
| }, |
| "0.95": { |
| "n": 193, |
| "errors": 13, |
| "error_rate": 0.06735751295336788 |
| }, |
| "0.99": { |
| "n": 154, |
| "errors": 4, |
| "error_rate": 0.025974025974025976 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5020576131687243, |
| "accuracy": 0.9836065573770492, |
| "confidence_cutoff": 0.9985267483650163 |
| }, |
| "0.8": { |
| "coverage": 0.8024691358024691, |
| "accuracy": 0.9230769230769231, |
| "confidence_cutoff": 0.946113674977154 |
| } |
| } |
| }, |
| "dns_off_task_reply": { |
| "n": 4, |
| "nll": 2.500105858228899, |
| "acc": 0.25, |
| "ece": 0.7354215562262277, |
| "brier": 1.1856881009881057, |
| "mean_conf": 0.7180991063824902, |
| "confident_error_rate": 0.25, |
| "coverage_at_0_9": 0.25, |
| "accuracy_at_0_9": 0.0, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 0.9375, |
| "error_rate_at_0_9": 1.0, |
| "confidence_bias": 0.4680991063824902, |
| "top_bins": { |
| "0.9": { |
| "n": 1, |
| "errors": 1, |
| "error_rate": 1.0 |
| }, |
| "0.95": { |
| "n": 1, |
| "errors": 1, |
| "error_rate": 1.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 1, |
| "error_rate": 1.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.0, |
| "confidence_cutoff": 0.7394568794916744 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 0.25, |
| "confidence_cutoff": 0.465355100312525 |
| } |
| } |
| }, |
| "dns_unhelpful_reply": { |
| "n": 4, |
| "nll": 0.6130712887407269, |
| "acc": 0.75, |
| "ece": 0.2694959031277442, |
| "brier": 0.34519095756377294, |
| "mean_conf": 0.7624970557751488, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.5, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.75, |
| "coverage_at_1pct_error": 0.75, |
| "aurc": 0.0625, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.012497055775148813, |
| "top_bins": { |
| "0.9": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9232891824527384 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.563985917805786 |
| } |
| } |
| }, |
| "dns_without_reply": { |
| "n": 4, |
| "nll": 0.6878185272523656, |
| "acc": 0.5, |
| "ece": 0.20221082782282854, |
| "brier": 0.5043315891024525, |
| "mean_conf": 0.7000800429377261, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.25, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.25, |
| "coverage_at_1pct_error": 0.25, |
| "aurc": 0.41666666666666663, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.20008004293772608, |
| "top_bins": { |
| "0.9": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.6943924514728984 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.5490827128017757 |
| } |
| } |
| }, |
| "none_absent": { |
| "n": 3, |
| "nll": 1.3777335821359413, |
| "acc": 0.6666666666666666, |
| "ece": 0.24885046106665176, |
| "brier": 0.5885968900899531, |
| "mean_conf": 0.9151737396988762, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.3333333333333333, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.6666666666666666, |
| "coverage_at_1pct_error": 0.6666666666666666, |
| "aurc": 0.1111111111111111, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.24850707303220954, |
| "top_bins": { |
| "0.9": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.6666666666666666, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.8739663040318306 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 0.6666666666666666, |
| "confidence_cutoff": 0.8720699971164613 |
| } |
| } |
| }, |
| "none_present": { |
| "n": 3, |
| "nll": 3.3971590445247757, |
| "acc": 0.0, |
| "ece": 0.806803931885096, |
| "brier": 1.589050038207996, |
| "mean_conf": 0.8068039318850961, |
| "confident_error_rate": 0.3333333333333333, |
| "coverage_at_0_9": 0.3333333333333333, |
| "accuracy_at_0_9": 0.0, |
| "coverage_at_5pct_error": 0.0, |
| "coverage_at_1pct_error": 0.0, |
| "aurc": 1.0, |
| "error_rate_at_0_9": 1.0, |
| "confidence_bias": 0.8068039318850961, |
| "top_bins": { |
| "0.9": { |
| "n": 1, |
| "errors": 1, |
| "error_rate": 1.0 |
| }, |
| "0.95": { |
| "n": 1, |
| "errors": 1, |
| "error_rate": 1.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.6666666666666666, |
| "accuracy": 0.0, |
| "confidence_cutoff": 0.8592945698759779 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 0.0, |
| "confidence_cutoff": 0.5949657569957464 |
| } |
| } |
| }, |
| "operator_terminated": { |
| "n": 4, |
| "nll": 1.0359158594833722, |
| "acc": 0.75, |
| "ece": 0.2925896655492872, |
| "brier": 0.4933779585120464, |
| "mean_conf": 0.9106235817198012, |
| "confident_error_rate": 0.25, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.6666666666666666, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.14583333333333331, |
| "error_rate_at_0_9": 0.3333333333333333, |
| "confidence_bias": 0.16062358171980118, |
| "top_bins": { |
| "0.9": { |
| "n": 3, |
| "errors": 1, |
| "error_rate": 0.3333333333333333 |
| }, |
| "0.95": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9713811910576418 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.736067832341028 |
| } |
| } |
| }, |
| "ordinary_resolution": { |
| "n": 4, |
| "nll": 0.04811475260764305, |
| "acc": 1.0, |
| "ece": 0.04485818654799875, |
| "brier": 0.011362447845748204, |
| "mean_conf": 0.9551418134520012, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.04485818654799878, |
| "top_bins": { |
| "0.9": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9973615074301722 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.848997818390719 |
| } |
| } |
| }, |
| "permuted": { |
| "n": 3, |
| "nll": 1.5849320188552285, |
| "acc": 0.3333333333333333, |
| "ece": 0.5115039138500523, |
| "brier": 0.948500922906351, |
| "mean_conf": 0.834218301404813, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.3333333333333333, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.3333333333333333, |
| "coverage_at_1pct_error": 0.3333333333333333, |
| "aurc": 0.38888888888888884, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.5008849680714798, |
| "top_bins": { |
| "0.9": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.6666666666666666, |
| "accuracy": 0.5, |
| "confidence_cutoff": 0.8701283299219073 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 0.3333333333333333, |
| "confidence_cutoff": 0.6484549929603911 |
| } |
| } |
| }, |
| "review_acknowledged": { |
| "n": 4, |
| "nll": 0.9430886337427746, |
| "acc": 0.75, |
| "ece": 0.26428833978133903, |
| "brier": 0.46621510579801406, |
| "mean_conf": 0.9076209540563065, |
| "confident_error_rate": 0.25, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.6666666666666666, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.14583333333333331, |
| "error_rate_at_0_9": 0.3333333333333333, |
| "confidence_bias": 0.1576209540563065, |
| "top_bins": { |
| "0.9": { |
| "n": 3, |
| "errors": 1, |
| "error_rate": 0.3333333333333333 |
| }, |
| "0.95": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9371813046912169 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.786665228549935 |
| } |
| } |
| } |
| }, |
| "heldout_tasks": { |
| "composition_held_and_or": { |
| "n": 32, |
| "nll": 0.24171629359001318, |
| "acc": 0.90625, |
| "ece": 0.08267706695037225, |
| "brier": 0.1580607438572586, |
| "mean_conf": 0.9889270669503722, |
| "confident_error_rate": 0.09375, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 0.90625, |
| "coverage_at_5pct_error": 0.9375, |
| "coverage_at_1pct_error": 0.90625, |
| "aurc": 0.0069010416666666664, |
| "error_rate_at_0_9": 0.09375, |
| "confidence_bias": 0.08267706695037225, |
| "top_bins": { |
| "0.9": { |
| "n": 32, |
| "errors": 3, |
| "error_rate": 0.09375 |
| }, |
| "0.95": { |
| "n": 30, |
| "errors": 1, |
| "error_rate": 0.03333333333333333 |
| }, |
| "0.99": { |
| "n": 26, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999364856833449 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9996934954887139 |
| } |
| } |
| }, |
| "contrastive_authorization": { |
| "n": 16, |
| "nll": 1.502653141036028e-05, |
| "acc": 1.0, |
| "ece": 1.50264042420023e-05, |
| "brier": 5.139828295030656e-10, |
| "mean_conf": 0.999984973595758, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -1.50264042420023e-05, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999862890472545 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999813198967473 |
| } |
| } |
| }, |
| "emotion": { |
| "n": 8, |
| "nll": 1.4194273414243654, |
| "acc": 0.625, |
| "ece": 0.3420785503418361, |
| "brier": 0.6845960520300963, |
| "mean_conf": 0.967078550341836, |
| "confident_error_rate": 0.25, |
| "coverage_at_0_9": 0.875, |
| "accuracy_at_0_9": 0.7142857142857143, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.12842261904761904, |
| "error_rate_at_0_9": 0.2857142857142857, |
| "confidence_bias": 0.342078550341836, |
| "top_bins": { |
| "0.9": { |
| "n": 7, |
| "errors": 2, |
| "error_rate": 0.2857142857142857 |
| }, |
| "0.95": { |
| "n": 5, |
| "errors": 1, |
| "error_rate": 0.2 |
| }, |
| "0.99": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9869563807629563 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.7142857142857143, |
| "confidence_cutoff": 0.936063568486461 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 8, |
| "nll": 1.3129749769552814, |
| "acc": 0.625, |
| "ece": 0.15765322690405312, |
| "brier": 0.6484351711065982, |
| "mean_conf": 0.7567759888850003, |
| "confident_error_rate": 0.125, |
| "coverage_at_0_9": 0.5, |
| "accuracy_at_0_9": 0.75, |
| "coverage_at_5pct_error": 0.375, |
| "coverage_at_1pct_error": 0.375, |
| "aurc": 0.22336309523809525, |
| "error_rate_at_0_9": 0.25, |
| "confidence_bias": 0.1317759888850003, |
| "top_bins": { |
| "0.9": { |
| "n": 4, |
| "errors": 1, |
| "error_rate": 0.25 |
| }, |
| "0.95": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.9327223999584052 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.5714285714285714, |
| "confidence_cutoff": 0.485476400811401 |
| } |
| } |
| }, |
| "otrf_sysmon_file": { |
| "n": 16, |
| "nll": 0.008605175298168915, |
| "acc": 1.0, |
| "ece": 0.008437849080092708, |
| "brier": 0.0005211762382657875, |
| "mean_conf": 0.9915621509199073, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.008437849080092708, |
| "top_bins": { |
| "0.9": { |
| "n": 16, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 15, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 13, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9989837483137961 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9925042879134804 |
| } |
| } |
| }, |
| "otrf_sysmon_join": { |
| "n": 16, |
| "nll": 0.12967885980971613, |
| "acc": 0.9375, |
| "ece": 0.0857233928404766, |
| "brier": 0.08146985002971384, |
| "mean_conf": 0.946506108546294, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.6875, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.9375, |
| "coverage_at_1pct_error": 0.9375, |
| "aurc": 0.00390625, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.009006108546294023, |
| "top_bins": { |
| "0.9": { |
| "n": 11, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 11, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 9, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9970209747654696 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.8702734279589676 |
| } |
| } |
| }, |
| "otrf_sysmon_process": { |
| "n": 24, |
| "nll": 0.16156109121321785, |
| "acc": 0.9583333333333334, |
| "ece": 0.05273169620250518, |
| "brier": 0.0899182770612567, |
| "mean_conf": 0.9768730534919899, |
| "confident_error_rate": 0.041666666666666664, |
| "coverage_at_0_9": 0.9583333333333334, |
| "accuracy_at_0_9": 0.9565217391304348, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9166666666666666, |
| "aurc": 0.0035477053140096615, |
| "error_rate_at_0_9": 0.043478260869565216, |
| "confidence_bias": 0.01853972015865657, |
| "top_bins": { |
| "0.9": { |
| "n": 23, |
| "errors": 1, |
| "error_rate": 0.043478260869565216 |
| }, |
| "0.95": { |
| "n": 22, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 19, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9994396085787187 |
| }, |
| "0.8": { |
| "coverage": 0.8333333333333334, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9832554759400715 |
| } |
| } |
| }, |
| "otrf_sysmon_registry": { |
| "n": 8, |
| "nll": 0.010723606364118954, |
| "acc": 1.0, |
| "ece": 0.010542385937563847, |
| "brier": 0.0006959554565011166, |
| "mean_conf": 0.9894576140624362, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.010542385937563847, |
| "top_bins": { |
| "0.9": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9972651302471073 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9840692701748682 |
| } |
| } |
| }, |
| "paws": { |
| "n": 8, |
| "nll": 1.0368545966100273, |
| "acc": 0.75, |
| "ece": 0.22498828525853157, |
| "brier": 0.34163600039007924, |
| "mean_conf": 0.9122532529823204, |
| "confident_error_rate": 0.125, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.8333333333333334, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.09494047619047619, |
| "error_rate_at_0_9": 0.16666666666666666, |
| "confidence_bias": 0.1622532529823204, |
| "top_bins": { |
| "0.9": { |
| "n": 6, |
| "errors": 1, |
| "error_rate": 0.16666666666666666 |
| }, |
| "0.95": { |
| "n": 6, |
| "errors": 1, |
| "error_rate": 0.16666666666666666 |
| }, |
| "0.99": { |
| "n": 6, |
| "errors": 1, |
| "error_rate": 0.16666666666666666 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9992691391151949 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.8571428571428571, |
| "confidence_cutoff": 0.749059870895155 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 8, |
| "nll": 0.005181466224742151, |
| "acc": 1.0, |
| "ece": 0.005129334884136472, |
| "brier": 0.00020524982613720548, |
| "mean_conf": 0.9948706651158635, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.005129334884136472, |
| "top_bins": { |
| "0.9": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 6, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999952371295075 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.98872862957213 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 8, |
| "nll": 0.00023928943016782004, |
| "acc": 1.0, |
| "ece": 0.0002391650142747892, |
| "brier": 4.955209174865095e-07, |
| "mean_conf": 0.9997608349857252, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0002391650142747892, |
| "top_bins": { |
| "0.9": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999911919978735 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9994259298933852 |
| } |
| } |
| }, |
| "sev_swarmtraces_manual_credential": { |
| "n": 40, |
| "nll": 0.9530052125004369, |
| "acc": 0.825, |
| "ece": 0.17316335764162732, |
| "brier": 0.3580423587545788, |
| "mean_conf": 0.9202306277592299, |
| "confident_error_rate": 0.15, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.8, |
| "coverage_at_5pct_error": 0.225, |
| "coverage_at_1pct_error": 0.225, |
| "aurc": 0.10718612602417658, |
| "error_rate_at_0_9": 0.2, |
| "confidence_bias": 0.09523062775922997, |
| "top_bins": { |
| "0.9": { |
| "n": 30, |
| "errors": 6, |
| "error_rate": 0.2 |
| }, |
| "0.95": { |
| "n": 26, |
| "errors": 3, |
| "error_rate": 0.11538461538461539 |
| }, |
| "0.99": { |
| "n": 15, |
| "errors": 1, |
| "error_rate": 0.06666666666666667 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9774245184450835 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8125, |
| "confidence_cutoff": 0.8542392185801473 |
| } |
| } |
| }, |
| "sev_swarmtraces_manual_flow": { |
| "n": 40, |
| "nll": 1.485896746743676, |
| "acc": 0.75, |
| "ece": 0.21977756775377905, |
| "brier": 0.43299049842888415, |
| "mean_conf": 0.9214803040956191, |
| "confident_error_rate": 0.15, |
| "coverage_at_0_9": 0.725, |
| "accuracy_at_0_9": 0.7931034482758621, |
| "coverage_at_5pct_error": 0.1, |
| "coverage_at_1pct_error": 0.1, |
| "aurc": 0.14190456225170725, |
| "error_rate_at_0_9": 0.20689655172413793, |
| "confidence_bias": 0.1714803040956191, |
| "top_bins": { |
| "0.9": { |
| "n": 29, |
| "errors": 6, |
| "error_rate": 0.20689655172413793 |
| }, |
| "0.95": { |
| "n": 27, |
| "errors": 6, |
| "error_rate": 0.2222222222222222 |
| }, |
| "0.99": { |
| "n": 21, |
| "errors": 2, |
| "error_rate": 0.09523809523809523 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9901066734665328 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8125, |
| "confidence_cutoff": 0.8291658959545783 |
| } |
| } |
| }, |
| "sev_swarmtraces_manual_prerequisite": { |
| "n": 3, |
| "nll": 1.0331257533433365e-05, |
| "acc": 1.0, |
| "ece": 1.0331183196821137e-05, |
| "brier": 2.5195789327524225e-10, |
| "mean_conf": 0.9999896688168032, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -1.0331183196821137e-05, |
| "top_bins": { |
| "0.9": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.6666666666666666, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999897312178962 |
| }, |
| "0.8": { |
| "coverage": 1.0, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999810314632929 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 8, |
| "nll": 0.7172428487540772, |
| "acc": 0.75, |
| "ece": 0.2179639808956667, |
| "brier": 0.44120571426799937, |
| "mean_conf": 0.9679639808956667, |
| "confident_error_rate": 0.25, |
| "coverage_at_0_9": 1.0, |
| "accuracy_at_0_9": 0.75, |
| "coverage_at_5pct_error": 0.5, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.09494047619047619, |
| "error_rate_at_0_9": 0.25, |
| "confidence_bias": 0.2179639808956667, |
| "top_bins": { |
| "0.9": { |
| "n": 8, |
| "errors": 2, |
| "error_rate": 0.25 |
| }, |
| "0.95": { |
| "n": 5, |
| "errors": 1, |
| "error_rate": 0.2 |
| }, |
| "0.99": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9897168920827885 |
| }, |
| "0.8": { |
| "coverage": 0.875, |
| "accuracy": 0.8571428571428571, |
| "confidence_cutoff": 0.940377631626033 |
| } |
| } |
| } |
| }, |
| "permutation": { |
| "n": 3, |
| "mean_max_delta": 0.04297798881434687, |
| "flip_rate": 0.0 |
| }, |
| "temperature": 1.6817928305074288, |
| "calibrated_clean": { |
| "n": 243, |
| "nll": 0.4460227427836622, |
| "acc": 0.8683127572016461, |
| "ece": 0.04490914804947624, |
| "brier": 0.21677989484802473, |
| "mean_conf": 0.9016047263613403, |
| "confident_error_rate": 0.024691358024691357, |
| "coverage_at_0_9": 0.6831275720164609, |
| "accuracy_at_0_9": 0.963855421686747, |
| "coverage_at_5pct_error": 0.7489711934156379, |
| "coverage_at_1pct_error": 0.4156378600823045, |
| "aurc": 0.03366327644063936, |
| "error_rate_at_0_9": 0.03614457831325301, |
| "confidence_bias": 0.033291969159694124, |
| "top_bins": { |
| "0.9": { |
| "n": 166, |
| "errors": 6, |
| "error_rate": 0.03614457831325301 |
| }, |
| "0.95": { |
| "n": 141, |
| "errors": 2, |
| "error_rate": 0.014184397163120567 |
| }, |
| "0.99": { |
| "n": 81, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5020576131687243, |
| "accuracy": 0.9836065573770492, |
| "confidence_cutoff": 0.9739727890189261 |
| }, |
| "0.8": { |
| "coverage": 0.8024691358024691, |
| "accuracy": 0.9282051282051282, |
| "confidence_cutoff": 0.8229542441861438 |
| } |
| } |
| }, |
| "metric_policy": { |
| "version": 2, |
| "selective_ties": "whole_confidence_groups", |
| "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", |
| "aurc": "right-step integral over whole confidence groups", |
| "confident_error_rate": "high-confidence errors divided by all questions", |
| "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", |
| "nll": "exact from logits when recorded; otherwise from floored probabilities", |
| "nll_floor": 1e-09, |
| "renormalize_returned_probabilities": true, |
| "raw_sums_outside_1e_5": 0, |
| "returned_zeros": 0 |
| }, |
| "coverage": { |
| "requested_records": 185, |
| "requested_questions": 284, |
| "evaluated_records": 185, |
| "evaluated_questions": 284, |
| "rejected_records": 0, |
| "truncated_records": 0 |
| }, |
| "latency_ms": { |
| "median": 49.463423999895895, |
| "p95": 189.5706840000457 |
| }, |
| "calibration": { |
| "inference_temperature": 1.0, |
| "additional_temperature": 1.6817928305074288, |
| "logits_recorded": true |
| }, |
| "suite_sha256": "adde94227c959c3bfd62ee2ca27d8a73d5b0fff96bf0500c627645e5a7d72f37" |
| }, |
| "provenance": { |
| "config": { |
| "epochs": 1, |
| "seed": 4, |
| "lr": 2.5e-06, |
| "lora": 16, |
| "accum": 2, |
| "batch": 4, |
| "perm_kl": 0, |
| "perm_frac": 0.3, |
| "ord_w": 0.0, |
| "p_none": 0, |
| "p_none_distract": 0, |
| "p_distract": 0, |
| "p_none_pair": 0, |
| "synthetic_repeat": 1, |
| "public_frac": 1, |
| "head_lr": 0.0, |
| "weight_decay": 0.01, |
| "anchor_w": 0.0, |
| "label_smoothing": 0.0, |
| "brier_w": 0.0, |
| "focal_gamma": 0.0, |
| "base": "Qwen/Qwen3.5-4B-Base", |
| "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b", |
| "init_from": "macmacmacmac/Sev-4B@a1824aefba305fda86e3503b895a7d9b3871f79a", |
| "lora_targets": "all", |
| "head_dim": 256, |
| "weights_dtype": "fp32", |
| "dtype": "bf16", |
| "checkpointing": 1, |
| "option_isolation": 0, |
| "special_embeddings": 0 |
| }, |
| "config_sha256": "01ce0c9f84977e5c32d40dbd5c012634208c7770615b9e7d9d6f5950c042706f", |
| "suite_sha256": "3b9a80dfd212b6932b66ea520c35ab1a3b19e132a81af1b6b7b306d776a70a02", |
| "source_hashes": { |
| "kev/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
| "kev/anchors.py": "6963eafdb276db5a6c94d939eae675c761555553b8d0f963ffd803448beaabbb", |
| "kev/api.py": "7bffacfb762c626b8bc2f670f350295af5ccb0883dbe90239c7d2f8e5ef56582", |
| "kev/autoresearch.py": "f7d7fabb9c0df7065bee3fec4aa4bee7028c5d1565847aefe2d74e3d7d40ed8a", |
| "kev/benchmark.py": "8d486659a9036102386b077fcbd197fc0d85ae7752a4711666d6b3a95622b26f", |
| "kev/calibrate.py": "c1eff4744fabd349e8abca86777a7aa0cbea44904c8223d3b61fca3d43741519", |
| "kev/checkpoint.py": "9f1dbc5d3d7d3050aca4ee34e0578c9ad76b7a2a7467dfc7a4ad65e3c471db59", |
| "kev/compare.py": "3606dbf98bb305d66420158498cb837304d2edb8dab01cdaf2ec42964bef3435", |
| "kev/composition.py": "f335ed17e18e0a544893db5e22b9a059e6ce1b2e14dbb863ac7d7bbf8f3e0536", |
| "kev/contrastive.py": "cbb979aa5d40265ad0e64695f94b281d91751fa405811ddfa8212fede111edcf", |
| "kev/data.py": "a5a5e200f56b60e4bf30a195a4cc87a74afddae15999ec48c7543a4f9b59f71c", |
| "kev/device.py": "d1677fd98ec0979c7284546306e34e0d09298ca4042fc39eb2b32a74f4e975c4", |
| "kev/evaluate.py": "999264f2837dcfbf2ec93601aa4745e701698ab850a43ad89324672a6604965c", |
| "kev/experiment.py": "1a5ca47d2b3d236e53c87e8c4197704f162c6692aa008521d31ea491945693f0", |
| "kev/jev.py": "acd4cc3f1844e438cc83a8d409c15ef78a5ef64d39ea10f583d75a6c7646b243", |
| "kev/metrics.py": "db0930d470c0ce16f16dcaf726d49a7a103973a5296cefc5259d1e68707d882b", |
| "kev/mlx_model.py": "f582428796faf6bf962b772a69a6227ac3259ccbd3215c0c8b55ce28dbbd0909", |
| "kev/model.py": "eac0f56c807b85b5c4077483d922e6bb03f07890de2835efd9372cf20cd01b8e", |
| "kev/plot.py": "0874bcff2885d8155a1cceade0de8a275a7163d3c3fd6aa0c294e8ccf5ff7e02", |
| "kev/predictors.py": "b2a66dd9f0f0a8026bc94603383bacfdd622e75d1c935173997175606626a9b3", |
| "kev/publish.py": "8abc9bcf4a01365697b05cf3c5ad0013462bd92d005f5616954e40ab1100c7aa", |
| "kev/serve.py": "c210b1f4598e64b49c399f73603ed372f4314c165ceda50e1340c0466ec1c735", |
| "kev/study_v3.py": "7891150aa623e4479c7c789d4dc4662f186132b37b7bac1e1f072b9e08ee13e3", |
| "kev/suite.py": "c5e5e27f2fed871b547a021d180b0a7f4a0542a488cce4ee02ad936f433441db", |
| "kev/train.py": "a2d85632d73760fb3efe51627ff4fde6af3100ed1c34e75d290c1b00c9899aa9", |
| "kev/transfer_v9.py": "0902409742151250a28af1fd8f42258b70fcc161f3deed3ee35fe3f473f3c763", |
| "modal_app.py": "639577258f4d581daefa51dad639961280ce92de9da256316ab4124f005a53fb", |
| "pyproject.toml": "39ec40f14c88598dee7ac98fb859a5a44e22337235a3064ebd5e89036ce81e30", |
| "uv.lock": "a9922dbb89acdef78299fd2b4a8c3f7f0fa1b2bc08b55595b6926fa785a9c466" |
| }, |
| "git_commit": "740b511bb562cf29430a85457b9e54389a3f21cd", |
| "platform": "Linux-4.19.0-gvisor-x86_64-with-glibc2.36", |
| "torch": "2.8.0+cu128", |
| "device": "cuda", |
| "gpu": "NVIDIA H100 80GB HBM3", |
| "legacy_checkpoint": false, |
| "measured_checkpoint": { |
| "requested": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint", |
| "resolved": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint", |
| "head_sha256": "506dcd3587847917a288d95757c497c15a2aef8e2d4bc6523b04ad991c961a2b", |
| "adapter_sha256": "fa1916b34fe1eeb10d15c36927c225df43115407d7c97b4f879bba3102c3ed21", |
| "inference_temperature": 1.0 |
| } |
| }, |
| "mechanism_checks": { |
| "n": 24, |
| "packed_max_delta": 1.923413947224617e-06, |
| "sibling_max_delta": 9.5367431640625e-07, |
| "tolerance": 0.001, |
| "passed": true |
| }, |
| "gates": { |
| "passed": true, |
| "checks": { |
| "complete_coverage": true, |
| "isolation_and_packing": true, |
| "transfer_complete": true, |
| "heldout_pairs_at_least_70pct": true, |
| "transfer_confident_errors_below_10pct": true |
| }, |
| "policy": "Research screening only: 0.001 isolation; 5pp task regression; 70% heldout pair correctness; <=10% confident errors; no automatic release." |
| }, |
| "calibration_fit": { |
| "temperature": 1.6817928305074288, |
| "aggregation": "micro", |
| "objective": "raw-logit NLL", |
| "split": "calibration", |
| "rows_sha256": "026b68705b7ac76bc4361bc2fda1cbfe74265b436a807ea68bca77f87d3cf9da", |
| "suite_sha256": "3b9a80dfd212b6932b66ea520c35ab1a3b19e132a81af1b6b7b306d776a70a02", |
| "n": 2674 |
| }, |
| "wall_seconds": 1285.6242614819998, |
| "promotable": false, |
| "test_evaluated": false, |
| "training_resources": { |
| "wall_seconds": 887.5156674385071, |
| "records_seen": 6577, |
| "requested_records": 6577, |
| "truncated_records": 0, |
| "rejected_records": 0, |
| "optimizer_steps": 823, |
| "forward_tokens": 1505379, |
| "peak_device_bytes": 22225622528, |
| "device": "cuda", |
| "dtype": "bf16", |
| "batch": 4, |
| "peak_rss_bytes": 31556661248 |
| } |
| } |
|
|
