diff --git "a/evaluation.json" "b/evaluation.json" new file mode 100644--- /dev/null +++ "b/evaluation.json" @@ -0,0 +1,8443 @@ +{ + "schema": "sev-behavioral-round-report-v1", + "created_at": "2026-09-25T16:37:28.519871+00:00", + "script_sha256": "5beee8c302040233e2950ebe2d3f8944e84d6202f08b95943d679690bcdad8d8", + "question": "operator_origin", + "suite": { + "path": "/Users/mac/Sev-release-20260925/evals/sev/behavioral-research-v1", + "manifest_sha256": "372d8709cce5fc50262afc5107cd4b3637614ff2b2a7e84a8ccfa4293a4b21c5", + "partition_sha256": { + "calibration": "eebb5ddeb322b16bbb312ddb6927b1d71226b72e4b01d323c698d1e2a3c4f38d", + "development": "bd76cba90825cfb2699ff8a4419c99f7b395541ed1cfe51c4a69305451f659e5" + } + }, + "prediction_files": { + "parent": { + "calibration": { + "path": "/Users/mac/Sev/runs/sev-r2-behavioral-4b-parents-v1/00-kev-4b@485ace8703592fcf405488b262449990824cfed1/calibration/rows.json", + "sha256": "e307a8170f6c1c159e252c8e6c55c88f51e37a784e0a4579bd5625c6742952cf" + }, + "development": { + "path": "/Users/mac/Sev/runs/sev-r2-behavioral-4b-parents-v1/00-kev-4b@485ace8703592fcf405488b262449990824cfed1/development/rows.json", + "sha256": "9ca8eeeb877ea120e5f4c5d54fa72f9bf31f200bbf460e14e6ec85c223162d56" + } + }, + "candidate": { + "calibration": { + "path": "/Users/mac/Sev/runs/sev-r2-research-4b-v1/00-trial-0/calibration/rows.json", + "sha256": "00909b4d6b838c534d8d403d84a578fea1ddaac5e30a2b488c2d1c138c0d05ab" + }, + "development": { + "path": "/Users/mac/Sev/runs/sev-r2-research-4b-v1/00-trial-0/development/rows.json", + "sha256": "994e39a231bc64037f392e606c5fbd3c455f79e68f7b34ba44517238b269b26a" + } + } + }, + "calibration_fits": { + "parent": { + "temperature": 4.0, + "split": "calibration", + "view": "full", + "windows": 182, + "episodes": 96, + "method": "kev.metrics.fit_temperature; macro task=episode; 121-point log grid 0.25..4", + "weighting": "equal episodes, then equal windows within each episode", + "experiment_fit_relationship": "Diagnostic refit; may differ from the experiment's ordinary row/task-weighted fit." + }, + "candidate": { + "temperature": 1.9999999999999998, + "split": "calibration", + "view": "full", + "windows": 182, + "episodes": 96, + "method": "kev.metrics.fit_temperature; macro task=episode; 121-point log grid 0.25..4", + "weighting": "equal episodes, then equal windows within each episode", + "experiment_fit_relationship": "Diagnostic refit; may differ from the experiment's ordinary row/task-weighted fit." + } + }, + "primary_panel": "development/full/iid/all", + "primary_metrics": [ + "acc", + "nll" + ], + "secondary_panels": [ + "development/full/heldout_task/all", + "development/full/all/all", + "loss views and transaction-count panels" + ], + "reporting_corrections": [ + "Earlier reporter metadata named combined development as primary. The preregistered primary is full-view IID development accuracy/NLL. All panel calculations remain unchanged; the earlier parent report is preserved." + ], + "test_read": false, + "panels": { + "calibration": { + "full/all/all": { + "windows": 182, + "episodes": 96, + "latent_groups": 32, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 36, + "2": 13, + "3": 24, + "4": 26, + "5": 41, + "6": 38, + "7": 4 + }, + "raw": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.7039948885030416, + "brier": 0.9366734114222882, + "ece": 0.3836640270561081 + }, + "candidate": { + "acc": 0.3342013888888889, + "nll": 1.0988623728398248, + "brier": 0.6670543370614523, + "ece": 0.06554797243603253 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0008680555555555802, + "ci95": [ + -0.0078125, + 0.010416666666666685 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3255208333333333, + 0.34375 + ] + }, + "nll": { + "candidate_minus_parent": -0.6051325156632168, + "ci95": [ + -0.6803591005740874, + -0.530793390037944 + ], + "parent_ci95": [ + 1.6258755139905854, + 1.7835939337351452 + ], + "candidate_ci95": [ + 1.0872959592876683, + 1.1104023477998943 + ] + }, + "brier": { + "candidate_minus_parent": -0.2696190743608359, + "ci95": [ + -0.2966854808852652, + -0.24314883056422498 + ], + "parent_ci95": [ + 0.9086695403765366, + 0.9664303635041863 + ], + "candidate_ci95": [ + 0.6585256055156145, + 0.6756684634800705 + ] + }, + "ece": { + "candidate_minus_parent": -0.31811605462007553, + "ci95": [ + -0.3467234011742797, + -0.2776120036720403 + ], + "parent_ci95": [ + 0.3658444671289674, + 0.4154924939910463 + ], + "candidate_ci95": [ + 0.055802842793600815, + 0.10411501636343842 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.1516241769020001, + "brier": 0.7011284844520596, + "ece": 0.1126117849166719 + }, + "candidate": { + "acc": 0.3342013888888889, + "nll": 1.094225810223573, + "brier": 0.6635708549658472, + "ece": 0.04216089485176933 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0008680555555555802, + "ci95": [ + -0.0078125, + 0.010416666666666685 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3255208333333333, + 0.34375 + ] + }, + "nll": { + "candidate_minus_parent": -0.05739836667842724, + "ci95": [ + -0.06962603452379744, + -0.04488653163402823 + ], + "parent_ci95": [ + 1.1374408274590326, + 1.166659736233086 + ], + "candidate_ci95": [ + 1.0877924378749446, + 1.1001794102062266 + ] + }, + "brier": { + "candidate_minus_parent": -0.03755762948621233, + "ci95": [ + -0.04590448711935209, + -0.02933488075617265 + ], + "parent_ci95": [ + 0.6916327369871181, + 0.7113507865114055 + ], + "candidate_ci95": [ + 0.6589866278321939, + 0.6679576727482305 + ] + }, + "ece": { + "candidate_minus_parent": -0.07045089006490257, + "ci95": [ + -0.10204058462593617, + -0.03478029248966403 + ], + "parent_ci95": [ + 0.10663147728974164, + 0.13809590546759193 + ], + "candidate_ci95": [ + 0.02485568463820158, + 0.07566025500259763 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/all/transactions_1_2": { + "windows": 49, + "episodes": 45, + "latent_groups": 26, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 36, + "2": 13 + }, + "raw": { + "parent": { + "acc": 0.3111111111111111, + "nll": 1.6603323212661212, + "brier": 0.8907029573817137, + "ece": 0.37799735259416234 + }, + "candidate": { + "acc": 0.3888888888888889, + "nll": 1.1006948393144855, + "brier": 0.6675216739911485, + "ece": 0.02197163304242618 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.07777777777777778, + "ci95": [ + -0.12229166666666667, + 0.26746313102665914 + ], + "parent_ci95": [ + 0.20512820512820512, + 0.425 + ], + "candidate_ci95": [ + 0.28888888888888886, + 0.48984374999999974 + ] + }, + "nll": { + "candidate_minus_parent": -0.5596374819516357, + "ci95": [ + -0.8551553776743378, + -0.2835412394003477 + ], + "parent_ci95": [ + 1.3960780027038617, + 1.9415518588113363 + ], + "candidate_ci95": [ + 1.07961907447766, + 1.1190005382885808 + ] + }, + "brier": { + "candidate_minus_parent": -0.2231812833905652, + "ci95": [ + -0.36429527240524406, + -0.07974528756374481 + ], + "parent_ci95": [ + 0.7571302612447247, + 1.0218668962200836 + ], + "candidate_ci95": [ + 0.6529239682286327, + 0.6800289575226035 + ] + }, + "ece": { + "candidate_minus_parent": -0.3560257195517362, + "ci95": [ + -0.4212366104265594, + -0.19688232190049682 + ], + "parent_ci95": [ + 0.27558167651761906, + 0.4812973135389953 + ], + "candidate_ci95": [ + 0.005189460689758951, + 0.13286403298320185 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 26, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3111111111111111, + "nll": 1.1519141587962758, + "brier": 0.6972118196346561, + "ece": 0.12505951037027957 + }, + "candidate": { + "acc": 0.3888888888888889, + "nll": 1.0987834540411583, + "brier": 0.6666360545746582, + "ece": 0.03892731596092908 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.07777777777777778, + "ci95": [ + -0.12229166666666667, + 0.26746313102665914 + ], + "parent_ci95": [ + 0.20512820512820512, + 0.425 + ], + "candidate_ci95": [ + 0.28888888888888886, + 0.48984374999999974 + ] + }, + "nll": { + "candidate_minus_parent": -0.0531307047551175, + "ci95": [ + -0.13131384096372054, + 0.02030876595375617 + ], + "parent_ci95": [ + 1.0836231878965614, + 1.2232119820149001 + ], + "candidate_ci95": [ + 1.087965251457569, + 1.10803281863799 + ] + }, + "brier": { + "candidate_minus_parent": -0.030575765059997817, + "ci95": [ + -0.08053924500142515, + 0.016278720227622917 + ], + "parent_ci95": [ + 0.6536499526341443, + 0.7418442945165746 + ], + "candidate_ci95": [ + 0.6592384181572574, + 0.6728962291231819 + ] + }, + "ece": { + "candidate_minus_parent": -0.0861321944093505, + "ci95": [ + -0.1587048303302882, + 0.024643558296109955 + ], + "parent_ci95": [ + 0.02664428234558302, + 0.22892663757370982 + ], + "candidate_ci95": [ + 0.002359617988633989, + 0.14107288255595657 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 26, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/all/transactions_3_plus": { + "windows": 133, + "episodes": 93, + "latent_groups": 32, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "3": 24, + "4": 26, + "5": 41, + "6": 38, + "7": 4 + }, + "raw": { + "parent": { + "acc": 0.3118279569892473, + "nll": 1.8007600408262734, + "brier": 0.9912122366031354, + "ece": 0.41777594278222113 + }, + "candidate": { + "acc": 0.34050179211469533, + "nll": 1.0897724637391926, + "brier": 0.6608081864876919, + "ece": 0.06945969138416776 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.02867383512544802, + "ci95": [ + -0.00347222222222221, + 0.06666666666666671 + ], + "parent_ci95": [ + 0.28888888888888886, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3263157894736842, + 0.35555555555555557 + ] + }, + "nll": { + "candidate_minus_parent": -0.7109875770870808, + "ci95": [ + -0.8249378106097072, + -0.5928019108611865 + ], + "parent_ci95": [ + 1.6800381011917274, + 1.9200895008447456 + ], + "candidate_ci95": [ + 1.0736534993392457, + 1.1068153124221158 + ] + }, + "brier": { + "candidate_minus_parent": -0.3304040501154435, + "ci95": [ + -0.3793116868541628, + -0.28256738524317954 + ], + "parent_ci95": [ + 0.9404112325282276, + 1.0398186656005743 + ], + "candidate_ci95": [ + 0.6494179878255043, + 0.6727965288233754 + ] + }, + "ece": { + "candidate_minus_parent": -0.3483162513980534, + "ci95": [ + -0.40399898613809715, + -0.29044166032303326 + ], + "parent_ci95": [ + 0.38626474035396524, + 0.473126989756557 + ], + "candidate_ci95": [ + 0.05996149153641762, + 0.11740475732652438 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3118279569892473, + "nll": 1.171633543085405, + "brier": 0.7154649932664467, + "ece": 0.14465601993335808 + }, + "candidate": { + "acc": 0.34050179211469533, + "nll": 1.088970247788087, + "brier": 0.6599466665496634, + "ece": 0.055631274925033 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.02867383512544802, + "ci95": [ + -0.00347222222222221, + 0.06666666666666671 + ], + "parent_ci95": [ + 0.28888888888888886, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3263157894736842, + 0.35555555555555557 + ] + }, + "nll": { + "candidate_minus_parent": -0.08266329529731808, + "ci95": [ + -0.10527160530287753, + -0.060813425309605006 + ], + "parent_ci95": [ + 1.1497915115579906, + 1.1953183319652614 + ], + "candidate_ci95": [ + 1.0798257200325903, + 1.0975630581184335 + ] + }, + "brier": { + "candidate_minus_parent": -0.05551832671678336, + "ci95": [ + -0.07142755322765607, + -0.040168620031008025 + ], + "parent_ci95": [ + 0.7003199621121581, + 0.7316456618117463 + ], + "candidate_ci95": [ + 0.6534735207307115, + 0.6661281632812587 + ] + }, + "ece": { + "candidate_minus_parent": -0.08902474500832508, + "ci95": [ + -0.15777008194974856, + -0.032028872805473535 + ], + "parent_ci95": [ + 0.11917374104868377, + 0.19726621817887172 + ], + "candidate_ci95": [ + 0.02223594121005041, + 0.1014834181087044 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/iid/all": { + "windows": 182, + "episodes": 96, + "latent_groups": 32, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 36, + "2": 13, + "3": 24, + "4": 26, + "5": 41, + "6": 38, + "7": 4 + }, + "raw": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.7039948885030416, + "brier": 0.9366734114222882, + "ece": 0.3836640270561081 + }, + "candidate": { + "acc": 0.3342013888888889, + "nll": 1.0988623728398248, + "brier": 0.6670543370614523, + "ece": 0.06554797243603253 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0008680555555555802, + "ci95": [ + -0.0078125, + 0.010416666666666685 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3255208333333333, + 0.34375 + ] + }, + "nll": { + "candidate_minus_parent": -0.6051325156632168, + "ci95": [ + -0.6803591005740874, + -0.530793390037944 + ], + "parent_ci95": [ + 1.6258755139905854, + 1.7835939337351452 + ], + "candidate_ci95": [ + 1.0872959592876683, + 1.1104023477998943 + ] + }, + "brier": { + "candidate_minus_parent": -0.2696190743608359, + "ci95": [ + -0.2966854808852652, + -0.24314883056422498 + ], + "parent_ci95": [ + 0.9086695403765366, + 0.9664303635041863 + ], + "candidate_ci95": [ + 0.6585256055156145, + 0.6756684634800705 + ] + }, + "ece": { + "candidate_minus_parent": -0.31811605462007553, + "ci95": [ + -0.3467234011742797, + -0.2776120036720403 + ], + "parent_ci95": [ + 0.3658444671289674, + 0.4154924939910463 + ], + "candidate_ci95": [ + 0.055802842793600815, + 0.10411501636343842 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.1516241769020001, + "brier": 0.7011284844520596, + "ece": 0.1126117849166719 + }, + "candidate": { + "acc": 0.3342013888888889, + "nll": 1.094225810223573, + "brier": 0.6635708549658472, + "ece": 0.04216089485176933 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0008680555555555802, + "ci95": [ + -0.0078125, + 0.010416666666666685 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3255208333333333, + 0.34375 + ] + }, + "nll": { + "candidate_minus_parent": -0.05739836667842724, + "ci95": [ + -0.06962603452379744, + -0.04488653163402823 + ], + "parent_ci95": [ + 1.1374408274590326, + 1.166659736233086 + ], + "candidate_ci95": [ + 1.0877924378749446, + 1.1001794102062266 + ] + }, + "brier": { + "candidate_minus_parent": -0.03755762948621233, + "ci95": [ + -0.04590448711935209, + -0.02933488075617265 + ], + "parent_ci95": [ + 0.6916327369871181, + 0.7113507865114055 + ], + "candidate_ci95": [ + 0.6589866278321939, + 0.6679576727482305 + ] + }, + "ece": { + "candidate_minus_parent": -0.07045089006490257, + "ci95": [ + -0.10204058462593617, + -0.03478029248966403 + ], + "parent_ci95": [ + 0.10663147728974164, + 0.13809590546759193 + ], + "candidate_ci95": [ + 0.02485568463820158, + 0.07566025500259763 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/iid/transactions_1_2": { + "windows": 49, + "episodes": 45, + "latent_groups": 26, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 36, + "2": 13 + }, + "raw": { + "parent": { + "acc": 0.3111111111111111, + "nll": 1.6603323212661212, + "brier": 0.8907029573817137, + "ece": 0.37799735259416234 + }, + "candidate": { + "acc": 0.3888888888888889, + "nll": 1.1006948393144855, + "brier": 0.6675216739911485, + "ece": 0.02197163304242618 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.07777777777777778, + "ci95": [ + -0.12229166666666667, + 0.26746313102665914 + ], + "parent_ci95": [ + 0.20512820512820512, + 0.425 + ], + "candidate_ci95": [ + 0.28888888888888886, + 0.48984374999999974 + ] + }, + "nll": { + "candidate_minus_parent": -0.5596374819516357, + "ci95": [ + -0.8551553776743378, + -0.2835412394003477 + ], + "parent_ci95": [ + 1.3960780027038617, + 1.9415518588113363 + ], + "candidate_ci95": [ + 1.07961907447766, + 1.1190005382885808 + ] + }, + "brier": { + "candidate_minus_parent": -0.2231812833905652, + "ci95": [ + -0.36429527240524406, + -0.07974528756374481 + ], + "parent_ci95": [ + 0.7571302612447247, + 1.0218668962200836 + ], + "candidate_ci95": [ + 0.6529239682286327, + 0.6800289575226035 + ] + }, + "ece": { + "candidate_minus_parent": -0.3560257195517362, + "ci95": [ + -0.4212366104265594, + -0.19688232190049682 + ], + "parent_ci95": [ + 0.27558167651761906, + 0.4812973135389953 + ], + "candidate_ci95": [ + 0.005189460689758951, + 0.13286403298320185 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 26, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3111111111111111, + "nll": 1.1519141587962758, + "brier": 0.6972118196346561, + "ece": 0.12505951037027957 + }, + "candidate": { + "acc": 0.3888888888888889, + "nll": 1.0987834540411583, + "brier": 0.6666360545746582, + "ece": 0.03892731596092908 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.07777777777777778, + "ci95": [ + -0.12229166666666667, + 0.26746313102665914 + ], + "parent_ci95": [ + 0.20512820512820512, + 0.425 + ], + "candidate_ci95": [ + 0.28888888888888886, + 0.48984374999999974 + ] + }, + "nll": { + "candidate_minus_parent": -0.0531307047551175, + "ci95": [ + -0.13131384096372054, + 0.02030876595375617 + ], + "parent_ci95": [ + 1.0836231878965614, + 1.2232119820149001 + ], + "candidate_ci95": [ + 1.087965251457569, + 1.10803281863799 + ] + }, + "brier": { + "candidate_minus_parent": -0.030575765059997817, + "ci95": [ + -0.08053924500142515, + 0.016278720227622917 + ], + "parent_ci95": [ + 0.6536499526341443, + 0.7418442945165746 + ], + "candidate_ci95": [ + 0.6592384181572574, + 0.6728962291231819 + ] + }, + "ece": { + "candidate_minus_parent": -0.0861321944093505, + "ci95": [ + -0.1587048303302882, + 0.024643558296109955 + ], + "parent_ci95": [ + 0.02664428234558302, + 0.22892663757370982 + ], + "candidate_ci95": [ + 0.002359617988633989, + 0.14107288255595657 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 26, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/iid/transactions_3_plus": { + "windows": 133, + "episodes": 93, + "latent_groups": 32, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "3": 24, + "4": 26, + "5": 41, + "6": 38, + "7": 4 + }, + "raw": { + "parent": { + "acc": 0.3118279569892473, + "nll": 1.8007600408262734, + "brier": 0.9912122366031354, + "ece": 0.41777594278222113 + }, + "candidate": { + "acc": 0.34050179211469533, + "nll": 1.0897724637391926, + "brier": 0.6608081864876919, + "ece": 0.06945969138416776 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.02867383512544802, + "ci95": [ + -0.00347222222222221, + 0.06666666666666671 + ], + "parent_ci95": [ + 0.28888888888888886, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3263157894736842, + 0.35555555555555557 + ] + }, + "nll": { + "candidate_minus_parent": -0.7109875770870808, + "ci95": [ + -0.8249378106097072, + -0.5928019108611865 + ], + "parent_ci95": [ + 1.6800381011917274, + 1.9200895008447456 + ], + "candidate_ci95": [ + 1.0736534993392457, + 1.1068153124221158 + ] + }, + "brier": { + "candidate_minus_parent": -0.3304040501154435, + "ci95": [ + -0.3793116868541628, + -0.28256738524317954 + ], + "parent_ci95": [ + 0.9404112325282276, + 1.0398186656005743 + ], + "candidate_ci95": [ + 0.6494179878255043, + 0.6727965288233754 + ] + }, + "ece": { + "candidate_minus_parent": -0.3483162513980534, + "ci95": [ + -0.40399898613809715, + -0.29044166032303326 + ], + "parent_ci95": [ + 0.38626474035396524, + 0.473126989756557 + ], + "candidate_ci95": [ + 0.05996149153641762, + 0.11740475732652438 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3118279569892473, + "nll": 1.171633543085405, + "brier": 0.7154649932664467, + "ece": 0.14465601993335808 + }, + "candidate": { + "acc": 0.34050179211469533, + "nll": 1.088970247788087, + "brier": 0.6599466665496634, + "ece": 0.055631274925033 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.02867383512544802, + "ci95": [ + -0.00347222222222221, + 0.06666666666666671 + ], + "parent_ci95": [ + 0.28888888888888886, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3263157894736842, + 0.35555555555555557 + ] + }, + "nll": { + "candidate_minus_parent": -0.08266329529731808, + "ci95": [ + -0.10527160530287753, + -0.060813425309605006 + ], + "parent_ci95": [ + 1.1497915115579906, + 1.1953183319652614 + ], + "candidate_ci95": [ + 1.0798257200325903, + 1.0975630581184335 + ] + }, + "brier": { + "candidate_minus_parent": -0.05551832671678336, + "ci95": [ + -0.07142755322765607, + -0.040168620031008025 + ], + "parent_ci95": [ + 0.7003199621121581, + 0.7316456618117463 + ], + "candidate_ci95": [ + 0.6534735207307115, + 0.6661281632812587 + ] + }, + "ece": { + "candidate_minus_parent": -0.08902474500832508, + "ci95": [ + -0.15777008194974856, + -0.032028872805473535 + ], + "parent_ci95": [ + 0.11917374104868377, + 0.19726621817887172 + ], + "candidate_ci95": [ + 0.02223594121005041, + 0.1014834181087044 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/all/all": { + "windows": 148, + "episodes": 93, + "latent_groups": 32, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 28, + "2": 23, + "3": 20, + "4": 41, + "5": 14, + "6": 15, + "7": 7 + }, + "raw": { + "parent": { + "acc": 0.3064516129032258, + "nll": 1.703684494187649, + "brier": 0.9537202801199026, + "ece": 0.4129881914164849 + }, + "candidate": { + "acc": 0.3333333333333333, + "nll": 1.091192212148124, + "brier": 0.6614718541960379, + "ece": 0.06541034126839912 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.026881720430107503, + "ci95": [ + -0.012152777777777736, + 0.07222222222222224 + ], + "parent_ci95": [ + 0.2777777777777778, + 0.328125 + ], + "candidate_ci95": [ + 0.3120567375886525, + 0.3518518518518518 + ] + }, + "nll": { + "candidate_minus_parent": -0.6124922820395251, + "ci95": [ + -0.7118571153712705, + -0.5197225076836384 + ], + "parent_ci95": [ + 1.6099049018770706, + 1.8069063026502517 + ], + "candidate_ci95": [ + 1.0755336874829131, + 1.108735959121408 + ] + }, + "brier": { + "candidate_minus_parent": -0.2922484259238647, + "ci95": [ + -0.3463368357881872, + -0.24465821940647325 + ], + "parent_ci95": [ + 0.906523256055364, + 1.006988521580945 + ], + "candidate_ci95": [ + 0.6498969445817571, + 0.6745257883818703 + ] + }, + "ece": { + "candidate_minus_parent": -0.3475778501480858, + "ci95": [ + -0.39278455794407824, + -0.2746709901865958 + ], + "parent_ci95": [ + 0.38440294829434885, + 0.4484787433230376 + ], + "candidate_ci95": [ + 0.04616834389587247, + 0.12818960972025695 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3064516129032258, + "nll": 1.1529331119038668, + "brier": 0.7033109417652029, + "ece": 0.13902595443424992 + }, + "candidate": { + "acc": 0.3333333333333333, + "nll": 1.0916634954036812, + "brier": 0.661974002661268, + "ece": 0.05073703452140449 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.026881720430107503, + "ci95": [ + -0.012152777777777736, + 0.07222222222222224 + ], + "parent_ci95": [ + 0.2777777777777778, + 0.328125 + ], + "candidate_ci95": [ + 0.3120567375886525, + 0.3518518518518518 + ] + }, + "nll": { + "candidate_minus_parent": -0.061269616500185586, + "ci95": [ + -0.0815360160872045, + -0.042796021724386325 + ], + "parent_ci95": [ + 1.1342530883801942, + 1.1746089245476712 + ], + "candidate_ci95": [ + 1.0841653219251408, + 1.099847160901972 + ] + }, + "brier": { + "candidate_minus_parent": -0.04133693910393488, + "ci95": [ + -0.054945952000430426, + -0.0283071065059587 + ], + "parent_ci95": [ + 0.6901203817927404, + 0.7179063393968591 + ], + "candidate_ci95": [ + 0.6566300081821984, + 0.6678751694064711 + ] + }, + "ece": { + "candidate_minus_parent": -0.08828891991284543, + "ci95": [ + -0.13567816051255593, + -0.03214176910943423 + ], + "parent_ci95": [ + 0.11565183132825865, + 0.17029153615689344 + ], + "candidate_ci95": [ + 0.020278667356217008, + 0.09905536546083844 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/all/transactions_1_2": { + "windows": 51, + "episodes": 49, + "latent_groups": 27, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 28, + "2": 23 + }, + "raw": { + "parent": { + "acc": 0.3469387755102041, + "nll": 1.5751052969851513, + "brier": 0.8802942551419999, + "ece": 0.35030273260170447 + }, + "candidate": { + "acc": 0.2857142857142857, + "nll": 1.1068816661995116, + "brier": 0.6720929561492037, + "ece": 0.08071820859513017 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.06122448979591838, + "ci95": [ + -0.20010204081632654, + 0.08510638297872342 + ], + "parent_ci95": [ + 0.2640947888589398, + 0.4318181818181818 + ], + "candidate_ci95": [ + 0.19990196078431374, + 0.36958534621578093 + ] + }, + "nll": { + "candidate_minus_parent": -0.4682236307856398, + "ci95": [ + -0.6874408585543053, + -0.25059914312634723 + ], + "parent_ci95": [ + 1.3660307447207953, + 1.7863481180513798 + ], + "candidate_ci95": [ + 1.0954611962384864, + 1.1205296358359653 + ] + }, + "brier": { + "candidate_minus_parent": -0.20820129899279616, + "ci95": [ + -0.3214504033746869, + -0.09218633279444037 + ], + "parent_ci95": [ + 0.7681875137354262, + 0.9905470857848545 + ], + "candidate_ci95": [ + 0.6642986501083062, + 0.6812307682914585 + ] + }, + "ece": { + "candidate_minus_parent": -0.2695845240065743, + "ci95": [ + -0.3904225741241218, + -0.13182069065959087 + ], + "parent_ci95": [ + 0.2684022133739064, + 0.4331878336118929 + ], + "candidate_ci95": [ + 0.015381404617433004, + 0.16679240726105588 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 27, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3469387755102041, + "nll": 1.1306501839467744, + "brier": 0.6866799099052595, + "ece": 0.1003196557889501 + }, + "candidate": { + "acc": 0.2857142857142857, + "nll": 1.101825014834903, + "brier": 0.6687795902311308, + "ece": 0.0639734338035598 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.06122448979591838, + "ci95": [ + -0.20010204081632654, + 0.08510638297872342 + ], + "parent_ci95": [ + 0.2640947888589398, + 0.4318181818181818 + ], + "candidate_ci95": [ + 0.19990196078431374, + 0.36958534621578093 + ] + }, + "nll": { + "candidate_minus_parent": -0.028825169111871496, + "ci95": [ + -0.08287051694186583, + 0.02922920870399774 + ], + "parent_ci95": [ + 1.0776980040903783, + 1.1807414922599382 + ], + "candidate_ci95": [ + 1.0959848691935723, + 1.1086465278602198 + ] + }, + "brier": { + "candidate_minus_parent": -0.01790031967412875, + "ci95": [ + -0.05386955921741074, + 0.020204736241254864 + ], + "parent_ci95": [ + 0.6515174635881781, + 0.7204922749907883 + ], + "candidate_ci95": [ + 0.6648958894226078, + 0.6733526771528309 + ] + }, + "ece": { + "candidate_minus_parent": -0.036346221985390295, + "ci95": [ + -0.1650347028291009, + 0.09366396409555656 + ], + "parent_ci95": [ + 0.024516912704449893, + 0.1972711957182588 + ], + "candidate_ci95": [ + 0.0035596289835879147, + 0.1498403229449324 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 27, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/all/transactions_3_plus": { + "windows": 97, + "episodes": 75, + "latent_groups": 29, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "3": 20, + "4": 41, + "5": 14, + "6": 15, + "7": 7 + }, + "raw": { + "parent": { + "acc": 0.28, + "nll": 1.7666116286336437, + "brier": 0.9763766869892648, + "ece": 0.4315626097972386 + }, + "candidate": { + "acc": 0.36666666666666664, + "nll": 1.0864975658938738, + "brier": 0.6587212588511653, + "ece": 0.044820232890561064 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.08666666666666661, + "ci95": [ + 0.024691358024691357, + 0.16911764705882354 + ], + "parent_ci95": [ + 0.23680340557275542, + 0.3125 + ], + "candidate_ci95": [ + 0.3291139240506329, + 0.41304347826086957 + ] + }, + "nll": { + "candidate_minus_parent": -0.6801140627397699, + "ci95": [ + -0.852694438368166, + -0.5364090061436311 + ], + "parent_ci95": [ + 1.6363811561091484, + 1.9277416812066965 + ], + "candidate_ci95": [ + 1.0565640880278935, + 1.1122708307393117 + ] + }, + "brier": { + "candidate_minus_parent": -0.31765542813809955, + "ci95": [ + -0.4055508030920121, + -0.24603262483477328 + ], + "parent_ci95": [ + 0.9125704809328496, + 1.0576784187561903 + ], + "candidate_ci95": [ + 0.6375226558255674, + 0.677367412345545 + ] + }, + "ece": { + "candidate_minus_parent": -0.3867423769066775, + "ci95": [ + -0.43813835904088466, + -0.299041074051707 + ], + "parent_ci95": [ + 0.38732821796041755, + 0.4880029459977231 + ], + "candidate_ci95": [ + 0.027281818992051166, + 0.12376763110665068 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 29, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.28, + "nll": 1.1695120669775345, + "brier": 0.7138176453650833, + "ece": 0.16388566024997767 + }, + "candidate": { + "acc": 0.36666666666666664, + "nll": 1.0880931619979777, + "brier": 0.6597052729288035, + "ece": 0.020713258604670517 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.08666666666666661, + "ci95": [ + 0.024691358024691357, + 0.16911764705882354 + ], + "parent_ci95": [ + 0.23680340557275542, + 0.3125 + ], + "candidate_ci95": [ + 0.3291139240506329, + 0.41304347826086957 + ] + }, + "nll": { + "candidate_minus_parent": -0.08141890497955684, + "ci95": [ + -0.12091422490095782, + -0.04948136248698705 + ], + "parent_ci95": [ + 1.143868559330941, + 1.2022605906182802 + ], + "candidate_ci95": [ + 1.0734995012679627, + 1.1004991802450863 + ] + }, + "brier": { + "candidate_minus_parent": -0.054112372436279754, + "ci95": [ + -0.08087458322037622, + -0.03322586306298517 + ], + "parent_ci95": [ + 0.6965512084324665, + 0.7354555338983582 + ], + "candidate_ci95": [ + 0.6493246096824813, + 0.6685547109622129 + ] + }, + "ece": { + "candidate_minus_parent": -0.14317240164530715, + "ci95": [ + -0.17492207253743788, + -0.08263253263041898 + ], + "parent_ci95": [ + 0.12860078933903504, + 0.20826683422962747 + ], + "candidate_ci95": [ + 0.009737212666042754, + 0.07540991325472185 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 29, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/iid/all": { + "windows": 148, + "episodes": 93, + "latent_groups": 32, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 28, + "2": 23, + "3": 20, + "4": 41, + "5": 14, + "6": 15, + "7": 7 + }, + "raw": { + "parent": { + "acc": 0.3064516129032258, + "nll": 1.703684494187649, + "brier": 0.9537202801199026, + "ece": 0.4129881914164849 + }, + "candidate": { + "acc": 0.3333333333333333, + "nll": 1.091192212148124, + "brier": 0.6614718541960379, + "ece": 0.06541034126839912 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.026881720430107503, + "ci95": [ + -0.012152777777777736, + 0.07222222222222224 + ], + "parent_ci95": [ + 0.2777777777777778, + 0.328125 + ], + "candidate_ci95": [ + 0.3120567375886525, + 0.3518518518518518 + ] + }, + "nll": { + "candidate_minus_parent": -0.6124922820395251, + "ci95": [ + -0.7118571153712705, + -0.5197225076836384 + ], + "parent_ci95": [ + 1.6099049018770706, + 1.8069063026502517 + ], + "candidate_ci95": [ + 1.0755336874829131, + 1.108735959121408 + ] + }, + "brier": { + "candidate_minus_parent": -0.2922484259238647, + "ci95": [ + -0.3463368357881872, + -0.24465821940647325 + ], + "parent_ci95": [ + 0.906523256055364, + 1.006988521580945 + ], + "candidate_ci95": [ + 0.6498969445817571, + 0.6745257883818703 + ] + }, + "ece": { + "candidate_minus_parent": -0.3475778501480858, + "ci95": [ + -0.39278455794407824, + -0.2746709901865958 + ], + "parent_ci95": [ + 0.38440294829434885, + 0.4484787433230376 + ], + "candidate_ci95": [ + 0.04616834389587247, + 0.12818960972025695 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3064516129032258, + "nll": 1.1529331119038668, + "brier": 0.7033109417652029, + "ece": 0.13902595443424992 + }, + "candidate": { + "acc": 0.3333333333333333, + "nll": 1.0916634954036812, + "brier": 0.661974002661268, + "ece": 0.05073703452140449 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.026881720430107503, + "ci95": [ + -0.012152777777777736, + 0.07222222222222224 + ], + "parent_ci95": [ + 0.2777777777777778, + 0.328125 + ], + "candidate_ci95": [ + 0.3120567375886525, + 0.3518518518518518 + ] + }, + "nll": { + "candidate_minus_parent": -0.061269616500185586, + "ci95": [ + -0.0815360160872045, + -0.042796021724386325 + ], + "parent_ci95": [ + 1.1342530883801942, + 1.1746089245476712 + ], + "candidate_ci95": [ + 1.0841653219251408, + 1.099847160901972 + ] + }, + "brier": { + "candidate_minus_parent": -0.04133693910393488, + "ci95": [ + -0.054945952000430426, + -0.0283071065059587 + ], + "parent_ci95": [ + 0.6901203817927404, + 0.7179063393968591 + ], + "candidate_ci95": [ + 0.6566300081821984, + 0.6678751694064711 + ] + }, + "ece": { + "candidate_minus_parent": -0.08828891991284543, + "ci95": [ + -0.13567816051255593, + -0.03214176910943423 + ], + "parent_ci95": [ + 0.11565183132825865, + 0.17029153615689344 + ], + "candidate_ci95": [ + 0.020278667356217008, + 0.09905536546083844 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/iid/transactions_1_2": { + "windows": 51, + "episodes": 49, + "latent_groups": 27, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 28, + "2": 23 + }, + "raw": { + "parent": { + "acc": 0.3469387755102041, + "nll": 1.5751052969851513, + "brier": 0.8802942551419999, + "ece": 0.35030273260170447 + }, + "candidate": { + "acc": 0.2857142857142857, + "nll": 1.1068816661995116, + "brier": 0.6720929561492037, + "ece": 0.08071820859513017 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.06122448979591838, + "ci95": [ + -0.20010204081632654, + 0.08510638297872342 + ], + "parent_ci95": [ + 0.2640947888589398, + 0.4318181818181818 + ], + "candidate_ci95": [ + 0.19990196078431374, + 0.36958534621578093 + ] + }, + "nll": { + "candidate_minus_parent": -0.4682236307856398, + "ci95": [ + -0.6874408585543053, + -0.25059914312634723 + ], + "parent_ci95": [ + 1.3660307447207953, + 1.7863481180513798 + ], + "candidate_ci95": [ + 1.0954611962384864, + 1.1205296358359653 + ] + }, + "brier": { + "candidate_minus_parent": -0.20820129899279616, + "ci95": [ + -0.3214504033746869, + -0.09218633279444037 + ], + "parent_ci95": [ + 0.7681875137354262, + 0.9905470857848545 + ], + "candidate_ci95": [ + 0.6642986501083062, + 0.6812307682914585 + ] + }, + "ece": { + "candidate_minus_parent": -0.2695845240065743, + "ci95": [ + -0.3904225741241218, + -0.13182069065959087 + ], + "parent_ci95": [ + 0.2684022133739064, + 0.4331878336118929 + ], + "candidate_ci95": [ + 0.015381404617433004, + 0.16679240726105588 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 27, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3469387755102041, + "nll": 1.1306501839467744, + "brier": 0.6866799099052595, + "ece": 0.1003196557889501 + }, + "candidate": { + "acc": 0.2857142857142857, + "nll": 1.101825014834903, + "brier": 0.6687795902311308, + "ece": 0.0639734338035598 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.06122448979591838, + "ci95": [ + -0.20010204081632654, + 0.08510638297872342 + ], + "parent_ci95": [ + 0.2640947888589398, + 0.4318181818181818 + ], + "candidate_ci95": [ + 0.19990196078431374, + 0.36958534621578093 + ] + }, + "nll": { + "candidate_minus_parent": -0.028825169111871496, + "ci95": [ + -0.08287051694186583, + 0.02922920870399774 + ], + "parent_ci95": [ + 1.0776980040903783, + 1.1807414922599382 + ], + "candidate_ci95": [ + 1.0959848691935723, + 1.1086465278602198 + ] + }, + "brier": { + "candidate_minus_parent": -0.01790031967412875, + "ci95": [ + -0.05386955921741074, + 0.020204736241254864 + ], + "parent_ci95": [ + 0.6515174635881781, + 0.7204922749907883 + ], + "candidate_ci95": [ + 0.6648958894226078, + 0.6733526771528309 + ] + }, + "ece": { + "candidate_minus_parent": -0.036346221985390295, + "ci95": [ + -0.1650347028291009, + 0.09366396409555656 + ], + "parent_ci95": [ + 0.024516912704449893, + 0.1972711957182588 + ], + "candidate_ci95": [ + 0.0035596289835879147, + 0.1498403229449324 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 27, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/iid/transactions_3_plus": { + "windows": 97, + "episodes": 75, + "latent_groups": 29, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "3": 20, + "4": 41, + "5": 14, + "6": 15, + "7": 7 + }, + "raw": { + "parent": { + "acc": 0.28, + "nll": 1.7666116286336437, + "brier": 0.9763766869892648, + "ece": 0.4315626097972386 + }, + "candidate": { + "acc": 0.36666666666666664, + "nll": 1.0864975658938738, + "brier": 0.6587212588511653, + "ece": 0.044820232890561064 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.08666666666666661, + "ci95": [ + 0.024691358024691357, + 0.16911764705882354 + ], + "parent_ci95": [ + 0.23680340557275542, + 0.3125 + ], + "candidate_ci95": [ + 0.3291139240506329, + 0.41304347826086957 + ] + }, + "nll": { + "candidate_minus_parent": -0.6801140627397699, + "ci95": [ + -0.852694438368166, + -0.5364090061436311 + ], + "parent_ci95": [ + 1.6363811561091484, + 1.9277416812066965 + ], + "candidate_ci95": [ + 1.0565640880278935, + 1.1122708307393117 + ] + }, + "brier": { + "candidate_minus_parent": -0.31765542813809955, + "ci95": [ + -0.4055508030920121, + -0.24603262483477328 + ], + "parent_ci95": [ + 0.9125704809328496, + 1.0576784187561903 + ], + "candidate_ci95": [ + 0.6375226558255674, + 0.677367412345545 + ] + }, + "ece": { + "candidate_minus_parent": -0.3867423769066775, + "ci95": [ + -0.43813835904088466, + -0.299041074051707 + ], + "parent_ci95": [ + 0.38732821796041755, + 0.4880029459977231 + ], + "candidate_ci95": [ + 0.027281818992051166, + 0.12376763110665068 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 29, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.28, + "nll": 1.1695120669775345, + "brier": 0.7138176453650833, + "ece": 0.16388566024997767 + }, + "candidate": { + "acc": 0.36666666666666664, + "nll": 1.0880931619979777, + "brier": 0.6597052729288035, + "ece": 0.020713258604670517 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.08666666666666661, + "ci95": [ + 0.024691358024691357, + 0.16911764705882354 + ], + "parent_ci95": [ + 0.23680340557275542, + 0.3125 + ], + "candidate_ci95": [ + 0.3291139240506329, + 0.41304347826086957 + ] + }, + "nll": { + "candidate_minus_parent": -0.08141890497955684, + "ci95": [ + -0.12091422490095782, + -0.04948136248698705 + ], + "parent_ci95": [ + 1.143868559330941, + 1.2022605906182802 + ], + "candidate_ci95": [ + 1.0734995012679627, + 1.1004991802450863 + ] + }, + "brier": { + "candidate_minus_parent": -0.054112372436279754, + "ci95": [ + -0.08087458322037622, + -0.03322586306298517 + ], + "parent_ci95": [ + 0.6965512084324665, + 0.7354555338983582 + ], + "candidate_ci95": [ + 0.6493246096824813, + 0.6685547109622129 + ] + }, + "ece": { + "candidate_minus_parent": -0.14317240164530715, + "ci95": [ + -0.17492207253743788, + -0.08263253263041898 + ], + "parent_ci95": [ + 0.12860078933903504, + 0.20826683422962747 + ], + "candidate_ci95": [ + 0.009737212666042754, + 0.07540991325472185 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 29, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/all/all": { + "windows": 152, + "episodes": 96, + "latent_groups": 32, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 27, + "2": 18, + "3": 24, + "4": 18, + "5": 42, + "6": 20, + "7": 3 + }, + "raw": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.6730827021190828, + "brier": 0.9271168690450743, + "ece": 0.38029185976877056 + }, + "candidate": { + "acc": 0.3489583333333333, + "nll": 1.0971521242555076, + "brier": 0.6658498861990656, + "ece": 0.046307694308096524 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.015625, + "ci95": [ + -0.00347222222222221, + 0.04171006944444442 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3298611111111111, + 0.37504340277777776 + ] + }, + "nll": { + "candidate_minus_parent": -0.5759305778635753, + "ci95": [ + -0.6646879978333292, + -0.4844753658625933 + ], + "parent_ci95": [ + 1.575759527226761, + 1.7691376887298536 + ], + "candidate_ci95": [ + 1.081862216974002, + 1.1148355704805166 + ] + }, + "brier": { + "candidate_minus_parent": -0.2612669828460087, + "ci95": [ + -0.2970389567438269, + -0.22634178418755768 + ], + "parent_ci95": [ + 0.8887463241160158, + 0.9676440292791192 + ], + "candidate_ci95": [ + 0.6550728672970835, + 0.6783244126276657 + ] + }, + "ece": { + "candidate_minus_parent": -0.33398416546067405, + "ci95": [ + -0.3670358118919271, + -0.2816153662787594 + ], + "parent_ci95": [ + 0.35101328311005003, + 0.42006858974136496 + ], + "candidate_ci95": [ + 0.024396575640987968, + 0.09070083025698743 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.1468427816323261, + "brier": 0.6981381709092208, + "ece": 0.10938489002382056 + }, + "candidate": { + "acc": 0.3489583333333333, + "nll": 1.0937687968736984, + "brier": 0.663354598517312, + "ece": 0.019037355871080977 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.015625, + "ci95": [ + -0.00347222222222221, + 0.04171006944444442 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3298611111111111, + 0.37504340277777776 + ] + }, + "nll": { + "candidate_minus_parent": -0.05307398475862768, + "ci95": [ + -0.06656368181703105, + -0.04064767349269174 + ], + "parent_ci95": [ + 1.131378684831578, + 1.165777015737088 + ], + "candidate_ci95": [ + 1.0857767794541715, + 1.102658732735975 + ] + }, + "brier": { + "candidate_minus_parent": -0.03478357239190877, + "ci95": [ + -0.04339052468546176, + -0.026690749871374275 + ], + "parent_ci95": [ + 0.6877235900544243, + 0.7106051033335131 + ], + "candidate_ci95": [ + 0.6576958496027633, + 0.66980743900186 + ] + }, + "ece": { + "candidate_minus_parent": -0.09034753415273958, + "ci95": [ + -0.13920480289089912, + -0.05692976019192213 + ], + "parent_ci95": [ + 0.10134604571995985, + 0.15887011140561966 + ], + "candidate_ci95": [ + 0.0056846758306226585, + 0.05005291241006455 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/all/transactions_1_2": { + "windows": 45, + "episodes": 42, + "latent_groups": 23, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 27, + "2": 18 + }, + "raw": { + "parent": { + "acc": 0.30952380952380953, + "nll": 1.706998903645994, + "brier": 0.9212914627123131, + "ece": 0.3949219477940342 + }, + "candidate": { + "acc": 0.40476190476190477, + "nll": 1.0960532065103286, + "brier": 0.6644578498966356, + "ece": 0.0515361557378001 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.09523809523809523, + "ci95": [ + -0.10256410256410253, + 0.27088068181818176 + ], + "parent_ci95": [ + 0.20512820512820512, + 0.4286525974025973 + ], + "candidate_ci95": [ + 0.309478021978022, + 0.5 + ] + }, + "nll": { + "candidate_minus_parent": -0.6109456971356655, + "ci95": [ + -0.8672339381424512, + -0.3446385051769663 + ], + "parent_ci95": [ + 1.4471843049380726, + 1.9533243982325792 + ], + "candidate_ci95": [ + 1.0794300015550526, + 1.1111172236579347 + ] + }, + "brier": { + "candidate_minus_parent": -0.2568336128156775, + "ci95": [ + -0.3817285469348027, + -0.10872128206234798 + ], + "parent_ci95": [ + 0.7821920851336587, + 1.044244265219641 + ], + "candidate_ci95": [ + 0.6532393456517698, + 0.6748604671268419 + ] + }, + "ece": { + "candidate_minus_parent": -0.3433857920562341, + "ci95": [ + -0.41543430112359964, + -0.24068696649838917 + ], + "parent_ci95": [ + 0.29114499760835, + 0.49447586904298535 + ], + "candidate_ci95": [ + 0.007441025650636024, + 0.1472801418978136 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 23, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.30952380952380953, + "nll": 1.158429211230326, + "brier": 0.7024608643728002, + "ece": 0.1313191003771914 + }, + "candidate": { + "acc": 0.40476190476190477, + "nll": 1.0963778788315839, + "brier": 0.665038046030096, + "ece": 0.05532362839396711 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.09523809523809523, + "ci95": [ + -0.10256410256410253, + 0.27088068181818176 + ], + "parent_ci95": [ + 0.20512820512820512, + 0.4286525974025973 + ], + "candidate_ci95": [ + 0.309478021978022, + 0.5 + ] + }, + "nll": { + "candidate_minus_parent": -0.062051332398742165, + "ci95": [ + -0.1297682515406164, + 0.008765126062274106 + ], + "parent_ci95": [ + 1.0925163708898833, + 1.2213072824062752 + ], + "candidate_ci95": [ + 1.0882109002505502, + 1.1040158253555261 + ] + }, + "brier": { + "candidate_minus_parent": -0.03742281834270422, + "ci95": [ + -0.08094726147904728, + 0.010873660088425609 + ], + "parent_ci95": [ + 0.6587283528455579, + 0.7432459424477127 + ], + "candidate_ci95": [ + 0.6594927791043819, + 0.6701795801066367 + ] + }, + "ece": { + "candidate_minus_parent": -0.0759954719832243, + "ci95": [ + -0.15320873787904174, + 0.022610033698028144 + ], + "parent_ci95": [ + 0.022382031820028063, + 0.23260449715420847 + ], + "candidate_ci95": [ + 0.0017389176176505234, + 0.1504230264430541 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 23, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/all/transactions_3_plus": { + "windows": 107, + "episodes": 82, + "latent_groups": 29, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "3": 24, + "4": 18, + "5": 42, + "6": 20, + "7": 3 + }, + "raw": { + "parent": { + "acc": 0.3048780487804878, + "nll": 1.7535274874757134, + "brier": 0.9723947060770417, + "ece": 0.3981191573176271 + }, + "candidate": { + "acc": 0.3556910569105691, + "nll": 1.0900251597170125, + "brier": 0.6611333300051692, + "ece": 0.05145502825345785 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.05081300813008133, + "ci95": [ + 0.007843137254901988, + 0.10685654008438818 + ], + "parent_ci95": [ + 0.26582278481012656, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3373004201680672, + 0.3782051282051282 + ] + }, + "nll": { + "candidate_minus_parent": -0.6635023277587009, + "ci95": [ + -0.8021584250061905, + -0.5285921340395728 + ], + "parent_ci95": [ + 1.6133860025871591, + 1.8956209466593967 + ], + "candidate_ci95": [ + 1.0718745522776276, + 1.1109697671138516 + ] + }, + "brier": { + "candidate_minus_parent": -0.31126137607187254, + "ci95": [ + -0.372040771277964, + -0.25304541752048104 + ], + "parent_ci95": [ + 0.9123559470337855, + 1.0348753860773976 + ], + "candidate_ci95": [ + 0.6480259058780655, + 0.6761502857169847 + ] + }, + "ece": { + "candidate_minus_parent": -0.3466641290641692, + "ci95": [ + -0.4069328888385726, + -0.27068127803160935 + ], + "parent_ci95": [ + 0.3522997946155402, + 0.45991410556399387 + ], + "candidate_ci95": [ + 0.03254665904019653, + 0.13149811142527223 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 29, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3048780487804878, + "nll": 1.1655763567531447, + "brier": 0.7114093944169242, + "ece": 0.13822239288209293 + }, + "candidate": { + "acc": 0.3556910569105691, + "nll": 1.0893491093060668, + "brier": 0.6603500514339845, + "ece": 0.018875857923770827 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.05081300813008133, + "ci95": [ + 0.007843137254901988, + 0.10685654008438818 + ], + "parent_ci95": [ + 0.26582278481012656, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3373004201680672, + 0.3782051282051282 + ] + }, + "nll": { + "candidate_minus_parent": -0.07622724744707798, + "ci95": [ + -0.10280872643452063, + -0.05400454009262088 + ], + "parent_ci95": [ + 1.1411099048409659, + 1.1933966733283992 + ], + "candidate_ci95": [ + 1.079626128827174, + 1.0995762112259768 + ] + }, + "brier": { + "candidate_minus_parent": -0.05105934298293968, + "ci95": [ + -0.06877127001477751, + -0.03603397210843332 + ], + "parent_ci95": [ + 0.694767814212102, + 0.7297861619784731 + ], + "candidate_ci95": [ + 0.6533625375594448, + 0.6676370103584927 + ] + }, + "ece": { + "candidate_minus_parent": -0.11934653495832209, + "ci95": [ + -0.17057654135184128, + -0.06356932675465357 + ], + "parent_ci95": [ + 0.1076936579270957, + 0.18985276072341845 + ], + "candidate_ci95": [ + 0.005858512718669687, + 0.054791348636670234 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 29, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/iid/all": { + "windows": 152, + "episodes": 96, + "latent_groups": 32, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 27, + "2": 18, + "3": 24, + "4": 18, + "5": 42, + "6": 20, + "7": 3 + }, + "raw": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.6730827021190828, + "brier": 0.9271168690450743, + "ece": 0.38029185976877056 + }, + "candidate": { + "acc": 0.3489583333333333, + "nll": 1.0971521242555076, + "brier": 0.6658498861990656, + "ece": 0.046307694308096524 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.015625, + "ci95": [ + -0.00347222222222221, + 0.04171006944444442 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3298611111111111, + 0.37504340277777776 + ] + }, + "nll": { + "candidate_minus_parent": -0.5759305778635753, + "ci95": [ + -0.6646879978333292, + -0.4844753658625933 + ], + "parent_ci95": [ + 1.575759527226761, + 1.7691376887298536 + ], + "candidate_ci95": [ + 1.081862216974002, + 1.1148355704805166 + ] + }, + "brier": { + "candidate_minus_parent": -0.2612669828460087, + "ci95": [ + -0.2970389567438269, + -0.22634178418755768 + ], + "parent_ci95": [ + 0.8887463241160158, + 0.9676440292791192 + ], + "candidate_ci95": [ + 0.6550728672970835, + 0.6783244126276657 + ] + }, + "ece": { + "candidate_minus_parent": -0.33398416546067405, + "ci95": [ + -0.3670358118919271, + -0.2816153662787594 + ], + "parent_ci95": [ + 0.35101328311005003, + 0.42006858974136496 + ], + "candidate_ci95": [ + 0.024396575640987968, + 0.09070083025698743 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.1468427816323261, + "brier": 0.6981381709092208, + "ece": 0.10938489002382056 + }, + "candidate": { + "acc": 0.3489583333333333, + "nll": 1.0937687968736984, + "brier": 0.663354598517312, + "ece": 0.019037355871080977 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.015625, + "ci95": [ + -0.00347222222222221, + 0.04171006944444442 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3298611111111111, + 0.37504340277777776 + ] + }, + "nll": { + "candidate_minus_parent": -0.05307398475862768, + "ci95": [ + -0.06656368181703105, + -0.04064767349269174 + ], + "parent_ci95": [ + 1.131378684831578, + 1.165777015737088 + ], + "candidate_ci95": [ + 1.0857767794541715, + 1.102658732735975 + ] + }, + "brier": { + "candidate_minus_parent": -0.03478357239190877, + "ci95": [ + -0.04339052468546176, + -0.026690749871374275 + ], + "parent_ci95": [ + 0.6877235900544243, + 0.7106051033335131 + ], + "candidate_ci95": [ + 0.6576958496027633, + 0.66980743900186 + ] + }, + "ece": { + "candidate_minus_parent": -0.09034753415273958, + "ci95": [ + -0.13920480289089912, + -0.05692976019192213 + ], + "parent_ci95": [ + 0.10134604571995985, + 0.15887011140561966 + ], + "candidate_ci95": [ + 0.0056846758306226585, + 0.05005291241006455 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/iid/transactions_1_2": { + "windows": 45, + "episodes": 42, + "latent_groups": 23, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 27, + "2": 18 + }, + "raw": { + "parent": { + "acc": 0.30952380952380953, + "nll": 1.706998903645994, + "brier": 0.9212914627123131, + "ece": 0.3949219477940342 + }, + "candidate": { + "acc": 0.40476190476190477, + "nll": 1.0960532065103286, + "brier": 0.6644578498966356, + "ece": 0.0515361557378001 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.09523809523809523, + "ci95": [ + -0.10256410256410253, + 0.27088068181818176 + ], + "parent_ci95": [ + 0.20512820512820512, + 0.4286525974025973 + ], + "candidate_ci95": [ + 0.309478021978022, + 0.5 + ] + }, + "nll": { + "candidate_minus_parent": -0.6109456971356655, + "ci95": [ + -0.8672339381424512, + -0.3446385051769663 + ], + "parent_ci95": [ + 1.4471843049380726, + 1.9533243982325792 + ], + "candidate_ci95": [ + 1.0794300015550526, + 1.1111172236579347 + ] + }, + "brier": { + "candidate_minus_parent": -0.2568336128156775, + "ci95": [ + -0.3817285469348027, + -0.10872128206234798 + ], + "parent_ci95": [ + 0.7821920851336587, + 1.044244265219641 + ], + "candidate_ci95": [ + 0.6532393456517698, + 0.6748604671268419 + ] + }, + "ece": { + "candidate_minus_parent": -0.3433857920562341, + "ci95": [ + -0.41543430112359964, + -0.24068696649838917 + ], + "parent_ci95": [ + 0.29114499760835, + 0.49447586904298535 + ], + "candidate_ci95": [ + 0.007441025650636024, + 0.1472801418978136 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 23, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.30952380952380953, + "nll": 1.158429211230326, + "brier": 0.7024608643728002, + "ece": 0.1313191003771914 + }, + "candidate": { + "acc": 0.40476190476190477, + "nll": 1.0963778788315839, + "brier": 0.665038046030096, + "ece": 0.05532362839396711 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.09523809523809523, + "ci95": [ + -0.10256410256410253, + 0.27088068181818176 + ], + "parent_ci95": [ + 0.20512820512820512, + 0.4286525974025973 + ], + "candidate_ci95": [ + 0.309478021978022, + 0.5 + ] + }, + "nll": { + "candidate_minus_parent": -0.062051332398742165, + "ci95": [ + -0.1297682515406164, + 0.008765126062274106 + ], + "parent_ci95": [ + 1.0925163708898833, + 1.2213072824062752 + ], + "candidate_ci95": [ + 1.0882109002505502, + 1.1040158253555261 + ] + }, + "brier": { + "candidate_minus_parent": -0.03742281834270422, + "ci95": [ + -0.08094726147904728, + 0.010873660088425609 + ], + "parent_ci95": [ + 0.6587283528455579, + 0.7432459424477127 + ], + "candidate_ci95": [ + 0.6594927791043819, + 0.6701795801066367 + ] + }, + "ece": { + "candidate_minus_parent": -0.0759954719832243, + "ci95": [ + -0.15320873787904174, + 0.022610033698028144 + ], + "parent_ci95": [ + 0.022382031820028063, + 0.23260449715420847 + ], + "candidate_ci95": [ + 0.0017389176176505234, + 0.1504230264430541 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 23, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/iid/transactions_3_plus": { + "windows": 107, + "episodes": 82, + "latent_groups": 29, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "3": 24, + "4": 18, + "5": 42, + "6": 20, + "7": 3 + }, + "raw": { + "parent": { + "acc": 0.3048780487804878, + "nll": 1.7535274874757134, + "brier": 0.9723947060770417, + "ece": 0.3981191573176271 + }, + "candidate": { + "acc": 0.3556910569105691, + "nll": 1.0900251597170125, + "brier": 0.6611333300051692, + "ece": 0.05145502825345785 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.05081300813008133, + "ci95": [ + 0.007843137254901988, + 0.10685654008438818 + ], + "parent_ci95": [ + 0.26582278481012656, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3373004201680672, + 0.3782051282051282 + ] + }, + "nll": { + "candidate_minus_parent": -0.6635023277587009, + "ci95": [ + -0.8021584250061905, + -0.5285921340395728 + ], + "parent_ci95": [ + 1.6133860025871591, + 1.8956209466593967 + ], + "candidate_ci95": [ + 1.0718745522776276, + 1.1109697671138516 + ] + }, + "brier": { + "candidate_minus_parent": -0.31126137607187254, + "ci95": [ + -0.372040771277964, + -0.25304541752048104 + ], + "parent_ci95": [ + 0.9123559470337855, + 1.0348753860773976 + ], + "candidate_ci95": [ + 0.6480259058780655, + 0.6761502857169847 + ] + }, + "ece": { + "candidate_minus_parent": -0.3466641290641692, + "ci95": [ + -0.4069328888385726, + -0.27068127803160935 + ], + "parent_ci95": [ + 0.3522997946155402, + 0.45991410556399387 + ], + "candidate_ci95": [ + 0.03254665904019653, + 0.13149811142527223 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 29, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3048780487804878, + "nll": 1.1655763567531447, + "brier": 0.7114093944169242, + "ece": 0.13822239288209293 + }, + "candidate": { + "acc": 0.3556910569105691, + "nll": 1.0893491093060668, + "brier": 0.6603500514339845, + "ece": 0.018875857923770827 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.05081300813008133, + "ci95": [ + 0.007843137254901988, + 0.10685654008438818 + ], + "parent_ci95": [ + 0.26582278481012656, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3373004201680672, + 0.3782051282051282 + ] + }, + "nll": { + "candidate_minus_parent": -0.07622724744707798, + "ci95": [ + -0.10280872643452063, + -0.05400454009262088 + ], + "parent_ci95": [ + 1.1411099048409659, + 1.1933966733283992 + ], + "candidate_ci95": [ + 1.079626128827174, + 1.0995762112259768 + ] + }, + "brier": { + "candidate_minus_parent": -0.05105934298293968, + "ci95": [ + -0.06877127001477751, + -0.03603397210843332 + ], + "parent_ci95": [ + 0.694767814212102, + 0.7297861619784731 + ], + "candidate_ci95": [ + 0.6533625375594448, + 0.6676370103584927 + ] + }, + "ece": { + "candidate_minus_parent": -0.11934653495832209, + "ci95": [ + -0.17057654135184128, + -0.06356932675465357 + ], + "parent_ci95": [ + 0.1076936579270957, + 0.18985276072341845 + ], + "candidate_ci95": [ + 0.005858512718669687, + 0.054791348636670234 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 29, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + } + }, + "development": { + "full/all/all": { + "windows": 475, + "episodes": 192, + "latent_groups": 64, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 78, + "2": 59, + "3": 68, + "4": 97, + "5": 107, + "6": 53, + "7": 13 + }, + "raw": { + "parent": { + "acc": 0.3346354166666667, + "nll": 1.7350104903591685, + "brier": 0.931591212015464, + "ece": 0.39189449934680637 + }, + "candidate": { + "acc": 0.33940972222222215, + "nll": 1.1081342789041253, + "brier": 0.6726546422375012, + "ece": 0.05665595823559088 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.004774305555555469, + "ci95": [ + -0.0039171006944444964, + 0.01388888888888895 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3372395833333333 + ], + "candidate_ci95": [ + 0.3311631944444445, + 0.3481011284722223 + ] + }, + "nll": { + "candidate_minus_parent": -0.6268762114550432, + "ci95": [ + -0.6790899859377322, + -0.5749093794570713 + ], + "parent_ci95": [ + 1.6811977782718885, + 1.7866107928562296 + ], + "candidate_ci95": [ + 1.096188224934354, + 1.1214289900464145 + ] + }, + "brier": { + "candidate_minus_parent": -0.25893656977796287, + "ci95": [ + -0.28024348498046525, + -0.23624394979708485 + ], + "parent_ci95": [ + 0.9092211056538766, + 0.9509438619550101 + ], + "candidate_ci95": [ + 0.6646108272384416, + 0.6815656878154813 + ] + }, + "ece": { + "candidate_minus_parent": -0.3352385411112155, + "ci95": [ + -0.35128286511395124, + -0.3108043566620138 + ], + "parent_ci95": [ + 0.3781534407523819, + 0.4056166546749857 + ], + "candidate_ci95": [ + 0.04569651446212632, + 0.07531915688931973 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 64, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3346354166666667, + "nll": 1.1449771461576341, + "brier": 0.6962720420695204, + "ece": 0.11982150416096793 + }, + "candidate": { + "acc": 0.33940972222222215, + "nll": 1.0986515810875779, + "brier": 0.6665575258233994, + "ece": 0.02460719607488689 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.004774305555555469, + "ci95": [ + -0.0039171006944444964, + 0.01388888888888895 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3372395833333333 + ], + "candidate_ci95": [ + 0.3311631944444445, + 0.3481011284722223 + ] + }, + "nll": { + "candidate_minus_parent": -0.04632556507005625, + "ci95": [ + -0.05491666451540952, + -0.03719116377048172 + ], + "parent_ci95": [ + 1.1365041969928562, + 1.1526221927426767 + ], + "candidate_ci95": [ + 1.0931171370730473, + 1.1043579084594954 + ] + }, + "brier": { + "candidate_minus_parent": -0.02971451624612098, + "ci95": [ + -0.0361325958427297, + -0.022840468111426974 + ], + "parent_ci95": [ + 0.6901428934681388, + 0.7015872230804511 + ], + "candidate_ci95": [ + 0.6627210641166765, + 0.670591035712894 + ] + }, + "ece": { + "candidate_minus_parent": -0.09521430808608104, + "ci95": [ + -0.10827761667473262, + -0.07017375862249017 + ], + "parent_ci95": [ + 0.11338467129999455, + 0.12711460228131327 + ], + "candidate_ci95": [ + 0.015289219037466687, + 0.047426377351587314 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 64, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/all/transactions_1_2": { + "windows": 137, + "episodes": 120, + "latent_groups": 55, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 78, + "2": 59 + }, + "raw": { + "parent": { + "acc": 0.3416666666666667, + "nll": 1.684599582747552, + "brier": 0.9005110955171775, + "ece": 0.36682455700518823 + }, + "candidate": { + "acc": 0.35, + "nll": 1.101116661104652, + "brier": 0.6683340927580801, + "ece": 0.013849288001388559 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.008333333333333304, + "ci95": [ + -0.0630993897122929, + 0.08622563718140923 + ], + "parent_ci95": [ + 0.30274651144861614, + 0.3793103448275862 + ], + "candidate_ci95": [ + 0.30251612272816103, + 0.39655172413793105 + ] + }, + "nll": { + "candidate_minus_parent": -0.5834829216429, + "ci95": [ + -0.7015545794752215, + -0.4644986033568784 + ], + "parent_ci95": [ + 1.5712100799782325, + 1.7988729386677607 + ], + "candidate_ci95": [ + 1.0929978575928703, + 1.1103106386867923 + ] + }, + "brier": { + "candidate_minus_parent": -0.23217700275909736, + "ci95": [ + -0.2852689584801007, + -0.18326137387219782 + ], + "parent_ci95": [ + 0.8520279193111497, + 0.9521012018246955 + ], + "candidate_ci95": [ + 0.6628975480894769, + 0.6745206693158341 + ] + }, + "ece": { + "candidate_minus_parent": -0.35297526900379966, + "ci95": [ + -0.3985098064069664, + -0.2794380675101525 + ], + "parent_ci95": [ + 0.32695543451399345, + 0.40934861556604873 + ], + "candidate_ci95": [ + 0.0010193241456591974, + 0.06189194519210392 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 55, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3416666666666667, + "nll": 1.1455507388151853, + "brier": 0.6944973866276738, + "ece": 0.10320926869972151 + }, + "candidate": { + "acc": 0.35, + "nll": 1.0991255871933145, + "brier": 0.6670069728868838, + "ece": 0.0015285428355831414 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.008333333333333304, + "ci95": [ + -0.0630993897122929, + 0.08622563718140923 + ], + "parent_ci95": [ + 0.30274651144861614, + 0.3793103448275862 + ], + "candidate_ci95": [ + 0.30251612272816103, + 0.39655172413793105 + ] + }, + "nll": { + "candidate_minus_parent": -0.046425151621870775, + "ci95": [ + -0.07472376843975126, + -0.01622713172066983 + ], + "parent_ci95": [ + 1.1183217393157816, + 1.1716799673702034 + ], + "candidate_ci95": [ + 1.0951166991154377, + 1.103752703626571 + ] + }, + "brier": { + "candidate_minus_parent": -0.027490413740789976, + "ci95": [ + -0.045069969488215435, + -0.009417163544500267 + ], + "parent_ci95": [ + 0.6786844067119482, + 0.7107307082378528 + ], + "candidate_ci95": [ + 0.6643381100021499, + 0.6700878067256317 + ] + }, + "ece": { + "candidate_minus_parent": -0.10168072586413837, + "ci95": [ + -0.1332521717224303, + -0.03987470346219365 + ], + "parent_ci95": [ + 0.06925090184928052, + 0.14806137845187706 + ], + "candidate_ci95": [ + 0.0008313869138119382, + 0.055754750153097166 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 55, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/all/transactions_3_plus": { + "windows": 338, + "episodes": 191, + "latent_groups": 64, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "3": 68, + "4": 97, + "5": 107, + "6": 53, + "7": 13 + }, + "raw": { + "parent": { + "acc": 0.33682373472949395, + "nll": 1.761452391457942, + "brier": 0.9442724162451026, + "ece": 0.3950515362300004 + }, + "candidate": { + "acc": 0.33726003490401396, + "nll": 1.1147148219060148, + "brier": 0.6763215604784556, + "ece": 0.0745558174365904 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.00043630017452001013, + "ci95": [ + -0.017454303297510847, + 0.01571561182009885 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.34219858156028377 + ], + "candidate_ci95": [ + 0.3223684210526316, + 0.3511308014288831 + ] + }, + "nll": { + "candidate_minus_parent": -0.6467375695519273, + "ci95": [ + -0.7267278992967698, + -0.5706812911968864 + ], + "parent_ci95": [ + 1.6806548862561872, + 1.8396636722233164 + ], + "candidate_ci95": [ + 1.0973732716449693, + 1.133455218416589 + ] + }, + "brier": { + "candidate_minus_parent": -0.267950855766647, + "ci95": [ + -0.29774336359130227, + -0.23610420078841077 + ], + "parent_ci95": [ + 0.9135761873571212, + 0.9723497753763009 + ], + "candidate_ci95": [ + 0.6646927948643595, + 0.6886955188028272 + ] + }, + "ece": { + "candidate_minus_parent": -0.32049571879341, + "ci95": [ + -0.34709872603547504, + -0.28679970747810446 + ], + "parent_ci95": [ + 0.3758519650838023, + 0.41288541640531323 + ], + "candidate_ci95": [ + 0.05472470170964465, + 0.10244079848991547 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 64, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.33682373472949395, + "nll": 1.1463297791564957, + "brier": 0.6983178566024099, + "ece": 0.12125128514353993 + }, + "candidate": { + "acc": 0.33726003490401396, + "nll": 1.1003470550538683, + "brier": 0.6675203677297942, + "ece": 0.03228487048620194 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.00043630017452001013, + "ci95": [ + -0.017454303297510847, + 0.01571561182009885 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.34219858156028377 + ], + "candidate_ci95": [ + 0.3223684210526316, + 0.3511308014288831 + ] + }, + "nll": { + "candidate_minus_parent": -0.04598272410262738, + "ci95": [ + -0.05915958918677809, + -0.031511613609357336 + ], + "parent_ci95": [ + 1.1334658573821124, + 1.1579355387417785 + ], + "candidate_ci95": [ + 1.0925136173057282, + 1.10792523882204 + ] + }, + "brier": { + "candidate_minus_parent": -0.030797488872615686, + "ci95": [ + -0.04002894682173265, + -0.020329667638605768 + ], + "parent_ci95": [ + 0.6893249796002362, + 0.7065566761962964 + ], + "candidate_ci95": [ + 0.6621069035390456, + 0.6727332808445797 + ] + }, + "ece": { + "candidate_minus_parent": -0.08896641465733798, + "ci95": [ + -0.11056724092783564, + -0.05915526650772133 + ], + "parent_ci95": [ + 0.1130570173875826, + 0.13525987313590213 + ], + "candidate_ci95": [ + 0.01819560542779522, + 0.06057786312676982 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 64, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/iid/all": { + "windows": 196, + "episodes": 96, + "latent_groups": 32, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 34, + "2": 16, + "3": 29, + "4": 36, + "5": 41, + "6": 33, + "7": 7 + }, + "raw": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.7124523573731933, + "brier": 0.9303227524837441, + "ece": 0.40316209525167973 + }, + "candidate": { + "acc": 0.34027777777777773, + "nll": 1.1141859538535557, + "brier": 0.6767587666407532, + "ece": 0.06582449248115932 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.00694444444444442, + "ci95": [ + 0.0, + 0.017361111111111105 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3333333333333333, + 0.3506944444444444 + ] + }, + "nll": { + "candidate_minus_parent": -0.5982664035196377, + "ci95": [ + -0.678565019721147, + -0.5198138791551941 + ], + "parent_ci95": [ + 1.6411516919873779, + 1.7821554572493035 + ], + "candidate_ci95": [ + 1.0932970868965697, + 1.137785374629282 + ] + }, + "brier": { + "candidate_minus_parent": -0.2535639858429909, + "ci95": [ + -0.28821447671246375, + -0.21795240813330616 + ], + "parent_ci95": [ + 0.9005605959813189, + 0.9600325280394629 + ], + "candidate_ci95": [ + 0.6623924005681695, + 0.6929750831612848 + ] + }, + "ece": { + "candidate_minus_parent": -0.3373376027705204, + "ci95": [ + -0.3670063589933437, + -0.3062944837628755 + ], + "parent_ci95": [ + 0.38457183848965565, + 0.4223828872836136 + ], + "candidate_ci95": [ + 0.04670505012772756, + 0.08779944049177654 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.1395642135708888, + "brier": 0.69244983786886, + "ece": 0.12259199189028035 + }, + "candidate": { + "acc": 0.34027777777777773, + "nll": 1.0997952972117553, + "brier": 0.6673327111796757, + "ece": 0.02908590049678272 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.00694444444444442, + "ci95": [ + 0.0, + 0.017361111111111105 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3333333333333333, + 0.3506944444444444 + ] + }, + "nll": { + "candidate_minus_parent": -0.03976891635913349, + "ci95": [ + -0.0541526606128827, + -0.025168846040066223 + ], + "parent_ci95": [ + 1.128130974471412, + 1.1506218033263493 + ], + "candidate_ci95": [ + 1.0898406047820128, + 1.110098733221803 + ] + }, + "brier": { + "candidate_minus_parent": -0.02511712668918431, + "ci95": [ + -0.035736637934652135, + -0.01491019414543638 + ], + "parent_ci95": [ + 0.6844439570327298, + 0.700775171664165 + ], + "candidate_ci95": [ + 0.6604923908009386, + 0.674458957910712 + ] + }, + "ece": { + "candidate_minus_parent": -0.09350609139349762, + "ci95": [ + -0.1122592755940263, + -0.06097575023830561 + ], + "parent_ci95": [ + 0.1151864856000442, + 0.13576747842445488 + ], + "candidate_ci95": [ + 0.016975309751749497, + 0.060832712514700425 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/iid/transactions_1_2": { + "windows": 50, + "episodes": 47, + "latent_groups": 24, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 34, + "2": 16 + }, + "raw": { + "parent": { + "acc": 0.3617021276595745, + "nll": 1.606278791640521, + "brier": 0.8668143694211096, + "ece": 0.31867378407842417 + }, + "candidate": { + "acc": 0.3617021276595745, + "nll": 1.0951264944863086, + "brier": 0.6643659084321083, + "ece": 0.0006462525101058933 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0, + "ci95": [ + -0.14583333333333337, + 0.14893617021276595 + ], + "parent_ci95": [ + 0.2856366459627329, + 0.4473684210526316 + ], + "candidate_ci95": [ + 0.27498774509803925, + 0.4524350649350649 + ] + }, + "nll": { + "candidate_minus_parent": -0.5111522971542124, + "ci95": [ + -0.6998260346042728, + -0.2966956346082348 + ], + "parent_ci95": [ + 1.4021310428301128, + 1.7895754867565086 + ], + "candidate_ci95": [ + 1.0799163978490631, + 1.1093613884301694 + ] + }, + "brier": { + "candidate_minus_parent": -0.20244846098900127, + "ci95": [ + -0.29625351971246794, + -0.10311579551210913 + ], + "parent_ci95": [ + 0.7727146159309324, + 0.9559588376055745 + ], + "candidate_ci95": [ + 0.6540730551374172, + 0.6741568414911062 + ] + }, + "ece": { + "candidate_minus_parent": -0.31802753156831826, + "ci95": [ + -0.37493368571558633, + -0.18510516049850553 + ], + "parent_ci95": [ + 0.24598050008228753, + 0.40578506275793336 + ], + "candidate_ci95": [ + 0.0009102569552842368, + 0.10216654573754363 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 24, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3617021276595745, + "nll": 1.1428425886049745, + "brier": 0.6920183105504162, + "ece": 0.08398588611848011 + }, + "candidate": { + "acc": 0.3617021276595745, + "nll": 1.0962314698157283, + "brier": 0.6650822310706771, + "ece": 0.013976683295737926 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0, + "ci95": [ + -0.14583333333333337, + 0.14893617021276595 + ], + "parent_ci95": [ + 0.2856366459627329, + 0.4473684210526316 + ], + "candidate_ci95": [ + 0.27498774509803925, + 0.4524350649350649 + ] + }, + "nll": { + "candidate_minus_parent": -0.046611118789246175, + "ci95": [ + -0.09561600806355856, + 0.005996069868429332 + ], + "parent_ci95": [ + 1.0945409291305586, + 1.1870865967481357 + ], + "candidate_ci95": [ + 1.088695192927692, + 1.1032106815760592 + ] + }, + "brier": { + "candidate_minus_parent": -0.026936079479739083, + "ci95": [ + -0.05750168546113095, + 0.00643858200710272 + ], + "parent_ci95": [ + 0.6620531653451835, + 0.7198120893298922 + ], + "candidate_ci95": [ + 0.6600092587271856, + 0.6698176402356754 + ] + }, + "ece": { + "candidate_minus_parent": -0.07000920282274219, + "ci95": [ + -0.14608908633724968, + 0.031151963204560304 + ], + "parent_ci95": [ + 0.025589163228596955, + 0.17726881365237537 + ], + "candidate_ci95": [ + 0.0009573572755788792, + 0.10847699460005204 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 24, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/iid/transactions_3_plus": { + "windows": 146, + "episodes": 95, + "latent_groups": 32, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "3": 29, + "4": 36, + "5": 41, + "6": 33, + "7": 7 + }, + "raw": { + "parent": { + "acc": 0.3368421052631579, + "nll": 1.7461010824909342, + "brier": 0.9415185117661182, + "ece": 0.4125392328057667 + }, + "candidate": { + "acc": 0.3368421052631579, + "nll": 1.1253460058972915, + "brier": 0.6834591680376513, + "ece": 0.08535066950123138 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0, + "ci95": [ + -0.02688172043010756, + 0.02105263157894738 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.34408602150537637 + ], + "candidate_ci95": [ + 0.3172043010752688, + 0.35789473684210527 + ] + }, + "nll": { + "candidate_minus_parent": -0.6207550765936427, + "ci95": [ + -0.7290660550006571, + -0.5173842578992225 + ], + "parent_ci95": [ + 1.6480276717710194, + 1.8467160693786326 + ], + "candidate_ci95": [ + 1.096204580918968, + 1.1579777731219882 + ] + }, + "brier": { + "candidate_minus_parent": -0.25805934372846695, + "ci95": [ + -0.30214923830076107, + -0.2142367363497392 + ], + "parent_ci95": [ + 0.901522270321034, + 0.9804060439548726 + ], + "candidate_ci95": [ + 0.6638689493859028, + 0.7057101836439521 + ] + }, + "ece": { + "candidate_minus_parent": -0.32718856330453533, + "ci95": [ + -0.3690141814092924, + -0.27888935367683226 + ], + "parent_ci95": [ + 0.38536173699373244, + 0.43754207835021325 + ], + "candidate_ci95": [ + 0.05528992404234269, + 0.1255413762737284 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3368421052631579, + "nll": 1.1390392002640022, + "brier": 0.6927147679259726, + "ece": 0.12558728946394143 + }, + "candidate": { + "acc": 0.3368421052631579, + "nll": 1.1027991703996407, + "brier": 0.6691990167866013, + "ece": 0.04061751773318837 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0, + "ci95": [ + -0.02688172043010756, + 0.02105263157894738 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.34408602150537637 + ], + "candidate_ci95": [ + 0.3172043010752688, + 0.35789473684210527 + ] + }, + "nll": { + "candidate_minus_parent": -0.03624002986436148, + "ci95": [ + -0.05793745836843981, + -0.01500581910141239 + ], + "parent_ci95": [ + 1.1229187761318027, + 1.1566543123195199 + ], + "candidate_ci95": [ + 1.0900328870684717, + 1.1166802658184218 + ] + }, + "brier": { + "candidate_minus_parent": -0.02351575113937132, + "ci95": [ + -0.03837682981567418, + -0.008820176281655752 + ], + "parent_ci95": [ + 0.6814368470195351, + 0.7046973404906302 + ], + "candidate_ci95": [ + 0.660377113283333, + 0.6787569507705161 + ] + }, + "ece": { + "candidate_minus_parent": -0.08496977173075307, + "ci95": [ + -0.11741964777848636, + -0.0519524894099493 + ], + "parent_ci95": [ + 0.11532377688961362, + 0.14439980032708952 + ], + "candidate_ci95": [ + 0.020732295614394415, + 0.07355313445514028 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/heldout_task/all": { + "windows": 279, + "episodes": 96, + "latent_groups": 32, + "sources": [ + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff" + ], + "transaction_count_histogram": { + "1": 44, + "2": 43, + "3": 39, + "4": 61, + "5": 66, + "6": 20, + "7": 6 + }, + "raw": { + "parent": { + "acc": 0.3359375, + "nll": 1.7575686233451444, + "brier": 0.9328596715471843, + "ece": 0.38062690344193295 + }, + "candidate": { + "acc": 0.3385416666666667, + "nll": 1.1020826039546938, + "brier": 0.668550517834249, + "ece": 0.06767651412584706 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.002604166666666685, + "ci95": [ + -0.012152777777777735, + 0.018229166666666685 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3411458333333333 + ], + "candidate_ci95": [ + 0.32463107638888883, + 0.3532986111111111 + ] + }, + "nll": { + "candidate_minus_parent": -0.6554860193904506, + "ci95": [ + -0.7191252619922539, + -0.5925014370566785 + ], + "parent_ci95": [ + 1.6936048834769095, + 1.8243095747652294 + ], + "candidate_ci95": [ + 1.0909955635083948, + 1.1148722338770993 + ] + }, + "brier": { + "candidate_minus_parent": -0.2643091537129353, + "ci95": [ + -0.28718910807802955, + -0.23881191694116374 + ], + "parent_ci95": [ + 0.9086730578648818, + 0.9557397760683705 + ], + "candidate_ci95": [ + 0.6602464516032025, + 0.6782263566624996 + ] + }, + "ece": { + "candidate_minus_parent": -0.3129503893160859, + "ci95": [ + -0.34660047272419636, + -0.2808933130120532 + ], + "parent_ci95": [ + 0.36558694479372333, + 0.39906087104179055 + ], + "candidate_ci95": [ + 0.04217660861029081, + 0.09521924116463996 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3359375, + "nll": 1.150390078744378, + "brier": 0.7000942462701806, + "ece": 0.11705101643165557 + }, + "candidate": { + "acc": 0.3385416666666667, + "nll": 1.0975078649634016, + "brier": 0.6657823404671229, + "ece": 0.03921704440404435 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.002604166666666685, + "ci95": [ + -0.012152777777777735, + 0.018229166666666685 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3411458333333333 + ], + "candidate_ci95": [ + 0.32463107638888883, + 0.3532986111111111 + ] + }, + "nll": { + "candidate_minus_parent": -0.05288221378097635, + "ci95": [ + -0.062443593811575986, + -0.04190493793031174 + ], + "parent_ci95": [ + 1.1410787694045743, + 1.1593915727679052 + ], + "candidate_ci95": [ + 1.091928362600967, + 1.1035442157202624 + ] + }, + "brier": { + "candidate_minus_parent": -0.03431190580305765, + "ci95": [ + -0.04143880164165249, + -0.02639595094126181 + ], + "parent_ci95": [ + 0.6935373129262572, + 0.706784911160115 + ], + "candidate_ci95": [ + 0.6617785284376593, + 0.6702643963140505 + ] + }, + "ece": { + "candidate_minus_parent": -0.07783397202761122, + "ci95": [ + -0.11444951609054546, + -0.04209215226387993 + ], + "parent_ci95": [ + 0.1088484189991902, + 0.13090558733426144 + ], + "candidate_ci95": [ + 0.010719860472098343, + 0.07335949517961429 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/heldout_task/transactions_1_2": { + "windows": 87, + "episodes": 73, + "latent_groups": 31, + "sources": [ + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff" + ], + "transaction_count_histogram": { + "1": 44, + "2": 43 + }, + "raw": { + "parent": { + "acc": 0.3287671232876712, + "nll": 1.7350252975698874, + "brier": 0.9222062479351931, + "ece": 0.3978257395744748 + }, + "candidate": { + "acc": 0.3424657534246575, + "nll": 1.104973343721942, + "brier": 0.6708889511597332, + "ece": 0.022349872495776038 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0136986301369863, + "ci95": [ + -0.0684931506849315, + 0.09999999999999998 + ], + "parent_ci95": [ + 0.27937091503267975, + 0.37142857142857144 + ], + "candidate_ci95": [ + 0.2875740466493891, + 0.39478905597326647 + ] + }, + "nll": { + "candidate_minus_parent": -0.6300519538479454, + "ci95": [ + -0.7871149224166658, + -0.4734227265150399 + ], + "parent_ci95": [ + 1.5816266494205868, + 1.8894676459462878 + ], + "candidate_ci95": [ + 1.0959260895443086, + 1.115504433417772 + ] + }, + "brier": { + "candidate_minus_parent": -0.2513172967754599, + "ci95": [ + -0.31816111614285403, + -0.19052235180482108 + ], + "parent_ci95": [ + 0.8620497635983301, + 0.9910377048377865 + ], + "candidate_ci95": [ + 0.6649016420955892, + 0.6780008413641927 + ] + }, + "ece": { + "candidate_minus_parent": -0.3754758670786988, + "ci95": [ + -0.43912023298958897, + -0.29252862461462614 + ], + "parent_ci95": [ + 0.35259500914090686, + 0.4548247635753083 + ], + "candidate_ci95": [ + 0.001328926908109236, + 0.0803031654645048 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 31, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3287671232876712, + "nll": 1.1472943423751842, + "brier": 0.6960935041020725, + "ece": 0.12388419826016156 + }, + "candidate": { + "acc": 0.3424657534246575, + "nll": 1.1009889230391574, + "brier": 0.668246190220606, + "ece": 0.006486013351091863 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0136986301369863, + "ci95": [ + -0.0684931506849315, + 0.09999999999999998 + ], + "parent_ci95": [ + 0.27937091503267975, + 0.37142857142857144 + ], + "candidate_ci95": [ + 0.2875740466493891, + 0.39478905597326647 + ] + }, + "nll": { + "candidate_minus_parent": -0.04630541933602683, + "ci95": [ + -0.08443920761641914, + -0.006965185387646984 + ], + "parent_ci95": [ + 1.110315934404331, + 1.1821887526257167 + ], + "candidate_ci95": [ + 1.096498439293462, + 1.10627108114853 + ] + }, + "brier": { + "candidate_minus_parent": -0.02784731388146644, + "ci95": [ + -0.050883589448877234, + -0.004163943536963318 + ], + "parent_ci95": [ + 0.6743476108077922, + 0.7173112637797384 + ], + "candidate_ci95": [ + 0.6652351601435463, + 0.6717803374395981 + ] + }, + "ece": { + "candidate_minus_parent": -0.1173981849090697, + "ci95": [ + -0.15950237235508946, + -0.03587379472521817 + ], + "parent_ci95": [ + 0.08056367869157598, + 0.17544693067276984 + ], + "candidate_ci95": [ + 0.0008863391142034598, + 0.0688510654467535 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 31, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "full/heldout_task/transactions_3_plus": { + "windows": 192, + "episodes": 96, + "latent_groups": 32, + "sources": [ + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff" + ], + "transaction_count_histogram": { + "3": 39, + "4": 61, + "5": 66, + "6": 20, + "7": 6 + }, + "raw": { + "parent": { + "acc": 0.3368055555555555, + "nll": 1.7766437909565445, + "brier": 0.9469976342190978, + "ece": 0.37774600316023155 + }, + "candidate": { + "acc": 0.33767361111111116, + "nll": 1.104194379414647, + "brier": 0.6692583029980016, + "ece": 0.08433455087754062 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0008680555555556357, + "ci95": [ + -0.01825086805555552, + 0.02172309027777771 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.34375 + ], + "candidate_ci95": [ + 0.3185763888888889, + 0.3567925347222223 + ] + }, + "nll": { + "candidate_minus_parent": -0.6724494115418975, + "ci95": [ + -0.7846533438325851, + -0.5642174507384142 + ], + "parent_ci95": [ + 1.669932285314896, + 1.895277090283664 + ], + "candidate_ci95": [ + 1.0893008766296481, + 1.1201245297548965 + ] + }, + "brier": { + "candidate_minus_parent": -0.2777393312210962, + "ci95": [ + -0.3149989606409033, + -0.23795002418428113 + ], + "parent_ci95": [ + 0.9083316689667666, + 0.9850896377946536 + ], + "candidate_ci95": [ + 0.6582977845907898, + 0.6809407849295769 + ] + }, + "ece": { + "candidate_minus_parent": -0.29341145228269094, + "ci95": [ + -0.3390404291788445, + -0.2490592554209443 + ], + "parent_ci95": [ + 0.35449375236020964, + 0.405508705226858 + ], + "candidate_ci95": [ + 0.048565854925756505, + 0.12331123938076059 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3368055555555555, + "nll": 1.1535444145188587, + "brier": 0.7038625797718008, + "ece": 0.11696044753480923 + }, + "candidate": { + "acc": 0.33767361111111116, + "nll": 1.0979204825762812, + "brier": 0.6658592046006618, + "ece": 0.05210446832070306 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0008680555555556357, + "ci95": [ + -0.01825086805555552, + 0.02172309027777771 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.34375 + ], + "candidate_ci95": [ + 0.3185763888888889, + 0.3567925347222223 + ] + }, + "nll": { + "candidate_minus_parent": -0.05562393194257753, + "ci95": [ + -0.07104980441804248, + -0.03889109071886262 + ], + "parent_ci95": [ + 1.1376932604226322, + 1.169471515491799 + ], + "candidate_ci95": [ + 1.0903705192268702, + 1.1055069702159943 + ] + }, + "brier": { + "candidate_minus_parent": -0.03800337517113894, + "ci95": [ + -0.04911188392942332, + -0.025785478747248913 + ], + "parent_ci95": [ + 0.6925366834896836, + 0.7152639596153512 + ], + "candidate_ci95": [ + 0.6605140359970081, + 0.6714526182780927 + ] + }, + "ece": { + "candidate_minus_parent": -0.06485597921410617, + "ci95": [ + -0.13292318496375544, + -0.019393761824628347 + ], + "parent_ci95": [ + 0.10688745756195825, + 0.15464785735718722 + ], + "candidate_ci95": [ + 0.013049959604135488, + 0.09738629236158458 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/all/all": { + "windows": 390, + "episodes": 191, + "latent_groups": 64, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 72, + "2": 62, + "3": 47, + "4": 97, + "5": 59, + "6": 43, + "7": 10 + }, + "raw": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.6254954852975034, + "brier": 0.9013043985586006, + "ece": 0.3597505684514547 + }, + "candidate": { + "acc": 0.33682373472949384, + "nll": 1.110266485995317, + "brier": 0.674771349616321, + "ece": 0.0596522660066575 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.003490401396160525, + "ci95": [ + -0.012347320724980352, + 0.02083333333333337 + ], + "parent_ci95": [ + 0.328125, + 0.3386243386243386 + ], + "candidate_ci95": [ + 0.3245149911816579, + 0.3524305555555555 + ] + }, + "nll": { + "candidate_minus_parent": -0.5152289993021864, + "ci95": [ + -0.5676505768552339, + -0.455085286081279 + ], + "parent_ci95": [ + 1.5684667289278473, + 1.6767592517653676 + ], + "candidate_ci95": [ + 1.1009605550043402, + 1.1207292524341428 + ] + }, + "brier": { + "candidate_minus_parent": -0.22653304894227955, + "ci95": [ + -0.250654103376682, + -0.1982401278563772 + ], + "parent_ci95": [ + 0.8760171160204571, + 0.9243132548480782 + ], + "candidate_ci95": [ + 0.6681397780663684, + 0.6824428981963307 + ] + }, + "ece": { + "candidate_minus_parent": -0.3000983024447972, + "ci95": [ + -0.32770194535473435, + -0.2727976391786125 + ], + "parent_ci95": [ + 0.3443639080838946, + 0.3816678968317757 + ], + "candidate_ci95": [ + 0.04335549755037792, + 0.07888579260595374 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 64, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.1407396953848146, + "brier": 0.6932489922362768, + "ece": 0.10409138149272482 + }, + "candidate": { + "acc": 0.33682373472949384, + "nll": 1.100190627600681, + "brier": 0.6676671025761404, + "ece": 0.02738107127463787 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.003490401396160525, + "ci95": [ + -0.012347320724980352, + 0.02083333333333337 + ], + "parent_ci95": [ + 0.328125, + 0.3386243386243386 + ], + "candidate_ci95": [ + 0.3245149911816579, + 0.3524305555555555 + ] + }, + "nll": { + "candidate_minus_parent": -0.040549067784133586, + "ci95": [ + -0.05069348217992202, + -0.027742540759487525 + ], + "parent_ci95": [ + 1.130108915954547, + 1.1501606809449922 + ], + "candidate_ci95": [ + 1.0957942757745658, + 1.1047245026075414 + ] + }, + "brier": { + "candidate_minus_parent": -0.025581889660136414, + "ci95": [ + -0.03237704094818407, + -0.01685801379643218 + ], + "parent_ci95": [ + 0.6860842681004884, + 0.6994286590282218 + ], + "candidate_ci95": [ + 0.664585723129944, + 0.6710185880855628 + ] + }, + "ece": { + "candidate_minus_parent": -0.07671031021808694, + "ci95": [ + -0.10213027408881704, + -0.05682353057788049 + ], + "parent_ci95": [ + 0.09783329602202052, + 0.12698925540903008 + ], + "candidate_ci95": [ + 0.013073190167377237, + 0.04491293661386301 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 64, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/all/transactions_1_2": { + "windows": 134, + "episodes": 121, + "latent_groups": 57, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 72, + "2": 62 + }, + "raw": { + "parent": { + "acc": 0.3140495867768595, + "nll": 1.6966350165378898, + "brier": 0.9281100519662293, + "ece": 0.39070076147798016 + }, + "candidate": { + "acc": 0.3305785123966942, + "nll": 1.1037310641630889, + "brier": 0.6700335760698483, + "ece": 0.03344612557316424 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.01652892561983471, + "ci95": [ + -0.06065340909090909, + 0.09823826741440374 + ], + "parent_ci95": [ + 0.2710146315305653, + 0.3566029900332226 + ], + "candidate_ci95": [ + 0.28203125, + 0.38098141695702664 + ] + }, + "nll": { + "candidate_minus_parent": -0.592903952374801, + "ci95": [ + -0.7578241062153577, + -0.4453913213688957 + ], + "parent_ci95": [ + 1.5539056137761456, + 1.8553697934380764 + ], + "candidate_ci95": [ + 1.0939309592978625, + 1.112084254804725 + ] + }, + "brier": { + "candidate_minus_parent": -0.25807647589638105, + "ci95": [ + -0.32915327406715944, + -0.19480923599509453 + ], + "parent_ci95": [ + 0.8666195602754322, + 0.9969135674266023 + ], + "candidate_ci95": [ + 0.6633850945977552, + 0.6756742502168676 + ] + }, + "ece": { + "candidate_minus_parent": -0.35725463590481593, + "ci95": [ + -0.4240395940757665, + -0.2692138271611469 + ], + "parent_ci95": [ + 0.3425586091205118, + 0.4420175276007634 + ], + "candidate_ci95": [ + 0.006504235656975715, + 0.08739692224084111 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 57, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3140495867768595, + "nll": 1.1504443915732265, + "brier": 0.6995600023788842, + "ece": 0.13361834319582644 + }, + "candidate": { + "acc": 0.3305785123966942, + "nll": 1.1004000643424592, + "brier": 0.6678439429866679, + "ece": 0.01795538869843083 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.01652892561983471, + "ci95": [ + -0.06065340909090909, + 0.09823826741440374 + ], + "parent_ci95": [ + 0.2710146315305653, + 0.3566029900332226 + ], + "candidate_ci95": [ + 0.28203125, + 0.38098141695702664 + ] + }, + "nll": { + "candidate_minus_parent": -0.05004432723076735, + "ci95": [ + -0.08637979380099192, + -0.01635323304221903 + ], + "parent_ci95": [ + 1.1184560351049317, + 1.1843136014307336 + ], + "candidate_ci95": [ + 1.0955398254088504, + 1.104605325738805 + ] + }, + "brier": { + "candidate_minus_parent": -0.03171605939221622, + "ci95": [ + -0.05485986502916156, + -0.010145809030184057 + ], + "parent_ci95": [ + 0.6797308847750702, + 0.7216905416020233 + ], + "candidate_ci95": [ + 0.6645759025183346, + 0.670651762712452 + ] + }, + "ece": { + "candidate_minus_parent": -0.11566295449739561, + "ci95": [ + -0.17571785910270954, + -0.04267184188290237 + ], + "parent_ci95": [ + 0.09250511763520454, + 0.18969974473657997 + ], + "candidate_ci95": [ + 0.0008354544586435885, + 0.0662625819853173 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 57, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/all/transactions_3_plus": { + "windows": 256, + "episodes": 171, + "latent_groups": 60, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "3": 47, + "4": 97, + "5": 59, + "6": 43, + "7": 10 + }, + "raw": { + "parent": { + "acc": 0.3362573099415205, + "nll": 1.614608076963336, + "brier": 0.8961514757519852, + "ece": 0.3541944709177928 + }, + "candidate": { + "acc": 0.3391812865497076, + "nll": 1.1112555150418164, + "brier": 0.6756569607510187, + "ece": 0.07742608781497969 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0029239766081871066, + "ci95": [ + -0.01845350620630618, + 0.025873411978221402 + ], + "parent_ci95": [ + 0.32514235596925467, + 0.3473053892215569 + ], + "candidate_ci95": [ + 0.3216374269005848, + 0.3588349771046143 + ] + }, + "nll": { + "candidate_minus_parent": -0.5033525619215196, + "ci95": [ + -0.5760397492442256, + -0.43086321079813233 + ], + "parent_ci95": [ + 1.5425398515836162, + 1.686522408649482 + ], + "candidate_ci95": [ + 1.0965239071870951, + 1.1262398187506255 + ] + }, + "brier": { + "candidate_minus_parent": -0.2204945150009665, + "ci95": [ + -0.2523669636611384, + -0.18626853044480648 + ], + "parent_ci95": [ + 0.8641108521283106, + 0.9272656271309757 + ], + "candidate_ci95": [ + 0.6651284837028695, + 0.6867827894711732 + ] + }, + "ece": { + "candidate_minus_parent": -0.27676838310281315, + "ci95": [ + -0.30600041253595806, + -0.2349006043734538 + ], + "parent_ci95": [ + 0.33462179449357915, + 0.3824874686221771 + ], + "candidate_ci95": [ + 0.059624438787811616, + 0.11708948440265422 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 60, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3362573099415205, + "nll": 1.1376072963375419, + "brier": 0.6911222461269452, + "ece": 0.10123581136238233 + }, + "candidate": { + "acc": 0.3391812865497076, + "nll": 1.0987252403049783, + "brier": 0.6666289153322652, + "ece": 0.03490897494051337 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0029239766081871066, + "ci95": [ + -0.01845350620630618, + 0.025873411978221402 + ], + "parent_ci95": [ + 0.32514235596925467, + 0.3473053892215569 + ], + "candidate_ci95": [ + 0.3216374269005848, + 0.3588349771046143 + ] + }, + "nll": { + "candidate_minus_parent": -0.03888205603256356, + "ci95": [ + -0.05334088872101057, + -0.023639423053128283 + ], + "parent_ci95": [ + 1.1234643869963845, + 1.151497368963336 + ], + "candidate_ci95": [ + 1.0912678426417426, + 1.1063863964992202 + ] + }, + "brier": { + "candidate_minus_parent": -0.02449333079467997, + "ci95": [ + -0.034341580370048086, + -0.014420557386953376 + ], + "parent_ci95": [ + 0.6821943361952916, + 0.7000595417618332 + ], + "candidate_ci95": [ + 0.6613182918503591, + 0.6721973513310285 + ] + }, + "ece": { + "candidate_minus_parent": -0.06632683642186896, + "ci95": [ + -0.09359934652536785, + -0.031535166093947214 + ], + "parent_ci95": [ + 0.09033077027996042, + 0.12980451404512994 + ], + "candidate_ci95": [ + 0.017559246375493297, + 0.06955580862493253 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 60, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/iid/all": { + "windows": 161, + "episodes": 95, + "latent_groups": 32, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 40, + "2": 24, + "3": 16, + "4": 18, + "5": 34, + "6": 24, + "7": 5 + }, + "raw": { + "parent": { + "acc": 0.3368421052631579, + "nll": 1.551428140539595, + "brier": 0.8809803470371403, + "ece": 0.3406805981123081 + }, + "candidate": { + "acc": 0.3263157894736842, + "nll": 1.1150678033880261, + "brier": 0.6783550595886768, + "ece": 0.07539941511313847 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.010526315789473661, + "ci95": [ + -0.03225806451612906, + 0.0 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.34408602150537637 + ], + "candidate_ci95": [ + 0.3118279569892473, + 0.3333333333333333 + ] + }, + "nll": { + "candidate_minus_parent": -0.43636033715156897, + "ci95": [ + -0.52810736804393, + -0.35009373129710897 + ], + "parent_ci95": [ + 1.4734442923463753, + 1.6369683513055724 + ], + "candidate_ci95": [ + 1.1016114662784458, + 1.1311523228494405 + ] + }, + "brier": { + "candidate_minus_parent": -0.20262528744846353, + "ci95": [ + -0.24471129020825447, + -0.16181052531141543 + ], + "parent_ci95": [ + 0.8460266474128768, + 0.9182401289315173 + ], + "candidate_ci95": [ + 0.669141068426313, + 0.6899658362473933 + ] + }, + "ece": { + "candidate_minus_parent": -0.26528118299916964, + "ci95": [ + -0.3059540812627167, + -0.22379753414226744 + ], + "parent_ci95": [ + 0.3192079421820447, + 0.3744168051211305 + ], + "candidate_ci95": [ + 0.05657089719716896, + 0.10463612356640144 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3368421052631579, + "nll": 1.1327163015658464, + "brier": 0.6883296731396646, + "ece": 0.09406803499836544 + }, + "candidate": { + "acc": 0.3263157894736842, + "nll": 1.1016997587704727, + "brier": 0.668802101067866, + "ece": 0.04215808199067718 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.010526315789473661, + "ci95": [ + -0.03225806451612906, + 0.0 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.34408602150537637 + ], + "candidate_ci95": [ + 0.3118279569892473, + 0.3333333333333333 + ] + }, + "nll": { + "candidate_minus_parent": -0.03101654279537369, + "ci95": [ + -0.05106434760087051, + -0.01233452322119823 + ], + "parent_ci95": [ + 1.116523591816024, + 1.1513212455338053 + ], + "candidate_ci95": [ + 1.09579883169201, + 1.1083455886338627 + ] + }, + "brier": { + "candidate_minus_parent": -0.01952757207179867, + "ci95": [ + -0.03254090267138453, + -0.00646752127970882 + ], + "parent_ci95": [ + 0.6778213296967531, + 0.7005165818006102 + ], + "candidate_ci95": [ + 0.6646066548969416, + 0.6736772012134988 + ] + }, + "ece": { + "candidate_minus_parent": -0.05190995300768826, + "ci95": [ + -0.10673619115606205, + -0.030280802684953916 + ], + "parent_ci95": [ + 0.08622401738775379, + 0.14015676652081643 + ], + "candidate_ci95": [ + 0.028781555727489527, + 0.06163593652826543 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/iid/transactions_1_2": { + "windows": 64, + "episodes": 60, + "latent_groups": 28, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 40, + "2": 24 + }, + "raw": { + "parent": { + "acc": 0.35, + "nll": 1.4883528089163451, + "brier": 0.8532207141394844, + "ece": 0.31972939169340536 + }, + "candidate": { + "acc": 0.3, + "nll": 1.1069882187922748, + "brier": 0.6722975273946177, + "ece": 0.06515841770023822 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.04999999999999999, + "ci95": [ + -0.1552146720757268, + 0.050021186440677934 + ], + "parent_ci95": [ + 0.2857142857142857, + 0.41509433962264153 + ], + "candidate_ci95": [ + 0.23214285714285715, + 0.35714285714285715 + ] + }, + "nll": { + "candidate_minus_parent": -0.3813645901240703, + "ci95": [ + -0.5156136554817409, + -0.23211164333094048 + ], + "parent_ci95": [ + 1.348725806837395, + 1.6150471936420967 + ], + "candidate_ci95": [ + 1.0928709149905234, + 1.1223015993915004 + ] + }, + "brier": { + "candidate_minus_parent": -0.18092318674486674, + "ci95": [ + -0.24475220111464377, + -0.10601283595766012 + ], + "parent_ci95": [ + 0.7791314208070413, + 0.9170941114976302 + ], + "candidate_ci95": [ + 0.6626849801685101, + 0.6826421567984873 + ] + }, + "ece": { + "candidate_minus_parent": -0.25457097399316714, + "ci95": [ + -0.3438415036852427, + -0.13863113460598453 + ], + "parent_ci95": [ + 0.25302953070124085, + 0.3843034725335647 + ], + "candidate_ci95": [ + 0.018820144360438072, + 0.1404389973818657 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 28, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.35, + "nll": 1.119540482516699, + "brier": 0.6805384450179266, + "ece": 0.07997479652210188 + }, + "candidate": { + "acc": 0.3, + "nll": 1.1019631899703175, + "brier": 0.6689075031173503, + "ece": 0.04907290355077636 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.04999999999999999, + "ci95": [ + -0.1552146720757268, + 0.050021186440677934 + ], + "parent_ci95": [ + 0.2857142857142857, + 0.41509433962264153 + ], + "candidate_ci95": [ + 0.23214285714285715, + 0.35714285714285715 + ] + }, + "nll": { + "candidate_minus_parent": -0.017577292546381518, + "ci95": [ + -0.05136216005923647, + 0.020599176977015705 + ], + "parent_ci95": [ + 1.0859349092471269, + 1.1508901652452113 + ], + "candidate_ci95": [ + 1.0948957293740824, + 1.1095137465094964 + ] + }, + "brier": { + "candidate_minus_parent": -0.011630941900576341, + "ci95": [ + -0.033248613554045965, + 0.012886900196317849 + ], + "parent_ci95": [ + 0.6590430467723951, + 0.7001408588651075 + ], + "candidate_ci95": [ + 0.6641347720982478, + 0.6740106780892229 + ] + }, + "ece": { + "candidate_minus_parent": -0.030901892971325513, + "ci95": [ + -0.1261490236410924, + 0.05221893446720805 + ], + "parent_ci95": [ + 0.0346809913133188, + 0.15421564281325367 + ], + "candidate_ci95": [ + 0.003870947592434356, + 0.11648710012476678 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 28, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/iid/transactions_3_plus": { + "windows": 97, + "episodes": 75, + "latent_groups": 28, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "3": 16, + "4": 18, + "5": 34, + "6": 24, + "7": 5 + }, + "raw": { + "parent": { + "acc": 0.3466666666666667, + "nll": 1.5680014012828554, + "brier": 0.8735192333226294, + "ece": 0.33890123655052196 + }, + "candidate": { + "acc": 0.3333333333333333, + "nll": 1.1236101735524817, + "brier": 0.684959148963268, + "ece": 0.11333522504688955 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.013333333333333364, + "ci95": [ + -0.06331142486205778, + 0.02857142857142858 + ], + "parent_ci95": [ + 0.32432432432432434, + 0.3699021526418786 + ], + "candidate_ci95": [ + 0.2972972972972973, + 0.3684210526315789 + ] + }, + "nll": { + "candidate_minus_parent": -0.44439122773037365, + "ci95": [ + -0.5961601664795805, + -0.3053275116294062 + ], + "parent_ci95": [ + 1.4424888114143912, + 1.7028062250692066 + ], + "candidate_ci95": [ + 1.1014624518540175, + 1.1451365874380015 + ] + }, + "brier": { + "candidate_minus_parent": -0.18856008435936134, + "ci95": [ + -0.2549635152100391, + -0.1225264364772087 + ], + "parent_ci95": [ + 0.8155310992154277, + 0.9311213349034488 + ], + "candidate_ci95": [ + 0.6690087344244944, + 0.7016534820846407 + ] + }, + "ece": { + "candidate_minus_parent": -0.22556601150363242, + "ci95": [ + -0.3077557235943786, + -0.12794339981102823 + ], + "parent_ci95": [ + 0.30080941430475944, + 0.3967571784864277 + ], + "candidate_ci95": [ + 0.06366678330424731, + 0.19623783341680856 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 28, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3466666666666667, + "nll": 1.131763207180215, + "brier": 0.6863143705776773, + "ece": 0.0877502271071839 + }, + "candidate": { + "acc": 0.3333333333333333, + "nll": 1.1035049943710618, + "brier": 0.6701746076014911, + "ece": 0.048907726748390154 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.013333333333333364, + "ci95": [ + -0.06331142486205778, + 0.02857142857142858 + ], + "parent_ci95": [ + 0.32432432432432434, + 0.3699021526418786 + ], + "candidate_ci95": [ + 0.2972972972972973, + 0.3684210526315789 + ] + }, + "nll": { + "candidate_minus_parent": -0.028258212809153305, + "ci95": [ + -0.05962702570649112, + 0.004027120527058854 + ], + "parent_ci95": [ + 1.1049092475977573, + 1.16036054023137 + ], + "candidate_ci95": [ + 1.0930108046380778, + 1.11365111458743 + ] + }, + "brier": { + "candidate_minus_parent": -0.016139762976186245, + "ci95": [ + -0.036028815996373954, + 0.006768874748515453 + ], + "parent_ci95": [ + 0.6687286849072276, + 0.7042716085169389 + ], + "candidate_ci95": [ + 0.6627336792543009, + 0.6777495706240455 + ] + }, + "ece": { + "candidate_minus_parent": -0.03884250035879375, + "ci95": [ + -0.1186266649620203, + 0.005548265333439938 + ], + "parent_ci95": [ + 0.0664247161683894, + 0.15976734020796518 + ], + "candidate_ci95": [ + 0.017378841072555818, + 0.08763460844669049 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 28, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/heldout_task/all": { + "windows": 229, + "episodes": 96, + "latent_groups": 32, + "sources": [ + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff" + ], + "transaction_count_histogram": { + "1": 32, + "2": 38, + "3": 31, + "4": 79, + "5": 25, + "6": 19, + "7": 5 + }, + "raw": { + "parent": { + "acc": 0.3298611111111111, + "nll": 1.698791295214183, + "brier": 0.921416741210046, + "ece": 0.3793138394559325 + }, + "candidate": { + "acc": 0.34722222222222215, + "nll": 1.105515182325449, + "brier": 0.6712249699561775, + "ece": 0.049518278502379665 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.01736111111111105, + "ci95": [ + 0.0, + 0.04513888888888884 + ], + "parent_ci95": [ + 0.3229166666666667, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3333333333333333, + 0.375 + ] + }, + "nll": { + "candidate_minus_parent": -0.593276112888734, + "ci95": [ + -0.6483202339957271, + -0.5433594179811996 + ], + "parent_ci95": [ + 1.6470309252925026, + 1.7596016187674182 + ], + "candidate_ci95": [ + 1.09282458371758, + 1.1184693544582691 + ] + }, + "brier": { + "candidate_minus_parent": -0.2501917712538685, + "ci95": [ + -0.27432881472656895, + -0.2296314348840799 + ], + "parent_ci95": [ + 0.9010265175449971, + 0.9494995621888368 + ], + "candidate_ci95": [ + 0.662110057644752, + 0.6806020442997136 + ] + }, + "ece": { + "candidate_minus_parent": -0.32979556095355284, + "ci95": [ + -0.357606466251182, + -0.2954258621683766 + ], + "parent_ci95": [ + 0.3635599452789377, + 0.40427440215523025 + ], + "candidate_ci95": [ + 0.025157637704134143, + 0.07769176694025634 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3298611111111111, + "nll": 1.1486795121848348, + "brier": 0.6981170684256327, + "ece": 0.11401031812776787 + }, + "candidate": { + "acc": 0.34722222222222215, + "nll": 1.098697216547241, + "brier": 0.6665439269853703, + "ece": 0.02349564669394129 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.01736111111111105, + "ci95": [ + 0.0, + 0.04513888888888884 + ], + "parent_ci95": [ + 0.3229166666666667, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3333333333333333, + 0.375 + ] + }, + "nll": { + "candidate_minus_parent": -0.04998229563759371, + "ci95": [ + -0.05935456056647385, + -0.0402773394943774 + ], + "parent_ci95": [ + 1.138189705911166, + 1.1595554900162879 + ], + "candidate_ci95": [ + 1.0919381937392876, + 1.105469476604823 + ] + }, + "brier": { + "candidate_minus_parent": -0.03157314144026235, + "ci95": [ + -0.038181326557168584, + -0.02542550191942243 + ], + "parent_ci95": [ + 0.691945596634429, + 0.7057214717840435 + ], + "candidate_ci95": [ + 0.6616884540923037, + 0.67147476440236 + ] + }, + "ece": { + "candidate_minus_parent": -0.09051467143382658, + "ci95": [ + -0.1264153633635337, + -0.06263934365622244 + ], + "parent_ci95": [ + 0.10808498983169598, + 0.14582065312155512 + ], + "candidate_ci95": [ + 0.00582896423162496, + 0.05429685894844026 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/heldout_task/transactions_1_2": { + "windows": 70, + "episodes": 61, + "latent_groups": 29, + "sources": [ + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff" + ], + "transaction_count_histogram": { + "1": 32, + "2": 38 + }, + "raw": { + "parent": { + "acc": 0.2786885245901639, + "nll": 1.9015027617394087, + "brier": 1.0017716957302407, + "ece": 0.4605086661841192 + }, + "candidate": { + "acc": 0.36065573770491804, + "nll": 1.1005273055114302, + "brier": 0.6678067387012225, + "ece": 0.002253707087517676 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.08196721311475413, + "ci95": [ + -0.045508658008657964, + 0.22033898305084745 + ], + "parent_ci95": [ + 0.21052631578947367, + 0.3442622950819672 + ], + "candidate_ci95": [ + 0.2878787878787879, + 0.4444444444444444 + ] + }, + "nll": { + "candidate_minus_parent": -0.8009754562279785, + "ci95": [ + -1.0944341558459387, + -0.5388799527475848 + ], + "parent_ci95": [ + 1.6494540797886756, + 2.1890376902151942 + ], + "candidate_ci95": [ + 1.0886223917630717, + 1.1124574888145347 + ] + }, + "brier": { + "candidate_minus_parent": -0.3339649570290182, + "ci95": [ + -0.4616763420076285, + -0.22106294852825745 + ], + "parent_ci95": [ + 0.8926639965028754, + 1.1242562183743325 + ], + "candidate_ci95": [ + 0.6597937031097704, + 0.6758501818214387 + ] + }, + "ece": { + "candidate_minus_parent": -0.4582549590966015, + "ci95": [ + -0.5064246196985348, + -0.31884644225687725 + ], + "parent_ci95": [ + 0.384158071834183, + 0.543965119130656 + ], + "candidate_ci95": [ + 0.001666947178718863, + 0.08851886178851265 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 29, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.2786885245901639, + "nll": 1.1808416791698115, + "brier": 0.7182697309306456, + "ece": 0.18638248746506375 + }, + "candidate": { + "acc": 0.36065573770491804, + "nll": 1.0988625637248937, + "brier": 0.6667978182679637, + "ece": 0.012652002959613952 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.08196721311475413, + "ci95": [ + -0.045508658008657964, + 0.22033898305084745 + ], + "parent_ci95": [ + 0.21052631578947367, + 0.3442622950819672 + ], + "candidate_ci95": [ + 0.2878787878787879, + 0.4444444444444444 + ] + }, + "nll": { + "candidate_minus_parent": -0.08197911544491787, + "ci95": [ + -0.147780102043434, + -0.021967052574866506 + ], + "parent_ci95": [ + 1.124289880805791, + 1.2425389171949466 + ], + "candidate_ci95": [ + 1.0929146615274494, + 1.1048799230089281 + ] + }, + "brier": { + "candidate_minus_parent": -0.05147191266268192, + "ci95": [ + -0.09260013917655888, + -0.013927776686896477 + ], + "parent_ci95": [ + 0.6834250582786119, + 0.757755005187276 + ], + "candidate_ci95": [ + 0.6628269859395758, + 0.6708305753576195 + ] + }, + "ece": { + "candidate_minus_parent": -0.1737304845054498, + "ci95": [ + -0.22450514563273516, + -0.07083593966067232 + ], + "parent_ci95": [ + 0.12131586807809672, + 0.2658403693964864 + ], + "candidate_ci95": [ + 0.0020626829750478443, + 0.09761867629624886 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 29, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "prefix_loss/heldout_task/transactions_3_plus": { + "windows": 159, + "episodes": 96, + "latent_groups": 32, + "sources": [ + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff" + ], + "transaction_count_histogram": { + "3": 31, + "4": 79, + "5": 25, + "6": 19, + "7": 5 + }, + "raw": { + "parent": { + "acc": 0.328125, + "nll": 1.6510195423387115, + "brier": 0.9138329151499195, + "ece": 0.3696085163632797 + }, + "candidate": { + "acc": 0.34375, + "nll": 1.1016034380803594, + "brier": 0.6683896262101987, + "ece": 0.07443605121945863 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.015625, + "ci95": [ + 0.0, + 0.041666666666666685 + ], + "parent_ci95": [ + 0.3177083333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3333333333333333, + 0.375 + ] + }, + "nll": { + "candidate_minus_parent": -0.5494161042583521, + "ci95": [ + -0.6179119075607382, + -0.48969539347820834 + ], + "parent_ci95": [ + 1.5799320401902752, + 1.735727089509863 + ], + "candidate_ci95": [ + 1.0803359724129493, + 1.1229626211532155 + ] + }, + "brier": { + "candidate_minus_parent": -0.24544328893972078, + "ci95": [ + -0.2750370542612782, + -0.22039742851750838 + ], + "parent_ci95": [ + 0.8864907256145825, + 0.9472722248543136 + ], + "candidate_ci95": [ + 0.6527527862224815, + 0.684133775429557 + ] + }, + "ece": { + "candidate_minus_parent": -0.29517246514382106, + "ci95": [ + -0.33283090348061833, + -0.2434390979172202 + ], + "parent_ci95": [ + 0.3498905546275224, + 0.39792126396644434 + ], + "candidate_ci95": [ + 0.04465048942192652, + 0.12030922907162292 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.328125, + "nll": 1.142172990991703, + "brier": 0.6948783988998107, + "ece": 0.11177142406175611 + }, + "candidate": { + "acc": 0.34375, + "nll": 1.0949910574408515, + "brier": 0.6638588432469327, + "ece": 0.04976727430001198 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.015625, + "ci95": [ + 0.0, + 0.041666666666666685 + ], + "parent_ci95": [ + 0.3177083333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3333333333333333, + 0.375 + ] + }, + "nll": { + "candidate_minus_parent": -0.04718193355085165, + "ci95": [ + -0.05846609201493588, + -0.03613562441133395 + ], + "parent_ci95": [ + 1.1293883233608148, + 1.1572591727556825 + ], + "candidate_ci95": [ + 1.0829251309123857, + 1.1064490416782484 + ] + }, + "brier": { + "candidate_minus_parent": -0.03101955565287806, + "ci95": [ + -0.03892913486669872, + -0.023066865238842306 + ], + "parent_ci95": [ + 0.6866324652855914, + 0.7047785964984155 + ], + "candidate_ci95": [ + 0.6551340116156167, + 0.6719950282068228 + ] + }, + "ece": { + "candidate_minus_parent": -0.06200414976174413, + "ci95": [ + -0.11525241336311848, + -0.012050859906388515 + ], + "parent_ci95": [ + 0.10303629178806634, + 0.14801943896591382 + ], + "candidate_ci95": [ + 0.013060497608093698, + 0.10147202300148765 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/all/all": { + "windows": 400, + "episodes": 192, + "latent_groups": 64, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 60, + "2": 60, + "3": 62, + "4": 74, + "5": 95, + "6": 41, + "7": 8 + }, + "raw": { + "parent": { + "acc": 0.3359375, + "nll": 1.7406074901136879, + "brier": 0.9401657076473695, + "ece": 0.38460155194384615 + }, + "candidate": { + "acc": 0.33550347222222227, + "nll": 1.1065801142748992, + "brier": 0.6719468244560088, + "ece": 0.05413551116313254 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.0004340277777777346, + "ci95": [ + -0.009548611111111107, + 0.008691406250000124 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3411458333333333 + ], + "candidate_ci95": [ + 0.3285590277777778, + 0.34375 + ] + }, + "nll": { + "candidate_minus_parent": -0.6340273758387887, + "ci95": [ + -0.6949692587650983, + -0.5763771561528886 + ], + "parent_ci95": [ + 1.683495559665557, + 1.8013055567222513 + ], + "candidate_ci95": [ + 1.096576736428186, + 1.1163791634819291 + ] + }, + "brier": { + "candidate_minus_parent": -0.26821888319136067, + "ci95": [ + -0.28957641859448985, + -0.24767413467914176 + ], + "parent_ci95": [ + 0.9192347540480064, + 0.9606597003615783 + ], + "candidate_ci95": [ + 0.6653425533094902, + 0.6783600881844497 + ] + }, + "ece": { + "candidate_minus_parent": -0.3304660407807136, + "ci95": [ + -0.3513774582804656, + -0.30953687802232904 + ], + "parent_ci95": [ + 0.37061568281624446, + 0.4036105845599263 + ], + "candidate_ci95": [ + 0.04543937738620642, + 0.07021350266475344 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 64, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3359375, + "nll": 1.1514101009185052, + "brier": 0.7007255316703395, + "ece": 0.1318957337257902 + }, + "candidate": { + "acc": 0.33550347222222227, + "nll": 1.0992675611044904, + "brier": 0.667040087036328, + "ece": 0.02521041186312405 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.0004340277777777346, + "ci95": [ + -0.009548611111111107, + 0.008691406250000124 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3411458333333333 + ], + "candidate_ci95": [ + 0.3285590277777778, + 0.34375 + ] + }, + "nll": { + "candidate_minus_parent": -0.05214253981401473, + "ci95": [ + -0.0611047205371555, + -0.042689065801267405 + ], + "parent_ci95": [ + 1.1420658463454039, + 1.1607064611272824 + ], + "candidate_ci95": [ + 1.094538571741435, + 1.1036078394289575 + ] + }, + "brier": { + "candidate_minus_parent": -0.03368544463401146, + "ci95": [ + -0.03992924731720826, + -0.027343189646057253 + ], + "parent_ci95": [ + 0.6942205689664265, + 0.7070023487666626 + ], + "candidate_ci95": [ + 0.6637669919722485, + 0.6700643112167214 + ] + }, + "ece": { + "candidate_minus_parent": -0.10668532186266615, + "ci95": [ + -0.13534219619205876, + -0.08066785804743311 + ], + "parent_ci95": [ + 0.11254079231923728, + 0.15998342150277217 + ], + "candidate_ci95": [ + 0.017516573977361352, + 0.04110521778685313 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 64, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/all/transactions_1_2": { + "windows": 120, + "episodes": 111, + "latent_groups": 53, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 60, + "2": 60 + }, + "raw": { + "parent": { + "acc": 0.35135135135135137, + "nll": 1.6960520792205063, + "brier": 0.9024482031715041, + "ece": 0.35711202622916166 + }, + "candidate": { + "acc": 0.34234234234234234, + "nll": 1.101448747899018, + "brier": 0.6684951405755102, + "ece": 0.0222143113151399 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.009009009009009028, + "ci95": [ + -0.09176435609921849, + 0.0762087912087912 + ], + "parent_ci95": [ + 0.30357142857142855, + 0.39814814814814814 + ], + "candidate_ci95": [ + 0.2935661183169883, + 0.3944954128440367 + ] + }, + "nll": { + "candidate_minus_parent": -0.5946033313214882, + "ci95": [ + -0.7581216779522638, + -0.44897828070830953 + ], + "parent_ci95": [ + 1.5569891830770557, + 1.8546291869076958 + ], + "candidate_ci95": [ + 1.0910651991677789, + 1.111816018257437 + ] + }, + "brier": { + "candidate_minus_parent": -0.2339530625959939, + "ci95": [ + -0.30292693258046355, + -0.16878874992495604 + ], + "parent_ci95": [ + 0.8386540155629556, + 0.9687019878832628 + ], + "candidate_ci95": [ + 0.6615977027563771, + 0.6754919205401331 + ] + }, + "ece": { + "candidate_minus_parent": -0.33489771491402176, + "ci95": [ + -0.39049483430670984, + -0.24899811558459425 + ], + "parent_ci95": [ + 0.3060085541080475, + 0.40839431898214706 + ], + "candidate_ci95": [ + 0.004200428773137663, + 0.07224785599092538 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 53, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.35135135135135137, + "nll": 1.1497208714203155, + "brier": 0.697066459482084, + "ece": 0.09289332764803161 + }, + "candidate": { + "acc": 0.34234234234234234, + "nll": 1.0992606854025573, + "brier": 0.6670804067459685, + "ece": 0.006460488670376252 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.009009009009009028, + "ci95": [ + -0.09176435609921849, + 0.0762087912087912 + ], + "parent_ci95": [ + 0.30357142857142855, + 0.39814814814814814 + ], + "candidate_ci95": [ + 0.2935661183169883, + 0.3944954128440367 + ] + }, + "nll": { + "candidate_minus_parent": -0.05046018601775826, + "ci95": [ + -0.08893543253969396, + -0.016119766171313037 + ], + "parent_ci95": [ + 1.1186675405813775, + 1.185234804225732 + ], + "candidate_ci95": [ + 1.0940675461919098, + 1.104439816406138 + ] + }, + "brier": { + "candidate_minus_parent": -0.029986052736115543, + "ci95": [ + -0.053591089547911044, + -0.008604571076369208 + ], + "parent_ci95": [ + 0.6774031678585275, + 0.7194912987064691 + ], + "candidate_ci95": [ + 0.6636148710664281, + 0.6705104599698513 + ] + }, + "ece": { + "candidate_minus_parent": -0.08643283897765536, + "ci95": [ + -0.1284744458005143, + -0.00567387954157532 + ], + "parent_ci95": [ + 0.04892543439467153, + 0.14708183067036984 + ], + "candidate_ci95": [ + 0.0007245024633393655, + 0.058453570890178434 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 53, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/all/transactions_3_plus": { + "windows": 280, + "episodes": 187, + "latent_groups": 63, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "3": 62, + "4": 74, + "5": 95, + "6": 41, + "7": 8 + }, + "raw": { + "parent": { + "acc": 0.3342245989304813, + "nll": 1.7555289505985587, + "brier": 0.9477640503549342, + "ece": 0.38925499483042736 + }, + "candidate": { + "acc": 0.3351158645276292, + "nll": 1.108188740668899, + "brier": 0.6725390323728005, + "ece": 0.066716952019375 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0008912655971479166, + "ci95": [ + -0.02216312056737585, + 0.024064171122994638 + ], + "parent_ci95": [ + 0.32432432432432434, + 0.34324324324324323 + ], + "candidate_ci95": [ + 0.3163970109060242, + 0.35294117647058826 + ] + }, + "nll": { + "candidate_minus_parent": -0.6473402099296597, + "ci95": [ + -0.733757998665238, + -0.5683007253955933 + ], + "parent_ci95": [ + 1.6771038718491593, + 1.8425302947893092 + ], + "candidate_ci95": [ + 1.0938231412925323, + 1.1228092182280043 + ] + }, + "brier": { + "candidate_minus_parent": -0.2752250179821336, + "ci95": [ + -0.3048612158832415, + -0.24746805643997863 + ], + "parent_ci95": [ + 0.9208384001562874, + 0.9762679410110967 + ], + "candidate_ci95": [ + 0.6630851718543517, + 0.6817288697755652 + ] + }, + "ece": { + "candidate_minus_parent": -0.32253804281105236, + "ci95": [ + -0.35615534554677647, + -0.2874507941358827 + ], + "parent_ci95": [ + 0.36944755188573547, + 0.4157692041247423 + ], + "candidate_ci95": [ + 0.04695683962256579, + 0.09947176928230858 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 63, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3342245989304813, + "nll": 1.150591689158398, + "brier": 0.7008946904316202, + "ece": 0.1363262357343515 + }, + "candidate": { + "acc": 0.3351158645276292, + "nll": 1.0988193944708307, + "brier": 0.6665887963196561, + "ece": 0.030538238453360175 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.0008912655971479166, + "ci95": [ + -0.02216312056737585, + 0.024064171122994638 + ], + "parent_ci95": [ + 0.32432432432432434, + 0.34324324324324323 + ], + "candidate_ci95": [ + 0.3163970109060242, + 0.35294117647058826 + ] + }, + "nll": { + "candidate_minus_parent": -0.05177229468756739, + "ci95": [ + -0.06603192468881637, + -0.03842071619828359 + ], + "parent_ci95": [ + 1.1367162320580266, + 1.1636937233838849 + ], + "candidate_ci95": [ + 1.0919966713264804, + 1.1051700107081668 + ] + }, + "brier": { + "candidate_minus_parent": -0.034305894111964164, + "ci95": [ + -0.043947105187773686, + -0.024996850755553735 + ], + "parent_ci95": [ + 0.6912483686237404, + 0.710424380972464 + ], + "candidate_ci95": [ + 0.6619212202937902, + 0.6709416587645569 + ] + }, + "ece": { + "candidate_minus_parent": -0.10578799728099131, + "ci95": [ + -0.14528332484454656, + -0.06599811211293838 + ], + "parent_ci95": [ + 0.11310969153121071, + 0.17550582691120856 + ], + "candidate_ci95": [ + 0.013988469825566535, + 0.059886216530795705 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 63, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/iid/all": { + "windows": 169, + "episodes": 96, + "latent_groups": 32, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 28, + "2": 19, + "3": 32, + "4": 23, + "5": 37, + "6": 25, + "7": 5 + }, + "raw": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.725757697659655, + "brier": 0.9401047180979623, + "ece": 0.4020367409320484 + }, + "candidate": { + "acc": 0.3368055555555555, + "nll": 1.1040626060473822, + "brier": 0.6702717561426815, + "ece": 0.0683276279449332 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.00347222222222221, + "ci95": [ + 0.0, + 0.010416666666666685 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3333333333333333, + 0.34375 + ] + }, + "nll": { + "candidate_minus_parent": -0.6216950916122728, + "ci95": [ + -0.717380983626561, + -0.5371541588570768 + ], + "parent_ci95": [ + 1.646108455710435, + 1.8141616324863405 + ], + "candidate_ci95": [ + 1.0878025017305053, + 1.1213471984146075 + ] + }, + "brier": { + "candidate_minus_parent": -0.2698329619552807, + "ci95": [ + -0.3036907003221278, + -0.23773415677566542 + ], + "parent_ci95": [ + 0.9117529183450316, + 0.9728276694058462 + ], + "candidate_ci95": [ + 0.6590794699959776, + 0.681685864765193 + ] + }, + "ece": { + "candidate_minus_parent": -0.3337091129871152, + "ci95": [ + -0.36812145646765443, + -0.2983093108505633 + ], + "parent_ci95": [ + 0.37645003059960197, + 0.42770359084715837 + ], + "candidate_ci95": [ + 0.04981607684558695, + 0.08984381483279809 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3333333333333333, + "nll": 1.1454640711429278, + "brier": 0.6964743796858682, + "ece": 0.13260620058724457 + }, + "candidate": { + "acc": 0.3368055555555555, + "nll": 1.0968550632456722, + "brier": 0.6653896553980921, + "ece": 0.02770294957160892 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.00347222222222221, + "ci95": [ + 0.0, + 0.010416666666666685 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3333333333333333 + ], + "candidate_ci95": [ + 0.3333333333333333, + 0.34375 + ] + }, + "nll": { + "candidate_minus_parent": -0.04860900789725564, + "ci95": [ + -0.06429208242319576, + -0.035905118558082774 + ], + "parent_ci95": [ + 1.133997196994798, + 1.1597081636865096 + ], + "candidate_ci95": [ + 1.0890255440357801, + 1.1043556773200416 + ] + }, + "brier": { + "candidate_minus_parent": -0.03108472428777609, + "ci95": [ + -0.042035188518183615, + -0.022463586773767412 + ], + "parent_ci95": [ + 0.6888515776201788, + 0.7064283693874138 + ], + "candidate_ci95": [ + 0.6599514223773825, + 0.6705037492073507 + ] + }, + "ece": { + "candidate_minus_parent": -0.10490325101563565, + "ci95": [ + -0.15451502648279997, + -0.07484636724625014 + ], + "parent_ci95": [ + 0.11485842839206964, + 0.18210567599060168 + ], + "candidate_ci95": [ + 0.018669307044195796, + 0.05262544209859075 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/iid/transactions_1_2": { + "windows": 47, + "episodes": 46, + "latent_groups": 24, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "1": 28, + "2": 19 + }, + "raw": { + "parent": { + "acc": 0.391304347826087, + "nll": 1.5129882776373453, + "brier": 0.8186307187425507, + "ece": 0.28917804828375704 + }, + "candidate": { + "acc": 0.32608695652173914, + "nll": 1.1015960837269156, + "brier": 0.668776812291835, + "ece": 0.03962849828397818 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.06521739130434784, + "ci95": [ + -0.20930232558139533, + 0.0667045454545454 + ], + "parent_ci95": [ + 0.3125, + 0.4865038115038115 + ], + "candidate_ci95": [ + 0.25, + 0.40009615384615377 + ] + }, + "nll": { + "candidate_minus_parent": -0.4113921939104297, + "ci95": [ + -0.6205679145739176, + -0.2101180318055222 + ], + "parent_ci95": [ + 1.3216366772080308, + 1.7139671734715645 + ], + "candidate_ci95": [ + 1.0852654892339724, + 1.1183779806860223 + ] + }, + "brier": { + "candidate_minus_parent": -0.1498539064507156, + "ci95": [ + -0.24867992033193947, + -0.043559138339383484 + ], + "parent_ci95": [ + 0.7190015074213191, + 0.9143874547417223 + ], + "candidate_ci95": [ + 0.6579463944990259, + 0.6799861375851854 + ] + }, + "ece": { + "candidate_minus_parent": -0.24954954999977885, + "ci95": [ + -0.3412014486136007, + -0.10647175778303841 + ], + "parent_ci95": [ + 0.20010121025847302, + 0.36941998397149045 + ], + "candidate_ci95": [ + 0.004974082473479452, + 0.11742284705331912 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 24, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.391304347826087, + "nll": 1.120753313789249, + "brier": 0.6776321501881324, + "ece": 0.040942654704029205 + }, + "candidate": { + "acc": 0.32608695652173914, + "nll": 1.09934156032772, + "brier": 0.6671793581123403, + "ece": 0.02327882216570413 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.06521739130434784, + "ci95": [ + -0.20930232558139533, + 0.0667045454545454 + ], + "parent_ci95": [ + 0.3125, + 0.4865038115038115 + ], + "candidate_ci95": [ + 0.25, + 0.40009615384615377 + ] + }, + "nll": { + "candidate_minus_parent": -0.021411753461529015, + "ci95": [ + -0.0742078488590243, + 0.031503542983940015 + ], + "parent_ci95": [ + 1.073362241891128, + 1.168284094132113 + ], + "candidate_ci95": [ + 1.0912596679638558, + 1.107740706599038 + ] + }, + "brier": { + "candidate_minus_parent": -0.010452792075792083, + "ci95": [ + -0.04319876483562449, + 0.024794671496791775 + ], + "parent_ci95": [ + 0.6464336025025834, + 0.7068355883848134 + ], + "candidate_ci95": [ + 0.6618441948255462, + 0.6727848577253428 + ] + }, + "ece": { + "candidate_minus_parent": -0.017663832538325076, + "ci95": [ + -0.10255297223563613, + 0.0762596890938096 + ], + "parent_ci95": [ + 0.0024123742064083987, + 0.12018630791428743 + ], + "candidate_ci95": [ + 0.001062319425773476, + 0.10029696521217593 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 24, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/iid/transactions_3_plus": { + "windows": 122, + "episodes": 91, + "latent_groups": 31, + "sources": [ + "sev_behavioral_artifact_handoff", + "sev_behavioral_authorized_export", + "sev_behavioral_lookup_catalog", + "sev_behavioral_update_export" + ], + "transaction_count_histogram": { + "3": 32, + "4": 23, + "5": 37, + "6": 25, + "7": 5 + }, + "raw": { + "parent": { + "acc": 0.32967032967032966, + "nll": 1.737592255031958, + "brier": 0.9498280924219143, + "ece": 0.41336371029633723 + }, + "candidate": { + "acc": 0.33516483516483514, + "nll": 1.1091061258354027, + "brier": 0.6728955720361526, + "ece": 0.086883523890482 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.005494505494505475, + "ci95": [ + -0.02777777777777779, + 0.03888888888888886 + ], + "parent_ci95": [ + 0.3111111111111111, + 0.34444444444444444 + ], + "candidate_ci95": [ + 0.3146067415730337, + 0.3532608695652174 + ] + }, + "nll": { + "candidate_minus_parent": -0.6284861291965553, + "ci95": [ + -0.7365761589792299, + -0.5299836225042109 + ], + "parent_ci95": [ + 1.6481255398405608, + 1.835985485646765 + ], + "candidate_ci95": [ + 1.0860489866208896, + 1.1354374741301707 + ] + }, + "brier": { + "candidate_minus_parent": -0.27693252038576177, + "ci95": [ + -0.32272262614292246, + -0.2379724134329891 + ], + "parent_ci95": [ + 0.9130870924640325, + 0.9881958378896699 + ], + "candidate_ci95": [ + 0.6577998309311673, + 0.6887578066446614 + ] + }, + "ece": { + "candidate_minus_parent": -0.32648018640585524, + "ci95": [ + -0.3761642813685217, + -0.25983029483154757 + ], + "parent_ci95": [ + 0.38081116400769355, + 0.4491761367813116 + ], + "candidate_ci95": [ + 0.05816060461883278, + 0.13675444810712686 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 31, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.32967032967032966, + "nll": 1.1438484259633617, + "brier": 0.695746062044986, + "ece": 0.14019819112070317 + }, + "candidate": { + "acc": 0.33516483516483514, + "nll": 1.0973027167047875, + "brier": 0.6654772404419866, + "ece": 0.03627728484177039 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.005494505494505475, + "ci95": [ + -0.02777777777777779, + 0.03888888888888886 + ], + "parent_ci95": [ + 0.3111111111111111, + 0.34444444444444444 + ], + "candidate_ci95": [ + 0.3146067415730337, + 0.3532608695652174 + ] + }, + "nll": { + "candidate_minus_parent": -0.04654570925857415, + "ci95": [ + -0.06762029227679757, + -0.026984482065314105 + ], + "parent_ci95": [ + 1.125887311305212, + 1.1627207604459122 + ], + "candidate_ci95": [ + 1.0864801553091392, + 1.1078573621419587 + ] + }, + "brier": { + "candidate_minus_parent": -0.030268821602999374, + "ci95": [ + -0.04420502400536354, + -0.01727495129952463 + ], + "parent_ci95": [ + 0.6846871811700046, + 0.7085095782172175 + ], + "candidate_ci95": [ + 0.6581439135738535, + 0.6725572237106584 + ] + }, + "ece": { + "candidate_minus_parent": -0.10392090627893277, + "ci95": [ + -0.17733269976239296, + -0.06015447573499777 + ], + "parent_ci95": [ + 0.11589770798809797, + 0.2102353079649942 + ], + "candidate_ci95": [ + 0.018755743706560857, + 0.06865304317626338 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 31, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/heldout_task/all": { + "windows": 231, + "episodes": 96, + "latent_groups": 32, + "sources": [ + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff" + ], + "transaction_count_histogram": { + "1": 32, + "2": 41, + "3": 30, + "4": 51, + "5": 58, + "6": 16, + "7": 3 + }, + "raw": { + "parent": { + "acc": 0.3385416666666667, + "nll": 1.7554572825677204, + "brier": 0.9402266971967772, + "ece": 0.367367651579607 + }, + "candidate": { + "acc": 0.33420138888888884, + "nll": 1.1090976225024158, + "brier": 0.6736218927693362, + "ece": 0.055902489562076274 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.004340277777777846, + "ci95": [ + -0.02256944444444442, + 0.01215277777777768 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3489583333333333 + ], + "candidate_ci95": [ + 0.3211805555555555, + 0.34722222222222227 + ] + }, + "nll": { + "candidate_minus_parent": -0.6463596600653045, + "ci95": [ + -0.7234564302872076, + -0.5713167884632395 + ], + "parent_ci95": [ + 1.678905602327295, + 1.8366894657861392 + ], + "candidate_ci95": [ + 1.099442596350794, + 1.1198317908611226 + ] + }, + "brier": { + "candidate_minus_parent": -0.26660480442744106, + "ci95": [ + -0.29319111488031036, + -0.240211311858877 + ], + "parent_ci95": [ + 0.9137964635276594, + 0.96882149817189 + ], + "candidate_ci95": [ + 0.6668101247461546, + 0.6816392474818874 + ] + }, + "ece": { + "candidate_minus_parent": -0.3114651620175307, + "ci95": [ + -0.3436920042267205, + -0.28518855784843594 + ], + "parent_ci95": [ + 0.35036920812646866, + 0.39004405490227895 + ], + "candidate_ci95": [ + 0.037515519119150936, + 0.07724749162261044 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3385416666666667, + "nll": 1.1573561306940823, + "brier": 0.7049766836548104, + "ece": 0.13118526686433576 + }, + "candidate": { + "acc": 0.33420138888888884, + "nll": 1.1016800589633087, + "brier": 0.6686905186745636, + "ece": 0.029678489842914945 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.004340277777777846, + "ci95": [ + -0.02256944444444442, + 0.01215277777777768 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3489583333333333 + ], + "candidate_ci95": [ + 0.3211805555555555, + 0.34722222222222227 + ] + }, + "nll": { + "candidate_minus_parent": -0.0556760717307736, + "ci95": [ + -0.06793569023212345, + -0.04323814096919579 + ], + "parent_ci95": [ + 1.1444930201587054, + 1.1710769227292688 + ], + "candidate_ci95": [ + 1.0968679616282462, + 1.1066588132568023 + ] + }, + "brier": { + "candidate_minus_parent": -0.03628616498024673, + "ci95": [ + -0.044616583193091544, + -0.028245525790847326 + ], + "parent_ci95": [ + 0.696442208712845, + 0.7142081513405784 + ], + "candidate_ci95": [ + 0.6652185510566648, + 0.6722788224530039 + ] + }, + "ece": { + "candidate_minus_parent": -0.10150677702142082, + "ci95": [ + -0.14658130666733923, + -0.06451661944813686 + ], + "parent_ci95": [ + 0.10649446224605773, + 0.17110719018645643 + ], + "candidate_ci95": [ + 0.012983333179381055, + 0.0535721921541909 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/heldout_task/transactions_1_2": { + "windows": 73, + "episodes": 65, + "latent_groups": 29, + "sources": [ + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff" + ], + "transaction_count_histogram": { + "1": 32, + "2": 41 + }, + "raw": { + "parent": { + "acc": 0.3230769230769231, + "nll": 1.82560492341782, + "brier": 0.9617651921519934, + "ece": 0.40518837985206335 + }, + "candidate": { + "acc": 0.35384615384615387, + "nll": 1.101344479466967, + "brier": 0.6682958036685728, + "ece": 0.009890425152577426 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.03076923076923077, + "ci95": [ + -0.07249040920716111, + 0.14516129032258063 + ], + "parent_ci95": [ + 0.26666666666666666, + 0.373134328358209 + ], + "candidate_ci95": [ + 0.2835820895522388, + 0.42309322033898306 + ] + }, + "nll": { + "candidate_minus_parent": -0.7242604439508531, + "ci95": [ + -0.957232645851412, + -0.525437308197973 + ], + "parent_ci95": [ + 1.6314021132204477, + 2.0514856371260306 + ], + "candidate_ci95": [ + 1.0882392331642454, + 1.1148529173040966 + ] + }, + "brier": { + "candidate_minus_parent": -0.29346938848342063, + "ci95": [ + -0.3908429661994472, + -0.20344320094316598 + ], + "parent_ci95": [ + 0.8735299166326217, + 1.0532749805380457 + ], + "candidate_ci95": [ + 0.6595671989614225, + 0.6775008638361005 + ] + }, + "ece": { + "candidate_minus_parent": -0.3952979546994859, + "ci95": [ + -0.441695614734844, + -0.2833377661075722 + ], + "parent_ci95": [ + 0.3438845701232617, + 0.4694929446306306 + ], + "candidate_ci95": [ + 0.00208642196278144, + 0.08624279806717236 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 29, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3230769230769231, + "nll": 1.170220989128455, + "brier": 0.7108199706747266, + "ece": 0.129658419269941 + }, + "candidate": { + "acc": 0.35384615384615387, + "nll": 1.0992034508401352, + "brier": 0.6670103796251514, + "ece": 0.005441716572471168 + }, + "paired": { + "acc": { + "candidate_minus_parent": 0.03076923076923077, + "ci95": [ + -0.07249040920716111, + 0.14516129032258063 + ], + "parent_ci95": [ + 0.26666666666666666, + 0.373134328358209 + ], + "candidate_ci95": [ + 0.2835820895522388, + 0.42309322033898306 + ] + }, + "nll": { + "candidate_minus_parent": -0.0710175382883198, + "ci95": [ + -0.12245128408798325, + -0.02299298460609988 + ], + "parent_ci95": [ + 1.1241940339495027, + 1.2172337233951056 + ], + "candidate_ci95": [ + 1.0926773710164572, + 1.105859396115898 + ] + }, + "brier": { + "candidate_minus_parent": -0.04380959104957516, + "ci95": [ + -0.07619559868356404, + -0.013991724849913031 + ], + "parent_ci95": [ + 0.682306683307591, + 0.7410407625903449 + ], + "candidate_ci95": [ + 0.6626434656543708, + 0.6714627601489861 + ] + }, + "ece": { + "candidate_minus_parent": -0.12421670269746984, + "ci95": [ + -0.16877106313977253, + -0.03649558296884139 + ], + "parent_ci95": [ + 0.08237175233001533, + 0.19524531874276807 + ], + "candidate_ci95": [ + 0.0009436284449954635, + 0.07885721568333505 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 29, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + }, + "sampled_loss/heldout_task/transactions_3_plus": { + "windows": 158, + "episodes": 96, + "latent_groups": 32, + "sources": [ + "sev_behavioral_retry_export", + "sev_behavioral_stale_handoff" + ], + "transaction_count_histogram": { + "3": 30, + "4": 51, + "5": 58, + "6": 16, + "7": 3 + }, + "raw": { + "parent": { + "acc": 0.3385416666666667, + "nll": 1.7725314432710662, + "brier": 0.9458075104789425, + "ece": 0.3679623325707097 + }, + "candidate": { + "acc": 0.3350694444444445, + "nll": 1.107319135979817, + "brier": 0.672201062483582, + "ece": 0.0675443921109179 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.00347222222222221, + "ci95": [ + -0.039930555555555525, + 0.03298611111111105 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3489583333333333 + ], + "candidate_ci95": [ + 0.3038194444444445, + 0.3697916666666667 + ] + }, + "nll": { + "candidate_minus_parent": -0.6652123072912492, + "ci95": [ + -0.8025975641357458, + -0.5516005543207456 + ], + "parent_ci95": [ + 1.660404920994282, + 1.9139018837933768 + ], + "candidate_ci95": [ + 1.0923167803855924, + 1.123221355468193 + ] + }, + "brier": { + "candidate_minus_parent": -0.2736064479953605, + "ci95": [ + -0.3138128898431603, + -0.23674963543085267 + ], + "parent_ci95": [ + 0.9083144203125041, + 0.9882132562854329 + ], + "candidate_ci95": [ + 0.6617359212586741, + 0.6836821973843982 + ] + }, + "ece": { + "candidate_minus_parent": -0.3004179404597918, + "ci95": [ + -0.3536256938359151, + -0.2540426675462728 + ], + "parent_ci95": [ + 0.3427788587563052, + 0.4034734267267112 + ], + "candidate_ci95": [ + 0.03351414265736119, + 0.10671035016969924 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + }, + "calibrated": { + "parent": { + "acc": 0.3385416666666667, + "nll": 1.1569837407286923, + "brier": 0.7057751610897839, + "ece": 0.13265594469103895 + }, + "candidate": { + "acc": 0.3350694444444445, + "nll": 1.100257078603226, + "brier": 0.6676424586620296, + "ece": 0.037382529727641785 + }, + "paired": { + "acc": { + "candidate_minus_parent": -0.00347222222222221, + "ci95": [ + -0.039930555555555525, + 0.03298611111111105 + ], + "parent_ci95": [ + 0.3333333333333333, + 0.3489583333333333 + ], + "candidate_ci95": [ + 0.3038194444444445, + 0.3697916666666667 + ] + }, + "nll": { + "candidate_minus_parent": -0.05672666212546629, + "ci95": [ + -0.0763045424395524, + -0.04000481087581666 + ], + "parent_ci95": [ + 1.1384890247157313, + 1.1790609577960465 + ], + "candidate_ci95": [ + 1.0924179530820977, + 1.1083763076809625 + ] + }, + "brier": { + "candidate_minus_parent": -0.03813270242775424, + "ci95": [ + -0.051958591175575164, + -0.026782914028508145 + ], + "parent_ci95": [ + 0.693406489808031, + 0.7212784754522964 + ], + "candidate_ci95": [ + 0.6621406053699213, + 0.6732328817854741 + ] + }, + "ece": { + "candidate_minus_parent": -0.09527341496339717, + "ci95": [ + -0.15369852233059553, + -0.03826622807493024 + ], + "parent_ci95": [ + 0.10441488602576846, + 0.17836289667024915 + ], + "candidate_ci95": [ + 0.008393098027866705, + 0.08302721483572066 + ] + } + }, + "bootstrap": { + "samples": 1000, + "seed": 0, + "latent_groups": 32, + "unit": "group_id alone; every source, episode and view in a group moves together", + "method": "paired cluster percentile bootstrap; weighted ECE recomputed each draw", + "temperature_uncertainty": "conditional on fixed calibration-fitted temperatures" + } + } + } + } + }, + "interpretation": [ + "The preregistered primary is equal-episode full-view IID development accuracy/NLL. Combined development, heldout-task/prior shift, loss views and transaction-count panels are secondary.", + "Every episode has equal weight within each panel; its eligible windows share that weight. Probabilities are not pooled into an episode ensemble.", + "Length is the number of observed transactions in _meta.request_ids, not a window's ordinal index. An episode can occur in both length panels.", + "Raw logits are restored using recorded inference_temperature; each model's one diagnostic temperature is fit only on its full-view calibration origin rows.", + "Calibration panels are in-sample for temperature fitting. Development labels never select temperatures; bootstrap intervals condition on the fitted temperatures.", + "Group resampling ignores source boundaries. Matched origins and every episode/window/view in a latent group remain dependent.", + "heldout_task combines unfamiliar tasks with a v2 prior distribution shift in the same controller implementation, not an implementation holdout.", + "Targets describe authored simulated policies. This is a development comparison, not a real-traffic accuracy claim or publication gate." + ] +}