{
  "generated": "2026-07-02",
  "prereg": "preregistration/2026-07-02_study1-replicate.md",
  "n_items": 60,
  "bootstrap": {
    "resamples": 10000,
    "seed": 20260702
  },
  "arms": {
    "A3_r0": {
      "n": 60,
      "accuracy": 0.85,
      "mean_confidence": 0.8889,
      "overconfidence": 0.0389,
      "ece": 0.0389,
      "mce": 0.1333,
      "brier": 0.0722,
      "auc": 0.8976
    },
    "A3_post": {
      "n": 60,
      "accuracy": 0.8833,
      "mean_confidence": 0.9167,
      "overconfidence": 0.0333,
      "ece": 0.0333,
      "mce": 0.0417,
      "brier": 0.0778,
      "auc": 0.8032
    },
    "B3": {
      "n": 60,
      "accuracy": 0.8833,
      "mean_confidence": 0.8889,
      "overconfidence": 0.0056,
      "ece": 0.0611,
      "mce": 0.2083,
      "brier": 0.0759,
      "auc": 0.8073
    },
    "A7_r0": {
      "n": 60,
      "accuracy": 0.8833,
      "mean_confidence": 0.8738,
      "overconfidence": -0.0095,
      "ece": 0.1048,
      "mce": 0.4286,
      "brier": 0.0673,
      "auc": 0.872
    },
    "A7_post": {
      "n": 60,
      "accuracy": 0.8833,
      "mean_confidence": 0.9167,
      "overconfidence": 0.0333,
      "ece": 0.0714,
      "mce": 0.4286,
      "brier": 0.0442,
      "auc": 0.9137
    },
    "B7": {
      "n": 60,
      "accuracy": 0.8833,
      "mean_confidence": 0.8786,
      "overconfidence": -0.0048,
      "ece": 0.081,
      "mce": 0.4286,
      "brier": 0.0565,
      "auc": 0.9016
    }
  },
  "deltas_ci95": {
    "P1_overconf_A3_post_minus_r0": {
      "mean": -0.0056,
      "lo": -0.05,
      "hi": 0.0333,
      "defined_in": 10000
    },
    "P1_overconf_A7_post_minus_r0": {
      "mean": 0.0427,
      "lo": 0.0095,
      "hi": 0.0786,
      "defined_in": 10000
    },
    "P2a_auc_A3_post_minus_B3": {
      "mean": -0.0026,
      "lo": -0.2263,
      "hi": 0.2127,
      "defined_in": 9984
    },
    "P2a_auc_A7_post_minus_B7": {
      "mean": 0.0125,
      "lo": -0.075,
      "hi": 0.1083,
      "defined_in": 9988
    },
    "P2b_brier_B7_minus_B3": {
      "mean": -0.0195,
      "lo": -0.0512,
      "hi": 0.0063,
      "defined_in": 10000
    },
    "P2b_brier_A7post_minus_A3post": {
      "mean": -0.0334,
      "lo": -0.0685,
      "hi": -0.0044,
      "defined_in": 10000
    },
    "P3_conf_A3_post_minus_B3": {
      "mean": 0.0278,
      "lo": -0.0222,
      "hi": 0.0778,
      "defined_in": 10000
    },
    "P3_conf_A7_post_minus_B7": {
      "mean": 0.0378,
      "lo": -0.0,
      "hi": 0.0762,
      "defined_in": 10000
    },
    "desc_acc_A3_post_minus_r0": {
      "mean": 0.0332,
      "lo": 0.0,
      "hi": 0.0833,
      "defined_in": 10000
    },
    "desc_acc_A7_post_minus_r0": {
      "mean": 0.0,
      "lo": 0.0,
      "hi": 0.0,
      "defined_in": 10000
    }
  },
  "verdicts": {
    "P1_A3": "NOT CONFIRMED",
    "P1_A7": "CONFIRMED",
    "P2a_A3vB3_gap_includes_zero": true,
    "P2b_B7_better_than_B3": true,
    "P2b_A7_better_than_A3": true,
    "P1_primary": "PARTIAL"
  },
  "round0_unanimity": {
    "A3": {
      "unanimous": 45,
      "n": 60,
      "share": 0.75
    },
    "A7": {
      "unanimous": 40,
      "n": 60,
      "share": 0.6667
    }
  },
  "diversity_by_round": {
    "A3": [
      1.3333,
      1.25
    ],
    "A7": [
      1.7167,
      1.4167
    ]
  },
  "per_source": {
    "gsm8k": {
      "n": 20,
      "A3_post": {
        "accuracy": 0.95,
        "overconfidence": 0.0
      },
      "B3": {
        "accuracy": 1.0,
        "overconfidence": -0.0667
      },
      "A7_post": {
        "accuracy": 0.95,
        "overconfidence": 0.0
      },
      "B7": {
        "accuracy": 1.0,
        "overconfidence": -0.1
      }
    },
    "mmlu": {
      "n": 20,
      "A3_post": {
        "accuracy": 0.95,
        "overconfidence": 0.0
      },
      "B3": {
        "accuracy": 1.0,
        "overconfidence": -0.0833
      },
      "A7_post": {
        "accuracy": 1.0,
        "overconfidence": -0.0286
      },
      "B7": {
        "accuracy": 0.95,
        "overconfidence": -0.0143
      }
    },
    "triviaqa": {
      "n": 20,
      "A3_post": {
        "accuracy": 0.75,
        "overconfidence": 0.1
      },
      "B3": {
        "accuracy": 0.65,
        "overconfidence": 0.1667
      },
      "A7_post": {
        "accuracy": 0.7,
        "overconfidence": 0.1286
      },
      "B7": {
        "accuracy": 0.7,
        "overconfidence": 0.1
      }
    }
  }
}