{
  "run": "002-bigger-model-smaller-gain",
  "as_of": "2026-07-13",
  "y": "MMLU, 5-shot (%)",
  "x": "log10(training compute in FLOP)",
  "n_models": 13,
  "split_flop": "1e25 (Epoch's GPT-4-scale line)",
  "compute_span_orders_of_mag": 3.2,
  "compute_span_x": 1592.0,
  "mmlu_span": [
    43.9,
    92.0
  ],
  "overall": {
    "n": 13,
    "slope_pts_per_decade": 12.52,
    "slope_ci95": [
      8.25,
      16.79
    ],
    "intercept": -235.49,
    "r2": 0.791,
    "p": 4.70935505232687e-05,
    "y_sd": 14.26,
    "resid_sd": 6.81
  },
  "early_lt_1e25": {
    "n": 6,
    "slope_pts_per_decade": 21.15,
    "slope_ci95": [
      5.44,
      36.85
    ],
    "intercept": -444.47,
    "r2": 0.777,
    "p": 0.020165957840447283,
    "y_sd": 13.0,
    "resid_sd": 6.86
  },
  "late_ge_1e25": {
    "n": 7,
    "slope_pts_per_decade": 3.35,
    "slope_ci95": [
      -0.35,
      7.05
    ],
    "intercept": 1.8,
    "r2": 0.52,
    "p": 0.0675750980981551,
    "y_sd": 2.8,
    "resid_sd": 2.13
  },
  "payoff_collapse_ratio": 6.3,
  "quadratic": {
    "a_x2": -4.94,
    "b_x": 260.35,
    "c": -3337.6,
    "concave": true
  },
  "human_expert_ceiling": 89.8,
  "models": [
    {
      "date": "2020-05",
      "model": "GPT-3 175B",
      "compute_flop": "3.14e+23",
      "mmlu": 43.9,
      "x": 23.5
    },
    {
      "date": "2022-04",
      "model": "Chinchilla 70B",
      "compute_flop": "5.76e+23",
      "mmlu": 67.5,
      "x": 23.76
    },
    {
      "date": "2021-12",
      "model": "Gopher 280B",
      "compute_flop": "6.31e+23",
      "mmlu": 60.0,
      "x": 23.8
    },
    {
      "date": "2022-04",
      "model": "PaLM 540B",
      "compute_flop": "2.53e+24",
      "mmlu": 69.3,
      "x": 24.4
    },
    {
      "date": "2023-07",
      "model": "Claude 2",
      "compute_flop": "3.87e+24",
      "mmlu": 78.5,
      "x": 24.59
    },
    {
      "date": "2023-05",
      "model": "PaLM 2 (L)",
      "compute_flop": "7.34e+24",
      "mmlu": 78.3,
      "x": 24.87
    },
    {
      "date": "2024-03",
      "model": "Claude 3 Opus",
      "compute_flop": "1.64e+25",
      "mmlu": 86.8,
      "x": 25.21
    },
    {
      "date": "2023-03",
      "model": "GPT-4",
      "compute_flop": "2.10e+25",
      "mmlu": 86.4,
      "x": 25.32
    },
    {
      "date": "2024-06",
      "model": "Claude 3.5 Sonnet",
      "compute_flop": "2.70e+25",
      "mmlu": 88.7,
      "x": 25.43
    },
    {
      "date": "2024-07",
      "model": "Llama 3.1 405B",
      "compute_flop": "3.80e+25",
      "mmlu": 88.6,
      "x": 25.58
    },
    {
      "date": "2023-12",
      "model": "Gemini 1.0 Ultra",
      "compute_flop": "5.00e+25",
      "mmlu": 83.7,
      "x": 25.7
    },
    {
      "date": "2025-02",
      "model": "GPT-4.5",
      "compute_flop": "3.80e+26",
      "mmlu": 90.8,
      "x": 26.58
    },
    {
      "date": "2025-07",
      "model": "Grok 4",
      "compute_flop": "5.00e+26",
      "mmlu": 92.0,
      "x": 26.7
    }
  ]
}