{
  "domain": "mathematics",
  "image": "python:3.12-slim",
  "data": "PutnamBench’s Lean 4 statements at a fixed commit, with Mathlib pinned in lake-manifest.json.",
  "benchmark": {
    "task": "Automated theorem proving",
    "metric": {
      "name": "problems proved",
      "better": "higher"
    },
    "cost": "GPU-hours",
    "subtasks": "topics",
    "guardrails": [
      "Each proof compiles against the pinned Mathlib with no sorry and no new axioms",
      "At most 64 proof attempts per problem",
      "Statements are not changed"
    ],
    "aggregate": "sum",
    "baseline": {
      "name": "Best-first tactic search",
      "score": 64,
      "cost": 180,
      "subtasks": {
        "algebra": 6,
        "analysis": 15,
        "number theory": 16,
        "combinatorics": 5,
        "geometry": 8,
        "linear algebra": 6,
        "probability": 2,
        "abstract algebra": 6
      }
    }
  }
}