{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://alloevil.github.io/llm-benchmarks-tracker/schema/benchmark.schema.json",
  "title": "Benchmark",
  "description": "One evaluation benchmark. File name must equal `id` (data/benchmarks/<id>.json).",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "id",
    "name",
    "layer",
    "released",
    "maintainer",
    "description",
    "domains",
    "metric",
    "status",
    "links",
    "description_zh"
  ],
  "properties": {
    "id": {
      "$ref": "#/$defs/slug"
    },
    "name": {
      "type": "string",
      "minLength": 2,
      "maxLength": 60
    },
    "full_name": {
      "type": "string",
      "maxLength": 160
    },
    "layer": {
      "description": "model = static prompt/response scoring; agent = interactive environment, tools, or multi-step execution.",
      "enum": [
        "model",
        "agent"
      ]
    },
    "released": {
      "$ref": "#/$defs/yearMonth"
    },
    "maintainer": {
      "type": "string",
      "minLength": 2,
      "maxLength": 120
    },
    "description": {
      "description": "What it tests, how it is scored, and why it matters. English. No score claims here; those live in results.",
      "type": "string",
      "minLength": 60,
      "maxLength": 600
    },
    "description_zh": {
      "description": "Simplified Chinese translation of `description`. Names and metrics stay in Latin script.",
      "type": "string",
      "minLength": 20,
      "maxLength": 400
    },
    "domains": {
      "type": "array",
      "minItems": 1,
      "uniqueItems": true,
      "items": {
        "enum": [
          "knowledge",
          "reasoning",
          "math",
          "science",
          "code",
          "software-engineering",
          "tool-use",
          "web",
          "computer-use",
          "multimodal",
          "long-context",
          "instruction-following",
          "factuality",
          "commonsense",
          "general-assistant",
          "ml-engineering",
          "research",
          "safety",
          "human-preference"
        ]
      }
    },
    "metric": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "name",
        "unit",
        "higher_is_better"
      ],
      "properties": {
        "name": {
          "description": "e.g. accuracy, pass@1, resolve rate, success rate",
          "type": "string"
        },
        "unit": {
          "enum": [
            "percent",
            "score",
            "elo"
          ]
        },
        "higher_is_better": {
          "type": "boolean"
        }
      }
    },
    "task_count": {
      "type": [
        "integer",
        "null"
      ],
      "minimum": 1
    },
    "splits": {
      "description": "Named evaluation subsets that results may reference via conditions.split.",
      "type": "array",
      "items": {
        "type": "object",
        "additionalProperties": false,
        "required": [
          "name"
        ],
        "properties": {
          "name": {
            "type": "string"
          },
          "task_count": {
            "type": [
              "integer",
              "null"
            ],
            "minimum": 1
          },
          "note": {
            "type": "string"
          }
        }
      }
    },
    "status": {
      "description": "active = still separates frontier systems; saturating = top scores within ~5 points of ceiling or human baseline; saturated = no longer discriminative; retired = maintainer stopped accepting results.",
      "enum": [
        "active",
        "saturating",
        "saturated",
        "retired"
      ]
    },
    "contamination_risk": {
      "description": "low = live/rolling or private test set; medium = public test set with mitigations; high = public, static, widely scraped.",
      "enum": [
        "low",
        "medium",
        "high"
      ]
    },
    "human_baseline": {
      "type": [
        "object",
        "null"
      ],
      "additionalProperties": false,
      "required": [
        "value",
        "population",
        "source"
      ],
      "properties": {
        "value": {
          "type": "number"
        },
        "population": {
          "description": "Who was measured, e.g. 'domain PhD experts', 'untrained crowd workers'.",
          "type": "string"
        },
        "source": {
          "type": "string",
          "format": "uri"
        }
      }
    },
    "supersedes": {
      "oneOf": [
        {
          "$ref": "#/$defs/slug"
        },
        {
          "type": "null"
        }
      ]
    },
    "superseded_by": {
      "oneOf": [
        {
          "$ref": "#/$defs/slug"
        },
        {
          "type": "null"
        }
      ]
    },
    "links": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "paper": {
          "type": [
            "string",
            "null"
          ],
          "format": "uri"
        },
        "website": {
          "type": [
            "string",
            "null"
          ],
          "format": "uri"
        },
        "leaderboard": {
          "type": [
            "string",
            "null"
          ],
          "format": "uri"
        },
        "dataset": {
          "type": [
            "string",
            "null"
          ],
          "format": "uri"
        },
        "code": {
          "type": [
            "string",
            "null"
          ],
          "format": "uri"
        }
      },
      "anyOf": [
        {
          "required": [
            "paper"
          ],
          "properties": {
            "paper": {
              "type": "string"
            }
          }
        },
        {
          "required": [
            "website"
          ],
          "properties": {
            "website": {
              "type": "string"
            }
          }
        }
      ]
    },
    "notes": {
      "type": "string",
      "maxLength": 800
    }
  },
  "$defs": {
    "slug": {
      "type": "string",
      "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$",
      "maxLength": 40
    },
    "yearMonth": {
      "type": "string",
      "pattern": "^(19|20)[0-9]{2}-(0[1-9]|1[0-2])$"
    }
  }
}