{
  "site": "https://inspect-evals-lint.generality.org/",
  "rules": [
    {
      "code": "IEFS001",
      "name": "package_location",
      "category": "file_structure",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": false,
      "summary": "The package exists at <source-root>/<name>/ with an __init__.py",
      "url": "https://inspect-evals-lint.generality.org/rules/IEFS001/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEFS001.md"
    },
    {
      "code": "IEFS002",
      "name": "main_file",
      "category": "file_structure",
      "scopes": [
        "eval"
      ],
      "allowlist": false,
      "summary": "<name>.py or tasks.py exists and defines at least one @task function",
      "url": "https://inspect-evals-lint.generality.org/rules/IEFS002/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEFS002.md"
    },
    {
      "code": "IEFS003",
      "name": "init_exports",
      "category": "file_structure",
      "scopes": [
        "eval"
      ],
      "allowlist": false,
      "summary": "__init__.py exports every @task function from the main file",
      "url": "https://inspect-evals-lint.generality.org/rules/IEFS003/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEFS003.md"
    },
    {
      "code": "IEFS004",
      "name": "registry",
      "category": "file_structure",
      "scopes": [
        "eval"
      ],
      "allowlist": false,
      "summary": "The evaluation is registered so inspect eval can find its tasks",
      "url": "https://inspect-evals-lint.generality.org/rules/IEFS004/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEFS004.md"
    },
    {
      "code": "IEFS005",
      "name": "eval_yaml",
      "category": "file_structure",
      "scopes": [
        "eval"
      ],
      "allowlist": false,
      "summary": "eval.yaml exists, is a mapping, and defines the required fields",
      "url": "https://inspect-evals-lint.generality.org/rules/IEFS005/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEFS005.md"
    },
    {
      "code": "IEFS006",
      "name": "readme",
      "category": "file_structure",
      "scopes": [
        "eval"
      ],
      "allowlist": false,
      "summary": "README.md exists and has no TODO markers",
      "url": "https://inspect-evals-lint.generality.org/rules/IEFS006/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEFS006.md"
    },
    {
      "code": "IECQ001",
      "name": "private_api_imports",
      "category": "code_quality",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": false,
      "summary": "No imports from private inspect_ai modules",
      "url": "https://inspect-evals-lint.generality.org/rules/IECQ001/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IECQ001.md"
    },
    {
      "code": "IECQ002",
      "name": "score_constants",
      "category": "code_quality",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": false,
      "summary": "Score() values use the CORRECT/INCORRECT constants, not string literals",
      "url": "https://inspect-evals-lint.generality.org/rules/IECQ002/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IECQ002.md"
    },
    {
      "code": "IECQ003",
      "name": "unscored_reason",
      "category": "code_quality",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": false,
      "summary": "Score.unscored() passes a reason= and the legacy unscored_reason metadata key is gone",
      "url": "https://inspect-evals-lint.generality.org/rules/IECQ003/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IECQ003.md"
    },
    {
      "code": "IECQ004",
      "name": "external_dependencies",
      "category": "code_quality",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": false,
      "summary": "Third-party imports are declared in pyproject.toml",
      "url": "https://inspect-evals-lint.generality.org/rules/IECQ004/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IECQ004.md"
    },
    {
      "code": "IETS001",
      "name": "tests_exist",
      "category": "tests",
      "scopes": [
        "eval"
      ],
      "allowlist": false,
      "summary": "A test directory exists for the evaluation",
      "url": "https://inspect-evals-lint.generality.org/rules/IETS001/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IETS001.md"
    },
    {
      "code": "IETS002",
      "name": "tests_init",
      "category": "tests",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": false,
      "summary": "The test directory and its sub-directories contain __init__.py",
      "url": "https://inspect-evals-lint.generality.org/rules/IETS002/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IETS002.md"
    },
    {
      "code": "IETS003",
      "name": "e2e_test",
      "category": "tests",
      "scopes": [
        "eval"
      ],
      "allowlist": false,
      "summary": "Some test runs eval() against mockllm/model",
      "url": "https://inspect-evals-lint.generality.org/rules/IETS003/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IETS003.md"
    },
    {
      "code": "IETS004",
      "name": "record_to_sample_test",
      "category": "tests",
      "scopes": [
        "eval"
      ],
      "allowlist": false,
      "summary": "record_to_sample is exercised by a test when the evaluation uses it",
      "url": "https://inspect-evals-lint.generality.org/rules/IETS004/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IETS004.md"
    },
    {
      "code": "IETS005",
      "name": "custom_solver_tests",
      "category": "tests",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": false,
      "summary": "Every @solver function name appears somewhere in the tests",
      "url": "https://inspect-evals-lint.generality.org/rules/IETS005/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IETS005.md"
    },
    {
      "code": "IETS006",
      "name": "custom_scorer_tests",
      "category": "tests",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": false,
      "summary": "Every @scorer function name appears somewhere in the tests",
      "url": "https://inspect-evals-lint.generality.org/rules/IETS006/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IETS006.md"
    },
    {
      "code": "IETS007",
      "name": "custom_tool_tests",
      "category": "tests",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": false,
      "summary": "Every @tool function name appears somewhere in the tests",
      "url": "https://inspect-evals-lint.generality.org/rules/IETS007/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IETS007.md"
    },
    {
      "code": "IEBP001",
      "name": "get_model_location",
      "category": "best_practices",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": false,
      "summary": "get_model() is only called inside @solver or @scorer functions",
      "url": "https://inspect-evals-lint.generality.org/rules/IEBP001/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEBP001.md"
    },
    {
      "code": "IEBP002",
      "name": "model_role_resolution",
      "category": "best_practices",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": true,
      "summary": "get_model(role=...) resolves deliberately: an explicit model, default= or required=True",
      "url": "https://inspect-evals-lint.generality.org/rules/IEBP002/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEBP002.md"
    },
    {
      "code": "IEBP003",
      "name": "sample_ids",
      "category": "best_practices",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": false,
      "summary": "Every Sample() passes id=",
      "url": "https://inspect-evals-lint.generality.org/rules/IEBP003/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEBP003.md"
    },
    {
      "code": "IEBP004",
      "name": "task_overridable_defaults",
      "category": "best_practices",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": false,
      "summary": "@task parameters naming a solver, scorer, metric, grader or model have defaults",
      "url": "https://inspect-evals-lint.generality.org/rules/IEBP004/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEBP004.md"
    },
    {
      "code": "IEBP005",
      "name": "sandbox_image_pinning",
      "category": "best_practices",
      "scopes": [
        "eval",
        "helper"
      ],
      "allowlist": true,
      "summary": "Registry images in compose files use an immutable tag or digest",
      "url": "https://inspect-evals-lint.generality.org/rules/IEBP005/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEBP005.md"
    },
    {
      "code": "IEBP006",
      "name": "gpu_sandbox_check",
      "category": "best_practices",
      "scopes": [
        "eval"
      ],
      "allowlist": false,
      "summary": "An evaluation requiring a GPU ships a maintenance sandbox check task",
      "url": "https://inspect-evals-lint.generality.org/rules/IEBP006/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEBP006.md"
    },
    {
      "code": "IEBP007",
      "name": "dockerfile_locking",
      "category": "best_practices",
      "scopes": [
        "eval"
      ],
      "allowlist": false,
      "summary": "Dockerfile builds consume locked inputs: committed locks, digest-pinned images, fixed sources",
      "url": "https://inspect-evals-lint.generality.org/rules/IEBP007/",
      "markdown": "https://inspect-evals-lint.generality.org/rules/IEBP007.md"
    }
  ]
}