{
  "inspect_ai": "0.3.283",
  "eval": {
    "status": "success",
    "metrics": {
      "flaky_judge": {
        "accuracy": 1.0
      }
    },
    "scored_samples": {
      "flaky_judge": 2
    },
    "unscored_samples": {
      "flaky_judge": 0
    },
    "samples": [
      {
        "id": 1,
        "error": null,
        "scores": {
          "flaky_judge": [
            "C",
            null
          ]
        },
        "model_events": 1
      },
      {
        "id": 2,
        "error": "RuntimeError('judge unavailable (simulated)')",
        "scores": {},
        "model_events": 1
      },
      {
        "id": 3,
        "error": "RuntimeError('judge unavailable (simulated)')",
        "scores": {},
        "model_events": 1
      },
      {
        "id": 4,
        "error": null,
        "scores": {
          "flaky_judge": [
            "C",
            null
          ]
        },
        "model_events": 1
      }
    ]
  },
  "rescore_working_judge": {
    "status": "success",
    "metrics": {
      "flaky_judge": {
        "accuracy": 1.0
      }
    },
    "scored_samples": {
      "flaky_judge": 4
    },
    "unscored_samples": {
      "flaky_judge": 0
    },
    "samples": [
      {
        "id": 1,
        "error": null,
        "scores": {
          "flaky_judge": [
            "C",
            null
          ]
        },
        "model_events": 1
      },
      {
        "id": 2,
        "error": "RuntimeError('judge unavailable (simulated)')",
        "scores": {
          "flaky_judge": [
            "C",
            null
          ]
        },
        "model_events": 1
      },
      {
        "id": 3,
        "error": "RuntimeError('judge unavailable (simulated)')",
        "scores": {
          "flaky_judge": [
            "C",
            null
          ]
        },
        "model_events": 1
      },
      {
        "id": 4,
        "error": null,
        "scores": {
          "flaky_judge": [
            "C",
            null
          ]
        },
        "model_events": 1
      }
    ]
  },
  "rescore_still_failing": "raised RuntimeError: judge unavailable (simulated)"
}