{
  "benchmark": "qiskitbench-hard",
  "display_name": "Qiskit-HumanEval-Hard",
  "tasks": 151,
  "run_count": 19,
  "models": [
    {
      "name": "GrayGate",
      "detail": "Verification pipeline",
      "color": "teal",
      "provider": "graygate",
      "model_id": "graygate",
      "mean": 91.6,
      "min": 90.7,
      "max": 92.7,
      "runs": [
        {
          "run_id": "114f676e433d",
          "completed_at": "2026-02-23 12:23:15",
          "passed": 140,
          "total": 151,
          "percent": 92.71523178807946,
          "config": {
            "parallel": 1
          },
          "qiskit_version": "2.3.0"
        },
        {
          "run_id": "39faaa7f97d6",
          "completed_at": "2026-02-27 19:12:25",
          "passed": 138,
          "total": 151,
          "percent": 91.3907284768212,
          "config": {
            "parallel": 1
          },
          "qiskit_version": "2.3.0"
        },
        {
          "run_id": "afa6dc5da65f",
          "completed_at": "2026-03-03 06:49:34",
          "passed": 137,
          "total": 151,
          "percent": 90.72847682119205,
          "config": {
            "parallel": 1
          },
          "qiskit_version": "2.3.0"
        }
      ],
      "run_count": 3
    },
    {
      "name": "Gemini 3 Pro",
      "detail": "Preview",
      "color": "blue",
      "provider": "google",
      "model_id": "gemini-3-pro-preview",
      "mean": 59.9,
      "min": 59.6,
      "max": 60.3,
      "runs": [
        {
          "run_id": "ca7be56f7ba9",
          "completed_at": "2026-02-17 05:40:54",
          "passed": 90,
          "total": 151,
          "percent": 59.602649006622514,
          "config": {
            "parallel": 10,
            "temperature": 0.0,
            "max_tokens": 8192
          },
          "qiskit_version": "2.0.0"
        },
        {
          "run_id": "ef4d271bf84f",
          "completed_at": "2026-02-27 01:20:28",
          "passed": 91,
          "total": 151,
          "percent": 60.264900662251655,
          "config": {
            "parallel": 1,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        }
      ],
      "run_count": 2
    },
    {
      "name": "Gemini 3 Flash",
      "detail": "Preview",
      "color": "blue",
      "provider": "google",
      "model_id": "gemini-3-flash-preview",
      "mean": 55.6,
      "min": 55.6,
      "max": 55.6,
      "runs": [
        {
          "run_id": "801a1dfb333f",
          "completed_at": "2026-02-24 23:20:47",
          "passed": 84,
          "total": 151,
          "percent": 55.629139072847686,
          "config": {
            "parallel": 7,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        },
        {
          "run_id": "0bede3656bdc",
          "completed_at": "2026-02-25 02:41:18",
          "passed": 84,
          "total": 151,
          "percent": 55.629139072847686,
          "config": {
            "parallel": 7,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        },
        {
          "run_id": "e0bf96ad33f9",
          "completed_at": "2026-02-25 02:52:22",
          "passed": 84,
          "total": 151,
          "percent": 55.629139072847686,
          "config": {
            "parallel": 7,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        }
      ],
      "run_count": 3
    },
    {
      "name": "GPT-5.2",
      "detail": "Direct model",
      "color": "amber",
      "provider": "openai",
      "model_id": "gpt-5.2",
      "mean": 46.1,
      "min": 42.4,
      "max": 49.0,
      "runs": [
        {
          "run_id": "7a0900fd9ba1",
          "completed_at": "2026-02-25 02:59:56",
          "passed": 71,
          "total": 151,
          "percent": 47.019867549668874,
          "config": {
            "parallel": 10,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        },
        {
          "run_id": "7546ea11e38e",
          "completed_at": "2026-02-25 03:07:16",
          "passed": 64,
          "total": 151,
          "percent": 42.384105960264904,
          "config": {
            "parallel": 10,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        },
        {
          "run_id": "356d051f4c00",
          "completed_at": "2026-02-25 03:14:06",
          "passed": 74,
          "total": 151,
          "percent": 49.00662251655629,
          "config": {
            "parallel": 10,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        }
      ],
      "run_count": 3
    },
    {
      "name": "Qwen 3.5",
      "detail": "397B A17B",
      "color": "coral",
      "provider": "openrouter",
      "model_id": "qwen/qwen3.5-397b-a17b",
      "mean": 38.4,
      "min": 37.7,
      "max": 39.1,
      "runs": [
        {
          "run_id": "cf53d5ece474",
          "completed_at": "2026-02-25 14:56:29",
          "passed": 57,
          "total": 151,
          "percent": 37.74834437086093,
          "config": {
            "parallel": 10,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        },
        {
          "run_id": "c20d396d6066",
          "completed_at": "2026-02-25 15:15:18",
          "passed": 58,
          "total": 151,
          "percent": 38.41059602649007,
          "config": {
            "parallel": 10,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        },
        {
          "run_id": "acf22b4c7727",
          "completed_at": "2026-02-25 15:29:46",
          "passed": 59,
          "total": 151,
          "percent": 39.0728476821192,
          "config": {
            "parallel": 10,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        }
      ],
      "run_count": 3
    },
    {
      "name": "DeepSeek Reasoner",
      "detail": "Direct model",
      "color": "blue",
      "provider": "deepseek",
      "model_id": "deepseek-reasoner",
      "mean": 32.5,
      "min": 31.1,
      "max": 33.8,
      "runs": [
        {
          "run_id": "7e84164d6dd7",
          "completed_at": "2026-02-25 03:33:53",
          "passed": 51,
          "total": 151,
          "percent": 33.77483443708609,
          "config": {
            "parallel": 10,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        },
        {
          "run_id": "fa08a44920b4",
          "completed_at": "2026-02-25 03:53:00",
          "passed": 47,
          "total": 151,
          "percent": 31.125827814569536,
          "config": {
            "parallel": 10,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        },
        {
          "run_id": "67cefdc021fb",
          "completed_at": "2026-02-25 04:11:20",
          "passed": 49,
          "total": 151,
          "percent": 32.450331125827816,
          "config": {
            "parallel": 10,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        }
      ],
      "run_count": 3
    },
    {
      "name": "Qiskit Qwen 2.5",
      "detail": "14B \u00b7 local model",
      "color": "amber",
      "provider": "ollama",
      "model_id": "hf.co/Qiskit/qwen2.5-coder-14b-qiskit-GGUF:latest",
      "mean": 25.8,
      "min": 25.8,
      "max": 25.8,
      "runs": [
        {
          "run_id": "bbb8b311b8ba",
          "completed_at": "2026-02-24 06:24:11",
          "passed": 39,
          "total": 151,
          "percent": 25.827814569536425,
          "config": {
            "parallel": 1,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        },
        {
          "run_id": "168f3f7ad40b",
          "completed_at": "2026-02-25 07:49:47",
          "passed": 39,
          "total": 151,
          "percent": 25.827814569536425,
          "config": {
            "parallel": 1,
            "temperature": 0.0,
            "max_tokens": 64000
          },
          "qiskit_version": "2.3.0"
        }
      ],
      "run_count": 2
    }
  ],
  "first_date": "2026-02-17",
  "last_date": "2026-03-03",
  "metric": "Arithmetic mean of passed_tasks / total_tasks across all completed 151-task runs per model in this snapshot.",
  "source": "data/graybench-results.db",
  "source_sha256": "a3710d04b607a78371399b2aca2b25c8bde8aaac90d6a895dd8590dfa32f8de7",
  "limitations": "Historical, internally recorded results; not independently reproduced. Configurations and Qiskit versions differ. GrayGate uses a repair pipeline; these are not equal-budget single-attempt comparisons. Dataset revision and GrayGate base model are not recorded in these run rows."
}