drivers/vdpa/mlx5/mlx5_vdpa_event.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <unistd.h>
   5 #include <stdint.h>
   6 #include <fcntl.h>
   7
   8 #include <rte_malloc.h>
   9 #include <rte_errno.h>
  10 #include <rte_lcore.h>
  11 #include <rte_atomic.h>
  12 #include <rte_common.h>
  13 #include <rte_io.h>
  14
  15 #include <mlx5_common.h>
  16
  17 #include "mlx5_vdpa_utils.h"
  18 #include "mlx5_vdpa.h"
  19
  20
  21 void
  22 mlx5_vdpa_event_qp_global_release(struct mlx5_vdpa_priv *priv)
  23 {
  24         if (priv->uar) {
  25                 mlx5_glue->devx_free_uar(priv->uar);
  26                 priv->uar = NULL;
  27         }
  28         if (priv->eventc) {
  29                 mlx5_glue->devx_destroy_event_channel(priv->eventc);
  30                 priv->eventc = NULL;
  31         }
  32         priv->eqn = 0;
  33 }
  34
  35 /* Prepare all the global resources for all the event objects.*/
  36 static int
  37 mlx5_vdpa_event_qp_global_prepare(struct mlx5_vdpa_priv *priv)
  38 {
  39         uint32_t lcore;
  40
  41         if (priv->eventc)
  42                 return 0;
  43         lcore = (uint32_t)rte_lcore_to_cpu_id(-1);
  44         if (mlx5_glue->devx_query_eqn(priv->ctx, lcore, &priv->eqn)) {
  45                 rte_errno = errno;
  46                 DRV_LOG(ERR, "Failed to query EQ number %d.", rte_errno);
  47                 return -1;
  48         }
  49         priv->eventc = mlx5_glue->devx_create_event_channel(priv->ctx,
  50                            MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA);
  51         if (!priv->eventc) {
  52                 rte_errno = errno;
  53                 DRV_LOG(ERR, "Failed to create event channel %d.",
  54                         rte_errno);
  55                 goto error;
  56         }
  57         priv->uar = mlx5_glue->devx_alloc_uar(priv->ctx, 0);
  58         if (!priv->uar) {
  59                 rte_errno = errno;
  60                 DRV_LOG(ERR, "Failed to allocate UAR.");
  61                 goto error;
  62         }
  63         return 0;
  64 error:
  65         mlx5_vdpa_event_qp_global_release(priv);
  66         return -1;
  67 }
  68
  69 static void
  70 mlx5_vdpa_cq_destroy(struct mlx5_vdpa_cq *cq)
  71 {
  72         if (cq->cq)
  73                 claim_zero(mlx5_devx_cmd_destroy(cq->cq));
  74         if (cq->umem_obj)
  75                 claim_zero(mlx5_glue->devx_umem_dereg(cq->umem_obj));
  76         if (cq->umem_buf)
  77                 rte_free((void *)(uintptr_t)cq->umem_buf);
  78         memset(cq, 0, sizeof(*cq));
  79 }
  80
  81 static inline void
  82 mlx5_vdpa_cq_arm(struct mlx5_vdpa_priv *priv, struct mlx5_vdpa_cq *cq)
  83 {
  84         uint32_t arm_sn = cq->arm_sn << MLX5_CQ_SQN_OFFSET;
  85         uint32_t cq_ci = cq->cq_ci & MLX5_CI_MASK;
  86         uint32_t doorbell_hi = arm_sn | MLX5_CQ_DBR_CMD_ALL | cq_ci;
  87         uint64_t doorbell = ((uint64_t)doorbell_hi << 32) | cq->cq->id;
  88         uint64_t db_be = rte_cpu_to_be_64(doorbell);
  89         uint32_t *addr = RTE_PTR_ADD(priv->uar->base_addr, MLX5_CQ_DOORBELL);
  90
  91         rte_io_wmb();
  92         cq->db_rec[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(doorbell_hi);
  93         rte_wmb();
  94 #ifdef RTE_ARCH_64
  95         *(uint64_t *)addr = db_be;
  96 #else
  97         *(uint32_t *)addr = db_be;
  98         rte_io_wmb();
  99         *((uint32_t *)addr + 1) = db_be >> 32;
 100 #endif
 101         cq->arm_sn++;
 102 }
 103
 104 static int
 105 mlx5_vdpa_cq_create(struct mlx5_vdpa_priv *priv, uint16_t log_desc_n,
 106                     int callfd, struct mlx5_vdpa_cq *cq)
 107 {
 108         struct mlx5_devx_cq_attr attr;
 109         size_t pgsize = sysconf(_SC_PAGESIZE);
 110         uint32_t umem_size;
 111         int ret;
 112         uint16_t event_nums[1] = {0};
 113
 114         cq->log_desc_n = log_desc_n;
 115         umem_size = sizeof(struct mlx5_cqe) * (1 << log_desc_n) +
 116                                                         sizeof(*cq->db_rec) * 2;
 117         cq->umem_buf = rte_zmalloc(__func__, umem_size, 4096);
 118         if (!cq->umem_buf) {
 119                 DRV_LOG(ERR, "Failed to allocate memory for CQ.");
 120                 rte_errno = ENOMEM;
 121                 return -ENOMEM;
 122         }
 123         cq->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
 124                                                 (void *)(uintptr_t)cq->umem_buf,
 125                                                 umem_size,
 126                                                 IBV_ACCESS_LOCAL_WRITE);
 127         if (!cq->umem_obj) {
 128                 DRV_LOG(ERR, "Failed to register umem for CQ.");
 129                 goto error;
 130         }
 131         attr.q_umem_valid = 1;
 132         attr.db_umem_valid = 1;
 133         attr.use_first_only = 0;
 134         attr.overrun_ignore = 0;
 135         attr.uar_page_id = priv->uar->page_id;
 136         attr.q_umem_id = cq->umem_obj->umem_id;
 137         attr.q_umem_offset = 0;
 138         attr.db_umem_id = cq->umem_obj->umem_id;
 139         attr.db_umem_offset = sizeof(struct mlx5_cqe) * (1 << log_desc_n);
 140         attr.eqn = priv->eqn;
 141         attr.log_cq_size = log_desc_n;
 142         attr.log_page_size = rte_log2_u32(pgsize);
 143         cq->cq = mlx5_devx_cmd_create_cq(priv->ctx, &attr);
 144         if (!cq->cq)
 145                 goto error;
 146         cq->db_rec = RTE_PTR_ADD(cq->umem_buf, (uintptr_t)attr.db_umem_offset);
 147         cq->cq_ci = 0;
 148         rte_spinlock_init(&cq->sl);
 149         /* Subscribe CQ event to the event channel controlled by the driver. */
 150         ret = mlx5_glue->devx_subscribe_devx_event(priv->eventc, cq->cq->obj,
 151                                                    sizeof(event_nums),
 152                                                    event_nums,
 153                                                    (uint64_t)(uintptr_t)cq);
 154         if (ret) {
 155                 DRV_LOG(ERR, "Failed to subscribe CQE event.");
 156                 rte_errno = errno;
 157                 goto error;
 158         }
 159         /* Subscribe CQ event to the guest FD only if it is not in poll mode. */
 160         if (callfd != -1) {
 161                 ret = mlx5_glue->devx_subscribe_devx_event_fd(priv->eventc,
 162                                                               callfd,
 163                                                               cq->cq->obj, 0);
 164                 if (ret) {
 165                         DRV_LOG(ERR, "Failed to subscribe CQE event fd.");
 166                         rte_errno = errno;
 167                         goto error;
 168                 }
 169         }
 170         /* First arming. */
 171         mlx5_vdpa_cq_arm(priv, cq);
 172         return 0;
 173 error:
 174         mlx5_vdpa_cq_destroy(cq);
 175         return -1;
 176 }
 177
 178 static inline void __rte_unused
 179 mlx5_vdpa_cq_poll(struct mlx5_vdpa_priv *priv __rte_unused,
 180                   struct mlx5_vdpa_cq *cq)
 181 {
 182         struct mlx5_vdpa_event_qp *eqp =
 183                                 container_of(cq, struct mlx5_vdpa_event_qp, cq);
 184         const unsigned int cq_size = 1 << cq->log_desc_n;
 185         const unsigned int cq_mask = cq_size - 1;
 186         int ret;
 187
 188         do {
 189                 volatile struct mlx5_cqe *cqe = cq->cqes + (cq->cq_ci &
 190                                                             cq_mask);
 191
 192                 ret = check_cqe(cqe, cq_size, cq->cq_ci);
 193                 switch (ret) {
 194                 case MLX5_CQE_STATUS_ERR:
 195                         cq->errors++;
 196                         /*fall-through*/
 197                 case MLX5_CQE_STATUS_SW_OWN:
 198                         cq->cq_ci++;
 199                         break;
 200                 case MLX5_CQE_STATUS_HW_OWN:
 201                 default:
 202                         break;
 203                 }
 204         } while (ret != MLX5_CQE_STATUS_HW_OWN);
 205         rte_io_wmb();
 206         /* Ring CQ doorbell record. */
 207         cq->db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
 208         rte_io_wmb();
 209         /* Ring SW QP doorbell record. */
 210         eqp->db_rec[0] = rte_cpu_to_be_32(cq->cq_ci + cq_size);
 211 }
 212
 213 static void
 214 mlx5_vdpa_interrupt_handler(void *cb_arg)
 215 {
 216 #ifndef HAVE_IBV_DEVX_EVENT
 217         (void)cb_arg;
 218         return;
 219 #else
 220         struct mlx5_vdpa_priv *priv = cb_arg;
 221         union {
 222                 struct mlx5dv_devx_async_event_hdr event_resp;
 223                 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128];
 224         } out;
 225
 226         while (mlx5_glue->devx_get_event(priv->eventc, &out.event_resp,
 227                                          sizeof(out.buf)) >=
 228                                        (ssize_t)sizeof(out.event_resp.cookie)) {
 229                 struct mlx5_vdpa_cq *cq = (struct mlx5_vdpa_cq *)
 230                                                (uintptr_t)out.event_resp.cookie;
 231                 rte_spinlock_lock(&cq->sl);
 232                 mlx5_vdpa_cq_poll(priv, cq);
 233                 mlx5_vdpa_cq_arm(priv, cq);
 234                 rte_spinlock_unlock(&cq->sl);
 235                 DRV_LOG(DEBUG, "CQ %d event: new cq_ci = %u.", cq->cq->id,
 236                         cq->cq_ci);
 237         }
 238 #endif /* HAVE_IBV_DEVX_ASYNC */
 239 }
 240
 241 int
 242 mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv)
 243 {
 244         int flags = fcntl(priv->eventc->fd, F_GETFL);
 245         int ret = fcntl(priv->eventc->fd, F_SETFL, flags | O_NONBLOCK);
 246         if (ret) {
 247                 DRV_LOG(ERR, "Failed to change event channel FD.");
 248                 rte_errno = errno;
 249                 return -rte_errno;
 250         }
 251         priv->intr_handle.fd = priv->eventc->fd;
 252         priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
 253         if (rte_intr_callback_register(&priv->intr_handle,
 254                                        mlx5_vdpa_interrupt_handler, priv)) {
 255                 priv->intr_handle.fd = 0;
 256                 DRV_LOG(ERR, "Failed to register CQE interrupt %d.", rte_errno);
 257                 return -rte_errno;
 258         }
 259         return 0;
 260 }
 261
 262 void
 263 mlx5_vdpa_cqe_event_unset(struct mlx5_vdpa_priv *priv)
 264 {
 265         int retries = MLX5_VDPA_INTR_RETRIES;
 266         int ret = -EAGAIN;
 267
 268         if (priv->intr_handle.fd) {
 269                 while (retries-- && ret == -EAGAIN) {
 270                         ret = rte_intr_callback_unregister(&priv->intr_handle,
 271                                                     mlx5_vdpa_interrupt_handler,
 272                                                     priv);
 273                         if (ret == -EAGAIN) {
 274                                 DRV_LOG(DEBUG, "Try again to unregister fd %d "
 275                                         "of CQ interrupt, retries = %d.",
 276                                         priv->intr_handle.fd, retries);
 277                                 usleep(MLX5_VDPA_INTR_RETRIES_USEC);
 278                         }
 279                 }
 280                 memset(&priv->intr_handle, 0, sizeof(priv->intr_handle));
 281         }
 282 }
 283
 284 void
 285 mlx5_vdpa_event_qp_destroy(struct mlx5_vdpa_event_qp *eqp)
 286 {
 287         if (eqp->sw_qp)
 288                 claim_zero(mlx5_devx_cmd_destroy(eqp->sw_qp));
 289         if (eqp->umem_obj)
 290                 claim_zero(mlx5_glue->devx_umem_dereg(eqp->umem_obj));
 291         if (eqp->umem_buf)
 292                 rte_free(eqp->umem_buf);
 293         if (eqp->fw_qp)
 294                 claim_zero(mlx5_devx_cmd_destroy(eqp->fw_qp));
 295         mlx5_vdpa_cq_destroy(&eqp->cq);
 296         memset(eqp, 0, sizeof(*eqp));
 297 }
 298
 299 static int
 300 mlx5_vdpa_qps2rts(struct mlx5_vdpa_event_qp *eqp)
 301 {
 302         if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_RST2INIT_QP,
 303                                           eqp->sw_qp->id)) {
 304                 DRV_LOG(ERR, "Failed to modify FW QP to INIT state(%u).",
 305                         rte_errno);
 306                 return -1;
 307         }
 308         if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_RST2INIT_QP,
 309                                           eqp->fw_qp->id)) {
 310                 DRV_LOG(ERR, "Failed to modify SW QP to INIT state(%u).",
 311                         rte_errno);
 312                 return -1;
 313         }
 314         if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_INIT2RTR_QP,
 315                                           eqp->sw_qp->id)) {
 316                 DRV_LOG(ERR, "Failed to modify FW QP to RTR state(%u).",
 317                         rte_errno);
 318                 return -1;
 319         }
 320         if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_INIT2RTR_QP,
 321                                           eqp->fw_qp->id)) {
 322                 DRV_LOG(ERR, "Failed to modify SW QP to RTR state(%u).",
 323                         rte_errno);
 324                 return -1;
 325         }
 326         if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_RTR2RTS_QP,
 327                                           eqp->sw_qp->id)) {
 328                 DRV_LOG(ERR, "Failed to modify FW QP to RTS state(%u).",
 329                         rte_errno);
 330                 return -1;
 331         }
 332         if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_RTR2RTS_QP,
 333                                           eqp->fw_qp->id)) {
 334                 DRV_LOG(ERR, "Failed to modify SW QP to RTS state(%u).",
 335                         rte_errno);
 336                 return -1;
 337         }
 338         return 0;
 339 }
 340
 341 int
 342 mlx5_vdpa_event_qp_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n,
 343                           int callfd, struct mlx5_vdpa_event_qp *eqp)
 344 {
 345         struct mlx5_devx_qp_attr attr = {0};
 346         uint16_t log_desc_n = rte_log2_u32(desc_n);
 347         uint32_t umem_size = (1 << log_desc_n) * MLX5_WSEG_SIZE +
 348                                                        sizeof(*eqp->db_rec) * 2;
 349
 350         if (mlx5_vdpa_event_qp_global_prepare(priv))
 351                 return -1;
 352         if (mlx5_vdpa_cq_create(priv, log_desc_n, callfd, &eqp->cq))
 353                 return -1;
 354         attr.pd = priv->pdn;
 355         eqp->fw_qp = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 356         if (!eqp->fw_qp) {
 357                 DRV_LOG(ERR, "Failed to create FW QP(%u).", rte_errno);
 358                 goto error;
 359         }
 360         eqp->umem_buf = rte_zmalloc(__func__, umem_size, 4096);
 361         if (!eqp->umem_buf) {
 362                 DRV_LOG(ERR, "Failed to allocate memory for SW QP.");
 363                 rte_errno = ENOMEM;
 364                 goto error;
 365         }
 366         eqp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
 367                                                (void *)(uintptr_t)eqp->umem_buf,
 368                                                umem_size,
 369                                                IBV_ACCESS_LOCAL_WRITE);
 370         if (!eqp->umem_obj) {
 371                 DRV_LOG(ERR, "Failed to register umem for SW QP.");
 372                 goto error;
 373         }
 374         attr.uar_index = priv->uar->page_id;
 375         attr.cqn = eqp->cq.cq->id;
 376         attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
 377         attr.rq_size = 1 << log_desc_n;
 378         attr.log_rq_stride = rte_log2_u32(MLX5_WSEG_SIZE);
 379         attr.sq_size = 0; /* No need SQ. */
 380         attr.dbr_umem_valid = 1;
 381         attr.wq_umem_id = eqp->umem_obj->umem_id;
 382         attr.wq_umem_offset = 0;
 383         attr.dbr_umem_id = eqp->umem_obj->umem_id;
 384         attr.dbr_address = (1 << log_desc_n) * MLX5_WSEG_SIZE;
 385         eqp->sw_qp = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 386         if (!eqp->sw_qp) {
 387                 DRV_LOG(ERR, "Failed to create SW QP(%u).", rte_errno);
 388                 goto error;
 389         }
 390         eqp->db_rec = RTE_PTR_ADD(eqp->umem_buf, (uintptr_t)attr.dbr_address);
 391         if (mlx5_vdpa_qps2rts(eqp))
 392                 goto error;
 393         /* First ringing. */
 394         rte_write32(rte_cpu_to_be_32(1 << log_desc_n), &eqp->db_rec[0]);
 395         return 0;
 396 error:
 397         mlx5_vdpa_event_qp_destroy(eqp);
 398         return -1;
 399 }