drivers/vdpa/mlx5/mlx5_vdpa_event.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <unistd.h>
   5 #include <stdint.h>
   6 #include <fcntl.h>
   7
   8 #include <rte_malloc.h>
   9 #include <rte_errno.h>
  10 #include <rte_lcore.h>
  11 #include <rte_atomic.h>
  12 #include <rte_common.h>
  13 #include <rte_io.h>
  14
  15 #include <mlx5_common.h>
  16
  17 #include "mlx5_vdpa_utils.h"
  18 #include "mlx5_vdpa.h"
  19
  20
  21 void
  22 mlx5_vdpa_event_qp_global_release(struct mlx5_vdpa_priv *priv)
  23 {
  24         if (priv->uar) {
  25                 mlx5_glue->devx_free_uar(priv->uar);
  26                 priv->uar = NULL;
  27         }
  28         if (priv->eventc) {
  29                 mlx5_glue->devx_destroy_event_channel(priv->eventc);
  30                 priv->eventc = NULL;
  31         }
  32         priv->eqn = 0;
  33 }
  34
  35 /* Prepare all the global resources for all the event objects.*/
  36 static int
  37 mlx5_vdpa_event_qp_global_prepare(struct mlx5_vdpa_priv *priv)
  38 {
  39         uint32_t lcore;
  40
  41         if (priv->eventc)
  42                 return 0;
  43         lcore = (uint32_t)rte_lcore_to_cpu_id(-1);
  44         if (mlx5_glue->devx_query_eqn(priv->ctx, lcore, &priv->eqn)) {
  45                 rte_errno = errno;
  46                 DRV_LOG(ERR, "Failed to query EQ number %d.", rte_errno);
  47                 return -1;
  48         }
  49         priv->eventc = mlx5_glue->devx_create_event_channel(priv->ctx,
  50                            MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA);
  51         if (!priv->eventc) {
  52                 rte_errno = errno;
  53                 DRV_LOG(ERR, "Failed to create event channel %d.",
  54                         rte_errno);
  55                 goto error;
  56         }
  57         priv->uar = mlx5_glue->devx_alloc_uar(priv->ctx, 0);
  58         if (!priv->uar) {
  59                 rte_errno = errno;
  60                 DRV_LOG(ERR, "Failed to allocate UAR.");
  61                 goto error;
  62         }
  63         return 0;
  64 error:
  65         mlx5_vdpa_event_qp_global_release(priv);
  66         return -1;
  67 }
  68
  69 static void
  70 mlx5_vdpa_cq_destroy(struct mlx5_vdpa_cq *cq)
  71 {
  72         if (cq->cq)
  73                 claim_zero(mlx5_devx_cmd_destroy(cq->cq));
  74         if (cq->umem_obj)
  75                 claim_zero(mlx5_glue->devx_umem_dereg(cq->umem_obj));
  76         if (cq->umem_buf)
  77                 rte_free((void *)(uintptr_t)cq->umem_buf);
  78         memset(cq, 0, sizeof(*cq));
  79 }
  80
  81 static inline void
  82 mlx5_vdpa_cq_arm(struct mlx5_vdpa_priv *priv, struct mlx5_vdpa_cq *cq)
  83 {
  84         const unsigned int cqe_mask = (1 << cq->log_desc_n) - 1;
  85         uint32_t arm_sn = cq->arm_sn << MLX5_CQ_SQN_OFFSET;
  86         uint32_t cq_ci = cq->cq_ci & MLX5_CI_MASK & cqe_mask;
  87         uint32_t doorbell_hi = arm_sn | MLX5_CQ_DBR_CMD_ALL | cq_ci;
  88         uint64_t doorbell = ((uint64_t)doorbell_hi << 32) | cq->cq->id;
  89         uint64_t db_be = rte_cpu_to_be_64(doorbell);
  90         uint32_t *addr = RTE_PTR_ADD(priv->uar->base_addr, MLX5_CQ_DOORBELL);
  91
  92         rte_io_wmb();
  93         cq->db_rec[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(doorbell_hi);
  94         rte_wmb();
  95 #ifdef RTE_ARCH_64
  96         *(uint64_t *)addr = db_be;
  97 #else
  98         *(uint32_t *)addr = db_be;
  99         rte_io_wmb();
 100         *((uint32_t *)addr + 1) = db_be >> 32;
 101 #endif
 102         cq->arm_sn++;
 103 }
 104
 105 static int
 106 mlx5_vdpa_cq_create(struct mlx5_vdpa_priv *priv, uint16_t log_desc_n,
 107                     int callfd, struct mlx5_vdpa_cq *cq)
 108 {
 109         struct mlx5_devx_cq_attr attr;
 110         size_t pgsize = sysconf(_SC_PAGESIZE);
 111         uint32_t umem_size;
 112         int ret;
 113         uint16_t event_nums[1] = {0};
 114
 115         cq->log_desc_n = log_desc_n;
 116         umem_size = sizeof(struct mlx5_cqe) * (1 << log_desc_n) +
 117                                                         sizeof(*cq->db_rec) * 2;
 118         cq->umem_buf = rte_zmalloc(__func__, umem_size, 4096);
 119         if (!cq->umem_buf) {
 120                 DRV_LOG(ERR, "Failed to allocate memory for CQ.");
 121                 rte_errno = ENOMEM;
 122                 return -ENOMEM;
 123         }
 124         cq->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
 125                                                 (void *)(uintptr_t)cq->umem_buf,
 126                                                 umem_size,
 127                                                 IBV_ACCESS_LOCAL_WRITE);
 128         if (!cq->umem_obj) {
 129                 DRV_LOG(ERR, "Failed to register umem for CQ.");
 130                 goto error;
 131         }
 132         attr.q_umem_valid = 1;
 133         attr.db_umem_valid = 1;
 134         attr.use_first_only = 0;
 135         attr.overrun_ignore = 0;
 136         attr.uar_page_id = priv->uar->page_id;
 137         attr.q_umem_id = cq->umem_obj->umem_id;
 138         attr.q_umem_offset = 0;
 139         attr.db_umem_id = cq->umem_obj->umem_id;
 140         attr.db_umem_offset = sizeof(struct mlx5_cqe) * (1 << log_desc_n);
 141         attr.eqn = priv->eqn;
 142         attr.log_cq_size = log_desc_n;
 143         attr.log_page_size = rte_log2_u32(pgsize);
 144         cq->cq = mlx5_devx_cmd_create_cq(priv->ctx, &attr);
 145         if (!cq->cq)
 146                 goto error;
 147         cq->db_rec = RTE_PTR_ADD(cq->umem_buf, (uintptr_t)attr.db_umem_offset);
 148         cq->cq_ci = 0;
 149         rte_spinlock_init(&cq->sl);
 150         /* Subscribe CQ event to the event channel controlled by the driver. */
 151         ret = mlx5_glue->devx_subscribe_devx_event(priv->eventc, cq->cq->obj,
 152                                                    sizeof(event_nums),
 153                                                    event_nums,
 154                                                    (uint64_t)(uintptr_t)cq);
 155         if (ret) {
 156                 DRV_LOG(ERR, "Failed to subscribe CQE event.");
 157                 rte_errno = errno;
 158                 goto error;
 159         }
 160         /* Subscribe CQ event to the guest FD only if it is not in poll mode. */
 161         if (callfd != -1) {
 162                 ret = mlx5_glue->devx_subscribe_devx_event_fd(priv->eventc,
 163                                                               callfd,
 164                                                               cq->cq->obj, 0);
 165                 if (ret) {
 166                         DRV_LOG(ERR, "Failed to subscribe CQE event fd.");
 167                         rte_errno = errno;
 168                         goto error;
 169                 }
 170         }
 171         /* First arming. */
 172         mlx5_vdpa_cq_arm(priv, cq);
 173         return 0;
 174 error:
 175         mlx5_vdpa_cq_destroy(cq);
 176         return -1;
 177 }
 178
 179 static inline void __rte_unused
 180 mlx5_vdpa_cq_poll(struct mlx5_vdpa_priv *priv __rte_unused,
 181                   struct mlx5_vdpa_cq *cq)
 182 {
 183         struct mlx5_vdpa_event_qp *eqp =
 184                                 container_of(cq, struct mlx5_vdpa_event_qp, cq);
 185         const unsigned int cqe_size = 1 << cq->log_desc_n;
 186         const unsigned int cqe_mask = cqe_size - 1;
 187         int ret;
 188
 189         do {
 190                 volatile struct mlx5_cqe *cqe = cq->cqes + (cq->cq_ci &
 191                                                             cqe_mask);
 192
 193                 ret = check_cqe(cqe, cqe_size, cq->cq_ci);
 194                 switch (ret) {
 195                 case MLX5_CQE_STATUS_ERR:
 196                         cq->errors++;
 197                         /*fall-through*/
 198                 case MLX5_CQE_STATUS_SW_OWN:
 199                         cq->cq_ci++;
 200                         break;
 201                 case MLX5_CQE_STATUS_HW_OWN:
 202                 default:
 203                         break;
 204                 }
 205         } while (ret != MLX5_CQE_STATUS_HW_OWN);
 206         rte_io_wmb();
 207         /* Ring CQ doorbell record. */
 208         cq->db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
 209         rte_io_wmb();
 210         /* Ring SW QP doorbell record. */
 211         eqp->db_rec[0] = rte_cpu_to_be_32(cq->cq_ci + cqe_size);
 212 }
 213
 214 static void
 215 mlx5_vdpa_interrupt_handler(void *cb_arg)
 216 {
 217 #ifndef HAVE_IBV_DEVX_EVENT
 218         (void)cb_arg;
 219         return;
 220 #else
 221         struct mlx5_vdpa_priv *priv = cb_arg;
 222         union {
 223                 struct mlx5dv_devx_async_event_hdr event_resp;
 224                 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128];
 225         } out;
 226
 227         while (mlx5_glue->devx_get_event(priv->eventc, &out.event_resp,
 228                                          sizeof(out.buf)) >=
 229                                        (ssize_t)sizeof(out.event_resp.cookie)) {
 230                 struct mlx5_vdpa_cq *cq = (struct mlx5_vdpa_cq *)
 231                                                (uintptr_t)out.event_resp.cookie;
 232                 rte_spinlock_lock(&cq->sl);
 233                 mlx5_vdpa_cq_poll(priv, cq);
 234                 mlx5_vdpa_cq_arm(priv, cq);
 235                 rte_spinlock_unlock(&cq->sl);
 236                 DRV_LOG(DEBUG, "CQ %d event: new cq_ci = %u.", cq->cq->id,
 237                         cq->cq_ci);
 238         }
 239 #endif /* HAVE_IBV_DEVX_ASYNC */
 240 }
 241
 242 int
 243 mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv)
 244 {
 245         int flags = fcntl(priv->eventc->fd, F_GETFL);
 246         int ret = fcntl(priv->eventc->fd, F_SETFL, flags | O_NONBLOCK);
 247         if (ret) {
 248                 DRV_LOG(ERR, "Failed to change event channel FD.");
 249                 rte_errno = errno;
 250                 return -rte_errno;
 251         }
 252         priv->intr_handle.fd = priv->eventc->fd;
 253         priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
 254         if (rte_intr_callback_register(&priv->intr_handle,
 255                                        mlx5_vdpa_interrupt_handler, priv)) {
 256                 priv->intr_handle.fd = 0;
 257                 DRV_LOG(ERR, "Failed to register CQE interrupt %d.", rte_errno);
 258                 return -rte_errno;
 259         }
 260         return 0;
 261 }
 262
 263 void
 264 mlx5_vdpa_cqe_event_unset(struct mlx5_vdpa_priv *priv)
 265 {
 266         int retries = MLX5_VDPA_INTR_RETRIES;
 267         int ret = -EAGAIN;
 268
 269         if (priv->intr_handle.fd) {
 270                 while (retries-- && ret == -EAGAIN) {
 271                         ret = rte_intr_callback_unregister(&priv->intr_handle,
 272                                                     mlx5_vdpa_interrupt_handler,
 273                                                     priv);
 274                         if (ret == -EAGAIN) {
 275                                 DRV_LOG(DEBUG, "Try again to unregister fd %d "
 276                                         "of CQ interrupt, retries = %d.",
 277                                         priv->intr_handle.fd, retries);
 278                                 usleep(MLX5_VDPA_INTR_RETRIES_USEC);
 279                         }
 280                 }
 281                 memset(&priv->intr_handle, 0, sizeof(priv->intr_handle));
 282         }
 283 }
 284
 285 void
 286 mlx5_vdpa_event_qp_destroy(struct mlx5_vdpa_event_qp *eqp)
 287 {
 288         if (eqp->sw_qp)
 289                 claim_zero(mlx5_devx_cmd_destroy(eqp->sw_qp));
 290         if (eqp->umem_obj)
 291                 claim_zero(mlx5_glue->devx_umem_dereg(eqp->umem_obj));
 292         if (eqp->umem_buf)
 293                 rte_free(eqp->umem_buf);
 294         if (eqp->fw_qp)
 295                 claim_zero(mlx5_devx_cmd_destroy(eqp->fw_qp));
 296         mlx5_vdpa_cq_destroy(&eqp->cq);
 297         memset(eqp, 0, sizeof(*eqp));
 298 }
 299
 300 static int
 301 mlx5_vdpa_qps2rts(struct mlx5_vdpa_event_qp *eqp)
 302 {
 303         if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_RST2INIT_QP,
 304                                           eqp->sw_qp->id)) {
 305                 DRV_LOG(ERR, "Failed to modify FW QP to INIT state(%u).",
 306                         rte_errno);
 307                 return -1;
 308         }
 309         if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_RST2INIT_QP,
 310                                           eqp->fw_qp->id)) {
 311                 DRV_LOG(ERR, "Failed to modify SW QP to INIT state(%u).",
 312                         rte_errno);
 313                 return -1;
 314         }
 315         if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_INIT2RTR_QP,
 316                                           eqp->sw_qp->id)) {
 317                 DRV_LOG(ERR, "Failed to modify FW QP to RTR state(%u).",
 318                         rte_errno);
 319                 return -1;
 320         }
 321         if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_INIT2RTR_QP,
 322                                           eqp->fw_qp->id)) {
 323                 DRV_LOG(ERR, "Failed to modify SW QP to RTR state(%u).",
 324                         rte_errno);
 325                 return -1;
 326         }
 327         if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_RTR2RTS_QP,
 328                                           eqp->sw_qp->id)) {
 329                 DRV_LOG(ERR, "Failed to modify FW QP to RTS state(%u).",
 330                         rte_errno);
 331                 return -1;
 332         }
 333         if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_RTR2RTS_QP,
 334                                           eqp->fw_qp->id)) {
 335                 DRV_LOG(ERR, "Failed to modify SW QP to RTS state(%u).",
 336                         rte_errno);
 337                 return -1;
 338         }
 339         return 0;
 340 }
 341
 342 int
 343 mlx5_vdpa_event_qp_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n,
 344                           int callfd, struct mlx5_vdpa_event_qp *eqp)
 345 {
 346         struct mlx5_devx_qp_attr attr = {0};
 347         uint16_t log_desc_n = rte_log2_u32(desc_n);
 348         uint32_t umem_size = (1 << log_desc_n) * MLX5_WSEG_SIZE +
 349                                                        sizeof(*eqp->db_rec) * 2;
 350
 351         if (mlx5_vdpa_event_qp_global_prepare(priv))
 352                 return -1;
 353         if (mlx5_vdpa_cq_create(priv, log_desc_n, callfd, &eqp->cq))
 354                 return -1;
 355         attr.pd = priv->pdn;
 356         eqp->fw_qp = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 357         if (!eqp->fw_qp) {
 358                 DRV_LOG(ERR, "Failed to create FW QP(%u).", rte_errno);
 359                 goto error;
 360         }
 361         eqp->umem_buf = rte_zmalloc(__func__, umem_size, 4096);
 362         if (!eqp->umem_buf) {
 363                 DRV_LOG(ERR, "Failed to allocate memory for SW QP.");
 364                 rte_errno = ENOMEM;
 365                 goto error;
 366         }
 367         eqp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
 368                                                (void *)(uintptr_t)eqp->umem_buf,
 369                                                umem_size,
 370                                                IBV_ACCESS_LOCAL_WRITE);
 371         if (!eqp->umem_obj) {
 372                 DRV_LOG(ERR, "Failed to register umem for SW QP.");
 373                 goto error;
 374         }
 375         attr.uar_index = priv->uar->page_id;
 376         attr.cqn = eqp->cq.cq->id;
 377         attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
 378         attr.rq_size = 1 << log_desc_n;
 379         attr.log_rq_stride = rte_log2_u32(MLX5_WSEG_SIZE);
 380         attr.sq_size = 0; /* No need SQ. */
 381         attr.dbr_umem_valid = 1;
 382         attr.wq_umem_id = eqp->umem_obj->umem_id;
 383         attr.wq_umem_offset = 0;
 384         attr.dbr_umem_id = eqp->umem_obj->umem_id;
 385         attr.dbr_address = (1 << log_desc_n) * MLX5_WSEG_SIZE;
 386         eqp->sw_qp = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 387         if (!eqp->sw_qp) {
 388                 DRV_LOG(ERR, "Failed to create SW QP(%u).", rte_errno);
 389                 goto error;
 390         }
 391         eqp->db_rec = RTE_PTR_ADD(eqp->umem_buf, (uintptr_t)attr.dbr_address);
 392         if (mlx5_vdpa_qps2rts(eqp))
 393                 goto error;
 394         /* First ringing. */
 395         rte_write32(rte_cpu_to_be_32(1 << log_desc_n), &eqp->db_rec[0]);
 396         return 0;
 397 error:
 398         mlx5_vdpa_event_qp_destroy(eqp);
 399         return -1;
 400 }