drivers/vdpa/mlx5/mlx5_vdpa_event.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <unistd.h>
   5 #include <stdint.h>
   6 #include <fcntl.h>
   7 #include <sys/eventfd.h>
   8
   9 #include <rte_malloc.h>
  10 #include <rte_errno.h>
  11 #include <rte_lcore.h>
  12 #include <rte_atomic.h>
  13 #include <rte_common.h>
  14 #include <rte_io.h>
  15 #include <rte_alarm.h>
  16
  17 #include <mlx5_common.h>
  18
  19 #include "mlx5_vdpa_utils.h"
  20 #include "mlx5_vdpa.h"
  21
  22
  23 #define MLX5_VDPA_DEFAULT_TIMER_DELAY_US 500u
  24 #define MLX5_VDPA_NO_TRAFFIC_TIME_S 2LLU
  25
  26 void
  27 mlx5_vdpa_event_qp_global_release(struct mlx5_vdpa_priv *priv)
  28 {
  29         if (priv->uar) {
  30                 mlx5_glue->devx_free_uar(priv->uar);
  31                 priv->uar = NULL;
  32         }
  33 #ifdef HAVE_IBV_DEVX_EVENT
  34         if (priv->eventc) {
  35                 union {
  36                         struct mlx5dv_devx_async_event_hdr event_resp;
  37                         uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr)
  38                                                                          + 128];
  39                 } out;
  40
  41                 /* Clean all pending events. */
  42                 while (mlx5_glue->devx_get_event(priv->eventc, &out.event_resp,
  43                        sizeof(out.buf)) >=
  44                        (ssize_t)sizeof(out.event_resp.cookie))
  45                         ;
  46                 mlx5_glue->devx_destroy_event_channel(priv->eventc);
  47                 priv->eventc = NULL;
  48         }
  49 #endif
  50         priv->eqn = 0;
  51 }
  52
  53 /* Prepare all the global resources for all the event objects.*/
  54 static int
  55 mlx5_vdpa_event_qp_global_prepare(struct mlx5_vdpa_priv *priv)
  56 {
  57         uint32_t lcore;
  58
  59         if (priv->eventc)
  60                 return 0;
  61         lcore = (uint32_t)rte_lcore_to_cpu_id(-1);
  62         if (mlx5_glue->devx_query_eqn(priv->ctx, lcore, &priv->eqn)) {
  63                 rte_errno = errno;
  64                 DRV_LOG(ERR, "Failed to query EQ number %d.", rte_errno);
  65                 return -1;
  66         }
  67         priv->eventc = mlx5_glue->devx_create_event_channel(priv->ctx,
  68                            MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA);
  69         if (!priv->eventc) {
  70                 rte_errno = errno;
  71                 DRV_LOG(ERR, "Failed to create event channel %d.",
  72                         rte_errno);
  73                 goto error;
  74         }
  75         priv->uar = mlx5_glue->devx_alloc_uar(priv->ctx, 0);
  76         if (!priv->uar) {
  77                 rte_errno = errno;
  78                 DRV_LOG(ERR, "Failed to allocate UAR.");
  79                 goto error;
  80         }
  81         return 0;
  82 error:
  83         mlx5_vdpa_event_qp_global_release(priv);
  84         return -1;
  85 }
  86
  87 static void
  88 mlx5_vdpa_cq_destroy(struct mlx5_vdpa_cq *cq)
  89 {
  90         if (cq->cq)
  91                 claim_zero(mlx5_devx_cmd_destroy(cq->cq));
  92         if (cq->umem_obj)
  93                 claim_zero(mlx5_glue->devx_umem_dereg(cq->umem_obj));
  94         if (cq->umem_buf)
  95                 rte_free((void *)(uintptr_t)cq->umem_buf);
  96         memset(cq, 0, sizeof(*cq));
  97 }
  98
  99 static inline void __rte_unused
 100 mlx5_vdpa_cq_arm(struct mlx5_vdpa_priv *priv, struct mlx5_vdpa_cq *cq)
 101 {
 102         uint32_t arm_sn = cq->arm_sn << MLX5_CQ_SQN_OFFSET;
 103         uint32_t cq_ci = cq->cq_ci & MLX5_CI_MASK;
 104         uint32_t doorbell_hi = arm_sn | MLX5_CQ_DBR_CMD_ALL | cq_ci;
 105         uint64_t doorbell = ((uint64_t)doorbell_hi << 32) | cq->cq->id;
 106         uint64_t db_be = rte_cpu_to_be_64(doorbell);
 107         uint32_t *addr = RTE_PTR_ADD(priv->uar->base_addr, MLX5_CQ_DOORBELL);
 108
 109         rte_io_wmb();
 110         cq->db_rec[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(doorbell_hi);
 111         rte_wmb();
 112 #ifdef RTE_ARCH_64
 113         *(uint64_t *)addr = db_be;
 114 #else
 115         *(uint32_t *)addr = db_be;
 116         rte_io_wmb();
 117         *((uint32_t *)addr + 1) = db_be >> 32;
 118 #endif
 119         cq->arm_sn++;
 120         cq->armed = 1;
 121 }
 122
 123 static int
 124 mlx5_vdpa_cq_create(struct mlx5_vdpa_priv *priv, uint16_t log_desc_n,
 125                     int callfd, struct mlx5_vdpa_cq *cq)
 126 {
 127         struct mlx5_devx_cq_attr attr;
 128         size_t pgsize = sysconf(_SC_PAGESIZE);
 129         uint32_t umem_size;
 130         int ret;
 131         uint16_t event_nums[1] = {0};
 132
 133         cq->log_desc_n = log_desc_n;
 134         umem_size = sizeof(struct mlx5_cqe) * (1 << log_desc_n) +
 135                                                         sizeof(*cq->db_rec) * 2;
 136         cq->umem_buf = rte_zmalloc(__func__, umem_size, 4096);
 137         if (!cq->umem_buf) {
 138                 DRV_LOG(ERR, "Failed to allocate memory for CQ.");
 139                 rte_errno = ENOMEM;
 140                 return -ENOMEM;
 141         }
 142         cq->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
 143                                                 (void *)(uintptr_t)cq->umem_buf,
 144                                                 umem_size,
 145                                                 IBV_ACCESS_LOCAL_WRITE);
 146         if (!cq->umem_obj) {
 147                 DRV_LOG(ERR, "Failed to register umem for CQ.");
 148                 goto error;
 149         }
 150         attr.q_umem_valid = 1;
 151         attr.db_umem_valid = 1;
 152         attr.use_first_only = 0;
 153         attr.overrun_ignore = 0;
 154         attr.uar_page_id = priv->uar->page_id;
 155         attr.q_umem_id = cq->umem_obj->umem_id;
 156         attr.q_umem_offset = 0;
 157         attr.db_umem_id = cq->umem_obj->umem_id;
 158         attr.db_umem_offset = sizeof(struct mlx5_cqe) * (1 << log_desc_n);
 159         attr.eqn = priv->eqn;
 160         attr.log_cq_size = log_desc_n;
 161         attr.log_page_size = rte_log2_u32(pgsize);
 162         cq->cq = mlx5_devx_cmd_create_cq(priv->ctx, &attr);
 163         if (!cq->cq)
 164                 goto error;
 165         cq->db_rec = RTE_PTR_ADD(cq->umem_buf, (uintptr_t)attr.db_umem_offset);
 166         cq->cq_ci = 0;
 167         rte_spinlock_init(&cq->sl);
 168         /* Subscribe CQ event to the event channel controlled by the driver. */
 169         ret = mlx5_glue->devx_subscribe_devx_event(priv->eventc, cq->cq->obj,
 170                                                    sizeof(event_nums),
 171                                                    event_nums,
 172                                                    (uint64_t)(uintptr_t)cq);
 173         if (ret) {
 174                 DRV_LOG(ERR, "Failed to subscribe CQE event.");
 175                 rte_errno = errno;
 176                 goto error;
 177         }
 178         if (callfd != -1) {
 179                 ret = mlx5_glue->devx_subscribe_devx_event_fd(priv->eventc,
 180                                                               callfd,
 181                                                               cq->cq->obj, 0);
 182                 if (ret) {
 183                         DRV_LOG(ERR, "Failed to subscribe CQE event fd.");
 184                         rte_errno = errno;
 185                         goto error;
 186                 }
 187         }
 188         cq->callfd = callfd;
 189         /* Init CQ to ones to be in HW owner in the start. */
 190         memset((void *)(uintptr_t)cq->umem_buf, 0xFF, attr.db_umem_offset);
 191         /* First arming. */
 192         mlx5_vdpa_cq_arm(priv, cq);
 193         return 0;
 194 error:
 195         mlx5_vdpa_cq_destroy(cq);
 196         return -1;
 197 }
 198
 199 static inline uint32_t
 200 mlx5_vdpa_cq_poll(struct mlx5_vdpa_cq *cq)
 201 {
 202         struct mlx5_vdpa_event_qp *eqp =
 203                                 container_of(cq, struct mlx5_vdpa_event_qp, cq);
 204         const unsigned int cq_size = 1 << cq->log_desc_n;
 205         const unsigned int cq_mask = cq_size - 1;
 206         uint32_t total = 0;
 207         int ret;
 208
 209         do {
 210                 volatile struct mlx5_cqe *cqe = cq->cqes + ((cq->cq_ci + total)
 211                                                             & cq_mask);
 212
 213                 ret = check_cqe(cqe, cq_size, cq->cq_ci + total);
 214                 switch (ret) {
 215                 case MLX5_CQE_STATUS_ERR:
 216                         cq->errors++;
 217                         /*fall-through*/
 218                 case MLX5_CQE_STATUS_SW_OWN:
 219                         total++;
 220                         break;
 221                 case MLX5_CQE_STATUS_HW_OWN:
 222                 default:
 223                         break;
 224                 }
 225         } while (ret != MLX5_CQE_STATUS_HW_OWN);
 226         rte_io_wmb();
 227         cq->cq_ci += total;
 228         /* Ring CQ doorbell record. */
 229         cq->db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
 230         rte_io_wmb();
 231         /* Ring SW QP doorbell record. */
 232         eqp->db_rec[0] = rte_cpu_to_be_32(cq->cq_ci + cq_size);
 233         return total;
 234 }
 235
 236 static void
 237 mlx5_vdpa_arm_all_cqs(struct mlx5_vdpa_priv *priv)
 238 {
 239         struct mlx5_vdpa_cq *cq;
 240         int i;
 241
 242         for (i = 0; i < priv->nr_virtqs; i++) {
 243                 cq = &priv->virtqs[i].eqp.cq;
 244                 if (cq->cq && !cq->armed)
 245                         mlx5_vdpa_cq_arm(priv, cq);
 246         }
 247 }
 248
 249 static void *
 250 mlx5_vdpa_poll_handle(void *arg)
 251 {
 252         struct mlx5_vdpa_priv *priv = arg;
 253         int i;
 254         struct mlx5_vdpa_cq *cq;
 255         uint32_t total;
 256         uint64_t current_tic;
 257
 258         pthread_mutex_lock(&priv->timer_lock);
 259         while (!priv->timer_on)
 260                 pthread_cond_wait(&priv->timer_cond, &priv->timer_lock);
 261         pthread_mutex_unlock(&priv->timer_lock);
 262         while (1) {
 263                 total = 0;
 264                 for (i = 0; i < priv->nr_virtqs; i++) {
 265                         cq = &priv->virtqs[i].eqp.cq;
 266                         if (cq->cq && !cq->armed) {
 267                                 uint32_t comp = mlx5_vdpa_cq_poll(cq);
 268
 269                                 if (comp) {
 270                                         /* Notify guest for descs consuming. */
 271                                         if (cq->callfd != -1)
 272                                                 eventfd_write(cq->callfd,
 273                                                               (eventfd_t)1);
 274                                         total += comp;
 275                                 }
 276                         }
 277                 }
 278                 current_tic = rte_rdtsc();
 279                 if (!total) {
 280                         /* No traffic ? stop timer and load interrupts. */
 281                         if (current_tic - priv->last_traffic_tic >=
 282                             rte_get_timer_hz() * MLX5_VDPA_NO_TRAFFIC_TIME_S) {
 283                                 DRV_LOG(DEBUG, "Device %s traffic was stopped.",
 284                                         priv->vdev->device->name);
 285                                 mlx5_vdpa_arm_all_cqs(priv);
 286                                 pthread_mutex_lock(&priv->timer_lock);
 287                                 priv->timer_on = 0;
 288                                 while (!priv->timer_on)
 289                                         pthread_cond_wait(&priv->timer_cond,
 290                                                           &priv->timer_lock);
 291                                 pthread_mutex_unlock(&priv->timer_lock);
 292                                 continue;
 293                         }
 294                 } else {
 295                         priv->last_traffic_tic = current_tic;
 296                 }
 297                 usleep(priv->timer_delay_us);
 298         }
 299         return NULL;
 300 }
 301
 302 static void
 303 mlx5_vdpa_interrupt_handler(void *cb_arg)
 304 {
 305         struct mlx5_vdpa_priv *priv = cb_arg;
 306 #ifdef HAVE_IBV_DEVX_EVENT
 307         union {
 308                 struct mlx5dv_devx_async_event_hdr event_resp;
 309                 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128];
 310         } out;
 311
 312         while (mlx5_glue->devx_get_event(priv->eventc, &out.event_resp,
 313                                          sizeof(out.buf)) >=
 314                                        (ssize_t)sizeof(out.event_resp.cookie)) {
 315                 struct mlx5_vdpa_cq *cq = (struct mlx5_vdpa_cq *)
 316                                                (uintptr_t)out.event_resp.cookie;
 317                 struct mlx5_vdpa_event_qp *eqp = container_of(cq,
 318                                                  struct mlx5_vdpa_event_qp, cq);
 319                 struct mlx5_vdpa_virtq *virtq = container_of(eqp,
 320                                                    struct mlx5_vdpa_virtq, eqp);
 321
 322                 mlx5_vdpa_cq_poll(cq);
 323                 /* Don't arm again - timer will take control. */
 324                 DRV_LOG(DEBUG, "Device %s virtq %d cq %d event was captured."
 325                         " Timer is %s, cq ci is %u.\n",
 326                         priv->vdev->device->name,
 327                         (int)virtq->index, cq->cq->id,
 328                         priv->timer_on ? "on" : "off", cq->cq_ci);
 329                 cq->armed = 0;
 330         }
 331 #endif
 332
 333         /* Traffic detected: make sure timer is on. */
 334         priv->last_traffic_tic = rte_rdtsc();
 335         pthread_mutex_lock(&priv->timer_lock);
 336         if (!priv->timer_on) {
 337                 priv->timer_on = 1;
 338                 pthread_cond_signal(&priv->timer_cond);
 339         }
 340         pthread_mutex_unlock(&priv->timer_lock);
 341 }
 342
 343 int
 344 mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv)
 345 {
 346         int flags;
 347         int ret;
 348
 349         if (!priv->eventc)
 350                 /* All virtqs are in poll mode. */
 351                 return 0;
 352         pthread_mutex_init(&priv->timer_lock, NULL);
 353         pthread_cond_init(&priv->timer_cond, NULL);
 354         priv->timer_on = 0;
 355         priv->timer_delay_us = MLX5_VDPA_DEFAULT_TIMER_DELAY_US;
 356         ret = pthread_create(&priv->timer_tid, NULL, mlx5_vdpa_poll_handle,
 357                              (void *)priv);
 358         if (ret) {
 359                 DRV_LOG(ERR, "Failed to create timer thread.");
 360                 return -1;
 361         }
 362         flags = fcntl(priv->eventc->fd, F_GETFL);
 363         ret = fcntl(priv->eventc->fd, F_SETFL, flags | O_NONBLOCK);
 364         if (ret) {
 365                 DRV_LOG(ERR, "Failed to change event channel FD.");
 366                 goto error;
 367         }
 368         priv->intr_handle.fd = priv->eventc->fd;
 369         priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
 370         if (rte_intr_callback_register(&priv->intr_handle,
 371                                        mlx5_vdpa_interrupt_handler, priv)) {
 372                 priv->intr_handle.fd = 0;
 373                 DRV_LOG(ERR, "Failed to register CQE interrupt %d.", rte_errno);
 374                 goto error;
 375         }
 376         return 0;
 377 error:
 378         mlx5_vdpa_cqe_event_unset(priv);
 379         return -1;
 380 }
 381
 382 void
 383 mlx5_vdpa_cqe_event_unset(struct mlx5_vdpa_priv *priv)
 384 {
 385         int retries = MLX5_VDPA_INTR_RETRIES;
 386         int ret = -EAGAIN;
 387         void *status;
 388
 389         if (priv->intr_handle.fd) {
 390                 while (retries-- && ret == -EAGAIN) {
 391                         ret = rte_intr_callback_unregister(&priv->intr_handle,
 392                                                     mlx5_vdpa_interrupt_handler,
 393                                                     priv);
 394                         if (ret == -EAGAIN) {
 395                                 DRV_LOG(DEBUG, "Try again to unregister fd %d "
 396                                         "of CQ interrupt, retries = %d.",
 397                                         priv->intr_handle.fd, retries);
 398                                 rte_pause();
 399                         }
 400                 }
 401                 memset(&priv->intr_handle, 0, sizeof(priv->intr_handle));
 402         }
 403         if (priv->timer_tid) {
 404                 pthread_cancel(priv->timer_tid);
 405                 pthread_join(priv->timer_tid, &status);
 406         }
 407         priv->timer_tid = 0;
 408 }
 409
 410 void
 411 mlx5_vdpa_event_qp_destroy(struct mlx5_vdpa_event_qp *eqp)
 412 {
 413         if (eqp->sw_qp)
 414                 claim_zero(mlx5_devx_cmd_destroy(eqp->sw_qp));
 415         if (eqp->umem_obj)
 416                 claim_zero(mlx5_glue->devx_umem_dereg(eqp->umem_obj));
 417         if (eqp->umem_buf)
 418                 rte_free(eqp->umem_buf);
 419         if (eqp->fw_qp)
 420                 claim_zero(mlx5_devx_cmd_destroy(eqp->fw_qp));
 421         mlx5_vdpa_cq_destroy(&eqp->cq);
 422         memset(eqp, 0, sizeof(*eqp));
 423 }
 424
 425 static int
 426 mlx5_vdpa_qps2rts(struct mlx5_vdpa_event_qp *eqp)
 427 {
 428         if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_RST2INIT_QP,
 429                                           eqp->sw_qp->id)) {
 430                 DRV_LOG(ERR, "Failed to modify FW QP to INIT state(%u).",
 431                         rte_errno);
 432                 return -1;
 433         }
 434         if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_RST2INIT_QP,
 435                                           eqp->fw_qp->id)) {
 436                 DRV_LOG(ERR, "Failed to modify SW QP to INIT state(%u).",
 437                         rte_errno);
 438                 return -1;
 439         }
 440         if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_INIT2RTR_QP,
 441                                           eqp->sw_qp->id)) {
 442                 DRV_LOG(ERR, "Failed to modify FW QP to RTR state(%u).",
 443                         rte_errno);
 444                 return -1;
 445         }
 446         if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_INIT2RTR_QP,
 447                                           eqp->fw_qp->id)) {
 448                 DRV_LOG(ERR, "Failed to modify SW QP to RTR state(%u).",
 449                         rte_errno);
 450                 return -1;
 451         }
 452         if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_RTR2RTS_QP,
 453                                           eqp->sw_qp->id)) {
 454                 DRV_LOG(ERR, "Failed to modify FW QP to RTS state(%u).",
 455                         rte_errno);
 456                 return -1;
 457         }
 458         if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_RTR2RTS_QP,
 459                                           eqp->fw_qp->id)) {
 460                 DRV_LOG(ERR, "Failed to modify SW QP to RTS state(%u).",
 461                         rte_errno);
 462                 return -1;
 463         }
 464         return 0;
 465 }
 466
 467 int
 468 mlx5_vdpa_event_qp_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n,
 469                           int callfd, struct mlx5_vdpa_event_qp *eqp)
 470 {
 471         struct mlx5_devx_qp_attr attr = {0};
 472         uint16_t log_desc_n = rte_log2_u32(desc_n);
 473         uint32_t umem_size = (1 << log_desc_n) * MLX5_WSEG_SIZE +
 474                                                        sizeof(*eqp->db_rec) * 2;
 475
 476         if (mlx5_vdpa_event_qp_global_prepare(priv))
 477                 return -1;
 478         if (mlx5_vdpa_cq_create(priv, log_desc_n, callfd, &eqp->cq))
 479                 return -1;
 480         attr.pd = priv->pdn;
 481         eqp->fw_qp = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 482         if (!eqp->fw_qp) {
 483                 DRV_LOG(ERR, "Failed to create FW QP(%u).", rte_errno);
 484                 goto error;
 485         }
 486         eqp->umem_buf = rte_zmalloc(__func__, umem_size, 4096);
 487         if (!eqp->umem_buf) {
 488                 DRV_LOG(ERR, "Failed to allocate memory for SW QP.");
 489                 rte_errno = ENOMEM;
 490                 goto error;
 491         }
 492         eqp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
 493                                                (void *)(uintptr_t)eqp->umem_buf,
 494                                                umem_size,
 495                                                IBV_ACCESS_LOCAL_WRITE);
 496         if (!eqp->umem_obj) {
 497                 DRV_LOG(ERR, "Failed to register umem for SW QP.");
 498                 goto error;
 499         }
 500         attr.uar_index = priv->uar->page_id;
 501         attr.cqn = eqp->cq.cq->id;
 502         attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
 503         attr.rq_size = 1 << log_desc_n;
 504         attr.log_rq_stride = rte_log2_u32(MLX5_WSEG_SIZE);
 505         attr.sq_size = 0; /* No need SQ. */
 506         attr.dbr_umem_valid = 1;
 507         attr.wq_umem_id = eqp->umem_obj->umem_id;
 508         attr.wq_umem_offset = 0;
 509         attr.dbr_umem_id = eqp->umem_obj->umem_id;
 510         attr.dbr_address = (1 << log_desc_n) * MLX5_WSEG_SIZE;
 511         eqp->sw_qp = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 512         if (!eqp->sw_qp) {
 513                 DRV_LOG(ERR, "Failed to create SW QP(%u).", rte_errno);
 514                 goto error;
 515         }
 516         eqp->db_rec = RTE_PTR_ADD(eqp->umem_buf, (uintptr_t)attr.dbr_address);
 517         if (mlx5_vdpa_qps2rts(eqp))
 518                 goto error;
 519         /* First ringing. */
 520         rte_write32(rte_cpu_to_be_32(1 << log_desc_n), &eqp->db_rec[0]);
 521         return 0;
 522 error:
 523         mlx5_vdpa_event_qp_destroy(eqp);
 524         return -1;
 525 }