drivers/net/mlx5/mlx5_rxq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2015 6WIND S.A.
   3  * Copyright 2015 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stddef.h>
   7 #include <errno.h>
   8 #include <string.h>
   9 #include <stdint.h>
  10 #include <fcntl.h>
  11 #include <sys/queue.h>
  12
  13 #include <rte_mbuf.h>
  14 #include <rte_malloc.h>
  15 #include <rte_ethdev_driver.h>
  16 #include <rte_common.h>
  17 #include <rte_interrupts.h>
  18 #include <rte_debug.h>
  19 #include <rte_io.h>
  20 #include <rte_eal_paging.h>
  21
  22 #include <mlx5_glue.h>
  23 #include <mlx5_devx_cmds.h>
  24 #include <mlx5_malloc.h>
  25
  26 #include "mlx5_defs.h"
  27 #include "mlx5.h"
  28 #include "mlx5_common_os.h"
  29 #include "mlx5_rxtx.h"
  30 #include "mlx5_utils.h"
  31 #include "mlx5_autoconf.h"
  32 #include "mlx5_flow.h"
  33
  34
  35 /* Default RSS hash key also used for ConnectX-3. */
  36 uint8_t rss_hash_default_key[] = {
  37         0x2c, 0xc6, 0x81, 0xd1,
  38         0x5b, 0xdb, 0xf4, 0xf7,
  39         0xfc, 0xa2, 0x83, 0x19,
  40         0xdb, 0x1a, 0x3e, 0x94,
  41         0x6b, 0x9e, 0x38, 0xd9,
  42         0x2c, 0x9c, 0x03, 0xd1,
  43         0xad, 0x99, 0x44, 0xa7,
  44         0xd9, 0x56, 0x3d, 0x59,
  45         0x06, 0x3c, 0x25, 0xf3,
  46         0xfc, 0x1f, 0xdc, 0x2a,
  47 };
  48
  49 /* Length of the default RSS hash key. */
  50 static_assert(MLX5_RSS_HASH_KEY_LEN ==
  51               (unsigned int)sizeof(rss_hash_default_key),
  52               "wrong RSS default key size.");
  53
  54 /**
  55  * Check whether Multi-Packet RQ can be enabled for the device.
  56  *
  57  * @param dev
  58  *   Pointer to Ethernet device.
  59  *
  60  * @return
  61  *   1 if supported, negative errno value if not.
  62  */
  63 inline int
  64 mlx5_check_mprq_support(struct rte_eth_dev *dev)
  65 {
  66         struct mlx5_priv *priv = dev->data->dev_private;
  67
  68         if (priv->config.mprq.enabled &&
  69             priv->rxqs_n >= priv->config.mprq.min_rxqs_num)
  70                 return 1;
  71         return -ENOTSUP;
  72 }
  73
  74 /**
  75  * Check whether Multi-Packet RQ is enabled for the Rx queue.
  76  *
  77  *  @param rxq
  78  *     Pointer to receive queue structure.
  79  *
  80  * @return
  81  *   0 if disabled, otherwise enabled.
  82  */
  83 inline int
  84 mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq)
  85 {
  86         return rxq->strd_num_n > 0;
  87 }
  88
  89 /**
  90  * Check whether Multi-Packet RQ is enabled for the device.
  91  *
  92  * @param dev
  93  *   Pointer to Ethernet device.
  94  *
  95  * @return
  96  *   0 if disabled, otherwise enabled.
  97  */
  98 inline int
  99 mlx5_mprq_enabled(struct rte_eth_dev *dev)
 100 {
 101         struct mlx5_priv *priv = dev->data->dev_private;
 102         uint32_t i;
 103         uint16_t n = 0;
 104         uint16_t n_ibv = 0;
 105
 106         if (mlx5_check_mprq_support(dev) < 0)
 107                 return 0;
 108         /* All the configured queues should be enabled. */
 109         for (i = 0; i < priv->rxqs_n; ++i) {
 110                 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
 111                 struct mlx5_rxq_ctrl *rxq_ctrl = container_of
 112                         (rxq, struct mlx5_rxq_ctrl, rxq);
 113
 114                 if (rxq == NULL || rxq_ctrl->type != MLX5_RXQ_TYPE_STANDARD)
 115                         continue;
 116                 n_ibv++;
 117                 if (mlx5_rxq_mprq_enabled(rxq))
 118                         ++n;
 119         }
 120         /* Multi-Packet RQ can't be partially configured. */
 121         MLX5_ASSERT(n == 0 || n == n_ibv);
 122         return n == n_ibv;
 123 }
 124
 125 /**
 126  * Allocate RX queue elements for Multi-Packet RQ.
 127  *
 128  * @param rxq_ctrl
 129  *   Pointer to RX queue structure.
 130  *
 131  * @return
 132  *   0 on success, a negative errno value otherwise and rte_errno is set.
 133  */
 134 static int
 135 rxq_alloc_elts_mprq(struct mlx5_rxq_ctrl *rxq_ctrl)
 136 {
 137         struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq;
 138         unsigned int wqe_n = 1 << rxq->elts_n;
 139         unsigned int i;
 140         int err;
 141
 142         /* Iterate on segments. */
 143         for (i = 0; i <= wqe_n; ++i) {
 144                 struct mlx5_mprq_buf *buf;
 145
 146                 if (rte_mempool_get(rxq->mprq_mp, (void **)&buf) < 0) {
 147                         DRV_LOG(ERR, "port %u empty mbuf pool", rxq->port_id);
 148                         rte_errno = ENOMEM;
 149                         goto error;
 150                 }
 151                 if (i < wqe_n)
 152                         (*rxq->mprq_bufs)[i] = buf;
 153                 else
 154                         rxq->mprq_repl = buf;
 155         }
 156         DRV_LOG(DEBUG,
 157                 "port %u Rx queue %u allocated and configured %u segments",
 158                 rxq->port_id, rxq->idx, wqe_n);
 159         return 0;
 160 error:
 161         err = rte_errno; /* Save rte_errno before cleanup. */
 162         wqe_n = i;
 163         for (i = 0; (i != wqe_n); ++i) {
 164                 if ((*rxq->mprq_bufs)[i] != NULL)
 165                         rte_mempool_put(rxq->mprq_mp,
 166                                         (*rxq->mprq_bufs)[i]);
 167                 (*rxq->mprq_bufs)[i] = NULL;
 168         }
 169         DRV_LOG(DEBUG, "port %u Rx queue %u failed, freed everything",
 170                 rxq->port_id, rxq->idx);
 171         rte_errno = err; /* Restore rte_errno. */
 172         return -rte_errno;
 173 }
 174
 175 /**
 176  * Allocate RX queue elements for Single-Packet RQ.
 177  *
 178  * @param rxq_ctrl
 179  *   Pointer to RX queue structure.
 180  *
 181  * @return
 182  *   0 on success, errno value on failure.
 183  */
 184 static int
 185 rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
 186 {
 187         const unsigned int sges_n = 1 << rxq_ctrl->rxq.sges_n;
 188         unsigned int elts_n = 1 << rxq_ctrl->rxq.elts_n;
 189         unsigned int i;
 190         int err;
 191
 192         /* Iterate on segments. */
 193         for (i = 0; (i != elts_n); ++i) {
 194                 struct rte_mbuf *buf;
 195
 196                 buf = rte_pktmbuf_alloc(rxq_ctrl->rxq.mp);
 197                 if (buf == NULL) {
 198                         DRV_LOG(ERR, "port %u empty mbuf pool",
 199                                 PORT_ID(rxq_ctrl->priv));
 200                         rte_errno = ENOMEM;
 201                         goto error;
 202                 }
 203                 /* Headroom is reserved by rte_pktmbuf_alloc(). */
 204                 MLX5_ASSERT(DATA_OFF(buf) == RTE_PKTMBUF_HEADROOM);
 205                 /* Buffer is supposed to be empty. */
 206                 MLX5_ASSERT(rte_pktmbuf_data_len(buf) == 0);
 207                 MLX5_ASSERT(rte_pktmbuf_pkt_len(buf) == 0);
 208                 MLX5_ASSERT(!buf->next);
 209                 /* Only the first segment keeps headroom. */
 210                 if (i % sges_n)
 211                         SET_DATA_OFF(buf, 0);
 212                 PORT(buf) = rxq_ctrl->rxq.port_id;
 213                 DATA_LEN(buf) = rte_pktmbuf_tailroom(buf);
 214                 PKT_LEN(buf) = DATA_LEN(buf);
 215                 NB_SEGS(buf) = 1;
 216                 (*rxq_ctrl->rxq.elts)[i] = buf;
 217         }
 218         /* If Rx vector is activated. */
 219         if (mlx5_rxq_check_vec_support(&rxq_ctrl->rxq) > 0) {
 220                 struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq;
 221                 struct rte_mbuf *mbuf_init = &rxq->fake_mbuf;
 222                 struct rte_pktmbuf_pool_private *priv =
 223                         (struct rte_pktmbuf_pool_private *)
 224                                 rte_mempool_get_priv(rxq_ctrl->rxq.mp);
 225                 int j;
 226
 227                 /* Initialize default rearm_data for vPMD. */
 228                 mbuf_init->data_off = RTE_PKTMBUF_HEADROOM;
 229                 rte_mbuf_refcnt_set(mbuf_init, 1);
 230                 mbuf_init->nb_segs = 1;
 231                 mbuf_init->port = rxq->port_id;
 232                 if (priv->flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
 233                         mbuf_init->ol_flags = EXT_ATTACHED_MBUF;
 234                 /*
 235                  * prevent compiler reordering:
 236                  * rearm_data covers previous fields.
 237                  */
 238                 rte_compiler_barrier();
 239                 rxq->mbuf_initializer =
 240                         *(rte_xmm_t *)&mbuf_init->rearm_data;
 241                 /* Padding with a fake mbuf for vectorized Rx. */
 242                 for (j = 0; j < MLX5_VPMD_DESCS_PER_LOOP; ++j)
 243                         (*rxq->elts)[elts_n + j] = &rxq->fake_mbuf;
 244         }
 245         DRV_LOG(DEBUG,
 246                 "port %u Rx queue %u allocated and configured %u segments"
 247                 " (max %u packets)",
 248                 PORT_ID(rxq_ctrl->priv), rxq_ctrl->rxq.idx, elts_n,
 249                 elts_n / (1 << rxq_ctrl->rxq.sges_n));
 250         return 0;
 251 error:
 252         err = rte_errno; /* Save rte_errno before cleanup. */
 253         elts_n = i;
 254         for (i = 0; (i != elts_n); ++i) {
 255                 if ((*rxq_ctrl->rxq.elts)[i] != NULL)
 256                         rte_pktmbuf_free_seg((*rxq_ctrl->rxq.elts)[i]);
 257                 (*rxq_ctrl->rxq.elts)[i] = NULL;
 258         }
 259         DRV_LOG(DEBUG, "port %u Rx queue %u failed, freed everything",
 260                 PORT_ID(rxq_ctrl->priv), rxq_ctrl->rxq.idx);
 261         rte_errno = err; /* Restore rte_errno. */
 262         return -rte_errno;
 263 }
 264
 265 /**
 266  * Allocate RX queue elements.
 267  *
 268  * @param rxq_ctrl
 269  *   Pointer to RX queue structure.
 270  *
 271  * @return
 272  *   0 on success, errno value on failure.
 273  */
 274 int
 275 rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl)
 276 {
 277         return mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ?
 278                rxq_alloc_elts_mprq(rxq_ctrl) : rxq_alloc_elts_sprq(rxq_ctrl);
 279 }
 280
 281 /**
 282  * Free RX queue elements for Multi-Packet RQ.
 283  *
 284  * @param rxq_ctrl
 285  *   Pointer to RX queue structure.
 286  */
 287 static void
 288 rxq_free_elts_mprq(struct mlx5_rxq_ctrl *rxq_ctrl)
 289 {
 290         struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq;
 291         uint16_t i;
 292
 293         DRV_LOG(DEBUG, "port %u Multi-Packet Rx queue %u freeing WRs",
 294                 rxq->port_id, rxq->idx);
 295         if (rxq->mprq_bufs == NULL)
 296                 return;
 297         MLX5_ASSERT(mlx5_rxq_check_vec_support(rxq) < 0);
 298         for (i = 0; (i != (1u << rxq->elts_n)); ++i) {
 299                 if ((*rxq->mprq_bufs)[i] != NULL)
 300                         mlx5_mprq_buf_free((*rxq->mprq_bufs)[i]);
 301                 (*rxq->mprq_bufs)[i] = NULL;
 302         }
 303         if (rxq->mprq_repl != NULL) {
 304                 mlx5_mprq_buf_free(rxq->mprq_repl);
 305                 rxq->mprq_repl = NULL;
 306         }
 307 }
 308
 309 /**
 310  * Free RX queue elements for Single-Packet RQ.
 311  *
 312  * @param rxq_ctrl
 313  *   Pointer to RX queue structure.
 314  */
 315 static void
 316 rxq_free_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl)
 317 {
 318         struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq;
 319         const uint16_t q_n = (1 << rxq->elts_n);
 320         const uint16_t q_mask = q_n - 1;
 321         uint16_t used = q_n - (rxq->rq_ci - rxq->rq_pi);
 322         uint16_t i;
 323
 324         DRV_LOG(DEBUG, "port %u Rx queue %u freeing WRs",
 325                 PORT_ID(rxq_ctrl->priv), rxq->idx);
 326         if (rxq->elts == NULL)
 327                 return;
 328         /**
 329          * Some mbuf in the Ring belongs to the application.  They cannot be
 330          * freed.
 331          */
 332         if (mlx5_rxq_check_vec_support(rxq) > 0) {
 333                 for (i = 0; i < used; ++i)
 334                         (*rxq->elts)[(rxq->rq_ci + i) & q_mask] = NULL;
 335                 rxq->rq_pi = rxq->rq_ci;
 336         }
 337         for (i = 0; (i != (1u << rxq->elts_n)); ++i) {
 338                 if ((*rxq->elts)[i] != NULL)
 339                         rte_pktmbuf_free_seg((*rxq->elts)[i]);
 340                 (*rxq->elts)[i] = NULL;
 341         }
 342 }
 343
 344 /**
 345  * Free RX queue elements.
 346  *
 347  * @param rxq_ctrl
 348  *   Pointer to RX queue structure.
 349  */
 350 static void
 351 rxq_free_elts(struct mlx5_rxq_ctrl *rxq_ctrl)
 352 {
 353         if (mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq))
 354                 rxq_free_elts_mprq(rxq_ctrl);
 355         else
 356                 rxq_free_elts_sprq(rxq_ctrl);
 357 }
 358
 359 /**
 360  * Returns the per-queue supported offloads.
 361  *
 362  * @param dev
 363  *   Pointer to Ethernet device.
 364  *
 365  * @return
 366  *   Supported Rx offloads.
 367  */
 368 uint64_t
 369 mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev)
 370 {
 371         struct mlx5_priv *priv = dev->data->dev_private;
 372         struct mlx5_dev_config *config = &priv->config;
 373         uint64_t offloads = (DEV_RX_OFFLOAD_SCATTER |
 374                              DEV_RX_OFFLOAD_TIMESTAMP |
 375                              DEV_RX_OFFLOAD_JUMBO_FRAME |
 376                              DEV_RX_OFFLOAD_RSS_HASH);
 377
 378         if (config->hw_fcs_strip)
 379                 offloads |= DEV_RX_OFFLOAD_KEEP_CRC;
 380
 381         if (config->hw_csum)
 382                 offloads |= (DEV_RX_OFFLOAD_IPV4_CKSUM |
 383                              DEV_RX_OFFLOAD_UDP_CKSUM |
 384                              DEV_RX_OFFLOAD_TCP_CKSUM);
 385         if (config->hw_vlan_strip)
 386                 offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
 387         if (MLX5_LRO_SUPPORTED(dev))
 388                 offloads |= DEV_RX_OFFLOAD_TCP_LRO;
 389         return offloads;
 390 }
 391
 392
 393 /**
 394  * Returns the per-port supported offloads.
 395  *
 396  * @return
 397  *   Supported Rx offloads.
 398  */
 399 uint64_t
 400 mlx5_get_rx_port_offloads(void)
 401 {
 402         uint64_t offloads = DEV_RX_OFFLOAD_VLAN_FILTER;
 403
 404         return offloads;
 405 }
 406
 407 /**
 408  * Verify if the queue can be released.
 409  *
 410  * @param dev
 411  *   Pointer to Ethernet device.
 412  * @param idx
 413  *   RX queue index.
 414  *
 415  * @return
 416  *   1 if the queue can be released
 417  *   0 if the queue can not be released, there are references to it.
 418  *   Negative errno and rte_errno is set if queue doesn't exist.
 419  */
 420 static int
 421 mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx)
 422 {
 423         struct mlx5_priv *priv = dev->data->dev_private;
 424         struct mlx5_rxq_ctrl *rxq_ctrl;
 425
 426         if (!(*priv->rxqs)[idx]) {
 427                 rte_errno = EINVAL;
 428                 return -rte_errno;
 429         }
 430         rxq_ctrl = container_of((*priv->rxqs)[idx], struct mlx5_rxq_ctrl, rxq);
 431         return (rte_atomic32_read(&rxq_ctrl->refcnt) == 1);
 432 }
 433
 434 /* Fetches and drops all SW-owned and error CQEs to synchronize CQ. */
 435 static void
 436 rxq_sync_cq(struct mlx5_rxq_data *rxq)
 437 {
 438         const uint16_t cqe_n = 1 << rxq->cqe_n;
 439         const uint16_t cqe_mask = cqe_n - 1;
 440         volatile struct mlx5_cqe *cqe;
 441         int ret, i;
 442
 443         i = cqe_n;
 444         do {
 445                 cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask];
 446                 ret = check_cqe(cqe, cqe_n, rxq->cq_ci);
 447                 if (ret == MLX5_CQE_STATUS_HW_OWN)
 448                         break;
 449                 if (ret == MLX5_CQE_STATUS_ERR) {
 450                         rxq->cq_ci++;
 451                         continue;
 452                 }
 453                 MLX5_ASSERT(ret == MLX5_CQE_STATUS_SW_OWN);
 454                 if (MLX5_CQE_FORMAT(cqe->op_own) != MLX5_COMPRESSED) {
 455                         rxq->cq_ci++;
 456                         continue;
 457                 }
 458                 /* Compute the next non compressed CQE. */
 459                 rxq->cq_ci += rte_be_to_cpu_32(cqe->byte_cnt);
 460
 461         } while (--i);
 462         /* Move all CQEs to HW ownership, including possible MiniCQEs. */
 463         for (i = 0; i < cqe_n; i++) {
 464                 cqe = &(*rxq->cqes)[i];
 465                 cqe->op_own = MLX5_CQE_INVALIDATE;
 466         }
 467         /* Resync CQE and WQE (WQ in RESET state). */
 468         rte_cio_wmb();
 469         *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
 470         rte_cio_wmb();
 471         *rxq->rq_db = rte_cpu_to_be_32(0);
 472         rte_cio_wmb();
 473 }
 474
 475 /**
 476  * Rx queue stop. Device queue goes to the RESET state,
 477  * all involved mbufs are freed from WQ.
 478  *
 479  * @param dev
 480  *   Pointer to Ethernet device structure.
 481  * @param idx
 482  *   RX queue index.
 483  *
 484  * @return
 485  *   0 on success, a negative errno value otherwise and rte_errno is set.
 486  */
 487 int
 488 mlx5_rx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t idx)
 489 {
 490         struct mlx5_priv *priv = dev->data->dev_private;
 491         struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx];
 492         struct mlx5_rxq_ctrl *rxq_ctrl =
 493                         container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 494         int ret;
 495
 496         MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
 497         if (rxq_ctrl->obj->type == MLX5_RXQ_OBJ_TYPE_IBV) {
 498                 struct ibv_wq_attr mod = {
 499                         .attr_mask = IBV_WQ_ATTR_STATE,
 500                         .wq_state = IBV_WQS_RESET,
 501                 };
 502
 503                 ret = mlx5_glue->modify_wq(rxq_ctrl->obj->wq, &mod);
 504         } else { /* rxq_ctrl->obj->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ. */
 505                 struct mlx5_devx_modify_rq_attr rq_attr;
 506
 507                 memset(&rq_attr, 0, sizeof(rq_attr));
 508                 rq_attr.rq_state = MLX5_RQC_STATE_RST;
 509                 rq_attr.state = MLX5_RQC_STATE_RDY;
 510                 ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, &rq_attr);
 511         }
 512         if (ret) {
 513                 DRV_LOG(ERR, "Cannot change Rx WQ state to RESET:  %s",
 514                         strerror(errno));
 515                 rte_errno = errno;
 516                 return ret;
 517         }
 518         /* Remove all processes CQEs. */
 519         rxq_sync_cq(rxq);
 520         /* Free all involved mbufs. */
 521         rxq_free_elts(rxq_ctrl);
 522         /* Set the actual queue state. */
 523         dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 524         return 0;
 525 }
 526
 527 /**
 528  * Rx queue stop. Device queue goes to the RESET state,
 529  * all involved mbufs are freed from WQ.
 530  *
 531  * @param dev
 532  *   Pointer to Ethernet device structure.
 533  * @param idx
 534  *   RX queue index.
 535  *
 536  * @return
 537  *   0 on success, a negative errno value otherwise and rte_errno is set.
 538  */
 539 int
 540 mlx5_rx_queue_stop(struct rte_eth_dev *dev, uint16_t idx)
 541 {
 542         eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
 543         int ret;
 544
 545         if (dev->data->rx_queue_state[idx] == RTE_ETH_QUEUE_STATE_HAIRPIN) {
 546                 DRV_LOG(ERR, "Hairpin queue can't be stopped");
 547                 rte_errno = EINVAL;
 548                 return -EINVAL;
 549         }
 550         if (dev->data->rx_queue_state[idx] == RTE_ETH_QUEUE_STATE_STOPPED)
 551                 return 0;
 552         /*
 553          * Vectorized Rx burst requires the CQ and RQ indices
 554          * synchronized, that might be broken on RQ restart
 555          * and cause Rx malfunction, so queue stopping is
 556          * not supported if vectorized Rx burst is engaged.
 557          * The routine pointer depends on the process
 558          * type, should perform check there.
 559          */
 560         if (pkt_burst == mlx5_rx_burst) {
 561                 DRV_LOG(ERR, "Rx queue stop is not supported "
 562                         "for vectorized Rx");
 563                 rte_errno = EINVAL;
 564                 return -EINVAL;
 565         }
 566         if (rte_eal_process_type() ==  RTE_PROC_SECONDARY) {
 567                 ret = mlx5_mp_os_req_queue_control(dev, idx,
 568                                                    MLX5_MP_REQ_QUEUE_RX_STOP);
 569         } else {
 570                 ret = mlx5_rx_queue_stop_primary(dev, idx);
 571         }
 572         return ret;
 573 }
 574
 575 /**
 576  * Rx queue start. Device queue goes to the ready state,
 577  * all required mbufs are allocated and WQ is replenished.
 578  *
 579  * @param dev
 580  *   Pointer to Ethernet device structure.
 581  * @param idx
 582  *   RX queue index.
 583  *
 584  * @return
 585  *   0 on success, a negative errno value otherwise and rte_errno is set.
 586  */
 587 int
 588 mlx5_rx_queue_start_primary(struct rte_eth_dev *dev, uint16_t idx)
 589 {
 590         struct mlx5_priv *priv = dev->data->dev_private;
 591         struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx];
 592         struct mlx5_rxq_ctrl *rxq_ctrl =
 593                         container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 594         int ret;
 595
 596         MLX5_ASSERT(rte_eal_process_type() ==  RTE_PROC_PRIMARY);
 597         /* Allocate needed buffers. */
 598         ret = rxq_alloc_elts(rxq_ctrl);
 599         if (ret) {
 600                 DRV_LOG(ERR, "Cannot reallocate buffers for Rx WQ");
 601                 rte_errno = errno;
 602                 return ret;
 603         }
 604         rte_cio_wmb();
 605         *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
 606         rte_cio_wmb();
 607         /* Reset RQ consumer before moving queue ro READY state. */
 608         *rxq->rq_db = rte_cpu_to_be_32(0);
 609         rte_cio_wmb();
 610         if (rxq_ctrl->obj->type == MLX5_RXQ_OBJ_TYPE_IBV) {
 611                 struct ibv_wq_attr mod = {
 612                         .attr_mask = IBV_WQ_ATTR_STATE,
 613                         .wq_state = IBV_WQS_RDY,
 614                 };
 615
 616                 ret = mlx5_glue->modify_wq(rxq_ctrl->obj->wq, &mod);
 617         } else { /* rxq_ctrl->obj->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ. */
 618                 struct mlx5_devx_modify_rq_attr rq_attr;
 619
 620                 memset(&rq_attr, 0, sizeof(rq_attr));
 621                 rq_attr.rq_state = MLX5_RQC_STATE_RDY;
 622                 rq_attr.state = MLX5_RQC_STATE_RST;
 623                 ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, &rq_attr);
 624         }
 625         if (ret) {
 626                 DRV_LOG(ERR, "Cannot change Rx WQ state to READY:  %s",
 627                         strerror(errno));
 628                 rte_errno = errno;
 629                 return ret;
 630         }
 631         /* Reinitialize RQ - set WQEs. */
 632         mlx5_rxq_initialize(rxq);
 633         rxq->err_state = MLX5_RXQ_ERR_STATE_NO_ERROR;
 634         /* Set actual queue state. */
 635         dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;
 636         return 0;
 637 }
 638
 639 /**
 640  * Rx queue start. Device queue goes to the ready state,
 641  * all required mbufs are allocated and WQ is replenished.
 642  *
 643  * @param dev
 644  *   Pointer to Ethernet device structure.
 645  * @param idx
 646  *   RX queue index.
 647  *
 648  * @return
 649  *   0 on success, a negative errno value otherwise and rte_errno is set.
 650  */
 651 int
 652 mlx5_rx_queue_start(struct rte_eth_dev *dev, uint16_t idx)
 653 {
 654         int ret;
 655
 656         if (dev->data->rx_queue_state[idx] == RTE_ETH_QUEUE_STATE_HAIRPIN) {
 657                 DRV_LOG(ERR, "Hairpin queue can't be started");
 658                 rte_errno = EINVAL;
 659                 return -EINVAL;
 660         }
 661         if (dev->data->rx_queue_state[idx] == RTE_ETH_QUEUE_STATE_STARTED)
 662                 return 0;
 663         if (rte_eal_process_type() ==  RTE_PROC_SECONDARY) {
 664                 ret = mlx5_mp_os_req_queue_control(dev, idx,
 665                                                    MLX5_MP_REQ_QUEUE_RX_START);
 666         } else {
 667                 ret = mlx5_rx_queue_start_primary(dev, idx);
 668         }
 669         return ret;
 670 }
 671
 672 /**
 673  * Rx queue presetup checks.
 674  *
 675  * @param dev
 676  *   Pointer to Ethernet device structure.
 677  * @param idx
 678  *   RX queue index.
 679  * @param desc
 680  *   Number of descriptors to configure in queue.
 681  *
 682  * @return
 683  *   0 on success, a negative errno value otherwise and rte_errno is set.
 684  */
 685 static int
 686 mlx5_rx_queue_pre_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t *desc)
 687 {
 688         struct mlx5_priv *priv = dev->data->dev_private;
 689
 690         if (!rte_is_power_of_2(*desc)) {
 691                 *desc = 1 << log2above(*desc);
 692                 DRV_LOG(WARNING,
 693                         "port %u increased number of descriptors in Rx queue %u"
 694                         " to the next power of two (%d)",
 695                         dev->data->port_id, idx, *desc);
 696         }
 697         DRV_LOG(DEBUG, "port %u configuring Rx queue %u for %u descriptors",
 698                 dev->data->port_id, idx, *desc);
 699         if (idx >= priv->rxqs_n) {
 700                 DRV_LOG(ERR, "port %u Rx queue index out of range (%u >= %u)",
 701                         dev->data->port_id, idx, priv->rxqs_n);
 702                 rte_errno = EOVERFLOW;
 703                 return -rte_errno;
 704         }
 705         if (!mlx5_rxq_releasable(dev, idx)) {
 706                 DRV_LOG(ERR, "port %u unable to release queue index %u",
 707                         dev->data->port_id, idx);
 708                 rte_errno = EBUSY;
 709                 return -rte_errno;
 710         }
 711         mlx5_rxq_release(dev, idx);
 712         return 0;
 713 }
 714
 715 /**
 716  *
 717  * @param dev
 718  *   Pointer to Ethernet device structure.
 719  * @param idx
 720  *   RX queue index.
 721  * @param desc
 722  *   Number of descriptors to configure in queue.
 723  * @param socket
 724  *   NUMA socket on which memory must be allocated.
 725  * @param[in] conf
 726  *   Thresholds parameters.
 727  * @param mp
 728  *   Memory pool for buffer allocations.
 729  *
 730  * @return
 731  *   0 on success, a negative errno value otherwise and rte_errno is set.
 732  */
 733 int
 734 mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 735                     unsigned int socket, const struct rte_eth_rxconf *conf,
 736                     struct rte_mempool *mp)
 737 {
 738         struct mlx5_priv *priv = dev->data->dev_private;
 739         struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx];
 740         struct mlx5_rxq_ctrl *rxq_ctrl =
 741                 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 742         int res;
 743
 744         res = mlx5_rx_queue_pre_setup(dev, idx, &desc);
 745         if (res)
 746                 return res;
 747         rxq_ctrl = mlx5_rxq_new(dev, idx, desc, socket, conf, mp);
 748         if (!rxq_ctrl) {
 749                 DRV_LOG(ERR, "port %u unable to allocate queue index %u",
 750                         dev->data->port_id, idx);
 751                 rte_errno = ENOMEM;
 752                 return -rte_errno;
 753         }
 754         DRV_LOG(DEBUG, "port %u adding Rx queue %u to list",
 755                 dev->data->port_id, idx);
 756         (*priv->rxqs)[idx] = &rxq_ctrl->rxq;
 757         return 0;
 758 }
 759
 760 /**
 761  *
 762  * @param dev
 763  *   Pointer to Ethernet device structure.
 764  * @param idx
 765  *   RX queue index.
 766  * @param desc
 767  *   Number of descriptors to configure in queue.
 768  * @param hairpin_conf
 769  *   Hairpin configuration parameters.
 770  *
 771  * @return
 772  *   0 on success, a negative errno value otherwise and rte_errno is set.
 773  */
 774 int
 775 mlx5_rx_hairpin_queue_setup(struct rte_eth_dev *dev, uint16_t idx,
 776                             uint16_t desc,
 777                             const struct rte_eth_hairpin_conf *hairpin_conf)
 778 {
 779         struct mlx5_priv *priv = dev->data->dev_private;
 780         struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx];
 781         struct mlx5_rxq_ctrl *rxq_ctrl =
 782                 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 783         int res;
 784
 785         res = mlx5_rx_queue_pre_setup(dev, idx, &desc);
 786         if (res)
 787                 return res;
 788         if (hairpin_conf->peer_count != 1 ||
 789             hairpin_conf->peers[0].port != dev->data->port_id ||
 790             hairpin_conf->peers[0].queue >= priv->txqs_n) {
 791                 DRV_LOG(ERR, "port %u unable to setup hairpin queue index %u "
 792                         " invalid hairpind configuration", dev->data->port_id,
 793                         idx);
 794                 rte_errno = EINVAL;
 795                 return -rte_errno;
 796         }
 797         rxq_ctrl = mlx5_rxq_hairpin_new(dev, idx, desc, hairpin_conf);
 798         if (!rxq_ctrl) {
 799                 DRV_LOG(ERR, "port %u unable to allocate queue index %u",
 800                         dev->data->port_id, idx);
 801                 rte_errno = ENOMEM;
 802                 return -rte_errno;
 803         }
 804         DRV_LOG(DEBUG, "port %u adding Rx queue %u to list",
 805                 dev->data->port_id, idx);
 806         (*priv->rxqs)[idx] = &rxq_ctrl->rxq;
 807         return 0;
 808 }
 809
 810 /**
 811  * DPDK callback to release a RX queue.
 812  *
 813  * @param dpdk_rxq
 814  *   Generic RX queue pointer.
 815  */
 816 void
 817 mlx5_rx_queue_release(void *dpdk_rxq)
 818 {
 819         struct mlx5_rxq_data *rxq = (struct mlx5_rxq_data *)dpdk_rxq;
 820         struct mlx5_rxq_ctrl *rxq_ctrl;
 821         struct mlx5_priv *priv;
 822
 823         if (rxq == NULL)
 824                 return;
 825         rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 826         priv = rxq_ctrl->priv;
 827         if (!mlx5_rxq_releasable(ETH_DEV(priv), rxq_ctrl->rxq.idx))
 828                 rte_panic("port %u Rx queue %u is still used by a flow and"
 829                           " cannot be removed\n",
 830                           PORT_ID(priv), rxq->idx);
 831         mlx5_rxq_release(ETH_DEV(priv), rxq_ctrl->rxq.idx);
 832 }
 833
 834 /**
 835  * Get an Rx queue Verbs/DevX object.
 836  *
 837  * @param dev
 838  *   Pointer to Ethernet device.
 839  * @param idx
 840  *   Queue index in DPDK Rx queue array
 841  *
 842  * @return
 843  *   The Verbs/DevX object if it exists.
 844  */
 845 static struct mlx5_rxq_obj *
 846 mlx5_rxq_obj_get(struct rte_eth_dev *dev, uint16_t idx)
 847 {
 848         struct mlx5_priv *priv = dev->data->dev_private;
 849         struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
 850         struct mlx5_rxq_ctrl *rxq_ctrl;
 851
 852         if (idx >= priv->rxqs_n)
 853                 return NULL;
 854         if (!rxq_data)
 855                 return NULL;
 856         rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
 857         if (rxq_ctrl->obj)
 858                 rte_atomic32_inc(&rxq_ctrl->obj->refcnt);
 859         return rxq_ctrl->obj;
 860 }
 861
 862 /**
 863  * Release the resources allocated for an RQ DevX object.
 864  *
 865  * @param rxq_ctrl
 866  *   DevX Rx queue object.
 867  */
 868 static void
 869 rxq_release_devx_rq_resources(struct mlx5_rxq_ctrl *rxq_ctrl)
 870 {
 871         if (rxq_ctrl->rxq.wqes) {
 872                 mlx5_free((void *)(uintptr_t)rxq_ctrl->rxq.wqes);
 873                 rxq_ctrl->rxq.wqes = NULL;
 874         }
 875         if (rxq_ctrl->wq_umem) {
 876                 mlx5_glue->devx_umem_dereg(rxq_ctrl->wq_umem);
 877                 rxq_ctrl->wq_umem = NULL;
 878         }
 879 }
 880
 881 /**
 882  * Release the resources allocated for the Rx CQ DevX object.
 883  *
 884  * @param rxq_ctrl
 885  *   DevX Rx queue object.
 886  */
 887 static void
 888 rxq_release_devx_cq_resources(struct mlx5_rxq_ctrl *rxq_ctrl)
 889 {
 890         if (rxq_ctrl->rxq.cqes) {
 891                 rte_free((void *)(uintptr_t)rxq_ctrl->rxq.cqes);
 892                 rxq_ctrl->rxq.cqes = NULL;
 893         }
 894         if (rxq_ctrl->cq_umem) {
 895                 mlx5_glue->devx_umem_dereg(rxq_ctrl->cq_umem);
 896                 rxq_ctrl->cq_umem = NULL;
 897         }
 898 }
 899
 900 /**
 901  * Release an Rx hairpin related resources.
 902  *
 903  * @param rxq_obj
 904  *   Hairpin Rx queue object.
 905  */
 906 static void
 907 rxq_obj_hairpin_release(struct mlx5_rxq_obj *rxq_obj)
 908 {
 909         struct mlx5_devx_modify_rq_attr rq_attr = { 0 };
 910
 911         MLX5_ASSERT(rxq_obj);
 912         rq_attr.state = MLX5_RQC_STATE_RST;
 913         rq_attr.rq_state = MLX5_RQC_STATE_RDY;
 914         mlx5_devx_cmd_modify_rq(rxq_obj->rq, &rq_attr);
 915         claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq));
 916 }
 917
 918 /**
 919  * Release an Rx verbs/DevX queue object.
 920  *
 921  * @param rxq_obj
 922  *   Verbs/DevX Rx queue object.
 923  *
 924  * @return
 925  *   1 while a reference on it exists, 0 when freed.
 926  */
 927 static int
 928 mlx5_rxq_obj_release(struct mlx5_rxq_obj *rxq_obj)
 929 {
 930         MLX5_ASSERT(rxq_obj);
 931         if (rte_atomic32_dec_and_test(&rxq_obj->refcnt)) {
 932                 switch (rxq_obj->type) {
 933                 case MLX5_RXQ_OBJ_TYPE_IBV:
 934                         MLX5_ASSERT(rxq_obj->wq);
 935                         MLX5_ASSERT(rxq_obj->ibv_cq);
 936                         rxq_free_elts(rxq_obj->rxq_ctrl);
 937                         claim_zero(mlx5_glue->destroy_wq(rxq_obj->wq));
 938                         claim_zero(mlx5_glue->destroy_cq(rxq_obj->ibv_cq));
 939                         if (rxq_obj->ibv_channel)
 940                                 claim_zero(mlx5_glue->destroy_comp_channel
 941                                            (rxq_obj->ibv_channel));
 942                         break;
 943                 case MLX5_RXQ_OBJ_TYPE_DEVX_RQ:
 944                         MLX5_ASSERT(rxq_obj->rq);
 945                         MLX5_ASSERT(rxq_obj->devx_cq);
 946                         rxq_free_elts(rxq_obj->rxq_ctrl);
 947                         claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq));
 948                         claim_zero(mlx5_devx_cmd_destroy(rxq_obj->devx_cq));
 949                         if (rxq_obj->devx_channel)
 950                                 mlx5_glue->devx_destroy_event_channel
 951                                                         (rxq_obj->devx_channel);
 952                         rxq_release_devx_rq_resources(rxq_obj->rxq_ctrl);
 953                         rxq_release_devx_cq_resources(rxq_obj->rxq_ctrl);
 954                         break;
 955                 case MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN:
 956                         MLX5_ASSERT(rxq_obj->rq);
 957                         rxq_obj_hairpin_release(rxq_obj);
 958                         break;
 959                 }
 960                 LIST_REMOVE(rxq_obj, next);
 961                 mlx5_free(rxq_obj);
 962                 return 0;
 963         }
 964         return 1;
 965 }
 966
 967 /**
 968  * Allocate queue vector and fill epoll fd list for Rx interrupts.
 969  *
 970  * @param dev
 971  *   Pointer to Ethernet device.
 972  *
 973  * @return
 974  *   0 on success, a negative errno value otherwise and rte_errno is set.
 975  */
 976 int
 977 mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev)
 978 {
 979         struct mlx5_priv *priv = dev->data->dev_private;
 980         unsigned int i;
 981         unsigned int rxqs_n = priv->rxqs_n;
 982         unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
 983         unsigned int count = 0;
 984         struct rte_intr_handle *intr_handle = dev->intr_handle;
 985
 986         if (!dev->data->dev_conf.intr_conf.rxq)
 987                 return 0;
 988         mlx5_rx_intr_vec_disable(dev);
 989         intr_handle->intr_vec = mlx5_malloc(0,
 990                                 n * sizeof(intr_handle->intr_vec[0]),
 991                                 0, SOCKET_ID_ANY);
 992         if (intr_handle->intr_vec == NULL) {
 993                 DRV_LOG(ERR,
 994                         "port %u failed to allocate memory for interrupt"
 995                         " vector, Rx interrupts will not be supported",
 996                         dev->data->port_id);
 997                 rte_errno = ENOMEM;
 998                 return -rte_errno;
 999         }
1000         intr_handle->type = RTE_INTR_HANDLE_EXT;
1001         for (i = 0; i != n; ++i) {
1002                 /* This rxq obj must not be released in this function. */
1003                 struct mlx5_rxq_obj *rxq_obj = mlx5_rxq_obj_get(dev, i);
1004                 int rc;
1005
1006                 /* Skip queues that cannot request interrupts. */
1007                 if (!rxq_obj || (!rxq_obj->ibv_channel &&
1008                                  !rxq_obj->devx_channel)) {
1009                         /* Use invalid intr_vec[] index to disable entry. */
1010                         intr_handle->intr_vec[i] =
1011                                 RTE_INTR_VEC_RXTX_OFFSET +
1012                                 RTE_MAX_RXTX_INTR_VEC_ID;
1013                         continue;
1014                 }
1015                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
1016                         DRV_LOG(ERR,
1017                                 "port %u too many Rx queues for interrupt"
1018                                 " vector size (%d), Rx interrupts cannot be"
1019                                 " enabled",
1020                                 dev->data->port_id, RTE_MAX_RXTX_INTR_VEC_ID);
1021                         mlx5_rx_intr_vec_disable(dev);
1022                         rte_errno = ENOMEM;
1023                         return -rte_errno;
1024                 }
1025                 rc = mlx5_os_set_nonblock_channel_fd(rxq_obj->fd);
1026                 if (rc < 0) {
1027                         rte_errno = errno;
1028                         DRV_LOG(ERR,
1029                                 "port %u failed to make Rx interrupt file"
1030                                 " descriptor %d non-blocking for queue index"
1031                                 " %d",
1032                                 dev->data->port_id, rxq_obj->fd, i);
1033                         mlx5_rx_intr_vec_disable(dev);
1034                         return -rte_errno;
1035                 }
1036                 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
1037                 intr_handle->efds[count] = rxq_obj->fd;
1038                 count++;
1039         }
1040         if (!count)
1041                 mlx5_rx_intr_vec_disable(dev);
1042         else
1043                 intr_handle->nb_efd = count;
1044         return 0;
1045 }
1046
1047 /**
1048  * Clean up Rx interrupts handler.
1049  *
1050  * @param dev
1051  *   Pointer to Ethernet device.
1052  */
1053 void
1054 mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev)
1055 {
1056         struct mlx5_priv *priv = dev->data->dev_private;
1057         struct rte_intr_handle *intr_handle = dev->intr_handle;
1058         unsigned int i;
1059         unsigned int rxqs_n = priv->rxqs_n;
1060         unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
1061
1062         if (!dev->data->dev_conf.intr_conf.rxq)
1063                 return;
1064         if (!intr_handle->intr_vec)
1065                 goto free;
1066         for (i = 0; i != n; ++i) {
1067                 struct mlx5_rxq_ctrl *rxq_ctrl;
1068                 struct mlx5_rxq_data *rxq_data;
1069
1070                 if (intr_handle->intr_vec[i] == RTE_INTR_VEC_RXTX_OFFSET +
1071                     RTE_MAX_RXTX_INTR_VEC_ID)
1072                         continue;
1073                 /**
1074                  * Need to access directly the queue to release the reference
1075                  * kept in mlx5_rx_intr_vec_enable().
1076                  */
1077                 rxq_data = (*priv->rxqs)[i];
1078                 rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
1079                 if (rxq_ctrl->obj)
1080                         mlx5_rxq_obj_release(rxq_ctrl->obj);
1081         }
1082 free:
1083         rte_intr_free_epoll_fd(intr_handle);
1084         if (intr_handle->intr_vec)
1085                 mlx5_free(intr_handle->intr_vec);
1086         intr_handle->nb_efd = 0;
1087         intr_handle->intr_vec = NULL;
1088 }
1089
1090 /**
1091  *  MLX5 CQ notification .
1092  *
1093  *  @param rxq
1094  *     Pointer to receive queue structure.
1095  *  @param sq_n_rxq
1096  *     Sequence number per receive queue .
1097  */
1098 static inline void
1099 mlx5_arm_cq(struct mlx5_rxq_data *rxq, int sq_n_rxq)
1100 {
1101         int sq_n = 0;
1102         uint32_t doorbell_hi;
1103         uint64_t doorbell;
1104         void *cq_db_reg = (char *)rxq->cq_uar + MLX5_CQ_DOORBELL;
1105
1106         sq_n = sq_n_rxq & MLX5_CQ_SQN_MASK;
1107         doorbell_hi = sq_n << MLX5_CQ_SQN_OFFSET | (rxq->cq_ci & MLX5_CI_MASK);
1108         doorbell = (uint64_t)doorbell_hi << 32;
1109         doorbell |= rxq->cqn;
1110         rxq->cq_db[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(doorbell_hi);
1111         mlx5_uar_write64(rte_cpu_to_be_64(doorbell),
1112                          cq_db_reg, rxq->uar_lock_cq);
1113 }
1114
1115 /**
1116  * DPDK callback for Rx queue interrupt enable.
1117  *
1118  * @param dev
1119  *   Pointer to Ethernet device structure.
1120  * @param rx_queue_id
1121  *   Rx queue number.
1122  *
1123  * @return
1124  *   0 on success, a negative errno value otherwise and rte_errno is set.
1125  */
1126 int
1127 mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1128 {
1129         struct mlx5_priv *priv = dev->data->dev_private;
1130         struct mlx5_rxq_data *rxq_data;
1131         struct mlx5_rxq_ctrl *rxq_ctrl;
1132
1133         rxq_data = (*priv->rxqs)[rx_queue_id];
1134         if (!rxq_data) {
1135                 rte_errno = EINVAL;
1136                 return -rte_errno;
1137         }
1138         rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
1139         if (rxq_ctrl->irq) {
1140                 struct mlx5_rxq_obj *rxq_obj;
1141
1142                 rxq_obj = mlx5_rxq_obj_get(dev, rx_queue_id);
1143                 if (!rxq_obj) {
1144                         rte_errno = EINVAL;
1145                         return -rte_errno;
1146                 }
1147                 mlx5_arm_cq(rxq_data, rxq_data->cq_arm_sn);
1148                 mlx5_rxq_obj_release(rxq_obj);
1149         }
1150         return 0;
1151 }
1152
1153 /**
1154  * DPDK callback for Rx queue interrupt disable.
1155  *
1156  * @param dev
1157  *   Pointer to Ethernet device structure.
1158  * @param rx_queue_id
1159  *   Rx queue number.
1160  *
1161  * @return
1162  *   0 on success, a negative errno value otherwise and rte_errno is set.
1163  */
1164 int
1165 mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1166 {
1167         struct mlx5_priv *priv = dev->data->dev_private;
1168         struct mlx5_rxq_data *rxq_data;
1169         struct mlx5_rxq_ctrl *rxq_ctrl;
1170         struct mlx5_rxq_obj *rxq_obj = NULL;
1171         struct ibv_cq *ev_cq;
1172         void *ev_ctx;
1173         int ret;
1174
1175         rxq_data = (*priv->rxqs)[rx_queue_id];
1176         if (!rxq_data) {
1177                 rte_errno = EINVAL;
1178                 return -rte_errno;
1179         }
1180         rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
1181         if (!rxq_ctrl->irq)
1182                 return 0;
1183         rxq_obj = mlx5_rxq_obj_get(dev, rx_queue_id);
1184         if (!rxq_obj) {
1185                 rte_errno = EINVAL;
1186                 return -rte_errno;
1187         }
1188         if (rxq_obj->type == MLX5_RXQ_OBJ_TYPE_IBV) {
1189                 ret = mlx5_glue->get_cq_event(rxq_obj->ibv_channel, &ev_cq,
1190                                               &ev_ctx);
1191                 if (ret < 0 || ev_cq != rxq_obj->ibv_cq)
1192                         goto exit;
1193                 mlx5_glue->ack_cq_events(rxq_obj->ibv_cq, 1);
1194         } else if (rxq_obj->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) {
1195 #ifdef HAVE_IBV_DEVX_EVENT
1196                 struct mlx5dv_devx_async_event_hdr *event_data = NULL;
1197
1198                 ret = mlx5_glue->devx_get_event
1199                                 (rxq_obj->devx_channel, event_data,
1200                                  sizeof(struct mlx5dv_devx_async_event_hdr));
1201                 if (ret < 0 || event_data->cookie !=
1202                                 (uint64_t)(uintptr_t)rxq_obj->devx_cq)
1203                         goto exit;
1204 #endif /* HAVE_IBV_DEVX_EVENT */
1205         }
1206         rxq_data->cq_arm_sn++;
1207         mlx5_rxq_obj_release(rxq_obj);
1208         return 0;
1209 exit:
1210         /**
1211          * For ret < 0 save the errno (may be EAGAIN which means the get_event
1212          * function was called before receiving one).
1213          */
1214         if (ret < 0)
1215                 rte_errno = errno;
1216         else
1217                 rte_errno = EINVAL;
1218         ret = rte_errno; /* Save rte_errno before cleanup. */
1219         if (rxq_obj)
1220                 mlx5_rxq_obj_release(rxq_obj);
1221         if (ret != EAGAIN)
1222                 DRV_LOG(WARNING, "port %u unable to disable interrupt on Rx queue %d",
1223                         dev->data->port_id, rx_queue_id);
1224         rte_errno = ret; /* Restore rte_errno. */
1225         return -rte_errno;
1226 }
1227
1228 /**
1229  * Create a CQ Verbs object.
1230  *
1231  * @param dev
1232  *   Pointer to Ethernet device.
1233  * @param priv
1234  *   Pointer to device private data.
1235  * @param rxq_data
1236  *   Pointer to Rx queue data.
1237  * @param cqe_n
1238  *   Number of CQEs in CQ.
1239  * @param rxq_obj
1240  *   Pointer to Rx queue object data.
1241  *
1242  * @return
1243  *   The Verbs object initialised, NULL otherwise and rte_errno is set.
1244  */
1245 static struct ibv_cq *
1246 mlx5_ibv_cq_new(struct rte_eth_dev *dev, struct mlx5_priv *priv,
1247                 struct mlx5_rxq_data *rxq_data,
1248                 unsigned int cqe_n, struct mlx5_rxq_obj *rxq_obj)
1249 {
1250         struct {
1251                 struct ibv_cq_init_attr_ex ibv;
1252                 struct mlx5dv_cq_init_attr mlx5;
1253         } cq_attr;
1254
1255         cq_attr.ibv = (struct ibv_cq_init_attr_ex){
1256                 .cqe = cqe_n,
1257                 .channel = rxq_obj->ibv_channel,
1258                 .comp_mask = 0,
1259         };
1260         cq_attr.mlx5 = (struct mlx5dv_cq_init_attr){
1261                 .comp_mask = 0,
1262         };
1263         if (priv->config.cqe_comp && !rxq_data->hw_timestamp &&
1264             !rxq_data->lro) {
1265                 cq_attr.mlx5.comp_mask |=
1266                                 MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;
1267 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
1268                 cq_attr.mlx5.cqe_comp_res_format =
1269                                 mlx5_rxq_mprq_enabled(rxq_data) ?
1270                                 MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX :
1271                                 MLX5DV_CQE_RES_FORMAT_HASH;
1272 #else
1273                 cq_attr.mlx5.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH;
1274 #endif
1275                 /*
1276                  * For vectorized Rx, it must not be doubled in order to
1277                  * make cq_ci and rq_ci aligned.
1278                  */
1279                 if (mlx5_rxq_check_vec_support(rxq_data) < 0)
1280                         cq_attr.ibv.cqe *= 2;
1281         } else if (priv->config.cqe_comp && rxq_data->hw_timestamp) {
1282                 DRV_LOG(DEBUG,
1283                         "port %u Rx CQE compression is disabled for HW"
1284                         " timestamp",
1285                         dev->data->port_id);
1286         } else if (priv->config.cqe_comp && rxq_data->lro) {
1287                 DRV_LOG(DEBUG,
1288                         "port %u Rx CQE compression is disabled for LRO",
1289                         dev->data->port_id);
1290         }
1291 #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD
1292         if (priv->config.cqe_pad) {
1293                 cq_attr.mlx5.comp_mask |= MLX5DV_CQ_INIT_ATTR_MASK_FLAGS;
1294                 cq_attr.mlx5.flags |= MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD;
1295         }
1296 #endif
1297         return mlx5_glue->cq_ex_to_cq(mlx5_glue->dv_create_cq(priv->sh->ctx,
1298                                                               &cq_attr.ibv,
1299                                                               &cq_attr.mlx5));
1300 }
1301
1302 /**
1303  * Create a WQ Verbs object.
1304  *
1305  * @param dev
1306  *   Pointer to Ethernet device.
1307  * @param priv
1308  *   Pointer to device private data.
1309  * @param rxq_data
1310  *   Pointer to Rx queue data.
1311  * @param idx
1312  *   Queue index in DPDK Rx queue array
1313  * @param wqe_n
1314  *   Number of WQEs in WQ.
1315  * @param rxq_obj
1316  *   Pointer to Rx queue object data.
1317  *
1318  * @return
1319  *   The Verbs object initialised, NULL otherwise and rte_errno is set.
1320  */
1321 static struct ibv_wq *
1322 mlx5_ibv_wq_new(struct rte_eth_dev *dev, struct mlx5_priv *priv,
1323                 struct mlx5_rxq_data *rxq_data, uint16_t idx,
1324                 unsigned int wqe_n, struct mlx5_rxq_obj *rxq_obj)
1325 {
1326         struct {
1327                 struct ibv_wq_init_attr ibv;
1328 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
1329                 struct mlx5dv_wq_init_attr mlx5;
1330 #endif
1331         } wq_attr;
1332
1333         wq_attr.ibv = (struct ibv_wq_init_attr){
1334                 .wq_context = NULL, /* Could be useful in the future. */
1335                 .wq_type = IBV_WQT_RQ,
1336                 /* Max number of outstanding WRs. */
1337                 .max_wr = wqe_n >> rxq_data->sges_n,
1338                 /* Max number of scatter/gather elements in a WR. */
1339                 .max_sge = 1 << rxq_data->sges_n,
1340                 .pd = priv->sh->pd,
1341                 .cq = rxq_obj->ibv_cq,
1342                 .comp_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING | 0,
1343                 .create_flags = (rxq_data->vlan_strip ?
1344                                  IBV_WQ_FLAGS_CVLAN_STRIPPING : 0),
1345         };
1346         /* By default, FCS (CRC) is stripped by hardware. */
1347         if (rxq_data->crc_present) {
1348                 wq_attr.ibv.create_flags |= IBV_WQ_FLAGS_SCATTER_FCS;
1349                 wq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
1350         }
1351         if (priv->config.hw_padding) {
1352 #if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING)
1353                 wq_attr.ibv.create_flags |= IBV_WQ_FLAG_RX_END_PADDING;
1354                 wq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
1355 #elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING)
1356                 wq_attr.ibv.create_flags |= IBV_WQ_FLAGS_PCI_WRITE_END_PADDING;
1357                 wq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
1358 #endif
1359         }
1360 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
1361         wq_attr.mlx5 = (struct mlx5dv_wq_init_attr){
1362                 .comp_mask = 0,
1363         };
1364         if (mlx5_rxq_mprq_enabled(rxq_data)) {
1365                 struct mlx5dv_striding_rq_init_attr *mprq_attr =
1366                                                 &wq_attr.mlx5.striding_rq_attrs;
1367
1368                 wq_attr.mlx5.comp_mask |= MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ;
1369                 *mprq_attr = (struct mlx5dv_striding_rq_init_attr){
1370                         .single_stride_log_num_of_bytes = rxq_data->strd_sz_n,
1371                         .single_wqe_log_num_of_strides = rxq_data->strd_num_n,
1372                         .two_byte_shift_en = MLX5_MPRQ_TWO_BYTE_SHIFT,
1373                 };
1374         }
1375         rxq_obj->wq = mlx5_glue->dv_create_wq(priv->sh->ctx, &wq_attr.ibv,
1376                                               &wq_attr.mlx5);
1377 #else
1378         rxq_obj->wq = mlx5_glue->create_wq(priv->sh->ctx, &wq_attr.ibv);
1379 #endif
1380         if (rxq_obj->wq) {
1381                 /*
1382                  * Make sure number of WRs*SGEs match expectations since a queue
1383                  * cannot allocate more than "desc" buffers.
1384                  */
1385                 if (wq_attr.ibv.max_wr != (wqe_n >> rxq_data->sges_n) ||
1386                     wq_attr.ibv.max_sge != (1u << rxq_data->sges_n)) {
1387                         DRV_LOG(ERR,
1388                                 "port %u Rx queue %u requested %u*%u but got"
1389                                 " %u*%u WRs*SGEs",
1390                                 dev->data->port_id, idx,
1391                                 wqe_n >> rxq_data->sges_n,
1392                                 (1 << rxq_data->sges_n),
1393                                 wq_attr.ibv.max_wr, wq_attr.ibv.max_sge);
1394                         claim_zero(mlx5_glue->destroy_wq(rxq_obj->wq));
1395                         rxq_obj->wq = NULL;
1396                         rte_errno = EINVAL;
1397                 }
1398         }
1399         return rxq_obj->wq;
1400 }
1401
1402 /**
1403  * Fill common fields of create RQ attributes structure.
1404  *
1405  * @param rxq_data
1406  *   Pointer to Rx queue data.
1407  * @param cqn
1408  *   CQ number to use with this RQ.
1409  * @param rq_attr
1410  *   RQ attributes structure to fill..
1411  */
1412 static void
1413 mlx5_devx_create_rq_attr_fill(struct mlx5_rxq_data *rxq_data, uint32_t cqn,
1414                               struct mlx5_devx_create_rq_attr *rq_attr)
1415 {
1416         rq_attr->state = MLX5_RQC_STATE_RST;
1417         rq_attr->vsd = (rxq_data->vlan_strip) ? 0 : 1;
1418         rq_attr->cqn = cqn;
1419         rq_attr->scatter_fcs = (rxq_data->crc_present) ? 1 : 0;
1420 }
1421
1422 /**
1423  * Fill common fields of DevX WQ attributes structure.
1424  *
1425  * @param priv
1426  *   Pointer to device private data.
1427  * @param rxq_ctrl
1428  *   Pointer to Rx queue control structure.
1429  * @param wq_attr
1430  *   WQ attributes structure to fill..
1431  */
1432 static void
1433 mlx5_devx_wq_attr_fill(struct mlx5_priv *priv, struct mlx5_rxq_ctrl *rxq_ctrl,
1434                        struct mlx5_devx_wq_attr *wq_attr)
1435 {
1436         wq_attr->end_padding_mode = priv->config.cqe_pad ?
1437                                         MLX5_WQ_END_PAD_MODE_ALIGN :
1438                                         MLX5_WQ_END_PAD_MODE_NONE;
1439         wq_attr->pd = priv->sh->pdn;
1440         wq_attr->dbr_addr = rxq_ctrl->rq_dbr_offset;
1441         wq_attr->dbr_umem_id = rxq_ctrl->rq_dbr_umem_id;
1442         wq_attr->dbr_umem_valid = 1;
1443         wq_attr->wq_umem_id = rxq_ctrl->wq_umem->umem_id;
1444         wq_attr->wq_umem_valid = 1;
1445 }
1446
1447 /**
1448  * Create a RQ object using DevX.
1449  *
1450  * @param dev
1451  *   Pointer to Ethernet device.
1452  * @param idx
1453  *   Queue index in DPDK Rx queue array
1454  * @param cqn
1455  *   CQ number to use with this RQ.
1456  *
1457  * @return
1458  *   The DevX object initialised, NULL otherwise and rte_errno is set.
1459  */
1460 static struct mlx5_devx_obj *
1461 mlx5_devx_rq_new(struct rte_eth_dev *dev, uint16_t idx, uint32_t cqn)
1462 {
1463         struct mlx5_priv *priv = dev->data->dev_private;
1464         struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
1465         struct mlx5_rxq_ctrl *rxq_ctrl =
1466                 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
1467         struct mlx5_devx_create_rq_attr rq_attr = { 0 };
1468         uint32_t wqe_n = 1 << (rxq_data->elts_n - rxq_data->sges_n);
1469         uint32_t wq_size = 0;
1470         uint32_t wqe_size = 0;
1471         uint32_t log_wqe_size = 0;
1472         void *buf = NULL;
1473         struct mlx5_devx_obj *rq;
1474
1475         /* Fill RQ attributes. */
1476         rq_attr.mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE;
1477         rq_attr.flush_in_error_en = 1;
1478         mlx5_devx_create_rq_attr_fill(rxq_data, cqn, &rq_attr);
1479         /* Fill WQ attributes for this RQ. */
1480         if (mlx5_rxq_mprq_enabled(rxq_data)) {
1481                 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ;
1482                 /*
1483                  * Number of strides in each WQE:
1484                  * 512*2^single_wqe_log_num_of_strides.
1485                  */
1486                 rq_attr.wq_attr.single_wqe_log_num_of_strides =
1487                                 rxq_data->strd_num_n -
1488                                 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1489                 /* Stride size = (2^single_stride_log_num_of_bytes)*64B. */
1490                 rq_attr.wq_attr.single_stride_log_num_of_bytes =
1491                                 rxq_data->strd_sz_n -
1492                                 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;
1493                 wqe_size = sizeof(struct mlx5_wqe_mprq);
1494         } else {
1495                 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC;
1496                 wqe_size = sizeof(struct mlx5_wqe_data_seg);
1497         }
1498         log_wqe_size = log2above(wqe_size) + rxq_data->sges_n;
1499         rq_attr.wq_attr.log_wq_stride = log_wqe_size;
1500         rq_attr.wq_attr.log_wq_sz = rxq_data->elts_n - rxq_data->sges_n;
1501         /* Calculate and allocate WQ memory space. */
1502         wqe_size = 1 << log_wqe_size; /* round up power of two.*/
1503         wq_size = wqe_n * wqe_size;
1504         size_t alignment = MLX5_WQE_BUF_ALIGNMENT;
1505         if (alignment == (size_t)-1) {
1506                 DRV_LOG(ERR, "Failed to get mem page size");
1507                 rte_errno = ENOMEM;
1508                 return NULL;
1509         }
1510         buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, wq_size,
1511                           alignment, rxq_ctrl->socket);
1512         if (!buf)
1513                 return NULL;
1514         rxq_data->wqes = buf;
1515         rxq_ctrl->wq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx,
1516                                                      buf, wq_size, 0);
1517         if (!rxq_ctrl->wq_umem) {
1518                 mlx5_free(buf);
1519                 return NULL;
1520         }
1521         mlx5_devx_wq_attr_fill(priv, rxq_ctrl, &rq_attr.wq_attr);
1522         rq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &rq_attr, rxq_ctrl->socket);
1523         if (!rq)
1524                 rxq_release_devx_rq_resources(rxq_ctrl);
1525         return rq;
1526 }
1527
1528 /**
1529  * Create a DevX CQ object for an Rx queue.
1530  *
1531  * @param dev
1532  *   Pointer to Ethernet device.
1533  * @param cqe_n
1534  *   Number of CQEs in CQ.
1535  * @param idx
1536  *   Queue index in DPDK Rx queue array
1537  * @param rxq_obj
1538  *   Pointer to Rx queue object data.
1539  *
1540  * @return
1541  *   The DevX object initialised, NULL otherwise and rte_errno is set.
1542  */
1543 static struct mlx5_devx_obj *
1544 mlx5_devx_cq_new(struct rte_eth_dev *dev, unsigned int cqe_n, uint16_t idx,
1545                  struct mlx5_rxq_obj *rxq_obj)
1546 {
1547         struct mlx5_devx_obj *cq_obj = 0;
1548         struct mlx5_devx_cq_attr cq_attr = { 0 };
1549         struct mlx5_priv *priv = dev->data->dev_private;
1550         struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
1551         struct mlx5_rxq_ctrl *rxq_ctrl =
1552                 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
1553         size_t page_size = rte_mem_page_size();
1554         uint32_t lcore = (uint32_t)rte_lcore_to_cpu_id(-1);
1555         uint32_t eqn = 0;
1556         void *buf = NULL;
1557         uint16_t event_nums[1] = {0};
1558         uint32_t log_cqe_n;
1559         uint32_t cq_size;
1560         int ret = 0;
1561
1562         if (page_size == (size_t)-1) {
1563                 DRV_LOG(ERR, "Failed to get page_size.");
1564                 goto error;
1565         }
1566         if (priv->config.cqe_comp && !rxq_data->hw_timestamp &&
1567             !rxq_data->lro) {
1568                 cq_attr.cqe_comp_en = MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;
1569 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
1570                 cq_attr.mini_cqe_res_format =
1571                                 mlx5_rxq_mprq_enabled(rxq_data) ?
1572                                 MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX :
1573                                 MLX5DV_CQE_RES_FORMAT_HASH;
1574 #else
1575                 cq_attr.mini_cqe_res_format = MLX5DV_CQE_RES_FORMAT_HASH;
1576 #endif
1577                 /*
1578                  * For vectorized Rx, it must not be doubled in order to
1579                  * make cq_ci and rq_ci aligned.
1580                  */
1581                 if (mlx5_rxq_check_vec_support(rxq_data) < 0)
1582                         cqe_n *= 2;
1583         } else if (priv->config.cqe_comp && rxq_data->hw_timestamp) {
1584                 DRV_LOG(DEBUG,
1585                         "port %u Rx CQE compression is disabled for HW"
1586                         " timestamp",
1587                         dev->data->port_id);
1588         } else if (priv->config.cqe_comp && rxq_data->lro) {
1589                 DRV_LOG(DEBUG,
1590                         "port %u Rx CQE compression is disabled for LRO",
1591                         dev->data->port_id);
1592         }
1593 #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD
1594         if (priv->config.cqe_pad)
1595                 cq_attr.cqe_size = MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD;
1596 #endif
1597         log_cqe_n = log2above(cqe_n);
1598         cq_size = sizeof(struct mlx5_cqe) * (1 << log_cqe_n);
1599         /* Query the EQN for this core. */
1600         if (mlx5_glue->devx_query_eqn(priv->sh->ctx, lcore, &eqn)) {
1601                 DRV_LOG(ERR, "Failed to query EQN for CQ.");
1602                 goto error;
1603         }
1604         cq_attr.eqn = eqn;
1605         buf = rte_calloc_socket(__func__, 1, cq_size, page_size,
1606                                 rxq_ctrl->socket);
1607         if (!buf) {
1608                 DRV_LOG(ERR, "Failed to allocate memory for CQ.");
1609                 goto error;
1610         }
1611         rxq_data->cqes = (volatile struct mlx5_cqe (*)[])(uintptr_t)buf;
1612         rxq_ctrl->cq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, buf,
1613                                                      cq_size,
1614                                                      IBV_ACCESS_LOCAL_WRITE);
1615         if (!rxq_ctrl->cq_umem) {
1616                 DRV_LOG(ERR, "Failed to register umem for CQ.");
1617                 goto error;
1618         }
1619         cq_attr.uar_page_id = priv->sh->devx_rx_uar->page_id;
1620         cq_attr.q_umem_id = rxq_ctrl->cq_umem->umem_id;
1621         cq_attr.q_umem_valid = 1;
1622         cq_attr.log_cq_size = log_cqe_n;
1623         cq_attr.log_page_size = rte_log2_u32(page_size);
1624         cq_attr.db_umem_offset = rxq_ctrl->cq_dbr_offset;
1625         cq_attr.db_umem_id = rxq_ctrl->cq_dbr_umem_id;
1626         cq_attr.db_umem_valid = rxq_ctrl->cq_dbr_umem_id_valid;
1627         cq_obj = mlx5_devx_cmd_create_cq(priv->sh->ctx, &cq_attr);
1628         if (!cq_obj)
1629                 goto error;
1630         rxq_data->cqe_n = log_cqe_n;
1631         rxq_data->cqn = cq_obj->id;
1632         if (rxq_obj->devx_channel) {
1633                 ret = mlx5_glue->devx_subscribe_devx_event
1634                                                 (rxq_obj->devx_channel,
1635                                                  cq_obj->obj,
1636                                                  sizeof(event_nums),
1637                                                  event_nums,
1638                                                  (uint64_t)(uintptr_t)cq_obj);
1639                 if (ret) {
1640                         DRV_LOG(ERR, "Fail to subscribe CQ to event channel.");
1641                         rte_errno = errno;
1642                         goto error;
1643                 }
1644         }
1645         /* Initialise CQ to 1's to mark HW ownership for all CQEs. */
1646         memset((void *)(uintptr_t)rxq_data->cqes, 0xFF, cq_size);
1647         return cq_obj;
1648 error:
1649         rxq_release_devx_cq_resources(rxq_ctrl);
1650         return NULL;
1651 }
1652
1653 /**
1654  * Create the Rx hairpin queue object.
1655  *
1656  * @param dev
1657  *   Pointer to Ethernet device.
1658  * @param idx
1659  *   Queue index in DPDK Rx queue array
1660  *
1661  * @return
1662  *   The hairpin DevX object initialised, NULL otherwise and rte_errno is set.
1663  */
1664 static struct mlx5_rxq_obj *
1665 mlx5_rxq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx)
1666 {
1667         struct mlx5_priv *priv = dev->data->dev_private;
1668         struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
1669         struct mlx5_rxq_ctrl *rxq_ctrl =
1670                 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
1671         struct mlx5_devx_create_rq_attr attr = { 0 };
1672         struct mlx5_rxq_obj *tmpl = NULL;
1673         uint32_t max_wq_data;
1674
1675         MLX5_ASSERT(rxq_data);
1676         MLX5_ASSERT(!rxq_ctrl->obj);
1677         tmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl), 0,
1678                            rxq_ctrl->socket);
1679         if (!tmpl) {
1680                 DRV_LOG(ERR,
1681                         "port %u Rx queue %u cannot allocate verbs resources",
1682                         dev->data->port_id, rxq_data->idx);
1683                 rte_errno = ENOMEM;
1684                 return NULL;
1685         }
1686         tmpl->type = MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN;
1687         tmpl->rxq_ctrl = rxq_ctrl;
1688         attr.hairpin = 1;
1689         max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz;
1690         /* Jumbo frames > 9KB should be supported, and more packets. */
1691         if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) {
1692                 if (priv->config.log_hp_size > max_wq_data) {
1693                         DRV_LOG(ERR, "total data size %u power of 2 is "
1694                                 "too large for hairpin",
1695                                 priv->config.log_hp_size);
1696                         mlx5_free(tmpl);
1697                         rte_errno = ERANGE;
1698                         return NULL;
1699                 }
1700                 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size;
1701         } else {
1702                 attr.wq_attr.log_hairpin_data_sz =
1703                                 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ?
1704                                  max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE;
1705         }
1706         /* Set the packets number to the maximum value for performance. */
1707         attr.wq_attr.log_hairpin_num_packets =
1708                         attr.wq_attr.log_hairpin_data_sz -
1709                         MLX5_HAIRPIN_QUEUE_STRIDE;
1710         tmpl->rq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &attr,
1711                                            rxq_ctrl->socket);
1712         if (!tmpl->rq) {
1713                 DRV_LOG(ERR,
1714                         "port %u Rx hairpin queue %u can't create rq object",
1715                         dev->data->port_id, idx);
1716                 mlx5_free(tmpl);
1717                 rte_errno = errno;
1718                 return NULL;
1719         }
1720         DRV_LOG(DEBUG, "port %u rxq %u updated with %p", dev->data->port_id,
1721                 idx, (void *)&tmpl);
1722         rte_atomic32_inc(&tmpl->refcnt);
1723         LIST_INSERT_HEAD(&priv->rxqsobj, tmpl, next);
1724         priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
1725         dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN;
1726         return tmpl;
1727 }
1728
1729 /**
1730  * Create the Rx queue Verbs/DevX object.
1731  *
1732  * @param dev
1733  *   Pointer to Ethernet device.
1734  * @param idx
1735  *   Queue index in DPDK Rx queue array
1736  * @param type
1737  *   Type of Rx queue object to create.
1738  *
1739  * @return
1740  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
1741  */
1742 struct mlx5_rxq_obj *
1743 mlx5_rxq_obj_new(struct rte_eth_dev *dev, uint16_t idx,
1744                  enum mlx5_rxq_obj_type type)
1745 {
1746         struct mlx5_priv *priv = dev->data->dev_private;
1747         struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
1748         struct mlx5_rxq_ctrl *rxq_ctrl =
1749                 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
1750         struct ibv_wq_attr mod;
1751         unsigned int cqe_n;
1752         unsigned int wqe_n = 1 << rxq_data->elts_n;
1753         struct mlx5_rxq_obj *tmpl = NULL;
1754         struct mlx5dv_cq cq_info;
1755         struct mlx5dv_rwq rwq;
1756         int ret = 0;
1757         struct mlx5dv_obj obj;
1758
1759         MLX5_ASSERT(rxq_data);
1760         MLX5_ASSERT(!rxq_ctrl->obj);
1761         if (type == MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN)
1762                 return mlx5_rxq_obj_hairpin_new(dev, idx);
1763         priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_RX_QUEUE;
1764         priv->verbs_alloc_ctx.obj = rxq_ctrl;
1765         tmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl), 0,
1766                            rxq_ctrl->socket);
1767         if (!tmpl) {
1768                 DRV_LOG(ERR,
1769                         "port %u Rx queue %u cannot allocate resources",
1770                         dev->data->port_id, rxq_data->idx);
1771                 rte_errno = ENOMEM;
1772                 goto error;
1773         }
1774         tmpl->type = type;
1775         tmpl->rxq_ctrl = rxq_ctrl;
1776         if (rxq_ctrl->irq) {
1777                 if (tmpl->type == MLX5_RXQ_OBJ_TYPE_IBV) {
1778                         tmpl->ibv_channel =
1779                                 mlx5_glue->create_comp_channel(priv->sh->ctx);
1780                         if (!tmpl->ibv_channel) {
1781                                 DRV_LOG(ERR, "port %u: comp channel creation "
1782                                         "failure", dev->data->port_id);
1783                                 rte_errno = ENOMEM;
1784                                 goto error;
1785                         }
1786                         tmpl->fd = tmpl->ibv_channel->fd;
1787                 } else if (tmpl->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) {
1788                         int devx_ev_flag =
1789                           MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA;
1790
1791                         tmpl->devx_channel =
1792                                 mlx5_glue->devx_create_event_channel
1793                                                                 (priv->sh->ctx,
1794                                                                  devx_ev_flag);
1795                         if (!tmpl->devx_channel) {
1796                                 rte_errno = errno;
1797                                 DRV_LOG(ERR,
1798                                         "Failed to create event channel %d.",
1799                                         rte_errno);
1800                                 goto error;
1801                         }
1802                         tmpl->fd = tmpl->devx_channel->fd;
1803                 }
1804         }
1805         if (mlx5_rxq_mprq_enabled(rxq_data))
1806                 cqe_n = wqe_n * (1 << rxq_data->strd_num_n) - 1;
1807         else
1808                 cqe_n = wqe_n - 1;
1809         DRV_LOG(DEBUG, "port %u device_attr.max_qp_wr is %d",
1810                 dev->data->port_id, priv->sh->device_attr.max_qp_wr);
1811         DRV_LOG(DEBUG, "port %u device_attr.max_sge is %d",
1812                 dev->data->port_id, priv->sh->device_attr.max_sge);
1813         if (tmpl->type == MLX5_RXQ_OBJ_TYPE_IBV) {
1814                 /* Create CQ using Verbs API. */
1815                 tmpl->ibv_cq = mlx5_ibv_cq_new(dev, priv, rxq_data, cqe_n,
1816                                                tmpl);
1817                 if (!tmpl->ibv_cq) {
1818                         DRV_LOG(ERR, "port %u Rx queue %u CQ creation failure",
1819                                 dev->data->port_id, idx);
1820                         rte_errno = ENOMEM;
1821                         goto error;
1822                 }
1823                 obj.cq.in = tmpl->ibv_cq;
1824                 obj.cq.out = &cq_info;
1825                 ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ);
1826                 if (ret) {
1827                         rte_errno = ret;
1828                         goto error;
1829                 }
1830                 if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {
1831                         DRV_LOG(ERR,
1832                                 "port %u wrong MLX5_CQE_SIZE environment "
1833                                 "variable value: it should be set to %u",
1834                                 dev->data->port_id, RTE_CACHE_LINE_SIZE);
1835                         rte_errno = EINVAL;
1836                         goto error;
1837                 }
1838                 /* Fill the rings. */
1839                 rxq_data->cqe_n = log2above(cq_info.cqe_cnt);
1840                 rxq_data->cq_db = cq_info.dbrec;
1841                 rxq_data->cqes =
1842                         (volatile struct mlx5_cqe (*)[])(uintptr_t)cq_info.buf;
1843                 rxq_data->cq_uar = cq_info.cq_uar;
1844                 rxq_data->cqn = cq_info.cqn;
1845                 /* Create WQ (RQ) using Verbs API. */
1846                 tmpl->wq = mlx5_ibv_wq_new(dev, priv, rxq_data, idx, wqe_n,
1847                                            tmpl);
1848                 if (!tmpl->wq) {
1849                         DRV_LOG(ERR, "port %u Rx queue %u WQ creation failure",
1850                                 dev->data->port_id, idx);
1851                         rte_errno = ENOMEM;
1852                         goto error;
1853                 }
1854                 /* Change queue state to ready. */
1855                 mod = (struct ibv_wq_attr){
1856                         .attr_mask = IBV_WQ_ATTR_STATE,
1857                         .wq_state = IBV_WQS_RDY,
1858                 };
1859                 ret = mlx5_glue->modify_wq(tmpl->wq, &mod);
1860                 if (ret) {
1861                         DRV_LOG(ERR,
1862                                 "port %u Rx queue %u WQ state to IBV_WQS_RDY"
1863                                 " failed", dev->data->port_id, idx);
1864                         rte_errno = ret;
1865                         goto error;
1866                 }
1867                 obj.rwq.in = tmpl->wq;
1868                 obj.rwq.out = &rwq;
1869                 ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_RWQ);
1870                 if (ret) {
1871                         rte_errno = ret;
1872                         goto error;
1873                 }
1874                 rxq_data->wqes = rwq.buf;
1875                 rxq_data->rq_db = rwq.dbrec;
1876         } else if (tmpl->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) {
1877                 struct mlx5_devx_modify_rq_attr rq_attr = { 0 };
1878                 struct mlx5_devx_dbr_page *dbr_page;
1879                 int64_t dbr_offset;
1880
1881                 /* Allocate CQ door-bell. */
1882                 dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs,
1883                                           &dbr_page);
1884                 if (dbr_offset < 0) {
1885                         DRV_LOG(ERR, "Failed to allocate CQ door-bell.");
1886                         goto error;
1887                 }
1888                 rxq_ctrl->cq_dbr_offset = dbr_offset;
1889                 rxq_ctrl->cq_dbr_umem_id = mlx5_os_get_umem_id(dbr_page->umem);
1890                 rxq_ctrl->cq_dbr_umem_id_valid = 1;
1891                 rxq_data->cq_db =
1892                         (uint32_t *)((uintptr_t)dbr_page->dbrs +
1893                                      (uintptr_t)rxq_ctrl->cq_dbr_offset);
1894                 rxq_data->cq_uar = priv->sh->devx_rx_uar->base_addr;
1895                 /* Create CQ using DevX API. */
1896                 tmpl->devx_cq = mlx5_devx_cq_new(dev, cqe_n, idx, tmpl);
1897                 if (!tmpl->devx_cq) {
1898                         DRV_LOG(ERR, "Failed to create CQ.");
1899                         goto error;
1900                 }
1901                 /* Allocate RQ door-bell. */
1902                 dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs,
1903                                           &dbr_page);
1904                 if (dbr_offset < 0) {
1905                         DRV_LOG(ERR, "Failed to allocate RQ door-bell.");
1906                         goto error;
1907                 }
1908                 rxq_ctrl->rq_dbr_offset = dbr_offset;
1909                 rxq_ctrl->rq_dbr_umem_id = mlx5_os_get_umem_id(dbr_page->umem);
1910                 rxq_ctrl->rq_dbr_umem_id_valid = 1;
1911                 rxq_data->rq_db =
1912                         (uint32_t *)((uintptr_t)dbr_page->dbrs +
1913                                      (uintptr_t)rxq_ctrl->rq_dbr_offset);
1914                 /* Create RQ using DevX API. */
1915                 tmpl->rq = mlx5_devx_rq_new(dev, idx, tmpl->devx_cq->id);
1916                 if (!tmpl->rq) {
1917                         DRV_LOG(ERR, "port %u Rx queue %u RQ creation failure",
1918                                 dev->data->port_id, idx);
1919                         rte_errno = ENOMEM;
1920                         goto error;
1921                 }
1922                 /* Change queue state to ready. */
1923                 rq_attr.rq_state = MLX5_RQC_STATE_RST;
1924                 rq_attr.state = MLX5_RQC_STATE_RDY;
1925                 ret = mlx5_devx_cmd_modify_rq(tmpl->rq, &rq_attr);
1926                 if (ret)
1927                         goto error;
1928         }
1929         rxq_data->cq_arm_sn = 0;
1930         mlx5_rxq_initialize(rxq_data);
1931         rxq_data->cq_ci = 0;
1932         DRV_LOG(DEBUG, "port %u rxq %u updated with %p", dev->data->port_id,
1933                 idx, (void *)&tmpl);
1934         rte_atomic32_inc(&tmpl->refcnt);
1935         LIST_INSERT_HEAD(&priv->rxqsobj, tmpl, next);
1936         priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
1937         dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;
1938         return tmpl;
1939 error:
1940         if (tmpl) {
1941                 ret = rte_errno; /* Save rte_errno before cleanup. */
1942                 if (tmpl->type == MLX5_RXQ_OBJ_TYPE_IBV) {
1943                         if (tmpl->wq)
1944                                 claim_zero(mlx5_glue->destroy_wq(tmpl->wq));
1945                         if (tmpl->ibv_cq)
1946                                 claim_zero(mlx5_glue->destroy_cq(tmpl->ibv_cq));
1947                         if (tmpl->ibv_channel)
1948                                 claim_zero(mlx5_glue->destroy_comp_channel
1949                                                         (tmpl->ibv_channel));
1950                 } else if (tmpl->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) {
1951                         if (tmpl->rq)
1952                                 claim_zero(mlx5_devx_cmd_destroy(tmpl->rq));
1953                         if (tmpl->devx_cq)
1954                                 claim_zero(mlx5_devx_cmd_destroy
1955                                                         (tmpl->devx_cq));
1956                         if (tmpl->devx_channel)
1957                                 mlx5_glue->devx_destroy_event_channel
1958                                                         (tmpl->devx_channel);
1959                 }
1960                 mlx5_free(tmpl);
1961                 rte_errno = ret; /* Restore rte_errno. */
1962         }
1963         if (type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) {
1964                 rxq_release_devx_rq_resources(rxq_ctrl);
1965                 rxq_release_devx_cq_resources(rxq_ctrl);
1966         }
1967         priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
1968         return NULL;
1969 }
1970
1971 /**
1972  * Verify the Rx queue objects list is empty
1973  *
1974  * @param dev
1975  *   Pointer to Ethernet device.
1976  *
1977  * @return
1978  *   The number of objects not released.
1979  */
1980 int
1981 mlx5_rxq_obj_verify(struct rte_eth_dev *dev)
1982 {
1983         struct mlx5_priv *priv = dev->data->dev_private;
1984         int ret = 0;
1985         struct mlx5_rxq_obj *rxq_obj;
1986
1987         LIST_FOREACH(rxq_obj, &priv->rxqsobj, next) {
1988                 DRV_LOG(DEBUG, "port %u Rx queue %u still referenced",
1989                         dev->data->port_id, rxq_obj->rxq_ctrl->rxq.idx);
1990                 ++ret;
1991         }
1992         return ret;
1993 }
1994
1995 /**
1996  * Callback function to initialize mbufs for Multi-Packet RQ.
1997  */
1998 static inline void
1999 mlx5_mprq_buf_init(struct rte_mempool *mp, void *opaque_arg,
2000                     void *_m, unsigned int i __rte_unused)
2001 {
2002         struct mlx5_mprq_buf *buf = _m;
2003         struct rte_mbuf_ext_shared_info *shinfo;
2004         unsigned int strd_n = (unsigned int)(uintptr_t)opaque_arg;
2005         unsigned int j;
2006
2007         memset(_m, 0, sizeof(*buf));
2008         buf->mp = mp;
2009         rte_atomic16_set(&buf->refcnt, 1);
2010         for (j = 0; j != strd_n; ++j) {
2011                 shinfo = &buf->shinfos[j];
2012                 shinfo->free_cb = mlx5_mprq_buf_free_cb;
2013                 shinfo->fcb_opaque = buf;
2014         }
2015 }
2016
2017 /**
2018  * Free mempool of Multi-Packet RQ.
2019  *
2020  * @param dev
2021  *   Pointer to Ethernet device.
2022  *
2023  * @return
2024  *   0 on success, negative errno value on failure.
2025  */
2026 int
2027 mlx5_mprq_free_mp(struct rte_eth_dev *dev)
2028 {
2029         struct mlx5_priv *priv = dev->data->dev_private;
2030         struct rte_mempool *mp = priv->mprq_mp;
2031         unsigned int i;
2032
2033         if (mp == NULL)
2034                 return 0;
2035         DRV_LOG(DEBUG, "port %u freeing mempool (%s) for Multi-Packet RQ",
2036                 dev->data->port_id, mp->name);
2037         /*
2038          * If a buffer in the pool has been externally attached to a mbuf and it
2039          * is still in use by application, destroying the Rx queue can spoil
2040          * the packet. It is unlikely to happen but if application dynamically
2041          * creates and destroys with holding Rx packets, this can happen.
2042          *
2043          * TODO: It is unavoidable for now because the mempool for Multi-Packet
2044          * RQ isn't provided by application but managed by PMD.
2045          */
2046         if (!rte_mempool_full(mp)) {
2047                 DRV_LOG(ERR,
2048                         "port %u mempool for Multi-Packet RQ is still in use",
2049                         dev->data->port_id);
2050                 rte_errno = EBUSY;
2051                 return -rte_errno;
2052         }
2053         rte_mempool_free(mp);
2054         /* Unset mempool for each Rx queue. */
2055         for (i = 0; i != priv->rxqs_n; ++i) {
2056                 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
2057
2058                 if (rxq == NULL)
2059                         continue;
2060                 rxq->mprq_mp = NULL;
2061         }
2062         priv->mprq_mp = NULL;
2063         return 0;
2064 }
2065
2066 /**
2067  * Allocate a mempool for Multi-Packet RQ. All configured Rx queues share the
2068  * mempool. If already allocated, reuse it if there're enough elements.
2069  * Otherwise, resize it.
2070  *
2071  * @param dev
2072  *   Pointer to Ethernet device.
2073  *
2074  * @return
2075  *   0 on success, negative errno value on failure.
2076  */
2077 int
2078 mlx5_mprq_alloc_mp(struct rte_eth_dev *dev)
2079 {
2080         struct mlx5_priv *priv = dev->data->dev_private;
2081         struct rte_mempool *mp = priv->mprq_mp;
2082         char name[RTE_MEMPOOL_NAMESIZE];
2083         unsigned int desc = 0;
2084         unsigned int buf_len;
2085         unsigned int obj_num;
2086         unsigned int obj_size;
2087         unsigned int strd_num_n = 0;
2088         unsigned int strd_sz_n = 0;
2089         unsigned int i;
2090         unsigned int n_ibv = 0;
2091
2092         if (!mlx5_mprq_enabled(dev))
2093                 return 0;
2094         /* Count the total number of descriptors configured. */
2095         for (i = 0; i != priv->rxqs_n; ++i) {
2096                 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
2097                 struct mlx5_rxq_ctrl *rxq_ctrl = container_of
2098                         (rxq, struct mlx5_rxq_ctrl, rxq);
2099
2100                 if (rxq == NULL || rxq_ctrl->type != MLX5_RXQ_TYPE_STANDARD)
2101                         continue;
2102                 n_ibv++;
2103                 desc += 1 << rxq->elts_n;
2104                 /* Get the max number of strides. */
2105                 if (strd_num_n < rxq->strd_num_n)
2106                         strd_num_n = rxq->strd_num_n;
2107                 /* Get the max size of a stride. */
2108                 if (strd_sz_n < rxq->strd_sz_n)
2109                         strd_sz_n = rxq->strd_sz_n;
2110         }
2111         MLX5_ASSERT(strd_num_n && strd_sz_n);
2112         buf_len = (1 << strd_num_n) * (1 << strd_sz_n);
2113         obj_size = sizeof(struct mlx5_mprq_buf) + buf_len + (1 << strd_num_n) *
2114                 sizeof(struct rte_mbuf_ext_shared_info) + RTE_PKTMBUF_HEADROOM;
2115         /*
2116          * Received packets can be either memcpy'd or externally referenced. In
2117          * case that the packet is attached to an mbuf as an external buffer, as
2118          * it isn't possible to predict how the buffers will be queued by
2119          * application, there's no option to exactly pre-allocate needed buffers
2120          * in advance but to speculatively prepares enough buffers.
2121          *
2122          * In the data path, if this Mempool is depleted, PMD will try to memcpy
2123          * received packets to buffers provided by application (rxq->mp) until
2124          * this Mempool gets available again.
2125          */
2126         desc *= 4;
2127         obj_num = desc + MLX5_MPRQ_MP_CACHE_SZ * n_ibv;
2128         /*
2129          * rte_mempool_create_empty() has sanity check to refuse large cache
2130          * size compared to the number of elements.
2131          * CACHE_FLUSHTHRESH_MULTIPLIER is defined in a C file, so using a
2132          * constant number 2 instead.
2133          */
2134         obj_num = RTE_MAX(obj_num, MLX5_MPRQ_MP_CACHE_SZ * 2);
2135         /* Check a mempool is already allocated and if it can be resued. */
2136         if (mp != NULL && mp->elt_size >= obj_size && mp->size >= obj_num) {
2137                 DRV_LOG(DEBUG, "port %u mempool %s is being reused",
2138                         dev->data->port_id, mp->name);
2139                 /* Reuse. */
2140                 goto exit;
2141         } else if (mp != NULL) {
2142                 DRV_LOG(DEBUG, "port %u mempool %s should be resized, freeing it",
2143                         dev->data->port_id, mp->name);
2144                 /*
2145                  * If failed to free, which means it may be still in use, no way
2146                  * but to keep using the existing one. On buffer underrun,
2147                  * packets will be memcpy'd instead of external buffer
2148                  * attachment.
2149                  */
2150                 if (mlx5_mprq_free_mp(dev)) {
2151                         if (mp->elt_size >= obj_size)
2152                                 goto exit;
2153                         else
2154                                 return -rte_errno;
2155                 }
2156         }
2157         snprintf(name, sizeof(name), "port-%u-mprq", dev->data->port_id);
2158         mp = rte_mempool_create(name, obj_num, obj_size, MLX5_MPRQ_MP_CACHE_SZ,
2159                                 0, NULL, NULL, mlx5_mprq_buf_init,
2160                                 (void *)(uintptr_t)(1 << strd_num_n),
2161                                 dev->device->numa_node, 0);
2162         if (mp == NULL) {
2163                 DRV_LOG(ERR,
2164                         "port %u failed to allocate a mempool for"
2165                         " Multi-Packet RQ, count=%u, size=%u",
2166                         dev->data->port_id, obj_num, obj_size);
2167                 rte_errno = ENOMEM;
2168                 return -rte_errno;
2169         }
2170         priv->mprq_mp = mp;
2171 exit:
2172         /* Set mempool for each Rx queue. */
2173         for (i = 0; i != priv->rxqs_n; ++i) {
2174                 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
2175                 struct mlx5_rxq_ctrl *rxq_ctrl = container_of
2176                         (rxq, struct mlx5_rxq_ctrl, rxq);
2177
2178                 if (rxq == NULL || rxq_ctrl->type != MLX5_RXQ_TYPE_STANDARD)
2179                         continue;
2180                 rxq->mprq_mp = mp;
2181         }
2182         DRV_LOG(INFO, "port %u Multi-Packet RQ is configured",
2183                 dev->data->port_id);
2184         return 0;
2185 }
2186
2187 #define MLX5_MAX_TCP_HDR_OFFSET ((unsigned int)(sizeof(struct rte_ether_hdr) + \
2188                                         sizeof(struct rte_vlan_hdr) * 2 + \
2189                                         sizeof(struct rte_ipv6_hdr)))
2190 #define MAX_TCP_OPTION_SIZE 40u
2191 #define MLX5_MAX_LRO_HEADER_FIX ((unsigned int)(MLX5_MAX_TCP_HDR_OFFSET + \
2192                                  sizeof(struct rte_tcp_hdr) + \
2193                                  MAX_TCP_OPTION_SIZE))
2194
2195 /**
2196  * Adjust the maximum LRO massage size.
2197  *
2198  * @param dev
2199  *   Pointer to Ethernet device.
2200  * @param idx
2201  *   RX queue index.
2202  * @param max_lro_size
2203  *   The maximum size for LRO packet.
2204  */
2205 static void
2206 mlx5_max_lro_msg_size_adjust(struct rte_eth_dev *dev, uint16_t idx,
2207                              uint32_t max_lro_size)
2208 {
2209         struct mlx5_priv *priv = dev->data->dev_private;
2210
2211         if (priv->config.hca_attr.lro_max_msg_sz_mode ==
2212             MLX5_LRO_MAX_MSG_SIZE_START_FROM_L4 && max_lro_size >
2213             MLX5_MAX_TCP_HDR_OFFSET)
2214                 max_lro_size -= MLX5_MAX_TCP_HDR_OFFSET;
2215         max_lro_size = RTE_MIN(max_lro_size, MLX5_MAX_LRO_SIZE);
2216         MLX5_ASSERT(max_lro_size >= MLX5_LRO_SEG_CHUNK_SIZE);
2217         max_lro_size /= MLX5_LRO_SEG_CHUNK_SIZE;
2218         if (priv->max_lro_msg_size)
2219                 priv->max_lro_msg_size =
2220                         RTE_MIN((uint32_t)priv->max_lro_msg_size, max_lro_size);
2221         else
2222                 priv->max_lro_msg_size = max_lro_size;
2223         DRV_LOG(DEBUG,
2224                 "port %u Rx Queue %u max LRO message size adjusted to %u bytes",
2225                 dev->data->port_id, idx,
2226                 priv->max_lro_msg_size * MLX5_LRO_SEG_CHUNK_SIZE);
2227 }
2228
2229 /**
2230  * Create a DPDK Rx queue.
2231  *
2232  * @param dev
2233  *   Pointer to Ethernet device.
2234  * @param idx
2235  *   RX queue index.
2236  * @param desc
2237  *   Number of descriptors to configure in queue.
2238  * @param socket
2239  *   NUMA socket on which memory must be allocated.
2240  *
2241  * @return
2242  *   A DPDK queue object on success, NULL otherwise and rte_errno is set.
2243  */
2244 struct mlx5_rxq_ctrl *
2245 mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
2246              unsigned int socket, const struct rte_eth_rxconf *conf,
2247              struct rte_mempool *mp)
2248 {
2249         struct mlx5_priv *priv = dev->data->dev_private;
2250         struct mlx5_rxq_ctrl *tmpl;
2251         unsigned int mb_len = rte_pktmbuf_data_room_size(mp);
2252         unsigned int mprq_stride_nums;
2253         unsigned int mprq_stride_size;
2254         unsigned int mprq_stride_cap;
2255         struct mlx5_dev_config *config = &priv->config;
2256         /*
2257          * Always allocate extra slots, even if eventually
2258          * the vector Rx will not be used.
2259          */
2260         uint16_t desc_n =
2261                 desc + config->rx_vec_en * MLX5_VPMD_DESCS_PER_LOOP;
2262         uint64_t offloads = conf->offloads |
2263                            dev->data->dev_conf.rxmode.offloads;
2264         unsigned int lro_on_queue = !!(offloads & DEV_RX_OFFLOAD_TCP_LRO);
2265         const int mprq_en = mlx5_check_mprq_support(dev) > 0;
2266         unsigned int max_rx_pkt_len = lro_on_queue ?
2267                         dev->data->dev_conf.rxmode.max_lro_pkt_size :
2268                         dev->data->dev_conf.rxmode.max_rx_pkt_len;
2269         unsigned int non_scatter_min_mbuf_size = max_rx_pkt_len +
2270                                                         RTE_PKTMBUF_HEADROOM;
2271         unsigned int max_lro_size = 0;
2272         unsigned int first_mb_free_size = mb_len - RTE_PKTMBUF_HEADROOM;
2273
2274         if (non_scatter_min_mbuf_size > mb_len && !(offloads &
2275                                                     DEV_RX_OFFLOAD_SCATTER)) {
2276                 DRV_LOG(ERR, "port %u Rx queue %u: Scatter offload is not"
2277                         " configured and no enough mbuf space(%u) to contain "
2278                         "the maximum RX packet length(%u) with head-room(%u)",
2279                         dev->data->port_id, idx, mb_len, max_rx_pkt_len,
2280                         RTE_PKTMBUF_HEADROOM);
2281                 rte_errno = ENOSPC;
2282                 return NULL;
2283         }
2284         tmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl) +
2285                            desc_n * sizeof(struct rte_mbuf *), 0, socket);
2286         if (!tmpl) {
2287                 rte_errno = ENOMEM;
2288                 return NULL;
2289         }
2290         tmpl->type = MLX5_RXQ_TYPE_STANDARD;
2291         if (mlx5_mr_btree_init(&tmpl->rxq.mr_ctrl.cache_bh,
2292                                MLX5_MR_BTREE_CACHE_N, socket)) {
2293                 /* rte_errno is already set. */
2294                 goto error;
2295         }
2296         tmpl->socket = socket;
2297         if (dev->data->dev_conf.intr_conf.rxq)
2298                 tmpl->irq = 1;
2299         mprq_stride_nums = config->mprq.stride_num_n ?
2300                 config->mprq.stride_num_n : MLX5_MPRQ_STRIDE_NUM_N;
2301         mprq_stride_size = non_scatter_min_mbuf_size <=
2302                 (1U << config->mprq.max_stride_size_n) ?
2303                 log2above(non_scatter_min_mbuf_size) : MLX5_MPRQ_STRIDE_SIZE_N;
2304         mprq_stride_cap = (config->mprq.stride_num_n ?
2305                 (1U << config->mprq.stride_num_n) : (1U << mprq_stride_nums)) *
2306                         (config->mprq.stride_size_n ?
2307                 (1U << config->mprq.stride_size_n) : (1U << mprq_stride_size));
2308         /*
2309          * This Rx queue can be configured as a Multi-Packet RQ if all of the
2310          * following conditions are met:
2311          *  - MPRQ is enabled.
2312          *  - The number of descs is more than the number of strides.
2313          *  - max_rx_pkt_len plus overhead is less than the max size
2314          *    of a stride or mprq_stride_size is specified by a user.
2315          *    Need to nake sure that there are enough stides to encap
2316          *    the maximum packet size in case mprq_stride_size is set.
2317          *  Otherwise, enable Rx scatter if necessary.
2318          */
2319         if (mprq_en && desc > (1U << mprq_stride_nums) &&
2320             (non_scatter_min_mbuf_size <=
2321              (1U << config->mprq.max_stride_size_n) ||
2322              (config->mprq.stride_size_n &&
2323               non_scatter_min_mbuf_size <= mprq_stride_cap))) {
2324                 /* TODO: Rx scatter isn't supported yet. */
2325                 tmpl->rxq.sges_n = 0;
2326                 /* Trim the number of descs needed. */
2327                 desc >>= mprq_stride_nums;
2328                 tmpl->rxq.strd_num_n = config->mprq.stride_num_n ?
2329                         config->mprq.stride_num_n : mprq_stride_nums;
2330                 tmpl->rxq.strd_sz_n = config->mprq.stride_size_n ?
2331                         config->mprq.stride_size_n : mprq_stride_size;
2332                 tmpl->rxq.strd_shift_en = MLX5_MPRQ_TWO_BYTE_SHIFT;
2333                 tmpl->rxq.strd_scatter_en =
2334                                 !!(offloads & DEV_RX_OFFLOAD_SCATTER);
2335                 tmpl->rxq.mprq_max_memcpy_len = RTE_MIN(first_mb_free_size,
2336                                 config->mprq.max_memcpy_len);
2337                 max_lro_size = RTE_MIN(max_rx_pkt_len,
2338                                        (1u << tmpl->rxq.strd_num_n) *
2339                                        (1u << tmpl->rxq.strd_sz_n));
2340                 DRV_LOG(DEBUG,
2341                         "port %u Rx queue %u: Multi-Packet RQ is enabled"
2342                         " strd_num_n = %u, strd_sz_n = %u",
2343                         dev->data->port_id, idx,
2344                         tmpl->rxq.strd_num_n, tmpl->rxq.strd_sz_n);
2345         } else if (max_rx_pkt_len <= first_mb_free_size) {
2346                 tmpl->rxq.sges_n = 0;
2347                 max_lro_size = max_rx_pkt_len;
2348         } else if (offloads & DEV_RX_OFFLOAD_SCATTER) {
2349                 unsigned int size = non_scatter_min_mbuf_size;
2350                 unsigned int sges_n;
2351
2352                 if (lro_on_queue && first_mb_free_size <
2353                     MLX5_MAX_LRO_HEADER_FIX) {
2354                         DRV_LOG(ERR, "Not enough space in the first segment(%u)"
2355                                 " to include the max header size(%u) for LRO",
2356                                 first_mb_free_size, MLX5_MAX_LRO_HEADER_FIX);
2357                         rte_errno = ENOTSUP;
2358                         goto error;
2359                 }
2360                 /*
2361                  * Determine the number of SGEs needed for a full packet
2362                  * and round it to the next power of two.
2363                  */
2364                 sges_n = log2above((size / mb_len) + !!(size % mb_len));
2365                 if (sges_n > MLX5_MAX_LOG_RQ_SEGS) {
2366                         DRV_LOG(ERR,
2367                                 "port %u too many SGEs (%u) needed to handle"
2368                                 " requested maximum packet size %u, the maximum"
2369                                 " supported are %u", dev->data->port_id,
2370                                 1 << sges_n, max_rx_pkt_len,
2371                                 1u << MLX5_MAX_LOG_RQ_SEGS);
2372                         rte_errno = ENOTSUP;
2373                         goto error;
2374                 }
2375                 tmpl->rxq.sges_n = sges_n;
2376                 max_lro_size = max_rx_pkt_len;
2377         }
2378         if (config->mprq.enabled && !mlx5_rxq_mprq_enabled(&tmpl->rxq))
2379                 DRV_LOG(WARNING,
2380                         "port %u MPRQ is requested but cannot be enabled\n"
2381                         " (requested: pkt_sz = %u, desc_num = %u,"
2382                         " rxq_num = %u, stride_sz = %u, stride_num = %u\n"
2383                         "  supported: min_rxqs_num = %u,"
2384                         " min_stride_sz = %u, max_stride_sz = %u).",
2385                         dev->data->port_id, non_scatter_min_mbuf_size,
2386                         desc, priv->rxqs_n,
2387                         config->mprq.stride_size_n ?
2388                                 (1U << config->mprq.stride_size_n) :
2389                                 (1U << mprq_stride_size),
2390                         config->mprq.stride_num_n ?
2391                                 (1U << config->mprq.stride_num_n) :
2392                                 (1U << mprq_stride_nums),
2393                         config->mprq.min_rxqs_num,
2394                         (1U << config->mprq.min_stride_size_n),
2395                         (1U << config->mprq.max_stride_size_n));
2396         DRV_LOG(DEBUG, "port %u maximum number of segments per packet: %u",
2397                 dev->data->port_id, 1 << tmpl->rxq.sges_n);
2398         if (desc % (1 << tmpl->rxq.sges_n)) {
2399                 DRV_LOG(ERR,
2400                         "port %u number of Rx queue descriptors (%u) is not a"
2401                         " multiple of SGEs per packet (%u)",
2402                         dev->data->port_id,
2403                         desc,
2404                         1 << tmpl->rxq.sges_n);
2405                 rte_errno = EINVAL;
2406                 goto error;
2407         }
2408         mlx5_max_lro_msg_size_adjust(dev, idx, max_lro_size);
2409         /* Toggle RX checksum offload if hardware supports it. */
2410         tmpl->rxq.csum = !!(offloads & DEV_RX_OFFLOAD_CHECKSUM);
2411         tmpl->rxq.hw_timestamp = !!(offloads & DEV_RX_OFFLOAD_TIMESTAMP);
2412         /* Configure VLAN stripping. */
2413         tmpl->rxq.vlan_strip = !!(offloads & DEV_RX_OFFLOAD_VLAN_STRIP);
2414         /* By default, FCS (CRC) is stripped by hardware. */
2415         tmpl->rxq.crc_present = 0;
2416         tmpl->rxq.lro = lro_on_queue;
2417         if (offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
2418                 if (config->hw_fcs_strip) {
2419                         /*
2420                          * RQs used for LRO-enabled TIRs should not be
2421                          * configured to scatter the FCS.
2422                          */
2423                         if (lro_on_queue)
2424                                 DRV_LOG(WARNING,
2425                                         "port %u CRC stripping has been "
2426                                         "disabled but will still be performed "
2427                                         "by hardware, because LRO is enabled",
2428                                         dev->data->port_id);
2429                         else
2430                                 tmpl->rxq.crc_present = 1;
2431                 } else {
2432                         DRV_LOG(WARNING,
2433                                 "port %u CRC stripping has been disabled but will"
2434                                 " still be performed by hardware, make sure MLNX_OFED"
2435                                 " and firmware are up to date",
2436                                 dev->data->port_id);
2437                 }
2438         }
2439         DRV_LOG(DEBUG,
2440                 "port %u CRC stripping is %s, %u bytes will be subtracted from"
2441                 " incoming frames to hide it",
2442                 dev->data->port_id,
2443                 tmpl->rxq.crc_present ? "disabled" : "enabled",
2444                 tmpl->rxq.crc_present << 2);
2445         /* Save port ID. */
2446         tmpl->rxq.rss_hash = !!priv->rss_conf.rss_hf &&
2447                 (!!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS));
2448         tmpl->rxq.port_id = dev->data->port_id;
2449         tmpl->priv = priv;
2450         tmpl->rxq.mp = mp;
2451         tmpl->rxq.elts_n = log2above(desc);
2452         tmpl->rxq.rq_repl_thresh =
2453                 MLX5_VPMD_RXQ_RPLNSH_THRESH(1 << tmpl->rxq.elts_n);
2454         tmpl->rxq.elts =
2455                 (struct rte_mbuf *(*)[1 << tmpl->rxq.elts_n])(tmpl + 1);
2456 #ifndef RTE_ARCH_64
2457         tmpl->rxq.uar_lock_cq = &priv->sh->uar_lock_cq;
2458 #endif
2459         tmpl->rxq.idx = idx;
2460         rte_atomic32_inc(&tmpl->refcnt);
2461         LIST_INSERT_HEAD(&priv->rxqsctrl, tmpl, next);
2462         return tmpl;
2463 error:
2464         mlx5_free(tmpl);
2465         return NULL;
2466 }
2467
2468 /**
2469  * Create a DPDK Rx hairpin queue.
2470  *
2471  * @param dev
2472  *   Pointer to Ethernet device.
2473  * @param idx
2474  *   RX queue index.
2475  * @param desc
2476  *   Number of descriptors to configure in queue.
2477  * @param hairpin_conf
2478  *   The hairpin binding configuration.
2479  *
2480  * @return
2481  *   A DPDK queue object on success, NULL otherwise and rte_errno is set.
2482  */
2483 struct mlx5_rxq_ctrl *
2484 mlx5_rxq_hairpin_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
2485                      const struct rte_eth_hairpin_conf *hairpin_conf)
2486 {
2487         struct mlx5_priv *priv = dev->data->dev_private;
2488         struct mlx5_rxq_ctrl *tmpl;
2489
2490         tmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl), 0,
2491                            SOCKET_ID_ANY);
2492         if (!tmpl) {
2493                 rte_errno = ENOMEM;
2494                 return NULL;
2495         }
2496         tmpl->type = MLX5_RXQ_TYPE_HAIRPIN;
2497         tmpl->socket = SOCKET_ID_ANY;
2498         tmpl->rxq.rss_hash = 0;
2499         tmpl->rxq.port_id = dev->data->port_id;
2500         tmpl->priv = priv;
2501         tmpl->rxq.mp = NULL;
2502         tmpl->rxq.elts_n = log2above(desc);
2503         tmpl->rxq.elts = NULL;
2504         tmpl->rxq.mr_ctrl.cache_bh = (struct mlx5_mr_btree) { 0 };
2505         tmpl->hairpin_conf = *hairpin_conf;
2506         tmpl->rxq.idx = idx;
2507         rte_atomic32_inc(&tmpl->refcnt);
2508         LIST_INSERT_HEAD(&priv->rxqsctrl, tmpl, next);
2509         return tmpl;
2510 }
2511
2512 /**
2513  * Get a Rx queue.
2514  *
2515  * @param dev
2516  *   Pointer to Ethernet device.
2517  * @param idx
2518  *   RX queue index.
2519  *
2520  * @return
2521  *   A pointer to the queue if it exists, NULL otherwise.
2522  */
2523 struct mlx5_rxq_ctrl *
2524 mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx)
2525 {
2526         struct mlx5_priv *priv = dev->data->dev_private;
2527         struct mlx5_rxq_ctrl *rxq_ctrl = NULL;
2528
2529         if ((*priv->rxqs)[idx]) {
2530                 rxq_ctrl = container_of((*priv->rxqs)[idx],
2531                                         struct mlx5_rxq_ctrl,
2532                                         rxq);
2533                 mlx5_rxq_obj_get(dev, idx);
2534                 rte_atomic32_inc(&rxq_ctrl->refcnt);
2535         }
2536         return rxq_ctrl;
2537 }
2538
2539 /**
2540  * Release a Rx queue.
2541  *
2542  * @param dev
2543  *   Pointer to Ethernet device.
2544  * @param idx
2545  *   RX queue index.
2546  *
2547  * @return
2548  *   1 while a reference on it exists, 0 when freed.
2549  */
2550 int
2551 mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx)
2552 {
2553         struct mlx5_priv *priv = dev->data->dev_private;
2554         struct mlx5_rxq_ctrl *rxq_ctrl;
2555
2556         if (!(*priv->rxqs)[idx])
2557                 return 0;
2558         rxq_ctrl = container_of((*priv->rxqs)[idx], struct mlx5_rxq_ctrl, rxq);
2559         MLX5_ASSERT(rxq_ctrl->priv);
2560         if (rxq_ctrl->obj && !mlx5_rxq_obj_release(rxq_ctrl->obj))
2561                 rxq_ctrl->obj = NULL;
2562         if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) {
2563                 if (rxq_ctrl->rq_dbr_umem_id_valid)
2564                         claim_zero(mlx5_release_dbr(&priv->dbrpgs,
2565                                                     rxq_ctrl->rq_dbr_umem_id,
2566                                                     rxq_ctrl->rq_dbr_offset));
2567                 if (rxq_ctrl->cq_dbr_umem_id_valid)
2568                         claim_zero(mlx5_release_dbr(&priv->dbrpgs,
2569                                                     rxq_ctrl->cq_dbr_umem_id,
2570                                                     rxq_ctrl->cq_dbr_offset));
2571                 if (rxq_ctrl->type == MLX5_RXQ_TYPE_STANDARD)
2572                         mlx5_mr_btree_free(&rxq_ctrl->rxq.mr_ctrl.cache_bh);
2573                 LIST_REMOVE(rxq_ctrl, next);
2574                 mlx5_free(rxq_ctrl);
2575                 (*priv->rxqs)[idx] = NULL;
2576                 return 0;
2577         }
2578         return 1;
2579 }
2580
2581 /**
2582  * Verify the Rx Queue list is empty
2583  *
2584  * @param dev
2585  *   Pointer to Ethernet device.
2586  *
2587  * @return
2588  *   The number of object not released.
2589  */
2590 int
2591 mlx5_rxq_verify(struct rte_eth_dev *dev)
2592 {
2593         struct mlx5_priv *priv = dev->data->dev_private;
2594         struct mlx5_rxq_ctrl *rxq_ctrl;
2595         int ret = 0;
2596
2597         LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) {
2598                 DRV_LOG(DEBUG, "port %u Rx Queue %u still referenced",
2599                         dev->data->port_id, rxq_ctrl->rxq.idx);
2600                 ++ret;
2601         }
2602         return ret;
2603 }
2604
2605 /**
2606  * Get a Rx queue type.
2607  *
2608  * @param dev
2609  *   Pointer to Ethernet device.
2610  * @param idx
2611  *   Rx queue index.
2612  *
2613  * @return
2614  *   The Rx queue type.
2615  */
2616 enum mlx5_rxq_type
2617 mlx5_rxq_get_type(struct rte_eth_dev *dev, uint16_t idx)
2618 {
2619         struct mlx5_priv *priv = dev->data->dev_private;
2620         struct mlx5_rxq_ctrl *rxq_ctrl = NULL;
2621
2622         if (idx < priv->rxqs_n && (*priv->rxqs)[idx]) {
2623                 rxq_ctrl = container_of((*priv->rxqs)[idx],
2624                                         struct mlx5_rxq_ctrl,
2625                                         rxq);
2626                 return rxq_ctrl->type;
2627         }
2628         return MLX5_RXQ_TYPE_UNDEFINED;
2629 }
2630
2631 /**
2632  * Create an indirection table.
2633  *
2634  * @param dev
2635  *   Pointer to Ethernet device.
2636  * @param queues
2637  *   Queues entering in the indirection table.
2638  * @param queues_n
2639  *   Number of queues in the array.
2640  *
2641  * @return
2642  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
2643  */
2644 static struct mlx5_ind_table_obj *
2645 mlx5_ind_table_obj_new(struct rte_eth_dev *dev, const uint16_t *queues,
2646                        uint32_t queues_n, enum mlx5_ind_tbl_type type)
2647 {
2648         struct mlx5_priv *priv = dev->data->dev_private;
2649         struct mlx5_ind_table_obj *ind_tbl;
2650         unsigned int i = 0, j = 0, k = 0;
2651
2652         ind_tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*ind_tbl) +
2653                               queues_n * sizeof(uint16_t), 0, SOCKET_ID_ANY);
2654         if (!ind_tbl) {
2655                 rte_errno = ENOMEM;
2656                 return NULL;
2657         }
2658         ind_tbl->type = type;
2659         if (ind_tbl->type == MLX5_IND_TBL_TYPE_IBV) {
2660                 const unsigned int wq_n = rte_is_power_of_2(queues_n) ?
2661                         log2above(queues_n) :
2662                         log2above(priv->config.ind_table_max_size);
2663                 struct ibv_wq *wq[1 << wq_n];
2664
2665                 for (i = 0; i != queues_n; ++i) {
2666                         struct mlx5_rxq_ctrl *rxq = mlx5_rxq_get(dev,
2667                                                                  queues[i]);
2668                         if (!rxq)
2669                                 goto error;
2670                         wq[i] = rxq->obj->wq;
2671                         ind_tbl->queues[i] = queues[i];
2672                 }
2673                 ind_tbl->queues_n = queues_n;
2674                 /* Finalise indirection table. */
2675                 k = i; /* Retain value of i for use in error case. */
2676                 for (j = 0; k != (unsigned int)(1 << wq_n); ++k, ++j)
2677                         wq[k] = wq[j];
2678                 ind_tbl->ind_table = mlx5_glue->create_rwq_ind_table
2679                         (priv->sh->ctx,
2680                          &(struct ibv_rwq_ind_table_init_attr){
2681                                 .log_ind_tbl_size = wq_n,
2682                                 .ind_tbl = wq,
2683                                 .comp_mask = 0,
2684                         });
2685                 if (!ind_tbl->ind_table) {
2686                         rte_errno = errno;
2687                         goto error;
2688                 }
2689         } else { /* ind_tbl->type == MLX5_IND_TBL_TYPE_DEVX */
2690                 struct mlx5_devx_rqt_attr *rqt_attr = NULL;
2691                 const unsigned int rqt_n =
2692                         1 << (rte_is_power_of_2(queues_n) ?
2693                               log2above(queues_n) :
2694                               log2above(priv->config.ind_table_max_size));
2695
2696                 rqt_attr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rqt_attr) +
2697                                       rqt_n * sizeof(uint32_t), 0,
2698                                       SOCKET_ID_ANY);
2699                 if (!rqt_attr) {
2700                         DRV_LOG(ERR, "port %u cannot allocate RQT resources",
2701                                 dev->data->port_id);
2702                         rte_errno = ENOMEM;
2703                         goto error;
2704                 }
2705                 rqt_attr->rqt_max_size = priv->config.ind_table_max_size;
2706                 rqt_attr->rqt_actual_size = rqt_n;
2707                 for (i = 0; i != queues_n; ++i) {
2708                         struct mlx5_rxq_ctrl *rxq = mlx5_rxq_get(dev,
2709                                                                  queues[i]);
2710                         if (!rxq)
2711                                 goto error;
2712                         rqt_attr->rq_list[i] = rxq->obj->rq->id;
2713                         ind_tbl->queues[i] = queues[i];
2714                 }
2715                 k = i; /* Retain value of i for use in error case. */
2716                 for (j = 0; k != rqt_n; ++k, ++j)
2717                         rqt_attr->rq_list[k] = rqt_attr->rq_list[j];
2718                 ind_tbl->rqt = mlx5_devx_cmd_create_rqt(priv->sh->ctx,
2719                                                         rqt_attr);
2720                 mlx5_free(rqt_attr);
2721                 if (!ind_tbl->rqt) {
2722                         DRV_LOG(ERR, "port %u cannot create DevX RQT",
2723                                 dev->data->port_id);
2724                         rte_errno = errno;
2725                         goto error;
2726                 }
2727                 ind_tbl->queues_n = queues_n;
2728         }
2729         rte_atomic32_inc(&ind_tbl->refcnt);
2730         LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next);
2731         return ind_tbl;
2732 error:
2733         for (j = 0; j < i; j++)
2734                 mlx5_rxq_release(dev, ind_tbl->queues[j]);
2735         mlx5_free(ind_tbl);
2736         DEBUG("port %u cannot create indirection table", dev->data->port_id);
2737         return NULL;
2738 }
2739
2740 /**
2741  * Get an indirection table.
2742  *
2743  * @param dev
2744  *   Pointer to Ethernet device.
2745  * @param queues
2746  *   Queues entering in the indirection table.
2747  * @param queues_n
2748  *   Number of queues in the array.
2749  *
2750  * @return
2751  *   An indirection table if found.
2752  */
2753 static struct mlx5_ind_table_obj *
2754 mlx5_ind_table_obj_get(struct rte_eth_dev *dev, const uint16_t *queues,
2755                        uint32_t queues_n)
2756 {
2757         struct mlx5_priv *priv = dev->data->dev_private;
2758         struct mlx5_ind_table_obj *ind_tbl;
2759
2760         LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
2761                 if ((ind_tbl->queues_n == queues_n) &&
2762                     (memcmp(ind_tbl->queues, queues,
2763                             ind_tbl->queues_n * sizeof(ind_tbl->queues[0]))
2764                      == 0))
2765                         break;
2766         }
2767         if (ind_tbl) {
2768                 unsigned int i;
2769
2770                 rte_atomic32_inc(&ind_tbl->refcnt);
2771                 for (i = 0; i != ind_tbl->queues_n; ++i)
2772                         mlx5_rxq_get(dev, ind_tbl->queues[i]);
2773         }
2774         return ind_tbl;
2775 }
2776
2777 /**
2778  * Release an indirection table.
2779  *
2780  * @param dev
2781  *   Pointer to Ethernet device.
2782  * @param ind_table
2783  *   Indirection table to release.
2784  *
2785  * @return
2786  *   1 while a reference on it exists, 0 when freed.
2787  */
2788 static int
2789 mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
2790                            struct mlx5_ind_table_obj *ind_tbl)
2791 {
2792         unsigned int i;
2793
2794         if (rte_atomic32_dec_and_test(&ind_tbl->refcnt)) {
2795                 if (ind_tbl->type == MLX5_IND_TBL_TYPE_IBV)
2796                         claim_zero(mlx5_glue->destroy_rwq_ind_table
2797                                                         (ind_tbl->ind_table));
2798                 else if (ind_tbl->type == MLX5_IND_TBL_TYPE_DEVX)
2799                         claim_zero(mlx5_devx_cmd_destroy(ind_tbl->rqt));
2800         }
2801         for (i = 0; i != ind_tbl->queues_n; ++i)
2802                 claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i]));
2803         if (!rte_atomic32_read(&ind_tbl->refcnt)) {
2804                 LIST_REMOVE(ind_tbl, next);
2805                 mlx5_free(ind_tbl);
2806                 return 0;
2807         }
2808         return 1;
2809 }
2810
2811 /**
2812  * Verify the Rx Queue list is empty
2813  *
2814  * @param dev
2815  *   Pointer to Ethernet device.
2816  *
2817  * @return
2818  *   The number of object not released.
2819  */
2820 int
2821 mlx5_ind_table_obj_verify(struct rte_eth_dev *dev)
2822 {
2823         struct mlx5_priv *priv = dev->data->dev_private;
2824         struct mlx5_ind_table_obj *ind_tbl;
2825         int ret = 0;
2826
2827         LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
2828                 DRV_LOG(DEBUG,
2829                         "port %u indirection table obj %p still referenced",
2830                         dev->data->port_id, (void *)ind_tbl);
2831                 ++ret;
2832         }
2833         return ret;
2834 }
2835
2836 /**
2837  * Create an Rx Hash queue.
2838  *
2839  * @param dev
2840  *   Pointer to Ethernet device.
2841  * @param rss_key
2842  *   RSS key for the Rx hash queue.
2843  * @param rss_key_len
2844  *   RSS key length.
2845  * @param hash_fields
2846  *   Verbs protocol hash field to make the RSS on.
2847  * @param queues
2848  *   Queues entering in hash queue. In case of empty hash_fields only the
2849  *   first queue index will be taken for the indirection table.
2850  * @param queues_n
2851  *   Number of queues.
2852  * @param tunnel
2853  *   Tunnel type.
2854  *
2855  * @return
2856  *   The Verbs/DevX object initialised index, 0 otherwise and rte_errno is set.
2857  */
2858 uint32_t
2859 mlx5_hrxq_new(struct rte_eth_dev *dev,
2860               const uint8_t *rss_key, uint32_t rss_key_len,
2861               uint64_t hash_fields,
2862               const uint16_t *queues, uint32_t queues_n,
2863               int tunnel __rte_unused)
2864 {
2865         struct mlx5_priv *priv = dev->data->dev_private;
2866         struct mlx5_hrxq *hrxq;
2867         uint32_t hrxq_idx = 0;
2868         struct ibv_qp *qp = NULL;
2869         struct mlx5_ind_table_obj *ind_tbl;
2870         int err;
2871         struct mlx5_devx_obj *tir = NULL;
2872         struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[queues[0]];
2873         struct mlx5_rxq_ctrl *rxq_ctrl =
2874                 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
2875
2876         queues_n = hash_fields ? queues_n : 1;
2877         ind_tbl = mlx5_ind_table_obj_get(dev, queues, queues_n);
2878         if (!ind_tbl) {
2879                 enum mlx5_ind_tbl_type type;
2880
2881                 type = rxq_ctrl->obj->type == MLX5_RXQ_OBJ_TYPE_IBV ?
2882                                 MLX5_IND_TBL_TYPE_IBV : MLX5_IND_TBL_TYPE_DEVX;
2883                 ind_tbl = mlx5_ind_table_obj_new(dev, queues, queues_n, type);
2884         }
2885         if (!ind_tbl) {
2886                 rte_errno = ENOMEM;
2887                 return 0;
2888         }
2889         if (ind_tbl->type == MLX5_IND_TBL_TYPE_IBV) {
2890 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
2891                 struct mlx5dv_qp_init_attr qp_init_attr;
2892
2893                 memset(&qp_init_attr, 0, sizeof(qp_init_attr));
2894                 if (tunnel) {
2895                         qp_init_attr.comp_mask =
2896                                 MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
2897                         qp_init_attr.create_flags =
2898                                 MLX5DV_QP_CREATE_TUNNEL_OFFLOADS;
2899                 }
2900 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
2901                 if (dev->data->dev_conf.lpbk_mode) {
2902                         /*
2903                          * Allow packet sent from NIC loop back
2904                          * w/o source MAC check.
2905                          */
2906                         qp_init_attr.comp_mask |=
2907                                 MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
2908                         qp_init_attr.create_flags |=
2909                                 MLX5DV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_UC;
2910                 }
2911 #endif
2912                 qp = mlx5_glue->dv_create_qp
2913                         (priv->sh->ctx,
2914                          &(struct ibv_qp_init_attr_ex){
2915                                 .qp_type = IBV_QPT_RAW_PACKET,
2916                                 .comp_mask =
2917                                         IBV_QP_INIT_ATTR_PD |
2918                                         IBV_QP_INIT_ATTR_IND_TABLE |
2919                                         IBV_QP_INIT_ATTR_RX_HASH,
2920                                 .rx_hash_conf = (struct ibv_rx_hash_conf){
2921                                         .rx_hash_function =
2922                                                 IBV_RX_HASH_FUNC_TOEPLITZ,
2923                                         .rx_hash_key_len = rss_key_len,
2924                                         .rx_hash_key =
2925                                                 (void *)(uintptr_t)rss_key,
2926                                         .rx_hash_fields_mask = hash_fields,
2927                                 },
2928                                 .rwq_ind_tbl = ind_tbl->ind_table,
2929                                 .pd = priv->sh->pd,
2930                           },
2931                           &qp_init_attr);
2932 #else
2933                 qp = mlx5_glue->create_qp_ex
2934                         (priv->sh->ctx,
2935                          &(struct ibv_qp_init_attr_ex){
2936                                 .qp_type = IBV_QPT_RAW_PACKET,
2937                                 .comp_mask =
2938                                         IBV_QP_INIT_ATTR_PD |
2939                                         IBV_QP_INIT_ATTR_IND_TABLE |
2940                                         IBV_QP_INIT_ATTR_RX_HASH,
2941                                 .rx_hash_conf = (struct ibv_rx_hash_conf){
2942                                         .rx_hash_function =
2943                                                 IBV_RX_HASH_FUNC_TOEPLITZ,
2944                                         .rx_hash_key_len = rss_key_len,
2945                                         .rx_hash_key =
2946                                                 (void *)(uintptr_t)rss_key,
2947                                         .rx_hash_fields_mask = hash_fields,
2948                                 },
2949                                 .rwq_ind_tbl = ind_tbl->ind_table,
2950                                 .pd = priv->sh->pd,
2951                          });
2952 #endif
2953                 if (!qp) {
2954                         rte_errno = errno;
2955                         goto error;
2956                 }
2957         } else { /* ind_tbl->type == MLX5_IND_TBL_TYPE_DEVX */
2958                 struct mlx5_devx_tir_attr tir_attr;
2959                 uint32_t i;
2960                 uint32_t lro = 1;
2961
2962                 /* Enable TIR LRO only if all the queues were configured for. */
2963                 for (i = 0; i < queues_n; ++i) {
2964                         if (!(*priv->rxqs)[queues[i]]->lro) {
2965                                 lro = 0;
2966                                 break;
2967                         }
2968                 }
2969                 memset(&tir_attr, 0, sizeof(tir_attr));
2970                 tir_attr.disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT;
2971                 tir_attr.rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ;
2972                 tir_attr.tunneled_offload_en = !!tunnel;
2973                 /* If needed, translate hash_fields bitmap to PRM format. */
2974                 if (hash_fields) {
2975 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
2976                         struct mlx5_rx_hash_field_select *rx_hash_field_select =
2977                                         hash_fields & IBV_RX_HASH_INNER ?
2978                                         &tir_attr.rx_hash_field_selector_inner :
2979                                         &tir_attr.rx_hash_field_selector_outer;
2980 #else
2981                         struct mlx5_rx_hash_field_select *rx_hash_field_select =
2982                                         &tir_attr.rx_hash_field_selector_outer;
2983 #endif
2984
2985                         /* 1 bit: 0: IPv4, 1: IPv6. */
2986                         rx_hash_field_select->l3_prot_type =
2987                                 !!(hash_fields & MLX5_IPV6_IBV_RX_HASH);
2988                         /* 1 bit: 0: TCP, 1: UDP. */
2989                         rx_hash_field_select->l4_prot_type =
2990                                 !!(hash_fields & MLX5_UDP_IBV_RX_HASH);
2991                         /* Bitmask which sets which fields to use in RX Hash. */
2992                         rx_hash_field_select->selected_fields =
2993                         ((!!(hash_fields & MLX5_L3_SRC_IBV_RX_HASH)) <<
2994                          MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) |
2995                         (!!(hash_fields & MLX5_L3_DST_IBV_RX_HASH)) <<
2996                          MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP |
2997                         (!!(hash_fields & MLX5_L4_SRC_IBV_RX_HASH)) <<
2998                          MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT |
2999                         (!!(hash_fields & MLX5_L4_DST_IBV_RX_HASH)) <<
3000                          MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT;
3001                 }
3002                 if (rxq_ctrl->obj->type == MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN)
3003                         tir_attr.transport_domain = priv->sh->td->id;
3004                 else
3005                         tir_attr.transport_domain = priv->sh->tdn;
3006                 memcpy(tir_attr.rx_hash_toeplitz_key, rss_key,
3007                        MLX5_RSS_HASH_KEY_LEN);
3008                 tir_attr.indirect_table = ind_tbl->rqt->id;
3009                 if (dev->data->dev_conf.lpbk_mode)
3010                         tir_attr.self_lb_block =
3011                                         MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
3012                 if (lro) {
3013                         tir_attr.lro_timeout_period_usecs =
3014                                         priv->config.lro.timeout;
3015                         tir_attr.lro_max_msg_sz = priv->max_lro_msg_size;
3016                         tir_attr.lro_enable_mask =
3017                                         MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
3018                                         MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO;
3019                 }
3020                 tir = mlx5_devx_cmd_create_tir(priv->sh->ctx, &tir_attr);
3021                 if (!tir) {
3022                         DRV_LOG(ERR, "port %u cannot create DevX TIR",
3023                                 dev->data->port_id);
3024                         rte_errno = errno;
3025                         goto error;
3026                 }
3027         }
3028         hrxq = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_HRXQ], &hrxq_idx);
3029         if (!hrxq)
3030                 goto error;
3031         hrxq->ind_table = ind_tbl;
3032         if (ind_tbl->type == MLX5_IND_TBL_TYPE_IBV) {
3033                 hrxq->qp = qp;
3034 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
3035                 hrxq->action =
3036                         mlx5_glue->dv_create_flow_action_dest_ibv_qp(hrxq->qp);
3037                 if (!hrxq->action) {
3038                         rte_errno = errno;
3039                         goto error;
3040                 }
3041 #endif
3042         } else { /* ind_tbl->type == MLX5_IND_TBL_TYPE_DEVX */
3043                 hrxq->tir = tir;
3044 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
3045                 hrxq->action = mlx5_glue->dv_create_flow_action_dest_devx_tir
3046                                                         (hrxq->tir->obj);
3047                 if (!hrxq->action) {
3048                         rte_errno = errno;
3049                         goto error;
3050                 }
3051 #endif
3052         }
3053         hrxq->rss_key_len = rss_key_len;
3054         hrxq->hash_fields = hash_fields;
3055         memcpy(hrxq->rss_key, rss_key, rss_key_len);
3056         rte_atomic32_inc(&hrxq->refcnt);
3057         ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_HRXQ], &priv->hrxqs, hrxq_idx,
3058                      hrxq, next);
3059         return hrxq_idx;
3060 error:
3061         err = rte_errno; /* Save rte_errno before cleanup. */
3062         mlx5_ind_table_obj_release(dev, ind_tbl);
3063         if (qp)
3064                 claim_zero(mlx5_glue->destroy_qp(qp));
3065         else if (tir)
3066                 claim_zero(mlx5_devx_cmd_destroy(tir));
3067         rte_errno = err; /* Restore rte_errno. */
3068         return 0;
3069 }
3070
3071 /**
3072  * Get an Rx Hash queue.
3073  *
3074  * @param dev
3075  *   Pointer to Ethernet device.
3076  * @param rss_conf
3077  *   RSS configuration for the Rx hash queue.
3078  * @param queues
3079  *   Queues entering in hash queue. In case of empty hash_fields only the
3080  *   first queue index will be taken for the indirection table.
3081  * @param queues_n
3082  *   Number of queues.
3083  *
3084  * @return
3085  *   An hash Rx queue index on success.
3086  */
3087 uint32_t
3088 mlx5_hrxq_get(struct rte_eth_dev *dev,
3089               const uint8_t *rss_key, uint32_t rss_key_len,
3090               uint64_t hash_fields,
3091               const uint16_t *queues, uint32_t queues_n)
3092 {
3093         struct mlx5_priv *priv = dev->data->dev_private;
3094         struct mlx5_hrxq *hrxq;
3095         uint32_t idx;
3096
3097         queues_n = hash_fields ? queues_n : 1;
3098         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_HRXQ], priv->hrxqs, idx,
3099                       hrxq, next) {
3100                 struct mlx5_ind_table_obj *ind_tbl;
3101
3102                 if (hrxq->rss_key_len != rss_key_len)
3103                         continue;
3104                 if (memcmp(hrxq->rss_key, rss_key, rss_key_len))
3105                         continue;
3106                 if (hrxq->hash_fields != hash_fields)
3107                         continue;
3108                 ind_tbl = mlx5_ind_table_obj_get(dev, queues, queues_n);
3109                 if (!ind_tbl)
3110                         continue;
3111                 if (ind_tbl != hrxq->ind_table) {
3112                         mlx5_ind_table_obj_release(dev, ind_tbl);
3113                         continue;
3114                 }
3115                 rte_atomic32_inc(&hrxq->refcnt);
3116                 return idx;
3117         }
3118         return 0;
3119 }
3120
3121 /**
3122  * Release the hash Rx queue.
3123  *
3124  * @param dev
3125  *   Pointer to Ethernet device.
3126  * @param hrxq
3127  *   Index to Hash Rx queue to release.
3128  *
3129  * @return
3130  *   1 while a reference on it exists, 0 when freed.
3131  */
3132 int
3133 mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
3134 {
3135         struct mlx5_priv *priv = dev->data->dev_private;
3136         struct mlx5_hrxq *hrxq;
3137
3138         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx);
3139         if (!hrxq)
3140                 return 0;
3141         if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
3142 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
3143                 mlx5_glue->destroy_flow_action(hrxq->action);
3144 #endif
3145                 if (hrxq->ind_table->type == MLX5_IND_TBL_TYPE_IBV)
3146                         claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
3147                 else /* hrxq->ind_table->type == MLX5_IND_TBL_TYPE_DEVX */
3148                         claim_zero(mlx5_devx_cmd_destroy(hrxq->tir));
3149                 mlx5_ind_table_obj_release(dev, hrxq->ind_table);
3150                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_HRXQ], &priv->hrxqs,
3151                              hrxq_idx, hrxq, next);
3152                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx);
3153                 return 0;
3154         }
3155         claim_nonzero(mlx5_ind_table_obj_release(dev, hrxq->ind_table));
3156         return 1;
3157 }
3158
3159 /**
3160  * Verify the Rx Queue list is empty
3161  *
3162  * @param dev
3163  *   Pointer to Ethernet device.
3164  *
3165  * @return
3166  *   The number of object not released.
3167  */
3168 int
3169 mlx5_hrxq_verify(struct rte_eth_dev *dev)
3170 {
3171         struct mlx5_priv *priv = dev->data->dev_private;
3172         struct mlx5_hrxq *hrxq;
3173         uint32_t idx;
3174         int ret = 0;
3175
3176         ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_HRXQ], priv->hrxqs, idx,
3177                       hrxq, next) {
3178                 DRV_LOG(DEBUG,
3179                         "port %u hash Rx queue %p still referenced",
3180                         dev->data->port_id, (void *)hrxq);
3181                 ++ret;
3182         }
3183         return ret;
3184 }
3185
3186 /**
3187  * Create a drop Rx queue Verbs/DevX object.
3188  *
3189  * @param dev
3190  *   Pointer to Ethernet device.
3191  *
3192  * @return
3193  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
3194  */
3195 static struct mlx5_rxq_obj *
3196 mlx5_rxq_obj_drop_new(struct rte_eth_dev *dev)
3197 {
3198         struct mlx5_priv *priv = dev->data->dev_private;
3199         struct ibv_context *ctx = priv->sh->ctx;
3200         struct ibv_cq *cq;
3201         struct ibv_wq *wq = NULL;
3202         struct mlx5_rxq_obj *rxq;
3203
3204         if (priv->drop_queue.rxq)
3205                 return priv->drop_queue.rxq;
3206         cq = mlx5_glue->create_cq(ctx, 1, NULL, NULL, 0);
3207         if (!cq) {
3208                 DEBUG("port %u cannot allocate CQ for drop queue",
3209                       dev->data->port_id);
3210                 rte_errno = errno;
3211                 goto error;
3212         }
3213         wq = mlx5_glue->create_wq(ctx,
3214                  &(struct ibv_wq_init_attr){
3215                         .wq_type = IBV_WQT_RQ,
3216                         .max_wr = 1,
3217                         .max_sge = 1,
3218                         .pd = priv->sh->pd,
3219                         .cq = cq,
3220                  });
3221         if (!wq) {
3222                 DEBUG("port %u cannot allocate WQ for drop queue",
3223                       dev->data->port_id);
3224                 rte_errno = errno;
3225                 goto error;
3226         }
3227         rxq = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq), 0, SOCKET_ID_ANY);
3228         if (!rxq) {
3229                 DEBUG("port %u cannot allocate drop Rx queue memory",
3230                       dev->data->port_id);
3231                 rte_errno = ENOMEM;
3232                 goto error;
3233         }
3234         rxq->ibv_cq = cq;
3235         rxq->wq = wq;
3236         priv->drop_queue.rxq = rxq;
3237         return rxq;
3238 error:
3239         if (wq)
3240                 claim_zero(mlx5_glue->destroy_wq(wq));
3241         if (cq)
3242                 claim_zero(mlx5_glue->destroy_cq(cq));
3243         return NULL;
3244 }
3245
3246 /**
3247  * Release a drop Rx queue Verbs/DevX object.
3248  *
3249  * @param dev
3250  *   Pointer to Ethernet device.
3251  *
3252  * @return
3253  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
3254  */
3255 static void
3256 mlx5_rxq_obj_drop_release(struct rte_eth_dev *dev)
3257 {
3258         struct mlx5_priv *priv = dev->data->dev_private;
3259         struct mlx5_rxq_obj *rxq = priv->drop_queue.rxq;
3260
3261         if (rxq->wq)
3262                 claim_zero(mlx5_glue->destroy_wq(rxq->wq));
3263         if (rxq->ibv_cq)
3264                 claim_zero(mlx5_glue->destroy_cq(rxq->ibv_cq));
3265         mlx5_free(rxq);
3266         priv->drop_queue.rxq = NULL;
3267 }
3268
3269 /**
3270  * Create a drop indirection table.
3271  *
3272  * @param dev
3273  *   Pointer to Ethernet device.
3274  *
3275  * @return
3276  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
3277  */
3278 static struct mlx5_ind_table_obj *
3279 mlx5_ind_table_obj_drop_new(struct rte_eth_dev *dev)
3280 {
3281         struct mlx5_priv *priv = dev->data->dev_private;
3282         struct mlx5_ind_table_obj *ind_tbl;
3283         struct mlx5_rxq_obj *rxq;
3284         struct mlx5_ind_table_obj tmpl;
3285
3286         rxq = mlx5_rxq_obj_drop_new(dev);
3287         if (!rxq)
3288                 return NULL;
3289         tmpl.ind_table = mlx5_glue->create_rwq_ind_table
3290                 (priv->sh->ctx,
3291                  &(struct ibv_rwq_ind_table_init_attr){
3292                         .log_ind_tbl_size = 0,
3293                         .ind_tbl = &rxq->wq,
3294                         .comp_mask = 0,
3295                  });
3296         if (!tmpl.ind_table) {
3297                 DEBUG("port %u cannot allocate indirection table for drop"
3298                       " queue",
3299                       dev->data->port_id);
3300                 rte_errno = errno;
3301                 goto error;
3302         }
3303         ind_tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*ind_tbl), 0,
3304                               SOCKET_ID_ANY);
3305         if (!ind_tbl) {
3306                 rte_errno = ENOMEM;
3307                 goto error;
3308         }
3309         ind_tbl->ind_table = tmpl.ind_table;
3310         return ind_tbl;
3311 error:
3312         mlx5_rxq_obj_drop_release(dev);
3313         return NULL;
3314 }
3315
3316 /**
3317  * Release a drop indirection table.
3318  *
3319  * @param dev
3320  *   Pointer to Ethernet device.
3321  */
3322 static void
3323 mlx5_ind_table_obj_drop_release(struct rte_eth_dev *dev)
3324 {
3325         struct mlx5_priv *priv = dev->data->dev_private;
3326         struct mlx5_ind_table_obj *ind_tbl = priv->drop_queue.hrxq->ind_table;
3327
3328         claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl->ind_table));
3329         mlx5_rxq_obj_drop_release(dev);
3330         mlx5_free(ind_tbl);
3331         priv->drop_queue.hrxq->ind_table = NULL;
3332 }
3333
3334 /**
3335  * Create a drop Rx Hash queue.
3336  *
3337  * @param dev
3338  *   Pointer to Ethernet device.
3339  *
3340  * @return
3341  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
3342  */
3343 struct mlx5_hrxq *
3344 mlx5_hrxq_drop_new(struct rte_eth_dev *dev)
3345 {
3346         struct mlx5_priv *priv = dev->data->dev_private;
3347         struct mlx5_ind_table_obj *ind_tbl = NULL;
3348         struct ibv_qp *qp = NULL;
3349         struct mlx5_hrxq *hrxq = NULL;
3350
3351         if (priv->drop_queue.hrxq) {
3352                 rte_atomic32_inc(&priv->drop_queue.hrxq->refcnt);
3353                 return priv->drop_queue.hrxq;
3354         }
3355         hrxq = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*hrxq), 0, SOCKET_ID_ANY);
3356         if (!hrxq) {
3357                 DRV_LOG(WARNING,
3358                         "port %u cannot allocate memory for drop queue",
3359                         dev->data->port_id);
3360                 rte_errno = ENOMEM;
3361                 goto error;
3362         }
3363         priv->drop_queue.hrxq = hrxq;
3364         ind_tbl = mlx5_ind_table_obj_drop_new(dev);
3365         if (!ind_tbl)
3366                 goto error;
3367         hrxq->ind_table = ind_tbl;
3368         qp = mlx5_glue->create_qp_ex(priv->sh->ctx,
3369                  &(struct ibv_qp_init_attr_ex){
3370                         .qp_type = IBV_QPT_RAW_PACKET,
3371                         .comp_mask =
3372                                 IBV_QP_INIT_ATTR_PD |
3373                                 IBV_QP_INIT_ATTR_IND_TABLE |
3374                                 IBV_QP_INIT_ATTR_RX_HASH,
3375                         .rx_hash_conf = (struct ibv_rx_hash_conf){
3376                                 .rx_hash_function =
3377                                         IBV_RX_HASH_FUNC_TOEPLITZ,
3378                                 .rx_hash_key_len = MLX5_RSS_HASH_KEY_LEN,
3379                                 .rx_hash_key = rss_hash_default_key,
3380                                 .rx_hash_fields_mask = 0,
3381                                 },
3382                         .rwq_ind_tbl = ind_tbl->ind_table,
3383                         .pd = priv->sh->pd
3384                  });
3385         if (!qp) {
3386                 DEBUG("port %u cannot allocate QP for drop queue",
3387                       dev->data->port_id);
3388                 rte_errno = errno;
3389                 goto error;
3390         }
3391         hrxq->qp = qp;
3392 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
3393         hrxq->action = mlx5_glue->dv_create_flow_action_dest_ibv_qp(hrxq->qp);
3394         if (!hrxq->action) {
3395                 rte_errno = errno;
3396                 goto error;
3397         }
3398 #endif
3399         rte_atomic32_set(&hrxq->refcnt, 1);
3400         return hrxq;
3401 error:
3402 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
3403         if (hrxq && hrxq->action)
3404                 mlx5_glue->destroy_flow_action(hrxq->action);
3405 #endif
3406         if (qp)
3407                 claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
3408         if (ind_tbl)
3409                 mlx5_ind_table_obj_drop_release(dev);
3410         if (hrxq) {
3411                 priv->drop_queue.hrxq = NULL;
3412                 mlx5_free(hrxq);
3413         }
3414         return NULL;
3415 }
3416
3417 /**
3418  * Release a drop hash Rx queue.
3419  *
3420  * @param dev
3421  *   Pointer to Ethernet device.
3422  */
3423 void
3424 mlx5_hrxq_drop_release(struct rte_eth_dev *dev)
3425 {
3426         struct mlx5_priv *priv = dev->data->dev_private;
3427         struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq;
3428
3429         if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
3430 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
3431                 mlx5_glue->destroy_flow_action(hrxq->action);
3432 #endif
3433                 claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
3434                 mlx5_ind_table_obj_drop_release(dev);
3435                 mlx5_free(hrxq);
3436                 priv->drop_queue.hrxq = NULL;
3437         }
3438 }
3439
3440
3441 /**
3442  * Set the Rx queue timestamp conversion parameters
3443  *
3444  * @param[in] dev
3445  *   Pointer to the Ethernet device structure.
3446  */
3447 void
3448 mlx5_rxq_timestamp_set(struct rte_eth_dev *dev)
3449 {
3450         struct mlx5_priv *priv = dev->data->dev_private;
3451         struct mlx5_dev_ctx_shared *sh = priv->sh;
3452         struct mlx5_rxq_data *data;
3453         unsigned int i;
3454
3455         for (i = 0; i != priv->rxqs_n; ++i) {
3456                 if (!(*priv->rxqs)[i])
3457                         continue;
3458                 data = (*priv->rxqs)[i];
3459                 data->sh = sh;
3460                 data->rt_timestamp = priv->config.rt_timestamp;
3461         }
3462 }