drivers/net/mlx5/linux/mlx5_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <stddef.h>
   6 #include <errno.h>
   7 #include <string.h>
   8 #include <stdint.h>
   9 #include <unistd.h>
  10 #include <inttypes.h>
  11 #include <sys/queue.h>
  12
  13 #include "mlx5_autoconf.h"
  14
  15 #include <rte_mbuf.h>
  16 #include <rte_malloc.h>
  17 #include <ethdev_driver.h>
  18 #include <rte_common.h>
  19
  20 #include <mlx5_glue.h>
  21 #include <mlx5_common.h>
  22 #include <mlx5_common_mr.h>
  23 #include <mlx5_verbs.h>
  24 #include <mlx5_rx.h>
  25 #include <mlx5_tx.h>
  26 #include <mlx5_utils.h>
  27 #include <mlx5_malloc.h>
  28
  29 /**
  30  * Register mr. Given protection domain pointer, pointer to addr and length
  31  * register the memory region.
  32  *
  33  * @param[in] pd
  34  *   Pointer to protection domain context.
  35  * @param[in] addr
  36  *   Pointer to memory start address.
  37  * @param[in] length
  38  *   Length of the memory to register.
  39  * @param[out] pmd_mr
  40  *   pmd_mr struct set with lkey, address, length and pointer to mr object
  41  *
  42  * @return
  43  *   0 on successful registration, -1 otherwise
  44  */
  45 static int
  46 mlx5_reg_mr(void *pd, void *addr, size_t length,
  47                  struct mlx5_pmd_mr *pmd_mr)
  48 {
  49         return mlx5_common_verbs_reg_mr(pd, addr, length, pmd_mr);
  50 }
  51
  52 /**
  53  * Deregister mr. Given the mlx5 pmd MR - deregister the MR
  54  *
  55  * @param[in] pmd_mr
  56  *   pmd_mr struct set with lkey, address, length and pointer to mr object
  57  *
  58  */
  59 static void
  60 mlx5_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
  61 {
  62         mlx5_common_verbs_dereg_mr(pmd_mr);
  63 }
  64
  65 /* verbs operations. */
  66 const struct mlx5_mr_ops mlx5_mr_verbs_ops = {
  67         .reg_mr = mlx5_reg_mr,
  68         .dereg_mr = mlx5_dereg_mr,
  69 };
  70
  71 /**
  72  * Modify Rx WQ vlan stripping offload
  73  *
  74  * @param rxq_obj
  75  *   Rx queue object.
  76  *
  77  * @return 0 on success, non-0 otherwise
  78  */
  79 static int
  80 mlx5_rxq_obj_modify_wq_vlan_strip(struct mlx5_rxq_obj *rxq_obj, int on)
  81 {
  82         uint16_t vlan_offloads =
  83                 (on ? IBV_WQ_FLAGS_CVLAN_STRIPPING : 0) |
  84                 0;
  85         struct ibv_wq_attr mod;
  86         mod = (struct ibv_wq_attr){
  87                 .attr_mask = IBV_WQ_ATTR_FLAGS,
  88                 .flags_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING,
  89                 .flags = vlan_offloads,
  90         };
  91
  92         return mlx5_glue->modify_wq(rxq_obj->wq, &mod);
  93 }
  94
  95 /**
  96  * Modifies the attributes for the specified WQ.
  97  *
  98  * @param rxq_obj
  99  *   Verbs Rx queue object.
 100  * @param type
 101  *   Type of change queue state.
 102  *
 103  * @return
 104  *   0 on success, a negative errno value otherwise and rte_errno is set.
 105  */
 106 static int
 107 mlx5_ibv_modify_wq(struct mlx5_rxq_obj *rxq_obj, uint8_t type)
 108 {
 109         struct ibv_wq_attr mod = {
 110                 .attr_mask = IBV_WQ_ATTR_STATE,
 111                 .wq_state = (enum ibv_wq_state)type,
 112         };
 113
 114         return mlx5_glue->modify_wq(rxq_obj->wq, &mod);
 115 }
 116
 117 /**
 118  * Modify QP using Verbs API.
 119  *
 120  * @param txq_obj
 121  *   Verbs Tx queue object.
 122  * @param type
 123  *   Type of change queue state.
 124  * @param dev_port
 125  *   IB device port number.
 126  *
 127  * @return
 128  *   0 on success, a negative errno value otherwise and rte_errno is set.
 129  */
 130 static int
 131 mlx5_ibv_modify_qp(struct mlx5_txq_obj *obj, enum mlx5_txq_modify_type type,
 132                    uint8_t dev_port)
 133 {
 134         struct ibv_qp_attr mod = {
 135                 .qp_state = IBV_QPS_RESET,
 136                 .port_num = dev_port,
 137         };
 138         int attr_mask = (IBV_QP_STATE | IBV_QP_PORT);
 139         int ret;
 140
 141         if (type != MLX5_TXQ_MOD_RST2RDY) {
 142                 ret = mlx5_glue->modify_qp(obj->qp, &mod, IBV_QP_STATE);
 143                 if (ret) {
 144                         DRV_LOG(ERR, "Cannot change Tx QP state to RESET %s",
 145                                 strerror(errno));
 146                         rte_errno = errno;
 147                         return ret;
 148                 }
 149                 if (type == MLX5_TXQ_MOD_RDY2RST)
 150                         return 0;
 151         }
 152         if (type == MLX5_TXQ_MOD_ERR2RDY)
 153                 attr_mask = IBV_QP_STATE;
 154         mod.qp_state = IBV_QPS_INIT;
 155         ret = mlx5_glue->modify_qp(obj->qp, &mod, attr_mask);
 156         if (ret) {
 157                 DRV_LOG(ERR, "Cannot change Tx QP state to INIT %s",
 158                         strerror(errno));
 159                 rte_errno = errno;
 160                 return ret;
 161         }
 162         mod.qp_state = IBV_QPS_RTR;
 163         ret = mlx5_glue->modify_qp(obj->qp, &mod, IBV_QP_STATE);
 164         if (ret) {
 165                 DRV_LOG(ERR, "Cannot change Tx QP state to RTR %s",
 166                         strerror(errno));
 167                 rte_errno = errno;
 168                 return ret;
 169         }
 170         mod.qp_state = IBV_QPS_RTS;
 171         ret = mlx5_glue->modify_qp(obj->qp, &mod, IBV_QP_STATE);
 172         if (ret) {
 173                 DRV_LOG(ERR, "Cannot change Tx QP state to RTS %s",
 174                         strerror(errno));
 175                 rte_errno = errno;
 176                 return ret;
 177         }
 178         return 0;
 179 }
 180
 181 /**
 182  * Create a CQ Verbs object.
 183  *
 184  * @param dev
 185  *   Pointer to Ethernet device.
 186  * @param idx
 187  *   Queue index in DPDK Rx queue array.
 188  *
 189  * @return
 190  *   The Verbs CQ object initialized, NULL otherwise and rte_errno is set.
 191  */
 192 static struct ibv_cq *
 193 mlx5_rxq_ibv_cq_create(struct rte_eth_dev *dev, uint16_t idx)
 194 {
 195         struct mlx5_priv *priv = dev->data->dev_private;
 196         struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
 197         struct mlx5_rxq_ctrl *rxq_ctrl =
 198                 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
 199         struct mlx5_rxq_obj *rxq_obj = rxq_ctrl->obj;
 200         unsigned int cqe_n = mlx5_rxq_cqe_num(rxq_data);
 201         struct {
 202                 struct ibv_cq_init_attr_ex ibv;
 203                 struct mlx5dv_cq_init_attr mlx5;
 204         } cq_attr;
 205
 206         cq_attr.ibv = (struct ibv_cq_init_attr_ex){
 207                 .cqe = cqe_n,
 208                 .channel = rxq_obj->ibv_channel,
 209                 .comp_mask = 0,
 210         };
 211         cq_attr.mlx5 = (struct mlx5dv_cq_init_attr){
 212                 .comp_mask = 0,
 213         };
 214         if (priv->config.cqe_comp && !rxq_data->hw_timestamp) {
 215                 cq_attr.mlx5.comp_mask |=
 216                                 MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;
 217                 rxq_data->byte_mask = UINT32_MAX;
 218 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 219                 if (mlx5_rxq_mprq_enabled(rxq_data)) {
 220                         cq_attr.mlx5.cqe_comp_res_format =
 221                                         MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX;
 222                         rxq_data->mcqe_format =
 223                                         MLX5_CQE_RESP_FORMAT_CSUM_STRIDX;
 224                 } else {
 225                         cq_attr.mlx5.cqe_comp_res_format =
 226                                         MLX5DV_CQE_RES_FORMAT_HASH;
 227                         rxq_data->mcqe_format =
 228                                         MLX5_CQE_RESP_FORMAT_HASH;
 229                 }
 230 #else
 231                 cq_attr.mlx5.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH;
 232                 rxq_data->mcqe_format = MLX5_CQE_RESP_FORMAT_HASH;
 233 #endif
 234                 /*
 235                  * For vectorized Rx, it must not be doubled in order to
 236                  * make cq_ci and rq_ci aligned.
 237                  */
 238                 if (mlx5_rxq_check_vec_support(rxq_data) < 0)
 239                         cq_attr.ibv.cqe *= 2;
 240         } else if (priv->config.cqe_comp && rxq_data->hw_timestamp) {
 241                 DRV_LOG(DEBUG,
 242                         "Port %u Rx CQE compression is disabled for HW"
 243                         " timestamp.",
 244                         dev->data->port_id);
 245         }
 246 #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD
 247         if (RTE_CACHE_LINE_SIZE == 128) {
 248                 cq_attr.mlx5.comp_mask |= MLX5DV_CQ_INIT_ATTR_MASK_FLAGS;
 249                 cq_attr.mlx5.flags |= MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD;
 250         }
 251 #endif
 252         return mlx5_glue->cq_ex_to_cq(mlx5_glue->dv_create_cq
 253                                                            (priv->sh->cdev->ctx,
 254                                                             &cq_attr.ibv,
 255                                                             &cq_attr.mlx5));
 256 }
 257
 258 /**
 259  * Create a WQ Verbs object.
 260  *
 261  * @param dev
 262  *   Pointer to Ethernet device.
 263  * @param idx
 264  *   Queue index in DPDK Rx queue array.
 265  *
 266  * @return
 267  *   The Verbs WQ object initialized, NULL otherwise and rte_errno is set.
 268  */
 269 static struct ibv_wq *
 270 mlx5_rxq_ibv_wq_create(struct rte_eth_dev *dev, uint16_t idx)
 271 {
 272         struct mlx5_priv *priv = dev->data->dev_private;
 273         struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
 274         struct mlx5_rxq_ctrl *rxq_ctrl =
 275                 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
 276         struct mlx5_rxq_obj *rxq_obj = rxq_ctrl->obj;
 277         unsigned int wqe_n = 1 << rxq_data->elts_n;
 278         struct {
 279                 struct ibv_wq_init_attr ibv;
 280 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 281                 struct mlx5dv_wq_init_attr mlx5;
 282 #endif
 283         } wq_attr;
 284
 285         wq_attr.ibv = (struct ibv_wq_init_attr){
 286                 .wq_context = NULL, /* Could be useful in the future. */
 287                 .wq_type = IBV_WQT_RQ,
 288                 /* Max number of outstanding WRs. */
 289                 .max_wr = wqe_n >> rxq_data->sges_n,
 290                 /* Max number of scatter/gather elements in a WR. */
 291                 .max_sge = 1 << rxq_data->sges_n,
 292                 .pd = priv->sh->pd,
 293                 .cq = rxq_obj->ibv_cq,
 294                 .comp_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING | 0,
 295                 .create_flags = (rxq_data->vlan_strip ?
 296                                  IBV_WQ_FLAGS_CVLAN_STRIPPING : 0),
 297         };
 298         /* By default, FCS (CRC) is stripped by hardware. */
 299         if (rxq_data->crc_present) {
 300                 wq_attr.ibv.create_flags |= IBV_WQ_FLAGS_SCATTER_FCS;
 301                 wq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
 302         }
 303         if (priv->config.hw_padding) {
 304 #if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING)
 305                 wq_attr.ibv.create_flags |= IBV_WQ_FLAG_RX_END_PADDING;
 306                 wq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
 307 #elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING)
 308                 wq_attr.ibv.create_flags |= IBV_WQ_FLAGS_PCI_WRITE_END_PADDING;
 309                 wq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
 310 #endif
 311         }
 312 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 313         wq_attr.mlx5 = (struct mlx5dv_wq_init_attr){
 314                 .comp_mask = 0,
 315         };
 316         if (mlx5_rxq_mprq_enabled(rxq_data)) {
 317                 struct mlx5dv_striding_rq_init_attr *mprq_attr =
 318                                                 &wq_attr.mlx5.striding_rq_attrs;
 319
 320                 wq_attr.mlx5.comp_mask |= MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ;
 321                 *mprq_attr = (struct mlx5dv_striding_rq_init_attr){
 322                         .single_stride_log_num_of_bytes = rxq_data->strd_sz_n,
 323                         .single_wqe_log_num_of_strides = rxq_data->strd_num_n,
 324                         .two_byte_shift_en = MLX5_MPRQ_TWO_BYTE_SHIFT,
 325                 };
 326         }
 327         rxq_obj->wq = mlx5_glue->dv_create_wq(priv->sh->cdev->ctx, &wq_attr.ibv,
 328                                               &wq_attr.mlx5);
 329 #else
 330         rxq_obj->wq = mlx5_glue->create_wq(priv->sh->cdev->ctx, &wq_attr.ibv);
 331 #endif
 332         if (rxq_obj->wq) {
 333                 /*
 334                  * Make sure number of WRs*SGEs match expectations since a queue
 335                  * cannot allocate more than "desc" buffers.
 336                  */
 337                 if (wq_attr.ibv.max_wr != (wqe_n >> rxq_data->sges_n) ||
 338                     wq_attr.ibv.max_sge != (1u << rxq_data->sges_n)) {
 339                         DRV_LOG(ERR,
 340                                 "Port %u Rx queue %u requested %u*%u but got"
 341                                 " %u*%u WRs*SGEs.",
 342                                 dev->data->port_id, idx,
 343                                 wqe_n >> rxq_data->sges_n,
 344                                 (1 << rxq_data->sges_n),
 345                                 wq_attr.ibv.max_wr, wq_attr.ibv.max_sge);
 346                         claim_zero(mlx5_glue->destroy_wq(rxq_obj->wq));
 347                         rxq_obj->wq = NULL;
 348                         rte_errno = EINVAL;
 349                 }
 350         }
 351         return rxq_obj->wq;
 352 }
 353
 354 /**
 355  * Create the Rx queue Verbs object.
 356  *
 357  * @param dev
 358  *   Pointer to Ethernet device.
 359  * @param idx
 360  *   Queue index in DPDK Rx queue array.
 361  *
 362  * @return
 363  *   0 on success, a negative errno value otherwise and rte_errno is set.
 364  */
 365 static int
 366 mlx5_rxq_ibv_obj_new(struct rte_eth_dev *dev, uint16_t idx)
 367 {
 368         struct mlx5_priv *priv = dev->data->dev_private;
 369         struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
 370         struct mlx5_rxq_ctrl *rxq_ctrl =
 371                 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
 372         struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj;
 373         struct mlx5dv_cq cq_info;
 374         struct mlx5dv_rwq rwq;
 375         int ret = 0;
 376         struct mlx5dv_obj obj;
 377
 378         MLX5_ASSERT(rxq_data);
 379         MLX5_ASSERT(tmpl);
 380         tmpl->rxq_ctrl = rxq_ctrl;
 381         if (rxq_ctrl->irq) {
 382                 tmpl->ibv_channel =
 383                         mlx5_glue->create_comp_channel(priv->sh->cdev->ctx);
 384                 if (!tmpl->ibv_channel) {
 385                         DRV_LOG(ERR, "Port %u: comp channel creation failure.",
 386                                 dev->data->port_id);
 387                         rte_errno = ENOMEM;
 388                         goto error;
 389                 }
 390                 tmpl->fd = ((struct ibv_comp_channel *)(tmpl->ibv_channel))->fd;
 391         }
 392         /* Create CQ using Verbs API. */
 393         tmpl->ibv_cq = mlx5_rxq_ibv_cq_create(dev, idx);
 394         if (!tmpl->ibv_cq) {
 395                 DRV_LOG(ERR, "Port %u Rx queue %u CQ creation failure.",
 396                         dev->data->port_id, idx);
 397                 rte_errno = ENOMEM;
 398                 goto error;
 399         }
 400         obj.cq.in = tmpl->ibv_cq;
 401         obj.cq.out = &cq_info;
 402         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ);
 403         if (ret) {
 404                 rte_errno = ret;
 405                 goto error;
 406         }
 407         if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {
 408                 DRV_LOG(ERR,
 409                         "Port %u wrong MLX5_CQE_SIZE environment "
 410                         "variable value: it should be set to %u.",
 411                         dev->data->port_id, RTE_CACHE_LINE_SIZE);
 412                 rte_errno = EINVAL;
 413                 goto error;
 414         }
 415         /* Fill the rings. */
 416         rxq_data->cqe_n = log2above(cq_info.cqe_cnt);
 417         rxq_data->cq_db = cq_info.dbrec;
 418         rxq_data->cqes = (volatile struct mlx5_cqe (*)[])(uintptr_t)cq_info.buf;
 419         rxq_data->cq_uar = cq_info.cq_uar;
 420         rxq_data->cqn = cq_info.cqn;
 421         /* Create WQ (RQ) using Verbs API. */
 422         tmpl->wq = mlx5_rxq_ibv_wq_create(dev, idx);
 423         if (!tmpl->wq) {
 424                 DRV_LOG(ERR, "Port %u Rx queue %u WQ creation failure.",
 425                         dev->data->port_id, idx);
 426                 rte_errno = ENOMEM;
 427                 goto error;
 428         }
 429         /* Change queue state to ready. */
 430         ret = mlx5_ibv_modify_wq(tmpl, IBV_WQS_RDY);
 431         if (ret) {
 432                 DRV_LOG(ERR,
 433                         "Port %u Rx queue %u WQ state to IBV_WQS_RDY failed.",
 434                         dev->data->port_id, idx);
 435                 rte_errno = ret;
 436                 goto error;
 437         }
 438         obj.rwq.in = tmpl->wq;
 439         obj.rwq.out = &rwq;
 440         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_RWQ);
 441         if (ret) {
 442                 rte_errno = ret;
 443                 goto error;
 444         }
 445         rxq_data->wqes = rwq.buf;
 446         rxq_data->rq_db = rwq.dbrec;
 447         rxq_data->cq_arm_sn = 0;
 448         mlx5_rxq_initialize(rxq_data);
 449         rxq_data->cq_ci = 0;
 450         dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;
 451         rxq_ctrl->wqn = ((struct ibv_wq *)(tmpl->wq))->wq_num;
 452         return 0;
 453 error:
 454         ret = rte_errno; /* Save rte_errno before cleanup. */
 455         if (tmpl->wq)
 456                 claim_zero(mlx5_glue->destroy_wq(tmpl->wq));
 457         if (tmpl->ibv_cq)
 458                 claim_zero(mlx5_glue->destroy_cq(tmpl->ibv_cq));
 459         if (tmpl->ibv_channel)
 460                 claim_zero(mlx5_glue->destroy_comp_channel(tmpl->ibv_channel));
 461         rte_errno = ret; /* Restore rte_errno. */
 462         return -rte_errno;
 463 }
 464
 465 /**
 466  * Release an Rx verbs queue object.
 467  *
 468  * @param rxq_obj
 469  *   Verbs Rx queue object.
 470  */
 471 static void
 472 mlx5_rxq_ibv_obj_release(struct mlx5_rxq_obj *rxq_obj)
 473 {
 474         MLX5_ASSERT(rxq_obj);
 475         MLX5_ASSERT(rxq_obj->wq);
 476         MLX5_ASSERT(rxq_obj->ibv_cq);
 477         claim_zero(mlx5_glue->destroy_wq(rxq_obj->wq));
 478         claim_zero(mlx5_glue->destroy_cq(rxq_obj->ibv_cq));
 479         if (rxq_obj->ibv_channel)
 480                 claim_zero(mlx5_glue->destroy_comp_channel
 481                                                         (rxq_obj->ibv_channel));
 482 }
 483
 484 /**
 485  * Get event for an Rx verbs queue object.
 486  *
 487  * @param rxq_obj
 488  *   Verbs Rx queue object.
 489  *
 490  * @return
 491  *   0 on success, a negative errno value otherwise and rte_errno is set.
 492  */
 493 static int
 494 mlx5_rx_ibv_get_event(struct mlx5_rxq_obj *rxq_obj)
 495 {
 496         struct ibv_cq *ev_cq;
 497         void *ev_ctx;
 498         int ret = mlx5_glue->get_cq_event(rxq_obj->ibv_channel,
 499                                           &ev_cq, &ev_ctx);
 500
 501         if (ret < 0 || ev_cq != rxq_obj->ibv_cq)
 502                 goto exit;
 503         mlx5_glue->ack_cq_events(rxq_obj->ibv_cq, 1);
 504         return 0;
 505 exit:
 506         if (ret < 0)
 507                 rte_errno = errno;
 508         else
 509                 rte_errno = EINVAL;
 510         return -rte_errno;
 511 }
 512
 513 /**
 514  * Creates a receive work queue as a filed of indirection table.
 515  *
 516  * @param dev
 517  *   Pointer to Ethernet device.
 518  * @param log_n
 519  *   Log of number of queues in the array.
 520  * @param ind_tbl
 521  *   Verbs indirection table object.
 522  *
 523  * @return
 524  *   0 on success, a negative errno value otherwise and rte_errno is set.
 525  */
 526 static int
 527 mlx5_ibv_ind_table_new(struct rte_eth_dev *dev, const unsigned int log_n,
 528                        struct mlx5_ind_table_obj *ind_tbl)
 529 {
 530         struct mlx5_priv *priv = dev->data->dev_private;
 531         struct ibv_wq *wq[1 << log_n];
 532         unsigned int i, j;
 533
 534         MLX5_ASSERT(ind_tbl);
 535         for (i = 0; i != ind_tbl->queues_n; ++i) {
 536                 struct mlx5_rxq_data *rxq = (*priv->rxqs)[ind_tbl->queues[i]];
 537                 struct mlx5_rxq_ctrl *rxq_ctrl =
 538                                 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 539
 540                 wq[i] = rxq_ctrl->obj->wq;
 541         }
 542         MLX5_ASSERT(i > 0);
 543         /* Finalise indirection table. */
 544         for (j = 0; i != (unsigned int)(1 << log_n); ++j, ++i)
 545                 wq[i] = wq[j];
 546         ind_tbl->ind_table = mlx5_glue->create_rwq_ind_table
 547                                         (priv->sh->cdev->ctx,
 548                                          &(struct ibv_rwq_ind_table_init_attr){
 549                                                  .log_ind_tbl_size = log_n,
 550                                                  .ind_tbl = wq,
 551                                                  .comp_mask = 0,
 552                                          });
 553         if (!ind_tbl->ind_table) {
 554                 rte_errno = errno;
 555                 return -rte_errno;
 556         }
 557         return 0;
 558 }
 559
 560 /**
 561  * Destroys the specified Indirection Table.
 562  *
 563  * @param ind_table
 564  *   Indirection table to release.
 565  */
 566 static void
 567 mlx5_ibv_ind_table_destroy(struct mlx5_ind_table_obj *ind_tbl)
 568 {
 569         claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl->ind_table));
 570 }
 571
 572 /**
 573  * Create an Rx Hash queue.
 574  *
 575  * @param dev
 576  *   Pointer to Ethernet device.
 577  * @param hrxq
 578  *   Pointer to Rx Hash queue.
 579  * @param tunnel
 580  *   Tunnel type.
 581  *
 582  * @return
 583  *   0 on success, a negative errno value otherwise and rte_errno is set.
 584  */
 585 static int
 586 mlx5_ibv_hrxq_new(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq,
 587                   int tunnel __rte_unused)
 588 {
 589         struct mlx5_priv *priv = dev->data->dev_private;
 590         struct ibv_qp *qp = NULL;
 591         struct mlx5_ind_table_obj *ind_tbl = hrxq->ind_table;
 592         const uint8_t *rss_key = hrxq->rss_key;
 593         uint64_t hash_fields = hrxq->hash_fields;
 594         int err;
 595 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 596         struct mlx5dv_qp_init_attr qp_init_attr;
 597
 598         memset(&qp_init_attr, 0, sizeof(qp_init_attr));
 599         if (tunnel) {
 600                 qp_init_attr.comp_mask =
 601                                        MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
 602                 qp_init_attr.create_flags = MLX5DV_QP_CREATE_TUNNEL_OFFLOADS;
 603         }
 604 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 605         if (dev->data->dev_conf.lpbk_mode) {
 606                 /* Allow packet sent from NIC loop back w/o source MAC check. */
 607                 qp_init_attr.comp_mask |=
 608                                 MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
 609                 qp_init_attr.create_flags |=
 610                                 MLX5DV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_UC;
 611         }
 612 #endif
 613         qp = mlx5_glue->dv_create_qp
 614                         (priv->sh->cdev->ctx,
 615                          &(struct ibv_qp_init_attr_ex){
 616                                 .qp_type = IBV_QPT_RAW_PACKET,
 617                                 .comp_mask =
 618                                         IBV_QP_INIT_ATTR_PD |
 619                                         IBV_QP_INIT_ATTR_IND_TABLE |
 620                                         IBV_QP_INIT_ATTR_RX_HASH,
 621                                 .rx_hash_conf = (struct ibv_rx_hash_conf){
 622                                         .rx_hash_function =
 623                                                 IBV_RX_HASH_FUNC_TOEPLITZ,
 624                                         .rx_hash_key_len = hrxq->rss_key_len,
 625                                         .rx_hash_key =
 626                                                 (void *)(uintptr_t)rss_key,
 627                                         .rx_hash_fields_mask = hash_fields,
 628                                 },
 629                                 .rwq_ind_tbl = ind_tbl->ind_table,
 630                                 .pd = priv->sh->pd,
 631                           },
 632                           &qp_init_attr);
 633 #else
 634         qp = mlx5_glue->create_qp_ex
 635                         (priv->sh->cdev->ctx,
 636                          &(struct ibv_qp_init_attr_ex){
 637                                 .qp_type = IBV_QPT_RAW_PACKET,
 638                                 .comp_mask =
 639                                         IBV_QP_INIT_ATTR_PD |
 640                                         IBV_QP_INIT_ATTR_IND_TABLE |
 641                                         IBV_QP_INIT_ATTR_RX_HASH,
 642                                 .rx_hash_conf = (struct ibv_rx_hash_conf){
 643                                         .rx_hash_function =
 644                                                 IBV_RX_HASH_FUNC_TOEPLITZ,
 645                                         .rx_hash_key_len = hrxq->rss_key_len,
 646                                         .rx_hash_key =
 647                                                 (void *)(uintptr_t)rss_key,
 648                                         .rx_hash_fields_mask = hash_fields,
 649                                 },
 650                                 .rwq_ind_tbl = ind_tbl->ind_table,
 651                                 .pd = priv->sh->pd,
 652                          });
 653 #endif
 654         if (!qp) {
 655                 rte_errno = errno;
 656                 goto error;
 657         }
 658         hrxq->qp = qp;
 659 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 660         hrxq->action = mlx5_glue->dv_create_flow_action_dest_ibv_qp(hrxq->qp);
 661         if (!hrxq->action) {
 662                 rte_errno = errno;
 663                 goto error;
 664         }
 665 #endif
 666         return 0;
 667 error:
 668         err = rte_errno; /* Save rte_errno before cleanup. */
 669         if (qp)
 670                 claim_zero(mlx5_glue->destroy_qp(qp));
 671         rte_errno = err; /* Restore rte_errno. */
 672         return -rte_errno;
 673 }
 674
 675 /**
 676  * Destroy a Verbs queue pair.
 677  *
 678  * @param hrxq
 679  *   Hash Rx queue to release its qp.
 680  */
 681 static void
 682 mlx5_ibv_qp_destroy(struct mlx5_hrxq *hrxq)
 683 {
 684         claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
 685 }
 686
 687 /**
 688  * Release a drop Rx queue Verbs object.
 689  *
 690  * @param dev
 691  *   Pointer to Ethernet device.
 692  */
 693 static void
 694 mlx5_rxq_ibv_obj_drop_release(struct rte_eth_dev *dev)
 695 {
 696         struct mlx5_priv *priv = dev->data->dev_private;
 697         struct mlx5_rxq_obj *rxq = priv->drop_queue.rxq;
 698
 699         if (rxq->wq)
 700                 claim_zero(mlx5_glue->destroy_wq(rxq->wq));
 701         if (rxq->ibv_cq)
 702                 claim_zero(mlx5_glue->destroy_cq(rxq->ibv_cq));
 703         mlx5_free(rxq);
 704         priv->drop_queue.rxq = NULL;
 705 }
 706
 707 /**
 708  * Create a drop Rx queue Verbs object.
 709  *
 710  * @param dev
 711  *   Pointer to Ethernet device.
 712  *
 713  * @return
 714  *   0 on success, a negative errno value otherwise and rte_errno is set.
 715  */
 716 static int
 717 mlx5_rxq_ibv_obj_drop_create(struct rte_eth_dev *dev)
 718 {
 719         struct mlx5_priv *priv = dev->data->dev_private;
 720         struct ibv_context *ctx = priv->sh->cdev->ctx;
 721         struct mlx5_rxq_obj *rxq = priv->drop_queue.rxq;
 722
 723         if (rxq)
 724                 return 0;
 725         rxq = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq), 0, SOCKET_ID_ANY);
 726         if (!rxq) {
 727                 DRV_LOG(DEBUG, "Port %u cannot allocate drop Rx queue memory.",
 728                       dev->data->port_id);
 729                 rte_errno = ENOMEM;
 730                 return -rte_errno;
 731         }
 732         priv->drop_queue.rxq = rxq;
 733         rxq->ibv_cq = mlx5_glue->create_cq(ctx, 1, NULL, NULL, 0);
 734         if (!rxq->ibv_cq) {
 735                 DRV_LOG(DEBUG, "Port %u cannot allocate CQ for drop queue.",
 736                       dev->data->port_id);
 737                 rte_errno = errno;
 738                 goto error;
 739         }
 740         rxq->wq = mlx5_glue->create_wq(ctx, &(struct ibv_wq_init_attr){
 741                                                     .wq_type = IBV_WQT_RQ,
 742                                                     .max_wr = 1,
 743                                                     .max_sge = 1,
 744                                                     .pd = priv->sh->pd,
 745                                                     .cq = rxq->ibv_cq,
 746                                               });
 747         if (!rxq->wq) {
 748                 DRV_LOG(DEBUG, "Port %u cannot allocate WQ for drop queue.",
 749                       dev->data->port_id);
 750                 rte_errno = errno;
 751                 goto error;
 752         }
 753         priv->drop_queue.rxq = rxq;
 754         return 0;
 755 error:
 756         mlx5_rxq_ibv_obj_drop_release(dev);
 757         return -rte_errno;
 758 }
 759
 760 /**
 761  * Create a Verbs drop action for Rx Hash queue.
 762  *
 763  * @param dev
 764  *   Pointer to Ethernet device.
 765  *
 766  * @return
 767  *   0 on success, a negative errno value otherwise and rte_errno is set.
 768  */
 769 static int
 770 mlx5_ibv_drop_action_create(struct rte_eth_dev *dev)
 771 {
 772         struct mlx5_priv *priv = dev->data->dev_private;
 773         struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq;
 774         struct ibv_rwq_ind_table *ind_tbl = NULL;
 775         struct mlx5_rxq_obj *rxq;
 776         int ret;
 777
 778         MLX5_ASSERT(hrxq && hrxq->ind_table);
 779         ret = mlx5_rxq_ibv_obj_drop_create(dev);
 780         if (ret < 0)
 781                 goto error;
 782         rxq = priv->drop_queue.rxq;
 783         ind_tbl = mlx5_glue->create_rwq_ind_table
 784                                 (priv->sh->cdev->ctx,
 785                                  &(struct ibv_rwq_ind_table_init_attr){
 786                                         .log_ind_tbl_size = 0,
 787                                         .ind_tbl = (struct ibv_wq **)&rxq->wq,
 788                                         .comp_mask = 0,
 789                                  });
 790         if (!ind_tbl) {
 791                 DRV_LOG(DEBUG, "Port %u"
 792                         " cannot allocate indirection table for drop queue.",
 793                         dev->data->port_id);
 794                 rte_errno = errno;
 795                 goto error;
 796         }
 797         hrxq->qp = mlx5_glue->create_qp_ex(priv->sh->cdev->ctx,
 798                  &(struct ibv_qp_init_attr_ex){
 799                         .qp_type = IBV_QPT_RAW_PACKET,
 800                         .comp_mask = IBV_QP_INIT_ATTR_PD |
 801                                      IBV_QP_INIT_ATTR_IND_TABLE |
 802                                      IBV_QP_INIT_ATTR_RX_HASH,
 803                         .rx_hash_conf = (struct ibv_rx_hash_conf){
 804                                 .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ,
 805                                 .rx_hash_key_len = MLX5_RSS_HASH_KEY_LEN,
 806                                 .rx_hash_key = rss_hash_default_key,
 807                                 .rx_hash_fields_mask = 0,
 808                                 },
 809                         .rwq_ind_tbl = ind_tbl,
 810                         .pd = priv->sh->pd
 811                  });
 812         if (!hrxq->qp) {
 813                 DRV_LOG(DEBUG, "Port %u cannot allocate QP for drop queue.",
 814                       dev->data->port_id);
 815                 rte_errno = errno;
 816                 goto error;
 817         }
 818 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 819         hrxq->action = mlx5_glue->dv_create_flow_action_dest_ibv_qp(hrxq->qp);
 820         if (!hrxq->action) {
 821                 rte_errno = errno;
 822                 goto error;
 823         }
 824 #endif
 825         hrxq->ind_table->ind_table = ind_tbl;
 826         return 0;
 827 error:
 828         if (hrxq->qp)
 829                 claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
 830         if (ind_tbl)
 831                 claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl));
 832         if (priv->drop_queue.rxq)
 833                 mlx5_rxq_ibv_obj_drop_release(dev);
 834         return -rte_errno;
 835 }
 836
 837 /**
 838  * Release a drop hash Rx queue.
 839  *
 840  * @param dev
 841  *   Pointer to Ethernet device.
 842  */
 843 static void
 844 mlx5_ibv_drop_action_destroy(struct rte_eth_dev *dev)
 845 {
 846         struct mlx5_priv *priv = dev->data->dev_private;
 847         struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq;
 848         struct ibv_rwq_ind_table *ind_tbl = hrxq->ind_table->ind_table;
 849
 850 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 851         claim_zero(mlx5_glue->destroy_flow_action(hrxq->action));
 852 #endif
 853         claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
 854         claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl));
 855         mlx5_rxq_ibv_obj_drop_release(dev);
 856 }
 857
 858 /**
 859  * Create a QP Verbs object.
 860  *
 861  * @param dev
 862  *   Pointer to Ethernet device.
 863  * @param idx
 864  *   Queue index in DPDK Tx queue array.
 865  *
 866  * @return
 867  *   The QP Verbs object, NULL otherwise and rte_errno is set.
 868  */
 869 static struct ibv_qp *
 870 mlx5_txq_ibv_qp_create(struct rte_eth_dev *dev, uint16_t idx)
 871 {
 872         struct mlx5_priv *priv = dev->data->dev_private;
 873         struct mlx5_txq_data *txq_data = (*priv->txqs)[idx];
 874         struct mlx5_txq_ctrl *txq_ctrl =
 875                         container_of(txq_data, struct mlx5_txq_ctrl, txq);
 876         struct ibv_qp *qp_obj = NULL;
 877         struct ibv_qp_init_attr_ex qp_attr = { 0 };
 878         const int desc = 1 << txq_data->elts_n;
 879
 880         MLX5_ASSERT(txq_ctrl->obj->cq);
 881         /* CQ to be associated with the send queue. */
 882         qp_attr.send_cq = txq_ctrl->obj->cq;
 883         /* CQ to be associated with the receive queue. */
 884         qp_attr.recv_cq = txq_ctrl->obj->cq;
 885         /* Max number of outstanding WRs. */
 886         qp_attr.cap.max_send_wr = ((priv->sh->device_attr.max_qp_wr < desc) ?
 887                                    priv->sh->device_attr.max_qp_wr : desc);
 888         /*
 889          * Max number of scatter/gather elements in a WR, must be 1 to prevent
 890          * libmlx5 from trying to affect must be 1 to prevent libmlx5 from
 891          * trying to affect too much memory. TX gather is not impacted by the
 892          * device_attr.max_sge limit and will still work properly.
 893          */
 894         qp_attr.cap.max_send_sge = 1;
 895         qp_attr.qp_type = IBV_QPT_RAW_PACKET,
 896         /* Do *NOT* enable this, completions events are managed per Tx burst. */
 897         qp_attr.sq_sig_all = 0;
 898         qp_attr.pd = priv->sh->pd;
 899         qp_attr.comp_mask = IBV_QP_INIT_ATTR_PD;
 900         if (txq_data->inlen_send)
 901                 qp_attr.cap.max_inline_data = txq_ctrl->max_inline_data;
 902         if (txq_data->tso_en) {
 903                 qp_attr.max_tso_header = txq_ctrl->max_tso_header;
 904                 qp_attr.comp_mask |= IBV_QP_INIT_ATTR_MAX_TSO_HEADER;
 905         }
 906         qp_obj = mlx5_glue->create_qp_ex(priv->sh->cdev->ctx, &qp_attr);
 907         if (qp_obj == NULL) {
 908                 DRV_LOG(ERR, "Port %u Tx queue %u QP creation failure.",
 909                         dev->data->port_id, idx);
 910                 rte_errno = errno;
 911         }
 912         return qp_obj;
 913 }
 914
 915 /**
 916  * Create the Tx queue Verbs object.
 917  *
 918  * @param dev
 919  *   Pointer to Ethernet device.
 920  * @param idx
 921  *   Queue index in DPDK Tx queue array.
 922  *
 923  * @return
 924  *   0 on success, a negative errno value otherwise and rte_errno is set.
 925  */
 926 int
 927 mlx5_txq_ibv_obj_new(struct rte_eth_dev *dev, uint16_t idx)
 928 {
 929         struct mlx5_priv *priv = dev->data->dev_private;
 930         struct mlx5_txq_data *txq_data = (*priv->txqs)[idx];
 931         struct mlx5_txq_ctrl *txq_ctrl =
 932                 container_of(txq_data, struct mlx5_txq_ctrl, txq);
 933         struct mlx5_txq_obj *txq_obj = txq_ctrl->obj;
 934         unsigned int cqe_n;
 935         struct mlx5dv_qp qp;
 936         struct mlx5dv_cq cq_info;
 937         struct mlx5dv_obj obj;
 938         const int desc = 1 << txq_data->elts_n;
 939         int ret = 0;
 940
 941         MLX5_ASSERT(txq_data);
 942         MLX5_ASSERT(txq_obj);
 943         txq_obj->txq_ctrl = txq_ctrl;
 944         if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) {
 945                 DRV_LOG(ERR, "Port %u MLX5_ENABLE_CQE_COMPRESSION "
 946                         "must never be set.", dev->data->port_id);
 947                 rte_errno = EINVAL;
 948                 return -rte_errno;
 949         }
 950         cqe_n = desc / MLX5_TX_COMP_THRESH +
 951                 1 + MLX5_TX_COMP_THRESH_INLINE_DIV;
 952         txq_obj->cq = mlx5_glue->create_cq(priv->sh->cdev->ctx, cqe_n,
 953                                            NULL, NULL, 0);
 954         if (txq_obj->cq == NULL) {
 955                 DRV_LOG(ERR, "Port %u Tx queue %u CQ creation failure.",
 956                         dev->data->port_id, idx);
 957                 rte_errno = errno;
 958                 goto error;
 959         }
 960         txq_obj->qp = mlx5_txq_ibv_qp_create(dev, idx);
 961         if (txq_obj->qp == NULL) {
 962                 rte_errno = errno;
 963                 goto error;
 964         }
 965         ret = mlx5_ibv_modify_qp(txq_obj, MLX5_TXQ_MOD_RST2RDY,
 966                                  (uint8_t)priv->dev_port);
 967         if (ret) {
 968                 DRV_LOG(ERR, "Port %u Tx queue %u QP state modifying failed.",
 969                         dev->data->port_id, idx);
 970                 rte_errno = errno;
 971                 goto error;
 972         }
 973         qp.comp_mask = MLX5DV_QP_MASK_UAR_MMAP_OFFSET;
 974 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 975         /* If using DevX, need additional mask to read tisn value. */
 976         if (priv->sh->devx && !priv->sh->tdn)
 977                 qp.comp_mask |= MLX5DV_QP_MASK_RAW_QP_HANDLES;
 978 #endif
 979         obj.cq.in = txq_obj->cq;
 980         obj.cq.out = &cq_info;
 981         obj.qp.in = txq_obj->qp;
 982         obj.qp.out = &qp;
 983         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_QP);
 984         if (ret != 0) {
 985                 rte_errno = errno;
 986                 goto error;
 987         }
 988         if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {
 989                 DRV_LOG(ERR,
 990                         "Port %u wrong MLX5_CQE_SIZE environment variable"
 991                         " value: it should be set to %u.",
 992                         dev->data->port_id, RTE_CACHE_LINE_SIZE);
 993                 rte_errno = EINVAL;
 994                 goto error;
 995         }
 996         txq_data->cqe_n = log2above(cq_info.cqe_cnt);
 997         txq_data->cqe_s = 1 << txq_data->cqe_n;
 998         txq_data->cqe_m = txq_data->cqe_s - 1;
 999         txq_data->qp_num_8s = ((struct ibv_qp *)txq_obj->qp)->qp_num << 8;
1000         txq_data->wqes = qp.sq.buf;
1001         txq_data->wqe_n = log2above(qp.sq.wqe_cnt);
1002         txq_data->wqe_s = 1 << txq_data->wqe_n;
1003         txq_data->wqe_m = txq_data->wqe_s - 1;
1004         txq_data->wqes_end = txq_data->wqes + txq_data->wqe_s;
1005         txq_data->qp_db = &qp.dbrec[MLX5_SND_DBR];
1006         txq_data->cq_db = cq_info.dbrec;
1007         txq_data->cqes = (volatile struct mlx5_cqe *)cq_info.buf;
1008         txq_data->cq_ci = 0;
1009         txq_data->cq_pi = 0;
1010         txq_data->wqe_ci = 0;
1011         txq_data->wqe_pi = 0;
1012         txq_data->wqe_comp = 0;
1013         txq_data->wqe_thres = txq_data->wqe_s / MLX5_TX_COMP_THRESH_INLINE_DIV;
1014 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
1015         /*
1016          * If using DevX need to query and store TIS transport domain value.
1017          * This is done once per port.
1018          * Will use this value on Rx, when creating matching TIR.
1019          */
1020         if (priv->sh->devx && !priv->sh->tdn) {
1021                 ret = mlx5_devx_cmd_qp_query_tis_td(txq_obj->qp, qp.tisn,
1022                                                     &priv->sh->tdn);
1023                 if (ret) {
1024                         DRV_LOG(ERR, "Fail to query port %u Tx queue %u QP TIS "
1025                                 "transport domain.", dev->data->port_id, idx);
1026                         rte_errno = EINVAL;
1027                         goto error;
1028                 } else {
1029                         DRV_LOG(DEBUG, "Port %u Tx queue %u TIS number %d "
1030                                 "transport domain %d.", dev->data->port_id,
1031                                 idx, qp.tisn, priv->sh->tdn);
1032                 }
1033         }
1034 #endif
1035         txq_ctrl->bf_reg = qp.bf.reg;
1036         if (qp.comp_mask & MLX5DV_QP_MASK_UAR_MMAP_OFFSET) {
1037                 txq_ctrl->uar_mmap_offset = qp.uar_mmap_offset;
1038                 DRV_LOG(DEBUG, "Port %u: uar_mmap_offset 0x%" PRIx64 ".",
1039                         dev->data->port_id, txq_ctrl->uar_mmap_offset);
1040         } else {
1041                 DRV_LOG(ERR,
1042                         "Port %u failed to retrieve UAR info, invalid"
1043                         " libmlx5.so",
1044                         dev->data->port_id);
1045                 rte_errno = EINVAL;
1046                 goto error;
1047         }
1048         txq_uar_init(txq_ctrl);
1049         dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;
1050         return 0;
1051 error:
1052         ret = rte_errno; /* Save rte_errno before cleanup. */
1053         if (txq_obj->cq)
1054                 claim_zero(mlx5_glue->destroy_cq(txq_obj->cq));
1055         if (txq_obj->qp)
1056                 claim_zero(mlx5_glue->destroy_qp(txq_obj->qp));
1057         rte_errno = ret; /* Restore rte_errno. */
1058         return -rte_errno;
1059 }
1060
1061 /*
1062  * Create the dummy QP with minimal resources for loopback.
1063  *
1064  * @param dev
1065  *   Pointer to Ethernet device.
1066  *
1067  * @return
1068  *   0 on success, a negative errno value otherwise and rte_errno is set.
1069  */
1070 int
1071 mlx5_rxq_ibv_obj_dummy_lb_create(struct rte_eth_dev *dev)
1072 {
1073 #if defined(HAVE_IBV_DEVICE_TUNNEL_SUPPORT) && defined(HAVE_IBV_FLOW_DV_SUPPORT)
1074         struct mlx5_priv *priv = dev->data->dev_private;
1075         struct mlx5_dev_ctx_shared *sh = priv->sh;
1076         struct ibv_context *ctx = sh->cdev->ctx;
1077         struct mlx5dv_qp_init_attr qp_init_attr = {0};
1078         struct {
1079                 struct ibv_cq_init_attr_ex ibv;
1080                 struct mlx5dv_cq_init_attr mlx5;
1081         } cq_attr = {{0}};
1082
1083         if (dev->data->dev_conf.lpbk_mode) {
1084                 /* Allow packet sent from NIC loop back w/o source MAC check. */
1085                 qp_init_attr.comp_mask |=
1086                                 MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
1087                 qp_init_attr.create_flags |=
1088                                 MLX5DV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_UC;
1089         } else {
1090                 return 0;
1091         }
1092         /* Only need to check refcnt, 0 after "sh" is allocated. */
1093         if (!!(__atomic_fetch_add(&sh->self_lb.refcnt, 1, __ATOMIC_RELAXED))) {
1094                 MLX5_ASSERT(sh->self_lb.ibv_cq && sh->self_lb.qp);
1095                 priv->lb_used = 1;
1096                 return 0;
1097         }
1098         cq_attr.ibv = (struct ibv_cq_init_attr_ex){
1099                 .cqe = 1,
1100                 .channel = NULL,
1101                 .comp_mask = 0,
1102         };
1103         cq_attr.mlx5 = (struct mlx5dv_cq_init_attr){
1104                 .comp_mask = 0,
1105         };
1106         /* Only CQ is needed, no WQ(RQ) is required in this case. */
1107         sh->self_lb.ibv_cq = mlx5_glue->cq_ex_to_cq(mlx5_glue->dv_create_cq(ctx,
1108                                                         &cq_attr.ibv,
1109                                                         &cq_attr.mlx5));
1110         if (!sh->self_lb.ibv_cq) {
1111                 DRV_LOG(ERR, "Port %u cannot allocate CQ for loopback.",
1112                         dev->data->port_id);
1113                 rte_errno = errno;
1114                 goto error;
1115         }
1116         sh->self_lb.qp = mlx5_glue->dv_create_qp(ctx,
1117                                 &(struct ibv_qp_init_attr_ex){
1118                                         .qp_type = IBV_QPT_RAW_PACKET,
1119                                         .comp_mask = IBV_QP_INIT_ATTR_PD,
1120                                         .pd = sh->pd,
1121                                         .send_cq = sh->self_lb.ibv_cq,
1122                                         .recv_cq = sh->self_lb.ibv_cq,
1123                                         .cap.max_recv_wr = 1,
1124                                 },
1125                                 &qp_init_attr);
1126         if (!sh->self_lb.qp) {
1127                 DRV_LOG(DEBUG, "Port %u cannot allocate QP for loopback.",
1128                         dev->data->port_id);
1129                 rte_errno = errno;
1130                 goto error;
1131         }
1132         priv->lb_used = 1;
1133         return 0;
1134 error:
1135         if (sh->self_lb.ibv_cq) {
1136                 claim_zero(mlx5_glue->destroy_cq(sh->self_lb.ibv_cq));
1137                 sh->self_lb.ibv_cq = NULL;
1138         }
1139         (void)__atomic_sub_fetch(&sh->self_lb.refcnt, 1, __ATOMIC_RELAXED);
1140         return -rte_errno;
1141 #else
1142         RTE_SET_USED(dev);
1143         return 0;
1144 #endif
1145 }
1146
1147 /*
1148  * Release the dummy queue resources for loopback.
1149  *
1150  * @param dev
1151  *   Pointer to Ethernet device.
1152  */
1153 void
1154 mlx5_rxq_ibv_obj_dummy_lb_release(struct rte_eth_dev *dev)
1155 {
1156 #if defined(HAVE_IBV_DEVICE_TUNNEL_SUPPORT) && defined(HAVE_IBV_FLOW_DV_SUPPORT)
1157         struct mlx5_priv *priv = dev->data->dev_private;
1158         struct mlx5_dev_ctx_shared *sh = priv->sh;
1159
1160         if (!priv->lb_used)
1161                 return;
1162         MLX5_ASSERT(__atomic_load_n(&sh->self_lb.refcnt, __ATOMIC_RELAXED));
1163         if (!(__atomic_sub_fetch(&sh->self_lb.refcnt, 1, __ATOMIC_RELAXED))) {
1164                 if (sh->self_lb.qp) {
1165                         claim_zero(mlx5_glue->destroy_qp(sh->self_lb.qp));
1166                         sh->self_lb.qp = NULL;
1167                 }
1168                 if (sh->self_lb.ibv_cq) {
1169                         claim_zero(mlx5_glue->destroy_cq(sh->self_lb.ibv_cq));
1170                         sh->self_lb.ibv_cq = NULL;
1171                 }
1172         }
1173         priv->lb_used = 0;
1174 #else
1175         RTE_SET_USED(dev);
1176         return;
1177 #endif
1178 }
1179
1180 /**
1181  * Release an Tx verbs queue object.
1182  *
1183  * @param txq_obj
1184  *   Verbs Tx queue object..
1185  */
1186 void
1187 mlx5_txq_ibv_obj_release(struct mlx5_txq_obj *txq_obj)
1188 {
1189         MLX5_ASSERT(txq_obj);
1190         claim_zero(mlx5_glue->destroy_qp(txq_obj->qp));
1191         claim_zero(mlx5_glue->destroy_cq(txq_obj->cq));
1192 }
1193
1194 struct mlx5_obj_ops ibv_obj_ops = {
1195         .rxq_obj_modify_vlan_strip = mlx5_rxq_obj_modify_wq_vlan_strip,
1196         .rxq_obj_new = mlx5_rxq_ibv_obj_new,
1197         .rxq_event_get = mlx5_rx_ibv_get_event,
1198         .rxq_obj_modify = mlx5_ibv_modify_wq,
1199         .rxq_obj_release = mlx5_rxq_ibv_obj_release,
1200         .ind_table_new = mlx5_ibv_ind_table_new,
1201         .ind_table_destroy = mlx5_ibv_ind_table_destroy,
1202         .hrxq_new = mlx5_ibv_hrxq_new,
1203         .hrxq_destroy = mlx5_ibv_qp_destroy,
1204         .drop_action_create = mlx5_ibv_drop_action_create,
1205         .drop_action_destroy = mlx5_ibv_drop_action_destroy,
1206         .txq_obj_new = mlx5_txq_ibv_obj_new,
1207         .txq_obj_modify = mlx5_ibv_modify_qp,
1208         .txq_obj_release = mlx5_txq_ibv_obj_release,
1209         .lb_dummy_queue_create = NULL,
1210         .lb_dummy_queue_release = NULL,
1211 };