drivers/net/mlx5/linux/mlx5_verbs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <stddef.h>
   6 #include <errno.h>
   7 #include <string.h>
   8 #include <stdint.h>
   9 #include <unistd.h>
  10 #include <inttypes.h>
  11 #include <sys/queue.h>
  12
  13 #include "mlx5_autoconf.h"
  14
  15 #include <rte_mbuf.h>
  16 #include <rte_malloc.h>
  17 #include <ethdev_driver.h>
  18 #include <rte_common.h>
  19
  20 #include <mlx5_glue.h>
  21 #include <mlx5_common.h>
  22 #include <mlx5_common_mr.h>
  23 #include <mlx5_verbs.h>
  24 #include <mlx5_rx.h>
  25 #include <mlx5_tx.h>
  26 #include <mlx5_utils.h>
  27 #include <mlx5_malloc.h>
  28
  29 /**
  30  * Modify Rx WQ vlan stripping offload
  31  *
  32  * @param rxq_obj
  33  *   Rx queue object.
  34  *
  35  * @return 0 on success, non-0 otherwise
  36  */
  37 static int
  38 mlx5_rxq_obj_modify_wq_vlan_strip(struct mlx5_rxq_obj *rxq_obj, int on)
  39 {
  40         uint16_t vlan_offloads =
  41                 (on ? IBV_WQ_FLAGS_CVLAN_STRIPPING : 0) |
  42                 0;
  43         struct ibv_wq_attr mod;
  44         mod = (struct ibv_wq_attr){
  45                 .attr_mask = IBV_WQ_ATTR_FLAGS,
  46                 .flags_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING,
  47                 .flags = vlan_offloads,
  48         };
  49
  50         return mlx5_glue->modify_wq(rxq_obj->wq, &mod);
  51 }
  52
  53 /**
  54  * Modifies the attributes for the specified WQ.
  55  *
  56  * @param rxq_obj
  57  *   Verbs Rx queue object.
  58  * @param type
  59  *   Type of change queue state.
  60  *
  61  * @return
  62  *   0 on success, a negative errno value otherwise and rte_errno is set.
  63  */
  64 static int
  65 mlx5_ibv_modify_wq(struct mlx5_rxq_obj *rxq_obj, uint8_t type)
  66 {
  67         struct ibv_wq_attr mod = {
  68                 .attr_mask = IBV_WQ_ATTR_STATE,
  69                 .wq_state = (enum ibv_wq_state)type,
  70         };
  71
  72         return mlx5_glue->modify_wq(rxq_obj->wq, &mod);
  73 }
  74
  75 /**
  76  * Modify QP using Verbs API.
  77  *
  78  * @param txq_obj
  79  *   Verbs Tx queue object.
  80  * @param type
  81  *   Type of change queue state.
  82  * @param dev_port
  83  *   IB device port number.
  84  *
  85  * @return
  86  *   0 on success, a negative errno value otherwise and rte_errno is set.
  87  */
  88 static int
  89 mlx5_ibv_modify_qp(struct mlx5_txq_obj *obj, enum mlx5_txq_modify_type type,
  90                    uint8_t dev_port)
  91 {
  92         struct ibv_qp_attr mod = {
  93                 .qp_state = IBV_QPS_RESET,
  94                 .port_num = dev_port,
  95         };
  96         int attr_mask = (IBV_QP_STATE | IBV_QP_PORT);
  97         int ret;
  98
  99         if (type != MLX5_TXQ_MOD_RST2RDY) {
 100                 ret = mlx5_glue->modify_qp(obj->qp, &mod, IBV_QP_STATE);
 101                 if (ret) {
 102                         DRV_LOG(ERR, "Cannot change Tx QP state to RESET %s",
 103                                 strerror(errno));
 104                         rte_errno = errno;
 105                         return ret;
 106                 }
 107                 if (type == MLX5_TXQ_MOD_RDY2RST)
 108                         return 0;
 109         }
 110         if (type == MLX5_TXQ_MOD_ERR2RDY)
 111                 attr_mask = IBV_QP_STATE;
 112         mod.qp_state = IBV_QPS_INIT;
 113         ret = mlx5_glue->modify_qp(obj->qp, &mod, attr_mask);
 114         if (ret) {
 115                 DRV_LOG(ERR, "Cannot change Tx QP state to INIT %s",
 116                         strerror(errno));
 117                 rte_errno = errno;
 118                 return ret;
 119         }
 120         mod.qp_state = IBV_QPS_RTR;
 121         ret = mlx5_glue->modify_qp(obj->qp, &mod, IBV_QP_STATE);
 122         if (ret) {
 123                 DRV_LOG(ERR, "Cannot change Tx QP state to RTR %s",
 124                         strerror(errno));
 125                 rte_errno = errno;
 126                 return ret;
 127         }
 128         mod.qp_state = IBV_QPS_RTS;
 129         ret = mlx5_glue->modify_qp(obj->qp, &mod, IBV_QP_STATE);
 130         if (ret) {
 131                 DRV_LOG(ERR, "Cannot change Tx QP state to RTS %s",
 132                         strerror(errno));
 133                 rte_errno = errno;
 134                 return ret;
 135         }
 136         return 0;
 137 }
 138
 139 /**
 140  * Create a CQ Verbs object.
 141  *
 142  * @param dev
 143  *   Pointer to Ethernet device.
 144  * @param idx
 145  *   Queue index in DPDK Rx queue array.
 146  *
 147  * @return
 148  *   The Verbs CQ object initialized, NULL otherwise and rte_errno is set.
 149  */
 150 static struct ibv_cq *
 151 mlx5_rxq_ibv_cq_create(struct rte_eth_dev *dev, uint16_t idx)
 152 {
 153         struct mlx5_priv *priv = dev->data->dev_private;
 154         struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
 155         struct mlx5_rxq_ctrl *rxq_ctrl =
 156                 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
 157         struct mlx5_rxq_obj *rxq_obj = rxq_ctrl->obj;
 158         unsigned int cqe_n = mlx5_rxq_cqe_num(rxq_data);
 159         struct {
 160                 struct ibv_cq_init_attr_ex ibv;
 161                 struct mlx5dv_cq_init_attr mlx5;
 162         } cq_attr;
 163
 164         cq_attr.ibv = (struct ibv_cq_init_attr_ex){
 165                 .cqe = cqe_n,
 166                 .channel = rxq_obj->ibv_channel,
 167                 .comp_mask = 0,
 168         };
 169         cq_attr.mlx5 = (struct mlx5dv_cq_init_attr){
 170                 .comp_mask = 0,
 171         };
 172         if (priv->config.cqe_comp && !rxq_data->hw_timestamp) {
 173                 cq_attr.mlx5.comp_mask |=
 174                                 MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;
 175                 rxq_data->byte_mask = UINT32_MAX;
 176 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 177                 if (mlx5_rxq_mprq_enabled(rxq_data)) {
 178                         cq_attr.mlx5.cqe_comp_res_format =
 179                                         MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX;
 180                         rxq_data->mcqe_format =
 181                                         MLX5_CQE_RESP_FORMAT_CSUM_STRIDX;
 182                 } else {
 183                         cq_attr.mlx5.cqe_comp_res_format =
 184                                         MLX5DV_CQE_RES_FORMAT_HASH;
 185                         rxq_data->mcqe_format =
 186                                         MLX5_CQE_RESP_FORMAT_HASH;
 187                 }
 188 #else
 189                 cq_attr.mlx5.cqe_comp_res_format = MLX5DV_CQE_RES_FORMAT_HASH;
 190                 rxq_data->mcqe_format = MLX5_CQE_RESP_FORMAT_HASH;
 191 #endif
 192                 /*
 193                  * For vectorized Rx, it must not be doubled in order to
 194                  * make cq_ci and rq_ci aligned.
 195                  */
 196                 if (mlx5_rxq_check_vec_support(rxq_data) < 0)
 197                         cq_attr.ibv.cqe *= 2;
 198         } else if (priv->config.cqe_comp && rxq_data->hw_timestamp) {
 199                 DRV_LOG(DEBUG,
 200                         "Port %u Rx CQE compression is disabled for HW"
 201                         " timestamp.",
 202                         dev->data->port_id);
 203         }
 204 #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD
 205         if (RTE_CACHE_LINE_SIZE == 128) {
 206                 cq_attr.mlx5.comp_mask |= MLX5DV_CQ_INIT_ATTR_MASK_FLAGS;
 207                 cq_attr.mlx5.flags |= MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD;
 208         }
 209 #endif
 210         return mlx5_glue->cq_ex_to_cq(mlx5_glue->dv_create_cq
 211                                                            (priv->sh->cdev->ctx,
 212                                                             &cq_attr.ibv,
 213                                                             &cq_attr.mlx5));
 214 }
 215
 216 /**
 217  * Create a WQ Verbs object.
 218  *
 219  * @param dev
 220  *   Pointer to Ethernet device.
 221  * @param idx
 222  *   Queue index in DPDK Rx queue array.
 223  *
 224  * @return
 225  *   The Verbs WQ object initialized, NULL otherwise and rte_errno is set.
 226  */
 227 static struct ibv_wq *
 228 mlx5_rxq_ibv_wq_create(struct rte_eth_dev *dev, uint16_t idx)
 229 {
 230         struct mlx5_priv *priv = dev->data->dev_private;
 231         struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
 232         struct mlx5_rxq_ctrl *rxq_ctrl =
 233                 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
 234         struct mlx5_rxq_obj *rxq_obj = rxq_ctrl->obj;
 235         unsigned int wqe_n = 1 << rxq_data->elts_n;
 236         struct {
 237                 struct ibv_wq_init_attr ibv;
 238 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 239                 struct mlx5dv_wq_init_attr mlx5;
 240 #endif
 241         } wq_attr;
 242
 243         wq_attr.ibv = (struct ibv_wq_init_attr){
 244                 .wq_context = NULL, /* Could be useful in the future. */
 245                 .wq_type = IBV_WQT_RQ,
 246                 /* Max number of outstanding WRs. */
 247                 .max_wr = wqe_n >> rxq_data->sges_n,
 248                 /* Max number of scatter/gather elements in a WR. */
 249                 .max_sge = 1 << rxq_data->sges_n,
 250                 .pd = priv->sh->cdev->pd,
 251                 .cq = rxq_obj->ibv_cq,
 252                 .comp_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING | 0,
 253                 .create_flags = (rxq_data->vlan_strip ?
 254                                  IBV_WQ_FLAGS_CVLAN_STRIPPING : 0),
 255         };
 256         /* By default, FCS (CRC) is stripped by hardware. */
 257         if (rxq_data->crc_present) {
 258                 wq_attr.ibv.create_flags |= IBV_WQ_FLAGS_SCATTER_FCS;
 259                 wq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
 260         }
 261         if (priv->config.hw_padding) {
 262 #if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING)
 263                 wq_attr.ibv.create_flags |= IBV_WQ_FLAG_RX_END_PADDING;
 264                 wq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
 265 #elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING)
 266                 wq_attr.ibv.create_flags |= IBV_WQ_FLAGS_PCI_WRITE_END_PADDING;
 267                 wq_attr.ibv.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
 268 #endif
 269         }
 270 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 271         wq_attr.mlx5 = (struct mlx5dv_wq_init_attr){
 272                 .comp_mask = 0,
 273         };
 274         if (mlx5_rxq_mprq_enabled(rxq_data)) {
 275                 struct mlx5dv_striding_rq_init_attr *mprq_attr =
 276                                                 &wq_attr.mlx5.striding_rq_attrs;
 277
 278                 wq_attr.mlx5.comp_mask |= MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ;
 279                 *mprq_attr = (struct mlx5dv_striding_rq_init_attr){
 280                         .single_stride_log_num_of_bytes = rxq_data->strd_sz_n,
 281                         .single_wqe_log_num_of_strides = rxq_data->strd_num_n,
 282                         .two_byte_shift_en = MLX5_MPRQ_TWO_BYTE_SHIFT,
 283                 };
 284         }
 285         rxq_obj->wq = mlx5_glue->dv_create_wq(priv->sh->cdev->ctx, &wq_attr.ibv,
 286                                               &wq_attr.mlx5);
 287 #else
 288         rxq_obj->wq = mlx5_glue->create_wq(priv->sh->cdev->ctx, &wq_attr.ibv);
 289 #endif
 290         if (rxq_obj->wq) {
 291                 /*
 292                  * Make sure number of WRs*SGEs match expectations since a queue
 293                  * cannot allocate more than "desc" buffers.
 294                  */
 295                 if (wq_attr.ibv.max_wr != (wqe_n >> rxq_data->sges_n) ||
 296                     wq_attr.ibv.max_sge != (1u << rxq_data->sges_n)) {
 297                         DRV_LOG(ERR,
 298                                 "Port %u Rx queue %u requested %u*%u but got"
 299                                 " %u*%u WRs*SGEs.",
 300                                 dev->data->port_id, idx,
 301                                 wqe_n >> rxq_data->sges_n,
 302                                 (1 << rxq_data->sges_n),
 303                                 wq_attr.ibv.max_wr, wq_attr.ibv.max_sge);
 304                         claim_zero(mlx5_glue->destroy_wq(rxq_obj->wq));
 305                         rxq_obj->wq = NULL;
 306                         rte_errno = EINVAL;
 307                 }
 308         }
 309         return rxq_obj->wq;
 310 }
 311
 312 /**
 313  * Create the Rx queue Verbs object.
 314  *
 315  * @param dev
 316  *   Pointer to Ethernet device.
 317  * @param idx
 318  *   Queue index in DPDK Rx queue array.
 319  *
 320  * @return
 321  *   0 on success, a negative errno value otherwise and rte_errno is set.
 322  */
 323 static int
 324 mlx5_rxq_ibv_obj_new(struct rte_eth_dev *dev, uint16_t idx)
 325 {
 326         struct mlx5_priv *priv = dev->data->dev_private;
 327         struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
 328         struct mlx5_rxq_ctrl *rxq_ctrl =
 329                 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
 330         struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj;
 331         struct mlx5dv_cq cq_info;
 332         struct mlx5dv_rwq rwq;
 333         int ret = 0;
 334         struct mlx5dv_obj obj;
 335
 336         MLX5_ASSERT(rxq_data);
 337         MLX5_ASSERT(tmpl);
 338         tmpl->rxq_ctrl = rxq_ctrl;
 339         if (rxq_ctrl->irq) {
 340                 tmpl->ibv_channel =
 341                         mlx5_glue->create_comp_channel(priv->sh->cdev->ctx);
 342                 if (!tmpl->ibv_channel) {
 343                         DRV_LOG(ERR, "Port %u: comp channel creation failure.",
 344                                 dev->data->port_id);
 345                         rte_errno = ENOMEM;
 346                         goto error;
 347                 }
 348                 tmpl->fd = ((struct ibv_comp_channel *)(tmpl->ibv_channel))->fd;
 349         }
 350         /* Create CQ using Verbs API. */
 351         tmpl->ibv_cq = mlx5_rxq_ibv_cq_create(dev, idx);
 352         if (!tmpl->ibv_cq) {
 353                 DRV_LOG(ERR, "Port %u Rx queue %u CQ creation failure.",
 354                         dev->data->port_id, idx);
 355                 rte_errno = ENOMEM;
 356                 goto error;
 357         }
 358         obj.cq.in = tmpl->ibv_cq;
 359         obj.cq.out = &cq_info;
 360         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ);
 361         if (ret) {
 362                 rte_errno = ret;
 363                 goto error;
 364         }
 365         if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {
 366                 DRV_LOG(ERR,
 367                         "Port %u wrong MLX5_CQE_SIZE environment "
 368                         "variable value: it should be set to %u.",
 369                         dev->data->port_id, RTE_CACHE_LINE_SIZE);
 370                 rte_errno = EINVAL;
 371                 goto error;
 372         }
 373         /* Fill the rings. */
 374         rxq_data->cqe_n = log2above(cq_info.cqe_cnt);
 375         rxq_data->cq_db = cq_info.dbrec;
 376         rxq_data->cqes = (volatile struct mlx5_cqe (*)[])(uintptr_t)cq_info.buf;
 377         rxq_data->cq_uar = cq_info.cq_uar;
 378         rxq_data->cqn = cq_info.cqn;
 379         /* Create WQ (RQ) using Verbs API. */
 380         tmpl->wq = mlx5_rxq_ibv_wq_create(dev, idx);
 381         if (!tmpl->wq) {
 382                 DRV_LOG(ERR, "Port %u Rx queue %u WQ creation failure.",
 383                         dev->data->port_id, idx);
 384                 rte_errno = ENOMEM;
 385                 goto error;
 386         }
 387         /* Change queue state to ready. */
 388         ret = mlx5_ibv_modify_wq(tmpl, IBV_WQS_RDY);
 389         if (ret) {
 390                 DRV_LOG(ERR,
 391                         "Port %u Rx queue %u WQ state to IBV_WQS_RDY failed.",
 392                         dev->data->port_id, idx);
 393                 rte_errno = ret;
 394                 goto error;
 395         }
 396         obj.rwq.in = tmpl->wq;
 397         obj.rwq.out = &rwq;
 398         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_RWQ);
 399         if (ret) {
 400                 rte_errno = ret;
 401                 goto error;
 402         }
 403         rxq_data->wqes = rwq.buf;
 404         rxq_data->rq_db = rwq.dbrec;
 405         rxq_data->cq_arm_sn = 0;
 406         mlx5_rxq_initialize(rxq_data);
 407         rxq_data->cq_ci = 0;
 408         dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;
 409         rxq_ctrl->wqn = ((struct ibv_wq *)(tmpl->wq))->wq_num;
 410         return 0;
 411 error:
 412         ret = rte_errno; /* Save rte_errno before cleanup. */
 413         if (tmpl->wq)
 414                 claim_zero(mlx5_glue->destroy_wq(tmpl->wq));
 415         if (tmpl->ibv_cq)
 416                 claim_zero(mlx5_glue->destroy_cq(tmpl->ibv_cq));
 417         if (tmpl->ibv_channel)
 418                 claim_zero(mlx5_glue->destroy_comp_channel(tmpl->ibv_channel));
 419         rte_errno = ret; /* Restore rte_errno. */
 420         return -rte_errno;
 421 }
 422
 423 /**
 424  * Release an Rx verbs queue object.
 425  *
 426  * @param rxq_obj
 427  *   Verbs Rx queue object.
 428  */
 429 static void
 430 mlx5_rxq_ibv_obj_release(struct mlx5_rxq_obj *rxq_obj)
 431 {
 432         MLX5_ASSERT(rxq_obj);
 433         MLX5_ASSERT(rxq_obj->wq);
 434         MLX5_ASSERT(rxq_obj->ibv_cq);
 435         claim_zero(mlx5_glue->destroy_wq(rxq_obj->wq));
 436         claim_zero(mlx5_glue->destroy_cq(rxq_obj->ibv_cq));
 437         if (rxq_obj->ibv_channel)
 438                 claim_zero(mlx5_glue->destroy_comp_channel
 439                                                         (rxq_obj->ibv_channel));
 440 }
 441
 442 /**
 443  * Get event for an Rx verbs queue object.
 444  *
 445  * @param rxq_obj
 446  *   Verbs Rx queue object.
 447  *
 448  * @return
 449  *   0 on success, a negative errno value otherwise and rte_errno is set.
 450  */
 451 static int
 452 mlx5_rx_ibv_get_event(struct mlx5_rxq_obj *rxq_obj)
 453 {
 454         struct ibv_cq *ev_cq;
 455         void *ev_ctx;
 456         int ret = mlx5_glue->get_cq_event(rxq_obj->ibv_channel,
 457                                           &ev_cq, &ev_ctx);
 458
 459         if (ret < 0 || ev_cq != rxq_obj->ibv_cq)
 460                 goto exit;
 461         mlx5_glue->ack_cq_events(rxq_obj->ibv_cq, 1);
 462         return 0;
 463 exit:
 464         if (ret < 0)
 465                 rte_errno = errno;
 466         else
 467                 rte_errno = EINVAL;
 468         return -rte_errno;
 469 }
 470
 471 /**
 472  * Creates a receive work queue as a filed of indirection table.
 473  *
 474  * @param dev
 475  *   Pointer to Ethernet device.
 476  * @param log_n
 477  *   Log of number of queues in the array.
 478  * @param ind_tbl
 479  *   Verbs indirection table object.
 480  *
 481  * @return
 482  *   0 on success, a negative errno value otherwise and rte_errno is set.
 483  */
 484 static int
 485 mlx5_ibv_ind_table_new(struct rte_eth_dev *dev, const unsigned int log_n,
 486                        struct mlx5_ind_table_obj *ind_tbl)
 487 {
 488         struct mlx5_priv *priv = dev->data->dev_private;
 489         struct ibv_wq *wq[1 << log_n];
 490         unsigned int i, j;
 491
 492         MLX5_ASSERT(ind_tbl);
 493         for (i = 0; i != ind_tbl->queues_n; ++i) {
 494                 struct mlx5_rxq_data *rxq = (*priv->rxqs)[ind_tbl->queues[i]];
 495                 struct mlx5_rxq_ctrl *rxq_ctrl =
 496                                 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 497
 498                 wq[i] = rxq_ctrl->obj->wq;
 499         }
 500         MLX5_ASSERT(i > 0);
 501         /* Finalise indirection table. */
 502         for (j = 0; i != (unsigned int)(1 << log_n); ++j, ++i)
 503                 wq[i] = wq[j];
 504         ind_tbl->ind_table = mlx5_glue->create_rwq_ind_table
 505                                         (priv->sh->cdev->ctx,
 506                                          &(struct ibv_rwq_ind_table_init_attr){
 507                                                  .log_ind_tbl_size = log_n,
 508                                                  .ind_tbl = wq,
 509                                                  .comp_mask = 0,
 510                                          });
 511         if (!ind_tbl->ind_table) {
 512                 rte_errno = errno;
 513                 return -rte_errno;
 514         }
 515         return 0;
 516 }
 517
 518 /**
 519  * Destroys the specified Indirection Table.
 520  *
 521  * @param ind_table
 522  *   Indirection table to release.
 523  */
 524 static void
 525 mlx5_ibv_ind_table_destroy(struct mlx5_ind_table_obj *ind_tbl)
 526 {
 527         claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl->ind_table));
 528 }
 529
 530 /**
 531  * Create an Rx Hash queue.
 532  *
 533  * @param dev
 534  *   Pointer to Ethernet device.
 535  * @param hrxq
 536  *   Pointer to Rx Hash queue.
 537  * @param tunnel
 538  *   Tunnel type.
 539  *
 540  * @return
 541  *   0 on success, a negative errno value otherwise and rte_errno is set.
 542  */
 543 static int
 544 mlx5_ibv_hrxq_new(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq,
 545                   int tunnel __rte_unused)
 546 {
 547         struct mlx5_priv *priv = dev->data->dev_private;
 548         struct ibv_qp *qp = NULL;
 549         struct mlx5_ind_table_obj *ind_tbl = hrxq->ind_table;
 550         const uint8_t *rss_key = hrxq->rss_key;
 551         uint64_t hash_fields = hrxq->hash_fields;
 552         int err;
 553 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 554         struct mlx5dv_qp_init_attr qp_init_attr;
 555
 556         memset(&qp_init_attr, 0, sizeof(qp_init_attr));
 557         if (tunnel) {
 558                 qp_init_attr.comp_mask =
 559                                        MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
 560                 qp_init_attr.create_flags = MLX5DV_QP_CREATE_TUNNEL_OFFLOADS;
 561         }
 562 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 563         if (dev->data->dev_conf.lpbk_mode) {
 564                 /* Allow packet sent from NIC loop back w/o source MAC check. */
 565                 qp_init_attr.comp_mask |=
 566                                 MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
 567                 qp_init_attr.create_flags |=
 568                                 MLX5DV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_UC;
 569         }
 570 #endif
 571         qp = mlx5_glue->dv_create_qp
 572                         (priv->sh->cdev->ctx,
 573                          &(struct ibv_qp_init_attr_ex){
 574                                 .qp_type = IBV_QPT_RAW_PACKET,
 575                                 .comp_mask =
 576                                         IBV_QP_INIT_ATTR_PD |
 577                                         IBV_QP_INIT_ATTR_IND_TABLE |
 578                                         IBV_QP_INIT_ATTR_RX_HASH,
 579                                 .rx_hash_conf = (struct ibv_rx_hash_conf){
 580                                         .rx_hash_function =
 581                                                 IBV_RX_HASH_FUNC_TOEPLITZ,
 582                                         .rx_hash_key_len = hrxq->rss_key_len,
 583                                         .rx_hash_key =
 584                                                 (void *)(uintptr_t)rss_key,
 585                                         .rx_hash_fields_mask = hash_fields,
 586                                 },
 587                                 .rwq_ind_tbl = ind_tbl->ind_table,
 588                                 .pd = priv->sh->cdev->pd,
 589                           },
 590                           &qp_init_attr);
 591 #else
 592         qp = mlx5_glue->create_qp_ex
 593                         (priv->sh->cdev->ctx,
 594                          &(struct ibv_qp_init_attr_ex){
 595                                 .qp_type = IBV_QPT_RAW_PACKET,
 596                                 .comp_mask =
 597                                         IBV_QP_INIT_ATTR_PD |
 598                                         IBV_QP_INIT_ATTR_IND_TABLE |
 599                                         IBV_QP_INIT_ATTR_RX_HASH,
 600                                 .rx_hash_conf = (struct ibv_rx_hash_conf){
 601                                         .rx_hash_function =
 602                                                 IBV_RX_HASH_FUNC_TOEPLITZ,
 603                                         .rx_hash_key_len = hrxq->rss_key_len,
 604                                         .rx_hash_key =
 605                                                 (void *)(uintptr_t)rss_key,
 606                                         .rx_hash_fields_mask = hash_fields,
 607                                 },
 608                                 .rwq_ind_tbl = ind_tbl->ind_table,
 609                                 .pd = priv->sh->cdev->pd,
 610                          });
 611 #endif
 612         if (!qp) {
 613                 rte_errno = errno;
 614                 goto error;
 615         }
 616         hrxq->qp = qp;
 617 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 618         hrxq->action = mlx5_glue->dv_create_flow_action_dest_ibv_qp(hrxq->qp);
 619         if (!hrxq->action) {
 620                 rte_errno = errno;
 621                 goto error;
 622         }
 623 #endif
 624         return 0;
 625 error:
 626         err = rte_errno; /* Save rte_errno before cleanup. */
 627         if (qp)
 628                 claim_zero(mlx5_glue->destroy_qp(qp));
 629         rte_errno = err; /* Restore rte_errno. */
 630         return -rte_errno;
 631 }
 632
 633 /**
 634  * Destroy a Verbs queue pair.
 635  *
 636  * @param hrxq
 637  *   Hash Rx queue to release its qp.
 638  */
 639 static void
 640 mlx5_ibv_qp_destroy(struct mlx5_hrxq *hrxq)
 641 {
 642         claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
 643 }
 644
 645 /**
 646  * Release a drop Rx queue Verbs object.
 647  *
 648  * @param dev
 649  *   Pointer to Ethernet device.
 650  */
 651 static void
 652 mlx5_rxq_ibv_obj_drop_release(struct rte_eth_dev *dev)
 653 {
 654         struct mlx5_priv *priv = dev->data->dev_private;
 655         struct mlx5_rxq_obj *rxq = priv->drop_queue.rxq;
 656
 657         if (rxq->wq)
 658                 claim_zero(mlx5_glue->destroy_wq(rxq->wq));
 659         if (rxq->ibv_cq)
 660                 claim_zero(mlx5_glue->destroy_cq(rxq->ibv_cq));
 661         mlx5_free(rxq);
 662         priv->drop_queue.rxq = NULL;
 663 }
 664
 665 /**
 666  * Create a drop Rx queue Verbs object.
 667  *
 668  * @param dev
 669  *   Pointer to Ethernet device.
 670  *
 671  * @return
 672  *   0 on success, a negative errno value otherwise and rte_errno is set.
 673  */
 674 static int
 675 mlx5_rxq_ibv_obj_drop_create(struct rte_eth_dev *dev)
 676 {
 677         struct mlx5_priv *priv = dev->data->dev_private;
 678         struct ibv_context *ctx = priv->sh->cdev->ctx;
 679         struct mlx5_rxq_obj *rxq = priv->drop_queue.rxq;
 680
 681         if (rxq)
 682                 return 0;
 683         rxq = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq), 0, SOCKET_ID_ANY);
 684         if (!rxq) {
 685                 DRV_LOG(DEBUG, "Port %u cannot allocate drop Rx queue memory.",
 686                       dev->data->port_id);
 687                 rte_errno = ENOMEM;
 688                 return -rte_errno;
 689         }
 690         priv->drop_queue.rxq = rxq;
 691         rxq->ibv_cq = mlx5_glue->create_cq(ctx, 1, NULL, NULL, 0);
 692         if (!rxq->ibv_cq) {
 693                 DRV_LOG(DEBUG, "Port %u cannot allocate CQ for drop queue.",
 694                       dev->data->port_id);
 695                 rte_errno = errno;
 696                 goto error;
 697         }
 698         rxq->wq = mlx5_glue->create_wq(ctx, &(struct ibv_wq_init_attr){
 699                                                     .wq_type = IBV_WQT_RQ,
 700                                                     .max_wr = 1,
 701                                                     .max_sge = 1,
 702                                                     .pd = priv->sh->cdev->pd,
 703                                                     .cq = rxq->ibv_cq,
 704                                               });
 705         if (!rxq->wq) {
 706                 DRV_LOG(DEBUG, "Port %u cannot allocate WQ for drop queue.",
 707                       dev->data->port_id);
 708                 rte_errno = errno;
 709                 goto error;
 710         }
 711         priv->drop_queue.rxq = rxq;
 712         return 0;
 713 error:
 714         mlx5_rxq_ibv_obj_drop_release(dev);
 715         return -rte_errno;
 716 }
 717
 718 /**
 719  * Create a Verbs drop action for Rx Hash queue.
 720  *
 721  * @param dev
 722  *   Pointer to Ethernet device.
 723  *
 724  * @return
 725  *   0 on success, a negative errno value otherwise and rte_errno is set.
 726  */
 727 static int
 728 mlx5_ibv_drop_action_create(struct rte_eth_dev *dev)
 729 {
 730         struct mlx5_priv *priv = dev->data->dev_private;
 731         struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq;
 732         struct ibv_rwq_ind_table *ind_tbl = NULL;
 733         struct mlx5_rxq_obj *rxq;
 734         int ret;
 735
 736         MLX5_ASSERT(hrxq && hrxq->ind_table);
 737         ret = mlx5_rxq_ibv_obj_drop_create(dev);
 738         if (ret < 0)
 739                 goto error;
 740         rxq = priv->drop_queue.rxq;
 741         ind_tbl = mlx5_glue->create_rwq_ind_table
 742                                 (priv->sh->cdev->ctx,
 743                                  &(struct ibv_rwq_ind_table_init_attr){
 744                                         .log_ind_tbl_size = 0,
 745                                         .ind_tbl = (struct ibv_wq **)&rxq->wq,
 746                                         .comp_mask = 0,
 747                                  });
 748         if (!ind_tbl) {
 749                 DRV_LOG(DEBUG, "Port %u"
 750                         " cannot allocate indirection table for drop queue.",
 751                         dev->data->port_id);
 752                 rte_errno = errno;
 753                 goto error;
 754         }
 755         hrxq->qp = mlx5_glue->create_qp_ex(priv->sh->cdev->ctx,
 756                  &(struct ibv_qp_init_attr_ex){
 757                         .qp_type = IBV_QPT_RAW_PACKET,
 758                         .comp_mask = IBV_QP_INIT_ATTR_PD |
 759                                      IBV_QP_INIT_ATTR_IND_TABLE |
 760                                      IBV_QP_INIT_ATTR_RX_HASH,
 761                         .rx_hash_conf = (struct ibv_rx_hash_conf){
 762                                 .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ,
 763                                 .rx_hash_key_len = MLX5_RSS_HASH_KEY_LEN,
 764                                 .rx_hash_key = rss_hash_default_key,
 765                                 .rx_hash_fields_mask = 0,
 766                                 },
 767                         .rwq_ind_tbl = ind_tbl,
 768                         .pd = priv->sh->cdev->pd
 769                  });
 770         if (!hrxq->qp) {
 771                 DRV_LOG(DEBUG, "Port %u cannot allocate QP for drop queue.",
 772                       dev->data->port_id);
 773                 rte_errno = errno;
 774                 goto error;
 775         }
 776 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 777         hrxq->action = mlx5_glue->dv_create_flow_action_dest_ibv_qp(hrxq->qp);
 778         if (!hrxq->action) {
 779                 rte_errno = errno;
 780                 goto error;
 781         }
 782 #endif
 783         hrxq->ind_table->ind_table = ind_tbl;
 784         return 0;
 785 error:
 786         if (hrxq->qp)
 787                 claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
 788         if (ind_tbl)
 789                 claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl));
 790         if (priv->drop_queue.rxq)
 791                 mlx5_rxq_ibv_obj_drop_release(dev);
 792         return -rte_errno;
 793 }
 794
 795 /**
 796  * Release a drop hash Rx queue.
 797  *
 798  * @param dev
 799  *   Pointer to Ethernet device.
 800  */
 801 static void
 802 mlx5_ibv_drop_action_destroy(struct rte_eth_dev *dev)
 803 {
 804         struct mlx5_priv *priv = dev->data->dev_private;
 805         struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq;
 806         struct ibv_rwq_ind_table *ind_tbl = hrxq->ind_table->ind_table;
 807
 808 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 809         claim_zero(mlx5_glue->destroy_flow_action(hrxq->action));
 810 #endif
 811         claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
 812         claim_zero(mlx5_glue->destroy_rwq_ind_table(ind_tbl));
 813         mlx5_rxq_ibv_obj_drop_release(dev);
 814 }
 815
 816 /**
 817  * Create a QP Verbs object.
 818  *
 819  * @param dev
 820  *   Pointer to Ethernet device.
 821  * @param idx
 822  *   Queue index in DPDK Tx queue array.
 823  *
 824  * @return
 825  *   The QP Verbs object, NULL otherwise and rte_errno is set.
 826  */
 827 static struct ibv_qp *
 828 mlx5_txq_ibv_qp_create(struct rte_eth_dev *dev, uint16_t idx)
 829 {
 830         struct mlx5_priv *priv = dev->data->dev_private;
 831         struct mlx5_txq_data *txq_data = (*priv->txqs)[idx];
 832         struct mlx5_txq_ctrl *txq_ctrl =
 833                         container_of(txq_data, struct mlx5_txq_ctrl, txq);
 834         struct ibv_qp *qp_obj = NULL;
 835         struct ibv_qp_init_attr_ex qp_attr = { 0 };
 836         const int desc = 1 << txq_data->elts_n;
 837
 838         MLX5_ASSERT(txq_ctrl->obj->cq);
 839         /* CQ to be associated with the send queue. */
 840         qp_attr.send_cq = txq_ctrl->obj->cq;
 841         /* CQ to be associated with the receive queue. */
 842         qp_attr.recv_cq = txq_ctrl->obj->cq;
 843         /* Max number of outstanding WRs. */
 844         qp_attr.cap.max_send_wr = ((priv->sh->device_attr.max_qp_wr < desc) ?
 845                                    priv->sh->device_attr.max_qp_wr : desc);
 846         /*
 847          * Max number of scatter/gather elements in a WR, must be 1 to prevent
 848          * libmlx5 from trying to affect must be 1 to prevent libmlx5 from
 849          * trying to affect too much memory. TX gather is not impacted by the
 850          * device_attr.max_sge limit and will still work properly.
 851          */
 852         qp_attr.cap.max_send_sge = 1;
 853         qp_attr.qp_type = IBV_QPT_RAW_PACKET,
 854         /* Do *NOT* enable this, completions events are managed per Tx burst. */
 855         qp_attr.sq_sig_all = 0;
 856         qp_attr.pd = priv->sh->cdev->pd;
 857         qp_attr.comp_mask = IBV_QP_INIT_ATTR_PD;
 858         if (txq_data->inlen_send)
 859                 qp_attr.cap.max_inline_data = txq_ctrl->max_inline_data;
 860         if (txq_data->tso_en) {
 861                 qp_attr.max_tso_header = txq_ctrl->max_tso_header;
 862                 qp_attr.comp_mask |= IBV_QP_INIT_ATTR_MAX_TSO_HEADER;
 863         }
 864         qp_obj = mlx5_glue->create_qp_ex(priv->sh->cdev->ctx, &qp_attr);
 865         if (qp_obj == NULL) {
 866                 DRV_LOG(ERR, "Port %u Tx queue %u QP creation failure.",
 867                         dev->data->port_id, idx);
 868                 rte_errno = errno;
 869         }
 870         return qp_obj;
 871 }
 872
 873 /**
 874  * Create the Tx queue Verbs object.
 875  *
 876  * @param dev
 877  *   Pointer to Ethernet device.
 878  * @param idx
 879  *   Queue index in DPDK Tx queue array.
 880  *
 881  * @return
 882  *   0 on success, a negative errno value otherwise and rte_errno is set.
 883  */
 884 int
 885 mlx5_txq_ibv_obj_new(struct rte_eth_dev *dev, uint16_t idx)
 886 {
 887         struct mlx5_priv *priv = dev->data->dev_private;
 888         struct mlx5_txq_data *txq_data = (*priv->txqs)[idx];
 889         struct mlx5_txq_ctrl *txq_ctrl =
 890                 container_of(txq_data, struct mlx5_txq_ctrl, txq);
 891         struct mlx5_txq_obj *txq_obj = txq_ctrl->obj;
 892         unsigned int cqe_n;
 893         struct mlx5dv_qp qp;
 894         struct mlx5dv_cq cq_info;
 895         struct mlx5dv_obj obj;
 896         const int desc = 1 << txq_data->elts_n;
 897         int ret = 0;
 898
 899         MLX5_ASSERT(txq_data);
 900         MLX5_ASSERT(txq_obj);
 901         txq_obj->txq_ctrl = txq_ctrl;
 902         if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) {
 903                 DRV_LOG(ERR, "Port %u MLX5_ENABLE_CQE_COMPRESSION "
 904                         "must never be set.", dev->data->port_id);
 905                 rte_errno = EINVAL;
 906                 return -rte_errno;
 907         }
 908         cqe_n = desc / MLX5_TX_COMP_THRESH +
 909                 1 + MLX5_TX_COMP_THRESH_INLINE_DIV;
 910         txq_obj->cq = mlx5_glue->create_cq(priv->sh->cdev->ctx, cqe_n,
 911                                            NULL, NULL, 0);
 912         if (txq_obj->cq == NULL) {
 913                 DRV_LOG(ERR, "Port %u Tx queue %u CQ creation failure.",
 914                         dev->data->port_id, idx);
 915                 rte_errno = errno;
 916                 goto error;
 917         }
 918         txq_obj->qp = mlx5_txq_ibv_qp_create(dev, idx);
 919         if (txq_obj->qp == NULL) {
 920                 rte_errno = errno;
 921                 goto error;
 922         }
 923         ret = mlx5_ibv_modify_qp(txq_obj, MLX5_TXQ_MOD_RST2RDY,
 924                                  (uint8_t)priv->dev_port);
 925         if (ret) {
 926                 DRV_LOG(ERR, "Port %u Tx queue %u QP state modifying failed.",
 927                         dev->data->port_id, idx);
 928                 rte_errno = errno;
 929                 goto error;
 930         }
 931         qp.comp_mask = MLX5DV_QP_MASK_UAR_MMAP_OFFSET;
 932 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 933         /* If using DevX, need additional mask to read tisn value. */
 934         if (priv->sh->devx && !priv->sh->tdn)
 935                 qp.comp_mask |= MLX5DV_QP_MASK_RAW_QP_HANDLES;
 936 #endif
 937         obj.cq.in = txq_obj->cq;
 938         obj.cq.out = &cq_info;
 939         obj.qp.in = txq_obj->qp;
 940         obj.qp.out = &qp;
 941         ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_QP);
 942         if (ret != 0) {
 943                 rte_errno = errno;
 944                 goto error;
 945         }
 946         if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {
 947                 DRV_LOG(ERR,
 948                         "Port %u wrong MLX5_CQE_SIZE environment variable"
 949                         " value: it should be set to %u.",
 950                         dev->data->port_id, RTE_CACHE_LINE_SIZE);
 951                 rte_errno = EINVAL;
 952                 goto error;
 953         }
 954         txq_data->cqe_n = log2above(cq_info.cqe_cnt);
 955         txq_data->cqe_s = 1 << txq_data->cqe_n;
 956         txq_data->cqe_m = txq_data->cqe_s - 1;
 957         txq_data->qp_num_8s = ((struct ibv_qp *)txq_obj->qp)->qp_num << 8;
 958         txq_data->wqes = qp.sq.buf;
 959         txq_data->wqe_n = log2above(qp.sq.wqe_cnt);
 960         txq_data->wqe_s = 1 << txq_data->wqe_n;
 961         txq_data->wqe_m = txq_data->wqe_s - 1;
 962         txq_data->wqes_end = txq_data->wqes + txq_data->wqe_s;
 963         txq_data->qp_db = &qp.dbrec[MLX5_SND_DBR];
 964         txq_data->cq_db = cq_info.dbrec;
 965         txq_data->cqes = (volatile struct mlx5_cqe *)cq_info.buf;
 966         txq_data->cq_ci = 0;
 967         txq_data->cq_pi = 0;
 968         txq_data->wqe_ci = 0;
 969         txq_data->wqe_pi = 0;
 970         txq_data->wqe_comp = 0;
 971         txq_data->wqe_thres = txq_data->wqe_s / MLX5_TX_COMP_THRESH_INLINE_DIV;
 972 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 973         /*
 974          * If using DevX need to query and store TIS transport domain value.
 975          * This is done once per port.
 976          * Will use this value on Rx, when creating matching TIR.
 977          */
 978         if (priv->sh->devx && !priv->sh->tdn) {
 979                 ret = mlx5_devx_cmd_qp_query_tis_td(txq_obj->qp, qp.tisn,
 980                                                     &priv->sh->tdn);
 981                 if (ret) {
 982                         DRV_LOG(ERR, "Fail to query port %u Tx queue %u QP TIS "
 983                                 "transport domain.", dev->data->port_id, idx);
 984                         rte_errno = EINVAL;
 985                         goto error;
 986                 } else {
 987                         DRV_LOG(DEBUG, "Port %u Tx queue %u TIS number %d "
 988                                 "transport domain %d.", dev->data->port_id,
 989                                 idx, qp.tisn, priv->sh->tdn);
 990                 }
 991         }
 992 #endif
 993         txq_ctrl->bf_reg = qp.bf.reg;
 994         if (qp.comp_mask & MLX5DV_QP_MASK_UAR_MMAP_OFFSET) {
 995                 txq_ctrl->uar_mmap_offset = qp.uar_mmap_offset;
 996                 DRV_LOG(DEBUG, "Port %u: uar_mmap_offset 0x%" PRIx64 ".",
 997                         dev->data->port_id, txq_ctrl->uar_mmap_offset);
 998         } else {
 999                 DRV_LOG(ERR,
1000                         "Port %u failed to retrieve UAR info, invalid"
1001                         " libmlx5.so",
1002                         dev->data->port_id);
1003                 rte_errno = EINVAL;
1004                 goto error;
1005         }
1006         txq_uar_init(txq_ctrl);
1007         dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;
1008         return 0;
1009 error:
1010         ret = rte_errno; /* Save rte_errno before cleanup. */
1011         if (txq_obj->cq)
1012                 claim_zero(mlx5_glue->destroy_cq(txq_obj->cq));
1013         if (txq_obj->qp)
1014                 claim_zero(mlx5_glue->destroy_qp(txq_obj->qp));
1015         rte_errno = ret; /* Restore rte_errno. */
1016         return -rte_errno;
1017 }
1018
1019 /*
1020  * Create the dummy QP with minimal resources for loopback.
1021  *
1022  * @param dev
1023  *   Pointer to Ethernet device.
1024  *
1025  * @return
1026  *   0 on success, a negative errno value otherwise and rte_errno is set.
1027  */
1028 int
1029 mlx5_rxq_ibv_obj_dummy_lb_create(struct rte_eth_dev *dev)
1030 {
1031 #if defined(HAVE_IBV_DEVICE_TUNNEL_SUPPORT) && defined(HAVE_IBV_FLOW_DV_SUPPORT)
1032         struct mlx5_priv *priv = dev->data->dev_private;
1033         struct mlx5_dev_ctx_shared *sh = priv->sh;
1034         struct ibv_context *ctx = sh->cdev->ctx;
1035         struct mlx5dv_qp_init_attr qp_init_attr = {0};
1036         struct {
1037                 struct ibv_cq_init_attr_ex ibv;
1038                 struct mlx5dv_cq_init_attr mlx5;
1039         } cq_attr = {{0}};
1040
1041         if (dev->data->dev_conf.lpbk_mode) {
1042                 /* Allow packet sent from NIC loop back w/o source MAC check. */
1043                 qp_init_attr.comp_mask |=
1044                                 MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
1045                 qp_init_attr.create_flags |=
1046                                 MLX5DV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_UC;
1047         } else {
1048                 return 0;
1049         }
1050         /* Only need to check refcnt, 0 after "sh" is allocated. */
1051         if (!!(__atomic_fetch_add(&sh->self_lb.refcnt, 1, __ATOMIC_RELAXED))) {
1052                 MLX5_ASSERT(sh->self_lb.ibv_cq && sh->self_lb.qp);
1053                 priv->lb_used = 1;
1054                 return 0;
1055         }
1056         cq_attr.ibv = (struct ibv_cq_init_attr_ex){
1057                 .cqe = 1,
1058                 .channel = NULL,
1059                 .comp_mask = 0,
1060         };
1061         cq_attr.mlx5 = (struct mlx5dv_cq_init_attr){
1062                 .comp_mask = 0,
1063         };
1064         /* Only CQ is needed, no WQ(RQ) is required in this case. */
1065         sh->self_lb.ibv_cq = mlx5_glue->cq_ex_to_cq(mlx5_glue->dv_create_cq(ctx,
1066                                                         &cq_attr.ibv,
1067                                                         &cq_attr.mlx5));
1068         if (!sh->self_lb.ibv_cq) {
1069                 DRV_LOG(ERR, "Port %u cannot allocate CQ for loopback.",
1070                         dev->data->port_id);
1071                 rte_errno = errno;
1072                 goto error;
1073         }
1074         sh->self_lb.qp = mlx5_glue->dv_create_qp(ctx,
1075                                 &(struct ibv_qp_init_attr_ex){
1076                                         .qp_type = IBV_QPT_RAW_PACKET,
1077                                         .comp_mask = IBV_QP_INIT_ATTR_PD,
1078                                         .pd = sh->cdev->pd,
1079                                         .send_cq = sh->self_lb.ibv_cq,
1080                                         .recv_cq = sh->self_lb.ibv_cq,
1081                                         .cap.max_recv_wr = 1,
1082                                 },
1083                                 &qp_init_attr);
1084         if (!sh->self_lb.qp) {
1085                 DRV_LOG(DEBUG, "Port %u cannot allocate QP for loopback.",
1086                         dev->data->port_id);
1087                 rte_errno = errno;
1088                 goto error;
1089         }
1090         priv->lb_used = 1;
1091         return 0;
1092 error:
1093         if (sh->self_lb.ibv_cq) {
1094                 claim_zero(mlx5_glue->destroy_cq(sh->self_lb.ibv_cq));
1095                 sh->self_lb.ibv_cq = NULL;
1096         }
1097         (void)__atomic_sub_fetch(&sh->self_lb.refcnt, 1, __ATOMIC_RELAXED);
1098         return -rte_errno;
1099 #else
1100         RTE_SET_USED(dev);
1101         return 0;
1102 #endif
1103 }
1104
1105 /*
1106  * Release the dummy queue resources for loopback.
1107  *
1108  * @param dev
1109  *   Pointer to Ethernet device.
1110  */
1111 void
1112 mlx5_rxq_ibv_obj_dummy_lb_release(struct rte_eth_dev *dev)
1113 {
1114 #if defined(HAVE_IBV_DEVICE_TUNNEL_SUPPORT) && defined(HAVE_IBV_FLOW_DV_SUPPORT)
1115         struct mlx5_priv *priv = dev->data->dev_private;
1116         struct mlx5_dev_ctx_shared *sh = priv->sh;
1117
1118         if (!priv->lb_used)
1119                 return;
1120         MLX5_ASSERT(__atomic_load_n(&sh->self_lb.refcnt, __ATOMIC_RELAXED));
1121         if (!(__atomic_sub_fetch(&sh->self_lb.refcnt, 1, __ATOMIC_RELAXED))) {
1122                 if (sh->self_lb.qp) {
1123                         claim_zero(mlx5_glue->destroy_qp(sh->self_lb.qp));
1124                         sh->self_lb.qp = NULL;
1125                 }
1126                 if (sh->self_lb.ibv_cq) {
1127                         claim_zero(mlx5_glue->destroy_cq(sh->self_lb.ibv_cq));
1128                         sh->self_lb.ibv_cq = NULL;
1129                 }
1130         }
1131         priv->lb_used = 0;
1132 #else
1133         RTE_SET_USED(dev);
1134         return;
1135 #endif
1136 }
1137
1138 /**
1139  * Release an Tx verbs queue object.
1140  *
1141  * @param txq_obj
1142  *   Verbs Tx queue object..
1143  */
1144 void
1145 mlx5_txq_ibv_obj_release(struct mlx5_txq_obj *txq_obj)
1146 {
1147         MLX5_ASSERT(txq_obj);
1148         claim_zero(mlx5_glue->destroy_qp(txq_obj->qp));
1149         claim_zero(mlx5_glue->destroy_cq(txq_obj->cq));
1150 }
1151
1152 struct mlx5_obj_ops ibv_obj_ops = {
1153         .rxq_obj_modify_vlan_strip = mlx5_rxq_obj_modify_wq_vlan_strip,
1154         .rxq_obj_new = mlx5_rxq_ibv_obj_new,
1155         .rxq_event_get = mlx5_rx_ibv_get_event,
1156         .rxq_obj_modify = mlx5_ibv_modify_wq,
1157         .rxq_obj_release = mlx5_rxq_ibv_obj_release,
1158         .ind_table_new = mlx5_ibv_ind_table_new,
1159         .ind_table_destroy = mlx5_ibv_ind_table_destroy,
1160         .hrxq_new = mlx5_ibv_hrxq_new,
1161         .hrxq_destroy = mlx5_ibv_qp_destroy,
1162         .drop_action_create = mlx5_ibv_drop_action_create,
1163         .drop_action_destroy = mlx5_ibv_drop_action_destroy,
1164         .txq_obj_new = mlx5_txq_ibv_obj_new,
1165         .txq_obj_modify = mlx5_ibv_modify_qp,
1166         .txq_obj_release = mlx5_txq_ibv_obj_release,
1167         .lb_dummy_queue_create = NULL,
1168         .lb_dummy_queue_release = NULL,
1169 };