net/mlx5: support upstream rdma-core
authorShachar Beiser <shacharbe@mellanox.com>
Tue, 26 Sep 2017 15:38:24 +0000 (17:38 +0200)
committerFerruh Yigit <ferruh.yigit@intel.com>
Fri, 6 Oct 2017 00:49:49 +0000 (02:49 +0200)
This removes the dependency on specific Mellanox OFED libraries by
using the upstream rdma-core and linux upstream community code.
Both rdma-core upstream and Mellanox OFED are Linux user-space packages:
  1. Rdma-core is Linux upstream user-space package.(Generic)
  2. Mellanox OFED is Mellanox's Linux user-space package.(Proprietary)
The difference between the two are the APIs towards the kernel.

Support for x86-32 is removed due to issues in rdma-core library.
ICC compilation will be supported as soon as the following patch is
integrated in rdma-core:
https://marc.info/?l=linux-rdma&m=150643474705690&w=2

Signed-off-by: Shachar Beiser <shacharbe@mellanox.com>
Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
18 files changed:
doc/guides/nics/features/mlx5.ini
doc/guides/nics/mlx5.rst
drivers/net/mlx5/Makefile
drivers/net/mlx5/mlx5.c
drivers/net/mlx5/mlx5.h
drivers/net/mlx5/mlx5_ethdev.c
drivers/net/mlx5/mlx5_fdir.c
drivers/net/mlx5/mlx5_flow.c
drivers/net/mlx5/mlx5_mac.c
drivers/net/mlx5/mlx5_prm.h
drivers/net/mlx5/mlx5_rxmode.c
drivers/net/mlx5/mlx5_rxq.c
drivers/net/mlx5/mlx5_rxtx.c
drivers/net/mlx5/mlx5_rxtx.h
drivers/net/mlx5/mlx5_rxtx_vec_sse.c
drivers/net/mlx5/mlx5_txq.c
drivers/net/mlx5/mlx5_vlan.c
mk/rte.app.mk

index d506249..4a2c3a6 100644 (file)
@@ -38,6 +38,5 @@ Stats per queue      = Y
 Other kdrv           = Y
 ARMv8                = Y
 Power8               = Y
-x86-32               = Y
 x86-64               = Y
 Usage doc            = Y
index c6a196c..be0e91c 100644 (file)
@@ -297,7 +297,7 @@ DPDK and must be installed separately:
   This library basically implements send/receive calls to the hardware
   queues.
 
-- **Kernel modules** (mlnx-ofed-kernel)
+- **Kernel modules**
 
   They provide the kernel-side Verbs API and low level device drivers that
   manage actual hardware initialization and resources sharing with user
@@ -324,9 +324,26 @@ DPDK and must be installed separately:
    Both libraries are BSD and GPL licensed. Linux kernel modules are GPL
    licensed.
 
-Currently supported by DPDK:
+Installation
+~~~~~~~~~~~~
 
-- Mellanox OFED version: **4.1**.
+Either RDMA Core library with a recent enough Linux kernel release
+(recommended) or Mellanox OFED, which provides compatibility with older
+releases.
+
+RMDA Core with Linux Kernel
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+- Minimal kernel version : 4.13-rc4 (see `Linux installation documentation`_)
+- Minimal rdma-core version: v15 (see `RDMA Core installation documentation`_)
+
+.. _`Linux installation documentation`: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git/plain/Documentation/admin-guide/README.rst
+.. _`RDMA Core installation documentation`: https://raw.githubusercontent.com/linux-rdma/rdma-core/master/README.md
+
+Mellanox OFED
+^^^^^^^^^^^^^
+
+- Mellanox OFED version: **4.2**.
 - firmware version:
 
   - ConnectX-4: **12.20.1010** and above.
@@ -334,9 +351,6 @@ Currently supported by DPDK:
   - ConnectX-5: **16.20.1010** and above.
   - ConnectX-5 Ex: **16.20.1010** and above.
 
-Getting Mellanox OFED
-~~~~~~~~~~~~~~~~~~~~~
-
 While these libraries and kernel modules are available on OpenFabrics
 Alliance's `website <https://www.openfabrics.org/>`__ and provided by package
 managers on most distributions, this PMD requires Ethernet extensions that
@@ -377,8 +391,8 @@ Supported NICs
 * Mellanox(R) ConnectX(R)-5 100G MCX556A-ECAT (2x100G)
 * Mellanox(R) ConnectX(R)-5 Ex EN 100G MCX516A-CDAT (2x100G)
 
-Quick Start Guide
------------------
+Quick Start Guide on OFED
+-------------------------
 
 1. Download latest Mellanox OFED. For more info check the  `prerequisites`_.
 
index 14b739a..f75d344 100644 (file)
@@ -63,7 +63,7 @@ CFLAGS += -D_DEFAULT_SOURCE
 CFLAGS += -D_XOPEN_SOURCE=600
 CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -Wno-strict-prototypes
-LDLIBS += -libverbs
+LDLIBS += -libverbs -lmlx5
 
 # A few warnings cannot be avoided in external headers.
 CFLAGS += -Wno-error=cast-qual
@@ -104,19 +104,19 @@ mlx5_autoconf.h.new: FORCE
 mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
        $Q $(RM) -f -- '$@'
        $Q sh -- '$<' '$@' \
-               HAVE_VERBS_IBV_EXP_CQ_COMPRESSED_CQE \
-               infiniband/verbs_exp.h \
-               enum IBV_EXP_CQ_COMPRESSED_CQE \
+               HAVE_IBV_DEVICE_VXLAN_SUPPORT \
+               infiniband/verbs.h \
+               enum IBV_DEVICE_VXLAN_SUPPORT \
                $(AUTOCONF_OUTPUT)
        $Q sh -- '$<' '$@' \
-               HAVE_VERBS_MLX5_ETH_VLAN_INLINE_HEADER_SIZE \
-               infiniband/mlx5_hw.h \
-               enum MLX5_ETH_VLAN_INLINE_HEADER_SIZE \
+               HAVE_IBV_WQ_FLAG_RX_END_PADDING \
+               infiniband/verbs.h \
+               enum IBV_WQ_FLAG_RX_END_PADDING \
                $(AUTOCONF_OUTPUT)
        $Q sh -- '$<' '$@' \
-               HAVE_VERBS_MLX5_OPCODE_TSO \
-               infiniband/mlx5_hw.h \
-               enum MLX5_OPCODE_TSO \
+               HAVE_IBV_MLX5_MOD_MPW \
+               infiniband/mlx5dv.h \
+               enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \
                $(AUTOCONF_OUTPUT)
        $Q sh -- '$<' '$@' \
                HAVE_ETHTOOL_LINK_MODE_25G \
@@ -133,11 +133,6 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
                /usr/include/linux/ethtool.h \
                enum ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT \
                $(AUTOCONF_OUTPUT)
-       $Q sh -- '$<' '$@' \
-               HAVE_UPDATE_CQ_CI \
-               infiniband/mlx5_hw.h \
-               func ibv_mlx5_exp_update_cq_ci \
-               $(AUTOCONF_OUTPUT)
 
 # Create mlx5_autoconf.h or update it in case it differs from the new one.
 
index 32e22df..a4b7184 100644 (file)
 /* Default PMD specific parameter value. */
 #define MLX5_ARG_UNSET (-1)
 
+#ifndef HAVE_IBV_MLX5_MOD_MPW
+#define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
+#define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
+#endif
+
 struct mlx5_args {
        int cqe_comp;
        int txq_inline;
@@ -247,10 +252,8 @@ static const struct eth_dev_ops mlx5_dev_ops = {
        .filter_ctrl = mlx5_dev_filter_ctrl,
        .rx_descriptor_status = mlx5_rx_descriptor_status,
        .tx_descriptor_status = mlx5_tx_descriptor_status,
-#ifdef HAVE_UPDATE_CQ_CI
        .rx_queue_intr_enable = mlx5_rx_intr_enable,
        .rx_queue_intr_disable = mlx5_rx_intr_disable,
-#endif
 };
 
 static struct {
@@ -442,12 +445,13 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
        struct ibv_device *ibv_dev;
        int err = 0;
        struct ibv_context *attr_ctx = NULL;
-       struct ibv_device_attr device_attr;
+       struct ibv_device_attr_ex device_attr;
        unsigned int sriov;
        unsigned int mps;
        unsigned int tunnel_en = 0;
        int idx;
        int i;
+       struct mlx5dv_context attrs_out;
 
        (void)pci_drv;
        assert(pci_drv == &mlx5_driver);
@@ -493,35 +497,24 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
                       PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) ||
                      (pci_dev->id.device_id ==
                       PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF));
-               /*
-                * Multi-packet send is supported by ConnectX-4 Lx PF as well
-                * as all ConnectX-5 devices.
-                */
                switch (pci_dev->id.device_id) {
                case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
                        tunnel_en = 1;
-                       mps = MLX5_MPW_DISABLED;
                        break;
                case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX:
-                       tunnel_en = 1;
-                       mps = MLX5_MPW;
-                       break;
                case PCI_DEVICE_ID_MELLANOX_CONNECTX5:
                case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
                case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX:
                case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
                        tunnel_en = 1;
-                       mps = MLX5_MPW_ENHANCED;
                        break;
                default:
-                       mps = MLX5_MPW_DISABLED;
+                       break;
                }
                INFO("PCI information matches, using device \"%s\""
-                    " (SR-IOV: %s, %sMPS: %s)",
+                    " (SR-IOV: %s)",
                     list[i]->name,
-                    sriov ? "true" : "false",
-                    mps == MLX5_MPW_ENHANCED ? "Enhanced " : "",
-                    mps != MLX5_MPW_DISABLED ? "true" : "false");
+                    sriov ? "true" : "false");
                attr_ctx = ibv_open_device(list[i]);
                err = errno;
                break;
@@ -542,11 +535,27 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
        ibv_dev = list[i];
 
        DEBUG("device opened");
-       if (ibv_query_device(attr_ctx, &device_attr))
+       /*
+        * Multi-packet send is supported by ConnectX-4 Lx PF as well
+        * as all ConnectX-5 devices.
+        */
+       mlx5dv_query_device(attr_ctx, &attrs_out);
+       if (attrs_out.flags & (MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW |
+                              MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED)) {
+               INFO("Enhanced MPW is detected\n");
+               mps = MLX5_MPW_ENHANCED;
+       } else if (attrs_out.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) {
+               INFO("MPW is detected\n");
+               mps = MLX5_MPW;
+       } else {
+               INFO("MPW is disabled\n");
+               mps = MLX5_MPW_DISABLED;
+       }
+       if (ibv_query_device_ex(attr_ctx, NULL, &device_attr))
                goto error;
-       INFO("%u port(s) detected", device_attr.phys_port_cnt);
+       INFO("%u port(s) detected", device_attr.orig_attr.phys_port_cnt);
 
-       for (i = 0; i < device_attr.phys_port_cnt; i++) {
+       for (i = 0; i < device_attr.orig_attr.phys_port_cnt; i++) {
                uint32_t port = i + 1; /* ports are indexed from one */
                uint32_t test = (1 << i);
                struct ibv_context *ctx = NULL;
@@ -554,7 +563,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
                struct ibv_pd *pd = NULL;
                struct priv *priv = NULL;
                struct rte_eth_dev *eth_dev;
-               struct ibv_exp_device_attr exp_device_attr;
+               struct ibv_device_attr_ex device_attr_ex;
                struct ether_addr mac;
                uint16_t num_vfs = 0;
                struct mlx5_args args = {
@@ -569,14 +578,6 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
                        .rx_vec_en = MLX5_ARG_UNSET,
                };
 
-               exp_device_attr.comp_mask =
-                       IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS |
-                       IBV_EXP_DEVICE_ATTR_RX_HASH |
-                       IBV_EXP_DEVICE_ATTR_VLAN_OFFLOADS |
-                       IBV_EXP_DEVICE_ATTR_RX_PAD_END_ALIGN |
-                       IBV_EXP_DEVICE_ATTR_TSO_CAPS |
-                       0;
-
                DEBUG("using port %u (%08" PRIx32 ")", port, test);
 
                ctx = ibv_open_device(ibv_dev);
@@ -642,26 +643,26 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
                        goto port_error;
                }
                mlx5_args_assign(priv, &args);
-               if (ibv_exp_query_device(ctx, &exp_device_attr)) {
-                       ERROR("ibv_exp_query_device() failed");
-                       err = ENODEV;
+               if (ibv_query_device_ex(ctx, NULL, &device_attr_ex)) {
+                       ERROR("ibv_query_device_ex() failed");
                        goto port_error;
                }
 
                priv->hw_csum =
-                       ((exp_device_attr.exp_device_cap_flags &
-                         IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT) &&
-                        (exp_device_attr.exp_device_cap_flags &
-                         IBV_EXP_DEVICE_RX_CSUM_IP_PKT));
+                       !!(device_attr_ex.device_cap_flags_ex &
+                          IBV_DEVICE_RAW_IP_CSUM);
                DEBUG("checksum offloading is %ssupported",
                      (priv->hw_csum ? "" : "not "));
 
+#ifdef HAVE_IBV_DEVICE_VXLAN_SUPPORT
                priv->hw_csum_l2tun = !!(exp_device_attr.exp_device_cap_flags &
-                                        IBV_EXP_DEVICE_VXLAN_SUPPORT);
+                                        IBV_DEVICE_VXLAN_SUPPORT);
+#endif
                DEBUG("L2 tunnel checksum offloads are %ssupported",
                      (priv->hw_csum_l2tun ? "" : "not "));
 
-               priv->ind_table_max_size = exp_device_attr.rx_hash_caps.max_rwq_indirection_table_size;
+               priv->ind_table_max_size =
+                       device_attr_ex.rss_caps.max_rwq_indirection_table_size;
                /* Remove this check once DPDK supports larger/variable
                 * indirection tables. */
                if (priv->ind_table_max_size >
@@ -669,29 +670,32 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
                        priv->ind_table_max_size = ETH_RSS_RETA_SIZE_512;
                DEBUG("maximum RX indirection table size is %u",
                      priv->ind_table_max_size);
-               priv->hw_vlan_strip = !!(exp_device_attr.wq_vlan_offloads_cap &
-                                        IBV_EXP_RECEIVE_WQ_CVLAN_STRIP);
+               priv->hw_vlan_strip = !!(device_attr_ex.raw_packet_caps &
+                                        IBV_RAW_PACKET_CAP_CVLAN_STRIPPING);
                DEBUG("VLAN stripping is %ssupported",
                      (priv->hw_vlan_strip ? "" : "not "));
 
-               priv->hw_fcs_strip = !!(exp_device_attr.exp_device_cap_flags &
-                                       IBV_EXP_DEVICE_SCATTER_FCS);
+               priv->hw_fcs_strip =
+                               !!(device_attr_ex.orig_attr.device_cap_flags &
+                               IBV_WQ_FLAGS_SCATTER_FCS);
                DEBUG("FCS stripping configuration is %ssupported",
                      (priv->hw_fcs_strip ? "" : "not "));
 
-               priv->hw_padding = !!exp_device_attr.rx_pad_end_addr_align;
+#ifdef HAVE_IBV_WQ_FLAG_RX_END_PADDING
+               priv->hw_padding = !!device_attr_ex.rx_pad_end_addr_align;
+#endif
                DEBUG("hardware RX end alignment padding is %ssupported",
                      (priv->hw_padding ? "" : "not "));
 
                priv_get_num_vfs(priv, &num_vfs);
                priv->sriov = (num_vfs || sriov);
                priv->tso = ((priv->tso) &&
-                           (exp_device_attr.tso_caps.max_tso > 0) &&
-                           (exp_device_attr.tso_caps.supported_qpts &
-                           (1 << IBV_QPT_RAW_ETH)));
+                           (device_attr_ex.tso_caps.max_tso > 0) &&
+                           (device_attr_ex.tso_caps.supported_qpts &
+                           (1 << IBV_QPT_RAW_PACKET)));
                if (priv->tso)
                        priv->max_tso_payload_sz =
-                               exp_device_attr.tso_caps.max_tso;
+                               device_attr_ex.tso_caps.max_tso;
                if (priv->mps && !mps) {
                        ERROR("multi-packet send not supported on this device"
                              " (" MLX5_TXQ_MPW_EN ")");
index e89aba8..ab03fe0 100644 (file)
@@ -89,7 +89,7 @@ struct mlx5_xstats_ctrl {
 struct priv {
        struct rte_eth_dev *dev; /* Ethernet device. */
        struct ibv_context *ctx; /* Verbs context. */
-       struct ibv_device_attr device_attr; /* Device properties. */
+       struct ibv_device_attr_ex device_attr; /* Device properties. */
        struct ibv_pd *pd; /* Protection Domain. */
        /*
         * MAC addresses array and configuration bit-field.
@@ -132,7 +132,7 @@ struct priv {
        struct rxq *(*rxqs)[]; /* RX queues. */
        struct txq *(*txqs)[]; /* TX queues. */
        /* Indirection tables referencing all RX WQs. */
-       struct ibv_exp_rwq_ind_table *(*ind_tables)[];
+       struct ibv_rwq_ind_table *(*ind_tables)[];
        unsigned int ind_tables_n; /* Number of indirection tables. */
        unsigned int ind_table_max_size; /* Maximum indirection table size. */
        /* Hash RX QPs feeding the indirection table. */
index a47fd2e..b87eb09 100644 (file)
@@ -660,8 +660,8 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
         * Since we need one CQ per QP, the limit is the minimum number
         * between the two values.
         */
-       max = ((priv->device_attr.max_cq > priv->device_attr.max_qp) ?
-              priv->device_attr.max_qp : priv->device_attr.max_cq);
+       max = RTE_MIN(priv->device_attr.orig_attr.max_cq,
+                     priv->device_attr.orig_attr.max_qp);
        /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */
        if (max >= 65535)
                max = 65535;
index ad256e4..acae668 100644 (file)
@@ -72,7 +72,7 @@ struct mlx5_fdir_filter {
        uint16_t queue; /* Queue assigned to if FDIR match. */
        enum rte_eth_fdir_behavior behavior;
        struct fdir_flow_desc desc;
-       struct ibv_exp_flow *flow;
+       struct ibv_flow *flow;
 };
 
 LIST_HEAD(fdir_filter_list, mlx5_fdir_filter);
@@ -238,19 +238,19 @@ priv_fdir_flow_add(struct priv *priv,
                   struct mlx5_fdir_filter *mlx5_fdir_filter,
                   struct fdir_queue *fdir_queue)
 {
-       struct ibv_exp_flow *flow;
+       struct ibv_flow *flow;
        struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc;
        enum rte_fdir_mode fdir_mode =
                priv->dev->data->dev_conf.fdir_conf.mode;
        struct rte_eth_fdir_masks *mask =
                &priv->dev->data->dev_conf.fdir_conf.mask;
        FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type));
-       struct ibv_exp_flow_attr *attr = &data->attr;
+       struct ibv_flow_attr *attr = &data->attr;
        uintptr_t spec_offset = (uintptr_t)&data->spec;
-       struct ibv_exp_flow_spec_eth *spec_eth;
-       struct ibv_exp_flow_spec_ipv4 *spec_ipv4;
-       struct ibv_exp_flow_spec_ipv6 *spec_ipv6;
-       struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp;
+       struct ibv_flow_spec_eth *spec_eth;
+       struct ibv_flow_spec_ipv4 *spec_ipv4;
+       struct ibv_flow_spec_ipv6 *spec_ipv6;
+       struct ibv_flow_spec_tcp_udp *spec_tcp_udp;
        struct mlx5_fdir_filter *iter_fdir_filter;
        unsigned int i;
 
@@ -272,10 +272,10 @@ priv_fdir_flow_add(struct priv *priv,
        priv_flow_attr(priv, attr, sizeof(data), desc->type);
 
        /* Set Ethernet spec */
-       spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset;
+       spec_eth = (struct ibv_flow_spec_eth *)spec_offset;
 
        /* The first specification must be Ethernet. */
-       assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH);
+       assert(spec_eth->type == IBV_FLOW_SPEC_ETH);
        assert(spec_eth->size == sizeof(*spec_eth));
 
        /* VLAN ID */
@@ -302,10 +302,10 @@ priv_fdir_flow_add(struct priv *priv,
                spec_offset += spec_eth->size;
 
                /* Set IP spec */
-               spec_ipv4 = (struct ibv_exp_flow_spec_ipv4 *)spec_offset;
+               spec_ipv4 = (struct ibv_flow_spec_ipv4 *)spec_offset;
 
                /* The second specification must be IP. */
-               assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4);
+               assert(spec_ipv4->type == IBV_FLOW_SPEC_IPV4);
                assert(spec_ipv4->size == sizeof(*spec_ipv4));
 
                spec_ipv4->val.src_ip =
@@ -329,10 +329,10 @@ priv_fdir_flow_add(struct priv *priv,
                spec_offset += spec_eth->size;
 
                /* Set IP spec */
-               spec_ipv6 = (struct ibv_exp_flow_spec_ipv6 *)spec_offset;
+               spec_ipv6 = (struct ibv_flow_spec_ipv6 *)spec_offset;
 
                /* The second specification must be IP. */
-               assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6);
+               assert(spec_ipv6->type == IBV_FLOW_SPEC_IPV6);
                assert(spec_ipv6->size == sizeof(*spec_ipv6));
 
                for (i = 0; i != RTE_DIM(desc->src_ip); ++i) {
@@ -362,11 +362,11 @@ priv_fdir_flow_add(struct priv *priv,
        }
 
        /* Set TCP/UDP flow specification. */
-       spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset;
+       spec_tcp_udp = (struct ibv_flow_spec_tcp_udp *)spec_offset;
 
        /* The third specification must be TCP/UDP. */
-       assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP ||
-              spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP);
+       assert(spec_tcp_udp->type == IBV_FLOW_SPEC_TCP ||
+              spec_tcp_udp->type == IBV_FLOW_SPEC_UDP);
        assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp));
 
        spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask;
@@ -380,7 +380,7 @@ priv_fdir_flow_add(struct priv *priv,
 create_flow:
 
        errno = 0;
-       flow = ibv_exp_create_flow(fdir_queue->qp, attr);
+       flow = ibv_create_flow(fdir_queue->qp, attr);
        if (flow == NULL) {
                /* It's not clear whether errno is always set in this case. */
                ERROR("%p: flow director configuration failed, errno=%d: %s",
@@ -416,16 +416,16 @@ priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue)
                assert(idx < priv->rxqs_n);
                if (fdir_queue == rxq_ctrl->fdir_queue &&
                    fdir_filter->flow != NULL) {
-                       claim_zero(ibv_exp_destroy_flow(fdir_filter->flow));
+                       claim_zero(ibv_destroy_flow(fdir_filter->flow));
                        fdir_filter->flow = NULL;
                }
        }
        assert(fdir_queue->qp);
        claim_zero(ibv_destroy_qp(fdir_queue->qp));
        assert(fdir_queue->ind_table);
-       claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table));
+       claim_zero(ibv_destroy_rwq_ind_table(fdir_queue->ind_table));
        if (fdir_queue->wq)
-               claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
+               claim_zero(ibv_destroy_wq(fdir_queue->wq));
        if (fdir_queue->cq)
                claim_zero(ibv_destroy_cq(fdir_queue->cq));
 #ifndef NDEBUG
@@ -447,7 +447,7 @@ priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue)
  *   Related flow director queue on success, NULL otherwise.
  */
 static struct fdir_queue *
-priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq,
+priv_fdir_queue_create(struct priv *priv, struct ibv_wq *wq,
                       unsigned int socket)
 {
        struct fdir_queue *fdir_queue;
@@ -461,21 +461,18 @@ priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq,
        assert(priv->pd);
        assert(priv->ctx);
        if (!wq) {
-               fdir_queue->cq = ibv_exp_create_cq(
-                       priv->ctx, 1, NULL, NULL, 0,
-                       &(struct ibv_exp_cq_init_attr){
-                               .comp_mask = 0,
-                       });
+               fdir_queue->cq = ibv_create_cq(
+                       priv->ctx, 1, NULL, NULL, 0);
                if (!fdir_queue->cq) {
                        ERROR("cannot create flow director CQ");
                        goto error;
                }
-               fdir_queue->wq = ibv_exp_create_wq(
+               fdir_queue->wq = ibv_create_wq(
                        priv->ctx,
-                       &(struct ibv_exp_wq_init_attr){
-                               .wq_type = IBV_EXP_WQT_RQ,
-                               .max_recv_wr = 1,
-                               .max_recv_sge = 1,
+                       &(struct ibv_wq_init_attr){
+                               .wq_type = IBV_WQT_RQ,
+                               .max_wr = 1,
+                               .max_sge = 1,
                                .pd = priv->pd,
                                .cq = fdir_queue->cq,
                        });
@@ -485,10 +482,9 @@ priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq,
                }
                wq = fdir_queue->wq;
        }
-       fdir_queue->ind_table = ibv_exp_create_rwq_ind_table(
+       fdir_queue->ind_table = ibv_create_rwq_ind_table(
                priv->ctx,
-               &(struct ibv_exp_rwq_ind_table_init_attr){
-                       .pd = priv->pd,
+               &(struct ibv_rwq_ind_table_init_attr){
                        .log_ind_tbl_size = 0,
                        .ind_tbl = &wq,
                        .comp_mask = 0,
@@ -497,24 +493,23 @@ priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq,
                ERROR("cannot create flow director indirection table");
                goto error;
        }
-       fdir_queue->qp = ibv_exp_create_qp(
+       fdir_queue->qp = ibv_create_qp_ex(
                priv->ctx,
-               &(struct ibv_exp_qp_init_attr){
+               &(struct ibv_qp_init_attr_ex){
                        .qp_type = IBV_QPT_RAW_PACKET,
                        .comp_mask =
-                               IBV_EXP_QP_INIT_ATTR_PD |
-                               IBV_EXP_QP_INIT_ATTR_PORT |
-                               IBV_EXP_QP_INIT_ATTR_RX_HASH,
-                       .pd = priv->pd,
-                       .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
+                               IBV_QP_INIT_ATTR_PD |
+                               IBV_QP_INIT_ATTR_IND_TABLE |
+                               IBV_QP_INIT_ATTR_RX_HASH,
+                       .rx_hash_conf = (struct ibv_rx_hash_conf){
                                .rx_hash_function =
-                                       IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
+                                       IBV_RX_HASH_FUNC_TOEPLITZ,
                                .rx_hash_key_len = rss_hash_default_key_len,
                                .rx_hash_key = rss_hash_default_key,
                                .rx_hash_fields_mask = 0,
-                               .rwq_ind_tbl = fdir_queue->ind_table,
                        },
-                       .port_num = priv->port,
+                       .rwq_ind_tbl = fdir_queue->ind_table,
+                       .pd = priv->pd,
                });
        if (!fdir_queue->qp) {
                ERROR("cannot create flow director hash RX QP");
@@ -525,10 +520,10 @@ error:
        assert(fdir_queue);
        assert(!fdir_queue->qp);
        if (fdir_queue->ind_table)
-               claim_zero(ibv_exp_destroy_rwq_ind_table
+               claim_zero(ibv_destroy_rwq_ind_table
                           (fdir_queue->ind_table));
        if (fdir_queue->wq)
-               claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
+               claim_zero(ibv_destroy_wq(fdir_queue->wq));
        if (fdir_queue->cq)
                claim_zero(ibv_destroy_cq(fdir_queue->cq));
        rte_free(fdir_queue);
@@ -673,13 +668,13 @@ priv_fdir_filter_flush(struct priv *priv)
        struct mlx5_fdir_filter *mlx5_fdir_filter;
 
        while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) {
-               struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
+               struct ibv_flow *flow = mlx5_fdir_filter->flow;
 
                DEBUG("%p: flushing flow director filter %p",
                      (void *)priv, (void *)mlx5_fdir_filter);
                LIST_REMOVE(mlx5_fdir_filter, next);
                if (flow != NULL)
-                       claim_zero(ibv_exp_destroy_flow(flow));
+                       claim_zero(ibv_destroy_flow(flow));
                rte_free(mlx5_fdir_filter);
        }
 }
@@ -712,7 +707,7 @@ priv_fdir_disable(struct priv *priv)
 
        /* Run on every flow director filter and destroy flow handle. */
        LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
-               struct ibv_exp_flow *flow;
+               struct ibv_flow *flow;
 
                /* Only valid elements should be in the list */
                assert(mlx5_fdir_filter != NULL);
@@ -720,7 +715,7 @@ priv_fdir_disable(struct priv *priv)
 
                /* Destroy flow handle */
                if (flow != NULL) {
-                       claim_zero(ibv_exp_destroy_flow(flow));
+                       claim_zero(ibv_destroy_flow(flow));
                        mlx5_fdir_filter->flow = NULL;
                }
        }
@@ -887,7 +882,7 @@ priv_fdir_filter_update(struct priv *priv,
 
        mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
        if (mlx5_fdir_filter != NULL) {
-               struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
+               struct ibv_flow *flow = mlx5_fdir_filter->flow;
                int err = 0;
 
                /* Update queue number. */
@@ -895,7 +890,7 @@ priv_fdir_filter_update(struct priv *priv,
 
                /* Destroy flow handle. */
                if (flow != NULL) {
-                       claim_zero(ibv_exp_destroy_flow(flow));
+                       claim_zero(ibv_destroy_flow(flow));
                        mlx5_fdir_filter->flow = NULL;
                }
                DEBUG("%p: flow director filter %p updated",
@@ -933,14 +928,14 @@ priv_fdir_filter_delete(struct priv *priv,
 
        mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
        if (mlx5_fdir_filter != NULL) {
-               struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
+               struct ibv_flow *flow = mlx5_fdir_filter->flow;
 
                /* Remove element from list. */
                LIST_REMOVE(mlx5_fdir_filter, next);
 
                /* Destroy flow handle. */
                if (flow != NULL) {
-                       claim_zero(ibv_exp_destroy_flow(flow));
+                       claim_zero(ibv_destroy_flow(flow));
                        mlx5_fdir_filter->flow = NULL;
                }
 
index 7dd3ebb..dbd241f 100644 (file)
@@ -89,11 +89,11 @@ mlx5_flow_create_vxlan(const struct rte_flow_item *item,
 
 struct rte_flow {
        TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
-       struct ibv_exp_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
-       struct ibv_exp_rwq_ind_table *ind_table; /**< Indirection table. */
+       struct ibv_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
+       struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
        struct ibv_qp *qp; /**< Verbs queue pair. */
-       struct ibv_exp_flow *ibv_flow; /**< Verbs flow. */
-       struct ibv_exp_wq *wq; /**< Verbs work queue. */
+       struct ibv_flow *ibv_flow; /**< Verbs flow. */
+       struct ibv_wq *wq; /**< Verbs work queue. */
        struct ibv_cq *cq; /**< Verbs completion queue. */
        uint16_t rxqs_n; /**< Number of queues in this flow, 0 if drop queue. */
        uint32_t mark:1; /**< Set if the flow is marked. */
@@ -172,7 +172,7 @@ static const struct mlx5_flow_items mlx5_flow_items[] = {
                .default_mask = &rte_flow_item_eth_mask,
                .mask_sz = sizeof(struct rte_flow_item_eth),
                .convert = mlx5_flow_create_eth,
-               .dst_sz = sizeof(struct ibv_exp_flow_spec_eth),
+               .dst_sz = sizeof(struct ibv_flow_spec_eth),
        },
        [RTE_FLOW_ITEM_TYPE_VLAN] = {
                .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
@@ -201,7 +201,7 @@ static const struct mlx5_flow_items mlx5_flow_items[] = {
                .default_mask = &rte_flow_item_ipv4_mask,
                .mask_sz = sizeof(struct rte_flow_item_ipv4),
                .convert = mlx5_flow_create_ipv4,
-               .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv4_ext),
+               .dst_sz = sizeof(struct ibv_flow_spec_ipv4_ext),
        },
        [RTE_FLOW_ITEM_TYPE_IPV6] = {
                .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
@@ -229,7 +229,7 @@ static const struct mlx5_flow_items mlx5_flow_items[] = {
                .default_mask = &rte_flow_item_ipv6_mask,
                .mask_sz = sizeof(struct rte_flow_item_ipv6),
                .convert = mlx5_flow_create_ipv6,
-               .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv6_ext),
+               .dst_sz = sizeof(struct ibv_flow_spec_ipv6),
        },
        [RTE_FLOW_ITEM_TYPE_UDP] = {
                .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
@@ -243,7 +243,7 @@ static const struct mlx5_flow_items mlx5_flow_items[] = {
                .default_mask = &rte_flow_item_udp_mask,
                .mask_sz = sizeof(struct rte_flow_item_udp),
                .convert = mlx5_flow_create_udp,
-               .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
+               .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
        },
        [RTE_FLOW_ITEM_TYPE_TCP] = {
                .actions = valid_actions,
@@ -256,7 +256,7 @@ static const struct mlx5_flow_items mlx5_flow_items[] = {
                .default_mask = &rte_flow_item_tcp_mask,
                .mask_sz = sizeof(struct rte_flow_item_tcp),
                .convert = mlx5_flow_create_tcp,
-               .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
+               .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
        },
        [RTE_FLOW_ITEM_TYPE_VXLAN] = {
                .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
@@ -267,13 +267,13 @@ static const struct mlx5_flow_items mlx5_flow_items[] = {
                .default_mask = &rte_flow_item_vxlan_mask,
                .mask_sz = sizeof(struct rte_flow_item_vxlan),
                .convert = mlx5_flow_create_vxlan,
-               .dst_sz = sizeof(struct ibv_exp_flow_spec_tunnel),
+               .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
        },
 };
 
 /** Structure to pass to the conversion function. */
 struct mlx5_flow {
-       struct ibv_exp_flow_attr *ibv_attr; /**< Verbs attribute. */
+       struct ibv_flow_attr *ibv_attr; /**< Verbs attribute. */
        unsigned int offset; /**< Offset in bytes in the ibv_attr buffer. */
        uint32_t inner; /**< Set once VXLAN is encountered. */
        uint64_t hash_fields; /**< Fields that participate in the hash. */
@@ -281,9 +281,9 @@ struct mlx5_flow {
 
 /** Structure for Drop queue. */
 struct rte_flow_drop {
-       struct ibv_exp_rwq_ind_table *ind_table; /**< Indirection table. */
+       struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
        struct ibv_qp *qp; /**< Verbs queue pair. */
-       struct ibv_exp_wq *wq; /**< Verbs work queue. */
+       struct ibv_wq *wq; /**< Verbs work queue. */
        struct ibv_cq *cq; /**< Verbs completion queue. */
 };
 
@@ -572,9 +572,9 @@ priv_flow_validate(struct priv *priv,
                }
        }
        if (action->mark && !flow->ibv_attr && !action->drop)
-               flow->offset += sizeof(struct ibv_exp_flow_spec_action_tag);
+               flow->offset += sizeof(struct ibv_flow_spec_action_tag);
        if (!flow->ibv_attr && action->drop)
-               flow->offset += sizeof(struct ibv_exp_flow_spec_action_drop);
+               flow->offset += sizeof(struct ibv_flow_spec_action_drop);
        if (!action->queue && !action->drop) {
                rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
                                   NULL, "no valid action");
@@ -606,7 +606,7 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
 {
        struct priv *priv = dev->data->dev_private;
        int ret;
-       struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr) };
+       struct mlx5_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
        struct mlx5_flow_action action = {
                .queue = 0,
                .drop = 0,
@@ -640,16 +640,16 @@ mlx5_flow_create_eth(const struct rte_flow_item *item,
        const struct rte_flow_item_eth *spec = item->spec;
        const struct rte_flow_item_eth *mask = item->mask;
        struct mlx5_flow *flow = (struct mlx5_flow *)data;
-       struct ibv_exp_flow_spec_eth *eth;
-       const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
+       struct ibv_flow_spec_eth *eth;
+       const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
        unsigned int i;
 
        ++flow->ibv_attr->num_of_specs;
        flow->ibv_attr->priority = 2;
        flow->hash_fields = 0;
        eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
-       *eth = (struct ibv_exp_flow_spec_eth) {
-               .type = flow->inner | IBV_EXP_FLOW_SPEC_ETH,
+       *eth = (struct ibv_flow_spec_eth) {
+               .type = flow->inner | IBV_FLOW_SPEC_ETH,
                .size = eth_size,
        };
        if (!spec)
@@ -689,8 +689,8 @@ mlx5_flow_create_vlan(const struct rte_flow_item *item,
        const struct rte_flow_item_vlan *spec = item->spec;
        const struct rte_flow_item_vlan *mask = item->mask;
        struct mlx5_flow *flow = (struct mlx5_flow *)data;
-       struct ibv_exp_flow_spec_eth *eth;
-       const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
+       struct ibv_flow_spec_eth *eth;
+       const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
 
        eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
        if (!spec)
@@ -721,29 +721,29 @@ mlx5_flow_create_ipv4(const struct rte_flow_item *item,
        const struct rte_flow_item_ipv4 *spec = item->spec;
        const struct rte_flow_item_ipv4 *mask = item->mask;
        struct mlx5_flow *flow = (struct mlx5_flow *)data;
-       struct ibv_exp_flow_spec_ipv4_ext *ipv4;
-       unsigned int ipv4_size = sizeof(struct ibv_exp_flow_spec_ipv4_ext);
+       struct ibv_flow_spec_ipv4_ext *ipv4;
+       unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
 
        ++flow->ibv_attr->num_of_specs;
        flow->ibv_attr->priority = 1;
-       flow->hash_fields = (IBV_EXP_RX_HASH_SRC_IPV4 |
-                            IBV_EXP_RX_HASH_DST_IPV4);
+       flow->hash_fields = (IBV_RX_HASH_SRC_IPV4 |
+                            IBV_RX_HASH_DST_IPV4);
        ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
-       *ipv4 = (struct ibv_exp_flow_spec_ipv4_ext) {
-               .type = flow->inner | IBV_EXP_FLOW_SPEC_IPV4_EXT,
+       *ipv4 = (struct ibv_flow_spec_ipv4_ext) {
+               .type = flow->inner | IBV_FLOW_SPEC_IPV4_EXT,
                .size = ipv4_size,
        };
        if (!spec)
                return 0;
        if (!mask)
                mask = default_mask;
-       ipv4->val = (struct ibv_exp_flow_ipv4_ext_filter){
+       ipv4->val = (struct ibv_flow_ipv4_ext_filter){
                .src_ip = spec->hdr.src_addr,
                .dst_ip = spec->hdr.dst_addr,
                .proto = spec->hdr.next_proto_id,
                .tos = spec->hdr.type_of_service,
        };
-       ipv4->mask = (struct ibv_exp_flow_ipv4_ext_filter){
+       ipv4->mask = (struct ibv_flow_ipv4_ext_filter){
                .src_ip = mask->hdr.src_addr,
                .dst_ip = mask->hdr.dst_addr,
                .proto = mask->hdr.next_proto_id,
@@ -775,17 +775,17 @@ mlx5_flow_create_ipv6(const struct rte_flow_item *item,
        const struct rte_flow_item_ipv6 *spec = item->spec;
        const struct rte_flow_item_ipv6 *mask = item->mask;
        struct mlx5_flow *flow = (struct mlx5_flow *)data;
-       struct ibv_exp_flow_spec_ipv6_ext *ipv6;
-       unsigned int ipv6_size = sizeof(struct ibv_exp_flow_spec_ipv6_ext);
+       struct ibv_flow_spec_ipv6 *ipv6;
+       unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
        unsigned int i;
 
        ++flow->ibv_attr->num_of_specs;
        flow->ibv_attr->priority = 1;
-       flow->hash_fields = (IBV_EXP_RX_HASH_SRC_IPV6 |
-                            IBV_EXP_RX_HASH_DST_IPV6);
+       flow->hash_fields = (IBV_RX_HASH_SRC_IPV6 |
+                            IBV_RX_HASH_DST_IPV6);
        ipv6 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
-       *ipv6 = (struct ibv_exp_flow_spec_ipv6_ext) {
-               .type = flow->inner | IBV_EXP_FLOW_SPEC_IPV6_EXT,
+       *ipv6 = (struct ibv_flow_spec_ipv6) {
+               .type = flow->inner | IBV_FLOW_SPEC_IPV6,
                .size = ipv6_size,
        };
        if (!spec)
@@ -832,16 +832,16 @@ mlx5_flow_create_udp(const struct rte_flow_item *item,
        const struct rte_flow_item_udp *spec = item->spec;
        const struct rte_flow_item_udp *mask = item->mask;
        struct mlx5_flow *flow = (struct mlx5_flow *)data;
-       struct ibv_exp_flow_spec_tcp_udp *udp;
-       unsigned int udp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
+       struct ibv_flow_spec_tcp_udp *udp;
+       unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 
        ++flow->ibv_attr->num_of_specs;
        flow->ibv_attr->priority = 0;
-       flow->hash_fields |= (IBV_EXP_RX_HASH_SRC_PORT_UDP |
-                             IBV_EXP_RX_HASH_DST_PORT_UDP);
+       flow->hash_fields |= (IBV_RX_HASH_SRC_PORT_UDP |
+                             IBV_RX_HASH_DST_PORT_UDP);
        udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
-       *udp = (struct ibv_exp_flow_spec_tcp_udp) {
-               .type = flow->inner | IBV_EXP_FLOW_SPEC_UDP,
+       *udp = (struct ibv_flow_spec_tcp_udp) {
+               .type = flow->inner | IBV_FLOW_SPEC_UDP,
                .size = udp_size,
        };
        if (!spec)
@@ -876,16 +876,16 @@ mlx5_flow_create_tcp(const struct rte_flow_item *item,
        const struct rte_flow_item_tcp *spec = item->spec;
        const struct rte_flow_item_tcp *mask = item->mask;
        struct mlx5_flow *flow = (struct mlx5_flow *)data;
-       struct ibv_exp_flow_spec_tcp_udp *tcp;
-       unsigned int tcp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
+       struct ibv_flow_spec_tcp_udp *tcp;
+       unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
 
        ++flow->ibv_attr->num_of_specs;
        flow->ibv_attr->priority = 0;
-       flow->hash_fields |= (IBV_EXP_RX_HASH_SRC_PORT_TCP |
-                             IBV_EXP_RX_HASH_DST_PORT_TCP);
+       flow->hash_fields |= (IBV_RX_HASH_SRC_PORT_TCP |
+                             IBV_RX_HASH_DST_PORT_TCP);
        tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
-       *tcp = (struct ibv_exp_flow_spec_tcp_udp) {
-               .type = flow->inner | IBV_EXP_FLOW_SPEC_TCP,
+       *tcp = (struct ibv_flow_spec_tcp_udp) {
+               .type = flow->inner | IBV_FLOW_SPEC_TCP,
                .size = tcp_size,
        };
        if (!spec)
@@ -920,8 +920,8 @@ mlx5_flow_create_vxlan(const struct rte_flow_item *item,
        const struct rte_flow_item_vxlan *spec = item->spec;
        const struct rte_flow_item_vxlan *mask = item->mask;
        struct mlx5_flow *flow = (struct mlx5_flow *)data;
-       struct ibv_exp_flow_spec_tunnel *vxlan;
-       unsigned int size = sizeof(struct ibv_exp_flow_spec_tunnel);
+       struct ibv_flow_spec_tunnel *vxlan;
+       unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
        union vni {
                uint32_t vlan_id;
                uint8_t vni[4];
@@ -931,11 +931,11 @@ mlx5_flow_create_vxlan(const struct rte_flow_item *item,
        flow->ibv_attr->priority = 0;
        id.vni[0] = 0;
        vxlan = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
-       *vxlan = (struct ibv_exp_flow_spec_tunnel) {
-               .type = flow->inner | IBV_EXP_FLOW_SPEC_VXLAN_TUNNEL,
+       *vxlan = (struct ibv_flow_spec_tunnel) {
+               .type = flow->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
                .size = size,
        };
-       flow->inner = IBV_EXP_FLOW_SPEC_INNER;
+       flow->inner = IBV_FLOW_SPEC_INNER;
        if (!spec)
                return 0;
        if (!mask)
@@ -960,12 +960,12 @@ mlx5_flow_create_vxlan(const struct rte_flow_item *item,
 static int
 mlx5_flow_create_flag_mark(struct mlx5_flow *flow, uint32_t mark_id)
 {
-       struct ibv_exp_flow_spec_action_tag *tag;
-       unsigned int size = sizeof(struct ibv_exp_flow_spec_action_tag);
+       struct ibv_flow_spec_action_tag *tag;
+       unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
 
        tag = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
-       *tag = (struct ibv_exp_flow_spec_action_tag){
-               .type = IBV_EXP_FLOW_SPEC_ACTION_TAG,
+       *tag = (struct ibv_flow_spec_action_tag){
+               .type = IBV_FLOW_SPEC_ACTION_TAG,
                .size = size,
                .tag_id = mlx5_flow_mark_set(mark_id),
        };
@@ -992,8 +992,8 @@ priv_flow_create_action_queue_drop(struct priv *priv,
                                   struct rte_flow_error *error)
 {
        struct rte_flow *rte_flow;
-       struct ibv_exp_flow_spec_action_drop *drop;
-       unsigned int size = sizeof(struct ibv_exp_flow_spec_action_drop);
+       struct ibv_flow_spec_action_drop *drop;
+       unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
 
        assert(priv->pd);
        assert(priv->ctx);
@@ -1005,18 +1005,18 @@ priv_flow_create_action_queue_drop(struct priv *priv,
        }
        rte_flow->drop = 1;
        drop = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
-       *drop = (struct ibv_exp_flow_spec_action_drop){
-                       .type = IBV_EXP_FLOW_SPEC_ACTION_DROP,
+       *drop = (struct ibv_flow_spec_action_drop){
+                       .type = IBV_FLOW_SPEC_ACTION_DROP,
                        .size = size,
        };
        ++flow->ibv_attr->num_of_specs;
-       flow->offset += sizeof(struct ibv_exp_flow_spec_action_drop);
+       flow->offset += sizeof(struct ibv_flow_spec_action_drop);
        rte_flow->ibv_attr = flow->ibv_attr;
        if (!priv->started)
                return rte_flow;
        rte_flow->qp = priv->flow_drop_queue->qp;
-       rte_flow->ibv_flow = ibv_exp_create_flow(rte_flow->qp,
-                                                rte_flow->ibv_attr);
+       rte_flow->ibv_flow = ibv_create_flow(rte_flow->qp,
+                                            rte_flow->ibv_attr);
        if (!rte_flow->ibv_flow) {
                rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
                                   NULL, "flow rule creation failure");
@@ -1054,7 +1054,7 @@ priv_flow_create_action_queue(struct priv *priv,
        unsigned int i;
        unsigned int j;
        const unsigned int wqs_n = 1 << log2above(action->queues_n);
-       struct ibv_exp_wq *wqs[wqs_n];
+       struct ibv_wq *wqs[wqs_n];
 
        assert(priv->pd);
        assert(priv->ctx);
@@ -1085,10 +1085,9 @@ priv_flow_create_action_queue(struct priv *priv,
        rte_flow->mark = action->mark;
        rte_flow->ibv_attr = flow->ibv_attr;
        rte_flow->hash_fields = flow->hash_fields;
-       rte_flow->ind_table = ibv_exp_create_rwq_ind_table(
+       rte_flow->ind_table = ibv_create_rwq_ind_table(
                priv->ctx,
-               &(struct ibv_exp_rwq_ind_table_init_attr){
-                       .pd = priv->pd,
+               &(struct ibv_rwq_ind_table_init_attr){
                        .log_ind_tbl_size = log2above(action->queues_n),
                        .ind_tbl = wqs,
                        .comp_mask = 0,
@@ -1098,24 +1097,23 @@ priv_flow_create_action_queue(struct priv *priv,
                                   NULL, "cannot allocate indirection table");
                goto error;
        }
-       rte_flow->qp = ibv_exp_create_qp(
+       rte_flow->qp = ibv_create_qp_ex(
                priv->ctx,
-               &(struct ibv_exp_qp_init_attr){
+               &(struct ibv_qp_init_attr_ex){
                        .qp_type = IBV_QPT_RAW_PACKET,
                        .comp_mask =
-                               IBV_EXP_QP_INIT_ATTR_PD |
-                               IBV_EXP_QP_INIT_ATTR_PORT |
-                               IBV_EXP_QP_INIT_ATTR_RX_HASH,
-                       .pd = priv->pd,
-                       .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
+                               IBV_QP_INIT_ATTR_PD |
+                               IBV_QP_INIT_ATTR_IND_TABLE |
+                               IBV_QP_INIT_ATTR_RX_HASH,
+                       .rx_hash_conf = (struct ibv_rx_hash_conf){
                                .rx_hash_function =
-                                       IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
+                                       IBV_RX_HASH_FUNC_TOEPLITZ,
                                .rx_hash_key_len = rss_hash_default_key_len,
                                .rx_hash_key = rss_hash_default_key,
                                .rx_hash_fields_mask = rte_flow->hash_fields,
-                               .rwq_ind_tbl = rte_flow->ind_table,
                        },
-                       .port_num = priv->port,
+                       .rwq_ind_tbl = rte_flow->ind_table,
+                       .pd = priv->pd
                });
        if (!rte_flow->qp) {
                rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
@@ -1124,8 +1122,8 @@ priv_flow_create_action_queue(struct priv *priv,
        }
        if (!priv->started)
                return rte_flow;
-       rte_flow->ibv_flow = ibv_exp_create_flow(rte_flow->qp,
-                                                rte_flow->ibv_attr);
+       rte_flow->ibv_flow = ibv_create_flow(rte_flow->qp,
+                                            rte_flow->ibv_attr);
        if (!rte_flow->ibv_flow) {
                rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
                                   NULL, "flow rule creation failure");
@@ -1137,7 +1135,7 @@ error:
        if (rte_flow->qp)
                ibv_destroy_qp(rte_flow->qp);
        if (rte_flow->ind_table)
-               ibv_exp_destroy_rwq_ind_table(rte_flow->ind_table);
+               ibv_destroy_rwq_ind_table(rte_flow->ind_table);
        rte_free(rte_flow);
        return NULL;
 }
@@ -1167,7 +1165,7 @@ priv_flow_create(struct priv *priv,
                 struct rte_flow_error *error)
 {
        struct rte_flow *rte_flow;
-       struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr), };
+       struct mlx5_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
        struct mlx5_flow_action action = {
                .queue = 0,
                .drop = 0,
@@ -1182,20 +1180,19 @@ priv_flow_create(struct priv *priv,
        if (err)
                goto exit;
        flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
-       flow.offset = sizeof(struct ibv_exp_flow_attr);
+       flow.offset = sizeof(struct ibv_flow_attr);
        if (!flow.ibv_attr) {
                rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
                                   NULL, "cannot allocate ibv_attr memory");
                goto exit;
        }
-       *flow.ibv_attr = (struct ibv_exp_flow_attr){
-               .type = IBV_EXP_FLOW_ATTR_NORMAL,
-               .size = sizeof(struct ibv_exp_flow_attr),
+       *flow.ibv_attr = (struct ibv_flow_attr){
+               .type = IBV_FLOW_ATTR_NORMAL,
+               .size = sizeof(struct ibv_flow_attr),
                .priority = attr->priority,
                .num_of_specs = 0,
                .port = 0,
                .flags = 0,
-               .reserved = 0,
        };
        flow.inner = 0;
        flow.hash_fields = 0;
@@ -1203,7 +1200,7 @@ priv_flow_create(struct priv *priv,
                                      error, &flow, &action));
        if (action.mark && !action.drop) {
                mlx5_flow_create_flag_mark(&flow, action.mark_id);
-               flow.offset += sizeof(struct ibv_exp_flow_spec_action_tag);
+               flow.offset += sizeof(struct ibv_flow_spec_action_tag);
        }
        if (action.drop)
                rte_flow =
@@ -1259,13 +1256,13 @@ priv_flow_destroy(struct priv *priv,
 {
        TAILQ_REMOVE(&priv->flows, flow, next);
        if (flow->ibv_flow)
-               claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
+               claim_zero(ibv_destroy_flow(flow->ibv_flow));
        if (flow->drop)
                goto free;
        if (flow->qp)
                claim_zero(ibv_destroy_qp(flow->qp));
        if (flow->ind_table)
-               claim_zero(ibv_exp_destroy_rwq_ind_table(flow->ind_table));
+               claim_zero(ibv_destroy_rwq_ind_table(flow->ind_table));
        if (flow->mark) {
                struct rte_flow *tmp;
                struct rxq *rxq;
@@ -1381,19 +1378,16 @@ priv_flow_create_drop_queue(struct priv *priv)
                WARN("cannot allocate memory for drop queue");
                goto error;
        }
-       fdq->cq = ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
-                       &(struct ibv_exp_cq_init_attr){
-                       .comp_mask = 0,
-                       });
+       fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
        if (!fdq->cq) {
                WARN("cannot allocate CQ for drop queue");
                goto error;
        }
-       fdq->wq = ibv_exp_create_wq(priv->ctx,
-                       &(struct ibv_exp_wq_init_attr){
-                       .wq_type = IBV_EXP_WQT_RQ,
-                       .max_recv_wr = 1,
-                       .max_recv_sge = 1,
+       fdq->wq = ibv_create_wq(priv->ctx,
+                       &(struct ibv_wq_init_attr){
+                       .wq_type = IBV_WQT_RQ,
+                       .max_wr = 1,
+                       .max_sge = 1,
                        .pd = priv->pd,
                        .cq = fdq->cq,
                        });
@@ -1401,9 +1395,8 @@ priv_flow_create_drop_queue(struct priv *priv)
                WARN("cannot allocate WQ for drop queue");
                goto error;
        }
-       fdq->ind_table = ibv_exp_create_rwq_ind_table(priv->ctx,
-                       &(struct ibv_exp_rwq_ind_table_init_attr){
-                       .pd = priv->pd,
+       fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx,
+                       &(struct ibv_rwq_ind_table_init_attr){
                        .log_ind_tbl_size = 0,
                        .ind_tbl = &fdq->wq,
                        .comp_mask = 0,
@@ -1412,24 +1405,23 @@ priv_flow_create_drop_queue(struct priv *priv)
                WARN("cannot allocate indirection table for drop queue");
                goto error;
        }
-       fdq->qp = ibv_exp_create_qp(priv->ctx,
-               &(struct ibv_exp_qp_init_attr){
+       fdq->qp = ibv_create_qp_ex(priv->ctx,
+               &(struct ibv_qp_init_attr_ex){
                        .qp_type = IBV_QPT_RAW_PACKET,
                        .comp_mask =
-                               IBV_EXP_QP_INIT_ATTR_PD |
-                               IBV_EXP_QP_INIT_ATTR_PORT |
-                               IBV_EXP_QP_INIT_ATTR_RX_HASH,
-                       .pd = priv->pd,
-                       .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
+                               IBV_QP_INIT_ATTR_PD |
+                               IBV_QP_INIT_ATTR_IND_TABLE |
+                               IBV_QP_INIT_ATTR_RX_HASH,
+                       .rx_hash_conf = (struct ibv_rx_hash_conf){
                                .rx_hash_function =
-                                       IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
+                                       IBV_RX_HASH_FUNC_TOEPLITZ,
                                .rx_hash_key_len = rss_hash_default_key_len,
                                .rx_hash_key = rss_hash_default_key,
                                .rx_hash_fields_mask = 0,
-                               .rwq_ind_tbl = fdq->ind_table,
                                },
-                       .port_num = priv->port,
-                       });
+                       .rwq_ind_tbl = fdq->ind_table,
+                       .pd = priv->pd
+               });
        if (!fdq->qp) {
                WARN("cannot allocate QP for drop queue");
                goto error;
@@ -1440,9 +1432,9 @@ error:
        if (fdq->qp)
                claim_zero(ibv_destroy_qp(fdq->qp));
        if (fdq->ind_table)
-               claim_zero(ibv_exp_destroy_rwq_ind_table(fdq->ind_table));
+               claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
        if (fdq->wq)
-               claim_zero(ibv_exp_destroy_wq(fdq->wq));
+               claim_zero(ibv_destroy_wq(fdq->wq));
        if (fdq->cq)
                claim_zero(ibv_destroy_cq(fdq->cq));
        if (fdq)
@@ -1467,9 +1459,9 @@ priv_flow_delete_drop_queue(struct priv *priv)
        if (fdq->qp)
                claim_zero(ibv_destroy_qp(fdq->qp));
        if (fdq->ind_table)
-               claim_zero(ibv_exp_destroy_rwq_ind_table(fdq->ind_table));
+               claim_zero(ibv_destroy_rwq_ind_table(fdq->ind_table));
        if (fdq->wq)
-               claim_zero(ibv_exp_destroy_wq(fdq->wq));
+               claim_zero(ibv_destroy_wq(fdq->wq));
        if (fdq->cq)
                claim_zero(ibv_destroy_cq(fdq->cq));
        rte_free(fdq);
@@ -1490,7 +1482,7 @@ priv_flow_stop(struct priv *priv)
        struct rte_flow *flow;
 
        TAILQ_FOREACH_REVERSE(flow, &priv->flows, mlx5_flows, next) {
-               claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
+               claim_zero(ibv_destroy_flow(flow->ibv_flow));
                flow->ibv_flow = NULL;
                if (flow->mark) {
                        unsigned int n;
@@ -1528,7 +1520,7 @@ priv_flow_start(struct priv *priv)
                        qp = priv->flow_drop_queue->qp;
                else
                        qp = flow->qp;
-               flow->ibv_flow = ibv_exp_create_flow(qp, flow->ibv_attr);
+               flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
                if (!flow->ibv_flow) {
                        DEBUG("Flow %p cannot be applied", (void *)flow);
                        rte_errno = EINVAL;
index b3c3fa2..086af58 100644 (file)
@@ -112,8 +112,8 @@ hash_rxq_del_mac_flow(struct hash_rxq *hash_rxq, unsigned int mac_index,
              (*mac)[0], (*mac)[1], (*mac)[2], (*mac)[3], (*mac)[4], (*mac)[5],
              mac_index,
              vlan_index);
-       claim_zero(ibv_exp_destroy_flow(hash_rxq->mac_flow
-                                       [mac_index][vlan_index]));
+       claim_zero(ibv_destroy_flow(hash_rxq->mac_flow
+                                   [mac_index][vlan_index]));
        hash_rxq->mac_flow[mac_index][vlan_index] = NULL;
 }
 
@@ -231,14 +231,14 @@ static int
 hash_rxq_add_mac_flow(struct hash_rxq *hash_rxq, unsigned int mac_index,
                      unsigned int vlan_index)
 {
-       struct ibv_exp_flow *flow;
+       struct ibv_flow *flow;
        struct priv *priv = hash_rxq->priv;
        const uint8_t (*mac)[ETHER_ADDR_LEN] =
                        (const uint8_t (*)[ETHER_ADDR_LEN])
                        priv->mac[mac_index].addr_bytes;
        FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, hash_rxq->type));
-       struct ibv_exp_flow_attr *attr = &data->attr;
-       struct ibv_exp_flow_spec_eth *spec = &data->spec;
+       struct ibv_flow_attr *attr = &data->attr;
+       struct ibv_flow_spec_eth *spec = &data->spec;
        unsigned int vlan_enabled = !!priv->vlan_filter_n;
        unsigned int vlan_id = priv->vlan_filter[vlan_index];
 
@@ -253,10 +253,10 @@ hash_rxq_add_mac_flow(struct hash_rxq *hash_rxq, unsigned int mac_index,
        assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec);
        priv_flow_attr(priv, attr, sizeof(data), hash_rxq->type);
        /* The first specification must be Ethernet. */
-       assert(spec->type == IBV_EXP_FLOW_SPEC_ETH);
+       assert(spec->type == IBV_FLOW_SPEC_ETH);
        assert(spec->size == sizeof(*spec));
-       *spec = (struct ibv_exp_flow_spec_eth){
-               .type = IBV_EXP_FLOW_SPEC_ETH,
+       *spec = (struct ibv_flow_spec_eth){
+               .type = IBV_FLOW_SPEC_ETH,
                .size = sizeof(*spec),
                .val = {
                        .dst_mac = {
@@ -284,7 +284,7 @@ hash_rxq_add_mac_flow(struct hash_rxq *hash_rxq, unsigned int mac_index,
              vlan_id);
        /* Create related flow. */
        errno = 0;
-       flow = ibv_exp_create_flow(hash_rxq->qp, attr);
+       flow = ibv_create_flow(hash_rxq->qp, attr);
        if (flow == NULL) {
                /* It's not clear whether errno is always set in this case. */
                ERROR("%p: flow configuration failed, errno=%d: %s",
index e00be81..2de310b 100644 (file)
@@ -41,7 +41,7 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
-#include <infiniband/mlx5_hw.h>
+#include <infiniband/mlx5dv.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -244,6 +244,46 @@ struct mlx5_cqe {
        uint8_t op_own;
 };
 
+/* Adding direct verbs to data-path. */
+
+/* CQ sequence number mask. */
+#define MLX5_CQ_SQN_MASK 0x3
+
+/* CQ sequence number index. */
+#define MLX5_CQ_SQN_OFFSET 28
+
+/* CQ doorbell index mask. */
+#define MLX5_CI_MASK 0xffffff
+
+/* CQ doorbell offset. */
+#define MLX5_CQ_ARM_DB 1
+
+/* CQ doorbell offset*/
+#define MLX5_CQ_DOORBELL 0x20
+
+/* CQE format value. */
+#define MLX5_COMPRESSED 0x3
+
+/* CQE format mask. */
+#define MLX5E_CQE_FORMAT_MASK 0xc
+
+/* MPW opcode. */
+#define MLX5_OPC_MOD_MPW 0x01
+
+/* Compressed Rx CQE structure. */
+struct mlx5_mini_cqe8 {
+       union {
+               uint32_t rx_hash_result;
+               uint32_t checksum;
+               struct {
+                       uint16_t wqe_counter;
+                       uint8_t  s_wqe_opcode;
+                       uint8_t  reserved;
+               } s_wqe_info;
+       };
+       uint32_t byte_cnt;
+};
+
 /**
  * Convert a user mark to flow mark.
  *
index db2e05b..e9ea2aa 100644 (file)
@@ -122,10 +122,10 @@ hash_rxq_special_flow_enable_vlan(struct hash_rxq *hash_rxq,
                                  unsigned int vlan_index)
 {
        struct priv *priv = hash_rxq->priv;
-       struct ibv_exp_flow *flow;
+       struct ibv_flow *flow;
        FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, hash_rxq->type));
-       struct ibv_exp_flow_attr *attr = &data->attr;
-       struct ibv_exp_flow_spec_eth *spec = &data->spec;
+       struct ibv_flow_attr *attr = &data->attr;
+       struct ibv_flow_spec_eth *spec = &data->spec;
        const uint8_t *mac;
        const uint8_t *mask;
        unsigned int vlan_enabled = (priv->vlan_filter_n &&
@@ -146,13 +146,13 @@ hash_rxq_special_flow_enable_vlan(struct hash_rxq *hash_rxq,
        assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec);
        priv_flow_attr(priv, attr, sizeof(data), hash_rxq->type);
        /* The first specification must be Ethernet. */
-       assert(spec->type == IBV_EXP_FLOW_SPEC_ETH);
+       assert(spec->type == IBV_FLOW_SPEC_ETH);
        assert(spec->size == sizeof(*spec));
 
        mac = special_flow_init[flow_type].dst_mac_val;
        mask = special_flow_init[flow_type].dst_mac_mask;
-       *spec = (struct ibv_exp_flow_spec_eth){
-               .type = IBV_EXP_FLOW_SPEC_ETH,
+       *spec = (struct ibv_flow_spec_eth){
+               .type = IBV_FLOW_SPEC_ETH,
                .size = sizeof(*spec),
                .val = {
                        .dst_mac = {
@@ -175,7 +175,7 @@ hash_rxq_special_flow_enable_vlan(struct hash_rxq *hash_rxq,
        };
 
        errno = 0;
-       flow = ibv_exp_create_flow(hash_rxq->qp, attr);
+       flow = ibv_create_flow(hash_rxq->qp, attr);
        if (flow == NULL) {
                /* It's not clear whether errno is always set in this case. */
                ERROR("%p: flow configuration failed, errno=%d: %s",
@@ -207,12 +207,12 @@ hash_rxq_special_flow_disable_vlan(struct hash_rxq *hash_rxq,
                                   enum hash_rxq_flow_type flow_type,
                                   unsigned int vlan_index)
 {
-       struct ibv_exp_flow *flow =
+       struct ibv_flow *flow =
                hash_rxq->special_flow[flow_type][vlan_index];
 
        if (flow == NULL)
                return;
-       claim_zero(ibv_exp_destroy_flow(flow));
+       claim_zero(ibv_destroy_flow(flow));
        hash_rxq->special_flow[flow_type][vlan_index] = NULL;
        DEBUG("%p: special flow %s (index %d) VLAN %u (index %u) disabled",
              (void *)hash_rxq, hash_rxq_flow_type_str(flow_type), flow_type,
index 437dc02..22448c9 100644 (file)
@@ -44,8 +44,7 @@
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
 #include <infiniband/verbs.h>
-#include <infiniband/arch.h>
-#include <infiniband/mlx5_hw.h>
+#include <infiniband/mlx5dv.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -56,6 +55,7 @@
 #include <rte_common.h>
 #include <rte_interrupts.h>
 #include <rte_debug.h>
+#include <rte_io.h>
 
 #include "mlx5.h"
 #include "mlx5_rxtx.h"
 /* Initialization data for hash RX queues. */
 const struct hash_rxq_init hash_rxq_init[] = {
        [HASH_RXQ_TCPV4] = {
-               .hash_fields = (IBV_EXP_RX_HASH_SRC_IPV4 |
-                               IBV_EXP_RX_HASH_DST_IPV4 |
-                               IBV_EXP_RX_HASH_SRC_PORT_TCP |
-                               IBV_EXP_RX_HASH_DST_PORT_TCP),
+               .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
+                               IBV_RX_HASH_DST_IPV4 |
+                               IBV_RX_HASH_SRC_PORT_TCP |
+                               IBV_RX_HASH_DST_PORT_TCP),
                .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
                .flow_priority = 0,
                .flow_spec.tcp_udp = {
-                       .type = IBV_EXP_FLOW_SPEC_TCP,
+                       .type = IBV_FLOW_SPEC_TCP,
                        .size = sizeof(hash_rxq_init[0].flow_spec.tcp_udp),
                },
                .underlayer = &hash_rxq_init[HASH_RXQ_IPV4],
        },
        [HASH_RXQ_UDPV4] = {
-               .hash_fields = (IBV_EXP_RX_HASH_SRC_IPV4 |
-                               IBV_EXP_RX_HASH_DST_IPV4 |
-                               IBV_EXP_RX_HASH_SRC_PORT_UDP |
-                               IBV_EXP_RX_HASH_DST_PORT_UDP),
+               .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
+                               IBV_RX_HASH_DST_IPV4 |
+                               IBV_RX_HASH_SRC_PORT_UDP |
+                               IBV_RX_HASH_DST_PORT_UDP),
                .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
                .flow_priority = 0,
                .flow_spec.tcp_udp = {
-                       .type = IBV_EXP_FLOW_SPEC_UDP,
+                       .type = IBV_FLOW_SPEC_UDP,
                        .size = sizeof(hash_rxq_init[0].flow_spec.tcp_udp),
                },
                .underlayer = &hash_rxq_init[HASH_RXQ_IPV4],
        },
        [HASH_RXQ_IPV4] = {
-               .hash_fields = (IBV_EXP_RX_HASH_SRC_IPV4 |
-                               IBV_EXP_RX_HASH_DST_IPV4),
+               .hash_fields = (IBV_RX_HASH_SRC_IPV4 |
+                               IBV_RX_HASH_DST_IPV4),
                .dpdk_rss_hf = (ETH_RSS_IPV4 |
                                ETH_RSS_FRAG_IPV4),
                .flow_priority = 1,
                .flow_spec.ipv4 = {
-                       .type = IBV_EXP_FLOW_SPEC_IPV4,
+                       .type = IBV_FLOW_SPEC_IPV4,
                        .size = sizeof(hash_rxq_init[0].flow_spec.ipv4),
                },
                .underlayer = &hash_rxq_init[HASH_RXQ_ETH],
        },
        [HASH_RXQ_TCPV6] = {
-               .hash_fields = (IBV_EXP_RX_HASH_SRC_IPV6 |
-                               IBV_EXP_RX_HASH_DST_IPV6 |
-                               IBV_EXP_RX_HASH_SRC_PORT_TCP |
-                               IBV_EXP_RX_HASH_DST_PORT_TCP),
+               .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
+                               IBV_RX_HASH_DST_IPV6 |
+                               IBV_RX_HASH_SRC_PORT_TCP |
+                               IBV_RX_HASH_DST_PORT_TCP),
                .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
                .flow_priority = 0,
                .flow_spec.tcp_udp = {
-                       .type = IBV_EXP_FLOW_SPEC_TCP,
+                       .type = IBV_FLOW_SPEC_TCP,
                        .size = sizeof(hash_rxq_init[0].flow_spec.tcp_udp),
                },
                .underlayer = &hash_rxq_init[HASH_RXQ_IPV6],
        },
        [HASH_RXQ_UDPV6] = {
-               .hash_fields = (IBV_EXP_RX_HASH_SRC_IPV6 |
-                               IBV_EXP_RX_HASH_DST_IPV6 |
-                               IBV_EXP_RX_HASH_SRC_PORT_UDP |
-                               IBV_EXP_RX_HASH_DST_PORT_UDP),
+               .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
+                               IBV_RX_HASH_DST_IPV6 |
+                               IBV_RX_HASH_SRC_PORT_UDP |
+                               IBV_RX_HASH_DST_PORT_UDP),
                .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
                .flow_priority = 0,
                .flow_spec.tcp_udp = {
-                       .type = IBV_EXP_FLOW_SPEC_UDP,
+                       .type = IBV_FLOW_SPEC_UDP,
                        .size = sizeof(hash_rxq_init[0].flow_spec.tcp_udp),
                },
                .underlayer = &hash_rxq_init[HASH_RXQ_IPV6],
        },
        [HASH_RXQ_IPV6] = {
-               .hash_fields = (IBV_EXP_RX_HASH_SRC_IPV6 |
-                               IBV_EXP_RX_HASH_DST_IPV6),
+               .hash_fields = (IBV_RX_HASH_SRC_IPV6 |
+                               IBV_RX_HASH_DST_IPV6),
                .dpdk_rss_hf = (ETH_RSS_IPV6 |
                                ETH_RSS_FRAG_IPV6),
                .flow_priority = 1,
                .flow_spec.ipv6 = {
-                       .type = IBV_EXP_FLOW_SPEC_IPV6,
+                       .type = IBV_FLOW_SPEC_IPV6,
                        .size = sizeof(hash_rxq_init[0].flow_spec.ipv6),
                },
                .underlayer = &hash_rxq_init[HASH_RXQ_ETH],
@@ -146,7 +146,7 @@ const struct hash_rxq_init hash_rxq_init[] = {
                .dpdk_rss_hf = 0,
                .flow_priority = 2,
                .flow_spec.eth = {
-                       .type = IBV_EXP_FLOW_SPEC_ETH,
+                       .type = IBV_FLOW_SPEC_ETH,
                        .size = sizeof(hash_rxq_init[0].flow_spec.eth),
                },
                .underlayer = NULL,
@@ -215,7 +215,7 @@ const size_t rss_hash_default_key_len = sizeof(rss_hash_default_key);
  *   Total size of the flow attribute buffer. No errors are defined.
  */
 size_t
-priv_flow_attr(struct priv *priv, struct ibv_exp_flow_attr *flow_attr,
+priv_flow_attr(struct priv *priv, struct ibv_flow_attr *flow_attr,
               size_t flow_attr_size, enum hash_rxq_type type)
 {
        size_t offset = sizeof(*flow_attr);
@@ -231,8 +231,8 @@ priv_flow_attr(struct priv *priv, struct ibv_exp_flow_attr *flow_attr,
                return offset;
        flow_attr_size = offset;
        init = &hash_rxq_init[type];
-       *flow_attr = (struct ibv_exp_flow_attr){
-               .type = IBV_EXP_FLOW_ATTR_NORMAL,
+       *flow_attr = (struct ibv_flow_attr){
+               .type = IBV_FLOW_ATTR_NORMAL,
                /* Priorities < 3 are reserved for flow director. */
                .priority = init->flow_priority + 3,
                .num_of_specs = 0,
@@ -338,13 +338,13 @@ priv_make_ind_table_init(struct priv *priv,
 int
 priv_create_hash_rxqs(struct priv *priv)
 {
-       struct ibv_exp_wq *wqs[priv->reta_idx_n];
+       struct ibv_wq *wqs[priv->reta_idx_n];
        struct ind_table_init ind_table_init[IND_TABLE_INIT_N];
        unsigned int ind_tables_n =
                priv_make_ind_table_init(priv, &ind_table_init);
        unsigned int hash_rxqs_n = 0;
        struct hash_rxq (*hash_rxqs)[] = NULL;
-       struct ibv_exp_rwq_ind_table *(*ind_tables)[] = NULL;
+       struct ibv_rwq_ind_table *(*ind_tables)[] = NULL;
        unsigned int i;
        unsigned int j;
        unsigned int k;
@@ -395,21 +395,20 @@ priv_create_hash_rxqs(struct priv *priv)
                goto error;
        }
        for (i = 0; (i != ind_tables_n); ++i) {
-               struct ibv_exp_rwq_ind_table_init_attr ind_init_attr = {
-                       .pd = priv->pd,
+               struct ibv_rwq_ind_table_init_attr ind_init_attr = {
                        .log_ind_tbl_size = 0, /* Set below. */
                        .ind_tbl = wqs,
                        .comp_mask = 0,
                };
                unsigned int ind_tbl_size = ind_table_init[i].max_size;
-               struct ibv_exp_rwq_ind_table *ind_table;
+               struct ibv_rwq_ind_table *ind_table;
 
                if (priv->reta_idx_n < ind_tbl_size)
                        ind_tbl_size = priv->reta_idx_n;
                ind_init_attr.log_ind_tbl_size = log2above(ind_tbl_size);
                errno = 0;
-               ind_table = ibv_exp_create_rwq_ind_table(priv->ctx,
-                                                        &ind_init_attr);
+               ind_table = ibv_create_rwq_ind_table(priv->ctx,
+                                                    &ind_init_attr);
                if (ind_table != NULL) {
                        (*ind_tables)[i] = ind_table;
                        continue;
@@ -437,8 +436,8 @@ priv_create_hash_rxqs(struct priv *priv)
                        hash_rxq_type_from_pos(&ind_table_init[j], k);
                struct rte_eth_rss_conf *priv_rss_conf =
                        (*priv->rss_conf)[type];
-               struct ibv_exp_rx_hash_conf hash_conf = {
-                       .rx_hash_function = IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
+               struct ibv_rx_hash_conf hash_conf = {
+                       .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ,
                        .rx_hash_key_len = (priv_rss_conf ?
                                            priv_rss_conf->rss_key_len :
                                            rss_hash_default_key_len),
@@ -446,23 +445,22 @@ priv_create_hash_rxqs(struct priv *priv)
                                        priv_rss_conf->rss_key :
                                        rss_hash_default_key),
                        .rx_hash_fields_mask = hash_rxq_init[type].hash_fields,
-                       .rwq_ind_tbl = (*ind_tables)[j],
                };
-               struct ibv_exp_qp_init_attr qp_init_attr = {
-                       .max_inl_recv = 0, /* Currently not supported. */
+               struct ibv_qp_init_attr_ex qp_init_attr = {
                        .qp_type = IBV_QPT_RAW_PACKET,
-                       .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
-                                     IBV_EXP_QP_INIT_ATTR_RX_HASH),
+                       .comp_mask = (IBV_QP_INIT_ATTR_PD |
+                                     IBV_QP_INIT_ATTR_IND_TABLE |
+                                     IBV_QP_INIT_ATTR_RX_HASH),
+                       .rx_hash_conf = hash_conf,
+                       .rwq_ind_tbl = (*ind_tables)[j],
                        .pd = priv->pd,
-                       .rx_hash_conf = &hash_conf,
-                       .port_num = priv->port,
                };
 
                DEBUG("using indirection table %u for hash RX queue %u type %d",
                      j, i, type);
                *hash_rxq = (struct hash_rxq){
                        .priv = priv,
-                       .qp = ibv_exp_create_qp(priv->ctx, &qp_init_attr),
+                       .qp = ibv_create_qp_ex(priv->ctx, &qp_init_attr),
                        .type = type,
                };
                if (hash_rxq->qp == NULL) {
@@ -497,12 +495,12 @@ error:
        }
        if (ind_tables != NULL) {
                for (j = 0; (j != ind_tables_n); ++j) {
-                       struct ibv_exp_rwq_ind_table *ind_table =
+                       struct ibv_rwq_ind_table *ind_table =
                                (*ind_tables)[j];
 
                        if (ind_table == NULL)
                                continue;
-                       claim_zero(ibv_exp_destroy_rwq_ind_table(ind_table));
+                       claim_zero(ibv_destroy_rwq_ind_table(ind_table));
                }
                rte_free(ind_tables);
        }
@@ -547,11 +545,11 @@ priv_destroy_hash_rxqs(struct priv *priv)
        rte_free(priv->hash_rxqs);
        priv->hash_rxqs = NULL;
        for (i = 0; (i != priv->ind_tables_n); ++i) {
-               struct ibv_exp_rwq_ind_table *ind_table =
+               struct ibv_rwq_ind_table *ind_table =
                        (*priv->ind_tables)[i];
 
                assert(ind_table != NULL);
-               claim_zero(ibv_exp_destroy_rwq_ind_table(ind_table));
+               claim_zero(ibv_destroy_rwq_ind_table(ind_table));
        }
        priv->ind_tables_n = 0;
        rte_free(priv->ind_tables);
@@ -765,7 +763,7 @@ rxq_cleanup(struct rxq_ctrl *rxq_ctrl)
        if (rxq_ctrl->fdir_queue != NULL)
                priv_fdir_queue_destroy(rxq_ctrl->priv, rxq_ctrl->fdir_queue);
        if (rxq_ctrl->wq != NULL)
-               claim_zero(ibv_exp_destroy_wq(rxq_ctrl->wq));
+               claim_zero(ibv_destroy_wq(rxq_ctrl->wq));
        if (rxq_ctrl->cq != NULL)
                claim_zero(ibv_destroy_cq(rxq_ctrl->cq));
        if (rxq_ctrl->channel != NULL)
@@ -788,16 +786,23 @@ static inline int
 rxq_setup(struct rxq_ctrl *tmpl)
 {
        struct ibv_cq *ibcq = tmpl->cq;
-       struct ibv_mlx5_cq_info cq_info;
-       struct mlx5_rwq *rwq = container_of(tmpl->wq, struct mlx5_rwq, wq);
+       struct mlx5dv_cq cq_info;
+       struct mlx5dv_rwq rwq;
        const uint16_t desc_n =
                (1 << tmpl->rxq.elts_n) + tmpl->priv->rx_vec_en *
                MLX5_VPMD_DESCS_PER_LOOP;
        struct rte_mbuf *(*elts)[desc_n] =
                rte_calloc_socket("RXQ", 1, sizeof(*elts), 0, tmpl->socket);
-       if (ibv_mlx5_exp_get_cq_info(ibcq, &cq_info)) {
-               ERROR("Unable to query CQ info. check your OFED.");
-               return ENOTSUP;
+       struct mlx5dv_obj obj;
+       int ret = 0;
+
+       obj.cq.in = ibcq;
+       obj.cq.out = &cq_info;
+       obj.rwq.in = tmpl->wq;
+       obj.rwq.out = &rwq;
+       ret = mlx5dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_RWQ);
+       if (ret != 0) {
+               return -EINVAL;
        }
        if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {
                ERROR("Wrong MLX5_CQE_SIZE environment variable value: "
@@ -806,7 +811,7 @@ rxq_setup(struct rxq_ctrl *tmpl)
        }
        if (elts == NULL)
                return ENOMEM;
-       tmpl->rxq.rq_db = rwq->rq.db;
+       tmpl->rxq.rq_db = rwq.dbrec;
        tmpl->rxq.cqe_n = log2above(cq_info.cqe_cnt);
        tmpl->rxq.cq_ci = 0;
        tmpl->rxq.rq_ci = 0;
@@ -814,11 +819,14 @@ rxq_setup(struct rxq_ctrl *tmpl)
        tmpl->rxq.cq_db = cq_info.dbrec;
        tmpl->rxq.wqes =
                (volatile struct mlx5_wqe_data_seg (*)[])
-               (uintptr_t)rwq->rq.buff;
+               (uintptr_t)rwq.buf;
        tmpl->rxq.cqes =
                (volatile struct mlx5_cqe (*)[])
                (uintptr_t)cq_info.buf;
        tmpl->rxq.elts = elts;
+       tmpl->rxq.cq_uar = cq_info.cq_uar;
+       tmpl->rxq.cqn = cq_info.cqn;
+       tmpl->rxq.cq_arm_sn = 0;
        return 0;
 }
 
@@ -856,11 +864,11 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
                        .rss_hash = priv->rxqs_n > 1,
                },
        };
-       struct ibv_exp_wq_attr mod;
+       struct ibv_wq_attr mod;
        union {
-               struct ibv_exp_cq_init_attr cq;
-               struct ibv_exp_wq_init_attr wq;
-               struct ibv_exp_cq_attr cq_attr;
+               struct ibv_cq_init_attr_ex cq;
+               struct ibv_wq_init_attr wq;
+               struct ibv_cq_ex cq_attr;
        } attr;
        unsigned int mb_len = rte_pktmbuf_data_room_size(mp);
        unsigned int cqe_n = desc - 1;
@@ -940,12 +948,12 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
                        goto error;
                }
        }
-       attr.cq = (struct ibv_exp_cq_init_attr){
+       attr.cq = (struct ibv_cq_init_attr_ex){
                .comp_mask = 0,
        };
        if (priv->cqe_comp) {
-               attr.cq.comp_mask |= IBV_EXP_CQ_INIT_ATTR_FLAGS;
-               attr.cq.flags |= IBV_EXP_CQ_COMPRESSED_CQE;
+               attr.cq.comp_mask |= IBV_CQ_INIT_ATTR_MASK_FLAGS;
+               attr.cq.flags |= MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE;
                /*
                 * For vectorized Rx, it must not be doubled in order to
                 * make cq_ci and rq_ci aligned.
@@ -953,8 +961,7 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
                if (rxq_check_vec_support(&tmpl.rxq) < 0)
                        cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */
        }
-       tmpl.cq = ibv_exp_create_cq(priv->ctx, cqe_n, NULL, tmpl.channel, 0,
-                                   &attr.cq);
+       tmpl.cq = ibv_create_cq(priv->ctx, cqe_n, NULL, tmpl.channel, 0);
        if (tmpl.cq == NULL) {
                ret = ENOMEM;
                ERROR("%p: CQ creation failure: %s",
@@ -962,35 +969,35 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
                goto error;
        }
        DEBUG("priv->device_attr.max_qp_wr is %d",
-             priv->device_attr.max_qp_wr);
+             priv->device_attr.orig_attr.max_qp_wr);
        DEBUG("priv->device_attr.max_sge is %d",
-             priv->device_attr.max_sge);
+             priv->device_attr.orig_attr.max_sge);
        /* Configure VLAN stripping. */
        tmpl.rxq.vlan_strip = (priv->hw_vlan_strip &&
                               !!dev->data->dev_conf.rxmode.hw_vlan_strip);
-       attr.wq = (struct ibv_exp_wq_init_attr){
+       attr.wq = (struct ibv_wq_init_attr){
                .wq_context = NULL, /* Could be useful in the future. */
-               .wq_type = IBV_EXP_WQT_RQ,
+               .wq_type = IBV_WQT_RQ,
                /* Max number of outstanding WRs. */
-               .max_recv_wr = desc >> tmpl.rxq.sges_n,
+               .max_wr = desc >> tmpl.rxq.sges_n,
                /* Max number of scatter/gather elements in a WR. */
-               .max_recv_sge = 1 << tmpl.rxq.sges_n,
+               .max_sge = 1 << tmpl.rxq.sges_n,
                .pd = priv->pd,
                .cq = tmpl.cq,
                .comp_mask =
-                       IBV_EXP_CREATE_WQ_VLAN_OFFLOADS |
+                       IBV_WQ_FLAGS_CVLAN_STRIPPING |
                        0,
-               .vlan_offloads = (tmpl.rxq.vlan_strip ?
-                                 IBV_EXP_RECEIVE_WQ_CVLAN_STRIP :
-                                 0),
+               .create_flags = (tmpl.rxq.vlan_strip ?
+                                IBV_WQ_FLAGS_CVLAN_STRIPPING :
+                                0),
        };
        /* By default, FCS (CRC) is stripped by hardware. */
        if (dev->data->dev_conf.rxmode.hw_strip_crc) {
                tmpl.rxq.crc_present = 0;
        } else if (priv->hw_fcs_strip) {
                /* Ask HW/Verbs to leave CRC in place when supported. */
-               attr.wq.flags |= IBV_EXP_CREATE_WQ_FLAG_SCATTER_FCS;
-               attr.wq.comp_mask |= IBV_EXP_CREATE_WQ_FLAGS;
+               attr.wq.create_flags |= IBV_WQ_FLAGS_SCATTER_FCS;
+               attr.wq.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
                tmpl.rxq.crc_present = 1;
        } else {
                WARN("%p: CRC stripping has been disabled but will still"
@@ -1004,20 +1011,22 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
              (void *)dev,
              tmpl.rxq.crc_present ? "disabled" : "enabled",
              tmpl.rxq.crc_present << 2);
+#ifdef HAVE_IBV_WQ_FLAG_RX_END_PADDING
        if (!mlx5_getenv_int("MLX5_PMD_ENABLE_PADDING"))
                ; /* Nothing else to do. */
        else if (priv->hw_padding) {
                INFO("%p: enabling packet padding on queue %p",
                     (void *)dev, (void *)rxq_ctrl);
-               attr.wq.flags |= IBV_EXP_CREATE_WQ_FLAG_RX_END_PADDING;
-               attr.wq.comp_mask |= IBV_EXP_CREATE_WQ_FLAGS;
+               attr.wq.create_flags |= IBV_WQ_FLAG_RX_END_PADDING;
+               attr.wq.comp_mask |= IBV_WQ_INIT_ATTR_FLAGS;
        } else
                WARN("%p: packet padding has been requested but is not"
                     " supported, make sure MLNX_OFED and firmware are"
                     " up to date",
                     (void *)dev);
+#endif
 
-       tmpl.wq = ibv_exp_create_wq(priv->ctx, &attr.wq);
+       tmpl.wq = ibv_create_wq(priv->ctx, &attr.wq);
        if (tmpl.wq == NULL) {
                ret = (errno ? errno : EINVAL);
                ERROR("%p: WQ creation failure: %s",
@@ -1028,12 +1037,12 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
         * Make sure number of WRs*SGEs match expectations since a queue
         * cannot allocate more than "desc" buffers.
         */
-       if (((int)attr.wq.max_recv_wr != (desc >> tmpl.rxq.sges_n)) ||
-           ((int)attr.wq.max_recv_sge != (1 << tmpl.rxq.sges_n))) {
+       if (((int)attr.wq.max_wr != (desc >> tmpl.rxq.sges_n)) ||
+           ((int)attr.wq.max_sge != (1 << tmpl.rxq.sges_n))) {
                ERROR("%p: requested %u*%u but got %u*%u WRs*SGEs",
                      (void *)dev,
                      (desc >> tmpl.rxq.sges_n), (1 << tmpl.rxq.sges_n),
-                     attr.wq.max_recv_wr, attr.wq.max_recv_sge);
+                     attr.wq.max_wr, attr.wq.max_sge);
                ret = EINVAL;
                goto error;
        }
@@ -1041,13 +1050,13 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
        tmpl.rxq.port_id = dev->data->port_id;
        DEBUG("%p: RTE port ID: %u", (void *)rxq_ctrl, tmpl.rxq.port_id);
        /* Change queue state to ready. */
-       mod = (struct ibv_exp_wq_attr){
-               .attr_mask = IBV_EXP_WQ_ATTR_STATE,
-               .wq_state = IBV_EXP_WQS_RDY,
+       mod = (struct ibv_wq_attr){
+               .attr_mask = IBV_WQ_ATTR_STATE,
+               .wq_state = IBV_WQS_RDY,
        };
-       ret = ibv_exp_modify_wq(tmpl.wq, &mod);
+       ret = ibv_modify_wq(tmpl.wq, &mod);
        if (ret) {
-               ERROR("%p: WQ state to IBV_EXP_WQS_RDY failed: %s",
+               ERROR("%p: WQ state to IBV_WQS_RDY failed: %s",
                      (void *)dev, strerror(ret));
                goto error;
        }
@@ -1311,7 +1320,30 @@ priv_rx_intr_vec_disable(struct priv *priv)
        intr_handle->intr_vec = NULL;
 }
 
-#ifdef HAVE_UPDATE_CQ_CI
+/**
+ *  MLX5 CQ notification .
+ *
+ *  @param rxq
+ *     Pointer to receive queue structure.
+ *  @param sq_n_rxq
+ *     Sequence number per receive queue .
+ */
+static inline void
+mlx5_arm_cq(struct rxq *rxq, int sq_n_rxq)
+{
+       int sq_n = 0;
+       uint32_t doorbell_hi;
+       uint64_t doorbell;
+       void *cq_db_reg = (char *)rxq->cq_uar + MLX5_CQ_DOORBELL;
+
+       sq_n = sq_n_rxq & MLX5_CQ_SQN_MASK;
+       doorbell_hi = sq_n << MLX5_CQ_SQN_OFFSET | (rxq->cq_ci & MLX5_CI_MASK);
+       doorbell = (uint64_t)doorbell_hi << 32;
+       doorbell |=  rxq->cqn;
+       rxq->cq_db[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(doorbell_hi);
+       rte_wmb();
+       rte_write64(rte_cpu_to_be_64(doorbell), cq_db_reg);
+}
 
 /**
  * DPDK callback for Rx queue interrupt enable.
@@ -1330,13 +1362,12 @@ mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
        struct priv *priv = mlx5_get_priv(dev);
        struct rxq *rxq = (*priv->rxqs)[rx_queue_id];
        struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq);
-       int ret;
+       int ret = 0;
 
        if (!rxq || !rxq_ctrl->channel) {
                ret = EINVAL;
        } else {
-               ibv_mlx5_exp_update_cq_ci(rxq_ctrl->cq, rxq->cq_ci);
-               ret = ibv_req_notify_cq(rxq_ctrl->cq, 0);
+               mlx5_arm_cq(rxq, rxq->cq_arm_sn);
        }
        if (ret)
                WARN("unable to arm interrupt on rx queue %d", rx_queue_id);
@@ -1368,6 +1399,7 @@ mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
                ret = EINVAL;
        } else {
                ret = ibv_get_cq_event(rxq_ctrl->cq->channel, &ev_cq, &ev_ctx);
+               rxq->cq_arm_sn++;
                if (ret || ev_cq != rxq_ctrl->cq)
                        ret = EINVAL;
        }
@@ -1378,5 +1410,3 @@ mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
                ibv_ack_cq_events(rxq_ctrl->cq, 1);
        return -ret;
 }
-
-#endif /* HAVE_UPDATE_CQ_CI */
index 674cce0..c45ebee 100644 (file)
@@ -42,8 +42,7 @@
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
 #include <infiniband/verbs.h>
-#include <infiniband/mlx5_hw.h>
-#include <infiniband/arch.h>
+#include <infiniband/mlx5dv.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
index 9375aa8..342c933 100644 (file)
@@ -43,7 +43,7 @@
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
 #include <infiniband/verbs.h>
-#include <infiniband/mlx5_hw.h>
+#include <infiniband/mlx5dv.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
@@ -81,8 +81,8 @@ struct mlx5_txq_stats {
 /* Flow director queue structure. */
 struct fdir_queue {
        struct ibv_qp *qp; /* Associated RX QP. */
-       struct ibv_exp_rwq_ind_table *ind_table; /* Indirection table. */
-       struct ibv_exp_wq *wq; /* Work queue. */
+       struct ibv_rwq_ind_table *ind_table; /* Indirection table. */
+       struct ibv_wq *wq; /* Work queue. */
        struct ibv_cq *cq; /* Completion queue. */
 };
 
@@ -124,13 +124,16 @@ struct rxq {
        struct mlx5_rxq_stats stats;
        uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */
        struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */
+       void *cq_uar; /* CQ user access region. */
+       uint32_t cqn; /* CQ number. */
+       uint8_t cq_arm_sn; /* CQ arm seq number. */
 } __rte_cache_aligned;
 
 /* RX queue control descriptor. */
 struct rxq_ctrl {
        struct priv *priv; /* Back pointer to private data. */
        struct ibv_cq *cq; /* Completion Queue. */
-       struct ibv_exp_wq *wq; /* Work Queue. */
+       struct ibv_wq *wq; /* Work Queue. */
        struct fdir_queue *fdir_queue; /* Flow director queue. */
        struct ibv_mr *mr; /* Memory Region (for mp). */
        struct ibv_comp_channel *channel;
@@ -152,8 +155,8 @@ enum hash_rxq_type {
 /* Flow structure with Ethernet specification. It is packed to prevent padding
  * between attr and spec as this layout is expected by libibverbs. */
 struct flow_attr_spec_eth {
-       struct ibv_exp_flow_attr attr;
-       struct ibv_exp_flow_spec_eth spec;
+       struct ibv_flow_attr attr;
+       struct ibv_flow_spec_eth spec;
 } __attribute__((packed));
 
 /* Define a struct flow_attr_spec_eth object as an array of at least
@@ -171,13 +174,13 @@ struct hash_rxq_init {
        unsigned int flow_priority; /* Flow priority to use. */
        union {
                struct {
-                       enum ibv_exp_flow_spec_type type;
+                       enum ibv_flow_spec_type type;
                        uint16_t size;
                } hdr;
-               struct ibv_exp_flow_spec_tcp_udp tcp_udp;
-               struct ibv_exp_flow_spec_ipv4 ipv4;
-               struct ibv_exp_flow_spec_ipv6 ipv6;
-               struct ibv_exp_flow_spec_eth eth;
+               struct ibv_flow_spec_tcp_udp tcp_udp;
+               struct ibv_flow_spec_ipv4 ipv4;
+               struct ibv_flow_spec_ipv6 ipv6;
+               struct ibv_flow_spec_eth eth;
        } flow_spec; /* Flow specification template. */
        const struct hash_rxq_init *underlayer; /* Pointer to underlayer. */
 };
@@ -231,9 +234,9 @@ struct hash_rxq {
        struct ibv_qp *qp; /* Hash RX QP. */
        enum hash_rxq_type type; /* Hash RX queue type. */
        /* MAC flow steering rules, one per VLAN ID. */
-       struct ibv_exp_flow *mac_flow
+       struct ibv_flow *mac_flow
                [MLX5_MAX_MAC_ADDRESSES][MLX5_MAX_VLAN_IDS];
-       struct ibv_exp_flow *special_flow
+       struct ibv_flow *special_flow
                [MLX5_MAX_SPECIAL_FLOWS][MLX5_MAX_VLAN_IDS];
 };
 
@@ -293,7 +296,7 @@ extern const unsigned int hash_rxq_init_n;
 extern uint8_t rss_hash_default_key[];
 extern const size_t rss_hash_default_key_len;
 
-size_t priv_flow_attr(struct priv *, struct ibv_exp_flow_attr *,
+size_t priv_flow_attr(struct priv *, struct ibv_flow_attr *,
                      size_t, enum hash_rxq_type);
 int priv_create_hash_rxqs(struct priv *);
 void priv_destroy_hash_rxqs(struct priv *);
@@ -305,10 +308,8 @@ int mlx5_rx_queue_setup(struct rte_eth_dev *, uint16_t, uint16_t, unsigned int,
 void mlx5_rx_queue_release(void *);
 int priv_rx_intr_vec_enable(struct priv *priv);
 void priv_rx_intr_vec_disable(struct priv *priv);
-#ifdef HAVE_UPDATE_CQ_CI
 int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
 int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
-#endif /* HAVE_UPDATE_CQ_CI */
 
 /* mlx5_txq.c */
 
index aff3359..33988e3 100644 (file)
@@ -43,8 +43,7 @@
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
 #include <infiniband/verbs.h>
-#include <infiniband/mlx5_hw.h>
-#include <infiniband/arch.h>
+#include <infiniband/mlx5dv.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
index b4c5b10..39a38c1 100644 (file)
@@ -162,13 +162,19 @@ txq_cleanup(struct txq_ctrl *txq_ctrl)
 static inline int
 txq_setup(struct txq_ctrl *tmpl, struct txq_ctrl *txq_ctrl)
 {
-       struct mlx5_qp *qp = to_mqp(tmpl->qp);
+       struct mlx5dv_qp qp;
        struct ibv_cq *ibcq = tmpl->cq;
-       struct ibv_mlx5_cq_info cq_info;
+       struct mlx5dv_cq cq_info;
+       struct mlx5dv_obj obj;
+       int ret = 0;
 
-       if (ibv_mlx5_exp_get_cq_info(ibcq, &cq_info)) {
-               ERROR("Unable to query CQ info. check your OFED.");
-               return ENOTSUP;
+       obj.cq.in = ibcq;
+       obj.cq.out = &cq_info;
+       obj.qp.in = tmpl->qp;
+       obj.qp.out = &qp;
+       ret = mlx5dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_QP);
+       if (ret != 0) {
+               return -EINVAL;
        }
        if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {
                ERROR("Wrong MLX5_CQE_SIZE environment variable value: "
@@ -176,11 +182,11 @@ txq_setup(struct txq_ctrl *tmpl, struct txq_ctrl *txq_ctrl)
                return EINVAL;
        }
        tmpl->txq.cqe_n = log2above(cq_info.cqe_cnt);
-       tmpl->txq.qp_num_8s = qp->ctrl_seg.qp_num << 8;
-       tmpl->txq.wqes = qp->gen_data.sqstart;
-       tmpl->txq.wqe_n = log2above(qp->sq.wqe_cnt);
-       tmpl->txq.qp_db = &qp->gen_data.db[MLX5_SND_DBR];
-       tmpl->txq.bf_reg = qp->gen_data.bf->reg;
+       tmpl->txq.qp_num_8s = tmpl->qp->qp_num << 8;
+       tmpl->txq.wqes = qp.sq.buf;
+       tmpl->txq.wqe_n = log2above(qp.sq.wqe_cnt);
+       tmpl->txq.qp_db = &qp.dbrec[MLX5_SND_DBR];
+       tmpl->txq.bf_reg = qp.bf.reg;
        tmpl->txq.cq_db = cq_info.dbrec;
        tmpl->txq.cqes =
                (volatile struct mlx5_cqe (*)[])
@@ -219,10 +225,10 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
                .socket = socket,
        };
        union {
-               struct ibv_exp_qp_init_attr init;
-               struct ibv_exp_cq_init_attr cq;
-               struct ibv_exp_qp_attr mod;
-               struct ibv_exp_cq_attr cq_attr;
+               struct ibv_qp_init_attr_ex init;
+               struct ibv_cq_init_attr_ex cq;
+               struct ibv_qp_attr mod;
+               struct ibv_cq_ex cq_attr;
        } attr;
        unsigned int cqe_n;
        const unsigned int max_tso_inline = ((MLX5_MAX_TSO_HEADER +
@@ -241,16 +247,16 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
        if (priv->mps == MLX5_MPW_ENHANCED)
                tmpl.txq.mpw_hdr_dseg = priv->mpw_hdr_dseg;
        /* MRs will be registered in mp2mr[] later. */
-       attr.cq = (struct ibv_exp_cq_init_attr){
+       attr.cq = (struct ibv_cq_init_attr_ex){
                .comp_mask = 0,
        };
        cqe_n = ((desc / MLX5_TX_COMP_THRESH) - 1) ?
                ((desc / MLX5_TX_COMP_THRESH) - 1) : 1;
        if (priv->mps == MLX5_MPW_ENHANCED)
                cqe_n += MLX5_TX_COMP_THRESH_INLINE_DIV;
-       tmpl.cq = ibv_exp_create_cq(priv->ctx,
-                                   cqe_n,
-                                   NULL, NULL, 0, &attr.cq);
+       tmpl.cq = ibv_create_cq(priv->ctx,
+                               cqe_n,
+                               NULL, NULL, 0);
        if (tmpl.cq == NULL) {
                ret = ENOMEM;
                ERROR("%p: CQ creation failure: %s",
@@ -258,19 +264,20 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
                goto error;
        }
        DEBUG("priv->device_attr.max_qp_wr is %d",
-             priv->device_attr.max_qp_wr);
+             priv->device_attr.orig_attr.max_qp_wr);
        DEBUG("priv->device_attr.max_sge is %d",
-             priv->device_attr.max_sge);
-       attr.init = (struct ibv_exp_qp_init_attr){
+             priv->device_attr.orig_attr.max_sge);
+       attr.init = (struct ibv_qp_init_attr_ex){
                /* CQ to be associated with the send queue. */
                .send_cq = tmpl.cq,
                /* CQ to be associated with the receive queue. */
                .recv_cq = tmpl.cq,
                .cap = {
                        /* Max number of outstanding WRs. */
-                       .max_send_wr = ((priv->device_attr.max_qp_wr < desc) ?
-                                       priv->device_attr.max_qp_wr :
-                                       desc),
+                       .max_send_wr =
+                        ((priv->device_attr.orig_attr.max_qp_wr < desc) ?
+                          priv->device_attr.orig_attr.max_qp_wr :
+                          desc),
                        /*
                         * Max number of scatter/gather elements in a WR,
                         * must be 1 to prevent libmlx5 from trying to affect
@@ -285,7 +292,7 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
                 * TX burst. */
                .sq_sig_all = 0,
                .pd = priv->pd,
-               .comp_mask = IBV_EXP_QP_INIT_ATTR_PD,
+               .comp_mask = IBV_QP_INIT_ATTR_PD,
        };
        if (priv->txq_inline && (priv->txqs_n >= priv->txqs_inline)) {
                unsigned int ds_cnt;
@@ -348,14 +355,14 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
        if (priv->tso) {
                attr.init.max_tso_header =
                        max_tso_inline * RTE_CACHE_LINE_SIZE;
-               attr.init.comp_mask |= IBV_EXP_QP_INIT_ATTR_MAX_TSO_HEADER;
+               attr.init.comp_mask |= IBV_QP_INIT_ATTR_MAX_TSO_HEADER;
                tmpl.txq.max_inline = RTE_MAX(tmpl.txq.max_inline,
                                              max_tso_inline);
                tmpl.txq.tso_en = 1;
        }
        if (priv->tunnel_en)
                tmpl.txq.tunnel_en = 1;
-       tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init);
+       tmpl.qp = ibv_create_qp_ex(priv->ctx, &attr.init);
        if (tmpl.qp == NULL) {
                ret = (errno ? errno : EINVAL);
                ERROR("%p: QP creation failure: %s",
@@ -367,14 +374,14 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
              attr.init.cap.max_send_wr,
              attr.init.cap.max_send_sge,
              attr.init.cap.max_inline_data);
-       attr.mod = (struct ibv_exp_qp_attr){
+       attr.mod = (struct ibv_qp_attr){
                /* Move the QP to this state. */
                .qp_state = IBV_QPS_INIT,
                /* Primary port number. */
                .port_num = priv->port
        };
-       ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod,
-                               (IBV_EXP_QP_STATE | IBV_EXP_QP_PORT));
+       ret = ibv_modify_qp(tmpl.qp, &attr.mod,
+                           (IBV_QP_STATE | IBV_QP_PORT));
        if (ret) {
                ERROR("%p: QP state to IBV_QPS_INIT failed: %s",
                      (void *)dev, strerror(ret));
@@ -387,17 +394,17 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
                goto error;
        }
        txq_alloc_elts(&tmpl, desc);
-       attr.mod = (struct ibv_exp_qp_attr){
+       attr.mod = (struct ibv_qp_attr){
                .qp_state = IBV_QPS_RTR
        };
-       ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE);
+       ret = ibv_modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE);
        if (ret) {
                ERROR("%p: QP state to IBV_QPS_RTR failed: %s",
                      (void *)dev, strerror(ret));
                goto error;
        }
        attr.mod.qp_state = IBV_QPS_RTS;
-       ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE);
+       ret = ibv_modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE);
        if (ret) {
                ERROR("%p: QP state to IBV_QPS_RTS failed: %s",
                      (void *)dev, strerror(ret));
index 353ae49..36ffbba 100644 (file)
@@ -139,20 +139,21 @@ priv_vlan_strip_queue_set(struct priv *priv, uint16_t idx, int on)
 {
        struct rxq *rxq = (*priv->rxqs)[idx];
        struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq);
-       struct ibv_exp_wq_attr mod;
+       struct ibv_wq_attr mod;
        uint16_t vlan_offloads =
-               (on ? IBV_EXP_RECEIVE_WQ_CVLAN_STRIP : 0) |
+               (on ? IBV_WQ_FLAGS_CVLAN_STRIPPING : 0) |
                0;
        int err;
 
        DEBUG("set VLAN offloads 0x%x for port %d queue %d",
              vlan_offloads, rxq->port_id, idx);
-       mod = (struct ibv_exp_wq_attr){
-               .attr_mask = IBV_EXP_WQ_ATTR_VLAN_OFFLOADS,
-               .vlan_offloads = vlan_offloads,
+       mod = (struct ibv_wq_attr){
+               .attr_mask = IBV_WQ_ATTR_FLAGS,
+               .flags_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING,
+               .flags = vlan_offloads,
        };
 
-       err = ibv_exp_modify_wq(rxq_ctrl->wq, &mod);
+       err = ibv_modify_wq(rxq_ctrl->wq, &mod);
        if (err) {
                ERROR("%p: failed to modified stripping mode: %s",
                      (void *)priv, strerror(err));
index c25fdd9..9415537 100644 (file)
@@ -129,7 +129,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI)        += -lrte_pmd_kni
 endif
 _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD)        += -lrte_pmd_lio
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD)       += -lrte_pmd_mlx4 -libverbs
-_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -libverbs
+_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD)       += -lrte_pmd_mlx5 -libverbs -lmlx5
 _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lrte_pmd_nfp
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL)       += -lrte_pmd_null
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_PCAP)       += -lrte_pmd_pcap -lpcap