drivers/net/mlx5/mlx5_rxtx_vec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdint.h>
   7 #include <string.h>
   8 #include <stdlib.h>
   9
  10 /* Verbs header. */
  11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  12 #ifdef PEDANTIC
  13 #pragma GCC diagnostic ignored "-Wpedantic"
  14 #endif
  15 #include <infiniband/verbs.h>
  16 #include <infiniband/mlx5dv.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_mbuf.h>
  22 #include <rte_mempool.h>
  23 #include <rte_prefetch.h>
  24
  25 #include <mlx5_prm.h>
  26
  27 #include "mlx5_defs.h"
  28 #include "mlx5.h"
  29 #include "mlx5_utils.h"
  30 #include "mlx5_rxtx.h"
  31 #include "mlx5_rxtx_vec.h"
  32 #include "mlx5_autoconf.h"
  33
  34 #if defined RTE_ARCH_X86_64
  35 #include "mlx5_rxtx_vec_sse.h"
  36 #elif defined RTE_ARCH_ARM64
  37 #include "mlx5_rxtx_vec_neon.h"
  38 #elif defined RTE_ARCH_PPC_64
  39 #include "mlx5_rxtx_vec_altivec.h"
  40 #else
  41 #error "This should not be compiled if SIMD instructions are not supported."
  42 #endif
  43
  44 /**
  45  * Skip error packets.
  46  *
  47  * @param rxq
  48  *   Pointer to RX queue structure.
  49  * @param[out] pkts
  50  *   Array to store received packets.
  51  * @param pkts_n
  52  *   Maximum number of packets in array.
  53  *
  54  * @return
  55  *   Number of packets successfully received (<= pkts_n).
  56  */
  57 static uint16_t
  58 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
  59                          uint16_t pkts_n)
  60 {
  61         uint16_t n = 0;
  62         unsigned int i;
  63 #ifdef MLX5_PMD_SOFT_COUNTERS
  64         uint32_t err_bytes = 0;
  65 #endif
  66
  67         for (i = 0; i < pkts_n; ++i) {
  68                 struct rte_mbuf *pkt = pkts[i];
  69
  70                 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) {
  71 #ifdef MLX5_PMD_SOFT_COUNTERS
  72                         err_bytes += PKT_LEN(pkt);
  73 #endif
  74                         rte_pktmbuf_free_seg(pkt);
  75                 } else {
  76                         pkts[n++] = pkt;
  77                 }
  78         }
  79         rxq->stats.idropped += (pkts_n - n);
  80 #ifdef MLX5_PMD_SOFT_COUNTERS
  81         /* Correct counters of errored completions. */
  82         rxq->stats.ipackets -= (pkts_n - n);
  83         rxq->stats.ibytes -= err_bytes;
  84 #endif
  85         mlx5_rx_err_handle(rxq, 1);
  86         return n;
  87 }
  88
  89 /**
  90  * DPDK callback for vectorized RX.
  91  *
  92  * @param dpdk_rxq
  93  *   Generic pointer to RX queue structure.
  94  * @param[out] pkts
  95  *   Array to store received packets.
  96  * @param pkts_n
  97  *   Maximum number of packets in array.
  98  *
  99  * @return
 100  *   Number of packets successfully received (<= pkts_n).
 101  */
 102 uint16_t
 103 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 104 {
 105         struct mlx5_rxq_data *rxq = dpdk_rxq;
 106         uint16_t nb_rx = 0;
 107         uint16_t tn = 0;
 108         uint64_t err = 0;
 109         bool no_cq = false;
 110
 111         do {
 112                 nb_rx = rxq_burst_v(rxq, pkts + tn, pkts_n - tn, &err, &no_cq);
 113                 if (unlikely(err | rxq->err_state))
 114                         nb_rx = rxq_handle_pending_error(rxq, pkts + tn, nb_rx);
 115                 tn += nb_rx;
 116                 if (unlikely(no_cq))
 117                         break;
 118         } while (tn != pkts_n);
 119         return tn;
 120 }
 121
 122 /**
 123  * Check a RX queue can support vectorized RX.
 124  *
 125  * @param rxq
 126  *   Pointer to RX queue.
 127  *
 128  * @return
 129  *   1 if supported, negative errno value if not.
 130  */
 131 int __rte_cold
 132 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)
 133 {
 134         struct mlx5_rxq_ctrl *ctrl =
 135                 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 136
 137         if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv)))
 138                 return -ENOTSUP;
 139         if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0)
 140                 return -ENOTSUP;
 141         if (rxq->lro)
 142                 return -ENOTSUP;
 143         return 1;
 144 }
 145
 146 /**
 147  * Check a device can support vectorized RX.
 148  *
 149  * @param dev
 150  *   Pointer to Ethernet device.
 151  *
 152  * @return
 153  *   1 if supported, negative errno value if not.
 154  */
 155 int __rte_cold
 156 mlx5_check_vec_rx_support(struct rte_eth_dev *dev)
 157 {
 158         struct mlx5_priv *priv = dev->data->dev_private;
 159         uint16_t i;
 160
 161         if (!priv->config.rx_vec_en)
 162                 return -ENOTSUP;
 163         if (mlx5_mprq_enabled(dev))
 164                 return -ENOTSUP;
 165         /* All the configured queues should support. */
 166         for (i = 0; i < priv->rxqs_n; ++i) {
 167                 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
 168
 169                 if (!rxq)
 170                         continue;
 171                 if (mlx5_rxq_check_vec_support(rxq) < 0)
 172                         break;
 173         }
 174         if (i != priv->rxqs_n)
 175                 return -ENOTSUP;
 176         return 1;
 177 }