drivers/net/mlx5/mlx5_rxtx_vec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <assert.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <stdlib.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #include <infiniband/mlx5dv.h>
  18 #ifdef PEDANTIC
  19 #pragma GCC diagnostic error "-Wpedantic"
  20 #endif
  21
  22 #include <rte_mbuf.h>
  23 #include <rte_mempool.h>
  24 #include <rte_prefetch.h>
  25
  26 #include <mlx5_prm.h>
  27
  28 #include "mlx5_defs.h"
  29 #include "mlx5.h"
  30 #include "mlx5_utils.h"
  31 #include "mlx5_rxtx.h"
  32 #include "mlx5_rxtx_vec.h"
  33 #include "mlx5_autoconf.h"
  34
  35 #if defined RTE_ARCH_X86_64
  36 #include "mlx5_rxtx_vec_sse.h"
  37 #elif defined RTE_ARCH_ARM64
  38 #include "mlx5_rxtx_vec_neon.h"
  39 #elif defined RTE_ARCH_PPC_64
  40 #include "mlx5_rxtx_vec_altivec.h"
  41 #else
  42 #error "This should not be compiled if SIMD instructions are not supported."
  43 #endif
  44
  45 /**
  46  * Skip error packets.
  47  *
  48  * @param rxq
  49  *   Pointer to RX queue structure.
  50  * @param[out] pkts
  51  *   Array to store received packets.
  52  * @param pkts_n
  53  *   Maximum number of packets in array.
  54  *
  55  * @return
  56  *   Number of packets successfully received (<= pkts_n).
  57  */
  58 static uint16_t
  59 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
  60                          uint16_t pkts_n)
  61 {
  62         uint16_t n = 0;
  63         unsigned int i;
  64 #ifdef MLX5_PMD_SOFT_COUNTERS
  65         uint32_t err_bytes = 0;
  66 #endif
  67
  68         for (i = 0; i < pkts_n; ++i) {
  69                 struct rte_mbuf *pkt = pkts[i];
  70
  71                 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) {
  72 #ifdef MLX5_PMD_SOFT_COUNTERS
  73                         err_bytes += PKT_LEN(pkt);
  74 #endif
  75                         rte_pktmbuf_free_seg(pkt);
  76                 } else {
  77                         pkts[n++] = pkt;
  78                 }
  79         }
  80         rxq->stats.idropped += (pkts_n - n);
  81 #ifdef MLX5_PMD_SOFT_COUNTERS
  82         /* Correct counters of errored completions. */
  83         rxq->stats.ipackets -= (pkts_n - n);
  84         rxq->stats.ibytes -= err_bytes;
  85 #endif
  86         mlx5_rx_err_handle(rxq, 1);
  87         return n;
  88 }
  89
  90 /**
  91  * DPDK callback for vectorized RX.
  92  *
  93  * @param dpdk_rxq
  94  *   Generic pointer to RX queue structure.
  95  * @param[out] pkts
  96  *   Array to store received packets.
  97  * @param pkts_n
  98  *   Maximum number of packets in array.
  99  *
 100  * @return
 101  *   Number of packets successfully received (<= pkts_n).
 102  */
 103 uint16_t
 104 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 105 {
 106         struct mlx5_rxq_data *rxq = dpdk_rxq;
 107         uint16_t nb_rx;
 108         uint64_t err = 0;
 109
 110         nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err);
 111         if (unlikely(err | rxq->err_state))
 112                 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx);
 113         return nb_rx;
 114 }
 115
 116 /**
 117  * Check a RX queue can support vectorized RX.
 118  *
 119  * @param rxq
 120  *   Pointer to RX queue.
 121  *
 122  * @return
 123  *   1 if supported, negative errno value if not.
 124  */
 125 int __attribute__((cold))
 126 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)
 127 {
 128         struct mlx5_rxq_ctrl *ctrl =
 129                 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 130
 131         if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv)))
 132                 return -ENOTSUP;
 133         if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0)
 134                 return -ENOTSUP;
 135         if (rxq->lro)
 136                 return -ENOTSUP;
 137         return 1;
 138 }
 139
 140 /**
 141  * Check a device can support vectorized RX.
 142  *
 143  * @param dev
 144  *   Pointer to Ethernet device.
 145  *
 146  * @return
 147  *   1 if supported, negative errno value if not.
 148  */
 149 int __attribute__((cold))
 150 mlx5_check_vec_rx_support(struct rte_eth_dev *dev)
 151 {
 152         struct mlx5_priv *priv = dev->data->dev_private;
 153         uint16_t i;
 154
 155         if (!priv->config.rx_vec_en)
 156                 return -ENOTSUP;
 157         if (mlx5_mprq_enabled(dev))
 158                 return -ENOTSUP;
 159         /* All the configured queues should support. */
 160         for (i = 0; i < priv->rxqs_n; ++i) {
 161                 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
 162
 163                 if (!rxq)
 164                         continue;
 165                 if (mlx5_rxq_check_vec_support(rxq) < 0)
 166                         break;
 167         }
 168         if (i != priv->rxqs_n)
 169                 return -ENOTSUP;
 170         return 1;
 171 }