drivers/net/mlx5/mlx5_rxtx_vec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <assert.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <stdlib.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #include <infiniband/mlx5dv.h>
  18 #ifdef PEDANTIC
  19 #pragma GCC diagnostic error "-Wpedantic"
  20 #endif
  21
  22 #include <rte_mbuf.h>
  23 #include <rte_mempool.h>
  24 #include <rte_prefetch.h>
  25
  26 #include "mlx5.h"
  27 #include "mlx5_utils.h"
  28 #include "mlx5_rxtx.h"
  29 #include "mlx5_rxtx_vec.h"
  30 #include "mlx5_autoconf.h"
  31 #include "mlx5_defs.h"
  32 #include "mlx5_prm.h"
  33
  34 #if defined RTE_ARCH_X86_64
  35 #include "mlx5_rxtx_vec_sse.h"
  36 #elif defined RTE_ARCH_ARM64
  37 #include "mlx5_rxtx_vec_neon.h"
  38 #else
  39 #error "This should not be compiled if SIMD instructions are not supported."
  40 #endif
  41
  42 /**
  43  * Skip error packets.
  44  *
  45  * @param rxq
  46  *   Pointer to RX queue structure.
  47  * @param[out] pkts
  48  *   Array to store received packets.
  49  * @param pkts_n
  50  *   Maximum number of packets in array.
  51  *
  52  * @return
  53  *   Number of packets successfully received (<= pkts_n).
  54  */
  55 static uint16_t
  56 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
  57                          uint16_t pkts_n)
  58 {
  59         uint16_t n = 0;
  60         unsigned int i;
  61 #ifdef MLX5_PMD_SOFT_COUNTERS
  62         uint32_t err_bytes = 0;
  63 #endif
  64
  65         for (i = 0; i < pkts_n; ++i) {
  66                 struct rte_mbuf *pkt = pkts[i];
  67
  68                 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) {
  69 #ifdef MLX5_PMD_SOFT_COUNTERS
  70                         err_bytes += PKT_LEN(pkt);
  71 #endif
  72                         rte_pktmbuf_free_seg(pkt);
  73                 } else {
  74                         pkts[n++] = pkt;
  75                 }
  76         }
  77         rxq->stats.idropped += (pkts_n - n);
  78 #ifdef MLX5_PMD_SOFT_COUNTERS
  79         /* Correct counters of errored completions. */
  80         rxq->stats.ipackets -= (pkts_n - n);
  81         rxq->stats.ibytes -= err_bytes;
  82 #endif
  83         mlx5_rx_err_handle(rxq, 1);
  84         return n;
  85 }
  86
  87 /**
  88  * DPDK callback for vectorized RX.
  89  *
  90  * @param dpdk_rxq
  91  *   Generic pointer to RX queue structure.
  92  * @param[out] pkts
  93  *   Array to store received packets.
  94  * @param pkts_n
  95  *   Maximum number of packets in array.
  96  *
  97  * @return
  98  *   Number of packets successfully received (<= pkts_n).
  99  */
 100 uint16_t
 101 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 102 {
 103         struct mlx5_rxq_data *rxq = dpdk_rxq;
 104         uint16_t nb_rx;
 105         uint64_t err = 0;
 106
 107         nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err);
 108         if (unlikely(err | rxq->err_state))
 109                 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx);
 110         return nb_rx;
 111 }
 112
 113 /**
 114  * Check a RX queue can support vectorized RX.
 115  *
 116  * @param rxq
 117  *   Pointer to RX queue.
 118  *
 119  * @return
 120  *   1 if supported, negative errno value if not.
 121  */
 122 int __attribute__((cold))
 123 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)
 124 {
 125         struct mlx5_rxq_ctrl *ctrl =
 126                 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 127
 128         if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv)))
 129                 return -ENOTSUP;
 130         if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0)
 131                 return -ENOTSUP;
 132         return 1;
 133 }
 134
 135 /**
 136  * Check a device can support vectorized RX.
 137  *
 138  * @param dev
 139  *   Pointer to Ethernet device.
 140  *
 141  * @return
 142  *   1 if supported, negative errno value if not.
 143  */
 144 int __attribute__((cold))
 145 mlx5_check_vec_rx_support(struct rte_eth_dev *dev)
 146 {
 147         struct mlx5_priv *priv = dev->data->dev_private;
 148         uint16_t i;
 149
 150         if (!priv->config.rx_vec_en)
 151                 return -ENOTSUP;
 152         if (mlx5_mprq_enabled(dev))
 153                 return -ENOTSUP;
 154         /* All the configured queues should support. */
 155         for (i = 0; i < priv->rxqs_n; ++i) {
 156                 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
 157
 158                 if (!rxq)
 159                         continue;
 160                 if (mlx5_rxq_check_vec_support(rxq) < 0)
 161                         break;
 162         }
 163         if (i != priv->rxqs_n)
 164                 return -ENOTSUP;
 165         return 1;
 166 }