drivers/net/mlx5/mlx5_rxtx_vec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdint.h>
   7 #include <string.h>
   8 #include <stdlib.h>
   9
  10 #include <rte_mbuf.h>
  11 #include <rte_mempool.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_vect.h>
  14
  15 #include <mlx5_glue.h>
  16 #include <mlx5_prm.h>
  17
  18 #include "mlx5_defs.h"
  19 #include "mlx5.h"
  20 #include "mlx5_utils.h"
  21 #include "mlx5_rxtx.h"
  22 #include "mlx5_rxtx_vec.h"
  23 #include "mlx5_autoconf.h"
  24
  25 #if defined RTE_ARCH_X86_64
  26 #include "mlx5_rxtx_vec_sse.h"
  27 #elif defined RTE_ARCH_ARM64
  28 #include "mlx5_rxtx_vec_neon.h"
  29 #elif defined RTE_ARCH_PPC_64
  30 #include "mlx5_rxtx_vec_altivec.h"
  31 #else
  32 #error "This should not be compiled if SIMD instructions are not supported."
  33 #endif
  34
  35 /**
  36  * Skip error packets.
  37  *
  38  * @param rxq
  39  *   Pointer to RX queue structure.
  40  * @param[out] pkts
  41  *   Array to store received packets.
  42  * @param pkts_n
  43  *   Maximum number of packets in array.
  44  *
  45  * @return
  46  *   Number of packets successfully received (<= pkts_n).
  47  */
  48 static uint16_t
  49 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
  50                          uint16_t pkts_n)
  51 {
  52         uint16_t n = 0;
  53         unsigned int i;
  54 #ifdef MLX5_PMD_SOFT_COUNTERS
  55         uint32_t err_bytes = 0;
  56 #endif
  57
  58         for (i = 0; i < pkts_n; ++i) {
  59                 struct rte_mbuf *pkt = pkts[i];
  60
  61                 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) {
  62 #ifdef MLX5_PMD_SOFT_COUNTERS
  63                         err_bytes += PKT_LEN(pkt);
  64 #endif
  65                         rte_pktmbuf_free_seg(pkt);
  66                 } else {
  67                         pkts[n++] = pkt;
  68                 }
  69         }
  70         rxq->stats.idropped += (pkts_n - n);
  71 #ifdef MLX5_PMD_SOFT_COUNTERS
  72         /* Correct counters of errored completions. */
  73         rxq->stats.ipackets -= (pkts_n - n);
  74         rxq->stats.ibytes -= err_bytes;
  75 #endif
  76         mlx5_rx_err_handle(rxq, 1);
  77         return n;
  78 }
  79
  80 /**
  81  * DPDK callback for vectorized RX.
  82  *
  83  * @param dpdk_rxq
  84  *   Generic pointer to RX queue structure.
  85  * @param[out] pkts
  86  *   Array to store received packets.
  87  * @param pkts_n
  88  *   Maximum number of packets in array.
  89  *
  90  * @return
  91  *   Number of packets successfully received (<= pkts_n).
  92  */
  93 uint16_t
  94 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
  95 {
  96         struct mlx5_rxq_data *rxq = dpdk_rxq;
  97         uint16_t nb_rx = 0;
  98         uint16_t tn = 0;
  99         uint64_t err = 0;
 100         bool no_cq = false;
 101
 102         do {
 103                 nb_rx = rxq_burst_v(rxq, pkts + tn, pkts_n - tn, &err, &no_cq);
 104                 if (unlikely(err | rxq->err_state))
 105                         nb_rx = rxq_handle_pending_error(rxq, pkts + tn, nb_rx);
 106                 tn += nb_rx;
 107                 if (unlikely(no_cq))
 108                         break;
 109         } while (tn != pkts_n);
 110         return tn;
 111 }
 112
 113 /**
 114  * Check a RX queue can support vectorized RX.
 115  *
 116  * @param rxq
 117  *   Pointer to RX queue.
 118  *
 119  * @return
 120  *   1 if supported, negative errno value if not.
 121  */
 122 int __rte_cold
 123 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)
 124 {
 125         struct mlx5_rxq_ctrl *ctrl =
 126                 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 127
 128         if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv)))
 129                 return -ENOTSUP;
 130         if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0)
 131                 return -ENOTSUP;
 132         if (rxq->lro)
 133                 return -ENOTSUP;
 134         return 1;
 135 }
 136
 137 /**
 138  * Check a device can support vectorized RX.
 139  *
 140  * @param dev
 141  *   Pointer to Ethernet device.
 142  *
 143  * @return
 144  *   1 if supported, negative errno value if not.
 145  */
 146 int __rte_cold
 147 mlx5_check_vec_rx_support(struct rte_eth_dev *dev)
 148 {
 149         struct mlx5_priv *priv = dev->data->dev_private;
 150         uint32_t i;
 151
 152         if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128)
 153                 return -ENOTSUP;
 154         if (!priv->config.rx_vec_en)
 155                 return -ENOTSUP;
 156         if (mlx5_mprq_enabled(dev))
 157                 return -ENOTSUP;
 158         /* All the configured queues should support. */
 159         for (i = 0; i < priv->rxqs_n; ++i) {
 160                 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
 161
 162                 if (!rxq)
 163                         continue;
 164                 if (mlx5_rxq_check_vec_support(rxq) < 0)
 165                         break;
 166         }
 167         if (i != priv->rxqs_n)
 168                 return -ENOTSUP;
 169         return 1;
 170 }