X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_rxtx_vec.c;h=1453f4ff63a0c79cf1a8840e1cfef48493d4f5b4;hb=db48f9db5d9ffdfc851161e12f05ae099df44de0;hp=edc663815b41a765c6aab7013d7b77de02438d11;hpb=3c2ddbd413e34698b66df65a57b0c760ffe0b3d6;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c index edc663815b..1453f4ff63 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec.c +++ b/drivers/net/mlx5/mlx5_rxtx_vec.c @@ -1,34 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright 2017 6WIND S.A. - * Copyright 2017 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd */ #include @@ -59,86 +31,63 @@ #include "mlx5_defs.h" #include "mlx5_prm.h" -#ifdef RTE_ARCH_X86_64 +#if defined RTE_ARCH_X86_64 #include "mlx5_rxtx_vec_sse.h" +#elif defined RTE_ARCH_ARM64 +#include "mlx5_rxtx_vec_neon.h" #else #error "This should not be compiled if SIMD instructions are not supported." #endif /** - * Count the number of continuous single segment packets. + * Count the number of packets having same ol_flags and same metadata (if + * PKT_TX_METADATA is set in ol_flags), and calculate cs_flags. * * @param pkts * Pointer to array of packets. * @param pkts_n * Number of packets. - * - * @return - * Number of continuous single segment packets. - */ -static inline unsigned int -txq_check_multiseg(struct rte_mbuf **pkts, uint16_t pkts_n) -{ - unsigned int pos; - - if (!pkts_n) - return 0; - /* Count the number of continuous single segment packets. */ - for (pos = 0; pos < pkts_n; ++pos) - if (NB_SEGS(pkts[pos]) > 1) - break; - return pos; -} - -/** - * Count the number of packets having same ol_flags and calculate cs_flags. - * - * @param txq - * Pointer to TX queue structure. - * @param pkts - * Pointer to array of packets. - * @param pkts_n - * Number of packets. * @param cs_flags * Pointer of flags to be returned. + * @param metadata + * Pointer of metadata to be returned. + * @param txq_offloads + * Offloads enabled on Tx queue * * @return - * Number of packets having same ol_flags. + * Number of packets having same ol_flags and metadata, if relevant. */ static inline unsigned int -txq_calc_offload(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, - uint16_t pkts_n, uint8_t *cs_flags) +txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, uint8_t *cs_flags, + rte_be32_t *metadata, const uint64_t txq_offloads) { unsigned int pos; - const uint64_t ol_mask = + const uint64_t cksum_ol_mask = PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE | PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM; + rte_be32_t p0_metadata, pn_metadata; if (!pkts_n) return 0; - /* Count the number of packets having same ol_flags. */ - for (pos = 1; pos < pkts_n; ++pos) - if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask) + p0_metadata = pkts[0]->ol_flags & PKT_TX_METADATA ? + pkts[0]->tx_metadata : 0; + /* Count the number of packets having same offload parameters. */ + for (pos = 1; pos < pkts_n; ++pos) { + /* Check if packet has same checksum flags. */ + if ((txq_offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP) && + ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & cksum_ol_mask)) break; - /* Should open another MPW session for the rest. */ - if (pkts[0]->ol_flags & - (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { - const uint64_t is_tunneled = - pkts[0]->ol_flags & - (PKT_TX_TUNNEL_GRE | - PKT_TX_TUNNEL_VXLAN); - - if (is_tunneled && txq->tunnel_en) { - *cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM | - MLX5_ETH_WQE_L4_INNER_CSUM; - if (pkts[0]->ol_flags & PKT_TX_OUTER_IP_CKSUM) - *cs_flags |= MLX5_ETH_WQE_L3_CSUM; - } else { - *cs_flags = MLX5_ETH_WQE_L3_CSUM | - MLX5_ETH_WQE_L4_CSUM; + /* Check if packet has same metadata. */ + if (txq_offloads & DEV_TX_OFFLOAD_MATCH_METADATA) { + pn_metadata = pkts[pos]->ol_flags & PKT_TX_METADATA ? + pkts[pos]->tx_metadata : 0; + if (pn_metadata != p0_metadata) + break; } } + *cs_flags = txq_ol_cksum_to_cs(pkts[0]); + *metadata = p0_metadata; return pos; } @@ -167,7 +116,7 @@ mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t ret; n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); - ret = txq_burst_v(txq, &pkts[nb_tx], n, 0); + ret = txq_burst_v(txq, &pkts[nb_tx], n, 0, 0); nb_tx += ret; if (!ret) break; @@ -198,19 +147,23 @@ mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) uint8_t cs_flags = 0; uint16_t n; uint16_t ret; + rte_be32_t metadata = 0; /* Transmit multi-seg packets in the head of pkts list. */ - if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) && + if ((txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) && NB_SEGS(pkts[nb_tx]) > 1) nb_tx += txq_scatter_v(txq, &pkts[nb_tx], pkts_n - nb_tx); n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); - if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS)) - n = txq_check_multiseg(&pkts[nb_tx], n); - if (!(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS)) - n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags); - ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags); + if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) + n = txq_count_contig_single_seg(&pkts[nb_tx], n); + if (txq->offloads & (MLX5_VEC_TX_CKSUM_OFFLOAD_CAP | + DEV_TX_OFFLOAD_MATCH_METADATA)) + n = txq_calc_offload(&pkts[nb_tx], n, + &cs_flags, &metadata, + txq->offloads); + ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags, metadata); nb_tx += ret; if (!ret) break; @@ -259,7 +212,6 @@ rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, rxq->stats.ipackets -= (pkts_n - n); rxq->stats.ibytes -= err_bytes; #endif - rxq->pending_err = 0; return n; } @@ -281,9 +233,10 @@ mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) { struct mlx5_rxq_data *rxq = dpdk_rxq; uint16_t nb_rx; + uint64_t err = 0; - nb_rx = rxq_burst_v(rxq, pkts, pkts_n); - if (unlikely(rxq->pending_err)) + nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err); + if (unlikely(err)) nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx); return nb_rx; } @@ -291,26 +244,19 @@ mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) /** * Check Tx queue flags are set for raw vectorized Tx. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * 1 if supported, negative errno value if not. */ int __attribute__((cold)) -priv_check_raw_vec_tx_support(struct priv *priv) +mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev) { - uint16_t i; - - /* All the configured queues should support. */ - for (i = 0; i < priv->txqs_n; ++i) { - struct mlx5_txq_data *txq = (*priv->txqs)[i]; + uint64_t offloads = dev->data->dev_conf.txmode.offloads; - if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) || - !(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS)) - break; - } - if (i != priv->txqs_n) + /* Doesn't support any offload. */ + if (offloads) return -ENOTSUP; return 1; } @@ -318,19 +264,22 @@ priv_check_raw_vec_tx_support(struct priv *priv) /** * Check a device can support vectorized TX. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * 1 if supported, negative errno value if not. */ int __attribute__((cold)) -priv_check_vec_tx_support(struct priv *priv) +mlx5_check_vec_tx_support(struct rte_eth_dev *dev) { - if (!priv->tx_vec_en || + struct priv *priv = dev->data->dev_private; + uint64_t offloads = dev->data->dev_conf.txmode.offloads; + + if (!priv->config.tx_vec_en || priv->txqs_n > MLX5_VPMD_MIN_TXQS || - priv->mps != MLX5_MPW_ENHANCED || - priv->tso) + priv->config.mps != MLX5_MPW_ENHANCED || + offloads & ~MLX5_VEC_TX_OFFLOAD_CAP) return -ENOTSUP; return 1; } @@ -345,12 +294,14 @@ priv_check_vec_tx_support(struct priv *priv) * 1 if supported, negative errno value if not. */ int __attribute__((cold)) -rxq_check_vec_support(struct mlx5_rxq_data *rxq) +mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) { struct mlx5_rxq_ctrl *ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); - if (!ctrl->priv->rx_vec_en || rxq->sges_n != 0) + if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv))) + return -ENOTSUP; + if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0) return -ENOTSUP; return 1; } @@ -358,18 +309,21 @@ rxq_check_vec_support(struct mlx5_rxq_data *rxq) /** * Check a device can support vectorized RX. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * 1 if supported, negative errno value if not. */ int __attribute__((cold)) -priv_check_vec_rx_support(struct priv *priv) +mlx5_check_vec_rx_support(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; uint16_t i; - if (!priv->rx_vec_en) + if (!priv->config.rx_vec_en) + return -ENOTSUP; + if (mlx5_mprq_enabled(dev)) return -ENOTSUP; /* All the configured queues should support. */ for (i = 0; i < priv->rxqs_n; ++i) { @@ -377,7 +331,7 @@ priv_check_vec_rx_support(struct priv *priv) if (!rxq) continue; - if (rxq_check_vec_support(rxq) < 0) + if (mlx5_rxq_check_vec_support(rxq) < 0) break; } if (i != priv->rxqs_n)