X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fi40e%2Fi40e_rxtx.c;h=61cb204be2b5a266c36cc983c590119cd1f8c95e;hb=76fd789cc7dddbaa2c08065b7c3ca915b5c07e7c;hp=46e9276be1fc9a2d3276ae1c81b83bd41f562cc6;hpb=6ada10deac667cd2f79bd08d25dd068c04747fe5;p=dpdk.git diff --git a/drivers/net/i40e/i40e_rxtx.c b/drivers/net/i40e/i40e_rxtx.c index 46e9276be1..61cb204be2 100644 --- a/drivers/net/i40e/i40e_rxtx.c +++ b/drivers/net/i40e/i40e_rxtx.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include @@ -92,7 +92,7 @@ i40e_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc) pmc->mask = rte_cpu_to_le_64(1 << I40E_RX_DESC_STATUS_DD_SHIFT); /* registers are 64-bit */ - pmc->data_sz = sizeof(uint64_t); + pmc->size = sizeof(uint64_t); return 0; } @@ -169,7 +169,7 @@ i40e_rxd_error_to_pkt_flags(uint64_t qword) flags |= PKT_RX_L4_CKSUM_GOOD; if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))) - flags |= PKT_RX_EIP_CKSUM_BAD; + flags |= PKT_RX_OUTER_IP_CKSUM_BAD; return flags; } @@ -1767,6 +1767,10 @@ i40e_dev_supported_ptypes_get(struct rte_eth_dev *dev) dev->rx_pkt_burst == i40e_recv_scattered_pkts || dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec || dev->rx_pkt_burst == i40e_recv_pkts_vec || +#ifdef CC_AVX512_SUPPORT + dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx512 || + dev->rx_pkt_burst == i40e_recv_pkts_vec_avx512 || +#endif dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx2 || dev->rx_pkt_burst == i40e_recv_pkts_vec_avx2) return ptypes; @@ -2529,6 +2533,25 @@ i40e_tx_queue_release_mbufs(struct i40e_tx_queue *txq) * vPMD tx will not set sw_ring's mbuf to NULL after free, * so need to free remains more carefully. */ +#ifdef CC_AVX512_SUPPORT + if (dev->tx_pkt_burst == i40e_xmit_pkts_vec_avx512) { + struct i40e_vec_tx_entry *swr = (void *)txq->sw_ring; + + i = txq->tx_next_dd - txq->tx_rs_thresh + 1; + if (txq->tx_tail < i) { + for (; i < txq->nb_tx_desc; i++) { + rte_pktmbuf_free_seg(swr[i].mbuf); + swr[i].mbuf = NULL; + } + i = 0; + } + for (; i < txq->tx_tail; i++) { + rte_pktmbuf_free_seg(swr[i].mbuf); + swr[i].mbuf = NULL; + } + return; + } +#endif if (dev->tx_pkt_burst == i40e_xmit_pkts_vec_avx2 || dev->tx_pkt_burst == i40e_xmit_pkts_vec) { i = txq->tx_next_dd - txq->tx_rs_thresh + 1; @@ -2822,23 +2845,23 @@ i40e_rx_queue_config(struct i40e_rx_queue *rxq) RTE_MIN((uint32_t)(hw->func_caps.rx_buf_chain_len * rxq->rx_buf_len), data->dev_conf.rxmode.max_rx_pkt_len); if (data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { - if (rxq->max_pkt_len <= RTE_ETHER_MAX_LEN || + if (rxq->max_pkt_len <= I40E_ETH_MAX_LEN || rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) { PMD_DRV_LOG(ERR, "maximum packet length must " "be larger than %u and smaller than %u," "as jumbo frame is enabled", - (uint32_t)RTE_ETHER_MAX_LEN, + (uint32_t)I40E_ETH_MAX_LEN, (uint32_t)I40E_FRAME_SIZE_MAX); return I40E_ERR_CONFIG; } } else { if (rxq->max_pkt_len < RTE_ETHER_MIN_LEN || - rxq->max_pkt_len > RTE_ETHER_MAX_LEN) { + rxq->max_pkt_len > I40E_ETH_MAX_LEN) { PMD_DRV_LOG(ERR, "maximum packet length must be " "larger than %u and smaller than %u, " "as jumbo frame is disabled", (uint32_t)RTE_ETHER_MIN_LEN, - (uint32_t)RTE_ETHER_MAX_LEN); + (uint32_t)I40E_ETH_MAX_LEN); return I40E_ERR_CONFIG; } } @@ -3162,6 +3185,7 @@ i40e_set_rx_function(struct rte_eth_dev *dev) I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); uint16_t rx_using_sse, i; bool use_avx2 = false; + bool use_avx512 = false; /* In order to allow Vector Rx there are a few configuration * conditions to be met and Rx Bulk Allocation should be allowed. */ @@ -3185,28 +3209,51 @@ i40e_set_rx_function(struct rte_eth_dev *dev) } } - use_avx2 = get_avx_supported(0); + use_avx512 = get_avx_supported(1); + + if (!use_avx512) + use_avx2 = get_avx_supported(0); } } if (ad->rx_vec_allowed && rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { if (dev->data->scattered_rx) { - PMD_INIT_LOG(DEBUG, - "Using %sVector Scattered Rx (port %d).", - use_avx2 ? "avx2 " : "", - dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - i40e_recv_scattered_pkts_vec_avx2 : - i40e_recv_scattered_pkts_vec; + if (use_avx512) { +#ifdef CC_AVX512_SUPPORT + PMD_DRV_LOG(NOTICE, + "Using AVX512 Vector Scattered Rx (port %d).", + dev->data->port_id); + dev->rx_pkt_burst = + i40e_recv_scattered_pkts_vec_avx512; +#endif + } else { + PMD_INIT_LOG(DEBUG, + "Using %sVector Scattered Rx (port %d).", + use_avx2 ? "avx2 " : "", + dev->data->port_id); + dev->rx_pkt_burst = use_avx2 ? + i40e_recv_scattered_pkts_vec_avx2 : + i40e_recv_scattered_pkts_vec; + } } else { - PMD_INIT_LOG(DEBUG, - "Using %sVector Rx (port %d).", - use_avx2 ? "avx2 " : "", - dev->data->port_id); - dev->rx_pkt_burst = use_avx2 ? - i40e_recv_pkts_vec_avx2 : - i40e_recv_pkts_vec; + if (use_avx512) { +#ifdef CC_AVX512_SUPPORT + PMD_DRV_LOG(NOTICE, + "Using AVX512 Vector Rx (port %d).", + dev->data->port_id); + dev->rx_pkt_burst = + i40e_recv_pkts_vec_avx512; +#endif + } else { + PMD_INIT_LOG(DEBUG, + "Using %sVector Rx (port %d).", + use_avx2 ? "avx2 " : "", + dev->data->port_id); + dev->rx_pkt_burst = use_avx2 ? + i40e_recv_pkts_vec_avx2 : + i40e_recv_pkts_vec; + } } } else if (!dev->data->scattered_rx && ad->rx_bulk_alloc_allowed) { PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " @@ -3229,6 +3276,10 @@ i40e_set_rx_function(struct rte_eth_dev *dev) rx_using_sse = (dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec || dev->rx_pkt_burst == i40e_recv_pkts_vec || +#ifdef CC_AVX512_SUPPORT + dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx512 || + dev->rx_pkt_burst == i40e_recv_pkts_vec_avx512 || +#endif dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx2 || dev->rx_pkt_burst == i40e_recv_pkts_vec_avx2); @@ -3249,6 +3300,10 @@ static const struct { { i40e_recv_pkts_bulk_alloc, "Scalar Bulk Alloc" }, { i40e_recv_pkts, "Scalar" }, #ifdef RTE_ARCH_X86 +#ifdef CC_AVX512_SUPPORT + { i40e_recv_scattered_pkts_vec_avx512, "Vector AVX512 Scattered" }, + { i40e_recv_pkts_vec_avx512, "Vector AVX512" }, +#endif { i40e_recv_scattered_pkts_vec_avx2, "Vector AVX2 Scattered" }, { i40e_recv_pkts_vec_avx2, "Vector AVX2" }, { i40e_recv_scattered_pkts_vec, "Vector SSE Scattered" }, @@ -3315,6 +3370,7 @@ i40e_set_tx_function(struct rte_eth_dev *dev) I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); int i; bool use_avx2 = false; + bool use_avx512 = false; if (rte_eal_process_type() == RTE_PROC_PRIMARY) { if (ad->tx_vec_allowed) { @@ -3328,19 +3384,30 @@ i40e_set_tx_function(struct rte_eth_dev *dev) } } - use_avx2 = get_avx_supported(0); + use_avx512 = get_avx_supported(1); + + if (!use_avx512) + use_avx2 = get_avx_supported(0); } } if (ad->tx_simple_allowed) { if (ad->tx_vec_allowed && rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { - PMD_INIT_LOG(DEBUG, "Using %sVector Tx (port %d).", - use_avx2 ? "avx2 " : "", - dev->data->port_id); - dev->tx_pkt_burst = use_avx2 ? - i40e_xmit_pkts_vec_avx2 : - i40e_xmit_pkts_vec; + if (use_avx512) { +#ifdef CC_AVX512_SUPPORT + PMD_DRV_LOG(NOTICE, "Using AVX512 Vector Tx (port %d).", + dev->data->port_id); + dev->tx_pkt_burst = i40e_xmit_pkts_vec_avx512; +#endif + } else { + PMD_INIT_LOG(DEBUG, "Using %sVector Tx (port %d).", + use_avx2 ? "avx2 " : "", + dev->data->port_id); + dev->tx_pkt_burst = use_avx2 ? + i40e_xmit_pkts_vec_avx2 : + i40e_xmit_pkts_vec; + } } else { PMD_INIT_LOG(DEBUG, "Simple tx finally be used."); dev->tx_pkt_burst = i40e_xmit_pkts_simple; @@ -3360,6 +3427,9 @@ static const struct { { i40e_xmit_pkts_simple, "Scalar Simple" }, { i40e_xmit_pkts, "Scalar" }, #ifdef RTE_ARCH_X86 +#ifdef CC_AVX512_SUPPORT + { i40e_xmit_pkts_vec_avx512, "Vector AVX512" }, +#endif { i40e_xmit_pkts_vec_avx2, "Vector AVX2" }, { i40e_xmit_pkts_vec, "Vector SSE" }, #elif defined(RTE_ARCH_ARM64) @@ -3459,58 +3529,6 @@ i40e_set_default_pctype_table(struct rte_eth_dev *dev) } } -#ifndef RTE_LIBRTE_I40E_INC_VECTOR -int -i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev) -{ - return -1; -} - -uint16_t -i40e_recv_pkts_vec( - void __rte_unused *rx_queue, - struct rte_mbuf __rte_unused **rx_pkts, - uint16_t __rte_unused nb_pkts) -{ - return 0; -} - -uint16_t -i40e_recv_scattered_pkts_vec( - void __rte_unused *rx_queue, - struct rte_mbuf __rte_unused **rx_pkts, - uint16_t __rte_unused nb_pkts) -{ - return 0; -} - -int -i40e_rxq_vec_setup(struct i40e_rx_queue __rte_unused *rxq) -{ - return -1; -} - -int -i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq) -{ - return -1; -} - -void -i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue __rte_unused*rxq) -{ - return; -} - -uint16_t -i40e_xmit_fixed_burst_vec(void __rte_unused * tx_queue, - struct rte_mbuf __rte_unused **tx_pkts, - uint16_t __rte_unused nb_pkts) -{ - return 0; -} -#endif /* ifndef RTE_LIBRTE_I40E_INC_VECTOR */ - #ifndef CC_AVX2_SUPPORT uint16_t i40e_recv_pkts_vec_avx2(void __rte_unused *rx_queue,