X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fi40e%2Fi40e_rxtx.c;h=61cb204be2b5a266c36cc983c590119cd1f8c95e;hb=31d7c6f7d424c533b0a4dd9b4408b814ac7852f1;hp=840b6f387fb017fd3d8eb7e96eae281f6e8a0af3;hpb=6cc330b709b0d0b72880d872edf7d13b95649566;p=dpdk.git diff --git a/drivers/net/i40e/i40e_rxtx.c b/drivers/net/i40e/i40e_rxtx.c index 840b6f387f..61cb204be2 100644 --- a/drivers/net/i40e/i40e_rxtx.c +++ b/drivers/net/i40e/i40e_rxtx.c @@ -17,12 +17,13 @@ #include #include #include -#include +#include #include #include #include #include #include +#include #include "i40e_logs.h" #include "base/i40e_prototype.h" @@ -71,6 +72,31 @@ #define I40E_TX_OFFLOAD_NOTSUP_MASK \ (PKT_TX_OFFLOAD_MASK ^ I40E_TX_OFFLOAD_MASK) +int +i40e_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc) +{ + struct i40e_rx_queue *rxq = rx_queue; + volatile union i40e_rx_desc *rxdp; + uint16_t desc; + + desc = rxq->rx_tail; + rxdp = &rxq->rx_ring[desc]; + /* watch for changes in status bit */ + pmc->addr = &rxdp->wb.qword1.status_error_len; + + /* + * we expect the DD bit to be set to 1 if this descriptor was already + * written to. + */ + pmc->val = rte_cpu_to_le_64(1 << I40E_RX_DESC_STATUS_DD_SHIFT); + pmc->mask = rte_cpu_to_le_64(1 << I40E_RX_DESC_STATUS_DD_SHIFT); + + /* registers are 64-bit */ + pmc->size = sizeof(uint64_t); + + return 0; +} + static inline void i40e_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union i40e_rx_desc *rxdp) { @@ -143,7 +169,7 @@ i40e_rxd_error_to_pkt_flags(uint64_t qword) flags |= PKT_RX_L4_CKSUM_GOOD; if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))) - flags |= PKT_RX_EIP_CKSUM_BAD; + flags |= PKT_RX_OUTER_IP_CKSUM_BAD; return flags; } @@ -760,7 +786,7 @@ i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) if (nb_hold > rxq->rx_free_thresh) { rx_id = (uint16_t) ((rx_id == 0) ? (rxq->nb_rx_desc - 1) : (rx_id - 1)); - I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + I40E_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); nb_hold = 0; } rxq->nb_rx_hold = nb_hold; @@ -938,7 +964,7 @@ i40e_recv_scattered_pkts(void *rx_queue, if (nb_hold > rxq->rx_free_thresh) { rx_id = (uint16_t)(rx_id == 0 ? (rxq->nb_rx_desc - 1) : (rx_id - 1)); - I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + I40E_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); nb_hold = 0; } rxq->nb_rx_hold = nb_hold; @@ -1248,8 +1274,8 @@ end_of_tx: (unsigned) txq->port_id, (unsigned) txq->queue_id, (unsigned) tx_id, (unsigned) nb_tx); - rte_cio_wmb(); - I40E_PCI_REG_WRITE_RELAXED(txq->qtx_tail, tx_id); + rte_io_wmb(); + I40E_PCI_REG_WC_WRITE_RELAXED(txq->qtx_tail, tx_id); txq->tx_tail = tx_id; return nb_tx; @@ -1400,7 +1426,7 @@ tx_xmit_pkts(struct i40e_tx_queue *txq, txq->tx_tail = 0; /* Update the tx tail register */ - I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + I40E_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail); return nb_pkts; } @@ -1741,6 +1767,10 @@ i40e_dev_supported_ptypes_get(struct rte_eth_dev *dev) dev->rx_pkt_burst == i40e_recv_scattered_pkts || dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec || dev->rx_pkt_burst == i40e_recv_pkts_vec || +#ifdef CC_AVX512_SUPPORT + dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx512 || + dev->rx_pkt_burst == i40e_recv_pkts_vec_avx512 || +#endif dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx2 || dev->rx_pkt_burst == i40e_recv_pkts_vec_avx2) return ptypes; @@ -2503,6 +2533,25 @@ i40e_tx_queue_release_mbufs(struct i40e_tx_queue *txq) * vPMD tx will not set sw_ring's mbuf to NULL after free, * so need to free remains more carefully. */ +#ifdef CC_AVX512_SUPPORT + if (dev->tx_pkt_burst == i40e_xmit_pkts_vec_avx512) { + struct i40e_vec_tx_entry *swr = (void *)txq->sw_ring; + + i = txq->tx_next_dd - txq->tx_rs_thresh + 1; + if (txq->tx_tail < i) { + for (; i < txq->nb_tx_desc; i++) { + rte_pktmbuf_free_seg(swr[i].mbuf); + swr[i].mbuf = NULL; + } + i = 0; + } + for (; i < txq->tx_tail; i++) { + rte_pktmbuf_free_seg(swr[i].mbuf); + swr[i].mbuf = NULL; + } + return; + } +#endif if (dev->tx_pkt_burst == i40e_xmit_pkts_vec_avx2 || dev->tx_pkt_burst == i40e_xmit_pkts_vec) { i = txq->tx_next_dd - txq->tx_rs_thresh + 1; @@ -2796,23 +2845,23 @@ i40e_rx_queue_config(struct i40e_rx_queue *rxq) RTE_MIN((uint32_t)(hw->func_caps.rx_buf_chain_len * rxq->rx_buf_len), data->dev_conf.rxmode.max_rx_pkt_len); if (data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { - if (rxq->max_pkt_len <= RTE_ETHER_MAX_LEN || + if (rxq->max_pkt_len <= I40E_ETH_MAX_LEN || rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) { PMD_DRV_LOG(ERR, "maximum packet length must " "be larger than %u and smaller than %u," "as jumbo frame is enabled", - (uint32_t)RTE_ETHER_MAX_LEN, + (uint32_t)I40E_ETH_MAX_LEN, (uint32_t)I40E_FRAME_SIZE_MAX); return I40E_ERR_CONFIG; } } else { if (rxq->max_pkt_len < RTE_ETHER_MIN_LEN || - rxq->max_pkt_len > RTE_ETHER_MAX_LEN) { + rxq->max_pkt_len > I40E_ETH_MAX_LEN) { PMD_DRV_LOG(ERR, "maximum packet length must be " "larger than %u and smaller than %u, " "as jumbo frame is disabled", (uint32_t)RTE_ETHER_MIN_LEN, - (uint32_t)RTE_ETHER_MAX_LEN); + (uint32_t)I40E_ETH_MAX_LEN); return I40E_ERR_CONFIG; } } @@ -2928,6 +2977,7 @@ i40e_dev_free_queues(struct rte_eth_dev *dev) continue; i40e_dev_rx_queue_release(dev->data->rx_queues[i]); dev->data->rx_queues[i] = NULL; + rte_eth_dma_zone_free(dev, "rx_ring", i); } for (i = 0; i < dev->data->nb_tx_queues; i++) { @@ -2935,19 +2985,17 @@ i40e_dev_free_queues(struct rte_eth_dev *dev) continue; i40e_dev_tx_queue_release(dev->data->tx_queues[i]); dev->data->tx_queues[i] = NULL; + rte_eth_dma_zone_free(dev, "tx_ring", i); } } -#define I40E_FDIR_NUM_TX_DESC I40E_MIN_RING_DESC -#define I40E_FDIR_NUM_RX_DESC I40E_MIN_RING_DESC - enum i40e_status_code i40e_fdir_setup_tx_resources(struct i40e_pf *pf) { struct i40e_tx_queue *txq; const struct rte_memzone *tz = NULL; - uint32_t ring_size; struct rte_eth_dev *dev; + uint32_t ring_size; if (!pf) { PMD_DRV_LOG(ERR, "PF is not available"); @@ -2987,12 +3035,14 @@ i40e_fdir_setup_tx_resources(struct i40e_pf *pf) txq->tx_ring_phys_addr = tz->iova; txq->tx_ring = (struct i40e_tx_desc *)tz->addr; + /* * don't need to allocate software ring and reset for the fdir * program queue just set the queue has been configured. */ txq->q_set = TRUE; pf->fdir.txq = txq; + pf->fdir.txq_available_buf_count = I40E_FDIR_PRG_PKT_CNT; return I40E_SUCCESS; } @@ -3093,41 +3143,49 @@ i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->conf.offloads = txq->offloads; } -static eth_rx_burst_t -i40e_get_latest_rx_vec(bool scatter) +static inline bool +get_avx_supported(bool request_avx512) { -#if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT) - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) - return scatter ? i40e_recv_scattered_pkts_vec_avx2 : - i40e_recv_pkts_vec_avx2; +#ifdef RTE_ARCH_X86 + if (request_avx512) { + if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512 && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1) +#ifdef CC_AVX512_SUPPORT + return true; +#else + PMD_DRV_LOG(NOTICE, + "AVX512 is not supported in build env"); + return false; #endif - return scatter ? i40e_recv_scattered_pkts_vec : - i40e_recv_pkts_vec; -} - -static eth_rx_burst_t -i40e_get_recommend_rx_vec(bool scatter) -{ -#if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT) - /* - * since AVX frequency can be different to base frequency, limit - * use of AVX2 version to later plaforms, not all those that could - * theoretically run it. - */ - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) - return scatter ? i40e_recv_scattered_pkts_vec_avx2 : - i40e_recv_pkts_vec_avx2; + } else { + if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256 && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) +#ifdef CC_AVX2_SUPPORT + return true; +#else + PMD_DRV_LOG(NOTICE, + "AVX2 is not supported in build env"); + return false; #endif - return scatter ? i40e_recv_scattered_pkts_vec : - i40e_recv_pkts_vec; + } +#else + RTE_SET_USED(request_avx512); +#endif /* RTE_ARCH_X86 */ + + return false; } + void __rte_cold i40e_set_rx_function(struct rte_eth_dev *dev) { struct i40e_adapter *ad = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); uint16_t rx_using_sse, i; + bool use_avx2 = false; + bool use_avx512 = false; /* In order to allow Vector Rx there are a few configuration * conditions to be met and Rx Bulk Allocation should be allowed. */ @@ -3150,19 +3208,53 @@ i40e_set_rx_function(struct rte_eth_dev *dev) break; } } + + use_avx512 = get_avx_supported(1); + + if (!use_avx512) + use_avx2 = get_avx_supported(0); } } - if (ad->rx_vec_allowed) { - /* Vec Rx path */ - PMD_INIT_LOG(DEBUG, "Vector Rx path will be used on port=%d.", - dev->data->port_id); - if (ad->use_latest_vec) - dev->rx_pkt_burst = - i40e_get_latest_rx_vec(dev->data->scattered_rx); - else - dev->rx_pkt_burst = - i40e_get_recommend_rx_vec(dev->data->scattered_rx); + if (ad->rx_vec_allowed && + rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { + if (dev->data->scattered_rx) { + if (use_avx512) { +#ifdef CC_AVX512_SUPPORT + PMD_DRV_LOG(NOTICE, + "Using AVX512 Vector Scattered Rx (port %d).", + dev->data->port_id); + dev->rx_pkt_burst = + i40e_recv_scattered_pkts_vec_avx512; +#endif + } else { + PMD_INIT_LOG(DEBUG, + "Using %sVector Scattered Rx (port %d).", + use_avx2 ? "avx2 " : "", + dev->data->port_id); + dev->rx_pkt_burst = use_avx2 ? + i40e_recv_scattered_pkts_vec_avx2 : + i40e_recv_scattered_pkts_vec; + } + } else { + if (use_avx512) { +#ifdef CC_AVX512_SUPPORT + PMD_DRV_LOG(NOTICE, + "Using AVX512 Vector Rx (port %d).", + dev->data->port_id); + dev->rx_pkt_burst = + i40e_recv_pkts_vec_avx512; +#endif + } else { + PMD_INIT_LOG(DEBUG, + "Using %sVector Rx (port %d).", + use_avx2 ? "avx2 " : "", + dev->data->port_id); + dev->rx_pkt_burst = use_avx2 ? + i40e_recv_pkts_vec_avx2 : + i40e_recv_pkts_vec; + } + } } else if (!dev->data->scattered_rx && ad->rx_bulk_alloc_allowed) { PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " "satisfied. Rx Burst Bulk Alloc function " @@ -3184,6 +3276,10 @@ i40e_set_rx_function(struct rte_eth_dev *dev) rx_using_sse = (dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec || dev->rx_pkt_burst == i40e_recv_pkts_vec || +#ifdef CC_AVX512_SUPPORT + dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx512 || + dev->rx_pkt_burst == i40e_recv_pkts_vec_avx512 || +#endif dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx2 || dev->rx_pkt_burst == i40e_recv_pkts_vec_avx2); @@ -3204,6 +3300,10 @@ static const struct { { i40e_recv_pkts_bulk_alloc, "Scalar Bulk Alloc" }, { i40e_recv_pkts, "Scalar" }, #ifdef RTE_ARCH_X86 +#ifdef CC_AVX512_SUPPORT + { i40e_recv_scattered_pkts_vec_avx512, "Vector AVX512 Scattered" }, + { i40e_recv_pkts_vec_avx512, "Vector AVX512" }, +#endif { i40e_recv_scattered_pkts_vec_avx2, "Vector AVX2 Scattered" }, { i40e_recv_pkts_vec_avx2, "Vector AVX2" }, { i40e_recv_scattered_pkts_vec, "Vector SSE Scattered" }, @@ -3263,37 +3363,14 @@ i40e_set_tx_function_flag(struct rte_eth_dev *dev, struct i40e_tx_queue *txq) txq->queue_id); } -static eth_tx_burst_t -i40e_get_latest_tx_vec(void) -{ -#if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT) - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) - return i40e_xmit_pkts_vec_avx2; -#endif - return i40e_xmit_pkts_vec; -} - -static eth_tx_burst_t -i40e_get_recommend_tx_vec(void) -{ -#if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT) - /* - * since AVX frequency can be different to base frequency, limit - * use of AVX2 version to later plaforms, not all those that could - * theoretically run it. - */ - if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) - return i40e_xmit_pkts_vec_avx2; -#endif - return i40e_xmit_pkts_vec; -} - void __rte_cold i40e_set_tx_function(struct rte_eth_dev *dev) { struct i40e_adapter *ad = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); int i; + bool use_avx2 = false; + bool use_avx512 = false; if (rte_eal_process_type() == RTE_PROC_PRIMARY) { if (ad->tx_vec_allowed) { @@ -3306,18 +3383,31 @@ i40e_set_tx_function(struct rte_eth_dev *dev) break; } } + + use_avx512 = get_avx_supported(1); + + if (!use_avx512) + use_avx2 = get_avx_supported(0); } } if (ad->tx_simple_allowed) { - if (ad->tx_vec_allowed) { - PMD_INIT_LOG(DEBUG, "Vector tx finally be used."); - if (ad->use_latest_vec) - dev->tx_pkt_burst = - i40e_get_latest_tx_vec(); - else - dev->tx_pkt_burst = - i40e_get_recommend_tx_vec(); + if (ad->tx_vec_allowed && + rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { + if (use_avx512) { +#ifdef CC_AVX512_SUPPORT + PMD_DRV_LOG(NOTICE, "Using AVX512 Vector Tx (port %d).", + dev->data->port_id); + dev->tx_pkt_burst = i40e_xmit_pkts_vec_avx512; +#endif + } else { + PMD_INIT_LOG(DEBUG, "Using %sVector Tx (port %d).", + use_avx2 ? "avx2 " : "", + dev->data->port_id); + dev->tx_pkt_burst = use_avx2 ? + i40e_xmit_pkts_vec_avx2 : + i40e_xmit_pkts_vec; + } } else { PMD_INIT_LOG(DEBUG, "Simple tx finally be used."); dev->tx_pkt_burst = i40e_xmit_pkts_simple; @@ -3337,6 +3427,9 @@ static const struct { { i40e_xmit_pkts_simple, "Scalar Simple" }, { i40e_xmit_pkts, "Scalar" }, #ifdef RTE_ARCH_X86 +#ifdef CC_AVX512_SUPPORT + { i40e_xmit_pkts_vec_avx512, "Vector AVX512" }, +#endif { i40e_xmit_pkts_vec_avx2, "Vector AVX2" }, { i40e_xmit_pkts_vec, "Vector SSE" }, #elif defined(RTE_ARCH_ARM64) @@ -3436,59 +3529,6 @@ i40e_set_default_pctype_table(struct rte_eth_dev *dev) } } -#ifndef RTE_LIBRTE_I40E_INC_VECTOR -/* Stubs needed for linkage when CONFIG_RTE_LIBRTE_I40E_INC_VECTOR is set to 'n' */ -int -i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev) -{ - return -1; -} - -uint16_t -i40e_recv_pkts_vec( - void __rte_unused *rx_queue, - struct rte_mbuf __rte_unused **rx_pkts, - uint16_t __rte_unused nb_pkts) -{ - return 0; -} - -uint16_t -i40e_recv_scattered_pkts_vec( - void __rte_unused *rx_queue, - struct rte_mbuf __rte_unused **rx_pkts, - uint16_t __rte_unused nb_pkts) -{ - return 0; -} - -int -i40e_rxq_vec_setup(struct i40e_rx_queue __rte_unused *rxq) -{ - return -1; -} - -int -i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq) -{ - return -1; -} - -void -i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue __rte_unused*rxq) -{ - return; -} - -uint16_t -i40e_xmit_fixed_burst_vec(void __rte_unused * tx_queue, - struct rte_mbuf __rte_unused **tx_pkts, - uint16_t __rte_unused nb_pkts) -{ - return 0; -} -#endif /* ifndef RTE_LIBRTE_I40E_INC_VECTOR */ - #ifndef CC_AVX2_SUPPORT uint16_t i40e_recv_pkts_vec_avx2(void __rte_unused *rx_queue,