#include <rte_udp.h>
#include <rte_ip.h>
#include <rte_net.h>
+#include <rte_vect.h>
#include "i40e_logs.h"
#include "base/i40e_prototype.h"
#define I40E_TX_OFFLOAD_NOTSUP_MASK \
(PKT_TX_OFFLOAD_MASK ^ I40E_TX_OFFLOAD_MASK)
+int
+i40e_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc)
+{
+ struct i40e_rx_queue *rxq = rx_queue;
+ volatile union i40e_rx_desc *rxdp;
+ uint16_t desc;
+
+ desc = rxq->rx_tail;
+ rxdp = &rxq->rx_ring[desc];
+ /* watch for changes in status bit */
+ pmc->addr = &rxdp->wb.qword1.status_error_len;
+
+ /*
+ * we expect the DD bit to be set to 1 if this descriptor was already
+ * written to.
+ */
+ pmc->val = rte_cpu_to_le_64(1 << I40E_RX_DESC_STATUS_DD_SHIFT);
+ pmc->mask = rte_cpu_to_le_64(1 << I40E_RX_DESC_STATUS_DD_SHIFT);
+
+ /* registers are 64-bit */
+ pmc->data_sz = sizeof(uint64_t);
+
+ return 0;
+}
+
static inline void
i40e_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union i40e_rx_desc *rxdp)
{
if (nb_hold > rxq->rx_free_thresh) {
rx_id = (uint16_t) ((rx_id == 0) ?
(rxq->nb_rx_desc - 1) : (rx_id - 1));
- I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
+ I40E_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
nb_hold = 0;
}
rxq->nb_rx_hold = nb_hold;
if (nb_hold > rxq->rx_free_thresh) {
rx_id = (uint16_t)(rx_id == 0 ?
(rxq->nb_rx_desc - 1) : (rx_id - 1));
- I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
+ I40E_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
nb_hold = 0;
}
rxq->nb_rx_hold = nb_hold;
(unsigned) txq->port_id, (unsigned) txq->queue_id,
(unsigned) tx_id, (unsigned) nb_tx);
- rte_cio_wmb();
- I40E_PCI_REG_WRITE_RELAXED(txq->qtx_tail, tx_id);
+ rte_io_wmb();
+ I40E_PCI_REG_WC_WRITE_RELAXED(txq->qtx_tail, tx_id);
txq->tx_tail = tx_id;
return nb_tx;
txq->tx_tail = 0;
/* Update the tx tail register */
- I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
+ I40E_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail);
return nb_pkts;
}
dev->rx_pkt_burst == i40e_recv_scattered_pkts ||
dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec ||
dev->rx_pkt_burst == i40e_recv_pkts_vec ||
+#ifdef CC_AVX512_SUPPORT
+ dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx512 ||
+ dev->rx_pkt_burst == i40e_recv_pkts_vec_avx512 ||
+#endif
dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx2 ||
dev->rx_pkt_burst == i40e_recv_pkts_vec_avx2)
return ptypes;
qinfo->conf.offloads = txq->offloads;
}
-static eth_rx_burst_t
-i40e_get_latest_rx_vec(bool scatter)
+static inline bool
+get_avx_supported(bool request_avx512)
{
-#if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
- return scatter ? i40e_recv_scattered_pkts_vec_avx2 :
- i40e_recv_pkts_vec_avx2;
+#ifdef RTE_ARCH_X86
+ if (request_avx512) {
+ if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512 &&
+ rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 &&
+ rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)
+#ifdef CC_AVX512_SUPPORT
+ return true;
+#else
+ PMD_DRV_LOG(NOTICE,
+ "AVX512 is not supported in build env");
+ return false;
#endif
- return scatter ? i40e_recv_scattered_pkts_vec :
- i40e_recv_pkts_vec;
-}
-
-static eth_rx_burst_t
-i40e_get_recommend_rx_vec(bool scatter)
-{
-#if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)
- /*
- * since AVX frequency can be different to base frequency, limit
- * use of AVX2 version to later plaforms, not all those that could
- * theoretically run it.
- */
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
- return scatter ? i40e_recv_scattered_pkts_vec_avx2 :
- i40e_recv_pkts_vec_avx2;
+ } else {
+ if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256 &&
+ rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 &&
+ rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1)
+#ifdef CC_AVX2_SUPPORT
+ return true;
+#else
+ PMD_DRV_LOG(NOTICE,
+ "AVX2 is not supported in build env");
+ return false;
#endif
- return scatter ? i40e_recv_scattered_pkts_vec :
- i40e_recv_pkts_vec;
+ }
+#else
+ RTE_SET_USED(request_avx512);
+#endif /* RTE_ARCH_X86 */
+
+ return false;
}
+
void __rte_cold
i40e_set_rx_function(struct rte_eth_dev *dev)
{
struct i40e_adapter *ad =
I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
uint16_t rx_using_sse, i;
+ bool use_avx2 = false;
+ bool use_avx512 = false;
/* In order to allow Vector Rx there are a few configuration
* conditions to be met and Rx Bulk Allocation should be allowed.
*/
break;
}
}
+
+ use_avx512 = get_avx_supported(1);
+
+ if (!use_avx512)
+ use_avx2 = get_avx_supported(0);
}
}
- if (ad->rx_vec_allowed) {
- /* Vec Rx path */
- PMD_INIT_LOG(DEBUG, "Vector Rx path will be used on port=%d.",
- dev->data->port_id);
- if (ad->use_latest_vec)
- dev->rx_pkt_burst =
- i40e_get_latest_rx_vec(dev->data->scattered_rx);
- else
- dev->rx_pkt_burst =
- i40e_get_recommend_rx_vec(dev->data->scattered_rx);
+ if (ad->rx_vec_allowed &&
+ rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
+ if (dev->data->scattered_rx) {
+ if (use_avx512) {
+#ifdef CC_AVX512_SUPPORT
+ PMD_DRV_LOG(NOTICE,
+ "Using AVX512 Vector Scattered Rx (port %d).",
+ dev->data->port_id);
+ dev->rx_pkt_burst =
+ i40e_recv_scattered_pkts_vec_avx512;
+#endif
+ } else {
+ PMD_INIT_LOG(DEBUG,
+ "Using %sVector Scattered Rx (port %d).",
+ use_avx2 ? "avx2 " : "",
+ dev->data->port_id);
+ dev->rx_pkt_burst = use_avx2 ?
+ i40e_recv_scattered_pkts_vec_avx2 :
+ i40e_recv_scattered_pkts_vec;
+ }
+ } else {
+ if (use_avx512) {
+#ifdef CC_AVX512_SUPPORT
+ PMD_DRV_LOG(NOTICE,
+ "Using AVX512 Vector Rx (port %d).",
+ dev->data->port_id);
+ dev->rx_pkt_burst =
+ i40e_recv_pkts_vec_avx512;
+#endif
+ } else {
+ PMD_INIT_LOG(DEBUG,
+ "Using %sVector Rx (port %d).",
+ use_avx2 ? "avx2 " : "",
+ dev->data->port_id);
+ dev->rx_pkt_burst = use_avx2 ?
+ i40e_recv_pkts_vec_avx2 :
+ i40e_recv_pkts_vec;
+ }
+ }
} else if (!dev->data->scattered_rx && ad->rx_bulk_alloc_allowed) {
PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
"satisfied. Rx Burst Bulk Alloc function "
rx_using_sse =
(dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec ||
dev->rx_pkt_burst == i40e_recv_pkts_vec ||
+#ifdef CC_AVX512_SUPPORT
+ dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx512 ||
+ dev->rx_pkt_burst == i40e_recv_pkts_vec_avx512 ||
+#endif
dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec_avx2 ||
dev->rx_pkt_burst == i40e_recv_pkts_vec_avx2);
{ i40e_recv_pkts_bulk_alloc, "Scalar Bulk Alloc" },
{ i40e_recv_pkts, "Scalar" },
#ifdef RTE_ARCH_X86
+#ifdef CC_AVX512_SUPPORT
+ { i40e_recv_scattered_pkts_vec_avx512, "Vector AVX512 Scattered" },
+ { i40e_recv_pkts_vec_avx512, "Vector AVX512" },
+#endif
{ i40e_recv_scattered_pkts_vec_avx2, "Vector AVX2 Scattered" },
{ i40e_recv_pkts_vec_avx2, "Vector AVX2" },
{ i40e_recv_scattered_pkts_vec, "Vector SSE Scattered" },
txq->queue_id);
}
-static eth_tx_burst_t
-i40e_get_latest_tx_vec(void)
-{
-#if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
- return i40e_xmit_pkts_vec_avx2;
-#endif
- return i40e_xmit_pkts_vec;
-}
-
-static eth_tx_burst_t
-i40e_get_recommend_tx_vec(void)
-{
-#if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)
- /*
- * since AVX frequency can be different to base frequency, limit
- * use of AVX2 version to later plaforms, not all those that could
- * theoretically run it.
- */
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
- return i40e_xmit_pkts_vec_avx2;
-#endif
- return i40e_xmit_pkts_vec;
-}
-
void __rte_cold
i40e_set_tx_function(struct rte_eth_dev *dev)
{
struct i40e_adapter *ad =
I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
int i;
+ bool use_avx2 = false;
+ bool use_avx512 = false;
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
if (ad->tx_vec_allowed) {
break;
}
}
+
+ use_avx512 = get_avx_supported(1);
+
+ if (!use_avx512)
+ use_avx2 = get_avx_supported(0);
}
}
if (ad->tx_simple_allowed) {
- if (ad->tx_vec_allowed) {
- PMD_INIT_LOG(DEBUG, "Vector tx finally be used.");
- if (ad->use_latest_vec)
- dev->tx_pkt_burst =
- i40e_get_latest_tx_vec();
- else
- dev->tx_pkt_burst =
- i40e_get_recommend_tx_vec();
+ if (ad->tx_vec_allowed &&
+ rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
+ if (use_avx512) {
+#ifdef CC_AVX512_SUPPORT
+ PMD_DRV_LOG(NOTICE, "Using AVX512 Vector Tx (port %d).",
+ dev->data->port_id);
+ dev->tx_pkt_burst = i40e_xmit_pkts_vec_avx512;
+#endif
+ } else {
+ PMD_INIT_LOG(DEBUG, "Using %sVector Tx (port %d).",
+ use_avx2 ? "avx2 " : "",
+ dev->data->port_id);
+ dev->tx_pkt_burst = use_avx2 ?
+ i40e_xmit_pkts_vec_avx2 :
+ i40e_xmit_pkts_vec;
+ }
} else {
PMD_INIT_LOG(DEBUG, "Simple tx finally be used.");
dev->tx_pkt_burst = i40e_xmit_pkts_simple;
{ i40e_xmit_pkts_simple, "Scalar Simple" },
{ i40e_xmit_pkts, "Scalar" },
#ifdef RTE_ARCH_X86
+#ifdef CC_AVX512_SUPPORT
+ { i40e_xmit_pkts_vec_avx512, "Vector AVX512" },
+#endif
{ i40e_xmit_pkts_vec_avx2, "Vector AVX2" },
{ i40e_xmit_pkts_vec, "Vector SSE" },
#elif defined(RTE_ARCH_ARM64)
}
#ifndef RTE_LIBRTE_I40E_INC_VECTOR
-/* Stubs needed for linkage when CONFIG_RTE_LIBRTE_I40E_INC_VECTOR is set to 'n' */
int
i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
{
git.droids-corp.org / dpdk.git / blobdiff