* Copyright(c) 2018 Intel Corporation
*/
-#include <rte_ethdev_driver.h>
+#include <ethdev_driver.h>
#include <rte_net.h>
#include <rte_vect.h>
#include "rte_pmd_ice.h"
#include "ice_rxtx.h"
+#include "ice_rxtx_vec_common.h"
#define ICE_TX_CKSUM_OFFLOAD_MASK ( \
PKT_TX_IP_CKSUM | \
uint32_t ring_size;
uint16_t len;
int use_def_burst_func = 1;
+ uint64_t offloads;
if (nb_desc % ICE_ALIGN_RING_DESC != 0 ||
nb_desc > ICE_MAX_RING_DESC ||
return -EINVAL;
}
+ offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
+
/* Free memory if needed */
if (dev->data->rx_queues[queue_idx]) {
ice_rx_queue_release(dev->data->rx_queues[queue_idx]);
rxq->nb_rx_desc = nb_desc;
rxq->rx_free_thresh = rx_conf->rx_free_thresh;
rxq->queue_id = queue_idx;
+ rxq->offloads = offloads;
rxq->reg_idx = vsi->base_queue + queue_idx;
rxq->port_id = dev->data->port_id;
flags |= PKT_RX_L4_CKSUM_GOOD;
if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S)))
- flags |= PKT_RX_EIP_CKSUM_BAD;
+ flags |= PKT_RX_OUTER_IP_CKSUM_BAD;
if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S)))
flags |= PKT_RX_OUTER_L4_CKSUM_BAD;
dev->rx_pkt_burst == ice_recv_scattered_pkts_vec ||
#ifdef CC_AVX512_SUPPORT
dev->rx_pkt_burst == ice_recv_pkts_vec_avx512 ||
+ dev->rx_pkt_burst == ice_recv_pkts_vec_avx512_offload ||
dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx512 ||
+ dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx512_offload ||
#endif
dev->rx_pkt_burst == ice_recv_pkts_vec_avx2 ||
dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx2)
desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
if (!(txd[desc_to_clean_to].cmd_type_offset_bsz &
rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))) {
- PMD_TX_FREE_LOG(DEBUG, "TX descriptor %4u is not done "
- "(port=%d queue=%d) value=0x%"PRIx64"\n",
- desc_to_clean_to,
- txq->port_id, txq->queue_id,
- txd[desc_to_clean_to].cmd_type_offset_bsz);
+ PMD_TX_LOG(DEBUG, "TX descriptor %4u is not done "
+ "(port=%d queue=%d) value=0x%"PRIx64"\n",
+ desc_to_clean_to,
+ txq->port_id, txq->queue_id,
+ txd[desc_to_clean_to].cmd_type_offset_bsz);
/* Failed to clean any descriptors */
return -1;
}
/* set RS bit on the last descriptor of one packet */
if (txq->nb_tx_used >= txq->tx_rs_thresh) {
- PMD_TX_FREE_LOG(DEBUG,
- "Setting RS bit on TXD id="
- "%4u (port=%d queue=%d)",
- tx_last, txq->port_id, txq->queue_id);
+ PMD_TX_LOG(DEBUG,
+ "Setting RS bit on TXD id="
+ "%4u (port=%d queue=%d)",
+ tx_last, txq->port_id, txq->queue_id);
td_cmd |= ICE_TX_DESC_CMD_RS;
#ifdef RTE_ARCH_X86
struct ice_rx_queue *rxq;
int i;
+ int rx_check_ret;
bool use_avx512 = false;
bool use_avx2 = false;
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
- if (!ice_rx_vec_dev_check(dev) && ad->rx_bulk_alloc_allowed &&
- rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
+ rx_check_ret = ice_rx_vec_dev_check(dev);
+ if (rx_check_ret >= 0 && ad->rx_bulk_alloc_allowed &&
+ rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
ad->rx_vec_allowed = true;
for (i = 0; i < dev->data->nb_rx_queues; i++) {
rxq = dev->data->rx_queues[i];
if (dev->data->scattered_rx) {
if (use_avx512) {
#ifdef CC_AVX512_SUPPORT
- PMD_DRV_LOG(NOTICE,
- "Using AVX512 Vector Scattered Rx (port %d).",
- dev->data->port_id);
- dev->rx_pkt_burst =
- ice_recv_scattered_pkts_vec_avx512;
+ if (rx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
+ PMD_DRV_LOG(NOTICE,
+ "Using AVX512 OFFLOAD Vector Scattered Rx (port %d).",
+ dev->data->port_id);
+ dev->rx_pkt_burst =
+ ice_recv_scattered_pkts_vec_avx512_offload;
+ } else {
+ PMD_DRV_LOG(NOTICE,
+ "Using AVX512 Vector Scattered Rx (port %d).",
+ dev->data->port_id);
+ dev->rx_pkt_burst =
+ ice_recv_scattered_pkts_vec_avx512;
+ }
#endif
} else {
PMD_DRV_LOG(DEBUG,
} else {
if (use_avx512) {
#ifdef CC_AVX512_SUPPORT
- PMD_DRV_LOG(NOTICE,
- "Using AVX512 Vector Rx (port %d).",
- dev->data->port_id);
- dev->rx_pkt_burst =
- ice_recv_pkts_vec_avx512;
+ if (rx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
+ PMD_DRV_LOG(NOTICE,
+ "Using AVX512 OFFLOAD Vector Rx (port %d).",
+ dev->data->port_id);
+ dev->rx_pkt_burst =
+ ice_recv_pkts_vec_avx512_offload;
+ } else {
+ PMD_DRV_LOG(NOTICE,
+ "Using AVX512 Vector Rx (port %d).",
+ dev->data->port_id);
+ dev->rx_pkt_burst =
+ ice_recv_pkts_vec_avx512;
+ }
#endif
} else {
PMD_DRV_LOG(DEBUG,
#ifdef RTE_ARCH_X86
#ifdef CC_AVX512_SUPPORT
{ ice_recv_scattered_pkts_vec_avx512, "Vector AVX512 Scattered" },
+ { ice_recv_scattered_pkts_vec_avx512_offload, "Offload Vector AVX512 Scattered" },
{ ice_recv_pkts_vec_avx512, "Vector AVX512" },
+ { ice_recv_pkts_vec_avx512_offload, "Offload Vector AVX512" },
#endif
{ ice_recv_scattered_pkts_vec_avx2, "Vector AVX2 Scattered" },
{ ice_recv_pkts_vec_avx2, "Vector AVX2" },
return i;
}
-#ifdef RTE_LIBRTE_ETHDEV_DEBUG
+#ifdef RTE_ETHDEV_DEBUG_TX
ret = rte_validate_tx_offload(m);
if (ret != 0) {
rte_errno = -ret;
#ifdef RTE_ARCH_X86
struct ice_tx_queue *txq;
int i;
+ int tx_check_ret;
bool use_avx512 = false;
bool use_avx2 = false;
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
- if (!ice_tx_vec_dev_check(dev) &&
- rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
+ tx_check_ret = ice_tx_vec_dev_check(dev);
+ if (tx_check_ret >= 0 &&
+ rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
ad->tx_vec_allowed = true;
for (i = 0; i < dev->data->nb_tx_queues; i++) {
txq = dev->data->tx_queues[i];
PMD_DRV_LOG(NOTICE,
"AVX512 is not supported in build env");
#endif
- if (!use_avx512 &&
+ if (!use_avx512 && tx_check_ret == ICE_VECTOR_PATH &&
(rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 ||
rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256)
use_avx2 = true;
+ if (!use_avx512 && tx_check_ret == ICE_VECTOR_OFFLOAD_PATH)
+ ad->tx_vec_allowed = false;
+
} else {
ad->tx_vec_allowed = false;
}
if (ad->tx_vec_allowed) {
if (use_avx512) {
#ifdef CC_AVX512_SUPPORT
- PMD_DRV_LOG(NOTICE, "Using AVX512 Vector Tx (port %d).",
- dev->data->port_id);
- dev->tx_pkt_burst = ice_xmit_pkts_vec_avx512;
+ if (tx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
+ PMD_DRV_LOG(NOTICE,
+ "Using AVX512 OFFLOAD Vector Tx (port %d).",
+ dev->data->port_id);
+ dev->tx_pkt_burst =
+ ice_xmit_pkts_vec_avx512_offload;
+ } else {
+ PMD_DRV_LOG(NOTICE,
+ "Using AVX512 Vector Tx (port %d).",
+ dev->data->port_id);
+ dev->tx_pkt_burst = ice_xmit_pkts_vec_avx512;
+ }
#endif
} else {
PMD_DRV_LOG(DEBUG, "Using %sVector Tx (port %d).",
#ifdef RTE_ARCH_X86
#ifdef CC_AVX512_SUPPORT
{ ice_xmit_pkts_vec_avx512, "Vector AVX512" },
+ { ice_xmit_pkts_vec_avx512_offload, "Offload Vector AVX512" },
#endif
{ ice_xmit_pkts_vec_avx2, "Vector AVX2" },
{ ice_xmit_pkts_vec, "Vector SSE" },