X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fbnxt%2Fbnxt_rxtx_vec_sse.c;h=7ae6f5c098ca2a6bbc4ff12db9cab7e07b7ed4bf;hb=255b8f86eb6e;hp=f71f4698528c4d2d8b598bdb9d9e2a20305376a1;hpb=369f6077c515e52f393e5b6e3226453d9f0ec166;p=dpdk.git diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c index f71f469852..7ae6f5c098 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2019-2020 Broadcom All rights reserved. */ +/* Copyright(c) 2019-2021 Broadcom All rights reserved. */ #include #include @@ -27,11 +27,11 @@ uint32_t tmp, of; \ \ of = _mm_extract_epi32((rss_flags), (pi)) | \ - bnxt_ol_flags_table[_mm_extract_epi32((ol_index), (pi))]; \ + rxr->ol_flags_table[_mm_extract_epi32((ol_index), (pi))]; \ \ tmp = _mm_extract_epi32((errors), (pi)); \ if (tmp) \ - of |= bnxt_ol_flags_err_table[tmp]; \ + of |= rxr->ol_flags_err_table[tmp]; \ (ol_flags) = of; \ } @@ -54,7 +54,8 @@ static inline void descs_to_mbufs(__m128i mm_rxcmp[4], __m128i mm_rxcmp1[4], - __m128i mbuf_init, struct rte_mbuf **mbuf) + __m128i mbuf_init, struct rte_mbuf **mbuf, + struct bnxt_rx_ring_info *rxr) { const __m128i shuf_msk = _mm_set_epi8(15, 14, 13, 12, /* rss */ @@ -63,31 +64,16 @@ descs_to_mbufs(__m128i mm_rxcmp[4], __m128i mm_rxcmp1[4], 0xFF, 0xFF, 3, 2, /* pkt_len */ 0xFF, 0xFF, 0xFF, 0xFF); /* pkt_type (zeroes) */ const __m128i flags_type_mask = - _mm_set_epi32(RX_PKT_CMPL_FLAGS_ITYPE_MASK, - RX_PKT_CMPL_FLAGS_ITYPE_MASK, - RX_PKT_CMPL_FLAGS_ITYPE_MASK, - RX_PKT_CMPL_FLAGS_ITYPE_MASK); + _mm_set1_epi32(RX_PKT_CMPL_FLAGS_ITYPE_MASK); const __m128i flags2_mask1 = - _mm_set_epi32(RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | - RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC, - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | - RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC, - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | - RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC, - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | - RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC); + _mm_set1_epi32(RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC); const __m128i flags2_mask2 = - _mm_set_epi32(RX_PKT_CMPL_FLAGS2_IP_TYPE, - RX_PKT_CMPL_FLAGS2_IP_TYPE, - RX_PKT_CMPL_FLAGS2_IP_TYPE, - RX_PKT_CMPL_FLAGS2_IP_TYPE); + _mm_set1_epi32(RX_PKT_CMPL_FLAGS2_IP_TYPE); const __m128i rss_mask = - _mm_set_epi32(RX_PKT_CMPL_FLAGS_RSS_VALID, - RX_PKT_CMPL_FLAGS_RSS_VALID, - RX_PKT_CMPL_FLAGS_RSS_VALID, - RX_PKT_CMPL_FLAGS_RSS_VALID); + _mm_set1_epi32(RX_PKT_CMPL_FLAGS_RSS_VALID); __m128i t0, t1, flags_type, flags2, index, errors, rss_flags; - __m128i ptype_idx; + __m128i ptype_idx, is_tunnel; uint32_t ol_flags; /* Compute packet type table indexes for four packets */ @@ -114,13 +100,17 @@ descs_to_mbufs(__m128i mm_rxcmp[4], __m128i mm_rxcmp1[4], t1 = _mm_unpackhi_epi32(mm_rxcmp1[2], mm_rxcmp1[3]); /* Compute ol_flags and checksum error indexes for four packets. */ - flags2 = _mm_and_si128(flags2, _mm_set_epi32(0x1F, 0x1F, 0x1F, 0x1F)); + is_tunnel = _mm_and_si128(flags2, _mm_set1_epi32(4)); + is_tunnel = _mm_slli_epi32(is_tunnel, 3); + flags2 = _mm_and_si128(flags2, _mm_set1_epi32(0x1F)); errors = _mm_srli_epi32(_mm_unpacklo_epi64(t0, t1), 4); - errors = _mm_and_si128(errors, _mm_set_epi32(0xF, 0xF, 0xF, 0xF)); + errors = _mm_and_si128(errors, _mm_set1_epi32(0xF)); errors = _mm_and_si128(errors, flags2); index = _mm_andnot_si128(errors, flags2); + errors = _mm_or_si128(errors, _mm_srli_epi32(is_tunnel, 1)); + index = _mm_or_si128(index, is_tunnel); /* Update mbuf rearm_data for four packets. */ GET_OL_FLAGS(rss_flags, index, errors, 0, ol_flags); @@ -165,16 +155,12 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t rx_ring_size = rxr->rx_ring_struct->ring_size; struct cmpl_base *cp_desc_ring = cpr->cp_desc_ring; uint64_t valid, desc_valid_mask = ~0ULL; - const __m128i info3_v_mask = _mm_set_epi32(CMPL_BASE_V, CMPL_BASE_V, - CMPL_BASE_V, CMPL_BASE_V); + const __m128i info3_v_mask = _mm_set1_epi32(CMPL_BASE_V); uint32_t raw_cons = cpr->cp_raw_cons; uint32_t cons, mbcons; int nb_rx_pkts = 0; const __m128i valid_target = - _mm_set_epi32(!!(raw_cons & cp_ring_size), - !!(raw_cons & cp_ring_size), - !!(raw_cons & cp_ring_size), - !!(raw_cons & cp_ring_size)); + _mm_set1_epi32(!!(raw_cons & cp_ring_size)); int i; /* If Rx Q was stopped return */ @@ -287,7 +273,8 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, goto out; } - descs_to_mbufs(rxcmp, rxcmp1, mbuf_init, &rx_pkts[nb_rx_pkts]); + descs_to_mbufs(rxcmp, rxcmp1, mbuf_init, &rx_pkts[nb_rx_pkts], + rxr); nb_rx_pkts += num_valid; if (num_valid < RTE_BNXT_DESCS_PER_LOOP) @@ -296,8 +283,7 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, out: if (nb_rx_pkts) { - rxr->rx_prod = - RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); + rxr->rx_raw_prod = RING_ADV(rxr->rx_raw_prod, nb_rx_pkts); rxq->rxrearm_nb += nb_rx_pkts; cpr->cp_raw_cons += 2 * nb_rx_pkts; @@ -340,7 +326,7 @@ bnxt_handle_tx_cp_vec(struct bnxt_tx_queue *txq) cpr->valid = !!(raw_cons & cp_ring_struct->ring_size); if (nb_tx_pkts) { if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) - bnxt_tx_cmp_fast(txq, nb_tx_pkts); + bnxt_tx_cmp_vec_fast(txq, nb_tx_pkts); else bnxt_tx_cmp_vec(txq, nb_tx_pkts); cpr->cp_raw_cons = raw_cons; @@ -370,11 +356,12 @@ bnxt_xmit_fixed_burst_vec(struct bnxt_tx_queue *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct bnxt_tx_ring_info *txr = txq->tx_ring; - uint16_t tx_prod = txr->tx_prod; + uint16_t tx_prod, tx_raw_prod = txr->tx_raw_prod; struct tx_bd_long *txbd; struct bnxt_sw_tx_bd *tx_buf; uint16_t to_send; + tx_prod = RING_IDX(txr->tx_ring_struct, tx_raw_prod); txbd = &txr->tx_desc_ring[tx_prod]; tx_buf = &txr->tx_buf_ring[tx_prod]; @@ -414,10 +401,10 @@ bnxt_xmit_fixed_burst_vec(struct bnxt_tx_queue *txq, struct rte_mbuf **tx_pkts, txbd[-1].opaque = nb_pkts; txbd[-1].flags_type &= ~TX_BD_LONG_FLAGS_NO_CMPL; - tx_prod = RING_ADV(txr->tx_ring_struct, tx_prod, nb_pkts); - bnxt_db_write(&txr->tx_db, tx_prod); + tx_raw_prod += nb_pkts; + bnxt_db_write(&txr->tx_db, tx_raw_prod); - txr->tx_prod = tx_prod; + txr->tx_raw_prod = tx_raw_prod; return nb_pkts; } @@ -454,8 +441,8 @@ bnxt_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, * Ensure that a ring wrap does not occur within a call to * bnxt_xmit_fixed_burst_vec(). */ - num = RTE_MIN(num, - ring_size - (txr->tx_prod & (ring_size - 1))); + num = RTE_MIN(num, ring_size - + (txr->tx_raw_prod & (ring_size - 1))); ret = bnxt_xmit_fixed_burst_vec(txq, &tx_pkts[nb_sent], num); nb_sent += ret; nb_pkts -= ret;