X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=sidebyside;f=drivers%2Fnet%2Fbnxt%2Fbnxt_rxtx_vec_neon.c;h=3d54d9d59d27bdd29302553a1d8152901642a231;hb=3740259eae2dcf0903d7e8a08909a22486be3744;hp=f49e29ccb1eb7cd8d3f4c7e9730c7a8fc35dc3f7;hpb=fc544b5460491195a2cdd1c75da9bbba3f6e0d98;p=dpdk.git diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c index f49e29ccb1..3d54d9d59d 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2019-2020 Broadcom All rights reserved. */ +/* Copyright(c) 2019-2021 Broadcom All rights reserved. */ #include #include @@ -27,11 +27,11 @@ uint32_t tmp, of; \ \ of = vgetq_lane_u32((rss_flags), (pi)) | \ - bnxt_ol_flags_table[vgetq_lane_u32((ol_idx), (pi))]; \ + rxr->ol_flags_table[vgetq_lane_u32((ol_idx), (pi))]; \ \ tmp = vgetq_lane_u32((errors), (pi)); \ if (tmp) \ - of |= bnxt_ol_flags_err_table[tmp]; \ + of |= rxr->ol_flags_err_table[tmp]; \ (ol_flags) = of; \ } @@ -58,7 +58,8 @@ static void descs_to_mbufs(uint32x4_t mm_rxcmp[4], uint32x4_t mm_rxcmp1[4], - uint64x2_t mb_init, struct rte_mbuf **mbuf) + uint64x2_t mb_init, struct rte_mbuf **mbuf, + struct bnxt_rx_ring_info *rxr) { const uint8x16_t shuf_msk = { 0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type (zeroes) */ @@ -67,42 +68,19 @@ descs_to_mbufs(uint32x4_t mm_rxcmp[4], uint32x4_t mm_rxcmp1[4], 0xFF, 0xFF, /* vlan_tci (zeroes) */ 12, 13, 14, 15 /* rss hash */ }; - const uint32x4_t flags_type_mask = { - RX_PKT_CMPL_FLAGS_ITYPE_MASK, - RX_PKT_CMPL_FLAGS_ITYPE_MASK, - RX_PKT_CMPL_FLAGS_ITYPE_MASK, - RX_PKT_CMPL_FLAGS_ITYPE_MASK - }; - const uint32x4_t flags2_mask1 = { - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | - RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC, - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | - RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC, - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | - RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC, - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | - RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC - }; - const uint32x4_t flags2_mask2 = { - RX_PKT_CMPL_FLAGS2_IP_TYPE, - RX_PKT_CMPL_FLAGS2_IP_TYPE, - RX_PKT_CMPL_FLAGS2_IP_TYPE, - RX_PKT_CMPL_FLAGS2_IP_TYPE - }; - const uint32x4_t rss_mask = { - RX_PKT_CMPL_FLAGS_RSS_VALID, - RX_PKT_CMPL_FLAGS_RSS_VALID, - RX_PKT_CMPL_FLAGS_RSS_VALID, - RX_PKT_CMPL_FLAGS_RSS_VALID - }; - const uint32x4_t flags2_index_mask = { - 0x1F, 0x1F, 0x1F, 0x1F - }; - const uint32x4_t flags2_error_mask = { - 0xF, 0xF, 0xF, 0xF - }; + const uint32x4_t flags_type_mask = + vdupq_n_u32(RX_PKT_CMPL_FLAGS_ITYPE_MASK); + const uint32x4_t flags2_mask1 = + vdupq_n_u32(RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC); + const uint32x4_t flags2_mask2 = + vdupq_n_u32(RX_PKT_CMPL_FLAGS2_IP_TYPE); + const uint32x4_t rss_mask = + vdupq_n_u32(RX_PKT_CMPL_FLAGS_RSS_VALID); + const uint32x4_t flags2_index_mask = vdupq_n_u32(0x1F); + const uint32x4_t flags2_error_mask = vdupq_n_u32(0x0F); uint32x4_t flags_type, flags2, index, errors, rss_flags; - uint32x4_t tmp, ptype_idx; + uint32x4_t tmp, ptype_idx, is_tunnel; uint64x2_t t0, t1; uint32_t ol_flags; @@ -139,10 +117,14 @@ descs_to_mbufs(uint32x4_t mm_rxcmp[4], uint32x4_t mm_rxcmp1[4], vget_low_u64(t1))); /* Compute ol_flags and checksum error indexes for four packets. */ + is_tunnel = vandq_u32(flags2, vdupq_n_u32(4)); + is_tunnel = vshlq_n_u32(is_tunnel, 3); errors = vandq_u32(vshrq_n_u32(errors, 4), flags2_error_mask); errors = vandq_u32(errors, flags2); index = vbicq_u32(flags2, errors); + errors = vorrq_u32(errors, vshrq_n_u32(is_tunnel, 1)); + index = vorrq_u32(index, is_tunnel); /* Update mbuf rearm_data for four packets. */ GET_OL_FLAGS(rss_flags, index, errors, 0, ol_flags); @@ -180,20 +162,13 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t rx_ring_size = rxr->rx_ring_struct->ring_size; struct cmpl_base *cp_desc_ring = cpr->cp_desc_ring; uint64_t valid, desc_valid_mask = ~0UL; - const uint32x4_t info3_v_mask = { - CMPL_BASE_V, CMPL_BASE_V, - CMPL_BASE_V, CMPL_BASE_V - }; + const uint32x4_t info3_v_mask = vdupq_n_u32(CMPL_BASE_V); uint32_t raw_cons = cpr->cp_raw_cons; uint32_t cons, mbcons; int nb_rx_pkts = 0; const uint64x2_t mb_init = {rxq->mbuf_initializer, 0}; - const uint32x4_t valid_target = { - !!(raw_cons & cp_ring_size), - !!(raw_cons & cp_ring_size), - !!(raw_cons & cp_ring_size), - !!(raw_cons & cp_ring_size) - }; + const uint32x4_t valid_target = + vdupq_n_u32(!!(raw_cons & cp_ring_size)); int i; /* If Rx Q was stopped return */ @@ -299,34 +274,19 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, num_valid = (sizeof(uint64_t) / sizeof(uint16_t)) - (__builtin_clzl(valid & desc_valid_mask) / 16); - switch (num_valid) { - case 4: - rxr->rx_buf_ring[mbcons + 3] = NULL; - /* FALLTHROUGH */ - case 3: - rxr->rx_buf_ring[mbcons + 2] = NULL; - /* FALLTHROUGH */ - case 2: - rxr->rx_buf_ring[mbcons + 1] = NULL; - /* FALLTHROUGH */ - case 1: - rxr->rx_buf_ring[mbcons + 0] = NULL; + if (num_valid == 0) break; - case 0: - goto out; - } - descs_to_mbufs(rxcmp, rxcmp1, mb_init, &rx_pkts[nb_rx_pkts]); + descs_to_mbufs(rxcmp, rxcmp1, mb_init, &rx_pkts[nb_rx_pkts], + rxr); nb_rx_pkts += num_valid; if (num_valid < RTE_BNXT_DESCS_PER_LOOP) break; } -out: if (nb_rx_pkts) { - rxr->rx_prod = - RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); + rxr->rx_raw_prod = RING_ADV(rxr->rx_raw_prod, nb_rx_pkts); rxq->rxrearm_nb += nb_rx_pkts; cpr->cp_raw_cons += 2 * nb_rx_pkts; @@ -383,7 +343,7 @@ bnxt_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, { struct bnxt_tx_queue *txq = tx_queue; struct bnxt_tx_ring_info *txr = txq->tx_ring; - uint16_t prod = txr->tx_prod; + uint16_t tx_prod, tx_raw_prod = txr->tx_raw_prod; struct rte_mbuf *tx_mbuf; struct tx_bd_long *txbd = NULL; struct bnxt_sw_tx_bd *tx_buf; @@ -400,16 +360,17 @@ bnxt_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_mbuf = *tx_pkts++; rte_prefetch0(tx_mbuf); - tx_buf = &txr->tx_buf_ring[prod]; + tx_prod = RING_IDX(txr->tx_ring_struct, tx_raw_prod); + tx_buf = &txr->tx_buf_ring[tx_prod]; tx_buf->mbuf = tx_mbuf; tx_buf->nr_bds = 1; - txbd = &txr->tx_desc_ring[prod]; + txbd = &txr->tx_desc_ring[tx_prod]; txbd->address = tx_mbuf->buf_iova + tx_mbuf->data_off; txbd->len = tx_mbuf->data_len; txbd->flags_type = bnxt_xmit_flags_len(tx_mbuf->data_len, TX_BD_FLAGS_NOCMPL); - prod = RING_NEXT(txr->tx_ring_struct, prod); + tx_raw_prod = RING_NEXT(tx_raw_prod); to_send--; } @@ -420,9 +381,9 @@ bnxt_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, } rte_compiler_barrier(); - bnxt_db_write(&txr->tx_db, prod); + bnxt_db_write(&txr->tx_db, tx_raw_prod); - txr->tx_prod = prod; + txr->tx_raw_prod = tx_raw_prod; return nb_pkts; }