X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fbnxt%2Fbnxt_rxtx_vec_neon.c;h=7f3eabcda16a62b4dd57a0f4ec0540593160b4b6;hb=cec43bbf3805911eec0081681faa6dea6a06e7a9;hp=488a0b4667ec7bc68bb7f0b60db5f1bc1b8300e4;hpb=398358341419ce2666fec38eff8fff98d5be5941;p=dpdk.git diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c index 488a0b4667..7f3eabcda1 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c @@ -13,9 +13,6 @@ #include "bnxt.h" #include "bnxt_cpr.h" #include "bnxt_ring.h" -#include "bnxt_rxr.h" -#include "bnxt_rxq.h" -#include "hsi_struct_def_dpdk.h" #include "bnxt_rxtx_vec_common.h" #include "bnxt_txq.h" @@ -29,30 +26,34 @@ static inline void bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) { struct rx_prod_pkt_bd *rxbds = &rxr->rx_desc_ring[rxq->rxrearm_start]; - struct bnxt_sw_rx_bd *rx_bufs = &rxr->rx_buf_ring[rxq->rxrearm_start]; + struct rte_mbuf **rx_bufs = &rxr->rx_buf_ring[rxq->rxrearm_start]; struct rte_mbuf *mb0, *mb1; - int i; + int nb, i; const uint64x2_t hdr_room = {0, RTE_PKTMBUF_HEADROOM}; const uint64x2_t addrmask = {0, UINT64_MAX}; - /* Pull RTE_BNXT_RXQ_REARM_THRESH more mbufs into the software ring */ - if (rte_mempool_get_bulk(rxq->mb_pool, - (void *)rx_bufs, - RTE_BNXT_RXQ_REARM_THRESH) < 0) { - rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += - RTE_BNXT_RXQ_REARM_THRESH; + /* + * Number of mbufs to allocate must be a multiple of two. The + * allocation must not go past the end of the ring. + */ + nb = RTE_MIN(rxq->rxrearm_nb & ~0x1, + rxq->nb_rx_desc - rxq->rxrearm_start); + + /* Allocate new mbufs into the software ring */ + if (rte_mempool_get_bulk(rxq->mb_pool, (void *)rx_bufs, nb) < 0) { + rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += nb; return; } /* Initialize the mbufs in vector, process 2 mbufs in one loop */ - for (i = 0; i < RTE_BNXT_RXQ_REARM_THRESH; i += 2, rx_bufs += 2) { + for (i = 0; i < nb; i += 2, rx_bufs += 2) { uint64x2_t buf_addr0, buf_addr1; uint64x2_t rxbd0, rxbd1; - mb0 = rx_bufs[0].mbuf; - mb1 = rx_bufs[1].mbuf; + mb0 = rx_bufs[0]; + mb1 = rx_bufs[1]; /* Load address fields from both mbufs */ buf_addr0 = vld1q_u64((uint64_t *)&mb0->buf_addr); @@ -83,12 +84,12 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr) vst1q_u64((uint64_t *)(rxbds++), rxbd1); } - rxq->rxrearm_start += RTE_BNXT_RXQ_REARM_THRESH; + rxq->rxrearm_start += nb; bnxt_db_write(&rxr->rx_db, rxq->rxrearm_start - 1); if (rxq->rxrearm_start >= rxq->nb_rx_desc) rxq->rxrearm_start = 0; - rxq->rxrearm_nb -= RTE_BNXT_RXQ_REARM_THRESH; + rxq->rxrearm_nb -= nb; } static uint32_t @@ -206,7 +207,6 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint32_t cons; int nb_rx_pkts = 0; struct rx_pkt_cmpl *rxcmp; - bool evt = false; const uint64x2_t mbuf_init = {rxq->mbuf_initializer, 0}; const uint8x16_t shuf_msk = { 0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type (zeroes) */ @@ -215,101 +215,85 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, 0xFF, 0xFF, /* vlan_tci (zeroes) */ 12, 13, 14, 15 /* rss hash */ }; + int i; /* If Rx Q was stopped return */ if (unlikely(!rxq->rx_started)) return 0; - if (rxq->rxrearm_nb >= RTE_BNXT_RXQ_REARM_THRESH) + if (rxq->rxrearm_nb >= rxq->rx_free_thresh) bnxt_rxq_rearm(rxq, rxr); /* Return no more than RTE_BNXT_MAX_RX_BURST per call. */ nb_pkts = RTE_MIN(nb_pkts, RTE_BNXT_MAX_RX_BURST); - /* Make nb_pkts an integer multiple of RTE_BNXT_DESCS_PER_LOOP */ + /* Make nb_pkts an integer multiple of RTE_BNXT_DESCS_PER_LOOP. */ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_BNXT_DESCS_PER_LOOP); if (!nb_pkts) return 0; /* Handle RX burst request */ - while (1) { + for (i = 0; i < nb_pkts; i++) { + struct rx_pkt_cmpl_hi *rxcmp1; + struct rte_mbuf *mbuf; + uint64x2_t mm_rxcmp; + uint8x16_t pkt_mb; + cons = RING_CMP(cpr->cp_ring_struct, raw_cons); rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; + rxcmp1 = (struct rx_pkt_cmpl_hi *)&cpr->cp_desc_ring[cons + 1]; - if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct)) + if (!CMP_VALID(rxcmp1, raw_cons + 1, cpr->cp_ring_struct)) break; - if (likely(CMP_TYPE(rxcmp) == RX_PKT_CMPL_TYPE_RX_L2)) { - struct rx_pkt_cmpl_hi *rxcmp1; - uint32_t tmp_raw_cons; - uint16_t cp_cons; - struct rte_mbuf *mbuf; - uint64x2_t mm_rxcmp; - uint8x16_t pkt_mb; - - tmp_raw_cons = NEXT_RAW_CMP(raw_cons); - cp_cons = RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); - rxcmp1 = (struct rx_pkt_cmpl_hi *) - &cpr->cp_desc_ring[cp_cons]; - - if (!CMP_VALID(rxcmp1, tmp_raw_cons, - cpr->cp_ring_struct)) - break; - - raw_cons = tmp_raw_cons; - cons = rxcmp->opaque; - - mbuf = rxr->rx_buf_ring[cons].mbuf; - rte_prefetch0(mbuf); - rxr->rx_buf_ring[cons].mbuf = NULL; - - /* Set constant fields from mbuf initializer. */ - vst1q_u64((uint64_t *)&mbuf->rearm_data, mbuf_init); - - /* Set mbuf pkt_len, data_len, and rss_hash fields. */ - mm_rxcmp = vld1q_u64((uint64_t *)rxcmp); - pkt_mb = vqtbl1q_u8(vreinterpretq_u8_u64(mm_rxcmp), - shuf_msk); - vst1q_u64((uint64_t *)&mbuf->rx_descriptor_fields1, - vreinterpretq_u64_u8(pkt_mb)); - - rte_compiler_barrier(); - - if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) - mbuf->ol_flags |= PKT_RX_RSS_HASH; - - if (rxcmp1->flags2 & - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { - mbuf->vlan_tci = rxcmp1->metadata & - (RX_PKT_CMPL_METADATA_VID_MASK | - RX_PKT_CMPL_METADATA_DE | - RX_PKT_CMPL_METADATA_PRI_MASK); - mbuf->ol_flags |= - PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; - } - - bnxt_parse_csum(mbuf, rxcmp1); - mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); - - rx_pkts[nb_rx_pkts++] = mbuf; - } else if (!BNXT_NUM_ASYNC_CPR(rxq->bp)) { - evt = - bnxt_event_hwrm_resp_handler(rxq->bp, - (struct cmpl_base *)rxcmp); + raw_cons += 2; + cons = rxcmp->opaque; + + mbuf = rxr->rx_buf_ring[cons]; + rte_prefetch0(mbuf); + rxr->rx_buf_ring[cons] = NULL; + + /* Set constant fields from mbuf initializer. */ + vst1q_u64((uint64_t *)&mbuf->rearm_data, mbuf_init); + + /* Set mbuf pkt_len, data_len, and rss_hash fields. */ + mm_rxcmp = vld1q_u64((uint64_t *)rxcmp); + pkt_mb = vqtbl1q_u8(vreinterpretq_u8_u64(mm_rxcmp), shuf_msk); + vst1q_u64((uint64_t *)&mbuf->rx_descriptor_fields1, + vreinterpretq_u64_u8(pkt_mb)); + + rte_compiler_barrier(); + + if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) + mbuf->ol_flags |= PKT_RX_RSS_HASH; + + if (rxcmp1->flags2 & + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { + mbuf->vlan_tci = rxcmp1->metadata & + (RX_PKT_CMPL_METADATA_VID_MASK | + RX_PKT_CMPL_METADATA_DE | + RX_PKT_CMPL_METADATA_PRI_MASK); + mbuf->ol_flags |= + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; } - raw_cons = NEXT_RAW_CMP(raw_cons); - if (nb_rx_pkts == nb_pkts || evt) - break; + bnxt_parse_csum(mbuf, rxcmp1); + mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); + + rx_pkts[nb_rx_pkts++] = mbuf; } - rxr->rx_prod = RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); - rxq->rxrearm_nb += nb_rx_pkts; - cpr->cp_raw_cons = raw_cons; - cpr->valid = !!(cpr->cp_raw_cons & cpr->cp_ring_struct->ring_size); - if (nb_rx_pkts || evt) + if (nb_rx_pkts) { + rxr->rx_prod = + RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); + + rxq->rxrearm_nb += nb_rx_pkts; + cpr->cp_raw_cons = raw_cons; + cpr->valid = + !!(cpr->cp_raw_cons & cpr->cp_ring_struct->ring_size); bnxt_db_cq(cpr); + } return nb_rx_pkts; } @@ -329,6 +313,8 @@ bnxt_tx_cmp_vec(struct bnxt_tx_queue *txq, int nr_pkts) tx_buf = &txr->tx_buf_ring[cons]; cons = RING_NEXT(txr->tx_ring_struct, cons); mbuf = rte_pktmbuf_prefree_seg(tx_buf->mbuf); + if (unlikely(mbuf == NULL)) + continue; tx_buf->mbuf = NULL; if (blk && mbuf->pool != free[0]->pool) {