From: Lance Richardson Date: Wed, 9 Sep 2020 15:52:57 +0000 (-0400) Subject: net/bnxt: require async completion ring for vector path X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=6d28c53bf7a9063caa3197c6cc481e2a69e3be96;p=dpdk.git net/bnxt: require async completion ring for vector path Disable support for vector mode when async completions can be placed in a receive completion ring and change the default for all platforms to use a dedicated async completion ring. Simplify completion handling in vector mode receive paths now that it no longer needs to handle async completions. Reviewed-by: Ajit Khaparde Signed-off-by: Lance Richardson --- diff --git a/drivers/net/bnxt/bnxt.h b/drivers/net/bnxt/bnxt.h index a190d78bdd..ef5824cf9a 100644 --- a/drivers/net/bnxt/bnxt.h +++ b/drivers/net/bnxt/bnxt.h @@ -119,20 +119,19 @@ (BNXT_CHIP_THOR(bp) ? TPA_MAX_SEGS_TH : \ TPA_MAX_SEGS) -#ifdef RTE_ARCH_ARM64 -#define BNXT_NUM_ASYNC_CPR(bp) (BNXT_STINGRAY(bp) ? 0 : 1) +/* + * Define the number of async completion rings to be used. Set to zero for + * configurations in which the maximum number of packet completion rings + * for packet completions is desired or when async completion handling + * cannot be interrupt-driven. + */ +#ifdef RTE_EXEC_ENV_FREEBSD +/* In FreeBSD OS, nic_uio driver does not support interrupts */ +#define BNXT_NUM_ASYNC_CPR(bp) 0 #else #define BNXT_NUM_ASYNC_CPR(bp) 1 #endif -/* In FreeBSD OS, nic_uio driver does not support interrupts */ -#ifdef RTE_EXEC_ENV_FREEBSD -#ifdef BNXT_NUM_ASYNC_CPR -#undef BNXT_NUM_ASYNC_CPR -#endif -#define BNXT_NUM_ASYNC_CPR(bp) 0 -#endif - #define BNXT_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET #define BNXT_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c index 26c3c1a2a3..0dddb658f6 100644 --- a/drivers/net/bnxt/bnxt_ethdev.c +++ b/drivers/net/bnxt/bnxt_ethdev.c @@ -1114,7 +1114,7 @@ bnxt_receive_function(struct rte_eth_dev *eth_dev) DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | DEV_RX_OFFLOAD_RSS_HASH | DEV_RX_OFFLOAD_VLAN_FILTER)) && - !BNXT_TRUFLOW_EN(bp)) { + !BNXT_TRUFLOW_EN(bp) && BNXT_NUM_ASYNC_CPR(bp)) { PMD_DRV_LOG(INFO, "Using vector mode receive for port %d\n", eth_dev->data->port_id); bp->flags |= BNXT_FLAG_RX_VECTOR_PKT_MODE; diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c index eff196f3a0..a212d46cbe 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_neon.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_neon.c @@ -206,7 +206,6 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint32_t cons; int nb_rx_pkts = 0; struct rx_pkt_cmpl *rxcmp; - bool evt = false; const uint64x2_t mbuf_init = {rxq->mbuf_initializer, 0}; const uint8x16_t shuf_msk = { 0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type (zeroes) */ @@ -215,6 +214,7 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, 0xFF, 0xFF, /* vlan_tci (zeroes) */ 12, 13, 14, 15 /* rss hash */ }; + int i; /* If Rx Q was stopped return */ if (unlikely(!rxq->rx_started)) @@ -226,90 +226,73 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, /* Return no more than RTE_BNXT_MAX_RX_BURST per call. */ nb_pkts = RTE_MIN(nb_pkts, RTE_BNXT_MAX_RX_BURST); - /* Make nb_pkts an integer multiple of RTE_BNXT_DESCS_PER_LOOP */ + /* Make nb_pkts an integer multiple of RTE_BNXT_DESCS_PER_LOOP. */ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_BNXT_DESCS_PER_LOOP); if (!nb_pkts) return 0; /* Handle RX burst request */ - while (1) { + for (i = 0; i < nb_pkts; i++) { + struct rx_pkt_cmpl_hi *rxcmp1; + struct rte_mbuf *mbuf; + uint64x2_t mm_rxcmp; + uint8x16_t pkt_mb; + cons = RING_CMP(cpr->cp_ring_struct, raw_cons); rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; + rxcmp1 = (struct rx_pkt_cmpl_hi *)&cpr->cp_desc_ring[cons + 1]; - if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct)) + if (!CMP_VALID(rxcmp1, raw_cons + 1, cpr->cp_ring_struct)) break; - if (likely(CMP_TYPE(rxcmp) == RX_PKT_CMPL_TYPE_RX_L2)) { - struct rx_pkt_cmpl_hi *rxcmp1; - uint32_t tmp_raw_cons; - uint16_t cp_cons; - struct rte_mbuf *mbuf; - uint64x2_t mm_rxcmp; - uint8x16_t pkt_mb; - - tmp_raw_cons = NEXT_RAW_CMP(raw_cons); - cp_cons = RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); - rxcmp1 = (struct rx_pkt_cmpl_hi *) - &cpr->cp_desc_ring[cp_cons]; - - if (!CMP_VALID(rxcmp1, tmp_raw_cons, - cpr->cp_ring_struct)) - break; - - raw_cons = tmp_raw_cons; - cons = rxcmp->opaque; - - mbuf = rxr->rx_buf_ring[cons]; - rte_prefetch0(mbuf); - rxr->rx_buf_ring[cons] = NULL; - - /* Set constant fields from mbuf initializer. */ - vst1q_u64((uint64_t *)&mbuf->rearm_data, mbuf_init); - - /* Set mbuf pkt_len, data_len, and rss_hash fields. */ - mm_rxcmp = vld1q_u64((uint64_t *)rxcmp); - pkt_mb = vqtbl1q_u8(vreinterpretq_u8_u64(mm_rxcmp), - shuf_msk); - vst1q_u64((uint64_t *)&mbuf->rx_descriptor_fields1, - vreinterpretq_u64_u8(pkt_mb)); - - rte_compiler_barrier(); - - if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) - mbuf->ol_flags |= PKT_RX_RSS_HASH; - - if (rxcmp1->flags2 & - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { - mbuf->vlan_tci = rxcmp1->metadata & - (RX_PKT_CMPL_METADATA_VID_MASK | - RX_PKT_CMPL_METADATA_DE | - RX_PKT_CMPL_METADATA_PRI_MASK); - mbuf->ol_flags |= - PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; - } - - bnxt_parse_csum(mbuf, rxcmp1); - mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); - - rx_pkts[nb_rx_pkts++] = mbuf; - } else if (!BNXT_NUM_ASYNC_CPR(rxq->bp)) { - evt = - bnxt_event_hwrm_resp_handler(rxq->bp, - (struct cmpl_base *)rxcmp); + raw_cons += 2; + cons = rxcmp->opaque; + + mbuf = rxr->rx_buf_ring[cons]; + rte_prefetch0(mbuf); + rxr->rx_buf_ring[cons] = NULL; + + /* Set constant fields from mbuf initializer. */ + vst1q_u64((uint64_t *)&mbuf->rearm_data, mbuf_init); + + /* Set mbuf pkt_len, data_len, and rss_hash fields. */ + mm_rxcmp = vld1q_u64((uint64_t *)rxcmp); + pkt_mb = vqtbl1q_u8(vreinterpretq_u8_u64(mm_rxcmp), shuf_msk); + vst1q_u64((uint64_t *)&mbuf->rx_descriptor_fields1, + vreinterpretq_u64_u8(pkt_mb)); + + rte_compiler_barrier(); + + if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) + mbuf->ol_flags |= PKT_RX_RSS_HASH; + + if (rxcmp1->flags2 & + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { + mbuf->vlan_tci = rxcmp1->metadata & + (RX_PKT_CMPL_METADATA_VID_MASK | + RX_PKT_CMPL_METADATA_DE | + RX_PKT_CMPL_METADATA_PRI_MASK); + mbuf->ol_flags |= + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; } - raw_cons = NEXT_RAW_CMP(raw_cons); - if (nb_rx_pkts == nb_pkts || evt) - break; + bnxt_parse_csum(mbuf, rxcmp1); + mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); + + rx_pkts[nb_rx_pkts++] = mbuf; } - rxr->rx_prod = RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); - rxq->rxrearm_nb += nb_rx_pkts; - cpr->cp_raw_cons = raw_cons; - cpr->valid = !!(cpr->cp_raw_cons & cpr->cp_ring_struct->ring_size); - if (nb_rx_pkts || evt) + if (nb_rx_pkts) { + rxr->rx_prod = + RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); + + rxq->rxrearm_nb += nb_rx_pkts; + cpr->cp_raw_cons = raw_cons; + cpr->valid = + !!(cpr->cp_raw_cons & cpr->cp_ring_struct->ring_size); bnxt_db_cq(cpr); + } return nb_rx_pkts; } diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c index 822e43343f..c00d7f6807 100644 --- a/drivers/net/bnxt/bnxt_rxtx_vec_sse.c +++ b/drivers/net/bnxt/bnxt_rxtx_vec_sse.c @@ -210,7 +210,6 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint32_t cons; int nb_rx_pkts = 0; struct rx_pkt_cmpl *rxcmp; - bool evt = false; const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer); const __m128i shuf_msk = _mm_set_epi8(15, 14, 13, 12, /* rss */ @@ -218,6 +217,7 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, 3, 2, /* data_len */ 0xFF, 0xFF, 3, 2, /* pkt_len */ 0xFF, 0xFF, 0xFF, 0xFF); /* pkt_type (zeroes) */ + int i; /* If Rx Q was stopped return */ if (unlikely(!rxq->rx_started)) @@ -238,83 +238,65 @@ bnxt_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, return 0; /* Handle RX burst request */ - while (1) { + for (i = 0; i < nb_pkts; i++) { + struct rx_pkt_cmpl_hi *rxcmp1; + struct rte_mbuf *mbuf; + __m128i mm_rxcmp, pkt_mb; + cons = RING_CMP(cpr->cp_ring_struct, raw_cons); rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; + rxcmp1 = (struct rx_pkt_cmpl_hi *)&cpr->cp_desc_ring[cons + 1]; - if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct)) + if (!CMP_VALID(rxcmp1, raw_cons + 1, cpr->cp_ring_struct)) break; - if (likely(CMP_TYPE(rxcmp) == RX_PKT_CMPL_TYPE_RX_L2)) { - struct rx_pkt_cmpl_hi *rxcmp1; - uint32_t tmp_raw_cons; - uint16_t cp_cons; - struct rte_mbuf *mbuf; - __m128i mm_rxcmp, pkt_mb; - - tmp_raw_cons = NEXT_RAW_CMP(raw_cons); - cp_cons = RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); - rxcmp1 = (struct rx_pkt_cmpl_hi *) - &cpr->cp_desc_ring[cp_cons]; - - if (!CMP_VALID(rxcmp1, tmp_raw_cons, - cpr->cp_ring_struct)) - break; - - raw_cons = tmp_raw_cons; - cons = rxcmp->opaque; - - mbuf = rxr->rx_buf_ring[cons]; - rte_prefetch0(mbuf); - rxr->rx_buf_ring[cons] = NULL; - - /* Set constant fields from mbuf initializer. */ - _mm_store_si128((__m128i *)&mbuf->rearm_data, - mbuf_init); - - /* Set mbuf pkt_len, data_len, and rss_hash fields. */ - mm_rxcmp = _mm_load_si128((__m128i *)rxcmp); - pkt_mb = _mm_shuffle_epi8(mm_rxcmp, shuf_msk); - _mm_storeu_si128((void *)&mbuf->rx_descriptor_fields1, - pkt_mb); - - rte_compiler_barrier(); - - if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) - mbuf->ol_flags |= PKT_RX_RSS_HASH; - - if (rxcmp1->flags2 & - RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { - mbuf->vlan_tci = rxcmp1->metadata & - (RX_PKT_CMPL_METADATA_VID_MASK | - RX_PKT_CMPL_METADATA_DE | - RX_PKT_CMPL_METADATA_PRI_MASK); - mbuf->ol_flags |= - PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; - } - - bnxt_parse_csum(mbuf, rxcmp1); - mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); - - rx_pkts[nb_rx_pkts++] = mbuf; - } else if (!BNXT_NUM_ASYNC_CPR(rxq->bp)) { - evt = - bnxt_event_hwrm_resp_handler(rxq->bp, - (struct cmpl_base *)rxcmp); + raw_cons += 2; + cons = rxcmp->opaque; + + mbuf = rxr->rx_buf_ring[cons]; + rte_prefetch0(mbuf); + rxr->rx_buf_ring[cons] = NULL; + + /* Set constant fields from mbuf initializer. */ + _mm_store_si128((__m128i *)&mbuf->rearm_data, mbuf_init); + + /* Set mbuf pkt_len, data_len, and rss_hash fields. */ + mm_rxcmp = _mm_load_si128((__m128i *)rxcmp); + pkt_mb = _mm_shuffle_epi8(mm_rxcmp, shuf_msk); + _mm_storeu_si128((void *)&mbuf->rx_descriptor_fields1, pkt_mb); + + rte_compiler_barrier(); + + if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) + mbuf->ol_flags |= PKT_RX_RSS_HASH; + + if (rxcmp1->flags2 & + RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { + mbuf->vlan_tci = rxcmp1->metadata & + (RX_PKT_CMPL_METADATA_VID_MASK | + RX_PKT_CMPL_METADATA_DE | + RX_PKT_CMPL_METADATA_PRI_MASK); + mbuf->ol_flags |= + PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; } - raw_cons = NEXT_RAW_CMP(raw_cons); - if (nb_rx_pkts == nb_pkts || evt) - break; + bnxt_parse_csum(mbuf, rxcmp1); + mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); + + rx_pkts[nb_rx_pkts++] = mbuf; } - rxr->rx_prod = RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); - rxq->rxrearm_nb += nb_rx_pkts; - cpr->cp_raw_cons = raw_cons; - cpr->valid = !!(cpr->cp_raw_cons & cpr->cp_ring_struct->ring_size); - if (nb_rx_pkts || evt) + if (nb_rx_pkts) { + rxr->rx_prod = + RING_ADV(rxr->rx_ring_struct, rxr->rx_prod, nb_rx_pkts); + + rxq->rxrearm_nb += nb_rx_pkts; + cpr->cp_raw_cons = raw_cons; + cpr->valid = + !!(cpr->cp_raw_cons & cpr->cp_ring_struct->ring_size); bnxt_db_cq(cpr); + } return nb_rx_pkts; }