/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2014-2018 Broadcom
+ * Copyright(c) 2014-2021 Broadcom
* All rights reserved.
*/
return mbuf;
}
+static void bnxt_tpa_get_metadata(struct bnxt *bp,
+ struct bnxt_tpa_info *tpa_info,
+ struct rx_tpa_start_cmpl *tpa_start,
+ struct rx_tpa_start_cmpl_hi *tpa_start1)
+{
+ tpa_info->cfa_code_valid = 0;
+ tpa_info->vlan_valid = 0;
+ tpa_info->hash_valid = 0;
+ tpa_info->l4_csum_valid = 0;
+
+ if (likely(tpa_start->flags_type &
+ rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS_RSS_VALID))) {
+ tpa_info->hash_valid = 1;
+ tpa_info->rss_hash = rte_le_to_cpu_32(tpa_start->rss_hash);
+ }
+
+ if (bp->vnic_cap_flags & BNXT_VNIC_CAP_RX_CMPL_V2) {
+ struct rx_tpa_start_v2_cmpl *v2_tpa_start = (void *)tpa_start;
+ struct rx_tpa_start_v2_cmpl_hi *v2_tpa_start1 =
+ (void *)tpa_start1;
+
+ if (v2_tpa_start->agg_id &
+ RX_TPA_START_V2_CMPL_METADATA1_VALID) {
+ tpa_info->vlan_valid = 1;
+ tpa_info->vlan =
+ rte_le_to_cpu_16(v2_tpa_start1->metadata0);
+ }
+
+ if (v2_tpa_start1->flags2 & RX_CMP_FLAGS2_L4_CSUM_ALL_OK_MASK)
+ tpa_info->l4_csum_valid = 1;
+
+ return;
+ }
+
+ tpa_info->cfa_code_valid = 1;
+ tpa_info->cfa_code = rte_le_to_cpu_16(tpa_start1->cfa_code);
+ if (tpa_start1->flags2 &
+ rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN)) {
+ tpa_info->vlan_valid = 1;
+ tpa_info->vlan = rte_le_to_cpu_32(tpa_start1->metadata);
+ }
+
+ if (likely(tpa_start1->flags2 &
+ rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_L4_CS_CALC)))
+ tpa_info->l4_csum_valid = 1;
+}
+
static void bnxt_tpa_start(struct bnxt_rx_queue *rxq,
struct rx_tpa_start_cmpl *tpa_start,
struct rx_tpa_start_cmpl_hi *tpa_start1)
mbuf->data_len = mbuf->pkt_len;
mbuf->port = rxq->port_id;
mbuf->ol_flags = PKT_RX_LRO;
- if (likely(tpa_start->flags_type &
- rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS_RSS_VALID))) {
- mbuf->hash.rss = rte_le_to_cpu_32(tpa_start->rss_hash);
+
+ bnxt_tpa_get_metadata(rxq->bp, tpa_info, tpa_start, tpa_start1);
+
+ if (likely(tpa_info->hash_valid)) {
+ mbuf->hash.rss = tpa_info->rss_hash;
mbuf->ol_flags |= PKT_RX_RSS_HASH;
- } else {
- mbuf->hash.fdir.id = rte_le_to_cpu_16(tpa_start1->cfa_code);
+ } else if (tpa_info->cfa_code_valid) {
+ mbuf->hash.fdir.id = tpa_info->cfa_code;
mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
}
- if (tpa_start1->flags2 &
- rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN)) {
- mbuf->vlan_tci = rte_le_to_cpu_32(tpa_start1->metadata);
+
+ if (tpa_info->vlan_valid) {
+ mbuf->vlan_tci = tpa_info->vlan;
mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
}
- if (likely(tpa_start1->flags2 &
- rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_L4_CS_CALC)))
+
+ if (likely(tpa_info->l4_csum_valid))
mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
/* recycle next mbuf */
*/
rte_bitmap_set(rxr->ag_bitmap, ag_cons);
}
+ last->next = NULL;
bnxt_prod_ag_mbuf(rxq);
return 0;
}
ip6 = i & (RX_PKT_CMPL_FLAGS2_IP_TYPE >> 7);
tun = i & (RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC >> 2);
- type = (i & 0x38) << 9;
+ type = (i & 0x78) << 9;
if (!tun && !ip6)
l3 = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
return bnxt_ptype_table[index];
}
-uint32_t
-bnxt_ol_flags_table[BNXT_OL_FLAGS_TBL_DIM] __rte_cache_aligned;
-
-uint32_t
-bnxt_ol_flags_err_table[BNXT_OL_FLAGS_ERR_TBL_DIM] __rte_cache_aligned;
-
static void __rte_cold
-bnxt_init_ol_flags_tables(void)
+bnxt_init_ol_flags_tables(struct bnxt_rx_queue *rxq)
{
- static bool initialized;
+ struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
+ struct rte_eth_conf *dev_conf;
+ bool outer_cksum_enabled;
+ uint64_t offloads;
uint32_t *pt;
int i;
- if (initialized)
- return;
+ dev_conf = &rxq->bp->eth_dev->data->dev_conf;
+ offloads = dev_conf->rxmode.offloads;
+
+ outer_cksum_enabled = !!(offloads & (DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
+ DEV_RX_OFFLOAD_OUTER_UDP_CKSUM));
/* Initialize ol_flags table. */
- pt = bnxt_ol_flags_table;
+ pt = rxr->ol_flags_table;
for (i = 0; i < BNXT_OL_FLAGS_TBL_DIM; i++) {
pt[i] = 0;
+
if (i & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN)
pt[i] |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
- if (i & RX_PKT_CMPL_FLAGS2_IP_CS_CALC)
- pt[i] |= PKT_RX_IP_CKSUM_GOOD;
+ if (i & (RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC << 3)) {
+ /* Tunnel case. */
+ if (outer_cksum_enabled) {
+ if (i & RX_PKT_CMPL_FLAGS2_IP_CS_CALC)
+ pt[i] |= PKT_RX_IP_CKSUM_GOOD;
+
+ if (i & RX_PKT_CMPL_FLAGS2_L4_CS_CALC)
+ pt[i] |= PKT_RX_L4_CKSUM_GOOD;
+
+ if (i & RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC)
+ pt[i] |= PKT_RX_OUTER_L4_CKSUM_GOOD;
+ } else {
+ if (i & RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)
+ pt[i] |= PKT_RX_IP_CKSUM_GOOD;
- if (i & RX_PKT_CMPL_FLAGS2_L4_CS_CALC)
- pt[i] |= PKT_RX_L4_CKSUM_GOOD;
+ if (i & RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC)
+ pt[i] |= PKT_RX_L4_CKSUM_GOOD;
+ }
+ } else {
+ /* Non-tunnel case. */
+ if (i & RX_PKT_CMPL_FLAGS2_IP_CS_CALC)
+ pt[i] |= PKT_RX_IP_CKSUM_GOOD;
- if (i & RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC)
- pt[i] |= PKT_RX_OUTER_L4_CKSUM_GOOD;
+ if (i & RX_PKT_CMPL_FLAGS2_L4_CS_CALC)
+ pt[i] |= PKT_RX_L4_CKSUM_GOOD;
+ }
}
/* Initialize checksum error table. */
- pt = bnxt_ol_flags_err_table;
+ pt = rxr->ol_flags_err_table;
for (i = 0; i < BNXT_OL_FLAGS_ERR_TBL_DIM; i++) {
pt[i] = 0;
- if (i & (RX_PKT_CMPL_ERRORS_IP_CS_ERROR >> 4))
- pt[i] |= PKT_RX_IP_CKSUM_BAD;
- if (i & (RX_PKT_CMPL_ERRORS_L4_CS_ERROR >> 4))
- pt[i] |= PKT_RX_L4_CKSUM_BAD;
+ if (i & (RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC << 2)) {
+ /* Tunnel case. */
+ if (outer_cksum_enabled) {
+ if (i & (RX_PKT_CMPL_ERRORS_IP_CS_ERROR >> 4))
+ pt[i] |= PKT_RX_IP_CKSUM_BAD;
- if (i & (RX_PKT_CMPL_ERRORS_T_IP_CS_ERROR >> 4))
- pt[i] |= PKT_RX_EIP_CKSUM_BAD;
+ if (i & (RX_PKT_CMPL_ERRORS_T_IP_CS_ERROR >> 4))
+ pt[i] |= PKT_RX_EIP_CKSUM_BAD;
- if (i & (RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR >> 4))
- pt[i] |= PKT_RX_OUTER_L4_CKSUM_BAD;
- }
+ if (i & (RX_PKT_CMPL_ERRORS_L4_CS_ERROR >> 4))
+ pt[i] |= PKT_RX_L4_CKSUM_BAD;
- initialized = true;
+ if (i & (RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR >> 4))
+ pt[i] |= PKT_RX_OUTER_L4_CKSUM_BAD;
+ } else {
+ if (i & (RX_PKT_CMPL_ERRORS_T_IP_CS_ERROR >> 4))
+ pt[i] |= PKT_RX_IP_CKSUM_BAD;
+
+ if (i & (RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR >> 4))
+ pt[i] |= PKT_RX_L4_CKSUM_BAD;
+ }
+ } else {
+ /* Non-tunnel case. */
+ if (i & (RX_PKT_CMPL_ERRORS_IP_CS_ERROR >> 4))
+ pt[i] |= PKT_RX_IP_CKSUM_BAD;
+
+ if (i & (RX_PKT_CMPL_ERRORS_L4_CS_ERROR >> 4))
+ pt[i] |= PKT_RX_L4_CKSUM_BAD;
+ }
+ }
}
static void
-bnxt_set_ol_flags(struct rx_pkt_cmpl *rxcmp, struct rx_pkt_cmpl_hi *rxcmp1,
- struct rte_mbuf *mbuf)
+bnxt_set_ol_flags(struct bnxt_rx_ring_info *rxr, struct rx_pkt_cmpl *rxcmp,
+ struct rx_pkt_cmpl_hi *rxcmp1, struct rte_mbuf *mbuf)
{
uint16_t flags_type, errors, flags;
uint64_t ol_flags;
RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC |
RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN);
+ flags |= (flags & RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC) << 3;
errors = rte_le_to_cpu_16(rxcmp1->errors_v2) &
(RX_PKT_CMPL_ERRORS_IP_CS_ERROR |
RX_PKT_CMPL_ERRORS_L4_CS_ERROR |
RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR);
errors = (errors >> 4) & flags;
- ol_flags = bnxt_ol_flags_table[flags & ~errors];
+ ol_flags = rxr->ol_flags_table[flags & ~errors];
- if (errors)
- ol_flags |= bnxt_ol_flags_err_table[errors];
+ if (unlikely(errors)) {
+ errors |= (flags & RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC) << 2;
+ ol_flags |= rxr->ol_flags_err_table[errors];
+ }
if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) {
mbuf->hash.rss = rte_le_to_cpu_32(rxcmp->rss_hash);
cpr->cp_ring_struct->ring_mask,
cpr->valid);
- if (cmp_type == RX_TPA_START_CMPL_TYPE_RX_TPA_START) {
+ if (cmp_type == RX_TPA_START_CMPL_TYPE_RX_TPA_START ||
+ cmp_type == RX_TPA_START_V2_CMPL_TYPE_RX_TPA_START_V2) {
bnxt_tpa_start(rxq, (struct rx_tpa_start_cmpl *)rxcmp,
(struct rx_tpa_start_cmpl_hi *)rxcmp1);
rc = -EINVAL; /* Continue w/o new mbuf */
return -EBUSY;
*rx_pkt = mbuf;
goto next_rx;
- } else if (cmp_type != 0x11) {
+ } else if ((cmp_type != CMPL_BASE_TYPE_RX_L2) &&
+ (cmp_type != CMPL_BASE_TYPE_RX_L2_V2)) {
rc = -EINVAL;
goto next_rx;
}
mbuf->data_len = mbuf->pkt_len;
mbuf->port = rxq->port_id;
- bnxt_set_ol_flags(rxcmp, rxcmp1, mbuf);
-
#ifdef RTE_LIBRTE_IEEE1588
if (unlikely((rte_le_to_cpu_16(rxcmp->flags_type) &
RX_PKT_CMPL_FLAGS_MASK) ==
}
#endif
+ if (cmp_type == CMPL_BASE_TYPE_RX_L2_V2) {
+ bnxt_parse_csum_v2(mbuf, rxcmp1);
+ bnxt_parse_pkt_type_v2(mbuf, rxcmp, rxcmp1);
+ bnxt_rx_vlan_v2(mbuf, rxcmp, rxcmp1);
+ /* TODO Add support for cfa_code parsing */
+ goto reuse_rx_mbuf;
+ }
+
+ bnxt_set_ol_flags(rxr, rxcmp, rxcmp1, mbuf);
+
+ mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1);
+
if (BNXT_TRUFLOW_EN(bp))
mark_id = bnxt_ulp_set_mark_in_mbuf(rxq->bp, rxcmp1, mbuf,
&vfr_flag);
else
bnxt_set_mark_in_mbuf(rxq->bp, rxcmp1, mbuf);
+reuse_rx_mbuf:
if (agg_buf)
bnxt_rx_pages(rxq, mbuf, &tmp_raw_cons, agg_buf, NULL);
- mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1);
-
#ifdef BNXT_DEBUG
if (rxcmp1->errors_v2 & RX_CMP_L2_ERRORS) {
/* Re-install the mbuf back to the rx ring */
uint16_t rx_raw_prod = rxr->rx_raw_prod;
uint16_t ag_raw_prod = rxr->ag_raw_prod;
uint32_t raw_cons = cpr->cp_raw_cons;
+ bool alloc_failed = false;
uint32_t cons;
int nb_rx_pkts = 0;
int nb_rep_rx_pkts = 0;
cpr->cp_ring_struct->ring_mask,
cpr->valid);
- /* TODO: Avoid magic numbers... */
- if ((CMP_TYPE(rxcmp) & 0x30) == 0x10) {
+ if ((CMP_TYPE(rxcmp) >= CMPL_BASE_TYPE_RX_TPA_START_V2) &&
+ (CMP_TYPE(rxcmp) <= RX_TPA_V2_ABUF_CMPL_TYPE_RX_TPA_AGG)) {
rc = bnxt_rx_pkt(&rx_pkts[nb_rx_pkts], rxq, &raw_cons);
- if (likely(!rc) || rc == -ENOMEM)
+ if (!rc)
nb_rx_pkts++;
- if (rc == -EBUSY) /* partial completion */
+ else if (rc == -EBUSY) /* partial completion */
break;
- if (rc == -ENODEV) /* completion for representor */
+ else if (rc == -ENODEV) /* completion for representor */
nb_rep_rx_pkts++;
+ else if (rc == -ENOMEM) {
+ nb_rx_pkts++;
+ alloc_failed = true;
+ }
} else if (!BNXT_NUM_ASYNC_CPR(rxq->bp)) {
evt =
bnxt_event_hwrm_resp_handler(rxq->bp,
goto done;
}
- rte_compiler_barrier();
+ /* Ring the completion queue doorbell. */
+ bnxt_db_cq(cpr);
+
+ /* Ring the receive descriptor doorbell. */
if (rx_raw_prod != rxr->rx_raw_prod)
bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod);
- rte_compiler_barrier();
/* Ring the AGG ring DB */
if (ag_raw_prod != rxr->ag_raw_prod)
bnxt_db_write(&rxr->ag_db, rxr->ag_raw_prod);
- bnxt_db_cq(cpr);
-
/* Attempt to alloc Rx buf in case of a previous allocation failure. */
- if (rc == -ENOMEM) {
- int i = RING_NEXT(rx_raw_prod);
- int cnt = nb_rx_pkts;
+ if (alloc_failed) {
+ uint16_t cnt;
- for (; nb_rx_pkts; i = RING_NEXT(i), cnt--) {
+ rx_raw_prod = RING_NEXT(rx_raw_prod);
+ for (cnt = 0; cnt < nb_rx_pkts + nb_rep_rx_pkts; cnt++) {
struct rte_mbuf **rx_buf;
- uint16_t rx_raw_prod = RING_IDX(rxr->rx_ring_struct, i);
+ uint16_t ndx;
- rx_buf = &rxr->rx_buf_ring[rx_raw_prod];
+ ndx = RING_IDX(rxr->rx_ring_struct, rx_raw_prod + cnt);
+ rx_buf = &rxr->rx_buf_ring[ndx];
/* Buffer already allocated for this index. */
if (*rx_buf != NULL && *rx_buf != &rxq->fake_mbuf)
continue;
/* This slot is empty. Alloc buffer for Rx */
- if (!bnxt_alloc_rx_data(rxq, rxr, i)) {
- rxr->rx_raw_prod = i;
+ if (!bnxt_alloc_rx_data(rxq, rxr, rx_raw_prod + cnt)) {
+ rxr->rx_raw_prod = rx_raw_prod + cnt;
bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod);
} else {
PMD_DRV_LOG(ERR, "Alloc mbuf failed\n");
/* Initialize packet type table. */
bnxt_init_ptype_table();
- /* Initialize offload flags parsing table. */
- bnxt_init_ol_flags_tables();
-
size = rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM;
size = RTE_MIN(BNXT_MAX_PKT_LEN, size);
ring = rxr->rx_ring_struct;
bnxt_init_rxbds(ring, type, size);
+ /* Initialize offload flags parsing table. */
+ bnxt_init_ol_flags_tables(rxq);
+
raw_prod = rxr->rx_raw_prod;
for (i = 0; i < ring->ring_size; i++) {
if (unlikely(!rxr->rx_buf_ring[i])) {
git.droids-corp.org / dpdk.git / blobdiff