X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fiavf%2Fiavf_rxtx.c;h=bb3d7ea4348f8478ade2a4dd2c4b33719ab76056;hb=149280731bf383e369262aa156b1a3d7e6fbbe6f;hp=59623ac8209465bbc3a20ebb87272d5961c86ee0;hpb=7be78d027918dbc846e502780faf94d5acdf5f75;p=dpdk.git diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c index 59623ac820..bb3d7ea434 100644 --- a/drivers/net/iavf/iavf_rxtx.c +++ b/drivers/net/iavf/iavf_rxtx.c @@ -230,8 +230,7 @@ reset_rx_queue(struct iavf_rx_queue *rxq) rxq->rx_tail = 0; rxq->nb_rx_hold = 0; - if (rxq->pkt_first_seg != NULL) - rte_pktmbuf_free(rxq->pkt_first_seg); + rte_pktmbuf_free(rxq->pkt_first_seg); rxq->pkt_first_seg = NULL; rxq->pkt_last_seg = NULL; @@ -363,12 +362,24 @@ release_txq_mbufs(struct iavf_tx_queue *txq) } } -static const struct iavf_rxq_ops def_rxq_ops = { - .release_mbufs = release_rxq_mbufs, +static const +struct iavf_rxq_ops iavf_rxq_release_mbufs_ops[] = { + [IAVF_REL_MBUFS_DEFAULT].release_mbufs = release_rxq_mbufs, +#ifdef RTE_ARCH_X86 + [IAVF_REL_MBUFS_SSE_VEC].release_mbufs = iavf_rx_queue_release_mbufs_sse, +#endif }; -static const struct iavf_txq_ops def_txq_ops = { - .release_mbufs = release_txq_mbufs, +static const +struct iavf_txq_ops iavf_txq_release_mbufs_ops[] = { + [IAVF_REL_MBUFS_DEFAULT].release_mbufs = release_txq_mbufs, +#ifdef RTE_ARCH_X86 + [IAVF_REL_MBUFS_SSE_VEC].release_mbufs = iavf_tx_queue_release_mbufs_sse, +#ifdef CC_AVX512_SUPPORT + [IAVF_REL_MBUFS_AVX512_VEC].release_mbufs = iavf_tx_queue_release_mbufs_avx512, +#endif +#endif + }; static inline void @@ -476,54 +487,60 @@ iavf_rxd_to_pkt_fields_by_comms_aux_v2(struct iavf_rx_queue *rxq, #endif } +static const +iavf_rxd_to_pkt_fields_t rxd_to_pkt_fields_ops[IAVF_RXDID_LAST + 1] = { + [IAVF_RXDID_LEGACY_0] = iavf_rxd_to_pkt_fields_by_comms_ovs, + [IAVF_RXDID_LEGACY_1] = iavf_rxd_to_pkt_fields_by_comms_ovs, + [IAVF_RXDID_COMMS_AUX_VLAN] = iavf_rxd_to_pkt_fields_by_comms_aux_v1, + [IAVF_RXDID_COMMS_AUX_IPV4] = iavf_rxd_to_pkt_fields_by_comms_aux_v1, + [IAVF_RXDID_COMMS_AUX_IPV6] = iavf_rxd_to_pkt_fields_by_comms_aux_v1, + [IAVF_RXDID_COMMS_AUX_IPV6_FLOW] = + iavf_rxd_to_pkt_fields_by_comms_aux_v1, + [IAVF_RXDID_COMMS_AUX_TCP] = iavf_rxd_to_pkt_fields_by_comms_aux_v1, + [IAVF_RXDID_COMMS_AUX_IP_OFFSET] = + iavf_rxd_to_pkt_fields_by_comms_aux_v2, + [IAVF_RXDID_COMMS_IPSEC_CRYPTO] = + iavf_rxd_to_pkt_fields_by_comms_aux_v2, + [IAVF_RXDID_COMMS_OVS_1] = iavf_rxd_to_pkt_fields_by_comms_ovs, +}; + static void iavf_select_rxd_to_pkt_fields_handler(struct iavf_rx_queue *rxq, uint32_t rxdid) { + rxq->rxdid = rxdid; + switch (rxdid) { case IAVF_RXDID_COMMS_AUX_VLAN: rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_vlan_mask; - rxq->rxd_to_pkt_fields = - iavf_rxd_to_pkt_fields_by_comms_aux_v1; break; case IAVF_RXDID_COMMS_AUX_IPV4: rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_ipv4_mask; - rxq->rxd_to_pkt_fields = - iavf_rxd_to_pkt_fields_by_comms_aux_v1; break; case IAVF_RXDID_COMMS_AUX_IPV6: rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_ipv6_mask; - rxq->rxd_to_pkt_fields = - iavf_rxd_to_pkt_fields_by_comms_aux_v1; break; case IAVF_RXDID_COMMS_AUX_IPV6_FLOW: rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_ipv6_flow_mask; - rxq->rxd_to_pkt_fields = - iavf_rxd_to_pkt_fields_by_comms_aux_v1; break; case IAVF_RXDID_COMMS_AUX_TCP: rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_tcp_mask; - rxq->rxd_to_pkt_fields = - iavf_rxd_to_pkt_fields_by_comms_aux_v1; break; case IAVF_RXDID_COMMS_AUX_IP_OFFSET: rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_ip_offset_mask; - rxq->rxd_to_pkt_fields = - iavf_rxd_to_pkt_fields_by_comms_aux_v2; break; case IAVF_RXDID_COMMS_IPSEC_CRYPTO: rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_ipsec_crypto_said_mask; - rxq->rxd_to_pkt_fields = - iavf_rxd_to_pkt_fields_by_comms_aux_v2; break; case IAVF_RXDID_COMMS_OVS_1: - rxq->rxd_to_pkt_fields = iavf_rxd_to_pkt_fields_by_comms_ovs; + case IAVF_RXDID_LEGACY_0: + case IAVF_RXDID_LEGACY_1: break; default: /* update this according to the RXDID for FLEX_DESC_NONE */ - rxq->rxd_to_pkt_fields = iavf_rxd_to_pkt_fields_by_comms_ovs; + rxq->rxdid = IAVF_RXDID_COMMS_OVS_1; break; } @@ -553,6 +570,9 @@ iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, PMD_INIT_FUNC_TRACE(); + if (ad->closed) + return -EIO; + offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; if (nb_desc % IAVF_ALIGN_RING_DESC != 0 || @@ -673,7 +693,7 @@ iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, rxq->q_set = true; dev->data->rx_queues[queue_idx] = rxq; rxq->qrx_tail = hw->hw_addr + IAVF_QRX_TAIL1(rxq->queue_id); - rxq->ops = &def_rxq_ops; + rxq->rel_mbufs_type = IAVF_REL_MBUFS_DEFAULT; if (check_rx_bulk_allow(rxq) == true) { PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " @@ -714,6 +734,9 @@ iavf_dev_tx_queue_setup(struct rte_eth_dev *dev, PMD_INIT_FUNC_TRACE(); + if (adapter->closed) + return -EIO; + offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; if (nb_desc % IAVF_ALIGN_RING_DESC != 0 || @@ -810,7 +833,7 @@ iavf_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->q_set = true; dev->data->tx_queues[queue_idx] = txq; txq->qtx_tail = hw->hw_addr + IAVF_QTX_TAIL1(queue_idx); - txq->ops = &def_txq_ops; + txq->rel_mbufs_type = IAVF_REL_MBUFS_DEFAULT; if (check_tx_vec_allow(txq) == false) { struct iavf_adapter *ad = @@ -881,6 +904,15 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) RTE_ETH_QUEUE_STATE_STARTED; } + if (dev->data->dev_conf.rxmode.offloads & + RTE_ETH_RX_OFFLOAD_TIMESTAMP) { + if (iavf_get_phc_time(rxq)) { + PMD_DRV_LOG(ERR, "get physical time failed"); + return err; + } + rxq->hw_time_update = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); + } + return err; } @@ -942,7 +974,7 @@ iavf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) } rxq = dev->data->rx_queues[rx_queue_id]; - rxq->ops->release_mbufs(rxq); + iavf_rxq_release_mbufs_ops[rxq->rel_mbufs_type].release_mbufs(rxq); reset_rx_queue(rxq); dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; @@ -970,7 +1002,7 @@ iavf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) } txq = dev->data->tx_queues[tx_queue_id]; - txq->ops->release_mbufs(txq); + iavf_txq_release_mbufs_ops[txq->rel_mbufs_type].release_mbufs(txq); reset_tx_queue(txq); dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; @@ -985,7 +1017,7 @@ iavf_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) if (!q) return; - q->ops->release_mbufs(q); + iavf_rxq_release_mbufs_ops[q->rel_mbufs_type].release_mbufs(q); rte_free(q->sw_ring); rte_memzone_free(q->mz); rte_free(q); @@ -999,7 +1031,7 @@ iavf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) if (!q) return; - q->ops->release_mbufs(q); + iavf_txq_release_mbufs_ops[q->rel_mbufs_type].release_mbufs(q); rte_free(q->sw_ring); rte_memzone_free(q->mz); rte_free(q); @@ -1033,7 +1065,7 @@ iavf_stop_queues(struct rte_eth_dev *dev) txq = dev->data->tx_queues[i]; if (!txq) continue; - txq->ops->release_mbufs(txq); + iavf_txq_release_mbufs_ops[txq->rel_mbufs_type].release_mbufs(txq); reset_tx_queue(txq); dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; } @@ -1041,7 +1073,7 @@ iavf_stop_queues(struct rte_eth_dev *dev) rxq = dev->data->rx_queues[i]; if (!rxq) continue; - rxq->ops->release_mbufs(rxq); + iavf_rxq_release_mbufs_ops[rxq->rel_mbufs_type].release_mbufs(rxq); reset_rx_queue(rxq); dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; } @@ -1417,6 +1449,7 @@ iavf_recv_pkts_flex_rxd(void *rx_queue, uint64_t dma_addr; uint64_t pkt_flags; const uint32_t *ptype_tbl; + uint64_t ts_ns; nb_rx = 0; nb_hold = 0; @@ -1425,6 +1458,16 @@ iavf_recv_pkts_flex_rxd(void *rx_queue, rx_ring = rxq->rx_ring; ptype_tbl = rxq->vsi->adapter->ptype_tbl; + if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) { + uint64_t sw_cur_time = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); + + if (sw_cur_time - rxq->hw_time_update > 4) { + if (iavf_get_phc_time(rxq)) + PMD_DRV_LOG(ERR, "get physical time failed"); + rxq->hw_time_update = sw_cur_time; + } + } + while (nb_rx < nb_pkts) { rxdp = (volatile union iavf_rx_flex_desc *)&rx_ring[rx_id]; rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0); @@ -1484,8 +1527,22 @@ iavf_recv_pkts_flex_rxd(void *rx_queue, iavf_flex_rxd_to_vlan_tci(rxm, &rxd); iavf_flex_rxd_to_ipsec_crypto_status(rxm, &rxd, &rxq->stats.ipsec_crypto); - rxq->rxd_to_pkt_fields(rxq, rxm, &rxd); + rxd_to_pkt_fields_ops[rxq->rxdid](rxq, rxm, &rxd); pkt_flags = iavf_flex_rxd_error_to_pkt_flags(rx_stat_err0); + + if (iavf_timestamp_dynflag > 0) { + ts_ns = iavf_tstamp_convert_32b_64b(rxq->phc_time, + rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high)); + + rxq->phc_time = ts_ns; + rxq->hw_time_update = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); + + *RTE_MBUF_DYNFIELD(rxm, + iavf_timestamp_dynfield_offset, + rte_mbuf_timestamp_t *) = ts_ns; + rxm->ol_flags |= iavf_timestamp_dynflag; + } + rxm->ol_flags |= pkt_flags; rx_pkts[nb_rx++] = rxm; @@ -1514,11 +1571,22 @@ iavf_recv_scattered_pkts_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t rx_stat_err0; uint64_t dma_addr; uint64_t pkt_flags; + uint64_t ts_ns; volatile union iavf_rx_desc *rx_ring = rxq->rx_ring; volatile union iavf_rx_flex_desc *rxdp; const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; + if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) { + uint64_t sw_cur_time = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); + + if (sw_cur_time - rxq->hw_time_update > 4) { + if (iavf_get_phc_time(rxq)) + PMD_DRV_LOG(ERR, "get physical time failed"); + rxq->hw_time_update = sw_cur_time; + } + } + while (nb_rx < nb_pkts) { rxdp = (volatile union iavf_rx_flex_desc *)&rx_ring[rx_id]; rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0); @@ -1628,9 +1696,22 @@ iavf_recv_scattered_pkts_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts, iavf_flex_rxd_to_vlan_tci(first_seg, &rxd); iavf_flex_rxd_to_ipsec_crypto_status(first_seg, &rxd, &rxq->stats.ipsec_crypto); - rxq->rxd_to_pkt_fields(rxq, first_seg, &rxd); + rxd_to_pkt_fields_ops[rxq->rxdid](rxq, first_seg, &rxd); pkt_flags = iavf_flex_rxd_error_to_pkt_flags(rx_stat_err0); + if (iavf_timestamp_dynflag > 0) { + ts_ns = iavf_tstamp_convert_32b_64b(rxq->phc_time, + rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high)); + + rxq->phc_time = ts_ns; + rxq->hw_time_update = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); + + *RTE_MBUF_DYNFIELD(first_seg, + iavf_timestamp_dynfield_offset, + rte_mbuf_timestamp_t *) = ts_ns; + first_seg->ol_flags |= iavf_timestamp_dynflag; + } + first_seg->ol_flags |= pkt_flags; /* Prefetch data of first segment, if configured to do so. */ @@ -1812,17 +1893,21 @@ iavf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, #define IAVF_LOOK_AHEAD 8 static inline int -iavf_rx_scan_hw_ring_flex_rxd(struct iavf_rx_queue *rxq) +iavf_rx_scan_hw_ring_flex_rxd(struct iavf_rx_queue *rxq, + struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) { volatile union iavf_rx_flex_desc *rxdp; struct rte_mbuf **rxep; struct rte_mbuf *mb; uint16_t stat_err0; uint16_t pkt_len; - int32_t s[IAVF_LOOK_AHEAD], nb_dd; + int32_t s[IAVF_LOOK_AHEAD], var, nb_dd; int32_t i, j, nb_rx = 0; + int32_t nb_staged = 0; uint64_t pkt_flags; const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; + uint64_t ts_ns; rxdp = (volatile union iavf_rx_flex_desc *)&rxq->rx_ring[rxq->rx_tail]; rxep = &rxq->sw_ring[rxq->rx_tail]; @@ -1833,6 +1918,16 @@ iavf_rx_scan_hw_ring_flex_rxd(struct iavf_rx_queue *rxq) if (!(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S))) return 0; + if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) { + uint64_t sw_cur_time = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); + + if (sw_cur_time - rxq->hw_time_update > 4) { + if (iavf_get_phc_time(rxq)) + PMD_DRV_LOG(ERR, "get physical time failed"); + rxq->hw_time_update = sw_cur_time; + } + } + /* Scan LOOK_AHEAD descriptors at a time to determine which * descriptors reference packets that are ready to be received. */ @@ -1842,13 +1937,30 @@ iavf_rx_scan_hw_ring_flex_rxd(struct iavf_rx_queue *rxq) for (j = IAVF_LOOK_AHEAD - 1; j >= 0; j--) s[j] = rte_le_to_cpu_16(rxdp[j].wb.status_error0); - rte_smp_rmb(); - - /* Compute how many status bits were set */ - for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++) - nb_dd += s[j] & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S); - - nb_rx += nb_dd; + /* This barrier is to order loads of different words in the descriptor */ + rte_atomic_thread_fence(__ATOMIC_ACQUIRE); + + /* Compute how many contiguous DD bits were set */ + for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++) { + var = s[j] & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S); +#ifdef RTE_ARCH_ARM + /* For Arm platforms, count only contiguous descriptors + * whose DD bit is set to 1. On Arm platforms, reads of + * descriptors can be reordered. Since the CPU may + * be reading the descriptors as the NIC updates them + * in memory, it is possbile that the DD bit for a + * descriptor earlier in the queue is read as not set + * while the DD bit for a descriptor later in the queue + * is read as set. + */ + if (var) + nb_dd += 1; + else + break; +#else + nb_dd += var; +#endif + } /* Translate descriptor info to mbuf parameters */ for (j = 0; j < nb_dd; j++) { @@ -1868,29 +1980,53 @@ iavf_rx_scan_hw_ring_flex_rxd(struct iavf_rx_queue *rxq) iavf_flex_rxd_to_vlan_tci(mb, &rxdp[j]); iavf_flex_rxd_to_ipsec_crypto_status(mb, &rxdp[j], &rxq->stats.ipsec_crypto); - rxq->rxd_to_pkt_fields(rxq, mb, &rxdp[j]); + rxd_to_pkt_fields_ops[rxq->rxdid](rxq, mb, &rxdp[j]); stat_err0 = rte_le_to_cpu_16(rxdp[j].wb.status_error0); pkt_flags = iavf_flex_rxd_error_to_pkt_flags(stat_err0); + if (iavf_timestamp_dynflag > 0) { + ts_ns = iavf_tstamp_convert_32b_64b(rxq->phc_time, + rte_le_to_cpu_32(rxdp[j].wb.flex_ts.ts_high)); + + rxq->phc_time = ts_ns; + rxq->hw_time_update = rte_get_timer_cycles() / + (rte_get_timer_hz() / 1000); + + *RTE_MBUF_DYNFIELD(mb, + iavf_timestamp_dynfield_offset, + rte_mbuf_timestamp_t *) = ts_ns; + mb->ol_flags |= iavf_timestamp_dynflag; + } + mb->ol_flags |= pkt_flags; - } - for (j = 0; j < IAVF_LOOK_AHEAD; j++) - rxq->rx_stage[i + j] = rxep[j]; + /* Put up to nb_pkts directly into buffers */ + if ((i + j) < nb_pkts) { + rx_pkts[i + j] = rxep[j]; + nb_rx++; + } else { + /* Stage excess pkts received */ + rxq->rx_stage[nb_staged] = rxep[j]; + nb_staged++; + } + } if (nb_dd != IAVF_LOOK_AHEAD) break; } + /* Update rxq->rx_nb_avail to reflect number of staged pkts */ + rxq->rx_nb_avail = nb_staged; + /* Clear software ring entries */ - for (i = 0; i < nb_rx; i++) + for (i = 0; i < (nb_rx + nb_staged); i++) rxq->sw_ring[rxq->rx_tail + i] = NULL; return nb_rx; } static inline int -iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq) +iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { volatile union iavf_rx_desc *rxdp; struct rte_mbuf **rxep; @@ -1898,8 +2034,9 @@ iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq) uint16_t pkt_len; uint64_t qword1; uint32_t rx_status; - int32_t s[IAVF_LOOK_AHEAD], nb_dd; + int32_t s[IAVF_LOOK_AHEAD], var, nb_dd; int32_t i, j, nb_rx = 0; + int32_t nb_staged = 0; uint64_t pkt_flags; const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; @@ -1927,13 +2064,30 @@ iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq) IAVF_RXD_QW1_STATUS_SHIFT; } - rte_smp_rmb(); - - /* Compute how many status bits were set */ - for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++) - nb_dd += s[j] & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT); - - nb_rx += nb_dd; + /* This barrier is to order loads of different words in the descriptor */ + rte_atomic_thread_fence(__ATOMIC_ACQUIRE); + + /* Compute how many contiguous DD bits were set */ + for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++) { + var = s[j] & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT); +#ifdef RTE_ARCH_ARM + /* For Arm platforms, count only contiguous descriptors + * whose DD bit is set to 1. On Arm platforms, reads of + * descriptors can be reordered. Since the CPU may + * be reading the descriptors as the NIC updates them + * in memory, it is possbile that the DD bit for a + * descriptor earlier in the queue is read as not set + * while the DD bit for a descriptor later in the queue + * is read as set. + */ + if (var) + nb_dd += 1; + else + break; +#else + nb_dd += var; +#endif + } /* Translate descriptor info to mbuf parameters */ for (j = 0; j < nb_dd; j++) { @@ -1963,17 +2117,26 @@ iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq) pkt_flags |= iavf_rxd_build_fdir(&rxdp[j], mb); mb->ol_flags |= pkt_flags; - } - for (j = 0; j < IAVF_LOOK_AHEAD; j++) - rxq->rx_stage[i + j] = rxep[j]; + /* Put up to nb_pkts directly into buffers */ + if ((i + j) < nb_pkts) { + rx_pkts[i + j] = rxep[j]; + nb_rx++; + } else { /* Stage excess pkts received */ + rxq->rx_stage[nb_staged] = rxep[j]; + nb_staged++; + } + } if (nb_dd != IAVF_LOOK_AHEAD) break; } + /* Update rxq->rx_nb_avail to reflect number of staged pkts */ + rxq->rx_nb_avail = nb_staged; + /* Clear software ring entries */ - for (i = 0; i < nb_rx; i++) + for (i = 0; i < (nb_rx + nb_staged); i++) rxq->sw_ring[rxq->rx_tail + i] = NULL; return nb_rx; @@ -2061,23 +2224,31 @@ rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) return iavf_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); if (rxq->rxdid >= IAVF_RXDID_FLEX_NIC && rxq->rxdid <= IAVF_RXDID_LAST) - nb_rx = (uint16_t)iavf_rx_scan_hw_ring_flex_rxd(rxq); + nb_rx = (uint16_t)iavf_rx_scan_hw_ring_flex_rxd(rxq, rx_pkts, nb_pkts); else - nb_rx = (uint16_t)iavf_rx_scan_hw_ring(rxq); + nb_rx = (uint16_t)iavf_rx_scan_hw_ring(rxq, rx_pkts, nb_pkts); + rxq->rx_next_avail = 0; - rxq->rx_nb_avail = nb_rx; - rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); + rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx + rxq->rx_nb_avail); if (rxq->rx_tail > rxq->rx_free_trigger) { if (iavf_rx_alloc_bufs(rxq) != 0) { - uint16_t i, j; + uint16_t i, j, nb_staged; /* TODO: count rx_mbuf_alloc_failed here */ + nb_staged = rxq->rx_nb_avail; rxq->rx_nb_avail = 0; - rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); - for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++) + + rxq->rx_tail = (uint16_t)(rxq->rx_tail - (nb_rx + nb_staged)); + for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++) { + rxq->sw_ring[j] = rx_pkts[i]; + rx_pkts[i] = NULL; + } + for (i = 0, j = rxq->rx_tail + nb_rx; i < nb_staged; i++, j++) { rxq->sw_ring[j] = rxq->rx_stage[i]; + rx_pkts[i] = NULL; + } return 0; } @@ -2090,10 +2261,7 @@ rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rxq->port_id, rxq->queue_id, rxq->rx_tail, nb_rx); - if (rxq->rx_nb_avail) - return iavf_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); - - return 0; + return nb_rx; } static uint16_t @@ -2439,6 +2607,14 @@ iavf_fill_data_desc(volatile struct iavf_tx_desc *desc, if (m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) hdrlen += ipseclen; bufsz = hdrlen + tlen; + } else if ((m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) && + (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | + RTE_MBUF_F_TX_UDP_SEG))) { + hdrlen += m->outer_l3_len + m->l3_len + ipseclen; + if (m->ol_flags & RTE_MBUF_F_TX_L4_MASK) + hdrlen += m->l4_len; + bufsz = hdrlen + tlen; + } else { bufsz = m->data_len; } @@ -2694,6 +2870,10 @@ iavf_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, struct iavf_tx_queue *txq = tx_queue; struct rte_eth_dev *dev = &rte_eth_devices[txq->port_id]; struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct iavf_adapter *adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); + + if (adapter->closed) + return 0; for (i = 0; i < nb_pkts; i++) { m = tx_pkts[i]; @@ -2750,14 +2930,27 @@ iavf_set_rx_function(struct rte_eth_dev *dev) struct iavf_adapter *adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + int i; + struct iavf_rx_queue *rxq; + bool use_flex = true; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + if (rxq->rxdid <= IAVF_RXDID_LEGACY_1) { + PMD_DRV_LOG(NOTICE, "request RXDID[%d] in Queue[%d] is legacy, " + "set rx_pkt_burst as legacy for all queues", rxq->rxdid, i); + use_flex = false; + } else if (!(vf->supported_rxdid & BIT(rxq->rxdid))) { + PMD_DRV_LOG(NOTICE, "request RXDID[%d] in Queue[%d] is not supported, " + "set rx_pkt_burst as legacy for all queues", rxq->rxdid, i); + use_flex = false; + } + } #ifdef RTE_ARCH_X86 - struct iavf_rx_queue *rxq; - int i; int check_ret; bool use_avx2 = false; bool use_avx512 = false; - bool use_flex = false; check_ret = iavf_rx_vec_dev_check(dev); if (check_ret >= 0 && @@ -2774,10 +2967,6 @@ iavf_set_rx_function(struct rte_eth_dev *dev) use_avx512 = true; #endif - if (vf->vf_res->vf_cap_flags & - VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) - use_flex = true; - for (i = 0; i < dev->data->nb_rx_queues; i++) { rxq = dev->data->rx_queues[i]; (void)iavf_rxq_vec_setup(rxq); @@ -2881,7 +3070,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) if (dev->data->scattered_rx) { PMD_DRV_LOG(DEBUG, "Using a Scattered Rx callback (port=%d).", dev->data->port_id); - if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) + if (use_flex) dev->rx_pkt_burst = iavf_recv_scattered_pkts_flex_rxd; else dev->rx_pkt_burst = iavf_recv_scattered_pkts; @@ -2892,7 +3081,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev) } else { PMD_DRV_LOG(DEBUG, "Using Basic Rx callback (port=%d).", dev->data->port_id); - if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) + if (use_flex) dev->rx_pkt_burst = iavf_recv_pkts_flex_rxd; else dev->rx_pkt_burst = iavf_recv_pkts;