From 5e64c8120c82c0c9d4de42a42f6d38922c3b5740 Mon Sep 17 00:00:00 2001 From: Pavan Nikhilesh Date: Wed, 18 Jul 2018 20:35:01 +0530 Subject: [PATCH] net/thunderx: enable Rx checksum offload Add L3/L4 Rx checksum offload and update capabilities. Signed-off-by: Pavan Nikhilesh Acked-by: Jerin Jacob --- drivers/net/thunderx/nicvf_ethdev.c | 33 ++++++++----- drivers/net/thunderx/nicvf_ethdev.h | 1 + drivers/net/thunderx/nicvf_rxtx.c | 73 +++++++++++++++++++++++++---- drivers/net/thunderx/nicvf_rxtx.h | 15 ++++-- drivers/net/thunderx/nicvf_struct.h | 27 ++++++----- 5 files changed, 113 insertions(+), 36 deletions(-) diff --git a/drivers/net/thunderx/nicvf_ethdev.c b/drivers/net/thunderx/nicvf_ethdev.c index 5e15a88a53..eba05fdf04 100644 --- a/drivers/net/thunderx/nicvf_ethdev.c +++ b/drivers/net/thunderx/nicvf_ethdev.c @@ -355,11 +355,9 @@ nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev) } memcpy((char *)ptypes + copied, &ptypes_end, sizeof(ptypes_end)); - if (dev->rx_pkt_burst == nicvf_recv_pkts || - dev->rx_pkt_burst == nicvf_recv_pkts_multiseg) - return ptypes; - return NULL; + /* All Ptypes are supported in all Rx functions. */ + return ptypes; } static void @@ -916,13 +914,18 @@ nicvf_set_tx_function(struct rte_eth_dev *dev) static void nicvf_set_rx_function(struct rte_eth_dev *dev) { - if (dev->data->scattered_rx) { - PMD_DRV_LOG(DEBUG, "Using multi-segment rx callback"); - dev->rx_pkt_burst = nicvf_recv_pkts_multiseg; - } else { - PMD_DRV_LOG(DEBUG, "Using single-segment rx callback"); - dev->rx_pkt_burst = nicvf_recv_pkts; - } + struct nicvf *nic = nicvf_pmd_priv(dev); + + const eth_rx_burst_t rx_burst_func[2][2] = { + /* [NORMAL/SCATTER] [NO_CKSUM/CKSUM] */ + [0][0] = nicvf_recv_pkts_no_offload, + [0][1] = nicvf_recv_pkts_cksum, + [1][0] = nicvf_recv_pkts_multiseg_no_offload, + [1][1] = nicvf_recv_pkts_multiseg_cksum, + }; + + dev->rx_pkt_burst = + rx_burst_func[dev->data->scattered_rx][nic->offload_cksum]; } static int @@ -1243,6 +1246,9 @@ nicvf_rxq_mbuf_setup(struct nicvf_rxq *rxq) offsetof(struct rte_mbuf, data_off) != 4); RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) - offsetof(struct rte_mbuf, data_off) != 6); + RTE_BUILD_BUG_ON(offsetof(struct nicvf_rxq, rxq_fastpath_data_end) - + offsetof(struct nicvf_rxq, + rxq_fastpath_data_start) > 128); mb_def.nb_segs = 1; mb_def.data_off = RTE_PKTMBUF_HEADROOM + (nic->skip_bytes); mb_def.port = rxq->port_id; @@ -1743,7 +1749,7 @@ nicvf_dev_start(struct rte_eth_dev *dev) return ret; } - /* Configure callbacks based on scatter mode */ + /* Configure callbacks based on offloads */ nicvf_set_tx_function(dev); nicvf_set_rx_function(dev); @@ -1962,6 +1968,9 @@ nicvf_dev_configure(struct rte_eth_dev *dev) } } + if (rxmode->offloads & DEV_RX_OFFLOAD_CHECKSUM) + nic->offload_cksum = 1; + PMD_INIT_LOG(DEBUG, "Configured ethdev port%d hwcap=0x%" PRIx64, dev->data->port_id, nicvf_hw_cap(nic)); diff --git a/drivers/net/thunderx/nicvf_ethdev.h b/drivers/net/thunderx/nicvf_ethdev.h index 9af508803f..ae440fef23 100644 --- a/drivers/net/thunderx/nicvf_ethdev.h +++ b/drivers/net/thunderx/nicvf_ethdev.h @@ -38,6 +38,7 @@ DEV_TX_OFFLOAD_MULTI_SEGS) #define NICVF_RX_OFFLOAD_CAPA ( \ + DEV_RX_OFFLOAD_CHECKSUM | \ DEV_RX_OFFLOAD_VLAN_STRIP | \ DEV_RX_OFFLOAD_CRC_STRIP | \ DEV_RX_OFFLOAD_JUMBO_FRAME | \ diff --git a/drivers/net/thunderx/nicvf_rxtx.c b/drivers/net/thunderx/nicvf_rxtx.c index 6e075e23cb..4980dab79d 100644 --- a/drivers/net/thunderx/nicvf_rxtx.c +++ b/drivers/net/thunderx/nicvf_rxtx.c @@ -331,6 +331,20 @@ nicvf_rx_classify_pkt(cqe_rx_word0_t cqe_rx_w0) return ptype_table[cqe_rx_w0.l3_type][cqe_rx_w0.l4_type]; } +static inline uint64_t __hot +nicvf_set_olflags(const cqe_rx_word0_t cqe_rx_w0) +{ + static const uint64_t flag_table[3] __rte_cache_aligned = { + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD, + PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_UNKNOWN, + PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD, + }; + + const uint8_t idx = (cqe_rx_w0.err_opcode == CQE_RX_ERR_L4_CHK) << 1 | + (cqe_rx_w0.err_opcode == CQE_RX_ERR_IP_CHK); + return flag_table[idx]; +} + static inline int __hot nicvf_fill_rbdr(struct nicvf_rxq *rxq, int to_fill) { @@ -389,11 +403,13 @@ nicvf_rx_offload(cqe_rx_word0_t cqe_rx_w0, cqe_rx_word2_t cqe_rx_w2, if (likely(cqe_rx_w0.rss_alg)) { pkt->hash.rss = cqe_rx_w2.rss_tag; pkt->ol_flags |= PKT_RX_RSS_HASH; + } } -uint16_t __hot -nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) +static __rte_always_inline uint16_t +nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, + const uint32_t flag) { uint32_t i, to_process; struct cqe_rx_t *cqe_rx; @@ -424,7 +440,11 @@ nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rb0_ptr = *((uint64_t *)cqe_rx + rbptr_offset); pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt (rb0_ptr - cqe_rx_w1.align_pad, mbuf_phys_off); - pkt->ol_flags = 0; + + if (flag & NICVF_RX_OFFLOAD_NONE) + pkt->ol_flags = 0; + if (flag & NICVF_RX_OFFLOAD_CKSUM) + pkt->ol_flags = nicvf_set_olflags(cqe_rx_w0); pkt->data_len = cqe_rx_w3.rb0_sz; pkt->pkt_len = cqe_rx_w3.rb0_sz; pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0); @@ -449,11 +469,27 @@ nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) return to_process; } -static inline uint16_t __hot +uint16_t __hot +nicvf_recv_pkts_no_offload(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + return nicvf_recv_pkts(rx_queue, rx_pkts, nb_pkts, + NICVF_RX_OFFLOAD_NONE); +} + +uint16_t __hot +nicvf_recv_pkts_cksum(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + return nicvf_recv_pkts(rx_queue, rx_pkts, nb_pkts, + NICVF_RX_OFFLOAD_CKSUM); +} + +static __rte_always_inline uint16_t __hot nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx, uint64_t mbuf_phys_off, struct rte_mbuf **rx_pkt, uint8_t rbptr_offset, - uint64_t mbuf_init) + uint64_t mbuf_init, const uint32_t flag) { struct rte_mbuf *pkt, *seg, *prev; cqe_rx_word0_t cqe_rx_w0; @@ -471,12 +507,15 @@ nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx, pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt (rb_ptr[0] - cqe_rx_w1.align_pad, mbuf_phys_off); - pkt->ol_flags = 0; pkt->pkt_len = cqe_rx_w1.pkt_len; pkt->data_len = rb_sz[nicvf_frag_num(0)]; nicvf_mbuff_init_mseg_update( pkt, mbuf_init, cqe_rx_w1.align_pad, nb_segs); pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0); + if (flag & NICVF_RX_OFFLOAD_NONE) + pkt->ol_flags = 0; + if (flag & NICVF_RX_OFFLOAD_CKSUM) + pkt->ol_flags = nicvf_set_olflags(cqe_rx_w0); nicvf_rx_offload(cqe_rx_w0, cqe_rx_w2, pkt); *rx_pkt = pkt; @@ -495,9 +534,9 @@ nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx, return nb_segs; } -uint16_t __hot +static __rte_always_inline uint16_t __hot nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts) + uint16_t nb_pkts, const uint32_t flag) { union cq_entry_t *cq_entry; struct cqe_rx_t *cqe_rx; @@ -519,7 +558,7 @@ nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts, cq_entry = &desc[cqe_head]; cqe_rx = (struct cqe_rx_t *)cq_entry; nb_segs = nicvf_process_cq_mseg_entry(cqe_rx, mbuf_phys_off, - rx_pkts + i, rbptr_offset, mbuf_init); + rx_pkts + i, rbptr_offset, mbuf_init, flag); buffers_consumed += nb_segs; cqe_head = (cqe_head + 1) & cqe_mask; nicvf_prefetch_store_keep(rx_pkts[i]); @@ -539,6 +578,22 @@ nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts, return to_process; } +uint16_t __hot +nicvf_recv_pkts_multiseg_no_offload(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + return nicvf_recv_pkts_multiseg(rx_queue, rx_pkts, nb_pkts, + NICVF_RX_OFFLOAD_NONE); +} + +uint16_t __hot +nicvf_recv_pkts_multiseg_cksum(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + return nicvf_recv_pkts_multiseg(rx_queue, rx_pkts, nb_pkts, + NICVF_RX_OFFLOAD_CKSUM); +} + uint32_t nicvf_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx) { diff --git a/drivers/net/thunderx/nicvf_rxtx.h b/drivers/net/thunderx/nicvf_rxtx.h index 8bdd582edd..72daffb60a 100644 --- a/drivers/net/thunderx/nicvf_rxtx.h +++ b/drivers/net/thunderx/nicvf_rxtx.h @@ -8,6 +8,9 @@ #include #include +#define NICVF_RX_OFFLOAD_NONE 0x1 +#define NICVF_RX_OFFLOAD_CKSUM 0x2 + #define NICVF_TX_OFFLOAD_MASK (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK) #ifndef __hot @@ -86,9 +89,15 @@ nicvf_mbuff_init_mseg_update(struct rte_mbuf *pkt, const uint64_t mbuf_init, uint32_t nicvf_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx); uint32_t nicvf_dev_rbdr_refill(struct rte_eth_dev *dev, uint16_t queue_idx); -uint16_t nicvf_recv_pkts(void *rxq, struct rte_mbuf **rx_pkts, uint16_t pkts); -uint16_t nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts, - uint16_t nb_pkts); +uint16_t nicvf_recv_pkts_no_offload(void *rxq, struct rte_mbuf **rx_pkts, + uint16_t pkts); +uint16_t nicvf_recv_pkts_cksum(void *rxq, struct rte_mbuf **rx_pkts, + uint16_t pkts); + +uint16_t nicvf_recv_pkts_multiseg_no_offload(void *rx_queue, + struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +uint16_t nicvf_recv_pkts_multiseg_cksum(void *rx_queue, + struct rte_mbuf **rx_pkts, uint16_t nb_pkts); uint16_t nicvf_xmit_pkts(void *txq, struct rte_mbuf **tx_pkts, uint16_t pkts); uint16_t nicvf_xmit_pkts_multiseg(void *txq, struct rte_mbuf **tx_pkts, diff --git a/drivers/net/thunderx/nicvf_struct.h b/drivers/net/thunderx/nicvf_struct.h index cf98f7c1ac..a770e6b54c 100644 --- a/drivers/net/thunderx/nicvf_struct.h +++ b/drivers/net/thunderx/nicvf_struct.h @@ -55,25 +55,27 @@ union mbuf_initializer { }; struct nicvf_rxq { + MARKER rxq_fastpath_data_start; + uint8_t rbptr_offset; + uint16_t rx_free_thresh; + uint32_t head; + uint32_t qlen_mask; + int32_t recv_buffers; + int32_t available_space; uint64_t mbuf_phys_off; uintptr_t cq_status; uintptr_t cq_door; - union mbuf_initializer mbuf_initializer; - nicvf_iova_addr_t phys; - union cq_entry_t *desc; struct nicvf_rbdr *shared_rbdr; - struct nicvf *nic; struct rte_mempool *pool; - uint32_t head; - uint32_t qlen_mask; - int32_t available_space; - int32_t recv_buffers; - uint16_t rx_free_thresh; - uint16_t queue_id; - uint16_t precharge_cnt; + union cq_entry_t *desc; + union mbuf_initializer mbuf_initializer; + MARKER rxq_fastpath_data_end; uint8_t rx_drop_en; + uint16_t precharge_cnt; uint16_t port_id; - uint8_t rbptr_offset; + uint16_t queue_id; + struct nicvf *nic; + nicvf_iova_addr_t phys; } __rte_cache_aligned; struct nicvf { @@ -85,6 +87,7 @@ struct nicvf { bool loopback_supported; bool pf_acked:1; bool pf_nacked:1; + bool offload_cksum:1; uint64_t hwcap; uint8_t link_up; uint8_t duplex; -- 2.20.1