net/thunderx: enable Rx checksum offload

Add L3/L4 Rx checksum offload and update capabilities.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@caviumnetworks.com>
Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

diff --git a/drivers/net/thunderx/nicvf_ethdev.c b/drivers/net/thunderx/nicvf_ethdev.c
index 5e15a88..eba05fd 100644 (file)
--- a/drivers/net/thunderx/nicvf_ethdev.c
+++ b/drivers/net/thunderx/nicvf_ethdev.c
@@ -355,11 +355,9 @@ nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev)
        }
 
        memcpy((char *)ptypes + copied, &ptypes_end, sizeof(ptypes_end));
-       if (dev->rx_pkt_burst == nicvf_recv_pkts ||
-               dev->rx_pkt_burst == nicvf_recv_pkts_multiseg)
-               return ptypes;
 
-       return NULL;
+       /* All Ptypes are supported in all Rx functions. */
+       return ptypes;
 }
 
 static void
@@ -916,13 +914,18 @@ nicvf_set_tx_function(struct rte_eth_dev *dev)
 static void
 nicvf_set_rx_function(struct rte_eth_dev *dev)
 {
-       if (dev->data->scattered_rx) {
-               PMD_DRV_LOG(DEBUG, "Using multi-segment rx callback");
-               dev->rx_pkt_burst = nicvf_recv_pkts_multiseg;
-       } else {
-               PMD_DRV_LOG(DEBUG, "Using single-segment rx callback");
-               dev->rx_pkt_burst = nicvf_recv_pkts;
-       }
+       struct nicvf *nic = nicvf_pmd_priv(dev);
+
+       const eth_rx_burst_t rx_burst_func[2][2] = {
+               /* [NORMAL/SCATTER] [NO_CKSUM/CKSUM] */
+               [0][0] = nicvf_recv_pkts_no_offload,
+               [0][1] = nicvf_recv_pkts_cksum,
+               [1][0] = nicvf_recv_pkts_multiseg_no_offload,
+               [1][1] = nicvf_recv_pkts_multiseg_cksum,
+       };
+
+       dev->rx_pkt_burst =
+               rx_burst_func[dev->data->scattered_rx][nic->offload_cksum];
 }
 
 static int
@@ -1243,6 +1246,9 @@ nicvf_rxq_mbuf_setup(struct nicvf_rxq *rxq)
                                offsetof(struct rte_mbuf, data_off) != 4);
        RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) -
                                offsetof(struct rte_mbuf, data_off) != 6);
+       RTE_BUILD_BUG_ON(offsetof(struct nicvf_rxq, rxq_fastpath_data_end) -
+                               offsetof(struct nicvf_rxq,
+                                       rxq_fastpath_data_start) > 128);
        mb_def.nb_segs = 1;
        mb_def.data_off = RTE_PKTMBUF_HEADROOM + (nic->skip_bytes);
        mb_def.port = rxq->port_id;
@@ -1743,7 +1749,7 @@ nicvf_dev_start(struct rte_eth_dev *dev)
                        return ret;
        }
 
-       /* Configure callbacks based on scatter mode */
+       /* Configure callbacks based on offloads */
        nicvf_set_tx_function(dev);
        nicvf_set_rx_function(dev);
 
@@ -1962,6 +1968,9 @@ nicvf_dev_configure(struct rte_eth_dev *dev)
                }
        }
 
+       if (rxmode->offloads & DEV_RX_OFFLOAD_CHECKSUM)
+               nic->offload_cksum = 1;
+
        PMD_INIT_LOG(DEBUG, "Configured ethdev port%d hwcap=0x%" PRIx64,
                dev->data->port_id, nicvf_hw_cap(nic));
 

diff --git a/drivers/net/thunderx/nicvf_ethdev.h b/drivers/net/thunderx/nicvf_ethdev.h
index 9af5088..ae440fe 100644 (file)
--- a/drivers/net/thunderx/nicvf_ethdev.h
+++ b/drivers/net/thunderx/nicvf_ethdev.h
@@ -38,6 +38,7 @@
        DEV_TX_OFFLOAD_MULTI_SEGS)
 
 #define NICVF_RX_OFFLOAD_CAPA ( \
+       DEV_RX_OFFLOAD_CHECKSUM    | \
        DEV_RX_OFFLOAD_VLAN_STRIP  | \
        DEV_RX_OFFLOAD_CRC_STRIP   | \
        DEV_RX_OFFLOAD_JUMBO_FRAME | \

diff --git a/drivers/net/thunderx/nicvf_rxtx.c b/drivers/net/thunderx/nicvf_rxtx.c
index 6e075e2..4980dab 100644 (file)
--- a/drivers/net/thunderx/nicvf_rxtx.c
+++ b/drivers/net/thunderx/nicvf_rxtx.c
@@ -331,6 +331,20 @@ nicvf_rx_classify_pkt(cqe_rx_word0_t cqe_rx_w0)
        return ptype_table[cqe_rx_w0.l3_type][cqe_rx_w0.l4_type];
 }
 
+static inline uint64_t __hot
+nicvf_set_olflags(const cqe_rx_word0_t cqe_rx_w0)
+{
+       static const uint64_t flag_table[3] __rte_cache_aligned = {
+               PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD,
+               PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_UNKNOWN,
+               PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD,
+       };
+
+       const uint8_t idx = (cqe_rx_w0.err_opcode == CQE_RX_ERR_L4_CHK) << 1 |
+               (cqe_rx_w0.err_opcode == CQE_RX_ERR_IP_CHK);
+       return flag_table[idx];
+}
+
 static inline int __hot
 nicvf_fill_rbdr(struct nicvf_rxq *rxq, int to_fill)
 {
@@ -389,11 +403,13 @@ nicvf_rx_offload(cqe_rx_word0_t cqe_rx_w0, cqe_rx_word2_t cqe_rx_w2,
        if (likely(cqe_rx_w0.rss_alg)) {
                pkt->hash.rss = cqe_rx_w2.rss_tag;
                pkt->ol_flags |= PKT_RX_RSS_HASH;
+
        }
 }
 
-uint16_t __hot
-nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+static __rte_always_inline uint16_t
+nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
+               const uint32_t flag)
 {
        uint32_t i, to_process;
        struct cqe_rx_t *cqe_rx;
@@ -424,7 +440,11 @@ nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
                rb0_ptr = *((uint64_t *)cqe_rx + rbptr_offset);
                pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt
                                (rb0_ptr - cqe_rx_w1.align_pad, mbuf_phys_off);
-               pkt->ol_flags = 0;
+
+               if (flag & NICVF_RX_OFFLOAD_NONE)
+                       pkt->ol_flags = 0;
+               if (flag & NICVF_RX_OFFLOAD_CKSUM)
+                       pkt->ol_flags = nicvf_set_olflags(cqe_rx_w0);
                pkt->data_len = cqe_rx_w3.rb0_sz;
                pkt->pkt_len = cqe_rx_w3.rb0_sz;
                pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0);
@@ -449,11 +469,27 @@ nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
        return to_process;
 }
 
-static inline uint16_t __hot
+uint16_t __hot
+nicvf_recv_pkts_no_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
+               uint16_t nb_pkts)
+{
+       return nicvf_recv_pkts(rx_queue, rx_pkts, nb_pkts,
+                       NICVF_RX_OFFLOAD_NONE);
+}
+
+uint16_t __hot
+nicvf_recv_pkts_cksum(void *rx_queue, struct rte_mbuf **rx_pkts,
+               uint16_t nb_pkts)
+{
+       return nicvf_recv_pkts(rx_queue, rx_pkts, nb_pkts,
+                       NICVF_RX_OFFLOAD_CKSUM);
+}
+
+static __rte_always_inline uint16_t __hot
 nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx,
                        uint64_t mbuf_phys_off,
                        struct rte_mbuf **rx_pkt, uint8_t rbptr_offset,
-                       uint64_t mbuf_init)
+                       uint64_t mbuf_init, const uint32_t flag)
 {
        struct rte_mbuf *pkt, *seg, *prev;
        cqe_rx_word0_t cqe_rx_w0;
@@ -471,12 +507,15 @@ nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx,
        pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt
                        (rb_ptr[0] - cqe_rx_w1.align_pad, mbuf_phys_off);
 
-       pkt->ol_flags = 0;
        pkt->pkt_len = cqe_rx_w1.pkt_len;
        pkt->data_len = rb_sz[nicvf_frag_num(0)];
        nicvf_mbuff_init_mseg_update(
                                pkt, mbuf_init, cqe_rx_w1.align_pad, nb_segs);
        pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0);
+       if (flag & NICVF_RX_OFFLOAD_NONE)
+               pkt->ol_flags = 0;
+       if (flag & NICVF_RX_OFFLOAD_CKSUM)
+               pkt->ol_flags = nicvf_set_olflags(cqe_rx_w0);
        nicvf_rx_offload(cqe_rx_w0, cqe_rx_w2, pkt);
 
        *rx_pkt = pkt;
@@ -495,9 +534,9 @@ nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx,
        return nb_segs;
 }
 
-uint16_t __hot
+static __rte_always_inline uint16_t __hot
 nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts,
-                        uint16_t nb_pkts)
+                        uint16_t nb_pkts, const uint32_t flag)
 {
        union cq_entry_t *cq_entry;
        struct cqe_rx_t *cqe_rx;
@@ -519,7 +558,7 @@ nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts,
                cq_entry = &desc[cqe_head];
                cqe_rx = (struct cqe_rx_t *)cq_entry;
                nb_segs = nicvf_process_cq_mseg_entry(cqe_rx, mbuf_phys_off,
-                       rx_pkts + i, rbptr_offset, mbuf_init);
+                       rx_pkts + i, rbptr_offset, mbuf_init, flag);
                buffers_consumed += nb_segs;
                cqe_head = (cqe_head + 1) & cqe_mask;
                nicvf_prefetch_store_keep(rx_pkts[i]);
@@ -539,6 +578,22 @@ nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts,
        return to_process;
 }
 
+uint16_t __hot
+nicvf_recv_pkts_multiseg_no_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
+               uint16_t nb_pkts)
+{
+       return nicvf_recv_pkts_multiseg(rx_queue, rx_pkts, nb_pkts,
+                       NICVF_RX_OFFLOAD_NONE);
+}
+
+uint16_t __hot
+nicvf_recv_pkts_multiseg_cksum(void *rx_queue, struct rte_mbuf **rx_pkts,
+               uint16_t nb_pkts)
+{
+       return nicvf_recv_pkts_multiseg(rx_queue, rx_pkts, nb_pkts,
+                       NICVF_RX_OFFLOAD_CKSUM);
+}
+
 uint32_t
 nicvf_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx)
 {

diff --git a/drivers/net/thunderx/nicvf_rxtx.h b/drivers/net/thunderx/nicvf_rxtx.h
index 8bdd582..72daffb 100644 (file)
--- a/drivers/net/thunderx/nicvf_rxtx.h
+++ b/drivers/net/thunderx/nicvf_rxtx.h
@@ -8,6 +8,9 @@
 #include <rte_byteorder.h>
 #include <rte_ethdev_driver.h>
 
+#define NICVF_RX_OFFLOAD_NONE           0x1
+#define NICVF_RX_OFFLOAD_CKSUM          0x2
+
 #define NICVF_TX_OFFLOAD_MASK (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK)
 
 #ifndef __hot
@@ -86,9 +89,15 @@ nicvf_mbuff_init_mseg_update(struct rte_mbuf *pkt, const uint64_t mbuf_init,
 uint32_t nicvf_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx);
 uint32_t nicvf_dev_rbdr_refill(struct rte_eth_dev *dev, uint16_t queue_idx);
 
-uint16_t nicvf_recv_pkts(void *rxq, struct rte_mbuf **rx_pkts, uint16_t pkts);
-uint16_t nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts,
-                                 uint16_t nb_pkts);
+uint16_t nicvf_recv_pkts_no_offload(void *rxq, struct rte_mbuf **rx_pkts,
+               uint16_t pkts);
+uint16_t nicvf_recv_pkts_cksum(void *rxq, struct rte_mbuf **rx_pkts,
+               uint16_t pkts);
+
+uint16_t nicvf_recv_pkts_multiseg_no_offload(void *rx_queue,
+               struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t nicvf_recv_pkts_multiseg_cksum(void *rx_queue,
+               struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
 
 uint16_t nicvf_xmit_pkts(void *txq, struct rte_mbuf **tx_pkts, uint16_t pkts);
 uint16_t nicvf_xmit_pkts_multiseg(void *txq, struct rte_mbuf **tx_pkts,

diff --git a/drivers/net/thunderx/nicvf_struct.h b/drivers/net/thunderx/nicvf_struct.h
index cf98f7c..a770e6b 100644 (file)
--- a/drivers/net/thunderx/nicvf_struct.h
+++ b/drivers/net/thunderx/nicvf_struct.h
@@ -55,25 +55,27 @@ union mbuf_initializer {
 };
 
 struct nicvf_rxq {
+       MARKER rxq_fastpath_data_start;
+       uint8_t  rbptr_offset;
+       uint16_t rx_free_thresh;
+       uint32_t head;
+       uint32_t qlen_mask;
+       int32_t recv_buffers;
+       int32_t available_space;
        uint64_t mbuf_phys_off;
        uintptr_t cq_status;
        uintptr_t cq_door;
-       union mbuf_initializer mbuf_initializer;
-       nicvf_iova_addr_t phys;
-       union cq_entry_t *desc;
        struct nicvf_rbdr *shared_rbdr;
-       struct nicvf *nic;
        struct rte_mempool *pool;
-       uint32_t head;
-       uint32_t qlen_mask;
-       int32_t available_space;
-       int32_t recv_buffers;
-       uint16_t rx_free_thresh;
-       uint16_t queue_id;
-       uint16_t precharge_cnt;
+       union cq_entry_t *desc;
+       union mbuf_initializer mbuf_initializer;
+       MARKER rxq_fastpath_data_end;
        uint8_t rx_drop_en;
+       uint16_t precharge_cnt;
        uint16_t port_id;
-       uint8_t  rbptr_offset;
+       uint16_t queue_id;
+       struct nicvf *nic;
+       nicvf_iova_addr_t phys;
 } __rte_cache_aligned;
 
 struct nicvf {
@@ -85,6 +87,7 @@ struct nicvf {
        bool loopback_supported;
        bool pf_acked:1;
        bool pf_nacked:1;
+       bool offload_cksum:1;
        uint64_t hwcap;
        uint8_t link_up;
        uint8_t duplex;