net/cnxk: support multi-segment vector Rx

[dpdk.git] / drivers / net / cnxk / cn9k_rx.h

diff --git a/drivers/net/cnxk/cn9k_rx.h b/drivers/net/cnxk/cn9k_rx.h
index 10ef5c6..5ae9e81 100644 (file)
--- a/drivers/net/cnxk/cn9k_rx.h
+++ b/drivers/net/cnxk/cn9k_rx.h
@@ -120,8 +120,15 @@ nix_cqe_xtract_mseg(const union nix_rx_parse_u *rx, struct rte_mbuf *mbuf,
 
        sg = *(const uint64_t *)(rx + 1);
        nb_segs = (sg >> 48) & 0x3;
-       mbuf->nb_segs = nb_segs;
+
+       if (nb_segs == 1) {
+               mbuf->next = NULL;
+               return;
+       }
+
+       mbuf->pkt_len = rx->pkt_lenm1 + 1;
        mbuf->data_len = sg & 0xFFFF;
+       mbuf->nb_segs = nb_segs;
        sg = sg >> 16;
 
        eol = ((const rte_iova_t *)(rx + 1) +
@@ -198,15 +205,14 @@ cn9k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag,
                        nix_update_match_id(rx->cn9k.match_id, ol_flags, mbuf);
 
        mbuf->ol_flags = ol_flags;
-       *(uint64_t *)(&mbuf->rearm_data) = val;
        mbuf->pkt_len = len;
+       mbuf->data_len = len;
+       *(uint64_t *)(&mbuf->rearm_data) = val;
 
-       if (flag & NIX_RX_MULTI_SEG_F) {
+       if (flag & NIX_RX_MULTI_SEG_F)
                nix_cqe_xtract_mseg(rx, mbuf, val);
-       } else {
-               mbuf->data_len = len;
+       else
                mbuf->next = NULL;
-       }
 }
 
 static inline uint16_t
@@ -484,16 +490,34 @@ cn9k_nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts,
                vst1q_u64((uint64_t *)mbuf2->rearm_data, rearm2);
                vst1q_u64((uint64_t *)mbuf3->rearm_data, rearm3);
 
-               /* Update that no more segments */
-               mbuf0->next = NULL;
-               mbuf1->next = NULL;
-               mbuf2->next = NULL;
-               mbuf3->next = NULL;
-
                /* Store the mbufs to rx_pkts */
                vst1q_u64((uint64_t *)&rx_pkts[packets], mbuf01);
                vst1q_u64((uint64_t *)&rx_pkts[packets + 2], mbuf23);
 
+               if (flags & NIX_RX_MULTI_SEG_F) {
+                       /* Multi segment is enable build mseg list for
+                        * individual mbufs in scalar mode.
+                        */
+                       nix_cqe_xtract_mseg((union nix_rx_parse_u *)
+                                           (cq0 + CQE_SZ(0) + 8), mbuf0,
+                                           mbuf_initializer);
+                       nix_cqe_xtract_mseg((union nix_rx_parse_u *)
+                                           (cq0 + CQE_SZ(1) + 8), mbuf1,
+                                           mbuf_initializer);
+                       nix_cqe_xtract_mseg((union nix_rx_parse_u *)
+                                           (cq0 + CQE_SZ(2) + 8), mbuf2,
+                                           mbuf_initializer);
+                       nix_cqe_xtract_mseg((union nix_rx_parse_u *)
+                                           (cq0 + CQE_SZ(3) + 8), mbuf3,
+                                           mbuf_initializer);
+               } else {
+                       /* Update that no more segments */
+                       mbuf0->next = NULL;
+                       mbuf1->next = NULL;
+                       mbuf2->next = NULL;
+                       mbuf3->next = NULL;
+               }
+
                /* Prefetch mbufs */
                roc_prefetch_store_keep(mbuf0);
                roc_prefetch_store_keep(mbuf1);
@@ -647,6 +671,9 @@ R(vlan_ts_mark_cksum_ptype_rss,     1, 1, 1, 1, 1, 1,                              \
                void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
                                                                               \
        uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_vec_##name(       \
+               void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
+                                                                              \
+       uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_vec_mseg_##name(  \
                void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);
 
 NIX_RX_FASTPATH_MODES