net/cnxk: add cn10k template Rx functions to build

Add cn10k segregated Rx and event dequeue functions to build,
add macros to make future modifications simpler.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Acked-by: Jerin Jacob <jerinj@marvell.com>

diff --git a/drivers/event/cnxk/cn10k_eventdev.c b/drivers/event/cnxk/cn10k_eventdev.c
index 7ee44ad61c67ddbe6bdbe6d5d80f6d1473c5fbce..d17a8e06467f8da5df3fbbf185ecbb997a3f2080 100644 (file)
--- a/drivers/event/cnxk/cn10k_eventdev.c
+++ b/drivers/event/cnxk/cn10k_eventdev.c
@@ -7,13 +7,7 @@
 #include "cnxk_worker.h"
 
 #define CN10K_SET_EVDEV_DEQ_OP(dev, deq_op, deq_ops)                           \
-       (deq_op = deq_ops[!!(dev->rx_offloads & NIX_RX_OFFLOAD_SECURITY_F)]    \
-                        [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]  \
-                        [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]      \
-                        [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)] \
-                        [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]    \
-                        [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]       \
-                        [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)])
+       deq_op = deq_ops[dev->rx_offloads & (NIX_RX_OFFLOAD_MAX - 1)]
 
 #define CN10K_SET_EVDEV_ENQ_OP(dev, enq_op, enq_ops)                           \
        (enq_op =                                                              \
@@ -291,92 +285,100 @@ static void
 cn10k_sso_fp_fns_set(struct rte_eventdev *event_dev)
 {
        struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
-       const event_dequeue_t sso_hws_deq[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                            \
-       [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_##name,
+       const event_dequeue_t sso_hws_deq[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_##name,
                NIX_RX_FASTPATH_MODES
 #undef R
        };
 
-       const event_dequeue_burst_t sso_hws_deq_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_burst_##name,
+       const event_dequeue_burst_t sso_hws_deq_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_burst_##name,
                NIX_RX_FASTPATH_MODES
 #undef R
        };
 
-       const event_dequeue_t sso_hws_deq_tmo[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_tmo_##name,
+       const event_dequeue_t sso_hws_deq_tmo[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_##name,
                NIX_RX_FASTPATH_MODES
 #undef R
        };
 
-       const event_dequeue_burst_t
-               sso_hws_deq_tmo_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_tmo_burst_##name,
+       const event_dequeue_burst_t sso_hws_deq_tmo_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_burst_##name,
                NIX_RX_FASTPATH_MODES
 #undef R
        };
 
-       const event_dequeue_t sso_hws_deq_ca[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_ca_##name,
+       const event_dequeue_t sso_hws_deq_ca[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_ca_##name,
                NIX_RX_FASTPATH_MODES
 #undef R
        };
 
-       const event_dequeue_burst_t
-               sso_hws_deq_ca_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_ca_burst_##name,
+       const event_dequeue_burst_t sso_hws_deq_ca_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_ca_burst_##name,
                NIX_RX_FASTPATH_MODES
 #undef R
        };
 
-       const event_dequeue_t sso_hws_deq_seg[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_seg_##name,
+       const event_dequeue_t sso_hws_deq_tmo_ca[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_ca_##name,
                NIX_RX_FASTPATH_MODES
 #undef R
        };
 
-       const event_dequeue_burst_t
-               sso_hws_deq_seg_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_seg_burst_##name,
+       const event_dequeue_burst_t sso_hws_deq_tmo_ca_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_ca_burst_##name,
                NIX_RX_FASTPATH_MODES
 #undef R
        };
 
-       const event_dequeue_t sso_hws_deq_tmo_seg[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_tmo_seg_##name,
+       const event_dequeue_t sso_hws_deq_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_seg_##name,
+
                NIX_RX_FASTPATH_MODES
 #undef R
        };
 
-       const event_dequeue_burst_t
-               sso_hws_deq_tmo_seg_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_tmo_seg_burst_##name,
+       const event_dequeue_burst_t sso_hws_deq_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_seg_burst_##name,
                        NIX_RX_FASTPATH_MODES
 #undef R
-               };
+       };
 
-       const event_dequeue_t sso_hws_deq_ca_seg[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_ca_seg_##name,
+       const event_dequeue_t sso_hws_deq_tmo_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_seg_##name,
                NIX_RX_FASTPATH_MODES
 #undef R
        };
 
-       const event_dequeue_burst_t
-               sso_hws_deq_ca_seg_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_ca_seg_burst_##name,
-                       NIX_RX_FASTPATH_MODES
+       const event_dequeue_burst_t sso_hws_deq_tmo_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_seg_burst_##name,
+               NIX_RX_FASTPATH_MODES
+#undef R
+       };
+
+       const event_dequeue_t sso_hws_deq_ca_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_ca_seg_##name,
+               NIX_RX_FASTPATH_MODES
+#undef R
+       };
+
+       const event_dequeue_burst_t sso_hws_deq_ca_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_ca_seg_burst_##name,
+               NIX_RX_FASTPATH_MODES
+#undef R
+       };
+
+       const event_dequeue_t sso_hws_deq_tmo_ca_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_ca_seg_##name,
+               NIX_RX_FASTPATH_MODES
+#undef R
+       };
+
+       const event_dequeue_burst_t sso_hws_deq_tmo_ca_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_ca_seg_burst_##name,
+               NIX_RX_FASTPATH_MODES
 #undef R
        };
 
@@ -418,6 +420,12 @@ cn10k_sso_fp_fns_set(struct rte_eventdev *event_dev)
                        CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
                                               sso_hws_deq_ca_seg_burst);
                }
+               if (dev->is_timeout_deq && dev->is_ca_internal_port) {
+                       CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+                                              sso_hws_deq_tmo_ca_seg);
+                       CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+                                              sso_hws_deq_tmo_ca_seg_burst);
+               }
        } else {
                CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue, sso_hws_deq);
                CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
@@ -434,6 +442,12 @@ cn10k_sso_fp_fns_set(struct rte_eventdev *event_dev)
                        CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
                                               sso_hws_deq_ca_burst);
                }
+               if (dev->is_timeout_deq && dev->is_ca_internal_port) {
+                       CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+                                              sso_hws_deq_tmo_ca);
+                       CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+                                              sso_hws_deq_tmo_ca_burst);
+               }
        }
        event_dev->ca_enqueue = cn10k_sso_hws_ca_enq;
 

diff --git a/drivers/event/cnxk/cn10k_worker.h b/drivers/event/cnxk/cn10k_worker.h
index f8331e88d7c83a84790adbdb341b27da7d643011..80f1955ccad6d5cc4a68fc171ee7d04d2ca8920b 100644 (file)
--- a/drivers/event/cnxk/cn10k_worker.h
+++ b/drivers/event/cnxk/cn10k_worker.h
@@ -352,7 +352,7 @@ uint16_t __rte_hot cn10k_sso_hws_enq_fwd_burst(void *port,
 uint16_t __rte_hot cn10k_sso_hws_ca_enq(void *port, struct rte_event ev[],
                                        uint16_t nb_events);
 
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
+#define R(name, flags)                                                         \
        uint16_t __rte_hot cn10k_sso_hws_deq_##name(                           \
                void *port, struct rte_event *ev, uint64_t timeout_ticks);     \
        uint16_t __rte_hot cn10k_sso_hws_deq_burst_##name(                     \
@@ -368,6 +368,11 @@ uint16_t __rte_hot cn10k_sso_hws_ca_enq(void *port, struct rte_event ev[],
        uint16_t __rte_hot cn10k_sso_hws_deq_ca_burst_##name(                  \
                void *port, struct rte_event ev[], uint16_t nb_events,         \
                uint64_t timeout_ticks);                                       \
+       uint16_t __rte_hot cn10k_sso_hws_deq_tmo_ca_##name(                    \
+               void *port, struct rte_event *ev, uint64_t timeout_ticks);     \
+       uint16_t __rte_hot cn10k_sso_hws_deq_tmo_ca_burst_##name(              \
+               void *port, struct rte_event ev[], uint16_t nb_events,         \
+               uint64_t timeout_ticks);                                       \
        uint16_t __rte_hot cn10k_sso_hws_deq_seg_##name(                       \
                void *port, struct rte_event *ev, uint64_t timeout_ticks);     \
        uint16_t __rte_hot cn10k_sso_hws_deq_seg_burst_##name(                 \
@@ -381,12 +386,74 @@ uint16_t __rte_hot cn10k_sso_hws_ca_enq(void *port, struct rte_event ev[],
        uint16_t __rte_hot cn10k_sso_hws_deq_ca_seg_##name(                    \
                void *port, struct rte_event *ev, uint64_t timeout_ticks);     \
        uint16_t __rte_hot cn10k_sso_hws_deq_ca_seg_burst_##name(              \
+               void *port, struct rte_event ev[], uint16_t nb_events,         \
+               uint64_t timeout_ticks);                                       \
+       uint16_t __rte_hot cn10k_sso_hws_deq_tmo_ca_seg_##name(                \
+               void *port, struct rte_event *ev, uint64_t timeout_ticks);     \
+       uint16_t __rte_hot cn10k_sso_hws_deq_tmo_ca_seg_burst_##name(          \
                void *port, struct rte_event ev[], uint16_t nb_events,         \
                uint64_t timeout_ticks);
 
 NIX_RX_FASTPATH_MODES
 #undef R
 
+#define SSO_DEQ(fn, flags)                                                     \
+       uint16_t __rte_hot fn(void *port, struct rte_event *ev,                \
+                             uint64_t timeout_ticks)                          \
+       {                                                                      \
+               struct cn10k_sso_hws *ws = port;                               \
+               RTE_SET_USED(timeout_ticks);                                   \
+               if (ws->swtag_req) {                                           \
+                       ws->swtag_req = 0;                                     \
+                       cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_WQE0);  \
+                       return 1;                                              \
+               }                                                              \
+               return cn10k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem);  \
+       }
+
+#define SSO_DEQ_SEG(fn, flags)   SSO_DEQ(fn, flags | NIX_RX_MULTI_SEG_F)
+#define SSO_DEQ_CA(fn, flags)    SSO_DEQ(fn, flags | CPT_RX_WQE_F)
+#define SSO_DEQ_CA_SEG(fn, flags) SSO_DEQ_SEG(fn, flags | CPT_RX_WQE_F)
+
+#define SSO_DEQ_TMO(fn, flags)                                                 \
+       uint16_t __rte_hot fn(void *port, struct rte_event *ev,                \
+                             uint64_t timeout_ticks)                          \
+       {                                                                      \
+               struct cn10k_sso_hws *ws = port;                               \
+               uint16_t ret = 1;                                              \
+               uint64_t iter;                                                 \
+               if (ws->swtag_req) {                                           \
+                       ws->swtag_req = 0;                                     \
+                       cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_WQE0);  \
+                       return ret;                                            \
+               }                                                              \
+               ret = cn10k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem);   \
+               for (iter = 1; iter < timeout_ticks && (ret == 0); iter++)     \
+                       ret = cn10k_sso_hws_get_work(ws, ev, flags,            \
+                                                    ws->lookup_mem);          \
+               return ret;                                                    \
+       }
+
+#define SSO_DEQ_TMO_SEG(fn, flags)    SSO_DEQ_TMO(fn, flags | NIX_RX_MULTI_SEG_F)
+#define SSO_DEQ_TMO_CA(fn, flags)     SSO_DEQ_TMO(fn, flags | CPT_RX_WQE_F)
+#define SSO_DEQ_TMO_CA_SEG(fn, flags) SSO_DEQ_TMO_SEG(fn, flags | CPT_RX_WQE_F)
+
+#define SSO_CMN_DEQ_BURST(fnb, fn, flags)                                      \
+       uint16_t __rte_hot fnb(void *port, struct rte_event ev[],              \
+                              uint16_t nb_events, uint64_t timeout_ticks)     \
+       {                                                                      \
+               RTE_SET_USED(nb_events);                                       \
+               return fn(port, ev, timeout_ticks);                            \
+       }
+
+#define SSO_CMN_DEQ_SEG_BURST(fnb, fn, flags)                                  \
+       uint16_t __rte_hot fnb(void *port, struct rte_event ev[],              \
+                              uint16_t nb_events, uint64_t timeout_ticks)     \
+       {                                                                      \
+               RTE_SET_USED(nb_events);                                       \
+               return fn(port, ev, timeout_ticks);                            \
+       }
+
 static __rte_always_inline struct cn10k_eth_txq *
 cn10k_sso_hws_xtract_meta(struct rte_mbuf *m,
                          const uint64_t txq_data[][RTE_MAX_QUEUES_PER_PORT])

diff --git a/drivers/event/cnxk/cn10k_worker_deq.c b/drivers/event/cnxk/cn10k_worker_deq.c
deleted file mode 100644 (file)
index 6083f69..0000000
--- a/drivers/event/cnxk/cn10k_worker_deq.c
+++ /dev/null
@@ -1,44 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn10k_worker.h"
-#include "cnxk_eventdev.h"
-#include "cnxk_worker.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       uint16_t __rte_hot cn10k_sso_hws_deq_##name(                           \
-               void *port, struct rte_event *ev, uint64_t timeout_ticks)      \
-       {                                                                      \
-               struct cn10k_sso_hws *ws = port;                               \
-                                                                              \
-               RTE_SET_USED(timeout_ticks);                                   \
-                                                                              \
-               if (ws->swtag_req) {                                           \
-                       ws->swtag_req = 0;                                     \
-                       cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_WQE0);  \
-                       return 1;                                              \
-               }                                                              \
-                                                                              \
-               return cn10k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem);  \
-       }                                                                      \
-                                                                              \
-       uint16_t __rte_hot cn10k_sso_hws_deq_seg_##name(                       \
-               void *port, struct rte_event *ev, uint64_t timeout_ticks)      \
-       {                                                                      \
-               struct cn10k_sso_hws *ws = port;                               \
-                                                                              \
-               RTE_SET_USED(timeout_ticks);                                   \
-                                                                              \
-               if (ws->swtag_req) {                                           \
-                       ws->swtag_req = 0;                                     \
-                       cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_WQE0);  \
-                       return 1;                                              \
-               }                                                              \
-                                                                              \
-               return cn10k_sso_hws_get_work(                                 \
-                       ws, ev, flags | NIX_RX_MULTI_SEG_F, ws->lookup_mem);   \
-       }
-
-NIX_RX_FASTPATH_MODES
-#undef R

diff --git a/drivers/event/cnxk/cn10k_worker_deq_burst.c b/drivers/event/cnxk/cn10k_worker_deq_burst.c
deleted file mode 100644 (file)
index 8539d5d..0000000
--- a/drivers/event/cnxk/cn10k_worker_deq_burst.c
+++ /dev/null
@@ -1,29 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn10k_worker.h"
-#include "cnxk_eventdev.h"
-#include "cnxk_worker.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                            \
-       uint16_t __rte_hot cn10k_sso_hws_deq_burst_##name(                     \
-               void *port, struct rte_event ev[], uint16_t nb_events,         \
-               uint64_t timeout_ticks)                                        \
-       {                                                                      \
-               RTE_SET_USED(nb_events);                                       \
-                                                                              \
-               return cn10k_sso_hws_deq_##name(port, ev, timeout_ticks);      \
-       }                                                                      \
-                                                                              \
-       uint16_t __rte_hot cn10k_sso_hws_deq_seg_burst_##name(                 \
-               void *port, struct rte_event ev[], uint16_t nb_events,         \
-               uint64_t timeout_ticks)                                        \
-       {                                                                      \
-               RTE_SET_USED(nb_events);                                       \
-                                                                              \
-               return cn10k_sso_hws_deq_seg_##name(port, ev, timeout_ticks);  \
-       }
-
-NIX_RX_FASTPATH_MODES
-#undef R

diff --git a/drivers/event/cnxk/cn10k_worker_deq_ca.c b/drivers/event/cnxk/cn10k_worker_deq_ca.c
deleted file mode 100644 (file)
index 15c698e..0000000
--- a/drivers/event/cnxk/cn10k_worker_deq_ca.c
+++ /dev/null
@@ -1,65 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn10k_worker.h"
-#include "cnxk_eventdev.h"
-#include "cnxk_worker.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       uint16_t __rte_hot cn10k_sso_hws_deq_ca_##name(                        \
-               void *port, struct rte_event *ev, uint64_t timeout_ticks)      \
-       {                                                                      \
-               struct cn10k_sso_hws *ws = port;                               \
-                                                                              \
-               RTE_SET_USED(timeout_ticks);                                   \
-                                                                              \
-               if (ws->swtag_req) {                                           \
-                       ws->swtag_req = 0;                                     \
-                       cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_WQE0);  \
-                       return 1;                                              \
-               }                                                              \
-                                                                              \
-               return cn10k_sso_hws_get_work(ws, ev, flags | CPT_RX_WQE_F,    \
-                                             ws->lookup_mem);                 \
-       }                                                                      \
-                                                                              \
-       uint16_t __rte_hot cn10k_sso_hws_deq_ca_burst_##name(                  \
-               void *port, struct rte_event ev[], uint16_t nb_events,         \
-               uint64_t timeout_ticks)                                        \
-       {                                                                      \
-               RTE_SET_USED(nb_events);                                       \
-                                                                              \
-               return cn10k_sso_hws_deq_ca_##name(port, ev, timeout_ticks);   \
-       }                                                                      \
-                                                                              \
-       uint16_t __rte_hot cn10k_sso_hws_deq_ca_seg_##name(                    \
-               void *port, struct rte_event *ev, uint64_t timeout_ticks)      \
-       {                                                                      \
-               struct cn10k_sso_hws *ws = port;                               \
-                                                                              \
-               RTE_SET_USED(timeout_ticks);                                   \
-                                                                              \
-               if (ws->swtag_req) {                                           \
-                       ws->swtag_req = 0;                                     \
-                       cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_WQE0);  \
-                       return 1;                                              \
-               }                                                              \
-                                                                              \
-               return cn10k_sso_hws_get_work(                                 \
-                       ws, ev, flags | NIX_RX_MULTI_SEG_F | CPT_RX_WQE_F,     \
-                       ws->lookup_mem);                                       \
-       }                                                                      \
-                                                                              \
-       uint16_t __rte_hot cn10k_sso_hws_deq_ca_seg_burst_##name(              \
-               void *port, struct rte_event ev[], uint16_t nb_events,         \
-               uint64_t timeout_ticks)                                        \
-       {                                                                      \
-               RTE_SET_USED(nb_events);                                       \
-                                                                              \
-               return cn10k_sso_hws_deq_ca_seg_##name(port, ev,               \
-                                                      timeout_ticks);         \
-       }
-
-NIX_RX_FASTPATH_MODES
-#undef R

diff --git a/drivers/event/cnxk/cn10k_worker_deq_tmo.c b/drivers/event/cnxk/cn10k_worker_deq_tmo.c
deleted file mode 100644 (file)
index 537ae37..0000000
--- a/drivers/event/cnxk/cn10k_worker_deq_tmo.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn10k_worker.h"
-#include "cnxk_eventdev.h"
-#include "cnxk_worker.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                            \
-       uint16_t __rte_hot cn10k_sso_hws_deq_tmo_##name(                       \
-               void *port, struct rte_event *ev, uint64_t timeout_ticks)      \
-       {                                                                      \
-               struct cn10k_sso_hws *ws = port;                               \
-               uint16_t ret = 1;                                              \
-               uint64_t iter;                                                 \
-                                                                              \
-               if (ws->swtag_req) {                                           \
-                       ws->swtag_req = 0;                                     \
-                       cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_WQE0);  \
-                       return ret;                                            \
-               }                                                              \
-                                                                              \
-               ret = cn10k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem);   \
-               for (iter = 1; iter < timeout_ticks && (ret == 0); iter++)     \
-                       ret = cn10k_sso_hws_get_work(ws, ev, flags,            \
-                                                    ws->lookup_mem);          \
-                                                                              \
-               return ret;                                                    \
-       }                                                                      \
-                                                                              \
-       uint16_t __rte_hot cn10k_sso_hws_deq_tmo_burst_##name(                 \
-               void *port, struct rte_event ev[], uint16_t nb_events,         \
-               uint64_t timeout_ticks)                                        \
-       {                                                                      \
-               RTE_SET_USED(nb_events);                                       \
-                                                                              \
-               return cn10k_sso_hws_deq_tmo_##name(port, ev, timeout_ticks);  \
-       }                                                                      \
-                                                                              \
-       uint16_t __rte_hot cn10k_sso_hws_deq_tmo_seg_##name(                   \
-               void *port, struct rte_event *ev, uint64_t timeout_ticks)      \
-       {                                                                      \
-               struct cn10k_sso_hws *ws = port;                               \
-               uint16_t ret = 1;                                              \
-               uint64_t iter;                                                 \
-                                                                              \
-               if (ws->swtag_req) {                                           \
-                       ws->swtag_req = 0;                                     \
-                       cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_WQE0);  \
-                       return ret;                                            \
-               }                                                              \
-                                                                              \
-               ret = cn10k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem);   \
-               for (iter = 1; iter < timeout_ticks && (ret == 0); iter++)     \
-                       ret = cn10k_sso_hws_get_work(ws, ev, flags,            \
-                                                    ws->lookup_mem);          \
-                                                                              \
-               return ret;                                                    \
-       }                                                                      \
-                                                                              \
-       uint16_t __rte_hot cn10k_sso_hws_deq_tmo_seg_burst_##name(             \
-               void *port, struct rte_event ev[], uint16_t nb_events,         \
-               uint64_t timeout_ticks)                                        \
-       {                                                                      \
-               RTE_SET_USED(nb_events);                                       \
-                                                                              \
-               return cn10k_sso_hws_deq_tmo_seg_##name(port, ev,              \
-                                                       timeout_ticks);        \
-       }
-
-NIX_RX_FASTPATH_MODES
-#undef R

diff --git a/drivers/event/cnxk/meson.build b/drivers/event/cnxk/meson.build
index c39515dadc9c765e66b4f59e786b3073da374dfc..ac452fe4517ee3ebac9d1b14e5cac859b9a13a1e 100644 (file)
--- a/drivers/event/cnxk/meson.build
+++ b/drivers/event/cnxk/meson.build
@@ -13,10 +13,6 @@ sources = files(
         'cn9k_worker.c',
         'cn10k_eventdev.c',
         'cn10k_worker.c',
-        'cn10k_worker_deq.c',
-        'cn10k_worker_deq_burst.c',
-        'cn10k_worker_deq_ca.c',
-        'cn10k_worker_deq_tmo.c',
         'cn10k_worker_tx_enq.c',
         'cn10k_worker_tx_enq_seg.c',
         'cnxk_eventdev.c',
@@ -327,6 +323,137 @@ sources += files(
         'tx/cn9k/tx_112_127_dual_seg.c',
 )
 
+sources += files(
+        'deq/cn10k/deq_0_15_burst.c',
+        'deq/cn10k/deq_16_31_burst.c',
+        'deq/cn10k/deq_32_47_burst.c',
+        'deq/cn10k/deq_48_63_burst.c',
+        'deq/cn10k/deq_64_79_burst.c',
+        'deq/cn10k/deq_80_95_burst.c',
+        'deq/cn10k/deq_96_111_burst.c',
+        'deq/cn10k/deq_112_127_burst.c',
+        'deq/cn10k/deq_0_15_seg_burst.c',
+        'deq/cn10k/deq_16_31_seg_burst.c',
+        'deq/cn10k/deq_32_47_seg_burst.c',
+        'deq/cn10k/deq_48_63_seg_burst.c',
+        'deq/cn10k/deq_64_79_seg_burst.c',
+        'deq/cn10k/deq_80_95_seg_burst.c',
+        'deq/cn10k/deq_96_111_seg_burst.c',
+        'deq/cn10k/deq_112_127_seg_burst.c',
+        'deq/cn10k/deq_0_15.c',
+        'deq/cn10k/deq_16_31.c',
+        'deq/cn10k/deq_32_47.c',
+        'deq/cn10k/deq_48_63.c',
+        'deq/cn10k/deq_64_79.c',
+        'deq/cn10k/deq_80_95.c',
+        'deq/cn10k/deq_96_111.c',
+        'deq/cn10k/deq_112_127.c',
+        'deq/cn10k/deq_0_15_seg.c',
+        'deq/cn10k/deq_16_31_seg.c',
+        'deq/cn10k/deq_32_47_seg.c',
+        'deq/cn10k/deq_48_63_seg.c',
+        'deq/cn10k/deq_64_79_seg.c',
+        'deq/cn10k/deq_80_95_seg.c',
+        'deq/cn10k/deq_96_111_seg.c',
+        'deq/cn10k/deq_112_127_seg.c',
+        'deq/cn10k/deq_0_15_tmo.c',
+        'deq/cn10k/deq_16_31_tmo.c',
+        'deq/cn10k/deq_32_47_tmo.c',
+        'deq/cn10k/deq_48_63_tmo.c',
+        'deq/cn10k/deq_64_79_tmo.c',
+        'deq/cn10k/deq_80_95_tmo.c',
+        'deq/cn10k/deq_96_111_tmo.c',
+        'deq/cn10k/deq_112_127_tmo.c',
+        'deq/cn10k/deq_0_15_tmo_burst.c',
+        'deq/cn10k/deq_16_31_tmo_burst.c',
+        'deq/cn10k/deq_32_47_tmo_burst.c',
+        'deq/cn10k/deq_48_63_tmo_burst.c',
+        'deq/cn10k/deq_64_79_tmo_burst.c',
+        'deq/cn10k/deq_80_95_tmo_burst.c',
+        'deq/cn10k/deq_96_111_tmo_burst.c',
+        'deq/cn10k/deq_112_127_tmo_burst.c',
+        'deq/cn10k/deq_0_15_tmo_seg.c',
+        'deq/cn10k/deq_16_31_tmo_seg.c',
+        'deq/cn10k/deq_32_47_tmo_seg.c',
+        'deq/cn10k/deq_48_63_tmo_seg.c',
+        'deq/cn10k/deq_64_79_tmo_seg.c',
+        'deq/cn10k/deq_80_95_tmo_seg.c',
+        'deq/cn10k/deq_96_111_tmo_seg.c',
+        'deq/cn10k/deq_112_127_tmo_seg.c',
+        'deq/cn10k/deq_0_15_tmo_seg_burst.c',
+        'deq/cn10k/deq_16_31_tmo_seg_burst.c',
+        'deq/cn10k/deq_32_47_tmo_seg_burst.c',
+        'deq/cn10k/deq_48_63_tmo_seg_burst.c',
+        'deq/cn10k/deq_64_79_tmo_seg_burst.c',
+        'deq/cn10k/deq_80_95_tmo_seg_burst.c',
+        'deq/cn10k/deq_96_111_tmo_seg_burst.c',
+        'deq/cn10k/deq_112_127_tmo_seg_burst.c',
+        'deq/cn10k/deq_0_15_ca.c',
+        'deq/cn10k/deq_16_31_ca.c',
+        'deq/cn10k/deq_32_47_ca.c',
+        'deq/cn10k/deq_48_63_ca.c',
+        'deq/cn10k/deq_64_79_ca.c',
+        'deq/cn10k/deq_80_95_ca.c',
+        'deq/cn10k/deq_96_111_ca.c',
+        'deq/cn10k/deq_112_127_ca.c',
+        'deq/cn10k/deq_0_15_ca_burst.c',
+        'deq/cn10k/deq_16_31_ca_burst.c',
+        'deq/cn10k/deq_32_47_ca_burst.c',
+        'deq/cn10k/deq_48_63_ca_burst.c',
+        'deq/cn10k/deq_64_79_ca_burst.c',
+        'deq/cn10k/deq_80_95_ca_burst.c',
+        'deq/cn10k/deq_96_111_ca_burst.c',
+        'deq/cn10k/deq_112_127_ca_burst.c',
+        'deq/cn10k/deq_0_15_ca_seg.c',
+        'deq/cn10k/deq_16_31_ca_seg.c',
+        'deq/cn10k/deq_32_47_ca_seg.c',
+        'deq/cn10k/deq_48_63_ca_seg.c',
+        'deq/cn10k/deq_64_79_ca_seg.c',
+        'deq/cn10k/deq_80_95_ca_seg.c',
+        'deq/cn10k/deq_96_111_ca_seg.c',
+        'deq/cn10k/deq_112_127_ca_seg.c',
+        'deq/cn10k/deq_0_15_ca_seg_burst.c',
+        'deq/cn10k/deq_16_31_ca_seg_burst.c',
+        'deq/cn10k/deq_32_47_ca_seg_burst.c',
+        'deq/cn10k/deq_48_63_ca_seg_burst.c',
+        'deq/cn10k/deq_64_79_ca_seg_burst.c',
+        'deq/cn10k/deq_80_95_ca_seg_burst.c',
+        'deq/cn10k/deq_96_111_ca_seg_burst.c',
+        'deq/cn10k/deq_112_127_ca_seg_burst.c',
+        'deq/cn10k/deq_0_15_ca_tmo.c',
+        'deq/cn10k/deq_16_31_ca_tmo.c',
+        'deq/cn10k/deq_32_47_ca_tmo.c',
+        'deq/cn10k/deq_48_63_ca_tmo.c',
+        'deq/cn10k/deq_64_79_ca_tmo.c',
+        'deq/cn10k/deq_80_95_ca_tmo.c',
+        'deq/cn10k/deq_96_111_ca_tmo.c',
+        'deq/cn10k/deq_112_127_ca_tmo.c',
+        'deq/cn10k/deq_0_15_ca_tmo_burst.c',
+        'deq/cn10k/deq_16_31_ca_tmo_burst.c',
+        'deq/cn10k/deq_32_47_ca_tmo_burst.c',
+        'deq/cn10k/deq_48_63_ca_tmo_burst.c',
+        'deq/cn10k/deq_64_79_ca_tmo_burst.c',
+        'deq/cn10k/deq_80_95_ca_tmo_burst.c',
+        'deq/cn10k/deq_96_111_ca_tmo_burst.c',
+        'deq/cn10k/deq_112_127_ca_tmo_burst.c',
+        'deq/cn10k/deq_0_15_ca_tmo_seg.c',
+        'deq/cn10k/deq_16_31_ca_tmo_seg.c',
+        'deq/cn10k/deq_32_47_ca_tmo_seg.c',
+        'deq/cn10k/deq_48_63_ca_tmo_seg.c',
+        'deq/cn10k/deq_64_79_ca_tmo_seg.c',
+        'deq/cn10k/deq_80_95_ca_tmo_seg.c',
+        'deq/cn10k/deq_96_111_ca_tmo_seg.c',
+        'deq/cn10k/deq_112_127_ca_tmo_seg.c',
+        'deq/cn10k/deq_0_15_ca_tmo_seg_burst.c',
+        'deq/cn10k/deq_16_31_ca_tmo_seg_burst.c',
+        'deq/cn10k/deq_32_47_ca_tmo_seg_burst.c',
+        'deq/cn10k/deq_48_63_ca_tmo_seg_burst.c',
+        'deq/cn10k/deq_64_79_ca_tmo_seg_burst.c',
+        'deq/cn10k/deq_80_95_ca_tmo_seg_burst.c',
+        'deq/cn10k/deq_96_111_ca_tmo_seg_burst.c',
+        'deq/cn10k/deq_112_127_ca_tmo_seg_burst.c',
+)
+
 extra_flags = ['-flax-vector-conversions', '-Wno-strict-aliasing']
 foreach flag: extra_flags
     if cc.has_argument(flag)

diff --git a/drivers/net/cnxk/cn10k_rx.h b/drivers/net/cnxk/cn10k_rx.h
index fe408907a6992dd03848fd2e71a29bd96babbcb3..8b00fcc66084ab5ee6346df7ac57b0dad1c5af9a 100644 (file)
--- a/drivers/net/cnxk/cn10k_rx.h
+++ b/drivers/net/cnxk/cn10k_rx.h
@@ -17,6 +17,7 @@
 #define NIX_RX_OFFLOAD_TSTAMP_F             BIT(4)
 #define NIX_RX_OFFLOAD_VLAN_STRIP_F  BIT(5)
 #define NIX_RX_OFFLOAD_SECURITY_F    BIT(6)
+#define NIX_RX_OFFLOAD_MAX          (NIX_RX_OFFLOAD_SECURITY_F << 1)
 
 /* Flags to control cqe_to_mbuf conversion function.
  * Defining it from backwards to denote its been
@@ -987,278 +988,211 @@ cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts,
 #define R_SEC_F   NIX_RX_OFFLOAD_SECURITY_F
 
 /* [R_SEC_F] [RX_VLAN_F] [TS] [MARK] [CKSUM] [PTYPE] [RSS] */
-#define NIX_RX_FASTPATH_MODES                                                 \
-R(no_offload,                  0, 0, 0, 0, 0, 0, 0,                           \
-               NIX_RX_OFFLOAD_NONE)                                           \
-R(rss,                         0, 0, 0, 0, 0, 0, 1,                           \
-               RSS_F)                                                         \
-R(ptype,                       0, 0, 0, 0, 0, 1, 0,                           \
-               PTYPE_F)                                                       \
-R(ptype_rss,                   0, 0, 0, 0, 0, 1, 1,                           \
-               PTYPE_F | RSS_F)                                               \
-R(cksum,                       0, 0, 0, 0, 1, 0, 0,                           \
-               CKSUM_F)                                                       \
-R(cksum_rss,                   0, 0, 0, 0, 1, 0, 1,                           \
-               CKSUM_F | RSS_F)                                               \
-R(cksum_ptype,                 0, 0, 0, 0, 1, 1, 0,                           \
-               CKSUM_F | PTYPE_F)                                             \
-R(cksum_ptype_rss,             0, 0, 0, 0, 1, 1, 1,                           \
-               CKSUM_F | PTYPE_F | RSS_F)                                     \
-R(mark,                                0, 0, 0, 1, 0, 0, 0,                           \
-               MARK_F)                                                        \
-R(mark_rss,                    0, 0, 0, 1, 0, 0, 1,                           \
-               MARK_F | RSS_F)                                                \
-R(mark_ptype,                  0, 0, 0, 1, 0, 1, 0,                           \
-               MARK_F | PTYPE_F)                                              \
-R(mark_ptype_rss,              0, 0, 0, 1, 0, 1, 1,                           \
-               MARK_F | PTYPE_F | RSS_F)                                      \
-R(mark_cksum,                  0, 0, 0, 1, 1, 0, 0,                           \
-               MARK_F | CKSUM_F)                                              \
-R(mark_cksum_rss,              0, 0, 0, 1, 1, 0, 1,                           \
-               MARK_F | CKSUM_F | RSS_F)                                      \
-R(mark_cksum_ptype,            0, 0, 0, 1, 1, 1, 0,                           \
-               MARK_F | CKSUM_F | PTYPE_F)                                    \
-R(mark_cksum_ptype_rss,                0, 0, 0, 1, 1, 1, 1,                           \
-               MARK_F | CKSUM_F | PTYPE_F | RSS_F)                            \
-R(ts,                          0, 0, 1, 0, 0, 0, 0,                           \
-               TS_F)                                                          \
-R(ts_rss,                      0, 0, 1, 0, 0, 0, 1,                           \
-               TS_F | RSS_F)                                                  \
-R(ts_ptype,                    0, 0, 1, 0, 0, 1, 0,                           \
-               TS_F | PTYPE_F)                                                \
-R(ts_ptype_rss,                        0, 0, 1, 0, 0, 1, 1,                           \
-               TS_F | PTYPE_F | RSS_F)                                        \
-R(ts_cksum,                    0, 0, 1, 0, 1, 0, 0,                           \
-               TS_F | CKSUM_F)                                                \
-R(ts_cksum_rss,                        0, 0, 1, 0, 1, 0, 1,                           \
-               TS_F | CKSUM_F | RSS_F)                                        \
-R(ts_cksum_ptype,              0, 0, 1, 0, 1, 1, 0,                           \
-               TS_F | CKSUM_F | PTYPE_F)                                      \
-R(ts_cksum_ptype_rss,          0, 0, 1, 0, 1, 1, 1,                           \
-               TS_F | CKSUM_F | PTYPE_F | RSS_F)                              \
-R(ts_mark,                     0, 0, 1, 1, 0, 0, 0,                           \
-               TS_F | MARK_F)                                                 \
-R(ts_mark_rss,                 0, 0, 1, 1, 0, 0, 1,                           \
-               TS_F | MARK_F | RSS_F)                                         \
-R(ts_mark_ptype,               0, 0, 1, 1, 0, 1, 0,                           \
-               TS_F | MARK_F | PTYPE_F)                                       \
-R(ts_mark_ptype_rss,           0, 0, 1, 1, 0, 1, 1,                           \
-               TS_F | MARK_F | PTYPE_F | RSS_F)                               \
-R(ts_mark_cksum,               0, 0, 1, 1, 1, 0, 0,                           \
-               TS_F | MARK_F | CKSUM_F)                                       \
-R(ts_mark_cksum_rss,           0, 0, 1, 1, 1, 0, 1,                           \
-               TS_F | MARK_F | CKSUM_F | RSS_F)                               \
-R(ts_mark_cksum_ptype,         0, 0, 1, 1, 1, 1, 0,                           \
-               TS_F | MARK_F | CKSUM_F | PTYPE_F)                             \
-R(ts_mark_cksum_ptype_rss,     0, 0, 1, 1, 1, 1, 1,                           \
-               TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)                     \
-R(vlan,                                0, 1, 0, 0, 0, 0, 0,                           \
-               RX_VLAN_F)                                                     \
-R(vlan_rss,                    0, 1, 0, 0, 0, 0, 1,                           \
-               RX_VLAN_F | RSS_F)                                             \
-R(vlan_ptype,                  0, 1, 0, 0, 0, 1, 0,                           \
-               RX_VLAN_F | PTYPE_F)                                           \
-R(vlan_ptype_rss,              0, 1, 0, 0, 0, 1, 1,                           \
-               RX_VLAN_F | PTYPE_F | RSS_F)                                   \
-R(vlan_cksum,                  0, 1, 0, 0, 1, 0, 0,                           \
-               RX_VLAN_F | CKSUM_F)                                           \
-R(vlan_cksum_rss,              0, 1, 0, 0, 1, 0, 1,                           \
-               RX_VLAN_F | CKSUM_F | RSS_F)                                   \
-R(vlan_cksum_ptype,            0, 1, 0, 0, 1, 1, 0,                           \
-               RX_VLAN_F | CKSUM_F | PTYPE_F)                                 \
-R(vlan_cksum_ptype_rss,                0, 1, 0, 0, 1, 1, 1,                           \
-               RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F)                         \
-R(vlan_mark,                   0, 1, 0, 1, 0, 0, 0,                           \
-               RX_VLAN_F | MARK_F)                                            \
-R(vlan_mark_rss,               0, 1, 0, 1, 0, 0, 1,                           \
-               RX_VLAN_F | MARK_F | RSS_F)                                    \
-R(vlan_mark_ptype,             0, 1, 0, 1, 0, 1, 0,                           \
-               RX_VLAN_F | MARK_F | PTYPE_F)                                  \
-R(vlan_mark_ptype_rss,         0, 1, 0, 1, 0, 1, 1,                           \
-               RX_VLAN_F | MARK_F | PTYPE_F | RSS_F)                          \
-R(vlan_mark_cksum,             0, 1, 0, 1, 1, 0, 0,                           \
-               RX_VLAN_F | MARK_F | CKSUM_F)                                  \
-R(vlan_mark_cksum_rss,         0, 1, 0, 1, 1, 0, 1,                           \
-               RX_VLAN_F | MARK_F | CKSUM_F | RSS_F)                          \
-R(vlan_mark_cksum_ptype,       0, 1, 0, 1, 1, 1, 0,                           \
-               RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F)                        \
-R(vlan_mark_cksum_ptype_rss,   0, 1, 0, 1, 1, 1, 1,                           \
-               RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)                \
-R(vlan_ts,                     0, 1, 1, 0, 0, 0, 0,                           \
-               RX_VLAN_F | TS_F)                                              \
-R(vlan_ts_rss,                 0, 1, 1, 0, 0, 0, 1,                           \
-               RX_VLAN_F | TS_F | RSS_F)                                      \
-R(vlan_ts_ptype,               0, 1, 1, 0, 0, 1, 0,                           \
-               RX_VLAN_F | TS_F | PTYPE_F)                                    \
-R(vlan_ts_ptype_rss,           0, 1, 1, 0, 0, 1, 1,                           \
-               RX_VLAN_F | TS_F | PTYPE_F | RSS_F)                            \
-R(vlan_ts_cksum,               0, 1, 1, 0, 1, 0, 0,                           \
-               RX_VLAN_F | TS_F | CKSUM_F)                                    \
-R(vlan_ts_cksum_rss,           0, 1, 1, 0, 1, 0, 1,                           \
-               RX_VLAN_F | TS_F | CKSUM_F | RSS_F)                            \
-R(vlan_ts_cksum_ptype,         0, 1, 1, 0, 1, 1, 0,                           \
-               RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F)                          \
-R(vlan_ts_cksum_ptype_rss,     0, 1, 1, 0, 1, 1, 1,                           \
-               RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F)                  \
-R(vlan_ts_mark,                        0, 1, 1, 1, 0, 0, 0,                           \
-               RX_VLAN_F | TS_F | MARK_F)                                     \
-R(vlan_ts_mark_rss,            0, 1, 1, 1, 0, 0, 1,                           \
-               RX_VLAN_F | TS_F | MARK_F | RSS_F)                             \
-R(vlan_ts_mark_ptype,          0, 1, 1, 1, 0, 1, 0,                           \
-               RX_VLAN_F | TS_F | MARK_F | PTYPE_F)                           \
-R(vlan_ts_mark_ptype_rss,      0, 1, 1, 1, 0, 1, 1,                           \
-               RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F)                   \
-R(vlan_ts_mark_cksum,          0, 1, 1, 1, 1, 0, 0,                           \
-               RX_VLAN_F | TS_F | MARK_F | CKSUM_F)                           \
-R(vlan_ts_mark_cksum_rss,      0, 1, 1, 1, 1, 0, 1,                           \
-               RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F)                   \
-R(vlan_ts_mark_cksum_ptype,    0, 1, 1, 1, 1, 1, 0,                           \
-               RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F)                 \
-R(vlan_ts_mark_cksum_ptype_rss,        0, 1, 1, 1, 1, 1, 1,                           \
-               RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)         \
-R(sec,                         1, 0, 0, 0, 0, 0, 0,                           \
-               R_SEC_F)                                                       \
-R(sec_rss,                     1, 0, 0, 0, 0, 0, 1,                           \
-               RSS_F)                                                         \
-R(sec_ptype,                   1, 0, 0, 0, 0, 1, 0,                           \
-               R_SEC_F | PTYPE_F)                                             \
-R(sec_ptype_rss,               1, 0, 0, 0, 0, 1, 1,                           \
-               R_SEC_F | PTYPE_F | RSS_F)                                     \
-R(sec_cksum,                   1, 0, 0, 0, 1, 0, 0,                           \
-               R_SEC_F | CKSUM_F)                                             \
-R(sec_cksum_rss,               1, 0, 0, 0, 1, 0, 1,                           \
-               R_SEC_F | CKSUM_F | RSS_F)                                     \
-R(sec_cksum_ptype,             1, 0, 0, 0, 1, 1, 0,                           \
-               R_SEC_F | CKSUM_F | PTYPE_F)                                   \
-R(sec_cksum_ptype_rss,         1, 0, 0, 0, 1, 1, 1,                           \
-               R_SEC_F | CKSUM_F | PTYPE_F | RSS_F)                           \
-R(sec_mark,                    1, 0, 0, 1, 0, 0, 0,                           \
-               R_SEC_F | MARK_F)                                              \
-R(sec_mark_rss,                        1, 0, 0, 1, 0, 0, 1,                           \
-               R_SEC_F | MARK_F | RSS_F)                                      \
-R(sec_mark_ptype,              1, 0, 0, 1, 0, 1, 0,                           \
-               R_SEC_F | MARK_F | PTYPE_F)                                    \
-R(sec_mark_ptype_rss,          1, 0, 0, 1, 0, 1, 1,                           \
-               R_SEC_F | MARK_F | PTYPE_F | RSS_F)                            \
-R(sec_mark_cksum,              1, 0, 0, 1, 1, 0, 0,                           \
-               R_SEC_F | MARK_F | CKSUM_F)                                    \
-R(sec_mark_cksum_rss,          1, 0, 0, 1, 1, 0, 1,                           \
-               R_SEC_F | MARK_F | CKSUM_F | RSS_F)                            \
-R(sec_mark_cksum_ptype,                1, 0, 0, 1, 1, 1, 0,                           \
-               R_SEC_F | MARK_F | CKSUM_F | PTYPE_F)                          \
-R(sec_mark_cksum_ptype_rss,    1, 0, 0, 1, 1, 1, 1,                           \
-               R_SEC_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)                  \
-R(sec_ts,                      1, 0, 1, 0, 0, 0, 0,                           \
-               R_SEC_F | TS_F)                                                \
-R(sec_ts_rss,                  1, 0, 1, 0, 0, 0, 1,                           \
-               R_SEC_F | TS_F | RSS_F)                                        \
-R(sec_ts_ptype,                        1, 0, 1, 0, 0, 1, 0,                           \
-               R_SEC_F | TS_F | PTYPE_F)                                      \
-R(sec_ts_ptype_rss,            1, 0, 1, 0, 0, 1, 1,                           \
-               R_SEC_F | TS_F | PTYPE_F | RSS_F)                              \
-R(sec_ts_cksum,                        1, 0, 1, 0, 1, 0, 0,                           \
-               R_SEC_F | TS_F | CKSUM_F)                                      \
-R(sec_ts_cksum_rss,            1, 0, 1, 0, 1, 0, 1,                           \
-               R_SEC_F | TS_F | CKSUM_F | RSS_F)                              \
-R(sec_ts_cksum_ptype,          1, 0, 1, 0, 1, 1, 0,                           \
-               R_SEC_F | TS_F | CKSUM_F | PTYPE_F)                            \
-R(sec_ts_cksum_ptype_rss,      1, 0, 1, 0, 1, 1, 1,                           \
-               R_SEC_F | TS_F | CKSUM_F | PTYPE_F | RSS_F)                    \
-R(sec_ts_mark,                 1, 0, 1, 1, 0, 0, 0,                           \
-               R_SEC_F | TS_F | MARK_F)                                       \
-R(sec_ts_mark_rss,             1, 0, 1, 1, 0, 0, 1,                           \
-               R_SEC_F | TS_F | MARK_F | RSS_F)                               \
-R(sec_ts_mark_ptype,           1, 0, 1, 1, 0, 1, 0,                           \
-               R_SEC_F | TS_F | MARK_F | PTYPE_F)                             \
-R(sec_ts_mark_ptype_rss,       1, 0, 1, 1, 0, 1, 1,                           \
-               R_SEC_F | TS_F | MARK_F | PTYPE_F | RSS_F)                     \
-R(sec_ts_mark_cksum,           1, 0, 1, 1, 1, 0, 0,                           \
-               R_SEC_F | TS_F | MARK_F | CKSUM_F)                             \
-R(sec_ts_mark_cksum_rss,       1, 0, 1, 1, 1, 0, 1,                           \
-               R_SEC_F | TS_F | MARK_F | CKSUM_F | RSS_F)                     \
-R(sec_ts_mark_cksum_ptype,     1, 0, 1, 1, 1, 1, 0,                           \
-               R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F)                   \
-R(sec_ts_mark_cksum_ptype_rss, 1, 0, 1, 1, 1, 1, 1,                           \
-               R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)           \
-R(sec_vlan,                    1, 1, 0, 0, 0, 0, 0,                           \
-               R_SEC_F | RX_VLAN_F)                                           \
-R(sec_vlan_rss,                        1, 1, 0, 0, 0, 0, 1,                           \
-               R_SEC_F | RX_VLAN_F | RSS_F)                                   \
-R(sec_vlan_ptype,              1, 1, 0, 0, 0, 1, 0,                           \
-               R_SEC_F | RX_VLAN_F | PTYPE_F)                                 \
-R(sec_vlan_ptype_rss,          1, 1, 0, 0, 0, 1, 1,                           \
-               R_SEC_F | RX_VLAN_F | PTYPE_F | RSS_F)                         \
-R(sec_vlan_cksum,              1, 1, 0, 0, 1, 0, 0,                           \
-               R_SEC_F | RX_VLAN_F | CKSUM_F)                                 \
-R(sec_vlan_cksum_rss,          1, 1, 0, 0, 1, 0, 1,                           \
-               R_SEC_F | RX_VLAN_F | CKSUM_F | RSS_F)                         \
-R(sec_vlan_cksum_ptype,                1, 1, 0, 0, 1, 1, 0,                           \
-               R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F)                       \
-R(sec_vlan_cksum_ptype_rss,    1, 1, 0, 0, 1, 1, 1,                           \
-               R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F)               \
-R(sec_vlan_mark,               1, 1, 0, 1, 0, 0, 0,                           \
-               R_SEC_F | RX_VLAN_F | MARK_F)                                  \
-R(sec_vlan_mark_rss,           1, 1, 0, 1, 0, 0, 1,                           \
-               R_SEC_F | RX_VLAN_F | MARK_F | RSS_F)                          \
-R(sec_vlan_mark_ptype,         1, 1, 0, 1, 0, 1, 0,                           \
-               R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F)                        \
-R(sec_vlan_mark_ptype_rss,     1, 1, 0, 1, 0, 1, 1,                           \
-               R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F | RSS_F)                \
-R(sec_vlan_mark_cksum,         1, 1, 0, 1, 1, 0, 0,                           \
-               R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F)                        \
-R(sec_vlan_mark_cksum_rss,     1, 1, 0, 1, 1, 0, 1,                           \
-               R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | RSS_F)                \
-R(sec_vlan_mark_cksum_ptype,   1, 1, 0, 1, 1, 1, 0,                           \
-               R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F)              \
-R(sec_vlan_mark_cksum_ptype_rss, 1, 1, 0, 1, 1, 1, 1,                         \
-               R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)      \
-R(sec_vlan_ts,                 1, 1, 1, 0, 0, 0, 0,                           \
-               R_SEC_F | RX_VLAN_F | TS_F)                                    \
-R(sec_vlan_ts_rss,             1, 1, 1, 0, 0, 0, 1,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | RSS_F)                            \
-R(sec_vlan_ts_ptype,           1, 1, 1, 0, 0, 1, 0,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F)                          \
-R(sec_vlan_ts_ptype_rss,       1, 1, 1, 0, 0, 1, 1,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F | RSS_F)                  \
-R(sec_vlan_ts_cksum,           1, 1, 1, 0, 1, 0, 0,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F)                          \
-R(sec_vlan_ts_cksum_rss,       1, 1, 1, 0, 1, 0, 1,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | RSS_F)                  \
-R(sec_vlan_ts_cksum_ptype,     1, 1, 1, 0, 1, 1, 0,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F)                \
-R(sec_vlan_ts_cksum_ptype_rss, 1, 1, 1, 0, 1, 1, 1,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F)        \
-R(sec_vlan_ts_mark,            1, 1, 1, 1, 0, 0, 0,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | MARK_F)                           \
-R(sec_vlan_ts_mark_rss,                1, 1, 1, 1, 0, 0, 1,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | MARK_F | RSS_F)                   \
-R(sec_vlan_ts_mark_ptype,      1, 1, 1, 1, 0, 1, 0,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F)                 \
-R(sec_vlan_ts_mark_ptype_rss,  1, 1, 1, 1, 0, 1, 1,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F)         \
-R(sec_vlan_ts_mark_cksum,      1, 1, 1, 1, 1, 0, 0,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F)                 \
-R(sec_vlan_ts_mark_cksum_rss,  1, 1, 1, 1, 1, 0, 1,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F)         \
-R(sec_vlan_ts_mark_cksum_ptype,        1, 1, 1, 1, 1, 1, 0,                           \
-               R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F)       \
-R(sec_vlan_ts_mark_cksum_ptype_rss,    1, 1, 1, 1, 1, 1, 1,                   \
-               R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                            \
+#define NIX_RX_FASTPATH_MODES_0_15                                             \
+       R(no_offload, NIX_RX_OFFLOAD_NONE)                                     \
+       R(rss, RSS_F)                                                          \
+       R(ptype, PTYPE_F)                                                      \
+       R(ptype_rss, PTYPE_F | RSS_F)                                          \
+       R(cksum, CKSUM_F)                                                      \
+       R(cksum_rss, CKSUM_F | RSS_F)                                          \
+       R(cksum_ptype, CKSUM_F | PTYPE_F)                                      \
+       R(cksum_ptype_rss, CKSUM_F | PTYPE_F | RSS_F)                          \
+       R(mark, MARK_F)                                                        \
+       R(mark_rss, MARK_F | RSS_F)                                            \
+       R(mark_ptype, MARK_F | PTYPE_F)                                        \
+       R(mark_ptype_rss, MARK_F | PTYPE_F | RSS_F)                            \
+       R(mark_cksum, MARK_F | CKSUM_F)                                        \
+       R(mark_cksum_rss, MARK_F | CKSUM_F | RSS_F)                            \
+       R(mark_cksum_ptype, MARK_F | CKSUM_F | PTYPE_F)                        \
+       R(mark_cksum_ptype_rss, MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_16_31                                            \
+       R(ts, TS_F)                                                            \
+       R(ts_rss, TS_F | RSS_F)                                                \
+       R(ts_ptype, TS_F | PTYPE_F)                                            \
+       R(ts_ptype_rss, TS_F | PTYPE_F | RSS_F)                                \
+       R(ts_cksum, TS_F | CKSUM_F)                                            \
+       R(ts_cksum_rss, TS_F | CKSUM_F | RSS_F)                                \
+       R(ts_cksum_ptype, TS_F | CKSUM_F | PTYPE_F)                            \
+       R(ts_cksum_ptype_rss, TS_F | CKSUM_F | PTYPE_F | RSS_F)                \
+       R(ts_mark, TS_F | MARK_F)                                              \
+       R(ts_mark_rss, TS_F | MARK_F | RSS_F)                                  \
+       R(ts_mark_ptype, TS_F | MARK_F | PTYPE_F)                              \
+       R(ts_mark_ptype_rss, TS_F | MARK_F | PTYPE_F | RSS_F)                  \
+       R(ts_mark_cksum, TS_F | MARK_F | CKSUM_F)                              \
+       R(ts_mark_cksum_rss, TS_F | MARK_F | CKSUM_F | RSS_F)                  \
+       R(ts_mark_cksum_ptype, TS_F | MARK_F | CKSUM_F | PTYPE_F)              \
+       R(ts_mark_cksum_ptype_rss, TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_32_47                                            \
+       R(vlan, RX_VLAN_F)                                                     \
+       R(vlan_rss, RX_VLAN_F | RSS_F)                                         \
+       R(vlan_ptype, RX_VLAN_F | PTYPE_F)                                     \
+       R(vlan_ptype_rss, RX_VLAN_F | PTYPE_F | RSS_F)                         \
+       R(vlan_cksum, RX_VLAN_F | CKSUM_F)                                     \
+       R(vlan_cksum_rss, RX_VLAN_F | CKSUM_F | RSS_F)                         \
+       R(vlan_cksum_ptype, RX_VLAN_F | CKSUM_F | PTYPE_F)                     \
+       R(vlan_cksum_ptype_rss, RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F)         \
+       R(vlan_mark, RX_VLAN_F | MARK_F)                                       \
+       R(vlan_mark_rss, RX_VLAN_F | MARK_F | RSS_F)                           \
+       R(vlan_mark_ptype, RX_VLAN_F | MARK_F | PTYPE_F)                       \
+       R(vlan_mark_ptype_rss, RX_VLAN_F | MARK_F | PTYPE_F | RSS_F)           \
+       R(vlan_mark_cksum, RX_VLAN_F | MARK_F | CKSUM_F)                       \
+       R(vlan_mark_cksum_rss, RX_VLAN_F | MARK_F | CKSUM_F | RSS_F)           \
+       R(vlan_mark_cksum_ptype, RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F)       \
+       R(vlan_mark_cksum_ptype_rss,                                           \
+         RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_48_63                                            \
+       R(vlan_ts, RX_VLAN_F | TS_F)                                           \
+       R(vlan_ts_rss, RX_VLAN_F | TS_F | RSS_F)                               \
+       R(vlan_ts_ptype, RX_VLAN_F | TS_F | PTYPE_F)                           \
+       R(vlan_ts_ptype_rss, RX_VLAN_F | TS_F | PTYPE_F | RSS_F)               \
+       R(vlan_ts_cksum, RX_VLAN_F | TS_F | CKSUM_F)                           \
+       R(vlan_ts_cksum_rss, RX_VLAN_F | TS_F | CKSUM_F | RSS_F)               \
+       R(vlan_ts_cksum_ptype, RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F)           \
+       R(vlan_ts_cksum_ptype_rss,                                             \
+         RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F)                        \
+       R(vlan_ts_mark, RX_VLAN_F | TS_F | MARK_F)                             \
+       R(vlan_ts_mark_rss, RX_VLAN_F | TS_F | MARK_F | RSS_F)                 \
+       R(vlan_ts_mark_ptype, RX_VLAN_F | TS_F | MARK_F | PTYPE_F)             \
+       R(vlan_ts_mark_ptype_rss, RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \
+       R(vlan_ts_mark_cksum, RX_VLAN_F | TS_F | MARK_F | CKSUM_F)             \
+       R(vlan_ts_mark_cksum_rss, RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \
+       R(vlan_ts_mark_cksum_ptype,                                            \
+         RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F)                       \
+       R(vlan_ts_mark_cksum_ptype_rss,                                        \
+         RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_64_79                                            \
+       R(sec, R_SEC_F)                                                        \
+       R(sec_rss, R_SEC_F | RSS_F)                                            \
+       R(sec_ptype, R_SEC_F | PTYPE_F)                                        \
+       R(sec_ptype_rss, R_SEC_F | PTYPE_F | RSS_F)                            \
+       R(sec_cksum, R_SEC_F | CKSUM_F)                                        \
+       R(sec_cksum_rss, R_SEC_F | CKSUM_F | RSS_F)                            \
+       R(sec_cksum_ptype, R_SEC_F | CKSUM_F | PTYPE_F)                        \
+       R(sec_cksum_ptype_rss, R_SEC_F | CKSUM_F | PTYPE_F | RSS_F)            \
+       R(sec_mark, R_SEC_F | MARK_F)                                          \
+       R(sec_mark_rss, R_SEC_F | MARK_F | RSS_F)                              \
+       R(sec_mark_ptype, R_SEC_F | MARK_F | PTYPE_F)                          \
+       R(sec_mark_ptype_rss, R_SEC_F | MARK_F | PTYPE_F | RSS_F)              \
+       R(sec_mark_cksum, R_SEC_F | MARK_F | CKSUM_F)                          \
+       R(sec_mark_cksum_rss, R_SEC_F | MARK_F | CKSUM_F | RSS_F)              \
+       R(sec_mark_cksum_ptype, R_SEC_F | MARK_F | CKSUM_F | PTYPE_F)          \
+       R(sec_mark_cksum_ptype_rss,                                            \
+         R_SEC_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_80_95                                            \
+       R(sec_ts, R_SEC_F | TS_F)                                              \
+       R(sec_ts_rss, R_SEC_F | TS_F | RSS_F)                                  \
+       R(sec_ts_ptype, R_SEC_F | TS_F | PTYPE_F)                              \
+       R(sec_ts_ptype_rss, R_SEC_F | TS_F | PTYPE_F | RSS_F)                  \
+       R(sec_ts_cksum, R_SEC_F | TS_F | CKSUM_F)                              \
+       R(sec_ts_cksum_rss, R_SEC_F | TS_F | CKSUM_F | RSS_F)                  \
+       R(sec_ts_cksum_ptype, R_SEC_F | TS_F | CKSUM_F | PTYPE_F)              \
+       R(sec_ts_cksum_ptype_rss, R_SEC_F | TS_F | CKSUM_F | PTYPE_F | RSS_F)  \
+       R(sec_ts_mark, R_SEC_F | TS_F | MARK_F)                                \
+       R(sec_ts_mark_rss, R_SEC_F | TS_F | MARK_F | RSS_F)                    \
+       R(sec_ts_mark_ptype, R_SEC_F | TS_F | MARK_F | PTYPE_F)                \
+       R(sec_ts_mark_ptype_rss, R_SEC_F | TS_F | MARK_F | PTYPE_F | RSS_F)    \
+       R(sec_ts_mark_cksum, R_SEC_F | TS_F | MARK_F | CKSUM_F)                \
+       R(sec_ts_mark_cksum_rss, R_SEC_F | TS_F | MARK_F | CKSUM_F | RSS_F)    \
+       R(sec_ts_mark_cksum_ptype,                                             \
+         R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F)                         \
+       R(sec_ts_mark_cksum_ptype_rss,                                         \
+         R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_96_111                                           \
+       R(sec_vlan, R_SEC_F | RX_VLAN_F)                                       \
+       R(sec_vlan_rss, R_SEC_F | RX_VLAN_F | RSS_F)                           \
+       R(sec_vlan_ptype, R_SEC_F | RX_VLAN_F | PTYPE_F)                       \
+       R(sec_vlan_ptype_rss, R_SEC_F | RX_VLAN_F | PTYPE_F | RSS_F)           \
+       R(sec_vlan_cksum, R_SEC_F | RX_VLAN_F | CKSUM_F)                       \
+       R(sec_vlan_cksum_rss, R_SEC_F | RX_VLAN_F | CKSUM_F | RSS_F)           \
+       R(sec_vlan_cksum_ptype, R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F)       \
+       R(sec_vlan_cksum_ptype_rss,                                            \
+         R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F)                     \
+       R(sec_vlan_mark, R_SEC_F | RX_VLAN_F | MARK_F)                         \
+       R(sec_vlan_mark_rss, R_SEC_F | RX_VLAN_F | MARK_F | RSS_F)             \
+       R(sec_vlan_mark_ptype, R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F)         \
+       R(sec_vlan_mark_ptype_rss,                                             \
+         R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F | RSS_F)                      \
+       R(sec_vlan_mark_cksum, R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F)         \
+       R(sec_vlan_mark_cksum_rss,                                             \
+         R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | RSS_F)                      \
+       R(sec_vlan_mark_cksum_ptype,                                           \
+         R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F)                    \
+       R(sec_vlan_mark_cksum_ptype_rss,                                       \
+         R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_112_127                                          \
+       R(sec_vlan_ts, R_SEC_F | RX_VLAN_F | TS_F)                             \
+       R(sec_vlan_ts_rss, R_SEC_F | RX_VLAN_F | TS_F | RSS_F)                 \
+       R(sec_vlan_ts_ptype, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F)             \
+       R(sec_vlan_ts_ptype_rss, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \
+       R(sec_vlan_ts_cksum, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F)             \
+       R(sec_vlan_ts_cksum_rss, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \
+       R(sec_vlan_ts_cksum_ptype,                                             \
+         R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F)                      \
+       R(sec_vlan_ts_cksum_ptype_rss,                                         \
+         R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F)              \
+       R(sec_vlan_ts_mark, R_SEC_F | RX_VLAN_F | TS_F | MARK_F)               \
+       R(sec_vlan_ts_mark_rss, R_SEC_F | RX_VLAN_F | TS_F | MARK_F | RSS_F)   \
+       R(sec_vlan_ts_mark_ptype,                                              \
+         R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F)                       \
+       R(sec_vlan_ts_mark_ptype_rss,                                          \
+         R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F)               \
+       R(sec_vlan_ts_mark_cksum,                                              \
+         R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F)                       \
+       R(sec_vlan_ts_mark_cksum_rss,                                          \
+         R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F)               \
+       R(sec_vlan_ts_mark_cksum_ptype,                                        \
+         R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F)             \
+       R(sec_vlan_ts_mark_cksum_ptype_rss,                                    \
+         R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES                                                  \
+       NIX_RX_FASTPATH_MODES_0_15                                             \
+       NIX_RX_FASTPATH_MODES_16_31                                            \
+       NIX_RX_FASTPATH_MODES_32_47                                            \
+       NIX_RX_FASTPATH_MODES_48_63                                            \
+       NIX_RX_FASTPATH_MODES_64_79                                            \
+       NIX_RX_FASTPATH_MODES_80_95                                            \
+       NIX_RX_FASTPATH_MODES_96_111                                           \
+       NIX_RX_FASTPATH_MODES_112_127
+
+#define R(name, flags)                                                         \
        uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_##name(          \
                void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
-                                                                              \
        uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_mseg_##name(     \
                void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
-                                                                              \
        uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_##name(      \
                void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);     \
-                                                                              \
        uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_mseg_##name( \
                void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);
 
 NIX_RX_FASTPATH_MODES
 #undef R
 
+#define NIX_RX_RECV(fn, flags)                                                 \
+       uint16_t __rte_noinline __rte_hot fn(                                  \
+               void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)      \
+       {                                                                      \
+               return cn10k_nix_recv_pkts(rx_queue, rx_pkts, pkts, (flags));  \
+       }
+
+#define NIX_RX_RECV_MSEG(fn, flags) NIX_RX_RECV(fn, flags | NIX_RX_MULTI_SEG_F)
+
+#define NIX_RX_RECV_VEC(fn, flags)                                             \
+       uint16_t __rte_noinline __rte_hot fn(                                  \
+               void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)      \
+       {                                                                      \
+               return cn10k_nix_recv_pkts_vector(rx_queue, rx_pkts, pkts,     \
+                                                 (flags), NULL, NULL, 0);     \
+       }
+
+#define NIX_RX_RECV_VEC_MSEG(fn, flags)                                        \
+       NIX_RX_RECV_VEC(fn, flags | NIX_RX_MULTI_SEG_F)
+
 #endif /* __CN10K_RX_H__ */

diff --git a/drivers/net/cnxk/cn10k_rx_mseg.c b/drivers/net/cnxk/cn10k_rx_mseg.c
deleted file mode 100644 (file)
index e7c2321..0000000
--- a/drivers/net/cnxk/cn10k_rx_mseg.c
+++ /dev/null
@@ -1,17 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn10k_ethdev.h"
-#include "cn10k_rx.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_mseg_##name(     \
-               void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)      \
-       {                                                                      \
-               return cn10k_nix_recv_pkts(rx_queue, rx_pkts, pkts,            \
-                                          (flags) | NIX_RX_MULTI_SEG_F);      \
-       }
-
-NIX_RX_FASTPATH_MODES
-#undef R

diff --git a/drivers/net/cnxk/cn10k_rx_select.c b/drivers/net/cnxk/cn10k_rx_select.c
new file mode 100644 (file)
index 0000000..5d9be0a
--- /dev/null
+++ b/drivers/net/cnxk/cn10k_rx_select.c
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include "cn10k_ethdev.h"
+#include "cn10k_rx.h"
+
+static inline void
+pick_rx_func(struct rte_eth_dev *eth_dev,
+            const eth_rx_burst_t rx_burst[NIX_RX_OFFLOAD_MAX])
+{
+       struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
+
+       /* [VLAN] [TSP] [MARK] [CKSUM] [PTYPE] [RSS] */
+       eth_dev->rx_pkt_burst =
+               rx_burst[dev->rx_offload_flags & (NIX_RX_OFFLOAD_MAX - 1)];
+
+       rte_atomic_thread_fence(__ATOMIC_RELEASE);
+}
+
+void
+cn10k_eth_set_rx_function(struct rte_eth_dev *eth_dev)
+{
+       struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
+
+       const eth_rx_burst_t nix_eth_rx_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_nix_recv_pkts_##name,
+
+               NIX_RX_FASTPATH_MODES
+#undef R
+       };
+
+       const eth_rx_burst_t nix_eth_rx_burst_mseg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_nix_recv_pkts_mseg_##name,
+
+               NIX_RX_FASTPATH_MODES
+#undef R
+       };
+
+       const eth_rx_burst_t nix_eth_rx_vec_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_nix_recv_pkts_vec_##name,
+
+               NIX_RX_FASTPATH_MODES
+#undef R
+       };
+
+       const eth_rx_burst_t nix_eth_rx_vec_burst_mseg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags)[flags] = cn10k_nix_recv_pkts_vec_mseg_##name,
+
+               NIX_RX_FASTPATH_MODES
+#undef R
+       };
+
+       /* Copy multi seg version with no offload for tear down sequence */
+       if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+               dev->rx_pkt_burst_no_offload = nix_eth_rx_burst_mseg[0];
+
+       if (dev->scalar_ena) {
+               if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
+                       return pick_rx_func(eth_dev, nix_eth_rx_burst_mseg);
+               return pick_rx_func(eth_dev, nix_eth_rx_burst);
+       }
+
+       if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
+               return pick_rx_func(eth_dev, nix_eth_rx_vec_burst_mseg);
+       return pick_rx_func(eth_dev, nix_eth_rx_vec_burst);
+}

diff --git a/drivers/net/cnxk/cn10k_rx_vec.c b/drivers/net/cnxk/cn10k_rx_vec.c
deleted file mode 100644 (file)
index 0ccc4df..0000000
--- a/drivers/net/cnxk/cn10k_rx_vec.c
+++ /dev/null
@@ -1,19 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn10k_ethdev.h"
-#include "cn10k_rx.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                            \
-       uint16_t __rte_noinline __rte_hot                                      \
-               cn10k_nix_recv_pkts_vec_##name(void *rx_queue,                 \
-                                              struct rte_mbuf **rx_pkts,      \
-                                              uint16_t pkts)                  \
-       {                                                                      \
-               return cn10k_nix_recv_pkts_vector(rx_queue, rx_pkts, pkts,     \
-                                                 (flags), NULL, NULL, 0);     \
-       }
-
-NIX_RX_FASTPATH_MODES
-#undef R

diff --git a/drivers/net/cnxk/cn10k_rx_vec_mseg.c b/drivers/net/cnxk/cn10k_rx_vec_mseg.c
deleted file mode 100644 (file)
index 38e0ec3..0000000
--- a/drivers/net/cnxk/cn10k_rx_vec_mseg.c
+++ /dev/null
@@ -1,18 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn10k_ethdev.h"
-#include "cn10k_rx.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
-       uint16_t __rte_noinline __rte_hot cn10k_nix_recv_pkts_vec_mseg_##name( \
-               void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)      \
-       {                                                                      \
-               return cn10k_nix_recv_pkts_vector(                             \
-                       rx_queue, rx_pkts, pkts, (flags) | NIX_RX_MULTI_SEG_F, \
-                       NULL, NULL, 0);                                        \
-       }
-
-NIX_RX_FASTPATH_MODES
-#undef R

diff --git a/drivers/net/cnxk/meson.build b/drivers/net/cnxk/meson.build
index 01bd5442b53febfdaa471549d5b8998bbe3d24ae..1fd388a1f00cbf38c370fac72dee5fe984837a8a 100644 (file)
--- a/drivers/net/cnxk/meson.build
+++ b/drivers/net/cnxk/meson.build
@@ -107,14 +107,46 @@ sources += files(
         'cn10k_ethdev.c',
         'cn10k_ethdev_sec.c',
         'cn10k_rte_flow.c',
-        'cn10k_rx.c',
-        'cn10k_rx_mseg.c',
-        'cn10k_rx_vec.c',
-        'cn10k_rx_vec_mseg.c',
         'cn10k_tx.c',
         'cn10k_tx_mseg.c',
         'cn10k_tx_vec.c',
         'cn10k_tx_vec_mseg.c',
+        'cn10k_rx_select.c',
+)
+
+sources += files(
+        'rx/cn10k/rx_0_15.c',
+        'rx/cn10k/rx_16_31.c',
+        'rx/cn10k/rx_32_47.c',
+        'rx/cn10k/rx_48_63.c',
+        'rx/cn10k/rx_64_79.c',
+        'rx/cn10k/rx_80_95.c',
+        'rx/cn10k/rx_96_111.c',
+        'rx/cn10k/rx_112_127.c',
+        'rx/cn10k/rx_0_15_mseg.c',
+        'rx/cn10k/rx_16_31_mseg.c',
+        'rx/cn10k/rx_32_47_mseg.c',
+        'rx/cn10k/rx_48_63_mseg.c',
+        'rx/cn10k/rx_64_79_mseg.c',
+        'rx/cn10k/rx_80_95_mseg.c',
+        'rx/cn10k/rx_96_111_mseg.c',
+        'rx/cn10k/rx_112_127_mseg.c',
+        'rx/cn10k/rx_0_15_vec.c',
+        'rx/cn10k/rx_16_31_vec.c',
+        'rx/cn10k/rx_32_47_vec.c',
+        'rx/cn10k/rx_48_63_vec.c',
+        'rx/cn10k/rx_64_79_vec.c',
+        'rx/cn10k/rx_80_95_vec.c',
+        'rx/cn10k/rx_96_111_vec.c',
+        'rx/cn10k/rx_112_127_vec.c',
+        'rx/cn10k/rx_0_15_vec_mseg.c',
+        'rx/cn10k/rx_16_31_vec_mseg.c',
+        'rx/cn10k/rx_32_47_vec_mseg.c',
+        'rx/cn10k/rx_48_63_vec_mseg.c',
+        'rx/cn10k/rx_64_79_vec_mseg.c',
+        'rx/cn10k/rx_80_95_vec_mseg.c',
+        'rx/cn10k/rx_96_111_vec_mseg.c',
+        'rx/cn10k/rx_112_127_vec_mseg.c',
 )
 
 deps += ['bus_pci', 'cryptodev', 'eventdev', 'security']