drivers/event/cnxk/cn10k_worker.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #ifndef __CN10K_WORKER_H__
   6 #define __CN10K_WORKER_H__
   7
   8 #include "cnxk_ethdev.h"
   9 #include "cnxk_eventdev.h"
  10 #include "cnxk_worker.h"
  11
  12 #include "cn10k_ethdev.h"
  13 #include "cn10k_rx.h"
  14
  15 /* SSO Operations */
  16
  17 static __rte_always_inline uint8_t
  18 cn10k_sso_hws_new_event(struct cn10k_sso_hws *ws, const struct rte_event *ev)
  19 {
  20         const uint32_t tag = (uint32_t)ev->event;
  21         const uint8_t new_tt = ev->sched_type;
  22         const uint64_t event_ptr = ev->u64;
  23         const uint16_t grp = ev->queue_id;
  24
  25         rte_atomic_thread_fence(__ATOMIC_ACQ_REL);
  26         if (ws->xaq_lmt <= *ws->fc_mem)
  27                 return 0;
  28
  29         cnxk_sso_hws_add_work(event_ptr, tag, new_tt, ws->grps_base[grp]);
  30         return 1;
  31 }
  32
  33 static __rte_always_inline void
  34 cn10k_sso_hws_fwd_swtag(struct cn10k_sso_hws *ws, const struct rte_event *ev)
  35 {
  36         const uint32_t tag = (uint32_t)ev->event;
  37         const uint8_t new_tt = ev->sched_type;
  38         const uint8_t cur_tt =
  39                 CNXK_TT_FROM_TAG(plt_read64(ws->base + SSOW_LF_GWS_WQE0));
  40
  41         /* CNXK model
  42          * cur_tt/new_tt     SSO_TT_ORDERED SSO_TT_ATOMIC SSO_TT_UNTAGGED
  43          *
  44          * SSO_TT_ORDERED        norm           norm             untag
  45          * SSO_TT_ATOMIC         norm           norm               untag
  46          * SSO_TT_UNTAGGED       norm           norm             NOOP
  47          */
  48
  49         if (new_tt == SSO_TT_UNTAGGED) {
  50                 if (cur_tt != SSO_TT_UNTAGGED)
  51                         cnxk_sso_hws_swtag_untag(ws->base +
  52                                                  SSOW_LF_GWS_OP_SWTAG_UNTAG);
  53         } else {
  54                 cnxk_sso_hws_swtag_norm(tag, new_tt,
  55                                         ws->base + SSOW_LF_GWS_OP_SWTAG_NORM);
  56         }
  57         ws->swtag_req = 1;
  58 }
  59
  60 static __rte_always_inline void
  61 cn10k_sso_hws_fwd_group(struct cn10k_sso_hws *ws, const struct rte_event *ev,
  62                         const uint16_t grp)
  63 {
  64         const uint32_t tag = (uint32_t)ev->event;
  65         const uint8_t new_tt = ev->sched_type;
  66
  67         plt_write64(ev->u64, ws->base + SSOW_LF_GWS_OP_UPD_WQP_GRP1);
  68         cnxk_sso_hws_swtag_desched(tag, new_tt, grp,
  69                                    ws->base + SSOW_LF_GWS_OP_SWTAG_DESCHED);
  70 }
  71
  72 static __rte_always_inline void
  73 cn10k_sso_hws_forward_event(struct cn10k_sso_hws *ws,
  74                             const struct rte_event *ev)
  75 {
  76         const uint8_t grp = ev->queue_id;
  77
  78         /* Group hasn't changed, Use SWTAG to forward the event */
  79         if (CNXK_GRP_FROM_TAG(plt_read64(ws->base + SSOW_LF_GWS_WQE0)) == grp)
  80                 cn10k_sso_hws_fwd_swtag(ws, ev);
  81         else
  82                 /*
  83                  * Group has been changed for group based work pipelining,
  84                  * Use deschedule/add_work operation to transfer the event to
  85                  * new group/core
  86                  */
  87                 cn10k_sso_hws_fwd_group(ws, ev, grp);
  88 }
  89
  90 static __rte_always_inline uint16_t
  91 cn10k_sso_hws_get_work(struct cn10k_sso_hws *ws, struct rte_event *ev)
  92 {
  93         union {
  94                 __uint128_t get_work;
  95                 uint64_t u64[2];
  96         } gw;
  97
  98         gw.get_work = ws->gw_wdata;
  99 #if defined(RTE_ARCH_ARM64) && !defined(__clang__)
 100         asm volatile(
 101                 PLT_CPU_FEATURE_PREAMBLE
 102                 "caspl %[wdata], %H[wdata], %[wdata], %H[wdata], [%[gw_loc]]\n"
 103                 : [wdata] "+r"(gw.get_work)
 104                 : [gw_loc] "r"(ws->base + SSOW_LF_GWS_OP_GET_WORK0)
 105                 : "memory");
 106 #else
 107         plt_write64(gw.u64[0], ws->base + SSOW_LF_GWS_OP_GET_WORK0);
 108         do {
 109                 roc_load_pair(gw.u64[0], gw.u64[1],
 110                               ws->base + SSOW_LF_GWS_WQE0);
 111         } while (gw.u64[0] & BIT_ULL(63));
 112 #endif
 113         gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 |
 114                     (gw.u64[0] & (0x3FFull << 36)) << 4 |
 115                     (gw.u64[0] & 0xffffffff);
 116
 117         ev->event = gw.u64[0];
 118         ev->u64 = gw.u64[1];
 119
 120         return !!gw.u64[1];
 121 }
 122
 123 /* Used in cleaning up workslot. */
 124 static __rte_always_inline uint16_t
 125 cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev)
 126 {
 127         union {
 128                 __uint128_t get_work;
 129                 uint64_t u64[2];
 130         } gw;
 131
 132 #ifdef RTE_ARCH_ARM64
 133         asm volatile(PLT_CPU_FEATURE_PREAMBLE
 134                      "          ldp %[tag], %[wqp], [%[tag_loc]]        \n"
 135                      "          tbz %[tag], 63, done%=                  \n"
 136                      "          sevl                                    \n"
 137                      "rty%=:    wfe                                     \n"
 138                      "          ldp %[tag], %[wqp], [%[tag_loc]]        \n"
 139                      "          tbnz %[tag], 63, rty%=                  \n"
 140                      "done%=:   dmb ld                                  \n"
 141                      : [tag] "=&r"(gw.u64[0]), [wqp] "=&r"(gw.u64[1])
 142                      : [tag_loc] "r"(ws->base + SSOW_LF_GWS_WQE0)
 143                      : "memory");
 144 #else
 145         do {
 146                 roc_load_pair(gw.u64[0], gw.u64[1],
 147                               ws->base + SSOW_LF_GWS_WQE0);
 148         } while (gw.u64[0] & BIT_ULL(63));
 149 #endif
 150
 151         gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 |
 152                     (gw.u64[0] & (0x3FFull << 36)) << 4 |
 153                     (gw.u64[0] & 0xffffffff);
 154
 155         ev->event = gw.u64[0];
 156         ev->u64 = gw.u64[1];
 157
 158         return !!gw.u64[1];
 159 }
 160
 161 /* CN10K Fastpath functions. */
 162 uint16_t __rte_hot cn10k_sso_hws_enq(void *port, const struct rte_event *ev);
 163 uint16_t __rte_hot cn10k_sso_hws_enq_burst(void *port,
 164                                            const struct rte_event ev[],
 165                                            uint16_t nb_events);
 166 uint16_t __rte_hot cn10k_sso_hws_enq_new_burst(void *port,
 167                                                const struct rte_event ev[],
 168                                                uint16_t nb_events);
 169 uint16_t __rte_hot cn10k_sso_hws_enq_fwd_burst(void *port,
 170                                                const struct rte_event ev[],
 171                                                uint16_t nb_events);
 172
 173 uint16_t __rte_hot cn10k_sso_hws_deq(void *port, struct rte_event *ev,
 174                                      uint64_t timeout_ticks);
 175 uint16_t __rte_hot cn10k_sso_hws_deq_burst(void *port, struct rte_event ev[],
 176                                            uint16_t nb_events,
 177                                            uint64_t timeout_ticks);
 178 uint16_t __rte_hot cn10k_sso_hws_tmo_deq(void *port, struct rte_event *ev,
 179                                          uint64_t timeout_ticks);
 180 uint16_t __rte_hot cn10k_sso_hws_tmo_deq_burst(void *port,
 181                                                struct rte_event ev[],
 182                                                uint16_t nb_events,
 183                                                uint64_t timeout_ticks);
 184
 185 #endif