drivers/event/cnxk/cn10k_worker.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #ifndef __CN10K_WORKER_H__
   6 #define __CN10K_WORKER_H__
   7
   8 #include "cnxk_eventdev.h"
   9 #include "cnxk_worker.h"
  10
  11 /* SSO Operations */
  12
  13 static __rte_always_inline uint8_t
  14 cn10k_sso_hws_new_event(struct cn10k_sso_hws *ws, const struct rte_event *ev)
  15 {
  16         const uint32_t tag = (uint32_t)ev->event;
  17         const uint8_t new_tt = ev->sched_type;
  18         const uint64_t event_ptr = ev->u64;
  19         const uint16_t grp = ev->queue_id;
  20
  21         rte_atomic_thread_fence(__ATOMIC_ACQ_REL);
  22         if (ws->xaq_lmt <= *ws->fc_mem)
  23                 return 0;
  24
  25         cnxk_sso_hws_add_work(event_ptr, tag, new_tt, ws->grps_base[grp]);
  26         return 1;
  27 }
  28
  29 static __rte_always_inline void
  30 cn10k_sso_hws_fwd_swtag(struct cn10k_sso_hws *ws, const struct rte_event *ev)
  31 {
  32         const uint32_t tag = (uint32_t)ev->event;
  33         const uint8_t new_tt = ev->sched_type;
  34         const uint8_t cur_tt = CNXK_TT_FROM_TAG(plt_read64(ws->tag_wqe_op));
  35
  36         /* CNXK model
  37          * cur_tt/new_tt     SSO_TT_ORDERED SSO_TT_ATOMIC SSO_TT_UNTAGGED
  38          *
  39          * SSO_TT_ORDERED        norm           norm             untag
  40          * SSO_TT_ATOMIC         norm           norm               untag
  41          * SSO_TT_UNTAGGED       norm           norm             NOOP
  42          */
  43
  44         if (new_tt == SSO_TT_UNTAGGED) {
  45                 if (cur_tt != SSO_TT_UNTAGGED)
  46                         cnxk_sso_hws_swtag_untag(ws->swtag_untag_op);
  47         } else {
  48                 cnxk_sso_hws_swtag_norm(tag, new_tt, ws->swtag_norm_op);
  49         }
  50         ws->swtag_req = 1;
  51 }
  52
  53 static __rte_always_inline void
  54 cn10k_sso_hws_fwd_group(struct cn10k_sso_hws *ws, const struct rte_event *ev,
  55                         const uint16_t grp)
  56 {
  57         const uint32_t tag = (uint32_t)ev->event;
  58         const uint8_t new_tt = ev->sched_type;
  59
  60         plt_write64(ev->u64, ws->updt_wqe_op);
  61         cnxk_sso_hws_swtag_desched(tag, new_tt, grp, ws->swtag_desched_op);
  62 }
  63
  64 static __rte_always_inline void
  65 cn10k_sso_hws_forward_event(struct cn10k_sso_hws *ws,
  66                             const struct rte_event *ev)
  67 {
  68         const uint8_t grp = ev->queue_id;
  69
  70         /* Group hasn't changed, Use SWTAG to forward the event */
  71         if (CNXK_GRP_FROM_TAG(plt_read64(ws->tag_wqe_op)) == grp)
  72                 cn10k_sso_hws_fwd_swtag(ws, ev);
  73         else
  74                 /*
  75                  * Group has been changed for group based work pipelining,
  76                  * Use deschedule/add_work operation to transfer the event to
  77                  * new group/core
  78                  */
  79                 cn10k_sso_hws_fwd_group(ws, ev, grp);
  80 }
  81
  82 static __rte_always_inline uint16_t
  83 cn10k_sso_hws_get_work(struct cn10k_sso_hws *ws, struct rte_event *ev)
  84 {
  85         union {
  86                 __uint128_t get_work;
  87                 uint64_t u64[2];
  88         } gw;
  89
  90         gw.get_work = ws->gw_wdata;
  91 #if defined(RTE_ARCH_ARM64) && !defined(__clang__)
  92         asm volatile(
  93                 PLT_CPU_FEATURE_PREAMBLE
  94                 "caspl %[wdata], %H[wdata], %[wdata], %H[wdata], [%[gw_loc]]\n"
  95                 : [wdata] "+r"(gw.get_work)
  96                 : [gw_loc] "r"(ws->getwrk_op)
  97                 : "memory");
  98 #else
  99         plt_write64(gw.u64[0], ws->getwrk_op);
 100         do {
 101                 roc_load_pair(gw.u64[0], gw.u64[1], ws->tag_wqe_op);
 102         } while (gw.u64[0] & BIT_ULL(63));
 103 #endif
 104         gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 |
 105                     (gw.u64[0] & (0x3FFull << 36)) << 4 |
 106                     (gw.u64[0] & 0xffffffff);
 107
 108         ev->event = gw.u64[0];
 109         ev->u64 = gw.u64[1];
 110
 111         return !!gw.u64[1];
 112 }
 113
 114 /* Used in cleaning up workslot. */
 115 static __rte_always_inline uint16_t
 116 cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev)
 117 {
 118         union {
 119                 __uint128_t get_work;
 120                 uint64_t u64[2];
 121         } gw;
 122
 123 #ifdef RTE_ARCH_ARM64
 124         asm volatile(PLT_CPU_FEATURE_PREAMBLE
 125                      "          ldp %[tag], %[wqp], [%[tag_loc]]        \n"
 126                      "          tbz %[tag], 63, done%=                  \n"
 127                      "          sevl                                    \n"
 128                      "rty%=:    wfe                                     \n"
 129                      "          ldp %[tag], %[wqp], [%[tag_loc]]        \n"
 130                      "          tbnz %[tag], 63, rty%=                  \n"
 131                      "done%=:   dmb ld                                  \n"
 132                      : [tag] "=&r"(gw.u64[0]), [wqp] "=&r"(gw.u64[1])
 133                      : [tag_loc] "r"(ws->tag_wqe_op)
 134                      : "memory");
 135 #else
 136         do {
 137                 roc_load_pair(gw.u64[0], gw.u64[1], ws->tag_wqe_op);
 138         } while (gw.u64[0] & BIT_ULL(63));
 139 #endif
 140
 141         gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 |
 142                     (gw.u64[0] & (0x3FFull << 36)) << 4 |
 143                     (gw.u64[0] & 0xffffffff);
 144
 145         ev->event = gw.u64[0];
 146         ev->u64 = gw.u64[1];
 147
 148         return !!gw.u64[1];
 149 }
 150
 151 /* CN10K Fastpath functions. */
 152 uint16_t __rte_hot cn10k_sso_hws_enq(void *port, const struct rte_event *ev);
 153 uint16_t __rte_hot cn10k_sso_hws_enq_burst(void *port,
 154                                            const struct rte_event ev[],
 155                                            uint16_t nb_events);
 156 uint16_t __rte_hot cn10k_sso_hws_enq_new_burst(void *port,
 157                                                const struct rte_event ev[],
 158                                                uint16_t nb_events);
 159 uint16_t __rte_hot cn10k_sso_hws_enq_fwd_burst(void *port,
 160                                                const struct rte_event ev[],
 161                                                uint16_t nb_events);
 162
 163 uint16_t __rte_hot cn10k_sso_hws_deq(void *port, struct rte_event *ev,
 164                                      uint64_t timeout_ticks);
 165 uint16_t __rte_hot cn10k_sso_hws_deq_burst(void *port, struct rte_event ev[],
 166                                            uint16_t nb_events,
 167                                            uint64_t timeout_ticks);
 168 uint16_t __rte_hot cn10k_sso_hws_tmo_deq(void *port, struct rte_event *ev,
 169                                          uint64_t timeout_ticks);
 170 uint16_t __rte_hot cn10k_sso_hws_tmo_deq_burst(void *port,
 171                                                struct rte_event ev[],
 172                                                uint16_t nb_events,
 173                                                uint64_t timeout_ticks);
 174
 175 #endif