drivers/event/octeontx2/otx2_worker.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #ifndef __OTX2_WORKER_H__
   6 #define __OTX2_WORKER_H__
   7
   8 #include <rte_common.h>
   9 #include <rte_branch_prediction.h>
  10
  11 #include <otx2_common.h>
  12 #include "otx2_evdev.h"
  13
  14 /* SSO Operations */
  15
  16 static __rte_always_inline uint16_t
  17 otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev,
  18                      const uint32_t flags, const void * const lookup_mem)
  19 {
  20         union otx2_sso_event event;
  21         uint64_t get_work1;
  22         uint64_t mbuf;
  23
  24         otx2_write64(BIT_ULL(16) | /* wait for work. */
  25                      1, /* Use Mask set 0. */
  26                      ws->getwrk_op);
  27
  28         if (flags & NIX_RX_OFFLOAD_PTYPE_F)
  29                 rte_prefetch_non_temporal(lookup_mem);
  30 #ifdef RTE_ARCH_ARM64
  31         asm volatile(
  32                         "               ldr %[tag], [%[tag_loc]]        \n"
  33                         "               ldr %[wqp], [%[wqp_loc]]        \n"
  34                         "               tbz %[tag], 63, done%=          \n"
  35                         "               sevl                            \n"
  36                         "rty%=:         wfe                             \n"
  37                         "               ldr %[tag], [%[tag_loc]]        \n"
  38                         "               ldr %[wqp], [%[wqp_loc]]        \n"
  39                         "               tbnz %[tag], 63, rty%=          \n"
  40                         "done%=:        dmb ld                          \n"
  41                         "               prfm pldl1keep, [%[wqp], #8]    \n"
  42                         "               sub %[mbuf], %[wqp], #0x80      \n"
  43                         "               prfm pldl1keep, [%[mbuf]]       \n"
  44                         : [tag] "=&r" (event.get_work0),
  45                           [wqp] "=&r" (get_work1),
  46                           [mbuf] "=&r" (mbuf)
  47                         : [tag_loc] "r" (ws->tag_op),
  48                           [wqp_loc] "r" (ws->wqp_op)
  49                         );
  50 #else
  51         event.get_work0 = otx2_read64(ws->tag_op);
  52         while ((BIT_ULL(63)) & event.get_work0)
  53                 event.get_work0 = otx2_read64(ws->tag_op);
  54
  55         get_work1 = otx2_read64(ws->wqp_op);
  56         rte_prefetch0((const void *)get_work1);
  57         mbuf = (uint64_t)((char *)get_work1 - sizeof(struct rte_mbuf));
  58         rte_prefetch0((const void *)mbuf);
  59 #endif
  60
  61         event.get_work0 = (event.get_work0 & (0x3ull << 32)) << 6 |
  62                 (event.get_work0 & (0x3FFull << 36)) << 4 |
  63                 (event.get_work0 & 0xffffffff);
  64         ws->cur_tt = event.sched_type;
  65         ws->cur_grp = event.queue_id;
  66
  67         if (event.sched_type != SSO_TT_EMPTY &&
  68             event.event_type == RTE_EVENT_TYPE_ETHDEV) {
  69                 otx2_wqe_to_mbuf(get_work1, mbuf, event.sub_event_type,
  70                                  (uint32_t) event.get_work0, flags, lookup_mem);
  71                 /* Extracting tstamp, if PTP enabled*/
  72                 otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp,
  73                                         flags);
  74                 get_work1 = mbuf;
  75         }
  76
  77         ev->event = event.get_work0;
  78         ev->u64 = get_work1;
  79
  80         return !!get_work1;
  81 }
  82
  83 /* Used in cleaning up workslot. */
  84 static __rte_always_inline uint16_t
  85 otx2_ssogws_get_work_empty(struct otx2_ssogws *ws, struct rte_event *ev,
  86                            const uint32_t flags)
  87 {
  88         union otx2_sso_event event;
  89         uint64_t get_work1;
  90         uint64_t mbuf;
  91
  92 #ifdef RTE_ARCH_ARM64
  93         asm volatile(
  94                         "               ldr %[tag], [%[tag_loc]]        \n"
  95                         "               ldr %[wqp], [%[wqp_loc]]        \n"
  96                         "               tbz %[tag], 63, done%=          \n"
  97                         "               sevl                            \n"
  98                         "rty%=:         wfe                             \n"
  99                         "               ldr %[tag], [%[tag_loc]]        \n"
 100                         "               ldr %[wqp], [%[wqp_loc]]        \n"
 101                         "               tbnz %[tag], 63, rty%=          \n"
 102                         "done%=:        dmb ld                          \n"
 103                         "               prfm pldl1keep, [%[wqp], #8]    \n"
 104                         "               sub %[mbuf], %[wqp], #0x80      \n"
 105                         "               prfm pldl1keep, [%[mbuf]]       \n"
 106                         : [tag] "=&r" (event.get_work0),
 107                           [wqp] "=&r" (get_work1),
 108                           [mbuf] "=&r" (mbuf)
 109                         : [tag_loc] "r" (ws->tag_op),
 110                           [wqp_loc] "r" (ws->wqp_op)
 111                         );
 112 #else
 113         event.get_work0 = otx2_read64(ws->tag_op);
 114         while ((BIT_ULL(63)) & event.get_work0)
 115                 event.get_work0 = otx2_read64(ws->tag_op);
 116
 117         get_work1 = otx2_read64(ws->wqp_op);
 118         rte_prefetch_non_temporal((const void *)get_work1);
 119         mbuf = (uint64_t)((char *)get_work1 - sizeof(struct rte_mbuf));
 120         rte_prefetch_non_temporal((const void *)mbuf);
 121 #endif
 122
 123         event.get_work0 = (event.get_work0 & (0x3ull << 32)) << 6 |
 124                 (event.get_work0 & (0x3FFull << 36)) << 4 |
 125                 (event.get_work0 & 0xffffffff);
 126         ws->cur_tt = event.sched_type;
 127         ws->cur_grp = event.queue_id;
 128
 129         if (event.sched_type != SSO_TT_EMPTY &&
 130             event.event_type == RTE_EVENT_TYPE_ETHDEV) {
 131                 otx2_wqe_to_mbuf(get_work1, mbuf, event.sub_event_type,
 132                                  (uint32_t) event.get_work0, flags, NULL);
 133                 /* Extracting tstamp, if PTP enabled*/
 134                 otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp,
 135                                         flags);
 136                 get_work1 = mbuf;
 137         }
 138
 139         ev->event = event.get_work0;
 140         ev->u64 = get_work1;
 141
 142         return !!get_work1;
 143 }
 144
 145 static __rte_always_inline void
 146 otx2_ssogws_add_work(struct otx2_ssogws *ws, const uint64_t event_ptr,
 147                      const uint32_t tag, const uint8_t new_tt,
 148                      const uint16_t grp)
 149 {
 150         uint64_t add_work0;
 151
 152         add_work0 = tag | ((uint64_t)(new_tt) << 32);
 153         otx2_store_pair(add_work0, event_ptr, ws->grps_base[grp]);
 154 }
 155
 156 static __rte_always_inline void
 157 otx2_ssogws_swtag_desched(struct otx2_ssogws *ws, uint32_t tag, uint8_t new_tt,
 158                           uint16_t grp)
 159 {
 160         uint64_t val;
 161
 162         val = tag | ((uint64_t)(new_tt & 0x3) << 32) | ((uint64_t)grp << 34);
 163         otx2_write64(val, ws->swtag_desched_op);
 164 }
 165
 166 static __rte_always_inline void
 167 otx2_ssogws_swtag_norm(struct otx2_ssogws *ws, uint32_t tag, uint8_t new_tt)
 168 {
 169         uint64_t val;
 170
 171         val = tag | ((uint64_t)(new_tt & 0x3) << 32);
 172         otx2_write64(val, ws->swtag_norm_op);
 173 }
 174
 175 static __rte_always_inline void
 176 otx2_ssogws_swtag_untag(struct otx2_ssogws *ws)
 177 {
 178         otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
 179                      SSOW_LF_GWS_OP_SWTAG_UNTAG);
 180         ws->cur_tt = SSO_SYNC_UNTAGGED;
 181 }
 182
 183 static __rte_always_inline void
 184 otx2_ssogws_swtag_flush(struct otx2_ssogws *ws)
 185 {
 186         otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
 187                      SSOW_LF_GWS_OP_SWTAG_FLUSH);
 188         ws->cur_tt = SSO_SYNC_EMPTY;
 189 }
 190
 191 static __rte_always_inline void
 192 otx2_ssogws_desched(struct otx2_ssogws *ws)
 193 {
 194         otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
 195                      SSOW_LF_GWS_OP_DESCHED);
 196 }
 197
 198 static __rte_always_inline void
 199 otx2_ssogws_swtag_wait(struct otx2_ssogws *ws)
 200 {
 201 #ifdef RTE_ARCH_ARM64
 202         uint64_t swtp;
 203
 204         asm volatile (
 205                         "       ldr %[swtb], [%[swtp_loc]]      \n"
 206                         "       cbz %[swtb], done%=             \n"
 207                         "       sevl                            \n"
 208                         "rty%=: wfe                             \n"
 209                         "       ldr %[swtb], [%[swtp_loc]]      \n"
 210                         "       cbnz %[swtb], rty%=             \n"
 211                         "done%=:                                \n"
 212                         : [swtb] "=&r" (swtp)
 213                         : [swtp_loc] "r" (ws->swtp_op)
 214                         );
 215 #else
 216         /* Wait for the SWTAG/SWTAG_FULL operation */
 217         while (otx2_read64(ws->swtp_op))
 218                 ;
 219 #endif
 220 }
 221
 222 #endif