drivers/event/octeontx2/otx2_worker.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #ifndef __OTX2_WORKER_H__
   6 #define __OTX2_WORKER_H__
   7
   8 #include <rte_common.h>
   9 #include <rte_branch_prediction.h>
  10
  11 #include <otx2_common.h>
  12 #include "otx2_evdev.h"
  13
  14 /* SSO Operations */
  15
  16 static __rte_always_inline uint16_t
  17 otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev,
  18                      const uint32_t flags, const void * const lookup_mem)
  19 {
  20         union otx2_sso_event event;
  21         uint64_t tstamp_ptr;
  22         uint64_t get_work1;
  23         uint64_t mbuf;
  24
  25         otx2_write64(BIT_ULL(16) | /* wait for work. */
  26                      1, /* Use Mask set 0. */
  27                      ws->getwrk_op);
  28
  29         if (flags & NIX_RX_OFFLOAD_PTYPE_F)
  30                 rte_prefetch_non_temporal(lookup_mem);
  31 #ifdef RTE_ARCH_ARM64
  32         asm volatile(
  33                         "               ldr %[tag], [%[tag_loc]]        \n"
  34                         "               ldr %[wqp], [%[wqp_loc]]        \n"
  35                         "               tbz %[tag], 63, done%=          \n"
  36                         "               sevl                            \n"
  37                         "rty%=:         wfe                             \n"
  38                         "               ldr %[tag], [%[tag_loc]]        \n"
  39                         "               ldr %[wqp], [%[wqp_loc]]        \n"
  40                         "               tbnz %[tag], 63, rty%=          \n"
  41                         "done%=:        dmb ld                          \n"
  42                         "               prfm pldl1keep, [%[wqp], #8]    \n"
  43                         "               sub %[mbuf], %[wqp], #0x80      \n"
  44                         "               prfm pldl1keep, [%[mbuf]]       \n"
  45                         : [tag] "=&r" (event.get_work0),
  46                           [wqp] "=&r" (get_work1),
  47                           [mbuf] "=&r" (mbuf)
  48                         : [tag_loc] "r" (ws->tag_op),
  49                           [wqp_loc] "r" (ws->wqp_op)
  50                         );
  51 #else
  52         event.get_work0 = otx2_read64(ws->tag_op);
  53         while ((BIT_ULL(63)) & event.get_work0)
  54                 event.get_work0 = otx2_read64(ws->tag_op);
  55
  56         get_work1 = otx2_read64(ws->wqp_op);
  57         rte_prefetch0((const void *)get_work1);
  58         mbuf = (uint64_t)((char *)get_work1 - sizeof(struct rte_mbuf));
  59         rte_prefetch0((const void *)mbuf);
  60 #endif
  61
  62         event.get_work0 = (event.get_work0 & (0x3ull << 32)) << 6 |
  63                 (event.get_work0 & (0x3FFull << 36)) << 4 |
  64                 (event.get_work0 & 0xffffffff);
  65         ws->cur_tt = event.sched_type;
  66         ws->cur_grp = event.queue_id;
  67
  68         if (event.sched_type != SSO_TT_EMPTY &&
  69             event.event_type == RTE_EVENT_TYPE_ETHDEV) {
  70                 otx2_wqe_to_mbuf(get_work1, mbuf, event.sub_event_type,
  71                                  (uint32_t) event.get_work0, flags, lookup_mem);
  72                 /* Extracting tstamp, if PTP enabled*/
  73                 tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *)get_work1)
  74                                              + OTX2_SSO_WQE_SG_PTR);
  75                 otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp,
  76                                         flags, (uint64_t *)tstamp_ptr);
  77                 get_work1 = mbuf;
  78         }
  79
  80         ev->event = event.get_work0;
  81         ev->u64 = get_work1;
  82
  83         return !!get_work1;
  84 }
  85
  86 /* Used in cleaning up workslot. */
  87 static __rte_always_inline uint16_t
  88 otx2_ssogws_get_work_empty(struct otx2_ssogws *ws, struct rte_event *ev,
  89                            const uint32_t flags)
  90 {
  91         union otx2_sso_event event;
  92         uint64_t tstamp_ptr;
  93         uint64_t get_work1;
  94         uint64_t mbuf;
  95
  96 #ifdef RTE_ARCH_ARM64
  97         asm volatile(
  98                         "               ldr %[tag], [%[tag_loc]]        \n"
  99                         "               ldr %[wqp], [%[wqp_loc]]        \n"
 100                         "               tbz %[tag], 63, done%=          \n"
 101                         "               sevl                            \n"
 102                         "rty%=:         wfe                             \n"
 103                         "               ldr %[tag], [%[tag_loc]]        \n"
 104                         "               ldr %[wqp], [%[wqp_loc]]        \n"
 105                         "               tbnz %[tag], 63, rty%=          \n"
 106                         "done%=:        dmb ld                          \n"
 107                         "               prfm pldl1keep, [%[wqp], #8]    \n"
 108                         "               sub %[mbuf], %[wqp], #0x80      \n"
 109                         "               prfm pldl1keep, [%[mbuf]]       \n"
 110                         : [tag] "=&r" (event.get_work0),
 111                           [wqp] "=&r" (get_work1),
 112                           [mbuf] "=&r" (mbuf)
 113                         : [tag_loc] "r" (ws->tag_op),
 114                           [wqp_loc] "r" (ws->wqp_op)
 115                         );
 116 #else
 117         event.get_work0 = otx2_read64(ws->tag_op);
 118         while ((BIT_ULL(63)) & event.get_work0)
 119                 event.get_work0 = otx2_read64(ws->tag_op);
 120
 121         get_work1 = otx2_read64(ws->wqp_op);
 122         rte_prefetch_non_temporal((const void *)get_work1);
 123         mbuf = (uint64_t)((char *)get_work1 - sizeof(struct rte_mbuf));
 124         rte_prefetch_non_temporal((const void *)mbuf);
 125 #endif
 126
 127         event.get_work0 = (event.get_work0 & (0x3ull << 32)) << 6 |
 128                 (event.get_work0 & (0x3FFull << 36)) << 4 |
 129                 (event.get_work0 & 0xffffffff);
 130         ws->cur_tt = event.sched_type;
 131         ws->cur_grp = event.queue_id;
 132
 133         if (event.sched_type != SSO_TT_EMPTY &&
 134             event.event_type == RTE_EVENT_TYPE_ETHDEV) {
 135                 otx2_wqe_to_mbuf(get_work1, mbuf, event.sub_event_type,
 136                                  (uint32_t) event.get_work0, flags, NULL);
 137                 /* Extracting tstamp, if PTP enabled*/
 138                 tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *)get_work1)
 139                                              + OTX2_SSO_WQE_SG_PTR);
 140                 otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp,
 141                                         flags, (uint64_t *)tstamp_ptr);
 142                 get_work1 = mbuf;
 143         }
 144
 145         ev->event = event.get_work0;
 146         ev->u64 = get_work1;
 147
 148         return !!get_work1;
 149 }
 150
 151 static __rte_always_inline void
 152 otx2_ssogws_add_work(struct otx2_ssogws *ws, const uint64_t event_ptr,
 153                      const uint32_t tag, const uint8_t new_tt,
 154                      const uint16_t grp)
 155 {
 156         uint64_t add_work0;
 157
 158         add_work0 = tag | ((uint64_t)(new_tt) << 32);
 159         otx2_store_pair(add_work0, event_ptr, ws->grps_base[grp]);
 160 }
 161
 162 static __rte_always_inline void
 163 otx2_ssogws_swtag_desched(struct otx2_ssogws *ws, uint32_t tag, uint8_t new_tt,
 164                           uint16_t grp)
 165 {
 166         uint64_t val;
 167
 168         val = tag | ((uint64_t)(new_tt & 0x3) << 32) | ((uint64_t)grp << 34);
 169         otx2_write64(val, ws->swtag_desched_op);
 170 }
 171
 172 static __rte_always_inline void
 173 otx2_ssogws_swtag_norm(struct otx2_ssogws *ws, uint32_t tag, uint8_t new_tt)
 174 {
 175         uint64_t val;
 176
 177         val = tag | ((uint64_t)(new_tt & 0x3) << 32);
 178         otx2_write64(val, ws->swtag_norm_op);
 179 }
 180
 181 static __rte_always_inline void
 182 otx2_ssogws_swtag_untag(struct otx2_ssogws *ws)
 183 {
 184         otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
 185                      SSOW_LF_GWS_OP_SWTAG_UNTAG);
 186         ws->cur_tt = SSO_SYNC_UNTAGGED;
 187 }
 188
 189 static __rte_always_inline void
 190 otx2_ssogws_swtag_flush(struct otx2_ssogws *ws)
 191 {
 192         otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
 193                      SSOW_LF_GWS_OP_SWTAG_FLUSH);
 194         ws->cur_tt = SSO_SYNC_EMPTY;
 195 }
 196
 197 static __rte_always_inline void
 198 otx2_ssogws_desched(struct otx2_ssogws *ws)
 199 {
 200         otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
 201                      SSOW_LF_GWS_OP_DESCHED);
 202 }
 203
 204 static __rte_always_inline void
 205 otx2_ssogws_swtag_wait(struct otx2_ssogws *ws)
 206 {
 207 #ifdef RTE_ARCH_ARM64
 208         uint64_t swtp;
 209
 210         asm volatile (
 211                         "       ldr %[swtb], [%[swtp_loc]]      \n"
 212                         "       cbz %[swtb], done%=             \n"
 213                         "       sevl                            \n"
 214                         "rty%=: wfe                             \n"
 215                         "       ldr %[swtb], [%[swtp_loc]]      \n"
 216                         "       cbnz %[swtb], rty%=             \n"
 217                         "done%=:                                \n"
 218                         : [swtb] "=&r" (swtp)
 219                         : [swtp_loc] "r" (ws->swtp_op)
 220                         );
 221 #else
 222         /* Wait for the SWTAG/SWTAG_FULL operation */
 223         while (otx2_read64(ws->swtp_op))
 224                 ;
 225 #endif
 226 }
 227
 228 static __rte_always_inline void
 229 otx2_ssogws_head_wait(struct otx2_ssogws *ws)
 230 {
 231 #ifdef RTE_ARCH_ARM64
 232         uint64_t tag;
 233
 234         asm volatile (
 235                         "       ldr %[tag], [%[tag_op]]         \n"
 236                         "       tbnz %[tag], 35, done%=         \n"
 237                         "       sevl                            \n"
 238                         "rty%=: wfe                             \n"
 239                         "       ldr %[tag], [%[tag_op]]         \n"
 240                         "       tbz %[tag], 35, rty%=           \n"
 241                         "done%=:                                \n"
 242                         : [tag] "=&r" (tag)
 243                         : [tag_op] "r" (ws->tag_op)
 244                         );
 245 #else
 246         /* Wait for the HEAD to be set */
 247         while (!(otx2_read64(ws->tag_op) & BIT_ULL(35)))
 248                 ;
 249 #endif
 250 }
 251
 252 static __rte_always_inline void
 253 otx2_ssogws_order(struct otx2_ssogws *ws, const uint8_t wait_flag)
 254 {
 255         if (wait_flag)
 256                 otx2_ssogws_head_wait(ws);
 257
 258         rte_cio_wmb();
 259 }
 260
 261 static __rte_always_inline const struct otx2_eth_txq *
 262 otx2_ssogws_xtract_meta(struct rte_mbuf *m)
 263 {
 264         return rte_eth_devices[m->port].data->tx_queues[
 265                         rte_event_eth_tx_adapter_txq_get(m)];
 266 }
 267
 268 static __rte_always_inline void
 269 otx2_ssogws_prepare_pkt(const struct otx2_eth_txq *txq, struct rte_mbuf *m,
 270                         uint64_t *cmd, const uint32_t flags)
 271 {
 272         otx2_lmt_mov(cmd, txq->cmd, otx2_nix_tx_ext_subs(flags));
 273         otx2_nix_xmit_prepare(m, cmd, flags);
 274 }
 275
 276 static __rte_always_inline uint16_t
 277 otx2_ssogws_event_tx(struct otx2_ssogws *ws, struct rte_event ev[],
 278                      uint64_t *cmd, const uint32_t flags)
 279 {
 280         struct rte_mbuf *m = ev[0].mbuf;
 281         const struct otx2_eth_txq *txq = otx2_ssogws_xtract_meta(m);
 282
 283         /* Perform header writes before barrier for TSO */
 284         otx2_nix_xmit_prepare_tso(m, flags);
 285         otx2_ssogws_order(ws, !ev->sched_type);
 286         otx2_ssogws_prepare_pkt(txq, m, cmd, flags);
 287
 288         if (flags & NIX_TX_MULTI_SEG_F) {
 289                 const uint16_t segdw = otx2_nix_prepare_mseg(m, cmd, flags);
 290                 otx2_nix_xmit_prepare_tstamp(cmd, &txq->cmd[0],
 291                                              m->ol_flags, segdw, flags);
 292                 otx2_nix_xmit_mseg_one(cmd, txq->lmt_addr, txq->io_addr, segdw);
 293         } else {
 294                 /* Passing no of segdw as 4: HDR + EXT + SG + SMEM */
 295                 otx2_nix_xmit_prepare_tstamp(cmd, &txq->cmd[0],
 296                                              m->ol_flags, 4, flags);
 297                 otx2_nix_xmit_one(cmd, txq->lmt_addr, txq->io_addr, flags);
 298         }
 299
 300         return 1;
 301 }
 302
 303 #endif