drivers/event/octeontx2/otx2_worker_dual.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #ifndef __OTX2_WORKER_DUAL_H__
   6 #define __OTX2_WORKER_DUAL_H__
   7
   8 #include <rte_branch_prediction.h>
   9 #include <rte_common.h>
  10
  11 #include <otx2_common.h>
  12 #include "otx2_evdev.h"
  13 #include "otx2_evdev_crypto_adptr_dp.h"
  14
  15 /* SSO Operations */
  16 static __rte_always_inline uint16_t
  17 otx2_ssogws_dual_get_work(struct otx2_ssogws_state *ws,
  18                           struct otx2_ssogws_state *ws_pair,
  19                           struct rte_event *ev, const uint32_t flags,
  20                           const void * const lookup_mem,
  21                           struct otx2_timesync_info * const tstamp)
  22 {
  23         const uint64_t set_gw = BIT_ULL(16) | 1;
  24         union otx2_sso_event event;
  25         uint64_t tstamp_ptr;
  26         uint64_t get_work1;
  27         uint64_t mbuf;
  28
  29         if (flags & NIX_RX_OFFLOAD_PTYPE_F)
  30                 rte_prefetch_non_temporal(lookup_mem);
  31 #ifdef RTE_ARCH_ARM64
  32         asm volatile(
  33                         "rty%=:                              \n"
  34                         "        ldr %[tag], [%[tag_loc]]    \n"
  35                         "        ldr %[wqp], [%[wqp_loc]]    \n"
  36                         "        tbnz %[tag], 63, rty%=      \n"
  37                         "done%=: str %[gw], [%[pong]]        \n"
  38                         "        dmb ld                      \n"
  39                         "        prfm pldl1keep, [%[wqp], #8]\n"
  40                         "        sub %[mbuf], %[wqp], #0x80  \n"
  41                         "        prfm pldl1keep, [%[mbuf]]   \n"
  42                         : [tag] "=&r" (event.get_work0),
  43                           [wqp] "=&r" (get_work1),
  44                           [mbuf] "=&r" (mbuf)
  45                         : [tag_loc] "r" (ws->tag_op),
  46                           [wqp_loc] "r" (ws->wqp_op),
  47                           [gw] "r" (set_gw),
  48                           [pong] "r" (ws_pair->getwrk_op)
  49                         );
  50 #else
  51         event.get_work0 = otx2_read64(ws->tag_op);
  52         while ((BIT_ULL(63)) & event.get_work0)
  53                 event.get_work0 = otx2_read64(ws->tag_op);
  54         get_work1 = otx2_read64(ws->wqp_op);
  55         otx2_write64(set_gw, ws_pair->getwrk_op);
  56
  57         rte_prefetch0((const void *)get_work1);
  58         mbuf = (uint64_t)((char *)get_work1 - sizeof(struct rte_mbuf));
  59         rte_prefetch0((const void *)mbuf);
  60 #endif
  61         event.get_work0 = (event.get_work0 & (0x3ull << 32)) << 6 |
  62                 (event.get_work0 & (0x3FFull << 36)) << 4 |
  63                 (event.get_work0 & 0xffffffff);
  64
  65         if (event.sched_type != SSO_TT_EMPTY) {
  66                 if ((flags & NIX_RX_OFFLOAD_SECURITY_F) &&
  67                     (event.event_type == RTE_EVENT_TYPE_CRYPTODEV)) {
  68                         get_work1 = otx2_handle_crypto_event(get_work1);
  69                 } else if (event.event_type == RTE_EVENT_TYPE_ETHDEV) {
  70                         uint8_t port = event.sub_event_type;
  71
  72                         event.sub_event_type = 0;
  73                         otx2_wqe_to_mbuf(get_work1, mbuf, port,
  74                                          event.flow_id, flags, lookup_mem);
  75                         /* Extracting tstamp, if PTP enabled. CGX will prepend
  76                          * the timestamp at starting of packet data and it can
  77                          * be derieved from WQE 9 dword which corresponds to SG
  78                          * iova.
  79                          * rte_pktmbuf_mtod_offset can be used for this purpose
  80                          * but it brings down the performance as it reads
  81                          * mbuf->buf_addr which is not part of cache in general
  82                          * fast path.
  83                          */
  84                         tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *)
  85                                                      get_work1) +
  86                                                      OTX2_SSO_WQE_SG_PTR);
  87                         otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, tstamp,
  88                                                 flags, (uint64_t *)tstamp_ptr);
  89                         get_work1 = mbuf;
  90                 }
  91         }
  92
  93         ev->event = event.get_work0;
  94         ev->u64 = get_work1;
  95
  96         return !!get_work1;
  97 }
  98
  99 static __rte_always_inline void
 100 otx2_ssogws_dual_add_work(struct otx2_ssogws_dual *ws, const uint64_t event_ptr,
 101                           const uint32_t tag, const uint8_t new_tt,
 102                           const uint16_t grp)
 103 {
 104         uint64_t add_work0;
 105
 106         add_work0 = tag | ((uint64_t)(new_tt) << 32);
 107         otx2_store_pair(add_work0, event_ptr, ws->grps_base[grp]);
 108 }
 109
 110 #endif