drivers/event/octeontx2/otx2_worker_dual.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include "otx2_worker_dual.h"
   6 #include "otx2_worker.h"
   7
   8 static __rte_noinline uint8_t
   9 otx2_ssogws_dual_new_event(struct otx2_ssogws_dual *ws,
  10                            const struct rte_event *ev)
  11 {
  12         const uint32_t tag = (uint32_t)ev->event;
  13         const uint8_t new_tt = ev->sched_type;
  14         const uint64_t event_ptr = ev->u64;
  15         const uint16_t grp = ev->queue_id;
  16
  17         if (ws->xaq_lmt <= *ws->fc_mem)
  18                 return 0;
  19
  20         otx2_ssogws_dual_add_work(ws, event_ptr, tag, new_tt, grp);
  21
  22         return 1;
  23 }
  24
  25 static __rte_always_inline void
  26 otx2_ssogws_dual_fwd_swtag(struct otx2_ssogws_state *ws,
  27                            const struct rte_event *ev)
  28 {
  29         const uint32_t tag = (uint32_t)ev->event;
  30         const uint8_t new_tt = ev->sched_type;
  31         const uint8_t cur_tt = ws->cur_tt;
  32
  33         /* 96XX model
  34          * cur_tt/new_tt     SSO_SYNC_ORDERED SSO_SYNC_ATOMIC SSO_SYNC_UNTAGGED
  35          *
  36          * SSO_SYNC_ORDERED        norm           norm             untag
  37          * SSO_SYNC_ATOMIC         norm           norm             untag
  38          * SSO_SYNC_UNTAGGED       norm           norm             NOOP
  39          */
  40         if (new_tt == SSO_SYNC_UNTAGGED) {
  41                 if (cur_tt != SSO_SYNC_UNTAGGED)
  42                         otx2_ssogws_swtag_untag((struct otx2_ssogws *)ws);
  43         } else {
  44                 otx2_ssogws_swtag_norm((struct otx2_ssogws *)ws, tag, new_tt);
  45         }
  46 }
  47
  48 static __rte_always_inline void
  49 otx2_ssogws_dual_fwd_group(struct otx2_ssogws_state *ws,
  50                            const struct rte_event *ev, const uint16_t grp)
  51 {
  52         const uint32_t tag = (uint32_t)ev->event;
  53         const uint8_t new_tt = ev->sched_type;
  54
  55         otx2_write64(ev->u64, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
  56                      SSOW_LF_GWS_OP_UPD_WQP_GRP1);
  57         rte_smp_wmb();
  58         otx2_ssogws_swtag_desched((struct otx2_ssogws *)ws, tag, new_tt, grp);
  59 }
  60
  61 static __rte_always_inline void
  62 otx2_ssogws_dual_forward_event(struct otx2_ssogws_dual *ws,
  63                                struct otx2_ssogws_state *vws,
  64                                const struct rte_event *ev)
  65 {
  66         const uint8_t grp = ev->queue_id;
  67
  68         /* Group hasn't changed, Use SWTAG to forward the event */
  69         if (vws->cur_grp == grp) {
  70                 otx2_ssogws_dual_fwd_swtag(vws, ev);
  71                 ws->swtag_req = 1;
  72         } else {
  73         /*
  74          * Group has been changed for group based work pipelining,
  75          * Use deschedule/add_work operation to transfer the event to
  76          * new group/core
  77          */
  78                 otx2_ssogws_dual_fwd_group(vws, ev, grp);
  79         }
  80 }
  81
  82 uint16_t __hot
  83 otx2_ssogws_dual_enq(void *port, const struct rte_event *ev)
  84 {
  85         struct otx2_ssogws_dual *ws = port;
  86         struct otx2_ssogws_state *vws = &ws->ws_state[!ws->vws];
  87
  88         switch (ev->op) {
  89         case RTE_EVENT_OP_NEW:
  90                 rte_smp_mb();
  91                 return otx2_ssogws_dual_new_event(ws, ev);
  92         case RTE_EVENT_OP_FORWARD:
  93                 otx2_ssogws_dual_forward_event(ws, vws, ev);
  94                 break;
  95         case RTE_EVENT_OP_RELEASE:
  96                 otx2_ssogws_swtag_flush((struct otx2_ssogws *)vws);
  97                 break;
  98         default:
  99                 return 0;
 100         }
 101
 102         return 1;
 103 }
 104
 105 uint16_t __hot
 106 otx2_ssogws_dual_enq_burst(void *port, const struct rte_event ev[],
 107                            uint16_t nb_events)
 108 {
 109         RTE_SET_USED(nb_events);
 110         return otx2_ssogws_dual_enq(port, ev);
 111 }
 112
 113 uint16_t __hot
 114 otx2_ssogws_dual_enq_new_burst(void *port, const struct rte_event ev[],
 115                                uint16_t nb_events)
 116 {
 117         struct otx2_ssogws_dual *ws = port;
 118         uint16_t i, rc = 1;
 119
 120         rte_smp_mb();
 121         if (ws->xaq_lmt <= *ws->fc_mem)
 122                 return 0;
 123
 124         for (i = 0; i < nb_events && rc; i++)
 125                 rc = otx2_ssogws_dual_new_event(ws, &ev[i]);
 126
 127         return nb_events;
 128 }
 129
 130 uint16_t __hot
 131 otx2_ssogws_dual_enq_fwd_burst(void *port, const struct rte_event ev[],
 132                                uint16_t nb_events)
 133 {
 134         struct otx2_ssogws_dual *ws = port;
 135         struct otx2_ssogws_state *vws = &ws->ws_state[!ws->vws];
 136
 137         RTE_SET_USED(nb_events);
 138         otx2_ssogws_dual_forward_event(ws, vws, ev);
 139
 140         return 1;
 141 }
 142
 143 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
 144 uint16_t __hot                                                          \
 145 otx2_ssogws_dual_deq_ ##name(void *port, struct rte_event *ev,          \
 146                              uint64_t timeout_ticks)                    \
 147 {                                                                       \
 148         struct otx2_ssogws_dual *ws = port;                             \
 149         uint8_t gw;                                                     \
 150                                                                         \
 151         RTE_SET_USED(timeout_ticks);                                    \
 152         if (ws->swtag_req) {                                            \
 153                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
 154                                        &ws->ws_state[!ws->vws]);        \
 155                 ws->swtag_req = 0;                                      \
 156                 return 1;                                               \
 157         }                                                               \
 158                                                                         \
 159         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
 160                                        &ws->ws_state[!ws->vws], ev,     \
 161                                        flags, ws->lookup_mem,           \
 162                                        ws->tstamp);                     \
 163         ws->vws = !ws->vws;                                             \
 164                                                                         \
 165         return gw;                                                      \
 166 }                                                                       \
 167                                                                         \
 168 uint16_t __hot                                                          \
 169 otx2_ssogws_dual_deq_burst_ ##name(void *port, struct rte_event ev[],   \
 170                                    uint16_t nb_events,                  \
 171                                    uint64_t timeout_ticks)              \
 172 {                                                                       \
 173         RTE_SET_USED(nb_events);                                        \
 174                                                                         \
 175         return otx2_ssogws_dual_deq_ ##name(port, ev, timeout_ticks);   \
 176 }                                                                       \
 177                                                                         \
 178 uint16_t __hot                                                          \
 179 otx2_ssogws_dual_deq_timeout_ ##name(void *port, struct rte_event *ev,  \
 180                                      uint64_t timeout_ticks)            \
 181 {                                                                       \
 182         struct otx2_ssogws_dual *ws = port;                             \
 183         uint64_t iter;                                                  \
 184         uint8_t gw;                                                     \
 185                                                                         \
 186         if (ws->swtag_req) {                                            \
 187                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
 188                                        &ws->ws_state[!ws->vws]);        \
 189                 ws->swtag_req = 0;                                      \
 190                 return 1;                                               \
 191         }                                                               \
 192                                                                         \
 193         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
 194                                        &ws->ws_state[!ws->vws], ev,     \
 195                                        flags, ws->lookup_mem,           \
 196                                        ws->tstamp);                     \
 197         ws->vws = !ws->vws;                                             \
 198         for (iter = 1; iter < timeout_ticks && (gw == 0); iter++) {     \
 199                 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],  \
 200                                                &ws->ws_state[!ws->vws], \
 201                                                ev, flags,               \
 202                                                ws->lookup_mem,          \
 203                                                ws->tstamp);             \
 204                 ws->vws = !ws->vws;                                     \
 205         }                                                               \
 206                                                                         \
 207         return gw;                                                      \
 208 }                                                                       \
 209                                                                         \
 210 uint16_t __hot                                                          \
 211 otx2_ssogws_dual_deq_timeout_burst_ ##name(void *port,                  \
 212                                            struct rte_event ev[],       \
 213                                            uint16_t nb_events,          \
 214                                            uint64_t timeout_ticks)      \
 215 {                                                                       \
 216         RTE_SET_USED(nb_events);                                        \
 217                                                                         \
 218         return otx2_ssogws_dual_deq_timeout_ ##name(port, ev,           \
 219                                                     timeout_ticks);     \
 220 }                                                                       \
 221                                                                         \
 222 uint16_t __hot                                                          \
 223 otx2_ssogws_dual_deq_seg_ ##name(void *port, struct rte_event *ev,      \
 224                                  uint64_t timeout_ticks)                \
 225 {                                                                       \
 226         struct otx2_ssogws_dual *ws = port;                             \
 227         uint8_t gw;                                                     \
 228                                                                         \
 229         RTE_SET_USED(timeout_ticks);                                    \
 230         if (ws->swtag_req) {                                            \
 231                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
 232                                        &ws->ws_state[!ws->vws]);        \
 233                 ws->swtag_req = 0;                                      \
 234                 return 1;                                               \
 235         }                                                               \
 236                                                                         \
 237         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
 238                                        &ws->ws_state[!ws->vws], ev,     \
 239                                        flags | NIX_RX_MULTI_SEG_F,      \
 240                                        ws->lookup_mem,                  \
 241                                        ws->tstamp);                     \
 242         ws->vws = !ws->vws;                                             \
 243                                                                         \
 244         return gw;                                                      \
 245 }                                                                       \
 246                                                                         \
 247 uint16_t __hot                                                          \
 248 otx2_ssogws_dual_deq_seg_burst_ ##name(void *port,                      \
 249                                        struct rte_event ev[],           \
 250                                        uint16_t nb_events,              \
 251                                        uint64_t timeout_ticks)          \
 252 {                                                                       \
 253         RTE_SET_USED(nb_events);                                        \
 254                                                                         \
 255         return otx2_ssogws_dual_deq_seg_ ##name(port, ev,               \
 256                                                 timeout_ticks);         \
 257 }                                                                       \
 258                                                                         \
 259 uint16_t __hot                                                          \
 260 otx2_ssogws_dual_deq_seg_timeout_ ##name(void *port,                    \
 261                                          struct rte_event *ev,          \
 262                                          uint64_t timeout_ticks)        \
 263 {                                                                       \
 264         struct otx2_ssogws_dual *ws = port;                             \
 265         uint64_t iter;                                                  \
 266         uint8_t gw;                                                     \
 267                                                                         \
 268         if (ws->swtag_req) {                                            \
 269                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
 270                                        &ws->ws_state[!ws->vws]);        \
 271                 ws->swtag_req = 0;                                      \
 272                 return 1;                                               \
 273         }                                                               \
 274                                                                         \
 275         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
 276                                        &ws->ws_state[!ws->vws], ev,     \
 277                                        flags | NIX_RX_MULTI_SEG_F,      \
 278                                        ws->lookup_mem,                  \
 279                                        ws->tstamp);                     \
 280         ws->vws = !ws->vws;                                             \
 281         for (iter = 1; iter < timeout_ticks && (gw == 0); iter++) {     \
 282                 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],  \
 283                                                &ws->ws_state[!ws->vws], \
 284                                                ev, flags |              \
 285                                                NIX_RX_MULTI_SEG_F,      \
 286                                                ws->lookup_mem,          \
 287                                                ws->tstamp);             \
 288                 ws->vws = !ws->vws;                                     \
 289         }                                                               \
 290                                                                         \
 291         return gw;                                                      \
 292 }                                                                       \
 293                                                                         \
 294 uint16_t __hot                                                          \
 295 otx2_ssogws_dual_deq_seg_timeout_burst_ ##name(void *port,              \
 296                                                struct rte_event ev[],   \
 297                                                uint16_t nb_events,      \
 298                                                uint64_t timeout_ticks)  \
 299 {                                                                       \
 300         RTE_SET_USED(nb_events);                                        \
 301                                                                         \
 302         return otx2_ssogws_dual_deq_seg_timeout_ ##name(port, ev,       \
 303                                                         timeout_ticks); \
 304 }
 305
 306 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 307 #undef R