drivers/event/octeontx2/otx2_worker_dual.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include "otx2_worker_dual.h"
   6 #include "otx2_worker.h"
   7
   8 static __rte_noinline uint8_t
   9 otx2_ssogws_dual_new_event(struct otx2_ssogws_dual *ws,
  10                            const struct rte_event *ev)
  11 {
  12         const uint32_t tag = (uint32_t)ev->event;
  13         const uint8_t new_tt = ev->sched_type;
  14         const uint64_t event_ptr = ev->u64;
  15         const uint16_t grp = ev->queue_id;
  16
  17         if (ws->xaq_lmt <= *ws->fc_mem)
  18                 return 0;
  19
  20         otx2_ssogws_dual_add_work(ws, event_ptr, tag, new_tt, grp);
  21
  22         return 1;
  23 }
  24
  25 static __rte_always_inline void
  26 otx2_ssogws_dual_fwd_swtag(struct otx2_ssogws_state *ws,
  27                            const struct rte_event *ev)
  28 {
  29         const uint32_t tag = (uint32_t)ev->event;
  30         const uint8_t new_tt = ev->sched_type;
  31         const uint8_t cur_tt = ws->cur_tt;
  32
  33         /* 96XX model
  34          * cur_tt/new_tt     SSO_SYNC_ORDERED SSO_SYNC_ATOMIC SSO_SYNC_UNTAGGED
  35          *
  36          * SSO_SYNC_ORDERED        norm           norm             untag
  37          * SSO_SYNC_ATOMIC         norm           norm             untag
  38          * SSO_SYNC_UNTAGGED       norm           norm             NOOP
  39          */
  40         if (new_tt == SSO_SYNC_UNTAGGED) {
  41                 if (cur_tt != SSO_SYNC_UNTAGGED)
  42                         otx2_ssogws_swtag_untag((struct otx2_ssogws *)ws);
  43         } else {
  44                 otx2_ssogws_swtag_norm((struct otx2_ssogws *)ws, tag, new_tt);
  45         }
  46 }
  47
  48 static __rte_always_inline void
  49 otx2_ssogws_dual_fwd_group(struct otx2_ssogws_state *ws,
  50                            const struct rte_event *ev, const uint16_t grp)
  51 {
  52         const uint32_t tag = (uint32_t)ev->event;
  53         const uint8_t new_tt = ev->sched_type;
  54
  55         otx2_write64(ev->u64, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
  56                      SSOW_LF_GWS_OP_UPD_WQP_GRP1);
  57         rte_smp_wmb();
  58         otx2_ssogws_swtag_desched((struct otx2_ssogws *)ws, tag, new_tt, grp);
  59 }
  60
  61 static __rte_always_inline void
  62 otx2_ssogws_dual_forward_event(struct otx2_ssogws_dual *ws,
  63                                struct otx2_ssogws_state *vws,
  64                                const struct rte_event *ev)
  65 {
  66         const uint8_t grp = ev->queue_id;
  67
  68         /* Group hasn't changed, Use SWTAG to forward the event */
  69         if (vws->cur_grp == grp) {
  70                 otx2_ssogws_dual_fwd_swtag(vws, ev);
  71                 ws->swtag_req = 1;
  72         } else {
  73         /*
  74          * Group has been changed for group based work pipelining,
  75          * Use deschedule/add_work operation to transfer the event to
  76          * new group/core
  77          */
  78                 otx2_ssogws_dual_fwd_group(vws, ev, grp);
  79         }
  80 }
  81
  82 uint16_t __hot
  83 otx2_ssogws_dual_enq(void *port, const struct rte_event *ev)
  84 {
  85         struct otx2_ssogws_dual *ws = port;
  86         struct otx2_ssogws_state *vws = &ws->ws_state[!ws->vws];
  87
  88         switch (ev->op) {
  89         case RTE_EVENT_OP_NEW:
  90                 rte_smp_mb();
  91                 return otx2_ssogws_dual_new_event(ws, ev);
  92         case RTE_EVENT_OP_FORWARD:
  93                 otx2_ssogws_dual_forward_event(ws, vws, ev);
  94                 break;
  95         case RTE_EVENT_OP_RELEASE:
  96                 otx2_ssogws_swtag_flush((struct otx2_ssogws *)vws);
  97                 break;
  98         default:
  99                 return 0;
 100         }
 101
 102         return 1;
 103 }
 104
 105 uint16_t __hot
 106 otx2_ssogws_dual_enq_burst(void *port, const struct rte_event ev[],
 107                            uint16_t nb_events)
 108 {
 109         RTE_SET_USED(nb_events);
 110         return otx2_ssogws_dual_enq(port, ev);
 111 }
 112
 113 uint16_t __hot
 114 otx2_ssogws_dual_enq_new_burst(void *port, const struct rte_event ev[],
 115                                uint16_t nb_events)
 116 {
 117         struct otx2_ssogws_dual *ws = port;
 118         uint16_t i, rc = 1;
 119
 120         rte_smp_mb();
 121         if (ws->xaq_lmt <= *ws->fc_mem)
 122                 return 0;
 123
 124         for (i = 0; i < nb_events && rc; i++)
 125                 rc = otx2_ssogws_dual_new_event(ws, &ev[i]);
 126
 127         return nb_events;
 128 }
 129
 130 uint16_t __hot
 131 otx2_ssogws_dual_enq_fwd_burst(void *port, const struct rte_event ev[],
 132                                uint16_t nb_events)
 133 {
 134         struct otx2_ssogws_dual *ws = port;
 135         struct otx2_ssogws_state *vws = &ws->ws_state[!ws->vws];
 136
 137         RTE_SET_USED(nb_events);
 138         otx2_ssogws_dual_forward_event(ws, vws, ev);
 139
 140         return 1;
 141 }
 142
 143 #define R(name, f5, f4, f3, f2, f1, f0, flags)                          \
 144 uint16_t __hot                                                          \
 145 otx2_ssogws_dual_deq_ ##name(void *port, struct rte_event *ev,          \
 146                              uint64_t timeout_ticks)                    \
 147 {                                                                       \
 148         struct otx2_ssogws_dual *ws = port;                             \
 149         uint8_t gw;                                                     \
 150                                                                         \
 151         RTE_SET_USED(timeout_ticks);                                    \
 152         if (ws->swtag_req) {                                            \
 153                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
 154                                        &ws->ws_state[!ws->vws]);        \
 155                 ws->swtag_req = 0;                                      \
 156                 return 1;                                               \
 157         }                                                               \
 158                                                                         \
 159         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
 160                                        &ws->ws_state[!ws->vws], ev,     \
 161                                        flags, ws->lookup_mem);          \
 162         ws->vws = !ws->vws;                                             \
 163                                                                         \
 164         return gw;                                                      \
 165 }                                                                       \
 166                                                                         \
 167 uint16_t __hot                                                          \
 168 otx2_ssogws_dual_deq_burst_ ##name(void *port, struct rte_event ev[],   \
 169                                    uint16_t nb_events,                  \
 170                                    uint64_t timeout_ticks)              \
 171 {                                                                       \
 172         RTE_SET_USED(nb_events);                                        \
 173                                                                         \
 174         return otx2_ssogws_dual_deq_ ##name(port, ev, timeout_ticks);   \
 175 }                                                                       \
 176                                                                         \
 177 uint16_t __hot                                                          \
 178 otx2_ssogws_dual_deq_timeout_ ##name(void *port, struct rte_event *ev,  \
 179                                      uint64_t timeout_ticks)            \
 180 {                                                                       \
 181         struct otx2_ssogws_dual *ws = port;                             \
 182         uint64_t iter;                                                  \
 183         uint8_t gw;                                                     \
 184                                                                         \
 185         if (ws->swtag_req) {                                            \
 186                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
 187                                        &ws->ws_state[!ws->vws]);        \
 188                 ws->swtag_req = 0;                                      \
 189                 return 1;                                               \
 190         }                                                               \
 191                                                                         \
 192         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
 193                                        &ws->ws_state[!ws->vws], ev,     \
 194                                        flags, ws->lookup_mem);          \
 195         ws->vws = !ws->vws;                                             \
 196         for (iter = 1; iter < timeout_ticks && (gw == 0); iter++) {     \
 197                 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],  \
 198                                                &ws->ws_state[!ws->vws], \
 199                                                ev, flags,               \
 200                                                ws->lookup_mem);         \
 201                 ws->vws = !ws->vws;                                     \
 202         }                                                               \
 203                                                                         \
 204         return gw;                                                      \
 205 }                                                                       \
 206                                                                         \
 207 uint16_t __hot                                                          \
 208 otx2_ssogws_dual_deq_timeout_burst_ ##name(void *port,                  \
 209                                            struct rte_event ev[],       \
 210                                            uint16_t nb_events,          \
 211                                            uint64_t timeout_ticks)      \
 212 {                                                                       \
 213         RTE_SET_USED(nb_events);                                        \
 214                                                                         \
 215         return otx2_ssogws_dual_deq_timeout_ ##name(port, ev,           \
 216                                                     timeout_ticks);     \
 217 }                                                                       \
 218                                                                         \
 219 uint16_t __hot                                                          \
 220 otx2_ssogws_dual_deq_seg_ ##name(void *port, struct rte_event *ev,      \
 221                                  uint64_t timeout_ticks)                \
 222 {                                                                       \
 223         struct otx2_ssogws_dual *ws = port;                             \
 224         uint8_t gw;                                                     \
 225                                                                         \
 226         RTE_SET_USED(timeout_ticks);                                    \
 227         if (ws->swtag_req) {                                            \
 228                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
 229                                        &ws->ws_state[!ws->vws]);        \
 230                 ws->swtag_req = 0;                                      \
 231                 return 1;                                               \
 232         }                                                               \
 233                                                                         \
 234         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
 235                                        &ws->ws_state[!ws->vws], ev,     \
 236                                        flags | NIX_RX_MULTI_SEG_F,      \
 237                                        ws->lookup_mem);                 \
 238         ws->vws = !ws->vws;                                             \
 239                                                                         \
 240         return gw;                                                      \
 241 }                                                                       \
 242                                                                         \
 243 uint16_t __hot                                                          \
 244 otx2_ssogws_dual_deq_seg_burst_ ##name(void *port,                      \
 245                                        struct rte_event ev[],           \
 246                                        uint16_t nb_events,              \
 247                                        uint64_t timeout_ticks)          \
 248 {                                                                       \
 249         RTE_SET_USED(nb_events);                                        \
 250                                                                         \
 251         return otx2_ssogws_dual_deq_seg_ ##name(port, ev,               \
 252                                                 timeout_ticks);         \
 253 }                                                                       \
 254                                                                         \
 255 uint16_t __hot                                                          \
 256 otx2_ssogws_dual_deq_seg_timeout_ ##name(void *port,                    \
 257                                          struct rte_event *ev,          \
 258                                          uint64_t timeout_ticks)        \
 259 {                                                                       \
 260         struct otx2_ssogws_dual *ws = port;                             \
 261         uint64_t iter;                                                  \
 262         uint8_t gw;                                                     \
 263                                                                         \
 264         if (ws->swtag_req) {                                            \
 265                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
 266                                        &ws->ws_state[!ws->vws]);        \
 267                 ws->swtag_req = 0;                                      \
 268                 return 1;                                               \
 269         }                                                               \
 270                                                                         \
 271         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
 272                                        &ws->ws_state[!ws->vws], ev,     \
 273                                        flags | NIX_RX_MULTI_SEG_F,      \
 274                                        ws->lookup_mem);                 \
 275         ws->vws = !ws->vws;                                             \
 276         for (iter = 1; iter < timeout_ticks && (gw == 0); iter++) {     \
 277                 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],  \
 278                                                &ws->ws_state[!ws->vws], \
 279                                                ev, flags |              \
 280                                                NIX_RX_MULTI_SEG_F,      \
 281                                                ws->lookup_mem);         \
 282                 ws->vws = !ws->vws;                                     \
 283         }                                                               \
 284                                                                         \
 285         return gw;                                                      \
 286 }                                                                       \
 287                                                                         \
 288 uint16_t __hot                                                          \
 289 otx2_ssogws_dual_deq_seg_timeout_burst_ ##name(void *port,              \
 290                                                struct rte_event ev[],   \
 291                                                uint16_t nb_events,      \
 292                                                uint64_t timeout_ticks)  \
 293 {                                                                       \
 294         RTE_SET_USED(nb_events);                                        \
 295                                                                         \
 296         return otx2_ssogws_dual_deq_seg_timeout_ ##name(port, ev,       \
 297                                                         timeout_ticks); \
 298 }
 299
 300 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 301 #undef R