drivers/event/octeontx2/otx2_worker_dual.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include "otx2_worker_dual.h"
   6 #include "otx2_worker.h"
   7
   8 static __rte_noinline uint8_t
   9 otx2_ssogws_dual_new_event(struct otx2_ssogws_dual *ws,
  10                            const struct rte_event *ev)
  11 {
  12         const uint32_t tag = (uint32_t)ev->event;
  13         const uint8_t new_tt = ev->sched_type;
  14         const uint64_t event_ptr = ev->u64;
  15         const uint16_t grp = ev->queue_id;
  16
  17         if (ws->xaq_lmt <= *ws->fc_mem)
  18                 return 0;
  19
  20         otx2_ssogws_dual_add_work(ws, event_ptr, tag, new_tt, grp);
  21
  22         return 1;
  23 }
  24
  25 static __rte_always_inline void
  26 otx2_ssogws_dual_fwd_swtag(struct otx2_ssogws_state *ws,
  27                            const struct rte_event *ev)
  28 {
  29         const uint32_t tag = (uint32_t)ev->event;
  30         const uint8_t new_tt = ev->sched_type;
  31         const uint8_t cur_tt = ws->cur_tt;
  32
  33         /* 96XX model
  34          * cur_tt/new_tt     SSO_SYNC_ORDERED SSO_SYNC_ATOMIC SSO_SYNC_UNTAGGED
  35          *
  36          * SSO_SYNC_ORDERED        norm           norm             untag
  37          * SSO_SYNC_ATOMIC         norm           norm             untag
  38          * SSO_SYNC_UNTAGGED       norm           norm             NOOP
  39          */
  40         if (new_tt == SSO_SYNC_UNTAGGED) {
  41                 if (cur_tt != SSO_SYNC_UNTAGGED)
  42                         otx2_ssogws_swtag_untag((struct otx2_ssogws *)ws);
  43         } else {
  44                 otx2_ssogws_swtag_norm((struct otx2_ssogws *)ws, tag, new_tt);
  45         }
  46 }
  47
  48 static __rte_always_inline void
  49 otx2_ssogws_dual_fwd_group(struct otx2_ssogws_state *ws,
  50                            const struct rte_event *ev, const uint16_t grp)
  51 {
  52         const uint32_t tag = (uint32_t)ev->event;
  53         const uint8_t new_tt = ev->sched_type;
  54
  55         otx2_write64(ev->u64, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
  56                      SSOW_LF_GWS_OP_UPD_WQP_GRP1);
  57         rte_smp_wmb();
  58         otx2_ssogws_swtag_desched((struct otx2_ssogws *)ws, tag, new_tt, grp);
  59 }
  60
  61 static __rte_always_inline void
  62 otx2_ssogws_dual_forward_event(struct otx2_ssogws_dual *ws,
  63                                struct otx2_ssogws_state *vws,
  64                                const struct rte_event *ev)
  65 {
  66         const uint8_t grp = ev->queue_id;
  67
  68         /* Group hasn't changed, Use SWTAG to forward the event */
  69         if (vws->cur_grp == grp) {
  70                 otx2_ssogws_dual_fwd_swtag(vws, ev);
  71                 ws->swtag_req = 1;
  72         } else {
  73         /*
  74          * Group has been changed for group based work pipelining,
  75          * Use deschedule/add_work operation to transfer the event to
  76          * new group/core
  77          */
  78                 otx2_ssogws_dual_fwd_group(vws, ev, grp);
  79         }
  80 }
  81
  82 uint16_t __rte_hot
  83 otx2_ssogws_dual_enq(void *port, const struct rte_event *ev)
  84 {
  85         struct otx2_ssogws_dual *ws = port;
  86         struct otx2_ssogws_state *vws = &ws->ws_state[!ws->vws];
  87
  88         switch (ev->op) {
  89         case RTE_EVENT_OP_NEW:
  90                 rte_smp_mb();
  91                 return otx2_ssogws_dual_new_event(ws, ev);
  92         case RTE_EVENT_OP_FORWARD:
  93                 otx2_ssogws_dual_forward_event(ws, vws, ev);
  94                 break;
  95         case RTE_EVENT_OP_RELEASE:
  96                 otx2_ssogws_swtag_flush((struct otx2_ssogws *)vws);
  97                 break;
  98         default:
  99                 return 0;
 100         }
 101
 102         return 1;
 103 }
 104
 105 uint16_t __rte_hot
 106 otx2_ssogws_dual_enq_burst(void *port, const struct rte_event ev[],
 107                            uint16_t nb_events)
 108 {
 109         RTE_SET_USED(nb_events);
 110         return otx2_ssogws_dual_enq(port, ev);
 111 }
 112
 113 uint16_t __rte_hot
 114 otx2_ssogws_dual_enq_new_burst(void *port, const struct rte_event ev[],
 115                                uint16_t nb_events)
 116 {
 117         struct otx2_ssogws_dual *ws = port;
 118         uint16_t i, rc = 1;
 119
 120         rte_smp_mb();
 121         if (ws->xaq_lmt <= *ws->fc_mem)
 122                 return 0;
 123
 124         for (i = 0; i < nb_events && rc; i++)
 125                 rc = otx2_ssogws_dual_new_event(ws, &ev[i]);
 126
 127         return nb_events;
 128 }
 129
 130 uint16_t __rte_hot
 131 otx2_ssogws_dual_enq_fwd_burst(void *port, const struct rte_event ev[],
 132                                uint16_t nb_events)
 133 {
 134         struct otx2_ssogws_dual *ws = port;
 135         struct otx2_ssogws_state *vws = &ws->ws_state[!ws->vws];
 136
 137         RTE_SET_USED(nb_events);
 138         otx2_ssogws_dual_forward_event(ws, vws, ev);
 139
 140         return 1;
 141 }
 142
 143 #define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                      \
 144 uint16_t __rte_hot                                                              \
 145 otx2_ssogws_dual_deq_ ##name(void *port, struct rte_event *ev,          \
 146                              uint64_t timeout_ticks)                    \
 147 {                                                                       \
 148         struct otx2_ssogws_dual *ws = port;                             \
 149         uint8_t gw;                                                     \
 150                                                                         \
 151         rte_prefetch_non_temporal(ws);                                  \
 152         RTE_SET_USED(timeout_ticks);                                    \
 153         if (ws->swtag_req) {                                            \
 154                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
 155                                        &ws->ws_state[!ws->vws]);        \
 156                 ws->swtag_req = 0;                                      \
 157                 return 1;                                               \
 158         }                                                               \
 159                                                                         \
 160         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
 161                                        &ws->ws_state[!ws->vws], ev,     \
 162                                        flags, ws->lookup_mem,           \
 163                                        ws->tstamp);                     \
 164         ws->vws = !ws->vws;                                             \
 165                                                                         \
 166         return gw;                                                      \
 167 }                                                                       \
 168                                                                         \
 169 uint16_t __rte_hot                                                              \
 170 otx2_ssogws_dual_deq_burst_ ##name(void *port, struct rte_event ev[],   \
 171                                    uint16_t nb_events,                  \
 172                                    uint64_t timeout_ticks)              \
 173 {                                                                       \
 174         RTE_SET_USED(nb_events);                                        \
 175                                                                         \
 176         return otx2_ssogws_dual_deq_ ##name(port, ev, timeout_ticks);   \
 177 }                                                                       \
 178                                                                         \
 179 uint16_t __rte_hot                                                              \
 180 otx2_ssogws_dual_deq_timeout_ ##name(void *port, struct rte_event *ev,  \
 181                                      uint64_t timeout_ticks)            \
 182 {                                                                       \
 183         struct otx2_ssogws_dual *ws = port;                             \
 184         uint64_t iter;                                                  \
 185         uint8_t gw;                                                     \
 186                                                                         \
 187         if (ws->swtag_req) {                                            \
 188                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
 189                                        &ws->ws_state[!ws->vws]);        \
 190                 ws->swtag_req = 0;                                      \
 191                 return 1;                                               \
 192         }                                                               \
 193                                                                         \
 194         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
 195                                        &ws->ws_state[!ws->vws], ev,     \
 196                                        flags, ws->lookup_mem,           \
 197                                        ws->tstamp);                     \
 198         ws->vws = !ws->vws;                                             \
 199         for (iter = 1; iter < timeout_ticks && (gw == 0); iter++) {     \
 200                 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],  \
 201                                                &ws->ws_state[!ws->vws], \
 202                                                ev, flags,               \
 203                                                ws->lookup_mem,          \
 204                                                ws->tstamp);             \
 205                 ws->vws = !ws->vws;                                     \
 206         }                                                               \
 207                                                                         \
 208         return gw;                                                      \
 209 }                                                                       \
 210                                                                         \
 211 uint16_t __rte_hot                                                              \
 212 otx2_ssogws_dual_deq_timeout_burst_ ##name(void *port,                  \
 213                                            struct rte_event ev[],       \
 214                                            uint16_t nb_events,          \
 215                                            uint64_t timeout_ticks)      \
 216 {                                                                       \
 217         RTE_SET_USED(nb_events);                                        \
 218                                                                         \
 219         return otx2_ssogws_dual_deq_timeout_ ##name(port, ev,           \
 220                                                     timeout_ticks);     \
 221 }                                                                       \
 222                                                                         \
 223 uint16_t __rte_hot                                                              \
 224 otx2_ssogws_dual_deq_seg_ ##name(void *port, struct rte_event *ev,      \
 225                                  uint64_t timeout_ticks)                \
 226 {                                                                       \
 227         struct otx2_ssogws_dual *ws = port;                             \
 228         uint8_t gw;                                                     \
 229                                                                         \
 230         RTE_SET_USED(timeout_ticks);                                    \
 231         if (ws->swtag_req) {                                            \
 232                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
 233                                        &ws->ws_state[!ws->vws]);        \
 234                 ws->swtag_req = 0;                                      \
 235                 return 1;                                               \
 236         }                                                               \
 237                                                                         \
 238         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
 239                                        &ws->ws_state[!ws->vws], ev,     \
 240                                        flags | NIX_RX_MULTI_SEG_F,      \
 241                                        ws->lookup_mem,                  \
 242                                        ws->tstamp);                     \
 243         ws->vws = !ws->vws;                                             \
 244                                                                         \
 245         return gw;                                                      \
 246 }                                                                       \
 247                                                                         \
 248 uint16_t __rte_hot                                                              \
 249 otx2_ssogws_dual_deq_seg_burst_ ##name(void *port,                      \
 250                                        struct rte_event ev[],           \
 251                                        uint16_t nb_events,              \
 252                                        uint64_t timeout_ticks)          \
 253 {                                                                       \
 254         RTE_SET_USED(nb_events);                                        \
 255                                                                         \
 256         return otx2_ssogws_dual_deq_seg_ ##name(port, ev,               \
 257                                                 timeout_ticks);         \
 258 }                                                                       \
 259                                                                         \
 260 uint16_t __rte_hot                                                              \
 261 otx2_ssogws_dual_deq_seg_timeout_ ##name(void *port,                    \
 262                                          struct rte_event *ev,          \
 263                                          uint64_t timeout_ticks)        \
 264 {                                                                       \
 265         struct otx2_ssogws_dual *ws = port;                             \
 266         uint64_t iter;                                                  \
 267         uint8_t gw;                                                     \
 268                                                                         \
 269         if (ws->swtag_req) {                                            \
 270                 otx2_ssogws_swtag_wait((struct otx2_ssogws *)           \
 271                                        &ws->ws_state[!ws->vws]);        \
 272                 ws->swtag_req = 0;                                      \
 273                 return 1;                                               \
 274         }                                                               \
 275                                                                         \
 276         gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],          \
 277                                        &ws->ws_state[!ws->vws], ev,     \
 278                                        flags | NIX_RX_MULTI_SEG_F,      \
 279                                        ws->lookup_mem,                  \
 280                                        ws->tstamp);                     \
 281         ws->vws = !ws->vws;                                             \
 282         for (iter = 1; iter < timeout_ticks && (gw == 0); iter++) {     \
 283                 gw = otx2_ssogws_dual_get_work(&ws->ws_state[ws->vws],  \
 284                                                &ws->ws_state[!ws->vws], \
 285                                                ev, flags |              \
 286                                                NIX_RX_MULTI_SEG_F,      \
 287                                                ws->lookup_mem,          \
 288                                                ws->tstamp);             \
 289                 ws->vws = !ws->vws;                                     \
 290         }                                                               \
 291                                                                         \
 292         return gw;                                                      \
 293 }                                                                       \
 294                                                                         \
 295 uint16_t __rte_hot                                                              \
 296 otx2_ssogws_dual_deq_seg_timeout_burst_ ##name(void *port,              \
 297                                                struct rte_event ev[],   \
 298                                                uint16_t nb_events,      \
 299                                                uint64_t timeout_ticks)  \
 300 {                                                                       \
 301         RTE_SET_USED(nb_events);                                        \
 302                                                                         \
 303         return otx2_ssogws_dual_deq_seg_timeout_ ##name(port, ev,       \
 304                                                         timeout_ticks); \
 305 }
 306
 307 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 308 #undef R
 309
 310 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags)                  \
 311 uint16_t __rte_hot                                                      \
 312 otx2_ssogws_dual_tx_adptr_enq_ ## name(void *port,                      \
 313                                        struct rte_event ev[],           \
 314                                        uint16_t nb_events)              \
 315 {                                                                       \
 316         struct otx2_ssogws_dual *ws = port;                             \
 317         struct otx2_ssogws *vws =                                       \
 318                 (struct otx2_ssogws *)&ws->ws_state[!ws->vws];          \
 319         uint64_t cmd[sz];                                               \
 320                                                                         \
 321         RTE_SET_USED(nb_events);                                        \
 322         return otx2_ssogws_event_tx(vws, ev, cmd, (const uint64_t       \
 323                                     (*)[RTE_MAX_QUEUES_PER_PORT])       \
 324                                     ws->tx_adptr_data,                  \
 325                                     flags);                             \
 326 }
 327 SSO_TX_ADPTR_ENQ_FASTPATH_FUNC
 328 #undef T
 329
 330 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags)                  \
 331 uint16_t __rte_hot                                                      \
 332 otx2_ssogws_dual_tx_adptr_enq_seg_ ## name(void *port,                  \
 333                                            struct rte_event ev[],       \
 334                                            uint16_t nb_events)          \
 335 {                                                                       \
 336         struct otx2_ssogws_dual *ws = port;                             \
 337         struct otx2_ssogws *vws =                                       \
 338                 (struct otx2_ssogws *)&ws->ws_state[!ws->vws];          \
 339         uint64_t cmd[(sz) + NIX_TX_MSEG_SG_DWORDS - 2];                 \
 340                                                                         \
 341         RTE_SET_USED(nb_events);                                        \
 342         return otx2_ssogws_event_tx(vws, ev, cmd, (const uint64_t       \
 343                                     (*)[RTE_MAX_QUEUES_PER_PORT])       \
 344                                     ws->tx_adptr_data,                  \
 345                                     (flags) | NIX_TX_MULTI_SEG_F);      \
 346 }
 347 SSO_TX_ADPTR_ENQ_FASTPATH_FUNC
 348 #undef T