drivers/event/octeontx2/otx2_worker.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include "otx2_worker.h"
   6
   7 static __rte_noinline uint8_t
   8 otx2_ssogws_new_event(struct otx2_ssogws *ws, const struct rte_event *ev)
   9 {
  10         const uint32_t tag = (uint32_t)ev->event;
  11         const uint8_t new_tt = ev->sched_type;
  12         const uint64_t event_ptr = ev->u64;
  13         const uint16_t grp = ev->queue_id;
  14
  15         if (ws->xaq_lmt <= *ws->fc_mem)
  16                 return 0;
  17
  18         otx2_ssogws_add_work(ws, event_ptr, tag, new_tt, grp);
  19
  20         return 1;
  21 }
  22
  23 static __rte_always_inline void
  24 otx2_ssogws_fwd_swtag(struct otx2_ssogws *ws, const struct rte_event *ev)
  25 {
  26         const uint32_t tag = (uint32_t)ev->event;
  27         const uint8_t new_tt = ev->sched_type;
  28         const uint8_t cur_tt = OTX2_SSOW_TT_FROM_TAG(otx2_read64(ws->tag_op));
  29
  30         /* 96XX model
  31          * cur_tt/new_tt     SSO_SYNC_ORDERED SSO_SYNC_ATOMIC SSO_SYNC_UNTAGGED
  32          *
  33          * SSO_SYNC_ORDERED        norm           norm             untag
  34          * SSO_SYNC_ATOMIC         norm           norm             untag
  35          * SSO_SYNC_UNTAGGED       norm           norm             NOOP
  36          */
  37
  38         if (new_tt == SSO_SYNC_UNTAGGED) {
  39                 if (cur_tt != SSO_SYNC_UNTAGGED)
  40                         otx2_ssogws_swtag_untag(ws);
  41         } else {
  42                 otx2_ssogws_swtag_norm(ws, tag, new_tt);
  43         }
  44
  45         ws->swtag_req = 1;
  46 }
  47
  48 static __rte_always_inline void
  49 otx2_ssogws_fwd_group(struct otx2_ssogws *ws, const struct rte_event *ev,
  50                       const uint16_t grp)
  51 {
  52         const uint32_t tag = (uint32_t)ev->event;
  53         const uint8_t new_tt = ev->sched_type;
  54
  55         otx2_write64(ev->u64, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
  56                      SSOW_LF_GWS_OP_UPD_WQP_GRP1);
  57         rte_smp_wmb();
  58         otx2_ssogws_swtag_desched(ws, tag, new_tt, grp);
  59 }
  60
  61 static __rte_always_inline void
  62 otx2_ssogws_forward_event(struct otx2_ssogws *ws, const struct rte_event *ev)
  63 {
  64         const uint8_t grp = ev->queue_id;
  65
  66         /* Group hasn't changed, Use SWTAG to forward the event */
  67         if (OTX2_SSOW_GRP_FROM_TAG(otx2_read64(ws->tag_op)) == grp)
  68                 otx2_ssogws_fwd_swtag(ws, ev);
  69         else
  70         /*
  71          * Group has been changed for group based work pipelining,
  72          * Use deschedule/add_work operation to transfer the event to
  73          * new group/core
  74          */
  75                 otx2_ssogws_fwd_group(ws, ev, grp);
  76 }
  77
  78 #define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                      \
  79 uint16_t __rte_hot                                                              \
  80 otx2_ssogws_deq_ ##name(void *port, struct rte_event *ev,               \
  81                         uint64_t timeout_ticks)                         \
  82 {                                                                       \
  83         struct otx2_ssogws *ws = port;                                  \
  84                                                                         \
  85         RTE_SET_USED(timeout_ticks);                                    \
  86                                                                         \
  87         if (ws->swtag_req) {                                            \
  88                 ws->swtag_req = 0;                                      \
  89                 otx2_ssogws_swtag_wait(ws);                             \
  90                 return 1;                                               \
  91         }                                                               \
  92                                                                         \
  93         return otx2_ssogws_get_work(ws, ev, flags, ws->lookup_mem);     \
  94 }                                                                       \
  95                                                                         \
  96 uint16_t __rte_hot                                                              \
  97 otx2_ssogws_deq_burst_ ##name(void *port, struct rte_event ev[],        \
  98                               uint16_t nb_events,                       \
  99                               uint64_t timeout_ticks)                   \
 100 {                                                                       \
 101         RTE_SET_USED(nb_events);                                        \
 102                                                                         \
 103         return otx2_ssogws_deq_ ##name(port, ev, timeout_ticks);        \
 104 }                                                                       \
 105                                                                         \
 106 uint16_t __rte_hot                                                              \
 107 otx2_ssogws_deq_timeout_ ##name(void *port, struct rte_event *ev,       \
 108                                 uint64_t timeout_ticks)                 \
 109 {                                                                       \
 110         struct otx2_ssogws *ws = port;                                  \
 111         uint16_t ret = 1;                                               \
 112         uint64_t iter;                                                  \
 113                                                                         \
 114         if (ws->swtag_req) {                                            \
 115                 ws->swtag_req = 0;                                      \
 116                 otx2_ssogws_swtag_wait(ws);                             \
 117                 return ret;                                             \
 118         }                                                               \
 119                                                                         \
 120         ret = otx2_ssogws_get_work(ws, ev, flags, ws->lookup_mem);      \
 121         for (iter = 1; iter < timeout_ticks && (ret == 0); iter++)      \
 122                 ret = otx2_ssogws_get_work(ws, ev, flags,               \
 123                                            ws->lookup_mem);             \
 124                                                                         \
 125         return ret;                                                     \
 126 }                                                                       \
 127                                                                         \
 128 uint16_t __rte_hot                                                              \
 129 otx2_ssogws_deq_timeout_burst_ ##name(void *port, struct rte_event ev[],\
 130                                       uint16_t nb_events,               \
 131                                       uint64_t timeout_ticks)           \
 132 {                                                                       \
 133         RTE_SET_USED(nb_events);                                        \
 134                                                                         \
 135         return otx2_ssogws_deq_timeout_ ##name(port, ev, timeout_ticks);\
 136 }                                                                       \
 137                                                                         \
 138 uint16_t __rte_hot                                                              \
 139 otx2_ssogws_deq_seg_ ##name(void *port, struct rte_event *ev,           \
 140                             uint64_t timeout_ticks)                     \
 141 {                                                                       \
 142         struct otx2_ssogws *ws = port;                                  \
 143                                                                         \
 144         RTE_SET_USED(timeout_ticks);                                    \
 145                                                                         \
 146         if (ws->swtag_req) {                                            \
 147                 ws->swtag_req = 0;                                      \
 148                 otx2_ssogws_swtag_wait(ws);                             \
 149                 return 1;                                               \
 150         }                                                               \
 151                                                                         \
 152         return otx2_ssogws_get_work(ws, ev, flags | NIX_RX_MULTI_SEG_F, \
 153                                     ws->lookup_mem);                    \
 154 }                                                                       \
 155                                                                         \
 156 uint16_t __rte_hot                                                              \
 157 otx2_ssogws_deq_seg_burst_ ##name(void *port, struct rte_event ev[],    \
 158                                   uint16_t nb_events,                   \
 159                                   uint64_t timeout_ticks)               \
 160 {                                                                       \
 161         RTE_SET_USED(nb_events);                                        \
 162                                                                         \
 163         return otx2_ssogws_deq_seg_ ##name(port, ev, timeout_ticks);    \
 164 }                                                                       \
 165                                                                         \
 166 uint16_t __rte_hot                                                              \
 167 otx2_ssogws_deq_seg_timeout_ ##name(void *port, struct rte_event *ev,   \
 168                                     uint64_t timeout_ticks)             \
 169 {                                                                       \
 170         struct otx2_ssogws *ws = port;                                  \
 171         uint16_t ret = 1;                                               \
 172         uint64_t iter;                                                  \
 173                                                                         \
 174         if (ws->swtag_req) {                                            \
 175                 ws->swtag_req = 0;                                      \
 176                 otx2_ssogws_swtag_wait(ws);                             \
 177                 return ret;                                             \
 178         }                                                               \
 179                                                                         \
 180         ret = otx2_ssogws_get_work(ws, ev, flags | NIX_RX_MULTI_SEG_F,  \
 181                                    ws->lookup_mem);                     \
 182         for (iter = 1; iter < timeout_ticks && (ret == 0); iter++)      \
 183                 ret = otx2_ssogws_get_work(ws, ev,                      \
 184                                            flags | NIX_RX_MULTI_SEG_F,  \
 185                                            ws->lookup_mem);             \
 186                                                                         \
 187         return ret;                                                     \
 188 }                                                                       \
 189                                                                         \
 190 uint16_t __rte_hot                                                              \
 191 otx2_ssogws_deq_seg_timeout_burst_ ##name(void *port,                   \
 192                                           struct rte_event ev[],        \
 193                                           uint16_t nb_events,           \
 194                                           uint64_t timeout_ticks)       \
 195 {                                                                       \
 196         RTE_SET_USED(nb_events);                                        \
 197                                                                         \
 198         return otx2_ssogws_deq_seg_timeout_ ##name(port, ev,            \
 199                                                    timeout_ticks);      \
 200 }
 201
 202 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 203 #undef R
 204
 205 uint16_t __rte_hot
 206 otx2_ssogws_enq(void *port, const struct rte_event *ev)
 207 {
 208         struct otx2_ssogws *ws = port;
 209
 210         switch (ev->op) {
 211         case RTE_EVENT_OP_NEW:
 212                 rte_smp_mb();
 213                 return otx2_ssogws_new_event(ws, ev);
 214         case RTE_EVENT_OP_FORWARD:
 215                 otx2_ssogws_forward_event(ws, ev);
 216                 break;
 217         case RTE_EVENT_OP_RELEASE:
 218                 otx2_ssogws_swtag_flush(ws->tag_op, ws->swtag_flush_op);
 219                 break;
 220         default:
 221                 return 0;
 222         }
 223
 224         return 1;
 225 }
 226
 227 uint16_t __rte_hot
 228 otx2_ssogws_enq_burst(void *port, const struct rte_event ev[],
 229                       uint16_t nb_events)
 230 {
 231         RTE_SET_USED(nb_events);
 232         return otx2_ssogws_enq(port, ev);
 233 }
 234
 235 uint16_t __rte_hot
 236 otx2_ssogws_enq_new_burst(void *port, const struct rte_event ev[],
 237                           uint16_t nb_events)
 238 {
 239         struct otx2_ssogws *ws = port;
 240         uint16_t i, rc = 1;
 241
 242         rte_smp_mb();
 243         if (ws->xaq_lmt <= *ws->fc_mem)
 244                 return 0;
 245
 246         for (i = 0; i < nb_events && rc; i++)
 247                 rc = otx2_ssogws_new_event(ws,  &ev[i]);
 248
 249         return nb_events;
 250 }
 251
 252 uint16_t __rte_hot
 253 otx2_ssogws_enq_fwd_burst(void *port, const struct rte_event ev[],
 254                           uint16_t nb_events)
 255 {
 256         struct otx2_ssogws *ws = port;
 257
 258         RTE_SET_USED(nb_events);
 259         otx2_ssogws_forward_event(ws,  ev);
 260
 261         return 1;
 262 }
 263
 264 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags)                  \
 265 uint16_t __rte_hot                                                      \
 266 otx2_ssogws_tx_adptr_enq_ ## name(void *port, struct rte_event ev[],    \
 267                                   uint16_t nb_events)                   \
 268 {                                                                       \
 269         struct otx2_ssogws *ws = port;                                  \
 270         uint64_t cmd[sz];                                               \
 271                                                                         \
 272         RTE_SET_USED(nb_events);                                        \
 273         return otx2_ssogws_event_tx(ws->base, &ev[0], cmd,              \
 274                                     (const uint64_t                     \
 275                                     (*)[RTE_MAX_QUEUES_PER_PORT])       \
 276                                     &ws->tx_adptr_data,                 \
 277                                     flags);                             \
 278 }
 279 SSO_TX_ADPTR_ENQ_FASTPATH_FUNC
 280 #undef T
 281
 282 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags)                  \
 283 uint16_t __rte_hot                                                      \
 284 otx2_ssogws_tx_adptr_enq_seg_ ## name(void *port, struct rte_event ev[],\
 285                                       uint16_t nb_events)               \
 286 {                                                                       \
 287         uint64_t cmd[(sz) + NIX_TX_MSEG_SG_DWORDS - 2];                 \
 288         struct otx2_ssogws *ws = port;                                  \
 289                                                                         \
 290         RTE_SET_USED(nb_events);                                        \
 291         return otx2_ssogws_event_tx(ws->base, &ev[0], cmd,              \
 292                                     (const uint64_t                     \
 293                                     (*)[RTE_MAX_QUEUES_PER_PORT])       \
 294                                     &ws->tx_adptr_data,                 \
 295                                     (flags) | NIX_TX_MULTI_SEG_F);      \
 296 }
 297 SSO_TX_ADPTR_ENQ_FASTPATH_FUNC
 298 #undef T
 299
 300 void
 301 ssogws_flush_events(struct otx2_ssogws *ws, uint8_t queue_id, uintptr_t base,
 302                     otx2_handle_event_t fn, void *arg)
 303 {
 304         uint64_t cq_ds_cnt = 1;
 305         uint64_t aq_cnt = 1;
 306         uint64_t ds_cnt = 1;
 307         struct rte_event ev;
 308         uint64_t enable;
 309         uint64_t val;
 310
 311         enable = otx2_read64(base + SSO_LF_GGRP_QCTL);
 312         if (!enable)
 313                 return;
 314
 315         val  = queue_id;        /* GGRP ID */
 316         val |= BIT_ULL(18);     /* Grouped */
 317         val |= BIT_ULL(16);     /* WAIT */
 318
 319         aq_cnt = otx2_read64(base + SSO_LF_GGRP_AQ_CNT);
 320         ds_cnt = otx2_read64(base + SSO_LF_GGRP_MISC_CNT);
 321         cq_ds_cnt = otx2_read64(base + SSO_LF_GGRP_INT_CNT);
 322         cq_ds_cnt &= 0x3FFF3FFF0000;
 323
 324         while (aq_cnt || cq_ds_cnt || ds_cnt) {
 325                 otx2_write64(val, ws->getwrk_op);
 326                 otx2_ssogws_get_work_empty(ws, &ev, 0);
 327                 if (fn != NULL && ev.u64 != 0)
 328                         fn(arg, ev);
 329                 if (ev.sched_type != SSO_TT_EMPTY)
 330                         otx2_ssogws_swtag_flush(ws->tag_op, ws->swtag_flush_op);
 331                 rte_mb();
 332                 aq_cnt = otx2_read64(base + SSO_LF_GGRP_AQ_CNT);
 333                 ds_cnt = otx2_read64(base + SSO_LF_GGRP_MISC_CNT);
 334                 cq_ds_cnt = otx2_read64(base + SSO_LF_GGRP_INT_CNT);
 335                 /* Extract cq and ds count */
 336                 cq_ds_cnt &= 0x3FFF3FFF0000;
 337         }
 338
 339         otx2_write64(0, OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op) +
 340                      SSOW_LF_GWS_OP_GWC_INVAL);
 341         rte_mb();
 342 }
 343
 344 void
 345 ssogws_reset(struct otx2_ssogws *ws)
 346 {
 347         uintptr_t base = OTX2_SSOW_GET_BASE_ADDR(ws->getwrk_op);
 348         uint64_t pend_state;
 349         uint8_t pend_tt;
 350         uint64_t tag;
 351
 352         /* Wait till getwork/swtp/waitw/desched completes. */
 353         do {
 354                 pend_state = otx2_read64(base + SSOW_LF_GWS_PENDSTATE);
 355                 rte_mb();
 356         } while (pend_state & (BIT_ULL(63) | BIT_ULL(62) | BIT_ULL(58)));
 357
 358         tag = otx2_read64(base + SSOW_LF_GWS_TAG);
 359         pend_tt = (tag >> 32) & 0x3;
 360         if (pend_tt != SSO_TT_EMPTY) { /* Work was pending */
 361                 if (pend_tt == SSO_SYNC_ATOMIC || pend_tt == SSO_SYNC_ORDERED)
 362                         otx2_ssogws_swtag_untag(ws);
 363                 otx2_ssogws_desched(ws);
 364         }
 365         rte_mb();
 366
 367         /* Wait for desched to complete. */
 368         do {
 369                 pend_state = otx2_read64(base + SSOW_LF_GWS_PENDSTATE);
 370                 rte_mb();
 371         } while (pend_state & BIT_ULL(58));
 372 }