drivers/event/cnxk/cn10k_eventdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include "cn10k_worker.h"
   6 #include "cnxk_eventdev.h"
   7 #include "cnxk_worker.h"
   8
   9 #define CN10K_SET_EVDEV_DEQ_OP(dev, deq_op, deq_ops)                           \
  10         deq_op = deq_ops[dev->rx_offloads & (NIX_RX_OFFLOAD_MAX - 1)]
  11
  12 #define CN10K_SET_EVDEV_ENQ_OP(dev, enq_op, enq_ops)                           \
  13         (enq_op =                                                              \
  14                  enq_ops[!!(dev->tx_offloads & NIX_TX_OFFLOAD_SECURITY_F)]     \
  15                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_TSTAMP_F)]       \
  16                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_TSO_F)]          \
  17                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_MBUF_NOFF_F)]    \
  18                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_VLAN_QINQ_F)]    \
  19                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)] \
  20                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_L3_L4_CSUM_F)])
  21
  22 static uint32_t
  23 cn10k_sso_gw_mode_wdata(struct cnxk_sso_evdev *dev)
  24 {
  25         uint32_t wdata = BIT(16) | 1;
  26
  27         switch (dev->gw_mode) {
  28         case CN10K_GW_MODE_NONE:
  29         default:
  30                 break;
  31         case CN10K_GW_MODE_PREF:
  32                 wdata |= BIT(19);
  33                 break;
  34         case CN10K_GW_MODE_PREF_WFE:
  35                 wdata |= BIT(20) | BIT(19);
  36                 break;
  37         }
  38
  39         return wdata;
  40 }
  41
  42 static void *
  43 cn10k_sso_init_hws_mem(void *arg, uint8_t port_id)
  44 {
  45         struct cnxk_sso_evdev *dev = arg;
  46         struct cn10k_sso_hws *ws;
  47
  48         /* Allocate event port memory */
  49         ws = rte_zmalloc("cn10k_ws",
  50                          sizeof(struct cn10k_sso_hws) + RTE_CACHE_LINE_SIZE,
  51                          RTE_CACHE_LINE_SIZE);
  52         if (ws == NULL) {
  53                 plt_err("Failed to alloc memory for port=%d", port_id);
  54                 return NULL;
  55         }
  56
  57         /* First cache line is reserved for cookie */
  58         ws = (struct cn10k_sso_hws *)((uint8_t *)ws + RTE_CACHE_LINE_SIZE);
  59         ws->base = roc_sso_hws_base_get(&dev->sso, port_id);
  60         ws->tx_base = ws->base;
  61         ws->hws_id = port_id;
  62         ws->swtag_req = 0;
  63         ws->gw_wdata = cn10k_sso_gw_mode_wdata(dev);
  64         ws->lmt_base = dev->sso.lmt_base;
  65
  66         return ws;
  67 }
  68
  69 static int
  70 cn10k_sso_hws_link(void *arg, void *port, uint16_t *map, uint16_t nb_link)
  71 {
  72         struct cnxk_sso_evdev *dev = arg;
  73         struct cn10k_sso_hws *ws = port;
  74
  75         return roc_sso_hws_link(&dev->sso, ws->hws_id, map, nb_link);
  76 }
  77
  78 static int
  79 cn10k_sso_hws_unlink(void *arg, void *port, uint16_t *map, uint16_t nb_link)
  80 {
  81         struct cnxk_sso_evdev *dev = arg;
  82         struct cn10k_sso_hws *ws = port;
  83
  84         return roc_sso_hws_unlink(&dev->sso, ws->hws_id, map, nb_link);
  85 }
  86
  87 static void
  88 cn10k_sso_hws_setup(void *arg, void *hws, uintptr_t grp_base)
  89 {
  90         struct cnxk_sso_evdev *dev = arg;
  91         struct cn10k_sso_hws *ws = hws;
  92         uint64_t val;
  93
  94         ws->grp_base = grp_base;
  95         ws->fc_mem = (uint64_t *)dev->fc_iova;
  96         ws->xaq_lmt = dev->xaq_lmt;
  97
  98         /* Set get_work timeout for HWS */
  99         val = NSEC2USEC(dev->deq_tmo_ns) - 1;
 100         plt_write64(val, ws->base + SSOW_LF_GWS_NW_TIM);
 101 }
 102
 103 static void
 104 cn10k_sso_hws_release(void *arg, void *hws)
 105 {
 106         struct cnxk_sso_evdev *dev = arg;
 107         struct cn10k_sso_hws *ws = hws;
 108         int i;
 109
 110         for (i = 0; i < dev->nb_event_queues; i++)
 111                 roc_sso_hws_unlink(&dev->sso, ws->hws_id, (uint16_t *)&i, 1);
 112         memset(ws, 0, sizeof(*ws));
 113 }
 114
 115 static void
 116 cn10k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base,
 117                            cnxk_handle_event_t fn, void *arg)
 118 {
 119         struct cn10k_sso_hws *ws = hws;
 120         uint64_t cq_ds_cnt = 1;
 121         uint64_t aq_cnt = 1;
 122         uint64_t ds_cnt = 1;
 123         struct rte_event ev;
 124         uint64_t val, req;
 125
 126         plt_write64(0, base + SSO_LF_GGRP_QCTL);
 127
 128         plt_write64(0, ws->base + SSOW_LF_GWS_OP_GWC_INVAL);
 129         req = queue_id;     /* GGRP ID */
 130         req |= BIT_ULL(18); /* Grouped */
 131         req |= BIT_ULL(16); /* WAIT */
 132
 133         aq_cnt = plt_read64(base + SSO_LF_GGRP_AQ_CNT);
 134         ds_cnt = plt_read64(base + SSO_LF_GGRP_MISC_CNT);
 135         cq_ds_cnt = plt_read64(base + SSO_LF_GGRP_INT_CNT);
 136         cq_ds_cnt &= 0x3FFF3FFF0000;
 137
 138         while (aq_cnt || cq_ds_cnt || ds_cnt) {
 139                 plt_write64(req, ws->base + SSOW_LF_GWS_OP_GET_WORK0);
 140                 cn10k_sso_hws_get_work_empty(ws, &ev);
 141                 if (fn != NULL && ev.u64 != 0)
 142                         fn(arg, ev);
 143                 if (ev.sched_type != SSO_TT_EMPTY)
 144                         cnxk_sso_hws_swtag_flush(
 145                                 ws->base + SSOW_LF_GWS_WQE0,
 146                                 ws->base + SSOW_LF_GWS_OP_SWTAG_FLUSH);
 147                 do {
 148                         val = plt_read64(ws->base + SSOW_LF_GWS_PENDSTATE);
 149                 } while (val & BIT_ULL(56));
 150                 aq_cnt = plt_read64(base + SSO_LF_GGRP_AQ_CNT);
 151                 ds_cnt = plt_read64(base + SSO_LF_GGRP_MISC_CNT);
 152                 cq_ds_cnt = plt_read64(base + SSO_LF_GGRP_INT_CNT);
 153                 /* Extract cq and ds count */
 154                 cq_ds_cnt &= 0x3FFF3FFF0000;
 155         }
 156
 157         plt_write64(0, ws->base + SSOW_LF_GWS_OP_GWC_INVAL);
 158         rte_mb();
 159 }
 160
 161 static void
 162 cn10k_sso_hws_reset(void *arg, void *hws)
 163 {
 164         struct cnxk_sso_evdev *dev = arg;
 165         struct cn10k_sso_hws *ws = hws;
 166         uintptr_t base = ws->base;
 167         uint64_t pend_state;
 168         union {
 169                 __uint128_t wdata;
 170                 uint64_t u64[2];
 171         } gw;
 172         uint8_t pend_tt;
 173
 174         plt_write64(0, ws->base + SSOW_LF_GWS_OP_GWC_INVAL);
 175         /* Wait till getwork/swtp/waitw/desched completes. */
 176         do {
 177                 pend_state = plt_read64(base + SSOW_LF_GWS_PENDSTATE);
 178         } while (pend_state & (BIT_ULL(63) | BIT_ULL(62) | BIT_ULL(58) |
 179                                BIT_ULL(56) | BIT_ULL(54)));
 180         pend_tt = CNXK_TT_FROM_TAG(plt_read64(base + SSOW_LF_GWS_WQE0));
 181         if (pend_tt != SSO_TT_EMPTY) { /* Work was pending */
 182                 if (pend_tt == SSO_TT_ATOMIC || pend_tt == SSO_TT_ORDERED)
 183                         cnxk_sso_hws_swtag_untag(base +
 184                                                  SSOW_LF_GWS_OP_SWTAG_UNTAG);
 185                 plt_write64(0, base + SSOW_LF_GWS_OP_DESCHED);
 186         }
 187
 188         /* Wait for desched to complete. */
 189         do {
 190                 pend_state = plt_read64(base + SSOW_LF_GWS_PENDSTATE);
 191         } while (pend_state & BIT_ULL(58));
 192
 193         switch (dev->gw_mode) {
 194         case CN10K_GW_MODE_PREF:
 195                 while (plt_read64(base + SSOW_LF_GWS_PRF_WQE0) & BIT_ULL(63))
 196                         ;
 197                 break;
 198         case CN10K_GW_MODE_PREF_WFE:
 199                 while (plt_read64(base + SSOW_LF_GWS_PRF_WQE0) &
 200                        SSOW_LF_GWS_TAG_PEND_GET_WORK_BIT)
 201                         continue;
 202                 plt_write64(0, base + SSOW_LF_GWS_OP_GWC_INVAL);
 203                 break;
 204         case CN10K_GW_MODE_NONE:
 205         default:
 206                 break;
 207         }
 208
 209         if (CNXK_TT_FROM_TAG(plt_read64(base + SSOW_LF_GWS_PRF_WQE0)) !=
 210             SSO_TT_EMPTY) {
 211                 plt_write64(BIT_ULL(16) | 1,
 212                             ws->base + SSOW_LF_GWS_OP_GET_WORK0);
 213                 do {
 214                         roc_load_pair(gw.u64[0], gw.u64[1],
 215                                       ws->base + SSOW_LF_GWS_WQE0);
 216                 } while (gw.u64[0] & BIT_ULL(63));
 217                 pend_tt = CNXK_TT_FROM_TAG(plt_read64(base + SSOW_LF_GWS_WQE0));
 218                 if (pend_tt != SSO_TT_EMPTY) { /* Work was pending */
 219                         if (pend_tt == SSO_TT_ATOMIC ||
 220                             pend_tt == SSO_TT_ORDERED)
 221                                 cnxk_sso_hws_swtag_untag(
 222                                         base + SSOW_LF_GWS_OP_SWTAG_UNTAG);
 223                         plt_write64(0, base + SSOW_LF_GWS_OP_DESCHED);
 224                 }
 225         }
 226
 227         plt_write64(0, base + SSOW_LF_GWS_OP_GWC_INVAL);
 228         rte_mb();
 229 }
 230
 231 static void
 232 cn10k_sso_set_rsrc(void *arg)
 233 {
 234         struct cnxk_sso_evdev *dev = arg;
 235
 236         dev->max_event_ports = dev->sso.max_hws;
 237         dev->max_event_queues =
 238                 dev->sso.max_hwgrp > RTE_EVENT_MAX_QUEUES_PER_DEV ?
 239                               RTE_EVENT_MAX_QUEUES_PER_DEV :
 240                               dev->sso.max_hwgrp;
 241 }
 242
 243 static int
 244 cn10k_sso_rsrc_init(void *arg, uint8_t hws, uint8_t hwgrp)
 245 {
 246         struct cnxk_sso_evdev *dev = arg;
 247
 248         return roc_sso_rsrc_init(&dev->sso, hws, hwgrp);
 249 }
 250
 251 static int
 252 cn10k_sso_updt_tx_adptr_data(const struct rte_eventdev *event_dev)
 253 {
 254         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 255         int i;
 256
 257         if (dev->tx_adptr_data == NULL)
 258                 return 0;
 259
 260         for (i = 0; i < dev->nb_event_ports; i++) {
 261                 struct cn10k_sso_hws *ws = event_dev->data->ports[i];
 262                 void *ws_cookie;
 263
 264                 ws_cookie = cnxk_sso_hws_get_cookie(ws);
 265                 ws_cookie = rte_realloc_socket(
 266                         ws_cookie,
 267                         sizeof(struct cnxk_sso_hws_cookie) +
 268                                 sizeof(struct cn10k_sso_hws) +
 269                                 (sizeof(uint64_t) * (dev->max_port_id + 1) *
 270                                  RTE_MAX_QUEUES_PER_PORT),
 271                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 272                 if (ws_cookie == NULL)
 273                         return -ENOMEM;
 274                 ws = RTE_PTR_ADD(ws_cookie, sizeof(struct cnxk_sso_hws_cookie));
 275                 memcpy(&ws->tx_adptr_data, dev->tx_adptr_data,
 276                        sizeof(uint64_t) * (dev->max_port_id + 1) *
 277                                RTE_MAX_QUEUES_PER_PORT);
 278                 event_dev->data->ports[i] = ws;
 279         }
 280
 281         return 0;
 282 }
 283
 284 static void
 285 cn10k_sso_fp_fns_set(struct rte_eventdev *event_dev)
 286 {
 287         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 288         const event_dequeue_t sso_hws_deq[NIX_RX_OFFLOAD_MAX] = {
 289 #define R(name, flags)[flags] = cn10k_sso_hws_deq_##name,
 290                 NIX_RX_FASTPATH_MODES
 291 #undef R
 292         };
 293
 294         const event_dequeue_burst_t sso_hws_deq_burst[NIX_RX_OFFLOAD_MAX] = {
 295 #define R(name, flags)[flags] = cn10k_sso_hws_deq_burst_##name,
 296                 NIX_RX_FASTPATH_MODES
 297 #undef R
 298         };
 299
 300         const event_dequeue_t sso_hws_deq_tmo[NIX_RX_OFFLOAD_MAX] = {
 301 #define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_##name,
 302                 NIX_RX_FASTPATH_MODES
 303 #undef R
 304         };
 305
 306         const event_dequeue_burst_t sso_hws_deq_tmo_burst[NIX_RX_OFFLOAD_MAX] = {
 307 #define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_burst_##name,
 308                 NIX_RX_FASTPATH_MODES
 309 #undef R
 310         };
 311
 312         const event_dequeue_t sso_hws_deq_ca[NIX_RX_OFFLOAD_MAX] = {
 313 #define R(name, flags)[flags] = cn10k_sso_hws_deq_ca_##name,
 314                 NIX_RX_FASTPATH_MODES
 315 #undef R
 316         };
 317
 318         const event_dequeue_burst_t sso_hws_deq_ca_burst[NIX_RX_OFFLOAD_MAX] = {
 319 #define R(name, flags)[flags] = cn10k_sso_hws_deq_ca_burst_##name,
 320                 NIX_RX_FASTPATH_MODES
 321 #undef R
 322         };
 323
 324         const event_dequeue_t sso_hws_deq_tmo_ca[NIX_RX_OFFLOAD_MAX] = {
 325 #define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_ca_##name,
 326                 NIX_RX_FASTPATH_MODES
 327 #undef R
 328         };
 329
 330         const event_dequeue_burst_t sso_hws_deq_tmo_ca_burst[NIX_RX_OFFLOAD_MAX] = {
 331 #define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_ca_burst_##name,
 332                 NIX_RX_FASTPATH_MODES
 333 #undef R
 334         };
 335
 336         const event_dequeue_t sso_hws_deq_seg[NIX_RX_OFFLOAD_MAX] = {
 337 #define R(name, flags)[flags] = cn10k_sso_hws_deq_seg_##name,
 338
 339                 NIX_RX_FASTPATH_MODES
 340 #undef R
 341         };
 342
 343         const event_dequeue_burst_t sso_hws_deq_seg_burst[NIX_RX_OFFLOAD_MAX] = {
 344 #define R(name, flags)[flags] = cn10k_sso_hws_deq_seg_burst_##name,
 345                         NIX_RX_FASTPATH_MODES
 346 #undef R
 347         };
 348
 349         const event_dequeue_t sso_hws_deq_tmo_seg[NIX_RX_OFFLOAD_MAX] = {
 350 #define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_seg_##name,
 351                 NIX_RX_FASTPATH_MODES
 352 #undef R
 353         };
 354
 355         const event_dequeue_burst_t sso_hws_deq_tmo_seg_burst[NIX_RX_OFFLOAD_MAX] = {
 356 #define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_seg_burst_##name,
 357                 NIX_RX_FASTPATH_MODES
 358 #undef R
 359         };
 360
 361         const event_dequeue_t sso_hws_deq_ca_seg[NIX_RX_OFFLOAD_MAX] = {
 362 #define R(name, flags)[flags] = cn10k_sso_hws_deq_ca_seg_##name,
 363                 NIX_RX_FASTPATH_MODES
 364 #undef R
 365         };
 366
 367         const event_dequeue_burst_t sso_hws_deq_ca_seg_burst[NIX_RX_OFFLOAD_MAX] = {
 368 #define R(name, flags)[flags] = cn10k_sso_hws_deq_ca_seg_burst_##name,
 369                 NIX_RX_FASTPATH_MODES
 370 #undef R
 371         };
 372
 373         const event_dequeue_t sso_hws_deq_tmo_ca_seg[NIX_RX_OFFLOAD_MAX] = {
 374 #define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_ca_seg_##name,
 375                 NIX_RX_FASTPATH_MODES
 376 #undef R
 377         };
 378
 379         const event_dequeue_burst_t sso_hws_deq_tmo_ca_seg_burst[NIX_RX_OFFLOAD_MAX] = {
 380 #define R(name, flags)[flags] = cn10k_sso_hws_deq_tmo_ca_seg_burst_##name,
 381                 NIX_RX_FASTPATH_MODES
 382 #undef R
 383         };
 384
 385         /* Tx modes */
 386         const event_tx_adapter_enqueue_t
 387                 sso_hws_tx_adptr_enq[2][2][2][2][2][2][2] = {
 388 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags)                         \
 389         [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_tx_adptr_enq_##name,
 390                         NIX_TX_FASTPATH_MODES
 391 #undef T
 392                 };
 393
 394         const event_tx_adapter_enqueue_t
 395                 sso_hws_tx_adptr_enq_seg[2][2][2][2][2][2][2] = {
 396 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags)                         \
 397         [f6][f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_tx_adptr_enq_seg_##name,
 398                         NIX_TX_FASTPATH_MODES
 399 #undef T
 400                 };
 401
 402         event_dev->enqueue = cn10k_sso_hws_enq;
 403         event_dev->enqueue_burst = cn10k_sso_hws_enq_burst;
 404         event_dev->enqueue_new_burst = cn10k_sso_hws_enq_new_burst;
 405         event_dev->enqueue_forward_burst = cn10k_sso_hws_enq_fwd_burst;
 406         if (dev->rx_offloads & NIX_RX_MULTI_SEG_F) {
 407                 CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
 408                                        sso_hws_deq_seg);
 409                 CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
 410                                        sso_hws_deq_seg_burst);
 411                 if (dev->is_timeout_deq) {
 412                         CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
 413                                                sso_hws_deq_tmo_seg);
 414                         CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
 415                                                sso_hws_deq_tmo_seg_burst);
 416                 }
 417                 if (dev->is_ca_internal_port) {
 418                         CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
 419                                                sso_hws_deq_ca_seg);
 420                         CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
 421                                                sso_hws_deq_ca_seg_burst);
 422                 }
 423                 if (dev->is_timeout_deq && dev->is_ca_internal_port) {
 424                         CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
 425                                                sso_hws_deq_tmo_ca_seg);
 426                         CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
 427                                                sso_hws_deq_tmo_ca_seg_burst);
 428                 }
 429         } else {
 430                 CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue, sso_hws_deq);
 431                 CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
 432                                        sso_hws_deq_burst);
 433                 if (dev->is_timeout_deq) {
 434                         CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
 435                                                sso_hws_deq_tmo);
 436                         CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
 437                                                sso_hws_deq_tmo_burst);
 438                 }
 439                 if (dev->is_ca_internal_port) {
 440                         CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
 441                                                sso_hws_deq_ca);
 442                         CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
 443                                                sso_hws_deq_ca_burst);
 444                 }
 445                 if (dev->is_timeout_deq && dev->is_ca_internal_port) {
 446                         CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
 447                                                sso_hws_deq_tmo_ca);
 448                         CN10K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
 449                                                sso_hws_deq_tmo_ca_burst);
 450                 }
 451         }
 452         event_dev->ca_enqueue = cn10k_sso_hws_ca_enq;
 453
 454         if (dev->tx_offloads & NIX_TX_MULTI_SEG_F)
 455                 CN10K_SET_EVDEV_ENQ_OP(dev, event_dev->txa_enqueue,
 456                                        sso_hws_tx_adptr_enq_seg);
 457         else
 458                 CN10K_SET_EVDEV_ENQ_OP(dev, event_dev->txa_enqueue,
 459                                        sso_hws_tx_adptr_enq);
 460
 461         event_dev->txa_enqueue_same_dest = event_dev->txa_enqueue;
 462 }
 463
 464 static void
 465 cn10k_sso_info_get(struct rte_eventdev *event_dev,
 466                    struct rte_event_dev_info *dev_info)
 467 {
 468         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 469
 470         dev_info->driver_name = RTE_STR(EVENTDEV_NAME_CN10K_PMD);
 471         cnxk_sso_info_get(dev, dev_info);
 472 }
 473
 474 static int
 475 cn10k_sso_dev_configure(const struct rte_eventdev *event_dev)
 476 {
 477         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 478         int rc;
 479
 480         rc = cnxk_sso_dev_validate(event_dev);
 481         if (rc < 0) {
 482                 plt_err("Invalid event device configuration");
 483                 return -EINVAL;
 484         }
 485
 486         rc = cn10k_sso_rsrc_init(dev, dev->nb_event_ports,
 487                                  dev->nb_event_queues);
 488         if (rc < 0) {
 489                 plt_err("Failed to initialize SSO resources");
 490                 return -ENODEV;
 491         }
 492
 493         rc = cnxk_sso_xaq_allocate(dev);
 494         if (rc < 0)
 495                 goto cnxk_rsrc_fini;
 496
 497         rc = cnxk_setup_event_ports(event_dev, cn10k_sso_init_hws_mem,
 498                                     cn10k_sso_hws_setup);
 499         if (rc < 0)
 500                 goto cnxk_rsrc_fini;
 501
 502         /* Restore any prior port-queue mapping. */
 503         cnxk_sso_restore_links(event_dev, cn10k_sso_hws_link);
 504
 505         dev->configured = 1;
 506         rte_mb();
 507
 508         return 0;
 509 cnxk_rsrc_fini:
 510         roc_sso_rsrc_fini(&dev->sso);
 511         dev->nb_event_ports = 0;
 512         return rc;
 513 }
 514
 515 static int
 516 cn10k_sso_port_setup(struct rte_eventdev *event_dev, uint8_t port_id,
 517                      const struct rte_event_port_conf *port_conf)
 518 {
 519
 520         RTE_SET_USED(port_conf);
 521         return cnxk_sso_port_setup(event_dev, port_id, cn10k_sso_hws_setup);
 522 }
 523
 524 static void
 525 cn10k_sso_port_release(void *port)
 526 {
 527         struct cnxk_sso_hws_cookie *gws_cookie = cnxk_sso_hws_get_cookie(port);
 528         struct cnxk_sso_evdev *dev;
 529
 530         if (port == NULL)
 531                 return;
 532
 533         dev = cnxk_sso_pmd_priv(gws_cookie->event_dev);
 534         if (!gws_cookie->configured)
 535                 goto free;
 536
 537         cn10k_sso_hws_release(dev, port);
 538         memset(gws_cookie, 0, sizeof(*gws_cookie));
 539 free:
 540         rte_free(gws_cookie);
 541 }
 542
 543 static int
 544 cn10k_sso_port_link(struct rte_eventdev *event_dev, void *port,
 545                     const uint8_t queues[], const uint8_t priorities[],
 546                     uint16_t nb_links)
 547 {
 548         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 549         uint16_t hwgrp_ids[nb_links];
 550         uint16_t link;
 551
 552         RTE_SET_USED(priorities);
 553         for (link = 0; link < nb_links; link++)
 554                 hwgrp_ids[link] = queues[link];
 555         nb_links = cn10k_sso_hws_link(dev, port, hwgrp_ids, nb_links);
 556
 557         return (int)nb_links;
 558 }
 559
 560 static int
 561 cn10k_sso_port_unlink(struct rte_eventdev *event_dev, void *port,
 562                       uint8_t queues[], uint16_t nb_unlinks)
 563 {
 564         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 565         uint16_t hwgrp_ids[nb_unlinks];
 566         uint16_t unlink;
 567
 568         for (unlink = 0; unlink < nb_unlinks; unlink++)
 569                 hwgrp_ids[unlink] = queues[unlink];
 570         nb_unlinks = cn10k_sso_hws_unlink(dev, port, hwgrp_ids, nb_unlinks);
 571
 572         return (int)nb_unlinks;
 573 }
 574
 575 static int
 576 cn10k_sso_start(struct rte_eventdev *event_dev)
 577 {
 578         int rc;
 579
 580         rc = cn10k_sso_updt_tx_adptr_data(event_dev);
 581         if (rc < 0)
 582                 return rc;
 583
 584         rc = cnxk_sso_start(event_dev, cn10k_sso_hws_reset,
 585                             cn10k_sso_hws_flush_events);
 586         if (rc < 0)
 587                 return rc;
 588         cn10k_sso_fp_fns_set(event_dev);
 589
 590         return rc;
 591 }
 592
 593 static void
 594 cn10k_sso_stop(struct rte_eventdev *event_dev)
 595 {
 596         cnxk_sso_stop(event_dev, cn10k_sso_hws_reset,
 597                       cn10k_sso_hws_flush_events);
 598 }
 599
 600 static int
 601 cn10k_sso_close(struct rte_eventdev *event_dev)
 602 {
 603         return cnxk_sso_close(event_dev, cn10k_sso_hws_unlink);
 604 }
 605
 606 static int
 607 cn10k_sso_selftest(void)
 608 {
 609         return cnxk_sso_selftest(RTE_STR(event_cn10k));
 610 }
 611
 612 static int
 613 cn10k_sso_rx_adapter_caps_get(const struct rte_eventdev *event_dev,
 614                               const struct rte_eth_dev *eth_dev, uint32_t *caps)
 615 {
 616         int rc;
 617
 618         RTE_SET_USED(event_dev);
 619         rc = strncmp(eth_dev->device->driver->name, "net_cn10k", 9);
 620         if (rc)
 621                 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP;
 622         else
 623                 *caps = RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT |
 624                         RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ |
 625                         RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID |
 626                         RTE_EVENT_ETH_RX_ADAPTER_CAP_EVENT_VECTOR;
 627
 628         return 0;
 629 }
 630
 631 static void
 632 cn10k_sso_set_priv_mem(const struct rte_eventdev *event_dev, void *lookup_mem,
 633                        void *tstmp_info)
 634 {
 635         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 636         int i;
 637
 638         for (i = 0; i < dev->nb_event_ports; i++) {
 639                 struct cn10k_sso_hws *ws = event_dev->data->ports[i];
 640                 ws->lookup_mem = lookup_mem;
 641                 ws->tstamp = tstmp_info;
 642         }
 643 }
 644
 645 static int
 646 cn10k_sso_rx_adapter_queue_add(
 647         const struct rte_eventdev *event_dev, const struct rte_eth_dev *eth_dev,
 648         int32_t rx_queue_id,
 649         const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
 650 {
 651         struct cn10k_eth_rxq *rxq;
 652         void *lookup_mem;
 653         void *tstmp_info;
 654         int rc;
 655
 656         rc = strncmp(eth_dev->device->driver->name, "net_cn10k", 8);
 657         if (rc)
 658                 return -EINVAL;
 659
 660         rc = cnxk_sso_rx_adapter_queue_add(event_dev, eth_dev, rx_queue_id,
 661                                            queue_conf);
 662         if (rc)
 663                 return -EINVAL;
 664         rxq = eth_dev->data->rx_queues[0];
 665         lookup_mem = rxq->lookup_mem;
 666         tstmp_info = rxq->tstamp;
 667         cn10k_sso_set_priv_mem(event_dev, lookup_mem, tstmp_info);
 668         cn10k_sso_fp_fns_set((struct rte_eventdev *)(uintptr_t)event_dev);
 669
 670         return 0;
 671 }
 672
 673 static int
 674 cn10k_sso_rx_adapter_queue_del(const struct rte_eventdev *event_dev,
 675                                const struct rte_eth_dev *eth_dev,
 676                                int32_t rx_queue_id)
 677 {
 678         int rc;
 679
 680         rc = strncmp(eth_dev->device->driver->name, "net_cn10k", 8);
 681         if (rc)
 682                 return -EINVAL;
 683
 684         return cnxk_sso_rx_adapter_queue_del(event_dev, eth_dev, rx_queue_id);
 685 }
 686
 687 static int
 688 cn10k_sso_rx_adapter_vector_limits(
 689         const struct rte_eventdev *dev, const struct rte_eth_dev *eth_dev,
 690         struct rte_event_eth_rx_adapter_vector_limits *limits)
 691 {
 692         struct cnxk_eth_dev *cnxk_eth_dev;
 693         int ret;
 694
 695         RTE_SET_USED(dev);
 696         ret = strncmp(eth_dev->device->driver->name, "net_cn10k", 8);
 697         if (ret)
 698                 return -ENOTSUP;
 699
 700         cnxk_eth_dev = cnxk_eth_pmd_priv(eth_dev);
 701         limits->log2_sz = true;
 702         limits->min_sz = 1 << ROC_NIX_VWQE_MIN_SIZE_LOG2;
 703         limits->max_sz = 1 << ROC_NIX_VWQE_MAX_SIZE_LOG2;
 704         limits->min_timeout_ns =
 705                 (roc_nix_get_vwqe_interval(&cnxk_eth_dev->nix) + 1) * 100;
 706         limits->max_timeout_ns = BITMASK_ULL(8, 0) * limits->min_timeout_ns;
 707
 708         return 0;
 709 }
 710
 711 static int
 712 cn10k_sso_tx_adapter_caps_get(const struct rte_eventdev *dev,
 713                               const struct rte_eth_dev *eth_dev, uint32_t *caps)
 714 {
 715         int ret;
 716
 717         RTE_SET_USED(dev);
 718         ret = strncmp(eth_dev->device->driver->name, "net_cn10k", 8);
 719         if (ret)
 720                 *caps = 0;
 721         else
 722                 *caps = RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT |
 723                         RTE_EVENT_ETH_TX_ADAPTER_CAP_EVENT_VECTOR;
 724
 725         return 0;
 726 }
 727
 728 static int
 729 cn10k_sso_tx_adapter_queue_add(uint8_t id, const struct rte_eventdev *event_dev,
 730                                const struct rte_eth_dev *eth_dev,
 731                                int32_t tx_queue_id)
 732 {
 733         int rc;
 734
 735         RTE_SET_USED(id);
 736         rc = cnxk_sso_tx_adapter_queue_add(event_dev, eth_dev, tx_queue_id);
 737         if (rc < 0)
 738                 return rc;
 739         rc = cn10k_sso_updt_tx_adptr_data(event_dev);
 740         if (rc < 0)
 741                 return rc;
 742         cn10k_sso_fp_fns_set((struct rte_eventdev *)(uintptr_t)event_dev);
 743
 744         return 0;
 745 }
 746
 747 static int
 748 cn10k_sso_tx_adapter_queue_del(uint8_t id, const struct rte_eventdev *event_dev,
 749                                const struct rte_eth_dev *eth_dev,
 750                                int32_t tx_queue_id)
 751 {
 752         int rc;
 753
 754         RTE_SET_USED(id);
 755         rc = cnxk_sso_tx_adapter_queue_del(event_dev, eth_dev, tx_queue_id);
 756         if (rc < 0)
 757                 return rc;
 758         return cn10k_sso_updt_tx_adptr_data(event_dev);
 759 }
 760
 761 static int
 762 cn10k_crypto_adapter_caps_get(const struct rte_eventdev *event_dev,
 763                               const struct rte_cryptodev *cdev, uint32_t *caps)
 764 {
 765         CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k");
 766         CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k");
 767
 768         *caps = RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD |
 769                 RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA;
 770
 771         return 0;
 772 }
 773
 774 static int
 775 cn10k_crypto_adapter_qp_add(const struct rte_eventdev *event_dev,
 776                             const struct rte_cryptodev *cdev,
 777                             int32_t queue_pair_id,
 778                             const struct rte_event *event)
 779 {
 780         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 781
 782         RTE_SET_USED(event);
 783
 784         CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k");
 785         CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k");
 786
 787         dev->is_ca_internal_port = 1;
 788         cn10k_sso_fp_fns_set((struct rte_eventdev *)(uintptr_t)event_dev);
 789
 790         return cnxk_crypto_adapter_qp_add(event_dev, cdev, queue_pair_id);
 791 }
 792
 793 static int
 794 cn10k_crypto_adapter_qp_del(const struct rte_eventdev *event_dev,
 795                             const struct rte_cryptodev *cdev,
 796                             int32_t queue_pair_id)
 797 {
 798         CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k");
 799         CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k");
 800
 801         return cnxk_crypto_adapter_qp_del(cdev, queue_pair_id);
 802 }
 803
 804 static struct eventdev_ops cn10k_sso_dev_ops = {
 805         .dev_infos_get = cn10k_sso_info_get,
 806         .dev_configure = cn10k_sso_dev_configure,
 807         .queue_def_conf = cnxk_sso_queue_def_conf,
 808         .queue_setup = cnxk_sso_queue_setup,
 809         .queue_release = cnxk_sso_queue_release,
 810         .port_def_conf = cnxk_sso_port_def_conf,
 811         .port_setup = cn10k_sso_port_setup,
 812         .port_release = cn10k_sso_port_release,
 813         .port_link = cn10k_sso_port_link,
 814         .port_unlink = cn10k_sso_port_unlink,
 815         .timeout_ticks = cnxk_sso_timeout_ticks,
 816
 817         .eth_rx_adapter_caps_get = cn10k_sso_rx_adapter_caps_get,
 818         .eth_rx_adapter_queue_add = cn10k_sso_rx_adapter_queue_add,
 819         .eth_rx_adapter_queue_del = cn10k_sso_rx_adapter_queue_del,
 820         .eth_rx_adapter_start = cnxk_sso_rx_adapter_start,
 821         .eth_rx_adapter_stop = cnxk_sso_rx_adapter_stop,
 822
 823         .eth_rx_adapter_vector_limits_get = cn10k_sso_rx_adapter_vector_limits,
 824
 825         .eth_tx_adapter_caps_get = cn10k_sso_tx_adapter_caps_get,
 826         .eth_tx_adapter_queue_add = cn10k_sso_tx_adapter_queue_add,
 827         .eth_tx_adapter_queue_del = cn10k_sso_tx_adapter_queue_del,
 828
 829         .timer_adapter_caps_get = cnxk_tim_caps_get,
 830
 831         .crypto_adapter_caps_get = cn10k_crypto_adapter_caps_get,
 832         .crypto_adapter_queue_pair_add = cn10k_crypto_adapter_qp_add,
 833         .crypto_adapter_queue_pair_del = cn10k_crypto_adapter_qp_del,
 834
 835         .dump = cnxk_sso_dump,
 836         .dev_start = cn10k_sso_start,
 837         .dev_stop = cn10k_sso_stop,
 838         .dev_close = cn10k_sso_close,
 839         .dev_selftest = cn10k_sso_selftest,
 840 };
 841
 842 static int
 843 cn10k_sso_init(struct rte_eventdev *event_dev)
 844 {
 845         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 846         int rc;
 847
 848         if (RTE_CACHE_LINE_SIZE != 64) {
 849                 plt_err("Driver not compiled for CN10K");
 850                 return -EFAULT;
 851         }
 852
 853         rc = roc_plt_init();
 854         if (rc < 0) {
 855                 plt_err("Failed to initialize platform model");
 856                 return rc;
 857         }
 858
 859         event_dev->dev_ops = &cn10k_sso_dev_ops;
 860         /* For secondary processes, the primary has done all the work */
 861         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 862                 cn10k_sso_fp_fns_set(event_dev);
 863                 return 0;
 864         }
 865
 866         rc = cnxk_sso_init(event_dev);
 867         if (rc < 0)
 868                 return rc;
 869
 870         cn10k_sso_set_rsrc(cnxk_sso_pmd_priv(event_dev));
 871         if (!dev->max_event_ports || !dev->max_event_queues) {
 872                 plt_err("Not enough eventdev resource queues=%d ports=%d",
 873                         dev->max_event_queues, dev->max_event_ports);
 874                 cnxk_sso_fini(event_dev);
 875                 return -ENODEV;
 876         }
 877
 878         plt_sso_dbg("Initializing %s max_queues=%d max_ports=%d",
 879                     event_dev->data->name, dev->max_event_queues,
 880                     dev->max_event_ports);
 881
 882         return 0;
 883 }
 884
 885 static int
 886 cn10k_sso_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 887 {
 888         return rte_event_pmd_pci_probe(pci_drv, pci_dev,
 889                                        sizeof(struct cnxk_sso_evdev),
 890                                        cn10k_sso_init);
 891 }
 892
 893 static const struct rte_pci_id cn10k_pci_sso_map[] = {
 894         CNXK_PCI_ID(PCI_SUBSYSTEM_DEVID_CN10KA, PCI_DEVID_CNXK_RVU_SSO_TIM_PF),
 895         CNXK_PCI_ID(PCI_SUBSYSTEM_DEVID_CN10KAS, PCI_DEVID_CNXK_RVU_SSO_TIM_PF),
 896         CNXK_PCI_ID(PCI_SUBSYSTEM_DEVID_CNF10KA, PCI_DEVID_CNXK_RVU_SSO_TIM_PF),
 897         CNXK_PCI_ID(PCI_SUBSYSTEM_DEVID_CN10KA, PCI_DEVID_CNXK_RVU_SSO_TIM_VF),
 898         CNXK_PCI_ID(PCI_SUBSYSTEM_DEVID_CN10KAS, PCI_DEVID_CNXK_RVU_SSO_TIM_VF),
 899         CNXK_PCI_ID(PCI_SUBSYSTEM_DEVID_CNF10KA, PCI_DEVID_CNXK_RVU_SSO_TIM_VF),
 900         {
 901                 .vendor_id = 0,
 902         },
 903 };
 904
 905 static struct rte_pci_driver cn10k_pci_sso = {
 906         .id_table = cn10k_pci_sso_map,
 907         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
 908         .probe = cn10k_sso_probe,
 909         .remove = cnxk_sso_remove,
 910 };
 911
 912 RTE_PMD_REGISTER_PCI(event_cn10k, cn10k_pci_sso);
 913 RTE_PMD_REGISTER_PCI_TABLE(event_cn10k, cn10k_pci_sso_map);
 914 RTE_PMD_REGISTER_KMOD_DEP(event_cn10k, "vfio-pci");
 915 RTE_PMD_REGISTER_PARAM_STRING(event_cn10k, CNXK_SSO_XAE_CNT "=<int>"
 916                               CNXK_SSO_GGRP_QOS "=<string>"
 917                               CNXK_SSO_FORCE_BP "=1"
 918                               CN10K_SSO_GW_MODE "=<int>"
 919                               CNXK_TIM_DISABLE_NPA "=1"
 920                               CNXK_TIM_CHNK_SLOTS "=<int>"
 921                               CNXK_TIM_RINGS_LMT "=<int>"
 922                               CNXK_TIM_STATS_ENA "=1"
 923                               CNXK_TIM_EXT_CLK "=<string>");