drivers/event/cnxk/cn10k_eventdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include "cn10k_worker.h"
   6 #include "cnxk_eventdev.h"
   7 #include "cnxk_worker.h"
   8
   9 static uint32_t
  10 cn10k_sso_gw_mode_wdata(struct cnxk_sso_evdev *dev)
  11 {
  12         uint32_t wdata = BIT(16) | 1;
  13
  14         switch (dev->gw_mode) {
  15         case CN10K_GW_MODE_NONE:
  16         default:
  17                 break;
  18         case CN10K_GW_MODE_PREF:
  19                 wdata |= BIT(19);
  20                 break;
  21         case CN10K_GW_MODE_PREF_WFE:
  22                 wdata |= BIT(20) | BIT(19);
  23                 break;
  24         }
  25
  26         return wdata;
  27 }
  28
  29 static void *
  30 cn10k_sso_init_hws_mem(void *arg, uint8_t port_id)
  31 {
  32         struct cnxk_sso_evdev *dev = arg;
  33         struct cn10k_sso_hws *ws;
  34
  35         /* Allocate event port memory */
  36         ws = rte_zmalloc("cn10k_ws",
  37                          sizeof(struct cn10k_sso_hws) + RTE_CACHE_LINE_SIZE,
  38                          RTE_CACHE_LINE_SIZE);
  39         if (ws == NULL) {
  40                 plt_err("Failed to alloc memory for port=%d", port_id);
  41                 return NULL;
  42         }
  43
  44         /* First cache line is reserved for cookie */
  45         ws = (struct cn10k_sso_hws *)((uint8_t *)ws + RTE_CACHE_LINE_SIZE);
  46         ws->base = roc_sso_hws_base_get(&dev->sso, port_id);
  47         ws->tx_base = ws->base;
  48         ws->hws_id = port_id;
  49         ws->swtag_req = 0;
  50         ws->gw_wdata = cn10k_sso_gw_mode_wdata(dev);
  51         ws->lmt_base = dev->sso.lmt_base;
  52
  53         return ws;
  54 }
  55
  56 static int
  57 cn10k_sso_hws_link(void *arg, void *port, uint16_t *map, uint16_t nb_link)
  58 {
  59         struct cnxk_sso_evdev *dev = arg;
  60         struct cn10k_sso_hws *ws = port;
  61
  62         return roc_sso_hws_link(&dev->sso, ws->hws_id, map, nb_link);
  63 }
  64
  65 static int
  66 cn10k_sso_hws_unlink(void *arg, void *port, uint16_t *map, uint16_t nb_link)
  67 {
  68         struct cnxk_sso_evdev *dev = arg;
  69         struct cn10k_sso_hws *ws = port;
  70
  71         return roc_sso_hws_unlink(&dev->sso, ws->hws_id, map, nb_link);
  72 }
  73
  74 static void
  75 cn10k_sso_hws_setup(void *arg, void *hws, uintptr_t *grps_base)
  76 {
  77         struct cnxk_sso_evdev *dev = arg;
  78         struct cn10k_sso_hws *ws = hws;
  79         uint64_t val;
  80
  81         rte_memcpy(ws->grps_base, grps_base,
  82                    sizeof(uintptr_t) * CNXK_SSO_MAX_HWGRP);
  83         ws->fc_mem = dev->fc_mem;
  84         ws->xaq_lmt = dev->xaq_lmt;
  85
  86         /* Set get_work timeout for HWS */
  87         val = NSEC2USEC(dev->deq_tmo_ns) - 1;
  88         plt_write64(val, ws->base + SSOW_LF_GWS_NW_TIM);
  89 }
  90
  91 static void
  92 cn10k_sso_hws_release(void *arg, void *hws)
  93 {
  94         struct cnxk_sso_evdev *dev = arg;
  95         struct cn10k_sso_hws *ws = hws;
  96         int i;
  97
  98         for (i = 0; i < dev->nb_event_queues; i++)
  99                 roc_sso_hws_unlink(&dev->sso, ws->hws_id, (uint16_t *)&i, 1);
 100         memset(ws, 0, sizeof(*ws));
 101 }
 102
 103 static void
 104 cn10k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base,
 105                            cnxk_handle_event_t fn, void *arg)
 106 {
 107         struct cn10k_sso_hws *ws = hws;
 108         uint64_t cq_ds_cnt = 1;
 109         uint64_t aq_cnt = 1;
 110         uint64_t ds_cnt = 1;
 111         struct rte_event ev;
 112         uint64_t val, req;
 113
 114         plt_write64(0, base + SSO_LF_GGRP_QCTL);
 115
 116         req = queue_id;     /* GGRP ID */
 117         req |= BIT_ULL(18); /* Grouped */
 118         req |= BIT_ULL(16); /* WAIT */
 119
 120         aq_cnt = plt_read64(base + SSO_LF_GGRP_AQ_CNT);
 121         ds_cnt = plt_read64(base + SSO_LF_GGRP_MISC_CNT);
 122         cq_ds_cnt = plt_read64(base + SSO_LF_GGRP_INT_CNT);
 123         cq_ds_cnt &= 0x3FFF3FFF0000;
 124
 125         while (aq_cnt || cq_ds_cnt || ds_cnt) {
 126                 plt_write64(req, ws->base + SSOW_LF_GWS_OP_GET_WORK0);
 127                 cn10k_sso_hws_get_work_empty(ws, &ev);
 128                 if (fn != NULL && ev.u64 != 0)
 129                         fn(arg, ev);
 130                 if (ev.sched_type != SSO_TT_EMPTY)
 131                         cnxk_sso_hws_swtag_flush(
 132                                 ws->base + SSOW_LF_GWS_WQE0,
 133                                 ws->base + SSOW_LF_GWS_OP_SWTAG_FLUSH);
 134                 do {
 135                         val = plt_read64(ws->base + SSOW_LF_GWS_PENDSTATE);
 136                 } while (val & BIT_ULL(56));
 137                 aq_cnt = plt_read64(base + SSO_LF_GGRP_AQ_CNT);
 138                 ds_cnt = plt_read64(base + SSO_LF_GGRP_MISC_CNT);
 139                 cq_ds_cnt = plt_read64(base + SSO_LF_GGRP_INT_CNT);
 140                 /* Extract cq and ds count */
 141                 cq_ds_cnt &= 0x3FFF3FFF0000;
 142         }
 143
 144         plt_write64(0, ws->base + SSOW_LF_GWS_OP_GWC_INVAL);
 145         rte_mb();
 146 }
 147
 148 static void
 149 cn10k_sso_hws_reset(void *arg, void *hws)
 150 {
 151         struct cnxk_sso_evdev *dev = arg;
 152         struct cn10k_sso_hws *ws = hws;
 153         uintptr_t base = ws->base;
 154         uint64_t pend_state;
 155         union {
 156                 __uint128_t wdata;
 157                 uint64_t u64[2];
 158         } gw;
 159         uint8_t pend_tt;
 160
 161         /* Wait till getwork/swtp/waitw/desched completes. */
 162         do {
 163                 pend_state = plt_read64(base + SSOW_LF_GWS_PENDSTATE);
 164         } while (pend_state & (BIT_ULL(63) | BIT_ULL(62) | BIT_ULL(58) |
 165                                BIT_ULL(56) | BIT_ULL(54)));
 166         pend_tt = CNXK_TT_FROM_TAG(plt_read64(base + SSOW_LF_GWS_WQE0));
 167         if (pend_tt != SSO_TT_EMPTY) { /* Work was pending */
 168                 if (pend_tt == SSO_TT_ATOMIC || pend_tt == SSO_TT_ORDERED)
 169                         cnxk_sso_hws_swtag_untag(base +
 170                                                  SSOW_LF_GWS_OP_SWTAG_UNTAG);
 171                 plt_write64(0, base + SSOW_LF_GWS_OP_DESCHED);
 172         }
 173
 174         /* Wait for desched to complete. */
 175         do {
 176                 pend_state = plt_read64(base + SSOW_LF_GWS_PENDSTATE);
 177         } while (pend_state & BIT_ULL(58));
 178
 179         switch (dev->gw_mode) {
 180         case CN10K_GW_MODE_PREF:
 181                 while (plt_read64(base + SSOW_LF_GWS_PRF_WQE0) & BIT_ULL(63))
 182                         ;
 183                 break;
 184         case CN10K_GW_MODE_PREF_WFE:
 185                 while (plt_read64(base + SSOW_LF_GWS_PRF_WQE0) &
 186                        SSOW_LF_GWS_TAG_PEND_GET_WORK_BIT)
 187                         continue;
 188                 plt_write64(0, base + SSOW_LF_GWS_OP_GWC_INVAL);
 189                 break;
 190         case CN10K_GW_MODE_NONE:
 191         default:
 192                 break;
 193         }
 194
 195         if (CNXK_TT_FROM_TAG(plt_read64(base + SSOW_LF_GWS_PRF_WQE0)) !=
 196             SSO_TT_EMPTY) {
 197                 plt_write64(BIT_ULL(16) | 1,
 198                             ws->base + SSOW_LF_GWS_OP_GET_WORK0);
 199                 do {
 200                         roc_load_pair(gw.u64[0], gw.u64[1],
 201                                       ws->base + SSOW_LF_GWS_WQE0);
 202                 } while (gw.u64[0] & BIT_ULL(63));
 203                 pend_tt = CNXK_TT_FROM_TAG(plt_read64(base + SSOW_LF_GWS_WQE0));
 204                 if (pend_tt != SSO_TT_EMPTY) { /* Work was pending */
 205                         if (pend_tt == SSO_TT_ATOMIC ||
 206                             pend_tt == SSO_TT_ORDERED)
 207                                 cnxk_sso_hws_swtag_untag(
 208                                         base + SSOW_LF_GWS_OP_SWTAG_UNTAG);
 209                         plt_write64(0, base + SSOW_LF_GWS_OP_DESCHED);
 210                 }
 211         }
 212
 213         plt_write64(0, base + SSOW_LF_GWS_OP_GWC_INVAL);
 214         rte_mb();
 215 }
 216
 217 static void
 218 cn10k_sso_set_rsrc(void *arg)
 219 {
 220         struct cnxk_sso_evdev *dev = arg;
 221
 222         dev->max_event_ports = dev->sso.max_hws;
 223         dev->max_event_queues =
 224                 dev->sso.max_hwgrp > RTE_EVENT_MAX_QUEUES_PER_DEV ?
 225                               RTE_EVENT_MAX_QUEUES_PER_DEV :
 226                               dev->sso.max_hwgrp;
 227 }
 228
 229 static int
 230 cn10k_sso_rsrc_init(void *arg, uint8_t hws, uint8_t hwgrp)
 231 {
 232         struct cnxk_sso_evdev *dev = arg;
 233
 234         return roc_sso_rsrc_init(&dev->sso, hws, hwgrp);
 235 }
 236
 237 static int
 238 cn10k_sso_updt_tx_adptr_data(const struct rte_eventdev *event_dev)
 239 {
 240         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 241         int i;
 242
 243         if (dev->tx_adptr_data == NULL)
 244                 return 0;
 245
 246         for (i = 0; i < dev->nb_event_ports; i++) {
 247                 struct cn10k_sso_hws *ws = event_dev->data->ports[i];
 248                 void *ws_cookie;
 249
 250                 ws_cookie = cnxk_sso_hws_get_cookie(ws);
 251                 ws_cookie = rte_realloc_socket(
 252                         ws_cookie,
 253                         sizeof(struct cnxk_sso_hws_cookie) +
 254                                 sizeof(struct cn10k_sso_hws) +
 255                                 (sizeof(uint64_t) * (dev->max_port_id + 1) *
 256                                  RTE_MAX_QUEUES_PER_PORT),
 257                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 258                 if (ws_cookie == NULL)
 259                         return -ENOMEM;
 260                 ws = RTE_PTR_ADD(ws_cookie, sizeof(struct cnxk_sso_hws_cookie));
 261                 memcpy(&ws->tx_adptr_data, dev->tx_adptr_data,
 262                        sizeof(uint64_t) * (dev->max_port_id + 1) *
 263                                RTE_MAX_QUEUES_PER_PORT);
 264                 event_dev->data->ports[i] = ws;
 265         }
 266
 267         return 0;
 268 }
 269
 270 static void
 271 cn10k_sso_fp_fns_set(struct rte_eventdev *event_dev)
 272 {
 273         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 274         const event_dequeue_t sso_hws_deq[2][2][2][2][2][2] = {
 275 #define R(name, f5, f4, f3, f2, f1, f0, flags)                                 \
 276         [f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_##name,
 277                 NIX_RX_FASTPATH_MODES
 278 #undef R
 279         };
 280
 281         const event_dequeue_burst_t sso_hws_deq_burst[2][2][2][2][2][2] = {
 282 #define R(name, f5, f4, f3, f2, f1, f0, flags)                                 \
 283         [f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_burst_##name,
 284                 NIX_RX_FASTPATH_MODES
 285 #undef R
 286         };
 287
 288         const event_dequeue_t sso_hws_tmo_deq[2][2][2][2][2][2] = {
 289 #define R(name, f5, f4, f3, f2, f1, f0, flags)                                 \
 290         [f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_tmo_##name,
 291                 NIX_RX_FASTPATH_MODES
 292 #undef R
 293         };
 294
 295         const event_dequeue_burst_t sso_hws_tmo_deq_burst[2][2][2][2][2][2] = {
 296 #define R(name, f5, f4, f3, f2, f1, f0, flags)                                 \
 297         [f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_tmo_burst_##name,
 298                 NIX_RX_FASTPATH_MODES
 299 #undef R
 300         };
 301
 302         const event_dequeue_t sso_hws_deq_seg[2][2][2][2][2][2] = {
 303 #define R(name, f5, f4, f3, f2, f1, f0, flags)                                 \
 304         [f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_seg_##name,
 305                 NIX_RX_FASTPATH_MODES
 306 #undef R
 307         };
 308
 309         const event_dequeue_burst_t sso_hws_deq_seg_burst[2][2][2][2][2][2] = {
 310 #define R(name, f5, f4, f3, f2, f1, f0, flags)                                 \
 311         [f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_seg_burst_##name,
 312                 NIX_RX_FASTPATH_MODES
 313 #undef R
 314         };
 315
 316         const event_dequeue_t sso_hws_tmo_deq_seg[2][2][2][2][2][2] = {
 317 #define R(name, f5, f4, f3, f2, f1, f0, flags)                                 \
 318         [f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_tmo_seg_##name,
 319                 NIX_RX_FASTPATH_MODES
 320 #undef R
 321         };
 322
 323         const event_dequeue_burst_t
 324                 sso_hws_tmo_deq_seg_burst[2][2][2][2][2][2] = {
 325 #define R(name, f5, f4, f3, f2, f1, f0, flags)                                 \
 326         [f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_deq_tmo_seg_burst_##name,
 327                         NIX_RX_FASTPATH_MODES
 328 #undef R
 329                 };
 330
 331         /* Tx modes */
 332         const event_tx_adapter_enqueue
 333                 sso_hws_tx_adptr_enq[2][2][2][2][2][2] = {
 334 #define T(name, f5, f4, f3, f2, f1, f0, sz, flags)                             \
 335         [f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_tx_adptr_enq_##name,
 336                         NIX_TX_FASTPATH_MODES
 337 #undef T
 338                 };
 339
 340         const event_tx_adapter_enqueue
 341                 sso_hws_tx_adptr_enq_seg[2][2][2][2][2][2] = {
 342 #define T(name, f5, f4, f3, f2, f1, f0, sz, flags)                             \
 343         [f5][f4][f3][f2][f1][f0] = cn10k_sso_hws_tx_adptr_enq_seg_##name,
 344                         NIX_TX_FASTPATH_MODES
 345 #undef T
 346                 };
 347
 348         event_dev->enqueue = cn10k_sso_hws_enq;
 349         event_dev->enqueue_burst = cn10k_sso_hws_enq_burst;
 350         event_dev->enqueue_new_burst = cn10k_sso_hws_enq_new_burst;
 351         event_dev->enqueue_forward_burst = cn10k_sso_hws_enq_fwd_burst;
 352         if (dev->rx_offloads & NIX_RX_MULTI_SEG_F) {
 353                 event_dev->dequeue = sso_hws_deq_seg
 354                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 355                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 356                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 357                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
 358                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 359                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 360                 event_dev->dequeue_burst = sso_hws_deq_seg_burst
 361                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 362                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 363                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 364                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
 365                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 366                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 367                 if (dev->is_timeout_deq) {
 368                         event_dev->dequeue = sso_hws_tmo_deq_seg
 369                                 [!!(dev->rx_offloads &
 370                                     NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 371                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 372                                 [!!(dev->rx_offloads &
 373                                     NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 374                                 [!!(dev->rx_offloads &
 375                                     NIX_RX_OFFLOAD_CHECKSUM_F)]
 376                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 377                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 378                         event_dev->dequeue_burst = sso_hws_tmo_deq_seg_burst
 379                                 [!!(dev->rx_offloads &
 380                                     NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 381                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 382                                 [!!(dev->rx_offloads &
 383                                     NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 384                                 [!!(dev->rx_offloads &
 385                                     NIX_RX_OFFLOAD_CHECKSUM_F)]
 386                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 387                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 388                 }
 389         } else {
 390                 event_dev->dequeue = sso_hws_deq
 391                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 392                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 393                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 394                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
 395                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 396                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 397                 event_dev->dequeue_burst = sso_hws_deq_burst
 398                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 399                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 400                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 401                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)]
 402                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 403                         [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 404                 if (dev->is_timeout_deq) {
 405                         event_dev->dequeue = sso_hws_tmo_deq
 406                                 [!!(dev->rx_offloads &
 407                                     NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 408                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 409                                 [!!(dev->rx_offloads &
 410                                     NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 411                                 [!!(dev->rx_offloads &
 412                                     NIX_RX_OFFLOAD_CHECKSUM_F)]
 413                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 414                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 415                         event_dev->dequeue_burst = sso_hws_tmo_deq_burst
 416                                 [!!(dev->rx_offloads &
 417                                     NIX_RX_OFFLOAD_VLAN_STRIP_F)]
 418                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)]
 419                                 [!!(dev->rx_offloads &
 420                                     NIX_RX_OFFLOAD_MARK_UPDATE_F)]
 421                                 [!!(dev->rx_offloads &
 422                                     NIX_RX_OFFLOAD_CHECKSUM_F)]
 423                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)]
 424                                 [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)];
 425                 }
 426         }
 427
 428         if (dev->tx_offloads & NIX_TX_MULTI_SEG_F) {
 429                 /* [SEC] [TSMP] [MBUF_NOFF] [VLAN] [OL3_L4_CSUM] [L3_L4_CSUM] */
 430                 event_dev->txa_enqueue = sso_hws_tx_adptr_enq_seg
 431                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_TSTAMP_F)]
 432                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_TSO_F)]
 433                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_MBUF_NOFF_F)]
 434                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_VLAN_QINQ_F)]
 435                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)]
 436                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_L3_L4_CSUM_F)];
 437         } else {
 438                 event_dev->txa_enqueue = sso_hws_tx_adptr_enq
 439                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_TSTAMP_F)]
 440                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_TSO_F)]
 441                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_MBUF_NOFF_F)]
 442                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_VLAN_QINQ_F)]
 443                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)]
 444                         [!!(dev->tx_offloads & NIX_TX_OFFLOAD_L3_L4_CSUM_F)];
 445         }
 446
 447         event_dev->txa_enqueue_same_dest = event_dev->txa_enqueue;
 448 }
 449
 450 static void
 451 cn10k_sso_info_get(struct rte_eventdev *event_dev,
 452                    struct rte_event_dev_info *dev_info)
 453 {
 454         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 455
 456         dev_info->driver_name = RTE_STR(EVENTDEV_NAME_CN10K_PMD);
 457         cnxk_sso_info_get(dev, dev_info);
 458 }
 459
 460 static int
 461 cn10k_sso_dev_configure(const struct rte_eventdev *event_dev)
 462 {
 463         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 464         int rc;
 465
 466         rc = cnxk_sso_dev_validate(event_dev);
 467         if (rc < 0) {
 468                 plt_err("Invalid event device configuration");
 469                 return -EINVAL;
 470         }
 471
 472         roc_sso_rsrc_fini(&dev->sso);
 473
 474         rc = cn10k_sso_rsrc_init(dev, dev->nb_event_ports,
 475                                  dev->nb_event_queues);
 476         if (rc < 0) {
 477                 plt_err("Failed to initialize SSO resources");
 478                 return -ENODEV;
 479         }
 480
 481         rc = cnxk_sso_xaq_allocate(dev);
 482         if (rc < 0)
 483                 goto cnxk_rsrc_fini;
 484
 485         rc = cnxk_setup_event_ports(event_dev, cn10k_sso_init_hws_mem,
 486                                     cn10k_sso_hws_setup);
 487         if (rc < 0)
 488                 goto cnxk_rsrc_fini;
 489
 490         /* Restore any prior port-queue mapping. */
 491         cnxk_sso_restore_links(event_dev, cn10k_sso_hws_link);
 492
 493         dev->configured = 1;
 494         rte_mb();
 495
 496         return 0;
 497 cnxk_rsrc_fini:
 498         roc_sso_rsrc_fini(&dev->sso);
 499         dev->nb_event_ports = 0;
 500         return rc;
 501 }
 502
 503 static int
 504 cn10k_sso_port_setup(struct rte_eventdev *event_dev, uint8_t port_id,
 505                      const struct rte_event_port_conf *port_conf)
 506 {
 507
 508         RTE_SET_USED(port_conf);
 509         return cnxk_sso_port_setup(event_dev, port_id, cn10k_sso_hws_setup);
 510 }
 511
 512 static void
 513 cn10k_sso_port_release(void *port)
 514 {
 515         struct cnxk_sso_hws_cookie *gws_cookie = cnxk_sso_hws_get_cookie(port);
 516         struct cnxk_sso_evdev *dev;
 517
 518         if (port == NULL)
 519                 return;
 520
 521         dev = cnxk_sso_pmd_priv(gws_cookie->event_dev);
 522         if (!gws_cookie->configured)
 523                 goto free;
 524
 525         cn10k_sso_hws_release(dev, port);
 526         memset(gws_cookie, 0, sizeof(*gws_cookie));
 527 free:
 528         rte_free(gws_cookie);
 529 }
 530
 531 static int
 532 cn10k_sso_port_link(struct rte_eventdev *event_dev, void *port,
 533                     const uint8_t queues[], const uint8_t priorities[],
 534                     uint16_t nb_links)
 535 {
 536         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 537         uint16_t hwgrp_ids[nb_links];
 538         uint16_t link;
 539
 540         RTE_SET_USED(priorities);
 541         for (link = 0; link < nb_links; link++)
 542                 hwgrp_ids[link] = queues[link];
 543         nb_links = cn10k_sso_hws_link(dev, port, hwgrp_ids, nb_links);
 544
 545         return (int)nb_links;
 546 }
 547
 548 static int
 549 cn10k_sso_port_unlink(struct rte_eventdev *event_dev, void *port,
 550                       uint8_t queues[], uint16_t nb_unlinks)
 551 {
 552         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 553         uint16_t hwgrp_ids[nb_unlinks];
 554         uint16_t unlink;
 555
 556         for (unlink = 0; unlink < nb_unlinks; unlink++)
 557                 hwgrp_ids[unlink] = queues[unlink];
 558         nb_unlinks = cn10k_sso_hws_unlink(dev, port, hwgrp_ids, nb_unlinks);
 559
 560         return (int)nb_unlinks;
 561 }
 562
 563 static int
 564 cn10k_sso_start(struct rte_eventdev *event_dev)
 565 {
 566         int rc;
 567
 568         rc = cn10k_sso_updt_tx_adptr_data(event_dev);
 569         if (rc < 0)
 570                 return rc;
 571
 572         rc = cnxk_sso_start(event_dev, cn10k_sso_hws_reset,
 573                             cn10k_sso_hws_flush_events);
 574         if (rc < 0)
 575                 return rc;
 576         cn10k_sso_fp_fns_set(event_dev);
 577
 578         return rc;
 579 }
 580
 581 static void
 582 cn10k_sso_stop(struct rte_eventdev *event_dev)
 583 {
 584         cnxk_sso_stop(event_dev, cn10k_sso_hws_reset,
 585                       cn10k_sso_hws_flush_events);
 586 }
 587
 588 static int
 589 cn10k_sso_close(struct rte_eventdev *event_dev)
 590 {
 591         return cnxk_sso_close(event_dev, cn10k_sso_hws_unlink);
 592 }
 593
 594 static int
 595 cn10k_sso_selftest(void)
 596 {
 597         return cnxk_sso_selftest(RTE_STR(event_cn10k));
 598 }
 599
 600 static int
 601 cn10k_sso_rx_adapter_caps_get(const struct rte_eventdev *event_dev,
 602                               const struct rte_eth_dev *eth_dev, uint32_t *caps)
 603 {
 604         int rc;
 605
 606         RTE_SET_USED(event_dev);
 607         rc = strncmp(eth_dev->device->driver->name, "net_cn10k", 9);
 608         if (rc)
 609                 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP;
 610         else
 611                 *caps = RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT |
 612                         RTE_EVENT_ETH_RX_ADAPTER_CAP_MULTI_EVENTQ |
 613                         RTE_EVENT_ETH_RX_ADAPTER_CAP_OVERRIDE_FLOW_ID;
 614
 615         return 0;
 616 }
 617
 618 static void
 619 cn10k_sso_set_priv_mem(const struct rte_eventdev *event_dev, void *lookup_mem,
 620                        void *tstmp_info)
 621 {
 622         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 623         int i;
 624
 625         for (i = 0; i < dev->nb_event_ports; i++) {
 626                 struct cn10k_sso_hws *ws = event_dev->data->ports[i];
 627                 ws->lookup_mem = lookup_mem;
 628                 ws->tstamp = tstmp_info;
 629         }
 630 }
 631
 632 static int
 633 cn10k_sso_rx_adapter_queue_add(
 634         const struct rte_eventdev *event_dev, const struct rte_eth_dev *eth_dev,
 635         int32_t rx_queue_id,
 636         const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
 637 {
 638         struct cn10k_eth_rxq *rxq;
 639         void *lookup_mem;
 640         void *tstmp_info;
 641         int rc;
 642
 643         rc = strncmp(eth_dev->device->driver->name, "net_cn10k", 8);
 644         if (rc)
 645                 return -EINVAL;
 646
 647         rc = cnxk_sso_rx_adapter_queue_add(event_dev, eth_dev, rx_queue_id,
 648                                            queue_conf);
 649         if (rc)
 650                 return -EINVAL;
 651         rxq = eth_dev->data->rx_queues[0];
 652         lookup_mem = rxq->lookup_mem;
 653         tstmp_info = rxq->tstamp;
 654         cn10k_sso_set_priv_mem(event_dev, lookup_mem, tstmp_info);
 655         cn10k_sso_fp_fns_set((struct rte_eventdev *)(uintptr_t)event_dev);
 656
 657         return 0;
 658 }
 659
 660 static int
 661 cn10k_sso_rx_adapter_queue_del(const struct rte_eventdev *event_dev,
 662                                const struct rte_eth_dev *eth_dev,
 663                                int32_t rx_queue_id)
 664 {
 665         int rc;
 666
 667         rc = strncmp(eth_dev->device->driver->name, "net_cn10k", 8);
 668         if (rc)
 669                 return -EINVAL;
 670
 671         return cnxk_sso_rx_adapter_queue_del(event_dev, eth_dev, rx_queue_id);
 672 }
 673
 674 static int
 675 cn10k_sso_tx_adapter_caps_get(const struct rte_eventdev *dev,
 676                               const struct rte_eth_dev *eth_dev, uint32_t *caps)
 677 {
 678         int ret;
 679
 680         RTE_SET_USED(dev);
 681         ret = strncmp(eth_dev->device->driver->name, "net_cn10k", 8);
 682         if (ret)
 683                 *caps = 0;
 684         else
 685                 *caps = RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT;
 686
 687         return 0;
 688 }
 689
 690 static int
 691 cn10k_sso_tx_adapter_queue_add(uint8_t id, const struct rte_eventdev *event_dev,
 692                                const struct rte_eth_dev *eth_dev,
 693                                int32_t tx_queue_id)
 694 {
 695         int rc;
 696
 697         RTE_SET_USED(id);
 698         rc = cnxk_sso_tx_adapter_queue_add(event_dev, eth_dev, tx_queue_id);
 699         if (rc < 0)
 700                 return rc;
 701         rc = cn10k_sso_updt_tx_adptr_data(event_dev);
 702         if (rc < 0)
 703                 return rc;
 704         cn10k_sso_fp_fns_set((struct rte_eventdev *)(uintptr_t)event_dev);
 705
 706         return 0;
 707 }
 708
 709 static int
 710 cn10k_sso_tx_adapter_queue_del(uint8_t id, const struct rte_eventdev *event_dev,
 711                                const struct rte_eth_dev *eth_dev,
 712                                int32_t tx_queue_id)
 713 {
 714         int rc;
 715
 716         RTE_SET_USED(id);
 717         rc = cnxk_sso_tx_adapter_queue_del(event_dev, eth_dev, tx_queue_id);
 718         if (rc < 0)
 719                 return rc;
 720         return cn10k_sso_updt_tx_adptr_data(event_dev);
 721 }
 722
 723 static struct rte_eventdev_ops cn10k_sso_dev_ops = {
 724         .dev_infos_get = cn10k_sso_info_get,
 725         .dev_configure = cn10k_sso_dev_configure,
 726         .queue_def_conf = cnxk_sso_queue_def_conf,
 727         .queue_setup = cnxk_sso_queue_setup,
 728         .queue_release = cnxk_sso_queue_release,
 729         .port_def_conf = cnxk_sso_port_def_conf,
 730         .port_setup = cn10k_sso_port_setup,
 731         .port_release = cn10k_sso_port_release,
 732         .port_link = cn10k_sso_port_link,
 733         .port_unlink = cn10k_sso_port_unlink,
 734         .timeout_ticks = cnxk_sso_timeout_ticks,
 735
 736         .eth_rx_adapter_caps_get = cn10k_sso_rx_adapter_caps_get,
 737         .eth_rx_adapter_queue_add = cn10k_sso_rx_adapter_queue_add,
 738         .eth_rx_adapter_queue_del = cn10k_sso_rx_adapter_queue_del,
 739         .eth_rx_adapter_start = cnxk_sso_rx_adapter_start,
 740         .eth_rx_adapter_stop = cnxk_sso_rx_adapter_stop,
 741
 742         .eth_tx_adapter_caps_get = cn10k_sso_tx_adapter_caps_get,
 743         .eth_tx_adapter_queue_add = cn10k_sso_tx_adapter_queue_add,
 744         .eth_tx_adapter_queue_del = cn10k_sso_tx_adapter_queue_del,
 745
 746         .timer_adapter_caps_get = cnxk_tim_caps_get,
 747
 748         .dump = cnxk_sso_dump,
 749         .dev_start = cn10k_sso_start,
 750         .dev_stop = cn10k_sso_stop,
 751         .dev_close = cn10k_sso_close,
 752         .dev_selftest = cn10k_sso_selftest,
 753 };
 754
 755 static int
 756 cn10k_sso_init(struct rte_eventdev *event_dev)
 757 {
 758         struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
 759         int rc;
 760
 761         if (RTE_CACHE_LINE_SIZE != 64) {
 762                 plt_err("Driver not compiled for CN9K");
 763                 return -EFAULT;
 764         }
 765
 766         rc = roc_plt_init();
 767         if (rc < 0) {
 768                 plt_err("Failed to initialize platform model");
 769                 return rc;
 770         }
 771
 772         event_dev->dev_ops = &cn10k_sso_dev_ops;
 773         /* For secondary processes, the primary has done all the work */
 774         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 775                 cn10k_sso_fp_fns_set(event_dev);
 776                 return 0;
 777         }
 778
 779         rc = cnxk_sso_init(event_dev);
 780         if (rc < 0)
 781                 return rc;
 782
 783         cn10k_sso_set_rsrc(cnxk_sso_pmd_priv(event_dev));
 784         if (!dev->max_event_ports || !dev->max_event_queues) {
 785                 plt_err("Not enough eventdev resource queues=%d ports=%d",
 786                         dev->max_event_queues, dev->max_event_ports);
 787                 cnxk_sso_fini(event_dev);
 788                 return -ENODEV;
 789         }
 790
 791         plt_sso_dbg("Initializing %s max_queues=%d max_ports=%d",
 792                     event_dev->data->name, dev->max_event_queues,
 793                     dev->max_event_ports);
 794
 795         return 0;
 796 }
 797
 798 static int
 799 cn10k_sso_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 800 {
 801         return rte_event_pmd_pci_probe(pci_drv, pci_dev,
 802                                        sizeof(struct cnxk_sso_evdev),
 803                                        cn10k_sso_init);
 804 }
 805
 806 static const struct rte_pci_id cn10k_pci_sso_map[] = {
 807         CNXK_PCI_ID(PCI_SUBSYSTEM_DEVID_CN10KA, PCI_DEVID_CNXK_RVU_SSO_TIM_PF),
 808         CNXK_PCI_ID(PCI_SUBSYSTEM_DEVID_CN10KAS, PCI_DEVID_CNXK_RVU_SSO_TIM_PF),
 809         CNXK_PCI_ID(PCI_SUBSYSTEM_DEVID_CN10KA, PCI_DEVID_CNXK_RVU_SSO_TIM_VF),
 810         CNXK_PCI_ID(PCI_SUBSYSTEM_DEVID_CN10KAS, PCI_DEVID_CNXK_RVU_SSO_TIM_VF),
 811         {
 812                 .vendor_id = 0,
 813         },
 814 };
 815
 816 static struct rte_pci_driver cn10k_pci_sso = {
 817         .id_table = cn10k_pci_sso_map,
 818         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
 819         .probe = cn10k_sso_probe,
 820         .remove = cnxk_sso_remove,
 821 };
 822
 823 RTE_PMD_REGISTER_PCI(event_cn10k, cn10k_pci_sso);
 824 RTE_PMD_REGISTER_PCI_TABLE(event_cn10k, cn10k_pci_sso_map);
 825 RTE_PMD_REGISTER_KMOD_DEP(event_cn10k, "vfio-pci");
 826 RTE_PMD_REGISTER_PARAM_STRING(event_cn10k, CNXK_SSO_XAE_CNT "=<int>"
 827                               CNXK_SSO_GGRP_QOS "=<string>"
 828                               CNXK_SSO_FORCE_BP "=1"
 829                               CN10K_SSO_GW_MODE "=<int>"
 830                               CNXK_TIM_DISABLE_NPA "=1"
 831                               CNXK_TIM_CHNK_SLOTS "=<int>"
 832                               CNXK_TIM_RINGS_LMT "=<int>"
 833                               CNXK_TIM_STATS_ENA "=1");